diff --git a/.clang_complete b/.clang_complete index d7cda0ff..e88dbc7b 100644 --- a/.clang_complete +++ b/.clang_complete @@ -1,5 +1,5 @@ -xc++ --std=c++14 +-std=c++17 -iquote . -iquote tdtl/ -iquote tl/ diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..78c6ddee --- /dev/null +++ b/.editorconfig @@ -0,0 +1,8 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +indent_style = space +indent_size = 2 diff --git a/.github/script/amd64-18.04.Dockerfile b/.github/script/amd64-18.04.Dockerfile deleted file mode 100644 index 3e98b026..00000000 --- a/.github/script/amd64-18.04.Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM ubuntu:18.04 - -RUN apt update -RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata -RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git curl libreadline-dev ccache libmicrohttpd-dev ninja-build - -WORKDIR / - -ARG BRANCH -RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH - -WORKDIR /ton -RUN mkdir /ton/build -WORKDIR /ton/build -ENV CC clang -ENV CXX clang++ -ENV CCACHE_DISABLE 1 -RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. -RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client \ No newline at end of file diff --git a/.github/script/amd64-20.04.Dockerfile b/.github/script/amd64-20.04.Dockerfile deleted file mode 100644 index ab71bbf2..00000000 --- a/.github/script/amd64-20.04.Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM ubuntu:20.04 - -RUN apt update -RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata -RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git curl libreadline-dev ccache libmicrohttpd-dev ninja-build - -WORKDIR / - -ARG BRANCH -RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH - -WORKDIR /ton -RUN mkdir /ton/build -WORKDIR /ton/build -ENV CC clang -ENV CXX clang++ -ENV CCACHE_DISABLE 1 -RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. -RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client \ No newline at end of file diff --git a/.github/script/amd64-22.04.Dockerfile b/.github/script/amd64-22.04.Dockerfile deleted file mode 100644 index 0479aa0b..00000000 --- a/.github/script/amd64-22.04.Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM ubuntu:22.04 - -RUN apt update -RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata -RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git curl libreadline-dev ccache libmicrohttpd-dev ninja-build - -WORKDIR / - -ARG BRANCH -RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH - -WORKDIR /ton -RUN mkdir /ton/build -WORKDIR /ton/build -ENV CC clang -ENV CXX clang++ -ENV CCACHE_DISABLE 1 -RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. -RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client \ No newline at end of file diff --git a/.github/script/arm64-18.04.Dockerfile b/.github/script/arm64-18.04.Dockerfile deleted file mode 100644 index 6c527a45..00000000 --- a/.github/script/arm64-18.04.Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM ubuntu:18.04 - -RUN apt update -RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata -RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git curl libreadline-dev ccache libmicrohttpd-dev ninja-build - -WORKDIR / - -ARG BRANCH -RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH - -WORKDIR /ton -RUN mkdir /ton/build -WORKDIR /ton/build -ENV CC clang -ENV CXX clang++ -ENV CCACHE_DISABLE 1 -RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= .. -RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id dht-server lite-client \ No newline at end of file diff --git a/.github/script/arm64-20.04.Dockerfile b/.github/script/arm64-20.04.Dockerfile deleted file mode 100644 index 7b2348fd..00000000 --- a/.github/script/arm64-20.04.Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM ubuntu:20.04 - -RUN apt update -RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata -RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git curl libreadline-dev ccache libmicrohttpd-dev ninja-build - -WORKDIR / - -ARG BRANCH -RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH - -WORKDIR /ton -RUN mkdir /ton/build -WORKDIR /ton/build -ENV CC clang -ENV CXX clang++ -ENV CCACHE_DISABLE 1 -RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= .. -RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id dht-server lite-client \ No newline at end of file diff --git a/.github/script/arm64-22.04.Dockerfile b/.github/script/arm64-22.04.Dockerfile deleted file mode 100644 index d0ea491b..00000000 --- a/.github/script/arm64-22.04.Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM ubuntu:22.04 - -RUN apt update -RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata -RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git curl libreadline-dev ccache libmicrohttpd-dev ninja-build - -WORKDIR / - -ARG BRANCH -RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH - -WORKDIR /ton -RUN mkdir /ton/build -WORKDIR /ton/build -ENV CC clang -ENV CXX clang++ -ENV CCACHE_DISABLE 1 -RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= .. -RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id dht-server lite-client \ No newline at end of file diff --git a/.github/script/fift-func-wasm-build-ubuntu.sh b/.github/script/fift-func-wasm-build-ubuntu.sh deleted file mode 100755 index 505ce137..00000000 --- a/.github/script/fift-func-wasm-build-ubuntu.sh +++ /dev/null @@ -1,79 +0,0 @@ -# The script build funcfift compiler to WASM - -# dependencies: -#sudo apt-get install -y build-essential git make cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev python3-pip nodejs - -export CC=$(which clang) -export CXX=$(which clang++) -export CCACHE_DISABLE=1 - -git clone https://github.com/openssl/openssl.git -cd openssl -git checkout OpenSSL_1_1_1j - -./config -make -j4 - -OPENSSL_DIR=`pwd` - -cd .. - -git clone https://github.com/madler/zlib.git -cd zlib -ZLIB_DIR=`pwd` - -cd .. - -# clone ton repo -git clone --recursive https://github.com/the-ton-tech/ton-blockchain.git - -# only to generate auto-block.cpp - -cd ton-blockchain -git pull -git checkout 1566a23b2bece49fd1de9ab2f35e88297d22829f -mkdir build -cd build -cmake -DCMAKE_BUILD_TYPE=Release -DZLIB_LIBRARY=/usr/lib/x86_64-linux-gnu/libz.so -DZLIB_INCLUDE_DIR=$ZLIB_DIR -DOPENSSL_ROOT_DIR=$OPENSSL_DIR -DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include -DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.so -DOPENSSL_SSL_LIBRARY=$OPENSSL_DIR/libssl.so .. -make -j4 fift - -rm -rf * - -cd ../.. - -git clone https://github.com/emscripten-core/emsdk.git -cd emsdk -./emsdk install latest -./emsdk activate latest -EMSDK_DIR=`pwd` - -source $EMSDK_DIR/emsdk_env.sh -export CC=$(which emcc) -export CXX=$(which em++) -export CCACHE_DISABLE=1 - -cd ../zlib - -emconfigure ./configure --static -emmake make -j4 -ZLIB_DIR=`pwd` - -cd ../openssl - -make clean -emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test no-ui -sed -i 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile -sed -i 's/-ldl//g' Makefile -sed -i 's/-O3/-Os/g' Makefile -emmake make depend -emmake make -j4 - -cd ../ton-blockchain - -cd build - -emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release -DZLIB_LIBRARY=$ZLIB_DIR/libz.a -DZLIB_INCLUDE_DIR=$ZLIB_DIR -DOPENSSL_ROOT_DIR=$OPENSSL_DIR -DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include -DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.a -DOPENSSL_SSL_LIBRARY=$OPENSSL_DIR/libssl.a -DCMAKE_TOOLCHAIN_FILE=$EMSDK_DIR/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -DCMAKE_CXX_FLAGS="-pthread -sUSE_ZLIB=1" .. - -cp -R ../crypto/smartcont ../crypto/fift/lib crypto - -emmake make -j4 funcfiftlib diff --git a/.github/workflows/build-ton-linux-android-tonlib.yml b/.github/workflows/build-ton-linux-android-tonlib.yml new file mode 100644 index 00000000..b1ef5281 --- /dev/null +++ b/.github/workflows/build-ton-linux-android-tonlib.yml @@ -0,0 +1,34 @@ +name: Tonlib Android + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + runs-on: ubuntu-22.04 + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Install system libraries + run: | + sudo apt-get update + sudo apt-get install -y build-essential git cmake ninja-build automake libtool texinfo autoconf libgflags-dev \ + zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev \ + libtool autoconf libsodium-dev libsecp256k1-dev liblz4-dev + + - name: Build TON + run: | + git submodule sync --recursive + git submodule update + cp assembly/android/build-android-tonlib.sh . + chmod +x build-android-tonlib.sh + ./build-android-tonlib.sh -a + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-android-tonlib + path: artifacts diff --git a/.github/workflows/build-ton-linux-arm64-appimage.yml b/.github/workflows/build-ton-linux-arm64-appimage.yml new file mode 100644 index 00000000..d464d8a2 --- /dev/null +++ b/.github/workflows/build-ton-linux-arm64-appimage.yml @@ -0,0 +1,57 @@ +name: Ubuntu TON build (AppImages, arm64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + runs-on: ubuntu-22.04-arm + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Install system libraries + run: | + sudo apt update + sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev + sudo apt remove libgsl-dev + + - name: Install clang-16 + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 16 all + + - name: Build TON + run: | + git submodule sync --recursive + git submodule update + cp assembly/native/build-ubuntu-appimages.sh . + chmod +x build-ubuntu-appimages.sh + ./build-ubuntu-appimages.sh -a + + - name: Make AppImages + run: | + cp assembly/appimage/create-appimages.sh . + cp assembly/appimage/AppRun . + cp assembly/appimage/ton.png . + chmod +x create-appimages.sh + ./create-appimages.sh aarch64 + rm -rf artifacts + + + - name: Build TON libs + run: | + cp assembly/native/build-ubuntu-portable-libs.sh . + chmod +x build-ubuntu-portable-libs.sh + ./build-ubuntu-portable-libs.sh -a + cp ./artifacts/libtonlibjson.so appimages/artifacts/ + cp ./artifacts/libemulator.so appimages/artifacts/ + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-arm64-linux + path: appimages/artifacts diff --git a/.github/workflows/build-ton-linux-arm64-shared.yml b/.github/workflows/build-ton-linux-arm64-shared.yml new file mode 100644 index 00000000..6433df0b --- /dev/null +++ b/.github/workflows/build-ton-linux-arm64-shared.yml @@ -0,0 +1,43 @@ +name: Ubuntu TON build (shared, arm64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + strategy: + fail-fast: false + matrix: + os: [ubuntu-22.04-arm, ubuntu-24.04-arm] + runs-on: ${{ matrix.os }} + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Install system libraries + run: | + sudo apt-get update + sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev + + - if: matrix.os != 'ubuntu-24.04-arm' + name: Install llvm-16 + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 16 all + + - name: Build TON + run: | + git submodule sync --recursive + git submodule update + cp assembly/native/build-ubuntu-shared.sh . + chmod +x build-ubuntu-shared.sh + ./build-ubuntu-shared.sh -t -a + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-binaries-${{ matrix.os }} + path: artifacts diff --git a/.github/workflows/build-ton-linux-x86-64-appimage.yml b/.github/workflows/build-ton-linux-x86-64-appimage.yml new file mode 100644 index 00000000..4f78ece9 --- /dev/null +++ b/.github/workflows/build-ton-linux-x86-64-appimage.yml @@ -0,0 +1,63 @@ +name: Ubuntu TON build (AppImages, x86-64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + runs-on: ubuntu-20.04 + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Install system libraries + run: | + sudo apt update + sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev + sudo apt remove libgsl-dev + + - name: Install gcc-11 g++-11 + run: | + sudo apt install -y manpages-dev software-properties-common + sudo add-apt-repository ppa:ubuntu-toolchain-r/test + sudo apt update && sudo apt install gcc-11 g++-11 + + - name: Install clang-16 + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 16 all + + - name: Build TON + run: | + git submodule sync --recursive + git submodule update + cp assembly/native/build-ubuntu-appimages.sh . + chmod +x build-ubuntu-appimages.sh + ./build-ubuntu-appimages.sh -a + + - name: Make AppImages + run: | + cp assembly/appimage/create-appimages.sh . + cp assembly/appimage/AppRun . + cp assembly/appimage/ton.png . + chmod +x create-appimages.sh + ./create-appimages.sh x86_64 + rm -rf artifacts + + + - name: Build TON libs + run: | + cp assembly/native/build-ubuntu-portable-libs.sh . + chmod +x build-ubuntu-portable-libs.sh + ./build-ubuntu-portable-libs.sh -a + cp ./artifacts/libtonlibjson.so appimages/artifacts/ + cp ./artifacts/libemulator.so appimages/artifacts/ + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-x86_64-linux + path: appimages/artifacts diff --git a/.github/workflows/build-ton-linux-x86-64-shared.yml b/.github/workflows/build-ton-linux-x86-64-shared.yml new file mode 100644 index 00000000..bf78f7df --- /dev/null +++ b/.github/workflows/build-ton-linux-x86-64-shared.yml @@ -0,0 +1,48 @@ +name: Ubuntu TON build (shared, x86-64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04] + runs-on: ${{ matrix.os }} + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Install system libraries + run: | + sudo apt-get update + sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev + + - if: matrix.os == 'ubuntu-20.04' + run: | + sudo apt install -y manpages-dev software-properties-common + sudo add-apt-repository ppa:ubuntu-toolchain-r/test + sudo apt update && sudo apt install gcc-11 g++-11 + + - if: matrix.os != 'ubuntu-24.04' + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 16 all + + - name: Build TON + run: | + git submodule sync --recursive + git submodule update + cp assembly/native/build-ubuntu-shared.sh . + chmod +x build-ubuntu-shared.sh + ./build-ubuntu-shared.sh -t -a + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-binaries-${{ matrix.os }} + path: artifacts diff --git a/.github/workflows/build-ton-macos-13-x86-64-portable.yml b/.github/workflows/build-ton-macos-13-x86-64-portable.yml new file mode 100644 index 00000000..5e50a468 --- /dev/null +++ b/.github/workflows/build-ton-macos-13-x86-64-portable.yml @@ -0,0 +1,27 @@ +name: MacOS-13 TON build (portable, x86-64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + runs-on: macos-13 + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Build TON + run: | + git submodule sync --recursive + git submodule update + cp assembly/native/build-macos-portable.sh . + chmod +x build-macos-portable.sh + ./build-macos-portable.sh -t -a + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-x86_64-macos + path: artifacts diff --git a/.github/workflows/build-ton-macos-14-arm64-portable.yml b/.github/workflows/build-ton-macos-14-arm64-portable.yml new file mode 100644 index 00000000..8eb3af70 --- /dev/null +++ b/.github/workflows/build-ton-macos-14-arm64-portable.yml @@ -0,0 +1,27 @@ +name: MacOS-14 TON build (portable, arm64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + runs-on: macos-14 + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Build TON + run: | + git submodule sync --recursive + git submodule update + cp assembly/native/build-macos-portable.sh . + chmod +x build-macos-portable.sh + ./build-macos-portable.sh -t -a + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-arm64-macos + path: artifacts diff --git a/.github/workflows/build-ton-macos-15-arm64-shared.yml b/.github/workflows/build-ton-macos-15-arm64-shared.yml new file mode 100644 index 00000000..00d8f639 --- /dev/null +++ b/.github/workflows/build-ton-macos-15-arm64-shared.yml @@ -0,0 +1,27 @@ +name: MacOS-15 TON build (shared, arm64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + runs-on: macos-15 + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Build TON + run: | + git submodule sync --recursive + git submodule update + cp assembly/native/build-macos-shared.sh . + chmod +x build-macos-shared.sh + ./build-macos-shared.sh -t -a + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-binaries-macos-15 + path: artifacts diff --git a/.github/workflows/build-ton-macos-arm64-shared.yml b/.github/workflows/build-ton-macos-arm64-shared.yml new file mode 100644 index 00000000..7481f9ff --- /dev/null +++ b/.github/workflows/build-ton-macos-arm64-shared.yml @@ -0,0 +1,27 @@ +name: MacOS-14 TON build (shared, arm64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + runs-on: macos-14 + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Build TON + run: | + git submodule sync --recursive + git submodule update + cp assembly/native/build-macos-shared.sh . + chmod +x build-macos-shared.sh + ./build-macos-shared.sh -t -a + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-binaries-macos-14 + path: artifacts diff --git a/.github/workflows/build-ton-macos-x86-64-shared.yml b/.github/workflows/build-ton-macos-x86-64-shared.yml new file mode 100644 index 00000000..6a69b2e3 --- /dev/null +++ b/.github/workflows/build-ton-macos-x86-64-shared.yml @@ -0,0 +1,27 @@ +name: MacOS TON build (shared, x86-64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + runs-on: macos-13 + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Build TON + run: | + git submodule sync --recursive + git submodule update + cp assembly/native/build-macos-shared.sh . + chmod +x build-macos-shared.sh + ./build-macos-shared.sh -t -a + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-binaries-macos-13 + path: artifacts diff --git a/.github/workflows/build-ton-wasm-emscripten.yml b/.github/workflows/build-ton-wasm-emscripten.yml new file mode 100644 index 00000000..bac0cf98 --- /dev/null +++ b/.github/workflows/build-ton-wasm-emscripten.yml @@ -0,0 +1,51 @@ +name: Emscripten TON build (wasm) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + runs-on: ubuntu-22.04 + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Install system libraries + run: | + sudo apt-get update + sudo apt-get install -y build-essential git openssl cmake ninja-build zlib1g-dev libssl-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev + + - name: Build TON WASM artifacts + run: | + git submodule sync --recursive + git submodule update + cp assembly/wasm/fift-func-wasm-build-ubuntu.sh . + chmod +x fift-func-wasm-build-ubuntu.sh + ./fift-func-wasm-build-ubuntu.sh -a + + - name: Prepare test + run: | + cp assembly/wasm/*.fc . + git clone https://github.com/ton-community/func-js.git + cd func-js + npm install + npm run build + npm link + + - name: Test TON WASM artifacts + run: | + base64 -w 0 artifacts/funcfiftlib.wasm > artifacts/funcfiftlib.wasm.js + printf "module.exports = { FuncFiftLibWasm: '" | cat - artifacts/funcfiftlib.wasm.js > temp.txt && mv temp.txt artifacts/funcfiftlib.wasm.js + echo "'}" >> artifacts/funcfiftlib.wasm.js + cp artifacts/funcfiftlib.wasm.js func-js/node_modules/@ton-community/func-js-bin/dist/funcfiftlib.wasm.js + cp artifacts/funcfiftlib.js func-js/node_modules/@ton-community/func-js-bin/dist/funcfiftlib.js + npx func-js stdlib.fc intrinsics.fc --fift ./output.f + + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-wasm + path: artifacts diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 089af583..05a3db26 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -4,6 +4,9 @@ on: [workflow_dispatch] permissions: write-all +env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + jobs: create-release: runs-on: ubuntu-22.04 @@ -11,75 +14,148 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Show all artifacts - run: | - mkdir artifacts - ls -lart artifacts - - - name: Download Ubuntu x86-64 artifacts - uses: dawidd6/action-download-artifact@v2 + - name: Download Linux arm64 artifacts + uses: dawidd6/action-download-artifact@v6 with: - workflow: ubuntu-compile.yml + workflow: build-ton-linux-arm64-appimage.yml path: artifacts workflow_conclusion: success + branch: master skip_unpack: true - - name: Download Ubuntu arm64 artifacts - uses: dawidd6/action-download-artifact@v2 + - name: Download and unzip Linux arm64 artifacts + uses: dawidd6/action-download-artifact@v6 with: - workflow: docker-compile-ubuntu.yml + workflow: build-ton-linux-arm64-appimage.yml path: artifacts workflow_conclusion: success + branch: master + skip_unpack: false + + - name: Download Linux x86-64 artifacts + uses: dawidd6/action-download-artifact@v6 + with: + workflow: build-ton-linux-x86-64-appimage.yml + path: artifacts + workflow_conclusion: success + branch: master skip_unpack: true - - name: Download MacOS 11.7 artifacts - uses: dawidd6/action-download-artifact@v2 + - name: Download and unzip Linux x86-64 artifacts + uses: dawidd6/action-download-artifact@v6 with: - workflow: macos-11.7-compile.yml + workflow: build-ton-linux-x86-64-appimage.yml path: artifacts workflow_conclusion: success + branch: master + skip_unpack: false + + - name: Download Mac x86-64 artifacts + uses: dawidd6/action-download-artifact@v6 + with: + workflow: build-ton-macos-13-x86-64-portable.yml + path: artifacts + workflow_conclusion: success + branch: master skip_unpack: true - - name: Download MacOS 12.6 artifacts - uses: dawidd6/action-download-artifact@v2 + - name: Download Mac arm64 artifacts + uses: dawidd6/action-download-artifact@v6 with: - workflow: macos-12.6-compile.yml + workflow: build-ton-macos-14-arm64-portable.yml path: artifacts workflow_conclusion: success + branch: master skip_unpack: true + - name: Download and unzip Mac x86-64 artifacts + uses: dawidd6/action-download-artifact@v6 + with: + workflow: build-ton-macos-13-x86-64-portable.yml + path: artifacts + workflow_conclusion: success + branch: master + skip_unpack: false + + - name: Download and unzip arm64 artifacts + uses: dawidd6/action-download-artifact@v6 + with: + workflow: build-ton-macos-14-arm64-portable.yml + path: artifacts + workflow_conclusion: success + branch: master + skip_unpack: false + - name: Download Windows artifacts - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v6 with: - workflow: win-2019-compile.yml + workflow: ton-x86-64-windows.yml path: artifacts workflow_conclusion: success + branch: master + skip_unpack: true + + - name: Download and unzip Windows artifacts + uses: dawidd6/action-download-artifact@v6 + with: + workflow: ton-x86-64-windows.yml + path: artifacts + workflow_conclusion: success + branch: master + skip_unpack: false + + - name: Download WASM artifacts + uses: dawidd6/action-download-artifact@v6 + with: + workflow: build-ton-wasm-emscripten.yml + path: artifacts + workflow_conclusion: success + branch: master + skip_unpack: true + + - name: Download Android Tonlib artifacts + uses: dawidd6/action-download-artifact@v6 + with: + workflow: build-ton-linux-android-tonlib.yml + path: artifacts + workflow_conclusion: success + branch: master skip_unpack: true - name: Show all artifacts run: | tree artifacts + -# create release + # create release - name: Read Changelog.md and use it as a body of new release id: read_release shell: bash run: | - r=$(cat Changelog.md) + r=$(cat recent_changelog.md) r="${r//'%'/'%25'}" r="${r//$'\n'/'%0A'}" r="${r//$'\r'/'%0D'}" echo "::set-output name=CHANGELOG_BODY::$r" - - name: Get current date - id: date - run: echo "::set-output name=date::$(date +'%Y.%m')" + - name: Get next tag + id: tag + run: | + git fetch --all --tags + git tag -l + NEW_TAG=v$(date +'%Y.%m') + FOUND=$(git tag -l | grep $NEW_TAG | wc -l) + if [ $FOUND -eq 0 ]; then + echo "TAG=$NEW_TAG" >> $GITHUB_OUTPUT + else + echo "TAG=$NEW_TAG-$FOUND" >> $GITHUB_OUTPUT + fi - name: Get registration token id: getRegToken run: | - curl -X POST -H \"Accept: application/vnd.github+json\" -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' https://api.github.com/repos/neodix42/HardTestDevelopment/actions/runners/registration-token + curl -X POST -H \"Accept: application/vnd.github+json\" -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' https://api.github.com/repos/ton-blockchain/ton/actions/runners/registration-token - name: Create release id: create_release @@ -87,81 +163,568 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - tag_name: v${{ steps.date.outputs.date }} - release_name: v${{ steps.date.outputs.date }} + tag_name: ${{ steps.tag.outputs.TAG }} + release_name: TON ${{ steps.tag.outputs.TAG }} body: | ${{ steps.read_release.outputs.CHANGELOG_BODY }} draft: false prerelease: false + # upload + + # win + - name: Upload Windows 2019 artifacts uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-win-binaries.zip - asset_name: ton-windows-2019-x86-64.zip - tag: v${{ steps.date.outputs.date }} + file: artifacts/ton-x86-64-windows.zip + asset_name: ton-win-x86-64.zip + tag: ${{ steps.tag.outputs.TAG }} - - name: Upload MacOS 11.7 x86-64 artifacts + - name: Upload Windows 2019 single artifact - fift uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-macos-11.7.zip - asset_name: ton-macos-11.7-x86-64.zip - tag: v${{ steps.date.outputs.date }} + file: artifacts/ton-x86-64-windows/fift.exe + asset_name: fift.exe + tag: ${{ steps.tag.outputs.TAG }} - - name: Upload MacOS 12.6 x86-64 artifacts + - name: Upload Windows 2019 single artifact - func uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-macos-12.6.zip - asset_name: ton-macos-12.6-x86-64.zip - tag: v${{ steps.date.outputs.date }} + file: artifacts/ton-x86-64-windows/func.exe + asset_name: func.exe + tag: ${{ steps.tag.outputs.TAG }} - - name: Upload Ubuntu 18.04 x86-64 artifacts + - name: Upload Windows 2019 single artifact - tolk uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-binaries-ubuntu-18.04.zip - asset_name: ton-ubuntu-18.04-x86-64.zip - tag: v${{ steps.date.outputs.date }} + file: artifacts/ton-x86-64-windows/tolk.exe + asset_name: tolk.exe + tag: ${{ steps.tag.outputs.TAG }} - - name: Upload Ubuntu 20.04 x86-64 artifacts + - name: Upload Windows 2019 single artifact - lite-client uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-binaries-ubuntu-20.04.zip - asset_name: ton-ubuntu-20.04-x86-64.zip - tag: v${{ steps.date.outputs.date }} + file: artifacts/ton-x86-64-windows/lite-client.exe + asset_name: lite-client.exe + tag: ${{ steps.tag.outputs.TAG }} - - name: Upload Ubuntu 22.04 x86-64 artifacts + - name: Upload Windows 2019 single artifact - proxy-liteserver uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-binaries-ubuntu-22.04.zip - asset_name: ton-ubuntu-22.04-x86-64.zip - tag: v${{ steps.date.outputs.date }} + file: artifacts/ton-x86-64-windows/proxy-liteserver.exe + asset_name: proxy-liteserver.exe + tag: ${{ steps.tag.outputs.TAG }} - - name: Upload Ubuntu 18.04 arm64 artifacts + - name: Upload Windows 2019 single artifact - rldp-http-proxy uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-ubuntu-18.04-arm64.zip - asset_name: ton-ubuntu-18.04-arm64.zip - tag: v${{ steps.date.outputs.date }} + file: artifacts/ton-x86-64-windows/rldp-http-proxy.exe + asset_name: rldp-http-proxy.exe + tag: ${{ steps.tag.outputs.TAG }} - - name: Upload Ubuntu 20.04 arm64 artifacts + - name: Upload Windows 2019 single artifact - http-proxy uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-ubuntu-20.04-arm64.zip - asset_name: ton-ubuntu-20.04-arm64.zip - tag: v${{ steps.date.outputs.date }} + file: artifacts/ton-x86-64-windows/http-proxy.exe + asset_name: http-proxy.exe + tag: ${{ steps.tag.outputs.TAG }} - - name: Upload Ubuntu 22.04 arm64 artifacts + - name: Upload Windows 2019 single artifact - storage-daemon-cli uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-ubuntu-22.04-arm64.zip - asset_name: ton-ubuntu-22.04-arm64.zip - tag: v${{ steps.date.outputs.date }} \ No newline at end of file + file: artifacts/ton-x86-64-windows/storage-daemon-cli.exe + asset_name: storage-daemon-cli.exe + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Windows 2019 single artifact - storage-daemon + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86-64-windows/storage-daemon.exe + asset_name: storage-daemon.exe + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Windows 2019 single artifact - tonlibjson + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86-64-windows/tonlibjson.dll + asset_name: tonlibjson.dll + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Windows 2019 single artifact - libemulator + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86-64-windows/emulator.dll + asset_name: libemulator.dll + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Windows 2019 single artifact - tonlib-cli + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86-64-windows/tonlib-cli.exe + asset_name: tonlib-cli.exe + tag: ${{ steps.tag.outputs.TAG }} + + # mac x86-64 + + - name: Upload Mac x86-64 artifacts + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-macos.zip + asset_name: ton-mac-x86-64.zip + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac x86-64 single artifact - fift + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-macos/fift + asset_name: fift-mac-x86-64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac x86-64 single artifact - func + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-macos/func + asset_name: func-mac-x86-64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac x86-64 single artifact - tolk + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-macos/tolk + asset_name: tolk-mac-x86-64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac x86-64 single artifact - lite-client + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-macos/lite-client + asset_name: lite-client-mac-x86-64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac x86-64 single artifact - proxy-liteserver + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-macos/proxy-liteserver + asset_name: proxy-liteserver-mac-x86-64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac x86-64 single artifact - rldp-http-proxy + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-macos/rldp-http-proxy + asset_name: rldp-http-proxy-mac-x86-64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac x86-64 single artifact - http-proxy + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-macos/http-proxy + asset_name: http-proxy-mac-x86-64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac x86-64 single artifact - storage-daemon-cli + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-macos/storage-daemon-cli + asset_name: storage-daemon-cli-mac-x86-64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac x86-64 single artifact - storage-daemon + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-macos/storage-daemon + asset_name: storage-daemon-mac-x86-64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac x86-64 single artifact - tonlibjson + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-macos/libtonlibjson.dylib + asset_name: tonlibjson-mac-x86-64.dylib + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac x86-64 single artifact - libemulator + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-macos/libemulator.dylib + asset_name: libemulator-mac-x86-64.dylib + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac x86-64 single artifact - tonlib-cli + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-macos/tonlib-cli + asset_name: tonlib-cli-mac-x86-64 + tag: ${{ steps.tag.outputs.TAG }} + + + # mac arm64 + + - name: Upload Mac arm64 artifacts + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos.zip + asset_name: ton-mac-arm64.zip + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - fift + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos/fift + asset_name: fift-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - func + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos/func + asset_name: func-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - tolk + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos/tolk + asset_name: tolk-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - lite-client + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos/lite-client + asset_name: lite-client-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - proxy-liteserver + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos/proxy-liteserver + asset_name: proxy-liteserver-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - rldp-http-proxy + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos/rldp-http-proxy + asset_name: rldp-http-proxy-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - http-proxy + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos/http-proxy + asset_name: http-proxy-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - storage-daemon-cli + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos/storage-daemon-cli + asset_name: storage-daemon-cli-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - storage-daemon + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos/storage-daemon + asset_name: storage-daemon-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - tonlibjson + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos/libtonlibjson.dylib + asset_name: tonlibjson-mac-arm64.dylib + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - libemulator + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos/libemulator.dylib + asset_name: libemulator-mac-arm64.dylib + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - tonlib-cli + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos/tonlib-cli + asset_name: tonlib-cli-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + # linux x86-64 + + - name: Upload Linux x86-64 artifacts + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-linux.zip + asset_name: ton-linux-x86_64.zip + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload generic smartcont+lib artifact + run: | + mkdir smartcont_lib + cd smartcont_lib + cp -r ../artifacts/ton-x86_64-linux/{smartcont,lib} . + zip -r smartcont_lib.zip . + gh release upload ${{ steps.tag.outputs.TAG }} smartcont_lib.zip + + - name: Upload Linux x86-64 single artifact - fift + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-linux/fift + asset_name: fift-linux-x86_64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux x86-64 single artifact - func + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-linux/func + asset_name: func-linux-x86_64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux x86-64 single artifact - tolk + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-linux/tolk + asset_name: tolk-linux-x86_64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux x86-64 single artifact - lite-client + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-linux/lite-client + asset_name: lite-client-linux-x86_64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux x86-64 single artifact - proxy-liteserver + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-linux/proxy-liteserver + asset_name: proxy-liteserver-linux-x86_64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux x86-64 single artifact - rldp-http-proxy + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-linux/rldp-http-proxy + asset_name: rldp-http-proxy-linux-x86_64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux x86-64 single artifact - http-proxy + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-linux/http-proxy + asset_name: http-proxy-linux-x86_64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux x86-64 single artifact - storage-daemon-cli + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-linux/storage-daemon-cli + asset_name: storage-daemon-cli-linux-x86_64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux x86-64 single artifact - storage-daemon + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-linux/storage-daemon + asset_name: storage-daemon-linux-x86_64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux x86-64 single artifact - tonlibjson + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-linux/libtonlibjson.so + asset_name: tonlibjson-linux-x86_64.so + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux x86-64 single artifact - libemulator + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-linux/libemulator.so + asset_name: libemulator-linux-x86_64.so + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux x86-64 single artifact - tonlib-cli + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-linux/tonlib-cli + asset_name: tonlib-cli-linux-x86_64 + tag: ${{ steps.tag.outputs.TAG }} + + + # linux arm64 + + - name: Upload Linux arm64 artifacts + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux.zip + asset_name: ton-linux-arm64.zip + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - fift + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/fift + asset_name: fift-linux-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - func + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/func + asset_name: func-linux-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - tolk + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/tolk + asset_name: tolk-linux-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - lite-client + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/lite-client + asset_name: lite-client-linux-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - proxy-liteserver + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/proxy-liteserver + asset_name: proxy-liteserver-linux-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - rldp-http-proxy + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/rldp-http-proxy + asset_name: rldp-http-proxy-linux-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - http-proxy + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/http-proxy + asset_name: http-proxy-linux-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - storage-daemon-cli + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/storage-daemon-cli + asset_name: storage-daemon-cli-linux-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - storage-daemon + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/storage-daemon + asset_name: storage-daemon-linux-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - tonlibjson + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/libtonlibjson.so + asset_name: tonlibjson-linux-arm64.so + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - libemulator + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/libemulator.so + asset_name: libemulator-linux-arm64.so + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - tonlib-cli + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/tonlib-cli + asset_name: tonlib-cli-linux-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + + - name: Upload WASM artifacts + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-wasm.zip + asset_name: ton-wasm.zip + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Android Tonlib artifacts + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-android-tonlib.zip + asset_name: ton-android-tonlib.zip + tag: ${{ steps.tag.outputs.TAG }} diff --git a/.github/workflows/create-tolk-release.yml b/.github/workflows/create-tolk-release.yml new file mode 100644 index 00000000..fb8438a1 --- /dev/null +++ b/.github/workflows/create-tolk-release.yml @@ -0,0 +1,154 @@ +name: Create tolk release + +on: + workflow_dispatch: + inputs: + tag: + description: 'tolk release and tag name' + required: true + +permissions: write-all + +jobs: + create-release: + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v3 + + - name: Download and unzip Linux arm64 artifacts + uses: dawidd6/action-download-artifact@v6 + with: + workflow: build-ton-linux-arm64-appimage.yml + path: artifacts + workflow_conclusion: success + branch: master + skip_unpack: false + + - name: Download and unzip Linux x86-64 artifacts + uses: dawidd6/action-download-artifact@v6 + with: + workflow: build-ton-linux-x86-64-appimage.yml + path: artifacts + workflow_conclusion: success + branch: master + skip_unpack: false + + - name: Download and unzip Mac x86-64 artifacts + uses: dawidd6/action-download-artifact@v6 + with: + workflow: build-ton-macos-13-x86-64-portable.yml + path: artifacts + workflow_conclusion: success + branch: master + skip_unpack: false + + - name: Download and unzip arm64 artifacts + uses: dawidd6/action-download-artifact@v6 + with: + workflow: build-ton-macos-14-arm64-portable.yml + path: artifacts + workflow_conclusion: success + branch: master + skip_unpack: false + + - name: Download and unzip Windows artifacts + uses: dawidd6/action-download-artifact@v6 + with: + workflow: ton-x86-64-windows.yml + path: artifacts + workflow_conclusion: success + branch: master + skip_unpack: false + + - name: Download WASM artifacts + uses: dawidd6/action-download-artifact@v6 + with: + workflow: build-ton-wasm-emscripten.yml + path: artifacts + workflow_conclusion: success + branch: master + skip_unpack: true + + - name: Show all artifacts + run: | + tree artifacts + + + # create release + - name: Get registration token + id: getRegToken + run: | + curl -X POST -H \"Accept: application/vnd.github+json\" -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' https://api.github.com/repos/ton-blockchain/ton/actions/runners/registration-token + + - name: Create release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ inputs.tag }} + release_name: ${{ inputs.tag }} + draft: false + prerelease: false + + # upload + + # win + + - name: Upload Windows 2019 single artifact - tolk + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86-64-windows/tolk.exe + asset_name: tolk.exe + tag: ${{ inputs.tag }} + + # mac x86-64 + + - name: Upload Mac x86-64 single artifact - tolk + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-macos/tolk + asset_name: tolk-mac-x86-64 + tag: ${{ inputs.tag }} + + # mac arm64 + + - name: Upload Mac arm64 single artifact - tolk + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos/tolk + asset_name: tolk-mac-arm64 + tag: ${{ inputs.tag }} + + # linux x86-64 + + - name: Upload Linux x86-64 single artifact - tolk + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-linux/tolk + asset_name: tolk-linux-x86_64 + tag: ${{ inputs.tag }} + + # linux arm64 + + - name: Upload Linux arm64 single artifact - tolk + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/tolk + asset_name: tolk-linux-arm64 + tag: ${{ inputs.tag }} + + - name: Upload WASM artifacts + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-wasm.zip + asset_name: ton-wasm.zip + tag: ${{ inputs.tag }} + diff --git a/.github/workflows/docker-compile-ubuntu.yml b/.github/workflows/docker-compile-ubuntu.yml deleted file mode 100644 index 08caab23..00000000 --- a/.github/workflows/docker-compile-ubuntu.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Docker Ubuntu Compile arm64 - -on: [push,workflow_dispatch,workflow_call] - -jobs: - build: - strategy: - fail-fast: false - max-parallel: 3 - matrix: - arch: [arm64] - ver: [22.04, 18.04, 20.04 ] - - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Set output - id: vars - run: echo ::set-output name=short_ref::${GITHUB_REF#refs/*/} - - - name: Check output - run: echo branch ${{ steps.vars.outputs.short_ref }} - - - name: Build with docker buildx - run: | - mkdir build-${{matrix.ver}}-${{matrix.arch}} - - docker buildx build --build-arg BRANCH=${{ steps.vars.outputs.short_ref }} --platform=linux/${{matrix.arch}} --progress=plain --load . -t build-${{matrix.ver}}-${{matrix.arch}} -f .github/script/${{matrix.arch}}-${{matrix.ver}}.Dockerfile - container_id=$(docker create --platform=linux/${{matrix.arch}} build-${{matrix.ver}}-${{matrix.arch}}) - docker cp $container_id:/ton/build/dht-server/dht-server build-${{matrix.ver}}-${{matrix.arch}}/ - docker cp -a $container_id:/ton/build/validator-engine/validator-engine build-${{matrix.ver}}-${{matrix.arch}}/ - docker cp -a $container_id:/ton/build/validator-engine-console/validator-engine-console build-${{matrix.ver}}-${{matrix.arch}}/ - docker cp -a $container_id:/ton/build/storage/storage-daemon/storage-daemon build-${{matrix.ver}}-${{matrix.arch}}/ - docker cp -a $container_id:/ton/build/storage/storage-daemon/storage-daemon-cli build-${{matrix.ver}}-${{matrix.arch}}/ - docker cp -a $container_id:/ton/build/crypto/fift build-${{matrix.ver}}-${{matrix.arch}}/ - docker cp -a $container_id:/ton/build/crypto/func build-${{matrix.ver}}-${{matrix.arch}}/ - docker cp -a $container_id:/ton/build/crypto/create-state build-${{matrix.ver}}-${{matrix.arch}}/ - docker cp -a $container_id:/ton/build/blockchain-explorer/blockchain-explorer build-${{matrix.ver}}-${{matrix.arch}}/ - docker cp -a $container_id:/ton/build/lite-client/lite-client build-${{matrix.ver}}-${{matrix.arch}}/ - docker cp -a $container_id:/ton/build/utils/generate-random-id build-${{matrix.ver}}-${{matrix.arch}}/ - docker cp -a $container_id:/ton/build/tonlib/libtonlibjson.so.0.5 build-${{matrix.ver}}-${{matrix.arch}}/tonlibjson.so - docker cp -a $container_id:/ton/crypto/smartcont build-${{matrix.ver}}-${{matrix.arch}}/ - docker cp -a $container_id:/ton/crypto/fift/lib build-${{matrix.ver}}-${{matrix.arch}}/ - - - name: Upload artifacts - uses: actions/upload-artifact@v1 - with: - name: ton-ubuntu-${{matrix.ver}}-${{matrix.arch}} - path: build-${{matrix.ver}}-${{matrix.arch}} diff --git a/.github/workflows/docker-ubuntu-branch-image.yml b/.github/workflows/docker-ubuntu-branch-image.yml new file mode 100644 index 00000000..00aa5015 --- /dev/null +++ b/.github/workflows/docker-ubuntu-branch-image.yml @@ -0,0 +1,61 @@ +name: Docker Ubuntu 22.04 branch image + +on: + workflow_dispatch: + push: + branches-ignore: + - master + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push: + runs-on: ubuntu-22.04 + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.5.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.10.0 + with: + driver-opts: image=moby/buildkit:v0.11.0 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and export to Docker + uses: docker/build-push-action@v6 + with: + load: true + context: ./ + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test + + - name: Test + run: | + docker run --rm -e "TEST=1" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test + + - name: Get tag as branch name + id: tag + run: | + echo "TAG=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT + + - name: Build and push + id: docker_build + uses: docker/build-push-action@v6 + with: + platforms: linux/amd64,linux/arm64 + push: true + context: ./ + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }} diff --git a/.github/workflows/docker-ubuntu-image.yml b/.github/workflows/docker-ubuntu-image.yml index ca754078..aa4eaeef 100644 --- a/.github/workflows/docker-ubuntu-image.yml +++ b/.github/workflows/docker-ubuntu-image.yml @@ -1,4 +1,4 @@ -name: Docker Ubuntu 20.04 image +name: Docker Ubuntu 22.04 image on: workflow_dispatch: @@ -12,28 +12,57 @@ env: jobs: build-and-push: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - - name: Checkout + - name: Check out repository uses: actions/checkout@v3 + with: + submodules: 'recursive' - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - + uses: docker/setup-qemu-action@v3.5.0 + - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3.10.0 - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and export to Docker + uses: docker/build-push-action@v6 + with: + load: true + context: ./ + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test + + - name: Test + run: | + docker run --rm -e "TEST=1" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test + + - name: Get next tag + id: tag + run: | + git fetch --all --tags + git tag -l + NEW_TAG=v$(date +'%Y.%m') + FOUND=$(git tag -l | grep $NEW_TAG | wc -l) + if [ $FOUND -eq 0 ]; then + echo "TAG=$NEW_TAG" >> $GITHUB_OUTPUT + else + echo "TAG=$NEW_TAG-$FOUND" >> $GITHUB_OUTPUT + fi + - name: Build and push id: docker_build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v6 with: + platforms: linux/amd64,linux/arm64 push: true - context: ./docker - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + context: ./ + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }} diff --git a/.github/workflows/macos-11.7-compile.yml b/.github/workflows/macos-11.7-compile.yml deleted file mode 100644 index 80a57e22..00000000 --- a/.github/workflows/macos-11.7-compile.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: MacOS 11.7 Big Sur x86-64 Compile - -on: [push,workflow_dispatch,workflow_call] - -jobs: - build: - - runs-on: macos-11 - - steps: - - name: Check out repository - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - name: Compile OpenSSL - run: | - git clone https://github.com/openssl/openssl openssl_1_1_1 - cd openssl_1_1_1 - git checkout OpenSSL_1_1_1-stable - ./Configure --prefix=/usr/local/macos darwin64-x86_64-cc -static -mmacosx-version-min=11.7 - make build_libs -j4 - - - name: Build all - run: | - export NONINTERACTIVE=1 - brew install ninja - rootPath=`pwd` - mkdir build - cd build - cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=11.7 -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_BUILD_TYPE=Release .. - ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc - - - name: Find & copy binaries - run: | - mkdir artifacts - cp build/storage/storage-daemon/storage-daemon artifacts/ - cp build/storage/storage-daemon/storage-daemon-cli artifacts/ - cp build/crypto/fift artifacts/ - cp build/crypto/func artifacts/ - cp build/crypto/create-state artifacts/ - cp build/crypto/tlbc artifacts/ - cp build/validator-engine-console/validator-engine-console artifacts/ - cp build/tonlib/tonlib-cli artifacts/ - cp build/tonlib/libtonlibjson.0.5.dylib artifacts/ - cp build/http/http-proxy artifacts/ - cp build/rldp-http-proxy/rldp-http-proxy artifacts/ - cp build/dht-server/dht-server artifacts/ - cp build/lite-client/lite-client artifacts/ - cp build/validator-engine/validator-engine artifacts/ - cp build/utils/generate-random-id artifacts/ - cp build/utils/json2tlo artifacts/ - cp build/adnl/adnl-proxy artifacts/ - rsync -r crypto/smartcont artifacts/ - rsync -r crypto/fift/lib artifacts/ - ls -laRt artifacts - - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-macos-11.7 - path: artifacts diff --git a/.github/workflows/macos-12.6-compile.yml b/.github/workflows/macos-12.6-compile.yml deleted file mode 100644 index 2656de0a..00000000 --- a/.github/workflows/macos-12.6-compile.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: MacOS 12.6 Monterey x86-64 Compile - -on: [push,workflow_dispatch,workflow_call] - -jobs: - build: - - runs-on: macos-12 - - steps: - - name: Check out repository - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - name: Compile OpenSSL - run: | - git clone https://github.com/openssl/openssl openssl_1_1_1 - cd openssl_1_1_1 - git checkout OpenSSL_1_1_1-stable - ./Configure --prefix=/usr/local/macos darwin64-x86_64-cc -static -mmacosx-version-min=12.6 - make build_libs -j4 - - - name: Build all - run: | - export NONINTERACTIVE=1 - brew install ninja - rootPath=`pwd` - mkdir build - cd build - cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=12.6 -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_BUILD_TYPE=Release .. - ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc - - - name: Find & copy binaries - run: | - mkdir artifacts - cp build/storage/storage-daemon/storage-daemon artifacts/ - cp build/storage/storage-daemon/storage-daemon-cli artifacts/ - cp build/crypto/fift artifacts/ - cp build/crypto/func artifacts/ - cp build/crypto/create-state artifacts/ - cp build/crypto/tlbc artifacts/ - cp build/validator-engine-console/validator-engine-console artifacts/ - cp build/tonlib/tonlib-cli artifacts/ - cp build/tonlib/libtonlibjson.0.5.dylib artifacts/ - cp build/http/http-proxy artifacts/ - cp build/rldp-http-proxy/rldp-http-proxy artifacts/ - cp build/dht-server/dht-server artifacts/ - cp build/lite-client/lite-client artifacts/ - cp build/validator-engine/validator-engine artifacts/ - cp build/utils/generate-random-id artifacts/ - cp build/utils/json2tlo artifacts/ - cp build/adnl/adnl-proxy artifacts/ - rsync -r crypto/smartcont artifacts/ - rsync -r crypto/fift/lib artifacts/ - ls -laRt artifacts - - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-macos-12.6 - path: artifacts diff --git a/.github/workflows/ubuntu-18.04-ton-ccpcheck.yml b/.github/workflows/ton-ccpcheck.yml similarity index 81% rename from .github/workflows/ubuntu-18.04-ton-ccpcheck.yml rename to .github/workflows/ton-ccpcheck.yml index f440d7a5..95bef5f3 100644 --- a/.github/workflows/ubuntu-18.04-ton-ccpcheck.yml +++ b/.github/workflows/ton-ccpcheck.yml @@ -1,11 +1,10 @@ -name: TON ccpcheck +name: TON Static Code Analysis on: [push,workflow_dispatch,workflow_call] jobs: build: - - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 steps: - name: Check out repository @@ -21,7 +20,7 @@ jobs: generate report: true - name: Upload report - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@master with: name: ton-ccpcheck-report path: output diff --git a/.github/workflows/ton-x86-64-windows.yml b/.github/workflows/ton-x86-64-windows.yml new file mode 100644 index 00000000..baaad778 --- /dev/null +++ b/.github/workflows/ton-x86-64-windows.yml @@ -0,0 +1,36 @@ +name: Windows TON build (portable, x86-64) + +on: [push,workflow_dispatch,workflow_call] + +defaults: + run: + shell: cmd + +jobs: + build: + + runs-on: windows-2019 + + steps: + - name: Get Current OS version + run: | + systeminfo | findstr /B /C:"OS Name" /C:"OS Version" + + - name: Check out current repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Build TON + run: | + git submodule sync --recursive + git submodule update + copy assembly\native\build-windows-github-2019.bat . + copy assembly\native\build-windows-2019.bat . + build-windows-github-2019.bat Enterprise + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-x86-64-windows + path: artifacts diff --git a/.github/workflows/tonlib-android-jni.yml b/.github/workflows/tonlib-android-jni.yml deleted file mode 100644 index cdc16841..00000000 --- a/.github/workflows/tonlib-android-jni.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Tonlib Android JNI - -on: [push,workflow_dispatch,workflow_call] - -jobs: - build: - - runs-on: ubuntu-22.04 - - steps: - - name: Check out repository - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - - name: Install libraries - run: | - sudo apt update - sudo apt install -y build-essential git make cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev ninja-build - - - name: Configure & Build - run: | - wget https://dl.google.com/android/repository/android-ndk-r25b-linux.zip - unzip android-ndk-r25b-linux.zip - export JAVA_AWT_LIBRARY=NotNeeded - export JAVA_JVM_LIBRARY=NotNeeded - export JAVA_INCLUDE_PATH=${JAVA_HOME}/include - export JAVA_AWT_INCLUDE_PATH=${JAVA_HOME}/include - export JAVA_INCLUDE_PATH2=${JAVA_HOME}/include/linux - - export ANDROID_NDK_ROOT=$(pwd)/android-ndk-r25b - export OPENSSL_DIR=$(pwd)/example/android/third_party/crypto - - rm -rf example/android/src/drinkless/org/ton/TonApi.java - cd example/android/ - cmake -GNinja -DTON_ONLY_TONLIB=ON . - ninja prepare_cross_compiling - rm CMakeCache.txt - ./build-all.sh - ../../android-ndk-r25b/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip libs/x86/libnative-lib.so - ../../android-ndk-r25b/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip libs/x86_64/libnative-lib.so - ../../android-ndk-r25b/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip libs/armeabi-v7a/libnative-lib.so - ../../android-ndk-r25b/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip libs/arm64-v8a/libnative-lib.so - find . -name "*.debug" -type f -delete - - - name: Find & copy binaries - run: | - mkdir -p artifacts/tonlib-android-jni - cp example/android/src/drinkless/org/ton/TonApi.java artifacts/tonlib-android-jni/ - cp -R example/android/libs/* artifacts/tonlib-android-jni/ - - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: Tonlib JNI libraries for Android - path: artifacts \ No newline at end of file diff --git a/.github/workflows/ubuntu-18.04-compile.yml b/.github/workflows/ubuntu-18.04-compile.yml deleted file mode 100644 index c661a683..00000000 --- a/.github/workflows/ubuntu-18.04-compile.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Ubuntu 18.04 Compile - -on: [push,workflow_dispatch,workflow_call] - -jobs: - build: - - runs-on: ubuntu-18.04 - - steps: - - name: Check out repository - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - - name: Install libraries - run: | - sudo apt update - sudo apt install -y build-essential git make cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev ninja-build - - - name: Show CPU flags - run: | - cat /proc/cpuinfo - - - name: Configure & Build - run: | - export CC=$(which clang) - export CXX=$(which clang++) - export CCACHE_DISABLE=1 - mkdir build - cd build - cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. - ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state - - - name: Find & copy binaries - run: | - mkdir artifacts - cp build/storage/storage-daemon/storage-daemon build/storage/storage-daemon/storage-daemon-cli build/crypto/fift build/crypto/tlbc build/crypto/func build/crypto/create-state build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli build/tonlib/libtonlibjson.so.0.5 build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy artifacts - cp -R crypto/smartcont artifacts/ - cp -R crypto/fift/lib artifacts/ - - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-ubuntu-binaries - path: artifacts diff --git a/.github/workflows/ubuntu-compile.yml b/.github/workflows/ubuntu-compile.yml deleted file mode 100644 index a91e7128..00000000 --- a/.github/workflows/ubuntu-compile.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Ubuntu Compile x86-64 - -on: [push,workflow_dispatch,workflow_call] - -jobs: - build: - strategy: - fail-fast: false - matrix: - os: [ubuntu-18.04, ubuntu-20.04, ubuntu-22.04] - runs-on: ${{ matrix.os }} - - steps: - - name: Check out repository - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - - name: Install libraries - run: | - sudo apt update - sudo apt install -y build-essential git make cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev ninja-build - - - name: Show CPU flags - run: | - cat /proc/cpuinfo - - - name: Configure & Build - run: | - export CC=$(which clang) - export CXX=$(which clang++) - export CCACHE_DISABLE=1 - mkdir build-${{ matrix.os }} - cd build-${{ matrix.os }} - cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. - ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork - - - name: Find & copy binaries - run: | - mkdir artifacts-${{ matrix.os }} - cp build-${{ matrix.os }}/storage/storage-daemon/storage-daemon build-${{ matrix.os }}/storage/storage-daemon/storage-daemon-cli build-${{ matrix.os }}/crypto/fift build-${{ matrix.os }}/crypto/tlbc build-${{ matrix.os }}/crypto/func build-${{ matrix.os }}/crypto/create-state build-${{ matrix.os }}/validator-engine-console/validator-engine-console build-${{ matrix.os }}/tonlib/tonlib-cli build-${{ matrix.os }}/tonlib/libtonlibjson.so.0.5 build-${{ matrix.os }}/http/http-proxy build-${{ matrix.os }}/rldp-http-proxy/rldp-http-proxy build-${{ matrix.os }}/dht-server/dht-server build-${{ matrix.os }}/lite-client/lite-client build-${{ matrix.os }}/validator-engine/validator-engine build-${{ matrix.os }}/utils/generate-random-id build-${{ matrix.os }}/utils/json2tlo build-${{ matrix.os }}/adnl/adnl-proxy artifacts-${{ matrix.os }} - cp -R crypto/smartcont artifacts-${{ matrix.os }} - cp -R crypto/fift/lib artifacts-${{ matrix.os }} - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-binaries-${{ matrix.os }} - path: artifacts-${{ matrix.os }} diff --git a/.github/workflows/win-2019-compile.yml b/.github/workflows/win-2019-compile.yml deleted file mode 100644 index e648dbb7..00000000 --- a/.github/workflows/win-2019-compile.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: Windows Server 2019 x64 Compile - -on: [push,workflow_dispatch,workflow_call] - -defaults: - run: - shell: cmd - -jobs: - build: - - runs-on: windows-2019 - - steps: - - name: Get Current OS version - run: | - systeminfo | findstr /B /C:"OS Name" /C:"OS Version" - - - name: Check out current repository - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - - name: Check out zlib repository - uses: actions/checkout@v3 - with: - repository: desktop-app/zlib - path: zlib - - - name: Setup msbuild.exe - uses: microsoft/setup-msbuild@v1.0.2 - - - name: Compile zlib Win64 - run: | - cd zlib\contrib\vstudio\vc14 - msbuild zlibstat.vcxproj /p:Configuration=ReleaseWithoutAsm /p:platform=x64 -p:PlatformToolset=v142 - - - name: Install pre-compiled OpenSSL Win64 - run: | - curl -Lo openssl-1.1.1o.zip https://github.com/neodiX42/precompiled-openssl-win64/raw/main/openssl-1.1.1o.zip - jar xf openssl-1.1.1o.zip - - - name: Install pre-compiled libmicrohttpd Win64 - run: | - curl -Lo libmicrohttpd-latest-w32-bin.zip https://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-latest-w32-bin.zip - unzip libmicrohttpd-latest-w32-bin.zip - - - name: Install pre-compiled Readline Win64 - run: | - curl -Lo readline-5.0-1-lib.zip https://github.com/neodiX42/precompiled-openssl-win64/raw/main/readline-5.0-1-lib.zip - unzip readline-5.0-1-lib.zip - - - name: Compile - run: | - set root=%cd% - echo %root% - mkdir build - cd build - cmake -DREADLINE_INCLUDE_DIR=%root%\readline-5.0-1-lib\include\readline -DREADLINE_LIBRARY=%root%\readline-5.0-1-lib\lib\readline.lib -DZLIB_FOUND=1 -DMHD_FOUND=1 -DMHD_LIBRARY=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static\libmicrohttpd.lib -DMHD_INCLUDE_DIR=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static -DZLIB_INCLUDE_DIR=%root%\zlib -DZLIB_LIBRARY=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=%root%/openssl-1.1/x64/include -DOPENSSL_CRYPTO_LIBRARY=%root%/openssl-1.1/x64/lib/libcrypto.lib -DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj /W0" .. - cmake --build . --target storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork --config Release - - - name: Show executables - run: | - cd build - del Release\test-* - dir *.exe /a-D /S /B - dir *.dll /a-D /S /B - - - name: Check if validator-engine.exe exists - run: | - set root=%cd% - copy %root%\build\validator-engine\Release\validator-engine.exe test - - - name: Find & copy binaries - run: | - mkdir artifacts - mkdir artifacts\smartcont - mkdir artifacts\lib - - for %%I in (build\storage\storage-daemon\Release\storage-daemon.exe build\storage\storage-daemon\Release\storage-daemon-cli.exe build\crypto\Release\fift.exe build\crypto\Release\tlbc.exe build\crypto\Release\func.exe build\crypto\Release\create-state.exe build\validator-engine-console\Release\validator-engine-console.exe build\tonlib\Release\tonlib-cli.exe build\tonlib\Release\tonlibjson.dll build\http\Release\http-proxy.exe build\rldp-http-proxy\Release\rldp-http-proxy.exe build\dht-server\Release\dht-server.exe build\lite-client\Release\lite-client.exe build\validator-engine\Release\validator-engine.exe build\utils\Release\generate-random-id.exe build\utils\Release\json2tlo.exe build\adnl\Release\adnl-proxy.exe) do copy %%I artifacts\ - xcopy /e /k /h /i crypto\smartcont artifacts\smartcont - xcopy /e /k /h /i crypto\fift\lib artifacts\lib - - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-win-binaries - path: artifacts diff --git a/.gitignore b/.gitignore index 54d9ffc7..e5bb366c 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,15 @@ test/regression-tests.cache/ *.swp **/*build*/ .idea -.vscode \ No newline at end of file +.vscode +.DS_Store +dev/ +zlib/ +libsodium/ +libmicrohttpd-0.9.77-w32-bin/ +readline-5.0-1-lib/ +openssl-3.1.4/ +libsodium-1.0.18-stable-msvc.zip +libmicrohttpd-0.9.77-w32-bin.zip +openssl-3.1.4.zip +readline-5.0-1-lib.zip diff --git a/.gitmodules b/.gitmodules index e6a47e8b..325e3f4e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,10 @@ [submodule "third-party/libraptorq"] path = third-party/libraptorq url = https://github.com/ton-blockchain/libRaptorQ +[submodule "third-party/blst"] + path = third-party/blst + url = https://github.com/supranational/blst.git +[submodule "third-party/secp256k1"] + path = third-party/secp256k1 + url = https://github.com/bitcoin-core/secp256k1 + branch = v0.3.2 diff --git a/CMake/BuildBLST.cmake b/CMake/BuildBLST.cmake new file mode 100644 index 00000000..1cf2366c --- /dev/null +++ b/CMake/BuildBLST.cmake @@ -0,0 +1,30 @@ +set(BLST_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third-party/blst) +set(BLST_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/blst) +set(BLST_INCLUDE_DIR ${BLST_SOURCE_DIR}/bindings) + +if (NOT BLST_LIB) + if (WIN32) + set(BLST_LIB ${BLST_BINARY_DIR}/blst.lib) + set(BLST_BUILD_COMMAND ${BLST_SOURCE_DIR}/build.bat) + else() + set(BLST_LIB ${BLST_BINARY_DIR}/libblst.a) + if (PORTABLE) + set(BLST_BUILD_COMMAND ${BLST_SOURCE_DIR}/build.sh -D__BLST_PORTABLE__) + else() + set(BLST_BUILD_COMMAND ${BLST_SOURCE_DIR}/build.sh) + endif() + endif() + + file(MAKE_DIRECTORY ${BLST_BINARY_DIR}) + add_custom_command( + WORKING_DIRECTORY ${BLST_BINARY_DIR} + COMMAND ${BLST_BUILD_COMMAND} + COMMENT "Build blst" + DEPENDS ${BLST_SOURCE_DIR} + OUTPUT ${BLST_LIB} + ) +else() + message(STATUS "Use BLST: ${BLST_LIB}") +endif() + +add_custom_target(blst DEPENDS ${BLST_LIB}) diff --git a/CMake/BuildSECP256K1.cmake b/CMake/BuildSECP256K1.cmake new file mode 100644 index 00000000..f8b3c8ca --- /dev/null +++ b/CMake/BuildSECP256K1.cmake @@ -0,0 +1,55 @@ +if (NOT SECP256K1_LIBRARY) + + set(SECP256K1_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third-party/secp256k1) + set(SECP256K1_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/secp256k1) + set(SECP256K1_INCLUDE_DIR ${SECP256K1_BINARY_DIR}/include) + + file(MAKE_DIRECTORY ${SECP256K1_BINARY_DIR}) + file(MAKE_DIRECTORY "${SECP256K1_BINARY_DIR}/include") + + if (MSVC) + set(SECP256K1_BINARY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third-party/secp256k1) + set(SECP256K1_LIBRARY ${SECP256K1_SOURCE_DIR}/build/src/Release/libsecp256k1.lib) + set(SECP256K1_INCLUDE_DIR ${SECP256K1_BINARY_DIR}/include) + add_custom_command( + WORKING_DIRECTORY ${SECP256K1_SOURCE_DIR} + COMMAND cmake -E env CFLAGS="/WX" cmake -A x64 -B build -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DSECP256K1_ENABLE_MODULE_EXTRAKEYS=ON -DSECP256K1_BUILD_EXAMPLES=OFF -DBUILD_SHARED_LIBS=OFF + COMMAND cmake --build build --config Release + COMMENT "Build Secp256k1" + DEPENDS ${SECP256K1_SOURCE_DIR} + OUTPUT ${SECP256K1_LIBRARY} + ) + elseif (EMSCRIPTEN) + set(SECP256K1_BINARY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third-party/secp256k1) + set(SECP256K1_LIBRARY ${SECP256K1_BINARY_DIR}/.libs/libsecp256k1.a) + set(SECP256K1_INCLUDE_DIR ${SECP256K1_SOURCE_DIR}/include) + add_custom_command( + WORKING_DIRECTORY ${SECP256K1_SOURCE_DIR} + COMMAND ./autogen.sh + COMMAND emconfigure ./configure --enable-module-recovery --enable-module-extrakeys --disable-tests --disable-benchmark + COMMAND emmake make clean + COMMAND emmake make + COMMENT "Build Secp256k1 with emscripten" + DEPENDS ${SECP256K1_SOURCE_DIR} + OUTPUT ${SECP256K1_LIBRARY} + ) + else() + if (NOT NIX) + set(SECP256K1_LIBRARY ${SECP256K1_BINARY_DIR}/lib/libsecp256k1.a) + add_custom_command( + WORKING_DIRECTORY ${SECP256K1_SOURCE_DIR} + COMMAND ./autogen.sh + COMMAND ./configure -q --disable-option-checking --enable-module-recovery --enable-module-extrakeys --prefix ${SECP256K1_BINARY_DIR} --with-pic --disable-shared --enable-static --disable-tests --disable-benchmark + COMMAND make -j16 + COMMAND make install + COMMENT "Build secp256k1" + DEPENDS ${SECP256K1_SOURCE_DIR} + OUTPUT ${SECP256K1_LIBRARY} + ) + endif() + endif() +else() + message(STATUS "Use Secp256k1: ${SECP256K1_LIBRARY}") +endif() + +add_custom_target(secp256k1 DEPENDS ${SECP256K1_LIBRARY}) diff --git a/CMake/FindMHD.cmake b/CMake/FindMHD.cmake index 822714a2..7d6dd5fd 100644 --- a/CMake/FindMHD.cmake +++ b/CMake/FindMHD.cmake @@ -2,37 +2,27 @@ # Once done this will define # # MHD_FOUND - system has MHD -# MHD_INCLUDE_DIRS - the MHD include directory +# MHD_INCLUDE_DIR - the MHD include directory # MHD_LIBRARY - Link these to use MHD -find_path( - MHD_INCLUDE_DIR - NAMES microhttpd.h - DOC "microhttpd include dir" -) - -find_library( - MHD_LIBRARY - NAMES microhttpd microhttpd-10 libmicrohttpd libmicrohttpd-dll - DOC "microhttpd library" -) - -set(MHD_INCLUDE_DIRS ${MHD_INCLUDE_DIR}) -set(MHD_LIBRARIES ${MHD_LIBRARY}) - -# debug library on windows -# same naming convention as in qt (appending debug library with d) -# boost is using the same "hack" as us with "optimized" and "debug" -# official MHD project actually uses _d suffix -if (MSVC) - find_library( - MHD_LIBRARY_DEBUG - NAMES microhttpd_d microhttpd-10_d libmicrohttpd_d libmicrohttpd-dll_d - DOC "mhd debug library" +if (NOT MHD_LIBRARY) + find_path( + MHD_INCLUDE_DIR + NAMES microhttpd.h + DOC "microhttpd include dir" ) - set(MHD_LIBRARIES optimized ${MHD_LIBRARIES} debug ${MHD_LIBRARY_DEBUG}) + + find_library( + MHD_LIBRARY + NAMES microhttpd microhttpd-10 libmicrohttpd libmicrohttpd-dll + DOC "microhttpd library" + ) +endif() + +if (MHD_LIBRARY) + message(STATUS "Found MHD: ${MHD_LIBRARY}") endif() include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(mhd DEFAULT_MSG MHD_INCLUDE_DIR MHD_LIBRARY) +find_package_handle_standard_args(MHD DEFAULT_MSG MHD_INCLUDE_DIR MHD_LIBRARY) mark_as_advanced(MHD_INCLUDE_DIR MHD_LIBRARY) diff --git a/CMake/FindSecp256k1.cmake b/CMake/FindSecp256k1.cmake new file mode 100644 index 00000000..1f776796 --- /dev/null +++ b/CMake/FindSecp256k1.cmake @@ -0,0 +1,27 @@ +# - Try to find Secp256k1 +# Once done this will define +# +# SECP256K1_INCLUDE_DIR - the Secp256k1 include directory +# SECP256K1_LIBRARY - Link these to use Secp256k1 + +if (NOT SECP256K1_LIBRARY) + find_path( + SECP256K1_INCLUDE_DIR + NAMES secp256k1_recovery.h + DOC "secp256k1_recovery.h include dir" + ) + + find_library( + SECP256K1_LIBRARY + NAMES secp256k1 libsecp256k1 + DOC "secp256k1 library" + ) +endif() + +if (SECP256K1_LIBRARY) + message(STATUS "Found Secp256k1: ${SECP256K1_LIBRARY}") +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Secp256k1 DEFAULT_MSG SECP256K1_INCLUDE_DIR SECP256K1_LIBRARY) +mark_as_advanced(SECP256K1_INCLUDE_DIR SECP256K1_LIBRARY) diff --git a/CMake/FindSodium.cmake b/CMake/FindSodium.cmake new file mode 100644 index 00000000..3818ef89 --- /dev/null +++ b/CMake/FindSodium.cmake @@ -0,0 +1,306 @@ +# Written in 2016 by Henrik Steffen Gaßmann +# +# To the extent possible under law, the author(s) have dedicated all +# copyright and related and neighboring rights to this software to the +# public domain worldwide. This software is distributed without any warranty. +# +# You should have received a copy of the CC0 Public Domain Dedication +# along with this software. If not, see +# +# http://creativecommons.org/publicdomain/zero/1.0/ +# +######################################################################## +# Tries to find the local libsodium installation. +# +# On Windows the SODIUM_DIR environment variable is used as a default +# hint which can be overridden by setting the corresponding cmake variable. +# +# Once done the following variables will be defined: +# +# SODIUM_FOUND +# SODIUM_INCLUDE_DIR +# SODIUM_LIBRARY_DEBUG +# SODIUM_LIBRARY_RELEASE +# +# +# Furthermore an imported "sodium" target is created. +# + + +if (CMAKE_C_COMPILER_ID STREQUAL "GNU" + OR CMAKE_C_COMPILER_ID STREQUAL "Clang") + set(_GCC_COMPATIBLE 1) +endif() + +# static library option +if (NOT DEFINED SODIUM_USE_STATIC_LIBS) + option(SODIUM_USE_STATIC_LIBS "enable to statically link against sodium" OFF) +endif() +if(NOT (SODIUM_USE_STATIC_LIBS EQUAL SODIUM_USE_STATIC_LIBS_LAST)) + if (NOT SODIUM_LIBRARY_RELEASE) + unset(sodium_LIBRARY CACHE) + unset(SODIUM_LIBRARY_DEBUG CACHE) + unset(SODIUM_LIBRARY_RELEASE CACHE) + unset(sodium_DLL_DEBUG CACHE) + unset(sodium_DLL_RELEASE CACHE) + set(SODIUM_USE_STATIC_LIBS_LAST ${SODIUM_USE_STATIC_LIBS} CACHE INTERNAL "internal change tracking variable") + endif() +endif() + + +######################################################################## +# UNIX +if (UNIX) + # import pkg-config + find_package(PkgConfig QUIET) + if (PKG_CONFIG_FOUND) + pkg_check_modules(sodium_PKG QUIET libsodium) + endif() + + if(SODIUM_USE_STATIC_LIBS) + foreach(_libname ${sodium_PKG_STATIC_LIBRARIES}) + if (NOT _libname MATCHES "^lib.*\\.a$") # ignore strings already ending with .a + list(INSERT sodium_PKG_STATIC_LIBRARIES 0 "lib${_libname}.a") + endif() + endforeach() + list(REMOVE_DUPLICATES sodium_PKG_STATIC_LIBRARIES) + + # if pkgconfig for libsodium doesn't provide + # static lib info, then override PKG_STATIC here.. + if (NOT sodium_PKG_STATIC_FOUND) + set(sodium_PKG_STATIC_LIBRARIES libsodium.a) + endif() + + set(XPREFIX sodium_PKG_STATIC) + else() + if (NOT sodium_PKG_FOUND) + set(sodium_PKG_LIBRARIES sodium) + endif() + + set(XPREFIX sodium_PKG) + endif() + + find_path(SODIUM_INCLUDE_DIR sodium.h + HINTS ${${XPREFIX}_INCLUDE_DIRS} + ) + find_library(SODIUM_LIBRARY_DEBUG NAMES ${${XPREFIX}_LIBRARIES} + HINTS ${${XPREFIX}_LIBRARY_DIRS} + ) + find_library(SODIUM_LIBRARY_RELEASE NAMES ${${XPREFIX}_LIBRARIES} + HINTS ${${XPREFIX}_LIBRARY_DIRS} + ) + + ######################################################################## + # Windows +elseif (WIN32) + set(SODIUM_DIR "$ENV{SODIUM_DIR}" CACHE FILEPATH "sodium install directory") + mark_as_advanced(SODIUM_DIR) + + find_path(SODIUM_INCLUDE_DIR + NAMES sodium.h + HINTS ${SODIUM_DIR} + PATH_SUFFIXES include + ) + + if (MSVC) + # detect target architecture + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/arch.cpp" [=[ + #if defined _M_IX86 + #error ARCH_VALUE x86_32 + #elif defined _M_X64 + #error ARCH_VALUE x86_64 + #endif + #error ARCH_VALUE unknown + ]=]) + try_compile(_UNUSED_VAR "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/arch.cpp" + OUTPUT_VARIABLE _COMPILATION_LOG + ) + string(REGEX REPLACE ".*ARCH_VALUE ([a-zA-Z0-9_]+).*" "\\1" _TARGET_ARCH "${_COMPILATION_LOG}") + + # construct library path + if (_TARGET_ARCH STREQUAL "x86_32") + string(APPEND _PLATFORM_PATH "Win32") + elseif(_TARGET_ARCH STREQUAL "x86_64") + string(APPEND _PLATFORM_PATH "x64") + else() + message(FATAL_ERROR "the ${_TARGET_ARCH} architecture is not supported by Findsodium.cmake.") + endif() + string(APPEND _PLATFORM_PATH "/$$CONFIG$$") + + message(STATUS "MSVC_VERSION ${MSVC_VERSION}") + if (MSVC_VERSION LESS 1900) + math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 60") + else() + if (MSVC_VERSION EQUAL 1941) + math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 51") + else() + math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 50") + endif() + + endif() + string(APPEND _PLATFORM_PATH "/v${_VS_VERSION}") + + if (SODIUM_USE_STATIC_LIBS) + string(APPEND _PLATFORM_PATH "/static") + else() + string(APPEND _PLATFORM_PATH "/dynamic") + endif() + + string(REPLACE "$$CONFIG$$" "Debug" _DEBUG_PATH_SUFFIX "${_PLATFORM_PATH}") + string(REPLACE "$$CONFIG$$" "Release" _RELEASE_PATH_SUFFIX "${_PLATFORM_PATH}") + + find_library(SODIUM_LIBRARY_DEBUG libsodium.lib + HINTS ${SODIUM_DIR} + PATH_SUFFIXES ${_DEBUG_PATH_SUFFIX} + ) + find_library(SODIUM_LIBRARY_RELEASE libsodium.lib + HINTS ${SODIUM_DIR} + PATH_SUFFIXES ${_RELEASE_PATH_SUFFIX} + ) + if (NOT SODIUM_USE_STATIC_LIBS) + set(CMAKE_FIND_LIBRARY_SUFFIXES_BCK ${CMAKE_FIND_LIBRARY_SUFFIXES}) + set(CMAKE_FIND_LIBRARY_SUFFIXES ".dll") + find_library(sodium_DLL_DEBUG libsodium + HINTS ${SODIUM_DIR} + PATH_SUFFIXES ${_DEBUG_PATH_SUFFIX} + ) + find_library(sodium_DLL_RELEASE libsodium + HINTS ${SODIUM_DIR} + PATH_SUFFIXES ${_RELEASE_PATH_SUFFIX} + ) + set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_BCK}) + endif() + + elseif(_GCC_COMPATIBLE) + if (SODIUM_USE_STATIC_LIBS) + find_library(SODIUM_LIBRARY_DEBUG libsodium.a + HINTS ${SODIUM_DIR} + PATH_SUFFIXES lib + ) + find_library(SODIUM_LIBRARY_RELEASE libsodium.a + HINTS ${SODIUM_DIR} + PATH_SUFFIXES lib + ) + else() + find_library(SODIUM_LIBRARY_DEBUG libsodium.dll.a + HINTS ${SODIUM_DIR} + PATH_SUFFIXES lib + ) + find_library(SODIUM_LIBRARY_RELEASE libsodium.dll.a + HINTS ${SODIUM_DIR} + PATH_SUFFIXES lib + ) + + file(GLOB _DLL + LIST_DIRECTORIES false + RELATIVE "${SODIUM_DIR}/bin" + "${SODIUM_DIR}/bin/libsodium*.dll" + ) + find_library(sodium_DLL_DEBUG ${_DLL} libsodium + HINTS ${SODIUM_DIR} + PATH_SUFFIXES bin + ) + find_library(sodium_DLL_RELEASE ${_DLL} libsodium + HINTS ${SODIUM_DIR} + PATH_SUFFIXES bin + ) + endif() + else() + message(FATAL_ERROR "this platform is not supported by FindSodium.cmake") + endif() + + + ######################################################################## + # unsupported +else() + message(FATAL_ERROR "this platform is not supported by FindSodium.cmake") +endif() + + +######################################################################## +# common stuff + +# extract sodium version +if (SODIUM_INCLUDE_DIR) + set(_VERSION_HEADER "${_INCLUDE_DIR}/sodium/version.h") + if (EXISTS _VERSION_HEADER) + file(READ "${_VERSION_HEADER}" _VERSION_HEADER_CONTENT) + string(REGEX REPLACE ".*#[ \t]*define[ \t]*SODIUM_VERSION_STRING[ \t]*\"([^\n]*)\".*" "\\1" + sodium_VERSION "${_VERSION_HEADER_CONTENT}") + set(sodium_VERSION "${sodium_VERSION}" PARENT_SCOPE) + endif() +endif() + +# communicate results +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + Sodium # The name must be either uppercase or match the filename case. + REQUIRED_VARS + SODIUM_LIBRARY_RELEASE + SODIUM_LIBRARY_DEBUG + SODIUM_INCLUDE_DIR + VERSION_VAR + sodium_VERSION +) + +if(SODIUM_FOUND) + set(SODIUM_LIBRARIES + optimized ${SODIUM_LIBRARY_RELEASE} debug ${SODIUM_LIBRARY_DEBUG}) +endif() + +# mark file paths as advanced +mark_as_advanced(SODIUM_INCLUDE_DIR) +mark_as_advanced(SODIUM_LIBRARY_DEBUG) +mark_as_advanced(SODIUM_LIBRARY_RELEASE) +if (WIN32) + mark_as_advanced(sodium_DLL_DEBUG) + mark_as_advanced(sodium_DLL_RELEASE) +endif() + +# create imported target +if(SODIUM_USE_STATIC_LIBS) + set(_LIB_TYPE STATIC) +else() + set(_LIB_TYPE SHARED) +endif() + +if(NOT TARGET sodium) + add_library(sodium ${_LIB_TYPE} IMPORTED) +endif() + +set_target_properties(sodium PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${SODIUM_INCLUDE_DIR}" + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + ) + +if (SODIUM_USE_STATIC_LIBS) + set_target_properties(sodium PROPERTIES + INTERFACE_COMPILE_DEFINITIONS "SODIUM_STATIC" + IMPORTED_LOCATION "${SODIUM_LIBRARY_RELEASE}" + IMPORTED_LOCATION_DEBUG "${SODIUM_LIBRARY_DEBUG}" + ) +else() + if (UNIX) + set_target_properties(sodium PROPERTIES + IMPORTED_LOCATION "${SODIUM_LIBRARY_RELEASE}" + IMPORTED_LOCATION_DEBUG "${SODIUM_LIBRARY_DEBUG}" + ) + elseif (WIN32) + set_target_properties(sodium PROPERTIES + IMPORTED_IMPLIB "${SODIUM_LIBRARY_RELEASE}" + IMPORTED_IMPLIB_DEBUG "${SODIUM_LIBRARY_DEBUG}" + ) + if (NOT (sodium_DLL_DEBUG MATCHES ".*-NOTFOUND")) + set_target_properties(sodium PROPERTIES + IMPORTED_LOCATION_DEBUG "${sodium_DLL_DEBUG}" + ) + endif() + if (NOT (sodium_DLL_RELEASE MATCHES ".*-NOTFOUND")) + set_target_properties(sodium PROPERTIES + IMPORTED_LOCATION_RELWITHDEBINFO "${sodium_DLL_RELEASE}" + IMPORTED_LOCATION_MINSIZEREL "${sodium_DLL_RELEASE}" + IMPORTED_LOCATION_RELEASE "${sodium_DLL_RELEASE}" + ) + endif() + endif() +endif() diff --git a/CMake/FindJeMalloc.cmake b/CMake/Findjemalloc.cmake similarity index 100% rename from CMake/FindJeMalloc.cmake rename to CMake/Findjemalloc.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 1e52f969..cea3fc7e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.1 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) project(TON VERSION 0.5 LANGUAGES C CXX) set(CMAKE_POSITION_INDEPENDENT_CODE ON) @@ -79,11 +79,12 @@ else() set(HAVE_SSE42 FALSE) endif() -set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED TRUE) set(CMAKE_CXX_EXTENSIONS FALSE) #BEGIN internal +option(BUILD_SHARED_LIBS "Use \"ON\" to build shared libraries instead of static where it's not specified (not recommended)" OFF) option(USE_EMSCRIPTEN "Use \"ON\" for config building wasm." OFF) option(TON_ONLY_TONLIB "Use \"ON\" to build only tonlib." OFF) if (USE_EMSCRIPTEN) @@ -108,7 +109,8 @@ set(TON_ARCH "native" CACHE STRING "Architecture, will be passed to -march=") #BEGIN M1 support EXECUTE_PROCESS( COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE ARCHITECTURE ) -if ((ARCHITECTURE MATCHES "arm64") AND (CMAKE_SYSTEM_NAME STREQUAL "Darwin")) +if ((ARCHITECTURE MATCHES "arm64") AND (CMAKE_SYSTEM_NAME STREQUAL "Darwin") AND + (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 13.0)) # only clang 13+ supports cpu=apple-m1 set(TON_ARCH "apple-m1") endif() #END M1 support @@ -135,7 +137,16 @@ set(CRC32C_BUILD_BENCHMARKS OFF CACHE BOOL "Build CRC32C's benchmarks") set(CRC32C_USE_GLOG OFF CACHE BOOL "Build CRC32C's tests with Google Logging") set(CRC32C_INSTALL OFF CACHE BOOL "Install CRC32C's header and library") message("Add crc32c") -add_subdirectory(third-party/crc32c EXCLUDE_FROM_ALL) +if (NOT MSVC) + set(OLD_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) + # fix aarch64 build @ crc32c/src/crc32c_arm64_linux_check.h + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=address") + add_subdirectory(third-party/crc32c EXCLUDE_FROM_ALL) + set(CMAKE_CXX_FLAGS ${OLD_CMAKE_CXX_FLAGS}) + unset(OLD_CMAKE_CXX_FLAGS) +else() + add_subdirectory(third-party/crc32c EXCLUDE_FROM_ALL) +endif() set(CRC32C_FOUND 1) if (TON_USE_ROCKSDB) @@ -172,6 +183,9 @@ endif() message("Add ton") set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH}) +include(BuildBLST) +include(BuildSECP256K1) + # Configure CCache if available find_program(CCACHE_FOUND ccache) #set(CCACHE_FOUND 0) @@ -198,7 +212,14 @@ include(CheckCXXCompilerFlag) set(CMAKE_THREAD_PREFER_PTHREAD ON) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -find_package(ZLIB REQUIRED) +find_package(PkgConfig REQUIRED) + +if (NOT ZLIB_FOUND) + find_package(ZLIB REQUIRED) +else() + message(STATUS "Using zlib ${ZLIB_LIBRARIES}") +endif() + if (TON_ARCH AND NOT MSVC) CHECK_CXX_COMPILER_FLAG( "-march=${TON_ARCH}" COMPILER_OPT_ARCH_SUPPORTED ) @@ -215,9 +236,14 @@ if (THREADS_HAVE_PTHREAD_ARG) endif() if (TON_USE_JEMALLOC) - find_package(JeMalloc REQUIRED) + find_package(jemalloc REQUIRED) endif() +if (NIX) + find_package(Secp256k1 REQUIRED) +endif() + + set(MEMPROF "" CACHE STRING "Use one of \"ON\", \"FAST\" or \"SAFE\" to enable memory profiling. \ Works under macOS and Linux when compiled using glibc. \ In FAST mode stack is unwinded only using frame pointers, which may fail. \ @@ -242,6 +268,9 @@ if (MSVC) add_definitions(-D_SCL_SECURE_NO_WARNINGS -D_CRT_SECURE_NO_WARNINGS) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /MP /W4 /wd4100 /wd4127 /wd4324 /wd4456 /wd4457 /wd4458 /wd4505 /wd4702") elseif (CLANG OR GCC) + if (GCC) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fstrong-eval-order=some") + endif() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer") if (APPLE) #use "-Wl,-exported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/export_list" for exported symbols @@ -311,6 +340,10 @@ add_cxx_compiler_flag("-Wno-sign-conversion") add_cxx_compiler_flag("-Qunused-arguments") add_cxx_compiler_flag("-Wno-unused-private-field") add_cxx_compiler_flag("-Wno-redundant-move") + +#add_cxx_compiler_flag("-Wno-unused-function") +#add_cxx_compiler_flag("-Wno-unused-variable") +#add_cxx_compiler_flag("-Wno-shorten-64-to-32") #add_cxx_compiler_flag("-Werror") #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isystem /usr/include/c++/v1") @@ -355,6 +388,9 @@ if (LATEX_FOUND) add_latex_document(doc/fiftbase.tex TARGET_NAME fift_basic_description) add_latex_document(doc/catchain.tex TARGET_NAME catchain_consensus_description) endif() +if (NOT LATEX_FOUND) + message(STATUS "Could NOT find LATEX (this is NOT an error)") +endif() #END internal function(target_link_libraries_system target) @@ -383,6 +419,8 @@ add_subdirectory(tl-utils) add_subdirectory(adnl) add_subdirectory(crypto) add_subdirectory(lite-client) +add_subdirectory(emulator) +add_subdirectory(tolk) #BEGIN tonlib add_subdirectory(tonlib) @@ -433,6 +471,10 @@ target_link_libraries(test-smartcont PRIVATE smc-envelope fift-lib ton_db) add_executable(test-bigint ${BIGINT_TEST_SOURCE}) target_link_libraries(test-bigint PRIVATE ton_crypto) +if (WINGETOPT_FOUND) + target_link_libraries_system(test-bigint wingetopt) +endif() + add_executable(test-cells test/test-td-main.cpp ${CELLS_TEST_SOURCE}) target_link_libraries(test-cells PRIVATE ton_crypto) @@ -454,10 +496,10 @@ target_link_libraries(test-net PRIVATE tdnet tdutils ${CMAKE_THREAD_LIBS_INIT}) #BEGIN tonlib add_executable(test-tonlib ${TONLIB_ONLINE_TEST_SOURCE}) -target_link_libraries(test-tonlib tdutils tdactor adnllite tl_api ton_crypto ton_block tl_tonlib_api tonlib) +target_link_libraries(test-tonlib tdactor adnllite tl_api ton_crypto tl_tonlib_api tonlib) add_executable(test-tonlib-offline test/test-td-main.cpp ${TONLIB_OFFLINE_TEST_SOURCE}) -target_link_libraries(test-tonlib-offline tdutils tdactor adnllite tl_api ton_crypto ton_block fift-lib tl_tonlib_api tonlib) +target_link_libraries(test-tonlib-offline tdactor adnllite tl_api ton_crypto fift-lib tl_tonlib_api tonlib) if (NOT CMAKE_CROSSCOMPILING) add_dependencies(test-tonlib-offline gen_fif) @@ -499,30 +541,21 @@ target_link_libraries(test-rldp2 adnl adnltest dht rldp2 tl_api) add_executable(test-validator-session-state test/test-validator-session-state.cpp) target_link_libraries(test-validator-session-state adnl dht rldp validatorsession tl_api) -#add_executable(test-node test/test-node.cpp) -#target_link_libraries(test-node overlay tdutils tdactor adnl tl_api dht -# catchain validatorsession) - +add_executable(test-overlay test/test-overlay.cpp) +target_link_libraries(test-overlay overlay tdutils tdactor adnl adnltest tl_api dht ) add_executable(test-catchain test/test-catchain.cpp) target_link_libraries(test-catchain overlay tdutils tdactor adnl adnltest rldp tl_api dht catchain ) -#add_executable(test-validator-session test/test-validator-session.cpp) -#target_link_libraries(test-validator-session overlay tdutils tdactor adnl tl_api dht -# catchain validatorsession) add_executable(test-ton-collator test/test-ton-collator.cpp) target_link_libraries(test-ton-collator overlay tdutils tdactor adnl tl_api dht catchain validatorsession validator-disk ton_validator validator-disk ) -#add_executable(test-validator test/test-validator.cpp) -#target_link_libraries(test-validator overlay tdutils tdactor adnl tl_api dht -# rldp catchain validatorsession ton-node validator ton_validator validator memprof ${JEMALLOC_LIBRARIES}) -#add_executable(test-ext-server test/test-ext-server.cpp) -#target_link_libraries(test-ext-server tdutils tdactor adnl tl_api dht ) -#add_executable(test-ext-client test/test-ext-client.cpp) -#target_link_libraries(test-ext-client tdutils tdactor adnl tl_api tl-lite-utils) add_executable(test-http test/test-http.cpp) target_link_libraries(test-http PRIVATE tonhttp) +add_executable(test-emulator test/test-td-main.cpp emulator/test/emulator-tests.cpp) +target_link_libraries(test-emulator PRIVATE emulator) + get_directory_property(HAS_PARENT PARENT_DIRECTORY) if (HAS_PARENT) set(ALL_TEST_SOURCE @@ -554,19 +587,84 @@ add_test(test-cells test-cells ${TEST_OPTIONS}) add_test(test-smartcont test-smartcont) add_test(test-net test-net) add_test(test-actors test-tdactor) +add_test(test-emulator test-emulator) #BEGIN tonlib add_test(test-tdutils test-tdutils) add_test(test-tonlib-offline test-tonlib-offline) #END tonlib +# FunC tests +if (NOT NIX) + if (MSVC) + set(PYTHON_VER "python") + else() + set(PYTHON_VER "python3") + endif() + add_test( + NAME test-func + COMMAND ${PYTHON_VER} run_tests.py tests/ + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/crypto/func/auto-tests) + if (WIN32) + set_property(TEST test-func PROPERTY ENVIRONMENT + "FUNC_EXECUTABLE=${CMAKE_CURRENT_BINARY_DIR}/crypto/func.exe" + "FIFT_EXECUTABLE=${CMAKE_CURRENT_BINARY_DIR}/crypto/fift.exe" + "FIFTPATH=${CMAKE_CURRENT_SOURCE_DIR}/crypto/fift/lib/") + else() + set_property(TEST test-func PROPERTY ENVIRONMENT + "FUNC_EXECUTABLE=${CMAKE_CURRENT_BINARY_DIR}/crypto/func" + "FIFT_EXECUTABLE=${CMAKE_CURRENT_BINARY_DIR}/crypto/fift" + "FIFTPATH=${CMAKE_CURRENT_SOURCE_DIR}/crypto/fift/lib/") + endif() + + add_test( + NAME test-func-legacy + COMMAND ${PYTHON_VER} legacy_tester.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/crypto/func/auto-tests) + if (WIN32) + set_property(TEST test-func-legacy PROPERTY ENVIRONMENT + "FUNC_EXECUTABLE=${CMAKE_CURRENT_BINARY_DIR}/crypto/func.exe" + "FIFT_EXECUTABLE=${CMAKE_CURRENT_BINARY_DIR}/crypto/fift.exe" + "FIFTPATH=${CMAKE_CURRENT_SOURCE_DIR}/crypto/fift/lib/") + else() + set_property(TEST test-func-legacy PROPERTY ENVIRONMENT + "FUNC_EXECUTABLE=${CMAKE_CURRENT_BINARY_DIR}/crypto/func" + "FIFT_EXECUTABLE=${CMAKE_CURRENT_BINARY_DIR}/crypto/fift" + "FIFTPATH=${CMAKE_CURRENT_SOURCE_DIR}/crypto/fift/lib/") + endif() +endif() + +# Tolk tests +if (NOT NIX) + if (MSVC) + set(PYTHON_VER "python") + else() + set(PYTHON_VER "python3") + endif() + add_test( + NAME test-tolk + COMMAND ${PYTHON_VER} tolk-tester.py tests/ + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tolk-tester) + if (WIN32) + set_property(TEST test-tolk PROPERTY ENVIRONMENT + "TOLK_EXECUTABLE=${CMAKE_CURRENT_BINARY_DIR}/tolk/tolk.exe" + "FIFT_EXECUTABLE=${CMAKE_CURRENT_BINARY_DIR}/crypto/fift.exe" + "FIFTPATH=${CMAKE_CURRENT_SOURCE_DIR}/crypto/fift/lib/") + else() + set_property(TEST test-tolk PROPERTY ENVIRONMENT + "TOLK_EXECUTABLE=${CMAKE_CURRENT_BINARY_DIR}/tolk/tolk" + "FIFT_EXECUTABLE=${CMAKE_CURRENT_BINARY_DIR}/crypto/fift" + "FIFTPATH=${CMAKE_CURRENT_SOURCE_DIR}/crypto/fift/lib/") + endif() +endif() + #BEGIN internal if (NOT TON_ONLY_TONLIB) add_test(test-adnl test-adnl) add_test(test-dht test-dht) add_test(test-rldp test-rldp) add_test(test-rldp2 test-rldp2) -#add_test(test-validator-session-state test-validator-session-state) +add_test(test-validator-session-state test-validator-session-state) add_test(test-catchain test-catchain) add_test(test-fec test-fec) diff --git a/Changelog.md b/Changelog.md index f3c76217..4dce39fc 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,45 +1,211 @@ -## 05.2022 Update -* Initial synchronization improved: adjusted timeouts for state download and the way of choosing which state to download. Nodes with low network speed and/or bad connectivity will synchronize faster and consistently. -* Improved peer-to-peer network stability and DDoS resistance: now peers will only relay valid messages to the network. Large messages, which require splitting for relaying, will be retranslated as well, but only after the node gets all parts, and reassembles and checks them. Validators may sign certificates for network peers, which allow relaying large messages by parts without checks. It is used now by validators to faster relay new blocks. Sign and import certificate commands are exposed via `validator-engine-console`. -* Fixed some rare edge cases in TVM arithmetic operations related to big numbers (`2**63+`) -* Improved fixes used to combat wrong activate-destruct-activate contract behavior last November. -* Improved tonlib: support libraries (with client-side caching), getmethods completely fill c7 register, getmethods support slice arguments, improved messages listing for transactions, added extended block header params, added getConfig method. -* RocksDB updated to a newer version. -* Improved persistent state serialization: memory usage during serialization was optimized; the start of serialization on different nodes was sparsed. -* FunC update: support for string literals and constants (including precompiled constant expressions), semver, `include` expressions. -* Fixed rarely manifested bugs in `Asm.fif`. -* LiteClient supports key as cli parameter. -* Improved Liteserver DoS resistance for running getmethods. +## 2025.03 Update +1. New extracurrency behavior introduced, check [GlobalVersions.md](./doc/GlobalVersions.md#version-10) +2. Optmization of validation process, in particular CellStorageStat. +3. Flag for speeding up broadcasts in various overlays. +4. Fixes for static builds for emulator and tonlibjson +5. Improving getstats output: adds + * Liteserver queries count + * Collated/validated blocks count, number of active sessions + * Persistent state sizes + * Initial sync progress +6. Fixes in logging, TON Storage, external message checking, persistent state downloading, UB in tonlib -Besides the work of the core team, this update is based on the efforts of @tvorogme (added support for slice arguments and noted bugs in Asm.fif), @akifoq (fixed bug in Asm.fif), @cryshado (noted strange behavior of LS, which, upon inspection, turned out to be a vector of DoS attack). -## 08.2022 Update -* Blockchain state serialization now works via separate db-handler which simplfies memory clearing after serialization -* CellDB now works asynchronously which substantially increase database access throughput -* Abseil-cpp and crc32 updated: solve issues with compilation on recent OS distributives -* Fixed a series of UBs and issues for exotic endianness hosts -* Added detailed network stats for overlays (can be accessed via `validator-console`) -* Improved auto-builds for wide range of systems. -* Added extended error information for unaccepted external messages: `exit_code` and TVM trace (where applicable). -* [Improved catchain DoS resistance](https://github.com/ton-blockchain/ton/blob/master/doc/catchain-dos.md) -* A series of FunC improvements, summarized [here](https://github.com/ton-blockchain/ton/pull/378) -#### Update delay -Update coincided with persistent state serialization event which lead to block production speed deterioration (issue substantially mitigated in update itself). This phenomena was aggravated by the fact that after update some validators lost ability to participate in block creation. The last was caused by threshold based hardcoded protocol version bump, where threshold was set in such manner (based on block height with value higher than 9m), that it eluded detection in private net tests. The update was temporarily paused and resumed after persistent state serialization ended and issues with block creation were resolved. +Besides the work of the core team, this update is based on the efforts of @Sild from StonFi(UB in tonlib). -Besides the work of the core team, this update is based on the efforts of @awesome-doge (help with abseil-cpp upgrade), @rec00rsiff (noted issues for exotic endianess and implemented network stats) and third-party security auditors. +## 2025.02 Update +1. Series of improvement/fixes for `Config8.version >= 9`, check [GlobalVersions.md](./doc/GlobalVersions.md) +2. Fix for better discovery of updated nodes' (validators') IPs: retry dht queries +3. Series of improvements for extra currency adoption: fixed c7 in rungetmethod, reserve modes +4. TVM: Fix processing continuation control data on deep jump +5. A few fixes of tl-b schemes: crc computation, incorrect tag for merkle proofs, advance_ext, NatWidth print +6. Emulator improvements: fix setting libraries, extracurrency support +7. Increase of gas limit for unlocking highload-v2 wallets locked in the beginning of 2024 +8. Validator console improvement: dashed names, better shard formats -## 10.2022 Update -* Added extended block creation and general perfomance stats gathering -* Forbidden report data on blocks not committed to the master chain for LS -* Improved debug in TVM -* FunC 0.3.0: multi-line asms, bitwise operations for constants, duplication of identical definition for constants and asms now allowed -* New tonlib methods: sendMessageReturnHash, getTransactionsV2, getMasterchainBlockSignatures, getShardBlockProof, getLibraries. -* Fixed bugs related to invalid TVM output (c4, c5, libaries) and non-validated network data; avoided too deep recursion in libraries loading -* Fixed multiple undefined behavior issues -* Added build of FunC and Fift to WASM -Besides the work of the core team, this update is based on the efforts of @tvorogme (debug improvements), @AlexeyFSL (WASM builds) and third-party security auditors. +Besides the work of the core team, this update is based on the efforts of @dbaranovstonfi from StonFi(libraries in emulator), @Rexagon (ret on deep jumps), @tvorogme from DTon (`advance_ext`), Nan from Zellic (`stk_und` and JNI) -## 12.2022 Update +## 2024.12 Update + +1. FunC 0.4.6: Fix in try/catch handling, fixing pure flag for functions stored in variables +2. Merging parts of Accelerator: support of specific shard monitoring, archive/liteserver slice format, support for partial liteservers, proxy liteserver, on-demand neighbour queue loading +3. Fix of asynchronous cell loading +4. Various improvements: caching certificates checks, better block overloading detection, `_malloc` in emulator +5. Introduction of telemetry in overlays +6. Use non-null local-id for tonlib-LS interaction - mitigates MitM attack. +7. Adding `SECP256K1_XONLY_PUBKEY_TWEAK_ADD`, `SETCONTCTRMANY` instructions to TVM (activated by `Config8.version >= 9`) +8. Private keys export via validator-engine-console - required for better backups +9. Fix proof checking in tonlib, `hash` in `raw.Message` in tonlib_api + +Besides the work of the core team, this update is based on the efforts of OtterSec and LayerZero (FunC), tg:@throwunless (FunC), Aviv Frenkel and Dima Kogan from Fordefi (LS MitM), @hacker-volodya (Tonlib), OKX team (async cell loading), @krigga (emulator) + +## 2024.10 Update + +1. Parallel write to celldb: substantial improvement of sync and GC speed, especially with slow disks. +2. Decreased network traffic: only first block candidate is sent optimistically. +3. Improved channel creation and dht lookups, introduction of semi-private overlays +4. New LS dispatch queue related methods and improvement security +5. Fixing recursion in TVM continuations +6. Improved stats for actors, validator sessions, perf counters, overlays, adnl, rocksdb +7. Migration to C++20 +8. Improved block size estimates: account for depth in various structures +9. Fix bug with `<<` optimization in FunC +10. Minor changes of TVM which will be activated by `Config8.version >= 9` +11. Multiple minor improvements + +Besides the work of the core team, this update is based on the efforts of @krigga (emulator), Arayz @ TonBit (LS security, TVM recursion), @ret2happy (UB in BLST). + +## 2024.08 Update + +1. Introduction of dispatch queues, message envelopes with transaction chain metadata, and explicitly stored msg_queue size, which will be activated by `Config8.version >= 8` and new `Config8.capabilities` bits: `capStoreOutMsgQueueSize`, `capMsgMetadata`, `capDeferMessages`. +2. A number of changes to transaction executor which will activated for `Config8.version >= 8`: + - Check mode on invalid `action_send_msg`. Ignore action if `IGNORE_ERROR` (+2) bit is set, bounce if `BOUNCE_ON_FAIL` (+16) bit is set. + - Slightly change random seed generation to fix mix of `addr_rewrite` and `addr`. + - Fill in `skipped_actions` for both invalid and valid messages with `IGNORE_ERROR` mode that can't be sent. + - Allow unfreeze through external messages. + - Don't use user-provided `fwd_fee` and `ihr_fee` for internal messages. +3. A few issues with broadcasts were fixed: stop on receiving last piece, response to AdnlMessageCreateChannel +4. A number of fixes and improvements for emulator and tonlib: correct work with config_addr, not accepted externals, bounces, debug ops gas consumption, added version and c5 dump, fixed tonlib crashes +5. Added new flags and commands to the node, in particular `--fast-state-serializer`, `getcollatoroptionsjson`, `setcollatoroptionsjson` + +Besides the work of the core team, this update is based on the efforts of @krigga (emulator), stonfi team, in particular @dbaranovstonfi and @hey-researcher (emulator), and @loeul, @xiaoxianBoy, @simlecode (typos in comments and docs). + + + +## 2024.06 Update + +1. Make Jemalloc default allocator +2. Add candidate broadcasting and caching +3. Limit per address speed for external messages broadcast by reasonably large number +4. Overlay improvements: fix dropping peers in small custom overlays, fix wrong certificate on missed keyblocks +5. Extended statistics and logs for celldb usage, session stats, persistent state serialization +6. Tonlib and explorer fixes +7. Flags for precize control of Celldb: `--celldb-cache-size`, `--celldb-direct-io` and `--celldb-preload-all` +8. Add valiator-console command to stop persistent state serialization +9. Use `@` path separator for defining include path in fift and create-state utilities on Windows only. + + +## 2024.04 Update + +1. Emulator: Single call optimized runGetMethod added +2. Tonlib: a series of proof improvements, also breaking Change in `liteServer.getAllShardsInfo` method (see below) +3. DB: usage statistics now collected, outdated persistent states are not serialized +4. LS: fast `getOutMsgQueueSizes` added, preliminary support of non-final block requests +5. Network: lz4 compression of block candidates (disabled by default). +6. Overlays: add custom overlays +7. Transaction Executor: fixed issue with due_payment collection + +* `liteServer.getAllShardsInfo` method was updated for better efficiency. Previously, field proof contained BoC with two roots: one for BlockState from block's root and another for ShardHashes from BlockState. Now, it returns a single-root proof BoC, specifically the merkle proof of ShardHashes directly from the block's root, streamlining data access and integrity. Checking of the proof requires to check that ShardHashes in the `data` correspond to ShardHashes from the block. + +Besides the work of the core team, this update is based on the efforts of @akifoq (due_payment issue). + +## 2024.03 Update + +1. Preparatory (not enabled yet) code for pre-compiled smart-contract. +2. Minor fixes for fee-related opcodes. + +## 2024.02 Update + +1. Improvement of validator synchronisation: + * Better handling of block broadcasts -> faster sync + * Additional separate overlay among validators as second option for synchronisation +2. Improvements in LS: + * c7 and library context is fully filled up for server-side rungetmethod + * Cache for runmethods and successfull external messages + * Logging of LS requests statistic +3. Precise control of open files: + * almost instantaneous validator start + * `--max-archive-fd` option + * autoremoval of not used temp archive files + * `--archive-preload-period` option +4. Preparatory (not enabled yet) code for addition on new TVM instructions for cheaper fee calculation onchain. + +## 2024.01 Update + +1. Fixes in how gas in transactions on special accounts is accounted in block limit. Previously, gas was counted as usual, so to conduct elections that costs >30m gas block limit in masterchain was set to 37m gas. To lower the limit for safety reasons it is proposed to caunt gas on special accounts separately. Besides `gas_max` is set to `special_gas_limit` for all types of transactions on special accounts. New behavior is activated through setting `version >= 5` in `ConfigParam 8;`. + * Besides update of config temporally increases gas limit on `EQD_v9j1rlsuHHw2FIhcsCFFSD367ldfDdCKcsNmNpIRzUlu` to `special_gas_limit`, see [details](https://t.me/tonstatus/88). +2. Improvements in LS behavior + * Improved detection of the state with all shards applied to decrease rate of `Block is not applied` error + * Better error logs: `block not in db` and `block is not applied` separation + * Fix error in proof generation for blocks after merge + * Fix most of `block is not applied` issues related to sending too recent block in Proofs + * LS now check external messages till `accept_message` (`set_gas`). +3. Improvements in DHT work and storage, CellDb, config.json amendment, peer misbehavior detection, validator session stats collection, emulator. +4. Change in CTOS and XLOAD behavior activated through setting `version >= 5` in `ConfigParam 8;`: + * Loading "nested libraries" (i.e. a library cell that points to another library cell) throws an exception. + * Loading a library consumes gas for cell load only once (for the library cell), not twice (both for the library cell and the cell in the library). + * `XLOAD` now works differently. When it takes a library cell, it returns the cell that it points to. This allows loading "nested libraries", if needed. + +Besides the work of the Core team, this update is based on the efforts of @XaBbl4 (peer misbehavior detection) and @akifoq (CTOS behavior and gas limit scheme for special accounts). + +## 2023.12 Update + +1. Optimized message queue handling, now queue cleaning speed doesn't depend on total queue size + * Cleaning delivered messages using lt augmentation instead of random search / consecutive walk + * Keeping root cell of queue message in memory until outdated (caching) +2. Changes to block collation/validation limits +3. Stop accepting new external message if message queue is overloaded +4. Introducing conditions for shard split/merge based on queue size + +Read [more](https://blog.ton.org/technical-report-december-5-inscriptions-launch-on-ton) on that update. + +## 2023.11 Update + +1. New TVM Functionality. (Disabled by default) +2. A series of emulator improvements: libraries support, higher max stack size, etc +3. A series of tonlib and tonlib-cli improvements: wallet-v4 support, getconfig, showtransactions, etc +4. Changes to public libraries: now contract can not publish more than 256 libraries (config parameter) and contracts can not be deployed with public libraries in initstate (instead contracts need explicitly publish all libraries) +5. Changes to storage due payment: now due payment is collected in Storage Phase, however for bouncable messages fee amount can not exceed balance of account prior to message. + + +Besides the work of the core team, this update is based on the efforts of @aleksej-paschenko (emulator improvements), @akifoq (security improvements), Trail of Bits auditor as well as all participants of [TEP-88 discussion](https://github.com/ton-blockchain/TEPs/pull/88). + +## 2023.10 Update +1. A series of additional security checks in node: special cells in action list, init state in external messages, peers data prior to saving to disk. +2. Human-readable timestamps in explorer + +Besides the work of the core team, this update is based on the efforts of @akifoq and @mr-tron. + +## 2023.06 Update +1. (disabled by default) New deflation mechanisms: partial fee burning and blackhole address +2. Storage-contract improvement + +Besides the work of the core team, this update is based on the efforts of @DearJohnDoe from Tonbyte (Storage-contract improvement). + +## 2023.05 Update +1. Archive manager optimization +2. A series of catchain (basic consensus protocol) security improvements +3. Update for Fift libraries and FunC: better error-handling, fixes for `catch` stack recovery +4. A series of out message queue handling optimization (already deployed during emergency upgrades between releases) +5. Improvement of binaries portability + +Besides the work of the core team, this update is based on the efforts of @aleksej-paschenko (portability improvement), [Disintar team](https://github.com/disintar/) (archive manager optimization) and [sec3-service](https://github.com/sec3-service) security auditors (funC improvements). + +## 2023.04 Update +1. CPU load optimization: previous DHT reconnect policy was too aggressive +2. Network throughput improvements: granular control on external message broadcast, optimize celldb GC, adjust state serialization and block downloading timings, rldp2 for states and archives +3. Update for Fift (namespaces) and Fift libraries (list of improvements: https://github.com/ton-blockchain/ton/issues/631) +4. Better handling of incorrect inputs in funC: fix UB and prevent crashes on some inputs, improve optimizing int consts and unused variables in FunC, fix analyzing repeat loop. FunC version is increase to 0.4.3. +5. `listBlockTransactionsExt` in liteserver added +6. Tvm emulator improvements + +Besides the work of the core team, this update is based on the efforts of @krigga (tvm emulator improvement), @ex3ndr (`PUSHSLICE` fift-asm improvement) and [sec3-service](https://github.com/sec3-service) security auditors (funC improvements). + +## 2023.03 Update +1. Improvement of ADNL connection stability +2. Transaction emulator support and getAccountStateByTransaction method +3. Fixes of typos, undefined behavior and timer warnings +4. Handling incorrect integer literal values in funC; funC version bumped to 0.4.2 +5. FunC Mathlib + +## 2023.01 Update +1. Added ConfigParam 44: `SuspendedAddressList`. Upon being set this config suspends initialisation of **uninit** addresses from the list for given time. +2. FunC: `v0.4.1` added pragmas for precise control of computation order +3. FunC: fixed compiler crashes for some exotic inputs +4. FunC: added legacy tester, a collection of smart-contracts which is used to check whether compilator update change compilation result +5. Improved archive manager: proper handling of recently garbage-collected blocks + +## 2022.12 Update Node update: 1. Improvements of ton-proxy: fixed few bugs, improved stability 2. Improved collator/validator checks, added optimization of storage stat calculation, generation and validation of new blocks is made safer @@ -54,9 +220,46 @@ Node update: Besides the work of the core team, this update is based on the efforts of @vtamara (help with abseil-cpp upgrade), @krigga(in-place modification of global variables) and third-party security auditors. -## 01.2023 Update -1. Added ConfigParam 44: `SuspendedAddressList`. Upon being set this config suspends initialisation of **uninit** addresses from the list for given time. -2. FunC: `v0.4.1` added pragmas for precise control of computation order -3. FunC: fixed compiler crashes for some exotic inputs -4. FunC: added legacy tester, a collection of smart-contracts which is used to check whether compilator update change compilation result -5. Improved archive manager: proper handling of recently garbage-collected blocks +## 2022.10 Update +* Added extended block creation and general perfomance stats gathering +* Forbidden report data on blocks not committed to the master chain for LS +* Improved debug in TVM +* FunC 0.3.0: multi-line asms, bitwise operations for constants, duplication of identical definition for constants and asms now allowed +* New tonlib methods: sendMessageReturnHash, getTransactionsV2, getMasterchainBlockSignatures, getShardBlockProof, getLibraries. +* Fixed bugs related to invalid TVM output (c4, c5, libaries) and non-validated network data; avoided too deep recursion in libraries loading +* Fixed multiple undefined behavior issues +* Added build of FunC and Fift to WASM + +Besides the work of the core team, this update is based on the efforts of @tvorogme (debug improvements), @AlexeyFSL (WASM builds) and third-party security auditors. + +## 2022.08 Update +* Blockchain state serialization now works via separate db-handler which simplifies memory clearing after serialization +* CellDB now works asynchronously which substantially increase database access throughput +* Abseil-cpp and crc32 updated: solve issues with compilation on recent OS distributives +* Fixed a series of UBs and issues for exotic endianness hosts +* Added detailed network stats for overlays (can be accessed via `validator-console`) +* Improved auto-builds for wide range of systems. +* Added extended error information for unaccepted external messages: `exit_code` and TVM trace (where applicable). +* [Improved catchain DoS resistance](https://github.com/ton-blockchain/ton/blob/master/doc/catchain-dos.md) +* A series of FunC improvements, summarized [here](https://github.com/ton-blockchain/ton/pull/378) +#### Update delay +Update coincided with persistent state serialization event which lead to block production speed deterioration (issue substantially mitigated in update itself). This phenomena was aggravated by the fact that after update some validators lost ability to participate in block creation. The last was caused by threshold based hardcoded protocol version bump, where threshold was set in such manner (based on block height with value higher than 9m), that it eluded detection in private net tests. The update was temporarily paused and resumed after persistent state serialization ended and issues with block creation were resolved. + +Besides the work of the core team, this update is based on the efforts of @awesome-doge (help with abseil-cpp upgrade), @rec00rsiff (noted issues for exotic endianess and implemented network stats) and third-party security auditors. + +## 2022.05 Update +* Initial synchronization improved: adjusted timeouts for state download and the way of choosing which state to download. Nodes with low network speed and/or bad connectivity will synchronize faster and consistently. +* Improved peer-to-peer network stability and DDoS resistance: now peers will only relay valid messages to the network. Large messages, which require splitting for relaying, will be retranslated as well, but only after the node gets all parts, and reassembles and checks them. Validators may sign certificates for network peers, which allow relaying large messages by parts without checks. It is used now by validators to faster relay new blocks. Sign and import certificate commands are exposed via `validator-engine-console`. +* Fixed some rare edge cases in TVM arithmetic operations related to big numbers (`2**63+`) +* Improved fixes used to combat wrong activate-destruct-activate contract behavior last November. +* Improved tonlib: support libraries (with client-side caching), getmethods completely fill c7 register, getmethods support slice arguments, improved messages listing for transactions, added extended block header params, added getConfig method. +* RocksDB updated to a newer version. +* Improved persistent state serialization: memory usage during serialization was optimized; the start of serialization on different nodes was sparsed. +* FunC update: support for string literals and constants (including precompiled constant expressions), semver, `include` expressions. +* Fixed rarely manifested bugs in `Asm.fif`. +* LiteClient supports key as cli parameter. +* Improved Liteserver DoS resistance for running getmethods. + +Besides the work of the core team, this update is based on the efforts of @tvorogme (added support for slice arguments and noted bugs in Asm.fif), @akifoq (fixed bug in Asm.fif), @cryshado (noted strange behavior of LS, which, upon inspection, turned out to be a vector of DoS attack). + + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..f1b836bf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,69 @@ +FROM ubuntu:22.04 AS builder +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + rm /var/lib/dpkg/info/libc-bin.* && \ + apt-get clean && \ + apt-get update && \ + apt install libc-bin && \ + apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git \ + ninja-build libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool \ + libjemalloc-dev lsb-release software-properties-common gnupg + +RUN wget https://apt.llvm.org/llvm.sh && \ + chmod +x llvm.sh && \ + ./llvm.sh 16 all && \ + rm -rf /var/lib/apt/lists/* + +ENV CC=/usr/bin/clang-16 +ENV CXX=/usr/bin/clang++-16 +ENV CCACHE_DISABLE=1 + +WORKDIR / +RUN mkdir ton +WORKDIR /ton + +COPY ./ ./ + +RUN mkdir build && \ + cd build && \ + cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DTON_USE_JEMALLOC=ON .. && \ + ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console \ + generate-random-id dht-server lite-client tolk rldp-http-proxy dht-server proxy-liteserver create-state \ + blockchain-explorer emulator tonlibjson http-proxy adnl-proxy + +FROM ubuntu:22.04 +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y wget curl libatomic1 openssl libsodium-dev libmicrohttpd-dev liblz4-dev libjemalloc-dev htop \ + net-tools netcat iptraf-ng jq tcpdump pv plzip && \ + rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /var/ton-work/db /var/ton-work/scripts /usr/share/ton/smartcont/auto /usr/lib/fift/ + +COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon /usr/local/bin/ +COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon-cli /usr/local/bin/ +COPY --from=builder /ton/build/lite-client/lite-client /usr/local/bin/ +COPY --from=builder /ton/build/validator-engine/validator-engine /usr/local/bin/ +COPY --from=builder /ton/build/validator-engine-console/validator-engine-console /usr/local/bin/ +COPY --from=builder /ton/build/utils/generate-random-id /usr/local/bin/ +COPY --from=builder /ton/build/blockchain-explorer/blockchain-explorer /usr/local/bin/ +COPY --from=builder /ton/build/crypto/create-state /usr/local/bin/ +COPY --from=builder /ton/build/utils/proxy-liteserver /usr/local/bin/ +COPY --from=builder /ton/build/dht-server/dht-server /usr/local/bin/ +COPY --from=builder /ton/build/rldp-http-proxy/rldp-http-proxy /usr/local/bin/ +COPY --from=builder /ton/build/http/http-proxy /usr/local/bin/ +COPY --from=builder /ton/build/adnl/adnl-proxy /usr/local/bin/ +COPY --from=builder /ton/build/tonlib/libtonlibjson.so /usr/local/bin/ +COPY --from=builder /ton/build/emulator/libemulator.so /usr/local/bin/ +COPY --from=builder /ton/build/tolk/tolk /usr/local/bin/ +COPY --from=builder /ton/build/crypto/fift /usr/local/bin/ +COPY --from=builder /ton/build/crypto/func /usr/local/bin/ +COPY --from=builder /ton/crypto/smartcont/* /usr/share/ton/smartcont/ +COPY --from=builder /ton/crypto/smartcont/auto/* /usr/share/ton/smartcont/auto/ +COPY --from=builder /ton/crypto/fift/lib/* /usr/lib/fift/ + +WORKDIR /var/ton-work/db +COPY ./docker/init.sh ./docker/control.template /var/ton-work/scripts/ +RUN chmod +x /var/ton-work/scripts/init.sh + +ENTRYPOINT ["/var/ton-work/scripts/init.sh"] diff --git a/README.md b/README.md index 893717ba..897ba809 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,57 @@
- - - TON logo - + + + + TON logo + + +

Reference implementation of TON Node and tools

-[![TON Overflow Group][ton-overflow-badge]][ton-overflow-url] -[![Stack Overflow Group][stack-overflow-badge]][stack-overflow-url] -[![Telegram Foundation Group][telegram-foundation-badge]][telegram-foundation-url] -[![Telegram Community Group][telegram-community-badge]][telegram-community-url] -[![Twitter Group][twitter-badge]][twitter-url] +## -[telegram-foundation-badge]: https://img.shields.io/badge/-TON%20Foundation-2CA5E0?style=flat&logo=telegram&logoColor=white -[telegram-community-badge]: https://img.shields.io/badge/-TON%20Community-2CA5E0?style=flat&logo=telegram&logoColor=white -[telegram-foundation-url]: https://t.me/tonblockchain -[telegram-community-url]: https://t.me/toncoin -[twitter-badge]: https://img.shields.io/twitter/follow/ton_blockchain -[twitter-url]: https://twitter.com/ton_blockchain -[stack-overflow-badge]: https://img.shields.io/badge/-Stack%20Overflow-FE7A16?style=flat&logo=stack-overflow&logoColor=white -[stack-overflow-url]: https://stackoverflow.com/questions/tagged/ton -[ton-overflow-badge]: https://img.shields.io/badge/-TON%20Overflow-FE7A16?style=flat&logo=stack-overflow&logoColor=white -[ton-overflow-url]: https://answers.ton.org +

+ + Ton Research + + + Telegram Community Group + + + Telegram Foundation Group + + + Telegram Community Chat + +

+ +

+ + Twitter Group + + + TON Overflow Group + + + Stack Overflow Group + +

Main TON monorepo, which includes the code of the node/validator, lite-client, tonlib, FunC compiler, etc. -## Updates flow: +## The Open Network + +__The Open Network (TON)__ is a fast, secure, scalable blockchain focused on handling _millions of transactions per second_ (TPS) with the goal of reaching hundreds of millions of blockchain users. +- To learn more about different aspects of TON blockchain and its underlying ecosystem check [documentation](https://ton.org/docs) +- To run node, validator or lite-server check [Participate section](https://ton.org/docs/participate/nodes/run-node) +- To develop decentralised apps check [Tutorials](https://docs.ton.org/v3/guidelines/smart-contracts/guidelines), [FunC docs](https://ton.org/docs/develop/func/overview) and [DApp tutorials](https://docs.ton.org/v3/guidelines/dapps/overview) +- To work on TON check [wallets](https://ton.app/wallets), [explorers](https://ton.app/explorers), [DEXes](https://ton.app/dex) and [utilities](https://ton.app/utilities) +- To interact with TON check [APIs](https://docs.ton.org/v3/guidelines/dapps/apis-sdks/overview) + +## Updates flow * **master branch** - mainnet is running on this stable branch. @@ -45,8 +69,83 @@ Usually, the response to your pull request will indicate which section it falls * Thou shall not merge your own PRs, at least one person should review the PR and merge it (4-eyes rule) * Thou shall make sure that workflows are cleanly completed for your PR before considering merge -## Workflows responsibility -If a CI workflow fails not because of your changes but workflow issues, try to fix it yourself or contact one of the persons listed below via Telegram messenger: +## Build TON blockchain -* **C/C++ CI (ccpp-linux.yml)**: TBD -* **C/C++ CI Win64 Compile (ccpp-win64.yml)**: TBD +### Ubuntu 20.4, 22.04, 24.04 (x86-64, aarch64) +Install additional system libraries +```bash + sudo apt-get update + sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev + + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 16 all +``` +Compile TON binaries +```bash + cp assembly/native/build-ubuntu-shared.sh . + chmod +x build-ubuntu-shared.sh + ./build-ubuntu-shared.sh +``` + +### MacOS 11, 12 (x86-64, aarch64) +```bash + cp assembly/native/build-macos-shared.sh . + chmod +x build-macos-shared.sh + ./build-macos-shared.sh +``` + +### Windows 10, 11, Server (x86-64) +You need to install `MS Visual Studio 2022` first. +Go to https://www.visualstudio.com/downloads/ and download `MS Visual Studio 2022 Community`. + +Launch installer and select `Desktop development with C++`. +After installation, also make sure that `cmake` is globally available by adding +`C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin` to the system `PATH` (adjust the path per your needs). + +Open an elevated (Run as Administrator) `x86-64 Native Tools Command Prompt for VS 2022`, go to the root folder and execute: +```bash + copy assembly\native\build-windows.bat . + build-windows.bat +``` + +### Building TON to WebAssembly +Install additional system libraries on Ubuntu +```bash + sudo apt-get update + sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev + + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 16 all +``` +Compile TON binaries with emscripten +```bash + cd assembly/wasm + chmod +x fift-func-wasm-build-ubuntu.sh + ./fift-func-wasm-build-ubuntu.sh +``` + +### Building TON tonlib library for Android (arm64-v8a, armeabi-v7a, x86, x86-64) +Install additional system libraries on Ubuntu +```bash + sudo apt-get update + sudo apt-get install -y build-essential git cmake ninja-build automake libtool texinfo autoconf libgflags-dev \ + zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev \ + libtool autoconf libsodium-dev libsecp256k1-dev +``` +Compile TON tonlib library +```bash + cp assembly/android/build-android-tonlib.sh . + chmod +x build-android-tonlib.sh + ./build-android-tonlib.sh +``` + +### TON portable binaries + +Linux portable binaries are wrapped into AppImages, at the same time MacOS portable binaries are statically linked executables. +Linux and MacOS binaries are available for both x86-64 and arm64 architectures. + +## Running tests + +Tests are executed by running `ctest` in the build directory. See `doc/Tests.md` for more information. diff --git a/adnl/CMakeLists.txt b/adnl/CMakeLists.txt index 954db27b..111c4c50 100644 --- a/adnl/CMakeLists.txt +++ b/adnl/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) #BEGIN internal if (NOT TON_ONLY_TONLIB) @@ -88,17 +88,17 @@ target_link_libraries(adnl PUBLIC tdactor ton_crypto tl_api tdnet tddb keys keyr add_executable(adnl-proxy ${ADNL_PROXY_SOURCE}) target_include_directories(adnl-proxy PUBLIC $) -target_link_libraries(adnl-proxy PUBLIC tdactor ton_crypto tl_api tdnet common - tl-utils git) +target_link_libraries(adnl-proxy PUBLIC tdactor ton_crypto tl_api tdnet common tl-utils git) add_executable(adnl-pong adnl-pong.cpp) target_include_directories(adnl-pong PUBLIC $) -target_link_libraries(adnl-pong PUBLIC tdactor ton_crypto tl_api tdnet common - tl-utils adnl dht git) +target_link_libraries(adnl-pong PUBLIC tdactor ton_crypto tl_api tdnet common tl-utils adnl dht git) add_library(adnltest STATIC ${ADNL_TEST_SOURCE}) target_include_directories(adnltest PUBLIC $) -target_link_libraries(adnltest PUBLIC adnl ) +target_link_libraries(adnltest PUBLIC adnl) + +install(TARGETS adnl-proxy RUNTIME DESTINATION bin) endif() #END internal diff --git a/adnl/adnl-channel.cpp b/adnl/adnl-channel.cpp index 5c8229ca..4da9d2ee 100644 --- a/adnl/adnl-channel.cpp +++ b/adnl/adnl-channel.cpp @@ -112,16 +112,16 @@ void AdnlChannelImpl::send_message(td::uint32 priority, td::actor::ActorId R) { - if (R.is_error()) { - VLOG(ADNL_WARNING) << id << ": dropping IN message: can not decrypt: " << R.move_as_error(); - } else { - auto packet = R.move_as_ok(); - packet.set_remote_addr(addr); - td::actor::send_closure(peer, &AdnlPeerPair::receive_packet_from_channel, channel_id, std::move(packet)); - } - }); + auto P = td::PromiseCreator::lambda([peer = peer_pair_, channel_id = channel_in_id_, addr, id = print_id(), + size = data.size()](td::Result R) { + if (R.is_error()) { + VLOG(ADNL_WARNING) << id << ": dropping IN message: can not decrypt: " << R.move_as_error(); + } else { + auto packet = R.move_as_ok(); + packet.set_remote_addr(addr); + td::actor::send_closure(peer, &AdnlPeerPair::receive_packet_from_channel, channel_id, std::move(packet), size); + } + }); decrypt(std::move(data), std::move(P)); } diff --git a/adnl/adnl-ext-client.hpp b/adnl/adnl-ext-client.hpp index 13339725..1dd7d2ba 100644 --- a/adnl/adnl-ext-client.hpp +++ b/adnl/adnl-ext-client.hpp @@ -43,7 +43,10 @@ class AdnlOutboundConnection : public AdnlExtConnection { public: AdnlOutboundConnection(td::SocketFd fd, std::unique_ptr callback, AdnlNodeIdFull dst, td::actor::ActorId ext_client) - : AdnlExtConnection(std::move(fd), std::move(callback), true), dst_(std::move(dst)), ext_client_(ext_client) { + : AdnlExtConnection(std::move(fd), std::move(callback), true) + , dst_(std::move(dst)) + , local_id_(privkeys::Ed25519::random()) + , ext_client_(ext_client) { } AdnlOutboundConnection(td::SocketFd fd, std::unique_ptr callback, AdnlNodeIdFull dst, PrivateKey local_id, td::actor::ActorId ext_client) diff --git a/adnl/adnl-ext-server.cpp b/adnl/adnl-ext-server.cpp index ed04469c..162a53af 100644 --- a/adnl/adnl-ext-server.cpp +++ b/adnl/adnl-ext-server.cpp @@ -91,7 +91,7 @@ td::Status AdnlInboundConnection::process_custom_packet(td::BufferSlice &data, b auto F = fetch_tl_object(data.clone(), true); if (F.is_ok()) { if (nonce_.size() > 0 || !remote_id_.is_zero()) { - return td::Status::Error(ErrorCode::protoviolation, "duplicate authentificate"); + return td::Status::Error(ErrorCode::protoviolation, "duplicate authenticate"); } auto f = F.move_as_ok(); nonce_ = td::SecureString{f->nonce_.size() + 256}; diff --git a/adnl/adnl-local-id.cpp b/adnl/adnl-local-id.cpp index b4818276..e0c62de7 100644 --- a/adnl/adnl-local-id.cpp +++ b/adnl/adnl-local-id.cpp @@ -41,20 +41,34 @@ AdnlAddressList AdnlLocalId::get_addr_list() const { } void AdnlLocalId::receive(td::IPAddress addr, td::BufferSlice data) { - auto P = td::PromiseCreator::lambda( - [peer_table = peer_table_, dst = short_id_, addr, id = print_id()](td::Result R) { - if (R.is_error()) { - VLOG(ADNL_WARNING) << id << ": dropping IN message: cannot decrypt: " << R.move_as_error(); - } else { - auto packet = R.move_as_ok(); - packet.set_remote_addr(addr); - td::actor::send_closure(peer_table, &AdnlPeerTable::receive_decrypted_packet, dst, std::move(packet)); - } - }); - + InboundRateLimiter& rate_limiter = inbound_rate_limiter_[addr]; + if (!rate_limiter.rate_limiter.take()) { + VLOG(ADNL_NOTICE) << this << ": dropping IN message: rate limit exceeded"; + add_dropped_packet_stats(addr); + return; + } + ++rate_limiter.currently_decrypting_packets; + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), peer_table = peer_table_, dst = short_id_, addr, + id = print_id(), size = data.size()](td::Result R) { + td::actor::send_closure(SelfId, &AdnlLocalId::decrypt_packet_done, addr); + if (R.is_error()) { + VLOG(ADNL_WARNING) << id << ": dropping IN message: cannot decrypt: " << R.move_as_error(); + } else { + auto packet = R.move_as_ok(); + packet.set_remote_addr(addr); + td::actor::send_closure(peer_table, &AdnlPeerTable::receive_decrypted_packet, dst, std::move(packet), size); + } + }); decrypt(std::move(data), std::move(P)); } +void AdnlLocalId::decrypt_packet_done(td::IPAddress addr) { + auto it = inbound_rate_limiter_.find(addr); + CHECK(it != inbound_rate_limiter_.end()); + --it->second.currently_decrypting_packets; + add_decrypted_packet_stats(addr); +} + void AdnlLocalId::deliver(AdnlNodeIdShort src, td::BufferSlice data) { auto s = std::move(data); for (auto &cb : cb_) { @@ -292,6 +306,72 @@ void AdnlLocalId::update_packet(AdnlPacket packet, bool update_id, bool sign, td } } +void AdnlLocalId::get_stats(bool all, td::Promise> promise) { + auto stats = create_tl_object(); + stats->short_id_ = short_id_.bits256_value(); + for (auto &[ip, x] : inbound_rate_limiter_) { + if (x.currently_decrypting_packets != 0) { + stats->current_decrypt_.push_back(create_tl_object( + ip.is_valid() ? PSTRING() << ip.get_ip_str() << ":" << ip.get_port() : "", x.currently_decrypting_packets)); + } + } + prepare_packet_stats(); + stats->packets_recent_ = packet_stats_prev_.tl(); + stats->packets_total_ = packet_stats_total_.tl(all); + stats->packets_total_->ts_start_ = (double)Adnl::adnl_start_time(); + stats->packets_total_->ts_end_ = td::Clocks::system(); + promise.set_result(std::move(stats)); +} + +void AdnlLocalId::add_decrypted_packet_stats(td::IPAddress addr) { + prepare_packet_stats(); + packet_stats_cur_.decrypted_packets[addr].inc(); + packet_stats_total_.decrypted_packets[addr].inc(); +} + +void AdnlLocalId::add_dropped_packet_stats(td::IPAddress addr) { + prepare_packet_stats(); + packet_stats_cur_.dropped_packets[addr].inc(); + packet_stats_total_.dropped_packets[addr].inc(); +} + +void AdnlLocalId::prepare_packet_stats() { + double now = td::Clocks::system(); + if (now >= packet_stats_cur_.ts_end) { + packet_stats_prev_ = std::move(packet_stats_cur_); + packet_stats_cur_ = {}; + auto now_int = (int)td::Clocks::system(); + packet_stats_cur_.ts_start = (double)(now_int / 60 * 60); + packet_stats_cur_.ts_end = packet_stats_cur_.ts_start + 60.0; + if (packet_stats_prev_.ts_end < now - 60.0) { + packet_stats_prev_ = {}; + packet_stats_prev_.ts_end = packet_stats_cur_.ts_start; + packet_stats_prev_.ts_start = packet_stats_prev_.ts_end - 60.0; + } + } +} + +tl_object_ptr AdnlLocalId::PacketStats::tl(bool all) const { + double threshold = all ? -1.0 : td::Clocks::system() - 600.0; + auto obj = create_tl_object(); + obj->ts_start_ = ts_start; + obj->ts_end_ = ts_end; + for (const auto &[ip, packets] : decrypted_packets) { + if (packets.last_packet_ts >= threshold) { + obj->decrypted_packets_.push_back(create_tl_object( + ip.is_valid() ? PSTRING() << ip.get_ip_str() << ":" << ip.get_port() : "", packets.packets)); + } + } + for (const auto &[ip, packets] : dropped_packets) { + if (packets.last_packet_ts >= threshold) { + obj->dropped_packets_.push_back(create_tl_object( + ip.is_valid() ? PSTRING() << ip.get_ip_str() << ":" << ip.get_port() : "", packets.packets)); + } + } + return obj; +} + + } // namespace adnl } // namespace ton diff --git a/adnl/adnl-local-id.h b/adnl/adnl-local-id.h index c9ecfff1..fa7f7f74 100644 --- a/adnl/adnl-local-id.h +++ b/adnl/adnl-local-id.h @@ -55,6 +55,7 @@ class AdnlLocalId : public td::actor::Actor { void deliver(AdnlNodeIdShort src, td::BufferSlice data); void deliver_query(AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); void receive(td::IPAddress addr, td::BufferSlice data); + void decrypt_packet_done(td::IPAddress addr); void subscribe(std::string prefix, std::unique_ptr callback); void unsubscribe(std::string prefix); @@ -77,6 +78,8 @@ class AdnlLocalId : public td::actor::Actor { void update_packet(AdnlPacket packet, bool update_id, bool sign, td::int32 update_addr_list_if, td::int32 update_priority_addr_list_if, td::Promise promise); + void get_stats(bool all, td::Promise> promise); + td::uint32 get_mode() { return mode_; } @@ -101,6 +104,32 @@ class AdnlLocalId : public td::actor::Actor { td::uint32 mode_; + struct InboundRateLimiter { + RateLimiter rate_limiter = RateLimiter(75, 0.33); + td::uint64 currently_decrypting_packets = 0; + }; + std::map inbound_rate_limiter_; + struct PacketStats { + double ts_start = 0.0, ts_end = 0.0; + + struct Counter { + td::uint64 packets = 0; + double last_packet_ts = 0.0; + + void inc() { + ++packets; + last_packet_ts = td::Clocks::system(); + } + }; + std::map decrypted_packets; + std::map dropped_packets; + + tl_object_ptr tl(bool all = true) const; + } packet_stats_cur_, packet_stats_prev_, packet_stats_total_; + void add_decrypted_packet_stats(td::IPAddress addr); + void add_dropped_packet_stats(td::IPAddress addr); + void prepare_packet_stats(); + void publish_address_list(); }; diff --git a/adnl/adnl-peer-table.cpp b/adnl/adnl-peer-table.cpp index 54891515..b8aab567 100644 --- a/adnl/adnl-peer-table.cpp +++ b/adnl/adnl-peer-table.cpp @@ -84,7 +84,7 @@ void AdnlPeerTableImpl::receive_packet(td::IPAddress addr, AdnlCategoryMask cat_ << " (len=" << (data.size() + 32) << ")"; } -void AdnlPeerTableImpl::receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket packet) { +void AdnlPeerTableImpl::receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket packet, td::uint64 serialized_size) { packet.run_basic_checks().ensure(); if (!packet.inited_from_short()) { @@ -119,7 +119,7 @@ void AdnlPeerTableImpl::receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket return; } td::actor::send_closure(it->second, &AdnlPeer::receive_packet, dst, it2->second.mode, it2->second.local_id.get(), - std::move(packet)); + std::move(packet), serialized_size); } void AdnlPeerTableImpl::add_peer(AdnlNodeIdShort local_id, AdnlNodeIdFull id, AdnlAddressList addr_list) { @@ -385,6 +385,88 @@ void AdnlPeerTableImpl::get_conn_ip_str(AdnlNodeIdShort l_id, AdnlNodeIdShort p_ td::actor::send_closure(it->second, &AdnlPeer::get_conn_ip_str, l_id, std::move(promise)); } +void AdnlPeerTableImpl::get_stats(bool all, td::Promise> promise) { + class Cb : public td::actor::Actor { + public: + explicit Cb(td::Promise> promise) : promise_(std::move(promise)) { + } + + void got_local_id_stats(tl_object_ptr local_id) { + auto &local_id_stats = local_id_stats_[local_id->short_id_]; + if (local_id_stats) { + local_id->peers_ = std::move(local_id_stats->peers_); + } + local_id_stats = std::move(local_id); + dec_pending(); + } + + void got_peer_stats(std::vector> peer_pairs) { + for (auto &peer_pair : peer_pairs) { + auto &local_id_stats = local_id_stats_[peer_pair->local_id_]; + if (local_id_stats == nullptr) { + local_id_stats = create_tl_object(); + local_id_stats->short_id_ = peer_pair->local_id_; + } + local_id_stats->peers_.push_back(std::move(peer_pair)); + } + dec_pending(); + } + + void inc_pending() { + ++pending_; + } + + void dec_pending() { + CHECK(pending_ > 0); + --pending_; + if (pending_ == 0) { + auto stats = create_tl_object(); + stats->timestamp_ = td::Clocks::system(); + for (auto &[id, local_id_stats] : local_id_stats_) { + stats->local_ids_.push_back(std::move(local_id_stats)); + } + promise_.set_result(std::move(stats)); + stop(); + } + } + + private: + td::Promise> promise_; + size_t pending_ = 1; + + std::map> local_id_stats_; + }; + auto callback = td::actor::create_actor("adnlstats", std::move(promise)).release(); + + for (auto &[id, local_id] : local_ids_) { + td::actor::send_closure(callback, &Cb::inc_pending); + td::actor::send_closure(local_id.local_id, &AdnlLocalId::get_stats, all, + [id = id, callback](td::Result> R) { + if (R.is_error()) { + VLOG(ADNL_NOTICE) + << "failed to get stats for local id " << id << " : " << R.move_as_error(); + td::actor::send_closure(callback, &Cb::dec_pending); + } else { + td::actor::send_closure(callback, &Cb::got_local_id_stats, R.move_as_ok()); + } + }); + } + for (auto &[id, peer] : peers_) { + td::actor::send_closure(callback, &Cb::inc_pending); + td::actor::send_closure( + peer, &AdnlPeer::get_stats, all, + [id = id, callback](td::Result>> R) { + if (R.is_error()) { + VLOG(ADNL_NOTICE) << "failed to get stats for peer " << id << " : " << R.move_as_error(); + td::actor::send_closure(callback, &Cb::dec_pending); + } else { + td::actor::send_closure(callback, &Cb::got_peer_stats, R.move_as_ok()); + } + }); + } + td::actor::send_closure(callback, &Cb::dec_pending); +} + } // namespace adnl } // namespace ton diff --git a/adnl/adnl-peer-table.h b/adnl/adnl-peer-table.h index cb7da613..055f32ac 100644 --- a/adnl/adnl-peer-table.h +++ b/adnl/adnl-peer-table.h @@ -90,7 +90,7 @@ class AdnlPeerTable : public Adnl { virtual void answer_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlQueryId query_id, td::BufferSlice data) = 0; virtual void receive_packet(td::IPAddress addr, AdnlCategoryMask cat_mask, td::BufferSlice data) = 0; - virtual void receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket packet) = 0; + virtual void receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket packet, td::uint64 serialized_size) = 0; virtual void send_message_in(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlMessage message, td::uint32 flags) = 0; virtual void register_channel(AdnlChannelIdShort id, AdnlNodeIdShort local_id, diff --git a/adnl/adnl-peer-table.hpp b/adnl/adnl-peer-table.hpp index 2a27a802..9ad61b65 100644 --- a/adnl/adnl-peer-table.hpp +++ b/adnl/adnl-peer-table.hpp @@ -44,7 +44,7 @@ class AdnlPeerTableImpl : public AdnlPeerTable { void add_static_nodes_from_config(AdnlNodesList nodes) override; void receive_packet(td::IPAddress addr, AdnlCategoryMask cat_mask, td::BufferSlice data) override; - void receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket data) override; + void receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket data, td::uint64 serialized_size) override; void send_message_in(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlMessage message, td::uint32 flags) override; void send_message(AdnlNodeIdShort src, AdnlNodeIdShort dst, td::BufferSlice data) override { send_message_ex(src, dst, std::move(data), 0); @@ -77,6 +77,10 @@ class AdnlPeerTableImpl : public AdnlPeerTable { td::actor::ActorId channel) override; void unregister_channel(AdnlChannelIdShort id) override; + void check_id_exists(AdnlNodeIdShort id, td::Promise promise) override { + promise.set_value(local_ids_.count(id)); + } + void write_new_addr_list_to_db(AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id, AdnlDbItem node, td::Promise promise) override; void get_addr_list_from_db(AdnlNodeIdShort local_id, AdnlNodeIdShort peer_id, @@ -104,6 +108,8 @@ class AdnlPeerTableImpl : public AdnlPeerTable { td::Promise, AdnlAddress>> promise) override; void get_conn_ip_str(AdnlNodeIdShort l_id, AdnlNodeIdShort p_id, td::Promise promise) override; + void get_stats(bool all, td::Promise> promise) override; + struct PrintId {}; PrintId print_id() const { return PrintId{}; diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index 99356ed5..ab460058 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -26,6 +26,7 @@ #include "td/utils/base64.h" #include "td/utils/Random.h" #include "auto/tl/ton_api.h" +#include "td/utils/overloaded.h" namespace ton { @@ -50,9 +51,13 @@ void AdnlPeerPairImpl::start_up() { } void AdnlPeerPairImpl::alarm() { - if (next_dht_query_at_ && next_dht_query_at_.is_in_past()) { - next_dht_query_at_ = td::Timestamp::never(); - discover(); + if (!disable_dht_query_) { + disable_dht_query_ = true; + if (next_dht_query_at_ && next_dht_query_at_.is_in_past()) { + next_dht_query_at_ = td::Timestamp::never(); + discover(); + } + alarm_timestamp().relax(next_dht_query_at_); } if (next_db_update_at_ && next_db_update_at_.is_in_past()) { if (received_from_db_ && received_from_static_nodes_ && !peer_id_.empty()) { @@ -68,9 +73,8 @@ void AdnlPeerPairImpl::alarm() { } if (retry_send_at_ && retry_send_at_.is_in_past()) { retry_send_at_ = td::Timestamp::never(); - send_messages_in(std::move(pending_messages_), false); + send_messages_from_queue(); } - alarm_timestamp().relax(next_dht_query_at_); alarm_timestamp().relax(next_db_update_at_); alarm_timestamp().relax(retry_send_at_); } @@ -113,6 +117,9 @@ void AdnlPeerPairImpl::discover() { } void AdnlPeerPairImpl::receive_packet_checked(AdnlPacket packet) { + last_received_packet_ = td::Timestamp::now(); + try_reinit_at_ = td::Timestamp::never(); + drop_addr_list_at_ = td::Timestamp::never(); request_reverse_ping_after_ = td::Timestamp::in(15.0); auto d = Adnl::adnl_start_time(); if (packet.dst_reinit_date() > d) { @@ -203,16 +210,24 @@ void AdnlPeerPairImpl::receive_packet_checked(AdnlPacket packet) { } } -void AdnlPeerPairImpl::receive_packet_from_channel(AdnlChannelIdShort id, AdnlPacket packet) { +void AdnlPeerPairImpl::receive_packet_from_channel(AdnlChannelIdShort id, AdnlPacket packet, + td::uint64 serialized_size) { + add_packet_stats(serialized_size, /* in = */ true, /* channel = */ true); if (id != channel_in_id_) { VLOG(ADNL_NOTICE) << this << ": dropping IN message: outdated channel id" << id; return; } - channel_ready_ = true; + if (channel_inited_ && !channel_ready_) { + channel_ready_ = true; + if (!out_messages_queue_.empty()) { + td::actor::send_closure(actor_id(this), &AdnlPeerPairImpl::send_messages_from_queue); + } + } receive_packet_checked(std::move(packet)); } -void AdnlPeerPairImpl::receive_packet(AdnlPacket packet) { +void AdnlPeerPairImpl::receive_packet(AdnlPacket packet, td::uint64 serialized_size) { + add_packet_stats(serialized_size, /* in = */ true, /* channel = */ false); packet.run_basic_checks().ensure(); if (!encryptor_) { @@ -233,126 +248,132 @@ void AdnlPeerPairImpl::deliver_message(AdnlMessage message) { message.visit([&](const auto &obj) { this->process_message(obj); }); } -void AdnlPeerPairImpl::send_messages_in(std::vector messages, bool allow_postpone) { - for (td::int32 idx = 0; idx < 2; idx++) { - std::vector not_sent; +void AdnlPeerPairImpl::send_messages_from_queue() { + while (!out_messages_queue_.empty() && out_messages_queue_.front().second.is_in_past()) { + out_messages_queue_total_size_ -= out_messages_queue_.front().first.size(); + add_expired_msg_stats(out_messages_queue_.front().first.size()); + out_messages_queue_.pop(); + VLOG(ADNL_NOTICE) << this << ": dropping OUT message: message in queue expired"; + } + if (out_messages_queue_.empty()) { + return; + } - auto connR = get_conn(idx == 1); - if (connR.is_error()) { - if (!allow_postpone) { - VLOG(ADNL_NOTICE) << this << ": dropping OUT messages: cannot get conn: " << connR.move_as_error(); - return; - } - VLOG(ADNL_INFO) << this << ": delaying OUT messages: cannot get conn: " << connR.move_as_error(); - if (!retry_send_at_) { - retry_send_at_.relax(td::Timestamp::in(10.0)); - alarm_timestamp().relax(retry_send_at_); - } - for (auto &m : messages) { - pending_messages_.push_back(std::move(m)); - } + auto connR = get_conn(); + if (connR.is_error()) { + disable_dht_query_ = false; + retry_send_at_.relax(td::Timestamp::in(message_in_queue_ttl_ - 1.0)); + alarm_timestamp().relax(retry_send_at_); + VLOG(ADNL_INFO) << this << ": delaying OUT messages: cannot get conn: " << connR.move_as_error(); + return; + } + disable_dht_query_ = true; + auto C = connR.move_as_ok(); + auto conn = std::move(C.first); + bool is_direct = C.second; + + bool first = !skip_init_packet_; + while (!out_messages_queue_.empty()) { + bool try_reinit = try_reinit_at_ && try_reinit_at_.is_in_past(); + bool via_channel = channel_ready_ && !try_reinit; + if (!via_channel && !nochannel_rate_limiter_.take()) { + alarm_timestamp().relax(retry_send_at_ = nochannel_rate_limiter_.ready_at()); return; } - auto C = connR.move_as_ok(); - bool is_direct = C.second; - auto conn = std::move(C.first); - if (idx == 1) { - CHECK(is_direct); + if (try_reinit) { + try_reinit_at_ = td::Timestamp::in(td::Random::fast(0.5, 1.5)); + } + respond_with_nop_after_ = td::Timestamp::in(td::Random::fast(1.0, 2.0)); + + size_t s = (via_channel ? channel_packet_header_max_size() : packet_header_max_size()); + if (first) { + s += 2 * addr_list_max_size(); } - size_t ptr = 0; - bool first = true; - do { - size_t s = (channel_ready_ ? channel_packet_header_max_size() : packet_header_max_size()); - if (first) { - s += 2 * addr_list_max_size(); - } + AdnlPacket packet; + packet.set_seqno(++out_seqno_); + packet.set_confirm_seqno(in_seqno_); - AdnlPacket packet; - packet.set_seqno(++out_seqno_); - packet.set_confirm_seqno(in_seqno_); - - if (first) { - if (!channel_inited_) { - auto M = adnlmessage::AdnlMessageCreateChannel{channel_pub_, channel_pk_date_}; - s += M.size(); - packet.add_message(std::move(M)); - } else if (!channel_ready_) { - auto M = adnlmessage::AdnlMessageConfirmChannel{channel_pub_, peer_channel_pub_, channel_pk_date_}; - s += M.size(); - packet.add_message(std::move(M)); - } + if (first) { + if (!channel_inited_) { + auto M = adnlmessage::AdnlMessageCreateChannel{channel_pub_, channel_pk_date_}; + s += M.size(); + packet.add_message(std::move(M)); + } else if (!channel_ready_) { + auto M = adnlmessage::AdnlMessageConfirmChannel{channel_pub_, peer_channel_pub_, channel_pk_date_}; + s += M.size(); + packet.add_message(std::move(M)); } + } - if (!addr_list_.empty()) { - packet.set_received_addr_list_version(addr_list_.version()); - } - if (!priority_addr_list_.empty()) { - packet.set_received_priority_addr_list_version(priority_addr_list_.version()); - } + if (!addr_list_.empty()) { + packet.set_received_addr_list_version(addr_list_.version()); + } + if (!priority_addr_list_.empty()) { + packet.set_received_priority_addr_list_version(priority_addr_list_.version()); + } - while (ptr < messages.size()) { - auto &M = messages[ptr]; - if (!is_direct && (M.flags() & Adnl::SendFlags::direct_only)) { - not_sent.push_back(std::move(M)); - continue; - } - CHECK(M.size() <= get_mtu()); + skip_init_packet_ = true; + while (!out_messages_queue_.empty()) { + auto &M = out_messages_queue_.front().first; + if (!is_direct && (M.flags() & Adnl::SendFlags::direct_only)) { + out_messages_queue_total_size_ -= M.size(); + out_messages_queue_.pop(); + continue; + } + CHECK(M.size() <= get_mtu()); + if (s + M.size() <= AdnlNetworkManager::get_mtu()) { + s += M.size(); + out_messages_queue_total_size_ -= M.size(); + packet.add_message(M.release()); + out_messages_queue_.pop(); + skip_init_packet_ = false; + } else { + break; + } + } + + if (!via_channel) { + packet.set_reinit_date(Adnl::adnl_start_time(), reinit_date_); + packet.set_source(local_id_); + } + + if (!first) { + if (!channel_inited_) { + auto M = adnlmessage::AdnlMessageCreateChannel{channel_pub_, channel_pk_date_}; if (s + M.size() <= AdnlNetworkManager::get_mtu()) { s += M.size(); - packet.add_message(M.release()); - ptr++; - } else { - break; + packet.add_message(std::move(M)); + } + } else if (!channel_ready_) { + auto M = adnlmessage::AdnlMessageConfirmChannel{channel_pub_, peer_channel_pub_, channel_pk_date_}; + if (s + M.size() <= AdnlNetworkManager::get_mtu()) { + s += M.size(); + packet.add_message(std::move(M)); } } - - if (!channel_ready_) { - packet.set_reinit_date(Adnl::adnl_start_time(), reinit_date_); - packet.set_source(local_id_); - } - - if (!first) { - if (!channel_inited_) { - auto M = adnlmessage::AdnlMessageCreateChannel{channel_pub_, channel_pk_date_}; - if (s + M.size() <= AdnlNetworkManager::get_mtu()) { - s += M.size(); - packet.add_message(std::move(M)); - } - } else if (!channel_ready_) { - auto M = adnlmessage::AdnlMessageConfirmChannel{channel_pub_, peer_channel_pub_, channel_pk_date_}; - if (s + M.size() <= AdnlNetworkManager::get_mtu()) { - s += M.size(); - packet.add_message(std::move(M)); - } - } - } - - packet.run_basic_checks().ensure(); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), conn, id = print_id(), - via_channel = channel_ready_](td::Result res) { - if (res.is_error()) { - LOG(ERROR) << id << ": dropping OUT message: error while creating packet: " << res.move_as_error(); - } else { - td::actor::send_closure(SelfId, &AdnlPeerPairImpl::send_packet_continue, res.move_as_ok(), conn, via_channel); - } - }); - - td::actor::send_closure(local_actor_, &AdnlLocalId::update_packet, std::move(packet), - !channel_ready_ && ack_seqno_ == 0 && in_seqno_ == 0, !channel_ready_, - (first || s + addr_list_max_size() <= AdnlNetworkManager::get_mtu()) - ? peer_recv_addr_list_version_ - : 0x7fffffff, - (first || s + 2 * addr_list_max_size() <= AdnlNetworkManager::get_mtu()) - ? peer_recv_priority_addr_list_version_ - : 0x7fffffff, - std::move(P)); - first = false; - } while (ptr < messages.size()); - messages = std::move(not_sent); - if (!messages.size()) { - break; } + + packet.run_basic_checks().ensure(); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), conn, id = print_id(), + via_channel](td::Result res) { + if (res.is_error()) { + LOG(ERROR) << id << ": dropping OUT message: error while creating packet: " << res.move_as_error(); + } else { + td::actor::send_closure(SelfId, &AdnlPeerPairImpl::send_packet_continue, res.move_as_ok(), conn, via_channel); + } + }); + + td::actor::send_closure(local_actor_, &AdnlLocalId::update_packet, std::move(packet), + (!channel_ready_ && ack_seqno_ == 0 && in_seqno_ == 0) || try_reinit, !via_channel, + (first || s + addr_list_max_size() <= AdnlNetworkManager::get_mtu()) + ? (try_reinit ? 0 : peer_recv_addr_list_version_) + : 0x7fffffff, + (first || s + 2 * addr_list_max_size() <= AdnlNetworkManager::get_mtu()) + ? peer_recv_priority_addr_list_version_ + : 0x7fffffff, + std::move(P)); + first = false; } } @@ -383,15 +404,26 @@ void AdnlPeerPairImpl::send_messages(std::vector messages) } } } - send_messages_in(std::move(new_vec), true); + for (auto &m : new_vec) { + out_messages_queue_total_size_ += m.size(); + out_messages_queue_.emplace(std::move(m), td::Timestamp::in(message_in_queue_ttl_)); + } + send_messages_from_queue(); } void AdnlPeerPairImpl::send_packet_continue(AdnlPacket packet, td::actor::ActorId conn, bool via_channel) { + if (!try_reinit_at_ && last_received_packet_ < td::Timestamp::in(-5.0)) { + try_reinit_at_ = td::Timestamp::in(10.0); + } + if (!drop_addr_list_at_ && last_received_packet_ < td::Timestamp::in(-60.0 * 9.0)) { + drop_addr_list_at_ = td::Timestamp::in(60.0); + } packet.run_basic_checks().ensure(); auto B = serialize_tl_object(packet.tl(), true); if (via_channel) { if (channel_ready_) { + add_packet_stats(B.size(), /* in = */ false, /* channel = */ true); td::actor::send_closure(channel_, &AdnlChannel::send_message, priority_, conn, std::move(B)); } else { VLOG(ADNL_WARNING) << this << ": dropping OUT message [" << local_id_ << "->" << peer_id_short_ @@ -419,6 +451,7 @@ void AdnlPeerPairImpl::send_packet_continue(AdnlPacket packet, td::actor::ActorI S.remove_prefix(32); S.copy_from(X.as_slice()); + add_packet_stats(B.size(), /* in = */ false, /* channel = */ false); td::actor::send_closure(conn, &AdnlNetworkConnection::send, local_id_, peer_id_short_, priority_, std::move(enc)); } @@ -505,10 +538,14 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageConfirmChan VLOG(ADNL_NOTICE) << this << ": received adnl.message.confirmChannel with old key"; return; } - channel_ready_ = true; + if (!channel_ready_) { + channel_ready_ = true; + send_messages_from_queue(); + } } void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageCustom &message) { + respond_with_nop(); td::actor::send_closure(local_actor_, &AdnlLocalId::deliver, peer_id_short_, message.data()); } @@ -521,6 +558,7 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageReinit &mes } void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageQuery &message) { + respond_with_nop(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), query_id = message.query_id(), flags = static_cast(0)](td::Result R) { if (R.is_error()) { @@ -539,6 +577,7 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageQuery &mess } void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageAnswer &message) { + respond_with_nop(); auto Q = out_queries_.find(message.query_id()); if (Q == out_queries_.end()) { @@ -556,6 +595,7 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageAnswer &mes } void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessagePart &message) { + respond_with_nop(); auto size = message.total_size(); if (size > huge_packet_max_size()) { VLOG(ADNL_WARNING) << this << ": dropping too big huge message: size=" << size; @@ -618,6 +658,14 @@ void AdnlPeerPairImpl::delete_query(AdnlQueryId id) { } } +void AdnlPeerPairImpl::respond_with_nop() { + if (respond_with_nop_after_.is_in_past()) { + std::vector messages; + messages.emplace_back(adnlmessage::AdnlMessageNop{}, 0); + send_messages(std::move(messages)); + } +} + void AdnlPeerPairImpl::reinit(td::int32 date) { if (reinit_date_ == 0) { reinit_date_ = date; @@ -647,7 +695,17 @@ void AdnlPeerPairImpl::reinit(td::int32 date) { } } -td::Result, bool>> AdnlPeerPairImpl::get_conn(bool direct_only) { +td::Result, bool>> AdnlPeerPairImpl::get_conn() { + if (drop_addr_list_at_ && drop_addr_list_at_.is_in_past()) { + drop_addr_list_at_ = td::Timestamp::never(); + priority_addr_list_ = AdnlAddressList{}; + priority_conns_.clear(); + addr_list_ = AdnlAddressList{}; + conns_.clear(); + has_reverse_addr_ = false; + return td::Status::Error(ErrorCode::notready, "no active connections"); + } + if (!priority_addr_list_.empty() && priority_addr_list_.expire_at() < td::Clocks::system()) { priority_addr_list_ = AdnlAddressList{}; priority_conns_.clear(); @@ -665,14 +723,18 @@ td::Result, bool>> AdnlPeerP } } - for (auto &conn : priority_conns_) { - if (conn.ready() && (!direct_only || conn.is_direct())) { - return std::make_pair(conn.conn.get(), conn.is_direct()); + for (int direct_only = 1; direct_only >= 0; --direct_only) { + for (auto &conn : priority_conns_) { + if (conn.ready() && (!direct_only || conn.is_direct())) { + return std::make_pair(conn.conn.get(), conn.is_direct()); + } } } - for (auto &conn : conns_) { - if (conn.ready() && (!direct_only || conn.is_direct())) { - return std::make_pair(conn.conn.get(), conn.is_direct()); + for (int direct_only = 1; direct_only >= 0; --direct_only) { + for (auto &conn : conns_) { + if (conn.ready() && (!direct_only || conn.is_direct())) { + return std::make_pair(conn.conn.get(), conn.is_direct()); + } } } return td::Status::Error(ErrorCode::notready, "no active connections"); @@ -760,6 +822,55 @@ void AdnlPeerPairImpl::get_conn_ip_str(td::Promise promise) { promise.set_value("undefined"); } +void AdnlPeerPairImpl::get_stats(bool all, td::Promise> promise) { + if (!all) { + double threshold = td::Clocks::system() - 600.0; + if (last_in_packet_ts_ < threshold && last_out_packet_ts_ < threshold) { + promise.set_value(nullptr); + return; + } + } + + auto stats = create_tl_object(); + stats->local_id_ = local_id_.bits256_value(); + stats->peer_id_ = peer_id_short_.bits256_value(); + for (const AdnlAddress &addr : addr_list_.addrs()) { + ton_api::downcast_call(*addr->tl(), td::overloaded( + [&](const ton_api::adnl_address_udp &obj) { + stats->ip_str_ = PSTRING() << td::IPAddress::ipv4_to_str(obj.ip_) << ":" + << obj.port_; + }, + [&](const auto &) {})); + if (!stats->ip_str_.empty()) { + break; + } + } + + prepare_packet_stats(); + stats->last_in_packet_ts_ = last_in_packet_ts_; + stats->last_out_packet_ts_ = last_out_packet_ts_; + stats->packets_total_ = packet_stats_total_.tl(); + stats->packets_total_->ts_start_ = started_ts_; + stats->packets_total_->ts_end_ = td::Clocks::system(); + stats->packets_recent_ = packet_stats_prev_.tl(); + + if (channel_ready_) { + stats->channel_status_ = 2; + } else if (channel_inited_) { + stats->channel_status_ = 1; + } else { + stats->channel_status_ = 0; + } + stats->try_reinit_at_ = (try_reinit_at_ ? try_reinit_at_.at_unix() : 0.0); + stats->connection_ready_ = + std::any_of(conns_.begin(), conns_.end(), [](const Conn &conn) { return conn.ready(); }) || + std::any_of(priority_conns_.begin(), priority_conns_.end(), [](const Conn &conn) { return conn.ready(); }); + stats->out_queue_messages_ = out_messages_queue_.size(); + stats->out_queue_bytes_ = out_messages_queue_total_size_; + + promise.set_result(std::move(stats)); +} + void AdnlPeerImpl::update_id(AdnlNodeIdFull id) { CHECK(id.compute_short_id() == peer_id_short_); if (!peer_id_.empty()) { @@ -783,8 +894,8 @@ void AdnlPeerPairImpl::Conn::create_conn(td::actor::ActorId pe void AdnlPeerPairImpl::conn_change_state(AdnlConnectionIdShort id, bool ready) { if (ready) { - if (pending_messages_.size() > 0) { - send_messages_in(std::move(pending_messages_), true); + if (out_messages_queue_.empty()) { + send_messages_from_queue(); } } } @@ -806,7 +917,7 @@ td::actor::ActorOwn AdnlPeer::create(td::actor::ActorId dst_actor, - AdnlPacket packet) { + AdnlPacket packet, td::uint64 serialized_size) { if (packet.inited_from()) { update_id(packet.from()); } @@ -824,7 +935,7 @@ void AdnlPeerImpl::receive_packet(AdnlNodeIdShort dst, td::uint32 dst_mode, td:: } } - td::actor::send_closure(it->second.get(), &AdnlPeerPair::receive_packet, std::move(packet)); + td::actor::send_closure(it->second.get(), &AdnlPeerPair::receive_packet, std::move(packet), serialized_size); } void AdnlPeerImpl::send_messages(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, @@ -904,6 +1015,58 @@ void AdnlPeerImpl::update_addr_list(AdnlNodeIdShort local_id, td::uint32 local_m td::actor::send_closure(it->second, &AdnlPeerPair::update_addr_list, std::move(addr_list)); } +void AdnlPeerImpl::get_stats(bool all, td::Promise>> promise) { + class Cb : public td::actor::Actor { + public: + explicit Cb(td::Promise>> promise) + : promise_(std::move(promise)) { + } + + void got_peer_pair_stats(tl_object_ptr peer_pair) { + if (peer_pair) { + result_.push_back(std::move(peer_pair)); + } + dec_pending(); + } + + void inc_pending() { + ++pending_; + } + + void dec_pending() { + CHECK(pending_ > 0); + --pending_; + if (pending_ == 0) { + promise_.set_result(std::move(result_)); + stop(); + } + } + + private: + td::Promise>> promise_; + size_t pending_ = 1; + std::vector> result_; + }; + auto callback = td::actor::create_actor("adnlpeerstats", std::move(promise)).release(); + + for (auto &[local_id, peer_pair] : peer_pairs_) { + td::actor::send_closure(callback, &Cb::inc_pending); + td::actor::send_closure(peer_pair, &AdnlPeerPair::get_stats, all, + [local_id = local_id, peer_id = peer_id_short_, + callback](td::Result> R) { + if (R.is_error()) { + VLOG(ADNL_NOTICE) << "failed to get stats for peer pair " << peer_id << "->" << local_id + << " : " << R.move_as_error(); + td::actor::send_closure(callback, &Cb::dec_pending); + } else { + td::actor::send_closure(callback, &Cb::got_peer_pair_stats, R.move_as_ok()); + } + }); + } + td::actor::send_closure(callback, &Cb::dec_pending); +} + + void AdnlPeerPairImpl::got_data_from_db(td::Result R) { received_from_db_ = false; if (R.is_error()) { @@ -987,6 +1150,66 @@ void AdnlPeerPairImpl::request_reverse_ping_result(td::Result R) { } } +void AdnlPeerPairImpl::add_packet_stats(td::uint64 bytes, bool in, bool channel) { + prepare_packet_stats(); + auto add_stats = [&](PacketStats &stats) { + if (in) { + ++stats.in_packets; + stats.in_bytes += bytes; + if (channel) { + ++stats.in_packets_channel; + stats.in_bytes_channel += bytes; + } + } else { + ++stats.out_packets; + stats.out_bytes += bytes; + if (channel) { + ++stats.out_packets_channel; + stats.out_bytes_channel += bytes; + } + } + }; + add_stats(packet_stats_cur_); + add_stats(packet_stats_total_); + if (in) { + last_in_packet_ts_ = td::Clocks::system(); + } else { + last_out_packet_ts_ = td::Clocks::system(); + } +} + +void AdnlPeerPairImpl::add_expired_msg_stats(td::uint64 bytes) { + prepare_packet_stats(); + auto add_stats = [&](PacketStats &stats) { + ++stats.out_expired_messages; + stats.out_expired_bytes += bytes; + }; + add_stats(packet_stats_cur_); + add_stats(packet_stats_total_); +} + +void AdnlPeerPairImpl::prepare_packet_stats() { + double now = td::Clocks::system(); + if (now >= packet_stats_cur_.ts_end) { + packet_stats_prev_ = std::move(packet_stats_cur_); + packet_stats_cur_ = {}; + auto now_int = (int)now; + packet_stats_cur_.ts_start = (double)(now_int / 60 * 60); + packet_stats_cur_.ts_end = packet_stats_cur_.ts_start + 60.0; + if (packet_stats_prev_.ts_end < now - 60.0) { + packet_stats_prev_ = {}; + packet_stats_prev_.ts_end = packet_stats_cur_.ts_start; + packet_stats_prev_.ts_start = packet_stats_prev_.ts_end - 60.0; + } + } +} + +tl_object_ptr AdnlPeerPairImpl::PacketStats::tl() const { + return create_tl_object(ts_start, ts_end, in_packets, in_bytes, in_packets_channel, + in_bytes_channel, out_packets, out_bytes, out_packets_channel, + out_bytes_channel, out_expired_messages, out_expired_bytes); +} + } // namespace adnl } // namespace ton diff --git a/adnl/adnl-peer.h b/adnl/adnl-peer.h index 8488e82e..1215f71d 100644 --- a/adnl/adnl-peer.h +++ b/adnl/adnl-peer.h @@ -39,9 +39,9 @@ class AdnlPeer; class AdnlPeerPair : public td::actor::Actor { public: - virtual void receive_packet_from_channel(AdnlChannelIdShort id, AdnlPacket packet) = 0; + virtual void receive_packet_from_channel(AdnlChannelIdShort id, AdnlPacket packet, td::uint64 serialized_size) = 0; virtual void receive_packet_checked(AdnlPacket packet) = 0; - virtual void receive_packet(AdnlPacket packet) = 0; + virtual void receive_packet(AdnlPacket packet, td::uint64 serialized_size) = 0; virtual void send_messages(std::vector message) = 0; inline void send_message(OutboundAdnlMessage message) { @@ -59,6 +59,7 @@ class AdnlPeerPair : public td::actor::Actor { virtual void update_peer_id(AdnlNodeIdFull id) = 0; virtual void update_addr_list(AdnlAddressList addr_list) = 0; virtual void get_conn_ip_str(td::Promise promise) = 0; + virtual void get_stats(bool all, td::Promise> promise) = 0; static td::actor::ActorOwn create(td::actor::ActorId network_manager, td::actor::ActorId peer_table, td::uint32 local_mode, @@ -71,7 +72,7 @@ class AdnlPeerPair : public td::actor::Actor { class AdnlPeer : public td::actor::Actor { public: virtual void receive_packet(AdnlNodeIdShort dst, td::uint32 dst_mode, td::actor::ActorId dst_actor, - AdnlPacket message) = 0; + AdnlPacket message, td::uint64 serialized_size) = 0; virtual void send_messages(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, std::vector messages) = 0; virtual void send_query(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, @@ -100,6 +101,7 @@ class AdnlPeer : public td::actor::Actor { td::actor::ActorId local_actor, AdnlAddressList addr_list) = 0; virtual void update_dht_node(td::actor::ActorId dht_node) = 0; virtual void get_conn_ip_str(AdnlNodeIdShort l_id, td::Promise promise) = 0; + virtual void get_stats(bool all, td::Promise>> promise) = 0; }; } // namespace adnl diff --git a/adnl/adnl-peer.hpp b/adnl/adnl-peer.hpp index 0efe827d..243974ba 100644 --- a/adnl/adnl-peer.hpp +++ b/adnl/adnl-peer.hpp @@ -20,6 +20,7 @@ #include #include +#include #include "adnl-peer.h" #include "adnl-peer-table.h" @@ -66,12 +67,12 @@ class AdnlPeerPairImpl : public AdnlPeerPair { void discover(); - void receive_packet_from_channel(AdnlChannelIdShort id, AdnlPacket packet) override; + void receive_packet_from_channel(AdnlChannelIdShort id, AdnlPacket packet, td::uint64 serialized_size) override; void receive_packet_checked(AdnlPacket packet) override; - void receive_packet(AdnlPacket packet) override; + void receive_packet(AdnlPacket packet, td::uint64 serialized_size) override; void deliver_message(AdnlMessage message); - void send_messages_in(std::vector messages, bool allow_postpone); + void send_messages_from_queue(); void send_messages(std::vector messages) override; void send_packet_continue(AdnlPacket packet, td::actor::ActorId conn, bool via_channel); void send_query(std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice data, @@ -89,6 +90,7 @@ class AdnlPeerPairImpl : public AdnlPeerPair { void update_peer_id(AdnlNodeIdFull id) override; void get_conn_ip_str(td::Promise promise) override; + void get_stats(bool all, td::Promise> promise) override; void got_data_from_db(td::Result R); void got_data_from_static_nodes(td::Result R); @@ -122,8 +124,9 @@ class AdnlPeerPairImpl : public AdnlPeerPair { } private: + void respond_with_nop(); void reinit(td::int32 date); - td::Result, bool>> get_conn(bool direct_only); + td::Result, bool>> get_conn(); void create_channel(pubkeys::Ed25519 pub, td::uint32 date); bool received_packet(td::uint64 seqno) const { @@ -182,11 +185,11 @@ class AdnlPeerPairImpl : public AdnlPeerPair { Conn() { } - bool ready() { + bool ready() const { return !conn.empty() && conn.get_actor_unsafe().is_active(); } - bool is_direct() { + bool is_direct() const { return addr->is_public(); } @@ -194,7 +197,14 @@ class AdnlPeerPairImpl : public AdnlPeerPair { td::actor::ActorId adnl); }; - std::vector pending_messages_; + // Messages waiting for connection or for nochannel rate limiter + std::queue> out_messages_queue_; + td::uint64 out_messages_queue_total_size_ = 0; + RateLimiter nochannel_rate_limiter_ = RateLimiter(50, 0.5); // max 50, period = 0.5s + td::Timestamp retry_send_at_ = td::Timestamp::never(); + bool disable_dht_query_ = false; + bool skip_init_packet_ = false; + double message_in_queue_ttl_ = 10.0; td::actor::ActorId network_manager_; td::actor::ActorId peer_table_; @@ -214,6 +224,7 @@ class AdnlPeerPairImpl : public AdnlPeerPair { pubkeys::Ed25519 channel_pub_; td::int32 channel_pk_date_; td::actor::ActorOwn channel_; + td::Timestamp respond_with_nop_after_; td::uint64 in_seqno_ = 0; td::uint64 out_seqno_ = 0; @@ -252,17 +263,34 @@ class AdnlPeerPairImpl : public AdnlPeerPair { td::Timestamp next_dht_query_at_ = td::Timestamp::never(); td::Timestamp next_db_update_at_ = td::Timestamp::never(); - td::Timestamp retry_send_at_ = td::Timestamp::never(); + + td::Timestamp last_received_packet_ = td::Timestamp::never(); + td::Timestamp try_reinit_at_ = td::Timestamp::never(); + td::Timestamp drop_addr_list_at_ = td::Timestamp::never(); bool has_reverse_addr_ = false; td::Timestamp request_reverse_ping_after_ = td::Timestamp::now(); bool request_reverse_ping_active_ = false; + + struct PacketStats { + double ts_start = 0.0, ts_end = 0.0; + td::uint64 in_packets = 0, in_bytes = 0, in_packets_channel = 0, in_bytes_channel = 0; + td::uint64 out_packets = 0, out_bytes = 0, out_packets_channel = 0, out_bytes_channel = 0; + td::uint64 out_expired_messages = 0, out_expired_bytes = 0; + + tl_object_ptr tl() const; + } packet_stats_cur_, packet_stats_prev_, packet_stats_total_; + double last_in_packet_ts_ = 0.0, last_out_packet_ts_ = 0.0; + double started_ts_ = td::Clocks::system(); + void add_packet_stats(td::uint64 bytes, bool in, bool channel); + void add_expired_msg_stats(td::uint64 bytes); + void prepare_packet_stats(); }; class AdnlPeerImpl : public AdnlPeer { public: void receive_packet(AdnlNodeIdShort dst, td::uint32 dst_mode, td::actor::ActorId dst_actor, - AdnlPacket packet) override; + AdnlPacket packet, td::uint64 serialized_size) override; void send_messages(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, std::vector messages) override; void send_query(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, std::string name, @@ -275,6 +303,7 @@ class AdnlPeerImpl : public AdnlPeer { AdnlAddressList addr_list) override; void update_dht_node(td::actor::ActorId dht_node) override; void get_conn_ip_str(AdnlNodeIdShort l_id, td::Promise promise) override; + void get_stats(bool all, td::Promise>> promise) override; //void check_signature(td::BufferSlice data, td::BufferSlice signature, td::Promise promise) override; AdnlPeerImpl(td::actor::ActorId network_manager, td::actor::ActorId peer_table, diff --git a/adnl/adnl-query.cpp b/adnl/adnl-query.cpp index e098c134..89dbbfc4 100644 --- a/adnl/adnl-query.cpp +++ b/adnl/adnl-query.cpp @@ -25,7 +25,7 @@ namespace ton { namespace adnl { void AdnlQuery::alarm() { - set_error(td::Status::Error(ErrorCode::timeout, "adnl query timeout")); + set_error(td::Status::Error(ErrorCode::timeout, PSTRING() << "timeout for adnl query " << name_)); } void AdnlQuery::result(td::BufferSlice data) { promise_.set_value(std::move(data)); diff --git a/adnl/adnl.h b/adnl/adnl.h index b7dad216..b49581a9 100644 --- a/adnl/adnl.h +++ b/adnl/adnl.h @@ -97,6 +97,8 @@ class Adnl : public AdnlSenderInterface { virtual void add_id_ex(AdnlNodeIdFull id, AdnlAddressList addr_list, td::uint8 cat, td::uint32 mode) = 0; virtual void del_id(AdnlNodeIdShort id, td::Promise promise) = 0; + virtual void check_id_exists(AdnlNodeIdShort id, td::Promise promise) = 0; + // subscribe to (some) messages(+queries) to this local id virtual void subscribe(AdnlNodeIdShort dst, std::string prefix, std::unique_ptr callback) = 0; virtual void unsubscribe(AdnlNodeIdShort dst, std::string prefix) = 0; @@ -119,6 +121,8 @@ class Adnl : public AdnlSenderInterface { virtual void create_tunnel(AdnlNodeIdShort dst, td::uint32 size, td::Promise, AdnlAddress>> promise) = 0; + virtual void get_stats(bool all, td::Promise> promise) = 0; + static td::actor::ActorOwn create(std::string db, td::actor::ActorId keyring); static std::string int_to_bytestring(td::int32 id) { diff --git a/adnl/utils.hpp b/adnl/utils.hpp index 50aec2ef..18d3f207 100644 --- a/adnl/utils.hpp +++ b/adnl/utils.hpp @@ -40,6 +40,40 @@ inline bool adnl_node_is_older(AdnlNode &a, AdnlNode &b) { return a.addr_list().version() < b.addr_list().version(); } +class RateLimiter { +public: + explicit RateLimiter(td::uint32 capacity, double period) : capacity_(capacity), period_(period), remaining_(capacity) { + } + + bool take() { + while (remaining_ < capacity_ && increment_at_.is_in_past()) { + ++remaining_; + increment_at_ += period_; + } + if (remaining_) { + --remaining_; + if (increment_at_.is_in_past()) { + increment_at_ = td::Timestamp::in(period_); + } + return true; + } + return false; + } + + td::Timestamp ready_at() const { + if (remaining_) { + return td::Timestamp::now(); + } + return increment_at_; + } + +private: + td::uint32 capacity_; + double period_; + td::uint32 remaining_; + td::Timestamp increment_at_ = td::Timestamp::never(); +}; + } // namespace adnl } // namespace ton diff --git a/assembly/android/build-android-tonlib.sh b/assembly/android/build-android-tonlib.sh new file mode 100644 index 00000000..29dbffc8 --- /dev/null +++ b/assembly/android/build-android-tonlib.sh @@ -0,0 +1,60 @@ +with_artifacts=false + +while getopts 'a' flag; do + case "${flag}" in + a) with_artifacts=true ;; + *) break + ;; + esac +done + +export CC=$(which clang-16) +export CXX=$(which clang++-16) +export CCACHE_DISABLE=1 + +if [ ! -d android-ndk-r25b ]; then + rm android-ndk-r25b-linux.zip + echo "Downloading https://dl.google.com/android/repository/android-ndk-r25b-linux.zip" + wget -q https://dl.google.com/android/repository/android-ndk-r25b-linux.zip + unzip -q android-ndk-r25b-linux.zip + test $? -eq 0 || { echo "Can't unzip android-ndk-r25b-linux.zip"; exit 1; } + echo "Android NDK extracted" +else + echo "Using extracted Android NDK" +fi + +export JAVA_AWT_LIBRARY=NotNeeded +export JAVA_JVM_LIBRARY=NotNeeded +export JAVA_INCLUDE_PATH=${JAVA_HOME}/include +export JAVA_AWT_INCLUDE_PATH=${JAVA_HOME}/include +export JAVA_INCLUDE_PATH2=${JAVA_HOME}/include/linux + +export ANDROID_NDK_ROOT=$(pwd)/android-ndk-r25b +export NDK_PLATFORM="android-21" +export ANDROID_PLATFORM="android-21" +export OPENSSL_DIR=$(pwd)/example/android/third_party/crypto + +rm -rf example/android/src/drinkless/org/ton/TonApi.java +cd example/android/ + +rm CMakeCache.txt .ninja_* +cmake -GNinja -DTON_ONLY_TONLIB=ON . + +test $? -eq 0 || { echo "Can't configure TON"; exit 1; } + +ninja prepare_cross_compiling + +test $? -eq 0 || { echo "Can't compile prepare_cross_compiling"; exit 1; } + +rm CMakeCache.txt .ninja_* + +. ./build-all.sh + +find . -name "*.debug" -type f -delete + +if [ "$with_artifacts" = true ]; then + cd ../.. + mkdir -p artifacts/tonlib-android-jni + cp example/android/src/drinkless/org/ton/TonApi.java artifacts/tonlib-android-jni/ + cp -R example/android/libs/* artifacts/tonlib-android-jni/ +fi diff --git a/assembly/appimage/AppRun b/assembly/appimage/AppRun new file mode 100644 index 00000000..c7f147b3 --- /dev/null +++ b/assembly/appimage/AppRun @@ -0,0 +1,3 @@ +#!/bin/sh +export LD_LIBRARY_PATH="${APPDIR}/usr/lib:${LD_LIBRARY_PATH}" +exec "$(dirname $0)"/usr/bin/app "$@" diff --git a/assembly/appimage/create-appimages.sh b/assembly/appimage/create-appimages.sh new file mode 100644 index 00000000..2a8cd0ec --- /dev/null +++ b/assembly/appimage/create-appimages.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +if [ ! -d "artifacts" ]; then + echo "No artifacts found." + exit 2 +fi +# x86_64 or aarch64 +ARCH=$1 + +rm -rf appimages + +mkdir -p appimages/artifacts + +wget -nc https://github.com/AppImage/appimagetool/releases/download/continuous/appimagetool-$ARCH.AppImage +chmod +x ./appimagetool-$ARCH.AppImage + +cd appimages +for file in ../artifacts/*; do + if [[ -f "$file" && "$file" != *.so ]]; then + appName=$(basename "$file") + echo $appName + # prepare AppDir + mkdir -p $appName.AppDir/usr/{bin,lib} + cp ../AppRun $appName.AppDir/AppRun + sed -i "s/app/$appName/g" $appName.AppDir/AppRun + chmod +x ./$appName.AppDir/AppRun + printf '[Desktop Entry]\nName='$appName'\nExec='$appName'\nIcon='$appName'\nType=Application\nCategories=Utility;\n' > $appName.AppDir/$appName.desktop + cp ../ton.png $appName.AppDir/$appName.png + cp $file $appName.AppDir/usr/bin/ + cp ../build/openssl_3/libcrypto.so.3 \ + /lib/$ARCH-linux-gnu/libatomic.so.1 \ + /lib/$ARCH-linux-gnu/libsodium.so.23 \ + /lib/$ARCH-linux-gnu/libz.so.1 \ + /lib/$ARCH-linux-gnu/liblz4.so.1 \ + /lib/$ARCH-linux-gnu/libmicrohttpd.so.12 \ + /lib/$ARCH-linux-gnu/libreadline.so.8 \ + /lib/$ARCH-linux-gnu/libstdc++.so.6 \ + $appName.AppDir/usr/lib/ + + chmod +x ./$appName.AppDir/usr/bin/$appName + # create AppImage + ./../appimagetool-$ARCH.AppImage -l $appName.AppDir + mv $appName-$ARCH.AppImage artifacts/$appName + fi +done + +ls -larth artifacts +cp -r ../artifacts/{smartcont,lib} artifacts/ +pwd +ls -larth artifacts diff --git a/assembly/appimage/ton.png b/assembly/appimage/ton.png new file mode 100644 index 00000000..2a25c863 Binary files /dev/null and b/assembly/appimage/ton.png differ diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh new file mode 100644 index 00000000..b785339e --- /dev/null +++ b/assembly/native/build-macos-portable.sh @@ -0,0 +1,189 @@ +#/bin/bash + +with_tests=false +with_artifacts=false +OSX_TARGET=10.15 + + +while getopts 'tao:' flag; do + case "${flag}" in + t) with_tests=true ;; + a) with_artifacts=true ;; + o) OSX_TARGET=${OPTARG} ;; + *) break + ;; + esac +done + +if [ ! -d "build" ]; then + mkdir build + cd build +else + cd build + rm -rf .ninja* CMakeCache.txt +fi + +export NONINTERACTIVE=1 +brew install ninja pkg-config automake libtool autoconf texinfo +brew install llvm@16 + + +if [ -f /opt/homebrew/opt/llvm@16/bin/clang ]; then + export CC=/opt/homebrew/opt/llvm@16/bin/clang + export CXX=/opt/homebrew/opt/llvm@16/bin/clang++ +else + export CC=/usr/local/opt/llvm@16/bin/clang + export CXX=/usr/local/opt/llvm@16/bin/clang++ +fi +export CCACHE_DISABLE=1 + +if [ ! -d "lz4" ]; then +git clone https://github.com/lz4/lz4.git +cd lz4 +lz4Path=`pwd` +git checkout v1.9.4 +make -j12 +test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } +cd .. +# ./lib/liblz4.a +# ./lib +else + lz4Path=$(pwd)/lz4 + echo "Using compiled lz4" +fi + +if [ ! -d "libsodium" ]; then + export LIBSODIUM_FULL_BUILD=1 + git clone https://github.com/jedisct1/libsodium.git + cd libsodium + sodiumPath=`pwd` + git checkout 1.0.18 + ./autogen.sh + ./configure --with-pic --enable-static + make -j12 + test $? -eq 0 || { echo "Can't compile libsodium"; exit 1; } + cd .. +else + sodiumPath=$(pwd)/libsodium + echo "Using compiled libsodium" +fi + +if [ ! -d "openssl_3" ]; then + git clone https://github.com/openssl/openssl openssl_3 + cd openssl_3 + opensslPath=`pwd` + git checkout openssl-3.1.4 + ./config + make build_libs -j12 + test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } + cd .. +else + opensslPath=$(pwd)/openssl_3 + echo "Using compiled openssl_3" +fi + +if [ ! -d "zlib" ]; then + git clone https://github.com/madler/zlib.git + cd zlib + zlibPath=`pwd` + ./configure --static + make -j12 + test $? -eq 0 || { echo "Can't compile zlib"; exit 1; } + cd .. +else + zlibPath=$(pwd)/zlib + echo "Using compiled zlib" +fi + +if [ ! -d "libmicrohttpd" ]; then + git clone https://git.gnunet.org/libmicrohttpd.git + cd libmicrohttpd + libmicrohttpdPath=`pwd` + ./autogen.sh + ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic + make -j12 + test $? -eq 0 || { echo "Can't compile libmicrohttpd"; exit 1; } + cd .. +else + libmicrohttpdPath=$(pwd)/libmicrohttpd + echo "Using compiled libmicrohttpd" +fi + +cmake -GNinja .. \ +-DPORTABLE=1 \ +-DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=$OSX_TARGET \ +-DCMAKE_CXX_FLAGS="-stdlib=libc++" \ +-DCMAKE_BUILD_TYPE=Release \ +-DOPENSSL_FOUND=1 \ +-DOPENSSL_INCLUDE_DIR=$opensslPath/include \ +-DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a \ +-DZLIB_FOUND=1 \ +-DZLIB_INCLUDE_DIR=$zlibPath \ +-DZLIB_LIBRARIES=$zlibPath/libz.a \ +-DSODIUM_FOUND=1 \ +-DSODIUM_INCLUDE_DIR=$sodiumPath/src/libsodium/include \ +-DSODIUM_LIBRARY_RELEASE=$sodiumPath/src/libsodium/.libs/libsodium.a \ +-DMHD_FOUND=1 \ +-DMHD_INCLUDE_DIR=$libmicrohttpdPath/src/include \ +-DMHD_LIBRARY=$libmicrohttpdPath/src/microhttpd/.libs/libmicrohttpd.a \ +-DLZ4_FOUND=1 \ +-DLZ4_INCLUDE_DIRS=$lz4Path/lib \ +-DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a + + +test $? -eq 0 || { echo "Can't configure ton"; exit 1; } + +if [ "$with_tests" = true ]; then + ninja storage-daemon storage-daemon-cli blockchain-explorer \ + tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ + lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \ + http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ + test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont \ + test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ + test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver + test $? -eq 0 || { echo "Can't compile ton"; exit 1; } +else + ninja storage-daemon storage-daemon-cli blockchain-explorer \ + tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ + lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \ + http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver + test $? -eq 0 || { echo "Can't compile ton"; exit 1; } +fi + +cd .. + +if [ "$with_artifacts" = true ]; then + echo Creating artifacts... + rm -rf artifacts + mkdir artifacts + cp build/storage/storage-daemon/storage-daemon artifacts/ + cp build/storage/storage-daemon/storage-daemon-cli artifacts/ + cp build/blockchain-explorer/blockchain-explorer artifacts/ + cp build/crypto/fift artifacts/ + cp build/crypto/func artifacts/ + cp build/tolk/tolk artifacts/ + cp build/crypto/create-state artifacts/ + cp build/crypto/tlbc artifacts/ + cp build/validator-engine-console/validator-engine-console artifacts/ + cp build/tonlib/tonlib-cli artifacts/ + cp build/tonlib/libtonlibjson.0.5.dylib artifacts/libtonlibjson.dylib + cp build/http/http-proxy artifacts/ + cp build/rldp-http-proxy/rldp-http-proxy artifacts/ + cp build/dht-server/dht-server artifacts/ + cp build/lite-client/lite-client artifacts/ + cp build/validator-engine/validator-engine artifacts/ + cp build/utils/generate-random-id artifacts/ + cp build/utils/json2tlo artifacts/ + cp build/utils/proxy-liteserver artifacts/ + cp build/adnl/adnl-proxy artifacts/ + cp build/emulator/libemulator.dylib artifacts/ + rsync -r crypto/smartcont artifacts/ + rsync -r crypto/fift/lib artifacts/ + chmod -R +x artifacts/* +fi + +if [ "$with_tests" = true ]; then + cd build +# ctest --output-on-failure -E "test-catchain|test-actors" + ctest --output-on-failure +fi diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh new file mode 100644 index 00000000..5c4a5fe0 --- /dev/null +++ b/assembly/native/build-macos-shared.sh @@ -0,0 +1,117 @@ +#/bin/bash + +with_tests=false +with_artifacts=false +OSX_TARGET=10.15 + + +while getopts 'tao:' flag; do + case "${flag}" in + t) with_tests=true ;; + a) with_artifacts=true ;; + o) OSX_TARGET=${OPTARG} ;; + *) break + ;; + esac +done + +if [ ! -d "build" ]; then + mkdir build + cd build +else + cd build + rm -rf .ninja* CMakeCache.txt +fi + +export NONINTERACTIVE=1 +brew install ninja libsodium libmicrohttpd pkg-config automake libtool autoconf gnutls +brew install llvm@16 + +if [ -f /opt/homebrew/opt/llvm@16/bin/clang ]; then + export CC=/opt/homebrew/opt/llvm@16/bin/clang + export CXX=/opt/homebrew/opt/llvm@16/bin/clang++ +else + export CC=/usr/local/opt/llvm@16/bin/clang + export CXX=/usr/local/opt/llvm@16/bin/clang++ +fi +export CCACHE_DISABLE=1 + +if [ ! -d "lz4" ]; then + git clone https://github.com/lz4/lz4 + cd lz4 + lz4Path=`pwd` + git checkout v1.9.4 + make -j12 + test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } + cd .. +else + lz4Path=$(pwd)/lz4 + echo "Using compiled lz4" +fi + +brew unlink openssl@1.1 +brew install openssl@3 +brew unlink openssl@3 && brew link --overwrite openssl@3 + +cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \ +-DCMAKE_CXX_FLAGS="-stdlib=libc++" \ +-DLZ4_FOUND=1 \ +-DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a \ +-DLZ4_INCLUDE_DIRS=$lz4Path/lib + +test $? -eq 0 || { echo "Can't configure ton"; exit 1; } + +if [ "$with_tests" = true ]; then + ninja storage-daemon storage-daemon-cli blockchain-explorer \ + tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ + lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \ + http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ + test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont \ + test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ + test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver + test $? -eq 0 || { echo "Can't compile ton"; exit 1; } +else + ninja storage-daemon storage-daemon-cli blockchain-explorer \ + tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ + lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \ + http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver + test $? -eq 0 || { echo "Can't compile ton"; exit 1; } +fi + +cd .. + +if [ "$with_artifacts" = true ]; then + echo Creating artifacts... + rm -rf artifacts + mkdir artifacts + cp build/storage/storage-daemon/storage-daemon artifacts/ + cp build/storage/storage-daemon/storage-daemon-cli artifacts/ + cp build/blockchain-explorer/blockchain-explorer artifacts/ + cp build/crypto/fift artifacts/ + cp build/crypto/func artifacts/ + cp build/tolk/tolk artifacts/ + cp build/crypto/create-state artifacts/ + cp build/crypto/tlbc artifacts/ + cp build/validator-engine-console/validator-engine-console artifacts/ + cp build/tonlib/tonlib-cli artifacts/ + cp build/tonlib/libtonlibjson.0.5.dylib artifacts/libtonlibjson.dylib + cp build/http/http-proxy artifacts/ + cp build/rldp-http-proxy/rldp-http-proxy artifacts/ + cp build/dht-server/dht-server artifacts/ + cp build/lite-client/lite-client artifacts/ + cp build/validator-engine/validator-engine artifacts/ + cp build/utils/generate-random-id artifacts/ + cp build/utils/json2tlo artifacts/ + cp build/utils/proxy-liteserver artifacts/ + cp build/adnl/adnl-proxy artifacts/ + cp build/emulator/libemulator.dylib artifacts/ + cp -R crypto/smartcont artifacts/ + cp -R crypto/fift/lib artifacts/ + chmod -R +x artifacts/* +fi + +if [ "$with_tests" = true ]; then + cd build +# ctest --output-on-failure -E "test-catchain|test-actors" + ctest --output-on-failure --timeout 1800 +fi diff --git a/assembly/native/build-ubuntu-appimages.sh b/assembly/native/build-ubuntu-appimages.sh new file mode 100644 index 00000000..4e63234d --- /dev/null +++ b/assembly/native/build-ubuntu-appimages.sh @@ -0,0 +1,109 @@ +#/bin/bash + +with_tests=false +with_artifacts=false + + +while getopts 'ta' flag; do + case "${flag}" in + t) with_tests=true ;; + a) with_artifacts=true ;; + *) break + ;; + esac +done + +if [ ! -d "build" ]; then + mkdir build + cd build +else + cd build + rm -rf .ninja* CMakeCache.txt +fi + +export CC=$(which clang-16) +export CXX=$(which clang++-16) +export CCACHE_DISABLE=1 + +if [ ! -d "openssl_3" ]; then + git clone https://github.com/openssl/openssl openssl_3 + cd openssl_3 + opensslPath=`pwd` + git checkout openssl-3.1.4 + ./config + make build_libs -j12 + test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } + cd .. +else + opensslPath=$(pwd)/openssl_3 + echo "Using compiled openssl_3" +fi + +cmake -GNinja .. \ +-DCMAKE_BUILD_TYPE=Release \ +-DPORTABLE=1 \ +-DOPENSSL_ROOT_DIR=$opensslPath \ +-DOPENSSL_INCLUDE_DIR=$opensslPath/include \ +-DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.so + + +test $? -eq 0 || { echo "Can't configure ton"; exit 1; } + +if [ "$with_tests" = true ]; then +ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ + validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ + adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ + test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ + test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ + test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver + test $? -eq 0 || { echo "Can't compile ton"; exit 1; } +else +ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ + validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ + adnl-proxy create-state emulator proxy-liteserver + test $? -eq 0 || { echo "Can't compile ton"; exit 1; } +fi + +# simple binaries' test +./storage/storage-daemon/storage-daemon -V || exit 1 +./validator-engine/validator-engine -V || exit 1 +./lite-client/lite-client -V || exit 1 +./crypto/fift -V || exit 1 + +echo validator-engine +ldd ./validator-engine/validator-engine || exit 1 +ldd ./validator-engine-console/validator-engine-console || exit 1 +ldd ./crypto/fift || exit 1 +echo blockchain-explorer +ldd ./blockchain-explorer/blockchain-explorer || exit 1 +echo libtonlibjson.so +ldd ./tonlib/libtonlibjson.so.0.5 || exit 1 +echo libemulator.so +ldd ./emulator/libemulator.so || exit 1 + +cd .. + +if [ "$with_artifacts" = true ]; then + rm -rf artifacts + mkdir artifacts + mv build/tonlib/libtonlibjson.so.0.5 build/tonlib/libtonlibjson.so + cp build/storage/storage-daemon/storage-daemon build/storage/storage-daemon/storage-daemon-cli \ + build/crypto/fift build/crypto/tlbc build/crypto/func build/tolk/tolk build/crypto/create-state build/blockchain-explorer/blockchain-explorer \ + build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli build/utils/proxy-liteserver \ + build/tonlib/libtonlibjson.so build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy \ + build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine \ + build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.so \ + artifacts + test $? -eq 0 || { echo "Can't copy final binaries"; exit 1; } + cp -R crypto/smartcont artifacts + cp -R crypto/fift/lib artifacts + chmod -R +x artifacts/* +fi + +if [ "$with_tests" = true ]; then + cd build +# ctest --output-on-failure -E "test-catchain|test-actors|test-smartcont|test-adnl|test-validator-session-state|test-dht|test-rldp" + ctest --output-on-failure --timeout 1800 +fi diff --git a/assembly/native/build-ubuntu-portable-libs.sh b/assembly/native/build-ubuntu-portable-libs.sh new file mode 100644 index 00000000..2f0a1ba4 --- /dev/null +++ b/assembly/native/build-ubuntu-portable-libs.sh @@ -0,0 +1,132 @@ +#/bin/bash + +#sudo apt-get update +#sudo apt-get install -y build-essential git cmake ninja-build automake libtool texinfo autoconf libc++-dev libc++abi-dev + +with_artifacts=false + +while getopts 'ta' flag; do + case "${flag}" in + a) with_artifacts=true ;; + *) break + ;; + esac +done + +if [ ! -d "build" ]; then + mkdir build + cd build +else + cd build + rm -rf .ninja* CMakeCache.txt +fi + +export CC=$(which clang) +export CXX=$(which clang++) +export CCACHE_DISABLE=1 + +if [ ! -d "lz4" ]; then +git clone https://github.com/lz4/lz4.git +cd lz4 +lz4Path=`pwd` +git checkout v1.9.4 +CFLAGS="-fPIC" make -j12 +test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } +cd .. +# ./lib/liblz4.a +# ./lib +else + lz4Path=$(pwd)/lz4 + echo "Using compiled lz4" +fi + +if [ ! -d "libsodium" ]; then + export LIBSODIUM_FULL_BUILD=1 + git clone https://github.com/jedisct1/libsodium.git + cd libsodium + sodiumPath=`pwd` + git checkout 1.0.18 + ./autogen.sh + ./configure --with-pic --enable-static + make -j12 + test $? -eq 0 || { echo "Can't compile libsodium"; exit 1; } + cd .. +else + sodiumPath=$(pwd)/libsodium + echo "Using compiled libsodium" +fi + +if [ ! -d "openssl_3" ]; then + git clone https://github.com/openssl/openssl openssl_3 + cd openssl_3 + opensslPath=`pwd` + git checkout openssl-3.1.4 + ./config + make build_libs -j12 + test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } + cd .. +else + opensslPath=$(pwd)/openssl_3 + echo "Using compiled openssl_3" +fi + +if [ ! -d "zlib" ]; then + git clone https://github.com/madler/zlib.git + cd zlib + zlibPath=`pwd` + ./configure --static + make -j12 + test $? -eq 0 || { echo "Can't compile zlib"; exit 1; } + cd .. +else + zlibPath=$(pwd)/zlib + echo "Using compiled zlib" +fi + +if [ ! -d "libmicrohttpd" ]; then + git clone https://git.gnunet.org/libmicrohttpd.git + cd libmicrohttpd + libmicrohttpdPath=`pwd` + ./autogen.sh + ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic + make -j12 + test $? -eq 0 || { echo "Can't compile libmicrohttpd"; exit 1; } + cd .. +else + libmicrohttpdPath=$(pwd)/libmicrohttpd + echo "Using compiled libmicrohttpd" +fi + +cmake -GNinja .. \ +-DPORTABLE=1 \ +-DCMAKE_BUILD_TYPE=Release \ +-DOPENSSL_FOUND=1 \ +-DOPENSSL_INCLUDE_DIR=$opensslPath/include \ +-DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a \ +-DZLIB_FOUND=1 \ +-DZLIB_INCLUDE_DIR=$zlibPath \ +-DZLIB_LIBRARIES=$zlibPath/libz.a \ +-DSODIUM_FOUND=1 \ +-DSODIUM_INCLUDE_DIR=$sodiumPath/src/libsodium/include \ +-DSODIUM_LIBRARY_RELEASE=$sodiumPath/src/libsodium/.libs/libsodium.a \ +-DMHD_FOUND=1 \ +-DMHD_INCLUDE_DIR=$libmicrohttpdPath/src/include \ +-DMHD_LIBRARY=$libmicrohttpdPath/src/microhttpd/.libs/libmicrohttpd.a \ +-DLZ4_FOUND=1 \ +-DLZ4_INCLUDE_DIRS=$lz4Path/lib \ +-DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a + + + +test $? -eq 0 || { echo "Can't configure ton"; exit 1; } + +ninja tonlibjson emulator +test $? -eq 0 || { echo "Can't compile ton"; exit 1; } + +cd .. + +mkdir artifacts +mv build/tonlib/libtonlibjson.so.0.5 build/tonlib/libtonlibjson.so +cp build/tonlib/libtonlibjson.so \ + build/emulator/libemulator.so \ + artifacts diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh new file mode 100644 index 00000000..16e77ac8 --- /dev/null +++ b/assembly/native/build-ubuntu-portable.sh @@ -0,0 +1,172 @@ +#/bin/bash + +#sudo apt-get update +#sudo apt-get install -y build-essential git cmake ninja-build automake libtool texinfo autoconf libc++-dev libc++abi-dev + +with_tests=false +with_artifacts=false + + +while getopts 'ta' flag; do + case "${flag}" in + t) with_tests=true ;; + a) with_artifacts=true ;; + *) break + ;; + esac +done + +if [ ! -d "build" ]; then + mkdir build + cd build +else + cd build + rm -rf .ninja* CMakeCache.txt +fi + +export CC=$(which clang) +export CXX=$(which clang++) +export CCACHE_DISABLE=1 + +if [ ! -d "lz4" ]; then +git clone https://github.com/lz4/lz4.git +cd lz4 +lz4Path=`pwd` +git checkout v1.9.4 +CFLAGS="-fPIC" make -j12 +test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } +cd .. +# ./lib/liblz4.a +# ./lib +else + lz4Path=$(pwd)/lz4 + echo "Using compiled lz4" +fi + +if [ ! -d "libsodium" ]; then + export LIBSODIUM_FULL_BUILD=1 + git clone https://github.com/jedisct1/libsodium.git + cd libsodium + sodiumPath=`pwd` + git checkout 1.0.18 + ./autogen.sh + ./configure --with-pic --enable-static + make -j12 + test $? -eq 0 || { echo "Can't compile libsodium"; exit 1; } + cd .. +else + sodiumPath=$(pwd)/libsodium + echo "Using compiled libsodium" +fi + +if [ ! -d "openssl_3" ]; then + git clone https://github.com/openssl/openssl openssl_3 + cd openssl_3 + opensslPath=`pwd` + git checkout openssl-3.1.4 + ./config + make build_libs -j12 + test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } + cd .. +else + opensslPath=$(pwd)/openssl_3 + echo "Using compiled openssl_3" +fi + +if [ ! -d "zlib" ]; then + git clone https://github.com/madler/zlib.git + cd zlib + zlibPath=`pwd` + ./configure --static + make -j12 + test $? -eq 0 || { echo "Can't compile zlib"; exit 1; } + cd .. +else + zlibPath=$(pwd)/zlib + echo "Using compiled zlib" +fi + +if [ ! -d "libmicrohttpd" ]; then + git clone https://git.gnunet.org/libmicrohttpd.git + cd libmicrohttpd + libmicrohttpdPath=`pwd` + ./autogen.sh + ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic + make -j12 + test $? -eq 0 || { echo "Can't compile libmicrohttpd"; exit 1; } + cd .. +else + libmicrohttpdPath=$(pwd)/libmicrohttpd + echo "Using compiled libmicrohttpd" +fi + +cmake -GNinja .. \ +-DPORTABLE=1 \ +-DCMAKE_BUILD_TYPE=Release \ +-DOPENSSL_FOUND=1 \ +-DOPENSSL_INCLUDE_DIR=$opensslPath/include \ +-DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a \ +-DZLIB_FOUND=1 \ +-DZLIB_INCLUDE_DIR=$zlibPath \ +-DZLIB_LIBRARIES=$zlibPath/libz.a \ +-DSODIUM_FOUND=1 \ +-DSODIUM_INCLUDE_DIR=$sodiumPath/src/libsodium/include \ +-DSODIUM_LIBRARY_RELEASE=$sodiumPath/src/libsodium/.libs/libsodium.a \ +-DMHD_FOUND=1 \ +-DMHD_INCLUDE_DIR=$libmicrohttpdPath/src/include \ +-DMHD_LIBRARY=$libmicrohttpdPath/src/microhttpd/.libs/libmicrohttpd.a \ +-DLZ4_FOUND=1 \ +-DLZ4_INCLUDE_DIRS=$lz4Path/lib \ +-DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a + + + +test $? -eq 0 || { echo "Can't configure ton"; exit 1; } + +if [ "$with_tests" = true ]; then +ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ + validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ + adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ + test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ + test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ + test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver + test $? -eq 0 || { echo "Can't compile ton"; exit 1; } +else +ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ + validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ + adnl-proxy create-state emulator proxy-liteserver + test $? -eq 0 || { echo "Can't compile ton"; exit 1; } +fi + +# simple binaries' test +./storage/storage-daemon/storage-daemon -V || exit 1 +./validator-engine/validator-engine -V || exit 1 +./lite-client/lite-client -V || exit 1 +./crypto/fift -V || exit 1 + +cd .. + +if [ "$with_artifacts" = true ]; then + rm -rf artifacts + mkdir artifacts + mv build/tonlib/libtonlibjson.so.0.5 build/tonlib/libtonlibjson.so + cp build/storage/storage-daemon/storage-daemon build/storage/storage-daemon/storage-daemon-cli \ + build/crypto/fift build/crypto/tlbc build/crypto/func build/tolk/tolk build/crypto/create-state build/blockchain-explorer/blockchain-explorer \ + build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli build/utils/proxy-liteserver \ + build/tonlib/libtonlibjson.so build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy \ + build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine \ + build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.so \ + artifacts + test $? -eq 0 || { echo "Can't copy final binaries"; exit 1; } + cp -R crypto/smartcont artifacts + cp -R crypto/fift/lib artifacts + chmod -R +x artifacts/* +fi + +if [ "$with_tests" = true ]; then + cd build +# ctest --output-on-failure -E "test-catchain|test-actors|test-smartcont|test-adnl|test-validator-session-state|test-dht|test-rldp" + ctest --output-on-failure -E "test-adnl" +fi diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh new file mode 100644 index 00000000..49cc8e1e --- /dev/null +++ b/assembly/native/build-ubuntu-shared.sh @@ -0,0 +1,102 @@ +#/bin/bash + +#sudo apt-get update +#sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev + +with_tests=false +with_artifacts=false + + +while getopts 'ta' flag; do + case "${flag}" in + t) with_tests=true ;; + a) with_artifacts=true ;; + *) break + ;; + esac +done + +if [ ! -d "build" ]; then + mkdir build + cd build +else + cd build + rm -rf .ninja* CMakeCache.txt +fi + +export CC=$(which clang-16) +export CXX=$(which clang++-16) +export CCACHE_DISABLE=1 + +if [ ! -d "openssl_3" ]; then + git clone https://github.com/openssl/openssl openssl_3 + cd openssl_3 + opensslPath=`pwd` + git checkout openssl-3.1.4 + ./config + make build_libs -j12 + test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } + cd .. +else + opensslPath=$(pwd)/openssl_3 + echo "Using compiled openssl_3" +fi + +cmake -GNinja -DTON_USE_JEMALLOC=ON .. \ +-DCMAKE_BUILD_TYPE=Release \ +-DOPENSSL_ROOT_DIR=$opensslPath \ +-DOPENSSL_INCLUDE_DIR=$opensslPath/include \ +-DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.so + + +test $? -eq 0 || { echo "Can't configure ton"; exit 1; } + +if [ "$with_tests" = true ]; then +ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ + validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ + adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ + test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ + test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ + test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver + test $? -eq 0 || { echo "Can't compile ton"; exit 1; } +else +ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ + validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ + adnl-proxy create-state emulator proxy-liteserver + test $? -eq 0 || { echo "Can't compile ton"; exit 1; } +fi + +# simple binaries' test +./storage/storage-daemon/storage-daemon -V || exit 1 +./validator-engine/validator-engine -V || exit 1 +./lite-client/lite-client -V || exit 1 +./crypto/fift -V || exit 1 + +ldd ./validator-engine/validator-engine || exit 1 + +cd .. + +if [ "$with_artifacts" = true ]; then + rm -rf artifacts + mkdir artifacts + mv build/tonlib/libtonlibjson.so.0.5 build/tonlib/libtonlibjson.so + cp build/storage/storage-daemon/storage-daemon build/storage/storage-daemon/storage-daemon-cli \ + build/crypto/fift build/crypto/tlbc build/crypto/func build/tolk/tolk build/crypto/create-state build/blockchain-explorer/blockchain-explorer \ + build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli build/utils/proxy-liteserver \ + build/tonlib/libtonlibjson.so build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy \ + build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine \ + build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.so \ + artifacts + test $? -eq 0 || { echo "Can't copy final binaries"; exit 1; } + cp -R crypto/smartcont artifacts + cp -R crypto/fift/lib artifacts + chmod -R +x artifacts/* +fi + +if [ "$with_tests" = true ]; then + cd build +# ctest --output-on-failure -E "test-catchain|test-actors|test-smartcont|test-adnl|test-validator-session-state|test-dht|test-rldp" + ctest --output-on-failure --timeout 1800 +fi diff --git a/assembly/native/build-windows-2019.bat b/assembly/native/build-windows-2019.bat new file mode 100644 index 00000000..844c09fc --- /dev/null +++ b/assembly/native/build-windows-2019.bat @@ -0,0 +1,208 @@ +REM execute this script inside elevated (Run as Administrator) console "x64 Native Tools Command Prompt for VS 2019" + +echo off + +echo Installing chocolatey windows package manager... +@"%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell.exe" -NoProfile -InputFormat None -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" && SET "PATH=%PATH%;%ALLUSERSPROFILE%\chocolatey\bin" +choco -? +IF %errorlevel% NEQ 0 ( + echo Can't install chocolatey + exit /b %errorlevel% +) + +choco feature enable -n allowEmptyChecksums + +echo Installing pkgconfiglite... +choco install -y pkgconfiglite +IF %errorlevel% NEQ 0 ( + echo Can't install pkgconfiglite + exit /b %errorlevel% +) + +echo Installing ninja... +choco install -y ninja +IF %errorlevel% NEQ 0 ( + echo Can't install ninja + exit /b %errorlevel% +) + +echo Installing nasm... +choco install -y nasm +where nasm +SET PATH=%PATH%;C:\Program Files\NASM +IF %errorlevel% NEQ 0 ( + echo Can't install nasm + exit /b %errorlevel% +) + +mkdir third_libs +cd third_libs + +set third_libs=%cd% +echo %third_libs% + +if not exist "zlib" ( + git clone https://github.com/madler/zlib.git + cd zlib + git checkout v1.3.1 + cd contrib\vstudio\vc14 + msbuild zlibstat.vcxproj /p:Configuration=ReleaseWithoutAsm /p:platform=x64 -p:PlatformToolset=v142 + cd ..\..\..\.. +) else ( + echo Using zlib... +) + +if not exist "lz4" ( + git clone https://github.com/lz4/lz4.git + cd lz4 + git checkout v1.9.4 + cd build\VS2017\liblz4 + msbuild liblz4.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v142 + cd ..\..\..\.. +) else ( + echo Using lz4... +) + +if not exist "libsodium" ( + git clone https://github.com/jedisct1/libsodium + cd libsodium + git checkout 1.0.18-RELEASE + msbuild libsodium.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v142 + cd .. +) else ( + echo Using libsodium... +) + +if not exist "openssl" ( + git clone https://github.com/openssl/openssl.git + cd openssl + git checkout openssl-3.1.4 + where perl + perl Configure VC-WIN64A + IF %errorlevel% NEQ 0 ( + echo Can't configure openssl + exit /b %errorlevel% + ) + nmake + cd .. +) else ( + echo Using openssl... +) + +if not exist "libmicrohttpd" ( + git clone https://github.com/Karlson2k/libmicrohttpd.git + cd libmicrohttpd + git checkout v1.0.1 + cd w32\VS2019 + msbuild libmicrohttpd.vcxproj /p:Configuration=Release-static /p:platform=x64 -p:PlatformToolset=v142 + IF %errorlevel% NEQ 0 ( + echo Can't compile libmicrohttpd + exit /b %errorlevel% + ) + cd ../../.. +) else ( + echo Using libmicrohttpd... +) + +cd .. +echo Current dir %cd% + +mkdir build +cd build +cmake -GNinja -DCMAKE_BUILD_TYPE=Release ^ +-DPORTABLE=1 ^ +-DSODIUM_USE_STATIC_LIBS=1 ^ +-DSODIUM_LIBRARY_RELEASE=%third_libs%\libsodium\Build\Release\x64\libsodium.lib ^ +-DSODIUM_LIBRARY_DEBUG=%third_libs%\libsodium\Build\Release\x64\libsodium.lib ^ +-DSODIUM_INCLUDE_DIR=%third_libs%\libsodium\src\libsodium\include ^ +-DLZ4_FOUND=1 ^ +-DLZ4_INCLUDE_DIRS=%third_libs%\lz4\lib ^ +-DLZ4_LIBRARIES=%third_libs%\lz4\build\VS2017\liblz4\bin\x64_Release\liblz4_static.lib ^ +-DMHD_FOUND=1 ^ +-DMHD_LIBRARY=%third_libs%\libmicrohttpd\w32\VS2019\Output\x64\libmicrohttpd.lib ^ +-DMHD_INCLUDE_DIR=%third_libs%\libmicrohttpd\src\include ^ +-DZLIB_FOUND=1 ^ +-DZLIB_INCLUDE_DIR=%third_libs%\zlib ^ +-DZLIB_LIBRARIES=%third_libs%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib ^ +-DOPENSSL_FOUND=1 ^ +-DOPENSSL_INCLUDE_DIR=%third_libs%\openssl\include ^ +-DOPENSSL_CRYPTO_LIBRARY=%third_libs%\openssl\libcrypto_static.lib ^ +-DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj" .. + +IF %errorlevel% NEQ 0 ( + echo Can't configure TON + exit /b %errorlevel% +) + +IF "%1"=="-t" ( +ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ +tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^ +json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ +test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net ^ +test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ +test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver +IF %errorlevel% NEQ 0 ( + echo Can't compile TON + exit /b %errorlevel% +) +) else ( +ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ +tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^ +json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver +IF %errorlevel% NEQ 0 ( + echo Can't compile TON + exit /b %errorlevel% +) +) + +copy validator-engine\validator-engine.exe test +IF %errorlevel% NEQ 0 ( + echo validator-engine.exe does not exist + exit /b %errorlevel% +) + +IF "%1"=="-t" ( + echo Running tests... +REM ctest -C Release --output-on-failure -E "test-catchain|test-actors|test-validator-session-state" + ctest -C Release --output-on-failure -E "test-bigint" --timeout 1800 + IF %errorlevel% NEQ 0 ( + echo Some tests failed + exit /b %errorlevel% + ) +) + +echo Strip and copy artifacts +cd .. +echo where strip +where strip +mkdir artifacts +mkdir artifacts\smartcont +mkdir artifacts\lib + +for %%I in (build\storage\storage-daemon\storage-daemon.exe ^ + build\storage\storage-daemon\storage-daemon-cli.exe ^ + build\blockchain-explorer\blockchain-explorer.exe ^ + build\crypto\fift.exe ^ + build\crypto\tlbc.exe ^ + build\crypto\func.exe ^ + build\tolk\tolk.exe ^ + build\crypto\create-state.exe ^ + build\validator-engine-console\validator-engine-console.exe ^ + build\tonlib\tonlib-cli.exe ^ + build\tonlib\tonlibjson.dll ^ + build\http\http-proxy.exe ^ + build\rldp-http-proxy\rldp-http-proxy.exe ^ + build\dht-server\dht-server.exe ^ + build\lite-client\lite-client.exe ^ + build\validator-engine\validator-engine.exe ^ + build\utils\generate-random-id.exe ^ + build\utils\json2tlo.exe ^ + build\utils\proxy-liteserver.exe ^ + build\adnl\adnl-proxy.exe ^ + build\emulator\emulator.dll) do ( + echo strip -s %%I & copy %%I artifacts\ + strip -s %%I & copy %%I artifacts\ +) + +xcopy /e /k /h /i crypto\smartcont artifacts\smartcont +xcopy /e /k /h /i crypto\fift\lib artifacts\lib diff --git a/assembly/native/build-windows-github-2019.bat b/assembly/native/build-windows-github-2019.bat new file mode 100644 index 00000000..4f7eee05 --- /dev/null +++ b/assembly/native/build-windows-github-2019.bat @@ -0,0 +1,2 @@ +call "C:\Program Files (x86)\Microsoft Visual Studio\2019\%1\VC\Auxiliary\Build\vcvars64.bat" +call build-windows-2019.bat -t diff --git a/assembly/native/build-windows-github.bat b/assembly/native/build-windows-github.bat new file mode 100644 index 00000000..bfa2d336 --- /dev/null +++ b/assembly/native/build-windows-github.bat @@ -0,0 +1,2 @@ +call "C:\Program Files\Microsoft Visual Studio\2022\%1\VC\Auxiliary\Build\vcvars64.bat" +call build-windows.bat -t diff --git a/assembly/native/build-windows.bat b/assembly/native/build-windows.bat new file mode 100644 index 00000000..68f83c39 --- /dev/null +++ b/assembly/native/build-windows.bat @@ -0,0 +1,208 @@ +REM execute this script inside elevated (Run as Administrator) console "x64 Native Tools Command Prompt for VS 2022" + +echo off + +echo Installing chocolatey windows package manager... +@"%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell.exe" -NoProfile -InputFormat None -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" && SET "PATH=%PATH%;%ALLUSERSPROFILE%\chocolatey\bin" +choco -? +IF %errorlevel% NEQ 0 ( + echo Can't install chocolatey + exit /b %errorlevel% +) + +choco feature enable -n allowEmptyChecksums + +echo Installing pkgconfiglite... +choco install -y pkgconfiglite +IF %errorlevel% NEQ 0 ( + echo Can't install pkgconfiglite + exit /b %errorlevel% +) + +echo Installing ninja... +choco install -y ninja +IF %errorlevel% NEQ 0 ( + echo Can't install ninja + exit /b %errorlevel% +) + +echo Installing nasm... +choco install -y nasm +where nasm +SET PATH=%PATH%;C:\Program Files\NASM +IF %errorlevel% NEQ 0 ( + echo Can't install nasm + exit /b %errorlevel% +) + +mkdir third_libs +cd third_libs + +set third_libs=%cd% +echo %third_libs% + +if not exist "zlib" ( + git clone https://github.com/madler/zlib.git + cd zlib + git checkout v1.3.1 + cd contrib\vstudio\vc14 + msbuild zlibstat.vcxproj /p:Configuration=ReleaseWithoutAsm /p:platform=x64 -p:PlatformToolset=v143 + cd ..\..\..\.. +) else ( + echo Using zlib... +) + +if not exist "lz4" ( + git clone https://github.com/lz4/lz4.git + cd lz4 + git checkout v1.9.4 + cd build\VS2022\liblz4 + msbuild liblz4.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v143 + cd ..\..\..\.. +) else ( + echo Using lz4... +) + +if not exist "libsodium" ( + git clone https://github.com/jedisct1/libsodium + cd libsodium + git checkout 1.0.18-RELEASE + msbuild libsodium.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v143 + cd .. +) else ( + echo Using libsodium... +) + +if not exist "openssl" ( + git clone https://github.com/openssl/openssl.git + cd openssl + git checkout openssl-3.1.4 + where perl + perl Configure VC-WIN64A + IF %errorlevel% NEQ 0 ( + echo Can't configure openssl + exit /b %errorlevel% + ) + nmake + cd .. +) else ( + echo Using openssl... +) + +if not exist "libmicrohttpd" ( + git clone https://github.com/Karlson2k/libmicrohttpd.git + cd libmicrohttpd + git checkout v1.0.1 + cd w32\VS2022 + msbuild libmicrohttpd.vcxproj /p:Configuration=Release-static /p:platform=x64 -p:PlatformToolset=v143 + IF %errorlevel% NEQ 0 ( + echo Can't compile libmicrohttpd + exit /b %errorlevel% + ) + cd ../../.. +) else ( + echo Using libmicrohttpd... +) + +cd .. +echo Current dir %cd% + +mkdir build +cd build +cmake -GNinja -DCMAKE_BUILD_TYPE=Release ^ +-DPORTABLE=1 ^ +-DSODIUM_USE_STATIC_LIBS=1 ^ +-DSODIUM_LIBRARY_RELEASE=%third_libs%\libsodium\Build\Release\x64\libsodium.lib ^ +-DSODIUM_LIBRARY_DEBUG=%third_libs%\libsodium\Build\Release\x64\libsodium.lib ^ +-DSODIUM_INCLUDE_DIR=%third_libs%\libsodium\src\libsodium\include ^ +-DLZ4_FOUND=1 ^ +-DLZ4_INCLUDE_DIRS=%third_libs%\lz4\lib ^ +-DLZ4_LIBRARIES=%third_libs%\lz4\build\VS2022\liblz4\bin\x64_Release\liblz4_static.lib ^ +-DMHD_FOUND=1 ^ +-DMHD_LIBRARY=%third_libs%\libmicrohttpd\w32\VS2022\Output\x64\libmicrohttpd.lib ^ +-DMHD_INCLUDE_DIR=%third_libs%\libmicrohttpd\src\include ^ +-DZLIB_FOUND=1 ^ +-DZLIB_INCLUDE_DIR=%third_libs%\zlib ^ +-DZLIB_LIBRARIES=%third_libs%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib ^ +-DOPENSSL_FOUND=1 ^ +-DOPENSSL_INCLUDE_DIR=%third_libs%\openssl\include ^ +-DOPENSSL_CRYPTO_LIBRARY=%third_libs%\openssl\libcrypto_static.lib ^ +-DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj" .. + +IF %errorlevel% NEQ 0 ( + echo Can't configure TON + exit /b %errorlevel% +) + +IF "%1"=="-t" ( +ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ +tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^ +json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ +test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net ^ +test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ +test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver +IF %errorlevel% NEQ 0 ( + echo Can't compile TON + exit /b %errorlevel% +) +) else ( +ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ +tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^ +json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver +IF %errorlevel% NEQ 0 ( + echo Can't compile TON + exit /b %errorlevel% +) +) + +copy validator-engine\validator-engine.exe test +IF %errorlevel% NEQ 0 ( + echo validator-engine.exe does not exist + exit /b %errorlevel% +) + +IF "%1"=="-t" ( + echo Running tests... +REM ctest -C Release --output-on-failure -E "test-catchain|test-actors|test-validator-session-state" + ctest -C Release --output-on-failure -E "test-bigint" --timeout 1800 + IF %errorlevel% NEQ 0 ( + echo Some tests failed + exit /b %errorlevel% + ) +) + +echo Strip and copy artifacts +cd .. +echo where strip +where strip +mkdir artifacts +mkdir artifacts\smartcont +mkdir artifacts\lib + +for %%I in (build\storage\storage-daemon\storage-daemon.exe ^ + build\storage\storage-daemon\storage-daemon-cli.exe ^ + build\blockchain-explorer\blockchain-explorer.exe ^ + build\crypto\fift.exe ^ + build\crypto\tlbc.exe ^ + build\crypto\func.exe ^ + build\tolk\tolk.exe ^ + build\crypto\create-state.exe ^ + build\validator-engine-console\validator-engine-console.exe ^ + build\tonlib\tonlib-cli.exe ^ + build\tonlib\tonlibjson.dll ^ + build\http\http-proxy.exe ^ + build\rldp-http-proxy\rldp-http-proxy.exe ^ + build\dht-server\dht-server.exe ^ + build\lite-client\lite-client.exe ^ + build\validator-engine\validator-engine.exe ^ + build\utils\generate-random-id.exe ^ + build\utils\json2tlo.exe ^ + build\utils\proxy-liteserver.exe ^ + build\adnl\adnl-proxy.exe ^ + build\emulator\emulator.dll) do ( + echo strip -s %%I & copy %%I artifacts\ + strip -s %%I & copy %%I artifacts\ +) + +xcopy /e /k /h /i crypto\smartcont artifacts\smartcont +xcopy /e /k /h /i crypto\fift\lib artifacts\lib diff --git a/assembly/nix/build-linux-arm64-nix.sh b/assembly/nix/build-linux-arm64-nix.sh new file mode 100644 index 00000000..6fc2fab2 --- /dev/null +++ b/assembly/nix/build-linux-arm64-nix.sh @@ -0,0 +1,38 @@ +#/bin/bash + +nix-build --version +test $? -eq 0 || { echo "Nix is not installed!"; exit 1; } + +with_tests=false + + +while getopts 't' flag; do + case "${flag}" in + t) with_tests=true ;; + *) break + ;; + esac +done + +cp assembly/nix/linux-arm64* . +export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz + +if [ "$with_tests" = true ]; then + nix-build linux-arm64-static.nix --arg testing true +else + nix-build linux-arm64-static.nix +fi + +mkdir -p artifacts/lib +cp ./result/bin/* artifacts/ +test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; } +chmod +x artifacts/* +rm -rf result + +nix-build linux-arm64-tonlib.nix + +cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so +cp ./result/lib/libemulator.so artifacts/ +cp ./result/lib/fift/* artifacts/lib/ +cp -r ./result/share/ton/smartcont artifacts/ +chmod -R +x artifacts diff --git a/assembly/nix/build-linux-x86-64-nix.sh b/assembly/nix/build-linux-x86-64-nix.sh new file mode 100644 index 00000000..30ab79f7 --- /dev/null +++ b/assembly/nix/build-linux-x86-64-nix.sh @@ -0,0 +1,38 @@ +#/bin/bash + +nix-build --version +test $? -eq 0 || { echo "Nix is not installed!"; exit 1; } + +with_tests=false + + +while getopts 't' flag; do + case "${flag}" in + t) with_tests=true ;; + *) break + ;; + esac +done + +cp assembly/nix/linux-x86-64* . +export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz + +if [ "$with_tests" = true ]; then + nix-build linux-x86-64-static.nix --arg testing true +else + nix-build linux-x86-64-static.nix +fi + +mkdir -p artifacts/lib +cp ./result/bin/* artifacts/ +test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; } +chmod +x artifacts/* +rm -rf result + +nix-build linux-x86-64-tonlib.nix + +cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so +cp ./result/lib/libemulator.so artifacts/ +cp ./result/lib/fift/* artifacts/lib/ +cp -r ./result/share/ton/smartcont artifacts/ +chmod -R +x artifacts diff --git a/assembly/nix/build-macos-nix.sh b/assembly/nix/build-macos-nix.sh new file mode 100644 index 00000000..8a07bea2 --- /dev/null +++ b/assembly/nix/build-macos-nix.sh @@ -0,0 +1,38 @@ +#/bin/bash + +nix-build --version +test $? -eq 0 || { echo "Nix is not installed!"; exit 1; } + +with_tests=false + + +while getopts 't' flag; do + case "${flag}" in + t) with_tests=true ;; + *) break + ;; + esac +done + +cp assembly/nix/macos-* . +export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz + +if [ "$with_tests" = true ]; then + nix-build macos-static.nix --arg testing true +else + nix-build macos-static.nix +fi + +mkdir -p artifacts/lib +cp ./result-bin/bin/* artifacts/ +test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; } +chmod +x artifacts/* +rm -rf result-bin + +nix-build macos-tonlib.nix + +cp ./result/lib/libtonlibjson.dylib artifacts/ +cp ./result/lib/libemulator.dylib artifacts/ +cp ./result/lib/fift/* artifacts/lib/ +cp -r ./result/share/ton/smartcont artifacts/ +chmod -R +x artifacts diff --git a/assembly/nix/flakes/flake.lock b/assembly/nix/flakes/flake.lock new file mode 100644 index 00000000..d22f15d3 --- /dev/null +++ b/assembly/nix/flakes/flake.lock @@ -0,0 +1,94 @@ +{ + "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1681202837, + "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "cfacdce06f30d2b68473a46042957675eebb3401", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs-stable": { + "locked": { + "lastModified": 1698846319, + "narHash": "sha256-4jyW/dqFBVpWFnhl0nvP6EN4lP7/ZqPxYRjl6var0Oc=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "50fc86b75d2744e1ab3837ef74b53f103a9b55a0", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-23.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-trunk": { + "locked": { + "lastModified": 1683098912, + "narHash": "sha256-bFHOixPoHZ5y44FvFgpEuZV0UYTQPNDZK/XqeUi1Lbs=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "abc97d3572dec126eba9fec358eb2f359a944683", + "type": "github" + }, + "original": { + "owner": "nixos", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-compat": "flake-compat", + "flake-utils": "flake-utils", + "nixpkgs-stable": "nixpkgs-stable", + "nixpkgs-trunk": "nixpkgs-trunk" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} \ No newline at end of file diff --git a/assembly/nix/flakes/flake.nix b/assembly/nix/flakes/flake.nix new file mode 100644 index 00000000..4e993ac5 --- /dev/null +++ b/assembly/nix/flakes/flake.nix @@ -0,0 +1,158 @@ +{ + inputs = { + nixpkgs-stable.url = "github:nixos/nixpkgs/nixos-23.05"; + nixpkgs-trunk.url = "github:nixos/nixpkgs"; + flake-compat = { + url = "github:edolstra/flake-compat"; + flake = false; + }; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs-stable, nixpkgs-trunk, flake-compat, flake-utils }: + let + ton = { host, pkgs ? host, stdenv ? pkgs.stdenv, staticGlibc ? false + , staticMusl ? false, staticExternalDeps ? staticGlibc }: + with host.lib; + stdenv.mkDerivation { + pname = "ton"; + version = "dev"; + + src = ./.; + + nativeBuildInputs = with host; + [ cmake ninja pkg-config git ] ++ + optionals stdenv.isLinux [ dpkg rpm createrepo_c pacman ]; + buildInputs = with pkgs; + # at some point nixpkgs' pkgsStatic will build with static glibc + # then we can skip these manual overrides + # and switch between pkgsStatic and pkgsStatic.pkgsMusl for static glibc and musl builds + if !staticExternalDeps then [ + openssl + zlib + libmicrohttpd + libsodium + secp256k1 + ] else + [ + (openssl.override { static = true; }).dev + (zlib.override { shared = false; }).dev + ] + ++ optionals (!stdenv.isDarwin) [ pkgsStatic.libmicrohttpd.dev pkgsStatic.libsodium.dev secp256k1 ] + ++ optionals stdenv.isDarwin [ (libiconv.override { enableStatic = true; enableShared = false; }) ] + ++ optionals stdenv.isDarwin (forEach [ libmicrohttpd.dev libsodium.dev secp256k1 gmp.dev nettle.dev (gnutls.override { withP11-kit = false; }).dev libtasn1.dev libidn2.dev libunistring.dev gettext ] (x: x.overrideAttrs(oldAttrs: rec { configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-shared" ]; dontDisableStatic = true; }))) + ++ optionals staticGlibc [ glibc.static ]; + + dontAddStaticConfigureFlags = stdenv.isDarwin; + + cmakeFlags = [ "-DTON_USE_ABSEIL=OFF" "-DNIX=ON" ] ++ optionals staticMusl [ + "-DCMAKE_CROSSCOMPILING=OFF" # pkgsStatic sets cross + ] ++ optionals (staticGlibc || staticMusl) [ + "-DCMAKE_LINK_SEARCH_START_STATIC=ON" + "-DCMAKE_LINK_SEARCH_END_STATIC=ON" + ] ++ optionals (stdenv.isDarwin) [ + "-DCMAKE_CXX_FLAGS=-stdlib=libc++" "-DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=11.7" + ]; + + LDFLAGS = optional staticExternalDeps (concatStringsSep " " [ + (optionalString stdenv.cc.isGNU "-static-libgcc") + (optionalString stdenv.isDarwin "-framework CoreFoundation") + "-static-libstdc++" + ]); + + GIT_REVISION = if self ? rev then self.rev else "dirty"; + GIT_REVISION_DATE = (builtins.concatStringsSep "-" (builtins.match "(.{4})(.{2})(.{2}).*" self.lastModifiedDate)) + " " + (builtins.concatStringsSep ":" (builtins.match "^........(.{2})(.{2})(.{2}).*" self.lastModifiedDate)); + + postInstall = '' + moveToOutput bin "$bin" + ''; + + preFixup = optionalString stdenv.isDarwin '' + for fn in "$bin"/bin/* "$out"/lib/*.dylib; do + echo Fixing libc++ in "$fn" + install_name_tool -change "$(otool -L "$fn" | grep libc++.1 | cut -d' ' -f1 | xargs)" libc++.1.dylib "$fn" + install_name_tool -change "$(otool -L "$fn" | grep libc++abi.1 | cut -d' ' -f1 | xargs)" libc++abi.dylib "$fn" + done + ''; + + outputs = [ "bin" "out" ]; + }; + hostPkgs = system: + import nixpkgs-stable { + inherit system; + overlays = [ + (self: super: { + zchunk = nixpkgs-trunk.legacyPackages.${system}.zchunk; + }) + ]; + }; + in with flake-utils.lib; + (nixpkgs-stable.lib.recursiveUpdate + (eachSystem (with system; [ x86_64-linux aarch64-linux ]) (system: + let + host = hostPkgs system; + # look out for https://github.com/NixOS/nixpkgs/issues/129595 for progress on better infra for this + # + # nixos 19.09 ships with glibc 2.27 + # we could also just override glibc source to a particular release + # but then we'd need to port patches as well + nixos1909 = (import (builtins.fetchTarball { + url = "https://channels.nixos.org/nixos-19.09/nixexprs.tar.xz"; + sha256 = "1vp1h2gkkrckp8dzkqnpcc6xx5lph5d2z46sg2cwzccpr8ay58zy"; + }) { inherit system; }); + glibc227 = nixos1909.glibc // { pname = "glibc"; }; + stdenv227 = let + cc = host.wrapCCWith { + cc = nixos1909.buildPackages.gcc-unwrapped; + libc = glibc227; + bintools = host.binutils.override { libc = glibc227; }; + }; + in (host.overrideCC host.stdenv cc); + in rec { + packages = rec { + ton-normal = ton { inherit host; }; + ton-static = ton { + inherit host; + stdenv = host.makeStatic host.stdenv; + staticGlibc = true; + }; + ton-musl = + let pkgs = nixpkgs-stable.legacyPackages.${system}.pkgsStatic; + in ton { + inherit host; + inherit pkgs; + stdenv = + pkgs.gcc12Stdenv; # doesn't build on aarch64-linux w/default GCC 9 + staticMusl = true; + }; + ton-oldglibc = (ton { + inherit host; + stdenv = stdenv227; + staticExternalDeps = true; + }); + ton-oldglibc_staticbinaries = host.symlinkJoin { + name = "ton"; + paths = [ ton-musl.bin ton-oldglibc.out ]; + }; + }; + devShells.default = + host.mkShell { inputsFrom = [ packages.ton-normal ]; }; + })) (eachSystem (with system; [ x86_64-darwin aarch64-darwin ]) (system: + let host = hostPkgs system; + in rec { + packages = rec { + ton-normal = ton { inherit host; }; + ton-static = ton { + inherit host; + stdenv = host.makeStatic host.stdenv; + staticExternalDeps = true; + }; + ton-staticbin-dylib = host.symlinkJoin { + name = "ton"; + paths = [ ton-static.bin ton-static.out ]; + }; + }; + devShells.default = + host.mkShell { inputsFrom = [ packages.ton-normal ]; }; + }))); +} \ No newline at end of file diff --git a/assembly/nix/flakes/shell.nix b/assembly/nix/flakes/shell.nix new file mode 100644 index 00000000..6234bb4d --- /dev/null +++ b/assembly/nix/flakes/shell.nix @@ -0,0 +1,10 @@ +(import + ( + let lock = builtins.fromJSON (builtins.readFile ./flake.lock); in + fetchTarball { + url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; + sha256 = lock.nodes.flake-compat.locked.narHash; + } + ) + { src = ./.; } +).shellNix diff --git a/assembly/nix/linux-arm64-static.nix b/assembly/nix/linux-arm64-static.nix new file mode 100644 index 00000000..536152d2 --- /dev/null +++ b/assembly/nix/linux-arm64-static.nix @@ -0,0 +1,62 @@ +# export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz + +{ pkgs ? import { system = builtins.currentSystem; } +, lib ? pkgs.lib +, stdenv ? pkgs.stdenv +, testing ? false +}: +let + staticOptions = pkg: pkg.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ]; + }); + + secp256k1Static = (staticOptions pkgs.secp256k1); + libsodiumStatic = (staticOptions pkgs.libsodium); + jemallocStatic = (staticOptions pkgs.jemalloc); + + microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ]; + }); + +in +stdenv.mkDerivation { + pname = "ton"; + version = "dev-bin"; + + src = ./.; + + nativeBuildInputs = with pkgs; + [ cmake ninja git pkg-config ]; + + buildInputs = with pkgs; + [ + (openssl.override { static = true; }).dev + microhttpdStatic.dev + (zlib.override { shared = false; }).dev + (lz4.override { enableStatic = true; enableShared = false; }).dev + jemallocStatic + secp256k1Static + libsodiumStatic.dev + glibc.static + ]; + + cmakeFlags = [ + "-DTON_USE_ABSEIL=OFF" + "-DNIX=ON" + "-DBUILD_SHARED_LIBS=OFF" + "-DCMAKE_LINK_SEARCH_START_STATIC=ON" + "-DCMAKE_LINK_SEARCH_END_STATIC=ON" + "-DTON_USE_JEMALLOC=ON" + ]; + + makeStatic = true; + doCheck = testing; + + LDFLAGS = [ + "-static-libgcc" "-static-libstdc++" "-static" + ]; +} diff --git a/assembly/nix/linux-arm64-tonlib.nix b/assembly/nix/linux-arm64-tonlib.nix new file mode 100644 index 00000000..9d251e02 --- /dev/null +++ b/assembly/nix/linux-arm64-tonlib.nix @@ -0,0 +1,61 @@ +# export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz +{ + pkgs ? import { system = builtins.currentSystem; } +, lib ? pkgs.lib +, stdenv ? pkgs.stdenv +}: +let + staticOptions = pkg: pkg.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ]; + }); + + secp256k1Static = (staticOptions pkgs.secp256k1); + libsodiumStatic = (staticOptions pkgs.libsodium); + + microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ]; + }); +in +pkgs.llvmPackages_16.stdenv.mkDerivation { + pname = "ton"; + version = "dev-lib"; + + src = ./.; + + nativeBuildInputs = with pkgs; + [ + cmake ninja git pkg-config + ]; + + buildInputs = with pkgs; + [ + (openssl.override { static = true; }).dev + microhttpdStatic.dev + (zlib.override { shared = false; }).dev + (lz4.override { enableStatic = true; enableShared = false; }).dev + secp256k1Static + libsodiumStatic.dev + ]; + + dontAddStaticConfigureFlags = false; + doCheck = false; + doInstallCheck = false; + + cmakeFlags = [ + "-DTON_USE_ABSEIL=ON" + "-DNIX=ON" + "-DTON_ONLY_TONLIB=ON" + ]; + + LDFLAGS = [ + "-static-libgcc" "-static-libstdc++" "-fPIC" "-fcommon" + ]; + + ninjaFlags = [ + "tonlibjson" "emulator" + ]; +} diff --git a/assembly/nix/linux-x86-64-static.nix b/assembly/nix/linux-x86-64-static.nix new file mode 100644 index 00000000..a96a9e86 --- /dev/null +++ b/assembly/nix/linux-x86-64-static.nix @@ -0,0 +1,62 @@ +# export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz + +{ pkgs ? import { system = builtins.currentSystem; } +, lib ? pkgs.lib +, stdenv ? pkgs.stdenv +, testing ? false +}: +let + staticOptions = pkg: pkg.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ]; + }); + + secp256k1Static = (staticOptions pkgs.secp256k1); + libsodiumStatic = (staticOptions pkgs.libsodium); + jemallocStatic = (staticOptions pkgs.jemalloc); + + microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ]; + }); + +in +stdenv.mkDerivation { + pname = "ton"; + version = "dev-bin"; + + src = ./.; + + nativeBuildInputs = with pkgs; + [ cmake ninja git pkg-config ]; + + buildInputs = with pkgs; + [ + (openssl.override { static = true; }).dev + microhttpdStatic.dev + (zlib.override { shared = false; }).dev + (lz4.override { enableStatic = true; enableShared = false; }).dev + jemallocStatic + secp256k1Static + libsodiumStatic.dev + glibc.static + ]; + + cmakeFlags = [ + "-DTON_USE_ABSEIL=OFF" + "-DNIX=ON" + "-DBUILD_SHARED_LIBS=OFF" + "-DCMAKE_LINK_SEARCH_START_STATIC=ON" + "-DCMAKE_LINK_SEARCH_END_STATIC=ON" + "-DTON_USE_JEMALLOC=ON" + ]; + + makeStatic = true; + doCheck = testing; + + LDFLAGS = [ + "-static-libgcc" "-static-libstdc++" "-fPIC" + ]; +} diff --git a/assembly/nix/linux-x86-64-tonlib.nix b/assembly/nix/linux-x86-64-tonlib.nix new file mode 100644 index 00000000..64ceaaea --- /dev/null +++ b/assembly/nix/linux-x86-64-tonlib.nix @@ -0,0 +1,77 @@ +# export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.11.tar.gz +# copy linux-x86-64-tonlib.nix to git root directory and execute: +# nix-build linux-x86-64-tonlib.nix +{ + pkgs ? import { system = builtins.currentSystem; } +, lib ? pkgs.lib +, stdenv ? pkgs.stdenv +}: +let + system = builtins.currentSystem; + + staticOptions = pkg: pkg.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ]; + }); + + secp256k1Static = (staticOptions pkgs.secp256k1); + libsodiumStatic = (staticOptions pkgs.libsodium); + + microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ]; + }); + + nixos1909 = (import (builtins.fetchTarball { + url = "https://channels.nixos.org/nixos-19.09/nixexprs.tar.xz"; + sha256 = "1vp1h2gkkrckp8dzkqnpcc6xx5lph5d2z46sg2cwzccpr8ay58zy"; + }) { inherit system; }); + glibc227 = nixos1909.glibc // { pname = "glibc"; }; + stdenv227 = let + cc = pkgs.wrapCCWith { + cc = nixos1909.buildPackages.gcc-unwrapped; + libc = glibc227; + bintools = pkgs.binutils.override { libc = glibc227; }; + }; + in (pkgs.overrideCC pkgs.stdenv cc); + +in +stdenv227.mkDerivation { + pname = "ton"; + version = "dev-lib"; + + src = ./.; + + nativeBuildInputs = with pkgs; + [ cmake ninja git pkg-config ]; + + buildInputs = with pkgs; + [ + (openssl.override { static = true; }).dev + microhttpdStatic.dev + (zlib.override { shared = false; }).dev + (lz4.override { enableStatic = true; enableShared = false; }).dev + secp256k1Static + libsodiumStatic.dev + ]; + + dontAddStaticConfigureFlags = false; + doCheck = false; + doInstallCheck = false; + + cmakeFlags = [ + "-DTON_USE_ABSEIL=ON" + "-DNIX=ON" + "-DTON_ONLY_TONLIB=ON" + ]; + + LDFLAGS = [ + "-static-libgcc" "-static-libstdc++" "-fPIC" + ]; + + ninjaFlags = [ + "tonlibjson" "emulator" + ]; +} diff --git a/assembly/nix/macos-static.nix b/assembly/nix/macos-static.nix new file mode 100644 index 00000000..2fd0b3a6 --- /dev/null +++ b/assembly/nix/macos-static.nix @@ -0,0 +1,67 @@ +# export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz + +{ pkgs ? import { system = builtins.currentSystem; } +, lib ? pkgs.lib +, stdenv ? pkgs.stdenv +, testing ? false +}: + +pkgs.llvmPackages_14.stdenv.mkDerivation { + pname = "ton"; + version = "dev-bin"; + + src = ./.; + + nativeBuildInputs = with pkgs; + [ cmake ninja git pkg-config ]; + + buildInputs = with pkgs; + lib.forEach [ + secp256k1 libsodium.dev libmicrohttpd.dev gmp.dev nettle.dev libtasn1.dev libidn2.dev libunistring.dev gettext jemalloc (gnutls.override { withP11-kit = false; }).dev + ] + (x: x.overrideAttrs(oldAttrs: rec { configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-shared" "--disable-tests" ]; dontDisableStatic = true; })) + ++ [ + darwin.apple_sdk.frameworks.CoreFoundation + (openssl.override { static = true; }).dev + (zlib.override { shared = false; }).dev + (libiconv.override { enableStatic = true; enableShared = false; }) + (lz4.override { enableStatic = true; enableShared = false; }).dev + ]; + + + dontAddStaticConfigureFlags = true; + makeStatic = true; + doCheck = testing; + + configureFlags = []; + + cmakeFlags = [ + "-DTON_USE_ABSEIL=OFF" + "-DNIX=ON" + "-DTON_USE_JEMALLOC=ON" + "-DCMAKE_CROSSCOMPILING=OFF" + "-DCMAKE_LINK_SEARCH_START_STATIC=ON" + "-DCMAKE_LINK_SEARCH_END_STATIC=ON" + "-DBUILD_SHARED_LIBS=OFF" + "-DCMAKE_CXX_FLAGS=-stdlib=libc++" + "-DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=11.3" + ]; + + LDFLAGS = [ + "-static-libstdc++" + "-framework CoreFoundation" + ]; + + postInstall = '' + moveToOutput bin "$bin" + ''; + + preFixup = '' + for fn in "$bin"/bin/* "$out"/lib/*.dylib; do + echo Fixing libc++ in "$fn" + install_name_tool -change "$(otool -L "$fn" | grep libc++.1 | cut -d' ' -f1 | xargs)" libc++.1.dylib "$fn" + install_name_tool -change "$(otool -L "$fn" | grep libc++abi.1 | cut -d' ' -f1 | xargs)" libc++abi.dylib "$fn" + done + ''; + outputs = [ "bin" "out" ]; +} diff --git a/assembly/nix/macos-tonlib.nix b/assembly/nix/macos-tonlib.nix new file mode 100644 index 00000000..4f7b7620 --- /dev/null +++ b/assembly/nix/macos-tonlib.nix @@ -0,0 +1,56 @@ +# export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz + +{ pkgs ? import { system = builtins.currentSystem; } +, lib ? pkgs.lib +, stdenv ? pkgs.stdenv +}: + +pkgs.llvmPackages_14.stdenv.mkDerivation { + pname = "ton"; + version = "dev-lib"; + + src = ./.; + + nativeBuildInputs = with pkgs; + [ cmake ninja git pkg-config ]; + + buildInputs = with pkgs; + lib.forEach [ + secp256k1 libsodium.dev libmicrohttpd.dev gmp.dev nettle.dev libtasn1.dev libidn2.dev libunistring.dev gettext (gnutls.override { withP11-kit = false; }).dev + ] (x: x.overrideAttrs(oldAttrs: rec { configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-shared" "--disable-tests" ]; dontDisableStatic = true; })) + ++ [ + darwin.apple_sdk.frameworks.CoreFoundation + (openssl.override { static = true; }).dev + (zlib.override { shared = false; }).dev + (libiconv.override { enableStatic = true; enableShared = false; }) + (lz4.override { enableStatic = true; enableShared = false; }).dev + ]; + + dontAddStaticConfigureFlags = true; + + configureFlags = []; + + cmakeFlags = [ + "-DTON_USE_ABSEIL=OFF" + "-DNIX=ON" + "-DCMAKE_CXX_FLAGS=-stdlib=libc++" + "-DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=11.3" + ]; + + LDFLAGS = [ + "-static-libstdc++" + "-framework CoreFoundation" + ]; + + ninjaFlags = [ + "tonlibjson" "emulator" + ]; + + preFixup = '' + for fn in $out/bin/* $out/lib/*.dylib; do + echo Fixing libc++ in "$fn" + install_name_tool -change "$(otool -L "$fn" | grep libc++.1 | cut -d' ' -f1 | xargs)" libc++.1.dylib "$fn" + install_name_tool -change "$(otool -L "$fn" | grep libc++abi.1 | cut -d' ' -f1 | xargs)" libc++abi.dylib "$fn" + done + ''; +} diff --git a/assembly/wasm/fift-func-wasm-build-ubuntu.sh b/assembly/wasm/fift-func-wasm-build-ubuntu.sh new file mode 100644 index 00000000..a463c02a --- /dev/null +++ b/assembly/wasm/fift-func-wasm-build-ubuntu.sh @@ -0,0 +1,182 @@ +# Execute these prerequisites first +# sudo apt update +# sudo apt install -y build-essential git make cmake ninja-build clang libgflags-dev zlib1g-dev libssl-dev \ +# libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev python3-pip \ +# nodejs libsodium-dev automake libtool libjemalloc-dev + +# wget https://apt.llvm.org/llvm.sh +# chmod +x llvm.sh +# sudo ./llvm.sh 16 all + +with_artifacts=false +scratch_new=false + +while getopts 'af' flag; do + case "${flag}" in + a) with_artifacts=true ;; + f) scratch_new=true ;; + *) break + ;; + esac +done + +export CC=$(which clang-16) +export CXX=$(which clang++-16) +export CCACHE_DISABLE=1 + +echo `pwd` +if [ "$scratch_new" = true ]; then + echo Compiling openssl zlib lz4 emsdk libsodium emsdk ton + rm -rf openssl zlib lz4 emsdk libsodium build openssl_em +fi + + +if [ ! -d "openssl" ]; then + git clone https://github.com/openssl/openssl.git + cp -r openssl openssl_em + cd openssl + git checkout openssl-3.1.4 + ./config + make -j16 + OPENSSL_DIR=`pwd` + cd .. +else + OPENSSL_DIR=`pwd`/openssl + echo Using compiled openssl at $OPENSSL_DIR +fi + +if [ ! -d "build" ]; then + mkdir build + cd build + cmake -GNinja -DTON_USE_JEMALLOC=ON .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DOPENSSL_ROOT_DIR=$OPENSSL_DIR \ + -DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \ + -DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.so + + test $? -eq 0 || { echo "Can't configure TON build"; exit 1; } + ninja fift smc-envelope + test $? -eq 0 || { echo "Can't compile fift "; exit 1; } + rm -rf * .ninja* CMakeCache.txt + cd .. +else + echo cleaning build... + rm -rf build/* build/.ninja* build/CMakeCache.txt +fi + +if [ ! -d "emsdk" ]; then + git clone https://github.com/emscripten-core/emsdk.git +echo + echo Using cloned emsdk +fi + +cd emsdk +./emsdk install 3.1.19 +./emsdk activate 3.1.19 +EMSDK_DIR=`pwd` + +. $EMSDK_DIR/emsdk_env.sh +export CC=$(which emcc) +export CXX=$(which em++) +export CCACHE_DISABLE=1 + +cd .. + +if [ ! -f "openssl_em/openssl_em" ]; then + cd openssl_em + emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test no-tests no-fuzz-afl no-fuzz-libfuzzer + sed -i 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile + sed -i 's/-ldl//g' Makefile + sed -i 's/-O3/-Os/g' Makefile + emmake make depend + emmake make -j16 + test $? -eq 0 || { echo "Can't compile OpenSSL with emmake "; exit 1; } + OPENSSL_DIR=`pwd` + touch openssl_em + cd .. +else + OPENSSL_DIR=`pwd`/openssl_em + echo Using compiled with empscripten openssl at $OPENSSL_DIR +fi + +if [ ! -d "zlib" ]; then + git clone https://github.com/madler/zlib.git + cd zlib + git checkout v1.3.1 + ZLIB_DIR=`pwd` + emconfigure ./configure --static + emmake make -j16 + test $? -eq 0 || { echo "Can't compile zlib with emmake "; exit 1; } + cd .. +else + ZLIB_DIR=`pwd`/zlib + echo Using compiled zlib with emscripten at $ZLIB_DIR +fi + +if [ ! -d "lz4" ]; then + git clone https://github.com/lz4/lz4.git + cd lz4 + git checkout v1.9.4 + LZ4_DIR=`pwd` + emmake make -j16 + test $? -eq 0 || { echo "Can't compile lz4 with emmake "; exit 1; } + cd .. +else + LZ4_DIR=`pwd`/lz4 + echo Using compiled lz4 with emscripten at $LZ4_DIR +fi + +if [ ! -d "libsodium" ]; then + git clone https://github.com/jedisct1/libsodium + cd libsodium + git checkout 1.0.18-RELEASE + SODIUM_DIR=`pwd` + emconfigure ./configure --disable-ssp + emmake make -j16 + test $? -eq 0 || { echo "Can't compile libsodium with emmake "; exit 1; } + cd .. +else + SODIUM_DIR=`pwd`/libsodium + echo Using compiled libsodium with emscripten at $SODIUM_DIR +fi + +cd build + +emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ +-DZLIB_FOUND=1 \ +-DZLIB_LIBRARIES=$ZLIB_DIR/libz.a \ +-DZLIB_INCLUDE_DIR=$ZLIB_DIR \ +-DLZ4_FOUND=1 \ +-DLZ4_LIBRARIES=$LZ4_DIR/lib/liblz4.a \ +-DLZ4_INCLUDE_DIRS=$LZ4_DIR/lib \ +-DOPENSSL_FOUND=1 \ +-DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \ +-DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.a \ +-DCMAKE_TOOLCHAIN_FILE=$EMSDK_DIR/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake \ +-DCMAKE_CXX_FLAGS="-sUSE_ZLIB=1" \ +-DSODIUM_FOUND=1 \ +-DSODIUM_INCLUDE_DIR=$SODIUM_DIR/src/libsodium/include \ +-DSODIUM_USE_STATIC_LIBS=1 \ +-DSODIUM_LIBRARY_RELEASE=$SODIUM_DIR/src/libsodium/.libs/libsodium.a \ +.. + +test $? -eq 0 || { echo "Can't configure TON with emmake "; exit 1; } +cp -R ../crypto/smartcont ../crypto/fift/lib crypto + +emmake make -j16 funcfiftlib func fift tlbc emulator-emscripten + +test $? -eq 0 || { echo "Can't compile TON with emmake "; exit 1; } + +if [ "$with_artifacts" = true ]; then + echo "Creating artifacts..." + cd .. + rm -rf artifacts + mkdir artifacts + ls build/crypto + cp build/crypto/fift* artifacts + cp build/crypto/func* artifacts + cp build/crypto/tlbc* artifacts + cp build/emulator/emulator-emscripten* artifacts + cp -R crypto/smartcont artifacts + cp -R crypto/fift/lib artifacts +fi diff --git a/assembly/wasm/intrinsics.fc b/assembly/wasm/intrinsics.fc new file mode 100644 index 00000000..14a4498d --- /dev/null +++ b/assembly/wasm/intrinsics.fc @@ -0,0 +1,61 @@ +#pragma allow-post-modification; +#pragma compute-asm-ltr; + +(slice, slice) __tact_load_address(slice cs) inline { + slice raw = cs~load_msg_addr(); + return (cs, raw); +} + +slice __gen_slice1 () asm """ + B{b5ee9c72410101010005000006abcdefe1e98884} B>boc boc boc boc boc tuple cons(X head, tuple tail) asm "CONS"; + +;;; Extracts the head and the tail of lisp-style list. +forall X -> (X, tuple) uncons(tuple list) asm "UNCONS"; + +;;; Extracts the tail and the head of lisp-style list. +forall X -> (tuple, X) list_next(tuple list) asm(-> 1 0) "UNCONS"; + +;;; Returns the head of lisp-style list. +forall X -> X car(tuple list) asm "CAR"; + +;;; Returns the tail of lisp-style list. +tuple cdr(tuple list) asm "CDR"; + +;;; Creates tuple with zero elements. +tuple empty_tuple() asm "NIL"; + +;;; Appends a value `x` to a `Tuple t = (x1, ..., xn)`, but only if the resulting `Tuple t' = (x1, ..., xn, x)` +;;; is of length at most 255. Otherwise throws a type check exception. +forall X -> tuple tpush(tuple t, X value) asm "TPUSH"; +forall X -> (tuple, ()) ~tpush(tuple t, X value) asm "TPUSH"; + +;;; Creates a tuple of length one with given argument as element. +forall X -> [X] single(X x) asm "SINGLE"; + +;;; Unpacks a tuple of length one +forall X -> X unsingle([X] t) asm "UNSINGLE"; + +;;; Creates a tuple of length two with given arguments as elements. +forall X, Y -> [X, Y] pair(X x, Y y) asm "PAIR"; + +;;; Unpacks a tuple of length two +forall X, Y -> (X, Y) unpair([X, Y] t) asm "UNPAIR"; + +;;; Creates a tuple of length three with given arguments as elements. +forall X, Y, Z -> [X, Y, Z] triple(X x, Y y, Z z) asm "TRIPLE"; + +;;; Unpacks a tuple of length three +forall X, Y, Z -> (X, Y, Z) untriple([X, Y, Z] t) asm "UNTRIPLE"; + +;;; Creates a tuple of length four with given arguments as elements. +forall X, Y, Z, W -> [X, Y, Z, W] tuple4(X x, Y y, Z z, W w) asm "4 TUPLE"; + +;;; Unpacks a tuple of length four +forall X, Y, Z, W -> (X, Y, Z, W) untuple4([X, Y, Z, W] t) asm "4 UNTUPLE"; + +;;; Returns the first element of a tuple (with unknown element types). +forall X -> X first(tuple t) asm "FIRST"; + +;;; Returns the second element of a tuple (with unknown element types). +forall X -> X second(tuple t) asm "SECOND"; + +;;; Returns the third element of a tuple (with unknown element types). +forall X -> X third(tuple t) asm "THIRD"; + +;;; Returns the fourth element of a tuple (with unknown element types). +forall X -> X fourth(tuple t) asm "3 INDEX"; + +;;; Returns the first element of a pair tuple. +forall X, Y -> X pair_first([X, Y] p) asm "FIRST"; + +;;; Returns the second element of a pair tuple. +forall X, Y -> Y pair_second([X, Y] p) asm "SECOND"; + +;;; Returns the first element of a triple tuple. +forall X, Y, Z -> X triple_first([X, Y, Z] p) asm "FIRST"; + +;;; Returns the second element of a triple tuple. +forall X, Y, Z -> Y triple_second([X, Y, Z] p) asm "SECOND"; + +;;; Returns the third element of a triple tuple. +forall X, Y, Z -> Z triple_third([X, Y, Z] p) asm "THIRD"; + + +;;; Push null element (casted to given type) +;;; By the TVM type `Null` FunC represents absence of a value of some atomic type. +;;; So `null` can actually have any atomic type. +forall X -> X null() asm "PUSHNULL"; + +;;; Moves a variable [x] to the top of the stack +forall X -> (X, ()) ~impure_touch(X x) impure asm "NOP"; + + + +;;; Returns the current Unix time as an Integer +int now() asm "NOW"; + +;;; Returns the internal address of the current smart contract as a Slice with a `MsgAddressInt`. +;;; If necessary, it can be parsed further using primitives such as [parse_std_addr]. +slice my_address() asm "MYADDR"; + +;;; Returns the balance of the smart contract as a tuple consisting of an int +;;; (balance in nanotoncoins) and a `cell` +;;; (a dictionary with 32-bit keys representing the balance of "extra currencies") +;;; at the start of Computation Phase. +;;; Note that RAW primitives such as [send_raw_message] do not update this field. +[int, cell] get_balance() asm "BALANCE"; + +;;; Returns the logical time of the current transaction. +int cur_lt() asm "LTIME"; + +;;; Returns the starting logical time of the current block. +int block_lt() asm "BLOCKLT"; + +;;; Computes the representation hash of a `cell` [c] and returns it as a 256-bit unsigned integer `x`. +;;; Useful for signing and checking signatures of arbitrary entities represented by a tree of cells. +int cell_hash(cell c) asm "HASHCU"; + +;;; Computes the hash of a `slice s` and returns it as a 256-bit unsigned integer `x`. +;;; The result is the same as if an ordinary cell containing only data and references from `s` had been created +;;; and its hash computed by [cell_hash]. +int slice_hash(slice s) asm "HASHSU"; + +;;; Computes sha256 of the data bits of `slice` [s]. If the bit length of `s` is not divisible by eight, +;;; throws a cell underflow exception. The hash value is returned as a 256-bit unsigned integer `x`. +int string_hash(slice s) asm "SHA256U"; + +{- + # Signature checks +-} + +;;; Checks the Ed25519-`signature` of a `hash` (a 256-bit unsigned integer, usually computed as the hash of some data) +;;; using [public_key] (also represented by a 256-bit unsigned integer). +;;; The signature must contain at least 512 data bits; only the first 512 bits are used. +;;; The result is `−1` if the signature is valid, `0` otherwise. +;;; Note that `CHKSIGNU` creates a 256-bit slice with the hash and calls `CHKSIGNS`. +;;; That is, if [hash] is computed as the hash of some data, these data are hashed twice, +;;; the second hashing occurring inside `CHKSIGNS`. +int check_signature(int hash, slice signature, int public_key) asm "CHKSIGNU"; + +;;; Checks whether [signature] is a valid Ed25519-signature of the data portion of `slice data` using `public_key`, +;;; similarly to [check_signature]. +;;; If the bit length of [data] is not divisible by eight, throws a cell underflow exception. +;;; The verification of Ed25519 signatures is the standard one, +;;; with sha256 used to reduce [data] to the 256-bit number that is actually signed. +int check_data_signature(slice data, slice signature, int public_key) asm "CHKSIGNS"; + +{--- + # Computation of boc size + The primitives below may be useful for computing storage fees of user-provided data. +-} + +;;; Returns `(x, y, z, -1)` or `(null, null, null, 0)`. +;;; Recursively computes the count of distinct cells `x`, data bits `y`, and cell references `z` +;;; in the DAG rooted at `cell` [c], effectively returning the total storage used by this DAG taking into account +;;; the identification of equal cells. +;;; The values of `x`, `y`, and `z` are computed by a depth-first traversal of this DAG, +;;; with a hash table of visited cell hashes used to prevent visits of already-visited cells. +;;; The total count of visited cells `x` cannot exceed non-negative [max_cells]; +;;; otherwise the computation is aborted before visiting the `(max_cells + 1)`-st cell and +;;; a zero flag is returned to indicate failure. If [c] is `null`, returns `x = y = z = 0`. +(int, int, int) compute_data_size(cell c, int max_cells) impure asm "CDATASIZE"; + +;;; Similar to [compute_data_size?], but accepting a `slice` [s] instead of a `cell`. +;;; The returned value of `x` does not take into account the cell that contains the `slice` [s] itself; +;;; however, the data bits and the cell references of [s] are accounted for in `y` and `z`. +(int, int, int) slice_compute_data_size(slice s, int max_cells) impure asm "SDATASIZE"; + +;;; A non-quiet version of [compute_data_size?] that throws a cell overflow exception (`8`) on failure. +(int, int, int, int) compute_data_size?(cell c, int max_cells) asm "CDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT"; + +;;; A non-quiet version of [slice_compute_data_size?] that throws a cell overflow exception (8) on failure. +(int, int, int, int) slice_compute_data_size?(cell c, int max_cells) asm "SDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT"; + +;;; Throws an exception with exit_code excno if cond is not 0 (commented since implemented in compilator) +;; () throw_if(int excno, int cond) impure asm "THROWARGIF"; + +{-- + # Debug primitives + Only works for local TVM execution with debug level verbosity +-} +;;; Dumps the stack (at most the top 255 values) and shows the total stack depth. +() dump_stack() impure asm "DUMPSTK"; + +{- + # Persistent storage save and load +-} + +;;; Returns the persistent contract storage cell. It can be parsed or modified with slice and builder primitives later. +cell get_data() asm "c4 PUSH"; + +;;; Sets `cell` [c] as persistent contract data. You can update persistent contract storage with this primitive. +() set_data(cell c) impure asm "c4 POP"; + +{- + # Continuation primitives +-} +;;; Usually `c3` has a continuation initialized by the whole code of the contract. It is used for function calls. +;;; The primitive returns the current value of `c3`. +cont get_c3() impure asm "c3 PUSH"; + +;;; Updates the current value of `c3`. Usually, it is used for updating smart contract code in run-time. +;;; Note that after execution of this primitive the current code +;;; (and the stack of recursive function calls) won't change, +;;; but any other function call will use a function from the new code. +() set_c3(cont c) impure asm "c3 POP"; + +;;; Transforms a `slice` [s] into a simple ordinary continuation `c`, with `c.code = s` and an empty stack and savelist. +cont bless(slice s) impure asm "BLESS"; + +{--- + # Gas related primitives +-} + +;;; Sets current gas limit `gl` to its maximal allowed value `gm`, and resets the gas credit `gc` to zero, +;;; decreasing the value of `gr` by `gc` in the process. +;;; In other words, the current smart contract agrees to buy some gas to finish the current transaction. +;;; This action is required to process external messages, which bring no value (hence no gas) with themselves. +;;; +;;; For more details check [accept_message effects](https://ton.org/docs/#/smart-contracts/accept). +() accept_message() impure asm "ACCEPT"; + +;;; Sets current gas limit `gl` to the minimum of limit and `gm`, and resets the gas credit `gc` to zero. +;;; If the gas consumed so far (including the present instruction) exceeds the resulting value of `gl`, +;;; an (unhandled) out of gas exception is thrown before setting new gas limits. +;;; Notice that [set_gas_limit] with an argument `limit ≥ 2^63 − 1` is equivalent to [accept_message]. +() set_gas_limit(int limit) impure asm "SETGASLIMIT"; + +;;; Commits the current state of registers `c4` (“persistent data”) and `c5` (“actions”) +;;; so that the current execution is considered “successful” with the saved values even if an exception +;;; in Computation Phase is thrown later. +() commit() impure asm "COMMIT"; + +;;; Not implemented +;;; Computes the amount of gas that can be bought for `amount` nanoTONs, +;;; and sets `gl` accordingly in the same way as [set_gas_limit]. +;;() buy_gas(int amount) impure asm "BUYGAS"; + +;;; Computes the minimum of two integers [x] and [y]. +int min(int x, int y) asm "MIN"; + +;;; Computes the maximum of two integers [x] and [y]. +int max(int x, int y) asm "MAX"; + +;;; Sorts two integers. +(int, int) minmax(int x, int y) asm "MINMAX"; + +;;; Computes the absolute value of an integer [x]. +int abs(int x) asm "ABS"; + +{- + # Slice primitives + + It is said that a primitive _loads_ some data, + if it returns the data and the remainder of the slice + (so it can also be used as [modifying method](https://ton.org/docs/#/func/statements?id=modifying-methods)). + + It is said that a primitive _preloads_ some data, if it returns only the data + (it can be used as [non-modifying method](https://ton.org/docs/#/func/statements?id=non-modifying-methods)). + + Unless otherwise stated, loading and preloading primitives read the data from a prefix of the slice. +-} + + +;;; Converts a `cell` [c] into a `slice`. Notice that [c] must be either an ordinary cell, +;;; or an exotic cell (see [TVM.pdf](https://ton-blockchain.github.io/docs/tvm.pdf), 3.1.2) +;;; which is automatically loaded to yield an ordinary cell `c'`, converted into a `slice` afterwards. +slice begin_parse(cell c) asm "CTOS"; + +;;; Checks if [s] is empty. If not, throws an exception. +() end_parse(slice s) impure asm "ENDS"; + +;;; Loads the first reference from the slice. +(slice, cell) load_ref(slice s) asm(-> 1 0) "LDREF"; + +;;; Preloads the first reference from the slice. +cell preload_ref(slice s) asm "PLDREF"; + +{- Functions below are commented because are implemented on compilator level for optimisation -} + +;;; Loads a signed [len]-bit integer from a slice [s]. +;; (slice, int) ~load_int(slice s, int len) asm(s len -> 1 0) "LDIX"; + +;;; Loads an unsigned [len]-bit integer from a slice [s]. +;; (slice, int) ~load_uint(slice s, int len) asm( -> 1 0) "LDUX"; + +;;; Preloads a signed [len]-bit integer from a slice [s]. +;; int preload_int(slice s, int len) asm "PLDIX"; + +;;; Preloads an unsigned [len]-bit integer from a slice [s]. +;; int preload_uint(slice s, int len) asm "PLDUX"; + +;;; Loads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate `slice s''`. +;; (slice, slice) load_bits(slice s, int len) asm(s len -> 1 0) "LDSLICEX"; + +;;; Preloads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate `slice s''`. +;; slice preload_bits(slice s, int len) asm "PLDSLICEX"; + +;;; Loads serialized amount of TonCoins (any unsigned integer up to `2^120 - 1`). +(slice, int) load_grams(slice s) asm(-> 1 0) "LDGRAMS"; +(slice, int) load_coins(slice s) asm(-> 1 0) "LDVARUINT16"; + +(slice, int) load_varint16(slice s) asm(-> 1 0) "LDVARINT16"; +(slice, int) load_varint32(slice s) asm(-> 1 0) "LDVARINT32"; +(slice, int) load_varuint16(slice s) asm(-> 1 0) "LDVARUINT16"; +(slice, int) load_varuint32(slice s) asm(-> 1 0) "LDVARUINT32"; + +;;; Returns all but the first `0 ≤ len ≤ 1023` bits of `slice` [s]. +slice skip_bits(slice s, int len) asm "SDSKIPFIRST"; +(slice, ()) ~skip_bits(slice s, int len) asm "SDSKIPFIRST"; + +;;; Returns the first `0 ≤ len ≤ 1023` bits of `slice` [s]. +slice first_bits(slice s, int len) asm "SDCUTFIRST"; + +;;; Returns all but the last `0 ≤ len ≤ 1023` bits of `slice` [s]. +slice skip_last_bits(slice s, int len) asm "SDSKIPLAST"; +(slice, ()) ~skip_last_bits(slice s, int len) asm "SDSKIPLAST"; + +;;; Returns the last `0 ≤ len ≤ 1023` bits of `slice` [s]. +slice slice_last(slice s, int len) asm "SDCUTLAST"; + +;;; Loads a dictionary `D` (HashMapE) from `slice` [s]. +;;; (returns `null` if `nothing` constructor is used). +(slice, cell) load_dict(slice s) asm(-> 1 0) "LDDICT"; + +;;; Preloads a dictionary `D` from `slice` [s]. +cell preload_dict(slice s) asm "PLDDICT"; + +;;; Loads a dictionary as [load_dict], but returns only the remainder of the slice. +slice skip_dict(slice s) asm "SKIPDICT"; +(slice, ()) ~skip_dict(slice s) asm "SKIPDICT"; + +;;; Loads (Maybe ^Cell) from `slice` [s]. +;;; In other words loads 1 bit and if it is true +;;; loads first ref and return it with slice remainder +;;; otherwise returns `null` and slice remainder +(slice, cell) load_maybe_ref(slice s) asm(-> 1 0) "LDOPTREF"; + +;;; Preloads (Maybe ^Cell) from `slice` [s]. +cell preload_maybe_ref(slice s) asm "PLDOPTREF"; + + +;;; Returns the depth of `cell` [c]. +;;; If [c] has no references, then return `0`; +;;; otherwise the returned value is one plus the maximum of depths of cells referred to from [c]. +;;; If [c] is a `null` instead of a cell, returns zero. +int cell_depth(cell c) asm "CDEPTH"; + + +{- + # Slice size primitives +-} + +;;; Returns the number of references in `slice` [s]. +int slice_refs(slice s) asm "SREFS"; + +;;; Returns the number of data bits in `slice` [s]. +int slice_bits(slice s) asm "SBITS"; + +;;; Returns both the number of data bits and the number of references in `slice` [s]. +(int, int) slice_bits_refs(slice s) asm "SBITREFS"; + +;;; Checks whether a `slice` [s] is empty (i.e., contains no bits of data and no cell references). +int slice_empty?(slice s) asm "SEMPTY"; + +;;; Checks whether `slice` [s] has no bits of data. +int slice_data_empty?(slice s) asm "SDEMPTY"; + +;;; Checks whether `slice` [s] has no references. +int slice_refs_empty?(slice s) asm "SREMPTY"; + +;;; Returns the depth of `slice` [s]. +;;; If [s] has no references, then returns `0`; +;;; otherwise the returned value is one plus the maximum of depths of cells referred to from [s]. +int slice_depth(slice s) asm "SDEPTH"; + +{- + # Builder size primitives +-} + +;;; Returns the number of cell references already stored in `builder` [b] +int builder_refs(builder b) asm "BREFS"; + +;;; Returns the number of data bits already stored in `builder` [b]. +int builder_bits(builder b) asm "BBITS"; + +;;; Returns the depth of `builder` [b]. +;;; If no cell references are stored in [b], then returns 0; +;;; otherwise the returned value is one plus the maximum of depths of cells referred to from [b]. +int builder_depth(builder b) asm "BDEPTH"; + +{- + # Builder primitives + It is said that a primitive _stores_ a value `x` into a builder `b` + if it returns a modified version of the builder `b'` with the value `x` stored at the end of it. + It can be used as [non-modifying method](https://ton.org/docs/#/func/statements?id=non-modifying-methods). + + All the primitives below first check whether there is enough space in the `builder`, + and only then check the range of the value being serialized. +-} + +;;; Creates a new empty `builder`. +builder begin_cell() asm "NEWC"; + +;;; Converts a `builder` into an ordinary `cell`. +cell end_cell(builder b) asm "ENDC"; + +;;; Stores a reference to `cell` [c] into `builder` [b]. +builder store_ref(builder b, cell c) asm(c b) "STREF"; + +;;; Stores an unsigned [len]-bit integer `x` into `b` for `0 ≤ len ≤ 256`. +;; builder store_uint(builder b, int x, int len) asm(x b len) "STUX"; + +;;; Stores a signed [len]-bit integer `x` into `b` for` 0 ≤ len ≤ 257`. +;; builder store_int(builder b, int x, int len) asm(x b len) "STIX"; + + +;;; Stores `slice` [s] into `builder` [b] +builder store_slice(builder b, slice s) asm "STSLICER"; + +;;; Stores (serializes) an integer [x] in the range `0..2^120 − 1` into `builder` [b]. +;;; The serialization of [x] consists of a 4-bit unsigned big-endian integer `l`, +;;; which is the smallest integer `l ≥ 0`, such that `x < 2^8l`, +;;; followed by an `8l`-bit unsigned big-endian representation of [x]. +;;; If [x] does not belong to the supported range, a range check exception is thrown. +;;; +;;; Store amounts of TonCoins to the builder as VarUInteger 16 +builder store_grams(builder b, int x) asm "STGRAMS"; +builder store_coins(builder b, int x) asm "STVARUINT16"; + +builder store_varint16(builder b, int x) asm "STVARINT16"; +builder store_varint32(builder b, int x) asm "STVARINT32"; +builder store_varuint16(builder b, int x) asm "STVARUINT16"; +builder store_varuint32(builder b, int x) asm "STVARUINT32"; + +;;; Stores dictionary `D` represented by `cell` [c] or `null` into `builder` [b]. +;;; In other words, stores a `1`-bit and a reference to [c] if [c] is not `null` and `0`-bit otherwise. +builder store_dict(builder b, cell c) asm(c b) "STDICT"; + +;;; Stores (Maybe ^Cell) to builder: +;;; if cell is null store 1 zero bit +;;; otherwise store 1 true bit and ref to cell +builder store_maybe_ref(builder b, cell c) asm(c b) "STOPTREF"; + + +{- + # Address manipulation primitives + The address manipulation primitives listed below serialize and deserialize values according to the following TL-B scheme: + ```TL-B + addr_none$00 = MsgAddressExt; + addr_extern$01 len:(## 8) external_address:(bits len) + = MsgAddressExt; + anycast_info$_ depth:(#<= 30) { depth >= 1 } + rewrite_pfx:(bits depth) = Anycast; + addr_std$10 anycast:(Maybe Anycast) + workchain_id:int8 address:bits256 = MsgAddressInt; + addr_var$11 anycast:(Maybe Anycast) addr_len:(## 9) + workchain_id:int32 address:(bits addr_len) = MsgAddressInt; + _ _:MsgAddressInt = MsgAddress; + _ _:MsgAddressExt = MsgAddress; + + int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool + src:MsgAddress dest:MsgAddressInt + value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams + created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; + ext_out_msg_info$11 src:MsgAddress dest:MsgAddressExt + created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; + ``` + A deserialized `MsgAddress` is represented by a tuple `t` as follows: + + - `addr_none` is represented by `t = (0)`, + i.e., a tuple containing exactly one integer equal to zero. + - `addr_extern` is represented by `t = (1, s)`, + where slice `s` contains the field `external_address`. In other words, ` + t` is a pair (a tuple consisting of two entries), containing an integer equal to one and slice `s`. + - `addr_std` is represented by `t = (2, u, x, s)`, + where `u` is either a `null` (if `anycast` is absent) or a slice `s'` containing `rewrite_pfx` (if anycast is present). + Next, integer `x` is the `workchain_id`, and slice `s` contains the address. + - `addr_var` is represented by `t = (3, u, x, s)`, + where `u`, `x`, and `s` have the same meaning as for `addr_std`. +-} + +;;; Loads from slice [s] the only prefix that is a valid `MsgAddress`, +;;; and returns both this prefix `s'` and the remainder `s''` of [s] as slices. +(slice, slice) load_msg_addr(slice s) asm(-> 1 0) "LDMSGADDR"; + +;;; Decomposes slice [s] containing a valid `MsgAddress` into a `tuple t` with separate fields of this `MsgAddress`. +;;; If [s] is not a valid `MsgAddress`, a cell deserialization exception is thrown. +tuple parse_addr(slice s) asm "PARSEMSGADDR"; + +;;; Parses slice [s] containing a valid `MsgAddressInt` (usually a `msg_addr_std`), +;;; applies rewriting from the anycast (if present) to the same-length prefix of the address, +;;; and returns both the workchain and the 256-bit address as integers. +;;; If the address is not 256-bit, or if [s] is not a valid serialization of `MsgAddressInt`, +;;; throws a cell deserialization exception. +(int, int) parse_std_addr(slice s) asm "REWRITESTDADDR"; + +;;; A variant of [parse_std_addr] that returns the (rewritten) address as a slice [s], +;;; even if it is not exactly 256 bit long (represented by a `msg_addr_var`). +(int, slice) parse_var_addr(slice s) asm "REWRITEVARADDR"; + +{- + # Dictionary primitives +-} + + +;;; Sets the value associated with [key_len]-bit key signed index in dictionary [dict] to [value] (cell), +;;; and returns the resulting dictionary. +cell idict_set_ref(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTISETREF"; +(cell, ()) ~idict_set_ref(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTISETREF"; + +;;; Sets the value associated with [key_len]-bit key unsigned index in dictionary [dict] to [value] (cell), +;;; and returns the resulting dictionary. +cell udict_set_ref(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTUSETREF"; +(cell, ()) ~udict_set_ref(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTUSETREF"; + +cell idict_get_ref(cell dict, int key_len, int index) asm(index dict key_len) "DICTIGETOPTREF"; +(cell, int) idict_get_ref?(cell dict, int key_len, int index) asm(index dict key_len) "DICTIGETREF" "NULLSWAPIFNOT"; +(cell, int) udict_get_ref?(cell dict, int key_len, int index) asm(index dict key_len) "DICTUGETREF" "NULLSWAPIFNOT"; +(cell, cell) idict_set_get_ref(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTISETGETOPTREF"; +(cell, cell) udict_set_get_ref(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTUSETGETOPTREF"; +(cell, int) idict_delete?(cell dict, int key_len, int index) asm(index dict key_len) "DICTIDEL"; +(cell, int) udict_delete?(cell dict, int key_len, int index) asm(index dict key_len) "DICTUDEL"; +(slice, int) idict_get?(cell dict, int key_len, int index) asm(index dict key_len) "DICTIGET" "NULLSWAPIFNOT"; +(slice, int) udict_get?(cell dict, int key_len, int index) asm(index dict key_len) "DICTUGET" "NULLSWAPIFNOT"; +(cell, slice, int) idict_delete_get?(cell dict, int key_len, int index) asm(index dict key_len) "DICTIDELGET" "NULLSWAPIFNOT"; +(cell, slice, int) udict_delete_get?(cell dict, int key_len, int index) asm(index dict key_len) "DICTUDELGET" "NULLSWAPIFNOT"; +(cell, (slice, int)) ~idict_delete_get?(cell dict, int key_len, int index) asm(index dict key_len) "DICTIDELGET" "NULLSWAPIFNOT"; +(cell, (slice, int)) ~udict_delete_get?(cell dict, int key_len, int index) asm(index dict key_len) "DICTUDELGET" "NULLSWAPIFNOT"; +(cell, cell, int) idict_delete_get_ref?(cell dict, int key_len, int index) asm(index dict key_len) "DICTIDELGETREF" "NULLSWAPIFNOT"; +(cell, cell, int) udict_delete_get_ref?(cell dict, int key_len, int index) asm(index dict key_len) "DICTUDELGETREF" "NULLSWAPIFNOT"; +(cell, (cell, int)) ~idict_delete_get_ref?(cell dict, int key_len, int index) asm(index dict key_len) "DICTIDELGETREF" "NULLSWAPIFNOT"; +(cell, (cell, int)) ~udict_delete_get_ref?(cell dict, int key_len, int index) asm(index dict key_len) "DICTUDELGETREF" "NULLSWAPIFNOT"; +cell udict_set(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTUSET"; +(cell, ()) ~udict_set(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTUSET"; +cell idict_set(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTISET"; +(cell, ()) ~idict_set(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTISET"; +cell dict_set(cell dict, int key_len, slice index, slice value) asm(value index dict key_len) "DICTSET"; +(cell, ()) ~dict_set(cell dict, int key_len, slice index, slice value) asm(value index dict key_len) "DICTSET"; +(cell, int) udict_add?(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTUADD"; +(cell, int) udict_replace?(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTUREPLACE"; +(cell, int) udict_replace_ref?(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTUREPLACEREF"; +(cell, slice, int) udict_replaceget?(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTUREPLACEGET" "NULLSWAPIFNOT"; +(cell, cell, int) udict_replaceget_ref?(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTUREPLACEGETREF" "NULLSWAPIFNOT"; +(cell, (slice, int)) ~udict_replaceget?(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTUREPLACEGET" "NULLSWAPIFNOT"; +(cell, (cell, int)) ~udict_replaceget_ref?(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTUREPLACEGETREF" "NULLSWAPIFNOT"; +(cell, int) idict_add?(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTIADD"; +(cell, int) idict_replace?(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTIREPLACE"; +(cell, int) idict_replace_ref?(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTIREPLACEREF"; +(cell, slice, int) idict_replaceget?(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTIREPLACEGET" "NULLSWAPIFNOT"; +(cell, cell, int) idict_replaceget_ref?(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTIREPLACEGETREF" "NULLSWAPIFNOT"; +(cell, (slice, int)) ~idict_replaceget?(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTIREPLACEGET" "NULLSWAPIFNOT"; +(cell, (cell, int)) ~idict_replaceget_ref?(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTIREPLACEGETREF" "NULLSWAPIFNOT"; +cell udict_set_builder(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTUSETB"; +(cell, ()) ~udict_set_builder(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTUSETB"; +cell idict_set_builder(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTISETB"; +(cell, ()) ~idict_set_builder(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTISETB"; +cell dict_set_builder(cell dict, int key_len, slice index, builder value) asm(value index dict key_len) "DICTSETB"; +(cell, ()) ~dict_set_builder(cell dict, int key_len, slice index, builder value) asm(value index dict key_len) "DICTSETB"; +(cell, int) dict_replace_builder?(cell dict, int key_len, slice index, builder value) asm(value index dict key_len) "DICTREPLACEB"; +(cell, builder, int) dict_replaceget_builder?(cell dict, int key_len, slice index, builder value) asm(value index dict key_len) "DICTREPLACEGETB" "NULLSWAPIFNOT"; +(cell, slice, int) dict_replaceget?(cell dict, int key_len, slice index, slice value) asm(value index dict key_len) "DICTREPLACEGET" "NULLSWAPIFNOT"; +(cell, (builder, int)) ~dict_replaceget_builder?(cell dict, int key_len, slice index, builder value) asm(value index dict key_len) "DICTREPLACEGETB" "NULLSWAPIFNOT"; +(cell, (slice, int)) ~dict_replaceget?(cell dict, int key_len, slice index, slice value) asm(value index dict key_len) "DICTREPLACEGET" "NULLSWAPIFNOT"; +(cell, int) udict_add_builder?(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTUADDB"; +(cell, int) udict_replace_builder?(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTUREPLACEB"; +(cell, builder, int) udict_replaceget_builder?(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTUREPLACEGETB" "NULLSWAPIFNOT"; +(cell, (builder, int)) ~udict_replaceget_builder?(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTUREPLACEGETB" "NULLSWAPIFNOT"; +(cell, int) idict_add_builder?(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTIADDB"; +(cell, int) idict_replace_builder?(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTIREPLACEB"; +(cell, builder, int) idict_replaceget_builder?(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTIREPLACEGETB" "NULLSWAPIFNOT"; +(cell, (builder, int)) ~idict_replaceget_builder?(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTIREPLACEGETB" "NULLSWAPIFNOT"; +(cell, int, slice, int) udict_delete_get_min(cell dict, int key_len) asm(-> 0 2 1 3) "DICTUREMMIN" "NULLSWAPIFNOT2"; +(cell, (int, slice, int)) ~udict::delete_get_min(cell dict, int key_len) asm(-> 0 2 1 3) "DICTUREMMIN" "NULLSWAPIFNOT2"; +(cell, int, slice, int) idict_delete_get_min(cell dict, int key_len) asm(-> 0 2 1 3) "DICTIREMMIN" "NULLSWAPIFNOT2"; +(cell, (int, slice, int)) ~idict::delete_get_min(cell dict, int key_len) asm(-> 0 2 1 3) "DICTIREMMIN" "NULLSWAPIFNOT2"; +(cell, slice, slice, int) dict_delete_get_min(cell dict, int key_len) asm(-> 0 2 1 3) "DICTREMMIN" "NULLSWAPIFNOT2"; +(cell, (slice, slice, int)) ~dict::delete_get_min(cell dict, int key_len) asm(-> 0 2 1 3) "DICTREMMIN" "NULLSWAPIFNOT2"; +(cell, int, slice, int) udict_delete_get_max(cell dict, int key_len) asm(-> 0 2 1 3) "DICTUREMMAX" "NULLSWAPIFNOT2"; +(cell, (int, slice, int)) ~udict::delete_get_max(cell dict, int key_len) asm(-> 0 2 1 3) "DICTUREMMAX" "NULLSWAPIFNOT2"; +(cell, int, slice, int) idict_delete_get_max(cell dict, int key_len) asm(-> 0 2 1 3) "DICTIREMMAX" "NULLSWAPIFNOT2"; +(cell, (int, slice, int)) ~idict::delete_get_max(cell dict, int key_len) asm(-> 0 2 1 3) "DICTIREMMAX" "NULLSWAPIFNOT2"; +(cell, slice, slice, int) dict_delete_get_max(cell dict, int key_len) asm(-> 0 2 1 3) "DICTREMMAX" "NULLSWAPIFNOT2"; +(cell, (slice, slice, int)) ~dict::delete_get_max(cell dict, int key_len) asm(-> 0 2 1 3) "DICTREMMAX" "NULLSWAPIFNOT2"; +(int, slice, int) udict_get_min?(cell dict, int key_len) asm (-> 1 0 2) "DICTUMIN" "NULLSWAPIFNOT2"; +(int, slice, int) udict_get_max?(cell dict, int key_len) asm (-> 1 0 2) "DICTUMAX" "NULLSWAPIFNOT2"; +(int, cell, int) udict_get_min_ref?(cell dict, int key_len) asm (-> 1 0 2) "DICTUMINREF" "NULLSWAPIFNOT2"; +(int, cell, int) udict_get_max_ref?(cell dict, int key_len) asm (-> 1 0 2) "DICTUMAXREF" "NULLSWAPIFNOT2"; +(int, slice, int) idict_get_min?(cell dict, int key_len) asm (-> 1 0 2) "DICTIMIN" "NULLSWAPIFNOT2"; +(int, slice, int) idict_get_max?(cell dict, int key_len) asm (-> 1 0 2) "DICTIMAX" "NULLSWAPIFNOT2"; +(int, cell, int) idict_get_min_ref?(cell dict, int key_len) asm (-> 1 0 2) "DICTIMINREF" "NULLSWAPIFNOT2"; +(int, cell, int) idict_get_max_ref?(cell dict, int key_len) asm (-> 1 0 2) "DICTIMAXREF" "NULLSWAPIFNOT2"; +(int, slice, int) udict_get_next?(cell dict, int key_len, int pivot) asm(pivot dict key_len -> 1 0 2) "DICTUGETNEXT" "NULLSWAPIFNOT2"; +(int, slice, int) udict_get_nexteq?(cell dict, int key_len, int pivot) asm(pivot dict key_len -> 1 0 2) "DICTUGETNEXTEQ" "NULLSWAPIFNOT2"; +(int, slice, int) udict_get_prev?(cell dict, int key_len, int pivot) asm(pivot dict key_len -> 1 0 2) "DICTUGETPREV" "NULLSWAPIFNOT2"; +(int, slice, int) udict_get_preveq?(cell dict, int key_len, int pivot) asm(pivot dict key_len -> 1 0 2) "DICTUGETPREVEQ" "NULLSWAPIFNOT2"; +(int, slice, int) idict_get_next?(cell dict, int key_len, int pivot) asm(pivot dict key_len -> 1 0 2) "DICTIGETNEXT" "NULLSWAPIFNOT2"; +(int, slice, int) idict_get_nexteq?(cell dict, int key_len, int pivot) asm(pivot dict key_len -> 1 0 2) "DICTIGETNEXTEQ" "NULLSWAPIFNOT2"; +(int, slice, int) idict_get_prev?(cell dict, int key_len, int pivot) asm(pivot dict key_len -> 1 0 2) "DICTIGETPREV" "NULLSWAPIFNOT2"; +(int, slice, int) idict_get_preveq?(cell dict, int key_len, int pivot) asm(pivot dict key_len -> 1 0 2) "DICTIGETPREVEQ" "NULLSWAPIFNOT2"; + +;;; Creates an empty dictionary, which is actually a null value. Equivalent to PUSHNULL +cell new_dict() asm "NEWDICT"; +;;; Checks whether a dictionary is empty. Equivalent to cell_null?. +int dict_empty?(cell c) asm "DICTEMPTY"; + + +{- Prefix dictionary primitives -} +(slice, slice, slice, int) pfxdict_get?(cell dict, int key_len, slice key) asm(key dict key_len) "PFXDICTGETQ" "NULLSWAPIFNOT2"; +(cell, int) pfxdict_set?(cell dict, int key_len, slice key, slice value) asm(value key dict key_len) "PFXDICTSET"; +(cell, int) pfxdict_delete?(cell dict, int key_len, slice key) asm(key dict key_len) "PFXDICTDEL"; + +;;; Returns the value of the global configuration parameter with integer index `i` as a `cell` or `null` value. +cell config_param(int x) asm "CONFIGOPTPARAM"; +;;; Checks whether c is a null. Note, that FunC also has polymorphic null? built-in. +int cell_null?(cell c) asm "ISNULL"; + +;;; Creates an output action which would reserve exactly amount nanotoncoins (if mode = 0), at most amount nanotoncoins (if mode = 2), or all but amount nanotoncoins (if mode = 1 or mode = 3), from the remaining balance of the account. It is roughly equivalent to creating an outbound message carrying amount nanotoncoins (or b − amount nanotoncoins, where b is the remaining balance) to oneself, so that the subsequent output actions would not be able to spend more money than the remainder. Bit +2 in mode means that the external action does not fail if the specified amount cannot be reserved; instead, all remaining balance is reserved. Bit +8 in mode means `amount <- -amount` before performing any further actions. Bit +4 in mode means that amount is increased by the original balance of the current account (before the compute phase), including all extra currencies, before performing any other checks and actions. Currently, amount must be a non-negative integer, and mode must be in the range 0..15. +() raw_reserve(int amount, int mode) impure asm "RAWRESERVE"; +;;; Similar to raw_reserve, but also accepts a dictionary extra_amount (represented by a cell or null) with extra currencies. In this way currencies other than TonCoin can be reserved. +() raw_reserve_extra(int amount, cell extra_amount, int mode) impure asm "RAWRESERVEX"; +;;; Sends a raw message contained in msg, which should contain a correctly serialized object Message X, with the only exception that the source address is allowed to have dummy value addr_none (to be automatically replaced with the current smart contract address), and ihr_fee, fwd_fee, created_lt and created_at fields can have arbitrary values (to be rewritten with correct values during the action phase of the current transaction). Integer parameter mode contains the flags. Currently mode = 0 is used for ordinary messages; mode = 128 is used for messages that are to carry all the remaining balance of the current smart contract (instead of the value originally indicated in the message); mode = 64 is used for messages that carry all the remaining value of the inbound message in addition to the value initially indicated in the new message (if bit 0 is not set, the gas fees are deducted from this amount); mode' = mode + 1 means that the sender wants to pay transfer fees separately; mode' = mode + 2 means that any errors arising while processing this message during the action phase should be ignored. Finally, mode' = mode + 32 means that the current account must be destroyed if its resulting balance is zero. This flag is usually employed together with +128. +() send_raw_message(cell msg, int mode) impure asm "SENDRAWMSG"; +;;; Creates an output action that would change this smart contract code to that given by cell new_code. Notice that this change will take effect only after the successful termination of the current run of the smart contract +() set_code(cell new_code) impure asm "SETCODE"; + +;;; Generates a new pseudo-random unsigned 256-bit integer x. The algorithm is as follows: if r is the old value of the random seed, considered as a 32-byte array (by constructing the big-endian representation of an unsigned 256-bit integer), then its sha512(r) is computed; the first 32 bytes of this hash are stored as the new value r' of the random seed, and the remaining 32 bytes are returned as the next random value x. +int random() impure asm "RANDU256"; +;;; Generates a new pseudo-random integer z in the range 0..range−1 (or range..−1, if range < 0). More precisely, an unsigned random value x is generated as in random; then z := x * range / 2^256 is computed. +int rand(int range) impure asm "RAND"; +;;; Returns the current random seed as an unsigned 256-bit Integer. +int get_seed() impure asm "RANDSEED"; +;;; Sets the random seed to unsigned 256-bit seed. +() set_seed(int x) impure asm "SETRAND"; +;;; Mixes unsigned 256-bit integer x into the random seed r by setting the random seed to sha256 of the concatenation of two 32-byte strings: the first with the big-endian representation of the old seed r, and the second with the big-endian representation of x. +() randomize(int x) impure asm "ADDRAND"; +;;; Equivalent to randomize(cur_lt());. +() randomize_lt() impure asm "LTIME" "ADDRAND"; + +;;; Checks whether the data parts of two slices coinside +int equal_slices_bits(slice a, slice b) asm "SDEQ"; +;;; Checks whether b is a null. Note, that FunC also has polymorphic null? built-in. +int builder_null?(builder b) asm "ISNULL"; +;;; Concatenates two builders +builder store_builder(builder to, builder from) asm "STBR"; + +;; CUSTOM: + +;; TVM UPGRADE 2023-07 https://docs.ton.org/learn/tvm-instructions/tvm-upgrade-2023-07 +;; In mainnet since 20 Dec 2023 https://t.me/tonblockchain/226 + +;;; Retrieves code of smart-contract from c7 + +cell my_code() asm "MYCODE"; diff --git a/blockchain-explorer/CMakeLists.txt b/blockchain-explorer/CMakeLists.txt index 0d02f01f..86cb3e74 100644 --- a/blockchain-explorer/CMakeLists.txt +++ b/blockchain-explorer/CMakeLists.txt @@ -1,22 +1,42 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) +option(NIX "Use \"ON\" for a static build." OFF) -find_package(MHD) - -if (MHD_FOUND) - - set(BLOCHAIN_EXPLORER_SOURCE +set(BLOCHAIN_EXPLORER_SOURCE blockchain-explorer.cpp blockchain-explorer.hpp blockchain-explorer-http.cpp blockchain-explorer-http.hpp blockchain-explorer-query.cpp blockchain-explorer-query.hpp - ) +) - add_executable(blockchain-explorer ${BLOCHAIN_EXPLORER_SOURCE}) - target_include_directories(blockchain-explorer PUBLIC ${MHD_INCLUDE_DIRS}) - target_link_libraries(blockchain-explorer tdutils tdactor adnllite tl_lite_api tl-lite-utils - ton_crypto ton_block ${MHD_LIBRARY}) +add_executable(blockchain-explorer ${BLOCHAIN_EXPLORER_SOURCE}) +if (NIX) + if (MHD_FOUND) + target_include_directories(blockchain-explorer PUBLIC ${MHD_INCLUDE_DIR}) + target_link_libraries(blockchain-explorer tdactor adnllite tl_lite_api tl-lite-utils ton_crypto ${MHD_LIBRARY}) + else() + find_package(PkgConfig REQUIRED) + pkg_check_modules(MHD libmicrohttpd) + target_include_directories(blockchain-explorer PUBLIC ${MHD_INCLUDE_DIR} ${MHD_STATIC_INCLUDE_DIRS}) + target_link_libraries(blockchain-explorer tdactor adnllite tl_lite_api tl-lite-utils ton_crypto ${MHD_LIBRARIES} ${MHD_STATIC_LIBRARIES}) + endif() +else() + if (MHD_FOUND) + target_include_directories(blockchain-explorer PUBLIC ${MHD_INCLUDE_DIR}) + target_link_libraries(blockchain-explorer tdactor adnllite tl_lite_api tl-lite-utils ton_crypto ${MHD_LIBRARY}) + else() + find_package(MHD) + target_include_directories(blockchain-explorer PUBLIC ${MHD_INCLUDE_DIR}) + target_link_libraries(blockchain-explorer tdactor adnllite tl_lite_api tl-lite-utils ton_crypto ${MHD_LIBRARY}) + endif() endif() + +target_include_directories(blockchain-explorer PUBLIC ${MHD_INCLUDE_DIR}) +target_link_libraries(blockchain-explorer tdactor adnllite tl_lite_api tl-lite-utils ton_crypto ${MHD_LIBRARY}) +target_link_libraries(blockchain-explorer lite-client-common) + +install(TARGETS blockchain-explorer RUNTIME DESTINATION bin) + diff --git a/blockchain-explorer/blockchain-explorer-http.cpp b/blockchain-explorer/blockchain-explorer-http.cpp index 8642e900..e5203cb7 100644 --- a/blockchain-explorer/blockchain-explorer-http.cpp +++ b/blockchain-explorer/blockchain-explorer-http.cpp @@ -35,9 +35,38 @@ #include "vm/cells/MerkleProof.h" #include "block/mc-config.h" #include "ton/ton-shard.h" +#include "td/utils/date.h" bool local_scripts{false}; +static std::string time_to_human(unsigned ts) { + td::StringBuilder sb; + sb << date::format("%F %T", + std::chrono::time_point{std::chrono::seconds(ts)}) + << ", "; + auto now = (unsigned)td::Clocks::system(); + bool past = now >= ts; + unsigned x = past ? now - ts : ts - now; + if (!past) { + sb << "in "; + } + if (x < 60) { + sb << x << "s"; + } else if (x < 3600) { + sb << x / 60 << "m " << x % 60 << "s"; + } else if (x < 3600 * 24) { + x /= 60; + sb << x / 60 << "h " << x % 60 << "m"; + } else { + x /= 3600; + sb << x / 24 << "d " << x % 24 << "h"; + } + if (past) { + sb << " ago"; + } + return sb.as_cslice().str(); +} + HttpAnswer& HttpAnswer::operator<<(AddressCell addr_c) { ton::WorkchainId wc; ton::StdSmcAddress addr; @@ -84,7 +113,7 @@ HttpAnswer& HttpAnswer::operator<<(MessageCell msg) { << "source" << AddressCell{info.src} << "\n" << "destinationNONE\n" << "lt" << info.created_lt << "\n" - << "time" << info.created_at << "\n"; + << "time" << info.created_at << " (" << time_to_human(info.created_at) << ")\n"; break; } case block::gen::CommonMsgInfo::int_msg_info: { @@ -93,9 +122,8 @@ HttpAnswer& HttpAnswer::operator<<(MessageCell msg) { abort("cannot unpack internal message"); return *this; } - td::RefInt256 value; - td::Ref extra; - if (!block::unpack_CurrencyCollection(info.value, value, extra)) { + block::CurrencyCollection currency_collection; + if (!currency_collection.unpack(info.value)) { abort("cannot unpack message value"); return *this; } @@ -103,8 +131,8 @@ HttpAnswer& HttpAnswer::operator<<(MessageCell msg) { << "source" << AddressCell{info.src} << "\n" << "destination" << AddressCell{info.dest} << "\n" << "lt" << info.created_lt << "\n" - << "time" << info.created_at << "\n" - << "value" << value << "\n"; + << "time" << info.created_at << " (" << time_to_human(info.created_at) << ")\n" + << "value" << currency_collection.to_str()<< "\n"; break; } default: @@ -277,7 +305,7 @@ HttpAnswer& HttpAnswer::operator<<(TransactionCell trans_c) { << "account" << trans_c.addr.rserialize(true) << "" << "hash" << trans_c.root->get_hash().to_hex() << "\n" << "lt" << trans.lt << "\n" - << "time" << trans.now << "\n" + << "time" << trans.now << " (" << time_to_human(trans.now) << ")\n" << "out messages"; vm::Dictionary dict{trans.r1.out_msgs, 15}; for (td::int32 i = 0; i < trans.outmsg_cnt; i++) { @@ -336,6 +364,7 @@ HttpAnswer& HttpAnswer::operator<<(AccountCell acc_c) { ton::LogicalTime last_trans_lt = 0; ton::Bits256 last_trans_hash; last_trans_hash.set_zero(); + block::CurrencyCollection balance = block::CurrencyCollection::zero(); try { auto state_root = vm::MerkleProof::virtualize(acc_c.q_roots[1], 1); if (state_root.is_null()) { @@ -368,6 +397,20 @@ HttpAnswer& HttpAnswer::operator<<(AccountCell acc_c) { } last_trans_hash = acc_info.last_trans_hash; last_trans_lt = acc_info.last_trans_lt; + block::gen::Account::Record_account acc; + block::gen::AccountStorage::Record storage_rec; + if (!tlb::unpack_cell(acc_c.root, acc)) { + abort("cannot unpack Account"); + return *this; + } + if (!tlb::csr_unpack(acc.storage, storage_rec)) { + abort("cannot unpack AccountStorage"); + return *this; + } + if (!balance.unpack(storage_rec.balance)) { + abort("cannot unpack account balance"); + return *this; + } } else if (acc_c.root.not_null()) { abort(PSTRING() << "account state proof shows that account state for " << acc_c.addr.workchain << ":" << acc_c.addr.addr.to_hex() << " must be empty, but it is not"); @@ -405,6 +448,7 @@ HttpAnswer& HttpAnswer::operator<<(AccountCell acc_c) { *this << "workchain" << acc_c.addr.workchain << ""; *this << "account hex" << acc_c.addr.addr.to_hex() << ""; *this << "account" << acc_c.addr.rserialize(true) << ""; + *this << "balance" << balance.to_str() << ""; if (last_trans_lt > 0) { *this << "last transaction" << "lt=" << last_trans_lt @@ -456,7 +500,7 @@ HttpAnswer& HttpAnswer::operator<<(BlockHeaderCell head_c) { << "block" << block_id.id.to_str() << "\n" << "roothash" << block_id.root_hash.to_hex() << "\n" << "filehash" << block_id.file_hash.to_hex() << "\n" - << "time" << info.gen_utime << "\n" + << "time" << info.gen_utime << " (" << time_to_human(info.gen_utime) << ")\n" << "lt" << info.start_lt << " .. " << info.end_lt << "\n" << "global_id" << blk.global_id << "\n" << "version" << info.version << "\n" @@ -543,7 +587,8 @@ HttpAnswer& HttpAnswer::operator<<(BlockShardsCell shards_c) { ton::ShardIdFull shard{id.workchain, id.shard}; if (ref.not_null()) { *this << "" << shard.to_str() << "top_block_id()} - << "\">" << ref->top_block_id().id.seqno << "" << ref->created_at() << "" + << "\">" << ref->top_block_id().id.seqno << "created_at()) << "\">" << ref->created_at() << "" << "" << ref->want_split_ << "" << "" << ref->want_merge_ << "" << "" << ref->before_split_ << "" diff --git a/blockchain-explorer/blockchain-explorer-query.cpp b/blockchain-explorer/blockchain-explorer-query.cpp index b53e7969..26a6787e 100644 --- a/blockchain-explorer/blockchain-explorer-query.cpp +++ b/blockchain-explorer/blockchain-explorer-query.cpp @@ -50,32 +50,6 @@ #include "vm/vm.h" #include "vm/cp0.h" -namespace { - -td::Ref prepare_vm_c7(ton::UnixTime now, ton::LogicalTime lt, td::Ref my_addr, - const block::CurrencyCollection &balance) { - td::BitArray<256> rand_seed; - td::RefInt256 rand_seed_int{true}; - td::Random::secure_bytes(rand_seed.as_slice()); - if (!rand_seed_int.unique_write().import_bits(rand_seed.cbits(), 256, false)) { - return {}; - } - auto tuple = vm::make_tuple_ref(td::make_refint(0x076ef1ea), // [ magic:0x076ef1ea - td::make_refint(0), // actions:Integer - td::make_refint(0), // msgs_sent:Integer - td::make_refint(now), // unixtime:Integer - td::make_refint(lt), // block_lt:Integer - td::make_refint(lt), // trans_lt:Integer - std::move(rand_seed_int), // rand_seed:Integer - balance.as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)] - my_addr, // myself:MsgAddressInt - vm::StackEntry()); // global_config:(Maybe Cell) ] = SmartContractInfo; - LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple).to_string(); - return vm::make_tuple_ref(std::move(tuple)); -} - -} // namespace - td::Result parse_block_id(std::map &opts, bool allow_empty) { if (allow_empty) { if (opts.count("workchain") == 0 && opts.count("shard") == 0 && opts.count("seqno") == 0) { @@ -1343,111 +1317,71 @@ void HttpQueryRunMethod::start_up_query() { if (R.is_error()) { td::actor::send_closure(SelfId, &HttpQueryRunMethod::abort_query, R.move_as_error_prefix("litequery failed: ")); } else { - td::actor::send_closure(SelfId, &HttpQueryRunMethod::got_account, R.move_as_ok()); + td::actor::send_closure(SelfId, &HttpQueryRunMethod::got_result, R.move_as_ok()); } }); + auto a = ton::create_tl_object(addr_.workchain, addr_.addr); - auto query = ton::serialize_tl_object(ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), std::move(a)), - true); + td::int64 method_id = (td::crc16(td::Slice{method_name_}) & 0xffff) | 0x10000; + + // serialize params + vm::CellBuilder cb; + td::Ref cell; + if (!(vm::Stack{params_}.serialize(cb) && cb.finalize_to(cell))) { + return abort_query(td::Status::Error("cannot serialize stack with get-method parameters")); + } + auto params_serialized = vm::std_boc_serialize(std::move(cell)); + if (params_serialized.is_error()) { + return abort_query(params_serialized.move_as_error_prefix("cannot serialize stack with get-method parameters : ")); + } + + auto query = ton::serialize_tl_object( + ton::create_tl_object( + 0x17, ton::create_tl_lite_block_id(block_id_), std::move(a), method_id, params_serialized.move_as_ok()), + true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, std::move(query), std::move(P)); } -void HttpQueryRunMethod::got_account(td::BufferSlice data) { - auto F = ton::fetch_tl_object(std::move(data), true); +void HttpQueryRunMethod::got_result(td::BufferSlice data) { + auto F = ton::fetch_tl_object(std::move(data), true); if (F.is_error()) { - abort_query(F.move_as_error()); - return; + return abort_query(F.move_as_error()); } - auto f = F.move_as_ok(); - data_ = std::move(f->state_); - proof_ = std::move(f->proof_); - shard_proof_ = std::move(f->shard_proof_); - block_id_ = ton::create_block_id(f->id_); - res_block_id_ = ton::create_block_id(f->shardblk_); - - finish_query(); -} - -void HttpQueryRunMethod::finish_query() { - if (promise_) { - auto page = [&]() -> std::string { - HttpAnswer A{"account", prefix_}; - A.set_account_id(addr_); - A.set_block_id(res_block_id_); - - block::AccountState account_state; - account_state.blk = block_id_; - account_state.shard_blk = res_block_id_; - account_state.shard_proof = std::move(shard_proof_); - account_state.proof = std::move(proof_); - account_state.state = std::move(data_); - auto r_info = account_state.validate(block_id_, addr_); - if (r_info.is_error()) { - A.abort(r_info.move_as_error()); - return A.finish(); - } - auto info = r_info.move_as_ok(); - if (info.root.is_null()) { - A.abort(PSTRING() << "account state of " << addr_ << " is empty (cannot run method `" << method_name_ << "`)"); - return A.finish(); - } - block::gen::Account::Record_account acc; - block::gen::AccountStorage::Record store; - block::CurrencyCollection balance; - if (!(tlb::unpack_cell(info.root, acc) && tlb::csr_unpack(acc.storage, store) && - balance.validate_unpack(store.balance))) { - A.abort("error unpacking account state"); - return A.finish(); - } - int tag = block::gen::t_AccountState.get_tag(*store.state); - switch (tag) { - case block::gen::AccountState::account_uninit: - A.abort(PSTRING() << "account " << addr_ << " not initialized yet (cannot run any methods)"); - return A.finish(); - case block::gen::AccountState::account_frozen: - A.abort(PSTRING() << "account " << addr_ << " frozen (cannot run any methods)"); - return A.finish(); - } - - CHECK(store.state.write().fetch_ulong(1) == 1); // account_init$1 _:StateInit = AccountState; - block::gen::StateInit::Record state_init; - CHECK(tlb::csr_unpack(store.state, state_init)); - auto code = state_init.code->prefetch_ref(); - auto data = state_init.data->prefetch_ref(); - auto stack = td::make_ref(std::move(params_)); - td::int64 method_id = (td::crc16(td::Slice{method_name_}) & 0xffff) | 0x10000; - stack.write().push_smallint(method_id); - long long gas_limit = vm::GasLimits::infty; - // OstreamLogger ostream_logger(ctx.error_stream); - // auto log = create_vm_log(ctx.error_stream ? &ostream_logger : nullptr); - vm::GasLimits gas{gas_limit}; - LOG(DEBUG) << "creating VM"; - vm::VmState vm{code, std::move(stack), gas, 1, data, vm::VmLog()}; - vm.set_c7(prepare_vm_c7(info.gen_utime, info.gen_lt, acc.addr, balance)); // tuple with SmartContractInfo - // vm.incr_stack_trace(1); // enable stack dump after each step - int exit_code = ~vm.run(); - if (exit_code != 0) { - A.abort(PSTRING() << "VM terminated with error code " << exit_code); - return A.finish(); - } - stack = vm.get_stack_ref(); - { - std::ostringstream os; - os << "result: "; - stack->dump(os, 3); - - A << HttpAnswer::CodeBlock{os.str()}; - } - + auto page = [&]() -> std::string { + HttpAnswer A{"account", prefix_}; + A.set_account_id(addr_); + A.set_block_id(ton::create_block_id(f->id_)); + if (f->exit_code_ != 0) { + A.abort(PSTRING() << "VM terminated with error code " << f->exit_code_); return A.finish(); - }(); - auto R = MHD_create_response_from_buffer(page.length(), const_cast(page.c_str()), MHD_RESPMEM_MUST_COPY); - MHD_add_response_header(R, "Content-Type", "text/html"); - promise_.set_value(std::move(R)); - } + } + + std::ostringstream os; + os << "result: "; + if (f->result_.empty()) { + os << ""; + } else { + auto r_cell = vm::std_boc_deserialize(f->result_); + if (r_cell.is_error()) { + A.abort(PSTRING() << "cannot deserialize VM result boc: " << r_cell.move_as_error()); + return A.finish(); + } + auto cs = vm::load_cell_slice(r_cell.move_as_ok()); + td::Ref stack; + if (!(vm::Stack::deserialize_to(cs, stack, 0) && cs.empty_ext())) { + A.abort("VM result boc cannot be deserialized"); + return A.finish(); + } + stack->dump(os, 3); + } + A << HttpAnswer::CodeBlock{os.str()}; + return A.finish(); + }(); + auto R = MHD_create_response_from_buffer(page.length(), const_cast(page.c_str()), MHD_RESPMEM_MUST_COPY); + MHD_add_response_header(R, "Content-Type", "text/html"); + promise_.set_value(std::move(R)); stop(); } HttpQueryStatus::HttpQueryStatus(std::string prefix, td::Promise promise) @@ -1498,7 +1432,7 @@ void HttpQueryStatus::finish_query() { for (td::uint32 i = 0; i < results_.ips.size(); i++) { A << ""; if (results_.ips[i].is_valid()) { - A << "" << results_.ips[i] << ""; + A << "" << results_.ips[i].get_ip_str() << ":" << results_.ips[i].get_port() << ""; } else { A << "hidden"; } diff --git a/blockchain-explorer/blockchain-explorer-query.hpp b/blockchain-explorer/blockchain-explorer-query.hpp index 0ce52af6..29501265 100644 --- a/blockchain-explorer/blockchain-explorer-query.hpp +++ b/blockchain-explorer/blockchain-explorer-query.hpp @@ -311,22 +311,14 @@ class HttpQueryRunMethod : public HttpQueryCommon { std::vector params, std::string prefix, td::Promise promise); HttpQueryRunMethod(std::map opts, std::string prefix, td::Promise promise); - void finish_query(); - void start_up_query() override; - void got_account(td::BufferSlice result); + void got_result(td::BufferSlice result); private: ton::BlockIdExt block_id_; block::StdAddress addr_; - std::string method_name_; std::vector params_; - - td::BufferSlice data_; - td::BufferSlice proof_; - td::BufferSlice shard_proof_; - ton::BlockIdExt res_block_id_; }; class HttpQueryStatus : public HttpQueryCommon { diff --git a/blockchain-explorer/blockchain-explorer.cpp b/blockchain-explorer/blockchain-explorer.cpp index 035b84ea..ca50d526 100644 --- a/blockchain-explorer/blockchain-explorer.cpp +++ b/blockchain-explorer/blockchain-explorer.cpp @@ -52,11 +52,12 @@ #include "vm/boc.h" #include "vm/cellops.h" #include "vm/cells/MerkleProof.h" -#include "vm/cp0.h" +#include "vm/vm.h" #include "auto/tl/lite_api.h" #include "ton/lite-tl.hpp" #include "tl-utils/lite-utils.hpp" +#include "lite-client/ext-client.h" #include @@ -103,30 +104,31 @@ class HttpQueryRunner { Self->finish(nullptr); } }); - mutex_.lock(); scheduler_ptr->run_in_context_external([&]() { func(std::move(P)); }); } void finish(MHD_Response* response) { + std::unique_lock lock(mutex_); response_ = response; - mutex_.unlock(); + cond.notify_all(); } MHD_Response* wait() { - mutex_.lock(); - mutex_.unlock(); + std::unique_lock lock(mutex_); + cond.wait(lock, [&]() { return response_ != nullptr; }); return response_; } private: std::function)> func_; - MHD_Response* response_; + MHD_Response* response_ = nullptr; std::mutex mutex_; + std::condition_variable cond; }; class CoreActor : public CoreActorInterface { private: std::string global_config_ = "ton-global.config"; - std::vector> clients_; + td::actor::ActorOwn client_; td::uint32 http_port_ = 80; MHD_Daemon* daemon_ = nullptr; @@ -136,35 +138,29 @@ class CoreActor : public CoreActorInterface { bool hide_ips_ = false; - std::unique_ptr make_callback(td::uint32 idx) { - class Callback : public ton::adnl::AdnlExtClient::Callback { + td::unique_ptr make_callback() { + class Callback : public liteclient::ExtClient::Callback { public: - void on_ready() override { - td::actor::send_closure(id_, &CoreActor::conn_ready, idx_); - } - void on_stop_ready() override { - td::actor::send_closure(id_, &CoreActor::conn_closed, idx_); - } - Callback(td::actor::ActorId id, td::uint32 idx) : id_(std::move(id)), idx_(idx) { + Callback(td::actor::ActorId id) : id_(std::move(id)) { } private: td::actor::ActorId id_; - td::uint32 idx_; }; - return std::make_unique(actor_id(this), idx); + return td::make_unique(actor_id(this)); } std::shared_ptr new_result_; td::int32 attempt_ = 0; td::int32 waiting_ = 0; - std::vector ready_; + size_t n_servers_ = 0; void run_queries(); - void got_result(td::uint32 idx, td::int32 attempt, td::Result data); - void send_query(td::uint32 idx); + void got_servers_ready(td::int32 attempt, std::vector ready); + void send_ping(td::uint32 idx); + void got_ping_result(td::uint32 idx, td::int32 attempt, td::Result data); void add_result() { if (new_result_) { @@ -195,12 +191,6 @@ class CoreActor : public CoreActorInterface { static CoreActor* instance_; td::actor::ActorId self_id_; - void conn_ready(td::uint32 idx) { - ready_.at(idx) = true; - } - void conn_closed(td::uint32 idx) { - ready_.at(idx) = false; - } void set_global_config(std::string str) { global_config_ = str; } @@ -225,10 +215,7 @@ class CoreActor : public CoreActorInterface { hide_ips_ = value; } - void send_lite_query(td::uint32 idx, td::BufferSlice query, td::Promise promise); - void send_lite_query(td::BufferSlice data, td::Promise promise) override { - return send_lite_query(0, std::move(data), std::move(promise)); - } + void send_lite_query(td::BufferSlice query, td::Promise promise) override; void get_last_result(td::Promise> promise) override { } void get_results(td::uint32 max, td::Promise promise) override { @@ -448,33 +435,27 @@ class CoreActor : public CoreActorInterface { } void run() { + std::vector servers; if (remote_public_key_.empty()) { auto G = td::read_file(global_config_).move_as_ok(); auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); ton::ton_api::liteclient_config_global gc; ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); - - CHECK(gc.liteservers_.size() > 0); - td::uint32 size = static_cast(gc.liteservers_.size()); - ready_.resize(size, false); - - for (td::uint32 i = 0; i < size; i++) { - auto& cli = gc.liteservers_[i]; - td::IPAddress addr; - addr.init_host_port(td::IPAddress::ipv4_to_str(cli->ip_), cli->port_).ensure(); - addrs_.push_back(addr); - clients_.emplace_back(ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull::create(cli->id_).move_as_ok(), - addr, make_callback(i))); + auto r_servers = liteclient::LiteServerConfig::parse_global_config(gc); + r_servers.ensure(); + servers = r_servers.move_as_ok(); + for (const auto& serv : servers) { + addrs_.push_back(serv.addr); } } else { if (!remote_addr_.is_valid()) { LOG(FATAL) << "remote addr not set"; } - ready_.resize(1, false); addrs_.push_back(remote_addr_); - clients_.emplace_back(ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull{remote_public_key_}, - remote_addr_, make_callback(0))); + servers.push_back(liteclient::LiteServerConfig{ton::adnl::AdnlNodeIdFull{remote_public_key_}, remote_addr_}); } + n_servers_ = servers.size(); + client_ = liteclient::ExtClient::create(std::move(servers), make_callback(), true); daemon_ = MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, static_cast(http_port_), nullptr, nullptr, &process_http_request, nullptr, MHD_OPTION_NOTIFY_COMPLETED, request_completed, nullptr, MHD_OPTION_THREAD_POOL_SIZE, 16, MHD_OPTION_END); @@ -482,7 +463,46 @@ class CoreActor : public CoreActorInterface { } }; -void CoreActor::got_result(td::uint32 idx, td::int32 attempt, td::Result R) { +void CoreActor::run_queries() { + waiting_ = 0; + new_result_ = std::make_shared(n_servers_, td::Timestamp::at_unix(attempt_ * 60)); + td::actor::send_closure(client_, &liteclient::ExtClient::get_servers_status, + [SelfId = actor_id(this), attempt = attempt_](td::Result> R) { + R.ensure(); + td::actor::send_closure(SelfId, &CoreActor::got_servers_ready, attempt, R.move_as_ok()); + }); +} + +void CoreActor::got_servers_ready(td::int32 attempt, std::vector ready) { + if (attempt != attempt_) { + return; + } + CHECK(ready.size() == n_servers_); + for (td::uint32 i = 0; i < n_servers_; i++) { + if (ready[i]) { + send_ping(i); + } + } + CHECK(waiting_ >= 0); + if (waiting_ == 0) { + add_result(); + } +} + +void CoreActor::send_ping(td::uint32 idx) { + waiting_++; + auto query = ton::create_tl_object(); + auto q = ton::create_tl_object(serialize_tl_object(query, true)); + + auto P = + td::PromiseCreator::lambda([SelfId = actor_id(this), idx, attempt = attempt_](td::Result R) { + td::actor::send_closure(SelfId, &CoreActor::got_ping_result, idx, attempt, std::move(R)); + }); + td::actor::send_closure(client_, &liteclient::ExtClient::send_query_to_server, "query", serialize_tl_object(q, true), + idx, td::Timestamp::in(10.0), std::move(P)); +} + +void CoreActor::got_ping_result(td::uint32 idx, td::int32 attempt, td::Result R) { if (attempt != attempt_) { return; } @@ -523,39 +543,7 @@ void CoreActor::got_result(td::uint32 idx, td::int32 attempt, td::Result(); - auto q = ton::create_tl_object(serialize_tl_object(query, true)); - - auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), idx, attempt = attempt_](td::Result R) { - td::actor::send_closure(SelfId, &CoreActor::got_result, idx, attempt, std::move(R)); - }); - td::actor::send_closure(clients_[idx], &ton::adnl::AdnlExtClient::send_query, "query", serialize_tl_object(q, true), - td::Timestamp::in(10.0), std::move(P)); -} - -void CoreActor::run_queries() { - waiting_ = 0; - new_result_ = std::make_shared(ready_.size(), td::Timestamp::at_unix(attempt_ * 60)); - for (td::uint32 i = 0; i < ready_.size(); i++) { - send_query(i); - } - CHECK(waiting_ >= 0); - if (waiting_ == 0) { - add_result(); - } -} - -void CoreActor::send_lite_query(td::uint32 idx, td::BufferSlice query, td::Promise promise) { - if (!ready_[idx]) { - promise.set_error(td::Status::Error(ton::ErrorCode::notready, "ext conn not ready")); - return; - } +void CoreActor::send_lite_query(td::BufferSlice query, td::Promise promise) { auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); @@ -573,7 +561,7 @@ void CoreActor::send_lite_query(td::uint32 idx, td::BufferSlice query, td::Promi promise.set_value(std::move(B)); }); auto q = ton::create_tl_object(std::move(query)); - td::actor::send_closure(clients_[idx], &ton::adnl::AdnlExtClient::send_query, "query", serialize_tl_object(q, true), + td::actor::send_closure(client_, &liteclient::ExtClient::send_query, "query", serialize_tl_object(q, true), td::Timestamp::in(10.0), std::move(P)); } @@ -654,7 +642,7 @@ int main(int argc, char* argv[]) { }); #endif - vm::init_op_cp0(); + vm::init_vm().ensure(); td::actor::Scheduler scheduler({2}); scheduler_ptr = &scheduler; diff --git a/catchain/CMakeLists.txt b/catchain/CMakeLists.txt index a57d3788..3f766688 100644 --- a/catchain/CMakeLists.txt +++ b/catchain/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) @@ -34,6 +34,5 @@ target_include_directories(overlay PUBLIC $/.. ${OPENSSL_INCLUDE_DIR} ) -target_link_libraries(catchain PRIVATE tdutils tdactor adnl tl_api dht tdfec - overlay) +target_link_libraries(catchain PRIVATE tdutils tdactor adnl tl_api dht tdfec overlay) diff --git a/catchain/catchain-receiver-interface.h b/catchain/catchain-receiver-interface.h index c8f1ef66..bc02832a 100644 --- a/catchain/catchain-receiver-interface.h +++ b/catchain/catchain-receiver-interface.h @@ -52,6 +52,7 @@ class CatChainReceiverInterface : public td::actor::Actor { td::BufferSlice query, td::uint64 max_answer_size, td::actor::ActorId via) = 0; virtual void send_custom_message_data(const PublicKeyHash &dst, td::BufferSlice query) = 0; + virtual void on_blame_processed(td::uint32 source_id) = 0; virtual void destroy() = 0; diff --git a/catchain/catchain-receiver-source.cpp b/catchain/catchain-receiver-source.cpp index 6a9e0777..e758b335 100644 --- a/catchain/catchain-receiver-source.cpp +++ b/catchain/catchain-receiver-source.cpp @@ -60,15 +60,15 @@ td::Result> CatChainReceiverSource::crea void CatChainReceiverSourceImpl::blame(td::uint32 fork, CatChainBlockHeight height) { blame(); - if (!blamed_heights_.empty()) { - if (blamed_heights_.size() <= fork) { - blamed_heights_.resize(fork + 1, 0); - } - if (blamed_heights_[fork] == 0 || blamed_heights_[fork] > height) { - VLOG(CATCHAIN_INFO) << this << ": blamed at " << fork << " " << height; - blamed_heights_[fork] = height; - } + // if (!blamed_heights_.empty()) { + if (blamed_heights_.size() <= fork) { + blamed_heights_.resize(fork + 1, 0); } + if (blamed_heights_[fork] == 0 || blamed_heights_[fork] > height) { + VLOG(CATCHAIN_INFO) << this << ": blamed at " << fork << " " << height; + blamed_heights_[fork] = height; + } + // } } void CatChainReceiverSourceImpl::blame() { @@ -144,7 +144,7 @@ void CatChainReceiverSourceImpl::on_new_block(CatChainReceivedBlock *block) { on_found_fork_proof(create_serialize_tl_object(block->export_tl_dep(), it->second->export_tl_dep()) .as_slice()); - chain_->add_prepared_event(fork_proof()); + chain_->on_found_fork_proof(id_, fork_proof()); } blame(); return; @@ -162,6 +162,15 @@ void CatChainReceiverSourceImpl::on_found_fork_proof(const td::Slice &proof) { } } +bool CatChainReceiverSourceImpl::allow_send_block(CatChainBlockHash hash) { + td::uint32 count = ++block_requests_count_[hash]; + if (count > MAX_BLOCK_REQUESTS) { + VLOG(CATCHAIN_INFO) << this << ": node requested block " << hash << " " << count << " times"; + return false; + } + return true; +} + } // namespace catchain } // namespace ton diff --git a/catchain/catchain-receiver-source.h b/catchain/catchain-receiver-source.h index e805c2a5..136906a0 100644 --- a/catchain/catchain-receiver-source.h +++ b/catchain/catchain-receiver-source.h @@ -61,6 +61,9 @@ class CatChainReceiverSource { virtual td::BufferSlice fork_proof() const = 0; virtual bool fork_is_found() const = 0; + // One block can be sent to one node in catchain.getDifference only a limited number of times to prevent DoS + virtual bool allow_send_block(CatChainBlockHash hash) = 0; + static td::Result> create(CatChainReceiver *chain, PublicKey pub_key, adnl::AdnlNodeIdShort adnl_id, td::uint32 id); diff --git a/catchain/catchain-receiver-source.hpp b/catchain/catchain-receiver-source.hpp index 0785d8c2..cf08c421 100644 --- a/catchain/catchain-receiver-source.hpp +++ b/catchain/catchain-receiver-source.hpp @@ -111,6 +111,8 @@ class CatChainReceiverSourceImpl : public CatChainReceiverSource { return chain_; } + bool allow_send_block(CatChainBlockHash hash) override; + CatChainReceiverSourceImpl(CatChainReceiver *chain, PublicKey source, adnl::AdnlNodeIdShort adnl_id, td::uint32 id); private: @@ -130,6 +132,11 @@ class CatChainReceiverSourceImpl : public CatChainReceiverSource { CatChainBlockHeight delivered_height_ = 0; CatChainBlockHeight received_height_ = 0; + + std::map block_requests_count_; + // One block can be sent to one node up to 5 times + + static const td::uint32 MAX_BLOCK_REQUESTS = 5; }; } // namespace catchain diff --git a/catchain/catchain-receiver.cpp b/catchain/catchain-receiver.cpp index 488fbb9d..a6160383 100644 --- a/catchain/catchain-receiver.cpp +++ b/catchain/catchain-receiver.cpp @@ -27,6 +27,8 @@ #include "catchain-receiver.hpp" +#include "td/utils/ThreadSafeCounter.h" + namespace ton { namespace catchain { @@ -168,13 +170,6 @@ void CatChainReceiverImpl::receive_message_from_overlay(adnl::AdnlNodeIdShort sr return; } - /*auto S = get_source_by_hash(src); - CHECK(S != nullptr); - - if (S->blamed()) { - VLOG(CATCHAIN_INFO) << this << ": dropping block update from blamed source " << src; - return; - }*/ if (data.size() > opts_.max_serialized_block_size) { VLOG(CATCHAIN_WARNING) << this << ": dropping broken block from " << src << ": too big (size=" << data.size() << ", limit=" << opts_.max_serialized_block_size << ")"; @@ -197,19 +192,6 @@ void CatChainReceiverImpl::receive_broadcast_from_overlay(const PublicKeyHash &s callback_->on_broadcast(src, std::move(data)); } -/*void CatChainReceiverImpl::send_block(const PublicKeyHash &src, tl_object_ptr block, - td::BufferSlice payload) { - CHECK(read_db_); - CHECK(src == local_id_); - - validate_block_sync(block, payload.as_slice()).ensure(); - auto B = create_block(std::move(block), td::SharedSlice{payload.as_slice()}); - CHECK(B != nullptr); - - run_scheduler(); - CHECK(B->delivered()); -}*/ - CatChainReceivedBlock *CatChainReceiverImpl::create_block(tl_object_ptr block, td::SharedSlice payload) { if (block->height_ == 0) { @@ -267,21 +249,16 @@ td::Status CatChainReceiverImpl::validate_block_sync(const tl_object_ptr &block, const td::Slice &payload) const { - //LOG(INFO) << ton_api::to_string(block); TRY_STATUS_PREFIX(CatChainReceivedBlock::pre_validate_block(this, block, payload), "failed to validate block: "); + // After pre_validate_block, block->height_ > 0 + auto id = CatChainReceivedBlock::block_id(this, block, payload); + td::BufferSlice B = serialize_tl_object(id, true); - if (block->height_ > 0) { - auto id = CatChainReceivedBlock::block_id(this, block, payload); - td::BufferSlice B = serialize_tl_object(id, true); - - CatChainReceiverSource *S = get_source_by_hash(PublicKeyHash{id->src_}); - CHECK(S != nullptr); - Encryptor *E = S->get_encryptor_sync(); - CHECK(E != nullptr); - return E->check_signature(B.as_slice(), block->signature_.as_slice()); - } else { - return td::Status::OK(); - } + CatChainReceiverSource *S = get_source_by_hash(PublicKeyHash{id->src_}); + CHECK(S != nullptr); + Encryptor *E = S->get_encryptor_sync(); + CHECK(E != nullptr); + return E->check_signature(B.as_slice(), block->signature_.as_slice()); } void CatChainReceiverImpl::run_scheduler() { @@ -312,7 +289,9 @@ void CatChainReceiverImpl::add_block_cont_3(tl_object_ptrdelivered()); + LOG_CHECK(last_sent_block_->delivered()) + << "source=" << last_sent_block_->get_source_id() << " ill=" << last_sent_block_->is_ill() + << " height=" << last_sent_block_->get_height(); } active_send_ = false; @@ -391,6 +370,12 @@ void CatChainReceiverImpl::add_block(td::BufferSlice payload, std::vectorheight_ + 1; + auto max_block_height = get_max_block_height(opts_, sources_.size()); + if (td::narrow_cast(height) > max_block_height) { + VLOG(CATCHAIN_WARNING) << this << ": cannot create block: max height exceeded (" << max_block_height << ")"; + active_send_ = false; + return; + } auto block_data = create_tl_object(std::move(prev), std::move(deps_arr)); auto block = create_tl_object(incarnation_, local_idx_, height, std::move(block_data), td::BufferSlice()); @@ -515,7 +500,6 @@ CatChainReceiverImpl::CatChainReceiverImpl(std::unique_ptr callback, } CHECK(local_idx_ != static_cast(ids.size())); - //std::sort(short_ids.begin(), short_ids.end()); auto F = create_tl_object(unique_hash, std::move(short_ids)); overlay_full_id_ = overlay::OverlayIdFull{serialize_tl_object(F, true)}; @@ -527,6 +511,8 @@ CatChainReceiverImpl::CatChainReceiverImpl(std::unique_ptr callback, blocks_[root_block_->get_hash()] = std::move(R); last_sent_block_ = root_block_; + blame_processed_.resize(sources_.size(), false); + choose_neighbours(); } @@ -540,9 +526,12 @@ void CatChainReceiverImpl::start_up() { for (td::uint32 i = 0; i < get_sources_cnt(); i++) { root_keys.emplace(get_source(i)->get_hash(), OVERLAY_MAX_ALLOWED_PACKET_SIZE); } - td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay, + overlay::OverlayOptions overlay_options; + overlay_options.broadcast_speed_multiplier_ = opts_.broadcast_speed_multiplier; + td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay_ex, get_source(local_idx_)->get_adnl_id(), overlay_full_id_.clone(), std::move(ids), - make_callback(), overlay::OverlayPrivacyRules{0, 0, std::move(root_keys)}); + make_callback(), overlay::OverlayPrivacyRules{0, 0, std::move(root_keys)}, + R"({ "type": "catchain" })", std::move(overlay_options)); CHECK(root_block_); @@ -700,11 +689,11 @@ void CatChainReceiverImpl::receive_query_from_overlay(adnl::AdnlNodeIdShort src, promise.set_error(td::Status::Error(ErrorCode::notready, "db not read")); return; } + TD_PERF_COUNTER(catchain_query_process); td::PerfWarningTimer t{"catchain query process", 0.05}; auto F = fetch_tl_object(data.clone(), true); if (F.is_error()) { callback_->on_custom_query(get_source_by_adnl_id(src)->get_hash(), std::move(data), std::move(promise)); - //LOG(WARNING) << this << ": unknown query from " << src; return; } auto f = F.move_as_ok(); @@ -717,70 +706,13 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat if (it == blocks_.end() || it->second->get_height() == 0 || !it->second->initialized()) { promise.set_value(serialize_tl_object(create_tl_object(), true)); } else { + CatChainReceiverSource *S = get_source_by_adnl_id(src); + CHECK(S != nullptr); promise.set_value(serialize_tl_object(create_tl_object(it->second->export_tl()), true, it->second->get_payload().as_slice())); } } -void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlocks query, - td::Promise promise) { - if (query.blocks_.size() > MAX_QUERY_BLOCKS) { - promise.set_error(td::Status::Error(ErrorCode::protoviolation, "too many blocks")); - return; - } - td::int32 cnt = 0; - for (const CatChainBlockHash &b : query.blocks_) { - auto it = blocks_.find(b); - if (it != blocks_.end() && it->second->get_height() > 0) { - auto block = create_tl_object(it->second->export_tl()); - CHECK(!it->second->get_payload().empty()); - td::BufferSlice B = serialize_tl_object(block, true, it->second->get_payload().clone()); - CHECK(B.size() <= opts_.max_serialized_block_size); - td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_message, src, - get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(B)); - cnt++; - } - } - promise.set_value(serialize_tl_object(create_tl_object(cnt), true)); -} - -void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlockHistory query, - td::Promise promise) { - int64_t h = query.height_; - if (h <= 0) { - promise.set_error(td::Status::Error(ErrorCode::protoviolation, "not-positive height")); - return; - } - if (h > MAX_QUERY_HEIGHT) { - h = MAX_QUERY_HEIGHT; - } - std::set s{query.stop_if_.begin(), query.stop_if_.end()}; - - CatChainReceivedBlock *B = get_block(query.block_); - if (B == nullptr) { - promise.set_value(serialize_tl_object(create_tl_object(0), true)); - return; - } - if (static_cast(h) > B->get_height()) { - h = B->get_height(); - } - td::uint32 cnt = 0; - while (h-- > 0) { - if (s.find(B->get_hash()) != s.end()) { - break; - } - auto block = create_tl_object(B->export_tl()); - CHECK(!B->get_payload().empty()); - td::BufferSlice BB = serialize_tl_object(block, true, B->get_payload().as_slice()); - CHECK(BB.size() <= opts_.max_serialized_block_size); - td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_message, src, - get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(BB)); - B = B->get_prev(); - cnt++; - } - promise.set_value(serialize_tl_object(create_tl_object(cnt), true)); -} - void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getDifference query, td::Promise promise) { auto &vt = query.rt_; @@ -832,6 +764,8 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat } } CHECK(right > 0); + CatChainReceiverSource *S0 = get_source_by_adnl_id(src); + CHECK(S0 != nullptr); for (td::uint32 i = 0; i < get_sources_cnt(); i++) { if (vt[i] >= 0 && my_vt[i] > vt[i]) { CatChainReceiverSource *S = get_source(i); @@ -839,12 +773,14 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat while (t-- > 0) { CatChainReceivedBlock *M = S->get_block(++vt[i]); CHECK(M != nullptr); - auto block = create_tl_object(M->export_tl()); - CHECK(!M->get_payload().empty()); - td::BufferSlice BB = serialize_tl_object(block, true, M->get_payload().as_slice()); - CHECK(BB.size() <= opts_.max_serialized_block_size); - td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_message, src, - get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(BB)); + if (S0->allow_send_block(M->get_hash())) { + auto block = create_tl_object(M->export_tl()); + CHECK(!M->get_payload().empty()); + td::BufferSlice BB = serialize_tl_object(block, true, M->get_payload().as_slice()); + CHECK(BB.size() <= opts_.max_serialized_block_size); + td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_message, src, + get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(BB)); + } } } } @@ -1031,10 +967,15 @@ void CatChainReceiverImpl::written_unsafe_root_block(CatChainReceivedBlock *bloc void CatChainReceiverImpl::alarm() { alarm_timestamp() = td::Timestamp::never(); - if (next_sync_ && next_sync_.is_in_past()) { + if (next_sync_ && next_sync_.is_in_past() && get_sources_cnt() > 1) { next_sync_ = td::Timestamp::in(td::Random::fast(SYNC_INTERVAL_MIN, SYNC_INTERVAL_MAX)); for (unsigned i = 0; i < SYNC_ITERATIONS; i++) { - CatChainReceiverSource *S = get_source(td::Random::fast(0, static_cast(get_sources_cnt()) - 1)); + auto idx = td::Random::fast(1, static_cast(get_sources_cnt()) - 1); + if (idx == static_cast(local_idx_)) { + idx = 0; + } + // idx is a random number in [0, get_sources_cnt-1] not equal to local_idx + CatChainReceiverSource *S = get_source(idx); CHECK(S != nullptr); if (!S->blamed()) { synchronize_with(S); @@ -1117,6 +1058,23 @@ static void destroy_db(const std::string& name, td::uint32 attempt) { } } +void CatChainReceiverImpl::on_found_fork_proof(td::uint32 source_id, td::BufferSlice data) { + if (blame_processed_[source_id]) { + add_block(std::move(data), std::vector()); + } else { + pending_fork_proofs_[source_id] = std::move(data); + } +} + +void CatChainReceiverImpl::on_blame_processed(td::uint32 source_id) { + blame_processed_[source_id] = true; + auto it = pending_fork_proofs_.find(source_id); + if (it != pending_fork_proofs_.end()) { + add_block(std::move(it->second), std::vector()); + pending_fork_proofs_.erase(it); + } +} + void CatChainReceiverImpl::destroy() { auto name = db_root_ + "/catchainreceiver" + db_suffix_ + td::base64url_encode(as_slice(incarnation_)); delay_action([name]() { destroy_db(name, 0); }, td::Timestamp::in(DESTROY_DB_DELAY)); diff --git a/catchain/catchain-receiver.h b/catchain/catchain-receiver.h index 2c959fbc..75c87351 100644 --- a/catchain/catchain-receiver.h +++ b/catchain/catchain-receiver.h @@ -56,7 +56,7 @@ class CatChainReceiver : public CatChainReceiverInterface { virtual void run_block(CatChainReceivedBlock *block) = 0; virtual void deliver_block(CatChainReceivedBlock *block) = 0; virtual td::uint32 add_fork() = 0; - virtual void add_prepared_event(td::BufferSlice data) = 0; + virtual void on_found_fork_proof(td::uint32 source_id, td::BufferSlice data) = 0; virtual void on_blame(td::uint32 source_id) = 0; virtual const CatChainOptions &opts() const = 0; diff --git a/catchain/catchain-receiver.hpp b/catchain/catchain-receiver.hpp index 5da4001c..5c4d3764 100644 --- a/catchain/catchain-receiver.hpp +++ b/catchain/catchain-receiver.hpp @@ -39,9 +39,6 @@ class CatChainReceiverImpl final : public CatChainReceiver { return PrintId{incarnation_, local_id_}; } - void add_prepared_event(td::BufferSlice data) override { - add_block(std::move(data), std::vector()); - } CatChainSessionId get_incarnation() const override { return incarnation_; } @@ -73,15 +70,10 @@ class CatChainReceiverImpl final : public CatChainReceiver { void receive_query_from_overlay(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlock query, td::Promise promise); - void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlocks query, - td::Promise promise); - void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlockHistory query, - td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getDifference query, td::Promise promise); template void process_query(adnl::AdnlNodeIdShort src, const T &query, td::Promise promise) { - //LOG(WARNING) << this << ": unknown query from " << src; callback_->on_custom_query(get_source_by_adnl_id(src)->get_hash(), serialize_tl_object(&query, true), std::move(promise)); } @@ -89,7 +81,6 @@ class CatChainReceiverImpl final : public CatChainReceiver { void receive_block(adnl::AdnlNodeIdShort src, tl_object_ptr block, td::BufferSlice payload); void receive_block_answer(adnl::AdnlNodeIdShort src, td::BufferSlice); - //void send_block(const PublicKeyHash &src, tl_object_ptr block, td::BufferSlice payload); CatChainReceivedBlock *create_block(tl_object_ptr block, td::SharedSlice payload) override; CatChainReceivedBlock *create_block(tl_object_ptr block) override; @@ -117,6 +108,8 @@ class CatChainReceiverImpl final : public CatChainReceiver { void on_blame(td::uint32 src) override { callback_->blame(src); } + void on_found_fork_proof(td::uint32 source_id, td::BufferSlice data) override; + void on_blame_processed(td::uint32 source_id) override; const CatChainOptions &opts() const override { return opts_; } @@ -204,9 +197,6 @@ class CatChainReceiverImpl final : public CatChainReceiver { std::vector neighbours_; - //std::queue> events_; - //std::queue raw_events_; - td::actor::ActorId keyring_; td::actor::ActorId adnl_; td::actor::ActorId overlay_manager_; @@ -231,6 +221,9 @@ class CatChainReceiverImpl final : public CatChainReceiver { bool started_{false}; std::list to_run_; + + std::vector blame_processed_; + std::map pending_fork_proofs_; }; } // namespace catchain diff --git a/catchain/catchain.cpp b/catchain/catchain.cpp index 001840fd..53ae16b5 100644 --- a/catchain/catchain.cpp +++ b/catchain/catchain.cpp @@ -108,11 +108,12 @@ void CatChainImpl::need_new_block(td::Timestamp t) { if (!receiver_started_) { return; } - if (!force_process_) { + if (!force_process_ || !active_process_) { VLOG(CATCHAIN_INFO) << this << ": forcing creation of new block"; } - force_process_ = true; - if (!active_process_) { + if (active_process_) { + force_process_ = true; + } else { alarm_timestamp().relax(t); } } @@ -197,6 +198,14 @@ void CatChainImpl::on_blame(td::uint32 src_id) { } } } + for (td::uint32 i = 0; i < process_deps_.size(); ++i) { + if (blocks_[process_deps_[i]]->source() == src_id) { + process_deps_[i] = process_deps_.back(); + process_deps_.pop_back(); + --i; + } + } + td::actor::send_closure(receiver_, &CatChainReceiverInterface::on_blame_processed, src_id); } void CatChainImpl::on_custom_query(const PublicKeyHash &src, td::BufferSlice data, td::Promise promise) { diff --git a/catchain/catchain.h b/catchain/catchain.h index 912957e5..c5c8af28 100644 --- a/catchain/catchain.h +++ b/catchain/catchain.h @@ -96,6 +96,7 @@ class CatChain : public td::actor::Actor { virtual void send_query_via(const PublicKeyHash &dst, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice query, td::uint64 max_answer_size, td::actor::ActorId via) = 0; + virtual void get_source_heights(td::Promise> promise) = 0; virtual void destroy() = 0; static td::actor::ActorOwn create(std::unique_ptr callback, const CatChainOptions &opts, diff --git a/catchain/catchain.hpp b/catchain/catchain.hpp index 8c8bb99a..586cf474 100644 --- a/catchain/catchain.hpp +++ b/catchain/catchain.hpp @@ -115,6 +115,15 @@ class CatChainImpl : public CatChain { td::actor::send_closure(receiver_, &CatChainReceiverInterface::send_custom_query_data_via, dst, name, std::move(promise), timeout, std::move(query), max_answer_size, via); } + void get_source_heights(td::Promise> promise) override { + std::vector heights(top_source_blocks_.size(), 0); + for (size_t i = 0; i < top_source_blocks_.size(); ++i) { + if (top_source_blocks_[i]) { + heights[i] = top_source_blocks_[i]->height(); + } + } + promise.set_result(std::move(heights)); + } void destroy() override; CatChainImpl(std::unique_ptr callback, const CatChainOptions &opts, td::actor::ActorId keyring, td::actor::ActorId adnl, diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 8fd70f7b..88a3671b 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) set(COMMON_SOURCE checksum.h @@ -8,7 +8,8 @@ set(COMMON_SOURCE errorlog.h errorlog.cpp -) + + global-version.h) add_library(common STATIC ${COMMON_SOURCE}) diff --git a/common/delay.h b/common/delay.h index 9b98c58e..3df8e7d8 100644 --- a/common/delay.h +++ b/common/delay.h @@ -49,4 +49,29 @@ template void delay_action(T promise, td::Timestamp timeout) { DelayedAction::create(std::move(promise), timeout); } + +template +class AsyncApply : public td::actor::Actor { + public: + AsyncApply(PromiseT promise, ValueT value) : promise_(std::move(promise)), value_(std::move(value)){ + } + + void start_up() override { + promise_(std::move(value_)); + stop(); + } + + static void create(td::Slice name, PromiseT promise, ValueT value ) { + td::actor::create_actor(PSLICE() << "async:" << name, std::move(promise), std::move(value)).release(); + } + + private: + PromiseT promise_; + ValueT value_; +}; + +template +void async_apply(td::Slice name, PromiseT &&promise, ValueT &&value) { + AsyncApply::create(name, std::forward(promise), std::forward(value)); +} } // namespace ton diff --git a/tonlib/tonlib/GenericAccount.h b/common/global-version.h similarity index 57% rename from tonlib/tonlib/GenericAccount.h rename to common/global-version.h index 4a36d78a..2308ce3e 100644 --- a/tonlib/tonlib/GenericAccount.h +++ b/common/global-version.h @@ -13,18 +13,12 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . - - Copyright 2017-2019 Telegram Systems LLP */ #pragma once -#include "vm/cells.h" -#include "block/block.h" -namespace tonlib { -class GenericAccount { - public: - static td::Ref get_init_state(td::Ref code, td::Ref data) noexcept; - static block::StdAddress get_address(ton::WorkchainId workchain_id, const td::Ref& init_state) noexcept; - static td::Ref create_ext_message(const block::StdAddress& address, td::Ref new_state, - td::Ref body) noexcept; -}; -} // namespace tonlib + +namespace ton { + +// See doc/GlobalVersions.md +constexpr int SUPPORTED_VERSION = 10; + +} diff --git a/create-hardfork/CMakeLists.txt b/create-hardfork/CMakeLists.txt index 3d78c118..3024603c 100644 --- a/create-hardfork/CMakeLists.txt +++ b/create-hardfork/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) @@ -10,8 +10,7 @@ set(CREATE_HARDFORK_SOURCE ) add_executable(create-hardfork ${CREATE_HARDFORK_SOURCE}) -target_link_libraries(create-hardfork overlay tdutils tdactor adnl tl_api dht - rldp catchain validatorsession full-node validator-hardfork ton_validator +target_link_libraries(create-hardfork overlay tdutils tdactor adnl tl_api dht rldp catchain validatorsession full-node validator-hardfork ton_validator validator-hardfork fift-lib memprof git ${JEMALLOC_LIBRARIES}) install(TARGETS create-hardfork RUNTIME DESTINATION bin) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 15533768..72bffae5 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -49,7 +49,7 @@ #include "validator/fabric.h" #include "validator/impl/collator.h" -#include "crypto/vm/cp0.h" +#include "crypto/vm/vm.h" #include "crypto/block/block-db.h" #include "common/errorlog.h" @@ -213,7 +213,7 @@ class HardforkCreator : public td::actor::Actor { ton::validator::ValidatorManagerHardforkFactory::create(opts, shard_, shard_top_block_id_, db_root_); for (auto &msg : ext_msgs_) { td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManager::new_external_message, - std::move(msg)); + std::move(msg), 0); } for (auto &topmsg : top_shard_descrs_) { td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManager::new_shard_block, ton::BlockIdExt{}, @@ -236,9 +236,8 @@ class HardforkCreator : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } - void add_shard(ton::ShardIdFull) override { - } - void del_shard(ton::ShardIdFull) override { + void on_new_masterchain_block(td::Ref state, + std::set shards_to_monitor) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } @@ -246,7 +245,10 @@ class HardforkCreator : public td::actor::Actor { } void send_shard_block_info(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::BufferSlice data) override { } - void send_broadcast(ton::BlockBroadcast broadcast) override { + void send_block_candidate(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override { + } + void send_broadcast(ton::BlockBroadcast broadcast, int mode) override { } void download_block(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override { @@ -267,13 +269,19 @@ class HardforkCreator : public td::actor::Actor { void get_next_key_blocks(ton::BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) override { } - void download_archive(ton::BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - - td::Promise promise) override { + void download_archive(ton::BlockSeqno masterchain_seqno, ton::ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override { + } + void download_out_msg_queue_proof( + ton::ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, + td::Timestamp timeout, td::Promise>> promise) override { } void new_key_block(ton::validator::BlockHandle handle) override { } + void send_validator_telemetry(ton::PublicKeyHash key, + ton::tl_object_ptr telemetry) override { + } }; td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::install_callback, @@ -307,7 +315,7 @@ int main(int argc, char *argv[]) { SET_VERBOSITY_LEVEL(verbosity_INFO); td::set_default_failure_signal_handler().ensure(); - CHECK(vm::init_op_cp0()); + vm::init_vm().ensure(); td::actor::ActorOwn x; diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index fab75bfc..06908338 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -1,10 +1,10 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() -set(TON_CRYPTO_SOURCE +set(TON_CRYPTO_CORE_SOURCE Ed25519.cpp common/bigint.cpp common/refcnt.cpp @@ -19,26 +19,8 @@ set(TON_CRYPTO_SOURCE openssl/bignum.cpp openssl/residue.cpp openssl/rand.cpp - vm/stack.cpp - vm/atom.cpp - vm/continuation.cpp - vm/dict.cpp - vm/memo.cpp - vm/dispatch.cpp - vm/opctable.cpp - vm/cp0.cpp - vm/stackops.cpp - vm/tupleops.cpp - vm/arithops.cpp - vm/cellops.cpp - vm/contops.cpp - vm/dictops.cpp - vm/debugops.cpp - vm/tonops.cpp vm/boc.cpp vm/large-boc-serializer.cpp - vm/utils.cpp - vm/vm.cpp tl/tlblib.cpp Ed25519.h @@ -66,34 +48,10 @@ set(TON_CRYPTO_SOURCE tl/tlbc-data.h tl/tlblib.hpp - vm/arithops.h - vm/atom.h - vm/boc.h - vm/boc-writers.h - vm/box.hpp - vm/cellops.h - vm/continuation.h - vm/contops.h - vm/cp0.h - vm/debugops.h - vm/dict.h - vm/dictops.h - vm/excno.hpp - vm/fmt.hpp - vm/log.h - vm/memo.h - vm/opctable.h - vm/stack.hpp - vm/stackops.h - vm/tupleops.h - vm/tonops.h - vm/vmstate.h - vm/utils.h - vm/vm.h - - vm/cells.h - vm/cellslice.h + keccak/keccak.h + keccak/keccak.cpp + vm/dict.cpp vm/cells/Cell.cpp vm/cells/CellBuilder.cpp vm/cells/CellHash.cpp @@ -106,6 +64,7 @@ set(TON_CRYPTO_SOURCE vm/cells/MerkleProof.cpp vm/cells/MerkleUpdate.cpp + vm/dict.h vm/cells/Cell.h vm/cells/CellBuilder.h vm/cells/CellHash.h @@ -124,9 +83,64 @@ set(TON_CRYPTO_SOURCE vm/cells/VirtualCell.h vm/cells/VirtualizationParameters.h + vm/cells.h + vm/cellslice.h + vm/db/StaticBagOfCellsDb.h vm/db/StaticBagOfCellsDb.cpp -) + + vm/Hasher.h + vm/Hasher.cpp + + ellcurve/secp256k1.h + ellcurve/secp256k1.cpp + ellcurve/p256.h + ellcurve/p256.cpp) + +set(TON_CRYPTO_SOURCE + vm/stack.cpp + vm/atom.cpp + vm/continuation.cpp + vm/memo.cpp + vm/dispatch.cpp + vm/opctable.cpp + vm/cp0.cpp + vm/stackops.cpp + vm/tupleops.cpp + vm/arithops.cpp + vm/cellops.cpp + vm/contops.cpp + vm/dictops.cpp + vm/debugops.cpp + vm/tonops.cpp + vm/utils.cpp + vm/vm.cpp + vm/bls.cpp + + vm/arithops.h + vm/atom.h + vm/boc.h + vm/boc-writers.h + vm/box.hpp + vm/cellops.h + vm/continuation.h + vm/contops.h + vm/cp0.h + vm/debugops.h + vm/dictops.h + vm/excno.hpp + vm/fmt.hpp + vm/log.h + vm/memo.h + vm/opctable.h + vm/stack.hpp + vm/stackops.h + vm/tupleops.h + vm/tonops.h + vm/vmstate.h + vm/utils.h + vm/vm.h + vm/bls.h) set(TON_DB_SOURCE vm/db/DynamicBagOfCellsDb.cpp @@ -137,12 +151,14 @@ set(TON_DB_SOURCE vm/db/CellHashTable.h vm/db/CellStorage.h vm/db/TonDb.h + vm/db/InMemoryBagOfCellsDb.cpp ) set(FIFT_SOURCE fift/Dictionary.cpp fift/Fift.cpp fift/IntCtx.cpp + fift/HashMap.cpp fift/Continuation.cpp fift/SourceLookup.cpp fift/utils.cpp @@ -151,6 +167,7 @@ set(FIFT_SOURCE fift/Dictionary.h fift/Fift.h fift/IntCtx.h + fift/HashMap.h fift/Continuation.h fift/SourceLookup.h fift/utils.h @@ -197,6 +214,7 @@ set(BLOCK_SOURCE block/mc-config.cpp block/output-queue-merger.cpp block/transaction.cpp + block/precompiled-smc/PrecompiledSmartContract.cpp ${TLB_BLOCK_AUTO} block/block-binlog.h @@ -207,6 +225,8 @@ set(BLOCK_SOURCE block/check-proof.h block/output-queue-merger.h block/transaction.h + block/precompiled-smc/PrecompiledSmartContract.h + block/precompiled-smc/common.h ) set(SMC_ENVELOPE_SOURCE @@ -220,6 +240,7 @@ set(SMC_ENVELOPE_SOURCE smc-envelope/SmartContractCode.cpp smc-envelope/WalletInterface.cpp smc-envelope/WalletV3.cpp + smc-envelope/WalletV4.cpp smc-envelope/GenericAccount.h smc-envelope/HighloadWallet.h @@ -230,6 +251,7 @@ set(SMC_ENVELOPE_SOURCE smc-envelope/SmartContractCode.h smc-envelope/WalletInterface.h smc-envelope/WalletV3.h + smc-envelope/WalletV4.h ) set(ED25519_TEST_SOURCE @@ -269,13 +291,24 @@ set(BIGINT_TEST_SOURCE set(USE_EMSCRIPTEN ${USE_EMSCRIPTEN} PARENT_SCOPE) +add_library(ton_crypto_core STATIC ${TON_CRYPTO_CORE_SOURCE}) +target_include_directories(ton_crypto_core PUBLIC $ + $) +target_link_libraries(ton_crypto_core PUBLIC ${OPENSSL_CRYPTO_LIBRARY} tdutils tddb_utils) +if (NOT WIN32) + target_link_libraries(ton_crypto_core PUBLIC dl z) +endif() +target_include_directories(ton_crypto_core SYSTEM PUBLIC $) add_library(ton_crypto STATIC ${TON_CRYPTO_SOURCE}) -target_include_directories(ton_crypto PUBLIC $ - $) -target_link_libraries(ton_crypto PUBLIC ${OPENSSL_CRYPTO_LIBRARY} tdutils tddb_utils) +target_include_directories(ton_crypto PUBLIC $ $) +target_link_libraries(ton_crypto PUBLIC ${OPENSSL_CRYPTO_LIBRARY} ton_crypto_core ton_block) +if (USE_EMSCRIPTEN) + target_link_options(ton_crypto PRIVATE -fexceptions) + target_compile_options(ton_crypto PRIVATE -fexceptions) +endif() if (NOT WIN32) - find_library(DL dl) + find_library(DL dl) if (DL) target_link_libraries(ton_crypto PUBLIC dl z) else() @@ -284,6 +317,36 @@ if (NOT WIN32) endif() target_include_directories(ton_crypto SYSTEM PUBLIC $) +add_dependencies(ton_crypto blst) +add_dependencies(ton_crypto_core secp256k1) + +target_include_directories(ton_crypto PRIVATE ${BLST_INCLUDE_DIR}) +target_link_libraries(ton_crypto PRIVATE ${BLST_LIB}) + +target_include_directories(ton_crypto_core PUBLIC $) + +if (MSVC) + find_package(Sodium REQUIRED) + target_compile_definitions(ton_crypto PUBLIC SODIUM_STATIC) + target_link_libraries(ton_crypto_core PUBLIC ${SECP256K1_LIBRARY}) + target_link_libraries(ton_crypto PUBLIC ${SECP256K1_LIBRARY}) +elseif (EMSCRIPTEN) + target_link_libraries(ton_crypto_core PUBLIC $) + target_link_libraries(ton_crypto PUBLIC $) +else() + if (NOT SODIUM_FOUND) + find_package(Sodium REQUIRED) + else() + message(STATUS "Using Sodium ${SODIUM_LIBRARY_RELEASE}") + endif() + target_compile_definitions(ton_crypto PUBLIC SODIUM_STATIC) + target_link_libraries(ton_crypto_core PUBLIC ${SECP256K1_LIBRARY}) + target_link_libraries(ton_crypto PUBLIC ${SECP256K1_LIBRARY}) +endif() + +target_include_directories(ton_crypto_core PUBLIC $) +target_link_libraries(ton_crypto PUBLIC ${SODIUM_LIBRARY_RELEASE}) + add_library(ton_db STATIC ${TON_DB_SOURCE}) target_include_directories(ton_db PUBLIC $ $) @@ -293,9 +356,14 @@ add_executable(test-ed25519-crypto test/test-ed25519-crypto.cpp) target_include_directories(test-ed25519-crypto PUBLIC $) target_link_libraries(test-ed25519-crypto PUBLIC ton_crypto) -add_library(fift-lib ${FIFT_SOURCE}) +add_library(fift-lib STATIC ${FIFT_SOURCE}) target_include_directories(fift-lib PUBLIC $) -target_link_libraries(fift-lib PUBLIC ton_crypto ton_db tdutils ton_block) +target_link_libraries(fift-lib PUBLIC ton_crypto) + +if (USE_EMSCRIPTEN) + target_link_options(fift-lib PRIVATE -fexceptions) + target_compile_options(fift-lib PRIVATE -fexceptions) +endif() set_target_properties(fift-lib PROPERTIES OUTPUT_NAME fift) add_executable(fift fift/fift-main.cpp) @@ -306,16 +374,20 @@ endif() add_library(src_parser ${PARSER_SOURCE}) target_include_directories(src_parser PUBLIC $) -target_link_libraries(src_parser PUBLIC ton_crypto) +target_link_libraries(src_parser PUBLIC ton_crypto_core) -add_library(ton_block ${BLOCK_SOURCE}) +add_library(ton_block STATIC ${BLOCK_SOURCE}) target_include_directories(ton_block PUBLIC $ $ $) -target_link_libraries(ton_block PUBLIC ton_crypto tdutils tdactor tl_api) +target_link_libraries(ton_block PUBLIC ton_crypto_core tdactor tl_api) +if (USE_EMSCRIPTEN) + target_link_options(ton_block PRIVATE -fexceptions) + target_compile_options(ton_block PRIVATE -fexceptions) +endif() add_executable(func func/func-main.cpp ${FUNC_LIB_SOURCE}) target_include_directories(func PUBLIC $) -target_link_libraries(func PUBLIC ton_crypto src_parser git ton_block) +target_link_libraries(func PUBLIC ton_crypto src_parser git) if (WINGETOPT_FOUND) target_link_libraries_system(func wingetopt) endif() @@ -324,38 +396,51 @@ if (USE_EMSCRIPTEN) add_executable(funcfiftlib funcfiftlib/funcfiftlib.cpp ${FUNC_LIB_SOURCE}) target_include_directories(funcfiftlib PUBLIC $) target_link_libraries(funcfiftlib PUBLIC fift-lib src_parser git) - target_link_options(funcfiftlib PRIVATE -sEXPORTED_RUNTIME_METHODS=FS,ccall,cwrap,_malloc,free,UTF8ToString,stringToUTF8) - target_link_options(funcfiftlib PRIVATE -sEXPORTED_FUNCTIONS=_func_compile,_version) + target_link_options(funcfiftlib PRIVATE -sEXPORTED_RUNTIME_METHODS=FS,ccall,cwrap,UTF8ToString,stringToUTF8,lengthBytesUTF8,addFunction,removeFunction,setValue) + target_link_options(funcfiftlib PRIVATE -sEXPORTED_FUNCTIONS=_func_compile,_version,_malloc,_free,_setThrew) target_link_options(funcfiftlib PRIVATE -sEXPORT_NAME=CompilerModule) target_link_options(funcfiftlib PRIVATE -sERROR_ON_UNDEFINED_SYMBOLS=0) - target_link_options(funcfiftlib PRIVATE -sFILESYSTEM=1) + target_link_options(funcfiftlib PRIVATE -sFILESYSTEM=1 -lnodefs.js) target_link_options(funcfiftlib PRIVATE -Oz) target_link_options(funcfiftlib PRIVATE -sIGNORE_MISSING_MAIN=1) target_link_options(funcfiftlib PRIVATE -sAUTO_NATIVE_LIBRARIES=0) target_link_options(funcfiftlib PRIVATE -sMODULARIZE=1) + target_link_options(funcfiftlib PRIVATE -sTOTAL_MEMORY=33554432) + target_link_options(funcfiftlib PRIVATE -sALLOW_MEMORY_GROWTH=1) + target_link_options(funcfiftlib PRIVATE -sALLOW_TABLE_GROWTH=1) target_link_options(funcfiftlib PRIVATE --embed-file ${CMAKE_CURRENT_SOURCE_DIR}/fift/lib@/fiftlib) - target_compile_options(funcfiftlib PRIVATE -sDISABLE_EXCEPTION_CATCHING=0) + target_link_options(funcfiftlib PRIVATE --pre-js ${CMAKE_CURRENT_SOURCE_DIR}/funcfiftlib/funcfiftlib-prejs.js) + target_link_options(funcfiftlib PRIVATE -fexceptions) + target_compile_options(funcfiftlib PRIVATE -fexceptions -fno-stack-protector) endif() add_executable(tlbc tl/tlbc.cpp) target_include_directories(tlbc PUBLIC $) -target_link_libraries(tlbc PUBLIC ton_crypto src_parser) +target_link_libraries(tlbc PUBLIC src_parser) if (WINGETOPT_FOUND) target_link_libraries_system(tlbc wingetopt) endif() add_library(pow-miner-lib util/Miner.cpp util/Miner.h) target_include_directories(pow-miner-lib PUBLIC $) -target_link_libraries(pow-miner-lib PUBLIC ton_crypto ton_block) +target_link_libraries(pow-miner-lib PUBLIC ton_crypto) add_executable(pow-miner util/pow-miner.cpp) -target_link_libraries(pow-miner PRIVATE ton_crypto ton_block pow-miner-lib git) +target_link_libraries(pow-miner PRIVATE ton_crypto pow-miner-lib git) if (WINGETOPT_FOUND) target_link_libraries_system(fift wingetopt) target_link_libraries_system(pow-miner wingetopt) endif() +add_executable(mintless-proof-generator util/mintless-proof-generator.cpp) +target_link_libraries(mintless-proof-generator PRIVATE ton_crypto git ${JEMALLOC_LIBRARIES}) + +if (JEMALLOC_FOUND) + target_include_directories(mintless-proof-generator PRIVATE ${JEMALLOC_INCLUDE_DIR}) + target_compile_definitions(mintless-proof-generator PRIVATE -DTON_USE_JEMALLOC=1) +endif() + set(TURN_OFF_LSAN cd .) if (TON_USE_ASAN AND NOT WIN32) set(TURN_OFF_LSAN export LSAN_OPTIONS=detect_leaks=0) @@ -403,10 +488,17 @@ if (NOT CMAKE_CROSSCOMPILING OR USE_EMSCRIPTEN) OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/${ARG_DEST_FIF} ) set(ARG_DEST_CPP "${ARG_DEST}.cpp") + + if (WIN32) + set(ARG_LIB_DIR "fift/lib@smartcont") + else() + set(ARG_LIB_DIR "fift/lib:smartcont") + endif() + add_custom_command( COMMENT "Generate ${ARG_DEST_CPP}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - COMMAND fift -Ifift/lib:smartcont -s asm-to-cpp.fif ${ARG_DEST_FIF} ${ARG_DEST_CPP} ${ARG_NAME} + COMMAND fift -I${ARG_LIB_DIR} -s asm-to-cpp.fif ${ARG_DEST_FIF} ${ARG_DEST_CPP} ${ARG_NAME} MAIN_DEPENDENCY ${ARG_SOURCE} DEPENDS fift ${ARG_DEST_FIF} smartcont/asm-to-cpp.fif fift/lib/Asm.fif OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/${ARG_DEST_CPP} @@ -436,7 +528,7 @@ if (NOT CMAKE_CROSSCOMPILING OR USE_EMSCRIPTEN) GenFif(DEST smartcont/auto/simple-wallet-ext-code SOURCE smartcont/simple-wallet-ext-code.fc NAME simple-wallet-ext) endif() -add_library(smc-envelope ${SMC_ENVELOPE_SOURCE}) +add_library(smc-envelope STATIC ${SMC_ENVELOPE_SOURCE}) target_include_directories(smc-envelope PUBLIC $) target_link_libraries(smc-envelope PUBLIC ton_crypto PRIVATE tdutils ton_block) if (NOT CMAKE_CROSSCOMPILING) @@ -447,12 +539,12 @@ add_executable(create-state block/create-state.cpp) target_include_directories(create-state PUBLIC $ $) if (INTERNAL_COMPILE) - target_link_libraries(create-state PUBLIC ton_crypto fift-lib ton_block tonlib git) + target_link_libraries(create-state PUBLIC ton_crypto fift-lib tonlib git) else() if (TONLIB_COMPILE) - target_link_libraries(create-state PUBLIC ton_crypto fift-lib ton_block tonlib git) + target_link_libraries(create-state PUBLIC ton_crypto fift-lib tonlib git) else() - target_link_libraries(create-state PUBLIC ton_crypto fift-lib ton_block git) + target_link_libraries(create-state PUBLIC ton_crypto fift-lib git) endif() endif() if (WINGETOPT_FOUND) @@ -462,7 +554,7 @@ endif() add_executable(dump-block block/dump-block.cpp) target_include_directories(dump-block PUBLIC $ $) -target_link_libraries(dump-block PUBLIC ton_crypto fift-lib ton_block git) +target_link_libraries(dump-block PUBLIC ton_crypto fift-lib git) if (WINGETOPT_FOUND) target_link_libraries_system(dump-block wingetopt) endif() @@ -470,7 +562,7 @@ endif() add_executable(adjust-block block/adjust-block.cpp) target_include_directories(adjust-block PUBLIC $ $) -target_link_libraries(adjust-block PUBLIC ton_crypto fift-lib ton_block git) +target_link_libraries(adjust-block PUBLIC ton_crypto fift-lib git) if (WINGETOPT_FOUND) target_link_libraries_system(dump-block wingetopt) target_link_libraries_system(adjust-block wingetopt) @@ -479,11 +571,11 @@ endif() add_executable(test-weight-distr block/test-weight-distr.cpp) target_include_directories(test-weight-distr PUBLIC $ $) -target_link_libraries(test-weight-distr PUBLIC ton_crypto fift-lib ton_block git) +target_link_libraries(test-weight-distr PUBLIC ton_crypto fift-lib git) if (WINGETOPT_FOUND) target_link_libraries_system(test-weight-distr wingetopt) endif() -install(TARGETS fift func pow-miner RUNTIME DESTINATION bin) +install(TARGETS fift func create-state tlbc RUNTIME DESTINATION bin) install(DIRECTORY fift/lib/ DESTINATION lib/fift) install(DIRECTORY smartcont DESTINATION share/ton) diff --git a/crypto/block/block-db.cpp b/crypto/block/block-db.cpp index 2be1f580..21c7c0a0 100644 --- a/crypto/block/block-db.cpp +++ b/crypto/block/block-db.cpp @@ -624,6 +624,7 @@ void BlockDbImpl::get_block_by_id(ton::BlockId blk_id, bool need_data, td::Promi } } promise(it->second); + return; } promise(td::Status::Error(-666, "block not found in database")); } @@ -642,6 +643,7 @@ void BlockDbImpl::get_state_by_id(ton::BlockId blk_id, bool need_data, td::Promi } } promise(it->second); + return; } if (zerostate.not_null() && blk_id == zerostate->blk.id) { LOG(DEBUG) << "get_state_by_id(): zerostate requested"; @@ -666,6 +668,7 @@ void BlockDbImpl::get_out_queue_info_by_id(ton::BlockId blk_id, td::Promisesecond->data.is_null()) { LOG(DEBUG) << "loading data for state " << blk_id.to_str(); @@ -679,6 +682,7 @@ void BlockDbImpl::get_out_queue_info_by_id(ton::BlockId blk_id, td::Promisesecond->data.clone(), options); @@ -707,10 +711,12 @@ void BlockDbImpl::get_out_queue_info_by_id(ton::BlockId blk_id, td::Promisesecond->blk.root_hash != state_root->get_hash().bits()) { promise(td::Status::Error( -668, std::string{"state for block "} + blk_id.to_str() + " is invalid : state root hash mismatch")); + return; } vm::CellSlice cs = vm::load_cell_slice(state_root); if (!cs.have(64, 1) || cs.prefetch_ulong(32) != 0x9023afde) { promise(td::Status::Error(-668, std::string{"state for block "} + blk_id.to_str() + " is invalid")); + return; } auto out_queue_info = cs.prefetch_ref(); promise(Ref{true, blk_id, it2->second->blk.root_hash.cbits(), state_root->get_hash().bits(), @@ -758,6 +764,7 @@ void BlockDbImpl::save_new_block(ton::BlockIdExt id, td::BufferSlice data, int a auto save_res = save_db_file(id.file_hash, data, FMode::chk_if_exists | FMode::overwrite | FMode::chk_file_hash); if (save_res.is_error()) { promise(std::move(save_res)); + return; } auto sz = data.size(); auto lev = bb.alloc(id.id, id.root_hash, id.file_hash, data.size(), authority & 0xff); @@ -780,6 +787,7 @@ void BlockDbImpl::save_new_state(ton::BlockIdExt id, td::BufferSlice data, int a auto save_res = save_db_file(id.file_hash, data, FMode::chk_if_exists | FMode::overwrite | FMode::chk_file_hash); if (save_res.is_error()) { promise(std::move(save_res)); + return; } auto sz = data.size(); auto lev = bb.alloc(id.id, id.root_hash, id.file_hash, data.size(), authority & 0xff); diff --git a/crypto/block/block-parse.cpp b/crypto/block/block-parse.cpp index c62854d4..50851c79 100644 --- a/crypto/block/block-parse.cpp +++ b/crypto/block/block-parse.cpp @@ -813,19 +813,45 @@ int IntermediateAddress::get_size(const vm::CellSlice& cs) const { const IntermediateAddress t_IntermediateAddress; bool MsgEnvelope::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { - return cs.fetch_ulong(4) == 4 // msg_envelope#4 - && t_IntermediateAddress.validate_skip(ops, cs, weak) // cur_addr:IntermediateAddress - && t_IntermediateAddress.validate_skip(ops, cs, weak) // next_addr:IntermediateAddress - && t_Grams.validate_skip(ops, cs, weak) // fwd_fee_remaining:Grams - && t_Ref_Message.validate_skip(ops, cs, weak); // msg:^Message + switch (get_tag(cs)) { + case 4: + return cs.fetch_ulong(4) == 4 // msg_envelope#4 + && t_IntermediateAddress.validate_skip(ops, cs, weak) // cur_addr:IntermediateAddress + && t_IntermediateAddress.validate_skip(ops, cs, weak) // next_addr:IntermediateAddress + && t_Grams.validate_skip(ops, cs, weak) // fwd_fee_remaining:Grams + && t_Ref_Message.validate_skip(ops, cs, weak); // msg:^Message + case 5: + return cs.fetch_ulong(4) == 5 // msg_envelope_v2#5 + && t_IntermediateAddress.validate_skip(ops, cs, weak) // cur_addr:IntermediateAddress + && t_IntermediateAddress.validate_skip(ops, cs, weak) // next_addr:IntermediateAddress + && t_Grams.validate_skip(ops, cs, weak) // fwd_fee_remaining:Grams + && t_Ref_Message.validate_skip(ops, cs, weak) // msg:^Message + && Maybe(64).validate_skip(ops, cs, weak) // emitted_lt:(Maybe uint64) + && Maybe().validate_skip(ops, cs, weak); // metadata:(Maybe MsgMetadata) + default: + return false; + } } bool MsgEnvelope::skip(vm::CellSlice& cs) const { - return cs.advance(4) // msg_envelope#4 - && t_IntermediateAddress.skip(cs) // cur_addr:IntermediateAddress - && t_IntermediateAddress.skip(cs) // next_addr:IntermediateAddress - && t_Grams.skip(cs) // fwd_fee_remaining:Grams - && t_Ref_Message.skip(cs); // msg:^Message + switch (get_tag(cs)) { + case 4: + return cs.advance(4) // msg_envelope#4 + && t_IntermediateAddress.skip(cs) // cur_addr:IntermediateAddress + && t_IntermediateAddress.skip(cs) // next_addr:IntermediateAddress + && t_Grams.skip(cs) // fwd_fee_remaining:Grams + && t_Ref_Message.skip(cs); // msg:^Message + case 5: + return cs.advance(4) // msg_envelope_v2#5 + && t_IntermediateAddress.skip(cs) // cur_addr:IntermediateAddress + && t_IntermediateAddress.skip(cs) // next_addr:IntermediateAddress + && t_Grams.skip(cs) // fwd_fee_remaining:Grams + && t_Ref_Message.skip(cs) // msg:^Message + && Maybe(64).skip(cs) // emitted_lt:(Maybe uint64) + && Maybe().skip(cs); // metadata:(Maybe MsgMetadata) + default: + return false; + } } bool MsgEnvelope::extract_fwd_fees_remaining(vm::CellSlice& cs) const { @@ -833,34 +859,101 @@ bool MsgEnvelope::extract_fwd_fees_remaining(vm::CellSlice& cs) const { } bool MsgEnvelope::unpack(vm::CellSlice& cs, MsgEnvelope::Record& data) const { - return cs.fetch_ulong(4) == 4 // msg_envelope#4 - && t_IntermediateAddress.fetch_to(cs, data.cur_addr) // cur_addr:IntermediateAddress - && t_IntermediateAddress.fetch_to(cs, data.next_addr) // next_addr:IntermediateAddress - && t_Grams.fetch_to(cs, data.fwd_fee_remaining) // fwd_fee_remaining:Grams - && cs.fetch_ref_to(data.msg); // msg:^Message + switch (get_tag(cs)) { + case 4: + return cs.fetch_ulong(4) == 4 // msg_envelope#4 + && t_IntermediateAddress.fetch_to(cs, data.cur_addr) // cur_addr:IntermediateAddress + && t_IntermediateAddress.fetch_to(cs, data.next_addr) // next_addr:IntermediateAddress + && t_Grams.fetch_to(cs, data.fwd_fee_remaining) // fwd_fee_remaining:Grams + && cs.fetch_ref_to(data.msg); // msg:^Message + case 5: + return cs.fetch_ulong(4) == 5 // msg_envelope_v2#5 + && t_IntermediateAddress.fetch_to(cs, data.cur_addr) // cur_addr:IntermediateAddress + && t_IntermediateAddress.fetch_to(cs, data.next_addr) // next_addr:IntermediateAddress + && t_Grams.fetch_to(cs, data.fwd_fee_remaining) // fwd_fee_remaining:Grams + && cs.fetch_ref_to(data.msg) // msg:^Message + && Maybe(64).skip(cs) // emitted_lt:(Maybe uint64) + && Maybe().skip(cs); // metadata:(Maybe MsgMetadata) + default: + return false; + } } bool MsgEnvelope::unpack(vm::CellSlice& cs, MsgEnvelope::Record_std& data) const { - return cs.fetch_ulong(4) == 4 // msg_envelope#4 - && t_IntermediateAddress.fetch_regular(cs, data.cur_addr) // cur_addr:IntermediateAddress - && t_IntermediateAddress.fetch_regular(cs, data.next_addr) // next_addr:IntermediateAddress - && t_Grams.as_integer_skip_to(cs, data.fwd_fee_remaining) // fwd_fee_remaining:Grams - && cs.fetch_ref_to(data.msg); // msg:^Message + data.emitted_lt = {}; + data.metadata = {}; + switch (get_tag(cs)) { + case 4: + return cs.fetch_ulong(4) == 4 // msg_envelope#4 + && t_IntermediateAddress.fetch_regular(cs, data.cur_addr) // cur_addr:IntermediateAddress + && t_IntermediateAddress.fetch_regular(cs, data.next_addr) // next_addr:IntermediateAddress + && t_Grams.as_integer_skip_to(cs, data.fwd_fee_remaining) // fwd_fee_remaining:Grams + && cs.fetch_ref_to(data.msg); // msg:^Message + case 5: { + bool with_metadata, with_emitted_lt; + return cs.fetch_ulong(4) == 5 // msg_envelope_v2#5 + && t_IntermediateAddress.fetch_regular(cs, data.cur_addr) // cur_addr:IntermediateAddress + && t_IntermediateAddress.fetch_regular(cs, data.next_addr) // next_addr:IntermediateAddress + && t_Grams.as_integer_skip_to(cs, data.fwd_fee_remaining) // fwd_fee_remaining:Grams + && cs.fetch_ref_to(data.msg) // msg:^Message + && cs.fetch_bool_to(with_emitted_lt) && + (!with_emitted_lt || cs.fetch_uint_to(64, data.emitted_lt.value_force())) // emitted_lt:(Maybe uint64) + && cs.fetch_bool_to(with_metadata) && + (!with_metadata || data.metadata.value_force().unpack(cs)); // metadata:(Maybe MsgMetadata) + } + default: + return false; + } } -bool MsgEnvelope::unpack_std(vm::CellSlice& cs, int& cur_a, int& nhop_a, Ref& msg) const { - return cs.fetch_ulong(4) == 4 // msg_envelope#4 - && t_IntermediateAddress.fetch_regular(cs, cur_a) // cur_addr:IntermediateAddress - && t_IntermediateAddress.fetch_regular(cs, nhop_a) // next_addr:IntermediateAddress - && cs.fetch_ref_to(msg); +bool MsgEnvelope::pack(vm::CellBuilder& cb, const Record_std& data) const { + bool v2 = (bool)data.metadata || (bool)data.emitted_lt; + if (!(cb.store_long_bool(v2 ? 5 : 4, 4) && // msg_envelope#4 / msg_envelope_v2#5 + cb.store_long_bool(data.cur_addr, 8) && // cur_addr:IntermediateAddress + cb.store_long_bool(data.next_addr, 8) && // next_addr:IntermediateAddress + t_Grams.store_integer_ref(cb, data.fwd_fee_remaining) && // fwd_fee_remaining:Grams + cb.store_ref_bool(data.msg))) { // msg:^Message + return false; + } + if (v2) { + if (!(cb.store_bool_bool((bool)data.emitted_lt) && + (!data.emitted_lt || cb.store_long_bool(data.emitted_lt.value(), 64)))) { // emitted_lt:(Maybe uint64) + return false; + } + if (!(cb.store_bool_bool((bool)data.metadata) && + (!data.metadata || data.metadata.value().pack(cb)))) { // metadata:(Maybe MsgMetadata) + return false; + } + } + return true; } -bool MsgEnvelope::get_created_lt(const vm::CellSlice& cs, unsigned long long& created_lt) const { +bool MsgEnvelope::pack_cell(td::Ref& cell, const Record_std& data) const { + vm::CellBuilder cb; + return pack(cb, data) && cb.finalize_to(cell); +} + +bool MsgEnvelope::get_emitted_lt(const vm::CellSlice& cs, unsigned long long& emitted_lt) const { + // Emitted lt is emitted_lt from MsgEnvelope (if present), otherwise created_lt if (!cs.size_refs()) { return false; } + if (get_tag(cs) == 5) { + vm::CellSlice cs2 = cs; + // msg_envelope_v2#5 cur_addr:IntermediateAddress + // next_addr:IntermediateAddress fwd_fee_remaining:Grams + // msg:^(Message Any) emitted_lt:(Maybe uint64) ... + bool have_emitted_lt; + if (!(cs2.skip_first(4) && t_IntermediateAddress.skip(cs2) && t_IntermediateAddress.skip(cs2) && + t_Grams.skip(cs2) && t_Ref_Message.skip(cs2) && cs2.fetch_bool_to(have_emitted_lt))) { + return false; + } + if (have_emitted_lt) { + return cs2.fetch_ulong_bool(64, emitted_lt); + } + } auto msg_cs = load_cell_slice(cs.prefetch_ref()); - return t_Message.get_created_lt(msg_cs, created_lt); + return t_Message.get_created_lt(msg_cs, emitted_lt); } const MsgEnvelope t_MsgEnvelope; @@ -1000,7 +1093,7 @@ bool Account::skip_copy_depth_balance(vm::CellBuilder& cb, vm::CellSlice& cs) co } const Account t_Account, t_AccountE{true}; -const RefTo t_Ref_Account; +const RefTo t_Ref_AccountE{true}; bool ShardAccount::extract_account_state(Ref cs_ref, Ref& acc_state) { if (cs_ref.is_null()) { @@ -1692,6 +1785,15 @@ bool InMsg::skip(vm::CellSlice& cs) const { && cs.advance(64) // transaction_id:uint64 && t_Grams.skip(cs) // fwd_fee:Grams && t_RefCell.skip(cs); // proof_delivered:^Cell + case msg_import_deferred_fin: + return cs.advance(5) // msg_import_deferred_fin$00100 + && t_Ref_MsgEnvelope.skip(cs) // in_msg:^MsgEnvelope + && t_Ref_Transaction.skip(cs) // transaction:^Transaction + && t_Grams.skip(cs); // fwd_fee:Grams + case msg_import_deferred_tr: + return cs.advance(5) // msg_import_deferred_tr$00101 + && t_Ref_MsgEnvelope.skip(cs) // in_msg:^MsgEnvelope + && t_Ref_MsgEnvelope.skip(cs); // out_msg:^MsgEnvelope } return false; } @@ -1734,12 +1836,22 @@ bool InMsg::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { && cs.advance(64) // transaction_id:uint64 && t_Grams.validate_skip(ops, cs, weak) // fwd_fee:Grams && t_RefCell.validate_skip(ops, cs, weak); // proof_delivered:^Cell + case msg_import_deferred_fin: + return cs.advance(5) // msg_import_deferred_fin$00100 + && t_Ref_MsgEnvelope.validate_skip(ops, cs, weak) // in_msg:^MsgEnvelope + && t_Ref_Transaction.validate_skip(ops, cs, weak) // transaction:^Transaction + && t_Grams.validate_skip(ops, cs, weak); // fwd_fee:Grams + case msg_import_deferred_tr: + return cs.advance(5) // msg_import_deferred_tr$00101 + && t_Ref_MsgEnvelope.validate_skip(ops, cs, weak) // in_msg:^MsgEnvelope + && t_Ref_MsgEnvelope.validate_skip(ops, cs, weak); // out_msg:^MsgEnvelope } return false; } bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const { - switch (get_tag(cs)) { + int tag = get_tag(cs); + switch (tag) { case msg_import_ext: // inbound external message return t_ImportFees.null_value(cb); // external messages have no value and no import fees case msg_import_ihr: // IHR-forwarded internal message to its final destination @@ -1765,8 +1877,9 @@ bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const { && t_CurrencyCollection.null_value(cb); // value_imported := 0 } return false; - case msg_import_fin: // internal message delivered to its final destination in this block - if (cs.advance(3) && cs.size_refs() >= 2) { + case msg_import_fin: // internal message delivered to its final destination in this block + case msg_import_deferred_fin: // internal message from DispatchQueue to its final destination in this block + if (cs.advance(tag == msg_import_fin ? 3 : 5) && cs.size_refs() >= 2) { auto msg_env_cs = load_cell_slice(cs.fetch_ref()); MsgEnvelope::Record in_msg; td::RefInt256 fwd_fee, fwd_fee_remaining, value_grams, ihr_fee; @@ -1787,13 +1900,14 @@ bool InMsg::get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const { msg_info.value.write()); // value_imported = msg.value + msg.ihr_fee + fwd_fee_remaining } return false; - case msg_import_tr: // transit internal message - if (cs.advance(3) && cs.size_refs() >= 2) { + case msg_import_tr: // transit internal message + case msg_import_deferred_tr: // internal message from DispatchQueue to OutMsgQueue + if (cs.advance(tag == msg_import_tr ? 3 : 5) && cs.size_refs() >= 2) { auto msg_env_cs = load_cell_slice(cs.fetch_ref()); MsgEnvelope::Record in_msg; - td::RefInt256 transit_fee, fwd_fee_remaining, value_grams, ihr_fee; + td::RefInt256 transit_fee = td::zero_refint(), fwd_fee_remaining, value_grams, ihr_fee; if (!(t_MsgEnvelope.unpack(msg_env_cs, in_msg) && cs.fetch_ref().not_null() && - t_Grams.as_integer_skip_to(cs, transit_fee) && + (tag == msg_import_deferred_tr || t_Grams.as_integer_skip_to(cs, transit_fee)) && (fwd_fee_remaining = t_Grams.as_integer(in_msg.fwd_fee_remaining)).not_null() && cmp(transit_fee, fwd_fee_remaining) <= 0)) { return false; @@ -1871,6 +1985,14 @@ bool OutMsg::skip(vm::CellSlice& cs) const { return cs.advance(3) // msg_export_tr_req$111 && t_Ref_MsgEnvelope.skip(cs) // out_msg:^MsgEnvelope && RefTo{}.skip(cs); // imported:^InMsg + case msg_export_new_defer: + return cs.advance(5) // msg_export_new_defer$10100 + && t_Ref_MsgEnvelope.skip(cs) // out_msg:^MsgEnvelope + && t_Ref_Transaction.skip(cs); // transaction:^Transaction + case msg_export_deferred_tr: + return cs.advance(5) // msg_export_deferred_tr$10101 + && t_Ref_MsgEnvelope.skip(cs) // out_msg:^MsgEnvelope + && RefTo{}.skip(cs); // imported:^InMsg } return false; } @@ -1910,12 +2032,21 @@ bool OutMsg::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return cs.advance(3) // msg_export_tr_req$111 && t_Ref_MsgEnvelope.validate_skip(ops, cs, weak) // out_msg:^MsgEnvelope && RefTo{}.validate_skip(ops, cs, weak); // imported:^InMsg + case msg_export_new_defer: + return cs.advance(5) // msg_export_new_defer$10100 + && t_Ref_MsgEnvelope.validate_skip(ops, cs, weak) // out_msg:^MsgEnvelope + && t_Ref_Transaction.validate_skip(ops, cs, weak); // transaction:^Transaction + case msg_export_deferred_tr: + return cs.advance(5) // msg_export_deferred_tr$10101 + && t_Ref_MsgEnvelope.validate_skip(ops, cs, weak) // out_msg:^MsgEnvelope + && RefTo{}.validate_skip(ops, cs, weak); // imported:^InMsg } return false; } bool OutMsg::get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const { - switch (get_tag(cs)) { + auto tag = get_tag(cs); + switch (tag) { case msg_export_ext: // external outbound message carries no value if (cs.have(3, 2)) { return t_CurrencyCollection.null_value(cb); @@ -1929,10 +2060,13 @@ bool OutMsg::get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const { return cs.have(4 + 63, 1) && t_CurrencyCollection.null_value(cb); case msg_export_deq_short: // dequeueing record for outbound message, no exported value return cs.have(4 + 256 + 32 + 64 + 64) && t_CurrencyCollection.null_value(cb); - case msg_export_new: // newly-generated outbound internal message, queued - case msg_export_tr: // transit internal message, queued - case msg_export_tr_req: // transit internal message, re-queued from this shardchain - if (cs.advance(3) && cs.size_refs() >= 2) { + case msg_export_new: // newly-generated outbound internal message, queued + case msg_export_tr: // transit internal message, queued + case msg_export_tr_req: // transit internal message, re-queued from this shardchain + case msg_export_new_defer: // newly-generated outbound internal message, deferred + case msg_export_deferred_tr: // internal message from DispatchQueue, queued + int tag_len = (tag == msg_export_new_defer || tag == msg_export_deferred_tr) ? 5 : 3; + if (cs.advance(tag_len) && cs.size_refs() >= 2) { auto msg_env_cs = load_cell_slice(cs.fetch_ref()); MsgEnvelope::Record out_msg; if (!(cs.fetch_ref().not_null() && t_MsgEnvelope.unpack(msg_env_cs, out_msg))) { @@ -1954,12 +2088,12 @@ bool OutMsg::get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const { return false; } -bool OutMsg::get_created_lt(vm::CellSlice& cs, unsigned long long& created_lt) const { +bool OutMsg::get_emitted_lt(vm::CellSlice& cs, unsigned long long& emitted_lt) const { switch (get_tag(cs)) { case msg_export_ext: if (cs.have(3, 1)) { auto msg_cs = load_cell_slice(cs.prefetch_ref()); - return t_Message.get_created_lt(msg_cs, created_lt); + return t_Message.get_created_lt(msg_cs, emitted_lt); } else { return false; } @@ -1970,9 +2104,11 @@ bool OutMsg::get_created_lt(vm::CellSlice& cs, unsigned long long& created_lt) c case msg_export_deq_short: case msg_export_deq_imm: case msg_export_tr_req: + case msg_export_new_defer: + case msg_export_deferred_tr: if (cs.have(3, 1)) { auto out_msg_cs = load_cell_slice(cs.prefetch_ref()); - return t_MsgEnvelope.get_created_lt(out_msg_cs, created_lt); + return t_MsgEnvelope.get_emitted_lt(out_msg_cs, emitted_lt); } else { return false; } @@ -2003,26 +2139,53 @@ bool Aug_OutMsgQueue::eval_empty(vm::CellBuilder& cb) const { bool Aug_OutMsgQueue::eval_leaf(vm::CellBuilder& cb, vm::CellSlice& cs) const { Ref msg_env; - unsigned long long created_lt; - return cs.fetch_ref_to(msg_env) && t_MsgEnvelope.get_created_lt(load_cell_slice(std::move(msg_env)), created_lt) && - cb.store_ulong_rchk_bool(created_lt, 64); + unsigned long long emitted_lt; + return cs.fetch_ref_to(msg_env) && t_MsgEnvelope.get_emitted_lt(load_cell_slice(std::move(msg_env)), emitted_lt) && + cb.store_ulong_rchk_bool(emitted_lt, 64); +} + +bool Aug_DispatchQueue::eval_fork(vm::CellBuilder& cb, vm::CellSlice& left_cs, vm::CellSlice& right_cs) const { + unsigned long long x, y; + return left_cs.fetch_ulong_bool(64, x) && right_cs.fetch_ulong_bool(64, y) && + cb.store_ulong_rchk_bool(std::min(x, y), 64); +} + +bool Aug_DispatchQueue::eval_empty(vm::CellBuilder& cb) const { + return cb.store_long_bool(0, 64); +} + +bool Aug_DispatchQueue::eval_leaf(vm::CellBuilder& cb, vm::CellSlice& cs) const { + Ref messages_root; + if (!cs.fetch_maybe_ref(messages_root)) { + return false; + } + vm::Dictionary messages{std::move(messages_root), 64}; + td::BitArray<64> key_buffer; + td::uint64 key; + if (messages.get_minmax_key(key_buffer.bits(), 64).is_null()) { + key = (td::uint64)-1; + } else { + key = key_buffer.to_ulong(); + } + return cb.store_long_bool(key, 64); } const Aug_OutMsgQueue aug_OutMsgQueue; +const Aug_DispatchQueue aug_DispatchQueue; const OutMsgQueue t_OutMsgQueue; const ProcessedUpto t_ProcessedUpto; const HashmapE t_ProcessedInfo{96, t_ProcessedUpto}; const HashmapE t_IhrPendingInfo{256, t_uint128}; -// _ out_queue:OutMsgQueue proc_info:ProcessedInfo = OutMsgQueueInfo; +// _ out_queue:OutMsgQueue proc_info:ProcessedInfo extra:(Maybe OutMsgQueueExtra) = OutMsgQueueInfo; bool OutMsgQueueInfo::skip(vm::CellSlice& cs) const { - return t_OutMsgQueue.skip(cs) && t_ProcessedInfo.skip(cs) && t_IhrPendingInfo.skip(cs); + return t_OutMsgQueue.skip(cs) && t_ProcessedInfo.skip(cs) && Maybe().skip(cs); } bool OutMsgQueueInfo::validate_skip(int* ops, vm::CellSlice& cs, bool weak) const { return t_OutMsgQueue.validate_skip(ops, cs, weak) && t_ProcessedInfo.validate_skip(ops, cs, weak) && - t_IhrPendingInfo.validate_skip(ops, cs, weak); + Maybe().validate_skip(ops, cs, weak); } const OutMsgQueueInfo t_OutMsgQueueInfo; @@ -2292,5 +2455,37 @@ bool Aug_ShardFees::eval_leaf(vm::CellBuilder& cb, vm::CellSlice& cs) const { const Aug_ShardFees aug_ShardFees; +bool validate_message_libs(const td::Ref &cell) { + gen::Message::Record rec; + if (!type_unpack_cell(cell, gen::t_Message_Any, rec)) { + return false; + } + vm::CellSlice& state_init = rec.init.write(); + if (!state_init.fetch_long(1)) { + return true; + } + if (state_init.fetch_long(1)) { + return gen::t_StateInitWithLibs.validate_ref(state_init.prefetch_ref()); + } else { + return gen::t_StateInitWithLibs.validate_csr(rec.init); + } +} + +bool validate_message_relaxed_libs(const td::Ref &cell) { + gen::MessageRelaxed::Record rec; + if (!type_unpack_cell(cell, gen::t_MessageRelaxed_Any, rec)) { + return false; + } + vm::CellSlice& state_init = rec.init.write(); + if (!state_init.fetch_long(1)) { + return true; + } + if (state_init.fetch_long(1)) { + return gen::t_StateInitWithLibs.validate_ref(state_init.prefetch_ref()); + } else { + return gen::t_StateInitWithLibs.validate_csr(rec.init); + } +} + } // namespace tlb } // namespace block diff --git a/crypto/block/block-parse.h b/crypto/block/block-parse.h index 25476a64..65f8b91f 100644 --- a/crypto/block/block-parse.h +++ b/crypto/block/block-parse.h @@ -28,6 +28,7 @@ #include "td/utils/bits.h" #include "td/utils/StringBuilder.h" #include "ton/ton-types.h" +#include "block-auto.h" namespace block { @@ -469,11 +470,17 @@ struct MsgEnvelope final : TLB_Complex { int cur_addr, next_addr; td::RefInt256 fwd_fee_remaining; Ref msg; + td::optional emitted_lt; + td::optional metadata; }; bool unpack(vm::CellSlice& cs, Record& data) const; bool unpack(vm::CellSlice& cs, Record_std& data) const; - bool unpack_std(vm::CellSlice& cs, int& cur_a, int& nhop_a, Ref& msg) const; - bool get_created_lt(const vm::CellSlice& cs, unsigned long long& created_lt) const; + bool pack(vm::CellBuilder& cb, const Record_std& data) const; + bool pack_cell(td::Ref& cell, const Record_std& data) const; + bool get_emitted_lt(const vm::CellSlice& cs, unsigned long long& emitted_lt) const; + int get_tag(const vm::CellSlice& cs) const override { + return (int)cs.prefetch_ulong(4); + } }; extern const MsgEnvelope t_MsgEnvelope; @@ -536,7 +543,7 @@ struct Account final : TLB_Complex { }; extern const Account t_Account, t_AccountE; -extern const RefTo t_Ref_Account; +extern const RefTo t_Ref_AccountE; struct AccountStatus final : TLB { enum { acc_state_uninit, acc_state_frozen, acc_state_active, acc_state_nonexist }; @@ -572,7 +579,7 @@ struct ShardAccount final : TLB_Complex { return cs.advance_ext(0x140, 1); } bool validate_skip(int* ops, vm::CellSlice& cs, bool weak = false) const override { - return cs.advance(0x140) && t_Ref_Account.validate_skip(ops, cs, weak); + return cs.advance(0x140) && t_Ref_AccountE.validate_skip(ops, cs, weak); } static bool unpack(vm::CellSlice& cs, Record& info) { return info.unpack(cs); @@ -801,12 +808,18 @@ struct InMsg final : TLB_Complex { msg_import_fin = 4, msg_import_tr = 5, msg_discard_fin = 6, - msg_discard_tr = 7 + msg_discard_tr = 7, + msg_import_deferred_fin = 8, + msg_import_deferred_tr = 9 }; bool skip(vm::CellSlice& cs) const override; bool validate_skip(int* ops, vm::CellSlice& cs, bool weak = false) const override; int get_tag(const vm::CellSlice& cs) const override { - return (int)cs.prefetch_ulong(3); + int tag = (int)cs.prefetch_ulong(3); + if (tag != 1) { + return tag; + } + return (int)cs.prefetch_ulong(5) - 0b00100 + 8; } bool get_import_fees(vm::CellBuilder& cb, vm::CellSlice& cs) const; }; @@ -822,16 +835,24 @@ struct OutMsg final : TLB_Complex { msg_export_deq_imm = 4, msg_export_deq = 12, msg_export_deq_short = 13, - msg_export_tr_req = 7 + msg_export_tr_req = 7, + msg_export_new_defer = 20, // 0b10100 + msg_export_deferred_tr = 21 // 0b10101 }; bool skip(vm::CellSlice& cs) const override; bool validate_skip(int* ops, vm::CellSlice& cs, bool weak = false) const override; int get_tag(const vm::CellSlice& cs) const override { int t = (int)cs.prefetch_ulong(3); - return t != 6 ? t : (int)cs.prefetch_ulong(4); + if (t == 6) { + return (int)cs.prefetch_ulong(4); + } + if (t == 5) { + return (int)cs.prefetch_ulong(5); + } + return t; } bool get_export_value(vm::CellBuilder& cb, vm::CellSlice& cs) const; - bool get_created_lt(vm::CellSlice& cs, unsigned long long& created_lt) const; + bool get_emitted_lt(vm::CellSlice& cs, unsigned long long& emitted_lt) const; }; extern const OutMsg t_OutMsg; @@ -909,6 +930,16 @@ struct Aug_OutMsgQueue final : AugmentationCheckData { extern const Aug_OutMsgQueue aug_OutMsgQueue; +struct Aug_DispatchQueue final : AugmentationCheckData { + Aug_DispatchQueue() : AugmentationCheckData(gen::t_AccountDispatchQueue, t_uint64) { + } + bool eval_fork(vm::CellBuilder& cb, vm::CellSlice& left_cs, vm::CellSlice& right_cs) const override; + bool eval_empty(vm::CellBuilder& cb) const override; + bool eval_leaf(vm::CellBuilder& cb, vm::CellSlice& cs) const override; +}; + +extern const Aug_DispatchQueue aug_DispatchQueue; + struct OutMsgQueue final : TLB_Complex { HashmapAugE dict_type; OutMsgQueue() : dict_type(32 + 64 + 256, aug_OutMsgQueue){}; @@ -1113,5 +1144,9 @@ struct Aug_ShardFees final : AugmentationCheckData { extern const Aug_ShardFees aug_ShardFees; +// Validate dict of libraries in message: used when sending and receiving message +bool validate_message_libs(const td::Ref &cell); +bool validate_message_relaxed_libs(const td::Ref &cell); + } // namespace tlb } // namespace block diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index 9e2caef5..e0782240 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -28,6 +28,7 @@ #include "td/utils/tl_storers.h" #include "td/utils/misc.h" #include "td/utils/Random.h" +#include "vm/fmt.hpp" namespace block { using namespace std::literals::string_literals; @@ -359,7 +360,6 @@ MsgProcessedUptoCollection::MsgProcessedUptoCollection(ton::ShardIdFull _owner, z.shard = key.get_uint(64); z.mc_seqno = (unsigned)((key + 64).get_uint(32)); z.last_inmsg_lt = value.write().fetch_ulong(64); - // std::cerr << "ProcessedUpto shard " << std::hex << z.shard << std::dec << std::endl; return value.write().fetch_bits_to(z.last_inmsg_hash) && z.shard && ton::shard_contains(owner.shard, z.shard); }); } @@ -642,7 +642,11 @@ bool EnqueuedMsgDescr::unpack(vm::CellSlice& cs) { } cur_prefix_ = interpolate_addr(src_prefix_, dest_prefix_, env.cur_addr); next_prefix_ = interpolate_addr(src_prefix_, dest_prefix_, env.next_addr); - lt_ = info.created_lt; + unsigned long long lt; + if (!tlb::t_MsgEnvelope.get_emitted_lt(vm::load_cell_slice(enq.out_msg), lt)) { + return invalidate(); + } + lt_ = lt; enqueued_lt_ = enq.enqueued_lt; hash_ = env.msg->get_hash().bits(); msg_ = std::move(env.msg); @@ -655,6 +659,12 @@ bool EnqueuedMsgDescr::check_key(td::ConstBitPtr key) const { hash_ == key + 96; } +bool ImportedMsgQueueLimits::deserialize(vm::CellSlice& cs) { + return cs.fetch_ulong(8) == 0xd3 // imported_msg_queue_limits#d3 + && cs.fetch_uint_to(32, max_bytes) // max_bytes:# + && cs.fetch_uint_to(32, max_msgs); // max_msgs:# +} + bool ParamLimits::deserialize(vm::CellSlice& cs) { return cs.fetch_ulong(8) == 0xc3 // param_limits#c3 && cs.fetch_uint_to(32, limits_[0]) // underload:uint32 @@ -714,8 +724,8 @@ td::uint64 BlockLimitStatus::estimate_block_size(const vm::NewCellStorageStat::S if (extra) { sum += *extra; } - return 2000 + (sum.bits >> 3) + sum.cells * 12 + sum.internal_refs * 3 + sum.external_refs * 40 + accounts * 200 + - transactions * 200 + (extra ? 200 : 0); + return 2000 + (sum.bits >> 3) + sum.cells * 12 + sum.internal_refs * 3 + sum.external_refs * 40 + transactions * 200 + + (extra ? 200 : 0) + extra_out_msgs * 300 + public_library_diff * 700; } int BlockLimitStatus::classify() const { @@ -851,19 +861,29 @@ td::Status ShardState::unpack_out_msg_queue_info(Ref out_msg_queue_inf out_msg_queue_ = std::make_unique(std::move(qinfo.out_queue), 352, block::tlb::aug_OutMsgQueue); if (verbosity >= 3 * 1) { - LOG(DEBUG) << "unpacking ProcessedUpto of our previous block " << id_.to_str(); - block::gen::t_ProcessedInfo.print(std::cerr, qinfo.proc_info); + FLOG(DEBUG) { + sb << "unpacking ProcessedUpto of our previous block " << id_.to_str(); + block::gen::t_ProcessedInfo.print(sb, qinfo.proc_info); + }; } if (!block::gen::t_ProcessedInfo.validate_csr(1024, qinfo.proc_info)) { return td::Status::Error( -666, "ProcessedInfo in the state of "s + id_.to_str() + " is invalid according to automated validity checks"); } - if (!block::gen::t_IhrPendingInfo.validate_csr(1024, qinfo.ihr_pending)) { - return td::Status::Error( - -666, "IhrPendingInfo in the state of "s + id_.to_str() + " is invalid according to automated validity checks"); - } processed_upto_ = block::MsgProcessedUptoCollection::unpack(ton::ShardIdFull(id_), std::move(qinfo.proc_info)); - ihr_pending_ = std::make_unique(std::move(qinfo.ihr_pending), 320); + ihr_pending_ = std::make_unique(320); + if (qinfo.extra.write().fetch_long(1)) { + block::gen::OutMsgQueueExtra::Record extra; + if (!block::tlb::csr_unpack(qinfo.extra, extra)) { + return td::Status::Error(-666, "cannot unpack OutMsgQueueExtre in the state of "s + id_.to_str()); + } + dispatch_queue_ = std::make_unique(extra.dispatch_queue, 256, tlb::aug_DispatchQueue); + if (extra.out_queue_size.write().fetch_long(1)) { + out_msg_queue_size_ = extra.out_queue_size->prefetch_ulong(48); + } + } else { + dispatch_queue_ = std::make_unique(256, tlb::aug_DispatchQueue); + } auto shard1 = id_.shard_full(); td::BitArray<64> pfx{(long long)shard1.shard}; int pfx_len = shard_prefix_length(shard1); @@ -994,6 +1014,17 @@ td::Status ShardState::merge_with(ShardState& sib) { underload_history_ = overload_history_ = 0; // 10. compute vert_seqno vert_seqno_ = std::max(vert_seqno_, sib.vert_seqno_); + // 11. merge dispatch_queue (same as account dict) + if (!dispatch_queue_->combine_with(*sib.dispatch_queue_)) { + return td::Status::Error(-666, "cannot merge dispatch queues of the two ancestors"); + } + sib.dispatch_queue_.reset(); + // 11. merge out_msg_queue_size + if (out_msg_queue_size_ && sib.out_msg_queue_size_) { + out_msg_queue_size_.value() += sib.out_msg_queue_size_.value(); + } else { + out_msg_queue_size_ = {}; + } // Anything else? add here // ... @@ -1058,10 +1089,12 @@ td::Status ShardState::split(ton::ShardIdFull subshard) { auto shard1 = id_.shard_full(); CHECK(ton::shard_is_parent(shard1, subshard)); CHECK(out_msg_queue_); - int res1 = block::filter_out_msg_queue(*out_msg_queue_, shard1, subshard); + td::uint64 queue_size; + int res1 = block::filter_out_msg_queue(*out_msg_queue_, shard1, subshard, &queue_size); if (res1 < 0) { return td::Status::Error(-666, "error splitting OutMsgQueue of "s + id_.to_str()); } + out_msg_queue_size_ = queue_size; LOG(DEBUG) << "split counters: " << res1; // 3. processed_upto LOG(DEBUG) << "splitting ProcessedUpto"; @@ -1091,6 +1124,11 @@ td::Status ShardState::split(ton::ShardIdFull subshard) { // NB: if total_fees_extra will be allowed to be non-empty, split it here too // 7. reset overload/underload history overload_history_ = underload_history_ = 0; + // 8. split dispatch_queue (same as account dict) + LOG(DEBUG) << "splitting dispatch_queue"; + CHECK(dispatch_queue_); + CHECK(dispatch_queue_->cut_prefix_subdict(pfx.bits(), pfx_len)); + CHECK(dispatch_queue_->has_common_prefix(pfx.bits(), pfx_len)); // 999. anything else? id_.id.shard = subshard.shard; id_.file_hash.set_zero(); @@ -1098,8 +1136,12 @@ td::Status ShardState::split(ton::ShardIdFull subshard) { return td::Status::OK(); } -int filter_out_msg_queue(vm::AugmentedDictionary& out_queue, ton::ShardIdFull old_shard, ton::ShardIdFull subshard) { - return out_queue.filter([subshard, old_shard](vm::CellSlice& cs, td::ConstBitPtr key, int key_len) -> int { +int filter_out_msg_queue(vm::AugmentedDictionary& out_queue, ton::ShardIdFull old_shard, ton::ShardIdFull subshard, + td::uint64* queue_size) { + if (queue_size) { + *queue_size = 0; + } + return out_queue.filter([=](vm::CellSlice& cs, td::ConstBitPtr key, int key_len) -> int { CHECK(key_len == 352); LOG(DEBUG) << "scanning OutMsgQueue entry with key " << key.to_hex(key_len); block::tlb::MsgEnvelope::Record_std env; @@ -1122,7 +1164,11 @@ int filter_out_msg_queue(vm::AugmentedDictionary& out_queue, ton::ShardIdFull ol << " does not contain current address belonging to shard " << old_shard.to_str(); return -1; } - return ton::shard_contains(subshard, cur_prefix); + bool res = ton::shard_contains(subshard, cur_prefix); + if (res && queue_size) { + ++*queue_size; + } + return res; }); } @@ -1274,6 +1320,65 @@ CurrencyCollection CurrencyCollection::operator-(td::RefInt256 other_grams) cons } } +bool CurrencyCollection::clamp(const CurrencyCollection& other) { + if (!is_valid() || !other.is_valid()) { + return invalidate(); + } + grams = std::min(grams, other.grams); + vm::Dictionary dict1{extra, 32}, dict2(other.extra, 32); + bool ok = dict1.check_for_each([&](td::Ref cs1, td::ConstBitPtr key, int n) { + CHECK(n == 32); + td::Ref cs2 = dict2.lookup(key, 32); + td::RefInt256 val1 = tlb::t_VarUIntegerPos_32.as_integer(cs1); + if (val1.is_null()) { + return false; + } + td::RefInt256 val2 = cs2.is_null() ? td::zero_refint() : tlb::t_VarUIntegerPos_32.as_integer(cs2); + if (val2.is_null()) { + return false; + } + if (val1 > val2) { + if (val2->sgn() == 0) { + dict1.lookup_delete(key, 32); + } else { + dict1.set(key, 32, cs2); + } + } + return true; + }); + extra = dict1.get_root_cell(); + return ok || invalidate(); +} + +bool CurrencyCollection::check_extra_currency_limit(td::uint32 max_currencies) const { + td::uint32 count = 0; + return vm::Dictionary{extra, 32}.check_for_each([&](td::Ref, td::ConstBitPtr, int) { + ++count; + return count <= max_currencies; + }); +} + +bool CurrencyCollection::remove_zero_extra_currencies(Ref& root, td::uint32 max_currencies) { + td::uint32 count = 0; + vm::Dictionary dict{root, 32}; + int res = dict.filter([&](const vm::CellSlice& cs, td::ConstBitPtr, int) -> int { + ++count; + if (count > max_currencies) { + return -1; + } + td::RefInt256 val = tlb::t_VarUInteger_32.as_integer(cs); + if (val.is_null()) { + return -1; + } + return val->sgn() > 0; + }); + if (res < 0) { + return false; + } + root = dict.get_root_cell(); + return true; +} + bool CurrencyCollection::operator==(const CurrencyCollection& other) const { return is_valid() && other.is_valid() && !td::cmp(grams, other.grams) && (extra.not_null() == other.extra.not_null()) && @@ -1364,42 +1469,60 @@ std::ostream& operator<<(std::ostream& os, const CurrencyCollection& cc) { bool ValueFlow::set_zero() { return from_prev_blk.set_zero() && to_next_blk.set_zero() && imported.set_zero() && exported.set_zero() && fees_collected.set_zero() && fees_imported.set_zero() && recovered.set_zero() && created.set_zero() && - minted.set_zero(); + minted.set_zero() && burned.set_zero(); } bool ValueFlow::validate() const { return is_valid() && from_prev_blk + imported + fees_imported + created + minted + recovered == - to_next_blk + exported + fees_collected; + to_next_blk + exported + fees_collected + burned; } bool ValueFlow::store(vm::CellBuilder& cb) const { vm::CellBuilder cb2; - return cb.store_long_bool(block::gen::ValueFlow::cons_tag[0], 32) // value_flow ^[ - && from_prev_blk.store(cb2) // from_prev_blk:CurrencyCollection - && to_next_blk.store(cb2) // to_next_blk:CurrencyCollection - && imported.store(cb2) // imported:CurrencyCollection - && exported.store(cb2) // exported:CurrencyCollection - && cb.store_ref_bool(cb2.finalize()) // ] - && fees_collected.store(cb) // fees_collected:CurrencyCollection - && fees_imported.store(cb2) // ^[ fees_imported:CurrencyCollection - && recovered.store(cb2) // recovered:CurrencyCollection - && created.store(cb2) // created:CurrencyCollection - && minted.store(cb2) // minted:CurrencyCollection - && cb.store_ref_bool(cb2.finalize()); // ] = ValueFlow; + auto type = burned.is_zero() ? block::gen::ValueFlow::value_flow : block::gen::ValueFlow::value_flow_v2; + return cb.store_long_bool(block::gen::ValueFlow::cons_tag[type], 32) // ^[ + && from_prev_blk.store(cb2) // from_prev_blk:CurrencyCollection + && to_next_blk.store(cb2) // to_next_blk:CurrencyCollection + && imported.store(cb2) // imported:CurrencyCollection + && exported.store(cb2) // exported:CurrencyCollection + && cb.store_ref_bool(cb2.finalize()) // ] + && fees_collected.store(cb) // fees_collected:CurrencyCollection + && (burned.is_zero() || burned.store(cb)) // fees_burned:CurrencyCollection + && fees_imported.store(cb2) // ^[ fees_imported:CurrencyCollection + && recovered.store(cb2) // recovered:CurrencyCollection + && created.store(cb2) // created:CurrencyCollection + && minted.store(cb2) // minted:CurrencyCollection + && cb.store_ref_bool(cb2.finalize()); // ] = ValueFlow; } bool ValueFlow::fetch(vm::CellSlice& cs) { - block::gen::ValueFlow::Record f; - if (!(tlb::unpack(cs, f) && from_prev_blk.validate_unpack(std::move(f.r1.from_prev_blk)) && - to_next_blk.validate_unpack(std::move(f.r1.to_next_blk)) && - imported.validate_unpack(std::move(f.r1.imported)) && exported.validate_unpack(std::move(f.r1.exported)) && - fees_collected.validate_unpack(std::move(f.fees_collected)) && - fees_imported.validate_unpack(std::move(f.r2.fees_imported)) && - recovered.validate_unpack(std::move(f.r2.recovered)) && created.validate_unpack(std::move(f.r2.created)) && - minted.validate_unpack(std::move(f.r2.minted)))) { + if (cs.size() < 32) { return invalidate(); } - return true; + auto tag = cs.prefetch_ulong(32); + block::gen::ValueFlow::Record_value_flow f1; + if (tag == block::gen::ValueFlow::cons_tag[block::gen::ValueFlow::value_flow] && tlb::unpack(cs, f1) && + from_prev_blk.validate_unpack(std::move(f1.r1.from_prev_blk)) && + to_next_blk.validate_unpack(std::move(f1.r1.to_next_blk)) && + imported.validate_unpack(std::move(f1.r1.imported)) && exported.validate_unpack(std::move(f1.r1.exported)) && + fees_collected.validate_unpack(std::move(f1.fees_collected)) && burned.set_zero() && + fees_imported.validate_unpack(std::move(f1.r2.fees_imported)) && + recovered.validate_unpack(std::move(f1.r2.recovered)) && created.validate_unpack(std::move(f1.r2.created)) && + minted.validate_unpack(std::move(f1.r2.minted))) { + return true; + } + block::gen::ValueFlow::Record_value_flow_v2 f2; + if (tag == block::gen::ValueFlow::cons_tag[block::gen::ValueFlow::value_flow_v2] && tlb::unpack(cs, f2) && + from_prev_blk.validate_unpack(std::move(f2.r1.from_prev_blk)) && + to_next_blk.validate_unpack(std::move(f2.r1.to_next_blk)) && + imported.validate_unpack(std::move(f2.r1.imported)) && exported.validate_unpack(std::move(f2.r1.exported)) && + fees_collected.validate_unpack(std::move(f2.fees_collected)) && burned.validate_unpack(std::move(f2.burned)) && + fees_imported.validate_unpack(std::move(f2.r2.fees_imported)) && + recovered.validate_unpack(std::move(f2.r2.recovered)) && created.validate_unpack(std::move(f2.r2.created)) && + minted.validate_unpack(std::move(f2.r2.minted))) { + return true; + } + return invalidate(); } bool ValueFlow::unpack(Ref csr) { @@ -1424,7 +1547,8 @@ bool ValueFlow::show(std::ostream& os) const { show_one(os, " to_next_blk:", to_next_blk) && show_one(os, " imported:", imported) && show_one(os, " exported:", exported) && show_one(os, " fees_collected:", fees_collected) && show_one(os, " fees_imported:", fees_imported) && show_one(os, " recovered:", recovered) && - show_one(os, " created:", created) && show_one(os, " minted:", minted) && say(os, ")")) || + show_one(os, " created:", created) && show_one(os, " minted:", minted) && + (burned.is_zero() || show_one(os, " burned:", burned)) && say(os, ")")) || (say(os, "...)") && false); } @@ -2277,4 +2401,132 @@ bool parse_block_id_ext(td::Slice str, ton::BlockIdExt& blkid) { return parse_block_id_ext(str.begin(), str.end(), blkid); } +bool unpack_account_dispatch_queue(Ref csr, vm::Dictionary& dict, td::uint64& dict_size) { + if (csr.not_null()) { + block::gen::AccountDispatchQueue::Record rec; + if (!block::tlb::csr_unpack(std::move(csr), rec)) { + return false; + } + dict = vm::Dictionary{rec.messages, 64}; + dict_size = rec.count; + if (dict_size == 0 || dict.is_empty()) { + return false; + } + } else { + dict = vm::Dictionary{64}; + dict_size = 0; + } + return true; +} + +Ref pack_account_dispatch_queue(const vm::Dictionary& dict, td::uint64 dict_size) { + if (dict_size == 0) { + return {}; + } + // _ messages:(HashmapE 64 EnqueuedMsg) count:uint48 = AccountDispatchQueue; + vm::CellBuilder cb; + CHECK(dict.append_dict_to_bool(cb)); + cb.store_long(dict_size, 48); + return cb.as_cellslice_ref(); +} + +Ref get_dispatch_queue_min_lt_account(const vm::AugmentedDictionary& dispatch_queue, + ton::StdSmcAddress& addr) { + // TODO: This can be done more effectively + vm::AugmentedDictionary queue{dispatch_queue.get_root(), 256, tlb::aug_DispatchQueue}; + if (queue.is_empty()) { + return {}; + } + auto root_extra = queue.get_root_extra(); + if (root_extra.is_null()) { + return {}; + } + ton::LogicalTime min_lt = root_extra->prefetch_long(64); + while (true) { + td::Bits256 key; + int pfx_len = queue.get_common_prefix(key.bits(), 256); + if (pfx_len < 0) { + return {}; + } + if (pfx_len == 256) { + addr = key; + return queue.lookup(key); + } + key[pfx_len] = false; + vm::AugmentedDictionary queue_cut{queue.get_root(), 256, tlb::aug_DispatchQueue}; + if (!queue_cut.cut_prefix_subdict(key.bits(), pfx_len + 1)) { + return {}; + } + root_extra = queue_cut.get_root_extra(); + if (root_extra.is_null()) { + return {}; + } + ton::LogicalTime cut_min_lt = root_extra->prefetch_long(64); + if (cut_min_lt != min_lt) { + key[pfx_len] = true; + } + if (!queue.cut_prefix_subdict(key.bits(), pfx_len + 1)) { + return {}; + } + } +} + +bool remove_dispatch_queue_entry(vm::AugmentedDictionary& dispatch_queue, const ton::StdSmcAddress& addr, + ton::LogicalTime lt) { + auto account_dispatch_queue = dispatch_queue.lookup(addr); + if (account_dispatch_queue.is_null()) { + return false; + } + vm::Dictionary dict{64}; + td::uint64 dict_size; + if (!unpack_account_dispatch_queue(std::move(account_dispatch_queue), dict, dict_size)) { + return false; + } + td::BitArray<64> key; + key.store_ulong(lt); + auto entry = dict.lookup_delete(key); + if (entry.is_null()) { + return false; + } + --dict_size; + account_dispatch_queue = pack_account_dispatch_queue(dict, dict_size); + if (account_dispatch_queue.not_null()) { + dispatch_queue.set(addr, account_dispatch_queue); + } else { + dispatch_queue.lookup_delete(addr); + } + return true; +} + +bool MsgMetadata::unpack(vm::CellSlice& cs) { + // msg_metadata#0 depth:uint32 initiator_addr:MsgAddressInt initiator_lt:uint64 = MsgMetadata; + int tag; + return cs.fetch_int_to(4, tag) && tag == 0 && cs.fetch_uint_to(32, depth) && + cs.prefetch_ulong(3) == 0b100 && // std address, no anycast + tlb::t_MsgAddressInt.extract_std_address(cs, initiator_wc, initiator_addr) && + cs.fetch_uint_to(64, initiator_lt); +} + +bool MsgMetadata::pack(vm::CellBuilder& cb) const { + // msg_metadata#0 depth:uint32 initiator_addr:MsgAddressInt initiator_lt:uint64 = MsgMetadata; + return cb.store_long_bool(0, 4) && cb.store_long_bool(depth, 32) && + tlb::t_MsgAddressInt.store_std_address(cb, initiator_wc, initiator_addr) && + cb.store_long_bool(initiator_lt, 64); +} + +std::string MsgMetadata::to_str() const { + return PSTRING() << "[ depth=" << depth << " init=" << initiator_wc << ":" << initiator_addr.to_hex() << ":" + << initiator_lt << " ]"; +} + +bool MsgMetadata::operator==(const MsgMetadata& other) const { + return depth == other.depth && initiator_wc == other.initiator_wc && initiator_addr == other.initiator_addr && + initiator_lt == other.initiator_lt; +} + +bool MsgMetadata::operator!=(const MsgMetadata& other) const { + return !(*this == other); +} + + } // namespace block diff --git a/crypto/block/block.h b/crypto/block/block.h index 6c460e31..685005b4 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -216,6 +216,16 @@ static inline std::ostream& operator<<(std::ostream& os, const MsgProcessedUptoC return proc_coll.print(os); } +struct ImportedMsgQueueLimits { + // Default values + td::uint32 max_bytes = 1 << 16; + td::uint32 max_msgs = 30; + bool deserialize(vm::CellSlice& cs); + ImportedMsgQueueLimits operator*(td::uint32 x) const { + return {max_bytes * x, max_msgs * x}; + } +}; + struct ParamLimits { enum { limits_cnt = 4 }; enum { cl_underload = 0, cl_normal = 1, cl_soft = 2, cl_medium = 3, cl_hard = 4 }; @@ -239,6 +249,12 @@ struct ParamLimits { bool deserialize(vm::CellSlice& cs); int classify(td::uint64 value) const; bool fits(unsigned cls, td::uint64 value) const; + void multiply_by(double x) { + CHECK(x > 0.0); + for (td::uint32& y : limits_) { + y = (td::uint32)std::min(y * x, 1e9); + } + } private: std::array limits_; @@ -261,7 +277,8 @@ struct BlockLimitStatus { ton::LogicalTime cur_lt; td::uint64 gas_used{}; vm::NewCellStorageStat st_stat; - unsigned accounts{}, transactions{}; + unsigned accounts{}, transactions{}, extra_out_msgs{}; + unsigned public_library_diff{}; BlockLimitStatus(const BlockLimits& limits_, ton::LogicalTime lt = 0) : limits(limits_), cur_lt(std::max(limits_.start_lt, lt)) { } @@ -270,6 +287,8 @@ struct BlockLimitStatus { st_stat.set_zero(); transactions = accounts = 0; gas_used = 0; + extra_out_msgs = 0; + public_library_diff = 0; } td::uint64 estimate_block_size(const vm::NewCellStorageStat::Stat* extra = nullptr) const; int classify() const; @@ -371,6 +390,9 @@ struct CurrencyCollection { CurrencyCollection operator-(const CurrencyCollection& other) const; CurrencyCollection operator-(CurrencyCollection&& other) const; CurrencyCollection operator-(td::RefInt256 other_grams) const; + bool clamp(const CurrencyCollection& other); + bool check_extra_currency_limit(td::uint32 max_currencies) const; + static bool remove_zero_extra_currencies(Ref& root, td::uint32 max_currencies); bool store(vm::CellBuilder& cb) const; bool store_or_zero(vm::CellBuilder& cb) const; bool fetch(vm::CellSlice& cs); @@ -414,6 +436,8 @@ struct ShardState { std::unique_ptr ihr_pending_; std::unique_ptr block_create_stats_; std::shared_ptr processed_upto_; + std::unique_ptr dispatch_queue_; + td::optional out_msg_queue_size_; bool is_valid() const { return id_.is_valid(); @@ -455,7 +479,7 @@ struct ShardState { struct ValueFlow { struct SetZero {}; CurrencyCollection from_prev_blk, to_next_blk, imported, exported, fees_collected, fees_imported, recovered, created, - minted; + minted, burned; ValueFlow() = default; ValueFlow(SetZero) : from_prev_blk{0} @@ -466,7 +490,8 @@ struct ValueFlow { , fees_imported{0} , recovered{0} , created{0} - , minted{0} { + , minted{0} + , burned{0} { } bool is_valid() const { return from_prev_blk.is_valid() && minted.is_valid(); @@ -652,7 +677,8 @@ class MtCarloComputeShare { void gen_vset(); }; -int filter_out_msg_queue(vm::AugmentedDictionary& out_queue, ton::ShardIdFull old_shard, ton::ShardIdFull subshard); +int filter_out_msg_queue(vm::AugmentedDictionary& out_queue, ton::ShardIdFull old_shard, ton::ShardIdFull subshard, + td::uint64* queue_size = nullptr); std::ostream& operator<<(std::ostream& os, const ShardId& shard_id); @@ -743,4 +769,25 @@ bool parse_hex_hash(td::Slice str, td::Bits256& hash); bool parse_block_id_ext(const char* str, const char* end, ton::BlockIdExt& blkid); bool parse_block_id_ext(td::Slice str, ton::BlockIdExt& blkid); +bool unpack_account_dispatch_queue(Ref csr, vm::Dictionary& dict, td::uint64& dict_size); +Ref pack_account_dispatch_queue(const vm::Dictionary& dict, td::uint64 dict_size); +Ref get_dispatch_queue_min_lt_account(const vm::AugmentedDictionary& dispatch_queue, + ton::StdSmcAddress& addr); +bool remove_dispatch_queue_entry(vm::AugmentedDictionary& dispatch_queue, const ton::StdSmcAddress& addr, + ton::LogicalTime lt); + +struct MsgMetadata { + td::uint32 depth; + ton::WorkchainId initiator_wc; + ton::StdSmcAddress initiator_addr; + ton::LogicalTime initiator_lt; + + bool unpack(vm::CellSlice& cs); + bool pack(vm::CellBuilder& cb) const; + std::string to_str() const; + + bool operator==(const MsgMetadata& other) const; + bool operator!=(const MsgMetadata& other) const; +}; + } // namespace block diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 8d98197f..4a8bbc06 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -143,8 +143,13 @@ tick_tock$_ tick:Bool tock:Bool = TickTock; _ split_depth:(Maybe (## 5)) special:(Maybe TickTock) code:(Maybe ^Cell) data:(Maybe ^Cell) - library:(HashmapE 256 SimpleLib) = StateInit; - + library:(Maybe ^Cell) = StateInit; + +// StateInitWithLibs is used to validate sent and received messages +_ split_depth:(Maybe (## 5)) special:(Maybe TickTock) + code:(Maybe ^Cell) data:(Maybe ^Cell) + library:(HashmapE 256 SimpleLib) = StateInitWithLibs; + simple_lib$_ public:Bool root:^Cell = SimpleLib; message$_ {X:Type} info:CommonMsgInfo @@ -167,6 +172,12 @@ interm_addr_ext$11 workchain_id:int32 addr_pfx:uint64 msg_envelope#4 cur_addr:IntermediateAddress next_addr:IntermediateAddress fwd_fee_remaining:Grams msg:^(Message Any) = MsgEnvelope; +msg_metadata#0 depth:uint32 initiator_addr:MsgAddressInt initiator_lt:uint64 = MsgMetadata; +msg_envelope_v2#5 cur_addr:IntermediateAddress + next_addr:IntermediateAddress fwd_fee_remaining:Grams + msg:^(Message Any) + emitted_lt:(Maybe uint64) + metadata:(Maybe MsgMetadata) = MsgEnvelope; // msg_import_ext$000 msg:^(Message Any) transaction:^Transaction = InMsg; @@ -182,6 +193,9 @@ msg_discard_fin$110 in_msg:^MsgEnvelope transaction_id:uint64 fwd_fee:Grams = InMsg; msg_discard_tr$111 in_msg:^MsgEnvelope transaction_id:uint64 fwd_fee:Grams proof_delivered:^Cell = InMsg; +msg_import_deferred_fin$00100 in_msg:^MsgEnvelope + transaction:^Transaction fwd_fee:Grams = InMsg; +msg_import_deferred_tr$00101 in_msg:^MsgEnvelope out_msg:^MsgEnvelope = InMsg; // import_fees$_ fees_collected:Grams value_imported:CurrencyCollection = ImportFees; @@ -205,6 +219,10 @@ msg_export_tr_req$111 out_msg:^MsgEnvelope imported:^InMsg = OutMsg; msg_export_deq_imm$100 out_msg:^MsgEnvelope reimport:^InMsg = OutMsg; +msg_export_new_defer$10100 out_msg:^MsgEnvelope + transaction:^Transaction = OutMsg; +msg_export_deferred_tr$10101 out_msg:^MsgEnvelope + imported:^InMsg = OutMsg; _ enqueued_lt:uint64 out_msg:^MsgEnvelope = EnqueuedMsg; @@ -219,8 +237,15 @@ _ (HashmapE 96 ProcessedUpto) = ProcessedInfo; ihr_pending$_ import_lt:uint64 = IhrPendingSince; _ (HashmapE 320 IhrPendingSince) = IhrPendingInfo; +// key - created_lt +_ messages:(HashmapE 64 EnqueuedMsg) count:uint48 = AccountDispatchQueue; +// key - sender address, aug - min created_lt +_ (HashmapAugE 256 AccountDispatchQueue uint64) = DispatchQueue; + +out_msg_queue_extra#0 dispatch_queue:DispatchQueue out_queue_size:(Maybe uint48) = OutMsgQueueExtra; + _ out_queue:OutMsgQueue proc_info:ProcessedInfo - ihr_pending:IhrPendingInfo = OutMsgQueueInfo; + extra:(Maybe OutMsgQueueExtra) = OutMsgQueueInfo; // storage_used$_ cells:(VarUInteger 7) bits:(VarUInteger 7) public_cells:(VarUInteger 7) = StorageUsed; @@ -271,7 +296,7 @@ transaction$0111 account_addr:bits256 lt:uint64 total_fees:CurrencyCollection state_update:^(HASH_UPDATE Account) description:^TransactionDescr = Transaction; -!merkle_update#02 {X:Type} old_hash:bits256 new_hash:bits256 +!merkle_update#04 {X:Type} old_hash:bits256 new_hash:bits256 old_depth:uint16 new_depth:uint16 old:^X new:^X = MERKLE_UPDATE X; update_hashes#72 {X:Type} old_hash:bits256 new_hash:bits256 = HASH_UPDATE X; @@ -366,7 +391,7 @@ trans_merge_install$0111 split_info:SplitMergeInfo smc_info#076ef1ea actions:uint16 msgs_sent:uint16 unixtime:uint32 block_lt:uint64 trans_lt:uint64 rand_seed:bits256 balance_remaining:CurrencyCollection - myself:MsgAddressInt = SmartContractInfo; + myself:MsgAddressInt global_config:(Maybe Cell) = SmartContractInfo; // // out_list_empty$_ = OutList 0; @@ -379,7 +404,7 @@ action_reserve_currency#36e6b809 mode:(## 8) currency:CurrencyCollection = OutAction; libref_hash$0 lib_hash:bits256 = LibRef; libref_ref$1 library:^Cell = LibRef; -action_change_library#26fa1dd4 mode:(## 7) { mode <= 2 } +action_change_library#26fa1dd4 mode:(## 7) libref:LibRef = OutAction; out_list_node$_ prev:^Cell action:OutAction = OutListNode; @@ -467,6 +492,19 @@ value_flow#b8e48dfb ^[ from_prev_blk:CurrencyCollection minted:CurrencyCollection ] = ValueFlow; +value_flow_v2#3ebf98b7 ^[ from_prev_blk:CurrencyCollection + to_next_blk:CurrencyCollection + imported:CurrencyCollection + exported:CurrencyCollection ] + fees_collected:CurrencyCollection + burned:CurrencyCollection + ^[ + fees_imported:CurrencyCollection + recovered:CurrencyCollection + created:CurrencyCollection + minted:CurrencyCollection + ] = ValueFlow; + // // bt_leaf$0 {X:Type} leaf:X = BinTree X; @@ -590,6 +628,11 @@ _ minter_addr:bits256 = ConfigParam 2; // ConfigParam 0 is used if absent _ fee_collector_addr:bits256 = ConfigParam 3; // ConfigParam 1 is used if absent _ dns_root_addr:bits256 = ConfigParam 4; // root TON DNS resolver +burning_config#01 + blackhole_addr:(Maybe bits256) + fee_burn_num:# fee_burn_denom:# { fee_burn_num <= fee_burn_denom } { fee_burn_denom >= 1 } = BurningConfig; +_ BurningConfig = ConfigParam 5; + _ mint_new_price:Grams mint_add_price:Grams = ConfigParam 6; _ to_mint:ExtraCurrencyCollection = ConfigParam 7; @@ -623,15 +666,15 @@ wc_split_merge_timings#0 //workchain#a5 enabled_since:uint32 min_split:(## 8) max_split:(## 8) // { min_split <= max_split } { max_split <= 60 } -workchain#a6 enabled_since:uint32 actual_min_split:(## 8) - min_split:(## 8) max_split:(## 8) { actual_min_split <= min_split } +workchain#a6 enabled_since:uint32 monitor_min_split:(## 8) + min_split:(## 8) max_split:(## 8) { monitor_min_split <= min_split } basic:(## 1) active:Bool accept_msgs:Bool flags:(## 13) { flags = 0 } zerostate_root_hash:bits256 zerostate_file_hash:bits256 version:uint32 format:(WorkchainFormat basic) = WorkchainDescr; -workchain_v2#a7 enabled_since:uint32 actual_min_split:(## 8) - min_split:(## 8) max_split:(## 8) { actual_min_split <= min_split } +workchain_v2#a7 enabled_since:uint32 monitor_min_split:(## 8) + min_split:(## 8) max_split:(## 8) { monitor_min_split <= min_split } basic:(## 1) active:Bool accept_msgs:Bool flags:(## 13) { flags = 0 } zerostate_root_hash:bits256 zerostate_file_hash:bits256 version:uint32 format:(WorkchainFormat basic) @@ -663,6 +706,8 @@ _#cc utime_since:uint32 bit_price_ps:uint64 cell_price_ps:uint64 mc_bit_price_ps:uint64 mc_cell_price_ps:uint64 = StoragePrices; _ (Hashmap 32 StoragePrices) = ConfigParam 18; +_ global_id:int32 = ConfigParam 19; + gas_prices#dd gas_price:uint64 gas_limit:uint64 gas_credit:uint64 block_gas_limit:uint64 freeze_due_limit:uint64 delete_due_limit:uint64 = GasLimitsPrices; @@ -755,20 +800,25 @@ _ MisbehaviourPunishmentConfig = ConfigParam 40; size_limits_config#01 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16 max_ext_msg_size:uint32 max_ext_msg_depth:uint16 = SizeLimitsConfig; size_limits_config_v2#02 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16 - max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_acc_state_bits:uint32 = SizeLimitsConfig; + max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_acc_state_bits:uint32 + max_acc_public_libraries:uint32 defer_out_queue_size_limit:uint32 max_msg_extra_currencies:uint32 = SizeLimitsConfig; _ SizeLimitsConfig = ConfigParam 43; // key is [ wc:int32 addr:uint256 ] suspended_address_list#00 addresses:(HashmapE 288 Unit) suspended_until:uint32 = SuspendedAddressList; _ SuspendedAddressList = ConfigParam 44; +precompiled_smc#b0 gas_usage:uint64 = PrecompiledSmc; +precompiled_contracts_config#c0 list:(HashmapE 256 PrecompiledSmc) = PrecompiledContractsConfig; +_ PrecompiledContractsConfig = ConfigParam 45; + oracle_bridge_params#_ bridge_address:bits256 oracle_mutlisig_address:bits256 oracles:(HashmapE 256 uint256) external_chain_address:bits256 = OracleBridgeParams; _ OracleBridgeParams = ConfigParam 71; // Ethereum bridge _ OracleBridgeParams = ConfigParam 72; // Binance Smart Chain bridge _ OracleBridgeParams = ConfigParam 73; // Polygon bridge // Note that chains in which bridge, minter and jetton-wallet operate are fixated -jetton_bridge_prices#_ bridge_burn_fee:Coins bridge_mint_fee:Coins +jetton_bridge_prices#_ bridge_burn_fee:Coins bridge_mint_fee:Coins wallet_min_tons_for_storage:Coins wallet_gas_consumption:Coins minter_min_tons_for_storage:Coins @@ -778,8 +828,8 @@ jetton_bridge_params_v0#00 bridge_address:bits256 oracles_address:bits256 oracle jetton_bridge_params_v1#01 bridge_address:bits256 oracles_address:bits256 oracles:(HashmapE 256 uint256) state_flags:uint8 prices:^JettonBridgePrices external_chain_address:bits256 = JettonBridgeParams; _ JettonBridgeParams = ConfigParam 79; // ETH->TON token bridge -_ JettonBridgeParams = ConfigParam 80; // BNB->TON token bridge -_ JettonBridgeParams = ConfigParam 81; // Polygon->TON token bridge +_ JettonBridgeParams = ConfigParam 81; // BNB->TON token bridge +_ JettonBridgeParams = ConfigParam 82; // Polygon->TON token bridge // diff --git a/crypto/block/check-proof.cpp b/crypto/block/check-proof.cpp index 6720ad40..431a03fe 100644 --- a/crypto/block/check-proof.cpp +++ b/crypto/block/check-proof.cpp @@ -315,6 +315,113 @@ td::Result TransactionList::validate() const { return std::move(res); } +td::Result BlockTransaction::validate(bool check_proof) const { + if (root.is_null()) { + return td::Status::Error("transactions are expected to be non-empty"); + } + if (check_proof && proof->get_hash().bits().compare(root->get_hash().bits(), 256)) { + return td::Status::Error(PSLICE() << "transaction hash mismatch: Merkle proof expects " + << proof->get_hash().bits().to_hex(256) + << " but received data has " << root->get_hash().bits().to_hex(256)); + } + block::gen::Transaction::Record trans; + if (!tlb::unpack_cell(root, trans)) { + return td::Status::Error("cannot unpack transaction cell"); + } + Info res; + res.blkid = blkid; + res.now = trans.now; + res.lt = trans.lt; + res.hash = root->get_hash().bits(); + res.transaction = root; + return std::move(res); +} + +td::Result BlockTransactionList::validate(bool check_proof) const { + constexpr int max_answer_transactions = 256; + + TRY_RESULT_PREFIX(list, vm::std_boc_deserialize_multi(std::move(transactions_boc)), "cannot deserialize transactions boc: "); + std::vector> tx_proofs(list.size()); + + if (check_proof) { + try { + TRY_RESULT(proof_cell, vm::std_boc_deserialize(std::move(proof_boc))); + auto virt_root = vm::MerkleProof::virtualize(proof_cell, 1); + + if (blkid.root_hash != virt_root->get_hash().bits()) { + return td::Status::Error("Invalid block proof root hash"); + } + block::gen::Block::Record blk; + block::gen::BlockExtra::Record extra; + if (!(tlb::unpack_cell(virt_root, blk) && tlb::unpack_cell(std::move(blk.extra), extra))) { + return td::Status::Error("Error unpacking proof cell"); + } + vm::AugmentedDictionary acc_dict{vm::load_cell_slice_ref(extra.account_blocks), 256, + block::tlb::aug_ShardAccountBlocks}; + + bool eof = false; + ton::LogicalTime reverse = reverse_mode ? ~0ULL : 0; + ton::LogicalTime trans_lt = static_cast(start_lt); + td::Bits256 cur_addr = start_addr; + bool allow_same = true; + int count = 0; + while (!eof && count < req_count && count < max_answer_transactions) { + auto value = acc_dict.extract_value( + acc_dict.vm::DictionaryFixed::lookup_nearest_key(cur_addr.bits(), 256, !reverse, allow_same)); + if (value.is_null()) { + eof = true; + break; + } + allow_same = false; + if (cur_addr != start_addr) { + trans_lt = reverse; + } + + block::gen::AccountBlock::Record acc_blk; + if (!tlb::csr_unpack(std::move(value), acc_blk) || acc_blk.account_addr != cur_addr) { + return td::Status::Error("Error unpacking proof account block"); + } + vm::AugmentedDictionary trans_dict{vm::DictNonEmpty(), std::move(acc_blk.transactions), 64, + block::tlb::aug_AccountTransactions}; + td::BitArray<64> cur_trans{(long long)trans_lt}; + while (count < req_count && count < max_answer_transactions) { + auto tvalue = trans_dict.extract_value_ref( + trans_dict.vm::DictionaryFixed::lookup_nearest_key(cur_trans.bits(), 64, !reverse)); + if (tvalue.is_null()) { + trans_lt = reverse; + break; + } + if (static_cast(count) < tx_proofs.size()) { + tx_proofs[count] = std::move(tvalue); + } + count++; + } + } + if (static_cast(count) != list.size()) { + return td::Status::Error(PSLICE() << "Txs count mismatch in proof (" << count << ") and response (" << list.size() << ")"); + } + } catch (vm::VmError& err) { + return err.as_status("Couldn't verify proof: "); + } catch (vm::VmVirtError& err) { + return err.as_status("Couldn't verify proof: "); + } catch (...) { + return td::Status::Error("Unknown exception raised while verifying proof"); + } + } + + Info res; + for (int i = 0; i < static_cast(list.size()); i++) { + auto& root = list[i]; + BlockTransaction transaction; + transaction.root = root; + transaction.blkid = blkid; + transaction.proof = tx_proofs[i]; + TRY_RESULT(info, transaction.validate(check_proof)); + res.transactions.push_back(std::move(info)); + } + return std::move(res); +} + td::Status BlockProofLink::validate(td::uint32* save_utime) const { if (save_utime) { *save_utime = 0; @@ -362,7 +469,7 @@ td::Status BlockProofLink::validate(td::uint32* save_utime) const { if (to.seqno()) { TRY_STATUS(check_block_header(vd_root, to)); if (!(tlb::unpack_cell(vd_root, blk) && tlb::unpack_cell(blk.info, info))) { - return td::Status::Error("cannot unpack header for block "s + from.to_str()); + return td::Status::Error("cannot unpack header for block "s + to.to_str()); } if (info.key_block != is_key) { return td::Status::Error(PSTRING() << "incorrect is_key_block value " << is_key << " for destination block " diff --git a/crypto/block/check-proof.h b/crypto/block/check-proof.h index 527f3138..497a4eba 100644 --- a/crypto/block/check-proof.h +++ b/crypto/block/check-proof.h @@ -88,4 +88,36 @@ struct TransactionList { td::Result validate() const; }; +struct BlockTransaction { + ton::BlockIdExt blkid; + td::Ref root; + td::Ref proof; + + struct Info { + ton::BlockIdExt blkid; + td::uint32 now; + ton::LogicalTime lt; + ton::Bits256 hash; + td::Ref transaction; + }; + td::Result validate(bool check_proof) const; +}; + +struct BlockTransactionList { + ton::BlockIdExt blkid; + td::BufferSlice transactions_boc; + td::BufferSlice proof_boc; + ton::LogicalTime start_lt; + td::Bits256 start_addr; + bool reverse_mode; + int req_count; + + struct Info { + ton::BlockIdExt blkid; + std::vector transactions; + }; + + td::Result validate(bool check_proof) const; +}; + } // namespace block diff --git a/crypto/block/create-state.cpp b/crypto/block/create-state.cpp index 7a734c3a..c8c8b970 100644 --- a/crypto/block/create-state.cpp +++ b/crypto/block/create-state.cpp @@ -47,6 +47,7 @@ #include "fift/Fift.h" #include "fift/Dictionary.h" #include "fift/SourceLookup.h" +#include "fift/IntCtx.h" #include "fift/words.h" #include "td/utils/logging.h" @@ -308,7 +309,7 @@ td::RefInt256 create_smartcontract(td::RefInt256 smc_addr, Ref code, R THRERR("cannot create smart-contract AccountStorage"); Ref storage = cb.finalize(); vm::CellStorageStat stats; - PDO(stats.compute_used_storage(Ref(storage))); + PDO(stats.compute_used_storage(Ref(storage)).is_ok()); if (verbosity > 2) { std::cerr << "storage is:\n"; vm::load_cell_slice(storage).print_rec(std::cerr); @@ -425,7 +426,7 @@ bool store_validator_list_hash(vm::CellBuilder& cb) { LOG_CHECK(vset) << "unpacked validator set is empty"; auto ccvc = block::Config::unpack_catchain_validators_config(config_dict.lookup_ref(td::BitArray<32>{28})); ton::ShardIdFull shard{ton::masterchainId}; - auto nodes = block::Config::do_compute_validator_set(ccvc, shard, *vset, now, 0); + auto nodes = block::Config::do_compute_validator_set(ccvc, shard, *vset, 0); LOG_CHECK(!nodes.empty()) << "validator node list in unpacked validator set is empty"; auto vset_hash = block::compute_validator_set_hash(0, shard, std::move(nodes)); LOG(DEBUG) << "initial validator set hash is " << vset_hash; @@ -813,11 +814,16 @@ void usage(const char* progname) { void parse_include_path_set(std::string include_path_set, std::vector& res) { td::Parser parser(include_path_set); while (!parser.empty()) { - auto path = parser.read_till_nofail(':'); + #if TD_WINDOWS + auto path_separator = '@'; + #else + auto path_separator = ':'; + #endif + auto path = parser.read_till_nofail(path_separator); if (!path.empty()) { res.push_back(path.str()); } - parser.skip_nofail(':'); + parser.skip_nofail(path_separator); } } @@ -866,8 +872,9 @@ int main(int argc, char* const argv[]) { case 'v': new_verbosity_level = VERBOSITY_NAME(FATAL) + (verbosity = td::to_integer(td::Slice(optarg))); break; - case 'V': - std::cout << "create-state build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; + case 'V': + std::cout << "create-state build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; std::exit(0); break; case 'h': diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 9b43075a..0f019b06 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -163,8 +163,11 @@ td::Status ConfigInfo::unpack() { } gen::McStateExtra::Record extra_info; if (!tlb::unpack_cell(state_extra_root_, extra_info)) { - vm::load_cell_slice(state_extra_root_).print_rec(std::cerr); - block::gen::t_McStateExtra.print_ref(std::cerr, state_extra_root_); + FLOG(WARNING) { + sb << "state extra information is invalid: "; + vm::load_cell_slice(state_extra_root_).print_rec(sb); + block::gen::t_McStateExtra.print_ref(sb, state_extra_root_); + }; return td::Status::Error("state extra information is invalid"); } gen::ValidatorInfo::Record validator_info; @@ -320,7 +323,7 @@ ton::ValidatorSessionConfig Config::get_consensus_config() const { c.max_block_size = r.max_block_bytes; c.max_collated_data_size = r.max_collated_bytes; }; - auto set_v2 = [&] (auto& r) { + auto set_v2 = [&](auto& r) { set_v1(r); c.new_catchain_ids = r.new_catchain_ids; }; @@ -621,12 +624,14 @@ td::Result> Config::get_storage_prices() const { } vm::Dictionary dict{std::move(cell), 32}; if (!dict.check_for_each([&res](Ref cs_ref, td::ConstBitPtr key, int n) -> bool { - block::gen::StoragePrices::Record data; - if (!tlb::csr_unpack(std::move(cs_ref), data) || data.utime_since != key.get_uint(n)) { + auto r_prices = do_get_one_storage_prices(*cs_ref); + if (r_prices.is_error()) { + return false; + } + res.push_back(r_prices.move_as_ok()); + if (res.back().valid_since != key.get_uint(n)) { return false; } - res.emplace_back(data.utime_since, data.bit_price_ps, data.cell_price_ps, data.mc_bit_price_ps, - data.mc_cell_price_ps); return true; })) { return td::Status::Error("invalid storage prices dictionary in configuration parameter 18"); @@ -634,16 +639,25 @@ td::Result> Config::get_storage_prices() const { return std::move(res); } -td::Result Config::do_get_gas_limits_prices(td::Ref cell, int id) { +td::Result Config::do_get_one_storage_prices(vm::CellSlice cs) { + block::gen::StoragePrices::Record data; + if (!tlb::unpack(cs, data)) { + return td::Status::Error("invalid storage prices dictionary in configuration parameter 18"); + } + return StoragePrices{data.utime_since, data.bit_price_ps, data.cell_price_ps, data.mc_bit_price_ps, + data.mc_cell_price_ps}; +} + +td::Result Config::do_get_gas_limits_prices(vm::CellSlice cs, int id) { GasLimitsPrices res; - auto cs = vm::load_cell_slice(cell); + vm::CellSlice cs0 = cs; block::gen::GasLimitsPrices::Record_gas_flat_pfx flat; if (tlb::unpack(cs, flat)) { cs = *flat.other; res.flat_gas_limit = flat.flat_gas_limit; res.flat_gas_price = flat.flat_gas_price; } else { - cs = vm::load_cell_slice(cell); + cs = cs0; } auto f = [&](const auto& r, td::uint64 spec_limit) { res.gas_limit = r.gas_limit; @@ -658,7 +672,7 @@ td::Result Config::do_get_gas_limits_prices(td::Ref c f(rec, rec.special_gas_limit); } else { block::gen::GasLimitsPrices::Record_gas_prices rec0; - if (tlb::unpack(cs, rec0)) { + if (tlb::unpack(cs = cs0, rec0)) { f(rec0, rec0.gas_limit); } else { return td::Status::Error(PSLICE() << "configuration parameter " << id @@ -688,7 +702,7 @@ td::Result Config::get_gas_limits_prices(bool is_masterchain) c if (cell.is_null()) { return td::Status::Error(PSLICE() << "configuration parameter " << id << " with gas prices is absent"); } - return do_get_gas_limits_prices(std::move(cell), id); + return do_get_gas_limits_prices(vm::load_cell_slice(cell), id); } td::Result Config::get_msg_prices(bool is_masterchain) const { @@ -697,7 +711,10 @@ td::Result Config::get_msg_prices(bool is_masterchain) const { if (cell.is_null()) { return td::Status::Error(PSLICE() << "configuration parameter " << id << " with msg prices is absent"); } - auto cs = vm::load_cell_slice(std::move(cell)); + return do_get_msg_prices(vm::load_cell_slice(cell), id); +} + +td::Result Config::do_get_msg_prices(vm::CellSlice cs, int id) { block::gen::MsgForwardPrices::Record rec; if (!tlb::unpack(cs, rec)) { return td::Status::Error(PSLICE() << "configuration parameter " << id @@ -855,11 +872,12 @@ Ref McShardHash::from_block(Ref block_root, const ton::Fi ton::RootHash rhash = block_root->get_hash().bits(); CurrencyCollection fees_collected, funds_created; if (init_fees) { - block::gen::ValueFlow::Record flow; - if (!(tlb::unpack_cell(rec.value_flow, flow) && fees_collected.unpack(flow.fees_collected) && - funds_created.unpack(flow.r2.created))) { + block::ValueFlow flow; + if (!flow.unpack(vm::load_cell_slice_ref(rec.value_flow))) { return {}; } + fees_collected = flow.fees_collected; + funds_created = flow.created; } return Ref(true, ton::BlockId{ton::ShardIdFull(shard), (unsigned)info.seq_no}, info.start_lt, info.end_lt, info.gen_utime, rhash, fhash, fees_collected, funds_created, ~0U, @@ -909,11 +927,12 @@ Ref McShardDescr::from_block(Ref block_root, Refget_hash().bits(); CurrencyCollection fees_collected, funds_created; if (init_fees) { - block::gen::ValueFlow::Record flow; - if (!(tlb::unpack_cell(rec.value_flow, flow) && fees_collected.unpack(flow.fees_collected) && - funds_created.unpack(flow.r2.created))) { + block::ValueFlow flow; + if (!flow.unpack(vm::load_cell_slice_ref(rec.value_flow))) { return {}; } + fees_collected = flow.fees_collected; + funds_created = flow.created; } auto res = Ref(true, ton::BlockId{ton::ShardIdFull(shard), (unsigned)info.seq_no}, info.start_lt, info.end_lt, info.gen_utime, rhash, fhash, fees_collected, funds_created, ~0U, @@ -1051,7 +1070,6 @@ Ref ShardConfig::get_shard_hash(ton::ShardIdFull id, bool exact) co ton::ShardIdFull true_id; vm::CellSlice cs; if (get_shard_hash_raw(cs, id, true_id, exact)) { - // block::gen::t_ShardDescr.print(std::cerr, vm::CellSlice{cs}); return McShardHash::unpack(cs, true_id); } else { return {}; @@ -1621,8 +1639,10 @@ bool ShardConfig::set_shard_info(ton::ShardIdFull shard, Ref value) { if (!gen::t_BinTree_ShardDescr.validate_ref(1024, value)) { LOG(ERROR) << "attempting to store an invalid (BinTree ShardDescr) at shard configuration position " << shard.to_str(); - gen::t_BinTree_ShardDescr.print_ref(std::cerr, value); - vm::load_cell_slice(value).print_rec(std::cerr); + FLOG(WARNING) { + gen::t_BinTree_ShardDescr.print_ref(sb, value); + vm::load_cell_slice(value).print_rec(sb); + }; return false; } auto root = shard_hashes_dict_->lookup_ref(td::BitArray<32>{shard.workchain}); @@ -1730,7 +1750,7 @@ ton::CatchainSeqno ConfigInfo::get_shard_cc_seqno(ton::ShardIdFull shard) const std::vector Config::compute_validator_set(ton::ShardIdFull shard, const block::ValidatorSet& vset, ton::UnixTime time, ton::CatchainSeqno cc_seqno) const { - return do_compute_validator_set(get_catchain_validators_config(), shard, vset, time, cc_seqno); + return do_compute_validator_set(get_catchain_validators_config(), shard, vset, cc_seqno); } std::vector Config::compute_validator_set(ton::ShardIdFull shard, ton::UnixTime time, @@ -1757,7 +1777,7 @@ std::vector ConfigInfo::compute_validator_set_cc(ton::Shard if (cc_seqno_delta) { cc_seqno = *cc_seqno_delta += cc_seqno; } - return do_compute_validator_set(get_catchain_validators_config(), shard, vset, time, cc_seqno); + return do_compute_validator_set(get_catchain_validators_config(), shard, vset, cc_seqno); } std::vector ConfigInfo::compute_validator_set_cc(ton::ShardIdFull shard, ton::UnixTime time, @@ -1840,9 +1860,8 @@ int ValidatorSet::lookup_public_key(td::ConstBitPtr pubkey) const { return -1; } -std::vector Config::do_compute_validator_set(const block::CatchainValidatorsConfig& ccv_conf, - ton::ShardIdFull shard, - const block::ValidatorSet& vset, ton::UnixTime time, +std::vector Config::do_compute_validator_set(const CatchainValidatorsConfig& ccv_conf, + ton::ShardIdFull shard, const ValidatorSet& vset, ton::CatchainSeqno cc_seqno) { // LOG(DEBUG) << "in Config::do_compute_validator_set() for " << shard.to_str() << " ; cc_seqno=" << cc_seqno; std::vector nodes; @@ -1914,10 +1933,17 @@ std::vector Config::compute_total_validator_set(int next) c } td::Result Config::get_size_limits_config() const { - SizeLimitsConfig limits; td::Ref param = get_config_param(43); if (param.is_null()) { - return limits; + return do_get_size_limits_config({}); + } + return do_get_size_limits_config(vm::load_cell_slice_ref(param)); +} + +td::Result Config::do_get_size_limits_config(td::Ref cs) { + SizeLimitsConfig limits; + if (cs.is_null()) { + return limits; // default values } auto unpack_v1 = [&](auto& rec) { limits.max_msg_bits = rec.max_msg_bits; @@ -1932,12 +1958,15 @@ td::Result Config::get_size_limits_config() const { unpack_v1(rec); limits.max_acc_state_bits = rec.max_acc_state_bits; limits.max_acc_state_cells = rec.max_acc_state_cells; + limits.max_acc_public_libraries = rec.max_acc_public_libraries; + limits.defer_out_queue_size_limit = rec.defer_out_queue_size_limit; + limits.max_msg_extra_currencies = rec.max_msg_extra_currencies; }; gen::SizeLimitsConfig::Record_size_limits_config rec_v1; gen::SizeLimitsConfig::Record_size_limits_config_v2 rec_v2; - if (tlb::unpack_cell(param, rec_v1)) { + if (tlb::csr_unpack(cs, rec_v1)) { unpack_v1(rec_v1); - } else if (tlb::unpack_cell(param, rec_v2)) { + } else if (tlb::csr_unpack(cs, rec_v2)) { unpack_v2(rec_v2); } else { return td::Status::Error("configuration parameter 43 is invalid"); @@ -1954,6 +1983,71 @@ std::unique_ptr Config::get_suspended_addresses(ton::UnixTime no return std::make_unique(rec.addresses->prefetch_ref(), 288); } +BurningConfig Config::get_burning_config() const { + td::Ref param = get_config_param(5); + gen::BurningConfig::Record rec; + if (param.is_null() || !tlb::unpack_cell(param, rec)) { + return {}; + } + BurningConfig c; + c.fee_burn_num = rec.fee_burn_num; + c.fee_burn_denom = rec.fee_burn_denom; + vm::CellSlice& addr = rec.blackhole_addr.write(); + if (addr.fetch_long(1)) { + td::Bits256 x; + addr.fetch_bits_to(x.bits(), 256); + c.blackhole_addr = x; + } + return c; +} + +td::Ref Config::get_unpacked_config_tuple(ton::UnixTime now) const { + auto get_param = [&](td::int32 idx) -> vm::StackEntry { + auto cell = get_config_param(idx); + if (cell.is_null()) { + return {}; + } + return vm::load_cell_slice_ref(cell); + }; + auto get_current_storage_prices = [&]() -> vm::StackEntry { + auto cell = get_config_param(18); + if (cell.is_null()) { + return {}; + } + vm::StackEntry res; + vm::Dictionary dict{std::move(cell), 32}; + dict.check_for_each([&](Ref cs_ref, td::ConstBitPtr key, int n) -> bool { + auto utime_since = key.get_uint(n); + if (now >= utime_since) { + res = std::move(cs_ref); + return true; + } + return false; + }); + return res; + }; + std::vector tuple; + tuple.push_back(get_current_storage_prices()); // storage_prices + tuple.push_back(get_param(19)); // global_id + tuple.push_back(get_param(20)); // config_mc_gas_prices + tuple.push_back(get_param(21)); // config_gas_prices + tuple.push_back(get_param(24)); // config_mc_fwd_prices + tuple.push_back(get_param(25)); // config_fwd_prices + tuple.push_back(get_param(43)); // size_limits_config + return td::make_cnt_ref>(std::move(tuple)); +} + +PrecompiledContractsConfig Config::get_precompiled_contracts_config() const { + PrecompiledContractsConfig c; + td::Ref param = get_config_param(45); + gen::PrecompiledContractsConfig::Record rec; + if (param.is_null() || !tlb::unpack_cell(param, rec)) { + return c; + } + c.list = vm::Dictionary{rec.list->prefetch_ref(), 256}; + return c; +} + td::Result> Config::unpack_validator_set_start_stop(Ref vset_root) { if (vset_root.is_null()) { return td::Status::Error("validator set absent"); @@ -1985,7 +2079,7 @@ bool WorkchainInfo::unpack(ton::WorkchainId wc, vm::CellSlice& cs) { } auto unpack_v1 = [this](auto& info) { enabled_since = info.enabled_since; - actual_min_split = info.actual_min_split; + monitor_min_split = info.monitor_min_split; min_split = info.min_split; max_split = info.max_split; basic = info.basic; @@ -2200,4 +2294,74 @@ Ref ConfigInfo::lookup_library(td::ConstBitPtr root_hash) const { return lib; } +td::Result> ConfigInfo::get_prev_blocks_info() const { + // [ wc:Integer shard:Integer seqno:Integer root_hash:Integer file_hash:Integer] = BlockId; + // [ last_mc_blocks:[BlockId...] + // prev_key_block:BlockId + // last_mc_blocks_100[BlockId...] ] : PrevBlocksInfo + auto block_id_to_tuple = [](const ton::BlockIdExt& block_id) -> vm::Ref { + td::RefInt256 shard = td::make_refint(block_id.id.shard); + if (shard->sgn() < 0) { + shard &= ((td::make_refint(1) << 64) - 1); + } + return vm::make_tuple_ref(td::make_refint(block_id.id.workchain), std::move(shard), + td::make_refint(block_id.id.seqno), td::bits_to_refint(block_id.root_hash.bits(), 256), + td::bits_to_refint(block_id.file_hash.bits(), 256)); + }; + std::vector tuple; + + std::vector last_mc_blocks; + last_mc_blocks.push_back(block_id_to_tuple(block_id)); + for (ton::BlockSeqno seqno = block_id.id.seqno; seqno > 0 && last_mc_blocks.size() < 16;) { + --seqno; + ton::BlockIdExt id; + if (!get_old_mc_block_id(seqno, id)) { + return td::Status::Error("cannot fetch old mc block"); + } + last_mc_blocks.push_back(block_id_to_tuple(id)); + } + tuple.push_back(td::make_cnt_ref>(std::move(last_mc_blocks))); + + ton::BlockIdExt last_key_block; + ton::LogicalTime last_key_block_lt; + if (!get_last_key_block(last_key_block, last_key_block_lt)) { + return td::Status::Error("cannot fetch last key block"); + } + tuple.push_back(block_id_to_tuple(last_key_block)); + + if (get_global_version() >= 9) { + std::vector last_mc_blocks_100; + for (ton::BlockSeqno seqno = block_id.id.seqno / 100 * 100; last_mc_blocks_100.size() < 16;) { + ton::BlockIdExt id; + if (!get_old_mc_block_id(seqno, id)) { + return td::Status::Error("cannot fetch old mc block"); + } + last_mc_blocks_100.push_back(block_id_to_tuple(id)); + if (seqno < 100) { + break; + } + seqno -= 100; + } + tuple.push_back(td::make_cnt_ref>(std::move(last_mc_blocks_100))); + } + + return td::make_cnt_ref>(std::move(tuple)); +} + +td::optional PrecompiledContractsConfig::get_contract( + td::Bits256 code_hash) const { + auto list_copy = list; + auto cs = list_copy.lookup(code_hash); + if (cs.is_null()) { + return {}; + } + gen::PrecompiledSmc::Record rec; + if (!tlb::csr_unpack(cs, rec)) { + return {}; + } + Contract c; + c.gas_usage = rec.gas_usage; + return c; +} + } // namespace block diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index ad5999e5..98e6a26d 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -197,6 +197,7 @@ struct McShardHash : public McShardHashI { : blk_(blk), start_lt_(start_lt), end_lt_(end_lt) { } McShardHash(const McShardHash&) = default; + McShardHash& operator=(const McShardHash&) = default; bool is_valid() const { return blk_.is_valid(); } @@ -350,7 +351,11 @@ struct GasLimitsPrices { td::uint64 freeze_due_limit{0}; td::uint64 delete_due_limit{0}; - td::RefInt256 compute_gas_price(td::uint64 gas_used) const; + td::RefInt256 compute_gas_price(td::uint64 gas_used) const { + return gas_used <= flat_gas_limit + ? td::make_refint(flat_gas_price) + : td::rshift(td::make_refint(gas_price) * (gas_used - flat_gas_limit), 16, 1) + flat_gas_price; + } }; // msg_fwd_fees = (lump_price + ceil((bit_price * msg.bits + cell_price * msg.cells)/2^16)) nanograms @@ -365,6 +370,7 @@ struct MsgPrices { td::uint32 first_frac; td::uint32 next_frac; td::uint64 compute_fwd_fees(td::uint64 cells, td::uint64 bits) const; + td::RefInt256 compute_fwd_fees256(td::uint64 cells, td::uint64 bits) const; std::pair compute_fwd_ihr_fees(td::uint64 cells, td::uint64 bits, bool ihr_disabled = false) const; MsgPrices() = default; @@ -389,6 +395,9 @@ struct SizeLimitsConfig { ExtMsgLimits ext_msg_limits; td::uint32 max_acc_state_cells = 1 << 16; td::uint32 max_acc_state_bits = (1 << 16) * 1023; + td::uint32 max_acc_public_libraries = 256; + td::uint32 defer_out_queue_size_limit = 256; + td::uint32 max_msg_extra_currencies = 2; }; struct CatchainValidatorsConfig { @@ -407,7 +416,7 @@ struct CatchainValidatorsConfig { struct WorkchainInfo : public td::CntObject { ton::WorkchainId workchain{ton::workchainInvalid}; ton::UnixTime enabled_since; - td::uint32 actual_min_split; + td::uint32 monitor_min_split; td::uint32 min_split, max_split; bool basic; bool active; @@ -448,10 +457,11 @@ class ShardConfig { ShardConfig() = default; ShardConfig(const ShardConfig& other); ShardConfig(ShardConfig&& other) = default; - ShardConfig(Ref shard_hashes, Ref mc_shard_hash = {}) + explicit ShardConfig(Ref shard_hashes, Ref mc_shard_hash = {}) : shard_hashes_(std::move(shard_hashes)), mc_shard_hash_(std::move(mc_shard_hash)) { init(); } + ShardConfig& operator=(ShardConfig&& other) = default; bool is_valid() const { return valid_; } @@ -504,6 +514,31 @@ class ShardConfig { bool set_shard_info(ton::ShardIdFull shard, Ref value); }; +struct BurningConfig { + td::optional blackhole_addr; + td::uint32 fee_burn_num = 0, fee_burn_denom = 1; + + td::RefInt256 calculate_burned_fees(const td::RefInt256& x) const { + if (x.is_null()) { + return x; + } + return x * fee_burn_num / td::make_refint(fee_burn_denom); + } + + CurrencyCollection calculate_burned_fees(const CurrencyCollection& x) const { + return CurrencyCollection{calculate_burned_fees(x.grams)}; + } +}; + +struct PrecompiledContractsConfig { + struct Contract { + td::uint64 gas_usage; + }; + vm::Dictionary list{256}; + + td::optional get_contract(td::Bits256 code_hash) const; +}; + class Config { enum { default_mc_catchain_lifetime = 200, @@ -513,7 +548,10 @@ class Config { }; public: - enum { needValidatorSet = 16, needSpecialSmc = 32, needWorkchainInfo = 256, needCapabilities = 512 }; + static constexpr int needValidatorSet = 16; + static constexpr int needSpecialSmc = 32; + static constexpr int needWorkchainInfo = 256; + static constexpr int needCapabilities = 512; int mode{0}; ton::BlockIdExt block_id; @@ -587,9 +625,11 @@ class Config { bool is_special_smartcontract(const ton::StdSmcAddress& addr) const; static td::Result> unpack_validator_set(Ref valset_root); td::Result> get_storage_prices() const; + static td::Result do_get_one_storage_prices(vm::CellSlice cs); td::Result get_gas_limits_prices(bool is_masterchain = false) const; - static td::Result do_get_gas_limits_prices(td::Ref cell, int id); + static td::Result do_get_gas_limits_prices(vm::CellSlice cs, int id); td::Result get_msg_prices(bool is_masterchain = false) const; + static td::Result do_get_msg_prices(vm::CellSlice cs, int id); static CatchainValidatorsConfig unpack_catchain_validators_config(Ref cell); CatchainValidatorsConfig get_catchain_validators_config() const; td::Status visit_validator_params() const; @@ -616,10 +656,13 @@ class Config { ton::CatchainSeqno cc_seqno) const; std::vector compute_total_validator_set(int next) const; td::Result get_size_limits_config() const; + static td::Result do_get_size_limits_config(td::Ref cs); std::unique_ptr get_suspended_addresses(ton::UnixTime now) const; - static std::vector do_compute_validator_set(const block::CatchainValidatorsConfig& ccv_conf, - ton::ShardIdFull shard, - const block::ValidatorSet& vset, ton::UnixTime time, + BurningConfig get_burning_config() const; + td::Ref get_unpacked_config_tuple(ton::UnixTime now) const; + PrecompiledContractsConfig get_precompiled_contracts_config() const; + static std::vector do_compute_validator_set(const CatchainValidatorsConfig& ccv_conf, + ton::ShardIdFull shard, const ValidatorSet& vset, ton::CatchainSeqno cc_seqno); static td::Result> unpack_config(Ref config_root, @@ -632,7 +675,6 @@ class Config { static td::Result> unpack_param_dict(vm::Dictionary& dict); static td::Result> unpack_param_dict(Ref dict_root); - protected: Config(int _mode) : mode(_mode) { config_addr.set_zero(); } @@ -645,14 +687,12 @@ class Config { class ConfigInfo : public Config, public ShardConfig { public: - enum { - needStateRoot = 1, - needLibraries = 2, - needStateExtraRoot = 4, - needShardHashes = 8, - needAccountsRoot = 64, - needPrevBlocks = 128 - }; + static constexpr int needStateRoot = 1; + static constexpr int needLibraries = 2; + static constexpr int needStateExtraRoot = 4; + static constexpr int needShardHashes = 8; + static constexpr int needAccountsRoot = 64; + static constexpr int needPrevBlocks = 128; ton::BlockSeqno vert_seqno{~0U}; int global_id_{0}; ton::UnixTime utime{0}; @@ -722,6 +762,7 @@ class ConfigInfo : public Config, public ShardConfig { ton::CatchainSeqno* cc_seqno_delta = nullptr) const; std::vector compute_validator_set_cc(ton::ShardIdFull shard, ton::UnixTime time, ton::CatchainSeqno* cc_seqno_delta = nullptr) const; + td::Result> get_prev_blocks_info() const; static td::Result> extract_config(std::shared_ptr static_boc, int mode = 0); static td::Result> extract_config(Ref mc_state_root, int mode = 0); diff --git a/crypto/block/output-queue-merger.cpp b/crypto/block/output-queue-merger.cpp index 1084bb1a..7d258cfe 100644 --- a/crypto/block/output-queue-merger.cpp +++ b/crypto/block/output-queue-merger.cpp @@ -138,7 +138,6 @@ bool OutputQueueMerger::add_root(int src, Ref outmsg_root) { if (outmsg_root.is_null()) { return true; } - //block::gen::HashmapAug{352, block::gen::t_EnqueuedMsg, block::gen::t_uint64}.print_ref(std::cerr, outmsg_root); auto kv = std::make_unique(src, std::move(outmsg_root)); if (kv->replace_by_prefix(common_pfx.cbits(), common_pfx_len)) { heap.push_back(std::move(kv)); @@ -146,22 +145,30 @@ bool OutputQueueMerger::add_root(int src, Ref outmsg_root) { return true; } -OutputQueueMerger::OutputQueueMerger(ton::ShardIdFull _queue_for, std::vector _neighbors) +OutputQueueMerger::OutputQueueMerger(ton::ShardIdFull _queue_for, std::vector _neighbors) : queue_for(_queue_for), neighbors(std::move(_neighbors)), eof(false), failed(false) { init(); } +OutputQueueMerger::OutputQueueMerger(ton::ShardIdFull _queue_for, std::vector _neighbors) + : queue_for(_queue_for), eof(false), failed(false) { + for (auto& nb : _neighbors) { + neighbors.emplace_back(nb.top_block_id(), nb.outmsg_root, nb.is_disabled()); + } + init(); +} + void OutputQueueMerger::init() { common_pfx.bits().store_int(queue_for.workchain, 32); int l = queue_for.pfx_len(); td::bitstring::bits_store_long_top(common_pfx.bits() + 32, queue_for.shard, l); common_pfx_len = 32 + l; int i = 0; - for (block::McShardDescr& neighbor : neighbors) { - if (!neighbor.is_disabled()) { - LOG(DEBUG) << "adding " << (neighbor.outmsg_root.is_null() ? "" : "non-") << "empty output queue for neighbor #" - << i << " (" << neighbor.blk_.to_str() << ")"; - add_root(i++, neighbor.outmsg_root); + for (Neighbor& neighbor : neighbors) { + if (!neighbor.disabled_) { + LOG(DEBUG) << "adding " << (neighbor.outmsg_root_.is_null() ? "" : "non-") << "empty output queue for neighbor #" + << i << " (" << neighbor.block_id_.to_str() << ")"; + add_root(i++, neighbor.outmsg_root_); } else { LOG(DEBUG) << "skipping output queue for disabled neighbor #" << i; i++; diff --git a/crypto/block/output-queue-merger.h b/crypto/block/output-queue-merger.h index bf3d8586..07533f24 100644 --- a/crypto/block/output-queue-merger.h +++ b/crypto/block/output-queue-merger.h @@ -51,12 +51,22 @@ struct OutputQueueMerger { bool unpack_node(td::ConstBitPtr key_pfx, int key_pfx_len, Ref node); bool split(MsgKeyValue& second); }; + struct Neighbor { + ton::BlockIdExt block_id_; + td::Ref outmsg_root_; + bool disabled_; + Neighbor() = default; + Neighbor(ton::BlockIdExt block_id, td::Ref outmsg_root, bool disabled = false) + : block_id_(block_id), outmsg_root_(std::move(outmsg_root)), disabled_(disabled) { + } + }; // ton::ShardIdFull queue_for; std::vector> msg_list; - std::vector neighbors; + std::vector neighbors; public: + OutputQueueMerger(ton::ShardIdFull _queue_for, std::vector _neighbors); OutputQueueMerger(ton::ShardIdFull _queue_for, std::vector _neighbors); bool is_eof() const { return eof; diff --git a/crypto/block/precompiled-smc/PrecompiledSmartContract.cpp b/crypto/block/precompiled-smc/PrecompiledSmartContract.cpp new file mode 100644 index 00000000..6797216d --- /dev/null +++ b/crypto/block/precompiled-smc/PrecompiledSmartContract.cpp @@ -0,0 +1,170 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "common.h" +#include +#include "vm/memo.h" + +namespace block::precompiled { + +using namespace vm; + +Result PrecompiledSmartContract::run(td::Ref my_address, ton::UnixTime now, ton::LogicalTime cur_lt, + CurrencyCollection balance, td::Ref c4, vm::CellSlice msg_body, + td::Ref msg, CurrencyCollection msg_balance, bool is_external, + std::vector> libraries, int global_version, + td::uint16 max_data_depth, td::Ref my_code, + td::Ref unpacked_config, td::RefInt256 due_payment, + td::uint64 precompiled_gas_usage) { + my_address_ = std::move(my_address); + now_ = now; + cur_lt_ = cur_lt; + balance_ = std::move(balance); + c4_ = (c4.not_null() ? std::move(c4) : CellBuilder().finalize()); + in_msg_body_ = std::move(msg_body); + in_msg_ = std::move(msg); + in_msg_balance_ = std::move(msg_balance); + is_external_ = is_external; + my_code_ = std::move(my_code); + unpacked_config_ = std::move(unpacked_config); + due_payment_ = std::move(due_payment); + precompiled_gas_usage_ = precompiled_gas_usage; + + vm::DummyVmState vm_state{std::move(libraries), global_version}; + vm::VmStateInterface::Guard guard{&vm_state}; + + Result result; + try { + result = do_run(); + } catch (vm::VmError &e) { + result = Result::error(e.get_errno(), e.get_arg()); + } catch (Result &r) { + result = std::move(r); + } + + if (result.exit_code != 0 && result.exit_code != 1) { + // see VmState::try_commit() + if (c4_.is_null() || c4_->get_depth() > max_data_depth || c4_->get_level() != 0 || c5_.is_null() || + c5_->get_depth() > max_data_depth || c5_->get_level() != 0) { + result = Result::error(Excno::cell_ov, 0); + } + } + return result; +} + +void PrecompiledSmartContract::send_raw_message(const td::Ref &msg, int mode) { + CellBuilder cb; + if (!(cb.store_ref_bool(c5_) // out_list$_ {n:#} prev:^(OutList n) + && cb.store_long_bool(0x0ec3c86d, 32) // action_send_msg#0ec3c86d + && cb.store_long_bool(mode, 8) // mode:(## 8) + && cb.store_ref_bool(msg))) { + throw VmError{Excno::cell_ov, "cannot serialize raw output message into an output action cell"}; + } + c5_ = cb.finalize_novm(); +} + +void PrecompiledSmartContract::raw_reserve(const td::RefInt256 &amount, int mode) { + if (amount->sgn() < 0) { + throw VmError{Excno::range_chk, "amount of nanograms must be non-negative"}; + } + CellBuilder cb; + if (!(cb.store_ref_bool(c5_) // out_list$_ {n:#} prev:^(OutList n) + && cb.store_long_bool(0x36e6b809, 32) // action_reserve_currency#36e6b809 + && cb.store_long_bool(mode, 8) // mode:(## 8) + && util::store_coins(cb, std::move(amount), true) // + && cb.store_maybe_ref({}))) { + throw VmError{Excno::cell_ov, "cannot serialize raw reserved currency amount into an output action cell"}; + } + c5_ = cb.finalize_novm(); +} + +td::RefInt256 PrecompiledSmartContract::get_compute_fee(ton::WorkchainId wc, td::uint64 gas_used) { + if (gas_used >= (1ULL << 63)) { + throw VmError{Excno::range_chk}; + } + block::GasLimitsPrices prices = util::get_gas_prices(unpacked_config_, wc); + return util::check_finite(prices.compute_gas_price(gas_used)); +} + +td::RefInt256 PrecompiledSmartContract::get_forward_fee(ton::WorkchainId wc, td::uint64 bits, td::uint64 cells) { + if (bits >= (1ULL << 63) || cells >= (1ULL << 63)) { + throw VmError{Excno::range_chk}; + } + block::MsgPrices prices = util::get_msg_prices(unpacked_config_, wc); + return util::check_finite(prices.compute_fwd_fees256(cells, bits)); +} + +td::RefInt256 PrecompiledSmartContract::get_storage_fee(ton::WorkchainId wc, td::uint64 duration, td::uint64 bits, + td::uint64 cells) { + if (bits >= (1ULL << 63) || cells >= (1ULL << 63) || duration >= (1ULL << 63)) { + throw VmError{Excno::range_chk}; + } + td::optional maybe_prices = util::get_storage_prices(unpacked_config_); + return util::check_finite(util::calculate_storage_fee(maybe_prices, wc, duration, bits, cells)); +} + +td::RefInt256 PrecompiledSmartContract::get_simple_compute_fee(ton::WorkchainId wc, td::uint64 gas_used) { + if (gas_used >= (1ULL << 63)) { + throw VmError{Excno::range_chk}; + } + block::GasLimitsPrices prices = util::get_gas_prices(unpacked_config_, wc); + return util::check_finite(td::rshift(td::make_refint(prices.gas_price) * gas_used, 16, 1)); +} + +td::RefInt256 PrecompiledSmartContract::get_simple_forward_fee(ton::WorkchainId wc, td::uint64 bits, td::uint64 cells) { + if (bits >= (1ULL << 63) || cells >= (1ULL << 63)) { + throw VmError{Excno::range_chk}; + } + block::MsgPrices prices = util::get_msg_prices(unpacked_config_, wc); + return util::check_finite( + td::rshift(td::make_refint(prices.bit_price) * bits + td::make_refint(prices.cell_price) * cells, 16, 1)); +} + +td::RefInt256 PrecompiledSmartContract::get_original_fwd_fee(ton::WorkchainId wc, const td::RefInt256 &x) { + if (x->sgn() < 0) { + throw VmError{Excno::range_chk, "fwd_fee is negative"}; + } + block::MsgPrices prices = util::get_msg_prices(unpacked_config_, wc); + return util::check_finite(td::muldiv(x, td::make_refint(1 << 16), td::make_refint((1 << 16) - prices.first_frac))); +} + +static std::atomic_bool precompiled_execution_enabled{false}; + +std::unique_ptr get_implementation(td::Bits256 code_hash) { + if (!precompiled_execution_enabled) { + return nullptr; + } + static std::map (*)()> map = []() { + auto from_hex = [](td::Slice s) -> td::Bits256 { + td::Bits256 x; + CHECK(x.from_hex(s) == 256); + return x; + }; + std::map (*)()> map; +#define CONTRACT(hash, cls) \ + map[from_hex(hash)] = []() -> std::unique_ptr { return std::make_unique(); }; + // CONTRACT("CODE_HASH_HEX", ClassName); + return map; + }(); + auto it = map.find(code_hash); + return it == map.end() ? nullptr : it->second(); +} + +void set_precompiled_execution_enabled(bool value) { + precompiled_execution_enabled = value; +} + +} // namespace block::precompiled diff --git a/crypto/block/precompiled-smc/PrecompiledSmartContract.h b/crypto/block/precompiled-smc/PrecompiledSmartContract.h new file mode 100644 index 00000000..509ebf79 --- /dev/null +++ b/crypto/block/precompiled-smc/PrecompiledSmartContract.h @@ -0,0 +1,122 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "common/refcnt.hpp" +#include "common/refint.h" +#include "vm/cells.h" +#include "vm/cellslice.h" +#include "vm/dict.h" +#include "vm/boc.h" +#include +#include "tl/tlblib.hpp" +#include "td/utils/bits.h" +#include "ton/ton-types.h" +#include "block/block.h" +#include "block/mc-config.h" + +namespace block::precompiled { + +struct Result { + int exit_code = 0; + td::optional exit_arg; + bool accepted = true; + bool committed = false; + + static Result error(int code, long long arg = 0) { + Result res; + res.exit_code = code; + res.exit_arg = arg; + return res; + } + + static Result error(vm::Excno code, long long arg = 0) { + Result res; + res.exit_code = (int)code; + res.exit_arg = arg; + return res; + } + + static Result not_accepted(int code = 0) { + Result res; + res.exit_code = code; + res.accepted = false; + return res; + } + + static Result success() { + Result res; + res.committed = true; + return res; + } +}; + +class PrecompiledSmartContract { + public: + virtual ~PrecompiledSmartContract() = default; + + virtual std::string get_name() const = 0; + + virtual int required_version() const { + return 6; + } + + Result run(td::Ref my_address, ton::UnixTime now, ton::LogicalTime cur_lt, CurrencyCollection balance, + td::Ref c4, vm::CellSlice msg_body, td::Ref msg, CurrencyCollection msg_balance, + bool is_external, std::vector> libraries, int global_version, td::uint16 max_data_depth, + td::Ref my_code, td::Ref unpacked_config, td::RefInt256 due_payment, td::uint64 precompiled_gas_usage); + + td::Ref get_c4() const { + return c4_; + } + td::Ref get_c5() const { + return c5_; + } + + protected: + td::Ref my_address_; + ton::UnixTime now_; + ton::LogicalTime cur_lt_; + CurrencyCollection balance_; + vm::CellSlice in_msg_body_; + td::Ref in_msg_; + CurrencyCollection in_msg_balance_; + bool is_external_; + td::Ref my_code_; + td::Ref unpacked_config_; + td::RefInt256 due_payment_; + td::uint64 precompiled_gas_usage_; + + td::Ref c4_; + td::Ref c5_ = vm::CellBuilder().finalize_novm(); + + void send_raw_message(const td::Ref& msg, int mode); + void raw_reserve(const td::RefInt256& amount, int mode); + + td::RefInt256 get_compute_fee(ton::WorkchainId wc, td::uint64 gas_used); + td::RefInt256 get_forward_fee(ton::WorkchainId wc, td::uint64 bits, td::uint64 cells); + td::RefInt256 get_storage_fee(ton::WorkchainId wc, td::uint64 duration, td::uint64 bits, td::uint64 cells); + td::RefInt256 get_simple_compute_fee(ton::WorkchainId wc, td::uint64 gas_used); + td::RefInt256 get_simple_forward_fee(ton::WorkchainId wc, td::uint64 bits, td::uint64 cells); + td::RefInt256 get_original_fwd_fee(ton::WorkchainId wc, const td::RefInt256& x); + + virtual Result do_run() = 0; +}; + +std::unique_ptr get_implementation(td::Bits256 code_hash); +void set_precompiled_execution_enabled(bool value); // disabled by default + +} // namespace block::precompiled \ No newline at end of file diff --git a/crypto/block/precompiled-smc/common.h b/crypto/block/precompiled-smc/common.h new file mode 100644 index 00000000..0406d7de --- /dev/null +++ b/crypto/block/precompiled-smc/common.h @@ -0,0 +1,21 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "PrecompiledSmartContract.h" +#include "vm/arithops.h" +#include "vm/cellops.h" +#include "vm/tonops.h" \ No newline at end of file diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index adda48a5..34d23511 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -20,6 +20,7 @@ #include "block/block.h" #include "block/block-parse.h" #include "block/block-auto.h" +#include "crypto/openssl/rand.hpp" #include "td/utils/bits.h" #include "td/utils/uint128.h" #include "ton/ton-shard.h" @@ -27,9 +28,20 @@ #include "td/utils/Timer.h" namespace { +/** + * Logger that stores the tail of log messages. + * + * @param max_size The size of the buffer. Default is 256. + */ class StringLoggerTail : public td::LogInterface { public: explicit StringLoggerTail(size_t max_size = 256) : buf(max_size, '\0') {} + + /** + * Appends a slice of data to the buffer. + * + * @param slice The slice of data to be appended. + */ void append(td::CSlice slice) override { if (slice.size() > buf.size()) { slice.remove_prefix(slice.size() - buf.size()); @@ -45,6 +57,12 @@ class StringLoggerTail : public td::LogInterface { slice.remove_prefix(s); } } + + /** + * Retrieves the tail of the log. + * + * @returns The log as std::string. + */ std::string get_log() const { if (truncated) { std::string res = buf; @@ -54,6 +72,7 @@ class StringLoggerTail : public td::LogInterface { return buf.substr(0, pos); } } + private: std::string buf; size_t pos = 0; @@ -64,6 +83,13 @@ class StringLoggerTail : public td::LogInterface { namespace block { using td::Ref; +/** + * Looks up a library among public libraries. + * + * @param key A constant bit pointer representing the key of the library to lookup. + * + * @returns A reference to the library cell if found, null otherwise. + */ Ref ComputePhaseConfig::lookup_library(td::ConstBitPtr key) const { return libraries ? vm::lookup_library_in(key, libraries->get_root_cell()) : Ref{}; } @@ -74,12 +100,27 @@ Ref ComputePhaseConfig::lookup_library(td::ConstBitPtr key) const { * */ +/** + * Sets the address of the account. + * + * @param wc The workchain ID of the account. + * @param new_addr The new address of the account. + * + * @returns True if the address was successfully set, false otherwise. + */ bool Account::set_address(ton::WorkchainId wc, td::ConstBitPtr new_addr) { workchain = wc; addr = new_addr; return true; } +/** + * Sets the split depth of the account. + * + * @param new_split_depth The new split depth value to be set. + * + * @returns True if the split depth was successfully set, False otherwise. + */ bool Account::set_split_depth(int new_split_depth) { if (new_split_depth < 0 || new_split_depth > 30) { return false; // invalid value for split_depth @@ -93,11 +134,26 @@ bool Account::set_split_depth(int new_split_depth) { } } +/** + * Checks if the given split depth is valid for the Account. + * + * @param split_depth The split depth to be checked. + * + * @returns True if the split depth is valid, False otherwise. + */ bool Account::check_split_depth(int split_depth) const { return split_depth_set_ ? (split_depth == split_depth_) : (split_depth >= 0 && split_depth <= 30); } -// initializes split_depth and addr_rewrite +/** + * Parses anycast data of the account address. + * + * Initializes split_depth and addr_rewrite. + * + * @param cs The cell slice containing partially-parsed account addressa. + * + * @returns True if parsing was successful, false otherwise. + */ bool Account::parse_maybe_anycast(vm::CellSlice& cs) { int t = (int)cs.fetch_ulong(1); if (t < 0) { @@ -112,6 +168,13 @@ bool Account::parse_maybe_anycast(vm::CellSlice& cs) { && set_split_depth(depth); } +/** + * Stores the anycast information to a serialized account address. + * + * @param cb The vm::CellBuilder object to store the information in. + * + * @returns True if the anycast information was successfully stored, false otherwise. + */ bool Account::store_maybe_anycast(vm::CellBuilder& cb) const { if (!split_depth_set_ || !split_depth_) { return cb.store_bool_bool(false); @@ -121,6 +184,13 @@ bool Account::store_maybe_anycast(vm::CellBuilder& cb) const { && cb.store_bits_bool(addr_rewrite.cbits(), split_depth_); // rewrite_pfx:(bits depth) } +/** + * Unpacks the address from a given CellSlice. + * + * @param addr_cs The CellSlice containing the address. + * + * @returns True if the address was successfully unpacked, False otherwise. + */ bool Account::unpack_address(vm::CellSlice& addr_cs) { int addr_tag = block::gen::t_MsgAddressInt.get_tag(addr_cs); int new_wc = ton::workchainInvalid; @@ -171,6 +241,15 @@ bool Account::unpack_address(vm::CellSlice& addr_cs) { return true; } +/** + * Unpacks storage information from a CellSlice. + * + * Storage information is serialized using StorageInfo TLB-scheme. + * + * @param cs The CellSlice containing the storage information. + * + * @returns True if the unpacking is successful, false otherwise. + */ bool Account::unpack_storage_info(vm::CellSlice& cs) { block::gen::StorageInfo::Record info; block::gen::StorageUsed::Record used; @@ -197,7 +276,16 @@ bool Account::unpack_storage_info(vm::CellSlice& cs) { return (u != std::numeric_limits::max()); } -// initializes split_depth (from account state - StateInit) +/** + * Unpacks the state of an Account from a CellSlice. + * + * State is serialized using StateInit TLB-scheme. + * Initializes split_depth (from account state - StateInit) + * + * @param cs The CellSlice containing the serialized state. + * + * @returns True if the state was successfully unpacked, False otherwise. + */ bool Account::unpack_state(vm::CellSlice& cs) { block::gen::StateInit::Record state; if (!tlb::unpack_exact(cs, state)) { @@ -225,6 +313,13 @@ bool Account::unpack_state(vm::CellSlice& cs) { return true; } +/** + * Computes the address of the account. + * + * @param force If set to true, the address will be recomputed even if it already exists. + * + * @returns True if the address was successfully computed, false otherwise. + */ bool Account::compute_my_addr(bool force) { if (!force && my_addr.not_null() && my_addr_exact.not_null()) { return true; @@ -265,6 +360,15 @@ bool Account::compute_my_addr(bool force) { return true; } +/** + * Computes the address of the Account. + * + * @param tmp_addr A reference to the CellSlice for the result. + * @param split_depth The split depth for the address. + * @param orig_addr_rewrite Address prefox of length split_depth. + * + * @returns True if the address was successfully computed, false otherwise. + */ bool Account::recompute_tmp_addr(Ref& tmp_addr, int split_depth, td::ConstBitPtr orig_addr_rewrite) const { if (!split_depth && my_addr_exact.not_null()) { @@ -306,6 +410,14 @@ bool Account::recompute_tmp_addr(Ref& tmp_addr, int split_depth, (tmp_addr = vm::load_cell_slice_ref(std::move(cell))).not_null(); } +/** + * Sets address rewriting info for a newly-activated account. + * + * @param split_depth The split depth for the account address. + * @param orig_addr_rewrite Address frepix of length split_depth. + * + * @returns True if the rewriting info was successfully set, false otherwise. + */ bool Account::init_rewrite_addr(int split_depth, td::ConstBitPtr orig_addr_rewrite) { if (split_depth_set_ || !set_split_depth(split_depth)) { return false; @@ -316,16 +428,28 @@ bool Account::init_rewrite_addr(int split_depth, td::ConstBitPtr orig_addr_rewri return compute_my_addr(true); } -// used to unpack previously existing accounts -bool Account::unpack(Ref shard_account, Ref extra, ton::UnixTime now, bool special) { +/** + * Unpacks the account information from the provided CellSlice. + * + * Used to unpack previously existing accounts. + * + * @param shard_account The ShardAccount to unpack. + * @param now The current Unix time. + * @param special Flag indicating if the account is special. + * + * @returns True if the unpacking is successful, false otherwise. + */ +bool Account::unpack(Ref shard_account, ton::UnixTime now, bool special) { LOG(DEBUG) << "unpacking " << (special ? "special " : "") << "account " << addr.to_hex(); if (shard_account.is_null()) { LOG(ERROR) << "account " << addr.to_hex() << " does not have a valid ShardAccount to unpack"; return false; } if (verbosity > 2) { - shard_account->print_rec(std::cerr, 2); - block::gen::t_ShardAccount.print(std::cerr, *shard_account); + FLOG(INFO) { + shard_account->print_rec(sb, 2); + block::gen::t_ShardAccount.print(sb, shard_account); + }; } block::gen::ShardAccount::Record acc_info; if (!(block::tlb::t_ShardAccount.validate_csr(shard_account) && tlb::unpack_exact(shard_account.write(), acc_info))) { @@ -385,7 +509,13 @@ bool Account::unpack(Ref shard_account, Ref extra, return true; } -// used to initialize new accounts +/** + * Initializes a new Account object. + * + * @param now The current Unix time. + * + * @returns True if the initialization is successful, false otherwise. + */ bool Account::init_new(ton::UnixTime now) { // only workchain and addr are initialized at this point if (workchain == ton::workchainInvalid) { @@ -428,6 +558,11 @@ bool Account::init_new(ton::UnixTime now) { return true; } +/** + * Resets the split depth of the account. + * + * @returns True if the split depth was successfully reset, false otherwise. + */ bool Account::forget_split_depth() { split_depth_set_ = false; split_depth_ = 0; @@ -437,6 +572,11 @@ bool Account::forget_split_depth() { return true; } +/** + * Deactivates the account. + * + * @returns True if the account was successfully deactivated, false otherwise. + */ bool Account::deactivate() { if (status == acc_active) { return false; @@ -460,10 +600,26 @@ bool Account::deactivate() { return true; } +/** + * Checks if the account belongs to a specific shard. + * + * @param shard The shard to check against. + * + * @returns True if the account belongs to the shard, False otherwise. + */ bool Account::belongs_to_shard(ton::ShardIdFull shard) const { return workchain == shard.workchain && ton::shard_is_ancestor(shard.shard, addr); } +/** + * Adds the partial storage payment to the total sum. + * + * @param payment The total sum to be updated. + * @param delta The time delta for which the payment is calculated. + * @param prices The storage prices. + * @param storage Account storage statistics. + * @param is_mc A flag indicating whether the account is in the masterchain. + */ void add_partial_storage_payment(td::BigInt256& payment, ton::UnixTime delta, const block::StoragePrices& prices, const vm::CellStorageStat& storage, bool is_mc) { td::BigInt256 c{(long long)storage.cells}, b{(long long)storage.bits}; @@ -477,16 +633,28 @@ void add_partial_storage_payment(td::BigInt256& payment, ton::UnixTime delta, co b.mul_short(prices.bit_price); } b += c; - b.mul_short(delta); + b.mul_short(delta).normalize(); CHECK(b.sgn() >= 0); payment += b; } +/** + * Computes the storage fees based on the given parameters. + * + * @param now The current Unix time. + * @param pricing The vector of storage prices. + * @param storage_stat Account storage statistics. + * @param last_paid The Unix time when the last payment was made. + * @param is_special A flag indicating if the account is special. + * @param is_masterchain A flag indicating if the account is in the masterchain. + * + * @returns The computed storage fees as RefInt256. + */ td::RefInt256 StoragePrices::compute_storage_fees(ton::UnixTime now, const std::vector& pricing, const vm::CellStorageStat& storage_stat, ton::UnixTime last_paid, bool is_special, bool is_masterchain) { if (now <= last_paid || !last_paid || is_special || pricing.empty() || now <= pricing[0].valid_since) { - return {}; + return td::zero_refint(); } std::size_t n = pricing.size(), i = n; while (i && pricing[i - 1].valid_since > last_paid) { @@ -505,14 +673,33 @@ td::RefInt256 StoragePrices::compute_storage_fees(ton::UnixTime now, const std:: } upto = valid_until; } - total.unique_write().rshift(16, 1); // divide by 2^16 with ceil rounding to obtain nanograms - return total; + return td::rshift(total, 16, 1); // divide by 2^16 with ceil rounding to obtain nanograms } +/** + * Computes the storage fees for the account. + * + * @param now The current Unix time. + * @param pricing The vector of storage prices. + * + * @returns The computed storage fees as RefInt256. + */ td::RefInt256 Account::compute_storage_fees(ton::UnixTime now, const std::vector& pricing) const { return StoragePrices::compute_storage_fees(now, pricing, storage_stat, last_paid, is_special, is_masterchain()); } +namespace transaction { +/** + * Constructs a new Transaction object. + * + * @param _account The Account object. + * @param ttype The type of the transaction (see transaction.cpp#309). + * @param req_start_lt The minimal logical time of the transaction. + * @param _now The current Unix time. + * @param _inmsg The input message that caused the transaction. + * + * @returns None + */ Transaction::Transaction(const Account& _account, int ttype, ton::LogicalTime req_start_lt, ton::UnixTime _now, Ref _inmsg) : trans_type(ttype) @@ -539,14 +726,24 @@ Transaction::Transaction(const Account& _account, int ttype, ton::LogicalTime re } } +/** + * Unpacks the input message of a transaction. + * + * @param ihr_delivered A boolean indicating whether the message was delivered using IHR (Instant Hypercube Routing). + * @param cfg Action phase configuration. + * + * @returns A boolean indicating whether the unpacking was successful. + */ bool Transaction::unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* cfg) { if (in_msg.is_null() || in_msg_type) { return false; } if (verbosity > 2) { - fprintf(stderr, "unpacking inbound message for a new transaction: "); - block::gen::t_Message_Any.print_ref(std::cerr, in_msg); - load_cell_slice(in_msg).print_rec(std::cerr); + FLOG(INFO) { + sb << "unpacking inbound message for a new transaction: "; + block::gen::t_Message_Any.print_ref(sb, in_msg); + load_cell_slice(in_msg).print_rec(sb); + }; } auto cs = vm::load_cell_slice(in_msg); int tag = block::gen::t_CommonMsgInfo.get_tag(cs); @@ -588,15 +785,19 @@ bool Transaction::unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* in_msg_type = 2; in_msg_extern = true; // compute forwarding fees for this external message - vm::CellStorageStat sstat; // for message size - sstat.compute_used_storage(cs); // message body - sstat.bits -= cs.size(); // bits in the root cells are free - sstat.cells--; // the root cell itself is not counted as a cell + vm::CellStorageStat sstat; // for message size + auto cell_info = sstat.compute_used_storage(cs).move_as_ok(); // message body + sstat.bits -= cs.size(); // bits in the root cells are free + sstat.cells--; // the root cell itself is not counted as a cell LOG(DEBUG) << "storage paid for a message: " << sstat.cells << " cells, " << sstat.bits << " bits"; if (sstat.bits > cfg->size_limits.max_msg_bits || sstat.cells > cfg->size_limits.max_msg_cells) { LOG(DEBUG) << "inbound external message too large, invalid"; return false; } + if (cell_info.max_merkle_depth > max_allowed_merkle_depth) { + LOG(DEBUG) << "inbound external message has too big merkle depth, invalid"; + return false; + } // fetch message pricing info CHECK(cfg); const MsgPrices& msg_prices = cfg->fetch_msg_prices(account.is_masterchain()); @@ -629,14 +830,15 @@ bool Transaction::unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* vm::CellBuilder cb; if (!(cs.advance(2) && block::gen::t_StateInit.fetch_to(cs, state_init) && cb.append_cellslice_bool(std::move(state_init)) && cb.finalize_to(in_msg_state) && - block::gen::t_StateInit.validate_ref(in_msg_state))) { + block::gen::t_StateInitWithLibs.validate_ref(in_msg_state))) { LOG(DEBUG) << "cannot parse StateInit in inbound message"; return false; } break; } case 3: { // (just$1 (right$1 _:^StateInit )) - if (!(cs.advance(2) && cs.fetch_ref_to(in_msg_state) && block::gen::t_StateInit.validate_ref(in_msg_state))) { + if (!(cs.advance(2) && cs.fetch_ref_to(in_msg_state) && + block::gen::t_StateInitWithLibs.validate_ref(in_msg_state))) { LOG(DEBUG) << "cannot parse ^StateInit in inbound message"; return false; } @@ -665,14 +867,29 @@ bool Transaction::unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* return false; } total_fees += in_fwd_fee; + if (account.workchain == ton::masterchainId && cfg->mc_blackhole_addr && + cfg->mc_blackhole_addr.value() == account.addr) { + blackhole_burned.grams = msg_balance_remaining.grams; + msg_balance_remaining.grams = td::zero_refint(); + LOG(DEBUG) << "Burning " << blackhole_burned.grams << " nanoton (blackhole address)"; + } return true; } +/** + * Prepares the storage phase of a transaction. + * + * @param cfg The configuration for the storage phase. + * @param force_collect Flag indicating whether to collect fees for frozen accounts. + * @param adjust_msg_value Flag indicating whether to adjust the message value if the account balance becomes less than the message balance. + * + * @returns True if the storage phase was successfully prepared, false otherwise. + */ bool Transaction::prepare_storage_phase(const StoragePhaseConfig& cfg, bool force_collect, bool adjust_msg_value) { if (now < account.last_paid) { return false; } - auto to_pay = account.compute_storage_fees(now, *(cfg.pricing)); + auto to_pay = account.compute_storage_fees(now, *(cfg.pricing)) + due_payment; if (to_pay.not_null() && sgn(to_pay) < 0) { return false; } @@ -685,7 +902,10 @@ bool Transaction::prepare_storage_phase(const StoragePhaseConfig& cfg, bool forc res->fees_collected = to_pay; res->fees_due = td::zero_refint(); balance -= std::move(to_pay); - } else if (acc_status == Account::acc_frozen && !force_collect && to_pay + due_payment < cfg.delete_due_limit) { + if (cfg.global_version >= 7) { + due_payment = td::zero_refint(); + } + } else if (acc_status == Account::acc_frozen && !force_collect && to_pay < cfg.delete_due_limit) { // do not collect fee res->last_paid_updated = (res->is_special ? 0 : account.last_paid); res->fees_collected = res->fees_due = td::zero_refint(); @@ -694,11 +914,13 @@ bool Transaction::prepare_storage_phase(const StoragePhaseConfig& cfg, bool forc res->fees_due = std::move(to_pay) - std::move(balance.grams); balance.grams = td::zero_refint(); if (!res->is_special) { - auto total_due = res->fees_due + due_payment; + auto total_due = res->fees_due; switch (acc_status) { case Account::acc_uninit: case Account::acc_frozen: - if (total_due > cfg.delete_due_limit) { + if (total_due > cfg.delete_due_limit && balance.extra.is_null()) { + // Keeping accounts with non-null extras is a temporary measure before implementing proper collection of + // extracurrencies from deleted accounts res->deleted = true; acc_status = Account::acc_deleted; if (balance.extra.not_null()) { @@ -716,6 +938,9 @@ bool Transaction::prepare_storage_phase(const StoragePhaseConfig& cfg, bool forc } break; } + if (cfg.enable_due_payment) { + due_payment = total_due; + } } } if (adjust_msg_value && msg_balance_remaining.grams > balance.grams) { @@ -726,12 +951,25 @@ bool Transaction::prepare_storage_phase(const StoragePhaseConfig& cfg, bool forc return true; } +/** + * Prepares the credit phase of a transaction. + * + * This function creates a CreditPhase object and performs the necessary calculations + * to determine the amount to be credited in the credit phase. It updates the due payment, + * credit, balance, and total fees accordingly. + * + * @returns True if the credit phase is prepared successfully, false otherwise. + */ bool Transaction::prepare_credit_phase() { credit_phase = std::make_unique(); - auto collected = std::min(msg_balance_remaining.grams, due_payment); - credit_phase->due_fees_collected = collected; - due_payment -= collected; - credit_phase->credit = msg_balance_remaining -= collected; + // Due payment is only collected in storage phase. + // For messages with bounce flag, contract always receives the amount specified in message + // auto collected = std::min(msg_balance_remaining.grams, due_payment); + // credit_phase->due_fees_collected = collected; + // due_payment -= collected; + // credit_phase->credit = msg_balance_remaining -= collected; + credit_phase->due_fees_collected = td::zero_refint(); + credit_phase->credit = msg_balance_remaining; if (!msg_balance_remaining.is_valid()) { LOG(ERROR) << "cannot compute the amount to be credited in the credit phase of transaction"; return false; @@ -742,16 +980,35 @@ bool Transaction::prepare_credit_phase() { LOG(ERROR) << "cannot credit currency collection to account"; return false; } - total_fees += std::move(collected); + // total_fees += std::move(collected); return true; } +} // namespace transaction +/** + * Parses the gas limits and prices from a given cell. + * + * @param cell The cell containing the gas limits and prices serialized using GasLimitsPricing TLB-scheme. + * @param freeze_due_limit Reference to store the freeze due limit. + * @param delete_due_limit Reference to store the delete due limit. + * + * @returns True if the parsing is successful, false otherwise. + */ bool ComputePhaseConfig::parse_GasLimitsPrices(Ref cell, td::RefInt256& freeze_due_limit, td::RefInt256& delete_due_limit) { return cell.not_null() && parse_GasLimitsPrices(vm::load_cell_slice_ref(std::move(cell)), freeze_due_limit, delete_due_limit); } +/** + * Parses the gas limits and prices from a given cell slice. + * + * @param cs The cell slice containing the gas limits and prices serialized using GasLimitsPricing TLB-scheme. + * @param freeze_due_limit Reference to store the freeze due limit. + * @param delete_due_limit Reference to store the delete due limit. + * + * @returns True if the parsing is successful, false otherwise. + */ bool ComputePhaseConfig::parse_GasLimitsPrices(Ref cs, td::RefInt256& freeze_due_limit, td::RefInt256& delete_due_limit) { if (cs.is_null()) { @@ -766,6 +1023,17 @@ bool ComputePhaseConfig::parse_GasLimitsPrices(Ref cs, td::RefInt } } +/** + * Parses the gas limits and prices from a gas limits and prices record. + * + * @param cs The cell slice containing the gas limits and prices serialized using GasLimitsPricing TLB-scheme. + * @param freeze_due_limit A reference to store the freeze due limit. + * @param delete_due_limit A reference to store the delete due limit. + * @param _flat_gas_limit The flat gas limit. + * @param _flat_gas_price The flat gas price. + * + * @returns True if the parsing is successful, false otherwise. + */ bool ComputePhaseConfig::parse_GasLimitsPrices_internal(Ref cs, td::RefInt256& freeze_due_limit, td::RefInt256& delete_due_limit, td::uint64 _flat_gas_limit, td::uint64 _flat_gas_price) { @@ -794,6 +1062,14 @@ bool ComputePhaseConfig::parse_GasLimitsPrices_internal(Ref cs, t return true; } +/** + * Checks if an address is suspended according to the ConfigParam(44). + * + * @param wc The workchain ID. + * @param addr The account address address. + * + * @returns True if the address is suspended, False otherwise. + */ bool ComputePhaseConfig::is_address_suspended(ton::WorkchainId wc, td::Bits256 addr) const { if (!suspended_addresses) { return false; @@ -808,16 +1084,42 @@ bool ComputePhaseConfig::is_address_suspended(ton::WorkchainId wc, td::Bits256 a } } -void ComputePhaseConfig::compute_threshold() { - gas_price256 = td::make_refint(gas_price); +/** + * Computes the maximum gas fee based on the gas prices and limits. + * + * @param gas_price256 The gas price from config as RefInt256 + * @param gas_limit The gas limit from config + * @param flat_gas_limit The flat gas limit from config + * @param flat_gas_price The flat gas price from config + * + * @returns The maximum gas fee. + */ +static td::RefInt256 compute_max_gas_threshold(const td::RefInt256& gas_price256, td::uint64 gas_limit, + td::uint64 flat_gas_limit, td::uint64 flat_gas_price) { if (gas_limit > flat_gas_limit) { - max_gas_threshold = - td::rshift(gas_price256 * (gas_limit - flat_gas_limit), 16, 1) + td::make_bigint(flat_gas_price); + return td::rshift(gas_price256 * (gas_limit - flat_gas_limit), 16, 1) + td::make_bigint(flat_gas_price); } else { - max_gas_threshold = td::make_refint(flat_gas_price); + return td::make_refint(flat_gas_price); } } +/** + * Computes the maximum for gas fee based on the gas prices and limits. + * + * Updates max_gas_threshold. + */ +void ComputePhaseConfig::compute_threshold() { + gas_price256 = td::make_refint(gas_price); + max_gas_threshold = compute_max_gas_threshold(gas_price256, gas_limit, flat_gas_limit, flat_gas_price); +} + +/** + * Computes the amount of gas that can be bought for a given amount of nanograms. + * + * @param nanograms The amount of nanograms to compute gas for. + * + * @returns The amount of gas. + */ td::uint64 ComputePhaseConfig::gas_bought_for(td::RefInt256 nanograms) const { if (nanograms.is_null() || sgn(nanograms) < 0) { return 0; @@ -832,35 +1134,158 @@ td::uint64 ComputePhaseConfig::gas_bought_for(td::RefInt256 nanograms) const { return res->to_long() + flat_gas_limit; } +/** + * Computes the gas price. + * + * @param gas_used The amount of gas used. + * + * @returns The computed gas price. + */ td::RefInt256 ComputePhaseConfig::compute_gas_price(td::uint64 gas_used) const { return gas_used <= flat_gas_limit ? td::make_refint(flat_gas_price) : td::rshift(gas_price256 * (gas_used - flat_gas_limit), 16, 1) + flat_gas_price; } +namespace transaction { + +/** + * Checks if it is required to increase gas_limit (from GasLimitsPrices config) for the transaction + * + * In January 2024 a highload wallet of @wallet Telegram bot in mainnet was stuck because current gas limit (1M) is + * not enough to clean up old queires, thus locking funds inside. + * See comment in crypto/smartcont/highload-wallet-v2-code.fc for details on why this happened. + * Account address: EQD_v9j1rlsuHHw2FIhcsCFFSD367ldfDdCKcsNmNpIRzUlu + * It was proposed to validators to increase gas limit for this account to 70M for a limited amount + * of time (until 2024-02-29). + * It is activated by setting global version to 5 in ConfigParam 8. + * This config change also activates new behavior for special accounts in masterchain. + * + * In August 2024 it was decided to unlock other old highload wallets that got into the same situation. + * See https://t.me/tondev_news/129 + * It is activated by setting global version to 9. + * + * @param cfg The compute phase configuration. + * @param now The Unix time of the transaction. + * @param account The account of the transaction. + * + * @returns Overridden gas limit or empty td::optional + */ +static td::optional override_gas_limit(const ComputePhaseConfig& cfg, ton::UnixTime now, + const Account& account) { + struct OverridenGasLimit { + td::uint64 new_limit; + int from_version; + ton::UnixTime until; + }; + static std::map, OverridenGasLimit> accounts = []() { + auto parse_addr = [](const char* s) -> std::pair { + auto r_addr = StdAddress::parse(td::Slice(s)); + r_addr.ensure(); + return {r_addr.ok().workchain, r_addr.ok().addr}; + }; + std::map, OverridenGasLimit> accounts; + + // Increase limit for EQD_v9j1rlsuHHw2FIhcsCFFSD367ldfDdCKcsNmNpIRzUlu until 2024-02-29 00:00:00 UTC + accounts[parse_addr("0:FFBFD8F5AE5B2E1C7C3614885CB02145483DFAEE575F0DD08A72C366369211CD")] = { + .new_limit = 70'000'000, .from_version = 5, .until = 1709164800}; + + // Increase limit for multiple accounts (https://t.me/tondev_news/129) until 2025-03-01 00:00:00 UTC + accounts[parse_addr("UQBeSl-dumOHieZ3DJkNKVkjeso7wZ0VpzR4LCbLGTQ8xr57")] = { + .new_limit = 70'000'000, .from_version = 9, .until = 1740787200}; + accounts[parse_addr("EQC3VcQ-43klww9UfimR58TBjBzk7GPupXQ3CNuthoNp-uTR")] = { + .new_limit = 70'000'000, .from_version = 9, .until = 1740787200}; + accounts[parse_addr("EQBhwBb8jvokGvfreHRRoeVxI237PrOJgyrsAhLA-4rBC_H5")] = { + .new_limit = 70'000'000, .from_version = 9, .until = 1740787200}; + accounts[parse_addr("EQCkoRp4OE-SFUoMEnYfL3vF43T3AzNfW8jyTC4yzk8cJqMS")] = { + .new_limit = 70'000'000, .from_version = 9, .until = 1740787200}; + accounts[parse_addr("UQBN5ICras79U8FYEm71ws34n-ZNIQ0LRNpckOUsIV3OebnC")] = { + .new_limit = 70'000'000, .from_version = 9, .until = 1740787200}; + accounts[parse_addr("EQBDanbCeUqI4_v-xrnAN0_I2wRvEIaLg1Qg2ZN5c6Zl1KOh")] = { + .new_limit = 225'000'000, .from_version = 9, .until = 1740787200}; + return accounts; + }(); + auto it = accounts.find({account.workchain, account.addr}); + if (it == accounts.end() || cfg.global_version < it->second.from_version || now >= it->second.until) { + return {}; + } + return it->second.new_limit; +} + +/** + * Computes the amount of gas that can be bought for a given amount of nanograms. + * Usually equal to `cfg.gas_bought_for(nanograms)` + * However, it overrides gas_limit from config in special cases. + * + * @param cfg The compute phase configuration. + * @param nanograms The amount of nanograms to compute gas for. + * + * @returns The amount of gas. + */ +td::uint64 Transaction::gas_bought_for(const ComputePhaseConfig& cfg, td::RefInt256 nanograms) { + if (auto new_limit = override_gas_limit(cfg, now, account)) { + gas_limit_overridden = true; + // Same as ComputePhaseConfig::gas_bought for, but with other gas_limit and max_gas_threshold + auto gas_limit = new_limit.value(); + LOG(INFO) << "overridding gas limit for account " << account.workchain << ":" << account.addr.to_hex() << " to " + << gas_limit; + auto max_gas_threshold = + compute_max_gas_threshold(cfg.gas_price256, gas_limit, cfg.flat_gas_limit, cfg.flat_gas_price); + if (nanograms.is_null() || sgn(nanograms) < 0) { + return 0; + } + if (nanograms >= max_gas_threshold) { + return gas_limit; + } + if (nanograms < cfg.flat_gas_price) { + return 0; + } + auto res = td::div((std::move(nanograms) - cfg.flat_gas_price) << 16, cfg.gas_price256); + return res->to_long() + cfg.flat_gas_limit; + } + return cfg.gas_bought_for(nanograms); +} + +/** + * Computes the gas limits for a transaction. + * + * @param cp The ComputePhase object to store the computed gas limits. + * @param cfg The compute phase configuration. + * + * @returns True if the gas limits were successfully computed, false otherwise. + */ bool Transaction::compute_gas_limits(ComputePhase& cp, const ComputePhaseConfig& cfg) { // Compute gas limits if (account.is_special) { cp.gas_max = cfg.special_gas_limit; } else { - cp.gas_max = cfg.gas_bought_for(balance.grams); + cp.gas_max = gas_bought_for(cfg, balance.grams); } - cp.gas_credit = 0; - if (trans_type != tr_ord) { + if (trans_type != tr_ord || (account.is_special && cfg.special_gas_full)) { // may use all gas that can be bought using remaining balance cp.gas_limit = cp.gas_max; } else { // originally use only gas bought using remaining message balance // if the message is "accepted" by the smart contract, the gas limit will be set to gas_max - cp.gas_limit = std::min(cfg.gas_bought_for(msg_balance_remaining.grams), cp.gas_max); - if (!block::tlb::t_Message.is_internal(in_msg)) { - // external messages carry no balance, give them some credit to check whether they are accepted - cp.gas_credit = std::min(cfg.gas_credit, cp.gas_max); - } + cp.gas_limit = std::min(gas_bought_for(cfg, msg_balance_remaining.grams), cp.gas_max); + } + if (trans_type == tr_ord && !block::tlb::t_Message.is_internal(in_msg)) { + // external messages carry no balance, give them some credit to check whether they are accepted + cp.gas_credit = std::min(cfg.gas_credit, cp.gas_max); + } else { + cp.gas_credit = 0; } LOG(DEBUG) << "gas limits: max=" << cp.gas_max << ", limit=" << cp.gas_limit << ", credit=" << cp.gas_credit; return true; } +/** + * Prepares a TVM stack for a transaction. + * + * @param cp The compute phase object. + * + * @returns A reference to the prepared virtual machine stack. + * Returns an empty reference if the transaction type is invalid. + */ Ref Transaction::prepare_vm_stack(ComputePhase& cp) { Ref stack_ref{true}; td::RefInt256 acc_addr{true}; @@ -887,18 +1312,39 @@ Ref Transaction::prepare_vm_stack(ComputePhase& cp) { } } +/** + * Prepares a random seed for a transaction. + * + * @param rand_seed The output random seed. + * @param cfg The configuration for the compute phase. + * + * @returns True if the random seed was successfully prepared, false otherwise. + */ bool Transaction::prepare_rand_seed(td::BitArray<256>& rand_seed, const ComputePhaseConfig& cfg) const { // we might use SHA256(block_rand_seed . addr . trans_lt) // instead, we use SHA256(block_rand_seed . addr) // if the smart contract wants to randomize further, it can use RANDOMIZE instruction td::BitArray<256 + 256> data; data.bits().copy_from(cfg.block_rand_seed.cbits(), 256); - (data.bits() + 256).copy_from(account.addr_rewrite.cbits(), 256); + if (cfg.global_version >= 8) { + (data.bits() + 256).copy_from(account.addr.cbits(), 256); + } else { + (data.bits() + 256).copy_from(account.addr_rewrite.cbits(), 256); + } rand_seed.clear(); data.compute_sha256(rand_seed); return true; } +/** + * Prepares the c7 tuple (virtual machine context) for a compute phase of a transaction. + * + * @param cfg The configuration for the compute phase. + * + * @returns A reference to a Tuple object. + * + * @throws CollatorError if the rand_seed cannot be computed for the transaction. + */ Ref Transaction::prepare_vm_c7(const ComputePhaseConfig& cfg) const { td::BitArray<256> rand_seed; td::RefInt256 rand_seed_int{true}; @@ -907,7 +1353,7 @@ Ref Transaction::prepare_vm_c7(const ComputePhaseConfig& cfg) const { throw CollatorError{"cannot generate valid SmartContractInfo"}; return {}; } - auto tuple = vm::make_tuple_ref( + std::vector tuple = { td::make_refint(0x076ef1ea), // [ magic:0x076ef1ea td::zero_refint(), // actions:Integer td::zero_refint(), // msgs_sent:Integer @@ -916,22 +1362,73 @@ Ref Transaction::prepare_vm_c7(const ComputePhaseConfig& cfg) const { td::make_refint(start_lt), // trans_lt:Integer std::move(rand_seed_int), // rand_seed:Integer balance.as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)] - my_addr, // myself:MsgAddressInt - vm::StackEntry::maybe(cfg.global_config)); // global_config:(Maybe Cell) ] = SmartContractInfo; - LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple).to_string(); - return vm::make_tuple_ref(std::move(tuple)); + my_addr, // myself:MsgAddressInt + vm::StackEntry::maybe(cfg.global_config) // global_config:(Maybe Cell) ] = SmartContractInfo; + }; + if (cfg.global_version >= 4) { + tuple.push_back(vm::StackEntry::maybe(new_code)); // code:Cell + if (msg_balance_remaining.is_valid()) { + tuple.push_back(msg_balance_remaining.as_vm_tuple()); // in_msg_value:[Integer (Maybe Cell)] + } else { + tuple.push_back(block::CurrencyCollection::zero().as_vm_tuple()); + } + tuple.push_back(storage_phase->fees_collected); // storage_fees:Integer + + // See crypto/block/mc-config.cpp#2223 (get_prev_blocks_info) + // [ wc:Integer shard:Integer seqno:Integer root_hash:Integer file_hash:Integer] = BlockId; + // [ last_mc_blocks:[BlockId...] + // prev_key_block:BlockId + // last_mc_blocks_100:[BlockId...] ] : PrevBlocksInfo + // The only context where PrevBlocksInfo (13 parameter of c7) is null is inside emulator + // where it need to be set via transaction_emulator_set_prev_blocks_info (see emulator/emulator-extern.cpp) + // Inside validator, collator and liteserver checking external message contexts + // prev_blocks_info is always not null, since get_prev_blocks_info() + // may only return tuple or raise Error (See crypto/block/mc-config.cpp#2223) + tuple.push_back(vm::StackEntry::maybe(cfg.prev_blocks_info)); + } + if (cfg.global_version >= 6) { + tuple.push_back(vm::StackEntry::maybe(cfg.unpacked_config_tuple)); // unpacked_config_tuple:[...] + tuple.push_back(due_payment.not_null() ? due_payment : td::zero_refint()); // due_payment:Integer + tuple.push_back(compute_phase->precompiled_gas_usage + ? vm::StackEntry(td::make_refint(compute_phase->precompiled_gas_usage.value())) + : vm::StackEntry()); // precompiled_gas_usage:Integer + } + auto tuple_ref = td::make_cnt_ref>(std::move(tuple)); + LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple_ref).to_string(); + return vm::make_tuple_ref(std::move(tuple_ref)); } +/** + * Computes the number of output actions in a list. + * + * @param list c5 cell. + * + * @returns The number of output actions. + */ int output_actions_count(Ref list) { int i = -1; do { ++i; - list = load_cell_slice(std::move(list)).prefetch_ref(); + bool special = true; + auto cs = load_cell_slice_special(std::move(list), special); + if (special) { + break; + } + list = cs.prefetch_ref(); } while (list.not_null()); return i; } -bool Transaction::unpack_msg_state(bool lib_only) { +/** + * Unpacks the message StateInit. + * + * @param cfg The configuration for the compute phase. + * @param lib_only If true, only unpack libraries from the state. + * @param forbid_public_libs Don't allow public libraries in initstate. + * + * @returns True if the unpacking is successful, false otherwise. + */ +bool Transaction::unpack_msg_state(const ComputePhaseConfig& cfg, bool lib_only, bool forbid_public_libs) { block::gen::StateInit::Record state; if (in_msg_state.is_null() || !tlb::unpack_cell(in_msg_state, state)) { LOG(ERROR) << "cannot unpack StateInit from an inbound message"; @@ -955,12 +1452,32 @@ bool Transaction::unpack_msg_state(bool lib_only) { new_tock = z & 1; LOG(DEBUG) << "tick=" << new_tick << ", tock=" << new_tock; } + td::Ref old_code = new_code, old_data = new_data, old_library = new_library; new_code = state.code->prefetch_ref(); new_data = state.data->prefetch_ref(); new_library = state.library->prefetch_ref(); + auto size_limits = cfg.size_limits; + if (forbid_public_libs) { + size_limits.max_acc_public_libraries = 0; + } + auto S = check_state_limits(size_limits, false); + if (S.is_error()) { + LOG(DEBUG) << "Cannot unpack msg state: " << S.move_as_error(); + new_code = old_code; + new_data = old_data; + new_library = old_library; + return false; + } return true; } +/** + * Computes the set of libraries to be used during TVM execution. + * + * @param cfg The configuration for the compute phase. + * + * @returns A vector of hashmaps with libraries. + */ std::vector> Transaction::compute_vm_libraries(const ComputePhaseConfig& cfg) { std::vector> lib_set; if (in_msg_library.not_null()) { @@ -976,6 +1493,11 @@ std::vector> Transaction::compute_vm_libraries(const ComputePhaseC return lib_set; } +/** + * Checks if the input message StateInit hash corresponds to the account address. + * + * @returns True if the input message state hash is valid, False otherwise. + */ bool Transaction::check_in_msg_state_hash() { CHECK(in_msg_state.not_null()); CHECK(new_split_depth >= 0 && new_split_depth < 32); @@ -989,12 +1511,102 @@ bool Transaction::check_in_msg_state_hash() { return account.recompute_tmp_addr(my_addr, d, orig_addr_rewrite.bits()); } +/** + * Runs the precompiled smart contract and prepares the compute phase. + * + * @param cfg The configuration for the compute phase. + * @param impl Implementation of the smart contract + * + * @returns True if the contract was successfully executed, false otherwise. + */ +bool Transaction::run_precompiled_contract(const ComputePhaseConfig& cfg, precompiled::PrecompiledSmartContract& impl) { + ComputePhase& cp = *compute_phase; + CHECK(cp.precompiled_gas_usage); + td::uint64 gas_usage = cp.precompiled_gas_usage.value(); + td::Timer timer; + auto result = + impl.run(my_addr, now, start_lt, balance, new_data, *in_msg_body, in_msg, msg_balance_remaining, in_msg_extern, + compute_vm_libraries(cfg), cfg.global_version, cfg.max_vm_data_depth, new_code, + cfg.unpacked_config_tuple, due_payment.not_null() ? due_payment : td::zero_refint(), gas_usage); + double elapsed = timer.elapsed(); + cp.vm_init_state_hash = td::Bits256::zero(); + cp.exit_code = result.exit_code; + cp.out_of_gas = false; + cp.vm_final_state_hash = td::Bits256::zero(); + cp.vm_steps = 0; + cp.gas_used = gas_usage; + cp.accepted = result.accepted; + cp.success = (cp.accepted && result.committed); + LOG(INFO) << "Running precompiled smart contract " << impl.get_name() << ": exit_code=" << result.exit_code + << " accepted=" << result.accepted << " success=" << cp.success << " gas_used=" << gas_usage + << " time=" << elapsed << "s"; + if (cp.accepted & use_msg_state) { + was_activated = true; + acc_status = Account::acc_active; + } + if (cfg.with_vm_log) { + cp.vm_log = PSTRING() << "Running precompiled smart contract " << impl.get_name() + << ": exit_code=" << result.exit_code << " accepted=" << result.accepted + << " success=" << cp.success << " gas_used=" << gas_usage << " time=" << elapsed << "s"; + } + if (cp.success) { + cp.new_data = impl.get_c4(); + cp.actions = impl.get_c5(); + int out_act_num = output_actions_count(cp.actions); + if (verbosity > 2) { + FLOG(INFO) { + sb << "new smart contract data: "; + bool can_be_special = true; + load_cell_slice_special(cp.new_data, can_be_special).print_rec(sb); + sb << "output actions: "; + block::gen::OutList{out_act_num}.print_ref(sb, cp.actions); + }; + } + } + cp.mode = 0; + cp.exit_arg = 0; + if (!cp.success && result.exit_arg) { + auto value = td::narrow_cast_safe(result.exit_arg.value()); + if (value.is_ok()) { + cp.exit_arg = value.ok(); + } + } + if (cp.accepted) { + if (account.is_special) { + cp.gas_fees = td::zero_refint(); + } else { + cp.gas_fees = cfg.compute_gas_price(cp.gas_used); + total_fees += cp.gas_fees; + balance -= cp.gas_fees; + } + LOG(DEBUG) << "gas fees: " << cp.gas_fees->to_dec_string() << " = " << cfg.gas_price256->to_dec_string() << " * " + << cp.gas_used << " /2^16 ; price=" << cfg.gas_price << "; flat rate=[" << cfg.flat_gas_price << " for " + << cfg.flat_gas_limit << "]; remaining balance=" << balance.to_str(); + CHECK(td::sgn(balance.grams) >= 0); + } + return true; +} + +/** + * Prepares the compute phase of a transaction, which includes running TVM. + * + * @param cfg The configuration for the compute phase. + * + * @returns True if the compute phase was successfully prepared and executed, false otherwise. + */ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { // TODO: add more skip verifications + sometimes use state from in_msg to re-activate // ... compute_phase = std::make_unique(); ComputePhase& cp = *(compute_phase.get()); - original_balance -= total_fees; + if (cfg.global_version >= 9) { + original_balance = balance; + if (msg_balance_remaining.is_valid()) { + original_balance -= msg_balance_remaining; + } + } else { + original_balance -= total_fees; + } if (td::sgn(balance.grams) <= 0) { // no gas cp.skip_reason = ComputePhase::sk_no_gas; @@ -1013,7 +1625,6 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { if (in_msg_state.not_null()) { LOG(DEBUG) << "HASH(in_msg_state) = " << in_msg_state->get_hash().bits().to_hex(256) << ", account_state_hash = " << account.state_hash.to_hex(); - // vm::load_cell_slice(in_msg_state).print_rec(std::cerr); } else { LOG(DEBUG) << "in_msg_state is null"; } @@ -1026,7 +1637,9 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { return true; } use_msg_state = true; - if (!(unpack_msg_state() && account.check_split_depth(new_split_depth))) { + bool forbid_public_libs = + acc_status == Account::acc_uninit && account.is_masterchain(); // Forbid for deploying, allow for unfreezing + if (!(unpack_msg_state(cfg, false, forbid_public_libs) && account.check_split_depth(new_split_depth))) { LOG(DEBUG) << "cannot unpack in_msg_state, or it has bad split_depth; cannot init account state"; cp.skip_reason = ComputePhase::sk_bad_state; return true; @@ -1041,8 +1654,50 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { cp.skip_reason = in_msg_state.not_null() ? ComputePhase::sk_bad_state : ComputePhase::sk_no_state; return true; } else if (in_msg_state.not_null()) { - unpack_msg_state(true); // use only libraries + if (cfg.allow_external_unfreeze) { + if (in_msg_extern && account.addr != in_msg_state->get_hash().bits()) { + // only for external messages with non-zero initstate in active accounts + LOG(DEBUG) << "in_msg_state hash mismatch in external message"; + cp.skip_reason = ComputePhase::sk_bad_state; + return true; + } + } + unpack_msg_state(cfg, true); // use only libraries } + if (!cfg.allow_external_unfreeze) { + if (in_msg_extern && in_msg_state.not_null() && account.addr != in_msg_state->get_hash().bits()) { + LOG(DEBUG) << "in_msg_state hash mismatch in external message"; + cp.skip_reason = ComputePhase::sk_bad_state; + return true; + } + } + + td::optional precompiled; + if (new_code.not_null() && trans_type == tr_ord) { + precompiled = cfg.precompiled_contracts.get_contract(new_code->get_hash().bits()); + } + + vm::GasLimits gas{(long long)cp.gas_limit, (long long)cp.gas_max, (long long)cp.gas_credit}; + if (precompiled) { + td::uint64 gas_usage = precompiled.value().gas_usage; + cp.precompiled_gas_usage = gas_usage; + if (gas_usage > cp.gas_limit) { + cp.skip_reason = ComputePhase::sk_no_gas; + return true; + } + auto impl = precompiled::get_implementation(new_code->get_hash().bits()); + if (impl != nullptr && !cfg.dont_run_precompiled_ && impl->required_version() <= cfg.global_version) { + return run_precompiled_contract(cfg, *impl); + } + + // Contract is marked as precompiled in global config, but implementation is not available + // In this case we run TVM and override gas_used + LOG(INFO) << "Unknown precompiled contract (code_hash=" << new_code->get_hash().to_hex() + << ", gas_usage=" << gas_usage << "), running VM"; + long long limit = account.is_special ? cfg.special_gas_limit : cfg.gas_limit; + gas = vm::GasLimits{limit, limit, gas.gas_credit ? limit : 0}; + } + // initialize VM Ref stack = prepare_vm_stack(cp); if (stack.is_null()) { @@ -1051,19 +1706,39 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { } // OstreamLogger ostream_logger(error_stream); // auto log = create_vm_log(error_stream ? &ostream_logger : nullptr); - vm::GasLimits gas{(long long)cp.gas_limit, (long long)cp.gas_max, (long long)cp.gas_credit}; LOG(DEBUG) << "creating VM"; std::unique_ptr logger; auto vm_log = vm::VmLog(); if (cfg.with_vm_log) { - logger = std::make_unique(); + size_t log_max_size = 256; + if (cfg.vm_log_verbosity > 4) { + log_max_size = 32 << 20; + } else if (cfg.vm_log_verbosity > 0) { + log_max_size = 1 << 20; + } + logger = std::make_unique(log_max_size); vm_log.log_interface = logger.get(); vm_log.log_options = td::LogOptions(VERBOSITY_NAME(DEBUG), true, false); + if (cfg.vm_log_verbosity > 1) { + vm_log.log_mask |= vm::VmLog::ExecLocation; + if (cfg.vm_log_verbosity > 2) { + vm_log.log_mask |= vm::VmLog::GasRemaining; + if (cfg.vm_log_verbosity > 3) { + vm_log.log_mask |= vm::VmLog::DumpStack; + if (cfg.vm_log_verbosity > 4) { + vm_log.log_mask |= vm::VmLog::DumpStackVerbose; + vm_log.log_mask |= vm::VmLog::DumpC5; + } + } + } + } } - vm::VmState vm{new_code, std::move(stack), gas, 1, new_data, vm_log, compute_vm_libraries(cfg)}; + vm::VmState vm{new_code, cfg.global_version, std::move(stack), gas, 1, new_data, vm_log, compute_vm_libraries(cfg)}; vm.set_max_data_depth(cfg.max_vm_data_depth); vm.set_c7(prepare_vm_c7(cfg)); // tuple with SmartContractInfo + vm.set_chksig_always_succeed(cfg.ignore_chksig); + vm.set_stop_on_accept_message(cfg.stop_on_accept_message); // vm.incr_stack_trace(1); // enable stack dump after each step LOG(DEBUG) << "starting VM"; @@ -1084,6 +1759,15 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { was_activated = true; acc_status = Account::acc_active; } + if (precompiled) { + cp.gas_used = precompiled.value().gas_usage; + cp.vm_steps = 0; + cp.vm_init_state_hash = cp.vm_final_state_hash = td::Bits256::zero(); + if (cp.out_of_gas) { + LOG(ERROR) << "Precompiled smc got out_of_gas in TVM"; + return false; + } + } LOG(INFO) << "steps: " << vm.get_steps_count() << " gas: used=" << gas.gas_consumed() << ", max=" << gas.gas_max << ", limit=" << gas.gas_limit << ", credit=" << gas.gas_credit; LOG(INFO) << "out_of_gas=" << cp.out_of_gas << ", accepted=" << cp.accepted << ", success=" << cp.success @@ -1096,10 +1780,13 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { cp.actions = vm.get_committed_state().c5; // c5 -> action list int out_act_num = output_actions_count(cp.actions); if (verbosity > 2) { - std::cerr << "new smart contract data: "; - load_cell_slice(cp.new_data).print_rec(std::cerr); - std::cerr << "output actions: "; - block::gen::OutList{out_act_num}.print_ref(std::cerr, cp.actions); + FLOG(INFO) { + sb << "new smart contract data: "; + bool can_be_special = true; + load_cell_slice_special(cp.new_data, can_be_special).print_rec(sb); + sb << "output actions: "; + block::gen::OutList{out_act_num}.print_ref(sb, cp.actions); + }; } } cp.mode = 0; @@ -1126,6 +1813,13 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { return true; } +/** + * Prepares the action phase of a transaction. + * + * @param cfg The configuration for the action phase. + * + * @returns True if the action phase was prepared successfully, false otherwise. + */ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { if (!compute_phase || !compute_phase->success) { return false; @@ -1143,21 +1837,23 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { ap.total_fwd_fees = td::zero_refint(); ap.total_action_fees = td::zero_refint(); ap.reserved_balance.set_zero(); + ap.action_fine = td::zero_refint(); td::Ref old_code = new_code, old_data = new_data, old_library = new_library; - auto enforce_state_size_limits = [&]() { + auto enforce_state_limits = [&]() { if (account.is_special) { return true; } - if (!check_state_size_limit(cfg)) { + auto S = check_state_limits(cfg.size_limits); + if (S.is_error()) { // Rollback changes to state, fail action phase - LOG(INFO) << "Account state size exceeded limits"; + LOG(INFO) << "Account state size exceeded limits: " << S.move_as_error(); new_storage_stat.clear(); new_code = old_code; new_data = old_data; new_library = old_library; ap.result_code = 50; - ap.state_size_too_big = true; + ap.state_exceeds_limits = true; return false; } return true; @@ -1166,7 +1862,15 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { int n = 0; while (true) { ap.action_list.push_back(list); - auto cs = load_cell_slice(std::move(list)); + bool special = true; + auto cs = load_cell_slice_special(std::move(list), special); + if (special) { + ap.result_code = 32; // action list invalid + ap.result_arg = n; + ap.action_list_invalid = true; + LOG(DEBUG) << "action list invalid: special cell"; + return true; + } if (!cs.size_ext()) { break; } @@ -1193,6 +1897,26 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { for (int i = n - 1; i >= 0; --i) { ap.result_arg = n - 1 - i; if (!block::gen::t_OutListNode.validate_ref(ap.action_list[i])) { + if (cfg.message_skip_enabled) { + // try to read mode from action_send_msg even if out_msg scheme is violated + // action should at least contain 40 bits: 32bit tag and 8 bit mode + // if (mode & 2), that is ignore error mode, skip action even for invalid message + // if there is no (mode & 2) but (mode & 16) presents - enable bounce if possible + bool special = true; + auto cs = load_cell_slice_special(ap.action_list[i], special); + if (!special) { + if ((cs.size() >= 40) && ((int)cs.fetch_ulong(32) == 0x0ec3c86d)) { + int mode = (int)cs.fetch_ulong(8); + if (mode & 2) { + ap.skipped_actions++; + ap.action_list[i] = {}; + continue; + } else if ((mode & 16) && cfg.bounce_on_fail_enabled) { + ap.bounce = true; + } + } + } + } ap.result_code = 34; // action #i invalid or unsupported ap.action_list_invalid = true; LOG(DEBUG) << "invalid action " << ap.result_arg << " found while preprocessing action list: error code " @@ -1202,12 +1926,16 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { } ap.valid = true; for (int i = n - 1; i >= 0; --i) { + if(ap.action_list[i].is_null()) { + continue; + } ap.result_arg = n - 1 - i; vm::CellSlice cs = load_cell_slice(ap.action_list[i]); CHECK(cs.fetch_ref().not_null()); int tag = block::gen::t_OutAction.get_tag(cs); CHECK(tag >= 0); int err_code = 34; + ap.need_bounce_on_fail = false; switch (tag) { case block::gen::OutAction::action_set_code: err_code = try_action_set_code(cs, ap, cfg); @@ -1238,18 +1966,30 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { ap.no_funds = true; } LOG(DEBUG) << "invalid action " << ap.result_arg << " in action list: error code " << ap.result_code; - // This is reuqired here because changes to libraries are applied even if actipn phase fails - enforce_state_size_limits(); + // This is required here because changes to libraries are applied even if actipn phase fails + enforce_state_limits(); + if (cfg.action_fine_enabled) { + ap.action_fine = std::min(ap.action_fine, balance.grams); + ap.total_action_fees = ap.action_fine; + balance.grams -= ap.action_fine; + total_fees += ap.action_fine; + } + if (ap.need_bounce_on_fail) { + ap.bounce = true; + } return true; } } + if (cfg.action_fine_enabled) { + ap.total_action_fees += ap.action_fine; + } end_lt = ap.end_lt; if (ap.new_code.not_null()) { new_code = ap.new_code; } new_data = compute_phase->new_data; // tentative persistent data update applied - if (!enforce_state_size_limits()) { + if (!enforce_state_limits()) { return true; } @@ -1260,9 +2000,9 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { ap.remaining_balance += ap.reserved_balance; CHECK(ap.remaining_balance.is_valid()); if (ap.acc_delete_req) { - CHECK(ap.remaining_balance.is_zero()); + CHECK(cfg.extra_currency_v2 ? ap.remaining_balance.grams->sgn() == 0 : ap.remaining_balance.is_zero()); ap.acc_status_change = ActionPhase::acst_deleted; - acc_status = Account::acc_deleted; + acc_status = (ap.remaining_balance.is_zero() ? Account::acc_deleted : Account::acc_uninit); was_deleted = true; } ap.success = true; @@ -1273,6 +2013,15 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { return true; } +/** + * Tries to set the code for an account. + * + * @param cs The CellSlice containing the action data serialized as action_set_code TLB-scheme. + * @param ap The action phase object. + * @param cfg The action phase configuration. + * + * @returns 0 if the code was successfully set, -1 otherwise. + */ int Transaction::try_action_set_code(vm::CellSlice& cs, ActionPhase& ap, const ActionPhaseConfig& cfg) { block::gen::OutAction::Record_action_set_code rec; if (!tlb::unpack_exact(cs, rec)) { @@ -1284,12 +2033,35 @@ int Transaction::try_action_set_code(vm::CellSlice& cs, ActionPhase& ap, const A return 0; } +/** + * Tries to change the library in the transaction. + * + * @param cs The cell slice containing the action data serialized as action_change_library TLB-scheme. + * @param ap The action phase object. + * @param cfg The action phase configuration. + * + * @returns 0 if the action was successfully performed, + * -1 if there was an error unpacking the data or the mode is invalid, + * 41 if the library reference is required but is null, + * 43 if the number of cells in the library exceeds the limit, + * 42 if there was a VM error during the operation. + */ int Transaction::try_action_change_library(vm::CellSlice& cs, ActionPhase& ap, const ActionPhaseConfig& cfg) { block::gen::OutAction::Record_action_change_library rec; if (!tlb::unpack_exact(cs, rec)) { return -1; } - // mode: +0 = remove library, +1 = add private library, +2 = add public library + // mode: +0 = remove library, +1 = add private library, +2 = add public library, +16 - bounce on fail + if (rec.mode & 16) { + if (!cfg.bounce_on_fail_enabled) { + return -1; + } + ap.need_bounce_on_fail = true; + rec.mode &= ~16; + } + if (rec.mode > 2) { + return -1; + } Ref lib_ref = rec.libref->prefetch_ref(); ton::Bits256 hash; if (lib_ref.not_null()) { @@ -1322,8 +2094,8 @@ int Transaction::try_action_change_library(vm::CellSlice& cs, ActionPhase& ap, c return 41; } vm::CellStorageStat sstat; - sstat.compute_used_storage(lib_ref); - if (sstat.cells > cfg.size_limits.max_library_cells) { + auto cell_info = sstat.compute_used_storage(lib_ref).move_as_ok(); + if (sstat.cells > cfg.size_limits.max_library_cells || cell_info.max_merkle_depth > max_allowed_merkle_depth) { return 43; } vm::CellBuilder cb; @@ -1338,10 +2110,20 @@ int Transaction::try_action_change_library(vm::CellSlice& cs, ActionPhase& ap, c ap.spec_actions++; return 0; } +} // namespace transaction -// msg_fwd_fees = (lump_price + ceil((bit_price * msg.bits + cell_price * msg.cells)/2^16)) nanograms -// ihr_fwd_fees = ceil((msg_fwd_fees * ihr_price_factor)/2^16) nanograms -// bits in the root cell of a message are not included in msg.bits (lump_price pays for them) +/** + * Computes the forward fees for a message based on the number of cells and bits. + * + * msg_fwd_fees = (lump_price + ceil((bit_price * msg.bits + cell_price * msg.cells)/2^16)) nanograms + * ihr_fwd_fees = ceil((msg_fwd_fees * ihr_price_factor)/2^16) nanograms + * bits in the root cell of a message are not included in msg.bits (lump_price pays for them) + * + * @param cells The number of cells in the message. + * @param bits The number of bits in the message. + * + * @returns The computed forward fees for the message. + */ td::uint64 MsgPrices::compute_fwd_fees(td::uint64 cells, td::uint64 bits) const { return lump_price + td::uint128(bit_price) .mult(bits) @@ -1351,6 +2133,34 @@ td::uint64 MsgPrices::compute_fwd_fees(td::uint64 cells, td::uint64 bits) const .lo(); } +/** + * Computes the forward fees for a message based on the number of cells and bits. + * Return the result as td::RefInt256 + * + * msg_fwd_fees = (lump_price + ceil((bit_price * msg.bits + cell_price * msg.cells)/2^16)) nanograms + * ihr_fwd_fees = ceil((msg_fwd_fees * ihr_price_factor)/2^16) nanograms + * bits in the root cell of a message are not included in msg.bits (lump_price pays for them) + * + * @param cells The number of cells in the message. + * @param bits The number of bits in the message. + * + * @returns The computed forward fees for the message as td::RefInt256j. + */ +td::RefInt256 MsgPrices::compute_fwd_fees256(td::uint64 cells, td::uint64 bits) const { + return td::make_refint(lump_price) + + td::rshift(td::make_refint(bit_price) * bits + td::make_refint(cell_price) * cells, 16, + 1); // divide by 2^16 with ceil rounding +} + +/** + * Computes the forward fees and IHR fees for a message with the given number of cells and bits. + * + * @param cells The number of cells. + * @param bits The number of bits. + * @param ihr_disabled Flag indicating whether IHR is disabled. + * + * @returns A pair of values representing the forward fees and IHR fees. + */ std::pair MsgPrices::compute_fwd_ihr_fees(td::uint64 cells, td::uint64 bits, bool ihr_disabled) const { td::uint64 fwd = compute_fwd_fees(cells, bits); @@ -1360,18 +2170,47 @@ std::pair MsgPrices::compute_fwd_ihr_fees(td::uint64 cel return std::pair(fwd, td::uint128(fwd).mult(ihr_factor).shr(16).lo()); } +/** + * Computes the part of the fees that go to the total fees of the current block. + * + * @param total The amount of fees. + * + * @returns The the part of the fees that go to the total fees of the current block. + */ td::RefInt256 MsgPrices::get_first_part(td::RefInt256 total) const { return (std::move(total) * first_frac) >> 16; } +/** + * Computes the part of the fees that go to the total fees of the current block. + * + * @param total The amount of fees. + * + * @returns The the part of the fees that go to the total fees of the current block. + */ td::uint64 MsgPrices::get_first_part(td::uint64 total) const { return td::uint128(total).mult(first_frac).shr(16).lo(); } +/** + * Computes the part of the fees that go to the total fees of the transit block. + * + * @param total The amount of fees. + * + * @returns The the part of the fees that go to the total fees of the transit block. + */ td::RefInt256 MsgPrices::get_next_part(td::RefInt256 total) const { return (std::move(total) * next_frac) >> 16; } +namespace transaction { +/** + * Checks if the source address is addr_none and replaces is with the account address. + * + * @param src_addr A reference to the source address of the message. + * + * @returns True if the source address is addr_none or is equal to the account address. + */ bool Transaction::check_replace_src_addr(Ref& src_addr) const { int t = (int)src_addr->prefetch_ulong(2); if (!t && src_addr->size_ext() == 2) { @@ -1392,6 +2231,15 @@ bool Transaction::check_replace_src_addr(Ref& src_addr) const { return false; } +/** + * Checks the destination address of a message, rewrites it if it is an anycast address. + * + * @param dest_addr A reference to the destination address of the transaction. + * @param cfg The configuration for the action phase. + * @param is_mc A pointer to a boolean where it will be stored whether the destination is in the masterchain. + * + * @returns True if the destination address is valid, false otherwise. + */ bool Transaction::check_rewrite_dest_addr(Ref& dest_addr, const ActionPhaseConfig& cfg, bool* is_mc) const { if (!dest_addr->prefetch_ulong(1)) { @@ -1454,11 +2302,6 @@ bool Transaction::check_rewrite_dest_addr(Ref& dest_addr, const A } if (rec.anycast->size() > 1) { // destination address is an anycast - if (rec.workchain_id == ton::masterchainId) { - // anycast addresses disabled in masterchain - LOG(DEBUG) << "masterchain destination address has an anycast field"; - return false; - } vm::CellSlice cs{*rec.anycast}; int d = (int)cs.fetch_ulong(6) - 32; if (d <= 0 || d > 30) { @@ -1498,21 +2341,58 @@ bool Transaction::check_rewrite_dest_addr(Ref& dest_addr, const A return true; } +/** + * Tries to send a message. + * + * @param cs0 The cell slice containing the action data serialized as action_send_msg TLB-scheme. + * @param ap The action phase. + * @param cfg The action phase configuration. + * @param redoing The index of the attempt, starting from 0. On later attempts tries to move message body and StateInit to separate cells. + * + * @returns 0 if the message is successfully sent or if the error may be ignored, error code otherwise. + * Returns -2 if the action should be attempted again. + */ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, const ActionPhaseConfig& cfg, int redoing) { block::gen::OutAction::Record_action_send_msg act_rec; - // mode: +128 = attach all remaining balance, +64 = attach all remaining balance of the inbound message, +32 = delete smart contract if balance becomes zero, +1 = pay message fees, +2 = skip if message cannot be sent + // mode: + // +128 = attach all remaining balance + // +64 = attach all remaining balance of the inbound message + // +32 = delete smart contract if balance becomes zero + // +1 = pay message fees + // +2 = skip if message cannot be sent + // +16 = bounce if action fails vm::CellSlice cs{cs0}; - if (!tlb::unpack_exact(cs, act_rec) || (act_rec.mode & ~0xe3) || (act_rec.mode & 0xc0) == 0xc0) { + if (!tlb::unpack_exact(cs, act_rec)) { + return -1; + } + if ((act_rec.mode & 16) && cfg.bounce_on_fail_enabled) { + act_rec.mode &= ~16; + ap.need_bounce_on_fail = true; + } + if ((act_rec.mode & ~0xe3) || (act_rec.mode & 0xc0) == 0xc0) { return -1; } bool skip_invalid = (act_rec.mode & 2); + auto check_skip_invalid = [&](unsigned error_code) -> unsigned int { + if (skip_invalid) { + if (cfg.message_skip_enabled) { + ap.skipped_actions++; + } + return 0; + } + return error_code; + }; // try to parse suggested message in act_rec.out_msg td::RefInt256 fwd_fee, ihr_fee; block::gen::MessageRelaxed::Record msg; if (!tlb::type_unpack_cell(act_rec.out_msg, block::gen::t_MessageRelaxed_Any, msg)) { return -1; } + if (!block::tlb::validate_message_relaxed_libs(act_rec.out_msg)) { + LOG(DEBUG) << "outbound message has invalid libs in StateInit"; + return -1; + } if (redoing >= 1) { if (msg.init->size_refs() >= 2) { LOG(DEBUG) << "moving the StateInit of a suggested outbound message into a separate cell"; @@ -1526,7 +2406,7 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, && cb.store_long_bool(3, 2) // (just (right ... )) && cb.store_ref_bool(std::move(cell)) // z:^StateInit && cb.finalize_to(cell)); - msg.init = vm::load_cell_slice_ref(std::move(cell)); + msg.init = vm::load_cell_slice_ref(cell); } else { redoing = 2; } @@ -1543,7 +2423,7 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, && cb.store_long_bool(1, 1) // (right ... ) && cb.store_ref_bool(std::move(cell)) // x:^X && cb.finalize_to(cell)); - msg.body = vm::load_cell_slice_ref(std::move(cell)); + msg.body = vm::load_cell_slice_ref(cell); } block::gen::CommonMsgInfoRelaxed::Record_int_msg_info info; @@ -1569,8 +2449,12 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, if (!tlb::csr_unpack(msg.info, info) || !block::tlb::t_CurrencyCollection.validate_csr(info.value)) { return -1; } - fwd_fee = block::tlb::t_Grams.as_integer(info.fwd_fee); - ihr_fee = block::tlb::t_Grams.as_integer(info.ihr_fee); + if (cfg.disable_custom_fess) { + fwd_fee = ihr_fee = td::zero_refint(); + } else { + fwd_fee = block::tlb::t_Grams.as_integer(info.fwd_fee); + ihr_fee = block::tlb::t_Grams.as_integer(info.ihr_fee); + } } // set created_at and created_lt to correct values info.created_at = now; @@ -1586,24 +2470,103 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, bool to_mc = false; if (!check_rewrite_dest_addr(info.dest, cfg, &to_mc)) { LOG(DEBUG) << "invalid destination address in a proposed outbound message"; - return skip_invalid ? 0 : 36; // invalid destination address + return check_skip_invalid(36); // invalid destination address + } + if (cfg.extra_currency_v2) { + CurrencyCollection value; + if (!value.unpack(info.value)) { + LOG(DEBUG) << "invalid value:ExtraCurrencies in a proposed outbound message"; + return check_skip_invalid(37); // invalid value:CurrencyCollection + } + if (!CurrencyCollection::remove_zero_extra_currencies(value.extra, cfg.size_limits.max_msg_extra_currencies)) { + LOG(DEBUG) << "invalid value:ExtraCurrencies in a proposed outbound message: too many currencies (max " + << cfg.size_limits.max_msg_extra_currencies << ")"; + // Dict should be valid, since it was checked in t_OutListNode.validate_ref, so error here means limit exceeded + return check_skip_invalid(41); // invalid value:CurrencyCollection : too many extra currencies + } + info.value = value.pack(); } // fetch message pricing info const MsgPrices& msg_prices = cfg.fetch_msg_prices(to_mc || account.is_masterchain()); + // If action fails, account is required to pay fine_per_cell for every visited cell + // Number of visited cells is limited depending on available funds + unsigned max_cells = cfg.size_limits.max_msg_cells; + td::uint64 fine_per_cell = 0; + if (cfg.action_fine_enabled && !account.is_special) { + fine_per_cell = (msg_prices.cell_price >> 16) / 4; + td::RefInt256 funds = ap.remaining_balance.grams; + if (!ext_msg && !(act_rec.mode & 0x80) && !(act_rec.mode & 1)) { + if (!block::tlb::t_CurrencyCollection.validate_csr(info.value)) { + LOG(DEBUG) << "invalid value:CurrencyCollection in proposed outbound message"; + return check_skip_invalid(37); + } + block::CurrencyCollection value; + CHECK(value.unpack(info.value)); + CHECK(value.grams.not_null()); + td::RefInt256 new_funds = value.grams; + if (act_rec.mode & 0x40) { + if (msg_balance_remaining.is_valid()) { + new_funds += msg_balance_remaining.grams; + } + if (compute_phase) { + new_funds -= compute_phase->gas_fees; + } + new_funds -= ap.action_fine; + if (new_funds->sgn() < 0) { + LOG(DEBUG) + << "not enough value to transfer with the message: all of the inbound message value has been consumed"; + return check_skip_invalid(37); + } + } + funds = std::min(funds, new_funds); + } + if (funds->cmp(max_cells * fine_per_cell) < 0) { + max_cells = static_cast((funds / td::make_refint(fine_per_cell))->to_long()); + } + } // compute size of message - vm::CellStorageStat sstat; // for message size + vm::CellStorageStat sstat(max_cells); // for message size // preliminary storage estimation of the resulting message - sstat.add_used_storage(msg.init, true, 3); // message init - sstat.add_used_storage(msg.body, true, 3); // message body (the root cell itself is not counted) - if (!ext_msg) { - sstat.add_used_storage(info.value->prefetch_ref()); + unsigned max_merkle_depth = 0; + auto add_used_storage = [&](const auto& x, unsigned skip_root_count) -> td::Status { + if (x.not_null()) { + TRY_RESULT(res, sstat.add_used_storage(x, true, skip_root_count)); + max_merkle_depth = std::max(max_merkle_depth, res.max_merkle_depth); + } + return td::Status::OK(); + }; + add_used_storage(msg.init, 3); // message init + add_used_storage(msg.body, 3); // message body (the root cell itself is not counted) + if (!ext_msg && !cfg.extra_currency_v2) { + add_used_storage(info.value->prefetch_ref(), 0); + } + auto collect_fine = [&] { + if (cfg.action_fine_enabled && !account.is_special) { + td::uint64 fine = fine_per_cell * std::min(max_cells, sstat.cells); + if (ap.remaining_balance.grams->cmp(fine) < 0) { + fine = ap.remaining_balance.grams->to_long(); + } + ap.action_fine += fine; + ap.remaining_balance.grams -= fine; + } + }; + if (sstat.cells > max_cells && max_cells < cfg.size_limits.max_msg_cells) { + LOG(DEBUG) << "not enough funds to process a message (max_cells=" << max_cells << ")"; + collect_fine(); + return check_skip_invalid(40); + } + if (sstat.bits > cfg.size_limits.max_msg_bits || sstat.cells > max_cells) { + LOG(DEBUG) << "message too large, invalid"; + collect_fine(); + return check_skip_invalid(40); + } + if (max_merkle_depth > max_allowed_merkle_depth) { + LOG(DEBUG) << "message has too big merkle depth, invalid"; + collect_fine(); + return check_skip_invalid(40); } LOG(DEBUG) << "storage paid for a message: " << sstat.cells << " cells, " << sstat.bits << " bits"; - if (sstat.bits > cfg.size_limits.max_msg_bits || sstat.cells > cfg.size_limits.max_msg_cells) { - LOG(DEBUG) << "message too large, invalid"; - return skip_invalid ? 0 : 40; - } // compute forwarding fees auto fees_c = msg_prices.compute_fwd_ihr_fees(sstat.cells, sstat.bits, info.ihr_disabled); @@ -1632,7 +2595,8 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, // ... if (!block::tlb::t_CurrencyCollection.validate_csr(info.value)) { LOG(DEBUG) << "invalid value:CurrencyCollection in proposed outbound message"; - return skip_invalid ? 0 : 37; + collect_fine(); + return check_skip_invalid(37); } if (info.ihr_disabled) { // if IHR is disabled, IHR fees will be always zero @@ -1645,17 +2609,29 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, if (act_rec.mode & 0x80) { // attach all remaining balance to this message - req = ap.remaining_balance; + if (cfg.extra_currency_v2) { + req.grams = ap.remaining_balance.grams; + } else { + req = ap.remaining_balance; + } act_rec.mode &= ~1; // pay fees from attached value } else if (act_rec.mode & 0x40) { // attach all remaining balance of the inbound message (in addition to the original value) - req += msg_balance_remaining; - if (!(act_rec.mode & 1) && compute_phase) { - req -= compute_phase->gas_fees; + if (cfg.extra_currency_v2) { + req.grams += msg_balance_remaining.grams; + } else { + req += msg_balance_remaining; + } + if (!(act_rec.mode & 1)) { + req -= ap.action_fine; + if (compute_phase) { + req -= compute_phase->gas_fees; + } if (!req.is_valid()) { LOG(DEBUG) << "not enough value to transfer with the message: all of the inbound message value has been consumed"; - return skip_invalid ? 0 : 37; + collect_fine(); + return check_skip_invalid(37); } } } @@ -1670,7 +2646,8 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, // receiver pays the fees (but cannot) LOG(DEBUG) << "not enough value attached to the message to pay forwarding fees : have " << req.grams << ", need " << fees_total; - return skip_invalid ? 0 : 37; // not enough grams + collect_fine(); + return check_skip_invalid(37); // not enough grams } else { // decrease message value req.grams -= fees_total; @@ -1680,7 +2657,13 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, if (ap.remaining_balance.grams < req_grams_brutto) { LOG(DEBUG) << "not enough grams to transfer with the message : remaining balance is " << ap.remaining_balance.to_str() << ", need " << req_grams_brutto << " (including forwarding fees)"; - return skip_invalid ? 0 : 37; // not enough grams + collect_fine(); + return check_skip_invalid(37); // not enough grams + } + + if (cfg.extra_currency_v2 && !req.check_extra_currency_limit(cfg.size_limits.max_msg_extra_currencies)) { + LOG(DEBUG) << "too many extra currencies in the message : max " << cfg.size_limits.max_msg_extra_currencies; + return check_skip_invalid(41); // to many extra currencies } Ref new_extra; @@ -1689,7 +2672,8 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, LOG(DEBUG) << "not enough extra currency to send with the message: " << block::CurrencyCollection{0, req.extra}.to_str() << " required, only " << block::CurrencyCollection{0, ap.remaining_balance.extra}.to_str() << " available"; - return skip_invalid ? 0 : 38; // not enough (extra) funds + collect_fine(); + return check_skip_invalid(38); // not enough (extra) funds } if (ap.remaining_balance.extra.not_null() || req.extra.not_null()) { LOG(DEBUG) << "subtracting extra currencies: " @@ -1711,7 +2695,11 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, vm::CellBuilder cb; if (!tlb::type_pack(cb, block::gen::t_MessageRelaxed_Any, msg)) { LOG(DEBUG) << "outbound message does not fit into a cell after rewriting"; - return redoing < 2 ? -2 : (skip_invalid ? 0 : 39); + if (redoing == 2) { + collect_fine(); + return check_skip_invalid(39); + } + return -2; } new_msg_bits = cb.size(); @@ -1719,7 +2707,11 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, // clear msg_balance_remaining if it has been used if (act_rec.mode & 0xc0) { - msg_balance_remaining.set_zero(); + if (cfg.extra_currency_v2) { + msg_balance_remaining.grams = td::zero_refint(); + } else { + msg_balance_remaining.set_zero(); + } } // update balance @@ -1733,7 +2725,8 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, // external messages also have forwarding fees if (ap.remaining_balance.grams < fwd_fee) { LOG(DEBUG) << "not enough funds to pay for an outbound external message"; - return skip_invalid ? 0 : 37; // not enough grams + collect_fine(); + return check_skip_invalid(37); // not enough grams } // repack message // ext_out_msg_info$11 constructor of CommonMsgInfo @@ -1746,7 +2739,11 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, vm::CellBuilder cb; if (!tlb::type_pack(cb, block::gen::t_MessageRelaxed_Any, msg)) { LOG(DEBUG) << "outbound message does not fit into a cell after rewriting"; - return redoing < 2 ? -2 : (skip_invalid ? 0 : 39); + if (redoing == 2) { + collect_fine(); + return check_skip_invalid(39); + } + return -2; } new_msg_bits = cb.size(); @@ -1761,17 +2758,23 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, if (!block::tlb::t_Message.validate_ref(new_msg)) { LOG(ERROR) << "generated outbound message is not a valid (Message Any) according to hand-written check"; + collect_fine(); return -1; } if (!block::gen::t_Message_Any.validate_ref(new_msg)) { LOG(ERROR) << "generated outbound message is not a valid (Message Any) according to automated check"; - block::gen::t_Message_Any.print_ref(std::cerr, new_msg); - vm::load_cell_slice(new_msg).print_rec(std::cerr); + FLOG(INFO) { + block::gen::t_Message_Any.print_ref(sb, new_msg); + vm::load_cell_slice(new_msg).print_rec(sb); + }; + collect_fine(); return -1; } if (verbosity > 2) { - std::cerr << "converted outbound message: "; - block::gen::t_Message_Any.print_ref(std::cerr, new_msg); + FLOG(INFO) { + sb << "converted outbound message: "; + block::gen::t_Message_Any.print_ref(sb, new_msg); + }; } ap.msgs_created++; @@ -1782,8 +2785,13 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, ap.total_fwd_fees += fees_total; if ((act_rec.mode & 0xa0) == 0xa0) { - CHECK(ap.remaining_balance.is_zero()); - ap.acc_delete_req = ap.reserved_balance.is_zero(); + if (cfg.extra_currency_v2) { + CHECK(ap.remaining_balance.grams->sgn() == 0); + ap.acc_delete_req = ap.reserved_balance.grams->sgn() == 0; + } else { + CHECK(ap.remaining_balance.is_zero()); + ap.acc_delete_req = ap.reserved_balance.is_zero(); + } } ap.tot_msg_bits += sstat.bits + new_msg_bits; @@ -1792,9 +2800,25 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, return 0; } +/** + * Tries to reserve a currency an action phase. + * + * @param cs The cell slice containing the action data serialized as action_reserve_currency TLB-scheme. + * @param ap The action phase. + * @param cfg The action phase configuration. + * + * @returns 0 if the currency is successfully reserved, error code otherwise. + */ int Transaction::try_action_reserve_currency(vm::CellSlice& cs, ActionPhase& ap, const ActionPhaseConfig& cfg) { block::gen::OutAction::Record_action_reserve_currency rec; - if (!tlb::unpack_exact(cs, rec) || (rec.mode & ~15)) { + if (!tlb::unpack_exact(cs, rec)) { + return -1; + } + if ((rec.mode & 16) && cfg.bounce_on_fail_enabled) { + rec.mode &= ~16; + ap.need_bounce_on_fail = true; + } + if (rec.mode & ~15) { return -1; } int mode = rec.mode; @@ -1820,22 +2844,25 @@ int Transaction::try_action_reserve_currency(vm::CellSlice& cs, ActionPhase& ap, LOG(DEBUG) << "cannot reserve a negative amount: " << reserve.to_str(); return -1; } - if (reserve.grams > ap.remaining_balance.grams) { - if (mode & 2) { - reserve.grams = ap.remaining_balance.grams; + if (mode & 2) { + if (cfg.reserve_extra_enabled) { + if (!reserve.clamp(ap.remaining_balance)) { + LOG(DEBUG) << "failed to clamp reserve amount" << mode; + return -1; + } } else { - LOG(DEBUG) << "cannot reserve " << reserve.grams << " nanograms : only " << ap.remaining_balance.grams - << " available"; - return 37; // not enough grams + reserve.grams = std::min(reserve.grams, ap.remaining_balance.grams); } } + if (reserve.grams > ap.remaining_balance.grams) { + LOG(DEBUG) << "cannot reserve " << reserve.grams << " nanograms : only " << ap.remaining_balance.grams + << " available"; + return 37; // not enough grams + } if (!block::sub_extra_currency(ap.remaining_balance.extra, reserve.extra, newc.extra)) { LOG(DEBUG) << "not enough extra currency to reserve: " << block::CurrencyCollection{0, reserve.extra}.to_str() << " required, only " << block::CurrencyCollection{0, ap.remaining_balance.extra}.to_str() << " available"; - if (mode & 2) { - // TODO: process (mode & 2) correctly by setting res_extra := inf (reserve.extra, ap.remaining_balance.extra) - } return 38; // not enough (extra) funds } newc.grams = ap.remaining_balance.grams - reserve.grams; @@ -1855,7 +2882,62 @@ int Transaction::try_action_reserve_currency(vm::CellSlice& cs, ActionPhase& ap, return 0; } -bool Transaction::check_state_size_limit(const ActionPhaseConfig& cfg) { +/** + * Calculates the number of public libraries in the dictionary. + * + * @param libraries The dictionary of account libraries. + * + * @returns The number of public libraries in the dictionary. + */ +static td::uint32 get_public_libraries_count(const td::Ref& libraries) { + td::uint32 count = 0; + vm::Dictionary dict{libraries, 256}; + dict.check_for_each([&](td::Ref value, td::ConstBitPtr key, int) { + if (block::is_public_library(key, std::move(value))) { + ++count; + } + return true; + }); + return count; +} + +/** + * Calculates the number of changes of public libraries in the dictionary. + * + * @param old_libraries The dictionary of account libraries before the transaction. + * @param new_libraries The dictionary of account libraries after the transaction. + * + * @returns The number of changed public libraries. + */ +static td::uint32 get_public_libraries_diff_count(const td::Ref& old_libraries, + const td::Ref& new_libraries) { + td::uint32 count = 0; + vm::Dictionary dict1{old_libraries, 256}; + vm::Dictionary dict2{new_libraries, 256}; + dict1.scan_diff(dict2, [&](td::ConstBitPtr key, int n, Ref val1, Ref val2) -> bool { + CHECK(n == 256); + bool is_public1 = val1.not_null() && block::is_public_library(key, val1); + bool is_public2 = val2.not_null() && block::is_public_library(key, val2); + if (is_public1 != is_public2) { + ++count; + } + return true; + }); + return count; +} + +/** + * Checks that the new account state fits in the limits. + * This function is not called for special accounts. + * + * @param size_limits The size limits configuration. + * @param update_storage_stat Store storage stat in the Transaction's CellStorageStat. + * + * @returns A `td::Status` indicating the result of the check. + * - If the state limits are within the allowed range, returns OK. + * - If the state limits exceed the maximum allowed range, returns an error. + */ +td::Status Transaction::check_state_limits(const SizeLimitsConfig& size_limits, bool update_storage_stat) { auto cell_equal = [](const td::Ref& a, const td::Ref& b) -> bool { if (a.is_null()) { return b.is_null(); @@ -1867,23 +2949,59 @@ bool Transaction::check_state_size_limit(const ActionPhaseConfig& cfg) { }; if (cell_equal(account.code, new_code) && cell_equal(account.data, new_data) && cell_equal(account.library, new_library)) { - return true; + return td::Status::OK(); } - // new_storage_stat is used here beause these stats will be reused in compute_state() - new_storage_stat.limit_cells = cfg.size_limits.max_acc_state_cells; - new_storage_stat.limit_bits = cfg.size_limits.max_acc_state_bits; - new_storage_stat.add_used_storage(new_code); - new_storage_stat.add_used_storage(new_data); - new_storage_stat.add_used_storage(new_library); + vm::CellStorageStat storage_stat; + storage_stat.limit_cells = size_limits.max_acc_state_cells; + storage_stat.limit_bits = size_limits.max_acc_state_bits; + { + TD_PERF_COUNTER(transaction_storage_stat_a); + td::Timer timer; + auto add_used_storage = [&](const td::Ref& cell) -> td::Status { + if (cell.not_null()) { + TRY_RESULT(res, storage_stat.add_used_storage(cell)); + if (res.max_merkle_depth > max_allowed_merkle_depth) { + return td::Status::Error("too big merkle depth"); + } + } + return td::Status::OK(); + }; + TRY_STATUS(add_used_storage(new_code)); + TRY_STATUS(add_used_storage(new_data)); + TRY_STATUS(add_used_storage(new_library)); + if (timer.elapsed() > 0.1) { + LOG(INFO) << "Compute used storage took " << timer.elapsed() << "s"; + } + } + if (acc_status == Account::acc_active) { - new_storage_stat.clear_limit(); + storage_stat.clear_limit(); } else { - new_storage_stat.clear(); + storage_stat.clear(); } - return new_storage_stat.cells <= cfg.size_limits.max_acc_state_cells && - new_storage_stat.bits <= cfg.size_limits.max_acc_state_bits; + td::Status res; + if (storage_stat.cells > size_limits.max_acc_state_cells || storage_stat.bits > size_limits.max_acc_state_bits) { + res = td::Status::Error(PSTRING() << "account state is too big"); + } else if (account.is_masterchain() && !cell_equal(account.library, new_library) && + get_public_libraries_count(new_library) > size_limits.max_acc_public_libraries) { + res = td::Status::Error("too many public libraries"); + } else { + res = td::Status::OK(); + } + if (update_storage_stat) { + // storage_stat will be reused in compute_state() + new_storage_stat = std::move(storage_stat); + } + return res; } +/** + * Prepares the bounce phase of a transaction. + * + * @param cfg The configuration for the action phase. + * + * @returns True if the bounce phase was successfully prepared, false otherwise. + */ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { if (in_msg.is_null() || !bounce_enabled) { return false; @@ -1926,6 +3044,9 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { if (compute_phase && compute_phase->gas_fees.not_null()) { msg_balance.grams -= compute_phase->gas_fees; } + if (action_phase && action_phase->action_fine.not_null()) { + msg_balance.grams -= action_phase->action_fine; + } if ((msg_balance.grams < 0) || (msg_balance.grams->signed_fits_bits(64) && msg_balance.grams->to_long() < (long long)bp.fwd_fees)) { // not enough funds @@ -1941,7 +3062,8 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { bp.fwd_fees -= bp.fwd_fees_collected; total_fees += td::make_refint(bp.fwd_fees_collected); // serialize outbound message - info.created_lt = end_lt++; + info.created_lt = start_lt + 1 + out_msgs.size(); + end_lt++; info.created_at = now; vm::CellBuilder cb; CHECK(cb.store_long_bool(5, 4) // int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool @@ -1971,13 +3093,16 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { } CHECK(cb.finalize_to(bp.out_msg)); if (verbosity > 2) { - LOG(INFO) << "generated bounced message: "; - block::gen::t_Message_Any.print_ref(std::cerr, bp.out_msg); + FLOG(INFO) { + sb << "generated bounced message: "; + block::gen::t_Message_Any.print_ref(sb, bp.out_msg); + }; } out_msgs.push_back(bp.out_msg); bp.ok = true; return true; } +} // namespace transaction /* * @@ -1985,6 +3110,14 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { * */ +/** + * Stores the account status in a CellBuilder object. + * + * @param cb The CellBuilder object to store the account status in. + * @param acc_status The account status to store. + * + * @returns True if the account status was successfully stored, false otherwise. + */ bool Account::store_acc_status(vm::CellBuilder& cb, int acc_status) const { int v; switch (acc_status) { @@ -2007,6 +3140,18 @@ bool Account::store_acc_status(vm::CellBuilder& cb, int acc_status) const { return cb.store_long_bool(v, 2); } +/** + * Tries to update the storage statistics based on the old storage statistics and old account state without fully recomputing it. + * + * It succeeds if only root cell of AccountStorage is changed. + * old_cs and new_cell are AccountStorage without extra currencies (if global_version >= 10). + * + * @param old_stat The old storage statistics. + * @param old_cs The old AccountStorage. + * @param new_cell The new AccountStorage. + * + * @returns An optional value of type vm::CellStorageStat. If the update is successful, it returns the new storage statistics. Otherwise, it returns an empty optional. + */ static td::optional try_update_storage_stat(const vm::CellStorageStat& old_stat, td::Ref old_cs, td::Ref new_cell) { @@ -2033,7 +3178,48 @@ static td::optional try_update_storage_stat(const vm::CellS return new_stat; } -bool Transaction::compute_state() { +/** + * Removes extra currencies dict from AccountStorage. + * + * This is used for computing account storage stats. + * + * @param storage_cs AccountStorage as CellSlice. + * + * @returns AccountStorage without extra currencies as Cell. + */ +static td::Ref storage_without_extra_currencies(td::Ref storage_cs) { + block::gen::AccountStorage::Record rec; + if (!block::gen::csr_unpack(storage_cs, rec)) { + LOG(ERROR) << "failed to unpack AccountStorage"; + return {}; + } + if (rec.balance->size_refs() > 0) { + block::gen::CurrencyCollection::Record balance; + if (!block::gen::csr_unpack(rec.balance, balance)) { + LOG(ERROR) << "failed to unpack AccountStorage"; + return {}; + } + balance.other = vm::CellBuilder{}.store_zeroes(1).as_cellslice_ref(); + if (!block::gen::csr_pack(rec.balance, balance)) { + LOG(ERROR) << "failed to pack AccountStorage"; + return {}; + } + } + td::Ref cell; + if (!block::gen::pack_cell(cell, rec)) { + LOG(ERROR) << "failed to pack AccountStorage"; + return {}; + } + return cell; +} + +namespace transaction { +/** + * Computes the new state of the account. + * + * @returns True if the state computation is successful, false otherwise. + */ +bool Transaction::compute_state(const SerializeConfig& cfg) { if (new_total_state.not_null()) { return true; } @@ -2067,11 +3253,13 @@ bool Transaction::compute_state() { auto frozen_state = cb2.finalize(); frozen_hash = frozen_state->get_hash().bits(); if (verbosity >= 3 * 1) { // !!!DEBUG!!! - std::cerr << "freezing state of smart contract: "; - block::gen::t_StateInit.print_ref(std::cerr, frozen_state); - CHECK(block::gen::t_StateInit.validate_ref(frozen_state)); - CHECK(block::tlb::t_StateInit.validate_ref(frozen_state)); - std::cerr << "with hash " << frozen_hash.to_hex() << std::endl; + FLOG(INFO) { + sb << "freezing state of smart contract: "; + block::gen::t_StateInit.print_ref(sb, frozen_state); + CHECK(block::gen::t_StateInit.validate_ref(frozen_state)); + CHECK(block::tlb::t_StateInit.validate_ref(frozen_state)); + sb << "with hash " << frozen_hash.to_hex(); + }; } } new_code.clear(); @@ -2103,12 +3291,27 @@ bool Transaction::compute_state() { new_inner_state.clear(); } vm::CellStorageStat& stats = new_storage_stat; - auto new_stats = try_update_storage_stat(account.storage_stat, account.storage, storage); + td::Ref old_storage_for_stat = account.storage; + td::Ref new_storage_for_stat = storage; + if (cfg.extra_currency_v2) { + new_storage_for_stat = storage_without_extra_currencies(new_storage); + if (new_storage_for_stat.is_null()) { + return false; + } + if (old_storage_for_stat.not_null()) { + old_storage_for_stat = vm::load_cell_slice_ref(storage_without_extra_currencies(old_storage_for_stat)); + if (old_storage_for_stat.is_null()) { + return false; + } + } + } + auto new_stats = try_update_storage_stat(account.storage_stat, old_storage_for_stat, storage); if (new_stats) { stats = new_stats.unwrap(); } else { + TD_PERF_COUNTER(transaction_storage_stat_b); td::Timer timer; - CHECK(stats.add_used_storage(Ref(storage))); + stats.add_used_storage(new_storage_for_stat).ensure(); if (timer.elapsed() > 0.1) { LOG(INFO) << "Compute used storage took " << timer.elapsed() << "s"; } @@ -2128,18 +3331,27 @@ bool Transaction::compute_state() { CHECK(cb.append_data_cell_bool(std::move(storage))); new_total_state = cb.finalize(); if (verbosity > 2) { - std::cerr << "new account state: "; - block::gen::t_Account.print_ref(std::cerr, new_total_state); + FLOG(INFO) { + sb << "new account state: "; + block::gen::t_Account.print_ref(sb, new_total_state); + }; } CHECK(block::tlb::t_Account.validate_ref(new_total_state)); return true; } -bool Transaction::serialize() { +/** + * Serializes the transaction object using Transaction TLB-scheme. + * + * Updates root. + * + * @returns True if the serialization is successful, False otherwise. + */ +bool Transaction::serialize(const SerializeConfig& cfg) { if (root.not_null()) { return true; } - if (!compute_state()) { + if (!compute_state(cfg)) { return false; } vm::Dictionary dict{15}; @@ -2214,22 +3426,28 @@ bool Transaction::serialize() { return false; } if (verbosity >= 3 * 1) { - std::cerr << "new transaction: "; - block::gen::t_Transaction.print_ref(std::cerr, root); - vm::load_cell_slice(root).print_rec(std::cerr); + FLOG(INFO) { + sb << "new transaction: "; + block::gen::t_Transaction.print_ref(sb, root); + vm::load_cell_slice(root).print_rec(sb); + }; } - if (!block::gen::t_Transaction.validate_ref(root)) { + if (!block::gen::t_Transaction.validate_ref(4096, root)) { LOG(ERROR) << "newly-generated transaction failed to pass automated validation:"; - vm::load_cell_slice(root).print_rec(std::cerr); - block::gen::t_Transaction.print_ref(std::cerr, root); + FLOG(INFO) { + vm::load_cell_slice(root).print_rec(sb); + block::gen::t_Transaction.print_ref(sb, root); + }; root.clear(); return false; } - if (!block::tlb::t_Transaction.validate_ref(root)) { + if (!block::tlb::t_Transaction.validate_ref(4096, root)) { LOG(ERROR) << "newly-generated transaction failed to pass hand-written validation:"; - vm::load_cell_slice(root).print_rec(std::cerr); - block::gen::t_Transaction.print_ref(std::cerr, root); + FLOG(INFO) { + vm::load_cell_slice(root).print_rec(sb); + block::gen::t_Transaction.print_ref(sb, root); + }; root.clear(); return false; } @@ -2237,6 +3455,13 @@ bool Transaction::serialize() { return true; } +/** + * Serializes the storage phase of a transaction. + * + * @param cb The CellBuilder to store the serialized data. + * + * @returns True if the serialization is successful, false otherwise. + */ bool Transaction::serialize_storage_phase(vm::CellBuilder& cb) { if (!storage_phase) { return false; @@ -2260,6 +3485,13 @@ bool Transaction::serialize_storage_phase(vm::CellBuilder& cb) { return ok; } +/** + * Serializes the credit phase of a transaction. + * + * @param cb The CellBuilder to store the serialized data. + * + * @returns True if the credit phase was successfully serialized, false otherwise. + */ bool Transaction::serialize_credit_phase(vm::CellBuilder& cb) { if (!credit_phase) { return false; @@ -2269,6 +3501,13 @@ bool Transaction::serialize_credit_phase(vm::CellBuilder& cb) { return block::store_Maybe_Grams_nz(cb, cp.due_fees_collected) && cp.credit.store(cb); } +/** + * Serializes the compute phase of a transaction. + * + * @param cb The CellBuilder to store the serialized data. + * + * @returns True if the serialization was successful, false otherwise. + */ bool Transaction::serialize_compute_phase(vm::CellBuilder& cb) { if (!compute_phase) { return false; @@ -2311,6 +3550,13 @@ bool Transaction::serialize_compute_phase(vm::CellBuilder& cb) { return ok; } +/** + * Serializes the action phase of a transaction. + * + * @param cb The CellBuilder to store the serialized data. + * + * @returns True if the serialization is successful, false otherwise. + */ bool Transaction::serialize_action_phase(vm::CellBuilder& cb) { if (!action_phase) { return false; @@ -2335,6 +3581,13 @@ bool Transaction::serialize_action_phase(vm::CellBuilder& cb) { return ok; } +/** + * Serializes the bounce phase of a transaction. + * + * @param cb The CellBuilder to store the serialized data. + * + * @returns True if the bounce phase was successfully serialized, false otherwise. + */ bool Transaction::serialize_bounce_phase(vm::CellBuilder& cb) { if (!bounce_phase) { return false; @@ -2355,6 +3608,15 @@ bool Transaction::serialize_bounce_phase(vm::CellBuilder& cb) { } } +/** + * Estimates the block storage profile increment if the transaction is added to the block. + * + * @param store_stat The current storage statistics of the block. + * @param usage_tree The usage tree of the block. + * + * @returns The estimated block storage profile increment. + * Returns Error if the transaction is not serialized or if its new state is not computed. + */ td::Result Transaction::estimate_block_storage_profile_incr( const vm::NewCellStorageStat& store_stat, const vm::CellUsageTree* usage_tree) const { if (root.is_null()) { @@ -2366,33 +3628,30 @@ td::Result Transaction::estimate_block_storage_pro return store_stat.tentative_add_proof(new_total_state, usage_tree) + store_stat.tentative_add_cell(root); } -bool Transaction::update_block_storage_profile(vm::NewCellStorageStat& store_stat, - const vm::CellUsageTree* usage_tree) const { - if (root.is_null() || new_total_state.is_null()) { - return false; - } - store_stat.add_proof(new_total_state, usage_tree); - store_stat.add_cell(root); - return true; -} - -bool Transaction::would_fit(unsigned cls, const block::BlockLimitStatus& blimst) const { - auto res = estimate_block_storage_profile_incr(blimst.st_stat, blimst.limits.usage_tree); - if (res.is_error()) { - LOG(ERROR) << res.move_as_error(); - return false; - } - auto extra = res.move_as_ok(); - return blimst.would_fit(cls, end_lt, gas_used(), &extra); -} - -bool Transaction::update_limits(block::BlockLimitStatus& blimst, bool with_size) const { - if (!(blimst.update_lt(end_lt) && blimst.update_gas(gas_used()))) { +/** + * Updates the limits status of a block. + * + * @param blimst The block limit status object to update. + * @param with_size Flag indicating whether to update the size limits. + * + * @returns True if the limits were successfully updated, False otherwise. + */ +bool Transaction::update_limits(block::BlockLimitStatus& blimst, bool with_gas, bool with_size) const { + if (!(blimst.update_lt(end_lt) && blimst.update_gas(with_gas ? gas_used() : 0))) { return false; } if (with_size) { - return blimst.add_proof(new_total_state) && blimst.add_cell(root) && blimst.add_transaction() && - blimst.add_account(is_first); + if (!(blimst.add_proof(new_total_state) && blimst.add_cell(root) && blimst.add_transaction() && + blimst.add_account(is_first))) { + return false; + } + if (account.is_masterchain()) { + if (was_frozen || was_deleted) { + blimst.public_library_diff += get_public_libraries_count(account.orig_library); + } else { + blimst.public_library_diff += get_public_libraries_diff_count(account.orig_library, new_library); + } + } } return true; } @@ -2403,6 +3662,13 @@ bool Transaction::update_limits(block::BlockLimitStatus& blimst, bool with_size) * */ +/** + * Commits a transaction for a given account. + * + * @param acc The account to commit the transaction for. + * + * @returns A reference to the root cell of the serialized transaction. + */ Ref Transaction::commit(Account& acc) { CHECK(account.last_trans_end_lt_ <= start_lt && start_lt < end_lt); CHECK(root.not_null()); @@ -2447,24 +3713,57 @@ Ref Transaction::commit(Account& acc) { return root; } +/** + * Extracts the output message at the specified index from the transaction. + * + * @param i The index of the output message to extract. + * + * @returns A pair of the logical time and the extracted output message. + */ LtCellRef Transaction::extract_out_msg(unsigned i) { return {start_lt + i + 1, std::move(out_msgs.at(i))}; } +/** + * Extracts the output message at index i from the transaction. + * + * @param i The index of the output message to extract. + * + * @returns A triple of the logical time, the extracted output message and the transaction root. + */ NewOutMsg Transaction::extract_out_msg_ext(unsigned i) { - return {start_lt + i + 1, std::move(out_msgs.at(i)), root}; + return {start_lt + i + 1, std::move(out_msgs.at(i)), root, i}; } +/** + * Extracts the outgoing messages from the transaction and adds them to the given list. + * + * @param list The list to which the outgoing messages will be added. + */ void Transaction::extract_out_msgs(std::vector& list) { for (unsigned i = 0; i < out_msgs.size(); i++) { list.emplace_back(start_lt + i + 1, std::move(out_msgs[i])); } } +} // namespace transaction +/** + * Adds a transaction to the account's transaction list. + * + * @param trans_root The root of the transaction cell. + * @param trans_lt The logical time of the transaction. + */ void Account::push_transaction(Ref trans_root, ton::LogicalTime trans_lt) { transactions.emplace_back(trans_lt, std::move(trans_root)); } +/** + * Serializes an account block for the account using AccountBlock TLB-scheme. + * + * @param cb The CellBuilder used to store the serialized data. + * + * @returns True if the account block was successfully created, false otherwise. + */ bool Account::create_account_block(vm::CellBuilder& cb) { if (transactions.empty()) { return false; @@ -2493,6 +3792,11 @@ bool Account::create_account_block(vm::CellBuilder& cb) { && cb.store_ref_bool(cb2.finalize()); // state_update:^(HASH_UPDATE Account) } +/** + * Checks if the libraries stored in the account object have changed. + * + * @returns True if the libraries have changed, False otherwise. + */ bool Account::libraries_changed() const { bool s = orig_library.not_null(); bool t = library.not_null(); @@ -2503,4 +3807,151 @@ bool Account::libraries_changed() const { } } +/** + * Fetches and initializes various configuration parameters from masterchain config for transaction processing. + * + * @param config The masterchain configuration. + * @param old_mparams Pointer to store a dictionary of mandatory parameters (ConfigParam 9). + * @param storage_prices Pointer to store the storage prices. + * @param storage_phase_cfg Pointer to store the storage phase configuration. + * @param rand_seed Pointer to the random seed. Generates a new seed if the value is `td::Bits256::zero()`. + * @param compute_phase_cfg Pointer to store the compute phase configuration. + * @param action_phase_cfg Pointer to store the action phase configuration. + * @param serialize_cfg Pointer to store the serialize phase configuration. + * @param masterchain_create_fee Pointer to store the masterchain create fee. + * @param basechain_create_fee Pointer to store the basechain create fee. + * @param wc The workchain ID. + * @param now The current Unix time. + */ +td::Status FetchConfigParams::fetch_config_params( + const block::ConfigInfo& config, Ref* old_mparams, std::vector* storage_prices, + StoragePhaseConfig* storage_phase_cfg, td::BitArray<256>* rand_seed, ComputePhaseConfig* compute_phase_cfg, + ActionPhaseConfig* action_phase_cfg, SerializeConfig* serialize_cfg, td::RefInt256* masterchain_create_fee, + td::RefInt256* basechain_create_fee, ton::WorkchainId wc, ton::UnixTime now) { + auto prev_blocks_info = config.get_prev_blocks_info(); + if (prev_blocks_info.is_error()) { + return prev_blocks_info.move_as_error_prefix( + td::Status::Error(-668, "cannot fetch prev blocks info from masterchain configuration: ")); + } + return fetch_config_params(config, prev_blocks_info.move_as_ok(), old_mparams, storage_prices, storage_phase_cfg, + rand_seed, compute_phase_cfg, action_phase_cfg, serialize_cfg, masterchain_create_fee, + basechain_create_fee, wc, now); +} + +/** + * Fetches and initializes various configuration parameters from masterchain config for transaction processing. + * + * @param config The masterchain configuration. + * @param prev_blocks_info The tuple with information about previous blocks. + * @param old_mparams Pointer to store a dictionary of mandatory parameters (ConfigParam 9). + * @param storage_prices Pointer to store the storage prices. + * @param storage_phase_cfg Pointer to store the storage phase configuration. + * @param rand_seed Pointer to the random seed. Generates a new seed if the value is `td::Bits256::zero()`. + * @param compute_phase_cfg Pointer to store the compute phase configuration. + * @param action_phase_cfg Pointer to store the action phase configuration. + * @param serialize_cfg Pointer to store the serialize phase configuration. + * @param masterchain_create_fee Pointer to store the masterchain create fee. + * @param basechain_create_fee Pointer to store the basechain create fee. + * @param wc The workchain ID. + * @param now The current Unix time. + */ +td::Status FetchConfigParams::fetch_config_params( + const block::Config& config, td::Ref prev_blocks_info, Ref* old_mparams, + std::vector* storage_prices, StoragePhaseConfig* storage_phase_cfg, + td::BitArray<256>* rand_seed, ComputePhaseConfig* compute_phase_cfg, ActionPhaseConfig* action_phase_cfg, + SerializeConfig* serialize_cfg, td::RefInt256* masterchain_create_fee, td::RefInt256* basechain_create_fee, + ton::WorkchainId wc, ton::UnixTime now) { + *old_mparams = config.get_config_param(9); + { + auto res = config.get_storage_prices(); + if (res.is_error()) { + return res.move_as_error(); + } + *storage_prices = res.move_as_ok(); + } + if (rand_seed->is_zero()) { + // generate rand seed + prng::rand_gen().strong_rand_bytes(rand_seed->data(), 32); + LOG(DEBUG) << "block random seed set to " << rand_seed->to_hex(); + } + TRY_RESULT(size_limits, config.get_size_limits_config()); + { + // compute compute_phase_cfg / storage_phase_cfg + auto cell = config.get_config_param(wc == ton::masterchainId ? 20 : 21); + if (cell.is_null()) { + return td::Status::Error(-668, "cannot fetch current gas prices and limits from masterchain configuration"); + } + if (!compute_phase_cfg->parse_GasLimitsPrices(std::move(cell), storage_phase_cfg->freeze_due_limit, + storage_phase_cfg->delete_due_limit)) { + return td::Status::Error(-668, "cannot unpack current gas prices and limits from masterchain configuration"); + } + TRY_RESULT_PREFIX(mc_gas_prices, config.get_gas_limits_prices(true), + "cannot unpack masterchain gas prices and limits: "); + compute_phase_cfg->mc_gas_prices = std::move(mc_gas_prices); + compute_phase_cfg->special_gas_full = config.get_global_version() >= 5; + storage_phase_cfg->enable_due_payment = config.get_global_version() >= 4; + storage_phase_cfg->global_version = config.get_global_version(); + compute_phase_cfg->block_rand_seed = *rand_seed; + compute_phase_cfg->max_vm_data_depth = size_limits.max_vm_data_depth; + compute_phase_cfg->global_config = config.get_root_cell(); + compute_phase_cfg->global_version = config.get_global_version(); + if (compute_phase_cfg->global_version >= 4) { + compute_phase_cfg->prev_blocks_info = std::move(prev_blocks_info); + } + if (compute_phase_cfg->global_version >= 6) { + compute_phase_cfg->unpacked_config_tuple = config.get_unpacked_config_tuple(now); + } + compute_phase_cfg->suspended_addresses = config.get_suspended_addresses(now); + compute_phase_cfg->size_limits = size_limits; + compute_phase_cfg->precompiled_contracts = config.get_precompiled_contracts_config(); + compute_phase_cfg->allow_external_unfreeze = compute_phase_cfg->global_version >= 8; + } + { + // compute action_phase_cfg + block::gen::MsgForwardPrices::Record rec; + auto cell = config.get_config_param(24); + if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { + return td::Status::Error(-668, "cannot fetch masterchain message transfer prices from masterchain configuration"); + } + action_phase_cfg->fwd_mc = + block::MsgPrices{rec.lump_price, rec.bit_price, rec.cell_price, rec.ihr_price_factor, + (unsigned)rec.first_frac, (unsigned)rec.next_frac}; + cell = config.get_config_param(25); + if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { + return td::Status::Error(-668, "cannot fetch standard message transfer prices from masterchain configuration"); + } + action_phase_cfg->fwd_std = + block::MsgPrices{rec.lump_price, rec.bit_price, rec.cell_price, rec.ihr_price_factor, + (unsigned)rec.first_frac, (unsigned)rec.next_frac}; + action_phase_cfg->workchains = &config.get_workchain_list(); + action_phase_cfg->bounce_msg_body = (config.has_capability(ton::capBounceMsgBody) ? 256 : 0); + action_phase_cfg->size_limits = size_limits; + action_phase_cfg->action_fine_enabled = config.get_global_version() >= 4; + action_phase_cfg->bounce_on_fail_enabled = config.get_global_version() >= 4; + action_phase_cfg->message_skip_enabled = config.get_global_version() >= 8; + action_phase_cfg->disable_custom_fess = config.get_global_version() >= 8; + action_phase_cfg->reserve_extra_enabled = config.get_global_version() >= 9; + action_phase_cfg->mc_blackhole_addr = config.get_burning_config().blackhole_addr; + action_phase_cfg->extra_currency_v2 = config.get_global_version() >= 10; + } + { + serialize_cfg->extra_currency_v2 = config.get_global_version() >= 10; + } + { + // fetch block_grams_created + auto cell = config.get_config_param(14); + if (cell.is_null()) { + *basechain_create_fee = *masterchain_create_fee = td::zero_refint(); + } else { + block::gen::BlockCreateFees::Record create_fees; + if (!(tlb::unpack_cell(cell, create_fees) && + block::tlb::t_Grams.as_integer_to(create_fees.masterchain_block_fee, *masterchain_create_fee) && + block::tlb::t_Grams.as_integer_to(create_fees.basechain_block_fee, *basechain_create_fee))) { + return td::Status::Error(-668, "cannot unpack BlockCreateFees from configuration parameter #14"); + } + } + } + return td::Status::OK(); +} + } // namespace block diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 2560c010..8e612e6a 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -29,13 +29,17 @@ #include "ton/ton-types.h" #include "block/block.h" #include "block/mc-config.h" +#include "precompiled-smc/PrecompiledSmartContract.h" namespace block { using td::Ref; using LtCellRef = std::pair>; struct Account; + +namespace transaction { struct Transaction; +} // namespace transaction struct CollatorError { std::string msg; @@ -62,8 +66,11 @@ struct NewOutMsg { ton::LogicalTime lt; Ref msg; Ref trans; - NewOutMsg(ton::LogicalTime _lt, Ref _msg, Ref _trans) - : lt(_lt), msg(std::move(_msg)), trans(std::move(_trans)) { + unsigned msg_idx; + td::optional metadata; + td::Ref msg_env_from_dispatch_queue; // Not null if from dispatch queue; in this case lt is emitted_lt + NewOutMsg(ton::LogicalTime _lt, Ref _msg, Ref _trans, unsigned _msg_idx) + : lt(_lt), msg(std::move(_msg)), trans(std::move(_trans)), msg_idx(_msg_idx) { } bool operator<(const NewOutMsg& other) const& { return lt < other.lt || (lt == other.lt && msg->get_hash() < other.msg->get_hash()); @@ -77,6 +84,8 @@ struct StoragePhaseConfig { const std::vector* pricing{nullptr}; td::RefInt256 freeze_due_limit; td::RefInt256 delete_due_limit; + bool enable_due_payment{false}; + int global_version = 0; StoragePhaseConfig() = default; StoragePhaseConfig(const std::vector* _pricing, td::RefInt256 freeze_limit = {}, td::RefInt256 delete_limit = {}) @@ -100,21 +109,29 @@ struct ComputePhaseConfig { td::uint64 gas_credit; td::uint64 flat_gas_limit = 0; td::uint64 flat_gas_price = 0; + bool special_gas_full = false; + block::GasLimitsPrices mc_gas_prices; static constexpr td::uint64 gas_infty = (1ULL << 63) - 1; td::RefInt256 gas_price256; td::RefInt256 max_gas_threshold; std::unique_ptr libraries; Ref global_config; td::BitArray<256> block_rand_seed; + bool ignore_chksig{false}; bool with_vm_log{false}; td::uint16 max_vm_data_depth = 512; + int global_version = 0; + Ref prev_blocks_info; + Ref unpacked_config_tuple; std::unique_ptr suspended_addresses; - ComputePhaseConfig(td::uint64 _gas_price = 0, td::uint64 _gas_limit = 0, td::uint64 _gas_credit = 0) - : gas_price(_gas_price), gas_limit(_gas_limit), special_gas_limit(_gas_limit), gas_credit(_gas_credit) { - compute_threshold(); - } - ComputePhaseConfig(td::uint64 _gas_price, td::uint64 _gas_limit, td::uint64 _spec_gas_limit, td::uint64 _gas_credit) - : gas_price(_gas_price), gas_limit(_gas_limit), special_gas_limit(_spec_gas_limit), gas_credit(_gas_credit) { + SizeLimitsConfig size_limits; + int vm_log_verbosity = 0; + bool stop_on_accept_message = false; + PrecompiledContractsConfig precompiled_contracts; + bool dont_run_precompiled_ = false; + bool allow_external_unfreeze{false}; + + ComputePhaseConfig() : gas_price(0), gas_limit(0), special_gas_limit(0), gas_credit(0) { compute_threshold(); } void compute_threshold(); @@ -148,11 +165,22 @@ struct ActionPhaseConfig { MsgPrices fwd_mc; // from/to masterchain SizeLimitsConfig size_limits; const WorkchainSet* workchains{nullptr}; + bool action_fine_enabled{false}; + bool bounce_on_fail_enabled{false}; + bool message_skip_enabled{false}; + bool disable_custom_fess{false}; + bool reserve_extra_enabled{false}; + bool extra_currency_v2{false}; + td::optional mc_blackhole_addr; const MsgPrices& fetch_msg_prices(bool is_masterchain) const { return is_masterchain ? fwd_mc : fwd_std; } }; +struct SerializeConfig { + bool extra_currency_v2{false}; +}; + struct CreditPhase { td::RefInt256 due_fees_collected; block::CurrencyCollection credit; @@ -177,6 +205,7 @@ struct ComputePhase { Ref new_data; Ref actions; std::string vm_log; + td::optional precompiled_gas_usage; }; struct ActionPhase { @@ -186,7 +215,7 @@ struct ActionPhase { bool code_changed{false}; bool action_list_invalid{false}; bool acc_delete_req{false}; - bool state_size_too_big{false}; + bool state_exceeds_limits{false}; enum { acst_unchanged = 0, acst_frozen = 2, acst_deleted = 3 }; int acc_status_change{acst_unchanged}; td::RefInt256 total_fwd_fees; // all fees debited from the account @@ -204,6 +233,9 @@ struct ActionPhase { std::vector> out_msgs; ton::LogicalTime end_lt; unsigned long long tot_msg_bits{0}, tot_msg_cells{0}; + td::RefInt256 action_fine; + bool need_bounce_on_fail = false; + bool bounce = false; }; struct BouncePhase { @@ -255,7 +287,7 @@ struct Account { return balance; } bool set_address(ton::WorkchainId wc, td::ConstBitPtr new_addr); - bool unpack(Ref account, Ref extra, ton::UnixTime now, bool special = false); + bool unpack(Ref account, ton::UnixTime now, bool special); bool init_new(ton::UnixTime now); bool deactivate(); bool recompute_tmp_addr(Ref& tmp_addr, int split_depth, td::ConstBitPtr orig_addr_rewrite) const; @@ -273,7 +305,7 @@ struct Account { bool create_account_block(vm::CellBuilder& cb); // stores an AccountBlock with all transactions protected: - friend struct Transaction; + friend struct transaction::Transaction; bool set_split_depth(int split_depth); bool check_split_depth(int split_depth) const; bool forget_split_depth(); @@ -288,7 +320,9 @@ struct Account { bool compute_my_addr(bool force = false); }; +namespace transaction { struct Transaction { + static constexpr unsigned max_allowed_merkle_depth = 2; enum { tr_none, tr_ord, @@ -325,6 +359,7 @@ struct Transaction { td::RefInt256 due_payment; td::RefInt256 in_fwd_fee, msg_fwd_fees; block::CurrencyCollection total_fees{0}; + block::CurrencyCollection blackhole_burned{0}; ton::UnixTime last_paid; Ref root; Ref new_total_state; @@ -343,30 +378,31 @@ struct Transaction { std::unique_ptr action_phase; std::unique_ptr bounce_phase; vm::CellStorageStat new_storage_stat; + bool gas_limit_overridden{false}; Transaction(const Account& _account, int ttype, ton::LogicalTime req_start_lt, ton::UnixTime _now, Ref _inmsg = {}); bool unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* cfg); bool check_in_msg_state_hash(); bool prepare_storage_phase(const StoragePhaseConfig& cfg, bool force_collect = true, bool adjust_msg_value = false); bool prepare_credit_phase(); + td::uint64 gas_bought_for(const ComputePhaseConfig& cfg, td::RefInt256 nanograms); bool compute_gas_limits(ComputePhase& cp, const ComputePhaseConfig& cfg); Ref prepare_vm_stack(ComputePhase& cp); std::vector> compute_vm_libraries(const ComputePhaseConfig& cfg); + bool run_precompiled_contract(const ComputePhaseConfig& cfg, precompiled::PrecompiledSmartContract& precompiled); bool prepare_compute_phase(const ComputePhaseConfig& cfg); bool prepare_action_phase(const ActionPhaseConfig& cfg); - bool check_state_size_limit(const ActionPhaseConfig& cfg); + td::Status check_state_limits(const SizeLimitsConfig& size_limits, bool update_storage_stat = true); bool prepare_bounce_phase(const ActionPhaseConfig& cfg); - bool compute_state(); - bool serialize(); + bool compute_state(const SerializeConfig& cfg); + bool serialize(const SerializeConfig& cfg); td::uint64 gas_used() const { return compute_phase ? compute_phase->gas_used : 0; } td::Result estimate_block_storage_profile_incr( const vm::NewCellStorageStat& store_stat, const vm::CellUsageTree* usage_tree) const; - bool update_block_storage_profile(vm::NewCellStorageStat& store_stat, const vm::CellUsageTree* usage_tree) const; - bool would_fit(unsigned cls, const block::BlockLimitStatus& blk_lim_st) const; - bool update_limits(block::BlockLimitStatus& blk_lim_st, bool with_size = true) const; + bool update_limits(block::BlockLimitStatus& blk_lim_st, bool with_gas = true, bool with_size = true) const; Ref commit(Account& _account); // _account should point to the same account LtCellRef extract_out_msg(unsigned i); @@ -388,7 +424,23 @@ struct Transaction { bool serialize_compute_phase(vm::CellBuilder& cb); bool serialize_action_phase(vm::CellBuilder& cb); bool serialize_bounce_phase(vm::CellBuilder& cb); - bool unpack_msg_state(bool lib_only = false); + bool unpack_msg_state(const ComputePhaseConfig& cfg, bool lib_only = false, bool forbid_public_libs = false); +}; +} // namespace transaction + +struct FetchConfigParams { + static td::Status fetch_config_params(const block::ConfigInfo& config, Ref* old_mparams, + std::vector* storage_prices, + StoragePhaseConfig* storage_phase_cfg, td::BitArray<256>* rand_seed, + ComputePhaseConfig* compute_phase_cfg, ActionPhaseConfig* action_phase_cfg, + SerializeConfig* serialize_cfg, td::RefInt256* masterchain_create_fee, + td::RefInt256* basechain_create_fee, ton::WorkchainId wc, ton::UnixTime now); + static td::Status fetch_config_params(const block::Config& config, Ref prev_blocks_info, + Ref* old_mparams, std::vector* storage_prices, + StoragePhaseConfig* storage_phase_cfg, td::BitArray<256>* rand_seed, + ComputePhaseConfig* compute_phase_cfg, ActionPhaseConfig* action_phase_cfg, + SerializeConfig* serialize_cfg, td::RefInt256* masterchain_create_fee, + td::RefInt256* basechain_create_fee, ton::WorkchainId wc, ton::UnixTime now); }; } // namespace block diff --git a/crypto/common/bigint.hpp b/crypto/common/bigint.hpp index 5fa8e2da..4061f82d 100644 --- a/crypto/common/bigint.hpp +++ b/crypto/common/bigint.hpp @@ -169,8 +169,6 @@ class PropagateConstSpan { size_t size_{0}; }; -struct Normalize {}; - template class AnyIntView { public: @@ -290,6 +288,7 @@ class BigIntG { public: enum { word_bits = Tr::word_bits, word_shift = Tr::word_shift, max_bits = len, word_cnt = len / word_shift + 1 }; typedef typename Tr::word_t word_t; + typedef typename Tr::uword_t uword_t; typedef Tr Traits; typedef BigIntG DoubleInt; @@ -312,9 +311,6 @@ class BigIntG { BigIntG() : n(0) { } explicit BigIntG(word_t x) : n(1) { - digits[0] = x; - } - BigIntG(Normalize, word_t x) : n(1) { if (x >= -Tr::Half && x < Tr::Half) { digits[0] = x; } else if (len <= 1) { @@ -325,6 +321,25 @@ class BigIntG { digits[n++] = (x >> Tr::word_shift) + (digits[0] < 0); } } + explicit BigIntG(uword_t x) : n(1) { + if (x < (uword_t)Tr::Half) { + digits[0] = x; + } else if (len <= 1) { + digits[0] = x; + normalize_bool(); + } else { + digits[0] = ((x ^ Tr::Half) & (Tr::Base - 1)) - Tr::Half; + digits[n++] = (x >> Tr::word_shift) + (digits[0] < 0); + } + } + explicit BigIntG(unsigned x) : BigIntG(uword_t(x)) { + } + explicit BigIntG(int x) : BigIntG(word_t(x)) { + } + explicit BigIntG(unsigned long x) : BigIntG(uword_t(x)) { + } + explicit BigIntG(long x) : BigIntG(word_t(x)) { + } BigIntG(const BigIntG& x) : n(x.n) { std::memcpy(digits, x.digits, n * sizeof(word_t)); ///std::cout << "(BiCC " << (const void*)&x << "->" << (void*)this << ")"; @@ -2279,11 +2294,11 @@ std::string AnyIntView::to_dec_string_destroy_any() { stack.push_back(divmod_short_any(Tr::max_pow10)); } while (sgn()); char slice[word_bits * 97879 / 325147 + 2]; - std::sprintf(slice, "%lld", stack.back()); + std::snprintf(slice, sizeof(slice), "%lld", stack.back()); s += slice; stack.pop_back(); while (stack.size()) { - std::sprintf(slice, "%018lld", stack.back()); + std::snprintf(slice, sizeof(slice), "%018lld", stack.back()); s += slice; stack.pop_back(); } @@ -2556,7 +2571,7 @@ typedef BigIntG<257, BigIntInfo> BigInt256; template BigIntG make_bigint(long long x) { - return BigIntG{Normalize(), x}; + return BigIntG{x}; } namespace literals { diff --git a/crypto/common/bitstring.cpp b/crypto/common/bitstring.cpp index aabc6984..52e57c9a 100644 --- a/crypto/common/bitstring.cpp +++ b/crypto/common/bitstring.cpp @@ -130,7 +130,7 @@ void bits_memcpy(unsigned char* to, int to_offs, const unsigned char* from, int from_offs &= 7; to_offs &= 7; //fprintf(stderr, "bits_memcpy: from=%p (%02x) to=%p (%02x) from_offs=%d to_offs=%d count=%lu\n", from, *from, to, *to, from_offs, to_offs, bit_count); - int sz = (int)bit_count; + int sz = static_cast(bit_count); bit_count += from_offs; if (from_offs == to_offs) { if (bit_count < 8) { @@ -206,7 +206,7 @@ void bits_memset(unsigned char* to, int to_offs, bool val, std::size_t bit_count } to += (to_offs >> 3); to_offs &= 7; - int sz = (int)bit_count; + int sz = static_cast(bit_count); bit_count += to_offs; int c = *to; if (bit_count <= 8) { @@ -596,7 +596,7 @@ long parse_bitstring_hex_literal(unsigned char* buff, std::size_t buff_size, con unsigned char* ptr = buff; const char* rptr = str; while (rptr < str_end) { - int c = *rptr++; + char c = *rptr++; if (c == ' ' || c == '\t') { continue; } @@ -627,24 +627,24 @@ long parse_bitstring_hex_literal(unsigned char* buff, std::size_t buff_size, con if (cmpl && bits) { int t = (hex_digits_count & 1) ? (0x100 + *ptr) >> 4 : (0x100 + *--ptr); while (bits > 0) { + if (t == 1) { + t = 0x100 + *--ptr; + } --bits; if (t & 1) { break; } t >>= 1; - if (t == 1) { - t = 0x100 + *--ptr; - } } } return bits; } -long parse_bitstring_binary_literal(BitPtr buff, std::size_t buff_size, const char* str, const char* str_end) { +long parse_bitstring_binary_literal(BitPtr buff, std::size_t buff_size_bits, const char* str, const char* str_end) { const char* ptr = str; - while (ptr < str_end && buff_size && (*ptr == '0' || *ptr == '1')) { + while (ptr < str_end && buff_size_bits && (*ptr == '0' || *ptr == '1')) { *buff++ = (bool)(*ptr++ & 1); - --buff_size; + --buff_size_bits; } return td::narrow_cast(ptr == str_end ? ptr - str : str - ptr - 1); } diff --git a/crypto/common/bitstring.h b/crypto/common/bitstring.h index dc3a2fa5..12333522 100644 --- a/crypto/common/bitstring.h +++ b/crypto/common/bitstring.h @@ -58,7 +58,7 @@ unsigned long long bits_load_long_top(ConstBitPtr from, unsigned top_bits); long long bits_load_long(ConstBitPtr from, unsigned bits); unsigned long long bits_load_ulong(ConstBitPtr from, unsigned bits); long parse_bitstring_hex_literal(unsigned char* buff, std::size_t buff_size, const char* str, const char* str_end); -long parse_bitstring_binary_literal(BitPtr buff, std::size_t buff_size, const char* str, const char* str_end); +long parse_bitstring_binary_literal(BitPtr buff, std::size_t buff_size_bits, const char* str, const char* str_end); void bits_sha256(BitPtr to, ConstBitPtr from, std::size_t size); @@ -554,11 +554,7 @@ class BitArray { set_same(0); } void set_zero_s() { - volatile uint8* p = data(); - auto x = m; - while (x--) { - *p++ = 0; - } + as_slice().fill_zero_secure(); } void set_ones() { set_same(1); diff --git a/crypto/common/refcnt.cpp b/crypto/common/refcnt.cpp index 7ef0857c..c6d1b888 100644 --- a/crypto/common/refcnt.cpp +++ b/crypto/common/refcnt.cpp @@ -29,6 +29,7 @@ Ref CntObject::clone() const { namespace detail { struct SafeDeleter { public: + thread_local static td::int64 delete_count; void retire(const CntObject *ptr) { if (is_active_) { to_delete_.push_back(ptr); @@ -39,9 +40,11 @@ struct SafeDeleter { is_active_ = false; }; delete ptr; + delete_count++; while (!to_delete_.empty()) { auto *ptr = to_delete_.back(); to_delete_.pop_back(); + delete_count++; delete ptr; } } @@ -50,6 +53,7 @@ struct SafeDeleter { std::vector to_delete_; bool is_active_{false}; }; +thread_local td::int64 SafeDeleter::delete_count{0}; TD_THREAD_LOCAL SafeDeleter *deleter; void safe_delete(const CntObject *ptr) { @@ -57,4 +61,7 @@ void safe_delete(const CntObject *ptr) { deleter->retire(ptr); } } // namespace detail +int64 ref_get_delete_count() { + return detail::SafeDeleter::delete_count; +} } // namespace td diff --git a/crypto/common/refcnt.hpp b/crypto/common/refcnt.hpp index ef50c3b9..953cc779 100644 --- a/crypto/common/refcnt.hpp +++ b/crypto/common/refcnt.hpp @@ -472,5 +472,6 @@ template void swap(Ref& r1, Ref& r2) { r1.swap(r2); } +int64 ref_get_delete_count(); } // namespace td diff --git a/crypto/common/refint.cpp b/crypto/common/refint.cpp index 8e06da5f..a2ccdaa6 100644 --- a/crypto/common/refint.cpp +++ b/crypto/common/refint.cpp @@ -261,10 +261,6 @@ int sgn(RefInt256 x) { return x->sgn(); } -RefInt256 make_refint(long long x) { - return td::RefInt256{true, td::Normalize(), x}; -} - RefInt256 zero_refint() { // static RefInt256 Zero = td::RefInt256{true, 0}; // return Zero; diff --git a/crypto/common/refint.h b/crypto/common/refint.h index 488ae97f..e3305c2f 100644 --- a/crypto/common/refint.h +++ b/crypto/common/refint.h @@ -113,8 +113,6 @@ RefInt256 make_refint(Args&&... args) { return td::RefInt256{true, std::forward(args)...}; } -extern RefInt256 make_refint(long long x); - extern RefInt256 zero_refint(); extern RefInt256 bits_to_refint(td::ConstBitPtr bits, int n, bool sgnd = false); diff --git a/crypto/ellcurve/p256.cpp b/crypto/ellcurve/p256.cpp new file mode 100644 index 00000000..de539372 --- /dev/null +++ b/crypto/ellcurve/p256.cpp @@ -0,0 +1,91 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ + +#include "p256.h" +#include "td/utils/check.h" +#include "td/utils/misc.h" +#include +#include +#include + +namespace td { + +td::Status p256_check_signature(td::Slice data, td::Slice public_key, td::Slice signature) { + CHECK(public_key.size() == 33); + CHECK(signature.size() == 64); + + EVP_PKEY_CTX* pctx = EVP_PKEY_CTX_new_id(EVP_PKEY_EC, nullptr); + if (pctx == nullptr) { + return td::Status::Error("Can't create EVP_PKEY_CTX"); + } + SCOPE_EXIT { + EVP_PKEY_CTX_free(pctx); + }; + if (EVP_PKEY_paramgen_init(pctx) <= 0) { + return td::Status::Error("EVP_PKEY_paramgen_init failed"); + } + if (EVP_PKEY_CTX_set_ec_paramgen_curve_nid(pctx, NID_X9_62_prime256v1) <= 0) { + return td::Status::Error("EVP_PKEY_CTX_set_ec_paramgen_curve_nid failed"); + } + EVP_PKEY* pkey = nullptr; + if (EVP_PKEY_paramgen(pctx, &pkey) <= 0) { + return td::Status::Error("EVP_PKEY_paramgen failed"); + } + SCOPE_EXIT { + EVP_PKEY_free(pkey); + }; + if (EVP_PKEY_set1_tls_encodedpoint(pkey, public_key.ubegin(), public_key.size()) <= 0) { + return td::Status::Error("Failed to import public key"); + } + EVP_MD_CTX* md_ctx = EVP_MD_CTX_new(); + if (md_ctx == nullptr) { + return td::Status::Error("Can't create EVP_MD_CTX"); + } + SCOPE_EXIT { + EVP_MD_CTX_free(md_ctx); + }; + if (EVP_DigestVerifyInit(md_ctx, nullptr, nullptr, nullptr, pkey) <= 0) { + return td::Status::Error("Can't init DigestVerify"); + } + ECDSA_SIG* sig = ECDSA_SIG_new(); + SCOPE_EXIT { + ECDSA_SIG_free(sig); + }; + unsigned char buf[33]; + buf[0] = 0; + std::copy(signature.ubegin(), signature.ubegin() + 32, buf + 1); + BIGNUM* r = BN_bin2bn(buf, 33, nullptr); + std::copy(signature.ubegin() + 32, signature.ubegin() + 64, buf + 1); + BIGNUM* s = BN_bin2bn(buf, 33, nullptr); + if (ECDSA_SIG_set0(sig, r, s) != 1) { + return td::Status::Error("Invalid signature"); + } + unsigned char* signature_encoded = nullptr; + int signature_len = i2d_ECDSA_SIG(sig, &signature_encoded); + if (signature_len <= 0) { + return td::Status::Error("Invalid signature"); + } + SCOPE_EXIT { + OPENSSL_free(signature_encoded); + }; + if (EVP_DigestVerify(md_ctx, signature_encoded, signature_len, data.ubegin(), data.size()) == 1) { + return td::Status::OK(); + } + return td::Status::Error("Wrong signature"); +} + +} // namespace td diff --git a/crypto/ellcurve/p256.h b/crypto/ellcurve/p256.h new file mode 100644 index 00000000..22d08be4 --- /dev/null +++ b/crypto/ellcurve/p256.h @@ -0,0 +1,26 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "td/utils/Slice.h" +#include "td/utils/Status.h" + +namespace td { + +td::Status p256_check_signature(td::Slice data, td::Slice public_key, td::Slice signature); + +} diff --git a/crypto/ellcurve/secp256k1.cpp b/crypto/ellcurve/secp256k1.cpp new file mode 100644 index 00000000..b98eea7b --- /dev/null +++ b/crypto/ellcurve/secp256k1.cpp @@ -0,0 +1,69 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ + +#include "secp256k1.h" +#include "td/utils/check.h" +#include "td/utils/logging.h" + +#include +#include +#include + +namespace td::secp256k1 { + +static const secp256k1_context* get_context() { + static secp256k1_context* ctx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY); + LOG_CHECK(ctx) << "Failed to create secp256k1_context"; + return ctx; +} + +bool ecrecover(const unsigned char* hash, const unsigned char* signature, unsigned char* public_key) { + const secp256k1_context* ctx = get_context(); + secp256k1_ecdsa_recoverable_signature ecdsa_signature; + if (signature[64] > 3 || + !secp256k1_ecdsa_recoverable_signature_parse_compact(ctx, &ecdsa_signature, signature, signature[64])) { + return false; + } + secp256k1_pubkey pubkey; + if (!secp256k1_ecdsa_recover(ctx, &pubkey, &ecdsa_signature, hash)) { + return false; + } + size_t len = 65; + secp256k1_ec_pubkey_serialize(ctx, public_key, &len, &pubkey, SECP256K1_EC_UNCOMPRESSED); + CHECK(len == 65); + return true; +} + +bool xonly_pubkey_tweak_add(const unsigned char* xonly_pubkey_bytes, const unsigned char* tweak, + unsigned char* output_pubkey_bytes) { + const secp256k1_context* ctx = get_context(); + + secp256k1_xonly_pubkey xonly_pubkey; + secp256k1_pubkey output_pubkey; + if (!secp256k1_xonly_pubkey_parse(ctx, &xonly_pubkey, xonly_pubkey_bytes)) { + return false; + } + if (!secp256k1_xonly_pubkey_tweak_add(ctx, &output_pubkey, &xonly_pubkey, tweak)) { + return false; + } + size_t len = 65; + secp256k1_ec_pubkey_serialize(ctx, output_pubkey_bytes, &len, &output_pubkey, SECP256K1_EC_UNCOMPRESSED); + CHECK(len == 65); + return true; +} + +} // namespace td::secp256k1 diff --git a/crypto/ellcurve/secp256k1.h b/crypto/ellcurve/secp256k1.h new file mode 100644 index 00000000..20f0b66b --- /dev/null +++ b/crypto/ellcurve/secp256k1.h @@ -0,0 +1,25 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +namespace td::secp256k1 { + +bool ecrecover(const unsigned char* hash, const unsigned char* signature, unsigned char* public_key); +bool xonly_pubkey_tweak_add(const unsigned char* xonly_pubkey_bytes, const unsigned char* tweak, + unsigned char* output_pubkey_bytes); + +} // namespace td::secp256k1 diff --git a/crypto/fift/Continuation.cpp b/crypto/fift/Continuation.cpp index 7e3b5ea2..e895082f 100644 --- a/crypto/fift/Continuation.cpp +++ b/crypto/fift/Continuation.cpp @@ -27,7 +27,7 @@ namespace fift { // bool FiftCont::print_dict_name(std::ostream& os, const IntCtx& ctx) const { std::string word_name; - if (ctx.dictionary && ctx.dictionary->lookup_def(this, &word_name)) { + if (ctx.dictionary.lookup_def(this, &word_name)) { if (word_name.size() && word_name.back() == ' ') { word_name.pop_back(); } @@ -39,7 +39,7 @@ bool FiftCont::print_dict_name(std::ostream& os, const IntCtx& ctx) const { std::string FiftCont::get_dict_name(const IntCtx& ctx) const { std::string word_name; - if (ctx.dictionary && ctx.dictionary->lookup_def(this, &word_name)) { + if (ctx.dictionary.lookup_def(this, &word_name)) { if (word_name.size() && word_name.back() == ' ') { word_name.pop_back(); } @@ -63,6 +63,140 @@ bool FiftCont::dump(std::ostream& os, const IntCtx& ctx) const { return ok; } +// +// StackWord +// +Ref StackWord::run_tail(IntCtx& ctx) const { + f(ctx.stack); + return {}; +} + +// +// CtxWord +// +Ref CtxWord::run_tail(IntCtx& ctx) const { + f(ctx); + return {}; +} + +// +// CtxTailWord +// +Ref CtxTailWord::run_tail(IntCtx& ctx) const { + return f(ctx); +} + +// +// WordList +// +WordList::WordList(std::vector>&& _list) : list(std::move(_list)) { +} + +WordList::WordList(const std::vector>& _list) : list(_list) { +} + +WordList& WordList::push_back(Ref word_def) { + list.push_back(std::move(word_def)); + return *this; +} + +WordList& WordList::push_back(FiftCont& wd) { + list.emplace_back(&wd); + return *this; +} + +Ref WordList::run_tail(IntCtx& ctx) const { + if (list.empty()) { + return {}; + } + if (list.size() > 1) { + ctx.next = td::make_ref(std::move(ctx.next), Ref(this), 1); + } + return list[0]; +} + +void WordList::close() { + list.shrink_to_fit(); +} + +WordList& WordList::append(const std::vector>& other) { + list.insert(list.end(), other.begin(), other.end()); + return *this; +} + +WordList& WordList::append(const Ref* begin, const Ref* end) { + list.insert(list.end(), begin, end); + return *this; +} + +bool WordList::dump(std::ostream& os, const IntCtx& ctx) const { + os << "{"; + for (auto entry : list) { + os << ' '; + entry->print_name(os, ctx); + } + os << " }" << std::endl; + return true; +} + +// +// ListCont +// + +Ref ListCont::run_tail(IntCtx& ctx) const { + auto sz = list->size(); + if (pos >= sz) { + return std::move(ctx.next); + } else if (ctx.next.not_null()) { + ctx.next = td::make_ref(SeqCont::seq(next, std::move(ctx.next)), list, pos + 1); + } else if (pos + 1 == sz) { + ctx.next = next; + } else { + ctx.next = td::make_ref(next, list, pos + 1); + } + return list->at(pos); +} + +Ref ListCont::run_modify(IntCtx& ctx) { + auto sz = list->size(); + if (pos >= sz) { + return std::move(ctx.next); + } + auto cur = list->at(pos++); + if (ctx.next.not_null()) { + next = SeqCont::seq(next, std::move(ctx.next)); + } + if (pos == sz) { + ctx.next = std::move(next); + } else { + ctx.next = self(); + } + return cur; +} + +bool ListCont::dump(std::ostream& os, const IntCtx& ctx) const { + std::string dict_name = list->get_dict_name(ctx); + if (!dict_name.empty()) { + os << "[in " << dict_name << ":] "; + } + std::size_t sz = list->size(), i, a = (pos >= 16 ? pos - 16 : 0), b = std::min(pos + 16, sz); + if (a > 0) { + os << "... "; + } + for (i = a; i < b; i++) { + if (i == pos) { + os << "**HERE** "; + } + list->at(i)->print_name(os, ctx); + os << ' '; + } + if (b < sz) { + os << "..."; + } + os << std::endl; + return true; +} + // // QuitCont // @@ -295,12 +429,15 @@ bool GenericLitCont::print_name(std::ostream& os, const IntCtx& ctx) const { bool sp = false; for (auto entry : list) { if (sp) { - os << sp; + os << ' '; } sp = true; int tp = entry.type(); if (entry.is_int() || entry.is(vm::StackEntry::t_string) || entry.is(vm::StackEntry::t_bytes)) { entry.dump(os); + } else if (entry.is_atom()) { + os << '`'; + entry.dump(os); } else { auto cont_lit = entry.as_object(); if (cont_lit.not_null()) { diff --git a/crypto/fift/Continuation.h b/crypto/fift/Continuation.h index f2c44e7b..6623b642 100644 --- a/crypto/fift/Continuation.h +++ b/crypto/fift/Continuation.h @@ -17,6 +17,7 @@ Copyright 2020 Telegram Systems LLP */ #pragma once +#include #include "common/refcnt.hpp" #include "common/refint.h" #include "vm/stack.hpp" @@ -76,6 +77,101 @@ class FiftCont : public td::CntObject { } }; +typedef std::function StackWordFunc; +typedef std::function CtxWordFunc; +typedef std::function(IntCtx&)> CtxTailWordFunc; + +class NopWord : public FiftCont { + public: + NopWord() = default; + ~NopWord() override = default; + Ref run_tail(IntCtx& ctx) const override { + return {}; + } +}; + +class StackWord : public FiftCont { + StackWordFunc f; + + public: + StackWord(StackWordFunc _f) : f(std::move(_f)) { + } + ~StackWord() override = default; + Ref run_tail(IntCtx& ctx) const override; +}; + +class CtxWord : public FiftCont { + CtxWordFunc f; + + public: + CtxWord(CtxWordFunc _f) : f(std::move(_f)) { + } + ~CtxWord() override = default; + Ref run_tail(IntCtx& ctx) const override; +}; + +class CtxTailWord : public FiftCont { + CtxTailWordFunc f; + + public: + CtxTailWord(CtxTailWordFunc _f) : f(std::move(_f)) { + } + ~CtxTailWord() override = default; + Ref run_tail(IntCtx& ctx) const override; +}; + +class WordList : public FiftCont { + std::vector> list; + + public: + ~WordList() override = default; + WordList() = default; + WordList(std::vector>&& _list); + WordList(const std::vector>& _list); + WordList& push_back(Ref word_def); + WordList& push_back(FiftCont& wd); + Ref run_tail(IntCtx& ctx) const override; + void close(); + bool is_list() const override { + return true; + } + long long list_size() const override { + return (long long)list.size(); + } + std::size_t size() const { + return list.size(); + } + const Ref& at(std::size_t idx) const { + return list.at(idx); + } + const Ref* get_list() const override { + return list.data(); + } + WordList& append(const std::vector>& other); + WordList& append(const Ref* begin, const Ref* end); + WordList* make_copy() const override { + return new WordList(list); + } + bool dump(std::ostream& os, const IntCtx& ctx) const override; +}; + +class ListCont : public FiftCont { + Ref next; + Ref list; + std::size_t pos; + + public: + ListCont(Ref nxt, Ref wl, std::size_t p = 0) : next(std::move(nxt)), list(std::move(wl)), pos(p) { + } + ~ListCont() override = default; + Ref run_tail(IntCtx& ctx) const override; + Ref run_modify(IntCtx& ctx) override; + Ref up() const override { + return next; + } + bool dump(std::ostream& os, const IntCtx& ctx) const override; +}; + class QuitCont : public FiftCont { int exit_code; diff --git a/crypto/fift/Dictionary.cpp b/crypto/fift/Dictionary.cpp index 59da278f..d2eae0a3 100644 --- a/crypto/fift/Dictionary.cpp +++ b/crypto/fift/Dictionary.cpp @@ -17,143 +17,10 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "Dictionary.h" +#include "IntCtx.h" namespace fift { -// -// StackWord -// -Ref StackWord::run_tail(IntCtx& ctx) const { - f(ctx.stack); - return {}; -} - -// -// CtxWord -// -Ref CtxWord::run_tail(IntCtx& ctx) const { - f(ctx); - return {}; -} - -// -// CtxTailWord -// -Ref CtxTailWord::run_tail(IntCtx& ctx) const { - return f(ctx); -} - -// -// WordList -// -WordList::WordList(std::vector>&& _list) : list(std::move(_list)) { -} - -WordList::WordList(const std::vector>& _list) : list(_list) { -} - -WordList& WordList::push_back(Ref word_def) { - list.push_back(std::move(word_def)); - return *this; -} - -WordList& WordList::push_back(FiftCont& wd) { - list.emplace_back(&wd); - return *this; -} - -Ref WordList::run_tail(IntCtx& ctx) const { - if (list.empty()) { - return {}; - } - if (list.size() > 1) { - ctx.next = td::make_ref(std::move(ctx.next), Ref(this), 1); - } - return list[0]; -} - -void WordList::close() { - list.shrink_to_fit(); -} - -WordList& WordList::append(const std::vector>& other) { - list.insert(list.end(), other.begin(), other.end()); - return *this; -} - -WordList& WordList::append(const Ref* begin, const Ref* end) { - list.insert(list.end(), begin, end); - return *this; -} - -bool WordList::dump(std::ostream& os, const IntCtx& ctx) const { - os << "{"; - for (auto entry : list) { - os << ' '; - entry->print_name(os, ctx); - } - os << " }" << std::endl; - return true; -} - -// -// ListCont -// - -Ref ListCont::run_tail(IntCtx& ctx) const { - auto sz = list->size(); - if (pos >= sz) { - return std::move(ctx.next); - } else if (ctx.next.not_null()) { - ctx.next = td::make_ref(SeqCont::seq(next, std::move(ctx.next)), list, pos + 1); - } else if (pos + 1 == sz) { - ctx.next = next; - } else { - ctx.next = td::make_ref(next, list, pos + 1); - } - return list->at(pos); -} - -Ref ListCont::run_modify(IntCtx& ctx) { - auto sz = list->size(); - if (pos >= sz) { - return std::move(ctx.next); - } - auto cur = list->at(pos++); - if (ctx.next.not_null()) { - next = SeqCont::seq(next, std::move(ctx.next)); - } - if (pos == sz) { - ctx.next = std::move(next); - } else { - ctx.next = self(); - } - return cur; -} - -bool ListCont::dump(std::ostream& os, const IntCtx& ctx) const { - std::string dict_name = list->get_dict_name(ctx); - if (!dict_name.empty()) { - os << "[in " << dict_name << ":] "; - } - std::size_t sz = list->size(), i, a = (pos >= 16 ? pos - 16 : 0), b = std::min(pos + 16, sz); - if (a > 0) { - os << "... "; - } - for (i = a; i < b; i++) { - if (i == pos) { - os << "**HERE** "; - } - list->at(i)->print_name(os, ctx); - os << ' '; - } - if (b < sz) { - os << "..."; - } - os << std::endl; - return true; -} - // // DictEntry // @@ -167,15 +34,49 @@ DictEntry::DictEntry(CtxWordFunc func, bool _act) : def(Ref{true, std:: DictEntry::DictEntry(CtxTailWordFunc func, bool _act) : def(Ref{true, std::move(func)}), active(_act) { } +DictEntry DictEntry::create_from(vm::StackEntry se) { + if (se.is_tuple()) { + auto& tuple = *se.as_tuple(); + if (tuple.size() == 1) { + auto def = tuple[0].as_object(); + if (def.not_null()) { + return DictEntry{std::move(def), true}; + } + } + } else { + auto def = std::move(se).as_object(); + if (def.not_null()) { + return DictEntry{std::move(def)}; + } + } + return {}; +} + +DictEntry::operator vm::StackEntry() const& { + if (def.is_null()) { + return {}; + } else if (active) { + return vm::make_tuple_ref(vm::StackEntry{vm::from_object, def}); + } else { + return {vm::from_object, def}; + } +} + +DictEntry::operator vm::StackEntry() && { + if (def.is_null()) { + return {}; + } else if (active) { + return vm::make_tuple_ref(vm::StackEntry{vm::from_object, std::move(def)}); + } else { + return {vm::from_object, std::move(def)}; + } +} + // // Dictionary // -DictEntry* Dictionary::lookup(td::Slice name) { - auto it = words_.find(name); - if (it == words_.end()) { - return nullptr; - } - return &it->second; +DictEntry Dictionary::lookup(std::string name) const { + return DictEntry::create_from(words().get(name)); } void Dictionary::def_ctx_word(std::string name, CtxWordFunc func) { @@ -196,26 +97,27 @@ void Dictionary::def_ctx_tail_word(std::string name, CtxTailWordFunc func) { } void Dictionary::def_word(std::string name, DictEntry word) { - auto res = words_.emplace(name, std::move(word)); - LOG_IF(FATAL, !res.second) << "Cannot redefine word: " << name; + auto dict = words(); + dict.set(std::move(name), vm::StackEntry(std::move(word))); + set_words(dict); } -void Dictionary::undef_word(td::Slice name) { - auto it = words_.find(name); - if (it == words_.end()) { - return; +void Dictionary::undef_word(std::string name) { + auto dict = words(); + if (dict.remove(name)) { + set_words(dict); } - words_.erase(it); } bool Dictionary::lookup_def(const FiftCont* cont, std::string* word_ptr) const { if (!cont) { return false; } - for (const auto& entry : words_) { - if (entry.second.get_def().get() == cont) { + for (auto entry : words()) { + auto val = DictEntry::create_from(entry.value()); + if (val.get_def().get() == cont && entry.key().is_string()) { if (word_ptr) { - *word_ptr = entry.first; + *word_ptr = vm::StackEntry(entry.key()).as_string(); } return true; } @@ -223,35 +125,4 @@ bool Dictionary::lookup_def(const FiftCont* cont, std::string* word_ptr) const { return false; } -void interpret_nop(vm::Stack& stack) { -} - -Ref Dictionary::nop_word_def = Ref{true, interpret_nop}; - -// -// functions for wordef -// -Ref pop_exec_token(vm::Stack& stack) { - stack.check_underflow(1); - auto wd_ref = stack.pop().as_object(); - if (wd_ref.is_null()) { - throw IntError{"execution token expected"}; - } - return wd_ref; -} - -Ref pop_word_list(vm::Stack& stack) { - stack.check_underflow(1); - auto wl_ref = stack.pop().as_object(); - if (wl_ref.is_null()) { - throw IntError{"word list expected"}; - } - return wl_ref; -} - -void push_argcount(vm::Stack& stack, int args) { - stack.push_smallint(args); - stack.push({vm::from_object, Dictionary::nop_word_def}); -} - } // namespace fift diff --git a/crypto/fift/Dictionary.h b/crypto/fift/Dictionary.h index 7307cdbe..b24bc742 100644 --- a/crypto/fift/Dictionary.h +++ b/crypto/fift/Dictionary.h @@ -17,115 +17,27 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once - -#include -#include - -#include "IntCtx.h" #include "Continuation.h" +#include "HashMap.h" +#include "vm/box.hpp" namespace fift { using td::Ref; +struct IntCtx; + /* * * WORD CLASSES * */ -typedef std::function StackWordFunc; -typedef std::function CtxWordFunc; - -class StackWord : public FiftCont { - StackWordFunc f; - - public: - StackWord(StackWordFunc _f) : f(std::move(_f)) { - } - ~StackWord() override = default; - Ref run_tail(IntCtx& ctx) const override; -}; - -class CtxWord : public FiftCont { - CtxWordFunc f; - - public: - CtxWord(CtxWordFunc _f) : f(std::move(_f)) { - } - ~CtxWord() override = default; - Ref run_tail(IntCtx& ctx) const override; -}; - -typedef std::function(IntCtx&)> CtxTailWordFunc; - -class CtxTailWord : public FiftCont { - CtxTailWordFunc f; - - public: - CtxTailWord(CtxTailWordFunc _f) : f(std::move(_f)) { - } - ~CtxTailWord() override = default; - Ref run_tail(IntCtx& ctx) const override; -}; - -class WordList : public FiftCont { - std::vector> list; - - public: - ~WordList() override = default; - WordList() = default; - WordList(std::vector>&& _list); - WordList(const std::vector>& _list); - WordList& push_back(Ref word_def); - WordList& push_back(FiftCont& wd); - Ref run_tail(IntCtx& ctx) const override; - void close(); - bool is_list() const override { - return true; - } - long long list_size() const override { - return (long long)list.size(); - } - std::size_t size() const { - return list.size(); - } - const Ref& at(std::size_t idx) const { - return list.at(idx); - } - const Ref* get_list() const override { - return list.data(); - } - WordList& append(const std::vector>& other); - WordList& append(const Ref* begin, const Ref* end); - WordList* make_copy() const override { - return new WordList(list); - } - bool dump(std::ostream& os, const IntCtx& ctx) const override; -}; - -class ListCont : public FiftCont { - Ref next; - Ref list; - std::size_t pos; - - public: - ListCont(Ref nxt, Ref wl, std::size_t p = 0) : next(std::move(nxt)), list(std::move(wl)), pos(p) { - } - ~ListCont() override = default; - Ref run_tail(IntCtx& ctx) const override; - Ref run_modify(IntCtx& ctx) override; - Ref up() const override { - return next; - } - bool dump(std::ostream& os, const IntCtx& ctx) const override; -}; - class DictEntry { Ref def; - bool active; + bool active{false}; public: - DictEntry() = delete; + DictEntry() = default; DictEntry(const DictEntry& ref) = default; DictEntry(DictEntry&& ref) = default; DictEntry(Ref _def, bool _act = false) : def(std::move(_def)), active(_act) { @@ -137,6 +49,9 @@ class DictEntry { //DictEntry(std::vector>&& word_list); DictEntry& operator=(const DictEntry&) = default; DictEntry& operator=(DictEntry&&) = default; + static DictEntry create_from(vm::StackEntry se); + explicit operator vm::StackEntry() const&; + explicit operator vm::StackEntry() &&; Ref get_def() const& { return def; } @@ -146,16 +61,17 @@ class DictEntry { bool is_active() const { return active; } + bool empty() const { + return def.is_null(); + } + explicit operator bool() const { + return def.not_null(); + } + bool operator!() const { + return def.is_null(); + } }; -/* -DictEntry::DictEntry(const std::vector>& word_list) : def(Ref{true, word_list}) { -} - -DictEntry::DictEntry(std::vector>&& word_list) : def(Ref{true, std::move(word_list)}) { -} -*/ - /* * * DICTIONARIES @@ -164,37 +80,52 @@ DictEntry::DictEntry(std::vector>&& word_list) : def(Ref class Dictionary { public: - DictEntry* lookup(td::Slice name); + Dictionary() : box_(true) { + } + Dictionary(Ref box) : box_(std::move(box)) { + } + Dictionary(Ref hmap) : box_(true, vm::from_object, std::move(hmap)) { + } + + DictEntry lookup(std::string name) const; void def_ctx_word(std::string name, CtxWordFunc func); void def_ctx_tail_word(std::string name, CtxTailWordFunc func); void def_active_word(std::string name, CtxWordFunc func); void def_stack_word(std::string name, StackWordFunc func); void def_word(std::string name, DictEntry word); - void undef_word(td::Slice name); + void undef_word(std::string name); bool lookup_def(const FiftCont* cont, std::string* word_ptr = nullptr) const; bool lookup_def(Ref cont, std::string* word_ptr = nullptr) const { return lookup_def(cont.get(), word_ptr); } auto begin() const { - return words_.begin(); + return words().begin(); } auto end() const { - return words_.end(); + return words().end(); + } + HashmapKeeper words() const { + if (box_->empty()) { + return {}; + } else { + return box_->get().as_object(); + } + } + Ref get_box() const { + return box_; + } + void set_words(Ref new_words) { + box_->set(vm::StackEntry{vm::from_object, std::move(new_words)}); + } + bool operator==(const Dictionary& other) const { + return box_ == other.box_; + } + bool operator!=(const Dictionary& other) const { + return box_ != other.box_; } - static Ref nop_word_def; - private: - std::map> words_; + Ref box_; }; -/* - * - * AUX FUNCTIONS FOR WORD DEFS - * - */ - -Ref pop_exec_token(vm::Stack& stack); -Ref pop_word_list(vm::Stack& stack); -void push_argcount(vm::Stack& stack, int args); } // namespace fift diff --git a/crypto/fift/Fift.cpp b/crypto/fift/Fift.cpp index ef265657..e0faca57 100644 --- a/crypto/fift/Fift.cpp +++ b/crypto/fift/Fift.cpp @@ -17,7 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "Fift.h" - +#include "IntCtx.h" #include "words.h" #include "td/utils/PathView.h" @@ -49,9 +49,8 @@ td::Result Fift::interpret_istream(std::istream& stream, std::string curren } td::Result Fift::do_interpret(IntCtx& ctx, bool is_interactive) { - ctx.ton_db = &config_.ton_db; ctx.source_lookup = &config_.source_lookup; - ctx.dictionary = &config_.dictionary; + ctx.dictionary = ctx.main_dictionary = ctx.context = config_.dictionary; ctx.output_stream = config_.output_stream; ctx.error_stream = config_.error_stream; if (!ctx.output_stream) { @@ -71,7 +70,7 @@ td::Result Fift::do_interpret(IntCtx& ctx, bool is_interactive) { ctx.top_ctx(); ctx.clear_error(); ctx.stack.clear(); - ctx.load_next_line(); + ctx.parser->load_next_line(); continue; } } diff --git a/crypto/fift/Fift.h b/crypto/fift/Fift.h index ebcf2ef4..a1772799 100644 --- a/crypto/fift/Fift.h +++ b/crypto/fift/Fift.h @@ -19,7 +19,6 @@ #pragma once #include "SourceLookup.h" -#include "vm/db/TonDb.h" #include "Dictionary.h" #include "td/utils/Status.h" @@ -31,13 +30,11 @@ struct Fift { public: struct Config { fift::SourceLookup source_lookup; - vm::TonDb ton_db; fift::Dictionary dictionary; std::ostream* output_stream{&std::cout}; std::ostream* error_stream{&std::cerr}; bool show_backtrace{true}; }; - // Fift must own ton_db and dictionary, no concurrent access is allowed explicit Fift(Config config); td::Result interpret_file(std::string fname, std::string current_dir, bool interactive = false); diff --git a/crypto/fift/HashMap.cpp b/crypto/fift/HashMap.cpp new file mode 100644 index 00000000..a4ca0e9b --- /dev/null +++ b/crypto/fift/HashMap.cpp @@ -0,0 +1,371 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "HashMap.h" +#include "td/utils/Random.h" +#include "IntCtx.h" + +namespace fift { +using td::Ref; + +DictKey::DictKey(vm::StackEntry se) { + auto tp = tp_ = se.type(); + switch (tp) { + case Type::t_int: + ref_ = se.as_int(); + break; + case Type::t_atom: + ref_ = se.as_atom(); + break; + case Type::t_string: + ref_ = se.as_string_ref(); + break; + case Type::t_bytes: + ref_ = se.as_bytes_ref(); + break; + case Type::t_null: + break; + default: + throw IntError{"unsupported key type"}; + } + compute_hash(); +} + +DictKey::operator vm::StackEntry() const& { + switch (tp_) { + case Type::t_int: + return value(); + case Type::t_atom: + return value(); + case Type::t_string: + case Type::t_bytes: + return {value>(), tp_ == Type::t_bytes}; + default: + return {}; + } +} + +DictKey::operator vm::StackEntry() && { + switch (tp_) { + case Type::t_int: + return move_value(); + case Type::t_atom: + return move_value(); + case Type::t_string: + case Type::t_bytes: + return {move_value>(), tp_ == Type::t_bytes}; + default: + return {}; + } +} + +std::ostream& operator<<(std::ostream& os, const DictKey& dkey) { + return os << vm::StackEntry(dkey).to_string(); +} + +int DictKey::cmp_internal(const DictKey& other) const { + if (tp_ != other.tp_) { + return tp_ < other.tp_ ? -1 : 1; + } + switch (tp_) { + case Type::t_int: + return td::cmp(value(), other.value()); + case Type::t_atom: { + int u = value()->index(), v = other.value()->index(); + return u == v ? 0 : (u < v ? -1 : 1); + } + case Type::t_string: + case Type::t_bytes: + return value>()->compare(*other.value>()); + default: + return 0; + } +} + +int DictKey::cmp(const DictKey& other) const { + if (hash_ < other.hash_) { + return -1; + } else if (hash_ > other.hash_) { + return 1; + } else { + return cmp_internal(other); + } +} + +DictKey::keyhash_t DictKey::compute_str_hash(DictKey::keyhash_t h, const char* str, std::size_t len) { + const char* end = str + len; + while (str < end) { + h = h * StrHash + (unsigned char)*str++; + } + return h; +} + +DictKey::keyhash_t DictKey::compute_int_hash(td::AnyIntView<> x) { + keyhash_t h = IntHash0; + for (int i = 0; i < x.size(); i++) { + h = h * MixConst3 + x.digits[i]; + } + return h * MixConst4; +} + +DictKey::keyhash_t DictKey::compute_hash() { + switch (tp_) { + case Type::t_int: + return hash_ = compute_int_hash(value()->as_any_int()); + case Type::t_atom: + return hash_ = value()->index() * MixConst1 + MixConst2; + case Type::t_string: + case Type::t_bytes: { + auto ref = value>(); + return hash_ = compute_str_hash(tp_, ref->data(), ref->size()); + } + default: + return hash_ = 0; + } +} + +const Hashmap* Hashmap::lookup_key_aux(const Hashmap* root, const DictKey& key) { + if (key.is_null()) { + return nullptr; + } + while (root) { + int r = key.cmp(root->key_); + if (!r) { + break; + } + root = (r < 0 ? root->left_.get() : root->right_.get()); + } + return root; +} + +Ref Hashmap::lookup_key(Ref root, const DictKey& key) { + return Ref(lookup_key_aux(root.get(), key)); +} + +vm::StackEntry Hashmap::get_key(Ref root, const DictKey& key) { + auto node = lookup_key_aux(root.get(), key); + if (node) { + return node->value_; + } else { + return {}; + } +} + +std::pair, vm::StackEntry> Hashmap::get_remove_key(Ref root, const DictKey& key) { + if (root.is_null() || key.is_null()) { + return {std::move(root), {}}; + } + vm::StackEntry val; + auto res = root->get_remove_internal(key, val); + if (val.is_null()) { + return {std::move(root), {}}; + } else { + return {std::move(res), std::move(val)}; + } +} + +Ref Hashmap::remove_key(Ref root, const DictKey& key) { + if (root.is_null() || key.is_null()) { + return root; + } + vm::StackEntry val; + auto res = root->get_remove_internal(key, val); + if (val.is_null()) { + return root; + } else { + return res; + } +} + +Ref Hashmap::get_remove_internal(const DictKey& key, vm::StackEntry& val) const { + int r = key.cmp(key_); + if (!r) { + val = value_; + return merge(left_, right_); + } else if (r < 0) { + if (left_.is_null()) { + return {}; + } else { + auto res = left_->get_remove_internal(key, val); + if (val.is_null()) { + return res; + } else { + return td::make_ref(key_, value_, std::move(res), right_, y_); + } + } + } else if (right_.is_null()) { + return {}; + } else { + auto res = right_->get_remove_internal(key, val); + if (val.is_null()) { + return res; + } else { + return td::make_ref(key_, value_, left_, std::move(res), y_); + } + } +} + +Ref Hashmap::merge(Ref a, Ref b) { + if (a.is_null()) { + return b; + } else if (b.is_null()) { + return a; + } else if (a->y_ > b->y_) { + auto& aa = a.write(); + aa.right_ = merge(std::move(aa.right_), std::move(b)); + return a; + } else { + auto& bb = b.write(); + bb.left_ = merge(std::move(a), std::move(bb.left_)); + return b; + } +} + +Ref Hashmap::set(Ref root, const DictKey& key, vm::StackEntry value) { + if (!key.is_null() && !replace(root, key, value) && !value.is_null()) { + insert(root, key, value, new_y()); + } + return root; +} + +bool Hashmap::replace(Ref& root, const DictKey& key, vm::StackEntry value) { + if (root.is_null() || key.is_null()) { + return false; + } + if (value.is_null()) { + auto res = root->get_remove_internal(key, value); + if (value.is_null()) { + return false; + } else { + root = std::move(res); + return true; + } + } + bool found = false; + auto res = root->replace_internal(key, std::move(value), found); + if (found) { + root = std::move(res); + } + return found; +} + +Ref Hashmap::replace_internal(const DictKey& key, const vm::StackEntry& value, bool& found) const { + int r = key.cmp(key_); + if (!r) { + found = true; + return td::make_ref(key_, value, left_, right_, y_); + } else if (r < 0) { + if (left_.is_null()) { + found = false; + return {}; + } + auto res = left_->replace_internal(key, value, found); + if (!found) { + return {}; + } + return td::make_ref(key_, value_, std::move(res), right_, y_); + } else { + if (right_.is_null()) { + found = false; + return {}; + } + auto res = right_->replace_internal(key, value, found); + if (!found) { + return {}; + } + return td::make_ref(key_, value_, left_, std::move(res), y_); + } +} + +void Hashmap::insert(Ref& root, const DictKey& key, vm::StackEntry value, long long y) { + if (root.is_null()) { + root = td::make_ref(key, std::move(value), empty(), empty(), y); + return; + } + if (root->y_ <= y) { + auto res = split(std::move(root), key); + root = td::make_ref(key, std::move(value), std::move(res.first), std::move(res.second), y); + return; + } + int r = key.cmp(root->key_); + CHECK(r); + insert(r < 0 ? root.write().left_ : root.write().right_, key, std::move(value), y); +} + +std::pair, Ref> Hashmap::split(Ref root, const DictKey& key, bool cmpv) { + if (root.is_null()) { + return {{}, {}}; + } + int r = key.cmp(root->key_); + if (r < (int)cmpv) { + if (root->left_.is_null()) { + return {{}, std::move(root)}; + } + auto res = split(root->left_, key, cmpv); + return {std::move(res.first), + td::make_ref(root->key_, root->value_, std::move(res.second), root->right_, root->y_)}; + } else { + if (root->right_.is_null()) { + return {std::move(root), {}}; + } + auto res = split(root->right_, key, cmpv); + return {td::make_ref(root->key_, root->value_, root->left_, std::move(res.first), root->y_), + std::move(res.second)}; + } +} + +long long Hashmap::new_y() { + return td::Random::fast_uint64(); +} + +bool HashmapIterator::unwind(Ref root) { + if (root.is_null()) { + return false; + } + while (true) { + auto left = root->lr(down_); + if (left.is_null()) { + cur_ = std::move(root); + return true; + } + stack_.push_back(std::move(root)); + root = std::move(left); + } +} + +bool HashmapIterator::next() { + if (cur_.not_null()) { + cur_ = cur_->rl(down_); + if (cur_.not_null()) { + while (true) { + auto left = cur_->lr(down_); + if (left.is_null()) { + return true; + } + stack_.push_back(std::move(cur_)); + cur_ = std::move(left); + } + } + } + if (stack_.empty()) { + return false; + } + cur_ = std::move(stack_.back()); + stack_.pop_back(); + return true; +} + +} // namespace fift diff --git a/crypto/fift/HashMap.h b/crypto/fift/HashMap.h new file mode 100644 index 00000000..d3d9ffbe --- /dev/null +++ b/crypto/fift/HashMap.h @@ -0,0 +1,306 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "common/refcnt.hpp" +#include "vm/stack.hpp" +#include "vm/atom.h" + +namespace fift { + +using td::Ref; +using td::RefAny; + +class DictKey { + public: + typedef vm::StackEntry::Type Type; + typedef unsigned long long keyhash_t; + + private: + RefAny ref_; + Type tp_ = Type::t_null; + keyhash_t hash_ = 0; + + static constexpr keyhash_t IntHash0 = 0xce6ab89d724409ed, MixConst1 = 0xcd5c126501510979, + MixConst2 = 0xb8f44d7fd6274ad1, MixConst3 = 0xd08726ea2422e405, + MixConst4 = 0x6407d2aeb5039dfb, StrHash = 0x93ff128344add06d; + keyhash_t compute_hash(); + static keyhash_t compute_str_hash(DictKey::keyhash_t h, const char* str, std::size_t len); + static keyhash_t compute_int_hash(td::AnyIntView<> x); + int cmp_internal(const DictKey& other) const; + template + Ref value() const { + return Ref{td::static_cast_ref(), ref_}; + } + template + Ref move_value() { + return Ref{td::static_cast_ref(), std::move(ref_)}; + } + + public: + DictKey() : ref_(), tp_(Type::t_null) { + } + DictKey(const DictKey& other) = default; + DictKey(DictKey&& other) = default; + DictKey& operator=(const DictKey& other) = default; + DictKey& operator=(DictKey&& other) = default; + DictKey(Ref atom_ref) : ref_(std::move(atom_ref)), tp_(Type::t_atom) { + compute_hash(); + } + DictKey(td::RefInt256 int_ref) : ref_(std::move(int_ref)), tp_(Type::t_int) { + compute_hash(); + } + explicit DictKey(vm::StackEntry se); + DictKey(std::string str, bool bytes = false) : ref_(), tp_(bytes ? Type::t_bytes : Type::t_string) { + ref_ = Ref>{true, std::move(str)}; + compute_hash(); + } + Type type() const { + return tp_; + } + void swap(DictKey& other) { + ref_.swap(other.ref_); + std::swap(tp_, other.tp_); + } + + operator vm::StackEntry() const&; + operator vm::StackEntry() &&; + int cmp(const DictKey& other) const; + bool operator==(const DictKey& other) const { + return hash_ == other.hash_ && !cmp_internal(other); + } + bool operator!=(const DictKey& other) const { + return hash_ != other.hash_ || cmp_internal(other); + } + bool operator<(const DictKey& other) const { + return hash_ < other.hash_ || (hash_ == other.hash_ && cmp_internal(other) < 0); + } + bool is_null() const { + return tp_ == Type::t_null; + } + bool is_string() const { + return tp_ == Type::t_string; + } +}; + +std::ostream& operator<<(std::ostream& os, const DictKey& dkey); + +class Hashmap : public td::CntObject { + DictKey key_; + vm::StackEntry value_; + Ref left_; + Ref right_; + long long y_; + + public: + Hashmap(DictKey key, vm::StackEntry value, Ref left, Ref right, long long y) + : key_(std::move(key)), value_(std::move(value)), left_(std::move(left)), right_(std::move(right)), y_(y) { + } + Hashmap(const Hashmap& other) = default; + Hashmap(Hashmap&& other) = default; + virtual ~Hashmap() { + } + Hashmap* make_copy() const override { + return new Hashmap(*this); + } + const DictKey& key() const& { + return key_; + } + DictKey key() && { + return std::move(key_); + } + const vm::StackEntry& value() const& { + return value_; + } + vm::StackEntry value() && { + return std::move(value_); + } + Ref left() const { + return left_; + } + Ref right() const { + return right_; + } + Ref lr(bool branch) const { + return branch ? right_ : left_; + } + Ref rl(bool branch) const { + return branch ? left_ : right_; + } + static Ref lookup_key(Ref root, const DictKey& key); + template + static Ref lookup(Ref root, Args&&... args) { + return lookup_key(std::move(root), DictKey{std::forward(args)...}); + } + static vm::StackEntry get_key(Ref root, const DictKey& key); + template + static vm::StackEntry get(Ref root, Args&&... args) { + return get_key(std::move(root), DictKey{std::forward(args)...}); + } + static Ref remove_key(Ref root, const DictKey& key); + template + static Ref remove(Ref root, Args&&... args) { + return remove_key(std::move(root), DictKey{std::forward(args)...}); + } + static std::pair, vm::StackEntry> get_remove_key(Ref root, const DictKey& key); + template + static std::pair, vm::StackEntry> get_remove(Ref root, Args&&... args) { + return get_remove_key(std::move(root), DictKey{std::forward(args)...}); + } + static Ref set(Ref root, const DictKey& key, vm::StackEntry value); + static bool replace(Ref& root, const DictKey& key, vm::StackEntry value); + static std::pair, Ref> split(Ref root, const DictKey& key, bool eq_left = false); + static Ref empty() { + return {}; + } + + private: + static Ref merge(Ref a, Ref b); + static const Hashmap* lookup_key_aux(const Hashmap* root, const DictKey& key); + Ref get_remove_internal(const DictKey& key, vm::StackEntry& val) const; + Ref replace_internal(const DictKey& key, const vm::StackEntry& value, bool& found) const; + static void insert(Ref& root, const DictKey& key, vm::StackEntry value, long long y); + static long long new_y(); +}; + +struct HashmapIdx { + Ref& root_; + DictKey idx_; + template + HashmapIdx(Ref& root, Args&&... args) : root_(root), idx_(std::forward(args)...) { + } + operator vm::StackEntry() const { + return Hashmap::get(root_, idx_); + } + template + HashmapIdx& operator=(T&& value) { + root_ = Hashmap::set(root_, idx_, vm::StackEntry(std::forward(value))); + return *this; + } +}; + +class HashmapIterator { + std::vector> stack_; + Ref cur_; + const bool down_{false}; + bool unwind(Ref root); + + public: + HashmapIterator() = default; + HashmapIterator(Ref root, bool down = false) : down_(down) { + unwind(std::move(root)); + } + const Hashmap& operator*() const { + return *cur_; + } + const Hashmap* operator->() const { + return cur_.get(); + } + bool eof() { + return cur_.is_null(); + } + bool next(); + bool operator<(const HashmapIterator& other) const { + if (other.cur_.is_null()) { + return cur_.not_null(); + } else if (cur_.is_null()) { + return false; + } else { + return cur_->key().cmp(other.cur_->key()) * (down_ ? -1 : 1) < 0; + } + } + bool operator==(const HashmapIterator& other) const { + return other.cur_.is_null() ? cur_.is_null() : (cur_.not_null() && cur_->key() == other.cur_->key()); + } + bool operator!=(const HashmapIterator& other) const { + return other.cur_.is_null() ? cur_.not_null() : (cur_.is_null() || cur_->key() != other.cur_->key()); + } + HashmapIterator& operator++() { + next(); + return *this; + } +}; + +struct HashmapKeeper { + Ref root; + HashmapKeeper() = default; + HashmapKeeper(Ref _root) : root(std::move(_root)) { + } + Ref extract() { + return std::move(root); + } + operator Ref() const& { + return root; + } + operator Ref() && { + return std::move(root); + } + template + HashmapIdx operator[](Args&&... args) { + return HashmapIdx{root, DictKey{std::forward(args)...}}; + } + template + vm::StackEntry operator[](Args&&... args) const { + return Hashmap::get(root, DictKey{std::forward(args)...}); + } + vm::StackEntry get_key(const DictKey& key) const { + return Hashmap::get(root, key); + } + template + vm::StackEntry get(Args&&... args) const { + return Hashmap::get(root, DictKey{std::forward(args)...}); + } + vm::StackEntry get_remove_key(const DictKey& key) { + auto res = Hashmap::get_remove_key(root, key); + root = std::move(res.first); + return std::move(res.second); + } + template + vm::StackEntry get_remove(Args&&... args) { + return get_remove_key(DictKey{std::forward(args)...}); + } + bool remove_key(const DictKey& key) { + auto res = Hashmap::get_remove(root, key); + root = std::move(res.first); + return !res.second.is_null(); + } + template + bool remove(Args&&... args) { + return remove_key(DictKey{std::forward(args)...}); + } + template + void set(T key, vm::StackEntry value) { + root = Hashmap::set(root, DictKey(key), std::move(value)); + } + template + bool replace(T key, vm::StackEntry value) { + return Hashmap::replace(root, DictKey(key), std::move(value)); + } + HashmapIterator begin(bool reverse = false) const { + return HashmapIterator{root, reverse}; + } + HashmapIterator end() const { + return HashmapIterator{}; + } + HashmapIterator rbegin() const { + return HashmapIterator{root, true}; + } + HashmapIterator rend() const { + return HashmapIterator{}; + } +}; + +} // namespace fift diff --git a/crypto/fift/IntCtx.cpp b/crypto/fift/IntCtx.cpp index 9d623b06..2cac8849 100644 --- a/crypto/fift/IntCtx.cpp +++ b/crypto/fift/IntCtx.cpp @@ -20,7 +20,7 @@ namespace fift { -td::StringBuilder& operator<<(td::StringBuilder& os, const IntCtx& ctx) { +td::StringBuilder& operator<<(td::StringBuilder& os, const ParseCtx& ctx) { if (ctx.include_depth) { return os << ctx.filename << ":" << ctx.line_no << ": "; } else { @@ -28,7 +28,7 @@ td::StringBuilder& operator<<(td::StringBuilder& os, const IntCtx& ctx) { } } -std::ostream& operator<<(std::ostream& os, const IntCtx& ctx) { +std::ostream& operator<<(std::ostream& os, const ParseCtx& ctx) { return os << (PSLICE() << ctx).c_str(); } @@ -67,73 +67,7 @@ void CharClassifier::set_char_class(int c, int cl) { *p = static_cast((*p & ~mask) | cl); } -IntCtx::Savepoint::Savepoint(IntCtx& _ctx, std::string new_filename, std::string new_current_dir, - std::unique_ptr new_input_stream) - : ctx(_ctx) - , old_line_no(_ctx.line_no) - , old_need_line(_ctx.need_line) - , old_filename(_ctx.filename) - , old_current_dir(_ctx.currentd_dir) - , old_input_stream(_ctx.input_stream) - , old_input_stream_holder(std::move(_ctx.input_stream_holder)) - , old_curline(_ctx.str) - , old_curpos(_ctx.input_ptr - _ctx.str.c_str()) - , old_word(_ctx.word) { - ctx.line_no = 0; - ctx.filename = new_filename; - ctx.currentd_dir = new_current_dir; - ctx.input_stream = new_input_stream.get(); - ctx.input_stream_holder = std::move(new_input_stream); - ctx.str = ""; - ctx.input_ptr = 0; - ++(ctx.include_depth); -} - -bool IntCtx::Savepoint::restore(IntCtx& _ctx) { - if (restored || &ctx != &_ctx) { - return false; - } - ctx.line_no = old_line_no; - ctx.need_line = old_need_line; - ctx.filename = old_filename; - ctx.currentd_dir = old_current_dir; - ctx.input_stream = old_input_stream; - ctx.input_stream_holder = std::move(old_input_stream_holder); - ctx.str = old_curline; - ctx.input_ptr = ctx.str.c_str() + old_curpos; - ctx.word = old_word; - --(ctx.include_depth); - return restored = true; -} - -bool IntCtx::enter_ctx(std::string new_filename, std::string new_current_dir, - std::unique_ptr new_input_stream) { - if (!new_input_stream) { - return false; - } - ctx_save_stack.emplace_back(*this, std::move(new_filename), std::move(new_current_dir), std::move(new_input_stream)); - return true; -} - -bool IntCtx::leave_ctx() { - if (ctx_save_stack.empty()) { - return false; - } - bool ok = ctx_save_stack.back().restore(*this); - ctx_save_stack.pop_back(); - return ok; -} - -bool IntCtx::top_ctx() { - while (!ctx_save_stack.empty()) { - if (!leave_ctx()) { - return false; - } - } - return true; -} - -bool IntCtx::load_next_line() { +bool ParseCtx::load_next_line() { if (!std::getline(*input_stream, str)) { return false; } @@ -145,11 +79,11 @@ bool IntCtx::load_next_line() { return true; } -bool IntCtx::is_sb() const { +bool ParseCtx::is_sb() const { return !eof() && line_no == 1 && *input_ptr == '#' && input_ptr[1] == '!'; } -td::Slice IntCtx::scan_word_to(char delim, bool err_endl) { +td::Slice ParseCtx::scan_word_to(char delim, bool err_endl) { load_next_line_ifreq(); auto ptr = input_ptr; while (*ptr && *ptr != delim) { @@ -167,7 +101,7 @@ td::Slice IntCtx::scan_word_to(char delim, bool err_endl) { } } -td::Slice IntCtx::scan_word() { +td::Slice ParseCtx::scan_word() { skipspc(true); auto ptr = input_ptr; while (*ptr && *ptr != ' ' && *ptr != '\t' && *ptr != '\r') { @@ -179,7 +113,7 @@ td::Slice IntCtx::scan_word() { return td::Slice{ptr, ptr2}; } -td::Slice IntCtx::scan_word_ext(const CharClassifier& classifier) { +td::Slice ParseCtx::scan_word_ext(const CharClassifier& classifier) { skipspc(true); auto ptr = input_ptr; while (*ptr && *ptr != '\r' && *ptr != '\n') { @@ -196,7 +130,7 @@ td::Slice IntCtx::scan_word_ext(const CharClassifier& classifier) { return td::Slice{ptr, input_ptr}; } -void IntCtx::skipspc(bool skip_eol) { +void ParseCtx::skipspc(bool skip_eol) { do { while (*input_ptr == ' ' || *input_ptr == '\t' || *input_ptr == '\r') { ++input_ptr; @@ -207,6 +141,45 @@ void IntCtx::skipspc(bool skip_eol) { } while (load_next_line()); } +bool IntCtx::enter_ctx(std::unique_ptr new_parser) { + if (!new_parser) { + return false; + } + if (parser) { + parser_save_stack.push_back(std::move(parser)); + } + parser = std::move(new_parser); + return true; +} + +bool IntCtx::enter_ctx(std::string new_filename, std::string new_current_dir, + std::unique_ptr new_input_stream) { + if (!new_input_stream) { + return false; + } else { + return enter_ctx( + std::make_unique(std::move(new_input_stream), new_filename, new_current_dir, include_depth() + 1)); + } +} + +bool IntCtx::leave_ctx() { + if (parser_save_stack.empty()) { + return false; + } else { + parser = std::move(parser_save_stack.back()); + parser_save_stack.pop_back(); + return true; + } +} + +bool IntCtx::top_ctx() { + if (!parser_save_stack.empty()) { + parser = std::move(parser_save_stack[0]); + parser_save_stack.clear(); + } + return true; +} + void IntCtx::check_compile() const { if (state <= 0) { throw IntError{"compilation mode only"}; @@ -283,15 +256,20 @@ td::Result IntCtx::get_result() { } } +std::ostream& ParseCtx::show_context(std::ostream& os) const { + if (include_depth && line_no) { + os << filename << ":" << line_no << ":\t"; + } + if (!word.empty()) { + os << word << ":"; + } + return os; +} + td::Status IntCtx::add_error_loc(td::Status err) const { - if (err.is_error()) { + if (err.is_error() && parser) { std::ostringstream os; - if (include_depth && line_no) { - os << filename << ":" << line_no << ":\t"; - } - if (!word.empty()) { - os << word << ":"; - } + parser->show_context(os); return err.move_as_error_prefix(os.str()); } else { return err; diff --git a/crypto/fift/IntCtx.h b/crypto/fift/IntCtx.h index 807a5d01..5d574ae5 100644 --- a/crypto/fift/IntCtx.h +++ b/crypto/fift/IntCtx.h @@ -18,8 +18,8 @@ */ #pragma once -#include "crypto/vm/db/TonDb.h" // FIXME #include "crypto/vm/stack.hpp" +#include "crypto/vm/box.hpp" #include "crypto/common/bitstring.h" #include "td/utils/Status.h" @@ -32,6 +32,11 @@ #include #include +namespace vm { +class TonDbImpl; // from crypto/vm/db/TonDb.h +using TonDb = std::unique_ptr; +} // namespace vm + namespace fift { class Dictionary; class SourceLookup; @@ -68,71 +73,36 @@ class CharClassifier { } }; -struct IntCtx { - vm::Stack stack; - Ref next, exc_handler; - Ref exc_cont, exc_next; - int state{0}; +struct ParseCtx { int include_depth{0}; int line_no{0}; - int exit_code{0}; - td::Status error; bool need_line{true}; std::string filename; std::string currentd_dir; std::istream* input_stream{nullptr}; std::unique_ptr input_stream_holder; - std::ostream* output_stream{nullptr}; - std::ostream* error_stream{nullptr}; - - vm::TonDb* ton_db{nullptr}; - Dictionary* dictionary{nullptr}; - SourceLookup* source_lookup{nullptr}; - int* now{nullptr}; std::string word; private: std::string str; const char* input_ptr = nullptr; - class Savepoint { - IntCtx& ctx; - int old_line_no; - bool old_need_line; - bool restored{false}; - std::string old_filename; - std::string old_current_dir; - std::istream* old_input_stream; - std::unique_ptr old_input_stream_holder; - std::string old_curline; - std::ptrdiff_t old_curpos; - std::string old_word; - - public: - Savepoint(IntCtx& _ctx, std::string new_filename, std::string new_current_dir, - std::unique_ptr new_input_stream); - bool restore(IntCtx& _ctx); - }; - - std::vector ctx_save_stack; - public: - IntCtx() = default; - IntCtx(std::istream& _istream, std::string _filename, std::string _curdir = "", int _depth = 0) + ParseCtx() = default; + ParseCtx(std::istream& _istream, std::string _filename, std::string _curdir = "", int _depth = 0) : include_depth(_depth) , filename(std::move(_filename)) , currentd_dir(std::move(_curdir)) , input_stream(&_istream) { } - - operator vm::Stack&() { - return stack; + ParseCtx(std::unique_ptr _istream_ptr, std::string _filename, std::string _curdir = "", int _depth = 0) + : include_depth(_depth) + , filename(std::move(_filename)) + , currentd_dir(std::move(_curdir)) + , input_stream(_istream_ptr.get()) + , input_stream_holder(std::move(_istream_ptr)) { } - bool enter_ctx(std::string new_filename, std::string new_current_dir, std::unique_ptr new_input_stream); - bool leave_ctx(); - bool top_ctx(); - td::Slice scan_word_to(char delim, bool err_endl = true); td::Slice scan_word(); td::Slice scan_word_ext(const CharClassifier& classifier); @@ -165,6 +135,50 @@ struct IntCtx { bool is_sb() const; + std::ostream& show_context(std::ostream& os) const; +}; + +struct IntCtx { + vm::Stack stack; + Ref next, exc_handler; + Ref exc_cont, exc_next; + int state{0}; + int exit_code{0}; + td::Status error; + + std::unique_ptr parser; + std::vector> parser_save_stack; + + std::ostream* output_stream{nullptr}; // move to OutCtx? + std::ostream* error_stream{nullptr}; + + vm::TonDb* ton_db{nullptr}; + SourceLookup* source_lookup{nullptr}; + int* now{nullptr}; + + Dictionary dictionary, main_dictionary, context; + + public: + IntCtx() = default; + IntCtx(std::istream& _istream, std::string _filename, std::string _curdir = "", int _depth = 0) { + parser = std::make_unique(_istream, _filename, _curdir, _depth); + } + IntCtx(std::unique_ptr _istream, std::string _filename, std::string _curdir = "", int _depth = 0) { + parser = std::make_unique(std::move(_istream), _filename, _curdir, _depth); + } + + bool enter_ctx(std::unique_ptr new_ctx); + bool enter_ctx(std::string new_filename, std::string new_current_dir, std::unique_ptr new_input_stream); + bool leave_ctx(); + bool top_ctx(); + int include_depth() const { + return parser ? parser->include_depth : -1; + } + + operator vm::Stack &() { + return stack; + } + void clear() { state = 0; stack.clear(); @@ -194,6 +208,6 @@ struct IntCtx { td::Result run(Ref cont); }; -td::StringBuilder& operator<<(td::StringBuilder& os, const IntCtx& ctx); -std::ostream& operator<<(std::ostream& os, const IntCtx& ctx); +td::StringBuilder& operator<<(td::StringBuilder& os, const ParseCtx& ctx); +std::ostream& operator<<(std::ostream& os, const ParseCtx& ctx); } // namespace fift diff --git a/crypto/fift/fift-main.cpp b/crypto/fift/fift-main.cpp index ef833f43..cdc36fc0 100644 --- a/crypto/fift/fift-main.cpp +++ b/crypto/fift/fift-main.cpp @@ -46,8 +46,6 @@ #include "SourceLookup.h" #include "words.h" -#include "vm/db/TonDb.h" - #include "td/utils/logging.h" #include "td/utils/misc.h" #include "td/utils/Parser.h" @@ -62,10 +60,9 @@ void usage(const char* progname) { << " [-i] [-n] [-I ] {-L } ...\n"; std::cerr << "\t-n\tDo not preload standard preamble file `Fift.fif`\n" "\t-i\tForce interactive mode even if explicit source file names are indicated\n" - "\t-I\tSets colon-separated library source include path. If not indicated, " + "\t-I\tSets colon-separated (unix) or at-separated (windows) library source include path. If not indicated, " "$FIFTPATH is used instead.\n" "\t-L\tPre-loads a library source file\n" - "\t-d\tUse a ton database\n" "\t-s\tScript mode: use first argument as a fift source file and import remaining arguments as $n)\n" "\t-v\tSet verbosity level\n" "\t-V\tShow fift build information\n"; @@ -75,11 +72,16 @@ void usage(const char* progname) { void parse_include_path_set(std::string include_path_set, std::vector& res) { td::Parser parser(include_path_set); while (!parser.empty()) { - auto path = parser.read_till_nofail(':'); + #if TD_WINDOWS + auto path_separator = '@'; + #else + auto path_separator = ':'; + #endif + auto path = parser.read_till_nofail(path_separator); if (!path.empty()) { res.push_back(path.str()); } - parser.skip_nofail(':'); + parser.skip_nofail(path_separator); } } @@ -89,13 +91,12 @@ int main(int argc, char* const argv[]) { bool script_mode = false; std::vector library_source_files, source_list; std::vector source_include_path; - std::string ton_db_path; fift::Fift::Config config; int i; int new_verbosity_level = VERBOSITY_NAME(INFO); - while (!script_mode && (i = getopt(argc, argv, "hinI:L:d:sv:V")) != -1) { + while (!script_mode && (i = getopt(argc, argv, "hinI:L:sv:V")) != -1) { switch (i) { case 'i': interactive = true; @@ -110,9 +111,6 @@ int main(int argc, char* const argv[]) { case 'L': library_source_files.emplace_back(optarg); break; - case 'd': - ton_db_path = optarg; - break; case 's': script_mode = true; break; @@ -153,16 +151,6 @@ int main(int argc, char* const argv[]) { config.source_lookup.add_include_path(path); } - if (!ton_db_path.empty()) { - auto r_ton_db = vm::TonDbImpl::open(ton_db_path); - if (r_ton_db.is_error()) { - LOG(ERROR) << "Error opening ton database: " << r_ton_db.error().to_string(); - std::exit(2); - } - config.ton_db = r_ton_db.move_as_ok(); - // FIXME //std::atexit([&] { config.ton_db.reset(); }); - } - fift::init_words_common(config.dictionary); fift::init_words_vm(config.dictionary, true); // enable vm debug fift::init_words_ton(config.dictionary); diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index 01fb8dd0..976093f8 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -1,12 +1,16 @@ library TVM_Asm // simple TVM Assembler +namespace Asm +Asm definitions +"0.4.5" constant asm-fif-version + variable @atend variable @was-split false @was-split ! { "not in asm context" abort } @atend ! { `normal eq? not abort"must be terminated by }>" } : @normal? -{ @atend @ 1 { @atend ! @normal? } does @atend ! } : @pushatend -{ @pushatend { }> b> } : }>c @@ -28,6 +32,7 @@ false @was-split ! { 1 { = or abort"invalid slice padding" swap 1 1 u, 0 rot u, } : @scomplete -{ tuck sbitrefs swap 17 + swap @havebitrefs not +{ tuck sbitrefs swap 26 + swap @havebitrefs not { PUSHREFSLICE } { over sbitrefs 2dup 123 0 2x<= { drop tuck 4 + 3 >> swap x{8B} s, over 4 u, 3 roll s, @@ -324,38 +329,109 @@ x{A7} x{A8} @Defop(8i,alt) MULCONST ' SUBCONST : SUBINT ' MULCONST : MULINT x{A8} @Defop MUL + x{A904} @Defop DIV x{A905} @Defop DIVR x{A906} @Defop DIVC x{A908} @Defop MOD +x{A909} @Defop MODR +x{A90A} @Defop MODC x{A90C} @Defop DIVMOD x{A90D} @Defop DIVMODR x{A90E} @Defop DIVMODC +x{A900} @Defop ADDDIVMOD +x{A901} @Defop ADDDIVMODR +x{A902} @Defop ADDDIVMODC + x{A925} @Defop RSHIFTR x{A926} @Defop RSHIFTC +x{A928} @Defop MODPOW2 +x{A929} @Defop MODPOW2R +x{A92A} @Defop MODPOW2C +x{A92C} @Defop RSHIFTMOD +x{A92D} @Defop RSHIFTMODR +x{A92E} @Defop RSHIFTMODC +x{A920} @Defop ADDRSHIFTMOD +x{A921} @Defop ADDRSHIFTMODR +x{A922} @Defop ADDRSHIFTMODC + x{A935} @Defop(8u+1) RSHIFTR# x{A936} @Defop(8u+1) RSHIFTC# x{A938} @Defop(8u+1) MODPOW2# x{A939} @Defop(8u+1) MODPOW2R# x{A93A} @Defop(8u+1) MODPOW2C# +x{A93C} @Defop(8u+1) RSHIFT#MOD +x{A93D} @Defop(8u+1) RSHIFTR#MOD +x{A93E} @Defop(8u+1) RSHIFTC#MOD +x{A930} @Defop(8u+1) ADDRSHIFT#MOD +x{A931} @Defop(8u+1) ADDRSHIFTR#MOD +x{A932} @Defop(8u+1) ADDRSHIFTC#MOD + x{A984} @Defop MULDIV x{A985} @Defop MULDIVR +x{A986} @Defop MULDIVC x{A988} @Defop MULMOD +x{A989} @Defop MULMODR +x{A98A} @Defop MULMODC x{A98C} @Defop MULDIVMOD x{A98D} @Defop MULDIVMODR x{A98E} @Defop MULDIVMODC +x{A980} @Defop MULADDDIVMOD +x{A981} @Defop MULADDDIVMODR +x{A982} @Defop MULADDDIVMODC + x{A9A4} @Defop MULRSHIFT x{A9A5} @Defop MULRSHIFTR x{A9A6} @Defop MULRSHIFTC +x{A9A8} @Defop MULMODPOW2 +x{A9A9} @Defop MULMODPOW2R +x{A9AA} @Defop MULMODPOW2C +x{A9AC} @Defop MULRSHIFTMOD +x{A9AD} @Defop MULRSHIFTRMOD +x{A9AE} @Defop MULRSHIFTCMOD +x{A9A0} @Defop MULADDRSHIFTMOD +x{A9A1} @Defop MULADDRSHIFTRMOD +x{A9A2} @Defop MULADDRSHIFTCMOD + x{A9B4} @Defop(8u+1) MULRSHIFT# x{A9B5} @Defop(8u+1) MULRSHIFTR# x{A9B6} @Defop(8u+1) MULRSHIFTC# +x{A9B8} @Defop(8u+1) MULMODPOW2# +x{A9B9} @Defop(8u+1) MULMODPOW2R# +x{A9BA} @Defop(8u+1) MULMODPOW2C# +x{A9BC} @Defop(8u+1) MULRSHIFT#MOD +x{A9BD} @Defop(8u+1) MULRSHIFTR#MOD +x{A9BE} @Defop(8u+1) MULRSHIFTC#MOD +x{A9B0} @Defop(8u+1) MULADDRSHIFT#MOD +x{A9B1} @Defop(8u+1) MULADDRSHIFTR#MOD +x{A9B2} @Defop(8u+1) MULADDRSHIFTC#MOD + x{A9C4} @Defop LSHIFTDIV x{A9C5} @Defop LSHIFTDIVR x{A9C6} @Defop LSHIFTDIVC +x{A9C8} @Defop LSHIFTMOD +x{A9C9} @Defop LSHIFTMODR +x{A9CA} @Defop LSHIFTMODC +x{A9CC} @Defop LSHIFTDIVMOD +x{A9CD} @Defop LSHIFTDIVMODR +x{A9CE} @Defop LSHIFTDIVMODC +x{A9C0} @Defop LSHIFTADDDIVMOD +x{A9C1} @Defop LSHIFTADDDIVMODR +x{A9C2} @Defop LSHIFTADDDIVMODC + x{A9D4} @Defop(8u+1) LSHIFT#DIV x{A9D5} @Defop(8u+1) LSHIFT#DIVR x{A9D6} @Defop(8u+1) LSHIFT#DIVC +x{A9D8} @Defop(8u+1) LSHIFT#MOD +x{A9D9} @Defop(8u+1) LSHIFT#MODR +x{A9DA} @Defop(8u+1) LSHIFT#MODC +x{A9DC} @Defop(8u+1) LSHIFT#DIVMOD +x{A9DD} @Defop(8u+1) LSHIFT#DIVMODR +x{A9DE} @Defop(8u+1) LSHIFT#DIVMODC +x{A9D0} @Defop(8u+1) LSHIFT#ADDDIVMOD +x{A9D1} @Defop(8u+1) LSHIFT#ADDDIVMODR +x{A9D2} @Defop(8u+1) LSHIFT#ADDDIVMODC + x{AA} @Defop(8u+1) LSHIFT# x{AB} @Defop(8u+1) RSHIFT# x{AC} @Defop LSHIFT @@ -385,15 +461,109 @@ x{B7A3} @Defop QNEGATE x{B7A4} @Defop QINC x{B7A5} @Defop QDEC x{B7A8} @Defop QMUL + x{B7A904} @Defop QDIV x{B7A905} @Defop QDIVR x{B7A906} @Defop QDIVC x{B7A908} @Defop QMOD +x{B7A909} @Defop QMODR +x{B7A90A} @Defop QMODC x{B7A90C} @Defop QDIVMOD x{B7A90D} @Defop QDIVMODR x{B7A90E} @Defop QDIVMODC +x{B7A900} @Defop QADDDIVMOD +x{B7A901} @Defop QADDDIVMODR +x{B7A902} @Defop QADDDIVMODC + +x{B7A925} @Defop QRSHIFTR +x{B7A926} @Defop QRSHIFTC +x{B7A928} @Defop QMODPOW2 +x{B7A929} @Defop QMODPOW2R +x{B7A92A} @Defop QMODPOW2C +x{B7A92C} @Defop QRSHIFTMOD +x{B7A92D} @Defop QRSHIFTMODR +x{B7A92E} @Defop QRSHIFTMODC +x{B7A920} @Defop QADDRSHIFTMOD +x{B7A921} @Defop QADDRSHIFTMODR +x{B7A922} @Defop QADDRSHIFTMODC + +x{B7A935} @Defop(8u+1) QRSHIFTR# +x{B7A936} @Defop(8u+1) QRSHIFTC# +x{B7A938} @Defop(8u+1) QMODPOW2# +x{B7A939} @Defop(8u+1) QMODPOW2R# +x{B7A93A} @Defop(8u+1) QMODPOW2C# +x{B7A93C} @Defop(8u+1) QRSHIFT#MOD +x{B7A93D} @Defop(8u+1) QRSHIFTR#MOD +x{B7A93E} @Defop(8u+1) QRSHIFTC#MOD +x{B7A930} @Defop(8u+1) QADDRSHIFT#MOD +x{B7A931} @Defop(8u+1) QADDRSHIFTR#MOD +x{B7A932} @Defop(8u+1) QADDRSHIFTC#MOD + +x{B7A984} @Defop QMULDIV x{B7A985} @Defop QMULDIVR +x{B7A986} @Defop QMULDIVC +x{B7A988} @Defop QMULMOD +x{B7A989} @Defop QMULMODR +x{B7A98A} @Defop QMULMODC x{B7A98C} @Defop QMULDIVMOD +x{B7A98D} @Defop QMULDIVMODR +x{B7A98E} @Defop QMULDIVMODC +x{B7A980} @Defop QMULADDDIVMOD +x{B7A981} @Defop QMULADDDIVMODR +x{B7A982} @Defop QMULADDDIVMODC + +x{B7A9A4} @Defop QMULRSHIFT +x{B7A9A5} @Defop QMULRSHIFTR +x{B7A9A6} @Defop QMULRSHIFTC +x{B7A9A8} @Defop QMULMODPOW2 +x{B7A9A9} @Defop QMULMODPOW2R +x{B7A9AA} @Defop QMULMODPOW2C +x{B7A9AC} @Defop QMULRSHIFTMOD +x{B7A9AD} @Defop QMULRSHIFTRMOD +x{B7A9AE} @Defop QMULRSHIFTCMOD +x{B7A9A0} @Defop QMULADDRSHIFTMOD +x{B7A9A1} @Defop QMULADDRSHIFTRMOD +x{B7A9A2} @Defop QMULADDRSHIFTCMOD + +x{B7A9B4} @Defop(8u+1) QMULRSHIFT# +x{B7A9B5} @Defop(8u+1) QMULRSHIFTR# +x{B7A9B6} @Defop(8u+1) QMULRSHIFTC# +x{B7A9B8} @Defop(8u+1) QMULMODPOW2# +x{B7A9B9} @Defop(8u+1) QMULMODPOW2R# +x{B7A9BA} @Defop(8u+1) QMULMODPOW2C# +x{B7A9BC} @Defop(8u+1) QMULRSHIFT#MOD +x{B7A9BD} @Defop(8u+1) QMULRSHIFTR#MOD +x{B7A9BE} @Defop(8u+1) QMULRSHIFTC#MOD +x{B7A9B0} @Defop(8u+1) QMULADDRSHIFT#MOD +x{B7A9B1} @Defop(8u+1) QMULADDRSHIFTR#MOD +x{B7A9B2} @Defop(8u+1) QMULADDRSHIFTC#MOD + +x{B7A9C4} @Defop QLSHIFTDIV +x{B7A9C5} @Defop QLSHIFTDIVR +x{B7A9C6} @Defop QLSHIFTDIVC +x{B7A9C8} @Defop QLSHIFTMOD +x{B7A9C9} @Defop QLSHIFTMODR +x{B7A9CA} @Defop QLSHIFTMODC +x{B7A9CC} @Defop QLSHIFTDIVMOD +x{B7A9CD} @Defop QLSHIFTDIVMODR +x{B7A9CE} @Defop QLSHIFTDIVMODC +x{B7A9C0} @Defop QLSHIFTADDDIVMOD +x{B7A9C1} @Defop QLSHIFTADDDIVMODR +x{B7A9C2} @Defop QLSHIFTADDDIVMODC + +x{B7A9D4} @Defop(8u+1) QLSHIFT#DIV +x{B7A9D5} @Defop(8u+1) QLSHIFT#DIVR +x{B7A9D6} @Defop(8u+1) QLSHIFT#DIVC +x{B7A9D8} @Defop(8u+1) QLSHIFT#MOD +x{B7A9D9} @Defop(8u+1) QLSHIFT#MODR +x{B7A9DA} @Defop(8u+1) QLSHIFT#MODC +x{B7A9DC} @Defop(8u+1) QLSHIFT#DIVMOD +x{B7A9DD} @Defop(8u+1) QLSHIFT#DIVMODR +x{B7A9DE} @Defop(8u+1) QLSHIFT#DIVMODC +x{B7A9D0} @Defop(8u+1) QLSHIFT#ADDDIVMOD +x{B7A9D1} @Defop(8u+1) QLSHIFT#ADDDIVMODR +x{B7A9D2} @Defop(8u+1) QLSHIFT#ADDDIVMODC + x{B7AC} @Defop QLSHIFT x{B7AD} @Defop QRSHIFT x{B7AE} @Defop QPOW2 @@ -504,6 +674,7 @@ x{CF1E} @Defop STSLICERQ x{CF1F} dup @Defop STBRQ @Defop BCONCATQ x{CF20} @Defop(ref) STREFCONST { > tuck 3 u, 3 roll s, @@ -606,6 +777,9 @@ x{D733} @Defop SSKIPLAST x{D734} @Defop SUBSLICE x{D736} @Defop SPLIT x{D737} @Defop SPLITQ +x{D739} @Defop XCTOS +x{D73A} @Defop XLOAD +x{D73B} @Defop XLOADQ x{D741} @Defop SCHKBITS x{D742} @Defop SCHKREFS x{D743} @Defop SCHKBITREFS @@ -639,6 +813,12 @@ x{D761} @Defop LDONES x{D762} @Defop LDSAME x{D764} @Defop SDEPTH x{D765} @Defop CDEPTH +x{D766} @Defop CLEVEL +x{D767} @Defop CLEVELMASK +{ } : }END> { }END> b> } : }END>c { }END>c s +// This is the way how FunC assigns method_id for reserved functions. +// Note, that Tolk entrypoints have other names (`onInternalMessage`, etc.), +// but method_id is assigned not by Fift, but by Tolk code generation. 0 constant recv_internal -1 constant recv_external -2 constant run_ticktock @@ -1319,3 +1618,44 @@ forget @proclist forget @proccnt { spec } : hash>libref // ( c -- c' ) { hash hash>libref } : >libref + +{ dup "." $pos dup -1 = + { drop 0 } + { $| 1 $| nip swap (number) 1- abort"invalid version" + dup dup 0 < swap 999 > or abort"invalid version" + } + cond +} : parse-version-level + +{ + 0 swap + "." $+ + { swap 1000 * swap parse-version-level rot + swap } 3 times + "" $= not abort"invalid version" +} : parse-asm-fif-version + +{ + dup =: required-version parse-asm-fif-version + asm-fif-version parse-asm-fif-version + = 1+ { + "Required Asm.fif version: " @' required-version "; actual Asm.fif version: " asm-fif-version $+ $+ $+ abort + } if +} : require-asm-fif-version + +{ + dup =: required-version parse-asm-fif-version + asm-fif-version parse-asm-fif-version + swap + >= 1+ { + "Required Asm.fif version: " @' required-version "; actual Asm.fif version: " asm-fif-version $+ $+ $+ abort + } if +} : require-asm-fif-version>= + + +Fift definitions Asm +' <{ : <{ +' PROGRAM{ : PROGRAM{ +' asm-fif-version : asm-fif-version +' require-asm-fif-version : require-asm-fif-version +' require-asm-fif-version>= : require-asm-fif-version>= +Fift diff --git a/crypto/fift/lib/Disasm.fif b/crypto/fift/lib/Disasm.fif new file mode 100644 index 00000000..a46eb5b2 --- /dev/null +++ b/crypto/fift/lib/Disasm.fif @@ -0,0 +1,141 @@ +library TVM_Disasm +// simple TVM Disassembler +"Lists.fif" include + +variable 'disasm +{ 'disasm @ execute } : disasm // disassemble a slice +// usage: x{74B0} disasm + +variable @dismode @dismode 0! +{ rot over @ and rot xor swap ! } : andxor! +{ -2 0 @dismode andxor! } : stack-disasm // output 's1 s4 XCHG' +{ -2 1 @dismode andxor! } : std-disasm // output 'XCHG s1, s4' +{ -3 2 @dismode andxor! } : show-vm-code +{ -3 0 @dismode andxor! } : hide-vm-code +{ @dismode @ 1 and 0= } : stack-disasm? + +variable @indent @indent 0! +{ ' space @indent @ 2* times } : .indent +{ @indent 1+! } : +indent +{ @indent 1-! } : -indent + +{ " " $pos } : spc-pos +{ dup " " $pos swap "," $pos dup 0< { drop } { + over 0< { nip } { min } cond } cond +} : spc-comma-pos +{ { dup spc-pos 0= } { 1 $| nip } while } : -leading +{ -leading -trailing dup spc-pos dup 0< { + drop dup $len { atom single } { drop nil } cond } { + $| swap atom swap -leading 2 { over spc-comma-pos dup 0>= } { + swap 1+ -rot $| 1 $| nip -leading rot + } while drop tuple + } cond +} : parse-op +{ dup "s-1" $= { drop "s(-1)" true } { + dup "s-2" $= { drop "s(-2)" true } { + dup 1 $| swap "x" $= { nip "x{" swap $+ +"}" true } { + 2drop false } cond } cond } cond +} : adj-op-arg +{ over count over <= { drop } { 2dup [] adj-op-arg { swap []= } { drop } cond } cond } : adj-arg[] +{ 1 adj-arg[] 2 adj-arg[] 3 adj-arg[] + dup first + dup `XCHG eq? { + drop dup count 2 = { tpop swap "s0" , swap , } if } { + dup `LSHIFT eq? { + drop dup count 2 = stack-disasm? and { second `LSHIFT# swap pair } if } { + dup `RSHIFT eq? { + drop dup count 2 = stack-disasm? and { second `RSHIFT# swap pair } if } { + drop + } cond } cond } cond +} : adjust-op + +variable @cp @cp 0! +variable @curop +variable @contX variable @contY variable @cdict + +{ atom>$ type } : .atom +{ dup first .atom dup count 1 > { space 0 over count 2- { 1+ 2dup [] type .", " } swap times 1+ [] type } { drop } cond } : std-show-op +{ 0 over count 1- { 1+ 2dup [] type space } swap times drop first .atom } : stk-show-op +{ @dismode @ 2 and { .indent ."// " @curop @ csr. } if } : .curop? +{ .curop? .indent @dismode @ 1 and ' std-show-op ' stk-show-op cond cr +} : show-simple-op +{ dup 4 u@ 9 = { 8 u@+ swap 15 and 3 << s@ } { + dup 7 u@ 0x47 = { 7 u@+ nip 2 u@+ 7 u@+ -rot 3 << swap sr@ } { + dup 8 u@ 0x8A = { ref@ " cr } : show-cont-op +{ swap scont-swap ":<{" show-cont-bodyx scont-swap + "" show-cont-bodyx .indent ."}>" cr } : show-cont2-op + +{ @contX @ null? { "CONT" show-cont-op } ifnot +} : flush-contX +{ @contY @ null? { scont-swap "CONT" show-cont-op scont-swap } ifnot +} : flush-contY +{ flush-contY flush-contX } : flush-cont +{ @contX @ null? not } : have-cont? +{ @contY @ null? not } : have-cont2? +{ flush-contY @contY ! scont-swap } : save-cont-body + +{ @cdict ! } : save-const-dict +{ @cdict null! } : flush-dict +{ @cdict @ null? not } : have-dict? + +{ flush-cont .indent type .":<{" cr + @curop @ ref@ " cr +} : show-ref-op +{ flush-contY .indent rot type .":<{" cr + @curop @ ref@ " cr +} : show-cont-ref-op +{ flush-cont .indent swap type .":<{" cr + @curop @ ref@+ " cr +} : show-ref2-op + +{ flush-cont first atom>$ dup 5 $| drop "DICTI" $= swap + .indent type ." {" cr +indent @cdict @ @cdict null! unpair + rot { + swap .indent . ."=> <{" cr +indent disasm -indent .indent ."}>" cr true + } swap ' idictforeach ' dictforeach cond drop + -indent .indent ."}" cr +} : show-const-dict-op + +( `PUSHCONT `PUSHREFCONT ) constant @PushContL +( `REPEAT `UNTIL `IF `IFNOT `IFJMP `IFNOTJMP ) constant @CmdC1 +( `IFREF `IFNOTREF `IFJMPREF `IFNOTJMPREF `CALLREF `JMPREF ) constant @CmdR1 +( `DICTIGETJMP `DICTIGETJMPZ `DICTUGETJMP `DICTUGETJMPZ `DICTIGETEXEC `DICTUGETEXEC ) constant @JmpDictL +{ dup first `DICTPUSHCONST eq? { + flush-cont @curop @ get-const-dict save-const-dict show-simple-op } { + dup first @JmpDictL list-member? have-dict? and { + flush-cont show-const-dict-op } { + flush-dict + dup first @PushContL list-member? { + drop @curop @ get-cont-body save-cont-body } { + dup first @CmdC1 list-member? have-cont? and { + flush-contY first atom>$ .curop? show-cont-op } { + dup first @CmdR1 list-member? { + flush-cont first atom>$ dup $len 3 - $| drop .curop? show-ref-op } { + dup first `WHILE eq? have-cont2? and { + drop "WHILE" "}>DO<{" .curop? show-cont2-op } { + dup first `IFELSE eq? have-cont2? and { + drop "IF" "}>ELSE<{" .curop? show-cont2-op } { + dup first dup `IFREFELSE eq? swap `IFELSEREF eq? or have-cont? and { + first `IFREFELSE eq? "IF" "}>ELSE<{" rot .curop? show-cont-ref-op } { + dup first `IFREFELSEREF eq? { + drop "IF" "}>ELSE<{" .curop? show-ref2-op } { + flush-cont show-simple-op + } cond } cond } cond } cond } cond } cond } cond } cond } cond +} : show-op +{ dup @cp @ (vmoplen) dup 0> { 65536 /mod swap sr@+ swap dup @cp @ (vmopdump) parse-op swap s> true } { drop false } cond } : fetch-one-op +{ { fetch-one-op } { swap @curop ! adjust-op show-op } while } : disasm-slice +{ { disasm-slice dup sbitrefs 1- or 0= } { ref@ = -rot <= and } : s-fits? diff --git a/crypto/fift/lib/FiftExt.fif b/crypto/fift/lib/FiftExt.fif new file mode 100644 index 00000000..6ed677d7 --- /dev/null +++ b/crypto/fift/lib/FiftExt.fif @@ -0,0 +1,118 @@ +{ ?dup { 1+ { execute } { 0 swap } cond } + { (number) ?dup 0= abort"-?" 'nop } cond +} : (interpret-prepare) +{ { include-depth 0= (seekeof?) not } { + (word-prefix-find) (interpret-prepare) (execute) + } while +} : interpret +{ ({) + { 0 (seekeof?) abort"no }" (word-prefix-find) (interpret-prepare) (compile) over atom? not } until + (}) swap execute +} : begin-block +{ swap 0 'nop } : end-block +{ { 1 'nop } `{ begin-block } +{ { swap `{ eq? not abort"} without {" swap execute } end-block } +:: } :: { + +// if{ ... }then{ ... }elseif{ ... }then{ ... }else{ ... } +{ eq? not abort"unexpected" } : ?pairs +{ dup `if eq? swap `ifnot eq? over or not abort"without if{" } : if-ifnot? +// cond then ? -- exec +{ { ' if } { ' ifnot } cond rot ({) 0 rot (compile) -rot 1 swap (compile) (}) +} : (make-if) +// cond then else -- exec +{ rot ({) 0 rot (compile) -rot 2 ' cond (compile) (}) +} : (make-cond) +{ `noelse `if begin-block } :: if{ +{ `noelse `ifnot begin-block } :: ifnot{ +{ 1 ' end-block does } : end-block-does +{ { over `else eq? } { + nip rot if-ifnot? ' swap ifnot (make-cond) + } while + swap `noelse ?pairs 0 swap +} : finish-else-chain +{ swap dup if-ifnot? drop `then { + swap `then ?pairs + swap if-ifnot? (make-if) finish-else-chain + } `{ begin-block +} end-block-does :: }then{ +{ swap `{ ?pairs nip + swap `then eq? not abort"without }then{" `else +} : ?else-ok +{ ?else-ok { finish-else-chain } `{ begin-block } end-block-does :: }else{ +{ ?else-ok `if begin-block } end-block-does :: }elseif{ +{ ?else-ok `ifnot begin-block } end-block-does :: }elseifnot{ + +// while{ ... }do{ ... } +{ 2 ' while does } : (make-while) +{ `while begin-block } :: while{ +{ swap `while eq? not abort"without while{" `while-do { + swap `while-do ?pairs (make-while) 0 swap + } `{ begin-block +} end-block-does :: }do{ + +// repeat{ ... }until{ ... } +{ swap ({) 0 rot (compile) 0 rot (compile) (}) 1 ' until does } : (make-until) +{ `repeat begin-block } :: repeat{ +{ swap `repeat eq? not abort"without repeat{" `until { + swap `until ?pairs (make-until) 0 swap + } `{ begin-block +} end-block-does :: }until{ + +// def { ... } instead of { ... } : +{ bl word swap bl word "{" $cmp abort"{ expected" `def { + swap `def ?pairs -rot 3 ' (create) + } `{ begin-block +} : (def) +{ 0 (def) } :: def +{ 1 (def) } :: def:: + +// defrec { ... } instead of recursive { ... } swap ! +{ recursive bl word "{" $cmp abort"{ expected" `defrec { + swap `defrec ?pairs swap ! 0 'nop + } `{ begin-block +} :: defrec + +def .sgn { + if{ ?dup 0= }then{ + ."zero" + }elseif{ 0> }then{ + ."positive" + }else{ + ."negative" + } + cr +} +// equivalent to: { ?dup 0= { ."zero" } { 0> { ."positive" } { ."negative" } cond } cond cr } : .sgn + +defrec fact { + if{ dup }then{ + dup 1- fact * + }else{ + drop 1 + } +} +// equivalent to: recursive fact { dup { dup 1- fact * } { drop 1 } cond } swap ! + +// [[ ... ]] computes arbitrary constants inside definitions +// { [[ 5 dup * ]] + } : add25 +// is equivalent to +// { 25 + } : add25 +{ "without [[" abort } box constant ']] +{ ']] @ execute } : ]] +{ { ']] @ 2 { ']] ! call/cc } does ']] ! + interpret 'nop ']] ! "]] not found" abort + } call/cc + drop 1 'nop +} :: [[ + +{ { over @ swap 2 { call/cc } does swap ! + interpret "literal to eof" abort + } call/cc + drop execute 1 'nop +} : interpret-literal-to +// use next line only if Lists.fif is loaded (or move it to Lists.fif if FiftExt.fif becomes part of Fift.fif) +// { ( ') interpret-literal-to } :: '( +// then you can use list literals '( a b c ... ) inside definitions: +// { '( 1 2 3 ) } : test +// { '( ( `a { ."A" } ) ( `b { ."B" } ) ) assoc { cadr execute } { ."???" } cond } : test2 diff --git a/crypto/fift/utils.cpp b/crypto/fift/utils.cpp index b8c5c61f..6057b2dc 100644 --- a/crypto/fift/utils.cpp +++ b/crypto/fift/utils.cpp @@ -22,6 +22,8 @@ #include "td/utils/filesystem.h" #include "td/utils/misc.h" #include "td/utils/port/path.h" +#include "vm/boc.h" +#include namespace fift { namespace { @@ -59,6 +61,12 @@ td::Result load_GetOpt_fif(std::string dir = "") { td::Result load_wallet3_code_fif(std::string dir = "") { return td::read_file_str(smartcont_dir(dir) + "wallet-v3-code.fif"); } +td::Result load_FiftExt_fif(std::string dir = "") { + return load_source("FiftExt.fif", dir); +} +td::Result load_Disasm_fif(std::string dir = "") { + return load_source("Disasm.fif", dir); +} class MemoryFileLoader : public fift::FileLoader { public: @@ -106,9 +114,10 @@ class MemoryFileLoader : public fift::FileLoader { std::map> files_; }; -td::Result create_source_lookup(std::string main, bool need_preamble = true, bool need_asm = true, +td::Result create_source_lookup(std::string&& main, bool need_preamble = true, bool need_asm = true, bool need_ton_util = true, bool need_lisp = true, - bool need_w3_code = true, std::string dir = "") { + bool need_w3_code = true, bool need_fift_ext = true, + bool need_disasm = true, std::string dir = "") { auto loader = std::make_unique(); loader->add_file("/main.fif", std::move(main)); if (need_preamble) { @@ -141,6 +150,14 @@ td::Result create_source_lookup(std::string main, bool need_ TRY_RESULT(f, load_wallet3_code_fif(dir)); loader->add_file("/wallet-v3-code.fif", std::move(f)); } + if (need_fift_ext) { + TRY_RESULT(f, load_FiftExt_fif(dir)); + loader->add_file("/FiftExt.fif", std::move(f)); + } + if (need_disasm) { + TRY_RESULT(f, load_Disasm_fif(dir)); + loader->add_file("/Disasm.fif", std::move(f)); + } auto res = fift::SourceLookup(std::move(loader)); res.add_include_path("/"); return std::move(res); @@ -172,7 +189,7 @@ td::Result run_fift(fift::SourceLookup source_lookup, std::o } // namespace td::Result mem_run_fift(std::string source, std::vector args, std::string fift_dir) { std::stringstream ss; - TRY_RESULT(source_lookup, create_source_lookup(source, true, true, true, true, true, fift_dir)); + TRY_RESULT(source_lookup, create_source_lookup(std::move(source), true, true, true, true, true, true, true, fift_dir)); TRY_RESULT_ASSIGN(source_lookup, run_fift(std::move(source_lookup), &ss, true, std::move(args))); FiftOutput res; res.source_lookup = std::move(source_lookup); @@ -190,18 +207,43 @@ td::Result mem_run_fift(SourceLookup source_lookup, std::vector create_mem_source_lookup(std::string main, std::string fift_dir, bool need_preamble, bool need_asm, bool need_ton_util, bool need_lisp, bool need_w3_code) { - return create_source_lookup(main, need_preamble, need_asm, need_ton_util, need_lisp, need_w3_code, fift_dir); + return create_source_lookup(std::move(main), need_preamble, need_asm, need_ton_util, need_lisp, need_w3_code, false, false, + fift_dir); } -td::Result> compile_asm(td::Slice asm_code, std::string fift_dir, bool is_raw) { +td::Result> compile_asm(td::Slice asm_code) { std::stringstream ss; - TRY_RESULT(source_lookup, - create_source_lookup(PSTRING() << "\"Asm.fif\" include\n " << (is_raw ? "<{" : "") << asm_code << "\n" - << (is_raw ? "}>c" : "") << " boc>B \"res\" B>file", - true, true, true, false, false, fift_dir)); + std::string sb; + sb.reserve(asm_code.size() + 100); + sb.append("\"Asm.fif\" include\n <{\n"); + sb.append(asm_code.data(), asm_code.size()); + sb.append("\n}>c boc>B \"res\" B>file"); + + TRY_RESULT(source_lookup, create_source_lookup(std::move(sb), true, true, true, false, false, false, false)); TRY_RESULT(res, run_fift(std::move(source_lookup), &ss)); TRY_RESULT(boc, res.read_file("res")); return vm::std_boc_deserialize(std::move(boc.data)); } +td::Result compile_asm_program(std::string&& program_code, const std::string& fift_dir) { + std::string main_fif; + main_fif.reserve(program_code.size() + 100); + main_fif.append(program_code.data(), program_code.size()); + main_fif.append(R"( dup hashB B>X $>B "hex" B>file)"); // write codeHashHex to a file + main_fif.append(R"( boc>B B>base64 $>B "boc" B>file)"); // write codeBoc64 to a file + + std::stringstream fift_output_stream; + TRY_RESULT(source_lookup, create_source_lookup(std::move(main_fif), true, true, false, false, false, false, false, fift_dir)); + TRY_RESULT(res, run_fift(std::move(source_lookup), &fift_output_stream)); + + TRY_RESULT(boc, res.read_file("boc")); + TRY_RESULT(hex, res.read_file("hex")); + + return CompiledProgramOutput{ + std::move(program_code), + std::move(boc.data), + std::move(hex.data), + }; +} + } // namespace fift diff --git a/crypto/fift/utils.h b/crypto/fift/utils.h index dd434fe0..fab92c54 100644 --- a/crypto/fift/utils.h +++ b/crypto/fift/utils.h @@ -26,11 +26,21 @@ struct FiftOutput { SourceLookup source_lookup; std::string output; }; + +// given a valid Fift code PROGRAM{ ... }END>c, compile_asm_program() returns this output +// now it's used primarily for wasm output (see tolk-js, for example) +struct CompiledProgramOutput { + std::string fiftCode; + std::string codeBoc64; + std::string codeHashHex; +}; + td::Result create_mem_source_lookup(std::string main, std::string fift_dir = "", bool need_preamble = true, bool need_asm = true, bool need_ton_util = true, bool need_lisp = true, bool need_w3_code = true); td::Result mem_run_fift(std::string source, std::vector args = {}, std::string fift_dir = ""); td::Result mem_run_fift(SourceLookup source_lookup, std::vector args); -td::Result> compile_asm(td::Slice asm_code, std::string fift_dir = "", bool is_raw = true); +td::Result> compile_asm(td::Slice asm_code); +td::Result compile_asm_program(std::string&& program_code, const std::string& fift_dir); } // namespace fift diff --git a/crypto/fift/words.cpp b/crypto/fift/words.cpp index cc1595dc..cbe1dbca 100644 --- a/crypto/fift/words.cpp +++ b/crypto/fift/words.cpp @@ -21,6 +21,7 @@ #include "Dictionary.h" #include "IntCtx.h" #include "SourceLookup.h" +#include "HashMap.h" #include "common/refcnt.hpp" #include "common/bigint.hpp" @@ -43,6 +44,7 @@ #include "vm/atom.h" #include "block/block.h" +#include "common/global-version.h" #include "td/utils/filesystem.h" #include "td/utils/misc.h" @@ -58,12 +60,31 @@ namespace fift { -void show_total_cells(std::ostream& stream) { - stream << "total cells = " << vm::DataCell::get_total_data_cells() << std::endl; +const Ref nop_word_def = Ref{true}; + +// +// functions for wordef +// +Ref pop_exec_token(vm::Stack& stack) { + auto wd_ref = stack.pop_chk().as_object(); + if (wd_ref.is_null()) { + throw IntError{"execution token expected"}; + } + return wd_ref; } -void do_compile(vm::Stack& stack, Ref word_def); -void do_compile_literals(vm::Stack& stack, int count); +Ref pop_word_list(vm::Stack& stack) { + auto wl_ref = stack.pop_chk().as_object(); + if (wl_ref.is_null()) { + throw IntError{"word list expected"}; + } + return wl_ref; +} + +void push_argcount(vm::Stack& stack, int args) { + stack.push_smallint(args); + stack.push_object(nop_word_def); +} void interpret_dot(IntCtx& ctx, bool space_after) { *ctx.output_stream << dec_string2(ctx.stack.pop_int()) << (space_after ? " " : ""); @@ -121,7 +142,7 @@ void interpret_print_list(IntCtx& ctx) { } void interpret_dottc(IntCtx& ctx) { - show_total_cells(*ctx.output_stream); + *ctx.output_stream << "total cells = " << vm::DataCell::get_total_data_cells() << std::endl; } void interpret_dot_internal(vm::Stack& stack) { @@ -213,7 +234,7 @@ void interpret_cmp(vm::Stack& stack, const char opt[3]) { } void interpret_sgn(vm::Stack& stack, const char opt[3]) { - auto x = stack.pop_int(); + auto x = stack.pop_int_finite(); int r = x->sgn(); assert((unsigned)(r + 1) <= 2); stack.push_smallint(((const signed char*)opt)[r + 1]); @@ -461,7 +482,7 @@ void interpret_make_xchg(vm::Stack& stack) { if (x) { stack.push_object(td::Ref{true, std::bind(interpret_xchg, _1, x, y)}); } else if (y <= 1) { - stack.push_object(y ? swap_word_def : Dictionary::nop_word_def); + stack.push_object(y ? swap_word_def : nop_word_def); } else { stack.push_object(td::Ref{true, std::bind(interpret_xchg0, _1, y)}); } @@ -486,7 +507,7 @@ void interpret_make_pop(vm::Stack& stack) { } void interpret_is_string(vm::Stack& stack) { - stack.push_bool(stack.pop().type() == vm::StackEntry::t_string); + stack.push_bool(stack.pop_chk().type() == vm::StackEntry::t_string); } int make_utf8_char(char buffer[4], int x) { @@ -1080,6 +1101,31 @@ void interpret_fetch_bytes(vm::Stack& stack, int mode) { } } +void interpret_fetch_slice(vm::Stack& stack, int mode) { + unsigned refs = ((mode & 1) ? stack.pop_smallint_range(4) : 0); + unsigned bits = stack.pop_smallint_range(1023); + auto cs = stack.pop_cellslice(); + if (!cs->have(bits, refs)) { + if (mode & 2) { + stack.push(std::move(cs)); + } + stack.push_bool(false); + if (!(mode & 4)) { + throw IntError{"end of data while fetching subslice from cell"}; + } + } else { + if (mode & 2) { + stack.push(cs.write().fetch_subslice(bits, refs)); + stack.push(std::move(cs)); + } else { + stack.push(cs->prefetch_subslice(bits, refs)); + } + if (mode & 4) { + stack.push_bool(true); + } + } +} + void interpret_cell_empty(vm::Stack& stack) { auto cs = stack.pop_cellslice(); stack.push_bool(cs->empty_ext()); @@ -1285,7 +1331,7 @@ void interpret_atom_anon(vm::Stack& stack) { } void interpret_is_atom(vm::Stack& stack) { - stack.push_bool(stack.pop().is_atom()); + stack.push_bool(stack.pop_chk().is_atom()); } bool are_eqv(vm::StackEntry x, vm::StackEntry y) { @@ -1307,11 +1353,13 @@ bool are_eqv(vm::StackEntry x, vm::StackEntry y) { } void interpret_is_eqv(vm::Stack& stack) { + stack.check_underflow(2); auto y = stack.pop(), x = stack.pop(); stack.push_bool(are_eqv(std::move(x), std::move(y))); } void interpret_is_eq(vm::Stack& stack) { + stack.check_underflow(2); auto y = stack.pop(), x = stack.pop(); stack.push_bool(x == y); } @@ -1481,6 +1529,150 @@ void interpret_crc32c(vm::Stack& stack) { stack.push_smallint(td::crc32c(td::Slice{str})); } +// Fift hashmaps + +void push_hmap(vm::Stack& stack, Ref hmap) { + if (hmap.not_null()) { + stack.push_object(std::move(hmap)); + } else { + stack.push({}); + } +} + +void push_hmap(vm::Stack& stack, HashmapKeeper hmap_keep) { + push_hmap(stack, hmap_keep.extract()); +} + +Ref pop_hmap(vm::Stack& stack) { + stack.check_underflow(1); + auto se = stack.pop(); + if (se.is_null()) { + return {}; + } + auto hmap_ref = std::move(se).as_object(); + if (hmap_ref.is_null()) { + throw IntError{"hashmap expected"}; + } + return hmap_ref; +} + +HashmapKeeper pop_hmap_keeper(vm::Stack& stack) { + return HashmapKeeper{pop_hmap(stack)}; +} + +void interpret_hmap_new(vm::Stack& stack) { + stack.push({}); +} + +void interpret_hmap_fetch(vm::Stack& stack, int mode) { + auto hmap = pop_hmap(stack); + auto value = Hashmap::get(std::move(hmap), stack.pop_chk()); + bool found = !value.is_null(); + if ((mode & 8) && !found) { + throw IntError{"hashmap key not found"}; + } + if (mode & (2 << (int)found)) { + stack.push(std::move(value)); + } + if (mode & 1) { + stack.push_bool(found); + } +} + +void interpret_hmap_delete(vm::Stack& stack, int mode) { + auto hmap = pop_hmap(stack); + auto res = Hashmap::get_remove(std::move(hmap), stack.pop_chk()); + push_hmap(stack, std::move(res.first)); + bool found = !res.second.is_null(); + if ((mode & 8) && !found) { + throw IntError{"hashmap key not found"}; + } + if (mode & (2 << (int)found)) { + stack.push(std::move(res.second)); + } + if (mode & 1) { + stack.push_bool(found); + } +} + +void interpret_hmap_store(vm::Stack& stack, int mode) { + stack.check_underflow(3); + auto hmap = pop_hmap_keeper(stack); + auto key = stack.pop(), value = stack.pop(); + bool ok = true; + if (mode & 1) { + hmap.set(std::move(key), std::move(value)); + } else { + ok = hmap.replace(std::move(key), std::move(value)); + } + push_hmap(stack, std::move(hmap)); + if (mode & 2) { + stack.push_bool(ok); + } +} + +void interpret_hmap_is_empty(vm::Stack& stack) { + stack.push_bool(pop_hmap(stack).is_null()); +} + +void interpret_hmap_decompose(vm::Stack& stack, int mode) { + auto hmap = pop_hmap(stack); + if (hmap.is_null()) { + if (mode & 1) { + stack.push_bool(false); + } else { + throw IntError{"empty hmap"}; + } + return; + } + stack.push(hmap->key()); + stack.push(hmap->value()); + push_hmap(stack, hmap->left()); + push_hmap(stack, hmap->right()); + if (mode & 1) { + stack.push_bool(true); + } +} + +class HmapIterCont : public LoopCont { + HashmapIterator it; + bool ok; + + public: + HmapIterCont(Ref _func, Ref _after, HashmapIterator _it) + : LoopCont(std::move(_func), std::move(_after)), it(std::move(_it)), ok(true) { + } + HmapIterCont(const HmapIterCont&) = default; + HmapIterCont* make_copy() const override { + return new HmapIterCont(*this); + } + bool init(IntCtx& ctx) override { + return true; + } + bool pre_exec(IntCtx& ctx) override { + if (it.eof()) { + return false; + } else { + ctx.stack.push(it->key()); + ctx.stack.push(it->value()); + return true; + } + } + bool post_exec(IntCtx& ctx) override { + ok = ctx.stack.pop_bool(); + return ok && it.next(); + } + bool finalize(IntCtx& ctx) override { + ctx.stack.push_bool(ok); + return true; + } +}; + +Ref interpret_hmap_foreach(IntCtx& ctx, int mode) { + auto func = pop_exec_token(ctx); + return td::make_ref(std::move(func), std::move(ctx.next), pop_hmap_keeper(ctx).begin(mode & 1)); +} + // vm dictionaries void interpret_dict_new(vm::Stack& stack) { stack.push({}); @@ -1880,32 +2072,32 @@ void interpret_pfx_dict_get(vm::Stack& stack) { } void interpret_bitstring_hex_literal(IntCtx& ctx) { - auto s = ctx.scan_word_to('}'); + auto s = ctx.parser->scan_word_to('}'); unsigned char buff[128]; int bits = (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), s.begin(), s.end()); - if (bits < 0) { + vm::CellBuilder cb; + if (bits < 0 || !cb.store_bits_bool(td::ConstBitPtr{buff}, bits)) { throw IntError{"Invalid hex bitstring constant"}; } - auto cs = Ref{true, vm::CellBuilder().store_bits(td::ConstBitPtr{buff}, bits).finalize()}; - ctx.stack.push(std::move(cs)); - push_argcount(ctx.stack, 1); + ctx.stack.push(cb.as_cellslice_ref()); + push_argcount(ctx, 1); } void interpret_bitstring_binary_literal(IntCtx& ctx) { - auto s = ctx.scan_word_to('}'); + auto s = ctx.parser->scan_word_to('}'); unsigned char buff[128]; - int bits = (int)td::bitstring::parse_bitstring_binary_literal(buff, sizeof(buff), s.begin(), s.end()); - if (bits < 0) { + int bits = (int)td::bitstring::parse_bitstring_binary_literal(buff, sizeof(buff) * 8, s.begin(), s.end()); + vm::CellBuilder cb; + if (bits < 0 || !cb.store_bits_bool(td::ConstBitPtr{buff}, bits)) { throw IntError{"Invalid binary bitstring constant"}; } - auto cs = Ref{true, vm::CellBuilder().store_bits(td::ConstBitPtr{buff}, bits).finalize()}; - ctx.stack.push(std::move(cs)); - push_argcount(ctx.stack, 1); + ctx.stack.push(cb.as_cellslice_ref()); + push_argcount(ctx, 1); } void interpret_word(IntCtx& ctx) { char sep = (char)ctx.stack.pop_smallint_range(127); - auto word = (sep != ' ' ? ctx.scan_word_to(sep, true) : ctx.scan_word()); + auto word = (sep != ' ' ? ctx.parser->scan_word_to(sep, true) : ctx.parser->scan_word()); ctx.stack.push_string(word); } @@ -1913,17 +2105,17 @@ void interpret_word_ext(IntCtx& ctx) { int mode = ctx.stack.pop_smallint_range(11); auto delims = ctx.stack.pop_string(); if (mode & 8) { - ctx.skipspc(mode & 4); + ctx.parser->skipspc(mode & 4); } - ctx.stack.push_string(ctx.scan_word_ext(CharClassifier{delims, mode & 3})); + ctx.stack.push_string(ctx.parser->scan_word_ext(CharClassifier{delims, mode & 3})); } void interpret_skipspc(IntCtx& ctx) { - ctx.skipspc(); + ctx.parser->skipspc(); } void interpret_wordlist_begin_aux(vm::Stack& stack) { - stack.push({vm::from_object, Ref{true}}); + stack.push_make_object(); } void interpret_wordlist_begin(IntCtx& ctx) { @@ -1936,7 +2128,7 @@ void interpret_wordlist_begin(IntCtx& ctx) { void interpret_wordlist_end_aux(vm::Stack& stack) { Ref wordlist_ref = pop_word_list(stack); wordlist_ref.write().close(); - stack.push({vm::from_object, Ref{wordlist_ref}}); + stack.push_object(std::move(wordlist_ref)); } void interpret_wordlist_end(IntCtx& ctx) { @@ -1955,7 +2147,7 @@ void interpret_internal_interpret_begin(IntCtx& ctx) { void interpret_internal_interpret_end(IntCtx& ctx) { ctx.check_int_exec(); ctx.state = -ctx.state; - ctx.stack.push({vm::from_object, Dictionary::nop_word_def}); + ctx.stack.push_object(nop_word_def); } // (create) @@ -1972,18 +2164,12 @@ void interpret_create_aux(IntCtx& ctx, int mode) { if (!(mode & 2)) { word += ' '; } - bool active = (mode & 1); - auto entry = ctx.dictionary->lookup(word); - if (entry) { - *entry = DictEntry{std::move(wd_ref), active}; // redefine word - } else { - ctx.dictionary->def_word(std::move(word), {std::move(wd_ref), active}); - } + ctx.dictionary.def_word(std::move(word), {std::move(wd_ref), (bool)(mode & 1)}); } // { bl word 0 (create) } : create void interpret_create(IntCtx& ctx) { - auto word = ctx.scan_word(); + auto word = ctx.parser->scan_word(); if (!word.size()) { throw IntError{"non-empty word name expected"}; } @@ -1995,10 +2181,10 @@ Ref create_aux_wd{Ref{true, std::bind(interpret_create_aux, s // { bl word 2 ' (create) } :: : void interpret_colon(IntCtx& ctx, int mode) { - ctx.stack.push_string(ctx.scan_word()); + ctx.stack.push_string(ctx.parser->scan_word()); ctx.stack.push_smallint(mode); ctx.stack.push_smallint(2); - ctx.stack.push({vm::from_object, create_aux_wd}); + ctx.stack.push_object(create_aux_wd); //push_argcount(ctx, 2, create_wd); } @@ -2006,27 +2192,27 @@ void interpret_colon(IntCtx& ctx, int mode) { void interpret_forget_aux(IntCtx& ctx) { std::string s = ctx.stack.pop_string(); auto s_copy = s; - auto entry = ctx.dictionary->lookup(s); + auto entry = ctx.dictionary.lookup(s); if (!entry) { s += " "; - entry = ctx.dictionary->lookup(s); + entry = ctx.dictionary.lookup(s); } if (!entry) { throw IntError{"`" + s_copy + "` not found"}; } else { - ctx.dictionary->undef_word(s); + ctx.dictionary.undef_word(s); } } // { bl word (forget) } : forget void interpret_forget(IntCtx& ctx) { - ctx.stack.push_string(ctx.scan_word()); + ctx.stack.push_string(ctx.parser->scan_word()); interpret_forget_aux(ctx); } void interpret_quote_str(IntCtx& ctx) { - ctx.stack.push_string(ctx.scan_word_to('"')); - push_argcount(ctx.stack, 1); + ctx.stack.push_string(ctx.parser->scan_word_to('"')); + push_argcount(ctx, 1); } int str_utf8_code(const char* str, int& len) { @@ -2054,7 +2240,7 @@ int str_utf8_code(const char* str, int& len) { } void interpret_char(IntCtx& ctx) { - auto s = ctx.scan_word(); + auto s = ctx.parser->scan_word(); int len = (s.size() < 10 ? (int)s.size() : 10); int code = str_utf8_code(s.data(), len); if (code < 0 || s.size() != (unsigned)len) { @@ -2195,6 +2381,12 @@ Ref interpret_execute(IntCtx& ctx) { return pop_exec_token(ctx); } +Ref interpret_call_cc(IntCtx& ctx) { + auto next = pop_exec_token(ctx); + ctx.stack.push_object(std::move(ctx.next)); + return next; +} + Ref interpret_execute_times(IntCtx& ctx) { int count = ctx.stack.pop_smallint_range(1000000000); auto body = pop_exec_token(ctx); @@ -2249,30 +2441,46 @@ Ref interpret_until(IntCtx& ctx) { return body; } -void interpret_tick(IntCtx& ctx) { - std::string word = ctx.scan_word().str(); - auto entry = ctx.dictionary->lookup(word); - if (!entry) { - entry = ctx.dictionary->lookup(word + ' '); +DictEntry context_lookup(IntCtx& ctx, std::string word, bool append_space = true) { + if (append_space) { + auto entry = context_lookup(ctx, word, false); if (!entry) { - throw IntError{"word `" + word + "` undefined"}; + entry = context_lookup(ctx, word + ' ', false); } + return entry; } - ctx.stack.push({vm::from_object, entry->get_def()}); + auto entry = ctx.context.lookup(word); + if (!entry && ctx.context != ctx.dictionary) { + entry = ctx.dictionary.lookup(word); + } + if (!entry && ctx.main_dictionary != ctx.context && ctx.main_dictionary != ctx.dictionary) { + entry = ctx.main_dictionary.lookup(word); + } + return entry; +} + +void interpret_tick(IntCtx& ctx) { + std::string word = ctx.parser->scan_word().str(); + auto entry = context_lookup(ctx, word); + if (!entry) { + throw IntError{"word `" + word + "` undefined"}; + } + ctx.stack.push_object(entry.get_def()); push_argcount(ctx, 1); } -void interpret_find(IntCtx& ctx) { +void interpret_find(IntCtx& ctx, int mode) { std::string word = ctx.stack.pop_string(); - auto entry = ctx.dictionary->lookup(word); - if (!entry) { - entry = ctx.dictionary->lookup(word + ' '); - } + auto entry = context_lookup(ctx, word, !(mode & 2)); if (!entry) { ctx.stack.push_bool(false); } else { - ctx.stack.push({vm::from_object, entry->get_def()}); - ctx.stack.push_bool(true); + ctx.stack.push_object(entry.get_def()); + if (!(mode & 1) || !entry.is_active()) { + ctx.stack.push_bool(true); + } else { + ctx.stack.push_smallint(1); + } } } @@ -2282,9 +2490,13 @@ void interpret_leave_source(IntCtx& ctx) { } } +void interpret_include_depth(IntCtx& ctx) { + ctx.stack.push_smallint(ctx.include_depth()); +} + Ref interpret_include(IntCtx& ctx) { auto fname = ctx.stack.pop_string(); - auto r_file = ctx.source_lookup->lookup_source(fname, ctx.currentd_dir); + auto r_file = ctx.source_lookup->lookup_source(fname, ctx.parser->currentd_dir); if (r_file.is_error()) { throw IntError{"cannot locate file `" + fname + "`"}; } @@ -2312,12 +2524,32 @@ Ref interpret_skip_source(IntCtx& ctx) { } void interpret_words(IntCtx& ctx) { - for (const auto& x : *ctx.dictionary) { - *ctx.output_stream << x.first << " "; + for (const auto& x : ctx.dictionary) { + *ctx.output_stream << vm::StackEntry(x.key()).as_string() << " "; } *ctx.output_stream << std::endl; } +void interpret_get_current(IntCtx& ctx) { + ctx.stack.push(ctx.dictionary.get_box()); +} + +void interpret_set_current(IntCtx& ctx) { + ctx.dictionary = ctx.stack.pop_box(); +} + +void interpret_get_context(IntCtx& ctx) { + ctx.stack.push(ctx.context.get_box()); +} + +void interpret_set_context(IntCtx& ctx) { + ctx.context = ctx.stack.pop_box(); +} + +void interpret_set_context_to(IntCtx& ctx, Ref box) { + ctx.context = std::move(box); +} + void interpret_print_backtrace(IntCtx& ctx) { ctx.print_backtrace(*ctx.output_stream, ctx.next); } @@ -2425,13 +2657,15 @@ std::vector> get_vm_libraries() { // +128 = pop hard gas limit (enabled by ACCEPT) from stack as well // +256 = enable stack trace // +512 = enable debug instructions +// +1024 = load global_version from stack void interpret_run_vm(IntCtx& ctx, int mode) { if (mode < 0) { - mode = ctx.stack.pop_smallint_range(0x3ff); + mode = ctx.stack.pop_smallint_range(0x7ff); } bool with_data = mode & 4; Ref c7; Ref data, actions; + int global_version = (mode & 1024) ? ctx.stack.pop_smallint_range(ton::SUPPORTED_VERSION) : ton::SUPPORTED_VERSION; long long gas_max = (mode & 128) ? ctx.stack.pop_long_range(vm::GasLimits::infty) : vm::GasLimits::infty; long long gas_limit = (mode & 8) ? ctx.stack.pop_long_range(vm::GasLimits::infty) : vm::GasLimits::infty; if (!(mode & 128)) { @@ -2450,7 +2684,7 @@ void interpret_run_vm(IntCtx& ctx, int mode) { auto log = create_vm_log((mode & 64) && ctx.error_stream ? &ostream_logger : nullptr); vm::GasLimits gas{gas_limit, gas_max}; int res = vm::run_vm_code(cs, ctx.stack, (mode & 3) | ((mode & 0x300) >> 6), &data, log, nullptr, &gas, - get_vm_libraries(), std::move(c7), &actions); + get_vm_libraries(), std::move(c7), &actions, global_version); ctx.stack.push_smallint(res); if (with_data) { ctx.stack.push_cell(std::move(data)); @@ -2463,112 +2697,26 @@ void interpret_run_vm(IntCtx& ctx, int mode) { } } -void do_interpret_db_run_vm_parallel(std::ostream* stream, vm::Stack& stack, vm::TonDb* ton_db_ptr, int threads_n, - int tasks_n) { - if (!ton_db_ptr || !*ton_db_ptr) { - throw vm::VmError{vm::Excno::fatal, "Ton database is not available"}; +void interpret_vmop_len(vm::Stack& stack) { + int cp = stack.pop_smallint_range(0x7fffffff, -0x80000000); + auto cs = stack.pop_cellslice(); + auto dispatch = vm::DispatchTable::get_table(cp); + if (!dispatch) { + throw IntError{"unknown vm codepage"}; } - auto& ton_db = *ton_db_ptr; - auto txn = ton_db->begin_transaction(); - auto txn_abort = td::ScopeExit() + [&] { ton_db->abort_transaction(std::move(txn)); }; - - struct Task { - vm::Ref code; - vm::SmartContractDb smart; - td::optional diff; - td::unique_ptr guard; - Ref stack; - int res{0}; - Ref data; - std::string log; - }; - std::vector tasks(tasks_n); - std::vector threads(threads_n); - - for (auto& task : tasks) { - task.code = stack.pop_cellslice(); - auto smart_hash = td::serialize(stack.pop_smallint_range(1000000000)); - task.smart = txn->begin_smartcontract(smart_hash); - task.guard = td::create_lambda_guard([&] { txn->abort_smartcontract(std::move(task.smart)); }); - auto argsn = stack.pop_smallint_range(100); - task.stack = stack.split_top(argsn); - } - - std::atomic next_task_i{0}; - auto run_tasks = [&] { - while (true) { - auto task_i = next_task_i++; - if (task_i >= tasks_n) { - break; - } - auto& task = tasks[task_i]; - auto data = task.smart->get_root(); - - StringLogger logger; - vm::VmLog log = create_vm_log(stream ? &logger : nullptr); - - task.res = vm::run_vm_code(task.code, task.stack, 3, &data, std::move(log)); - task.smart->set_root(data); - task.diff = vm::SmartContractDiff(std::move(task.smart)); - task.data = std::move(data); - task.log = std::move(logger.res); - } - }; - - td::Timer timer; - for (auto& thread : threads) { - thread = td::thread(run_tasks); - } - run_tasks(); - for (auto& thread : threads) { - thread.join(); - } - - if (stream) { - int id = 0; - for (auto& task : tasks) { - id++; - *stream << "Task #" << id << " vm_log begin" << std::endl; - *stream << task.log; - *stream << "Task #" << id << " vm_log end" << std::endl; - } - } - - LOG(ERROR) << timer; - timer = {}; - - for (auto& task : tasks) { - auto retn = task.stack.write().pop_smallint_range(100, -1); - if (retn == -1) { - retn = task.stack->depth(); - } - stack.push_from_stack(std::move(*task.stack), retn); - stack.push_smallint(task.res); - stack.push_cell(std::move(task.data)); - task.guard->dismiss(); - if (task.diff) { - txn->commit_smartcontract(std::move(task.diff.value())); - } else { - txn->commit_smartcontract(std::move(task.smart)); - } - } - LOG(ERROR) << timer; - timer = {}; - - txn_abort.dismiss(); - ton_db->commit_transaction(std::move(txn)); - timer = {}; - LOG(INFO) << "TonDB stats: \n" << ton_db->stats(); + stack.push_smallint(dispatch->instr_len(*cs)); } -void interpret_db_run_vm(IntCtx& ctx) { - do_interpret_db_run_vm_parallel(ctx.error_stream, ctx.stack, ctx.ton_db, 0, 1); -} - -void interpret_db_run_vm_parallel(IntCtx& ctx) { - auto threads_n = ctx.stack.pop_smallint_range(32, 0); - auto tasks_n = ctx.stack.pop_smallint_range(1000000000); - do_interpret_db_run_vm_parallel(ctx.error_stream, ctx.stack, ctx.ton_db, threads_n, tasks_n); +void interpret_vmop_dump(vm::Stack& stack) { + int cp = stack.pop_smallint_range(0x7fffffff, -0x80000000); + auto cs = stack.pop_cellslice(); + auto dispatch = vm::DispatchTable::get_table(cp); + if (!dispatch) { + throw IntError{"unknown vm codepage"}; + } + auto dump = dispatch->dump_instr(cs.write()); + stack.push_cellslice(std::move(cs)); + stack.push_string(std::move(dump)); } void interpret_store_vm_cont(vm::Stack& stack) { @@ -2621,11 +2769,11 @@ Ref interpret_get_cmdline_arg(IntCtx& ctx) { interpret_get_fixed_cmdline_arg(ctx.stack, n); return {}; } - auto entry = ctx.dictionary->lookup("$0 "); + auto entry = ctx.dictionary.lookup("$0 "); if (!entry) { throw IntError{"-?"}; } else { - return entry->get_def(); + return entry.get_def(); } } @@ -2665,29 +2813,19 @@ Ref interpret_execute_internal(IntCtx& ctx) { return word_def; } -// wl x1 .. xn n 'w --> wl' -void interpret_compile_internal(vm::Stack& stack) { - Ref word_def = pop_exec_token(stack); - int count = stack.pop_smallint_range(255); - do_compile_literals(stack, count); - if (word_def != Dictionary::nop_word_def) { - do_compile(stack, word_def); - } -} - void do_compile(vm::Stack& stack, Ref word_def) { Ref wl_ref = pop_word_list(stack); - if (word_def != Dictionary::nop_word_def) { + if (word_def != nop_word_def) { auto list_size = word_def->list_size(); - if ((td::uint64)list_size <= 1) { - // inline short definitions + if (list_size >= 0 && (list_size <= 2 || word_def.is_unique())) { + // inline short and unique definitions auto list = word_def->get_list(); wl_ref.write().append(list, list + list_size); } else { - wl_ref.write().push_back(word_def); + wl_ref.write().push_back(std::move(word_def)); } } - stack.push({vm::from_object, wl_ref}); + stack.push_object(std::move(wl_ref)); } void compile_one_literal(WordList& wlist, vm::StackEntry val) { @@ -2715,12 +2853,146 @@ void do_compile_literals(vm::Stack& stack, int count) { } } stack.pop_many(count + 1); - stack.push({vm::from_object, wl_ref}); + stack.push_object(std::move(wl_ref)); +} + +// wl x1 .. xn n 'w --> wl' +void interpret_compile_internal(vm::Stack& stack) { + Ref word_def = pop_exec_token(stack); + int count = stack.pop_smallint_range(255); + do_compile_literals(stack, count); + if (word_def != nop_word_def) { + do_compile(stack, std::move(word_def)); + } +} + +Ref interpret_compile_execute(IntCtx& ctx) { + if (ctx.state > 0) { + interpret_compile_internal(ctx.stack); + return {}; + } else { + return interpret_execute_internal(ctx); + } +} + +void interpret_seekeof(IntCtx& ctx, int mode) { + if (mode == -1) { + mode = ctx.stack.pop_smallint_range(3, -1); + } + bool eof = true; + if (ctx.parser && (ctx.parser->get_input() || ctx.parser->load_next_line())) { + while (true) { + if (!ctx.parser->is_sb()) { + ctx.parser->skipspc(); + if (*ctx.parser->get_input()) { + eof = false; + break; + } + } + if (mode & 1) { + *ctx.output_stream << " ok" << std::endl; + } + if (!ctx.parser->load_next_line()) { + break; + } + } + } + ctx.stack.push_bool(eof); +} + +void interpret_word_prefix_find(IntCtx& ctx, int mode) { + const char *ptr = ctx.parser->get_input(), *start = ptr; + if (!ptr) { + ctx.stack.push_string(std::string{}); + ctx.stack.push_bool(false); + return; + } + while (*ptr && *ptr != ' ' && *ptr != '\t') { + ptr++; + } + std::string Word{start, ptr}; + Word.push_back(' '); + auto entry = context_lookup(ctx, Word, false); + Word.pop_back(); + if (entry) { + ctx.parser->set_input(ptr); + ctx.parser->skipspc(); + } else { + const char* ptr_end = ptr; + while (true) { + entry = context_lookup(ctx, Word, false); + if (entry) { + ctx.parser->set_input(ptr); + break; + } + if (ptr == start) { + Word = std::string{start, ptr_end}; + ctx.parser->set_input(ptr_end); + ctx.parser->skipspc(); + break; + } + Word.pop_back(); + --ptr; + } + } + ctx.parser->word = Word; + if (!(mode & 2) || !entry) { + ctx.stack.push_string(std::move(Word)); + } + if (mode & 1) { + if (!entry) { + ctx.stack.push_bool(false); + } else { + ctx.stack.push_object(entry.get_def()); + ctx.stack.push_smallint(entry.is_active() ? 1 : -1); + } + } +} + +// equivalent to +// { ?dup { 1+ { execute } { 0 swap } cond } { (number) ?dup 0= abort"-?" 'nop } cond +// } : (interpret-prepare) +Ref interpret_prepare(IntCtx& ctx) { + int found = ctx.stack.pop_smallint_range(1, -1); + if (!found) { + // numbers + interpret_parse_number(ctx); // (number) + interpret_cond_dup(ctx); // ?dup + auto res = ctx.stack.pop_int(); // 0= abort"-?" + if (res == 0) { + throw IntError{"-?"}; + } + ctx.stack.push_object(nop_word_def); // 'nop + return {}; + } else if (found == -1) { + // ordinary word + ctx.stack.push_smallint(0); // 0 + interpret_swap(ctx); // swap + return {}; + } else { + // active word + return pop_exec_token(ctx); // execute + } +} + +Ref InterpretCont::run_tail(IntCtx& ctx) const { + static Ref interpret_prepare_ref = td::make_ref(interpret_prepare); + static Ref compile_exec_ref = td::make_ref(interpret_compile_execute); + interpret_seekeof(ctx, !ctx.state && !ctx.include_depth()); // seekeof + if (ctx.stack.pop_bool()) { + exit_interpret->clear(); + return {}; // exit loop + } + exit_interpret->set({vm::from_object, ctx.next}); // set 'exit-interpret to current continuation + interpret_word_prefix_find(ctx, 3); // (word-prefix-find) + // (interpet-prepare) (compile-execute) and schedule next loop iteration + ctx.next = SeqCont::seq(compile_exec_ref, SeqCont::seq(self(), std::move(ctx.next))); + return interpret_prepare_ref; // (interpret-prepare) } void init_words_common(Dictionary& d) { using namespace std::placeholders; - d.def_word("nop ", Dictionary::nop_word_def); + d.def_word("nop ", nop_word_def); // stack print/dump words d.def_ctx_word(". ", std::bind(interpret_dot, _1, true)); d.def_ctx_word("._ ", std::bind(interpret_dot, _1, false)); @@ -2940,6 +3212,14 @@ void init_words_common(Dictionary& d) { d.def_stack_word("ref@+ ", std::bind(interpret_fetch_ref, _1, 2)); d.def_stack_word("ref@? ", std::bind(interpret_fetch_ref, _1, 4)); d.def_stack_word("ref@?+ ", std::bind(interpret_fetch_ref, _1, 6)); + d.def_stack_word("s@ ", std::bind(interpret_fetch_slice, _1, 0)); + d.def_stack_word("sr@ ", std::bind(interpret_fetch_slice, _1, 1)); + d.def_stack_word("s@+ ", std::bind(interpret_fetch_slice, _1, 2)); + d.def_stack_word("sr@+ ", std::bind(interpret_fetch_slice, _1, 3)); + d.def_stack_word("s@? ", std::bind(interpret_fetch_slice, _1, 4)); + d.def_stack_word("sr@? ", std::bind(interpret_fetch_slice, _1, 5)); + d.def_stack_word("s@?+ ", std::bind(interpret_fetch_slice, _1, 6)); + d.def_stack_word("sr@?+ ", std::bind(interpret_fetch_slice, _1, 7)); d.def_stack_word("s> ", interpret_cell_check_empty); d.def_stack_word("empty? ", interpret_cell_empty); d.def_stack_word("remaining ", interpret_cell_remaining); @@ -2968,6 +3248,18 @@ void init_words_common(Dictionary& d) { d.def_stack_word("crc16 ", interpret_crc16); d.def_stack_word("crc32 ", interpret_crc32); d.def_stack_word("crc32c ", interpret_crc32c); + // hashmaps + d.def_stack_word("hmapnew ", interpret_hmap_new); + d.def_stack_word("hmap@ ", std::bind(interpret_hmap_fetch, _1, 6)); + d.def_stack_word("hmap@? ", std::bind(interpret_hmap_fetch, _1, 5)); + d.def_stack_word("hmap- ", std::bind(interpret_hmap_delete, _1, 0)); + d.def_stack_word("hmap-? ", std::bind(interpret_hmap_delete, _1, 1)); + d.def_stack_word("hmap@- ", std::bind(interpret_hmap_delete, _1, 6)); + d.def_stack_word("hmap! ", std::bind(interpret_hmap_store, _1, 0)); + d.def_stack_word("hmap!+ ", std::bind(interpret_hmap_store, _1, 1)); + d.def_stack_word("hmapempty? ", interpret_hmap_is_empty); + d.def_stack_word("hmapunpack ", std::bind(interpret_hmap_decompose, _1, 1)); + d.def_ctx_tail_word("hmapforeach ", std::bind(interpret_hmap_foreach, _1, 0)); // vm dictionaries d.def_stack_word("dictnew ", interpret_dict_new); d.def_stack_word("dict>s ", interpret_dict_to_slice); @@ -3039,6 +3331,7 @@ void init_words_common(Dictionary& d) { d.def_stack_word("atom? ", interpret_is_atom); // execution control d.def_ctx_tail_word("execute ", interpret_execute); + d.def_ctx_tail_word("call/cc ", interpret_call_cc); d.def_ctx_tail_word("times ", interpret_execute_times); d.def_ctx_tail_word("if ", interpret_if); d.def_ctx_tail_word("ifnot ", interpret_ifnot); @@ -3054,10 +3347,12 @@ void init_words_common(Dictionary& d) { d.def_stack_word("(}) ", interpret_wordlist_end_aux); d.def_stack_word("(compile) ", interpret_compile_internal); d.def_ctx_tail_word("(execute) ", interpret_execute_internal); + d.def_ctx_tail_word("(interpret-prepare) ", interpret_prepare); d.def_active_word("' ", interpret_tick); - d.def_word("'nop ", LitCont::literal({vm::from_object, Dictionary::nop_word_def})); + d.def_word("'nop ", LitCont::literal({vm::from_object, nop_word_def})); // dictionary manipulation - d.def_ctx_word("find ", interpret_find); + d.def_ctx_word("find ", std::bind(interpret_find, _1, 1)); + d.def_ctx_word("(word-prefix-find) ", std::bind(interpret_word_prefix_find, _1, 3)); d.def_ctx_word("create ", interpret_create); d.def_ctx_word("(create) ", std::bind(interpret_create_aux, _1, -1)); d.def_active_word(": ", std::bind(interpret_colon, _1, 0)); @@ -3067,12 +3362,21 @@ void init_words_common(Dictionary& d) { d.def_ctx_word("(forget) ", interpret_forget_aux); d.def_ctx_word("forget ", interpret_forget); d.def_ctx_word("words ", interpret_words); + d.def_word("Fift-wordlist ", LitCont::literal(d.get_box())); + d.def_ctx_word("Fift ", std::bind(interpret_set_context_to, _1, d.get_box())); + d.def_ctx_word("current@ ", interpret_get_current); + d.def_ctx_word("current! ", interpret_set_current); + d.def_ctx_word("context@ ", interpret_get_context); + d.def_ctx_word("context! ", interpret_set_context); d.def_ctx_word(".bt ", interpret_print_backtrace); d.def_ctx_word("cont. ", interpret_print_continuation); // input parse d.def_ctx_word("word ", interpret_word); d.def_ctx_word("(word) ", interpret_word_ext); d.def_ctx_word("skipspc ", interpret_skipspc); + d.def_ctx_word("seekeof? ", std::bind(interpret_seekeof, _1, 1)); + d.def_ctx_word("(seekeof?) ", std::bind(interpret_seekeof, _1, -1)); + d.def_ctx_word("include-depth ", interpret_include_depth); d.def_ctx_tail_word("include ", interpret_include); d.def_ctx_tail_word("skip-to-eof ", interpret_skip_source); d.def_word("'exit-interpret ", LitCont::literal(exit_interpret)); @@ -3098,16 +3402,16 @@ void init_words_ton(Dictionary& d) { void init_words_vm(Dictionary& d, bool enable_debug) { using namespace std::placeholders; - vm::init_op_cp0(enable_debug); + vm::init_vm(enable_debug).ensure(); // vm run d.def_word("vmlibs ", LitCont::literal(vm_libraries)); // d.def_ctx_word("runvmcode ", std::bind(interpret_run_vm, _1, 0x40)); // d.def_ctx_word("runvm ", std::bind(interpret_run_vm, _1, 0x45)); d.def_ctx_word("runvmx ", std::bind(interpret_run_vm, _1, -1)); - d.def_ctx_word("dbrunvm ", interpret_db_run_vm); - d.def_ctx_word("dbrunvm-parallel ", interpret_db_run_vm_parallel); d.def_stack_word("vmcont, ", interpret_store_vm_cont); d.def_stack_word("vmcont@ ", interpret_fetch_vm_cont); + d.def_stack_word("(vmoplen) ", interpret_vmop_len); + d.def_stack_word("(vmopdump) ", interpret_vmop_dump); } void import_cmdline_args(Dictionary& d, std::string arg0, int n, const char* const argv[]) { @@ -3121,104 +3425,9 @@ void import_cmdline_args(Dictionary& d, std::string arg0, int n, const char* con cmdline_args->set(std::move(list)); for (int i = 1; i <= n; i++) { char buffer[14]; - sprintf(buffer, "$%d ", i); + snprintf(buffer, sizeof(buffer), "$%d ", i); d.def_stack_word(buffer, std::bind(interpret_get_fixed_cmdline_arg, _1, i)); } } -std::pair numeric_value_ext(std::string s, bool allow_frac = true) { - td::RefInt256 num, denom; - int res = parse_number(s, num, denom, allow_frac); - if (res <= 0) { - throw IntError{"-?"}; - } - return std::make_pair(std::move(num), res == 2 ? std::move(denom) : td::RefInt256{}); -} - -td::RefInt256 numeric_value(std::string s) { - td::RefInt256 num, denom; - int res = parse_number(s, num, denom, false); - if (res != 1) { - throw IntError{"-?"}; - } - return num; -} - -Ref interpret_compile_execute(IntCtx& ctx) { - if (ctx.state > 0) { - interpret_compile_internal(ctx.stack); - return {}; - } else { - return interpret_execute_internal(ctx); - } -} - -Ref InterpretCont::run_tail(IntCtx& ctx) const { - if (!ctx.get_input() && !ctx.load_next_line()) { - return {}; - } - while (true) { - if (!ctx.is_sb()) { - ctx.skipspc(); - if (*ctx.get_input()) { - break; - } - } - if (!ctx.state && !ctx.include_depth) { - *ctx.output_stream << " ok" << std::endl; - } - if (!ctx.load_next_line()) { - return {}; - } - } - const char* ptr = ctx.get_input(); - std::string Word; - Word.reserve(128); - auto entry = ctx.dictionary->lookup(""); - std::string entry_word; - const char* ptr_end = ptr; - while (*ptr && *ptr != ' ' && *ptr != '\t') { - Word += *ptr++; - auto cur = ctx.dictionary->lookup(Word); - if (cur) { - entry = cur; - entry_word = Word; - ptr_end = ptr; - } - } - auto cur = ctx.dictionary->lookup(Word + " "); - if (cur || !entry) { - entry = std::move(cur); - ctx.set_input(ptr); - ctx.skipspc(); - } else { - Word = entry_word; - ctx.set_input(ptr_end); - } - ctx.word = Word; - static Ref compile_exec_ref = td::make_ref(interpret_compile_execute); - if (!entry) { - // numbers - auto res = numeric_value_ext(Word); - ctx.stack.push(std::move(res.first)); - if (res.second.not_null()) { - ctx.stack.push(std::move(res.second)); - push_argcount(ctx, 2); - } else { - push_argcount(ctx, 1); - } - } else if (!entry->is_active()) { - // ordinary word - ctx.stack.push_smallint(0); - ctx.stack.push({vm::from_object, entry->get_def()}); - } else { - // active word - ctx.next = SeqCont::seq(compile_exec_ref, SeqCont::seq(self(), std::move(ctx.next))); - return entry->get_def(); - } - exit_interpret->set({vm::from_object, ctx.next}); - ctx.next = SeqCont::seq(self(), std::move(ctx.next)); - return compile_exec_ref; -} - } // namespace fift diff --git a/crypto/func/abscode.cpp b/crypto/func/abscode.cpp index 4c893f35..f1ffcfa4 100644 --- a/crypto/func/abscode.cpp +++ b/crypto/func/abscode.cpp @@ -38,6 +38,9 @@ TmpVar::TmpVar(var_idx_t _idx, int _cls, TypeExpr* _type, SymDef* sym, const Src if (!_type) { v_type = TypeExpr::new_hole(); } + if (cls == _Named) { + undefined = true; + } } void TmpVar::set_location(const SrcLocation& loc) { @@ -158,9 +161,9 @@ void VarDescr::set_const(td::RefInt256 value) { } else if (s > 0) { val |= _NonZero | _Pos | _Finite; } else if (!s) { - if (*int_const == 1) { - val |= _Bit; - } + //if (*int_const == 1) { + // val |= _Bit; + //} val |= _Zero | _Neg | _Pos | _Finite | _Bool | _Bit; } if (val & _Finite) { @@ -179,7 +182,7 @@ void VarDescr::set_const_nan() { void VarDescr::operator|=(const VarDescr& y) { val &= y.val; - if (is_int_const() && cmp(int_const, y.int_const) != 0) { + if (is_int_const() && y.is_int_const() && cmp(int_const, y.int_const) != 0) { val &= ~_Const; } if (!(val & _Const)) { diff --git a/crypto/func/analyzer.cpp b/crypto/func/analyzer.cpp index c001e3e8..9d59cf9a 100644 --- a/crypto/func/analyzer.cpp +++ b/crypto/func/analyzer.cpp @@ -17,6 +17,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "func.h" +#include "vm/boc.h" namespace funC { @@ -42,14 +43,14 @@ int CodeBlob::split_vars(bool strict) { } std::vector comp_types; int k = var.v_type->extract_components(comp_types); - assert(k <= 254 && n <= 0x7fff00); - assert((unsigned)k == comp_types.size()); + func_assert(k <= 254 && n <= 0x7fff00); + func_assert((unsigned)k == comp_types.size()); if (k != 1) { var.coord = ~((n << 8) + k); for (int i = 0; i < k; i++) { auto v = create_var(vars[j].cls, comp_types[i], 0, vars[j].where.get()); - assert(v == n + i); - assert(vars[v].idx == v); + func_assert(v == n + i); + func_assert(vars[v].idx == v); vars[v].name = vars[j].name; vars[v].coord = ((int)j << 8) + i + 1; } @@ -75,12 +76,12 @@ bool CodeBlob::compute_used_code_vars() { } bool CodeBlob::compute_used_code_vars(std::unique_ptr& ops_ptr, const VarDescrList& var_info, bool edit) const { - assert(ops_ptr); + func_assert(ops_ptr); if (!ops_ptr->next) { - assert(ops_ptr->cl == Op::_Nop); + func_assert(ops_ptr->cl == Op::_Nop); return ops_ptr->set_var_info(var_info); } - return compute_used_code_vars(ops_ptr->next, var_info, edit) | ops_ptr->compute_used_vars(*this, edit); + return int(compute_used_code_vars(ops_ptr->next, var_info, edit)) | int(ops_ptr->compute_used_vars(*this, edit)); } bool operator==(const VarDescrList& x, const VarDescrList& y) { @@ -346,7 +347,7 @@ bool Op::std_compute_used_vars(bool disabled) { } bool Op::compute_used_vars(const CodeBlob& code, bool edit) { - assert(next); + func_assert(next); const VarDescrList& next_var_info = next->var_info; if (cl == _Nop) { return set_var_info_except(next_var_info, left); @@ -379,7 +380,7 @@ bool Op::compute_used_vars(const CodeBlob& code, bool edit) { case _Let: { // left = right std::size_t cnt = next_var_info.count_used(left); - assert(left.size() == right.size()); + func_assert(left.size() == right.size()); auto l_it = left.cbegin(), r_it = right.cbegin(); VarDescrList new_var_info{next_var_info}; new_var_info -= left; @@ -388,7 +389,7 @@ bool Op::compute_used_vars(const CodeBlob& code, bool edit) { for (; l_it < left.cend(); ++l_it, ++r_it) { if (std::find(l_it + 1, left.cend(), *l_it) == left.cend()) { auto p = next_var_info[*l_it]; - new_var_info.add_var(*r_it, !p || p->is_unused()); + new_var_info.add_var(*r_it, edit && (!p || p->is_unused())); new_left.push_back(*l_it); new_right.push_back(*r_it); } @@ -500,7 +501,12 @@ bool Op::compute_used_vars(const CodeBlob& code, bool edit) { } changes = (new_var_info.size() == n); } while (changes <= edit); + func_assert(left.size() == 1); + bool last = new_var_info.count_used(left) == 0; new_var_info += left; + if (last) { + new_var_info[left[0]]->flags |= VarDescr::_Last; + } return set_var_info(std::move(new_var_info)); } case _Again: { @@ -578,7 +584,7 @@ bool prune_unreachable(std::unique_ptr& ops) { ops = std::move(op.block1); return prune_unreachable(ops); } else { - reach = prune_unreachable(op.block0) | prune_unreachable(op.block1); + reach = int(prune_unreachable(op.block0)) | int(prune_unreachable(op.block1)); } break; } @@ -650,11 +656,11 @@ bool prune_unreachable(std::unique_ptr& ops) { ops = std::move(op.block0); return false; } - reach = true; + reach = (op.cl != Op::_Again); break; } case Op::_TryCatch: { - reach = prune_unreachable(op.block0) | prune_unreachable(op.block1); + reach = int(prune_unreachable(op.block0)) | int(prune_unreachable(op.block1)); break; } default: @@ -679,7 +685,7 @@ void CodeBlob::prune_unreachable_code() { void CodeBlob::fwd_analyze() { VarDescrList values; - assert(ops && ops->cl == Op::_Import); + func_assert(ops && ops->cl == Op::_Import); for (var_idx_t i : ops->left) { values += i; if (vars[i].v_type->is_int()) { @@ -735,7 +741,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { res.emplace_back(i); } AsmOpList tmp; - func->compile(tmp, res, args); // abstract interpretation of res := f (args) + func->compile(tmp, res, args, where); // abstract interpretation of res := f (args) int j = 0; for (var_idx_t i : left) { values.add_newval(i).set_value(res[j++]); @@ -760,7 +766,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { break; case _Let: { std::vector old_val; - assert(left.size() == right.size()); + func_assert(left.size() == right.size()); for (std::size_t i = 0; i < right.size(); i++) { const VarDescr* ov = values[right[i]]; if (!ov && verbosity >= 5) { @@ -775,7 +781,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) { } std::cerr << std::endl; } - // assert(ov); + // func_assert(ov); if (ov) { old_val.push_back(*ov); } else { @@ -886,15 +892,15 @@ bool Op::mark_noreturn() { return set_noreturn(true); case _If: case _TryCatch: - return set_noreturn((block0->mark_noreturn() & (block1 && block1->mark_noreturn())) | next->mark_noreturn()); + return set_noreturn((int(block0->mark_noreturn()) & int(block1 && block1->mark_noreturn())) | int(next->mark_noreturn())); case _Again: block0->mark_noreturn(); - return set_noreturn(false); + return set_noreturn(true); case _Until: - return set_noreturn(block0->mark_noreturn() | next->mark_noreturn()); + return set_noreturn(int(block0->mark_noreturn()) | int(next->mark_noreturn())); case _While: block1->mark_noreturn(); - return set_noreturn(block0->mark_noreturn() | next->mark_noreturn()); + return set_noreturn(int(block0->mark_noreturn()) | int(next->mark_noreturn())); case _Repeat: block0->mark_noreturn(); return set_noreturn(next->mark_noreturn()); diff --git a/crypto/func/asmops.cpp b/crypto/func/asmops.cpp index ccc409d3..71ee58f6 100644 --- a/crypto/func/asmops.cpp +++ b/crypto/func/asmops.cpp @@ -166,7 +166,7 @@ AsmOp AsmOp::UnTuple(int a) { AsmOp AsmOp::IntConst(td::RefInt256 x) { if (x->signed_fits_bits(8)) { - return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT", x); + return AsmOp::Const(dec_string(x) + " PUSHINT", x); } if (!x->is_valid()) { return AsmOp::Const("PUSHNAN", x); @@ -184,9 +184,9 @@ AsmOp AsmOp::IntConst(td::RefInt256 x) { return AsmOp::Const(k, "PUSHNEGPOW2", x); } if (!x->mod_pow2_short(23)) { - return AsmOp::Const(dec_string(std::move(x)) + " PUSHINTX", x); + return AsmOp::Const(dec_string(x) + " PUSHINTX", x); } - return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT", x); + return AsmOp::Const(dec_string(x) + " PUSHINT", x); } AsmOp AsmOp::BoolConst(bool f) { diff --git a/crypto/func/auto-tests/legacy_tester.js b/crypto/func/auto-tests/legacy_tester.js new file mode 100644 index 00000000..57092d68 --- /dev/null +++ b/crypto/func/auto-tests/legacy_tester.js @@ -0,0 +1,27 @@ +const fs = require('fs/promises'); +const { compileWasm, compileFile } = require('./wasm_tests_common'); + +async function main() { + const tests = JSON.parse((await fs.readFile('../legacy_tests.json')).toString('utf-8')) + + for (const [filename, hashstr] of tests) { + if (filename.includes('storage-provider')) continue; + + const mod = await compileWasm() + + const response = await compileFile(mod, filename); + + if (response.status !== 'ok') { + console.error(response); + throw new Error('Could not compile ' + filename); + } + + if (BigInt('0x' + response.codeHashHex) !== BigInt(hashstr)) { + throw new Error('Compilation result is different for ' + filename); + } + + console.log(filename, 'ok'); + } +} + +main() \ No newline at end of file diff --git a/crypto/func/auto-tests/legacy_tester.py b/crypto/func/auto-tests/legacy_tester.py index 6db332e5..9a990501 100644 --- a/crypto/func/auto-tests/legacy_tester.py +++ b/crypto/func/auto-tests/legacy_tester.py @@ -8,37 +8,37 @@ import shutil add_pragmas = [] #["allow-post-modification", "compute-asm-ltr"]; tests = [ - # note, that deployed version of elector,config and multisig differ since it is compilled with func-0.1.0. - # Newer compillators optimize arithmetic and logic expression that can be calculated at the compile time - ["elector/elector-code.fc", 115226404411715505328583639896096915745686314074575650766750648324043316883483], - ["config/config-code.fc", 10913070768607625342121305745084703121685937915388357634624451844356456145601], - ["eth-bridge-multisig/multisig-code.fc", 101509909129354488841890823627011033360100627957439967918234053299675481277954], + # note, that deployed version of elector,config and multisig differ since it is compilled with func-0.1.0. + # Newer compillators optimize arithmetic and logic expression that can be calculated at the compile time + ["elector/elector-code.fc", 115226404411715505328583639896096915745686314074575650766750648324043316883483], + ["config/config-code.fc", 10913070768607625342121305745084703121685937915388357634624451844356456145601], + ["eth-bridge-multisig/multisig-code.fc", 101509909129354488841890823627011033360100627957439967918234053299675481277954], - ["bsc-bridge-collector/votes-collector.fc", 62190447221288642706570413295807615918589884489514159926097051017036969900417], - ["uni-lock-wallet/uni-lockup-wallet.fc", 61959738324779104851267145467044677651344601417998258530238254441977103654381], - ["nft-collection/nft-collection-editable.fc", 45561997735512210616567774035540357815786262097548276229169737015839077731274], - ["dns-collection/nft-collection.fc", 107999822699841936063083742021519765435859194241091312445235370766165379261859], + ["bsc-bridge-collector/votes-collector.fc", 62190447221288642706570413295807615918589884489514159926097051017036969900417], + ["uni-lock-wallet/uni-lockup-wallet.fc", 61959738324779104851267145467044677651344601417998258530238254441977103654381], + ["nft-collection/nft-collection-editable.fc", 45561997735512210616567774035540357815786262097548276229169737015839077731274], + ["dns-collection/nft-collection.fc", 107999822699841936063083742021519765435859194241091312445235370766165379261859], - # note, that deployed version of tele-nft-item differs since it is compilled with func-0.3.0. - # After introducing of try/catch construction, c2 register is not always the default one. - # Thus it is necessary to save it upon jumps, differences of deployed and below compilled is that - # "c2 SAVE" is added to the beginning of recv_internal. It does not change behavior. - ["tele-nft-item/nft-item.fc", 69777543125381987786450436977742010705076866061362104025338034583422166453344], + # note, that deployed version of tele-nft-item differs since it is compilled with func-0.3.0. + # After introducing of try/catch construction, c2 register is not always the default one. + # Thus it is necessary to save it upon jumps, differences of deployed and below compilled is that + # "c2 SAVE" is added to the beginning of recv_internal. It does not change behavior. + ["tele-nft-item/nft-item.fc", 69777543125381987786450436977742010705076866061362104025338034583422166453344], - ["storage/storage-contract.fc", 91377830060355733016937375216020277778264560226873154627574229667513068328151], - ["storage/storage-provider.fc", 13618336676213331164384407184540461509022654507176709588621016553953760588122], - ["nominator-pool/pool.fc", 69767057279163099864792356875696330339149706521019810113334238732928422055375], - ["jetton-minter/jetton-minter.fc", 9028309926287301331466371999814928201427184114165428257502393474125007156494], - ["gg-marketplace/nft-marketplace-v2.fc", 92199806964112524639740773542356508485601908152150843819273107618799016205930], - ["jetton-wallet/jetton-wallet.fc", 86251125787443633057458168028617933212663498001665054651523310772884328206542], - ["whales-nominators/nominators.fc", 8941364499854379927692172316865293429893094891593442801401542636695127885153], + ["storage/storage-contract.fc", 91377830060355733016937375216020277778264560226873154627574229667513068328151], + ["storage/storage-provider.fc", 13618336676213331164384407184540461509022654507176709588621016553953760588122], + ["nominator-pool/pool.fc", 69767057279163099864792356875696330339149706521019810113334238732928422055375], + ["jetton-minter/jetton-minter.fc", 9028309926287301331466371999814928201427184114165428257502393474125007156494], + ["gg-marketplace/nft-marketplace-v2.fc", 92199806964112524639740773542356508485601908152150843819273107618799016205930], + ["jetton-wallet/jetton-wallet.fc", 86251125787443633057458168028617933212663498001665054651523310772884328206542], + ["whales-nominators/nominators.fc", 8941364499854379927692172316865293429893094891593442801401542636695127885153], - ["tact-examples/treasure_Treasure.code.fc", 13962538639825790677138656603323869918938565499584297120566680287245364723897], - ["tact-examples/jetton_SampleJetton.code.fc", 94076762218493729104783735200107713211245710256802265203823917715299139499110], - ["tact-examples/jetton_JettonDefaultWallet.code.fc", 29421313492520031238091587108198906058157443241743283101866538036369069620563], - ["tact-examples/maps_MapTestContract.code.fc", 22556550222249123835909180266811414538971143565993192846012583552876721649744], + ["tact-examples/treasure_Treasure.code.fc", 13962538639825790677138656603323869918938565499584297120566680287245364723897], + ["tact-examples/jetton_SampleJetton.code.fc", 94076762218493729104783735200107713211245710256802265203823917715299139499110], + ["tact-examples/jetton_JettonDefaultWallet.code.fc", 29421313492520031238091587108198906058157443241743283101866538036369069620563], + ["tact-examples/maps_MapTestContract.code.fc", 22556550222249123835909180266811414538971143565993192846012583552876721649744], ] def getenv(name, default=None): @@ -51,7 +51,6 @@ def getenv(name, default=None): FUNC_EXECUTABLE = getenv("FUNC_EXECUTABLE", "func") FIFT_EXECUTABLE = getenv("FIFT_EXECUTABLE", "fift") -FIFT_LIBS = getenv("FIFTPATH") TMP_DIR = tempfile.mkdtemp() COMPILED_FIF = os.path.join(TMP_DIR, "compiled.fif") @@ -63,55 +62,61 @@ class ExecutionError(Exception): pass def pre_process_func(f): - shutil.copyfile(f, f+"_backup") - with open(f, "r") as src: - sources = src.read() - with open(f, "w") as src: - for pragma in add_pragmas: - src.write("#pragma %s;\n"%pragma) - src.write(sources) + shutil.copyfile(f, f+"_backup") + with open(f, "r") as src: + sources = src.read() + with open(f, "w") as src: + for pragma in add_pragmas: + src.write("#pragma %s;\n"%pragma) + src.write(sources) def post_process_func(f): - shutil.move(f+"_backup", f) + shutil.move(f+"_backup", f) def compile_func(f): res = None try: pre_process_func(f) if "storage-provider.fc" in f : - # This contract requires building of storage-contract to include it as ref - with open(f, "r") as src: - sources = src.read() + # This contract requires building of storage-contract to include it as ref + with open(f, "r") as src: + sources = src.read() + COMPILED_ST_BOC = os.path.join(TMP_DIR, "storage-contract-code.boc") + sources = sources.replace("storage-contract-code.boc", COMPILED_ST_BOC) + with open(f, "w") as src: + src.write(sources) + COMPILED_ST_FIF = os.path.join(TMP_DIR, "storage-contract.fif") COMPILED_ST_BOC = os.path.join(TMP_DIR, "storage-contract-code.boc") - sources = sources.replace("storage-contract-code.boc", COMPILED_ST_BOC) - with open(f, "w") as src: - src.write(sources) - COMPILED_ST_FIF = os.path.join(TMP_DIR, "storage-contract.fif") - COMPILED_ST_BOC = os.path.join(TMP_DIR, "storage-contract-code.boc") - COMPILED_BUILD_BOC = os.path.join(TMP_DIR, "build-boc.fif") - res = subprocess.run([FUNC_EXECUTABLE, "-o", COMPILED_ST_FIF, "-SPA", f.replace("storage-provider.fc","storage-contract.fc")], capture_output=False, timeout=10) - with open(COMPILED_BUILD_BOC, "w") as scr: - scr.write("\"%s\" include boc>B \"%s\" B>file "%(COMPILED_ST_FIF, COMPILED_ST_BOC)) - res = subprocess.run([FIFT_EXECUTABLE, COMPILED_BUILD_BOC ], capture_output=True, timeout=10) - - + COMPILED_BUILD_BOC = os.path.join(TMP_DIR, "build-boc.fif") + res = subprocess.run([FUNC_EXECUTABLE, "-o", COMPILED_ST_FIF, "-SPA", f.replace("storage-provider.fc","storage-contract.fc")], capture_output=False, timeout=10) + with open(COMPILED_BUILD_BOC, "w") as scr: + scr.write("\"%s\" include boc>B \"%s\" B>file "%(COMPILED_ST_FIF, COMPILED_ST_BOC)) + res = subprocess.run([FIFT_EXECUTABLE, COMPILED_BUILD_BOC ], capture_output=True, timeout=10) + + res = subprocess.run([FUNC_EXECUTABLE, "-o", COMPILED_FIF, "-SPA", f], capture_output=True, timeout=10) except Exception as e: - post_process_func(f) - raise e + post_process_func(f) + raise e else: - post_process_func(f) + post_process_func(f) if res.returncode != 0: raise ExecutionError(str(res.stderr, "utf-8")) def run_runner(): - res = subprocess.run([FIFT_EXECUTABLE, "-I", FIFT_LIBS, RUNNER_FIF], capture_output=True, timeout=10) + res = subprocess.run([FIFT_EXECUTABLE, RUNNER_FIF], capture_output=True, timeout=10) if res.returncode != 0: raise ExecutionError(str(res.stderr, "utf-8")) s = str(res.stdout, "utf-8") s = s.strip() return int(s) +def get_version(): + res = subprocess.run([FUNC_EXECUTABLE, "-s"], capture_output=True, timeout=10) + if res.returncode != 0: + raise ExecutionError(str(res.stderr, "utf-8")) + s = str(res.stdout, "utf-8") + return s.strip() success = 0 for ti, t in enumerate(tests): @@ -132,14 +137,15 @@ for ti, t in enumerate(tests): try: func_out = run_runner() if func_out != th: - raise ExecutionError("Error : expected '%d', found '%d'" % (th, func_out)) + raise ExecutionError("Error : expected '%d', found '%d'" % (th, func_out)) success += 1 except ExecutionError as e: print(e, file=sys.stderr) - #print("Compiled:", file=sys.stderr) - #with open(COMPILED_FIF, "r") as f: - # print(f.read(), file=sys.stderr) - #exit(2) + print("Compiled:", file=sys.stderr) + with open(COMPILED_FIF, "r") as f: + print(f.read(), file=sys.stderr) + exit(2) print(" OK ", file=sys.stderr) -print("Done: Success %d, Error: %d"%(success, len(tests)-success), file=sys.stderr) +print(get_version()) +print("Done: Success %d, Error: %d"%(success, len(tests)-success), file=sys.stderr) \ No newline at end of file diff --git a/crypto/func/auto-tests/legacy_tests.json b/crypto/func/auto-tests/legacy_tests.json new file mode 100644 index 00000000..61a433bb --- /dev/null +++ b/crypto/func/auto-tests/legacy_tests.json @@ -0,0 +1 @@ +[["elector/elector-code.fc", "115226404411715505328583639896096915745686314074575650766750648324043316883483"], ["config/config-code.fc", "10913070768607625342121305745084703121685937915388357634624451844356456145601"], ["eth-bridge-multisig/multisig-code.fc", "101509909129354488841890823627011033360100627957439967918234053299675481277954"], ["bsc-bridge-collector/votes-collector.fc", "62190447221288642706570413295807615918589884489514159926097051017036969900417"], ["uni-lock-wallet/uni-lockup-wallet.fc", "61959738324779104851267145467044677651344601417998258530238254441977103654381"], ["nft-collection/nft-collection-editable.fc", "45561997735512210616567774035540357815786262097548276229169737015839077731274"], ["dns-collection/nft-collection.fc", "107999822699841936063083742021519765435859194241091312445235370766165379261859"], ["tele-nft-item/nft-item.fc", "69777543125381987786450436977742010705076866061362104025338034583422166453344"], ["storage/storage-contract.fc", "91377830060355733016937375216020277778264560226873154627574229667513068328151"], ["storage/storage-provider.fc", "13618336676213331164384407184540461509022654507176709588621016553953760588122"], ["nominator-pool/pool.fc", "69767057279163099864792356875696330339149706521019810113334238732928422055375"], ["jetton-minter/jetton-minter.fc", "9028309926287301331466371999814928201427184114165428257502393474125007156494"], ["gg-marketplace/nft-marketplace-v2.fc", "92199806964112524639740773542356508485601908152150843819273107618799016205930"], ["jetton-wallet/jetton-wallet.fc", "86251125787443633057458168028617933212663498001665054651523310772884328206542"], ["whales-nominators/nominators.fc", "8941364499854379927692172316865293429893094891593442801401542636695127885153"], ["tact-examples/treasure_Treasure.code.fc", "13962538639825790677138656603323869918938565499584297120566680287245364723897"], ["tact-examples/jetton_SampleJetton.code.fc", "94076762218493729104783735200107713211245710256802265203823917715299139499110"], ["tact-examples/jetton_JettonDefaultWallet.code.fc", "29421313492520031238091587108198906058157443241743283101866538036369069620563"], ["tact-examples/maps_MapTestContract.code.fc", "22556550222249123835909180266811414538971143565993192846012583552876721649744"]] \ No newline at end of file diff --git a/crypto/func/auto-tests/run_tests.js b/crypto/func/auto-tests/run_tests.js new file mode 100644 index 00000000..f8e6c6a7 --- /dev/null +++ b/crypto/func/auto-tests/run_tests.js @@ -0,0 +1,77 @@ +const fs = require('fs/promises'); +const os = require('os'); +const path = require('path'); +const { compileWasm, compileFile } = require('./wasm_tests_common'); +const { execSync } = require('child_process'); + +async function main() { + const compiledPath = path.join(os.tmpdir(), 'compiled.fif'); + const runnerPath = path.join(os.tmpdir(), 'runner.fif'); + + const tests = (await fs.readdir('.')).filter(f => f.endsWith('.fc')).sort(); + + const mathChars = '0x123456789()+-*/<>'.split('') + + for (const testFile of tests) { + const mod = await compileWasm() + + const result = await compileFile(mod, testFile) + + if (result.status !== 'ok') { + console.error(result); + throw new Error('Could not compile ' + filename); + } + + const fileLines = (await fs.readFile(testFile)).toString('utf-8').split('\n'); + + const testCases = []; + + for (const line of fileLines) { + const parts = line.split('|').map(c => c.trim()); + + if (parts.length !== 4 || parts[0] !== 'TESTCASE') continue; + + const processedInputs = []; + + for (const input of parts[2].split(' ')) { + if (input.includes('x{')) { + processedInputs.push(input); + continue; + } + + if (input.length === 0) { + continue + } + + const replacedInput = input.split('').filter(c => mathChars.includes(c)).join('').replace('//', '/').replace(/([0-9a-f])($|[^0-9a-fx])/gmi, '$1n$2') + + processedInputs.push(eval(replacedInput).toString()); + } + + testCases.push([parts[1], processedInputs.join(' '), parts[3]]); + } + + await fs.writeFile(compiledPath, '"Asm.fif" include\n' + JSON.parse('"' + result.fiftCode + '"')); + await fs.writeFile(runnerPath, `"${compiledPath}" include `${t[1]} ${t[0]} code 1 runvmx abort"exitcode is not 0" .s cr { drop } depth 1- times`).join('\n')}`) + + const fiftResult = execSync(`${process.env.FIFT_EXECUTABLE || 'fift'} -I ${process.env.FIFT_LIBS} /tmp/runner.fif`, { + stdio: ['pipe', 'pipe', 'ignore'] + }).toString('utf-8') + + const testResults = fiftResult.split('\n').map(s => s.trim()).filter(s => s.length > 0) + + if (testResults.length !== testCases.length) { + throw new Error(`Got ${testResults.length} results but there are ${testCases.length} cases`) + } + + for (let i = 0; i < testResults.length; i++) { + if (testResults[i] !== testCases[i][2]) { + throw new Error(`Unequal result ${testResults[i]} and case ${testCases[i][2]}`) + } + } + + console.log(testFile, 'ok') + } +} + +main() \ No newline at end of file diff --git a/crypto/func/auto-tests/run_tests.py b/crypto/func/auto-tests/run_tests.py index ae9c990c..158e871b 100644 --- a/crypto/func/auto-tests/run_tests.py +++ b/crypto/func/auto-tests/run_tests.py @@ -4,6 +4,7 @@ import subprocess import sys import tempfile + def getenv(name, default=None): if name in os.environ: return os.environ[name] @@ -12,10 +13,9 @@ def getenv(name, default=None): exit(1) return default + FUNC_EXECUTABLE = getenv("FUNC_EXECUTABLE", "func") FIFT_EXECUTABLE = getenv("FIFT_EXECUTABLE", "fift") -#FUNC_STDLIB = getenv("FUNC_STDLIB") -FIFT_LIBS = getenv("FIFT_LIBS") TMP_DIR = tempfile.mkdtemp() COMPILED_FIF = os.path.join(TMP_DIR, "compiled.fif") RUNNER_FIF = os.path.join(TMP_DIR, "runner.fif") @@ -25,22 +25,26 @@ if len(sys.argv) != 2: exit(1) TESTS_DIR = sys.argv[1] + class ExecutionError(Exception): pass + def compile_func(f): res = subprocess.run([FUNC_EXECUTABLE, "-o", COMPILED_FIF, "-SPA", f], capture_output=True, timeout=10) if res.returncode != 0: raise ExecutionError(str(res.stderr, "utf-8")) + def run_runner(): - res = subprocess.run([FIFT_EXECUTABLE, "-I", FIFT_LIBS, RUNNER_FIF], capture_output=True, timeout=10) + res = subprocess.run([FIFT_EXECUTABLE, RUNNER_FIF], capture_output=True, timeout=10) if res.returncode != 0: raise ExecutionError(str(res.stderr, "utf-8")) s = str(res.stdout, "utf-8") s = [x.strip() for x in s.split("\n")] return [x for x in s if x != ""] + tests = [s for s in os.listdir(TESTS_DIR) if s.endswith(".fc")] tests.sort() print("Found", len(tests), "tests", file=sys.stderr) @@ -66,6 +70,21 @@ for ti, tf in enumerate(tests): print("Error: no test cases", file=sys.stderr) exit(2) + # preprocess arithmetics in input + for i in range(len(cases)): + inputs = cases[i][1].split(" ") + processed_inputs = "" + for in_arg in inputs: + if "x{" in in_arg: + processed_inputs += in_arg + continue + # filter and execute + # is it safe enough? + filtered_in = "".join(filter(lambda x: x in "0x123456789()+-*/<>", in_arg)) + if filtered_in: + processed_inputs += str(eval(filtered_in)) + " " + cases[i][1] = processed_inputs.strip() + with open(RUNNER_FIF, "w") as f: print("\"%s\" include > 0) == 1; +} + +int lshift_var(int i) { + return (1 << i) == 1; +} + +int rshift_var(int i) { + return (1 >> i) == 1; +} + +int main(int x) { + if (x == 0) { + return lshift(); + } elseif (x == 1) { + return rshift(); + } elseif (x == 2) { + return lshift_var(0); + } elseif (x == 3) { + return rshift_var(0); + } elseif (x == 4) { + return lshift_var(1); + } else { + return rshift_var(1); + } +} + +int is_claimed(int index) method_id(11) { + int claim_bit_index = index % 256; + int mask = 1 << claim_bit_index; + return (255 & mask) == mask; +} + + +{- + method_id | in | out +TESTCASE | 0 | 0 | -1 +TESTCASE | 0 | 1 | -1 +TESTCASE | 0 | 2 | -1 +TESTCASE | 0 | 3 | -1 +TESTCASE | 0 | 4 | 0 +TESTCASE | 0 | 5 | 0 +TESTCASE | 11 | 0 | -1 +TESTCASE | 11 | 1 | -1 +TESTCASE | 11 | 256 | -1 +TESTCASE | 11 | 8 | 0 +-} diff --git a/crypto/func/auto-tests/tests/test-math.fc b/crypto/func/auto-tests/tests/test-math.fc new file mode 100644 index 00000000..e9651dc3 --- /dev/null +++ b/crypto/func/auto-tests/tests/test-math.fc @@ -0,0 +1,275 @@ +#include "../../../smartcont/mathlib.fc"; + +forall X -> (tuple, ()) ~tset(tuple t, int idx, X val) asm(t val idx) "SETINDEXVAR"; + +;; computes 1-acos(x)/Pi by a very simple, extremely slow (~70k gas) and imprecise method +;; fixed256 acos_prepare_slow(fixed255 x); +int acos_prepare_slow_f255(int x) inline { + x -= (x == 0); + int t = 1; + repeat (255) { + t = t * sgn(x) * 2 + 1; ;; decode Gray code (sgn(x_0), sgn(x_1), ...) + x = (-1 << 255) - muldivr(x, - x, 1 << 254); ;; iterate x := 2*x^2 - 1 = cos(2*acos(x)) + } + return abs(t); +} + +;; extremely slow (~70k gas) and somewhat imprecise (very imprecise when x is small), for testing only +;; fixed254 acos_slow(fixed255 x); +int acos_slow_f255(int x) inline_ref { + int t = acos_prepare_slow_f255(x); + return - mulrshiftr256(t + (-1 << 256), Pi_const_f254()); +} + +;; fixed255 asin_slow(fixed255 x); +int asin_slow_f255(int x) inline_ref { + int t = acos_prepare_slow_f255(abs(x)) % (1 << 255); + return muldivr(t, Pi_const_f254(), 1 << 255) * sgn(x); +} + +tuple test_nrand(int n) inline_ref { + tuple t = empty_tuple(); + repeat (255) { + t~tpush(0); + } + repeat (n) { + int x = fixed248::nrand(); + int bucket = (abs(x) >> 243); ;; 255 buckets starting from x=0, each 1/32 wide + t~tset(bucket, t.at(bucket) + 1); + } + return t; +} + +int geom_mean_test(int x, int y) method_id(10000) { + return geom_mean(x, y); +} +int tan_f260_test(int x) method_id(10001) { + return tan_f260(x); +} +(int, int) sincosm1_f259_test(int x) method_id(10002) { + return sincosm1_f259(x); +} +(int, int) sincosn_f256_test(int x, int y) method_id(10003) { + return sincosn_f256(x, y); +} +(int, int) sincosm1_f256_test(int x) method_id(10004) { + return sincosm1_f256(x); +} +(int, int) tan_aux_f256_test(x) method_id(10005) { + return tan_aux_f256(x); +} +(int) fixed248::tan_test(x) method_id(10006) { + return fixed248::tan(x); +} +{- + (int) atanh_alt_f258_test(x) method_id(10007) { + return atanh_alt_f258(x); + } +-} +(int) atanh_f258_test(x, y) method_id(10008) { + return atanh_f258(x, y); +} +(int) atanh_f261_test(x, y) method_id(10009) { + return atanh_f261(x, y); +} + +(int, int) log2_aux_f256_test(x) method_id(10010) { + return log2_aux_f256(x); +} +(int, int) log_aux_f256_test(x) method_id(10011) { + return log_aux_f256(x); +} +int fixed248::pow_test(x, y) method_id(10012) { + return fixed248::pow(x, y); +} +int exp_log_div(x, y) method_id(10013) { + return fixed248::exp(fixed248::log(x << 248) ~/ y); +} +int fixed248::log_test(x) method_id(10014) { + return fixed248::log(x); +} +(int,int) log_aux_f257_test(x) method_id(10015) { + return log_aux_f257(x); +} +(int,int) fixed248::sincos_test(x) method_id(10016) { + return fixed248::sincos(x); +} +int fixed248::exp_test(x) method_id(10017) { + return fixed248::exp(x); +} +int fixed248::exp2_test(x) method_id(10018) { + return fixed248::exp2(x); +} +int expm1_f257_test(x) method_id(10019) { + return expm1_f257(x); +} +int atan_f255_test(x) method_id(10020) { + return atan_f255(x); +} +int atan_f259_test(x, n) method_id(10021) { + return atan_f259(x, n); +} +(int, int) atan_aux_f256_test(x) method_id(10022) { + return atan_aux_f256(x); +} +int asin_f255_test(x) method_id(10023) { + return asin_f255(x); +} +int asin_slow_f255_test(x) method_id(10024) { + return asin_slow_f255(x); +} +int acos_f255_test(x) method_id(10025) { + return acos_f255(x); +} +int acos_slow_f255_test(x) method_id(10026) { + return acos_slow_f255(x); +} +int fixed248::atan_test(x) method_id(10027) { + return fixed248::atan(x); +} +int fixed248::acot_test(x) method_id(10028) { + return fixed248::acot(x); +} + +_ main() { + int One = 1; + ;; repeat(76 / 4) { One *= 10000; } + int sqrt2 = geom_mean(One, 2 * One); + int sqrt3 = geom_mean(One, 3 * One); + ;; return geom_mean(-1 - (-1 << 256), -1 - (-1 << 256)); + ;; return geom_mean(-1 - (-1 << 256), -2 - (-1 << 256)); + ;; return geom_mean(-1 - (-1 << 256), 1 << 255); + ;; return (sqrt2, geom_mean(sqrt2, One)); ;; (sqrt(2), 2^(1/4)) + ;; return (sqrt3, geom_mean(sqrt3, One)); ;; (sqrt(3), 3^(1/4)) + ;; return geom_mean(3 << 254, 1 << 254); + ;; return geom_mean(3, 5); + ;; return tan_f260(115641670674223639132965820642403718536242645001775371762318060545014644837101 - 1); + ;; return tan_f260(15 << 252); ;; tan(15/256) * 2^260 + ;; return sincosm1_f259(1 << 255); ;; (sin,1-cos)(1/16) * 2^259 + ;; return sincosm1_f259(115641670674223639132965820642403718536242645001775371762318060545014644837101 - 1); + ;; return sincosm1_f256((1 << 255) - 1 + (1 << 255)); ;; (sin,1-cos)(1-2^(-256)) + ;; return sincosm1_f256(Pi_const_f254()); ;; (sin,1-cos)(Pi/4) + ;; return sincosn_f256(Pi_const_f254(), 0); ;; (sin,-cos)(Pi/4) + ;; return sincosn_f256((1 << 255) + 1, 0); ;; (sin,-cos)(1/2+1/2^256) + ;; return sincosn_f256(1 << 254, 0); + ;; return sincosn_f256(touch(15) << 252, 0); ;; (sin,-cos)(15/16) + ;; return sincosm1_f256(touch(15) << 252); ;; (sin,1-cos)(15/16) + ;; return sincosn_f256(60628596148627720713372490462954977108898896221398738326462025186323149077698, 0); ;; (sin,-cos)(Pi/6) + ;; return sincosm1_f256(60628596148627720713372490462954977108898896221398738326462025186323149077698); ;; (sin,1-cos)(Pi/6) + ;; return tan_aux_f256(1899 << 245); ;; (p,q) such that p/q=tan(1899/2048) + ;; return fixed248::tan(11 << 248); ;; tan(11) + ;; return atanh_alt_f258(1 << 252); ;; atanh(1/64) * 2^258 + ;; return atanh_f258(1 << 252, 18); ;; atanh(1/64) * 2^258 + ;; return atanh_f261(muldivr(64, 1 << 255, 55), 18); ;; atanh(1/55) * 2^261 + ;; return log2_aux_f256(1 << 255); + ;; return log2_aux_f256(-1 - (-1 << 256)); ;; log2(2-1/2^255))*2^256 ~ 2^256 - 1.43 + ;; return log_aux_f256(-1 - (-1 << 256)); + ;; return log_aux_f256(3); ;; log(3/2)*2^256 + ;; return fixed248::pow(3 << 248, 3 << 248); ;; 3^3 + ;; return fixed248::exp(fixed248::log(5 << 248) ~/ 7); ;; exp(log(5)/7) = 5^(1/7) + ;; return fixed248::log(Pi_const_f254() ~>> 6); ;; log(Pi) + ;; return atanh_alt_f258(1 << 255); ;; atanh(1/8) * 2^258 + ;; return atanh_f258(1 << 255, 37); ;; atanh(1/8) * 2^258 + ;; return atanh_f258(81877371507464127617551201542979628307507432471243237061821853600756754782485, 36); ;; atanh(sqrt(2)/8) * 2^258 + ;; return log_aux_f257(Pi_const_f254()); ;; log(Pi/4) + ;; return log_aux_f257(3 << 254); ;; log(3) + ;; return atanh_alt_f258(81877371507464127617551201542979628307507432471243237061821853600756754782485); ;; atanh(sqrt(2)/8) * 2^258 + ;; return fixed248::sincos(Pi_const_f254() ~/ (64 * 3)); ;; (sin,cos)(Pi/3) + ;; return fixed248::exp(3 << 248); ;; exp(3)*2^248 + ;; return fixed248::exp2((1 << 248) ~/ 5); ;; 2^(1/5)*2^248 + ;; return fixed248::pow(3 << 248, -3 << 247); ;; 3^(-1.5) + ;; return fixed248::pow(10 << 248, -70 << 248); ;; 10^(-70) + ;; return fixed248::pow(fixed248::Pi_const(), touch(3) << 248); ;; Pi^3 ~ 31.006, computed more precisely + ;; return fixed248::pow(fixed248::Pi_const(), fixed248::Pi_const()); ;; Pi^Pi, more precisely + ;; return fixed248::exp(fixed248::log(fixed248::Pi_const()) * 3); ;; Pi^3 ~ 31.006 + ;; return fixed248::exp(muldivr(fixed248::log(fixed248::Pi_const()), fixed248::Pi_const(), 1 << 248)); ;; Pi^Pi + ;; return fixed248::sin(fixed248::log(fixed248::exp(fixed248::Pi_const()))); ;; sin(log(e^Pi)) + ;; return expm1_f257(1 << 255); ;; (exp(1/4)-1)*2^256 + ;; return expm1_f257(-1 << 256); ;; (exp(-1/2)-1)*2^256 (argument out of range, will overflow) + ;; return expm1_f257(log2_const_f256()); ;; (exp(log(2)/2)-1)*2^256 + ;; return expm1_f257(- log2_const_f256()); ;; (exp(-log(2)/2)-1)*2^256 + ;; return tanh_f258(log2_const_f256(), 17); ;; tanh(log(2)/4)*2^258 + ;; return atan_f255(0xa0 << 247); + ;; return atan_f259(1 << 255, 26); ;; atan(1/16) + ;; return atan_f259(touch(2273) << 244, 26); ;; atan(2273/2^15) + ;; return atan_aux_f256(0xa0 << 248); + ;; return atan_aux_f256(-1 - (-1 << 256)); + ;; return atan_aux_f256(-1 << 256); + ;; return atan_aux_f256(1); ;; atan(1/2^256)*2^261 = 32 + ;;return fixed248::nrand(); + ;; return test_nrand(100000); + int One = touch(1 << 255); + ;; return asin_f255(One); + ;; return asin_f255(-2 * One ~/ -3); + int arg = muldivr(12, One, 17); ;; 12/17 + ;; return [ asin_slow_f255(arg), asin_f255(arg) ]; + ;; return [ acos_slow_f255(arg), acos_f255(arg) ]; + ;; return 4 * atan_f255(One ~/ 5) - atan_f255(One ~/ 239); ;; 4 * atan(1/5) - atan(1/239) = Pi/4 as fixed255 + int One = touch(1 << 248); + ;; return fixed248::atan(One) ~/ 5); ;; atan(1/5) + ;; return fixed248::acot(One ~/ 239); ;; atan(1/5) +} + +{- + method_id | in | out +TESTCASE | 10000 | -1-(-1<<256) -1-(-1<<256) | 115792089237316195423570985008687907853269984665640564039457584007913129639935 +TESTCASE | 10000 | -1-(-1<<256) -2-(-1<<256) | 115792089237316195423570985008687907853269984665640564039457584007913129639934 +TESTCASE | 10000 | -1-(-1<<256) 1<<255 | 81877371507464127617551201542979628307507432471243237061821853600756754782485 +TESTCASE | 10000 | 1 2 | 1 +TESTCASE | 10000 | 1 3 | 2 +TESTCASE | 10000 | 3<<254, 1<<254 | 50139445418395255283694704271811692336355250894665672355503583528635147053497 +TESTCASE | 10000 | 3 5 | 4 +TESTCASE | 10001 | 115641670674223639132965820642403718536242645001775371762318060545014644837101-1 | 115792089237316195423570985008687907853269984665640564039457584007913129639935 +TESTCASE | 10001 | 15<<252 | 108679485937549714997960660780289583146059954551846264494610741505469565211201 + +TESTCASE | 10002 | 1<<255 | 57858359242454268843682786479537198006144860419130642837770554273561536355094 28938600351875109040123440645416448095273333920390487381363947585666516031269 +TESTCASE | 10002 | 90942894222941581070058735694432465663348344332098107489693037779484723616546 | 90796875678616203090520439851979829600860326752181983760731669850687818036503 71369031536005973567205947792557760023823761636922618688720973932041901854510 +TESTCASE | 10002 | 115641670674223639132965820642403718536242645001775371762318060545014644837100 | 115341536360906404779899502576747487978354537254490211650198994186870666100480 115341536360906404779899502576747487978354537254490211650198994186870666100479 +TESTCASE | 10003 | 90942894222941581070058735694432465663348344332098107489693037779484723616546 0 | 81877371507464127617551201542979628307507432471243237061821853600756754782485 -81877371507464127617551201542979628307507432471243237061821853600756754782486 +TESTCASE | 10003 | (1<<255)+1 0 | 55513684748706254392157395574451324146997108788015526773113170656738693667657 -101617118319522600545601981648807607350213579319835970884288805016705398675944 +TESTCASE | 10003 | 1<<254 0 | 28647421327665059059430596260119789787021370826354543144805343654507971817712 -112192393597863122712065585177748900737784171216163716639418346853706594800924 +TESTCASE | 10003 | 15<<252 0 | 93337815620236900315136494926097782162348358704087992554326802765553037216157 -68526346066204767396483080633934170508153877799043171682610011603005473885083 +TESTCASE | 10004 | 15<<252 | 93337815620236900315136494926097782162348358704087992554326802765553037216158 94531486342222856054175808749507474690232213733194784713695144809815311509707 +TESTCASE | 10003 | 60628596148627720713372490462954977108898896221398738326462025186323149077698 0 | 57896044618658097711785492504343953926634992332820282019728792003956564819968 -100278890836790510567389408543623384672710501789331344711007167057270294106993 +TESTCASE | 10004 | 60628596148627720713372490462954977108898896221398738326462025186323149077698 | 57896044618658097711785492504343953926634992332820282019728792003956564819968 31026396801051369712363152930129046361118965752618438656900833901285671065886 +TESTCASE | 10005 | 1899<<245 | -115784979074977116522606932816046735344768048129666123117516779696532375620701 -86847621900007587791673148476644866514014227467564880140262768165345715058771 +TESTCASE | 10006 | 11<<248 | -102200470999497240398685962406597118965525125432278008915850368651878945159221 +TESTCASE | 10008 | 1<<252 18 | 7237594612640731814076778712183932891481921212865048737772958953246047977071 +TESTCASE! | 10009 | 64*(1<<255)//55 18 | 67377367986958444187782963285047188951340314639925508148698906136973510008513 +TESTCASE | 10010 | 1<<255 | 0 255 +TESTCASE | 10011 | -1-(-1<<256) | 80260960185991308862233904206310070533990667611589946606122867505419956976171 255 +TESTCASE | 10012 | 3<<248 3<<248 | 12212446911748192486079752325135052781399568695204278238536542063334587891712 +TESTCASE | 10013 | 5 7 | 569235245303856216139605450142923208167703167128528666640203654338408315932 +TESTCASE | 10014 | 1420982722233462204219667745225507275989817880189032929526453715304448806508 | 517776035526939558040896860590142614178014859368681705591403663865964112176 +TESTCASE | 10008 | 1<<255 37 | 58200445412255555045265806996802932280233368707362818578692888102488340124094 +TESTCASE | 10008 | 81877371507464127617551201542979628307507432471243237061821853600756754782485 36 | 82746618329032515754939514227666784789465120373484337368014239356561508382845 +TESTCASE | 10015 | 90942894222941581070058735694432465663348344332098107489693037779484723616546 | -55942510554172181731996424203087263676819062449594753161692794122306202470292 256 +TESTCASE | 10015 | 3<<254 | -66622616410625360568738677407433830899150908037353507097280251369610028875158 256 +TESTCASE | 10016 | 90942894222941581070058735694432465663348344332098107489693037779484723616546//(64*3) | 391714417331212931903864877123528846377775397614575565277371746317462086355 226156424291633194186662080095093570025917938800079226639565593765455331328 +TESTCASE | 10017 | 3<<248 | 9084946421051389814103830025729847734065792062362132089390904679466687950835 +TESTCASE | 10018 | (1<<248)//5 | 519571025111621076330285524602776985448579272766894385941850747946908706857 +TESTCASE | 10012 | 3<<248 -3<<247 | 87047648295825095978636639360784188083950088358794570061638165848324908079 +TESTCASE | 10012 | 10<<248 -70<<248 | 45231 +TESTCASE | 10012 | 1420982722233462204219667745225507275989817880189032929526453715304448806508 3<<248 | 14024537329227316173680050897643053638073167245065581681188087336877135047241 +TESTCASE | 10012 | 1420982722233462204219667745225507275989817880189032929526453715304448806508 1420982722233462204219667745225507275989817880189032929526453715304448806508 | 16492303277433924047657446877966346821161732581471802839855102123372676002295 +TESTCASE | 10019 | 1<<255 | 65775792789545756849501669218806308540691279864498696756901136302101823231959 +TESTCASE | 10019 | -1<<255 | -51226238931640701466578648374135745377468902266335737558089915608594425303282 + +TESTCASE | 10020 | 160<<247 | 32340690885082755723307749066376646841771751777398167772823878380310576779097 +TESTCASE | 10021 | 1<<255 26 | 57820835337111819566482910321201859268121322500887685881159030272507322418551 +TESTCASE | 10021 | 2273<<244 26 | 64153929153128256059565403901040178355488584937372975321150754259394300105908 +TESTCASE | 10022 | 160<<248 | 18 -13775317617017974742132028403521581424991093186766868001115299479309514610238 +TESTCASE | 10022 | -1-(-1<<256) | 25 16312150880916231694896252427912541090503675654570543195394548083530005073282 +TESTCASE | 10022 | -1<<256 | -25 -16312150880916231694896252427912541090503675654570543195394548083530005073298 +TESTCASE | 10022 | 1 | 0 32 + +TESTCASE | 10023 | 1<<255 | 90942894222941581070058735694432465663348344332098107489693037779484723616546 +TESTCASE | 10023 | (1-(1<<255))//-3 | 19675212872822715586637341573564384553677006914302429002469838095945333339604 +TESTCASE | 10023 | 12*(1<<255)//17 | 45371280744427205854111943101074857545572584208710061167826656461897302968384 +TESTCASE | 10024 | 12*(1<<255)//17 | 45371280744427205854111943101074857545572584208710061167826656461897302968387 +TESTCASE | 10025 | 12*(1<<255)//17 | 22785806739257187607973396296678804058887880061694023160933190658793710324081 +TESTCASE | 10026 | 12*(1<<255)//17 | 22785806739257187607973396296678804058887880061694023160933190658793710324080 + +TESTCASE | 10027 | (1<<248)//5 | 89284547973388213553327350968415123522888028497458323165947767504203347189 +TESTCASE | 10028 | (1<<248)//239 | 708598849781543798951441405045469962900811296151941404481049216461523216127 +-} diff --git a/crypto/func/auto-tests/tests/try-func.fc b/crypto/func/auto-tests/tests/try-func.fc new file mode 100644 index 00000000..05e7594c --- /dev/null +++ b/crypto/func/auto-tests/tests/try-func.fc @@ -0,0 +1,149 @@ +int foo(int x) { + try { + if (x == 7) { + throw(44); + } + return x; + } catch (_, _) { + return 2; + } +} + +int foo_inline(int x) inline { + try { + if (x == 7) { + throw(44); + } + return x; + } catch (_, _) { + return 2; + } +} + +int foo_inlineref(int x) inline_ref { + try { + if (x == 7) { + throw(44); + } + return x; + } catch (_, _) { + return 2; + } +} + +int test(int x, int y, int z) method_id(101) { + y = foo(y); + return x * 100 + y * 10 + z; +} + +int test_inline(int x, int y, int z) method_id(102) { + y = foo_inline(y); + return x * 100 + y * 10 + z; +} + +int test_inlineref(int x, int y, int z) method_id(103) { + y = foo_inlineref(y); + return x * 100 + y * 10 + z; +} + +int foo_inline_big( + int x1, int x2, int x3, int x4, int x5, int x6, int x7, int x8, int x9, int x10, + int x11, int x12, int x13, int x14, int x15, int x16, int x17, int x18, int x19, int x20 +) inline { + try { + if (x1 == 7) { + throw(44); + } + return x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + x10 + x11 + x12 + x13 + x14 + x15 + x16 + x17 + x18 + x19 + x20; + } catch (_, _) { + return 1; + } +} + +int test_inline_big(int x, int y, int z) method_id(104) { + y = foo_inline_big( + y, y + 1, y + 2, y + 3, y + 4, y + 5, y + 6, y + 7, y + 8, y + 9, + y + 10, y + 11, y + 12, y + 13, y + 14, y + 15, y + 16, y + 17, y + 18, y + 19); + return x * 1000000 + y * 1000 + z; +} + +int foo_big( + int x1, int x2, int x3, int x4, int x5, int x6, int x7, int x8, int x9, int x10, + int x11, int x12, int x13, int x14, int x15, int x16, int x17, int x18, int x19, int x20 +) { + try { + if (x1 == 7) { + throw(44); + } + return x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + x10 + x11 + x12 + x13 + x14 + x15 + x16 + x17 + x18 + x19 + x20; + } catch (_, _) { + return 1; + } +} + +int test_big(int x, int y, int z) method_id(105) { + y = foo_big( + y, y + 1, y + 2, y + 3, y + 4, y + 5, y + 6, y + 7, y + 8, y + 9, + y + 10, y + 11, y + 12, y + 13, y + 14, y + 15, y + 16, y + 17, y + 18, y + 19); + return x * 1000000 + y * 1000 + z; +} + + +() some_throwing(int op) impure { + if (op == 1) { + return (); + } elseif (op == 2) { + return (); + } else { + throw(1); + } +} + +int test106() method_id(106) { + try { + some_throwing(1337); + return 1337; + } catch(_, code) { + return code; + } + return -1; +} + +global int g_reg; + +(int, int) test107() method_id(107) { + int l_reg = 10; + g_reg = 10; + try { + ;; note, that regardless of assignment, an exception RESTORES them to previous (to 10) + ;; it's very unexpected, but is considered to be a TVM feature, not a bug + g_reg = 999; + l_reg = 999; + some_throwing(999); + } catch(_, _) { + } + ;; returns (10,10) because of an exception, see a comment above + return (g_reg, l_reg); +} + +() main() { +} + +{- + method_id | in | out +TESTCASE | 101 | 1 2 3 | 123 +TESTCASE | 101 | 3 8 9 | 389 +TESTCASE | 101 | 3 7 9 | 329 +TESTCASE | 102 | 1 2 3 | 123 +TESTCASE | 102 | 3 8 9 | 389 +TESTCASE | 102 | 3 7 9 | 329 +TESTCASE | 103 | 1 2 3 | 123 +TESTCASE | 103 | 3 8 9 | 389 +TESTCASE | 103 | 3 7 9 | 329 +TESTCASE | 104 | 4 8 9 | 4350009 +TESTCASE | 104 | 4 7 9 | 4001009 +TESTCASE | 105 | 4 8 9 | 4350009 +TESTCASE | 105 | 4 7 9 | 4001009 +TESTCASE | 106 | | 1 +TESTCASE | 107 | | 10 10 +-} diff --git a/crypto/func/auto-tests/tests/var-apply.fc b/crypto/func/auto-tests/tests/var-apply.fc new file mode 100644 index 00000000..e6eb4f7e --- /dev/null +++ b/crypto/func/auto-tests/tests/var-apply.fc @@ -0,0 +1,132 @@ +tuple empty_tuple() asm "NIL"; +forall X -> (tuple, ()) ~tpush(tuple t, X value) asm "TPUSH"; +builder begin_cell() asm "NEWC"; +cell end_cell(builder b) asm "ENDC"; +slice begin_parse(cell c) asm "CTOS"; + + +_ getBeginCell() { + return begin_cell; +} + +_ getBeginParse() { + return begin_parse; +} + +(int, int) test101() method_id(101) { + var (_, f_end_cell) = (0, end_cell); + builder b = (getBeginCell())().store_int(1, 32); + b~store_int(2, 32); + var s = (getBeginParse())(f_end_cell(b)); + return (s~load_int(32), s~load_int(32)); +} + +() my_throw_always() inline { + throw(1000); +} + +_ get_raiser() inline { + return my_throw_always; +} + +int test102() method_id(102) { + try { + var raiser = get_raiser(); + raiser(); ;; `some_var()` is always impure, the compiler has no considerations about its runtime value + return 0; + } catch (_, code) { + return code; + } +} + +int sum(int a, int b) impure inline { + throw_unless(1000, a + b < 24); + return a + b; +} + +int mul(int a, int b) impure inline { + throw_unless(1001, a * b < 24); + return a * b; +} + +int sum_pure(int a, int b) inline { + throw_unless(1000, a + b < 24); + return a + b; +} + +int mul_pure(int a, int b) inline { + throw_unless(1001, a * b < 24); + return a * b; +} + +int demo_handler(int op, int query_id, int a, int b) { + if (op == 0xF2) { + var func = query_id % 2 == 0 ? sum : mul; + int result = func(a, b); + return 0; ;; result not used, we test that func is nevertheless called + } + if (op == 0xF3) { + var func = query_id % 2 == 0 ? sum_pure : mul_pure; + int result = func(a, b); + return 0; ;; the same for sum_pure, since `some_var()` is always impure + } + if (op == 0xF4) { + var func = query_id % 2 == 0 ? sum : mul; + int result = func(a, b); + return result; + } + return -1; +} + +tuple test103() method_id(103) { + tuple t = empty_tuple(); + try { + t~tpush(demo_handler(0xF2, 122, 100, 200)); + } catch(_, code) { + t~tpush(code); + } + try { + t~tpush(demo_handler(0xF4, 122, 100, 200)); + } catch(_, code) { + t~tpush(code); + } + try { + t~tpush(demo_handler(0xF3, 122, 10, 10)); + } catch(_, code) { + t~tpush(code); + } + try { + t~tpush(demo_handler(0xF3, 123, 10, 10)); + } catch(_, code) { + t~tpush(code); + } + return t; +} + +() always_throw2(int x) impure { + throw (239 + x); +} + +global int -> () global_f; + +int test104() method_id(104) { + try { + global_f = always_throw2; + global_f(1); + return 0; + } catch (_, code) { + return code; + } +} + +() main() { +} + + +{- + method_id | in | out +TESTCASE | 101 | | 1 2 +TESTCASE | 102 | | 1000 +TESTCASE | 103 | | [ 1000 1000 0 1001 ] +TESTCASE | 104 | | 240 +-} diff --git a/crypto/func/auto-tests/wasm_tests_common.js b/crypto/func/auto-tests/wasm_tests_common.js new file mode 100644 index 00000000..84d8e0d5 --- /dev/null +++ b/crypto/func/auto-tests/wasm_tests_common.js @@ -0,0 +1,63 @@ +const fsSync = require('fs'); + +const copyToCString = (mod, str) => { + const len = mod.lengthBytesUTF8(str) + 1; + const ptr = mod._malloc(len); + mod.stringToUTF8(str, ptr, len); + return ptr; +}; + +const copyToCStringPtr = (mod, str, ptr) => { + const allocated = copyToCString(mod, str); + mod.setValue(ptr, allocated, '*'); + return allocated; +}; + +const copyFromCString = (mod, ptr) => { + return mod.UTF8ToString(ptr); +}; + +async function compileFile(mod, filename) { + const callbackPtr = mod.addFunction((_kind, _data, contents, error) => { + const kind = copyFromCString(mod, _kind); + const data = copyFromCString(mod, _data); + if (kind === 'realpath') { + copyToCStringPtr(mod, fsSync.realpathSync(data), contents); + } else if (kind === 'source') { + const path = fsSync.realpathSync(data); + try { + copyToCStringPtr(mod, fsSync.readFileSync(path).toString('utf-8'), contents); + } catch (err) { + copyToCStringPtr(mod, e.message, error); + } + } else { + copyToCStringPtr(mod, 'Unknown callback kind ' + kind, error); + } + }, 'viiii'); + + const config = { + optLevel: 2, + sources: [filename] + }; + + const configPtr = copyToCString(mod, JSON.stringify(config)); + + const responsePtr = mod._func_compile(configPtr, callbackPtr); + + return JSON.parse(copyFromCString(mod, responsePtr)); +} + +const wasmModule = require(process.env.FUNCFIFTLIB_MODULE) + +const wasmBinary = new Uint8Array(fsSync.readFileSync(process.env.FUNCFIFTLIB_WASM)) + +async function compileWasm() { + const mod = await wasmModule({ wasmBinary }) + + return mod +} + +module.exports = { + compileFile, + compileWasm +} diff --git a/crypto/func/builtins.cpp b/crypto/func/builtins.cpp index 36f240d7..cf3adf41 100644 --- a/crypto/func/builtins.cpp +++ b/crypto/func/builtins.cpp @@ -29,8 +29,12 @@ using namespace std::literals::string_literals; int glob_func_cnt, undef_func_cnt, glob_var_cnt, const_cnt; std::vector glob_func, glob_vars; +std::set prohibited_var_names; SymDef* predefine_builtin_func(std::string name, TypeExpr* func_type) { + if (name.back() == '_') { + prohibited_var_names.insert(name); + } sym_idx_t name_idx = sym::symbols.lookup(name, 1); if (sym::symbols.is_keyword(name_idx)) { std::cerr << "fatal: global function `" << name << "` already defined as a keyword" << std::endl; @@ -82,9 +86,10 @@ SymDef* define_builtin_const(std::string name, TypeExpr* const_type, Args&&... a define_builtin_func(name, TypeExpr::new_map(TypeExpr::new_unit(), const_type), std::forward(args)...)); } -bool SymValAsmFunc::compile(AsmOpList& dest, std::vector& out, std::vector& in) const { +bool SymValAsmFunc::compile(AsmOpList& dest, std::vector& out, std::vector& in, + const SrcLocation& where) const { if (simple_compile) { - return dest.append(simple_compile(out, in)); + return dest.append(simple_compile(out, in, where)); } else if (ext_compile) { return ext_compile(dest, out, in); } else { @@ -259,7 +264,7 @@ int emulate_lshift(int a, int b) { } int t = ((b & VarDescr::_NonZero) ? VarDescr::_Even : 0); t |= b & VarDescr::_Finite; - return emulate_mul(a, VarDescr::_Int | VarDescr::_Pos | VarDescr::_NonZero | VarDescr::_Even | t); + return emulate_mul(a, VarDescr::_Int | VarDescr::_Pos | VarDescr::_NonZero | t); } int emulate_div(int a, int b) { @@ -305,7 +310,7 @@ int emulate_rshift(int a, int b) { } int t = ((b & VarDescr::_NonZero) ? VarDescr::_Even : 0); t |= b & VarDescr::_Finite; - return emulate_div(a, VarDescr::_Int | VarDescr::_Pos | VarDescr::_NonZero | VarDescr::_Even | t); + return emulate_div(a, VarDescr::_Int | VarDescr::_Pos | VarDescr::_NonZero | t); } int emulate_mod(int a, int b, int round_mode = -1) { @@ -423,11 +428,14 @@ AsmOp push_const(td::RefInt256 x) { return AsmOp::IntConst(std::move(x)); } -AsmOp compile_add(std::vector& res, std::vector& args) { - assert(res.size() == 1 && args.size() == 2); +AsmOp compile_add(std::vector& res, std::vector& args, const SrcLocation& where) { + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const + y.int_const); + if (!r.int_const->is_valid()) { + throw src::ParseError(where, "integer overflow"); + } x.unused(); y.unused(); return push_const(r.int_const); @@ -462,11 +470,14 @@ AsmOp compile_add(std::vector& res, std::vector& args) { return exec_op("ADD", 2); } -AsmOp compile_sub(std::vector& res, std::vector& args) { - assert(res.size() == 1 && args.size() == 2); +AsmOp compile_sub(std::vector& res, std::vector& args, const SrcLocation& where) { + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const - y.int_const); + if (!r.int_const->is_valid()) { + throw src::ParseError(where, "integer overflow"); + } x.unused(); y.unused(); return push_const(r.int_const); @@ -492,11 +503,14 @@ AsmOp compile_sub(std::vector& res, std::vector& args) { return exec_op("SUB", 2); } -AsmOp compile_negate(std::vector& res, std::vector& args) { - assert(res.size() == 1 && args.size() == 1); +AsmOp compile_negate(std::vector& res, std::vector& args, const SrcLocation& where) { + func_assert(res.size() == 1 && args.size() == 1); VarDescr &r = res[0], &x = args[0]; if (x.is_int_const()) { r.set_const(-x.int_const); + if (!r.int_const->is_valid()) { + throw src::ParseError(where, "integer overflow"); + } x.unused(); return push_const(r.int_const); } @@ -504,8 +518,8 @@ AsmOp compile_negate(std::vector& res, std::vector& args) { return exec_op("NEGATE", 1); } -AsmOp compile_and(std::vector& res, std::vector& args) { - assert(res.size() == 1 && args.size() == 2); +AsmOp compile_and(std::vector& res, std::vector& args, const SrcLocation& where) { + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const & y.int_const); @@ -517,8 +531,8 @@ AsmOp compile_and(std::vector& res, std::vector& args) { return exec_op("AND", 2); } -AsmOp compile_or(std::vector& res, std::vector& args) { - assert(res.size() == 1 && args.size() == 2); +AsmOp compile_or(std::vector& res, std::vector& args, const SrcLocation& where) { + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const | y.int_const); @@ -530,8 +544,8 @@ AsmOp compile_or(std::vector& res, std::vector& args) { return exec_op("OR", 2); } -AsmOp compile_xor(std::vector& res, std::vector& args) { - assert(res.size() == 1 && args.size() == 2); +AsmOp compile_xor(std::vector& res, std::vector& args, const SrcLocation& where) { + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const ^ y.int_const); @@ -543,8 +557,8 @@ AsmOp compile_xor(std::vector& res, std::vector& args) { return exec_op("XOR", 2); } -AsmOp compile_not(std::vector& res, std::vector& args) { - assert(res.size() == 1 && args.size() == 1); +AsmOp compile_not(std::vector& res, std::vector& args, const SrcLocation& where) { + func_assert(res.size() == 1 && args.size() == 1); VarDescr &r = res[0], &x = args[0]; if (x.is_int_const()) { r.set_const(~x.int_const); @@ -555,9 +569,12 @@ AsmOp compile_not(std::vector& res, std::vector& args) { return exec_op("NOT", 1); } -AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y) { +AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, const SrcLocation& where) { if (x.is_int_const() && y.is_int_const()) { r.set_const(x.int_const * y.int_const); + if (!r.int_const->is_valid()) { + throw src::ParseError(where, "integer overflow"); + } x.unused(); y.unused(); return push_const(r.int_const); @@ -620,23 +637,23 @@ AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y) { return exec_op("MUL", 2); } -AsmOp compile_mul(std::vector& res, std::vector& args) { - assert(res.size() == 1 && args.size() == 2); - return compile_mul_internal(res[0], args[0], args[1]); +AsmOp compile_mul(std::vector& res, std::vector& args, const SrcLocation& where) { + func_assert(res.size() == 1 && args.size() == 2); + return compile_mul_internal(res[0], args[0], args[1], where); } -AsmOp compile_lshift(std::vector& res, std::vector& args) { - assert(res.size() == 1 && args.size() == 2); +AsmOp compile_lshift(std::vector& res, std::vector& args, const SrcLocation& where) { + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (y.is_int_const()) { auto yv = y.int_const->to_long(); if (yv < 0 || yv > 256) { - r.set_const_nan(); - x.unused(); - y.unused(); - return push_const(r.int_const); + throw src::ParseError(where, "lshift argument is out of range"); } else if (x.is_int_const()) { r.set_const(x.int_const << (int)yv); + if (!r.int_const->is_valid()) { + throw src::ParseError(where, "integer overflow"); + } x.unused(); y.unused(); return push_const(r.int_const); @@ -661,22 +678,20 @@ AsmOp compile_lshift(std::vector& res, std::vector& args) { } if (xv == -1) { x.unused(); - return exec_op("NEGPOW2", 1); + return exec_op("-1 PUSHINT SWAP LSHIFT", 1); } } return exec_op("LSHIFT", 2); } -AsmOp compile_rshift(std::vector& res, std::vector& args, int round_mode) { - assert(res.size() == 1 && args.size() == 2); +AsmOp compile_rshift(std::vector& res, std::vector& args, const SrcLocation& where, + int round_mode) { + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (y.is_int_const()) { auto yv = y.int_const->to_long(); if (yv < 0 || yv > 256) { - r.set_const_nan(); - x.unused(); - y.unused(); - return push_const(r.int_const); + throw src::ParseError(where, "rshift argument is out of range"); } else if (x.is_int_const()) { r.set_const(td::rshift(x.int_const, (int)yv, round_mode)); x.unused(); @@ -699,9 +714,12 @@ AsmOp compile_rshift(std::vector& res, std::vector& args, in return exec_op(rshift, 2); } -AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, int round_mode) { +AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, const SrcLocation& where, int round_mode) { if (x.is_int_const() && y.is_int_const()) { r.set_const(div(x.int_const, y.int_const, round_mode)); + if (!r.int_const->is_valid()) { + throw src::ParseError(where, *y.int_const == 0 ? "division by zero" : "integer overflow"); + } x.unused(); y.unused(); return push_const(r.int_const); @@ -709,10 +727,7 @@ AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, int round_mode r.val = emulate_div(x.val, y.val); if (y.is_int_const()) { if (*y.int_const == 0) { - x.unused(); - y.unused(); - r.set_const(div(y.int_const, y.int_const)); - return push_const(r.int_const); + throw src::ParseError(where, "division by zero"); } if (*y.int_const == 1 && x.always_finite()) { y.unused(); @@ -739,16 +754,20 @@ AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, int round_mode return exec_op(op, 2); } -AsmOp compile_div(std::vector& res, std::vector& args, int round_mode) { - assert(res.size() == 1 && args.size() == 2); - return compile_div_internal(res[0], args[0], args[1], round_mode); +AsmOp compile_div(std::vector& res, std::vector& args, const SrcLocation& where, int round_mode) { + func_assert(res.size() == 1 && args.size() == 2); + return compile_div_internal(res[0], args[0], args[1], where, round_mode); } -AsmOp compile_mod(std::vector& res, std::vector& args, int round_mode) { - assert(res.size() == 1 && args.size() == 2); +AsmOp compile_mod(std::vector& res, std::vector& args, const src::SrcLocation& where, + int round_mode) { + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { r.set_const(mod(x.int_const, y.int_const, round_mode)); + if (!r.int_const->is_valid()) { + throw src::ParseError(where, *y.int_const == 0 ? "division by zero" : "integer overflow"); + } x.unused(); y.unused(); return push_const(r.int_const); @@ -756,10 +775,7 @@ AsmOp compile_mod(std::vector& res, std::vector& args, int r r.val = emulate_mod(x.val, y.val); if (y.is_int_const()) { if (*y.int_const == 0) { - x.unused(); - y.unused(); - r.set_const(mod(y.int_const, y.int_const)); - return push_const(r.int_const); + throw src::ParseError(where, "division by zero"); } if ((*y.int_const == 1 || *y.int_const == -1) && x.always_finite()) { x.unused(); @@ -784,11 +800,15 @@ AsmOp compile_mod(std::vector& res, std::vector& args, int r return exec_op(op, 2); } -AsmOp compile_muldiv(std::vector& res, std::vector& args, int round_mode) { - assert(res.size() == 1 && args.size() == 3); +AsmOp compile_muldiv(std::vector& res, std::vector& args, const SrcLocation& where, + int round_mode) { + func_assert(res.size() == 1 && args.size() == 3); VarDescr &r = res[0], &x = args[0], &y = args[1], &z = args[2]; if (x.is_int_const() && y.is_int_const() && z.is_int_const()) { r.set_const(muldiv(x.int_const, y.int_const, z.int_const, round_mode)); + if (!r.int_const->is_valid()) { + throw src::ParseError(where, *z.int_const == 0 ? "division by zero" : "integer overflow"); + } x.unused(); y.unused(); z.unused(); @@ -806,24 +826,20 @@ AsmOp compile_muldiv(std::vector& res, std::vector& args, in r.val = emulate_div(emulate_mul(x.val, y.val), z.val); if (z.is_int_const()) { if (*z.int_const == 0) { - x.unused(); - y.unused(); - z.unused(); - r.set_const(div(z.int_const, z.int_const)); - return push_const(r.int_const); + throw src::ParseError(where, "division by zero"); } if (*z.int_const == 1) { z.unused(); - return compile_mul_internal(r, x, y); + return compile_mul_internal(r, x, y, where); } } if (y.is_int_const() && *y.int_const == 1) { y.unused(); - return compile_div_internal(r, x, z, round_mode); + return compile_div_internal(r, x, z, where, round_mode); } if (x.is_int_const() && *x.int_const == 1) { x.unused(); - return compile_div_internal(r, y, z, round_mode); + return compile_div_internal(r, y, z, where, round_mode); } if (z.is_int_const()) { int k = is_pos_pow2(z.int_const); @@ -907,8 +923,8 @@ int compute_compare(const VarDescr& x, const VarDescr& y, int mode) { } AsmOp compile_cmp_int(std::vector& res, std::vector& args, int mode) { - assert(mode >= 1 && mode <= 7); - assert(res.size() == 1 && args.size() == 2); + func_assert(mode >= 1 && mode <= 7); + func_assert(res.size() == 1 && args.size() == 2); VarDescr &r = res[0], &x = args[0], &y = args[1]; if (x.is_int_const() && y.is_int_const()) { int v = compute_compare(x.int_const, y.int_const, mode); @@ -919,7 +935,7 @@ AsmOp compile_cmp_int(std::vector& res, std::vector& args, i } int v = compute_compare(x, y, mode); // std::cerr << "compute_compare(" << x << ", " << y << ", " << mode << ") = " << v << std::endl; - assert(v); + func_assert(v); if (!(v & (v - 1))) { r.set_const(v - (v >> 2) - 2); x.unused(); @@ -954,8 +970,8 @@ AsmOp compile_cmp_int(std::vector& res, std::vector& args, i return exec_op(cmp_names[mode], 2); } -AsmOp compile_throw(std::vector& res, std::vector& args) { - assert(res.empty() && args.size() == 1); +AsmOp compile_throw(std::vector& res, std::vector& args, const SrcLocation&) { + func_assert(res.empty() && args.size() == 1); VarDescr& x = args[0]; if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { x.unused(); @@ -966,7 +982,7 @@ AsmOp compile_throw(std::vector& res, std::vector& args) { } AsmOp compile_cond_throw(std::vector& res, std::vector& args, bool mode) { - assert(res.empty() && args.size() == 2); + func_assert(res.empty() && args.size() == 2); VarDescr &x = args[0], &y = args[1]; std::string suff = (mode ? "IF" : "IFNOT"); bool skip_cond = false; @@ -986,8 +1002,8 @@ AsmOp compile_cond_throw(std::vector& res, std::vector& args } } -AsmOp compile_throw_arg(std::vector& res, std::vector& args) { - assert(res.empty() && args.size() == 2); +AsmOp compile_throw_arg(std::vector& res, std::vector& args, const SrcLocation&) { + func_assert(res.empty() && args.size() == 2); VarDescr &x = args[1]; if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { x.unused(); @@ -998,7 +1014,7 @@ AsmOp compile_throw_arg(std::vector& res, std::vector& args) } AsmOp compile_cond_throw_arg(std::vector& res, std::vector& args, bool mode) { - assert(res.empty() && args.size() == 3); + func_assert(res.empty() && args.size() == 3); VarDescr &x = args[1], &y = args[2]; std::string suff = (mode ? "IF" : "IFNOT"); bool skip_cond = false; @@ -1019,7 +1035,7 @@ AsmOp compile_cond_throw_arg(std::vector& res, std::vector& } AsmOp compile_bool_const(std::vector& res, std::vector& args, bool val) { - assert(res.size() == 1 && args.empty()); + func_assert(res.size() == 1 && args.empty()); VarDescr& r = res[0]; r.set_const(val ? -1 : 0); return AsmOp::Const(val ? "TRUE" : "FALSE"); @@ -1030,7 +1046,7 @@ AsmOp compile_bool_const(std::vector& res, std::vector& args // int preload_int(slice s, int len) asm "PLDIX"; // int preload_uint(slice s, int len) asm "PLDUX"; AsmOp compile_fetch_int(std::vector& res, std::vector& args, bool fetch, bool sgnd) { - assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); + func_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); auto &y = args[1], &r = res.back(); r.val = (sgnd ? VarDescr::FiniteInt : VarDescr::FiniteUInt); int v = -1; @@ -1053,7 +1069,7 @@ AsmOp compile_fetch_int(std::vector& res, std::vector& args, // builder store_uint(builder b, int x, int len) asm(x b len) "STUX"; // builder store_int(builder b, int x, int len) asm(x b len) "STIX"; AsmOp compile_store_int(std::vector& res, std::vector& args, bool sgnd) { - assert(args.size() == 3 && res.size() == 1); + func_assert(args.size() == 3 && res.size() == 1); auto& z = args[2]; if (z.is_int_const() && z.int_const > 0 && z.int_const <= 256) { z.unused(); @@ -1063,7 +1079,7 @@ AsmOp compile_store_int(std::vector& res, std::vector& args, } AsmOp compile_fetch_slice(std::vector& res, std::vector& args, bool fetch) { - assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); + func_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); auto& y = args[1]; int v = -1; if (y.is_int_const() && y.int_const > 0 && y.int_const <= 256) { @@ -1077,8 +1093,8 @@ AsmOp compile_fetch_slice(std::vector& res, std::vector& arg } // _at(tuple t, int index) asm "INDEXVAR"; -AsmOp compile_tuple_at(std::vector& res, std::vector& args) { - assert(args.size() == 2 && res.size() == 1); +AsmOp compile_tuple_at(std::vector& res, std::vector& args, const SrcLocation&) { + func_assert(args.size() == 2 && res.size() == 1); auto& y = args[1]; if (y.is_int_const() && y.int_const >= 0 && y.int_const < 16) { y.unused(); @@ -1088,8 +1104,8 @@ AsmOp compile_tuple_at(std::vector& res, std::vector& args) } // int null?(X arg) -AsmOp compile_is_null(std::vector& res, std::vector& args) { - assert(args.size() == 1 && res.size() == 1); +AsmOp compile_is_null(std::vector& res, std::vector& args, const SrcLocation&) { + func_assert(args.size() == 1 && res.size() == 1); auto &x = args[0], &r = res[0]; if (x.always_null() || x.always_not_null()) { x.unused(); @@ -1102,7 +1118,7 @@ AsmOp compile_is_null(std::vector& res, std::vector& args) { bool compile_run_method(AsmOpList& code, std::vector& res, std::vector& args, int n, bool has_value) { - assert(args.size() == (unsigned)n + 1 && res.size() == (unsigned)has_value); + func_assert(args.size() == (unsigned)n + 1 && res.size() == (unsigned)has_value); auto& x = args[0]; if (x.is_int_const() && x.int_const->unsigned_fits_bits(14)) { x.unused(); @@ -1149,21 +1165,21 @@ void define_builtins() { define_builtin_func("_-_", arith_bin_op, compile_sub); define_builtin_func("-_", arith_un_op, compile_negate); define_builtin_func("_*_", arith_bin_op, compile_mul); - define_builtin_func("_/_", arith_bin_op, std::bind(compile_div, _1, _2, -1)); - define_builtin_func("_~/_", arith_bin_op, std::bind(compile_div, _1, _2, 0)); - define_builtin_func("_^/_", arith_bin_op, std::bind(compile_div, _1, _2, 1)); - define_builtin_func("_%_", arith_bin_op, std::bind(compile_mod, _1, _2, -1)); - define_builtin_func("_~%_", arith_bin_op, std::bind(compile_mod, _1, _2, 0)); - define_builtin_func("_^%_", arith_bin_op, std::bind(compile_mod, _1, _2, 1)); + define_builtin_func("_/_", arith_bin_op, std::bind(compile_div, _1, _2, _3, -1)); + define_builtin_func("_~/_", arith_bin_op, std::bind(compile_div, _1, _2, _3, 0)); + define_builtin_func("_^/_", arith_bin_op, std::bind(compile_div, _1, _2, _3, 1)); + define_builtin_func("_%_", arith_bin_op, std::bind(compile_mod, _1, _2, _3, -1)); + define_builtin_func("_~%_", arith_bin_op, std::bind(compile_mod, _1, _2, _3, 0)); + define_builtin_func("_^%_", arith_bin_op, std::bind(compile_mod, _1, _2, _3, 1)); define_builtin_func("_/%_", TypeExpr::new_map(Int2, Int2), AsmOp::Custom("DIVMOD", 2, 2)); define_builtin_func("divmod", TypeExpr::new_map(Int2, Int2), AsmOp::Custom("DIVMOD", 2, 2)); define_builtin_func("~divmod", TypeExpr::new_map(Int2, Int2), AsmOp::Custom("DIVMOD", 2, 2)); define_builtin_func("moddiv", TypeExpr::new_map(Int2, Int2), AsmOp::Custom("DIVMOD", 2, 2), {}, {1, 0}); define_builtin_func("~moddiv", TypeExpr::new_map(Int2, Int2), AsmOp::Custom("DIVMOD", 2, 2), {}, {1, 0}); define_builtin_func("_<<_", arith_bin_op, compile_lshift); - define_builtin_func("_>>_", arith_bin_op, std::bind(compile_rshift, _1, _2, -1)); - define_builtin_func("_~>>_", arith_bin_op, std::bind(compile_rshift, _1, _2, 0)); - define_builtin_func("_^>>_", arith_bin_op, std::bind(compile_rshift, _1, _2, 1)); + define_builtin_func("_>>_", arith_bin_op, std::bind(compile_rshift, _1, _2, _3, -1)); + define_builtin_func("_~>>_", arith_bin_op, std::bind(compile_rshift, _1, _2, _3, 0)); + define_builtin_func("_^>>_", arith_bin_op, std::bind(compile_rshift, _1, _2, _3, 1)); define_builtin_func("_&_", arith_bin_op, compile_and); define_builtin_func("_|_", arith_bin_op, compile_or); define_builtin_func("_^_", arith_bin_op, compile_xor); @@ -1171,22 +1187,22 @@ void define_builtins() { define_builtin_func("^_+=_", arith_bin_op, compile_add); define_builtin_func("^_-=_", arith_bin_op, compile_sub); define_builtin_func("^_*=_", arith_bin_op, compile_mul); - define_builtin_func("^_/=_", arith_bin_op, std::bind(compile_div, _1, _2, -1)); - define_builtin_func("^_~/=_", arith_bin_op, std::bind(compile_div, _1, _2, 0)); - define_builtin_func("^_^/=_", arith_bin_op, std::bind(compile_div, _1, _2, 1)); - define_builtin_func("^_%=_", arith_bin_op, std::bind(compile_mod, _1, _2, -1)); - define_builtin_func("^_~%=_", arith_bin_op, std::bind(compile_mod, _1, _2, 0)); - define_builtin_func("^_^%=_", arith_bin_op, std::bind(compile_mod, _1, _2, 1)); + define_builtin_func("^_/=_", arith_bin_op, std::bind(compile_div, _1, _2, _3, -1)); + define_builtin_func("^_~/=_", arith_bin_op, std::bind(compile_div, _1, _2, _3, 0)); + define_builtin_func("^_^/=_", arith_bin_op, std::bind(compile_div, _1, _2, _3, 1)); + define_builtin_func("^_%=_", arith_bin_op, std::bind(compile_mod, _1, _2, _3, -1)); + define_builtin_func("^_~%=_", arith_bin_op, std::bind(compile_mod, _1, _2, _3, 0)); + define_builtin_func("^_^%=_", arith_bin_op, std::bind(compile_mod, _1, _2, _3, 1)); define_builtin_func("^_<<=_", arith_bin_op, compile_lshift); - define_builtin_func("^_>>=_", arith_bin_op, std::bind(compile_rshift, _1, _2, -1)); - define_builtin_func("^_~>>=_", arith_bin_op, std::bind(compile_rshift, _1, _2, 0)); - define_builtin_func("^_^>>=_", arith_bin_op, std::bind(compile_rshift, _1, _2, 1)); + define_builtin_func("^_>>=_", arith_bin_op, std::bind(compile_rshift, _1, _2, _3, -1)); + define_builtin_func("^_~>>=_", arith_bin_op, std::bind(compile_rshift, _1, _2, _3, 0)); + define_builtin_func("^_^>>=_", arith_bin_op, std::bind(compile_rshift, _1, _2, _3, 1)); define_builtin_func("^_&=_", arith_bin_op, compile_and); define_builtin_func("^_|=_", arith_bin_op, compile_or); define_builtin_func("^_^=_", arith_bin_op, compile_xor); - define_builtin_func("muldiv", TypeExpr::new_map(Int3, Int), std::bind(compile_muldiv, _1, _2, -1)); - define_builtin_func("muldivr", TypeExpr::new_map(Int3, Int), std::bind(compile_muldiv, _1, _2, 0)); - define_builtin_func("muldivc", TypeExpr::new_map(Int3, Int), std::bind(compile_muldiv, _1, _2, 1)); + define_builtin_func("muldiv", TypeExpr::new_map(Int3, Int), std::bind(compile_muldiv, _1, _2, _3, -1)); + define_builtin_func("muldivr", TypeExpr::new_map(Int3, Int), std::bind(compile_muldiv, _1, _2, _3, 0)); + define_builtin_func("muldivc", TypeExpr::new_map(Int3, Int), std::bind(compile_muldiv, _1, _2, _3, 1)); define_builtin_func("muldivmod", TypeExpr::new_map(Int3, Int2), AsmOp::Custom("MULDIVMOD", 3, 2)); define_builtin_func("_==_", arith_bin_op, std::bind(compile_cmp_int, _1, _2, 2)); define_builtin_func("_!=_", arith_bin_op, std::bind(compile_cmp_int, _1, _2, 5)); @@ -1232,16 +1248,18 @@ void define_builtins() { AsmOp::Custom("s0 DUMP", 1, 1), true); define_builtin_func("~strdump", TypeExpr::new_forall({X}, TypeExpr::new_map(X, TypeExpr::new_tensor({X, Unit}))), AsmOp::Custom("STRDUMP", 1, 1), true); - define_builtin_func("run_method0", TypeExpr::new_map(Int, Unit), - [](auto a, auto b, auto c) { return compile_run_method(a, b, c, 0, false); }, true); - define_builtin_func("run_method1", TypeExpr::new_forall({X}, TypeExpr::new_map(TypeExpr::new_tensor({Int, X}), Unit)), - [](auto a, auto b, auto c) { return compile_run_method(a, b, c, 1, false); }, {1, 0}, {}, true); + define_builtin_func( + "run_method0", TypeExpr::new_map(Int, Unit), + [](AsmOpList& a, auto b, auto c) { return compile_run_method(a, b, c, 0, false); }, true); + define_builtin_func( + "run_method1", TypeExpr::new_forall({X}, TypeExpr::new_map(TypeExpr::new_tensor({Int, X}), Unit)), + [](AsmOpList& a, auto b, auto c) { return compile_run_method(a, b, c, 1, false); }, {1, 0}, {}, true); define_builtin_func( "run_method2", TypeExpr::new_forall({X, Y}, TypeExpr::new_map(TypeExpr::new_tensor({Int, X, Y}), Unit)), - [](auto a, auto b, auto c) { return compile_run_method(a, b, c, 2, false); }, {1, 2, 0}, {}, true); + [](AsmOpList& a, auto b, auto c) { return compile_run_method(a, b, c, 2, false); }, {1, 2, 0}, {}, true); define_builtin_func( "run_method3", TypeExpr::new_forall({X, Y, Z}, TypeExpr::new_map(TypeExpr::new_tensor({Int, X, Y, Z}), Unit)), - [](auto a, auto b, auto c) { return compile_run_method(a, b, c, 3, false); }, {1, 2, 3, 0}, {}, true); + [](AsmOpList& a, auto b, auto c) { return compile_run_method(a, b, c, 3, false); }, {1, 2, 3, 0}, {}, true); } } // namespace funC diff --git a/crypto/func/codegen.cpp b/crypto/func/codegen.cpp index ac43d12a..aa2972c2 100644 --- a/crypto/func/codegen.cpp +++ b/crypto/func/codegen.cpp @@ -165,7 +165,7 @@ void Stack::push_new_const(var_idx_t idx, const_idx_t cidx) { void Stack::assign_var(var_idx_t new_idx, var_idx_t old_idx) { int i = find(old_idx); - assert(i >= 0 && "variable not found in stack"); + func_assert(i >= 0 && "variable not found in stack"); if (new_idx != old_idx) { at(i).first = new_idx; modified(); @@ -174,10 +174,10 @@ void Stack::assign_var(var_idx_t new_idx, var_idx_t old_idx) { void Stack::do_copy_var(var_idx_t new_idx, var_idx_t old_idx) { int i = find(old_idx); - assert(i >= 0 && "variable not found in stack"); + func_assert(i >= 0 && "variable not found in stack"); if (find(old_idx, i + 1) < 0) { issue_push(i); - assert(at(0).first == old_idx); + func_assert(at(0).first == old_idx); } assign_var(new_idx, old_idx); } @@ -199,21 +199,21 @@ void Stack::enforce_state(const StackLayout& req_stack) { j = 0; } issue_xchg(j, depth() - i - 1); - assert(s[i].first == x); + func_assert(s[i].first == x); } while (depth() > k) { issue_pop(0); } - assert(depth() == k); + func_assert(depth() == k); for (int i = 0; i < k; i++) { - assert(s[i].first == req_stack[i]); + func_assert(s[i].first == req_stack[i]); } } void Stack::merge_const(const Stack& req_stack) { - assert(s.size() == req_stack.s.size()); + func_assert(s.size() == req_stack.s.size()); for (std::size_t i = 0; i < s.size(); i++) { - assert(s[i].first == req_stack.s[i].first); + func_assert(s[i].first == req_stack.s[i].first); if (s[i].second != req_stack.s[i].second) { s[i].second = not_const; } @@ -251,15 +251,15 @@ void Stack::rearrange_top(const StackLayout& top, std::vector last) { if (last[i]) { // rearrange x to be at s(ss-1) issue_xchg(--ss, j); - assert(get(ss).first == x); + func_assert(get(ss).first == x); } else { // create a new copy of x issue_push(j); issue_xchg(0, ss); - assert(get(ss).first == x); + func_assert(get(ss).first == x); } } - assert(!ss); + func_assert(!ss); } void Stack::rearrange_top(var_idx_t top, bool last) { @@ -269,14 +269,13 @@ void Stack::rearrange_top(var_idx_t top, bool last) { } else { issue_push(i); } - assert(get(0).first == top); + func_assert(get(0).first == top); } bool Op::generate_code_step(Stack& stack) { stack.opt_show(); stack.drop_vars_except(var_info); stack.opt_show(); - const auto& next_var_info = next->var_info; bool inline_func = stack.mode & Stack::_InlineFunc; switch (cl) { case _Nop: @@ -291,7 +290,7 @@ bool Op::generate_code_step(Stack& stack) { return false; } case _IntConst: { - auto p = next_var_info[left[0]]; + auto p = next->var_info[left[0]]; if (!p || p->is_unused()) { return true; } @@ -301,13 +300,13 @@ bool Op::generate_code_step(Stack& stack) { stack.o << push_const(int_const); stack.push_new_const(left[0], cidx); } else { - assert(stack.at(i).second == cidx); + func_assert(stack.at(i).second == cidx); stack.do_copy_var(left[0], stack[i]); } return true; } case _SliceConst: { - auto p = next_var_info[left[0]]; + auto p = next->var_info[left[0]]; if (!p || p->is_unused()) { return true; } @@ -319,7 +318,7 @@ bool Op::generate_code_step(Stack& stack) { if (dynamic_cast(fun_ref->value)) { bool used = false; for (auto i : left) { - auto p = next_var_info[i]; + auto p = next->var_info[i]; if (p && !p->is_unused()) { used = true; } @@ -330,7 +329,7 @@ bool Op::generate_code_step(Stack& stack) { std::string name = sym::symbols.get_name(fun_ref->sym_idx); stack.o << AsmOp::Custom(name + " GETGLOB", 0, 1); if (left.size() != 1) { - assert(left.size() <= 15); + func_assert(left.size() <= 15); stack.o << AsmOp::UnTuple((int)left.size()); } for (auto i : left) { @@ -338,8 +337,8 @@ bool Op::generate_code_step(Stack& stack) { } return true; } else { - assert(left.size() == 1); - auto p = next_var_info[left[0]]; + func_assert(left.size() == 1); + auto p = next->var_info[left[0]]; if (!p || p->is_unused() || disabled()) { return true; } @@ -350,17 +349,17 @@ bool Op::generate_code_step(Stack& stack) { // TODO: create and compile a true lambda instead of this (so that arg_order and ret_order would work correctly) std::vector args0, res; TypeExpr::remove_indirect(func->sym_type); - assert(func->get_type()->is_map()); + func_assert(func->get_type()->is_map()); auto wr = func->get_type()->args.at(0)->get_width(); auto wl = func->get_type()->args.at(1)->get_width(); - assert(wl >= 0 && wr >= 0); + func_assert(wl >= 0 && wr >= 0); for (int i = 0; i < wl; i++) { res.emplace_back(0); } for (int i = 0; i < wr; i++) { args0.emplace_back(0); } - func->compile(stack.o, res, args0); // compile res := f (args0) + func->compile(stack.o, res, args0, where); // compile res := f (args0) } else { std::string name = sym::symbols.get_name(fun_ref->sym_idx); stack.o << AsmOp::Custom(name + " CALLDICT", (int)right.size(), (int)left.size()); @@ -371,13 +370,13 @@ bool Op::generate_code_step(Stack& stack) { return true; } case _Let: { - assert(left.size() == right.size()); + func_assert(left.size() == right.size()); int i = 0; std::vector active; active.reserve(left.size()); for (std::size_t k = 0; k < left.size(); k++) { var_idx_t y = left[k]; // "y" = "x" - auto p = next_var_info[y]; + auto p = next->var_info[y]; active.push_back(p && !p->is_unused()); } for (std::size_t k = 0; k < left.size(); k++) { @@ -421,13 +420,13 @@ bool Op::generate_code_step(Stack& stack) { stack.rearrange_top(right, std::move(last)); stack.opt_show(); int k = (int)stack.depth() - (int)right.size(); - assert(k >= 0); + func_assert(k >= 0); if (cl == _Tuple) { stack.o << AsmOp::Tuple((int)right.size()); - assert(left.size() == 1); + func_assert(left.size() == 1); } else { stack.o << AsmOp::UnTuple((int)left.size()); - assert(right.size() == 1); + func_assert(right.size() == 1); } stack.s.resize(k); for (int i = 0; i < (int)left.size(); i++) { @@ -443,16 +442,16 @@ bool Op::generate_code_step(Stack& stack) { SymValFunc* func = (fun_ref ? dynamic_cast(fun_ref->value) : nullptr); auto arg_order = (func ? func->get_arg_order() : nullptr); auto ret_order = (func ? func->get_ret_order() : nullptr); - assert(!arg_order || arg_order->size() == right.size()); - assert(!ret_order || ret_order->size() == left.size()); + func_assert(!arg_order || arg_order->size() == right.size()); + func_assert(!ret_order || ret_order->size() == left.size()); std::vector right1; if (args.size()) { - assert(args.size() == right.size()); + func_assert(args.size() == right.size()); for (int i = 0; i < (int)right.size(); i++) { int j = arg_order ? arg_order->at(i) : i; const VarDescr& arg = args.at(j); if (!arg.is_unused()) { - assert(var_info[arg.idx] && !var_info[arg.idx]->is_unused()); + func_assert(var_info[arg.idx] && !var_info[arg.idx]->is_unused()); right1.push_back(arg.idx); } } @@ -470,17 +469,25 @@ bool Op::generate_code_step(Stack& stack) { stack.rearrange_top(right1, std::move(last)); stack.opt_show(); int k = (int)stack.depth() - (int)right1.size(); - assert(k >= 0); + func_assert(k >= 0); for (int i = 0; i < (int)right1.size(); i++) { if (stack.s[k + i].first != right1[i]) { std::cerr << stack.o; } - assert(stack.s[k + i].first == right1[i]); + func_assert(stack.s[k + i].first == right1[i]); } + auto exec_callxargs = [&](int args, int ret) { + if (args <= 15 && ret <= 15) { + stack.o << exec_arg2_op("CALLXARGS", args, ret, args + 1, ret); + } else { + func_assert(args <= 254 && ret <= 254); + stack.o << AsmOp::Const(PSTRING() << args << " PUSHINT"); + stack.o << AsmOp::Const(PSTRING() << ret << " PUSHINT"); + stack.o << AsmOp::Custom("CALLXVARARGS", args + 3, ret); + } + }; if (cl == _CallInd) { - // TODO: replace with exec_arg2_op() - stack.o << exec_arg2_op("CALLXARGS", (int)right.size() - 1, (int)left.size(), (int)right.size(), - (int)left.size()); + exec_callxargs((int)right.size() - 1, (int)left.size()); } else { auto func = dynamic_cast(fun_ref->value); if (func) { @@ -489,13 +496,19 @@ bool Op::generate_code_step(Stack& stack) { for (var_idx_t i : left) { res.emplace_back(i); } - func->compile(stack.o, res, args); // compile res := f (args) + func->compile(stack.o, res, args, where); // compile res := f (args) } else { auto fv = dynamic_cast(fun_ref->value); std::string name = sym::symbols.get_name(fun_ref->sym_idx); bool is_inline = (fv && (fv->flags & 3)); - stack.o << AsmOp::Custom(name + (is_inline ? " INLINECALLDICT" : " CALLDICT"), (int)right.size(), - (int)left.size()); + if (is_inline) { + stack.o << AsmOp::Custom(name + " INLINECALLDICT", (int)right.size(), (int)left.size()); + } else if (fv && fv->code && fv->code->require_callxargs) { + stack.o << AsmOp::Custom(name + (" PREPAREDICT"), 0, 2); + exec_callxargs((int)right.size() + 1, (int)left.size()); + } else { + stack.o << AsmOp::Custom(name + " CALLDICT", (int)right.size(), (int)left.size()); + } } } stack.s.resize(k); @@ -506,7 +519,7 @@ bool Op::generate_code_step(Stack& stack) { return true; } case _SetGlob: { - assert(fun_ref && dynamic_cast(fun_ref->value)); + func_assert(fun_ref && dynamic_cast(fun_ref->value)); std::vector last; for (var_idx_t x : right) { last.push_back(var_info[x] && var_info[x]->is_last()); @@ -514,12 +527,12 @@ bool Op::generate_code_step(Stack& stack) { stack.rearrange_top(right, std::move(last)); stack.opt_show(); int k = (int)stack.depth() - (int)right.size(); - assert(k >= 0); + func_assert(k >= 0); for (int i = 0; i < (int)right.size(); i++) { if (stack.s[k + i].first != right[i]) { std::cerr << stack.o; } - assert(stack.s[k + i].first == right[i]); + func_assert(stack.s[k + i].first == right[i]); } if (right.size() > 1) { stack.o << AsmOp::Tuple((int)right.size()); @@ -540,7 +553,7 @@ bool Op::generate_code_step(Stack& stack) { } var_idx_t x = left[0]; stack.rearrange_top(x, var_info[x] && var_info[x]->is_last()); - assert(stack[0] == x); + func_assert(stack[0] == x); stack.opt_show(); stack.s.pop_back(); stack.modified(); @@ -652,7 +665,7 @@ bool Op::generate_code_step(Stack& stack) { var_idx_t x = left[0]; //stack.drop_vars_except(block0->var_info, x); stack.rearrange_top(x, var_info[x] && var_info[x]->is_last()); - assert(stack[0] == x); + func_assert(stack[0] == x); stack.opt_show(); stack.s.pop_back(); stack.modified(); @@ -817,6 +830,8 @@ bool Op::generate_code_step(Stack& stack) { catch_stack.push_new_var(left[1]); stack.rearrange_top(catch_vars, catch_last); stack.opt_show(); + stack.o << "c1 PUSH"; + stack.o << "c3 PUSH"; stack.o << "c4 PUSH"; stack.o << "c5 PUSH"; stack.o << "c7 PUSH"; @@ -833,6 +848,8 @@ bool Op::generate_code_step(Stack& stack) { stack.o << "c7 SETCONT"; stack.o << "c5 SETCONT"; stack.o << "c4 SETCONT"; + stack.o << "c3 SETCONT"; + stack.o << "c1 SETCONT"; for (size_t begin = catch_vars.size(), end = begin; end > 0; end = begin) { begin = end >= block_size ? end - block_size : 0; stack.o << std::to_string(end - begin) + " PUSHINT"; @@ -878,12 +895,13 @@ void Op::generate_code_all(Stack& stack) { void CodeBlob::generate_code(AsmOpList& out, int mode) { Stack stack{out, mode}; - assert(ops && ops->cl == Op::_Import); + func_assert(ops && ops->cl == Op::_Import); + auto args = (int)ops->left.size(); for (var_idx_t x : ops->left) { stack.push_new_var(x); } ops->generate_code_all(stack); - stack.apply_wrappers(); + stack.apply_wrappers(require_callxargs && (mode & Stack::_InlineAny) ? args : -1); if (!(mode & Stack::_DisableOpt)) { optimize_code(out); } diff --git a/crypto/func/func-main.cpp b/crypto/func/func-main.cpp index 45194ea3..c8208eee 100644 --- a/crypto/func/func-main.cpp +++ b/crypto/func/func-main.cpp @@ -109,7 +109,7 @@ int main(int argc, char* const argv[]) { std::unique_ptr fs; if (!output_filename.empty()) { - fs = std::make_unique(output_filename, fs->trunc | fs->out); + fs = std::make_unique(output_filename, std::fstream::trunc | std::fstream::out); if (!fs->is_open()) { std::cerr << "failed to create output file " << output_filename << '\n'; return 2; @@ -123,5 +123,7 @@ int main(int argc, char* const argv[]) { sources.push_back(std::string(argv[optind++])); } + funC::read_callback = funC::fs_read_callback; + return funC::func_proceed(sources, *outs, std::cerr); } diff --git a/crypto/func/func.cpp b/crypto/func/func.cpp index be44d2ea..39648c05 100644 --- a/crypto/func/func.cpp +++ b/crypto/func/func.cpp @@ -30,6 +30,8 @@ #include "parser/lexer.h" #include #include "git.h" +#include +#include "td/utils/port/path.h" namespace funC { @@ -39,6 +41,28 @@ bool interactive = false; GlobalPragma pragma_allow_post_modification{"allow-post-modification"}; GlobalPragma pragma_compute_asm_ltr{"compute-asm-ltr"}; std::string generated_from, boc_output_filename; +ReadCallback::Callback read_callback; + +td::Result fs_read_callback(ReadCallback::Kind kind, const char* query) { + switch (kind) { + case ReadCallback::Kind::ReadFile: { + std::ifstream ifs{query}; + if (ifs.fail()) { + auto msg = std::string{"cannot open source file `"} + query + "`"; + return td::Status::Error(msg); + } + std::stringstream ss; + ss << ifs.rdbuf(); + return ss.str(); + } + case ReadCallback::Kind::Realpath: { + return td::realpath(td::CSlice(query)); + } + default: { + return td::Status::Error("Unknown query kind"); + } + } +} /* * @@ -48,7 +72,7 @@ std::string generated_from, boc_output_filename; void generate_output_func(SymDef* func_sym, std::ostream &outs, std::ostream &errs) { SymValCodeFunc* func_val = dynamic_cast(func_sym->value); - assert(func_val); + func_assert(func_val); std::string name = sym::symbols.get_name(func_sym->sym_idx); if (verbosity >= 2) { errs << "\n\n=========================\nfunction " << name << " : " << func_val->get_type() << std::endl; @@ -121,6 +145,9 @@ void generate_output_func(SymDef* func_sym, std::ostream &outs, std::ostream &er if (fv && (fv->flags & 1) && code.ops->noreturn()) { mode |= Stack::_InlineFunc; } + if (fv && (fv->flags & 3)) { + mode |= Stack::_InlineAny; + } code.generate_code(outs, mode, indent + 1); outs << std::string(indent * 2, ' ') << "}>\n"; if (verbosity >= 2) { @@ -139,7 +166,7 @@ int generate_output(std::ostream &outs, std::ostream &errs) { } for (SymDef* func_sym : glob_func) { SymValCodeFunc* func_val = dynamic_cast(func_sym->value); - assert(func_val); + func_assert(func_val); std::string name = sym::symbols.get_name(func_sym->sym_idx); outs << std::string(indent * 2, ' '); if (func_val->method_id.is_null()) { @@ -149,7 +176,7 @@ int generate_output(std::ostream &outs, std::ostream &errs) { } } for (SymDef* gvar_sym : glob_vars) { - assert(dynamic_cast(gvar_sym->value)); + func_assert(dynamic_cast(gvar_sym->value)); std::string name = sym::symbols.get_name(gvar_sym->sym_idx); outs << std::string(indent * 2, ' ') << "DECLGLOBVAR " << name << "\n"; } diff --git a/crypto/func/func.h b/crypto/func/func.h index b979afb2..25711db8 100644 --- a/crypto/func/func.h +++ b/crypto/func/func.h @@ -19,6 +19,7 @@ #pragma once #include #include +#include #include #include #include @@ -30,6 +31,11 @@ #include "parser/srcread.h" #include "parser/lexer.h" #include "parser/symtable.h" +#include "td/utils/Status.h" + +#define func_assert(expr) \ + (bool(expr) ? void(0) \ + : throw src::Fatal(PSTRING() << "Assertion failed at " << __FILE__ << ":" << __LINE__ << ": " << #expr)) namespace funC { @@ -39,7 +45,7 @@ extern std::string generated_from; constexpr int optimize_depth = 20; -const std::string func_version{"0.4.1"}; +const std::string func_version{"0.4.6"}; enum Keyword { _Eof = -1, @@ -157,6 +163,7 @@ struct TypeExpr { int minw, maxw; static constexpr int w_inf = 1023; std::vector args; + bool was_forall_var = false; TypeExpr(te_type _constr, int _val = 0) : constr(_constr), value(_val), minw(0), maxw(w_inf) { } TypeExpr(te_type _constr, int _val, int width) : constr(_constr), value(_val), minw(width), maxw(width) { @@ -263,7 +270,7 @@ struct TypeExpr { return new TypeExpr{te_ForAll, body, std::move(list)}; } static bool remove_indirect(TypeExpr*& te, TypeExpr* forbidden = nullptr); - static bool remove_forall(TypeExpr*& te); + static std::vector remove_forall(TypeExpr*& te); static bool remove_forall_in(TypeExpr*& te, TypeExpr* te2, const std::vector& new_vars); }; @@ -307,6 +314,7 @@ struct TmpVar { int coord; std::unique_ptr where; std::vector> on_modification; + bool undefined = false; TmpVar(var_idx_t _idx, int _cls, TypeExpr* _type = 0, SymDef* sym = 0, const SrcLocation* loc = 0); void show(std::ostream& os, int omit_idx = 0) const; void dump(std::ostream& os) const; @@ -505,7 +513,7 @@ class ListIterator { ptr = ptr->next.get(); return *this; } - ListIterator& operator++(int) { + ListIterator operator++(int) { T* z = ptr; ptr = ptr->next.get(); return ListIterator{z}; @@ -691,6 +699,7 @@ struct CodeBlob { std::unique_ptr* cur_ops; std::stack*> cur_ops_stack; int flags = 0; + bool require_callxargs = false; CodeBlob(TypeExpr* ret = nullptr) : var_cnt(0), in_var_cnt(0), op_cnt(0), ret_type(ret), cur_ops(&ops) { } template @@ -838,6 +847,7 @@ struct SymValConst : sym::SymValBase { extern int glob_func_cnt, undef_func_cnt, glob_var_cnt; extern std::vector glob_func, glob_vars; +extern std::set prohibited_var_names; /* * @@ -845,6 +855,35 @@ extern std::vector glob_func, glob_vars; * */ +class ReadCallback { +public: + /// Noncopyable. + ReadCallback(ReadCallback const&) = delete; + ReadCallback& operator=(ReadCallback const&) = delete; + + enum class Kind + { + ReadFile, + Realpath + }; + + static std::string kindString(Kind _kind) + { + switch (_kind) + { + case Kind::ReadFile: + return "source"; + case Kind::Realpath: + return "realpath"; + default: + throw ""; // todo ? + } + } + + /// File reading or generic query callback. + using Callback = std::function(ReadCallback::Kind, const char*)>; +}; + // defined in parse-func.cpp bool parse_source(std::istream* is, const src::FileDescr* fdescr); bool parse_source_file(const char* filename, src::Lexem lex = {}, bool is_main = false); @@ -1144,7 +1183,7 @@ struct AsmOpList { } template AsmOpList& add(Args&&... args) { - list_.emplace_back(std::forward(args)...); + append(AsmOp(std::forward(args)...)); adjust_last(); return *this; } @@ -1531,8 +1570,8 @@ struct Stack { AsmOpList& o; enum { _StkCmt = 1, _CptStkCmt = 2, _DisableOpt = 4, _DisableOut = 128, _Shown = 256, - _InlineFunc = 512, _NeedRetAlt = 1024, - _ModeSave = _InlineFunc | _NeedRetAlt, + _InlineFunc = 512, _NeedRetAlt = 1024, _InlineAny = 2048, + _ModeSave = _InlineFunc | _NeedRetAlt | _InlineAny, _Garbage = -0x10000 }; int mode; @@ -1579,7 +1618,7 @@ struct Stack { if (i > 255) { throw src::Fatal{"Too deep stack"}; } - assert(i >= 0 && i < depth() && "invalid stack reference"); + func_assert(i >= 0 && i < depth() && "invalid stack reference"); } void modified() { mode &= ~_Shown; @@ -1610,14 +1649,24 @@ struct Stack { bool operator==(const Stack& y) const & { return s == y.s; } - void apply_wrappers() { + void apply_wrappers(int callxargs_count) { + bool is_inline = mode & _InlineFunc; if (o.retalt_) { o.insert(0, "SAMEALTSAVE"); o.insert(0, "c2 SAVE"); - if (mode & _InlineFunc) { - o.indent_all(); - o.insert(0, "CONT:<{"); - o << "}>"; + } + if (callxargs_count != -1 || (is_inline && o.retalt_)) { + o.indent_all(); + o.insert(0, "CONT:<{"); + o << "}>"; + if (callxargs_count != -1) { + if (callxargs_count <= 15) { + o << AsmOp::Custom(PSTRING() << callxargs_count << " -1 CALLXARGS"); + } else { + func_assert(callxargs_count <= 254); + o << AsmOp::Custom(PSTRING() << callxargs_count << " PUSHINT -1 PUSHINT CALLXVARARGS"); + } + } else { o << "EXECUTE"; } } @@ -1631,11 +1680,11 @@ struct Stack { * */ -typedef std::function&, std::vector&)> simple_compile_func_t; +typedef std::function&, std::vector&, const SrcLocation)> simple_compile_func_t; typedef std::function&, std::vector&)> compile_func_t; inline simple_compile_func_t make_simple_compile(AsmOp op) { - return [op](std::vector& out, std::vector& in) -> AsmOp { return op; }; + return [op](std::vector& out, std::vector& in, const SrcLocation&) -> AsmOp { return op; }; } inline compile_func_t make_ext_compile(std::vector ops) { @@ -1674,7 +1723,7 @@ struct SymValAsmFunc : SymValFunc { std::initializer_list ret_order = {}, bool impure = false) : SymValFunc(-1, ft, arg_order, ret_order, impure), ext_compile(std::move(_compile)) { } - bool compile(AsmOpList& dest, std::vector& out, std::vector& in) const; + bool compile(AsmOpList& dest, std::vector& out, std::vector& in, const SrcLocation& where) const; }; // defined in builtins.cpp @@ -1691,6 +1740,9 @@ void define_builtins(); extern int verbosity, indent, opt_level; extern bool stack_layout_comments, op_rewrite_comments, program_envelope, asm_preamble, interactive; extern std::string generated_from, boc_output_filename; +extern ReadCallback::Callback read_callback; + +td::Result fs_read_callback(ReadCallback::Kind kind, const char* query); class GlobalPragma { public: diff --git a/crypto/func/gen-abscode.cpp b/crypto/func/gen-abscode.cpp index a2b98878..7421dabb 100644 --- a/crypto/func/gen-abscode.cpp +++ b/crypto/func/gen-abscode.cpp @@ -92,7 +92,7 @@ bool Expr::deduce_type(const Lexem& lem) { return true; } case _VarApply: { - assert(args.size() == 2); + func_assert(args.size() == 2); TypeExpr* fun_type = TypeExpr::new_map(args[1]->e_type, TypeExpr::new_hole()); try { unify(fun_type, args[0]->e_type); @@ -107,7 +107,7 @@ bool Expr::deduce_type(const Lexem& lem) { return true; } case _Letop: { - assert(args.size() == 2); + func_assert(args.size() == 2); try { // std::cerr << "in assignment: " << args[0]->e_type << " from " << args[1]->e_type << std::endl; unify(args[0]->e_type, args[1]->e_type); @@ -122,7 +122,7 @@ bool Expr::deduce_type(const Lexem& lem) { return true; } case _LetFirst: { - assert(args.size() == 2); + func_assert(args.size() == 2); TypeExpr* rhs_type = TypeExpr::new_tensor({args[0]->e_type, TypeExpr::new_hole()}); try { // std::cerr << "in implicit assignment of a modifying method: " << rhs_type << " and " << args[1]->e_type << std::endl; @@ -140,7 +140,7 @@ bool Expr::deduce_type(const Lexem& lem) { return true; } case _CondExpr: { - assert(args.size() == 3); + func_assert(args.size() == 3); auto flag_type = TypeExpr::new_atomic(_Int); try { unify(args[0]->e_type, flag_type); @@ -204,7 +204,11 @@ int Expr::predefine_vars() { } case _Var: if (!sym) { - assert(val < 0 && here.defined()); + func_assert(val < 0 && here.defined()); + if (prohibited_var_names.count(sym::symbols.get_name(~val))) { + throw src::ParseError{ + here, PSTRING() << "symbol `" << sym::symbols.get_name(~val) << "` cannot be redefined as a variable"}; + } sym = sym::define_symbol(~val, false, here); // std::cerr << "predefining variable " << sym::symbols.get_name(~val) << std::endl; if (!sym) { @@ -270,7 +274,7 @@ std::vector pre_compile_tensor(const std::vector args, CodeBl arg_order.resize(args.size()); std::iota(arg_order.begin(), arg_order.end(), 0); } - assert(args.size() == arg_order.size()); + func_assert(args.size() == arg_order.size()); std::vector> res_lists(args.size()); struct ModifiedVar { @@ -306,7 +310,7 @@ std::vector pre_compile_tensor(const std::vector args, CodeBl } for (const auto& list : res_lists) { for (var_idx_t v : list) { - assert(!code.vars.at(v).on_modification.empty()); + func_assert(!code.vars.at(v).on_modification.empty()); code.vars.at(v).on_modification.pop_back(); } } @@ -335,7 +339,7 @@ std::vector Expr::pre_compile(CodeBlob& code, std::vector(sym->value); std::vector res; if (func && func->arg_order.size() == args.size() && !(code.flags & CodeBlob::_ComputeAsmLtr)) { @@ -376,7 +380,10 @@ std::vector Expr::pre_compile(CodeBlob& code, std::vector Expr::pre_compile(CodeBlob& code, std::vectorpre_compile(code); - assert(cond.size() == 1); + func_assert(cond.size() == 1); auto rvect = new_tmp_vect(code); Op& if_op = code.emplace_back(here, Op::_If, cond); code.push_set_cur(if_op.block0); diff --git a/crypto/func/optimize.cpp b/crypto/func/optimize.cpp index f8dc3d82..74bb97ec 100644 --- a/crypto/func/optimize.cpp +++ b/crypto/func/optimize.cpp @@ -61,9 +61,9 @@ void Optimizer::apply() { if (!p_ && !q_) { return; } - assert(p_ > 0 && p_ <= l_ && q_ >= 0 && q_ <= n && l_ <= n); + func_assert(p_ > 0 && p_ <= l_ && q_ >= 0 && q_ <= n && l_ <= n); for (int i = p_; i < l_; i++) { - assert(op_[i]); + func_assert(op_[i]); op_cons_[i]->car = std::move(op_[i]); op_cons_[i] = nullptr; } @@ -71,7 +71,7 @@ void Optimizer::apply() { code_ = std::move(code_->cdr); } for (int j = q_ - 1; j >= 0; j--) { - assert(oq_[j]); + func_assert(oq_[j]); oq_[j]->indent = indent_; code_ = AsmOpCons::cons(std::move(oq_[j]), std::move(code_)); } @@ -246,7 +246,7 @@ bool Optimizer::rewrite_const_push_xchgs() { } show_left(); auto c_op = std::move(op_[0]); - assert(c_op->is_gconst()); + func_assert(c_op->is_gconst()); StackTransform t; q_ = 0; int pos = 0; @@ -265,31 +265,31 @@ bool Optimizer::rewrite_const_push_xchgs() { if (b > pos) { oq_[q_]->b = b - 1; } - assert(apply_op(t, *oq_[q_])); + func_assert(apply_op(t, *oq_[q_])); ++q_; } } else { - assert(op_[i]->is_push(&a)); - assert(a != pos); + func_assert(op_[i]->is_push(&a)); + func_assert(a != pos); oq_[q_] = std::move(op_[i]); if (a > pos) { oq_[q_]->a = a - 1; } - assert(apply_op(t, *oq_[q_])); + func_assert(apply_op(t, *oq_[q_])); ++q_; ++pos; } } - assert(!pos); + func_assert(!pos); t.apply_push_newconst(); - assert(t <= tr_[p_ - 1]); + func_assert(t <= tr_[p_ - 1]); oq_[q_++] = std::move(c_op); show_right(); return true; } bool Optimizer::rewrite(int p, AsmOp&& new_op) { - assert(p > 0 && p <= l_); + func_assert(p > 0 && p <= l_); p_ = p; q_ = 1; show_left(); @@ -300,7 +300,7 @@ bool Optimizer::rewrite(int p, AsmOp&& new_op) { } bool Optimizer::rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2) { - assert(p > 1 && p <= l_); + func_assert(p > 1 && p <= l_); p_ = p; q_ = 2; show_left(); @@ -313,7 +313,7 @@ bool Optimizer::rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2) { } bool Optimizer::rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2, AsmOp&& new_op3) { - assert(p > 2 && p <= l_); + func_assert(p > 2 && p <= l_); p_ = p; q_ = 3; show_left(); @@ -328,7 +328,7 @@ bool Optimizer::rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2, AsmOp&& new_op3 } bool Optimizer::rewrite_nop() { - assert(p_ > 0 && p_ <= l_); + func_assert(p_ > 0 && p_ <= l_); q_ = 0; show_left(); show_right(); diff --git a/crypto/func/parse-func.cpp b/crypto/func/parse-func.cpp index 011947d5..4fc1cece 100644 --- a/crypto/func/parse-func.cpp +++ b/crypto/func/parse-func.cpp @@ -22,8 +22,6 @@ #include "openssl/digest.hpp" #include "block/block.h" #include "block-parse.h" -#include -#include "td/utils/port/path.h" namespace sym { @@ -174,6 +172,10 @@ FormalArg parse_formal_arg(Lexer& lex, int fa_idx) { lex.expect(_Ident, "formal parameter name"); } loc = lex.cur().loc; + if (prohibited_var_names.count(sym::symbols.get_name(lex.cur().val))) { + throw src::ParseError{ + loc, PSTRING() << "symbol `" << sym::symbols.get_name(lex.cur().val) << "` cannot be redefined as a variable"}; + } SymDef* new_sym_def = sym::define_symbol(lex.cur().val, true, loc); if (!new_sym_def) { lex.cur().error_at("cannot define symbol `", "`"); @@ -397,9 +399,9 @@ bool check_global_func(const Lexem& cur, sym_idx_t func_name = 0) { cur.loc.show_error(std::string{"undefined function `"} + symbols.get_name(func_name) + "`, defining a global function of unknown type"); def = sym::define_global_symbol(func_name, 0, cur.loc); - assert(def && "cannot define global function"); + func_assert(def && "cannot define global function"); ++undef_func_cnt; - make_new_glob_func(def, TypeExpr::new_hole()); // was: ... ::new_func() + make_new_glob_func(def, TypeExpr::new_func()); // was: ... ::new_func() return true; } SymVal* val = dynamic_cast(def->value); @@ -425,7 +427,7 @@ Expr* make_func_apply(Expr* fun, Expr* x) { res->flags = Expr::_IsRvalue | (fun->flags & Expr::_IsImpure); } else { res = new Expr{Expr::_VarApply, {fun, x}}; - res->flags = Expr::_IsRvalue; + res->flags = Expr::_IsRvalue | Expr::_IsImpure; // for `some_var()`, don't make any considerations about runtime value, it's impure } return res; } @@ -487,7 +489,7 @@ Expr* parse_expr100(Lexer& lex, CodeBlob& code, bool nv) { Expr* res = new Expr{Expr::_Const, lex.cur().loc}; res->flags = Expr::_IsRvalue; res->intval = td::string_to_int256(lex.cur().str); - if (res->intval.is_null()) { + if (res->intval.is_null() || !res->intval->signed_fits_bits(257)) { lex.cur().error_at("invalid integer constant `", "`"); } res->e_type = TypeExpr::new_atomic(_Int); @@ -1109,6 +1111,7 @@ blk_fl::val parse_do_stmt(Lexer& lex, CodeBlob& code) { } blk_fl::val parse_try_catch_stmt(Lexer& lex, CodeBlob& code) { + code.require_callxargs = true; lex.expect(_Try); Op& try_catch_op = code.emplace_back(lex.cur().loc, Op::_TryCatch); code.push_set_cur(try_catch_op.block0); @@ -1130,7 +1133,7 @@ blk_fl::val parse_try_catch_stmt(Lexer& lex, CodeBlob& code) { expr->predefine_vars(); expr->define_new_vars(code); try_catch_op.left = expr->pre_compile(code); - assert(try_catch_op.left.size() == 2); + func_assert(try_catch_op.left.size() == 2 || try_catch_op.left.size() == 1); blk_fl::val res1 = parse_block_stmt(lex, code); sym::close_scope(lex); code.close_pop_cur(lex.cur().loc); @@ -1293,7 +1296,7 @@ SymValAsmFunc* parse_asm_func_body(Lexer& lex, TypeExpr* func_type, const Formal } lex.next(); } - assert(arg_order.size() == (unsigned)tot_width); + func_assert(arg_order.size() == (unsigned)tot_width); } if (lex.tp() == _Mapsto) { lex.expect(_Mapsto); @@ -1371,6 +1374,10 @@ std::vector parse_type_var_list(Lexer& lex) { throw src::ParseError{lex.cur().loc, "free type identifier expected"}; } auto loc = lex.cur().loc; + if (prohibited_var_names.count(sym::symbols.get_name(lex.cur().val))) { + throw src::ParseError{loc, PSTRING() << "symbol `" << sym::symbols.get_name(lex.cur().val) + << "` cannot be redefined as a variable"}; + } SymDef* new_sym_def = sym::define_symbol(lex.cur().val, true, loc); if (!new_sym_def || new_sym_def->value) { lex.cur().error_at("redefined type variable `", "`"); @@ -1481,7 +1488,7 @@ void parse_func_def(Lexer& lex) { std::cerr << "function " << func_name.str << " : " << func_type << std::endl; } SymDef* func_sym = sym::define_global_symbol(func_name.val, 0, loc); - assert(func_sym); + func_assert(func_sym); SymValFunc* func_sym_val = dynamic_cast(func_sym->value); if (func_sym->value) { if (func_sym->value->type != SymVal::_Func || !func_sym_val) { @@ -1757,7 +1764,7 @@ bool parse_source_file(const char* filename, src::Lexem lex, bool is_main) { } } - auto path_res = td::realpath(td::CSlice(filename)); + auto path_res = read_callback(ReadCallback::Kind::Realpath, filename); if (path_res.is_error()) { auto error = path_res.move_as_error(); lex.error(error.message().c_str()); @@ -1784,17 +1791,19 @@ bool parse_source_file(const char* filename, src::Lexem lex, bool is_main) { source_files[real_filename] = cur_source; cur_source->is_main = is_main; source_fdescr.push_back(cur_source); - std::ifstream ifs{filename}; - if (ifs.fail()) { - auto msg = std::string{"cannot open source file `"} + filename + "`"; + auto file_res = read_callback(ReadCallback::Kind::ReadFile, filename); + if (file_res.is_error()) { + auto msg = file_res.move_as_error().message().str(); if (lex.tp) { lex.error(msg); } else { throw src::Fatal{msg}; } } + auto file_str = file_res.move_as_ok(); + std::stringstream ss{file_str}; inclusion_locations.push(lex.loc); - bool res = parse_source(&ifs, cur_source); + bool res = parse_source(&ss, cur_source); inclusion_locations.pop(); return res; } diff --git a/crypto/func/stack-transform.cpp b/crypto/func/stack-transform.cpp index 4d9b6a5f..b5290608 100644 --- a/crypto/func/stack-transform.cpp +++ b/crypto/func/stack-transform.cpp @@ -183,7 +183,7 @@ bool StackTransform::is_permutation() const { if (!is_valid() || d) { return false; } - assert(n <= max_n); + func_assert(n <= max_n); std::array X, Y; for (int i = 0; i < n; i++) { X[i] = A[i].first; diff --git a/crypto/func/unify-types.cpp b/crypto/func/unify-types.cpp index 517299e9..f9b639c0 100644 --- a/crypto/func/unify-types.cpp +++ b/crypto/func/unify-types.cpp @@ -132,7 +132,7 @@ void TypeExpr::replace_with(TypeExpr* te2) { } bool TypeExpr::remove_indirect(TypeExpr*& te, TypeExpr* forbidden) { - assert(te); + func_assert(te); while (te->constr == te_Indirect) { te = te->args[0]; } @@ -146,12 +146,9 @@ bool TypeExpr::remove_indirect(TypeExpr*& te, TypeExpr* forbidden) { return res; } -bool TypeExpr::remove_forall(TypeExpr*& te) { - assert(te); - if (te->constr != te_ForAll) { - return false; - } - assert(te->args.size() >= 1); +std::vector TypeExpr::remove_forall(TypeExpr*& te) { + func_assert(te && te->constr == te_ForAll); + func_assert(te->args.size() >= 1); std::vector new_vars; for (std::size_t i = 1; i < te->args.size(); i++) { new_vars.push_back(new_hole(1)); @@ -161,12 +158,12 @@ bool TypeExpr::remove_forall(TypeExpr*& te) { te = te->args[0]; remove_forall_in(te, te2, new_vars); // std::cerr << "-> " << te << std::endl; - return true; + return new_vars; } bool TypeExpr::remove_forall_in(TypeExpr*& te, TypeExpr* te2, const std::vector& new_vars) { - assert(te); - assert(te2 && te2->constr == te_ForAll); + func_assert(te); + func_assert(te2 && te2->constr == te_ForAll); if (te->constr == te_Var) { for (std::size_t i = 0; i < new_vars.size(); i++) { if (te == te2->args[i + 1]) { @@ -280,7 +277,7 @@ std::ostream& TypeExpr::print(std::ostream& os, int lex_level) { return os << "]"; } case te_Map: { - assert(args.size() == 2); + func_assert(args.size() == 2); if (lex_level > 0) { os << "("; } @@ -293,7 +290,7 @@ std::ostream& TypeExpr::print(std::ostream& os, int lex_level) { return os; } case te_ForAll: { - assert(args.size() >= 1); + func_assert(args.size() >= 1); if (lex_level > 0) { os << '('; } @@ -346,11 +343,11 @@ void check_update_widths(TypeExpr* te1, TypeExpr* te2) { check_width_compat(te1, te2); te1->minw = te2->minw = std::max(te1->minw, te2->minw); te1->maxw = te2->maxw = std::min(te1->maxw, te2->maxw); - assert(te1->minw <= te1->maxw); + func_assert(te1->minw <= te1->maxw); } void unify(TypeExpr*& te1, TypeExpr*& te2) { - assert(te1 && te2); + func_assert(te1 && te2); // std::cerr << "unify( " << te1 << " , " << te2 << " )\n"; while (te1->constr == TypeExpr::te_Indirect) { te1 = te1->args[0]; @@ -363,23 +360,37 @@ void unify(TypeExpr*& te1, TypeExpr*& te2) { } if (te1->constr == TypeExpr::te_ForAll) { TypeExpr* te = te1; - if (!TypeExpr::remove_forall(te)) { - throw UnifyError{te1, te2, "cannot remove universal type quantifier while performing type unification"}; + std::vector new_vars = TypeExpr::remove_forall(te); + for (TypeExpr* t : new_vars) { + t->was_forall_var = true; } unify(te, te2); + for (TypeExpr* t : new_vars) { + t->was_forall_var = false; + } return; } if (te2->constr == TypeExpr::te_ForAll) { TypeExpr* te = te2; - if (!TypeExpr::remove_forall(te)) { - throw UnifyError{te2, te1, "cannot remove universal type quantifier while performing type unification"}; + std::vector new_vars = TypeExpr::remove_forall(te); + for (TypeExpr* t : new_vars) { + t->was_forall_var = true; } unify(te1, te); + for (TypeExpr* t : new_vars) { + t->was_forall_var = false; + } return; } + if (te1->was_forall_var && te2->constr == TypeExpr::te_Tensor) { + throw UnifyError{te1, te2, "cannot unify generic type and tensor"}; + } + if (te2->was_forall_var && te1->constr == TypeExpr::te_Tensor) { + throw UnifyError{te2, te1, "cannot unify generic type and tensor"}; + } if (te1->constr == TypeExpr::te_Unknown) { if (te2->constr == TypeExpr::te_Unknown) { - assert(te1->value != te2->value); + func_assert(te1->value != te2->value); } if (!TypeExpr::remove_indirect(te2, te1)) { throw UnifyError{te1, te2, "type unification results in an infinite cyclic type"}; diff --git a/crypto/funcfiftlib/funcfiftlib-prejs.js b/crypto/funcfiftlib/funcfiftlib-prejs.js new file mode 100644 index 00000000..38326c38 --- /dev/null +++ b/crypto/funcfiftlib/funcfiftlib-prejs.js @@ -0,0 +1 @@ +var crypto = { getRandomValues: function(array) { for (var i = 0; i < array.length; i++) array[i] = (Math.random()*256)|0 } }; \ No newline at end of file diff --git a/crypto/funcfiftlib/funcfiftlib.cpp b/crypto/funcfiftlib/funcfiftlib.cpp index 6c8912bc..403c075d 100644 --- a/crypto/funcfiftlib/funcfiftlib.cpp +++ b/crypto/funcfiftlib/funcfiftlib.cpp @@ -30,38 +30,17 @@ #include "td/utils/JsonBuilder.h" #include "fift/utils.h" #include "td/utils/base64.h" +#include "td/utils/Status.h" #include #include - -std::string escape_json(const std::string &s) { - std::ostringstream o; - for (auto c = s.cbegin(); c != s.cend(); c++) { - switch (*c) { - case '"': o << "\\\""; break; - case '\\': o << "\\\\"; break; - case '\b': o << "\\b"; break; - case '\f': o << "\\f"; break; - case '\n': o << "\\n"; break; - case '\r': o << "\\r"; break; - case '\t': o << "\\t"; break; - default: - if ('\x00' <= *c && *c <= '\x1f') { - o << "\\u" - << std::hex << std::setw(4) << std::setfill('0') << static_cast(*c); - } else { - o << *c; - } - } - } - return o.str(); -} +#include "vm/boc.h" td::Result compile_internal(char *config_json) { TRY_RESULT(input_json, td::json_decode(td::MutableSlice(config_json))) - auto &obj = input_json.get_object(); + td::JsonObject& config = input_json.get_object(); - TRY_RESULT(opt_level, td::get_json_object_int_field(obj, "optLevel", false)); - TRY_RESULT(sources_obj, td::get_json_object_field(obj, "sources", td::JsonValue::Type::Array, false)); + TRY_RESULT(opt_level, td::get_json_object_int_field(config, "optLevel", false)); + TRY_RESULT(sources_obj, td::get_json_object_field(config, "sources", td::JsonValue::Type::Array, false)); auto &sources_arr = sources_obj.get_array(); @@ -73,32 +52,63 @@ td::Result compile_internal(char *config_json) { funC::opt_level = std::max(0, opt_level); funC::program_envelope = true; + funC::asm_preamble = true; funC::verbosity = 0; funC::indent = 1; std::ostringstream outs, errs; - auto compile_res = funC::func_proceed(sources, outs, errs); - - if (compile_res != 0) { - return td::Status::Error(std::string("Func compilation error: ") + errs.str()); + int funC_res = funC::func_proceed(sources, outs, errs); + if (funC_res != 0) { + return td::Status::Error("FunC compilation error: " + errs.str()); } - TRY_RESULT(code_cell, fift::compile_asm(outs.str(), "/fiftlib/", false)); - TRY_RESULT(boc, vm::std_boc_serialize(code_cell)); + TRY_RESULT(fift_res, fift::compile_asm_program(outs.str(), "/fiftlib/")); td::JsonBuilder result_json; - auto result_obj = result_json.enter_object(); - result_obj("status", "ok"); - result_obj("codeBoc", td::base64_encode(boc)); - result_obj("fiftCode", escape_json(outs.str())); - result_obj.leave(); - - outs.clear(); - errs.clear(); + auto obj = result_json.enter_object(); + obj("status", "ok"); + obj("fiftCode", std::move(fift_res.fiftCode)); + obj("codeBoc", std::move(fift_res.codeBoc64)); + obj("codeHashHex", std::move(fift_res.codeHashHex)); + obj.leave(); return result_json.string_builder().as_cslice().str(); } +/// Callback used to retrieve additional source files or data. +/// +/// @param _kind The kind of callback (a string). +/// @param _data The data for the callback (a string). +/// @param o_contents A pointer to the contents of the file, if found. Allocated via malloc(). +/// @param o_error A pointer to an error message, if there is one. Allocated via malloc(). +/// +/// The callback implementor must use malloc() to allocate storage for +/// contents or error. The callback implementor must use free() to free +/// said storage after func_compile returns. +/// +/// If the callback is not supported, *o_contents and *o_error must be set to NULL. +typedef void (*CStyleReadFileCallback)(char const* _kind, char const* _data, char** o_contents, char** o_error); + +funC::ReadCallback::Callback wrapReadCallback(CStyleReadFileCallback _readCallback) +{ + funC::ReadCallback::Callback readCallback; + if (_readCallback) { + readCallback = [=](funC::ReadCallback::Kind _kind, char const* _data) -> td::Result { + char* contents_c = nullptr; + char* error_c = nullptr; + _readCallback(funC::ReadCallback::kindString(_kind).data(), _data, &contents_c, &error_c); + if (!contents_c && !error_c) { + return td::Status::Error("Callback not supported"); + } + if (contents_c) { + return contents_c; + } + return td::Status::Error(std::string(error_c)); + }; + } + return readCallback; +} + extern "C" { const char* version() { @@ -111,7 +121,13 @@ const char* version() { return strdup(version_json.string_builder().as_cslice().c_str()); } -const char *func_compile(char *config_json) { +const char *func_compile(char *config_json, CStyleReadFileCallback callback) { + if (callback) { + funC::read_callback = wrapReadCallback(callback); + } else { + funC::read_callback = funC::fs_read_callback; + } + auto res = compile_internal(config_json); if (res.is_error()) { diff --git a/crypto/keccak/keccak.cpp b/crypto/keccak/keccak.cpp new file mode 100644 index 00000000..aac48ddc --- /dev/null +++ b/crypto/keccak/keccak.cpp @@ -0,0 +1,473 @@ +/* + * An implementation of the SHA3 (Keccak) hash function family. + * + * Algorithm specifications: http://keccak.noekeon.org/ + * NIST Announcement: + * http://csrc.nist.gov/groups/ST/hash/sha-3/winner_sha-3.html + * + * Written in 2013 by Fabrizio Tarizzo + * + * =================================================================== + * The contents of this file are dedicated to the public domain. To + * the extent that dedication to the public domain is not available, + * everyone is granted a worldwide, perpetual, royalty-free, + * non-exclusive license to exercise all rights associated with the + * contents of this file for any purpose whatsoever. + * No rights are reserved. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * =================================================================== +*/ + +#include +#include +#include +#include + +#define KECCAK_F1600_STATE 200 +#define MIN(x, y) ((x) < (y) ? (x) : (y)) +#define MAX(x, y) ((x) < (y) ? (x) : (y)) + +/** Standard errors common to all ciphers **/ +#define ERR_NULL 1 +#define ERR_MEMORY 2 +#define ERR_NOT_ENOUGH_DATA 3 +#define ERR_ENCRYPT 4 +#define ERR_DECRYPT 5 +#define ERR_KEY_SIZE 6 +#define ERR_NONCE_SIZE 7 +#define ERR_NR_ROUNDS 8 +#define ERR_DIGEST_SIZE 9 +#define ERR_MAX_DATA 10 +#define ERR_MAX_OFFSET 11 +#define ERR_BLOCK_SIZE 12 +#define ERR_TAG_SIZE 13 +#define ERR_VALUE 14 +#define ERR_EC_POINT 15 +#define ERR_EC_CURVE 16 +#define ERR_MODULUS 17 +#define ERR_UNKNOWN 32 + +typedef struct +{ + uint64_t state[25]; + + /* The buffer is as long as the state, + * but only 'rate' bytes will be used. + */ + uint8_t buf[KECCAK_F1600_STATE]; + + /* When absorbing, this is the number of bytes in buf that + * are coming from the message and outstanding. + * When squeezing, this is the remaining number of bytes + * that can be used as digest. + */ + unsigned valid_bytes; + + /* All values in bytes */ + unsigned capacity; + unsigned rate; + + uint8_t squeezing; + uint8_t rounds; +} keccak_state; + +#undef ROL64 +#define ROL64(x,y) ((((x) << (y)) | (x) >> (64-(y))) & 0xFFFFFFFFFFFFFFFFULL) + +static void keccak_function (uint64_t *state, unsigned rounds); + +int keccak_reset(keccak_state *state) +{ + if (NULL == state) + return ERR_NULL; + + memset(state->state, 0, sizeof(state->state)); + memset(state->buf, 0, sizeof(state->buf)); + state->valid_bytes = 0; + state->squeezing = 0; + + return 0; +} + +static void keccak_absorb_internal (keccak_state *self) +{ + unsigned i,j; + uint64_t d; + + for (i=j=0; j < self->rate; ++i, j += 8) { + d = *(const uint64_t*)(self->buf + j); + self->state[i] ^= d; + } +} + +static void +keccak_squeeze_internal (keccak_state *self) +{ + unsigned i, j; + + for (i=j=0; j < self->rate; ++i, j += 8) { + *(uint64_t*)(self->buf+j) = self->state[i]; + } +} + +int keccak_init (keccak_state **state, + size_t capacity_bytes, + uint8_t rounds) +{ + keccak_state *ks; + + if (NULL == state) { + return ERR_NULL; + } + + *state = ks = (keccak_state*) calloc(1, sizeof(keccak_state)); + if (NULL == ks) + return ERR_MEMORY; + + if (capacity_bytes >= KECCAK_F1600_STATE) + return ERR_DIGEST_SIZE; + + if ((rounds != 12) && (rounds != 24)) + return ERR_NR_ROUNDS; + + ks->capacity = (unsigned)capacity_bytes; + + ks->rate = KECCAK_F1600_STATE - ks->capacity; + + ks->squeezing = 0; + ks->rounds = rounds; + + return 0; +} + +int keccak_destroy(keccak_state *state) +{ + free(state); + return 0; +} + +int keccak_absorb (keccak_state *self, + const uint8_t *in, + size_t length) +{ + if (NULL==self || NULL==in) + return ERR_NULL; + + if (self->squeezing != 0) + return ERR_UNKNOWN; + + while (length > 0) { + unsigned tc; + unsigned left; + + left = self->rate - self->valid_bytes; + tc = (unsigned) MIN(length, left); + memcpy(self->buf + self->valid_bytes, in, tc); + + self->valid_bytes += tc; + in += tc; + length -= tc; + + if (self->valid_bytes == self->rate) { + keccak_absorb_internal (self); + keccak_function(self->state, self->rounds); + self->valid_bytes = 0; + } + } + + return 0; +} + +static void keccak_finish (keccak_state *self, uint8_t padding) +{ + assert(self->squeezing == 0); + assert(self->valid_bytes < self->rate); + + /* Padding */ + memset(self->buf + self->valid_bytes, 0, self->rate - self->valid_bytes); + self->buf[self->valid_bytes] = padding; + self->buf[self->rate-1] |= 0x80; + + /* Final absorb */ + keccak_absorb_internal (self); + keccak_function (self->state, self->rounds); + + /* First squeeze */ + self->squeezing = 1; + keccak_squeeze_internal (self); + self->valid_bytes = self->rate; +} + +int keccak_squeeze (keccak_state *self, uint8_t *out, size_t length, uint8_t padding) +{ + if ((NULL == self) || (NULL == out)) + return ERR_NULL; + + if (self->squeezing == 0) { + keccak_finish (self, padding); + } + + assert(self->squeezing == 1); + assert(self->valid_bytes > 0); + assert(self->valid_bytes <= self->rate); + + while (length > 0) { + unsigned tc; + + tc = (unsigned)MIN(self->valid_bytes, length); + memcpy(out, self->buf + (self->rate - self->valid_bytes), tc); + + self->valid_bytes -= tc; + out += tc; + length -= tc; + + if (self->valid_bytes == 0) { + keccak_function (self->state, self->rounds); + keccak_squeeze_internal (self); + self->valid_bytes = self->rate; + } + } + + return 0; +} + +int keccak_digest(keccak_state *state, uint8_t *digest, size_t len, uint8_t padding) +{ + keccak_state tmp; + + if ((NULL==state) || (NULL==digest)) + return ERR_NULL; + + if (2*len != state->capacity) + return ERR_UNKNOWN; + + tmp = *state; + return keccak_squeeze(&tmp, digest, len, padding); +} + +int keccak_copy(const keccak_state *src, keccak_state *dst) +{ + if (NULL == src || NULL == dst) { + return ERR_NULL; + } + + *dst = *src; + return 0; +} + +/* Keccak core function */ + +#define KECCAK_ROUNDS 24 + +#define ROT_01 36 +#define ROT_02 3 +#define ROT_03 41 +#define ROT_04 18 +#define ROT_05 1 +#define ROT_06 44 +#define ROT_07 10 +#define ROT_08 45 +#define ROT_09 2 +#define ROT_10 62 +#define ROT_11 6 +#define ROT_12 43 +#define ROT_13 15 +#define ROT_14 61 +#define ROT_15 28 +#define ROT_16 55 +#define ROT_17 25 +#define ROT_18 21 +#define ROT_19 56 +#define ROT_20 27 +#define ROT_21 20 +#define ROT_22 39 +#define ROT_23 8 +#define ROT_24 14 + +static const uint64_t roundconstants[KECCAK_ROUNDS] = { + 0x0000000000000001ULL, + 0x0000000000008082ULL, + 0x800000000000808aULL, + 0x8000000080008000ULL, + 0x000000000000808bULL, + 0x0000000080000001ULL, + 0x8000000080008081ULL, + 0x8000000000008009ULL, + 0x000000000000008aULL, + 0x0000000000000088ULL, + 0x0000000080008009ULL, + 0x000000008000000aULL, + 0x000000008000808bULL, + 0x800000000000008bULL, + 0x8000000000008089ULL, + 0x8000000000008003ULL, + 0x8000000000008002ULL, + 0x8000000000000080ULL, + 0x000000000000800aULL, + 0x800000008000000aULL, + 0x8000000080008081ULL, + 0x8000000000008080ULL, + 0x0000000080000001ULL, + 0x8000000080008008ULL +}; + +static void keccak_function (uint64_t *state, unsigned rounds) +{ + unsigned i; + unsigned start_round; + + /* Temporary variables to avoid indexing overhead */ + uint64_t a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12; + uint64_t a13, a14, a15, a16, a17, a18, a19, a20, a21, a22, a23, a24; + + uint64_t b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b11, b12; + uint64_t b13, b14, b15, b16, b17, b18, b19, b20, b21, b22, b23, b24; + + uint64_t c0, c1, c2, c3, c4, d; + + a0 = state[0]; + a1 = state[1]; + a2 = state[2]; + a3 = state[3]; + a4 = state[4]; + a5 = state[5]; + a6 = state[6]; + a7 = state[7]; + a8 = state[8]; + a9 = state[9]; + a10 = state[10]; + a11 = state[11]; + a12 = state[12]; + a13 = state[13]; + a14 = state[14]; + a15 = state[15]; + a16 = state[16]; + a17 = state[17]; + a18 = state[18]; + a19 = state[19]; + a20 = state[20]; + a21 = state[21]; + a22 = state[22]; + a23 = state[23]; + a24 = state[24]; + + if (rounds == 24) + start_round = 0; + else /* rounds == 12 */ + start_round = 12; + + for (i = start_round; i < KECCAK_ROUNDS; ++i) { + /* + Uses temporary variables and loop unrolling to + avoid array indexing and inner loops overhead + */ + + /* Prepare column parity for Theta step */ + c0 = a0 ^ a5 ^ a10 ^ a15 ^ a20; + c1 = a1 ^ a6 ^ a11 ^ a16 ^ a21; + c2 = a2 ^ a7 ^ a12 ^ a17 ^ a22; + c3 = a3 ^ a8 ^ a13 ^ a18 ^ a23; + c4 = a4 ^ a9 ^ a14 ^ a19 ^ a24; + + /* Theta + Rho + Pi steps */ + d = c4 ^ ROL64(c1, 1); + b0 = d ^ a0; + b16 = ROL64(d ^ a5, ROT_01); + b7 = ROL64(d ^ a10, ROT_02); + b23 = ROL64(d ^ a15, ROT_03); + b14 = ROL64(d ^ a20, ROT_04); + + d = c0 ^ ROL64(c2, 1); + b10 = ROL64(d ^ a1, ROT_05); + b1 = ROL64(d ^ a6, ROT_06); + b17 = ROL64(d ^ a11, ROT_07); + b8 = ROL64(d ^ a16, ROT_08); + b24 = ROL64(d ^ a21, ROT_09); + + d = c1 ^ ROL64(c3, 1); + b20 = ROL64(d ^ a2, ROT_10); + b11 = ROL64(d ^ a7, ROT_11); + b2 = ROL64(d ^ a12, ROT_12); + b18 = ROL64(d ^ a17, ROT_13); + b9 = ROL64(d ^ a22, ROT_14); + + d = c2 ^ ROL64(c4, 1); + b5 = ROL64(d ^ a3, ROT_15); + b21 = ROL64(d ^ a8, ROT_16); + b12 = ROL64(d ^ a13, ROT_17); + b3 = ROL64(d ^ a18, ROT_18); + b19 = ROL64(d ^ a23, ROT_19); + + d = c3 ^ ROL64(c0, 1); + b15 = ROL64(d ^ a4, ROT_20); + b6 = ROL64(d ^ a9, ROT_21); + b22 = ROL64(d ^ a14, ROT_22); + b13 = ROL64(d ^ a19, ROT_23); + b4 = ROL64(d ^ a24, ROT_24); + + /* Chi + Iota steps */ + a0 = b0 ^ (~b1 & b2) ^ roundconstants[i]; + a1 = b1 ^ (~b2 & b3); + a2 = b2 ^ (~b3 & b4); + a3 = b3 ^ (~b4 & b0); + a4 = b4 ^ (~b0 & b1); + + a5 = b5 ^ (~b6 & b7); + a6 = b6 ^ (~b7 & b8); + a7 = b7 ^ (~b8 & b9); + a8 = b8 ^ (~b9 & b5); + a9 = b9 ^ (~b5 & b6); + + a10 = b10 ^ (~b11 & b12); + a11 = b11 ^ (~b12 & b13); + a12 = b12 ^ (~b13 & b14); + a13 = b13 ^ (~b14 & b10); + a14 = b14 ^ (~b10 & b11); + + a15 = b15 ^ (~b16 & b17); + a16 = b16 ^ (~b17 & b18); + a17 = b17 ^ (~b18 & b19); + a18 = b18 ^ (~b19 & b15); + a19 = b19 ^ (~b15 & b16); + + a20 = b20 ^ (~b21 & b22); + a21 = b21 ^ (~b22 & b23); + a22 = b22 ^ (~b23 & b24); + a23 = b23 ^ (~b24 & b20); + a24 = b24 ^ (~b20 & b21); + } + + state[0] = a0; + state[1] = a1; + state[2] = a2; + state[3] = a3; + state[4] = a4; + state[5] = a5; + state[6] = a6; + state[7] = a7; + state[8] = a8; + state[9] = a9; + state[10] = a10; + state[11] = a11; + state[12] = a12; + state[13] = a13; + state[14] = a14; + state[15] = a15; + state[16] = a16; + state[17] = a17; + state[18] = a18; + state[19] = a19; + state[20] = a20; + state[21] = a21; + state[22] = a22; + state[23] = a23; + state[24] = a24; +} diff --git a/crypto/keccak/keccak.h b/crypto/keccak/keccak.h new file mode 100644 index 00000000..28fa66fb --- /dev/null +++ b/crypto/keccak/keccak.h @@ -0,0 +1,41 @@ +/* + * An implementation of the SHA3 (Keccak) hash function family. + * + * Algorithm specifications: http://keccak.noekeon.org/ + * NIST Announcement: + * http://csrc.nist.gov/groups/ST/hash/sha-3/winner_sha-3.html + * + * Written in 2013 by Fabrizio Tarizzo + * + * =================================================================== + * The contents of this file are dedicated to the public domain. To + * the extent that dedication to the public domain is not available, + * everyone is granted a worldwide, perpetual, royalty-free, + * non-exclusive license to exercise all rights associated with the + * contents of this file for any purpose whatsoever. + * No rights are reserved. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * =================================================================== +*/ + +#pragma once +#include +#include + +struct keccak_state; + +int keccak_reset(keccak_state *state); +int keccak_init(keccak_state **state, size_t capacity_bytes, uint8_t rounds); +int keccak_destroy(keccak_state *state); +int keccak_absorb(keccak_state *self, const uint8_t *in, size_t length); +int keccak_squeeze(keccak_state *self, uint8_t *out, size_t length, uint8_t padding); +int keccak_digest(keccak_state *state, uint8_t *digest, size_t len, uint8_t padding); +int keccak_copy(const keccak_state *src, keccak_state *dst); diff --git a/crypto/openssl/bignum.cpp b/crypto/openssl/bignum.cpp index 74dd6bbf..9b6bf637 100644 --- a/crypto/openssl/bignum.cpp +++ b/crypto/openssl/bignum.cpp @@ -232,6 +232,7 @@ Bignum& Bignum::import_lsb(const unsigned char* buffer, std::size_t size) { std::string Bignum::to_str() const { char* ptr = BN_bn2dec(val); + CHECK(ptr); std::string z(ptr); OPENSSL_free(ptr); return z; @@ -239,6 +240,7 @@ std::string Bignum::to_str() const { std::string Bignum::to_hex() const { char* ptr = BN_bn2hex(val); + CHECK(ptr); std::string z(ptr); OPENSSL_free(ptr); return z; @@ -255,7 +257,13 @@ std::istream& operator>>(std::istream& is, Bignum& x) { return is; } -bool is_prime(const Bignum& p, int nchecks, bool trial_div) { - return BN_is_prime_fasttest_ex(p.bn_ptr(), BN_prime_checks, get_ctx(), trial_div, 0); +bool is_prime(const Bignum& p) { +#if OPENSSL_VERSION_MAJOR >= 3 + int result = BN_check_prime(p.bn_ptr(), get_ctx(), nullptr); + LOG_IF(FATAL, result == -1); + return result; +#else + return BN_is_prime_fasttest_ex(p.bn_ptr(), BN_prime_checks, get_ctx(), true, 0); +#endif } } // namespace arith diff --git a/crypto/openssl/bignum.h b/crypto/openssl/bignum.h index 2a8dd8a0..032dbb02 100644 --- a/crypto/openssl/bignum.h +++ b/crypto/openssl/bignum.h @@ -335,7 +335,7 @@ const Bignum sqr(const Bignum& x); std::ostream& operator<<(std::ostream& os, const Bignum& x); std::istream& operator>>(std::istream& is, Bignum& x); -bool is_prime(const Bignum& p, int nchecks = 64, bool trial_div = true); +bool is_prime(const Bignum& p); inline int cmp(const Bignum& x, const Bignum& y) { return BN_cmp(x.bn_ptr(), y.bn_ptr()); diff --git a/crypto/openssl/digest.hpp b/crypto/openssl/digest.hpp index 5c232df9..2adeef1d 100644 --- a/crypto/openssl/digest.hpp +++ b/crypto/openssl/digest.hpp @@ -48,6 +48,7 @@ struct OpensslEVP_SHA512 { template class HashCtx { + EVP_MD_CTX *base_ctx{nullptr}; EVP_MD_CTX *ctx{nullptr}; void init(); void clear(); @@ -77,16 +78,20 @@ class HashCtx { template void HashCtx::init() { ctx = EVP_MD_CTX_create(); + base_ctx = EVP_MD_CTX_create(); + EVP_DigestInit_ex(base_ctx, H::get_evp(), 0); reset(); } template void HashCtx::reset() { - EVP_DigestInit_ex(ctx, H::get_evp(), 0); + EVP_MD_CTX_copy_ex(ctx, base_ctx); } template void HashCtx::clear() { + EVP_MD_CTX_destroy(base_ctx); + base_ctx = nullptr; EVP_MD_CTX_destroy(ctx); ctx = nullptr; } diff --git a/crypto/parser/lexer.cpp b/crypto/parser/lexer.cpp index 624d8dd2..117f1df5 100644 --- a/crypto/parser/lexer.cpp +++ b/crypto/parser/lexer.cpp @@ -250,7 +250,6 @@ const Lexem& Lexer::next() { } if (is_multiline_quote(src.get_ptr(), src.get_end_ptr())) { src.advance(multiline_quote.size()); - const char* begin = src.get_ptr(); const char* end = nullptr; SrcLocation here = src.here(); std::string body; diff --git a/crypto/parser/symtable.cpp b/crypto/parser/symtable.cpp index a8843da9..d52d9648 100644 --- a/crypto/parser/symtable.cpp +++ b/crypto/parser/symtable.cpp @@ -32,8 +32,8 @@ int scope_level; SymTable<100003> symbols; -SymDef* sym_def[symbols.hprime]; -SymDef* global_sym_def[symbols.hprime]; +SymDef* sym_def[symbols.hprime + 1]; +SymDef* global_sym_def[symbols.hprime + 1]; std::vector> symbol_stack; std::vector scope_opened_at; diff --git a/crypto/parser/symtable.h b/crypto/parser/symtable.h index 9489b2bc..51d59dfa 100644 --- a/crypto/parser/symtable.h +++ b/crypto/parser/symtable.h @@ -161,8 +161,8 @@ struct SymDef { } }; -extern SymDef* sym_def[symbols.hprime]; -extern SymDef* global_sym_def[symbols.hprime]; +extern SymDef* sym_def[symbols.hprime + 1]; +extern SymDef* global_sym_def[symbols.hprime + 1]; extern std::vector> symbol_stack; extern std::vector scope_opened_at; diff --git a/crypto/smartcont/LICENSE.LGPL b/crypto/smartcont/LICENSE.LGPL new file mode 100644 index 00000000..b482fc4e --- /dev/null +++ b/crypto/smartcont/LICENSE.LGPL @@ -0,0 +1,481 @@ + GNU LIBRARY GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1991 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the library GPL. It is + numbered 2 because it goes with version 2 of the ordinary GPL.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Library General Public License, applies to some +specially designated Free Software Foundation software, and to any +other libraries whose authors decide to use it. You can use it for +your libraries, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if +you distribute copies of the library, or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link a program with the library, you must provide +complete object files to the recipients so that they can relink them +with the library, after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + Our method of protecting your rights has two steps: (1) copyright +the library, and (2) offer you this license which gives you legal +permission to copy, distribute and/or modify the library. + + Also, for each distributor's protection, we want to make certain +that everyone understands that there is no warranty for this free +library. If the library is modified by someone else and passed on, we +want its recipients to know that what they have is not the original +version, so that any problems introduced by others will not reflect on +the original authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that companies distributing free +software will individually obtain patent licenses, thus in effect +transforming the program into proprietary software. To prevent this, +we have made it clear that any patent must be licensed for everyone's +free use or not licensed at all. + + Most GNU software, including some libraries, is covered by the ordinary +GNU General Public License, which was designed for utility programs. This +license, the GNU Library General Public License, applies to certain +designated libraries. This license is quite different from the ordinary +one; be sure to read it in full, and don't assume that anything in it is +the same as in the ordinary license. + + The reason we have a separate public license for some libraries is that +they blur the distinction we usually make between modifying or adding to a +program and simply using it. Linking a program with a library, without +changing the library, is in some sense simply using the library, and is +analogous to running a utility program or application program. However, in +a textual and legal sense, the linked executable is a combined work, a +derivative of the original library, and the ordinary General Public License +treats it as such. + + Because of this blurred distinction, using the ordinary General +Public License for libraries did not effectively promote software +sharing, because most developers did not use the libraries. We +concluded that weaker conditions might promote sharing better. + + However, unrestricted linking of non-free programs would deprive the +users of those programs of all benefit from the free status of the +libraries themselves. This Library General Public License is intended to +permit developers of non-free programs to use free libraries, while +preserving your freedom as a user of such programs to change the free +libraries that are incorporated in them. (We have not seen how to achieve +this as regards changes in header files, but we have achieved it as regards +changes in the actual functions of the Library.) The hope is that this +will lead to faster development of free libraries. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, while the latter only +works together with the library. + + Note that it is possible for a library to be covered by the ordinary +General Public License rather than by this special one. + + GNU LIBRARY GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library which +contains a notice placed by the copyright holder or other authorized +party saying it may be distributed under the terms of this Library +General Public License (also called "this License"). Each licensee is +addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also compile or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + c) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + d) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the source code distributed need not include anything that is normally +distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Library General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public + License as published by the Free Software Foundation; either + version 2 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Library General Public License for more details. + + You should have received a copy of the GNU Library General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! diff --git a/crypto/smartcont/config-code.fc b/crypto/smartcont/config-code.fc index a638144e..b3aa04c4 100644 --- a/crypto/smartcont/config-code.fc +++ b/crypto/smartcont/config-code.fc @@ -1,4 +1,6 @@ ;; Simple configuration smart contract +;; Currently deployed config-contract in mainnet can be found +;; on https://verifier.ton.org/Ef9VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVbxn () set_conf_param(int index, cell value) impure { var cs = get_data().begin_parse(); @@ -397,6 +399,7 @@ int register_voting_proposal(slice cs, int msg_value) impure inline_ref { hash = -0xcd506e6c; ;; cannot set mandatory parameter to null } } + ;; Note, in config contract currently deployed in mainnet, this limit is 256 if (param_val.cell_depth() >= 128) { hash = -0xc2616456; ;; bad value } @@ -601,7 +604,7 @@ _ unpack_proposal(slice pstatus) inline_ref { voters_list = cons(voter_id, voters_list); } } until (~ f); - ;; Note there is a bug in config contract currently deployed in testnet2: + ;; Note there is a bug in config contract currently deployed in mainnet: ;; wins and losses are messed up var (rounds_remaining, wins, losses) = (rest~load_uint(8), rest~load_uint(8), rest~load_uint(8)); rest.end_parse(); diff --git a/crypto/smartcont/highload-wallet-v2-code.fc b/crypto/smartcont/highload-wallet-v2-code.fc index 7dd65f9e..b7626bbe 100644 --- a/crypto/smartcont/highload-wallet-v2-code.fc +++ b/crypto/smartcont/highload-wallet-v2-code.fc @@ -3,6 +3,22 @@ ;; this version does not use seqno for replay protection; instead, it remembers all recent query_ids ;; in this way several external messages with different query_id can be sent in parallel + +;; Note, when dealing with highload-wallet the following limits need to be checked and taken into account: +;; 1) Storage size limit. Currently, size of contract storage should be less than 65535 cells. If size of +;; old_queries will grow above this limit, exception in ActionPhase will be thrown and transaction will fail. +;; Failed transaction may be replayed. +;; 2) Gas limit. Currently, gas limit is 1'000'000 gas units, that means that there is a limit of how much +;; old queries may be cleaned in one tx. If number of expired queries will be higher, contract will stuck. + +;; That means that it is not recommended to set too high expiration date: +;; number of queries during expiration timespan should not exceed 1000. +;; Also, number of expired queries cleaned in one transaction should be below 100. + +;; Such precautions are not easy to follow, so it is recommended to use highload contract +;; only when strictly necessary and the developer understands the above details. + + () recv_internal(slice in_msg) impure { ;; do nothing for internal messages } diff --git a/crypto/smartcont/mathlib.fc b/crypto/smartcont/mathlib.fc new file mode 100644 index 00000000..f2dfd73f --- /dev/null +++ b/crypto/smartcont/mathlib.fc @@ -0,0 +1,939 @@ +{- + - + - FunC fixed-point mathematical library + - + -} + +{- + This file is part of TON FunC Standard Library. + + FunC Standard Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + FunC Standard Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + +-} + +#include "stdlib.fc"; +#pragma version >=0.4.2; + +{---------------- HIGH-LEVEL FUNCTION DECLARATIONS -----------------} +{- + Most functions declared here work either with integers or with fixed-point numbers of type `fixed248`. + `fixedNNN` informally denotes an alias for type `int` used to represent fixed-point numbers with scale 2^NNN. + Prefix `fixedNNN::` is prepended to the names of high-level functions that accept arguments and return values of type `fixedNNN`. +-} + +{- function declarations have been commented out, otherwise they are not inlined by the current FunC compiler + +;; nearest integer to sqrt(a*b) for non-negative integers or fixed-point numbers a and b +int geom_mean(int a, int b) inline_ref; +;; integer square root +int sqrt(int a) inline; +;; fixed-point square root +;; fixed248 sqrt(fixed248 x) +int fixed248::sqrt(int x) inline; + +int fixed248::sqr(int x) inline; +const int fixed248::One; + +;; log(2) as fixed248 +int fixed248::log2_const() inline; +;; Pi as fixed248 +int fixed248::Pi_const() inline; + +;; fixed248 exp(fixed248 x) +int fixed248::exp(int x) inline_ref; +;; fixed248 exp2(fixed248 x) +int fixed248::exp2(int x) inline_ref; + +;; fixed248 log(fixed248 x) +int fixed248::log(int x) inline_ref; +;; fixed248 log2(fixed248 x) +int fixed248::log2(int x) inline; + +;; fixed248 pow(fixed248 x, fixed248 y) +int fixed248::pow(int x, int y) inline_ref; + +;; (fixed248, fixed248) sincos(fixed248 x); +(int, int) fixed248::sincos(int x) inline_ref; +;; fixed248 sin(fixed248 x); +int fixed248::sin(int x) inline; +;; fixed248 cos(fixed248 x); +int fixed248::cos(int x) inline; +;; fixed248 tan(fixed248 x); +int fixed248::tan(int x) inline_ref; +;; fixed248 cot(fixed248 x); +int fixed248::cot(int x) inline_ref; + + +;; fixed248 asin(fixed248 x); +int fixed248::asin(int x) inline; +;; fixed248 acos(fixed248 x); +int fixed248::acos(int x) inline; +;; fixed248 atan(fixed248 x); +int fixed248::atan(int x) inline_ref; +;; fixed248 acot(fixed248 x); +int fixed248::acot(int x) inline_ref; + +;; random number uniformly distributed in [0..1) +;; fixed248 random(); +int fixed248::random() impure inline; +;; random number with standard normal distribution (2100 gas on average) +;; fixed248 nrand(); +int fixed248::nrand() impure inline; +;; generates a random number approximately distributed according to the standard normal distribution (1200 gas) +;; (fails chi-squared test, but it is shorter and faster than fixed248::nrand()) +;; fixed248 nrand_fast(); +int fixed248::nrand_fast() impure inline; + +-} ;; end (declarations) + +{-------------------- INTERMEDIATE FUNCTIONS -----------------------} + +{- + Intermediate functions are used in the implementations of high-level `fixedNNN::...` functions + if necessary, they can be used to define additional high-level functions for other fixed-point types, such as fixed128, outside this library. They can be also used in a hypothetical floating-point FunC library. + For these reasons, the declarations of these functions are collected here. +-} + +{- function declarations have been commented out, otherwise they are not inlined by the current FunC compiler + +;; fixed258 tanh(fixed258 x, int steps); +int tanh_f258(int x, int n); + +;; computes exp(x)-1 for |x| <= log(2)/2. +;; fixed257 expm1(fixed257 x); +int expm1_f257(int x); + +;; computes (sin(x+xe),-cos(x+xe)) for |x| <= Pi/4, xe very small +;; this function is very accurate, error less than 0.7 ulp (consumes ~ 5500 gas) +;; (fixed256, fixed256) sincosn(fixed256 x, fixed259 xe) +(int, int) sincosn_f256(int x, int xe); + +;; compute (sin(x),1-cos(x)) in fixed256 for |x| < 16*atan(1/16) = 0.9987 +;; (fixed256, fixed257) sincosm1_f256(fixed256 x); +;; slightly less accurate than sincosn_f256() (error up to 3/2^256), but faster (~ 4k gas) and shorter +(int, int) sincosm1_f256(int x); + +;; compute (p, q) such that p/q = tan(x) for |x|<2*atan(1/2)=1899/2048=0.927 +;; (int, int) tan_aux(fixed256 x); +(int, int) tan_aux_f256(int x); + +;; returns (y, s) such that log(x) = y/2^256 + s*log(2) for positive integer x +;; this function is very precise (error less than 0.6 ulp) and consumes < 7k gas +;; (fixed256, int) log_aux_f256(int x); +(int, int) log_aux_f256(int x); + +;; returns (y, s) such that log2(x) = y/2^256 + s for positive integer x +;; this function is very precise (error less than 0.6 ulp) and consumes < 7k gas +;; (fixed256, int) log2_aux_f256(int x); +(int, int) log2_aux_f256(int x); + +;; compute (q, z) such that atan(x)=q*atan(1/32)+z for -1 <= x < 1 +;; this function is reasonably accurate (error < 7 ulp with ulp = 2^-261), but it consumes >7k gas +;; this is sufficient for most purposes +;; (int, fixed261) atan_aux(fixed256 x) +(int, int) atan_aux_f256(int x); + +;; fixed255 atan(fixed255 x); +int atan_f255(int x); + +;; for -1 <= x < 1 only +;; fixed256 atan_small(fixed256 x); +int atan_f256_small(int x); + +;; fixed255 asin(fixed255 x); +int asin_f255(int x); + +;; fixed254 acos(fixed255 x); +int acos_f255(int x); + +;; generates normally distributed pseudo-random number +;; fixed252 nrand(); +int nrand_f252(int x); + +;; a faster and shorter variant of nrand_f252() that fails chi-squared test +;; (should suffice for most purposes) +;; fixed252 nrand_fast(); +int nrand_fast_f252(int x); + +-} ;; end (declarations) + +{---------------- MISSING OPERATIONS AND BUILT-INS -----------------} + +int sgn(int x) asm "SGN"; + +;; compute floor(log2(x))+1 +int log2_floor_p1(int x) asm "UBITSIZE"; + +int mulrshiftr(int x, int y, int s) asm "MULRSHIFTR"; +int mulrshiftr256(int x, int y) asm "256 MULRSHIFTR#"; +(int, int) mulrshift256mod(int x, int y) asm "256 MULRSHIFT#MOD"; +(int, int) mulrshiftr256mod(int x, int y) asm "256 MULRSHIFTR#MOD"; +(int, int) mulrshiftr255mod(int x, int y) asm "255 MULRSHIFTR#MOD"; +(int, int) mulrshiftr248mod(int x, int y) asm "248 MULRSHIFTR#MOD"; +(int, int) mulrshiftr5mod(int x, int y) asm "5 MULRSHIFTR#MOD"; +(int, int) mulrshiftr6mod(int x, int y) asm "6 MULRSHIFTR#MOD"; +(int, int) mulrshiftr7mod(int x, int y) asm "7 MULRSHIFTR#MOD"; + +int lshift256divr(int x, int y) asm "256 LSHIFT#DIVR"; +(int, int) lshift256divmodr(int x, int y) asm "256 LSHIFT#DIVMODR"; +(int, int) lshift255divmodr(int x, int y) asm "255 LSHIFT#DIVMODR"; +(int, int) lshift2divmodr(int x, int y) asm "2 LSHIFT#DIVMODR"; +(int, int) lshift7divmodr(int x, int y) asm "7 LSHIFT#DIVMODR"; +(int, int) lshiftdivmodr(int x, int y, int s) asm "LSHIFTDIVMODR"; + +(int, int) rshiftr256mod(int x) asm "256 RSHIFTR#MOD"; +(int, int) rshiftr248mod(int x) asm "248 RSHIFTR#MOD"; +(int, int) rshiftr4mod(int x) asm "4 RSHIFTR#MOD"; +(int, int) rshift3mod(int x) asm "3 RSHIFT#MOD"; + +;; computes y - x (FunC compiler does not try to use this by itself) +int sub_rev(int x, int y) asm "SUBR"; + +int nan() asm "PUSHNAN"; +int is_nan(int x) asm "ISNAN"; + +{------------------------ SQUARE ROOTS ----------------------------} + +;; computes sqrt(a*b) exactly rounded to the nearest integer +;; for all 0 <= a, b <= 2^256-1 +;; may be used with b=1 or b=scale of fixed-point numbers +int geom_mean(int a, int b) inline_ref { + ifnot (min(a, b)) { + return 0; + } + int s = log2_floor_p1(a); ;; throws out of range error if a < 0 or b < 0 + int t = log2_floor_p1(b); + ;; NB: (a-b)/2+b == (a+b)/2, but without overflow for large a and b + int x = (s == t ? (a - b) / 2 + b : 1 << ((s + t) / 2)); + do { + ;; if always used with b=2^const, may be optimized to "const LSHIFTDIVC#" + ;; it is important to use `muldivc` here, not `muldiv` or `muldivr` + int q = (muldivc(a, b, x) - x) / 2; + x += q; + } until (q == 0); + return x; +} + +;; integer square root, computes round(sqrt(a)) for all a>=0. +;; note: `inline` is better than `inline_ref` for such simple functions +int sqrt(int a) inline { + return geom_mean(a, 1); +} + +;; version for fixed248 = fixed-point numbers with scale 2^248 +;; fixed248 sqrt(fixed248 x) +int fixed248::sqrt(int x) inline { + return geom_mean(x, 1 << 248); +} + +;; fixed255 sqrt(fixed255 x) +int fixed255::sqrt(int x) inline { + return geom_mean(x, 1 << 255); +} + +;; fixed248 sqr(fixed248 x); +int fixed248::sqr(int x) inline { + return muldivr(x, x, 1 << 248); +} + +;; fixed255 sqr(fixed255 x); +int fixed255::sqr(int x) inline { + return muldivr(x, x, 1 << 255); +} + +const int fixed248::One = (1 << 248); +const int fixed255::One = (1 << 255); + +{-------------------- USEFUL CONSTANTS --------------------} + +;; store huge constants in inline_ref functions for reuse +;; (y,z) where y=round(log(2)*2^256), z=round((log(2)*2^256-y)*2^128) +;; then log(2) = y/2^256 + z/2^384 +(int, int) log2_xconst_f256() inline_ref { + return (80260960185991308862233904206310070533990667611589946606122867505419956976172, -32272921378999278490133606779486332143); +} + +;; (y,z) where Pi = y/2^254 + z/2^382 +(int, int) Pi_xconst_f254() inline_ref { + return (90942894222941581070058735694432465663348344332098107489693037779484723616546, 108051869516004014909778934258921521947); +} + +;; atan(1/16) as fixed260 +int Atan1_16_f260() inline_ref { + return 115641670674223639132965820642403718536242645001775371762318060545014644837101; ;; true value is ...101.0089... +} + +;; atan(1/8) as fixed259 +int Atan1_8_f259() inline_ref { + return 115194597005316551477397594802136977648153890007566736408151129975021336532841; ;; correction -0.1687... +} + +;; atan(1/32) as fixed261 +int Atan1_32_f261() inline_ref { + return 115754418570128574501879331591757054405465733718902755858991306434399246026247; ;; correction 0.395... +} + +;; inline is better than inline_ref for such very small functions +int log2_const_f256() inline { + (int c, _) = log2_xconst_f256(); + return c; +} + +int fixed248::log2_const() inline { + return log2_const_f256() ~>> 8; +} + +int Pi_const_f254() inline { + (int c, _) = Pi_xconst_f254(); + return c; +} + +int fixed248::Pi_const() inline { + return Pi_const_f254() ~>> 6; +} + +{--------------- HYPERBOLIC TANGENT AND EXPONENT -------------------} + +;; hyperbolic tangent of small x via n+2 terms of Lambert's continued fraction +;; n=17: good for |x| < log(2)/4 = 0.173 +;; fixed258 tanh_f258(fixed258 x, int n) +int tanh_f258(int x, int n) inline_ref { + int x2 = muldivr(x, x, 1 << 255); ;; x^2 as fixed261 + int c = int a = (2 * n + 5) << 250; ;; a=2n+5 as fixed250 + int Two = (1 << 251); ;; 2. as fixed250 + repeat (n) { + a = (c -= Two) + muldivr(x2, 1 << 239, a); ;; a := 2k+1+x^2/a as fixed250, k=n+1,n,...,2 + } + a = (touch(3) << 254) + muldivr(x2, 1 << 243, a); ;; a := 3+x^2/a as fixed254 + ;; y = x/(1+a') = x - x*a'/(1+a') = x - x*x^2/(a+x^2) where a' = x^2/a + return x - (muldivr(x, x2, a + (x2 ~>> 7)) ~>> 7); +} + +;; fixed257 expm1_f257(fixed257 x) +;; computes exp(x)-1 for small x via 19 terms of Lambert's continued fraction for tanh(x/2) +;; good for |x| < log(2)/2 = 0.347 (n=17); consumes ~3500 gas +int expm1_f257(int x) inline_ref { + ;; (almost) compute tanh(x/2) first; x/2 as fixed258 = x as fixed257 + int x2 = muldivr(x, x, 1 << 255); ;; x^2 as fixed261 + int Two = (1 << 251); ;; 2. as fixed250 + int c = int a = touch(39) << 250; ;; a=2n+5 as fixed250 + repeat (17) { + a = (c -= Two) + muldivr(x2, 1 << 239, a); ;; a := 2k+1+x^2/a as fixed250, k=n+1,n,...,2 + } + a = (touch(3) << 254) + muldivr(x2, 1 << 243, a); ;; a := 3+x^2/a as fixed254 + ;; now tanh(x/2) = x/(1+a') where a'=x^2/a ; apply exp(x)-1=2*tanh(x/2)/(1-tanh(x/2)) + int t = (x ~>> 4) - a; ;; t:=x-a as fixed254 + return x - muldivr(x2, t / 2, a + mulrshiftr256(x, t) ~/ 4) ~/ 4; ;; x - x^2 * (x-a) / (a + x*(x-a)) +} + +;; expm1_f257() may be used to implement specific fixed-point exponentials +;; example: +;; fixed248 exp(fixed248 x) +int fixed248::exp(int x) inline_ref { + var (l2c, l2d) = log2_xconst_f256(); + ;; divide x by log(2) and convert to fixed257 + ;; (int q, x) = muldivmodr(x, 256, l2c); ;; unfortunately, no such built-in + (int q, x) = lshiftdivmodr(x, l2c, 8); + x = 2 * x - muldivr(q, l2d, 1 << 127); + int y = expm1_f257(x); + ;; result is (1 + y) * (2^q) --> ((1 << 257) + y) >> (9 - q) + return (y ~>> (9 - q)) - (-1 << (248 + q)); + ;; note that (y ~>> (9 - q)) + (1 << (248 + q)) leads to overflow when q=8 +} + +;; compute 2^x in fixed248 +;; fixed248 exp2(fixed248 x) +int fixed248::exp2(int x) inline_ref { + ;; (int q, x) = divmodr(x, 1 << 248); ;; no such built-in + (int q, x) = rshiftr248mod(x); + x = muldivr(x, log2_const_f256(), 1 << 247); + int y = expm1_f257(x); + return (y ~>> (9 - q)) - (-1 << (248 + q)); +} + +{--------------------- TRIGONOMETRIC FUNCTIONS -----------------------} + +;; fixed260 tan(fixed260 x); +;; computes tan(x) for small |x|> 10)) ~>> 9); +} + +;; fixed260 tan(fixed260 x); +int tan_f260(int x) inline_ref { + return tan_f260_inlined(x); +} + +;; fixed258 tan(fixed258 x); +;; computes tan(x) for small |x|> 6)) ~>> 5); +} + +;; fixed258 tan(fixed258 x); +int tan_f258(int x) inline_ref { + return tan_f258_inlined(x); +} + +;; (fixed259, fixed263) sincosm1(fixed259 x) +;; computes (sin(x), 1-cos(x)) for small |x|<2*atan(1/16) +(int, int) sincosm1_f259_inlined(int x) inline { + int t = tan_f260_inlined(x); ;; t=tan(x/2) as fixed260 + int tt = mulrshiftr256(t, t); ;; t^2 as fixed264 + int y = tt ~/ 512 + (1 << 255); ;; 1+t^2 as fixed255 + ;; 2*t/(1+t^2) as fixed259 and 2*t^2/(1+t^2) as fixed263 + ;; return (muldivr(t, 1 << 255, y), muldivr(tt, 1 << 255, y)); + return (t - muldivr(t / 2, tt, y) ~/ 256, tt - muldivr(tt / 2, tt, y) ~/ 256); +} + +(int, int) sincosm1_f259(int x) inline_ref { + return sincosm1_f259_inlined(x); +} + +;; computes (sin(x+xe),-cos(x+xe)) for |x| <= Pi/4, xe very small +;; this function is very accurate, error less than 0.7 ulp (consumes ~ 5500 gas) +;; (fixed256, fixed256) sincosn(fixed256 x, fixed259 xe) +(int, int) sincosn_f256(int x, int xe) inline_ref { + ;; var (q, x1) = muldivmodr(x, 8, Atan1_8_f259()); ;; no muldivmodr() builtin + var (q, x1) = lshift2divmodr(abs(x), Atan1_8_f259()); ;; reduce mod theta where theta=2*atan(1/8) + var (si, co) = sincosm1_f259(x1 * 2 + xe); + var (a, b, c) = (-1, 0, 1); + repeat (q) { ;; (a+b*I) *= (8+I)^2 = 63+16*I + (a, b, c) = (63 * a - 16 * b, 16 * a + 63 * b, 65 * c); + } + ;; now a/c = cos(q*theta), b/c = sin(q*theta) exactly(!) + ;; compute (a+b*I)*(1-co+si*I)/c + ;; (b, a) = (lshift256divr(b, c), lshift256divr(a, c)); + (b, int br) = lshift256divmodr(b, c); br = muldivr(br, 128, c); + (a, int ar) = lshift256divmodr(a, c); ar = muldivr(ar, 128, c); + return (sgn(x) * (((mulrshiftr256(b, co) - br) ~/ 16 - mulrshiftr256(a, si)) ~/ 8 - b), + a - ((mulrshiftr256(a, co) - ar) ~/ 16 + mulrshiftr256(b, si)) ~/ 8); +} + +;; compute (sin(x),1-cos(x)) in fixed256 for |x| < 16*atan(1/16) = 0.9987 +;; (fixed256, fixed257) sincosm1_f256(fixed256 x); +;; slightly less accurate than sincosn_f256() (error up to 3/2^256), but faster (~ 4k gas) and shorter +(int, int) sincosm1_f256(int x) inline_ref { + var (si, co) = sincosm1_f259_inlined(x); ;; compute (sin,1-cos)(x/8) in (fixed259,fixed263) + int r = 7; + repeat (r / 2) { + ;; 1-cos(2*x) = 2*sin(x)^2, sin(2*x) = 2*sin(x)*cos(x) + (co, si) = (mulrshiftr256(si, si), si - (mulrshiftr256(si, co) ~>> r)); + r -= 2; + } + return (si, co); +} + +;; compute (p, q) such that p/q = tan(x) for |x|<2*atan(1/2)=1899/2048=0.927 +;; (int, int) tan_aux(fixed256 x); +(int, int) tan_aux_f256(int x) inline_ref { + int t = tan_f258_inlined(x); ;; t=tan(x/4) as fixed258 + ;; t:=2*t/(1-t^2)=2*(t-t^3/(t^2-1)) + int tt = mulrshiftr256(t, t); ;; t^2 as fixed260 + t = muldivr(t, tt, tt ~/ 16 + (-1 << 256)) ~/ 16 - t; ;; now t=-tan(x/2) as fixed259 + return (t, mulrshiftr256(t, t) ~/ 4 + (-1 << 256)); ;; return (2*t, t^2-1) as fixed256 +} + +;; sincosm1_f256() and sincosn_f256() may be used to implement trigonometric functions for different fixed-point types +;; example: +;; (fixed248, fixed248) sincos(fixed248 x); +(int, int) fixed248::sincos(int x) inline_ref { + var (Pic, Pid) = Pi_xconst_f254(); + ;; (int q, x) = muldivmodr(x, 128, Pic); ;; no muldivmodr() builtin + (int q, x) = lshift7divmodr(x, Pic); ;; reduce mod Pi/2 + x = 2 * x - muldivr(q, Pid, 1 << 127); + (int si, int co) = sincosm1_f256(x); ;; doesn't make sense to use more accurate sincosn_f256() + co = (1 << 248) - (co ~>> 9); + si ~>>= 8; + repeat (q & 3) { + (si, co) = (co, - si); + } + return (si, co); +} + +;; fixed248 sin(fixed248 x); +;; inline is better than inline_ref for such simple functions +int fixed248::sin(int x) inline { + (int si, _) = fixed248::sincos(x); + return si; +} + +;; fixed248 cos(fixed248 x); +int fixed248::cos(int x) inline { + (_, int co) = fixed248::sincos(x); + return co; +} + +;; similarly, tan_aux_f256() may be used to implement tan() and cot() for specific fixed-point formats +;; fixed248 tan(fixed248 x); +;; not very accurate when |tan(x)| is very large (difficult to do better without floating-point numbers) +;; however, the relative accuracy is approximately 2^-247 in all cases, which is good enough for arguments given up to 2^-249 +int fixed248::tan(int x) inline_ref { + var (Pic, Pid) = Pi_xconst_f254(); + ;; (int q, x) = muldivmodr(x, 128, Pic); ;; no muldivmodr() builtin + (int q, x) = lshift7divmodr(x, Pic); ;; reduce mod Pi/2 + x = 2 * x - muldivr(q, Pid, 1 << 127); + var (a, b) = tan_aux_f256(x); ;; now a/b = tan(x') + if (q & 1) { + (a, b) = (b, - a); + } + return muldivr(a, 1 << 248, b); ;; either -b/a or a/b as fixed248 +} + +;; fixed248 cot(fixed248 x); +int fixed248::cot(int x) inline_ref { + var (Pic, Pid) = Pi_xconst_f254(); + (int q, x) = lshift7divmodr(x, Pic); ;; reduce mod Pi/2 + x = 2 * x - muldivr(q, Pid, 1 << 127); + var (b, a) = tan_aux_f256(x); ;; now b/a = tan(x') + if (q & 1) { + (a, b) = (b, - a); + } + return muldivr(a, 1 << 248, b); ;; either -b/a or a/b as fixed248 +} + +{----------------- INVERSE HYPERBOLIC TANGENT AND LOGARITHMS -----------------} + +;; inverse hyperbolic tangent of small x, evaluated by means of n terms of the continued fraction +;; valid for |x| < 2^-2.5 ~ 0.18 if n=37 (slightly less accurate with n=36) +;; |x| < 1/8 if n=32; |x| < 2^-3.5 if n=28; |x| < 1/16 if n=25 +;; |x| < 2^-4.5 if n=23; |x| < 1/32 if n=21; |x| < 1/64 if n=18 +;; fixed258 atanh(fixed258 x); +int atanh_f258(int x, int n) inline_ref { + int x2 = mulrshiftr256(x, x); ;; x^2 as fixed260 + int One = (1 << 254); + int a = One ~/ n + (1 << 255); ;; a := 2 + 1/n as fixed254 + repeat (n - 1) { + ;; a := 1 + (1 - x^2 / a)(1 + 1/n) as fixed254 + int t = One - muldivr(x2, 1 << 248, a); ;; t := 1 - x^2 / a + a = muldivr(t, n, (int n1 = n - 1)) + One; + n = n1; + } + ;; x / (1 - x^2 / a) = x / (1 - d) = x + x * d / (1 - d) for d = x^2 / a + ;; int d = muldivr(x2, 1 << 255, a - (x2 ~>> 6)); ;; d/(1-d) = x^2/(a-x^2) as fixed261 + ;; return x + (mulrshiftr256(x, d) ~>> 5); + return x + muldivr(x, x2 / 2, a - x2 ~/ 64) ~/ 32; +} + +;; number of terms n should be chosen as for atanh_f258() +;; fixed261 atanh(fixed261 x); +int atanh_f261_inlined(int x, int n) inline { + int x2 = mulrshiftr256(x, x); ;; x^2 as fixed266 + int One = (1 << 254); + int a = One ~/ n + (1 << 255); ;; a := 2 + 1/n as fixed254 + repeat (n - 1) { + ;; a := 1 + (1 - x^2 / a)(1 + 1/n) as fixed254 + int t = One - muldivr(x2, 1 << 242, a); ;; t := 1 - x^2 / a + a = muldivr(t, n, (int n1 = n - 1)) + One; + n = n1; + } + ;; x / (1 - x^2 / a) = x / (1 - d) = x + x * d / (1 - d) for d = x^2 / a + ;; int d = muldivr(x2, 1 << 255, a - (x2 ~>> 12)); ;; d/(1-d) = x^2/(a-x^2) as fixed267 + ;; return x + (mulrshiftr256(x, d) ~>> 11); + return x + muldivr(x, x2, a - x2 ~/ 4096) ~/ 4096; +} + +;; fixed261 atanh(fixed261 x); +int atanh_f261(int x, int n) inline_ref { + return atanh_f261_inlined(x, n); +} + +;; returns (y, s) such that log(x) = y/2^257 + s*log(2) for positive integer x +;; (fixed257, int) log_aux(int x) +(int, int) log_aux_f257(int x) inline_ref { + int s = log2_floor_p1(x); + x <<= 256 - s; + int t = touch(-1 << 256); + if ((x >> 249) <= 90) { + ;; t~touch(); + t >>= 1; + s -= 1; + } + x += t; + int 2x = 2 * x; + int y = lshift256divr(2x, (x >> 1) - t); + ;; y = 2x - (mulrshiftr256(2x, y) ~>> 2); ;; this line could improve precision on very rare occasions + return (atanh_f258(y, 36), s); +} + +;; computes 33^m for small m +int pow33(int m) inline { + int t = 1; + repeat (m) { t *= 33; } + return t; +} + +;; computes 33^m for small 0<=m<=22 +;; slightly faster than pow33() +int pow33b(int m) inline { + (int mh, int ml) = m /% 5; + int t = 1; + repeat (ml) { t *= 33; } + repeat (mh) { t *= 33 * 33 * 33 * 33 * 33; } + return t; +} + +;; returns (s, q, y) such that log(x) = s*log(2) + q*log(33/32) + y/2^260 for positive integer x +;; (int, int, fixed260) log_auxx_f260(int x); +(int, int, int) log_auxx_f260(int x) inline_ref { + int s = log2_floor_p1(x) - 1; + x <<= 255 - s; ;; rescale to 1 <= x < 2 as fixed255 + int t = touch(2873) << 244; ;; ~ (33/32)^11 ~ sqrt(2) as fixed255 + int x1 = (x - t) >> 1; + int q = muldivr(x1, 65, x1 + t) + 11; ;; crude approximation to round(log(x)/log(33/32)) + ;; t = 1; repeat (q) { t *= 33; } ;; t:=33^q, 0<=q<=22 + t = pow33b(q); + t <<= (51 - q) * 5; ;; t:=(33/32)^q as fixed255, nearest power of 33/32 to x + x -= t; + int y = lshift256divr(x << 4, (x >> 1) + t); ;; y = (x-t)/(x+t) as fixed261 + y = atanh_f261(y, 18); ;; atanh((x-t)/(x+t)) as fixed261, or log(x/t) as fixed260 + return (s, q, y); +} + +;; returns (y, s) such that log(x) = y/2^256 + s*log(2) for positive integer x +;; this function is very precise (error less than 0.6 ulp) and consumes < 7k gas +;; (fixed256, int) log_aux_f256(int x); +(int, int) log_aux_f256(int x) inline_ref { + var (s, q, y) = log_auxx_f260(x); + var (yh, yl) = rshiftr4mod(y); ;; y ~/% 16 , but FunC does not optimize this to RSHIFTR#MOD + ;; int Log33_32 = 3563114646320977386603103333812068872452913448227778071188132859183498739150; ;; log(33/32) as fixed256 + ;; int Log33_32_l = -3769; ;; log(33/32) = Log33_32 / 2^256 + Log33_32_l / 2^269 + yh += (yl * 512 + q * -3769) ~>> 13; ;; compensation, may be removed if slightly worse accuracy is acceptable + int Log33_32 = 3563114646320977386603103333812068872452913448227778071188132859183498739150; ;; log(33/32) as fixed256 + return (yh + q * Log33_32, s); +} + +;; returns (y, s) such that log2(x) = y/2^256 + s for positive integer x +;; this function is very precise (error less than 0.6 ulp) and consumes < 7k gas +;; (fixed256, int) log2_aux_f256(int x); +(int, int) log2_aux_f256(int x) inline_ref { + var (s, q, y) = log_auxx_f260(x); + y = lshift256divr(y, log2_const_f256()) ~>> 4; ;; y/log(2) as fixed256 + int Log33_32 = 5140487830366106860412008603913034462883915832139695448455767612111363481357; ;; log_2(33/32) as fixed256 + ;; Log33_32/2^256 happens to be a very precise approximation to log_2(33/32), no compensation required + return (y + q * Log33_32, s); +} + +;; functions log_aux_f256() and log2_aux_f256() may be used to implement specific fixed-point instances of log() and log2() + +;; fixed248 log(fixed248 x) +int fixed248::log(int x) inline_ref { + var (y, s) = log_aux_f256(x); + return muldivr(s - 248, log2_const_f256(), 1 << 8) + (y ~>> 8); + ;; return muldivr(s - 248, 80260960185991308862233904206310070533990667611589946606122867505419956976172, 1 << 8) + (y ~>> 8); +} + +;; fixed248 log2(fixed248 x) +int fixed248::log2(int x) inline { + var (y, s) = log2_aux_f256(x); + return ((s - 248) << 248) + (y ~>> 8); +} + +;; computes x^y as exp(y*log(x)), x >= 0 +;; fixed248 pow(fixed248 x, fixed248 y); +int fixed248::pow(int x, int y) inline_ref { + ifnot (y) { + return 1 << 248; ;; x^0 = 1 + } + if (x <= 0) { + int bad = (x | y) < 0; + return 0 >> bad; ;; 0^y = 0 if x=0 and y>=0; "out of range" exception otherwise + } + var (l, s) = log2_aux_f256(x); + s -= 248; ;; log_2(x) = s+l, l is fixed256, 0<=l<1 + ;; compute (s+l)*y = q+ll + var (q1, r1) = mulrshiftr248mod(s, y); ;; muldivmodr(s, y, 1 << 248) + var (q2, r2) = mulrshift256mod(l, y); + r2 >>= 247; + var (q3, r3) = rshiftr248mod(q2); ;; divmodr(q2, 1 << 248); + var (q, ll) = rshiftr248mod(r1 + r3); + ll = 512 * ll + r2; + q += q1 + q3; + ;; now log_2(x^y) = y*log_2(x) = q + ll, ss integer, ll fixed257, -1/2<=ll<1/2 + int sq = q + 248; + if (sq <= 0) { + return - (sq == 0); ;; underflow + } + int y = expm1_f257(mulrshiftr256(ll, log2_const_f256())); + return (y ~>> (9 - q)) - (-1 << sq); +} + +{--------------------- INVERSE TRIGONOMETRIC FUNCTIONS -------------------} + +;; number of terms n should be chosen as for atanh_f258() +;; fixed259 atan(fixed259 x); +int atan_f259(int x, int n) inline_ref { + int x2 = mulrshiftr256(x, x); ;; x^2 as fixed262 + int One = (1 << 254); + int a = One ~/ n + (1 << 255); ;; a := 2 + 1/n as fixed254 + repeat (n - 1) { + ;; a := 1 + (1 + x^2 / a)(1 + 1/n) as fixed254 + int t = One + muldivr(x2, 1 << 246, a); ;; t := 1 + x^2 / a + a = muldivr(t, n, (int n1 = n - 1)) + One; + n = n1; + } + ;; x / (1 + x^2 / a) = x / (1 + d) = x - x * d / (1 + d) = x - x * x^2/(a+x^2) for d = x^2 / a + return x - muldivr(x, x2, a + x2 ~/ 256) ~/ 256; +} + +;; number of terms n should be chosen as for atanh_f261() +;; fixed261 atan(fixed261 x); +int atan_f261_inlined(int x, int n) inline { + int x2 = mulrshiftr256(x, x); ;; x^2 as fixed266 + int One = (1 << 254); + int a = One ~/ n + (1 << 255); ;; a := 2 + 1/n as fixed254 + repeat (n - 1) { + ;; a := 1 + (1 + x^2 / a)(1 + 1/n) as fixed254 + int t = One + muldivr(x2, 1 << 242, a); ;; t := 1 + x^2 / a + a = muldivr(t, n, (int n1 = n - 1)) + One; + n = n1; + } + ;; x / (1 + x^2 / a) = x / (1 + d) = x - x * d / (1 + d) = x - x * x^2/(a+x^2) for d = x^2 / a + return x - muldivr(x, x2, a + x2 ~/ 4096) ~/ 4096; +} + +;; fixed261 atan(fixed261 x); +int atan_f261(int x, int n) inline_ref { + return atan_f261_inlined(x, n); +} + +;; computes (q,a,b) such that q is approximately atan(x)/atan(1/32) and a+b*I=(1+I/32)^q as fixed255 +;; then b/a=atan(q*atan(1/32)) exactly, and (a,b) is almost a unit vector pointing in the direction of (1,x) +;; must have |x|<1.1, x is fixed24 +;; (int, fixed255, fixed255) atan_aux_prereduce(fixed24 x); +(int, int, int) atan_aux_prereduce(int x) inline_ref { + int xu = abs(x); + int tc = 7214596; ;; tan(13*theta) as fixed24 where theta=atan(1/32) + int t1 = muldivr(xu - tc, 1 << 88, xu * tc + (1 << 48)); ;; tan(x') as fixed64 where x'=atan(x)-13*theta + ;; t1/(3+t1^2) * 3073/32 = x'/3 * 3072/32 = x' / (96/3072) = x' / theta + int q = muldivr(t1 * 3073, 1 << 59, t1 * t1 + (touch(3) << 128)) + 13; ;; approximately round(atan(x)/theta), 0<=q<=25 + var (pa, pb) = (33226912, 5232641); ;; (32+I)^5 + var (qh, ql) = q /% 5; + var (a, b) = (1 << (5 * (51 - q)), 0); ;; (1/32^q, 0) as fixed255 + repeat (ql) { ;; a+b*I *= 32+I + (a, b) = (sub_rev(touch(b), 32 * a), a + 32 * b); ;; same as (32 * a - b, 32 * b + a), but more efficient + } + repeat (qh) { ;; a+b*I *= (32+I)^5 = pa + pb*I + (a, b) = (a * pa - b * pb, a * pb + b * pa); + } + int xs = sgn(x); + return (xs * q, a, xs * b); +} + +;; compute (q, z) such that atan(x)=q*atan(1/32)+z for -1 <= x < 1 +;; this function is reasonably accurate (error < 7 ulp with ulp = 2^-261), but it consumes >7k gas +;; this is sufficient for most purposes +;; (int, fixed261) atan_aux(fixed256 x) +(int, int) atan_aux_f256(int x) inline_ref { + var (q, a, b) = atan_aux_prereduce(x ~>> 232); ;; convert x to fixed24 + ;; now b/a = tan(q*atan(1/32)) exactly, where q is near atan(x)/atan(1/32); so b/a is near x + ;; compute y = u/v = (a*x-b)/(a+b*x) as fixed261 ; then |y|<0.0167 = 1.07/64 and atan(x)=atan(y)+q*atan(1/32) + var (u, ul) = mulrshiftr256mod(a, x); + u = (ul ~>> 250) + ((u - b) << 6); ;; |u| < 1/32, convert fixed255 -> fixed261 + int v = a + mulrshiftr256(b, x); ;; v is scalar product of (a,b) and (1,x), it is approximately in [1..sqrt(2)] as fixed255 + int y = muldivr(u, 1 << 255, v); ;; y = u/v as fixed261 + int z = atan_f261_inlined(y, 18); ;; z = atan(x)-q*atan(1/32) + return (q, z); +} + +;; compute (q, z) such that atan(x)=q*atan(1/32)+z for -1 <= x < 1 +;; this function is very accurate (error < 2 ulp), but it consumes >7k gas +;; in most cases, faster function atan_aux_f256() should be used +;; (int, fixed261) atan_auxx(fixed256 x) +(int, int) atan_auxx_f256(int x) inline_ref { + var (q, a, b) = atan_aux_prereduce(x ~>> 232); ;; convert x to fixed24 + ;; now b/a = tan(q*atan(1/32)) exactly, where q is near atan(x)/atan(1/32); so b/a is near x + ;; compute y = (a*x-b)/(a+b*x) as fixed261 ; then |y|<0.0167 = 1.07/64 and atan(x)=atan(y)+q*atan(1/32) + ;; use sort of double precision arithmetic for this + var (u, ul) = mulrshiftr256mod(a, x); + ul /= 2; + u -= b; ;; |u| < 1/32 as fixed255 + var (v, vl) = mulrshiftr256mod(b, x); + vl /= 2; + v += a; ;; v is scalar product of (a,b) and (1,x), it is approximately in [1..sqrt(2)] as fixed255 + ;; y = (u + ul*eps) / (v + vl*eps) = u/v + (ul - vl * u/v)/v * eps where eps=1/2^255 + var (y, r) = lshift255divmodr(u, v); ;; y = u/v as fixed255 + int yl = muldivr(ul + r, 1 << 255, v) - muldivr(vl, y, v); ;; y/2^255 + yl/2^510 represent u/v + y = (yl ~>> 249) + (y << 6); ;; convert y to fixed261 + int z = atan_f261_inlined(y, 18); ;; z = atan(x)-q*atan(1/32) + return (q, z); +} + +;; consumes ~ 8k gas +;; fixed255 atan(fixed255 x); +int atan_f255(int x) inline_ref { + int s = (x ~>> 256); + touch(x); + if (s) { + x = lshift256divr(-1 << 255, x); ;; x:=-1/x as fixed256 + } else { + x *= 2; ;; convert to fixed256 + } + var (q, z) = atan_aux_f256(x); + ;; now atan(x) = z + q*atan(1/32) + s*(Pi/2), z is fixed261 + var (Pi_h, Pi_l) = Pi_xconst_f254(); ;; Pi/2 as fixed255 + fixed383 + var (qh, ql) = mulrshiftr6mod (q, Atan1_32_f261()); + return qh + s * Pi_h + (z + ql + muldivr(s, Pi_l, 1 << 122)) ~/ 64; +} + +;; computes atan(x) for -1 <= x < 1 only +;; fixed256 atan_small(fixed256 x); +int atan_f256_small(int x) inline_ref { + var (q, z) = atan_aux_f256(x); + ;; now atan(x) = z + q*atan(1/32), z is fixed261 + var (qh, ql) = mulrshiftr5mod (q, Atan1_32_f261()); + return qh + (z + ql) ~/ 32; +} + +;; fixed255 asin(fixed255 x); +int asin_f255(int x) inline_ref { + int a = fixed255::One - fixed255::sqr(x); ;; a:=1-x^2 + ifnot (a) { + return sgn(x) * Pi_const_f254(); ;; Pi/2 or -Pi/2 + } + int y = fixed255::sqrt(a); ;; sqrt(1-x^2) + int t = - lshift256divr(x, (-1 << 255) - y); ;; t = x/(1+sqrt(1-x^2)) avoiding overflow + return atan_f256_small(t); ;; asin(x)=2*atan(t) +} + +;; fixed254 acos(fixed255 x); +int acos_f255(int x) inline_ref { + int Pi = Pi_const_f254(); + if (x == (-1 << 255)) { + return Pi; ;; acos(-1) = Pi + } + Pi /= 2; + int y = fixed255::sqrt(fixed255::One - fixed255::sqr(x)); ;; sqrt(1-x^2) + int t = lshift256divr(x, (-1 << 255) - y); ;; t = -x/(1+sqrt(1-x^2)) avoiding overflow + return Pi + atan_f256_small(t) ~/ 2; ;; acos(x)=Pi/2 + 2*atan(t) +} + +;; consumes ~ 10k gas +;; fixed248 asin(fixed248 x) +int fixed248::asin(int x) inline { + return asin_f255(x << 7) ~>> 7; +} + +;; consumes ~ 10k gas +;; fixed248 acos(fixed248 x) +int fixed248::acos(int x) inline { + return acos_f255(x << 7) ~>> 6; +} + +;; consumes ~ 7500 gas +;; fixed248 atan(fixed248 x); +int fixed248::atan(int x) inline_ref { + int s = (x ~>> 249); + touch(x); + if (s) { + s = sgn(s); + x = lshift256divr(-1 << 248, x); ;; x:=-1/x as fixed256 + } else { + x <<= 8; ;; convert to fixed256 + } + var (q, z) = atan_aux_f256(x); + ;; now atan(x) = z + q*atan(1/32) + s*(Pi/2), z is fixed261 + return (z ~/ 64 + s * Pi_const_f254() + muldivr(q, Atan1_32_f261(), 64)) ~/ 128; ;; compute in fixed255, then convert +} + +;; fixed248 acot(fixed248 x); +int fixed248::acot(int x) inline_ref { + int s = (x ~>> 249); + touch(x); + if (s) { + x = lshift256divr(-1 << 248, x); ;; x:=-1/x as fixed256 + s = 0; + } else { + x <<= 8; ;; convert to fixed256 + s = sgn(x); + } + var (q, z) = atan_aux_f256(x); + ;; now acot(x) = - z - q*atan(1/32) + s*(Pi/2), z is fixed261 + return (s * Pi_const_f254() - z ~/ 64 - muldivr(q, Atan1_32_f261(), 64)) ~/ 128; ;; compute in fixed255, then convert +} + +{--------------------- PSEUDO-RANDOM NUMBERS -------------------} + +;; random number with standard normal distribution N(0,1) +;; generated by Kinderman--Monahan ratio method modified by J.Leva +;; spends ~ 2k..3k gas on average +;; fixed252 nrand(); +int nrand_f252() impure inline_ref { + var (x, s, t, A, B, r0) = (nan(), touch(29483) << 236, touch(-3167) << 239, 12845, 16693, 9043); + ;; 4/sqrt(e*Pi) = 1.369 loop iterations on average + do { + var (u, v) = (random() / 16 + 1, muldivr(random() - (1 << 255), 7027, 1 << 16)); ;; fixed252; 7027=ceil(sqrt(8/e)*2^12) + int va = abs(v); + var (u1, v1) = (u - s, va - t); ;; (u - 29483/2^16, abs(v) + 3167/2^13) as fixed252 + ;; Q := u1^2 + v1 * (A*v1 - B*u1) as fixed252 where A=12845/2^16, B=16693/2^16 + int Q = muldivr(u1, u1, 1 << 252) + muldivr(v1, muldivr(v1, A, 1 << 16) - muldivr(u1, B, 1 << 16), 1 << 252); + ;; must have 9043 / 2^15 < Q < 9125 / 2^15, otherwise accept if smaller, reject if larger + int Qd = (Q >> 237) - r0; + if ((Qd < 9125 - 9043) & (va / u < 16)) { + x = muldivr(v, 1 << 252, u); ;; x:=v/u as fixed252; reject immediately if |v/u| >= 16 + if (Qd >= 0) { ;; immediately accept if Qd < 0 + ;; rarely taken branch - 0.012 times per call on average + ;; check condition v^2 < -4*u^2*log(u), or equivalent condition u < exp(-x^2/4) for x=v/u + int xx = mulrshiftr256(x, x) ~/ 4; ;; x^2/4 as fixed248 + int ex = fixed248::exp(- xx) * 16; ;; exp(-x^2/4) as fixed252 + if (u > ex) { + x = nan(); ;; condition false, reject + } + } + } + } until (~ is_nan(x)); + return x; +} + +;; generates a random number approximately distributed according to the standard normal distribution +;; much faster than nrand_f252(), should be suitable for most purposes when only several random numbers are needed +;; fixed252 nrand_fast(); +int nrand_fast_f252() impure inline_ref { + int t = touch(-3) << 253; ;; -6. as fixed252 + repeat (12) { + t += random() / 16; ;; add together 12 uniformly random numbers + } + return t; +} + +;; random number uniformly distributed in [0..1) +;; fixed248 random(); +int fixed248::random() impure inline { + return random() >> 8; +} + +;; random number with standard normal distribution +;; fixed248 nrand(); +int fixed248::nrand() impure inline { + return nrand_f252() ~>> 4; +} + +;; generates a random number approximately distributed according to the standard normal distribution +;; fixed248 nrand_fast(); +int fixed248::nrand_fast() impure inline { + return nrand_fast_f252() ~>> 4; +} diff --git a/crypto/smartcont/stdlib.fc b/crypto/smartcont/stdlib.fc index 3531608a..8fb27a7e 100644 --- a/crypto/smartcont/stdlib.fc +++ b/crypto/smartcont/stdlib.fc @@ -1,6 +1,21 @@ ;; Standard library for funC ;; +{- + This file is part of TON FunC Standard Library. + + FunC Standard Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + FunC Standard Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + +-} + {- # Tuple manipulation primitives The names and the types are mostly self-explaining. @@ -229,7 +244,7 @@ cont bless(slice s) impure asm "BLESS"; ;;; In other words, the current smart contract agrees to buy some gas to finish the current transaction. ;;; This action is required to process external messages, which bring no value (hence no gas) with themselves. ;;; -;;; For more details check [accept_message effects](https://ton.org/docs/#/smart-contracts/accept). +;;; For more details check [accept_message effects](https://docs.ton.org/develop/smart-contracts/guidelines/accept). () accept_message() impure asm "ACCEPT"; ;;; Sets current gas limit `gl` to the minimum of limit and `gm`, and resets the gas credit `gc` to zero. @@ -267,10 +282,10 @@ int abs(int x) asm "ABS"; It is said that a primitive _loads_ some data, if it returns the data and the remainder of the slice - (so it can also be used as [modifying method](https://ton.org/docs/#/func/statements?id=modifying-methods)). + (so it can also be used as [modifying method](https://docs.ton.org/develop/func/statements#modifying-methods)). It is said that a primitive _preloads_ some data, if it returns only the data - (it can be used as [non-modifying method](https://ton.org/docs/#/func/statements?id=non-modifying-methods)). + (it can be used as [non-modifying method](https://docs.ton.org/develop/func/statements#non-modifying-methods)). Unless otherwise stated, loading and preloading primitives read the data from a prefix of the slice. -} @@ -310,7 +325,7 @@ cell preload_ref(slice s) asm "PLDREF"; ;;; Preloads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate `slice s''`. ;; slice preload_bits(slice s, int len) asm "PLDSLICEX"; -;;; Loads serialized amount of TonCoins (any unsigned integer up to `2^128 - 1`). +;;; Loads serialized amount of TonCoins (any unsigned integer up to `2^120 - 1`). (slice, int) load_grams(slice s) asm( -> 1 0) "LDGRAMS"; (slice, int) load_coins(slice s) asm( -> 1 0) "LDGRAMS"; @@ -401,7 +416,7 @@ int builder_depth(builder b) asm "BDEPTH"; # Builder primitives It is said that a primitive _stores_ a value `x` into a builder `b` if it returns a modified version of the builder `b'` with the value `x` stored at the end of it. - It can be used as [non-modifying method](https://ton.org/docs/#/func/statements?id=non-modifying-methods). + It can be used as [non-modifying method](https://docs.ton.org/develop/func/statements#non-modifying-methods). All the primitives below first check whether there is enough space in the `builder`, and only then check the range of the value being serialized. @@ -426,7 +441,7 @@ builder store_ref(builder b, cell c) asm(c b) "STREF"; ;;; Stores `slice` [s] into `builder` [b] builder store_slice(builder b, slice s) asm "STSLICER"; -;;; Stores (serializes) an integer [x] in the range `0..2^128 − 1` into `builder` [b]. +;;; Stores (serializes) an integer [x] in the range `0..2^120 − 1` into `builder` [b]. ;;; The serialization of [x] consists of a 4-bit unsigned big-endian integer `l`, ;;; which is the smallest integer `l ≥ 0`, such that `x < 2^8l`, ;;; followed by an `8l`-bit unsigned big-endian representation of [x]. diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk new file mode 100644 index 00000000..82757c22 --- /dev/null +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -0,0 +1,756 @@ +// Standard library for Tolk (LGPL licence). +// It contains common functions that are available out of the box, the user doesn't have to import anything. +// More specific functions are required to be imported explicitly, like "@stdlib/tvm-dicts". +tolk 0.9 + +/** + Tuple manipulation primitives. + Elements of a tuple can be of arbitrary type. + Note that atomic type `tuple` can't be cast to composite tuple type (e.g. `[int, cell]`) and vise versa. + */ + +/// Creates a tuple with zero elements. +@pure +fun createEmptyTuple(): tuple + asm "NIL"; + +/// Appends a value to tuple, resulting in `Tuple t' = (x1, ..., xn, value)`. +/// If its size exceeds 255, throws a type check exception. +@pure +fun tuplePush(mutate self: tuple, value: T): void + asm "TPUSH"; + +/// Returns the first element of a non-empty tuple. +/// `t.0` is actually the same as `t.tupleFirst()` +@pure +fun tupleFirst(self: tuple): T + asm "FIRST"; + +/// Returns the [`index`]-th element of a tuple. +/// `t.i` is actually the same as `t.tupleAt(i)` +@pure +fun tupleAt(self: tuple, index: int): T + builtin; + +/// Sets the [`index`]-th element of a tuple to a specified value +/// (element with this index must already exist, a new element isn't created). +/// `t.i = value` is actually the same as `t.tupleSetAt(value, i)` +@pure +fun tupleSetAt(mutate self: tuple, value: T, index: int): void + builtin; + +/// Returns the size of a tuple (elements count in it). +@pure +fun tupleSize(self: tuple): int + asm "TLEN"; + +/// Returns the last element of a non-empty tuple. +@pure +fun tupleLast(self: tuple): T + asm "LAST"; + + +/** + Mathematical primitives. + */ + +/// Computes the minimum of two integers. +@pure +fun min(x: int, y: int): int + asm "MIN"; + +/// Computes the maximum of two integers. +@pure +fun max(x: int, y: int): int + asm "MAX"; + +/// Sorts two integers. +@pure +fun minMax(x: int, y: int): (int, int) + asm "MINMAX"; + +/// Computes the absolute value of an integer. +@pure +fun abs(x: int): int + asm "ABS"; + +/// Returns the sign of an integer: `-1` if x < 0, `0` if x == 0, `1` if x > 0. +@pure +fun sign(x: int): int + asm "SGN"; + +/// Computes the quotient and remainder of [x] / [y]. Example: divMod(112,3) = (37,1) +@pure +fun divMod(x: int, y: int): (int, int) + asm "DIVMOD"; + +/// Computes the remainder and quotient of [x] / [y]. Example: modDiv(112,3) = (1,37) +@pure +fun modDiv(x: int, y: int): (int, int) + asm(-> 1 0) "DIVMOD"; + +/// Computes multiple-then-divide: floor([x] * [y] / [z]). +/// The intermediate result is stored in a 513-bit integer to prevent precision loss. +@pure +fun mulDivFloor(x: int, y: int, z: int): int + builtin; + +/// Similar to `mulDivFloor`, but rounds the result: round([x] * [y] / [z]). +@pure +fun mulDivRound(x: int, y: int, z: int): int + builtin; + +/// Similar to `mulDivFloor`, but ceils the result: ceil([x] * [y] / [z]). +@pure +fun mulDivCeil(x: int, y: int, z: int): int + builtin; + +/// Computes the quotient and remainder of ([x] * [y] / [z]). Example: mulDivMod(112,3,10) = (33,6) +@pure +fun mulDivMod(x: int, y: int, z: int): (int, int) + builtin; + + +/** + Global getters of environment and contract state. + */ + +const MASTERCHAIN = -1; +const BASECHAIN = 0; + +/// Returns current Unix timestamp (in seconds). +@pure +fun now(): int + asm "NOW"; + +/// Returns the internal address of the current smart contract as a Slice with a `MsgAddressInt`. +/// If necessary, it can be parsed further using primitives such as [parseStandardAddress]. +@pure +fun getMyAddress(): slice + asm "MYADDR"; + +/// Returns the balance (in nanotoncoins) of the smart contract at the start of Computation Phase. +/// Note that RAW primitives such as [sendMessage] do not update this field. +@pure +fun getMyOriginalBalance(): int + asm "BALANCE" "FIRST"; + +/// Same as [getMyOriginalBalance], but returns a tuple: +/// `int` — balance in nanotoncoins; +/// `cell` — a dictionary with 32-bit keys representing the balance of "extra currencies". +@pure +fun getMyOriginalBalanceWithExtraCurrencies(): [int, cell?] + asm "BALANCE"; + +/// Returns the logical time of the current transaction. +@pure +fun getLogicalTime(): int + asm "LTIME"; + +/// Returns the starting logical time of the current block. +@pure +fun getCurrentBlockLogicalTime(): int + asm "BLOCKLT"; + +/// Returns the value of the global configuration parameter with integer index `i` as a `cell` or `null` value. +@pure +fun getBlockchainConfigParam(x: int): cell? + asm "CONFIGOPTPARAM"; + +/// Returns the persistent contract storage cell. It can be parsed or modified with slice and builder primitives later. +@pure +fun getContractData(): cell + asm "c4 PUSH"; + +/// Sets `cell` [c] as persistent contract data. You can update persistent contract storage with this primitive. +fun setContractData(c: cell): void + asm "c4 POP"; + +/// Retrieves code of smart-contract from c7 +@pure +fun getContractCode(): cell + asm "MYCODE"; + +/// Creates an output action that would change this smart contract code to that given by cell [newCode]. +/// Notice that this change will take effect only after the successful termination of the current run of the smart contract. +fun setContractCodePostponed(newCode: cell): void + asm "SETCODE"; + +/// Commits the current state of registers `c4` (“persistent data”) and `c5` (“actions”) +/// so that the current execution is considered “successful” with the saved values even if an exception +/// in Computation Phase is thrown later. +fun commitContractDataAndActions(): void + asm "COMMIT"; + + +/** + Signature checks, hashing, cryptography. + */ + +/// Computes the representation hash of a `cell` [c] and returns it as a 256-bit unsigned integer `x`. +/// Useful for signing and checking signatures of arbitrary entities represented by a tree of cells. +@pure +fun cellHash(c: cell): int + asm "HASHCU"; + +/// Computes the hash of a `slice s` and returns it as a 256-bit unsigned integer `x`. +/// The result is the same as if an ordinary cell containing only data and references from `s` had been created +/// and its hash computed by [cellHash]. +@pure +fun sliceHash(s: slice): int + asm "HASHSU"; + +/// Computes sha256 of the data bits of `slice` [s]. If the bit length of `s` is not divisible by eight, +/// throws a cell underflow exception. The hash value is returned as a 256-bit unsigned integer `x`. +@pure +fun stringHash(s: slice): int + asm "SHA256U"; + +/// Checks the Ed25519-`signature` of a `hash` (a 256-bit unsigned integer, usually computed as the hash of some data) +/// using [publicKey] (also represented by a 256-bit unsigned integer). +/// The signature must contain at least 512 data bits; only the first 512 bits are used. +/// The result is `−1` if the signature is valid, `0` otherwise. +/// Note that `CHKSIGNU` creates a 256-bit slice with the hash and calls `CHKSIGNS`. +/// That is, if [hash] is computed as the hash of some data, these data are hashed twice, +/// the second hashing occurring inside `CHKSIGNS`. +@pure +fun isSignatureValid(hash: int, signature: slice, publicKey: int): bool + asm "CHKSIGNU"; + +/// Checks whether [signature] is a valid Ed25519-signature of the data portion of `slice data` using `publicKey`, +/// similarly to [isSignatureValid]. +/// If the bit length of [data] is not divisible by eight, throws a cell underflow exception. +/// The verification of Ed25519 signatures is the standard one, +/// with sha256 used to reduce [data] to the 256-bit number that is actually signed. +@pure +fun isSliceSignatureValid(data: slice, signature: slice, publicKey: int): bool + asm "CHKSIGNS"; + +/// Generates a new pseudo-random unsigned 256-bit integer x. +fun random(): int + asm "RANDU256"; + +/// Generates a new pseudo-random integer z in the range 0..range−1 (or range..−1, if range < 0). +/// More precisely, an unsigned random value x is generated as in random; then z := x * range / 2^256 is computed. +fun randomRange(range: int): int + asm "RAND"; + +/// Returns the current random seed as an unsigned 256-bit integer. +@pure +fun randomGetSeed(): int + asm "RANDSEED"; + +/// Sets the random seed to unsigned 256-bit seed. +fun randomSetSeed(seed: int): void + asm "SETRAND"; + +/// Initializes (mixes) random seed with unsigned 256-bit integer x. +fun randomizeBy(x: int): void + asm "ADDRAND"; + +/// Initializes random seed using current time. Don't forget to call this before calling `random`! +fun randomizeByLogicalTime(): void + asm "LTIME" "ADDRAND"; + + +/** + Size computation primitives. + They may be useful for computing storage fees of user-provided data. + */ + +/// Returns `(x, y, z, -1)` or `(null, null, null, 0)`. +/// Recursively computes the count of distinct cells `x`, data bits `y`, and cell references `z` +/// in the DAG rooted at `cell` [c], effectively returning the total storage used by this DAG taking into account +/// the identification of equal cells. +/// The values of `x`, `y`, and `z` are computed by a depth-first traversal of this DAG, +/// with a hash table of visited cell hashes used to prevent visits of already-visited cells. +/// The total count of visited cells `x` cannot exceed non-negative [maxCells]; +/// otherwise the computation is aborted before visiting the `(maxCells + 1)`-st cell and +/// a zero flag is returned to indicate failure. If [c] is `null`, returns `x = y = z = 0`. +@pure +fun calculateCellSize(c: cell, maxCells: int): (int, int, int, bool) + asm "CDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT"; + +/// Similar to [calculateCellSize], but accepting a `slice` [s] instead of a `cell`. +/// The returned value of `x` does not take into account the cell that contains the `slice` [s] itself; +/// however, the data bits and the cell references of [s] are accounted for in `y` and `z`. +@pure +fun calculateSliceSize(s: slice, maxCells: int): (int, int, int, bool) + asm "SDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT"; + +/// A non-quiet version of [calculateCellSize] that throws a cell overflow exception (`8`) on failure. +fun calculateCellSizeStrict(c: cell, maxCells: int): (int, int, int) + asm "CDATASIZE"; + +/// A non-quiet version of [calculateSliceSize] that throws a cell overflow exception (`8`) on failure. +fun calculateSliceSizeStrict(s: slice, maxCells: int): (int, int, int) + asm "SDATASIZE"; + +/// Returns the depth of `cell` [c]. +/// If [c] has no references, then return `0`; +/// otherwise the returned value is one plus the maximum of depths of cells referred to from [c]. +/// If [c] is a `null` instead of a cell, returns zero. +@pure +fun getCellDepth(c: cell?): int + asm "CDEPTH"; + +/// Returns the depth of `slice` [s]. +/// If [s] has no references, then returns `0`; +/// otherwise the returned value is one plus the maximum of depths of cells referred to from [s]. +@pure +fun getSliceDepth(s: slice): int + asm "SDEPTH"; + +/// Returns the depth of `builder` [b]. +/// If no cell references are stored in [b], then returns 0; +/// otherwise the returned value is one plus the maximum of depths of cells referred to from [b]. +@pure +fun getBuilderDepth(b: builder): int + asm "BDEPTH"; + + +/** + Debug primitives. + Only works for local TVM execution with debug level verbosity. + */ + +/// Dump a variable [x] to the debug log. +fun debugPrint(x: T): void + builtin; + +/// Dump a string [x] to the debug log. +fun debugPrintString(x: T): void + builtin; + +/// Dumps the stack (at most the top 255 values) and shows the total stack depth. +fun debugDumpStack(): void + builtin; + + +/** + Slice primitives: parsing cells. + When you _load_ some data, you mutate the slice (shifting an internal pointer on the stack). + When you _preload_ some data, you just get the result without mutating the slice. + */ + +/// Converts a `cell` [c] into a `slice`. Notice that [c] must be either an ordinary cell, +/// or an exotic cell (see [TVM.pdf](https://ton-blockchain.github.io/docs/tvm.pdf), 3.1.2) +/// which is automatically loaded to yield an ordinary cell `c'`, converted into a `slice` afterwards. +@pure +fun beginParse(c: cell): slice + asm "CTOS"; + +/// Checks if slice is empty. If not, throws an exception. +fun assertEndOfSlice(self: slice): void + asm "ENDS"; + +/// Loads the next reference from the slice. +@pure +fun loadRef(mutate self: slice): cell + asm( -> 1 0) "LDREF"; + +/// Preloads the next reference from the slice. +@pure +fun preloadRef(self: slice): cell + asm "PLDREF"; + +/// Loads a signed [len]-bit integer from a slice. +@pure +fun loadInt(mutate self: slice, len: int): int + builtin; + +/// Loads an unsigned [len]-bit integer from a slice. +@pure +fun loadUint(mutate self: slice, len: int): int + builtin; + +/// Loads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate slice `s''`. +@pure +fun loadBits(mutate self: slice, len: int): slice + builtin; + +/// Preloads a signed [len]-bit integer from a slice. +@pure +fun preloadInt(self: slice, len: int): int + builtin; + +/// Preloads an unsigned [len]-bit integer from a slice. +@pure +fun preloadUint(self: slice, len: int): int + builtin; + +/// Preloads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate slice. +@pure +fun preloadBits(self: slice, len: int): slice + builtin; + +/// Loads serialized amount of Toncoins (any unsigned integer up to `2^120 - 1`). +@pure +fun loadCoins(mutate self: slice): int + asm( -> 1 0) "LDGRAMS"; + +/// Loads bool (-1 or 0) from a slice +@pure +fun loadBool(mutate self: slice): bool + asm( -> 1 0) "1 LDI"; + +/// Shifts a slice pointer to [len] bits forward, mutating the slice. +@pure +fun skipBits(mutate self: slice, len: int): self + asm "SDSKIPFIRST"; + +/// Returns the first `0 ≤ len ≤ 1023` bits of a slice. +@pure +fun getFirstBits(self: slice, len: int): slice + asm "SDCUTFIRST"; + +/// Returns all but the last `0 ≤ len ≤ 1023` bits of a slice. +@pure +fun removeLastBits(mutate self: slice, len: int): self + asm "SDSKIPLAST"; + +/// Returns the last `0 ≤ len ≤ 1023` bits of a slice. +@pure +fun getLastBits(self: slice, len: int): slice + asm "SDCUTLAST"; + +/// Loads a dictionary (TL HashMapE structure, represented as TVM cell) from a slice. +/// Returns `null` if `nothing` constructor is used. +@pure +fun loadDict(mutate self: slice): cell? + asm( -> 1 0) "LDDICT"; + +/// Preloads a dictionary (cell) from a slice. +@pure +fun preloadDict(self: slice): cell? + asm "PLDDICT"; + +/// Loads a dictionary as [loadDict], but returns only the remainder of the slice. +@pure +fun skipDict(mutate self: slice): self + asm "SKIPDICT"; + +/// Loads (Maybe ^Cell) from a slice. +/// In other words, loads 1 bit: if it's true, loads the first ref, otherwise returns `null`. +@pure +fun loadMaybeRef(mutate self: slice): cell? + asm( -> 1 0) "LDOPTREF"; + +/// Preloads (Maybe ^Cell) from a slice. +@pure +fun preloadMaybeRef(self: slice): cell? + asm "PLDOPTREF"; + +/// Loads (Maybe ^Cell), but returns only the remainder of the slice. +@pure +fun skipMaybeRef(mutate self: slice): self + asm "SKIPOPTREF"; + +/** + Builder primitives: constructing cells. + When you _store_ some data, you mutate the builder (shifting an internal pointer on the stack). + All the primitives below first check whether there is enough space in the `builder`, + and only then check the range of the value being serialized. + */ + +/// Creates a new empty builder. +@pure +fun beginCell(): builder + asm "NEWC"; + +/// Converts a builder into an ordinary `cell`. +@pure +fun endCell(self: builder): cell + asm "ENDC"; + +/// Stores a reference to a cell into a builder. +@pure +fun storeRef(mutate self: builder, c: cell): self + asm(c self) "STREF"; + +/// Stores a signed [len]-bit integer into a builder (`0 ≤ len ≤ 257`). +@pure +fun storeInt(mutate self: builder, x: int, len: int): self + builtin; + +/// Stores an unsigned [len]-bit integer into a builder (`0 ≤ len ≤ 256`). +@pure +fun storeUint(mutate self: builder, x: int, len: int): self + builtin; + +/// Stores a slice into a builder. +@pure +fun storeSlice(mutate self: builder, s: slice): self + asm "STSLICER"; + +/// Stores amount of Toncoins into a builder. +@pure +fun storeCoins(mutate self: builder, x: int): self + asm "STGRAMS"; + +/// Stores bool (-1 or 0) into a builder. +/// Attention: true value is `-1`, not 1! If you pass `1` here, TVM will throw an exception. +@pure +fun storeBool(mutate self: builder, x: bool): self + asm(x self) "1 STI"; + +/// Stores dictionary (represented by TVM `cell` or `null`) into a builder. +/// In other words, stores a `1`-bit and a reference to [c] if [c] is not `null` and `0`-bit otherwise. +@pure +fun storeDict(mutate self: builder, c: cell?): self + asm(c self) "STDICT"; + +/// Stores (Maybe ^Cell) into a builder. +/// In other words, if cell is `null`, store '0' bit; otherwise, store '1' and a ref to [c]. +@pure +fun storeMaybeRef(mutate self: builder, c: cell?): self + asm(c self) "STOPTREF"; + +/// Concatenates two builders. +@pure +fun storeBuilder(mutate self: builder, from: builder): self + asm "STBR"; + +/// Stores a slice representing TL addr_none$00 (two `0` bits). +@pure +fun storeAddressNone(mutate self: builder): self + asm "b{00} STSLICECONST"; + + +/** + Slice size primitives. + */ + +/// Returns the number of references in a slice. +@pure +fun getRemainingRefsCount(self: slice): int + asm "SREFS"; + +/// Returns the number of data bits in a slice. +@pure +fun getRemainingBitsCount(self: slice): int + asm "SBITS"; + +/// Returns both the number of data bits and the number of references in a slice. +@pure +fun getRemainingBitsAndRefsCount(self: slice): (int, int) + asm "SBITREFS"; + +/// Checks whether a slice is empty (i.e., contains no bits of data and no cell references). +@pure +fun isEndOfSlice(self: slice): bool + asm "SEMPTY"; + +/// Checks whether a slice has no bits of data. +@pure +fun isEndOfSliceBits(self: slice): bool + asm "SDEMPTY"; + +/// Checks whether a slice has no references. +@pure +fun isEndOfSliceRefs(self: slice): bool + asm "SREMPTY"; + +/// Checks whether data parts of two slices coinside. +@pure +fun isSliceBitsEqual(self: slice, b: slice): bool + asm "SDEQ"; + +/// Returns the number of cell references already stored in a builder. +@pure +fun getBuilderRefsCount(self: builder): int + asm "BREFS"; + +/// Returns the number of data bits already stored in a builder. +@pure +fun getBuilderBitsCount(self: builder): int + asm "BBITS"; + + +/** + Address manipulation primitives. + The address manipulation primitives listed below serialize and deserialize values according to the following TL-B scheme: + ```TL-B + addr_none$00 = MsgAddressExt; + addr_extern$01 len:(## 8) external_address:(bits len) + = MsgAddressExt; + anycast_info$_ depth:(#<= 30) { depth >= 1 } + rewrite_pfx:(bits depth) = Anycast; + addr_std$10 anycast:(Maybe Anycast) + workchain_id:int8 address:bits256 = MsgAddressInt; + addr_var$11 anycast:(Maybe Anycast) addr_len:(## 9) + workchain_id:int32 address:(bits addr_len) = MsgAddressInt; + _ _:MsgAddressInt = MsgAddress; + _ _:MsgAddressExt = MsgAddress; + + int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool + src:MsgAddress dest:MsgAddressInt + value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams + created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; + ext_out_msg_info$11 src:MsgAddress dest:MsgAddressExt + created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; + ``` + A deserialized `MsgAddress` is represented by a tuple `t` as follows: + + - `addr_none` is represented by `t = (0)`, + i.e., a tuple containing exactly one integer equal to zero. + - `addr_extern` is represented by `t = (1, s)`, + where slice `s` contains the field `external_address`. In other words, ` + t` is a pair (a tuple consisting of two entries), containing an integer equal to one and slice `s`. + - `addr_std` is represented by `t = (2, u, x, s)`, + where `u` is either a `null` (if `anycast` is absent) or a slice `s'` containing `rewrite_pfx` (if anycast is present). + Next, integer `x` is the `workchain_id`, and slice `s` contains the address. + - `addr_var` is represented by `t = (3, u, x, s)`, + where `u`, `x`, and `s` have the same meaning as for `addr_std`. + */ + +/// Loads from slice [s] the only prefix that is a valid `MsgAddress`, +/// and returns both this prefix `s'` and the remainder `s''` of [s] as slices. +@pure +fun loadAddress(mutate self: slice): slice + asm( -> 1 0) "LDMSGADDR"; + +/// Decomposes slice [s] containing a valid `MsgAddress` into a `tuple t` with separate fields of this `MsgAddress`. +/// If [s] is not a valid `MsgAddress`, a cell deserialization exception is thrown. +@pure +fun parseAddress(s: slice): tuple + asm "PARSEMSGADDR"; + +/// Parses slice [s] containing a valid `MsgAddressInt` (usually a `msg_addr_std`), +/// applies rewriting from the anycast (if present) to the same-length prefix of the address, +/// and returns both the workchain and the 256-bit address as integers. +/// If the address is not 256-bit, or if [s] is not a valid serialization of `MsgAddressInt`, +/// throws a cell deserialization exception. +@pure +fun parseStandardAddress(s: slice): (int, int) + asm "REWRITESTDADDR"; + +/// Creates a slice representing TL addr_none$00 (two `0` bits). +@pure +fun createAddressNone(): slice + asm "b{00} PUSHSLICE"; + +/// Returns if a slice pointer contains an empty address. +/// In other words, a slice starts with two `0` bits (TL addr_none$00). +@pure +fun addressIsNone(s: slice): bool + asm "2 PLDU" "0 EQINT"; + + +/** + Reserving Toncoins on balance and its flags. + */ + +/// mode = 0: Reserve exact amount of nanotoncoins +const RESERVE_MODE_EXACT_AMOUNT = 0; +/// +1: Actually reserves all but amount, meaning `currentContractBalance - amount` +const RESERVE_MODE_ALL_BUT_AMOUNT = 1; +/// +2: Actually set `min(amount, currentContractBalance)` (without this mode, if amount is greater, the action will fail) +const RESERVE_MODE_AT_MOST = 2; +/// +4: [amount] is increased by the _original_ balance of the current account (before the compute phase). +const RESERVE_MODE_INCREASE_BY_ORIGINAL_BALANCE = 4; +/// +8: Actually sets `amount = -amount` before performing any further actions. +const RESERVE_MODE_NEGATE_AMOUNT = 8; +/// +16: If this action fails, the transaction will be bounced. +const RESERVE_MODE_BOUNCE_ON_ACTION_FAIL = 16; + +/// Creates an output action which would reserve Toncoins on balance. +/// For [reserveMode] consider constants above. +fun reserveToncoinsOnBalance(nanoTonCoins: int, reserveMode: int): void + asm "RAWRESERVE"; + +/// Similar to [reserveToncoinsOnBalance], but also accepts a dictionary extraAmount (represented by a cell or null) +/// with extra currencies. In this way currencies other than Toncoin can be reserved. +fun reserveExtraCurrenciesOnBalance(nanoTonCoins: int, extraAmount: cell?, reserveMode: int): void + asm "RAWRESERVEX"; + + +/** + Messages sending and parsing primitives. + Working with messages is low-level right now, but still, every contract should do that. + + `Message` structure, its header and so on are specified in TL-B scheme, particularly: + int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool ... = CommonMsgInfo; + */ + +/// 0b011000 tag - 0, ihr_disabled - 1, bounce - 1, bounced - 0, src = adr_none$00 +const BOUNCEABLE = 0x18; +/// 0b010000 tag - 0, ihr_disabled - 1, bounce - 0, bounced - 0, src = adr_none$00 +const NON_BOUNCEABLE = 0x10; + +/// Load msgFlags from incoming message body (4 bits). +@pure +fun loadMessageFlags(mutate self: slice): int + asm( -> 1 0) "4 LDU"; + +/// Having msgFlags (4 bits), check that a message is bounced. +/// Effectively, it's `msgFlags & 1` (the lowest bit present). +@pure +fun isMessageBounced(msgFlags: int): bool + asm "2 PUSHINT" "MODR"; + +/// Skip 0xFFFFFFFF prefix (when a message is bounced). +@pure +fun skipBouncedPrefix(mutate self: slice): self + asm "32 PUSHINT" "SDSKIPFIRST"; + +/// The guideline recommends to start the body of an internal message with uint32 `op` and uint64 `queryId`. +@pure +fun loadMessageOp(mutate self: slice): int + asm( -> 1 0) "32 LDU"; + +@pure +fun skipMessageOp(mutate self: slice): self + asm "32 PUSHINT" "SDSKIPFIRST"; + +@pure +fun storeMessageOp(mutate self: builder, op: int): self + asm(op self) "32 STU"; + +/// The guideline recommends that uint64 `queryId` should follow uint32 `op`. +@pure +fun loadMessageQueryId(mutate self: slice): int + asm( -> 1 0) "64 LDU"; + +@pure +fun skipMessageQueryId(mutate self: slice): self + asm "64 PUSHINT" "SDSKIPFIRST"; + +@pure +fun storeMessageQueryId(mutate self: builder, queryId: int): self + asm(queryId self) "64 STU"; + +/// SEND MODES - https://docs.ton.org/tvm.pdf page 137, SENDRAWMSG + +/// mode = 0 is used for ordinary messages; the gas fees are deducted from the senging amount; action phaes should NOT be ignored. +const SEND_MODE_REGULAR = 0; +/// +1 means that the sender wants to pay transfer fees separately. +const SEND_MODE_PAY_FEES_SEPARATELY = 1; +/// +2 means that any errors arising while processing this message during the action phase should be ignored. +const SEND_MODE_IGNORE_ERRORS = 2; +/// in the case of action fail - bounce transaction. No effect if SEND_MODE_IGNORE_ERRORS (+2) is used. TVM UPGRADE 2023-07. https://docs.ton.org/learn/tvm-instructions/tvm-upgrade-2023-07#sending-messages +const SEND_MODE_BOUNCE_ON_ACTION_FAIL = 16; +/// mode = 32 means that the current account must be destroyed if its resulting balance is zero. +const SEND_MODE_DESTROY = 32; +/// mode = 64 is used for messages that carry all the remaining value of the inbound message in addition to the value initially indicated in the new message. +const SEND_MODE_CARRY_ALL_REMAINING_MESSAGE_VALUE = 64; +/// mode = 128 is used for messages that are to carry all the remaining balance of the current smart contract (instead of the value originally indicated in the message). +const SEND_MODE_CARRY_ALL_BALANCE = 128; +/// do not create an action, only estimate fee. TVM UPGRADE 2023-07. https://docs.ton.org/learn/tvm-instructions/tvm-upgrade-2023-07#sending-messages +const SEND_MODE_ESTIMATE_FEE_ONLY = 1024; +/// Other modes affect the fee calculation as follows: +/// +64 substitutes the entire balance of the incoming message as an outcoming value (slightly inaccurate, gas expenses that cannot be estimated before the computation is completed are not taken into account). +/// +128 substitutes the value of the entire balance of the contract before the start of the computation phase (slightly inaccurate, since gas expenses that cannot be estimated before the completion of the computation phase are not taken into account). + +/// Sends a raw message — a correctly serialized TL object `Message X`. +/// For `mode`, see constants above (except SEND_MODE_ESTIMATE_FEE_ONLY). +/// This function is still available, but deprecated: consider using [sendMessage]. +@deprecated +fun sendRawMessage(msg: cell, mode: int): void + asm "SENDRAWMSG"; + +/// Creates an output action and returns a fee for creating a message. +/// Mode has the same effect as in the case of SENDRAWMSG. +/// For mode including SEND_MODE_ESTIMATE_FEE_ONLY it just returns estimated fee without sending a message. +fun sendMessage(msg: cell, mode: int): int + asm "SENDMSG"; diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk new file mode 100644 index 00000000..9873ca94 --- /dev/null +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -0,0 +1,69 @@ +// A part of standard library for Tolk +tolk 0.9 + +/** + Gas and payment related primitives. + */ + +/// Returns amount of gas (in gas units) consumed in current Computation Phase. +fun getGasConsumedAtTheMoment(): int + asm "GASCONSUMED"; + +/// This function is required to be called when you process an external message (from an outer world) +/// and "accept" it to blockchain. +/// Without calling this function, an external message would be discarded. +/// As an effect, the current smart contract agrees to buy some gas to finish the current transaction. +/// For more details, check [accept_message effects](https://ton.org/docs/#/smart-contracts/accept). +fun acceptExternalMessage(): void + asm "ACCEPT"; + +/// When processing an internal message, by default, the limit of gas consumption is determined by incoming message. +/// Functions [setGasLimit] and [setGasLimitToMaximum] allow you to change this behavior. +/// Sets current gas limit `gl` to its maximal allowed value `gm`, and resets the gas credit `gc` to zero, +/// decreasing the value of `gr` by `gc` in the process. +fun setGasLimitToMaximum(): void + asm "ACCEPT"; + +/// When processing an internal message, by default, the limit of gas consumption is determined by incoming message. +/// Functions [setGasLimit] and [setGasLimitToMaximum] allow you to change this behavior. +/// Sets current gas limit `gl` to the minimum of limit and `gm`, and resets the gas credit `gc` to zero. +/// If the gas consumed so far (including the present instruction) exceeds the resulting value of `gl`, +/// an (unhandled) out of gas exception is thrown before setting new gas limits. +fun setGasLimit(limit: int): void + asm "SETGASLIMIT"; + +/// Calculates fee (amount in nanotoncoins to be paid) for a transaction which consumed [gasUsed] gas units. +fun calculateGasFee(workchain: int, gasUsed: int): int + asm(gasUsed workchain) "GETGASFEE"; + +/// Same as [calculateGasFee], but without flat price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) +fun calculateGasFeeWithoutFlatPrice(workchain: int, gasUsed: int): int + asm(gasUsed workchain) "GETGASFEESIMPLE"; + +/// Calculates amount of nanotoncoins you should pay for storing a contract of provided size for [seconds]. +/// [bits] and [cells] represent contract state (code + data). +fun calculateStorageFee(workchain: int, seconds: int, bits: int, cells: int): int + asm(cells bits seconds workchain) "GETSTORAGEFEE"; + +/// Calculates amount of nanotoncoins you should pay to send a message of specified size. +fun calculateMessageFee(workchain: int, bits: int, cells: int): int + asm(cells bits workchain) "GETFORWARDFEE"; + +/// Same as [calculateMessageFee], but without lump price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) +fun calculateMessageFeeWithoutLumpPrice(workchain: int, bits: int, cells: int): int + asm(cells bits workchain) "GETFORWARDFEESIMPLE"; + +/// Calculates fee that was paid by the sender of an incoming internal message. +fun calculateOriginalMessageFee(workchain: int, incomingFwdFee: int): int + asm(incomingFwdFee workchain) "GETORIGINALFWDFEE"; + +/// Returns the amount of nanotoncoins current contract debts for storage. ("due" and "debt" are synonyms) +/// If it has no debt, `0` is returned. +fun getMyStorageDuePayment(): int + asm "DUEPAYMENT"; + +/// Returns the amount of nanotoncoins charged for storage. +/// (during storage phase preceeding to current computation phase) +@pure +fun getMyStoragePaidPayment(): int + asm "STORAGEFEES"; diff --git a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk new file mode 100644 index 00000000..e63438b5 --- /dev/null +++ b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk @@ -0,0 +1,39 @@ +// A part of standard library for Tolk +tolk 0.9 + +/** + Lisp-style lists are nested 2-elements tuples: `(1, (2, (3, null)))` represents list `[1, 2, 3]`. + Elements of a list can be of different types. + Empty list is conventionally represented as TVM `null` value. + */ + +@pure +fun createEmptyList(): tuple + asm "PUSHNULL"; + +/// Adds an element to the beginning of lisp-style list. +/// Note, that it does not mutate the list: instead, it returns a new one (it's a lisp pattern). +@pure +fun listPrepend(head: X, tail: tuple?): tuple + asm "CONS"; + +/// Extracts the head and the tail of lisp-style list. +@pure +fun listSplit(list: tuple): (X, tuple?) + asm "UNCONS"; + +/// Extracts the tail and the head of lisp-style list. +/// After extracting the last element, tuple is assigned to null. +@pure +fun listNext(mutate self: tuple?): X + asm( -> 1 0) "UNCONS"; + +/// Returns the head of lisp-style list. +@pure +fun listGetHead(list: tuple): X + asm "CAR"; + +/// Returns the tail of lisp-style list. +@pure +fun listGetTail(list: tuple): tuple? + asm "CDR"; diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk new file mode 100644 index 00000000..ee205687 --- /dev/null +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -0,0 +1,312 @@ +// A part of standard library for Tolk +tolk 0.9 + +/** + Dictionaries are represented as `cell` data type (cells can store anything, dicts in particular). + Currently, they have very low-level API very close to TVM internals. + Most of functions are duplicated for three common cases: + - iDict* - dicts with signed integer keys + - uDict* - dicts with unsigned integer keys + - sDict* - dicts with arbitrary slice keys + When accessing a dict element, you should not only provide a key, but provide keyLen, + since for optimization, key length is not stored in the dictionary itself. + Every dictionary object (`self` parameter) can be null. TVM NULL is essentially "empty dictionary". + */ + +/// Creates an empty dictionary, which is actually a null value. Equivalent to PUSHNULL +@pure +fun createEmptyDict(): cell? + asm "NEWDICT"; + +/// Checks whether a dictionary is empty. +@pure +fun dictIsEmpty(self: cell?): bool + asm "DICTEMPTY"; + + +@pure +fun iDictGet(self: cell?, keyLen: int, key: int): (slice?, bool) + asm(key self keyLen) "DICTIGET" "NULLSWAPIFNOT"; + +@pure +fun uDictGet(self: cell?, keyLen: int, key: int): (slice?, bool) + asm(key self keyLen) "DICTUGET" "NULLSWAPIFNOT"; + +@pure +fun sDictGet(self: cell?, keyLen: int, key: slice): (slice?, bool) + asm(key self keyLen) "DICTGET" "NULLSWAPIFNOT"; + + +@pure +fun iDictSet(mutate self: cell?, keyLen: int, key: int, value: slice): void + asm(value key self keyLen) "DICTISET"; + +@pure +fun uDictSet(mutate self: cell?, keyLen: int, key: int, value: slice): void + asm(value key self keyLen) "DICTUSET"; + +@pure +fun sDictSet(mutate self: cell?, keyLen: int, key: slice, value: slice): void + asm(value key self keyLen) "DICTSET"; + + +@pure +fun iDictSetRef(mutate self: cell?, keyLen: int, key: int, value: cell): void + asm(value key self keyLen) "DICTISETREF"; + +@pure +fun uDictSetRef(mutate self: cell?, keyLen: int, key: int, value: cell): void + asm(value key self keyLen) "DICTUSETREF"; + +@pure +fun sDictSetRef(mutate self: cell?, keyLen: int, key: slice, value: cell): void + asm(value key self keyLen) "DICTSETREF"; + + +@pure +fun iDictSetIfNotExists(mutate self: cell?, keyLen: int, key: int, value: slice): bool + asm(value key self keyLen) "DICTIADD"; + +@pure +fun uDictSetIfNotExists(mutate self: cell?, keyLen: int, key: int, value: slice): bool + asm(value key self keyLen) "DICTUADD"; + + +@pure +fun iDictSetIfExists(mutate self: cell?, keyLen: int, key: int, value: slice): bool + asm(value key self keyLen) "DICTIREPLACE"; + +@pure +fun uDictSetIfExists(mutate self: cell?, keyLen: int, key: int, value: slice): bool + asm(value key self keyLen) "DICTUREPLACE"; + + +@pure +fun iDictGetRef(self: cell?, keyLen: int, key: int): (cell?, bool) + asm(key self keyLen) "DICTIGETREF" "NULLSWAPIFNOT"; + +@pure +fun uDictGetRef(self: cell?, keyLen: int, key: int): (cell?, bool) + asm(key self keyLen) "DICTUGETREF" "NULLSWAPIFNOT"; + +@pure +fun sDictGetRef(self: cell?, keyLen: int, key: slice): (cell?, bool) + asm(key self keyLen) "DICTGETREF" "NULLSWAPIFNOT"; + + +@pure +fun iDictGetRefOrNull(self: cell?, keyLen: int, key: int): cell? + asm(key self keyLen) "DICTIGETOPTREF"; + +@pure +fun uDictGetRefOrNull(self: cell?, keyLen: int, key: int): cell? + asm(key self keyLen) "DICTUGETOPTREF"; + +@pure +fun sDictGetRefOrNull(self: cell?, keyLen: int, key: slice): cell? + asm(key self keyLen) "DICTGETOPTREF"; + + +@pure +fun iDictDelete(mutate self: cell?, keyLen: int, key: int): bool + asm(key self keyLen) "DICTIDEL"; + +@pure +fun uDictDelete(mutate self: cell?, keyLen: int, key: int): bool + asm(key self keyLen) "DICTUDEL"; + +@pure +fun sDictDelete(mutate self: cell?, keyLen: int, key: slice): bool + asm(key self keyLen) "DICTDEL"; + + +@pure +fun iDictSetAndGet(mutate self: cell?, keyLen: int, key: int, value: slice): (slice?, bool) + asm(value key self keyLen) "DICTISETGET" "NULLSWAPIFNOT"; + +@pure +fun uDictSetAndGet(mutate self: cell?, keyLen: int, key: int, value: slice): (slice?, bool) + asm(value key self keyLen) "DICTUSETGET" "NULLSWAPIFNOT"; + +@pure +fun sDictSetAndGet(mutate self: cell?, keyLen: int, key: slice, value: slice): (slice?, bool) + asm(value key self keyLen) "DICTSETGET" "NULLSWAPIFNOT"; + + +@pure +fun iDictSetAndGetRefOrNull(mutate self: cell?, keyLen: int, key: int, value: cell): cell? + asm(value key self keyLen) "DICTISETGETOPTREF"; + +@pure +fun uDictSetAndGetRefOrNull(mutate self: cell?, keyLen: int, key: int, value: cell): cell? + asm(value key self keyLen) "DICTUSETGETOPTREF"; + + +@pure +fun iDictDeleteAndGet(mutate self: cell?, keyLen: int, key: int): (slice?, bool) + asm(key self keyLen) "DICTIDELGET" "NULLSWAPIFNOT"; + +@pure +fun uDictDeleteAndGet(mutate self: cell?, keyLen: int, key: int): (slice?, bool) + asm(key self keyLen) "DICTUDELGET" "NULLSWAPIFNOT"; + +@pure +fun sDictDeleteAndGet(mutate self: cell?, keyLen: int, key: slice): (slice?, bool) + asm(key self keyLen) "DICTDELGET" "NULLSWAPIFNOT"; + + +@pure +fun iDictSetBuilder(mutate self: cell?, keyLen: int, key: int, value: builder): void + asm(value key self keyLen) "DICTISETB"; + +@pure +fun uDictSetBuilder(mutate self: cell?, keyLen: int, key: int, value: builder): void + asm(value key self keyLen) "DICTUSETB"; + +@pure +fun sDictSetBuilder(mutate self: cell?, keyLen: int, key: slice, value: builder): void + asm(value key self keyLen) "DICTSETB"; + + +@pure +fun iDictSetBuilderIfNotExists(mutate self: cell?, keyLen: int, key: int, value: builder): bool + asm(value key self keyLen) "DICTIADDB"; + +@pure +fun uDictSetBuilderIfNotExists(mutate self: cell?, keyLen: int, key: int, value: builder): bool + asm(value key self keyLen) "DICTUADDB"; + +@pure +fun iDictSetBuilderIfExists(mutate self: cell?, keyLen: int, key: int, value: builder): bool + asm(value key self keyLen) "DICTIREPLACEB"; + +@pure +fun uDictSetBuilderIfExists(mutate self: cell?, keyLen: int, key: int, value: builder): bool + asm(value key self keyLen) "DICTUREPLACEB"; + + +@pure +fun iDictDeleteFirstAndGet(mutate self: cell?, keyLen: int): (int?, slice?, bool) + asm(-> 0 2 1 3) "DICTIREMMIN" "NULLSWAPIFNOT2"; + +@pure +fun uDictDeleteFirstAndGet(mutate self: cell?, keyLen: int): (int?, slice?, bool) + asm(-> 0 2 1 3) "DICTUREMMIN" "NULLSWAPIFNOT2"; + +@pure +fun sDictDeleteFirstAndGet(mutate self: cell?, keyLen: int): (slice?, slice?, bool) + asm(-> 0 2 1 3) "DICTREMMIN" "NULLSWAPIFNOT2"; + + +@pure +fun iDictDeleteLastAndGet(mutate self: cell?, keyLen: int): (int?, slice?, bool) + asm(-> 0 2 1 3) "DICTIREMMAX" "NULLSWAPIFNOT2"; + +@pure +fun uDictDeleteLastAndGet(mutate self: cell?, keyLen: int): (int?, slice?, bool) + asm(-> 0 2 1 3) "DICTUREMMAX" "NULLSWAPIFNOT2"; + +@pure +fun sDictDeleteLastAndGet(mutate self: cell?, keyLen: int): (slice?, slice?, bool) + asm(-> 0 2 1 3) "DICTREMMAX" "NULLSWAPIFNOT2"; + + +@pure +fun iDictGetFirst(self: cell?, keyLen: int): (int?, slice?, bool) + asm (-> 1 0 2) "DICTIMIN" "NULLSWAPIFNOT2"; + +@pure +fun uDictGetFirst(self: cell?, keyLen: int): (int?, slice?, bool) + asm (-> 1 0 2) "DICTUMIN" "NULLSWAPIFNOT2"; + +@pure +fun sDictGetFirst(self: cell?, keyLen: int): (slice?, slice?, bool) + asm (-> 1 0 2) "DICTMIN" "NULLSWAPIFNOT2"; + +@pure +fun iDictGetFirstAsRef(self: cell?, keyLen: int): (int?, cell?, bool) + asm (-> 1 0 2) "DICTIMINREF" "NULLSWAPIFNOT2"; + +@pure +fun uDictGetFirstAsRef(self: cell?, keyLen: int): (int?, cell?, bool) + asm (-> 1 0 2) "DICTUMINREF" "NULLSWAPIFNOT2"; + +@pure +fun sDictGetFirstAsRef(self: cell?, keyLen: int): (slice?, cell?, bool) + asm (-> 1 0 2) "DICTMINREF" "NULLSWAPIFNOT2"; + + +@pure +fun iDictGetLast(self: cell?, keyLen: int): (int?, slice?, bool) + asm (-> 1 0 2) "DICTIMAX" "NULLSWAPIFNOT2"; + +@pure +fun uDictGetLast(self: cell?, keyLen: int): (int?, slice?, bool) + asm (-> 1 0 2) "DICTUMAX" "NULLSWAPIFNOT2"; + +@pure +fun sDictGetLast(self: cell?, keyLen: int): (slice?, slice?, bool) + asm (-> 1 0 2) "DICTMAX" "NULLSWAPIFNOT2"; + +@pure +fun iDictGetLastAsRef(self: cell?, keyLen: int): (int?, cell?, bool) + asm (-> 1 0 2) "DICTIMAXREF" "NULLSWAPIFNOT2"; + +@pure +fun uDictGetLastAsRef(self: cell?, keyLen: int): (int?, cell?, bool) + asm (-> 1 0 2) "DICTUMAXREF" "NULLSWAPIFNOT2"; + +@pure +fun sDictGetLastAsRef(self: cell?, keyLen: int): (slice?, cell?, bool) + asm (-> 1 0 2) "DICTMAXREF" "NULLSWAPIFNOT2"; + + +@pure +fun iDictGetNext(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXT" "NULLSWAPIFNOT2"; + +@pure +fun uDictGetNext(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXT" "NULLSWAPIFNOT2"; + +@pure +fun iDictGetNextOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXTEQ" "NULLSWAPIFNOT2"; + +@pure +fun uDictGetNextOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXTEQ" "NULLSWAPIFNOT2"; + + +@pure +fun iDictGetPrev(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTIGETPREV" "NULLSWAPIFNOT2"; + +@pure +fun uDictGetPrev(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTUGETPREV" "NULLSWAPIFNOT2"; + +@pure +fun iDictGetPrevOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTIGETPREVEQ" "NULLSWAPIFNOT2"; + +@pure +fun uDictGetPrevOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTUGETPREVEQ" "NULLSWAPIFNOT2"; + + +/** + Prefix dictionary primitives. + */ + +@pure +fun prefixDictGet(self: cell?, keyLen: int, key: slice): (slice, slice?, slice?, bool) + asm(key self keyLen) "PFXDICTGETQ" "NULLSWAPIFNOT2"; + +@pure +fun prefixDictSet(mutate self: cell?, keyLen: int, key: slice, value: slice): bool + asm(value key self keyLen) "PFXDICTSET"; + +@pure +fun prefixDictDelete(mutate self: cell?, keyLen: int, key: slice): bool + asm(key self keyLen) "PFXDICTDEL"; diff --git a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk new file mode 100644 index 00000000..136eaa4a --- /dev/null +++ b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk @@ -0,0 +1,25 @@ +// A part of standard library for Tolk +tolk 0.9 + +/// Usually `c3` has a continuation initialized by the whole code of the contract. It is used for function calls. +/// The primitive returns the current value of `c3`. +@pure +fun getTvmRegisterC3(): continuation + asm "c3 PUSH"; + +/// Updates the current value of `c3`. Usually, it is used for updating smart contract code in run-time. +/// Note that after execution of this primitive the current code +/// (and the stack of recursive function calls) won't change, +/// but any other function call will use a function from the new code. +fun setTvmRegisterC3(c: continuation): void + asm "c3 POP"; + +/// Transforms a `slice` [s] into a simple ordinary continuation `c`, with `c.code = s` and an empty stack and savelist. +@pure +fun transformSliceToContinuation(s: slice): continuation + asm "BLESS"; + +/// Moves a variable or a value [x] to the top of the stack. +@pure +fun stackMoveToTop(mutate self: X): void + asm "NOP"; diff --git a/crypto/smartcont/wallet-v3.fif b/crypto/smartcont/wallet-v3.fif index c090dc09..f44e0c92 100644 --- a/crypto/smartcont/wallet-v3.fif +++ b/crypto/smartcont/wallet-v3.fif @@ -13,7 +13,7 @@ variable extra-currencies { extra-currencies @ cc+ extra-currencies ! } : extra-cc+! begin-options - " [-x *] [-n|-b] [-t] [-B ] [-C ] []" +cr +tab + " [-x *] [-n|-b] [-t] [-B ] [-C ] [-I ] []" +cr +tab +"Creates a request to advanced wallet created by new-wallet-v3.fif, with private key loaded from file .pk " +"and address from .addr, and saves it into .boc ('wallet-query.boc' by default)" disable-digit-options generic-help-setopt @@ -29,6 +29,8 @@ begin-options "Sets the payload of the transfer message" option-help "C" "--comment" { =: comment } short-long-option-arg "Sets the comment to be sent in the transfer message" option-help + "I" "--with-init" { =: init-file } short-long-option-arg + "Indicates filename with BoC containing StateInit for internal message" option-help "m" "--mode" { parse-int =: send-mode } short-long-option-arg "Sets transfer mode (0..255) for SENDRAWMSG (" send-mode (.) $+ +" by default)" option-help @@ -57,14 +59,18 @@ file-base +".pk" load-keypair nip constant wallet_pk def? body-boc-file { @' body-boc-file file>B B>boc } { comment simple-transfer-body } cond constant body-cell +def? init-file { @' init-file file>B B>boc diff --git a/crypto/smc-envelope/GenericAccount.cpp b/crypto/smc-envelope/GenericAccount.cpp index 4cd6bf3f..4c9bd165 100644 --- a/crypto/smc-envelope/GenericAccount.cpp +++ b/crypto/smc-envelope/GenericAccount.cpp @@ -61,7 +61,8 @@ block::StdAddress GenericAccount::get_address(ton::WorkchainId workchain_id, return block::StdAddress(workchain_id, init_state->get_hash().bits(), true /*bounce*/); } -void GenericAccount::store_int_message(vm::CellBuilder& cb, const block::StdAddress& dest_address, td::int64 gramms) { +void GenericAccount::store_int_message(vm::CellBuilder& cb, const block::StdAddress& dest_address, td::int64 gramms, + td::Ref extra_currencies) { td::BigInt256 dest_addr; dest_addr.import_bits(dest_address.addr.as_bitslice()); cb.store_zeroes(1) @@ -73,7 +74,8 @@ void GenericAccount::store_int_message(vm::CellBuilder& cb, const block::StdAddr .store_long(dest_address.workchain, 8) .store_int256(dest_addr, 256); block::tlb::t_Grams.store_integer_value(cb, td::BigInt256(gramms)); - cb.store_zeroes(9 + 64 + 32); + cb.store_maybe_ref(extra_currencies); + cb.store_zeroes(8 + 64 + 32); } td::Ref GenericAccount::create_ext_message(const block::StdAddress& address, td::Ref new_state, @@ -155,7 +157,7 @@ td::Result GenericAccount::get_wallet_id(const SmartContract& sc) { return TRY_VM([&]() -> td::Result { auto answer = sc.run_get_method("wallet_id"); if (!answer.success) { - return td::Status::Error("seqno get method failed"); + return td::Status::Error("wallet_id get method failed"); } return static_cast(answer.stack.write().pop_long_range(std::numeric_limits::max())); }()); diff --git a/crypto/smc-envelope/GenericAccount.h b/crypto/smc-envelope/GenericAccount.h index 285553c7..93a059f9 100644 --- a/crypto/smc-envelope/GenericAccount.h +++ b/crypto/smc-envelope/GenericAccount.h @@ -36,7 +36,8 @@ class GenericAccount { static block::StdAddress get_address(ton::WorkchainId workchain_id, const td::Ref& init_state) noexcept; static td::Ref create_ext_message(const block::StdAddress& address, td::Ref new_state, td::Ref body) noexcept; - static void store_int_message(vm::CellBuilder& cb, const block::StdAddress& dest_address, td::int64 gramms); + static void store_int_message(vm::CellBuilder& cb, const block::StdAddress& dest_address, td::int64 gramms, + td::Ref extra_currencies); static td::Result get_public_key(const SmartContract& sc); static td::Result get_seqno(const SmartContract& sc); diff --git a/crypto/smc-envelope/ManualDns.cpp b/crypto/smc-envelope/ManualDns.cpp index 40ce3bac..617ab915 100644 --- a/crypto/smc-envelope/ManualDns.cpp +++ b/crypto/smc-envelope/ManualDns.cpp @@ -105,7 +105,7 @@ td::Result> DnsInterface::EntryData::as_cell() const { return error; } if (res.is_null()) { - return td::Status::Error("Entry data is emtpy"); + return td::Status::Error("Entry data is empty"); } return res; //dns_text#1eda _:Text = DNSRecord; diff --git a/crypto/smc-envelope/ManualDns.h b/crypto/smc-envelope/ManualDns.h index b5dee59a..d24cd023 100644 --- a/crypto/smc-envelope/ManualDns.h +++ b/crypto/smc-envelope/ManualDns.h @@ -305,7 +305,7 @@ class ManualDns : public ton::SmartContract, public DnsInterface { if (!info.known_category.insert(action.category).second) { continue; } - if (action.category == 0) { + if (action.category.is_zero()) { info.closed = true; auto old_actions = std::move(info.actions); bool is_empty = true; @@ -327,7 +327,7 @@ class ManualDns : public ton::SmartContract, public DnsInterface { if (info.closed) { CombinedActions ca; ca.name = it.first; - ca.category = 0; + ca.category = td::Bits256::zero(); if (!info.actions.empty() || info.non_empty) { ca.actions = std::move(info.actions); } diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index 9a273a08..4d860fba 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -40,21 +40,93 @@ int SmartContract::Answer::output_actions_count(td::Ref list) { } namespace { -td::Ref prepare_vm_stack(td::RefInt256 amount, td::Ref body) { +td::Ref build_internal_message(td::RefInt256 amount, td::Ref body, SmartContract::Args args) { + vm::CellBuilder cb; + if (args.address) { + td::BigInt256 dest_addr; + dest_addr.import_bits((*args.address).addr.as_bitslice()); + cb.store_ones(1) + .store_zeroes(2) + .store_long((*args.address).workchain, 8) + .store_int256(dest_addr, 256); + } + auto address = cb.finalize(); + + vm::CellBuilder b; + b.store_long(0b0110, 4); // 0 ihr_disabled:Bool bounce:Bool bounced:Bool + // use -1:00..00 as src:MsgAddressInt + // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; + b.store_long(0b100, 3); b.store_ones(8); b.store_zeroes(256); + b.append_cellslice(address); // dest:MsgAddressInt + unsigned len = (((unsigned)amount->bit_size(false) + 7) >> 3); + b.store_long_bool(len, 4) && b.store_int256_bool(*amount, len * 8, false); // grams:Grams + b.store_zeroes(1 + 4 + 4 + 64 + 32 + 1); // extre, ihr_fee, fwd_fee, created_lt, created_at, init + // body:(Either X ^X) + if (b.remaining_bits() >= 1 + (*body).size() && b.remaining_refs() >= (*body).size_refs()) { + b.store_zeroes(1); + b.append_cellslice(body); + } else { + b.store_ones(1); + b.store_ref(vm::CellBuilder().append_cellslice(body).finalize_novm()); + } + return b.finalize_novm(); +} + +td::Ref build_external_message(td::RefInt256 amount, td::Ref body, SmartContract::Args args) { + vm::CellBuilder cb; + if (args.address) { + td::BigInt256 dest_addr; + dest_addr.import_bits((*args.address).addr.as_bitslice()); + cb.store_ones(1) + .store_zeroes(2) + .store_long((*args.address).workchain, 8) + .store_int256(dest_addr, 256); + } + auto address = cb.finalize(); + + vm::CellBuilder b; + b.store_long(0b1000, 4); // ext_in_msg_info$10 src:MsgAddressExt + b.append_cellslice(address); // dest:MsgAddressInt + b.store_zeroes(4); //import_fee:Grams + b.store_zeroes(1); // init + // body:(Either X ^X) + if (b.remaining_bits() >= 1 + (*body).size() && b.remaining_refs() >= (*body).size_refs()) { + b.store_zeroes(1); + b.append_cellslice(body); + } else { + b.store_ones(1); + b.store_ref(vm::CellBuilder().append_cellslice(body).finalize_novm()); + } + return b.finalize_novm(); +} + +td::Ref prepare_vm_stack(td::RefInt256 amount, td::Ref body, SmartContract::Args args, int selector) { td::Ref stack_ref{true}; td::RefInt256 acc_addr{true}; //CHECK(acc_addr.write().import_bits(account.addr.cbits(), 256)); vm::Stack& stack = stack_ref.write(); - stack.push_int(td::make_refint(10000000000)); - stack.push_int(std::move(amount)); - stack.push_cell(vm::CellBuilder().finalize()); + if(args.balance) { + stack.push_int(td::make_refint(args.balance)); + } else { + stack.push_int(td::make_refint(10000000000)); + } + stack.push_int(amount); + if(selector == 0) { + stack.push_cell(build_internal_message(amount, body, args)); + } else { + stack.push_cell(build_external_message(amount, body, args)); + } stack.push_cellslice(std::move(body)); return stack_ref; } -td::Ref prepare_vm_c7(SmartContract::Args args) { +td::Ref prepare_vm_c7(SmartContract::Args args, td::Ref code) { td::BitArray<256> rand_seed; - rand_seed.as_slice().fill(0); + if (args.rand_seed) { + rand_seed = args.rand_seed.unwrap(); + } else { + rand_seed.as_slice().fill(0); + } td::RefInt256 rand_seed_int{true}; rand_seed_int.unique_write().import_bits(rand_seed.cbits(), 256, false); @@ -67,10 +139,7 @@ td::Ref prepare_vm_c7(SmartContract::Args args) { if (args.address) { td::BigInt256 dest_addr; dest_addr.import_bits((*args.address).addr.as_bitslice()); - cb.store_ones(1) - .store_zeroes(2) - .store_long((*args.address).workchain, 8) - .store_int256(dest_addr, 256); + cb.store_ones(1).store_zeroes(2).store_long((*args.address).workchain, 8).store_int256(dest_addr, 256); } auto address = cb.finalize(); auto config = td::Ref(); @@ -79,26 +148,51 @@ td::Ref prepare_vm_c7(SmartContract::Args args) { config = (*args.config)->get_root_cell(); } - auto tuple = vm::make_tuple_ref( - td::make_refint(0x076ef1ea), // [ magic:0x076ef1ea - td::make_refint(0), // actions:Integer - td::make_refint(0), // msgs_sent:Integer - td::make_refint(now), // unixtime:Integer - td::make_refint(0), //TODO: // block_lt:Integer - td::make_refint(0), //TODO: // trans_lt:Integer - std::move(rand_seed_int), // rand_seed:Integer - block::CurrencyCollection(args.balance).as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)] - vm::load_cell_slice_ref(address), // myself:MsgAddressInt - vm::StackEntry::maybe(config) //vm::StackEntry::maybe(td::Ref()) - ); // global_config:(Maybe Cell) ] = SmartContractInfo; + std::vector tuple = { + td::make_refint(0x076ef1ea), // [ magic:0x076ef1ea + td::make_refint(0), // actions:Integer + td::make_refint(0), // msgs_sent:Integer + td::make_refint(now), // unixtime:Integer + td::make_refint(0), // block_lt:Integer (TODO) + td::make_refint(0), // trans_lt:Integer (TODO) + std::move(rand_seed_int), // rand_seed:Integer + block::CurrencyCollection(args.balance, args.extra_currencies) + .as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)] + vm::load_cell_slice_ref(address), // myself:MsgAddressInt + vm::StackEntry::maybe(config) // vm::StackEntry::maybe(td::Ref()) + }; + if (args.config && args.config.value()->get_global_version() >= 4) { + tuple.push_back(vm::StackEntry::maybe(code)); // code:Cell + tuple.push_back(block::CurrencyCollection::zero().as_vm_tuple()); // in_msg_value:[Integer (Maybe Cell)] + tuple.push_back(td::zero_refint()); // storage_fees:Integer + + // See crypto/block/mc-config.cpp#2115 (get_prev_blocks_info) + // [ wc:Integer shard:Integer seqno:Integer root_hash:Integer file_hash:Integer] = BlockId; + // [ last_mc_blocks:[BlockId...] + // prev_key_block:BlockId ] : PrevBlocksInfo + tuple.push_back(args.prev_blocks_info ? args.prev_blocks_info.value() : vm::StackEntry{}); // prev_block_info + } + if (args.config && args.config.value()->get_global_version() >= 6) { + tuple.push_back(args.config.value()->get_unpacked_config_tuple(now)); // unpacked_config_tuple + tuple.push_back(td::zero_refint()); // due_payment + // precomiled_gas_usage:(Maybe Integer) + td::optional precompiled; + if (code.not_null()) { + precompiled = args.config.value()->get_precompiled_contracts_config().get_contract(code->get_hash().bits()); + } + tuple.push_back(precompiled ? td::make_refint(precompiled.value().gas_usage) : vm::StackEntry()); + } + auto tuple_ref = td::make_cnt_ref>(std::move(tuple)); //LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple).to_string(); - return vm::make_tuple_ref(std::move(tuple)); + return vm::make_tuple_ref(std::move(tuple_ref)); } SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref stack, td::Ref c7, - vm::GasLimits gas, bool ignore_chksig, td::Ref libraries) { + vm::GasLimits gas, bool ignore_chksig, td::Ref libraries, + int vm_log_verbosity, bool debug_enabled, + std::shared_ptr config) { auto gas_credit = gas.gas_credit; - vm::init_op_cp0(); + vm::init_vm(debug_enabled).ensure(); vm::DictionaryBase::get_empty_dictionary(); class Logger : public td::LogInterface { @@ -109,15 +203,18 @@ SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref= VERBOSITY_NAME(DEBUG)) { - log.log_options.level = 4; - log.log_options.fix_newlines = true; - log.log_mask |= vm::VmLog::DumpStack; - } else { - log.log_options.level = 0; - log.log_mask = 0; + vm::VmLog log{&logger, td::LogOptions(VERBOSITY_NAME(DEBUG), true, false)}; + if (vm_log_verbosity > 1) { + log.log_mask |= vm::VmLog::ExecLocation; + if (vm_log_verbosity > 2) { + log.log_mask |= vm::VmLog::GasRemaining; + if (vm_log_verbosity > 3) { + log.log_mask |= vm::VmLog::DumpStack; + if (vm_log_verbosity > 4) { + log.log_mask |= vm::VmLog::DumpStackVerbose; + } + } + } } SmartContract::Answer res; @@ -126,24 +223,31 @@ SmartContract::Answer run_smartcont(SmartContract::State state, td::Refdump(os, 2); LOG(DEBUG) << "VM stack:\n" << os.str(); } - vm::VmState vm{state.code, std::move(stack), gas, 1, state.data, log}; + int global_version = config ? config->get_global_version() : 0; + vm::VmState vm{state.code, global_version, std::move(stack), gas, 1, state.data, log}; vm.set_c7(std::move(c7)); vm.set_chksig_always_succeed(ignore_chksig); if (!libraries.is_null()) { vm.register_library_collection(libraries); } + if (config) { + auto r_limits = config->get_size_limits_config(); + if (r_limits.is_ok()) { + vm.set_max_data_depth(r_limits.ok().max_vm_data_depth); + } + } try { res.code = ~vm.run(); } catch (...) { LOG(FATAL) << "catch unhandled exception"; } - td::ConstBitPtr mlib = vm.get_missing_library(); res.new_state = std::move(state); res.stack = vm.get_stack_ref(); gas = vm.get_gas_limits(); res.gas_used = gas.gas_consumed(); res.accepted = gas.gas_credit == 0; - res.success = (res.accepted && (unsigned)res.code <= 1); + res.success = (res.accepted && vm.committed()); + res.vm_log = logger.res; if (GET_VERBOSITY_LEVEL() >= VERBOSITY_NAME(DEBUG)) { LOG(DEBUG) << "VM log\n" << logger.res; std::ostringstream os; @@ -153,9 +257,10 @@ SmartContract::Answer run_smartcont(SmartContract::State state, td::Ref SmartContract::get_init_state() const { SmartContract::Answer SmartContract::run_method(Args args) { if (!args.c7) { - args.c7 = prepare_vm_c7(args); + args.c7 = prepare_vm_c7(args, state_.code); } if (!args.limits) { bool is_internal = args.get_method_id().ok() == 0; @@ -219,14 +324,15 @@ SmartContract::Answer SmartContract::run_method(Args args) { args.stack.value().write().push_smallint(args.method_id.unwrap()); auto res = run_smartcont(get_state(), args.stack.unwrap(), args.c7.unwrap(), args.limits.unwrap(), args.ignore_chksig, - args.libraries ? args.libraries.unwrap().get_root_cell() : td::Ref{}); + args.libraries ? args.libraries.unwrap().get_root_cell() : td::Ref{}, + args.vm_log_verbosity_level, args.debug_enabled, args.config ? args.config.value() : nullptr); state_ = res.new_state; return res; } SmartContract::Answer SmartContract::run_get_method(Args args) const { if (!args.c7) { - args.c7 = prepare_vm_c7(args); + args.c7 = prepare_vm_c7(args, state_.code); } if (!args.limits) { args.limits = vm::GasLimits{1000000, 1000000}; @@ -237,7 +343,8 @@ SmartContract::Answer SmartContract::run_get_method(Args args) const { CHECK(args.method_id); args.stack.value().write().push_smallint(args.method_id.unwrap()); return run_smartcont(get_state(), args.stack.unwrap(), args.c7.unwrap(), args.limits.unwrap(), args.ignore_chksig, - args.libraries ? args.libraries.unwrap().get_root_cell() : td::Ref{}); + args.libraries ? args.libraries.unwrap().get_root_cell() : td::Ref{}, + args.vm_log_verbosity_level, args.debug_enabled, args.config ? args.config.value() : nullptr); } SmartContract::Answer SmartContract::run_get_method(td::Slice method, Args args) const { @@ -246,10 +353,10 @@ SmartContract::Answer SmartContract::run_get_method(td::Slice method, Args args) SmartContract::Answer SmartContract::send_external_message(td::Ref cell, Args args) { return run_method( - args.set_stack(prepare_vm_stack(td::make_refint(0), vm::load_cell_slice_ref(cell))).set_method_id(-1)); + args.set_stack(prepare_vm_stack(td::make_refint(0), vm::load_cell_slice_ref(cell), args, -1)).set_method_id(-1)); } SmartContract::Answer SmartContract::send_internal_message(td::Ref cell, Args args) { return run_method( - args.set_stack(prepare_vm_stack(td::make_refint(args.amount), vm::load_cell_slice_ref(cell))).set_method_id(0)); + args.set_stack(prepare_vm_stack(td::make_refint(args.amount), vm::load_cell_slice_ref(cell), args, 0)).set_method_id(0)); } } // namespace ton diff --git a/crypto/smc-envelope/SmartContract.h b/crypto/smc-envelope/SmartContract.h index 95dd2e84..49edb969 100644 --- a/crypto/smc-envelope/SmartContract.h +++ b/crypto/smc-envelope/SmartContract.h @@ -49,7 +49,8 @@ class SmartContract : public td::CntObject { td::Ref actions; td::int32 code; td::int64 gas_used; - td::ConstBitPtr missing_library{0}; + td::optional missing_library; + std::string vm_log; static int output_actions_count(td::Ref list); }; @@ -59,13 +60,18 @@ class SmartContract : public td::CntObject { td::optional> c7; td::optional> stack; td::optional now; + td::optional> rand_seed; bool ignore_chksig{false}; td::uint64 amount{0}; td::uint64 balance{0}; + td::Ref extra_currencies; + int vm_log_verbosity_level{0}; + bool debug_enabled{false}; td::optional address; td::optional> config; td::optional libraries; + td::optional> prev_blocks_info; Args() { } @@ -100,6 +106,10 @@ class SmartContract : public td::CntObject { this->stack = std::move(stack); return std::move(*this); } + Args&& set_rand_seed(td::BitArray<256> rand_seed) { + this->rand_seed = std::move(rand_seed); + return std::move(*this); + } Args&& set_ignore_chksig(bool ignore_chksig) { this->ignore_chksig = ignore_chksig; return std::move(*this); @@ -112,11 +122,15 @@ class SmartContract : public td::CntObject { this->balance = balance; return std::move(*this); } + Args&& set_extra_currencies(td::Ref extra_currencies) { + this->extra_currencies = std::move(extra_currencies); + return std::move(*this); + } Args&& set_address(block::StdAddress address) { this->address = address; return std::move(*this); } - Args&& set_config(std::shared_ptr& config) { + Args&& set_config(const std::shared_ptr& config) { this->config = config; return std::move(*this); } @@ -124,6 +138,22 @@ class SmartContract : public td::CntObject { this->libraries = libraries; return std::move(*this); } + Args&& set_prev_blocks_info(td::Ref tuple) { + if (tuple.is_null()) { + this->prev_blocks_info = {}; + } else { + this->prev_blocks_info = std::move(tuple); + } + return std::move(*this); + } + Args&& set_vm_verbosity_level(int vm_log_verbosity_level) { + this->vm_log_verbosity_level = vm_log_verbosity_level; + return std::move(*this); + } + Args&& set_debug_enabled(bool debug_enabled) { + this->debug_enabled = debug_enabled; + return std::move(*this); + } td::Result get_method_id() const { if (!method_id) { diff --git a/crypto/smc-envelope/SmartContractCode.cpp b/crypto/smc-envelope/SmartContractCode.cpp index d10c4b5c..585450f6 100644 --- a/crypto/smc-envelope/SmartContractCode.cpp +++ b/crypto/smc-envelope/SmartContractCode.cpp @@ -28,6 +28,7 @@ namespace { // WALLET_REVISION = 2; // WALLET2_REVISION = 2; // WALLET3_REVISION = 2; +// WALLET4_REVISION = 2; // HIGHLOAD_WALLET_REVISION = 2; // HIGHLOAD_WALLET2_REVISION = 2; // DNS_REVISION = 1; @@ -92,6 +93,20 @@ const auto& get_map() { "AAXrc52omhpn5jrhf/AABesePaiaGmPmOuFj8ABDbbYHwR7Z5AOAQm1B1tnkA4BTu1E0IEBQNch0x/" "0BNEC2zz4J28QAoAg9HtvpTGX+gAwoXC2CZEw4g8AOiGOETGA8/gzIG6SMHCU0NcLH+IB3yGSAaGSW3/iAAzTB9QC+wAAHssfFMsfEsv/yx/" "0AMntVA=="); + with_tvm_code( + "wallet-v4-r2", + "te6cckECFAEAAtQAART/APSkE/S88sgLAQIBIAIDAgFIBAUE+PKDCNcYINMf0x/THwL4I7vyZO1E0NMf0x/T//" + "QE0VFDuvKhUVG68qIF+QFUEGT5EPKj+AAkpMjLH1JAyx9SMMv/" + "UhD0AMntVPgPAdMHIcAAn2xRkyDXSpbTB9QC+wDoMOAhwAHjACHAAuMAAcADkTDjDQOkyMsfEssfy/" + "8QERITAubQAdDTAyFxsJJfBOAi10nBIJJfBOAC0x8hghBwbHVnvSKCEGRzdHK9sJJfBeAD+kAwIPpEAcjKB8v/" + "ydDtRNCBAUDXIfQEMFyBAQj0Cm+hMbOSXwfgBdM/" + "yCWCEHBsdWe6kjgw4w0DghBkc3RyupJfBuMNBgcCASAICQB4AfoA9AQw+CdvIjBQCqEhvvLgUIIQcGx1Z4MesXCAGFAEywUmzxZY+" + "gIZ9ADLaRfLH1Jgyz8gyYBA+wAGAIpQBIEBCPRZMO1E0IEBQNcgyAHPFvQAye1UAXKwjiOCEGRzdHKDHrFwgBhQBcsFUAPPFiP6AhPLassfyz/" + "JgED7AJJfA+ICASAKCwBZvSQrb2omhAgKBrkPoCGEcNQICEekk30pkQzmkD6f+YN4EoAbeBAUiYcVnzGEAgFYDA0AEbjJftRNDXCx+" + "AA9sp37UTQgQFA1yH0BDACyMoHy//J0AGBAQj0Cm+hMYAIBIA4PABmtznaiaEAga5Drhf/AABmvHfaiaEAQa5DrhY/AAG7SB/" + "oA1NQi+QAFyMoHFcv/ydB3dIAYyMsFywIizxZQBfoCFMtrEszMyXP7AMhAFIEBCPRR8qcCAHCBAQjXGPoA0z/" + "IVCBHgQEI9FHyp4IQbm90ZXB0gBjIywXLAlAGzxZQBPoCFMtqEssfyz/Jc/sAAgBsgQEI1xj6ANM/" + "MFIkgQEI9Fnyp4IQZHN0cnB0gBjIywXLAlAFzxZQA/oCE8tqyx8Syz/Jc/sAAAr0AMntVGliJeU="); return map; }(); return map; @@ -137,9 +152,12 @@ td::Span SmartContractCode::get_revisions(Type type) { static int res[] = {1}; return res; } + case Type::WalletV4: { + static int res[] = {2}; + return res; + } } UNREACHABLE(); - return {}; } td::Result SmartContractCode::validate_revision(Type type, int revision) { @@ -179,9 +197,10 @@ td::Ref SmartContractCode::get_code(Type type, int ext_revision) { return "payment-channel"; case Type::RestrictedWallet: return "restricted-wallet3"; + case Type::WalletV4: + return "wallet-v4"; } UNREACHABLE(); - return ""; }(type); if (revision == -1) { return load(basename).move_as_ok(); diff --git a/crypto/smc-envelope/SmartContractCode.h b/crypto/smc-envelope/SmartContractCode.h index 85be3531..be50d2a1 100644 --- a/crypto/smc-envelope/SmartContractCode.h +++ b/crypto/smc-envelope/SmartContractCode.h @@ -26,7 +26,16 @@ class SmartContractCode { public: static td::Result> load(td::Slice name); - enum Type { WalletV3 = 4, HighloadWalletV1, HighloadWalletV2, ManualDns, Multisig, PaymentChannel, RestrictedWallet }; + enum Type { + WalletV3 = 4, + HighloadWalletV1, + HighloadWalletV2, + ManualDns, + Multisig, + PaymentChannel, + RestrictedWallet, + WalletV4 + }; static td::Span get_revisions(Type type); static td::Result validate_revision(Type type, int revision); static td::Ref get_code(Type type, int revision = 0); diff --git a/crypto/smc-envelope/WalletInterface.cpp b/crypto/smc-envelope/WalletInterface.cpp index 4c2feca9..eecec435 100644 --- a/crypto/smc-envelope/WalletInterface.cpp +++ b/crypto/smc-envelope/WalletInterface.cpp @@ -48,27 +48,30 @@ td::Result> WalletInterface::get_init_message(const td::Ed2551 td::Ref WalletInterface::create_int_message(const Gift &gift) { vm::CellBuilder cbi; - GenericAccount::store_int_message(cbi, gift.destination, gift.gramms < 0 ? 0 : gift.gramms); + GenericAccount::store_int_message(cbi, gift.destination, gift.gramms < 0 ? 0 : gift.gramms, gift.extra_currencies); if (gift.init_state.not_null()) { cbi.store_ones(2); cbi.store_ref(gift.init_state); } else { cbi.store_zeroes(1); } - cbi.store_zeroes(1); store_gift_message(cbi, gift); return cbi.finalize(); } void WalletInterface::store_gift_message(vm::CellBuilder &cb, const Gift &gift) { if (gift.body.not_null()) { auto body = vm::load_cell_slice(gift.body); - //TODO: handle error - CHECK(cb.append_cellslice_bool(body)); + if (cb.can_extend_by(1 + body.size(), body.size_refs())) { + CHECK(cb.store_zeroes_bool(1) && cb.append_cellslice_bool(body)); + } else { + CHECK(cb.store_ones_bool(1) && cb.store_ref_bool(gift.body)); + } return; } + cb.store_zeroes(1); if (gift.is_encrypted) { - cb.store_long(1, 32); + cb.store_long(EncryptedCommentOp, 32); } else { cb.store_long(0, 32); } diff --git a/crypto/smc-envelope/WalletInterface.h b/crypto/smc-envelope/WalletInterface.h index 20141f5c..e88ca0a6 100644 --- a/crypto/smc-envelope/WalletInterface.h +++ b/crypto/smc-envelope/WalletInterface.h @@ -33,9 +33,11 @@ namespace ton { class WalletInterface : public SmartContract { public: + static constexpr uint32_t EncryptedCommentOp = 0x2167da4b; struct Gift { block::StdAddress destination; td::int64 gramms; + td::Ref extra_currencies; td::int32 send_mode{-1}; bool is_encrypted{false}; @@ -73,6 +75,8 @@ class WalletInterface : public SmartContract { td::uint32 valid_until = std::numeric_limits::max()) const; static td::Ref create_int_message(const Gift &gift); + + private: static void store_gift_message(vm::CellBuilder &cb, const Gift &gift); }; diff --git a/crypto/smc-envelope/WalletV4.cpp b/crypto/smc-envelope/WalletV4.cpp new file mode 100644 index 00000000..738fa9c7 --- /dev/null +++ b/crypto/smc-envelope/WalletV4.cpp @@ -0,0 +1,71 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "WalletV4.h" +#include "GenericAccount.h" +#include "SmartContractCode.h" + +#include "vm/boc.h" +#include "vm/cells/CellString.h" +#include "td/utils/base64.h" + +#include + +namespace ton { +td::Result> WalletV4::make_a_gift_message(const td::Ed25519::PrivateKey& private_key, + td::uint32 valid_until, td::Span gifts) const { + CHECK(gifts.size() <= get_max_gifts_size()); + TRY_RESULT(seqno, get_seqno()); + TRY_RESULT(wallet_id, get_wallet_id()); + vm::CellBuilder cb; + cb.store_long(wallet_id, 32).store_long(valid_until, 32).store_long(seqno, 32); + cb.store_long(0, 8); // The only difference with wallet-v3 + + for (auto& gift : gifts) { + td::int32 send_mode = 3; + if (gift.gramms == -1) { + send_mode += 128; + } + if (gift.send_mode > -1) { + send_mode = gift.send_mode; + } + cb.store_long(send_mode, 8).store_ref(create_int_message(gift)); + } + + auto message_outer = cb.finalize(); + auto signature = private_key.sign(message_outer->get_hash().as_slice()).move_as_ok(); + return vm::CellBuilder().store_bytes(signature).append_cellslice(vm::load_cell_slice(message_outer)).finalize(); +} + +td::Ref WalletV4::get_init_data(const InitData& init_data) noexcept { + return vm::CellBuilder() + .store_long(init_data.seqno, 32) + .store_long(init_data.wallet_id, 32) + .store_bytes(init_data.public_key) + .store_zeroes(1) // plugins dict + .finalize(); +} + +td::Result WalletV4::get_wallet_id() const { + return TRY_VM([&]() -> td::Result { + auto answer = run_get_method("get_subwallet_id"); + if (!answer.success) { + return td::Status::Error("get_subwallet_id get method failed"); + } + return static_cast(answer.stack.write().pop_long_range(std::numeric_limits::max())); + }()); +} +} // namespace ton diff --git a/tonlib/tonlib/TestWallet.h b/crypto/smc-envelope/WalletV4.h similarity index 53% rename from tonlib/tonlib/TestWallet.h rename to crypto/smc-envelope/WalletV4.h index ef726b55..721e8103 100644 --- a/tonlib/tonlib/TestWallet.h +++ b/crypto/smc-envelope/WalletV4.h @@ -13,28 +13,34 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . - - Copyright 2017-2019 Telegram Systems LLP */ #pragma once +#include "smc-envelope/SmartContract.h" +#include "smc-envelope/WalletInterface.h" #include "vm/cells.h" #include "Ed25519.h" #include "block/block.h" -#include "CellString.h" +#include "vm/cells/CellString.h" + +namespace ton { + +struct WalletV4Traits { + using InitData = WalletInterface::DefaultInitData; -namespace tonlib { -class TestWallet { - public: static constexpr unsigned max_message_size = vm::CellString::max_bytes; - static td::Ref get_init_state(const td::Ed25519::PublicKey& public_key) noexcept; - static td::Ref get_init_message(const td::Ed25519::PrivateKey& private_key) noexcept; - static td::Ref make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 seqno, - td::int64 gramms, td::Slice message, - const block::StdAddress& dest_address) noexcept; - - static td::Ref get_init_code() noexcept; - static vm::CellHash get_init_code_hash() noexcept; - static td::Ref get_init_data(const td::Ed25519::PublicKey& public_key) noexcept; + static constexpr unsigned max_gifts_size = 4; + static constexpr auto code_type = SmartContractCode::WalletV4; }; -} // namespace tonlib + +class WalletV4 : public WalletBase { + public: + explicit WalletV4(State state) : WalletBase(std::move(state)) { + } + td::Result> make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 valid_until, + td::Span gifts) const override; + static td::Ref get_init_data(const InitData& init_data) noexcept; + + td::Result get_wallet_id() const override; +}; +} // namespace ton \ No newline at end of file diff --git a/crypto/test/Ed25519.cpp b/crypto/test/Ed25519.cpp index 45e8891f..131bfe92 100644 --- a/crypto/test/Ed25519.cpp +++ b/crypto/test/Ed25519.cpp @@ -17,6 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "crypto/Ed25519.h" +#include "ellcurve/Ed25519.h" + #include "td/utils/logging.h" #include "td/utils/misc.h" #include "td/utils/Slice.h" @@ -24,6 +26,8 @@ #include "td/utils/JsonBuilder.h" #include "wycheproof.h" +#include "keys/keys.hpp" +#include "td/utils/benchmark.h" #include #include @@ -217,3 +221,36 @@ TEST(Crypto, almost_zero) { } } } + +BENCH(ed25519_sign, "ed25519_sign") { + auto private_key = td::Ed25519::generate_private_key().move_as_ok(); + std::string hash_to_sign(32, 'a'); + for (int i = 0; i < n; i++) { + private_key.sign(hash_to_sign).ensure(); + } +} + +BENCH(ed25519_shared_secret, "ed25519_shared_secret") { + auto private_key_a = td::Ed25519::generate_private_key().move_as_ok(); + auto private_key_b = td::Ed25519::generate_private_key().move_as_ok(); + auto public_key_b = private_key_a.get_public_key().move_as_ok(); + for (int i = 0; i < n; i++) { + td::Ed25519::compute_shared_secret(public_key_b, private_key_a).ensure(); + } +} + +BENCH(ed25519_verify, "ed25519_verify") { + auto private_key = td::Ed25519::generate_private_key().move_as_ok(); + std::string hash_to_sign(32, 'a'); + auto public_key = private_key.get_public_key().move_as_ok(); + auto signature = private_key.sign(hash_to_sign).move_as_ok(); + for (int i = 0; i < n; i++) { + public_key.verify_signature(hash_to_sign, signature).ensure(); + } +} + +TEST(Crypto, ed25519_benchmark) { + bench(ed25519_signBench()); + bench(ed25519_shared_secretBench()); + bench(ed25519_verifyBench()); +} \ No newline at end of file diff --git a/crypto/test/fift.cpp b/crypto/test/fift.cpp index 466ad4e7..9a92e0b1 100644 --- a/crypto/test/fift.cpp +++ b/crypto/test/fift.cpp @@ -33,9 +33,14 @@ std::string load_test(std::string name) { return td::read_file_str(current_dir() + "fift/" + name).move_as_ok(); } -td::Status run_fift(std::string name, bool preload_fift = true) { - TRY_RESULT(res, fift::mem_run_fift(load_test(name))); - REGRESSION_VERIFY(res.output); +td::Status run_fift(std::string name, bool expect_error = false) { + auto res = fift::mem_run_fift(load_test(name)); + if (expect_error) { + res.ensure_error(); + return td::Status::OK(); + } + res.ensure(); + REGRESSION_VERIFY(res.ok().output); return td::Status::OK(); } @@ -79,7 +84,7 @@ TEST(Fift, testvmprog) { run_fift("testvmprog.fif"); } TEST(Fift, bug) { - run_fift("bug.fif"); + run_fift("bug.fif", true); } TEST(Fift, contfrac) { run_fift("contfrac.fif"); @@ -110,3 +115,59 @@ TEST(Fift, test_sort) { TEST(Fift, test_sort2) { run_fift("sort2.fif"); } + +TEST(Fift, test_hmap) { + run_fift("hmap.fif"); +} + +TEST(Fift, test_disasm) { + run_fift("disasm.fif"); +} + +TEST(Fift, test_fiftext) { + run_fift("fift-ext.fif"); +} + +TEST(Fift, test_namespaces) { + run_fift("namespaces.fif"); +} + +TEST(Fift, test_asm_nested_program) { + run_fift("asm-nested-program.fif"); +} + +TEST(Fift, test_adddiv) { + run_fift("adddiv.fif"); +} + +TEST(Fift, test_tvm_runvm) { + run_fift("tvm_runvm.fif"); +} + +TEST(Fift, test_hash_ext) { + run_fift("hash_ext.fif"); +} + +TEST(Fift, test_deep_stack_ops) { + run_fift("deep_stack_ops.fif"); +} + +TEST(Fift, test_rist255) { + run_fift("rist255.fif"); +} + +TEST(Fift, test_bls) { + run_fift("bls.fif"); +} + +TEST(Fift, test_bls_ops) { + run_fift("bls_ops.fif"); +} + +TEST(Fift, test_levels) { + run_fift("levels.fif"); +} + +TEST(Fift, test_secp256k1) { + run_fift("secp256k1.fif"); +} diff --git a/crypto/test/fift/adddiv.fif b/crypto/test/fift/adddiv.fif new file mode 100644 index 00000000..96be6525 --- /dev/null +++ b/crypto/test/fift/adddiv.fif @@ -0,0 +1,144 @@ +{ + =: ans-r =: ans-q =: mode + =: z =: w =: y =: x + ."MULADDDIVMOD " @' x . @' y . @' w . @' z . @' mode . ."= " @' ans-q . @' ans-r . cr + @' x @' y @' w @' z + abort"Incorrect r" + @' ans-q <> abort"Incorrect q" +} : test-muladddivmod + +{ + =: ans-r =: ans-q =: mode + =: y =: w =: x + ."ADDDIVMOD " @' x . @' w . @' y . @' mode . ."= " @' ans-q . @' ans-r . cr + @' x @' w @' y + abort"Incorrect r" + @' ans-q <> abort"Incorrect q" +} : test-adddivmod + +{ + =: ans-r =: ans-q =: mode + =: z =: w =: y =: x + ."SHLADDDIVMOD " @' x . @' y . @' w . @' z . @' mode . ."= " @' ans-q . @' ans-r . cr + @' x @' w @' z @' y + abort"Incorrect r" + @' ans-q <> abort"Incorrect q" +} : test-shladddivmod + +{ + =: ans-r =: ans-q =: mode + =: z =: w =: y =: x + ."MULADDSHRMOD " @' x . @' y . @' w . @' z . @' mode . ."= " @' ans-q . @' ans-r . cr + @' x @' y @' w @' z + abort"Incorrect r" + @' ans-q <> abort"Incorrect q" +} : test-muladdshrmod + +729 -212 552 0 0 517 test-adddivmod +823 -139 -918 1 -1 -234 test-adddivmod +-470 977 47 2 11 -10 test-adddivmod +-5 -171 880 0 -1 704 test-adddivmod +605 699 -379 1 -3 167 test-adddivmod +982 -24 -267 2 -3 157 test-adddivmod +194 826 859 0 1 161 test-adddivmod +-553 33 -715 1 1 195 test-adddivmod +-423 -714 547 2 -2 -43 test-adddivmod +-806 266 637 0 -1 97 test-adddivmod +-487 863 90 1 4 16 test-adddivmod +444 659 232 2 5 -57 test-adddivmod +847 -700 -365 -798 0 743 -351 test-muladddivmod +494 -849 840 741 1 -565 99 test-muladddivmod +400 -324 -34 146 2 -887 -132 test-muladddivmod +-794 -276 -111 -353 0 -621 -180 test-muladddivmod +251 311 869 -582 1 -136 -222 test-muladddivmod +979 131 -24 -94 2 -1364 9 test-muladddivmod +772 67 -467 -873 0 -59 -250 test-muladddivmod +648 881 123 875 1 653 -364 test-muladddivmod +-972 -809 473 720 2 1093 -139 test-muladddivmod +-184 454 689 607 0 -137 312 test-muladddivmod +368 280 -998 253 1 403 83 test-muladddivmod +10 695 776 -318 2 -24 94 test-muladddivmod +-501 5 441 782 0 -20 49 test-shladddivmod +-872 3 878 162 1 -38 58 test-shladddivmod +-546 3 645 981 2 -3 -780 test-shladddivmod +-709 8 -83 -814 0 223 -65 test-shladddivmod +-836 5 792 40 1 -649 0 test-shladddivmod +910 7 -777 -383 2 -302 37 test-shladddivmod +128 4 447 -745 0 -4 -485 test-shladddivmod +121 5 668 888 1 5 100 test-shladddivmod +106 3 973 637 2 3 -90 test-shladddivmod +235 8 203 -411 0 -147 -54 test-shladddivmod +-89 1 221 634 1 0 43 test-shladddivmod +-212 5 178 -505 2 14 464 test-shladddivmod +-406 -624 -613 2 0 63182 3 test-muladdshrmod +-933 254 344 4 1 -14790 2 test-muladdshrmod +-25 -859 -817 10 2 21 -846 test-muladdshrmod +551 -734 795 2 0 -100910 1 test-muladdshrmod +891 -921 725 1 1 -409943 0 test-muladdshrmod +839 432 890 8 2 1420 -182 test-muladdshrmod +399 -199 715 8 0 -308 162 test-muladdshrmod +-436 68 662 3 1 -3623 -2 test-muladdshrmod +739 -560 833 10 2 -403 -335 test-muladdshrmod +207 690 945 6 0 2246 31 test-muladdshrmod +187 -437 -78 7 1 -639 -5 test-muladdshrmod +352 313 434 5 2 3457 -14 test-muladdshrmod +25527586395537789869570741399081402682256851060036756777914400562788085574414 -25113160904436633803354228324572450480033217030010944063278205598870671865288 -2256017777568044393040721989849804550406163786581604155567013427587080747425 0 -1 -1841592286466888326824208915340852348182529756555791440930818463669667038299 test-adddivmod +-32640246231411323106115073889225624429888478001529402430751550293198007894765 -37905632746827282747258884343685092209247121139851002852469398603088743361022 18753000550171967665955942657615296985943924316121913951069685581208211144632 1 -4 4466123222449264810449812397550471304640098123107250521057793428546093322741 test-adddivmod +-95549375212741069607980980863332591640631650412403896229315709698307081396373 106240530356563762252550465652293166037855203388135409179201250774369290711413 63027891874179850739525204890840444316414650119364284762072976150516424705738 2 1 -52336736730357158094955720101879869919191097143632771812187435074454215390698 test-adddivmod +51890526097631236478215460328003213527162540175278758108042092328342498521783 19183851048961131310064308531261717471288729454763519442604661504489273629484 37741726035775552009535280639145255965240492091225040562231509956705484553131 0 1 33332651110816815778744488220119675033210777538817236988415243876126287598136 test-adddivmod +94071222795948162024492074951491627130123473883831806178173544936543536671735 -79181277644495598779765208777007083667700785694819686730856530041854640922750 101162201634522410327363362416275476419529310370084833033537453927088908146625 1 0 14889945151452563244726866174484543462422688189012119447317014894688895748985 test-adddivmod +106849434942283600005926697698751951080751408318517871682753577855332568068854 -24045207490958235002714107597178464324559754040415520751984110097220649552695 74679027778960833059271215597415768445807741415021004423546293120031915119911 2 2 -66553828106596301115329841093258050135423828551939657916323118481951911723663 test-adddivmod +-111083603622141356205460042548478779196244111142279948844255415507465496179136 75885719295648796212066209648237285368659643103523469813991536525258102456646 17487288729375302809394363552184340106515810164188804948959885094558256876961 0 -3 17263981861633348434789257756311526491962962453809935816615776301467376908393 test-adddivmod +33525382765608498977714984645580937896464574678957613892455278004374765395791 -31203451453013366071092485074296252911543265223612191519873428434459391093162 8205361318951100784724737477089194897616580930791932431609075219761273143789 1 0 2321931312595132906622499571284684984921309455345422372581849569915374302629 test-adddivmod +-75999952514508567055051892339604431790575024284443576993806836353012574839934 60096525995012658905135417908615298741753181036991250015243181812480959008245 25156241733999421479698222967097698882671132665627779117308277826680836574301 2 0 -15903426519495908149916474430989133048821843247452326978563654540531615831689 test-adddivmod +-25810038415830521876908334245348683093953748145428320530928049623042163542952 13768870762468614895982523795103333718119339794093637483718383357794846902545 -23455630873759864343226329762160671724884569294252944494055101268304334927469 0 0 -12041167653361906980925810450245349375834408351334683047209666265247316640407 test-adddivmod +65295636229319469142829758722387755590067603179313024098599396192036031682932 -14508864012052681942310481748034005897814022127985455644382059031122725256639 101258243423658080118042170583591218535452530372858366205652259477702813508611 1 1 -50471471206391292917522893609237468843198949321530797751434922316789507082318 test-adddivmod +-9782778804673764937901196234199348720011483228792837975519798751976472498576 97723276666278794523978327321824380269179391405498018632142147588187660001006 -49342195947431856333198825767811410708861002865285674269877261690183407963360 2 -1 38598301914173173252878305319813620840306905311419506386745087146027779539070 test-adddivmod +69314796062889467182180663903322273169679119368960545268948720263471816924632 93795938161576335898249305761803240528541213785951419531969609595386829401314 -110346049538998400484346689828188917177083570083289016450597131164930681126587 81149113889296322255375738811288869158926250513403232273765830197055460701260 0 80117280566567035416625919712742621817262121093084845174084285957987697395276 42036691045037437094861959671408390901385540330966807308270651575596167392101 test-muladddivmod +-21971450167728355573106502799062118429867408154846289128369107182511643511483 45393555019045493054213057422940072801743824102220594624535868754509418047929 60641652588698494073802822494164156953515858330346793601084497228687965671868 65539824510463171124453489789915597533293103308691110446148835625201340366598 1 -15217651854984270515658273738098718647212153016787679186857576358263739302058 -29371413441460037863394670149687230435838766165616129771684695789862414338155 test-muladddivmod +73634008615213489758113683559933051942050608504790565710917645985405082665491 -80155625965090770138821073870266428949137773405175831429923552413761952441361 -38999294453130493506999209141746004309873017414282530140731810289885759492802 -107314816556101242244916895079818037889329087681097150645968648449136338086659 2 54998743344874712983373052080844956119323047070936199528619384334263523872368 13082689556615721830015990667012489975468766353417550442844496303849124272459 test-muladddivmod +68441985454806782074858763992903374541757416422223520705458179994903492447753 -61267036145362164840470993649039311748277914273628051012777836672446118682406 88886725386626308975645991554561163959863903429407091061340939714760192334434 -55192455525382179382269200386360902483303866088141254649003462525396159498006 0 75974833096374772090455746107157391488128832802648627140447901963962285642644 -35281244576421605907379124849976697589234314365299818483123805214351816431420 test-muladddivmod +13991345100229597325861930387540671073560101799369013558248988792137436010794 32261366615236683296917985913895811707233003589900325435836147185374329094359 -35081305217709330815376010427375742966580511272499524644418734096933080609103 34617350357576952655827361801899194862477556493235499468927833026852355915741 1 13039123706936329824826187692520278079826111394035475962656812793878481550617 15501972367754182150585081456806290271752173495030969288120604381172009339746 test-muladddivmod +54599312873997957959553141264956574941720752239162126863069720527738322459321 743784849758283770307843473486531088082399957077059004634332280604041992918 -82452755014987554458459818028526388807440160648566275282022897198718871469325 38291925157759929750140152840087355208836284729075633154257868030142639468974 2 1060540611514863450709563259060420401132951143137987621813620016383521959845 -25576601802989288764264589884413515994814497538433618559906725414501597729677 test-muladddivmod +-33715442018098691670021742265266162391174212035380017502572244718418822715623 104887034421874269674272525533704680767916752281317133765280341780898862714955 40882837978738774675184345709913504579827188391303805465905655071457659357026 56761521466376613401789060160037629513193949115508135629309104361041554577464 0 -62301232175317956930653590924038385722281926697603233811485244969539670459488 12675521639977150682490928392886170001456429479565150104484728889568554893493 test-muladddivmod +-109706830967107194245468623157302140481713559442721649770259230494532080435232 91091244832469326414116901981654442590329304825709696121493056753007506269108 25586919878947735367325917367337541421822321271842417480093738776553133015696 92030412800134708014057990497075561862656985932946512327644864855223778050176 1 -108587275612051383813143325045450855254642049763863834262763340362417287376196 14236417728543624555544633487823994857557292455110879135165049494799852613136 test-muladddivmod +-55349740795363582739414701155082958898639391946026926848599395086577482106561 5153084015148055450516513461735201914781593572521892141092888805185704024388 -65052157988798026904166625513790079385321361518951356077015576483034859919945 -102114306233907646655227677599549837347948442788318424043410967802211179284463 2 2793162633665004752332533372431053166813934025727573730237936408503456830715 70263858264069476764017250048942592796103230166090146586104085945931001951432 test-muladddivmod +-39413219476407785933419341321985162122574957075787517059529471719124009278433 -11397560582871643424486223364245599528148517163558642929375020427946855266926 72776685568498650781344357948907544676236244067805626443550533574512635929751 58457602475162312341105487265860863804074002837155658624106642883511552164475 0 7684450571493046035065075993797738633610800975055222799851151005079197271998 25447845742266018765952184307141986304437190746678223193451015171399298065659 test-muladddivmod +28902743859707224532426872535592986589584185136999529704427764834614586536146 30118285526375438691436500554083725474584606124188443987423181314058619658259 -112947693515558587819874917596387701093051538659457576158697245759111287251112 -20731780512952622217494754630018335482763668682808767673633845662171371751908 1 -41988727958917592867685340883133510770733630298849576855386410060863664014757 6354934151364300917311007564393521634968794138377927567288802927545928772346 test-muladddivmod +25071799008788838242365403068059094326298333378650328613993357620568407227394 -27895932377486960662691379394722126815225783254727720878708826335618109428175 -34783759350459778495483474263506062490621190762057336926096642777973708891199 -46562500545170684944465306233470472124382229989733955650634447552911654749066 2 15020696945874349483297599474537626462120625640258473403957682700588115661859 6123529058588613987393724235023234231421775050193606259696039938834907756545 test-muladddivmod +4637334961015630956409530377119128659169399268652032534941609159910059786532 174 -12797497544848087213503981396389428116533825336325463794725594822879690679136 93018556588730187003792638630891717650672350374200759319903911998812002342061 0 1193763005785115313715849486131901382204829468191839 86009298512437248420661887032254663876544001080814196765583913598468234097773 test-shladddivmod +63737576485202419511398757422488387344373087691300709744039454443359600742212 17 4434935617055335860270673726237887676208015615002198868506400573861444556411 2395017004775767547761160570739886212190100215618999647636254869255450984262 1 3488166 -825476601398469938219841341785340947550825081895127156723070705988041475817 test-shladddivmod +-21264162232365890547240808872203618534857207092355857230053229357857057625546 79 83558436280879393517752968647441646810997015153366115449062116401734553793990 -19667935035506461778339921177556508690003173951785474262950610395510252887673 2 653520430812207296424074 11997579569142787892139960093229284888868038054542931791709721193377458047744 test-shladddivmod +50327589756875372367121062573124726147569135695110368165104387616199145495643 159 -56582804136162261915738821952650758787670281453734279574878610404888378091325 41794171917267054297819695791653745099726306264896745093211576006595621865899 0 879953489426631344181134857312933722690648995542 7479251035838043422275284357505479089411783830647758441772426265279908213201 test-shladddivmod +-6790346796221361248085433832482913268295667476610056123099942839800062619360 99 -30655005682505720678929136056715874580886440550632733046480550204329894400037 107280989788626394506897862107199815360330755655673306259970134464147378777352 1 -40117951973353319220033820651 15419963034230519057842831193590832882628829971176441336761525930488437456435 test-shladddivmod +19046177038646178344769825269053799430308794840550024608332777213474587282651 33 3489319764096497921939317127917520109408688444737580375038679679318170964558 32230702261963164880326384192551720153568540700317784363615498612027586166919 2 5076073543 -31149228513135397726284948367312236333921327766221169520773046236565040396067 test-shladddivmod +-110976730374012300153801377038320854184737548404302035494280064993634691387718 177 62599745911697906087525399889373248851994389233059198427444302208529620264342 2939901734097150061182352721440114362417026502565316012346244994332394908737 0 -7231166201303287776306953132498157445481985675557698778 861631401006029279966827198914714166628754059441816200263218068486981128432 test-shladddivmod +31852390609689123349381354058350609996064615917952716537339963695989162024629 72 -96928192014181974842953135946922776102933173463512906699090906721752057028243 92656901870085089608820151790239083253300815942245860126341275046871599043017 1 1623394035183378246274 -40717467475217300542569194922405492138512784779596821986363220696851482878117 test-shladddivmod +108325270025037161417115474167849009240382377052459846115851642255437288685912 79 -47995506721499545563379889382854078032364619862028413664240909132079634758006 -6166659869789359796343211847537623762770094929722294841335533066616771405539 2 -10618164339787666672859802 2975209691732235673066443530343246725711182217991112340969803628522007294972 test-shladddivmod +-102030576317847732070693755030275793160072651528715762716678938494538431797856 162 92259248995288252713115460931891173987977282785918620735892298001910599707421 93069691103831234597581468228357432179178680731051507965844021657079407259554 0 -6408868561946334361312419431284710070541388837754 30090871674656727061174631170248864679275378922263881716308993939778213471313 test-shladddivmod +109919709354382523017092140618288318375226272062645130510340454988137017880658 32 11403889262848182921779993946145880893942415285082669079885795084154364295755 -35203479084324376157414643218816253627488669490337878341936871887579427112193 1 -13410650571 -9079152369425295070217928786983170793628233088577531145426537161436231255680 test-shladddivmod +32604320264365535336905506430047200856461747148561589244458238442999859570507 87 53600018835628474925583604580250089573381703851509998750334950185805444447855 -13714304553912094478086374486990066534941196173664347737943079140839510240 2 -367884070882655630785891877843 6730574288892465511219887918570728429135366732586091870751580223301793231 test-shladddivmod +-6528863440223970638249234714151581865561638514865295384371332680662061970029 -36913540142944821177073496030226577063991271774591429797367434456256316959902 24604605953723174243828240349210337384146509969508816161702610762299708087414 252 0 33301544418230523796798136129325191903276669555089137976430642606666380785097 1111119448521463468919386205636226935698167866010067725789944817681239062460 test-muladdshrmod +-84914121307006819861534520354024886732959304976576667909765018675779753895095 -33734797430686302202051663614563177864696792935288850099657207932669766616337 -102660654939881416004941779328139233368526378948134839417440263136258000263276 254 1 98955315519894417210680117251614014274294848178227094975298724352068617948436 11185753669896509258573194177216136324346588911755615485008785003988287318715 test-muladdshrmod +-4756805439354914657157062146720760601779363897648559488036119230046473414258 -25205069919224711211607424233708613506619570034044466450241534811277675992336 65841615838405859557610720340525261800430556950418299394523533586361728325556 253 2 8283509830821873572245889567787981786085701113981069889850529960221182899414 -8998763596547876718672658010959335659555767305449508909489610444294401222444 test-muladdshrmod +783982375644419567722751049236700086045042736024464491262223120876229442273 5175287637281731481295785519248883235846260036385254885568602894849814368811 -55298781556612762404255558682268367199188235546843731267393108383385576323488 255 0 70079645738212902632136650950755273036821422652512833810215828489102796342 57092783249763565645949150057908892679575599568561392603234275558727880866859 test-muladdshrmod +4070111884734287416608979312653292787122202104172152519474989991166732734990 34257747897608142079436828950758942168797726684869935923750014522979867949066 -61460807674782162344337010795962912420813084141671620888972984019328919535019 255 1 2408331480685488199417578349126812242697171581435356157803139449785540375610 -22381063802952741956988549732332921247382434293458568395759850191487471696159 test-muladdshrmod +46624156993487206784214341981008205045812193232799951047364876541247557155655 -63729769095463769066210525784474878773448422698647046890735133366216202823024 87326180659880540224448931705993651721770461620305276267209594753354088752298 254 2 -102644205801514086102164756532696646475956724447744150113359070359995022772818 -1428092989561848018204511966860198528552911262896258786227284617693530033510 test-muladdshrmod +39272829609755756484322638781158254787641099751271808569134728682168699365109 -11052442053174551837010800294891429615750956891563661801761360982769602119895 -283548082261175812447439483712866143310217576711284967538745642449188994625 253 0 -29988969117668461072095834210490978680262152778767346814972545689246419413980 2174529687955394421606608137904112203633356592266557030912275482323503849980 test-muladdshrmod +-6634646688578172858373436607694752472569893432402323211016655704885763912303 -50615746987061510852137844454230120888792101396398105566610972957006661885326 39885934214260485879330356646246293765018634098614146427190880259955614376279 255 1 5800354762566883489975672198380697469946049260633808174115670550957411425502 -28109144677804577765510965536534358513109159026887377398047363431999553081879 test-muladdshrmod +36957305118458084509865949238755863689212814576372103672502961803454095738285 -17426859900126562814159340928393951526114707618474312299911007064028697482925 61889497203091392721847485067819343116261217894188379032096092268147391639006 252 2 -88993959131783882197769064733391828196442843554551869076190294411733289296212 -933414186558473122333257942263787394321322401897335201713504555165014099467 test-muladdshrmod +72799360004808200591693183601526367934146450159257557514619233204077568035423 -43259113456489928181943264810324365449524922073847088046666461901052175252848 -20928308093701346202205365707157304905304946151034389446852806842785919722024 255 0 -54394661928130271541114188703398383016925401302362215366329066394036243394378 29938721175889548572374867166785136077053268439086595306734647725317227983176 test-muladdshrmod +92302382053431541957491748846080777012291600699830575887072283670295321137177 69013668157296288202299005784733240169135155398176346313070175201757132886117 89824190356716673951870631561712249743053597592009107233443818823508120467661 255 1 110026963104671419264446752630572096381373731021551359794924308423043643793015 -23908158790588303139174012402649398780932740824917520006353845804429534584150 test-muladdshrmod +19094862392572944474897897922146461521059803020953497046241755599584285859217 -9623105143646767554719516123478862168481718581670270810378662760058752527692 34811921737822512587681133088401450937187249507755006187477956368554274579224 255 2 -3173824217483389220453883003236448477412384215802980550708734379368313500289 -51434014928851488423168753814799098916141626164449965549730115348405330387188 test-muladdshrmod \ No newline at end of file diff --git a/crypto/test/fift/asm-nested-program.fif b/crypto/test/fift/asm-nested-program.fif new file mode 100644 index 00000000..ef604d72 --- /dev/null +++ b/crypto/test/fift/asm-nested-program.fif @@ -0,0 +1,93 @@ +"Asm.fif" include + +// Four programs: +// 4 contains 2 and 3, 2 contains 1 + +// Program #1 +PROGRAM{ + DECLPROC foo1 + DECLPROC foo2 + DECLPROC foo3 + DECLPROC main + foo1 PROC:<{ MUL INC }> + foo2 PROCINLINE:<{ PLDREF }> + foo3 PROC:<{ CTOS foo2 INLINECALLDICT CTOS 32 PLDU }> + main PROC:<{ 0 PUSHINT }> +}END>c constant code-1 + +// Program #2 +PROGRAM{ + DECLPROC foo3 + DECLPROC foo4 + DECLPROC main + foo3 PROC:<{ code-1 PUSHREF }> + foo4 PROC:<{ CTOS 8 PLDU }> + main PROC:<{ foo3 CALLDICT foo4 CALLDICT NEWC ROT STUX }> +}END>c constant code-2 + +// Program #3 +PROGRAM{ + DECLPROC foo1 + DECLPROC foo4 + DECLPROC main + foo1 PROC:<{ DUP 137 PUSHINT MUL PAIR }> + foo4 PROC:<{ UNPAIR SWAP DIV }> + main PROC:<{ 70 PUSHINT DIV }> +}END>c constant code-3 + +// Program #4 +PROGRAM{ + DECLPROC foo2 + DECLPROC foo3 + DECLPROC foo5 + DECLPROC main + foo2 PROC:<{ code-2 PUSHREF }> + foo3 PROC:<{ code-3 PUSHREF }> + foo5 PROC:<{ foo2 CALLDICT CTOS 8 PLDU 1 RSHIFT# }> + main PROC:<{ foo5 CALLDICT 5 MULCONST }> +}END>c + +.dump cr + +// Program #4, nested +PROGRAM{ + DECLPROC foo2 + DECLPROC foo3 + DECLPROC foo5 + DECLPROC main + foo2 PROC:<{ + PROGRAM{ + DECLPROC foo3 + DECLPROC foo4 + DECLPROC main + foo3 PROC:<{ + PROGRAM{ + DECLPROC foo1 + DECLPROC foo2 + DECLPROC foo3 + DECLPROC main + foo1 PROC:<{ MUL INC }> + foo2 PROCINLINE:<{ PLDREF }> + foo3 PROC:<{ CTOS foo2 INLINECALLDICT CTOS 32 PLDU }> + main PROC:<{ 0 PUSHINT }> + }END>c PUSHREF + }> + foo4 PROC:<{ CTOS 8 PLDU }> + main PROC:<{ foo3 CALLDICT foo4 CALLDICT NEWC ROT STUX }> + }END>c PUSHREF + }> + foo3 PROC:<{ + PROGRAM{ + DECLPROC foo1 + DECLPROC foo4 + DECLPROC main + foo1 PROC:<{ DUP 137 PUSHINT MUL PAIR }> + foo4 PROC:<{ UNPAIR SWAP DIV }> + main PROC:<{ 70 PUSHINT DIV }> + }END>c PUSHREF + }> + foo5 PROC:<{ foo2 CALLDICT CTOS 8 PLDU 1 RSHIFT# }> + main PROC:<{ foo5 CALLDICT 5 MULCONST }> +}END>c + +.dump cr \ No newline at end of file diff --git a/crypto/test/fift/bls.fif b/crypto/test/fift/bls.fif new file mode 100644 index 00000000..ca9a9d37 --- /dev/null +++ b/crypto/test/fift/bls.fif @@ -0,0 +1,954 @@ +// Based on https://github.com/ethereum/bls12-381-tests +"Asm.fif" include +"FiftExt.fif" include + +{ shash swap shash B= } : slices-eq? + +{ + =: expected-result + // ."Args: " .s + [[ <{ BLS_VERIFY }>s ]] 0 runvmx + abort"exitcode != 0" + ."Result: " dup . cr + ."Expected: " @' expected-result dup . cr cr + <> abort"wrong answer" +} : test-verify // pub msg signature exprected-result + +{ + =: expected-result + // ."Args: " .s + [[ <{ BLS_AGGREGATE }>s ]] 0 runvmx + { drop x{} } if + ."Result: " dup csr. + ."Expected: " @' expected-result dup csr. cr + slices-eq? not abort"wrong answer" +} : test-aggregate // sig[1] ... sig[n] n expected-result + +{ + =: expected-result + // ."Args: " .s + [[ <{ BLS_FASTAGGREGATEVERIFY }>s ]] 0 runvmx + abort"exitcode != 0" + ."Result: " dup . cr + ."Expected: " @' expected-result dup . cr cr + <> abort"wrong answer" +} : test-fast-aggregate-verify // pub[1] ... pub[n] n msg signature exprected-result + +{ + =: expected-result + // ."Args: " .s + [[ <{ BLS_AGGREGATEVERIFY }>s ]] 0 runvmx + abort"exitcode != 0" + ."Result: " dup . cr + ."Expected: " @' expected-result dup . cr cr + <> abort"wrong answer" +} : test-aggregate-verify // pub[1] msg[1] ... pub[n] msg[n] n signature exprected-result + +."Test verifycase_one_privkey_47117849458281be" cr +x{97f1d3a73197d7942695638c4fa9ac0fc3688c4f9774b905a14e3a3f171bac586c55e83ff97a1aeffb3af00adb22c6bb} +x{1212121212121212121212121212121212121212121212121212121212121212} +x{a42ae16f1c2a5fa69c04cb5998d2add790764ce8dd45bf25b29b4700829232052b52352dcff1cf255b3a7810ad7269601810f03b2bc8b68cf289cf295b206770605a190b6842583e47c3d1c0f73c54907bfb2a602157d46a4353a20283018763} +-1 +test-verify + +."Test verify_infinity_pubkey_and_infinity_signature" cr +x{c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000} +x{1212121212121212121212121212121212121212121212121212121212121212} +x{c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000} +0 +test-verify + +."Test verify_tampered_signature_case_195246ee3bd3b6ec" cr +x{b53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f} +x{abababababababababababababababababababababababababababababababab} +x{ae82747ddeefe4fd64cf9cedb9b04ae3e8a43420cd255e3c7cd06a8d88b7c7f8638543719981c5d16fa3527c468c25f0026704a6951bde891360c7e8d12ddee0559004ccdbe6046b55bae1b257ee97f7cdb955773d7cf29adf3ccbb9ffffffff} +0 +test-verify + +."Test verify_tampered_signature_case_2ea479adf8c40300" cr +x{a491d1b0ecd9bb917989f0e74f0dea0422eac4a873e5e2644f368dffb9a6e20fd6e10c1b77654d067c0618f6e5a7f79a} +x{5656565656565656565656565656565656565656565656565656565656565656} +x{882730e5d03f6b42c3abc26d3372625034e1d871b65a8a6b900a56dae22da98abbe1b68f85e49fe7652a55ec3d0591c20767677e33e5cbb1207315c41a9ac03be39c2e7668edc043d6cb1d9fd93033caa8a1c5b0e84bedaeb6c64972ffffffff} +0 +test-verify + +."Test verify_tampered_signature_case_2f09d443ab8a3ac2" cr +x{b301803f8b5ac4a1133581fc676dfedc60d891dd5fa99028805e5ea5b08d3491af75d0707adab3b70c6a6a580217bf81} +x{0000000000000000000000000000000000000000000000000000000000000000} +x{b23c46be3a001c63ca711f87a005c200cc550b9429d5f4eb38d74322144f1b63926da3388979e5321012fb1a0526bcd100b5ef5fe72628ce4cd5e904aeaa3279527843fae5ca9ca675f4f51ed8f83bbf7155da9ecc9663100a885d5dffffffff} +0 +test-verify + +."Test verify_tampered_signature_case_3208262581c8fc09" cr +x{b301803f8b5ac4a1133581fc676dfedc60d891dd5fa99028805e5ea5b08d3491af75d0707adab3b70c6a6a580217bf81} +x{5656565656565656565656565656565656565656565656565656565656565656} +x{af1390c3c47acdb37131a51216da683c509fce0e954328a59f93aebda7e4ff974ba208d9a4a2a2389f892a9d418d618418dd7f7a6bc7aa0da999a9d3a5b815bc085e14fd001f6a1948768a3f4afefc8b8240dda329f984cb345c6363ffffffff} +0 +test-verify + +."Test verify_tampered_signature_case_6b3b17f6962a490c" cr +x{b53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f} +x{5656565656565656565656565656565656565656565656565656565656565656} +x{a4efa926610b8bd1c8330c918b7a5e9bf374e53435ef8b7ec186abf62e1b1f65aeaaeb365677ac1d1172a1f5b44b4e6d022c252c58486c0a759fbdc7de15a756acc4d343064035667a594b4c2a6f0b0b421975977f297dba63ee2f63ffffffff} +0 +test-verify + +."Test verify_tampered_signature_case_6eeb7c52dfd9baf0" cr +x{b301803f8b5ac4a1133581fc676dfedc60d891dd5fa99028805e5ea5b08d3491af75d0707adab3b70c6a6a580217bf81} +x{abababababababababababababababababababababababababababababababab} +x{9674e2228034527f4c083206032b020310face156d4a4685e2fcaec2f6f3665aa635d90347b6ce124eb879266b1e801d185de36a0a289b85e9039662634f2eea1e02e670bc7ab849d006a70b2f93b84597558a05b879c8d445f387a5ffffffff} +0 +test-verify + +."Test verify_tampered_signature_case_8761a0b7e920c323" cr +x{a491d1b0ecd9bb917989f0e74f0dea0422eac4a873e5e2644f368dffb9a6e20fd6e10c1b77654d067c0618f6e5a7f79a} +x{abababababababababababababababababababababababababababababababab} +x{91347bccf740d859038fcdcaf233eeceb2a436bcaaee9b2aa3bfb70efe29dfb2677562ccbea1c8e061fb9971b0753c240622fab78489ce96768259fc01360346da5b9f579e5da0d941e4c6ba18a0e64906082375394f337fa1af2b71ffffffff} +0 +test-verify + +."Test verify_tampered_signature_case_d34885d766d5f705" cr +x{b53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f} +x{0000000000000000000000000000000000000000000000000000000000000000} +x{948a7cb99f76d616c2c564ce9bf4a519f1bea6b0a624a02276443c245854219fabb8d4ce061d255af5330b078d5380681751aa7053da2c98bae898edc218c75f07e24d8802a17cd1f6833b71e58f5eb5b94208b4d0bb3848cecb075effffffff} +0 +test-verify + +."Test verify_tampered_signature_case_e8a50c445c855360" cr +x{a491d1b0ecd9bb917989f0e74f0dea0422eac4a873e5e2644f368dffb9a6e20fd6e10c1b77654d067c0618f6e5a7f79a} +x{0000000000000000000000000000000000000000000000000000000000000000} +x{b6ed936746e01f8ecf281f020953fbf1f01debd5657c4a383940b020b26507f6076334f91e2366c96e9ab279fb5158090352ea1c5b0c9274504f4f0e7053af24802e51e4568d164fe986834f41e55c8e850ce1f98458c0cfc9ab380bffffffff} +0 +test-verify + +."Test verify_valid_case_195246ee3bd3b6ec" cr +x{b53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f} +x{abababababababababababababababababababababababababababababababab} +x{ae82747ddeefe4fd64cf9cedb9b04ae3e8a43420cd255e3c7cd06a8d88b7c7f8638543719981c5d16fa3527c468c25f0026704a6951bde891360c7e8d12ddee0559004ccdbe6046b55bae1b257ee97f7cdb955773d7cf29adf3ccbb9975e4eb9} +-1 +test-verify + +."Test verify_valid_case_2ea479adf8c40300" cr +x{a491d1b0ecd9bb917989f0e74f0dea0422eac4a873e5e2644f368dffb9a6e20fd6e10c1b77654d067c0618f6e5a7f79a} +x{5656565656565656565656565656565656565656565656565656565656565656} +x{882730e5d03f6b42c3abc26d3372625034e1d871b65a8a6b900a56dae22da98abbe1b68f85e49fe7652a55ec3d0591c20767677e33e5cbb1207315c41a9ac03be39c2e7668edc043d6cb1d9fd93033caa8a1c5b0e84bedaeb6c64972503a43eb} +-1 +test-verify + +."Test verify_valid_case_2f09d443ab8a3ac2" cr +x{b301803f8b5ac4a1133581fc676dfedc60d891dd5fa99028805e5ea5b08d3491af75d0707adab3b70c6a6a580217bf81} +x{0000000000000000000000000000000000000000000000000000000000000000} +x{b23c46be3a001c63ca711f87a005c200cc550b9429d5f4eb38d74322144f1b63926da3388979e5321012fb1a0526bcd100b5ef5fe72628ce4cd5e904aeaa3279527843fae5ca9ca675f4f51ed8f83bbf7155da9ecc9663100a885d5dc6df96d9} +-1 +test-verify + +."Test verify_valid_case_3208262581c8fc09" cr +x{b301803f8b5ac4a1133581fc676dfedc60d891dd5fa99028805e5ea5b08d3491af75d0707adab3b70c6a6a580217bf81} +x{5656565656565656565656565656565656565656565656565656565656565656} +x{af1390c3c47acdb37131a51216da683c509fce0e954328a59f93aebda7e4ff974ba208d9a4a2a2389f892a9d418d618418dd7f7a6bc7aa0da999a9d3a5b815bc085e14fd001f6a1948768a3f4afefc8b8240dda329f984cb345c6363272ba4fe} +-1 +test-verify + +."Test verify_valid_case_6b3b17f6962a490c" cr +x{b53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f} +x{5656565656565656565656565656565656565656565656565656565656565656} +x{a4efa926610b8bd1c8330c918b7a5e9bf374e53435ef8b7ec186abf62e1b1f65aeaaeb365677ac1d1172a1f5b44b4e6d022c252c58486c0a759fbdc7de15a756acc4d343064035667a594b4c2a6f0b0b421975977f297dba63ee2f63ffe47bb6} +-1 +test-verify + +."Test verify_valid_case_6eeb7c52dfd9baf0" cr +x{b301803f8b5ac4a1133581fc676dfedc60d891dd5fa99028805e5ea5b08d3491af75d0707adab3b70c6a6a580217bf81} +x{abababababababababababababababababababababababababababababababab} +x{9674e2228034527f4c083206032b020310face156d4a4685e2fcaec2f6f3665aa635d90347b6ce124eb879266b1e801d185de36a0a289b85e9039662634f2eea1e02e670bc7ab849d006a70b2f93b84597558a05b879c8d445f387a5d5b653df} +-1 +test-verify + +."Test verify_valid_case_8761a0b7e920c323" cr +x{a491d1b0ecd9bb917989f0e74f0dea0422eac4a873e5e2644f368dffb9a6e20fd6e10c1b77654d067c0618f6e5a7f79a} +x{abababababababababababababababababababababababababababababababab} +x{91347bccf740d859038fcdcaf233eeceb2a436bcaaee9b2aa3bfb70efe29dfb2677562ccbea1c8e061fb9971b0753c240622fab78489ce96768259fc01360346da5b9f579e5da0d941e4c6ba18a0e64906082375394f337fa1af2b7127b0d121} +-1 +test-verify + +."Test verify_valid_case_d34885d766d5f705" cr +x{b53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f} +x{0000000000000000000000000000000000000000000000000000000000000000} +x{948a7cb99f76d616c2c564ce9bf4a519f1bea6b0a624a02276443c245854219fabb8d4ce061d255af5330b078d5380681751aa7053da2c98bae898edc218c75f07e24d8802a17cd1f6833b71e58f5eb5b94208b4d0bb3848cecb075ea21be115} +-1 +test-verify + +."Test verify_valid_case_e8a50c445c855360" cr +x{a491d1b0ecd9bb917989f0e74f0dea0422eac4a873e5e2644f368dffb9a6e20fd6e10c1b77654d067c0618f6e5a7f79a} +x{0000000000000000000000000000000000000000000000000000000000000000} +x{b6ed936746e01f8ecf281f020953fbf1f01debd5657c4a383940b020b26507f6076334f91e2366c96e9ab279fb5158090352ea1c5b0c9274504f4f0e7053af24802e51e4568d164fe986834f41e55c8e850ce1f98458c0cfc9ab380b55285a55} +-1 +test-verify + +."Test verify_wrong_pubkey_case_195246ee3bd3b6ec" cr +x{b53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f} +x{abababababababababababababababababababababababababababababababab} +x{9674e2228034527f4c083206032b020310face156d4a4685e2fcaec2f6f3665aa635d90347b6ce124eb879266b1e801d185de36a0a289b85e9039662634f2eea1e02e670bc7ab849d006a70b2f93b84597558a05b879c8d445f387a5d5b653df} +0 +test-verify + +."Test verify_wrong_pubkey_case_2ea479adf8c40300" cr +x{a491d1b0ecd9bb917989f0e74f0dea0422eac4a873e5e2644f368dffb9a6e20fd6e10c1b77654d067c0618f6e5a7f79a} +x{5656565656565656565656565656565656565656565656565656565656565656} +x{a4efa926610b8bd1c8330c918b7a5e9bf374e53435ef8b7ec186abf62e1b1f65aeaaeb365677ac1d1172a1f5b44b4e6d022c252c58486c0a759fbdc7de15a756acc4d343064035667a594b4c2a6f0b0b421975977f297dba63ee2f63ffe47bb6} +0 +test-verify + +."Test verify_wrong_pubkey_case_2f09d443ab8a3ac2" cr +x{b301803f8b5ac4a1133581fc676dfedc60d891dd5fa99028805e5ea5b08d3491af75d0707adab3b70c6a6a580217bf81} +x{0000000000000000000000000000000000000000000000000000000000000000} +x{b6ed936746e01f8ecf281f020953fbf1f01debd5657c4a383940b020b26507f6076334f91e2366c96e9ab279fb5158090352ea1c5b0c9274504f4f0e7053af24802e51e4568d164fe986834f41e55c8e850ce1f98458c0cfc9ab380b55285a55} +0 +test-verify + +."Test verify_wrong_pubkey_case_3208262581c8fc09" cr +x{b301803f8b5ac4a1133581fc676dfedc60d891dd5fa99028805e5ea5b08d3491af75d0707adab3b70c6a6a580217bf81} +x{5656565656565656565656565656565656565656565656565656565656565656} +x{882730e5d03f6b42c3abc26d3372625034e1d871b65a8a6b900a56dae22da98abbe1b68f85e49fe7652a55ec3d0591c20767677e33e5cbb1207315c41a9ac03be39c2e7668edc043d6cb1d9fd93033caa8a1c5b0e84bedaeb6c64972503a43eb} +0 +test-verify + +."Test verify_wrong_pubkey_case_6b3b17f6962a490c" cr +x{b53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f} +x{5656565656565656565656565656565656565656565656565656565656565656} +x{af1390c3c47acdb37131a51216da683c509fce0e954328a59f93aebda7e4ff974ba208d9a4a2a2389f892a9d418d618418dd7f7a6bc7aa0da999a9d3a5b815bc085e14fd001f6a1948768a3f4afefc8b8240dda329f984cb345c6363272ba4fe} +0 +test-verify + +."Test verify_wrong_pubkey_case_6eeb7c52dfd9baf0" cr +x{b301803f8b5ac4a1133581fc676dfedc60d891dd5fa99028805e5ea5b08d3491af75d0707adab3b70c6a6a580217bf81} +x{abababababababababababababababababababababababababababababababab} +x{91347bccf740d859038fcdcaf233eeceb2a436bcaaee9b2aa3bfb70efe29dfb2677562ccbea1c8e061fb9971b0753c240622fab78489ce96768259fc01360346da5b9f579e5da0d941e4c6ba18a0e64906082375394f337fa1af2b7127b0d121} +0 +test-verify + +."Test verify_wrong_pubkey_case_8761a0b7e920c323" cr +x{a491d1b0ecd9bb917989f0e74f0dea0422eac4a873e5e2644f368dffb9a6e20fd6e10c1b77654d067c0618f6e5a7f79a} +x{abababababababababababababababababababababababababababababababab} +x{ae82747ddeefe4fd64cf9cedb9b04ae3e8a43420cd255e3c7cd06a8d88b7c7f8638543719981c5d16fa3527c468c25f0026704a6951bde891360c7e8d12ddee0559004ccdbe6046b55bae1b257ee97f7cdb955773d7cf29adf3ccbb9975e4eb9} +0 +test-verify + +."Test verify_wrong_pubkey_case_d34885d766d5f705" cr +x{b53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f} +x{0000000000000000000000000000000000000000000000000000000000000000} +x{b23c46be3a001c63ca711f87a005c200cc550b9429d5f4eb38d74322144f1b63926da3388979e5321012fb1a0526bcd100b5ef5fe72628ce4cd5e904aeaa3279527843fae5ca9ca675f4f51ed8f83bbf7155da9ecc9663100a885d5dc6df96d9} +0 +test-verify + +."Test verify_wrong_pubkey_case_e8a50c445c855360" cr +x{a491d1b0ecd9bb917989f0e74f0dea0422eac4a873e5e2644f368dffb9a6e20fd6e10c1b77654d067c0618f6e5a7f79a} +x{0000000000000000000000000000000000000000000000000000000000000000} +x{948a7cb99f76d616c2c564ce9bf4a519f1bea6b0a624a02276443c245854219fabb8d4ce061d255af5330b078d5380681751aa7053da2c98bae898edc218c75f07e24d8802a17cd1f6833b71e58f5eb5b94208b4d0bb3848cecb075ea21be115} +0 +test-verify + + +."Test aggregate_0x0000000000000000000000000000000000000000000000000000000000000000" cr +x{b6ed936746e01f8ecf281f020953fbf1f01debd5657c4a383940b020b26507f6076334f91e2366c96e9ab279fb5158090352ea1c5b0c9274504f4f0e7053af24802e51e4568d164fe986834f41e55c8e850ce1f98458c0cfc9ab380b55285a55} +x{b23c46be3a001c63ca711f87a005c200cc550b9429d5f4eb38d74322144f1b63926da3388979e5321012fb1a0526bcd100b5ef5fe72628ce4cd5e904aeaa3279527843fae5ca9ca675f4f51ed8f83bbf7155da9ecc9663100a885d5dc6df96d9} +x{948a7cb99f76d616c2c564ce9bf4a519f1bea6b0a624a02276443c245854219fabb8d4ce061d255af5330b078d5380681751aa7053da2c98bae898edc218c75f07e24d8802a17cd1f6833b71e58f5eb5b94208b4d0bb3848cecb075ea21be115} +3 +x{9683b3e6701f9a4b706709577963110043af78a5b41991b998475a3d3fd62abf35ce03b33908418efc95a058494a8ae504354b9f626231f6b3f3c849dfdeaf5017c4780e2aee1850ceaf4b4d9ce70971a3d2cfcd97b7e5ecf6759f8da5f76d31} +test-aggregate + +."Test aggregate_0x5656565656565656565656565656565656565656565656565656565656565656" cr +x{882730e5d03f6b42c3abc26d3372625034e1d871b65a8a6b900a56dae22da98abbe1b68f85e49fe7652a55ec3d0591c20767677e33e5cbb1207315c41a9ac03be39c2e7668edc043d6cb1d9fd93033caa8a1c5b0e84bedaeb6c64972503a43eb} +x{af1390c3c47acdb37131a51216da683c509fce0e954328a59f93aebda7e4ff974ba208d9a4a2a2389f892a9d418d618418dd7f7a6bc7aa0da999a9d3a5b815bc085e14fd001f6a1948768a3f4afefc8b8240dda329f984cb345c6363272ba4fe} +x{a4efa926610b8bd1c8330c918b7a5e9bf374e53435ef8b7ec186abf62e1b1f65aeaaeb365677ac1d1172a1f5b44b4e6d022c252c58486c0a759fbdc7de15a756acc4d343064035667a594b4c2a6f0b0b421975977f297dba63ee2f63ffe47bb6} +3 +x{ad38fc73846583b08d110d16ab1d026c6ea77ac2071e8ae832f56ac0cbcdeb9f5678ba5ce42bd8dce334cc47b5abcba40a58f7f1f80ab304193eb98836cc14d8183ec14cc77de0f80c4ffd49e168927a968b5cdaa4cf46b9805be84ad7efa77b} +test-aggregate + +."Test aggregate_0xabababababababababababababababababababababababababababababababab" cr +x{91347bccf740d859038fcdcaf233eeceb2a436bcaaee9b2aa3bfb70efe29dfb2677562ccbea1c8e061fb9971b0753c240622fab78489ce96768259fc01360346da5b9f579e5da0d941e4c6ba18a0e64906082375394f337fa1af2b7127b0d121} +x{9674e2228034527f4c083206032b020310face156d4a4685e2fcaec2f6f3665aa635d90347b6ce124eb879266b1e801d185de36a0a289b85e9039662634f2eea1e02e670bc7ab849d006a70b2f93b84597558a05b879c8d445f387a5d5b653df} +x{ae82747ddeefe4fd64cf9cedb9b04ae3e8a43420cd255e3c7cd06a8d88b7c7f8638543719981c5d16fa3527c468c25f0026704a6951bde891360c7e8d12ddee0559004ccdbe6046b55bae1b257ee97f7cdb955773d7cf29adf3ccbb9975e4eb9} +3 +x{9712c3edd73a209c742b8250759db12549b3eaf43b5ca61376d9f30e2747dbcf842d8b2ac0901d2a093713e20284a7670fcf6954e9ab93de991bb9b313e664785a075fc285806fa5224c82bde146561b446ccfc706a64b8579513cfc4ff1d930} +test-aggregate + +."Test aggregate_infinity_signature" cr +x{c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000} +1 +x{c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000} +test-aggregate + +."Test aggregate_na_signatures" cr +0 +x{} +test-aggregate + +."Test aggregate_single_signature" cr +x{b6ed936746e01f8ecf281f020953fbf1f01debd5657c4a383940b020b26507f6076334f91e2366c96e9ab279fb5158090352ea1c5b0c9274504f4f0e7053af24802e51e4568d164fe986834f41e55c8e850ce1f98458c0cfc9ab380b55285a55} +1 +x{b6ed936746e01f8ecf281f020953fbf1f01debd5657c4a383940b020b26507f6076334f91e2366c96e9ab279fb5158090352ea1c5b0c9274504f4f0e7053af24802e51e4568d164fe986834f41e55c8e850ce1f98458c0cfc9ab380b55285a55} +test-aggregate + +."Test fast_aggregate_verify_extra_pubkey_4f079f946446fabf" cr +x{a491d1b0ecd9bb917989f0e74f0dea0422eac4a873e5e2644f368dffb9a6e20fd6e10c1b77654d067c0618f6e5a7f79a} +x{b301803f8b5ac4a1133581fc676dfedc60d891dd5fa99028805e5ea5b08d3491af75d0707adab3b70c6a6a580217bf81} +x{b53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f} +3 +x{5656565656565656565656565656565656565656565656565656565656565656} +x{912c3615f69575407db9392eb21fee18fff797eeb2fbe1816366ca2a08ae574d8824dbfafb4c9eaa1cf61b63c6f9b69911f269b664c42947dd1b53ef1081926c1e82bb2a465f927124b08391a5249036146d6f3f1e17ff5f162f779746d830d1} +0 +test-fast-aggregate-verify + +."Test fast_aggregate_verify_extra_pubkey_5a38e6b4017fe4dd" cr +x{a491d1b0ecd9bb917989f0e74f0dea0422eac4a873e5e2644f368dffb9a6e20fd6e10c1b77654d067c0618f6e5a7f79a} +x{b301803f8b5ac4a1133581fc676dfedc60d891dd5fa99028805e5ea5b08d3491af75d0707adab3b70c6a6a580217bf81} +x{b53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f} +x{b53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f} +4 +x{abababababababababababababababababababababababababababababababab} +x{9712c3edd73a209c742b8250759db12549b3eaf43b5ca61376d9f30e2747dbcf842d8b2ac0901d2a093713e20284a7670fcf6954e9ab93de991bb9b313e664785a075fc285806fa5224c82bde146561b446ccfc706a64b8579513cfc4ff1d930} +0 +test-fast-aggregate-verify + +."Test fast_aggregate_verify_extra_pubkey_a698ea45b109f303" cr +x{a491d1b0ecd9bb917989f0e74f0dea0422eac4a873e5e2644f368dffb9a6e20fd6e10c1b77654d067c0618f6e5a7f79a} +x{b53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f} +2 +x{0000000000000000000000000000000000000000000000000000000000000000} +x{b6ed936746e01f8ecf281f020953fbf1f01debd5657c4a383940b020b26507f6076334f91e2366c96e9ab279fb5158090352ea1c5b0c9274504f4f0e7053af24802e51e4568d164fe986834f41e55c8e850ce1f98458c0cfc9ab380b55285a55} +0 +test-fast-aggregate-verify + +."Test fast_aggregate_verify_infinity_pubkey" cr +x{a491d1b0ecd9bb917989f0e74f0dea0422eac4a873e5e2644f368dffb9a6e20fd6e10c1b77654d067c0618f6e5a7f79a} +x{b301803f8b5ac4a1133581fc676dfedc60d891dd5fa99028805e5ea5b08d3491af75d0707adab3b70c6a6a580217bf81} +x{b53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f} +x{c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000} +4 +x{1212121212121212121212121212121212121212121212121212121212121212} +x{afcb4d980f079265caa61aee3e26bf48bebc5dc3e7f2d7346834d76cbc812f636c937b6b44a9323d8bc4b1cdf71d6811035ddc2634017faab2845308f568f2b9a0356140727356eae9eded8b87fd8cb8024b440c57aee06076128bb32921f584} +0 +test-fast-aggregate-verify + +."Test fast_aggregate_verify_na_pubkeys_and_infinity_signature" cr +0 +x{abababababababababababababababababababababababababababababababab} +x{c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000} +0 +test-fast-aggregate-verify + +."Test fast_aggregate_verify_na_pubkeys_and_na_signature" cr +0 +x{abababababababababababababababababababababababababababababababab} +x{000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000} +0 +test-fast-aggregate-verify + +."Test fast_aggregate_verify_tampered_signature_3d7576f3c0e3570a" cr +x{a491d1b0ecd9bb917989f0e74f0dea0422eac4a873e5e2644f368dffb9a6e20fd6e10c1b77654d067c0618f6e5a7f79a} +x{b301803f8b5ac4a1133581fc676dfedc60d891dd5fa99028805e5ea5b08d3491af75d0707adab3b70c6a6a580217bf81} +x{b53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f} +3 +x{abababababababababababababababababababababababababababababababab} +x{9712c3edd73a209c742b8250759db12549b3eaf43b5ca61376d9f30e2747dbcf842d8b2ac0901d2a093713e20284a7670fcf6954e9ab93de991bb9b313e664785a075fc285806fa5224c82bde146561b446ccfc706a64b8579513cfcffffffff} +0 +test-fast-aggregate-verify + +."Test fast_aggregate_verify_tampered_signature_5e745ad0c6199a6c" cr +x{a491d1b0ecd9bb917989f0e74f0dea0422eac4a873e5e2644f368dffb9a6e20fd6e10c1b77654d067c0618f6e5a7f79a} +1 +x{0000000000000000000000000000000000000000000000000000000000000000} +x{b6ed936746e01f8ecf281f020953fbf1f01debd5657c4a383940b020b26507f6076334f91e2366c96e9ab279fb5158090352ea1c5b0c9274504f4f0e7053af24802e51e4568d164fe986834f41e55c8e850ce1f98458c0cfc9ab380bffffffff} +0 +test-fast-aggregate-verify + +."Test fast_aggregate_verify_tampered_signature_652ce62f09290811" cr +x{a491d1b0ecd9bb917989f0e74f0dea0422eac4a873e5e2644f368dffb9a6e20fd6e10c1b77654d067c0618f6e5a7f79a} +x{b301803f8b5ac4a1133581fc676dfedc60d891dd5fa99028805e5ea5b08d3491af75d0707adab3b70c6a6a580217bf81} +2 +x{5656565656565656565656565656565656565656565656565656565656565656} +x{912c3615f69575407db9392eb21fee18fff797eeb2fbe1816366ca2a08ae574d8824dbfafb4c9eaa1cf61b63c6f9b69911f269b664c42947dd1b53ef1081926c1e82bb2a465f927124b08391a5249036146d6f3f1e17ff5f162f7797ffffffff} +0 +test-fast-aggregate-verify + +."Test fast_aggregate_verify_valid_3d7576f3c0e3570a" cr +x{a491d1b0ecd9bb917989f0e74f0dea0422eac4a873e5e2644f368dffb9a6e20fd6e10c1b77654d067c0618f6e5a7f79a} +x{b301803f8b5ac4a1133581fc676dfedc60d891dd5fa99028805e5ea5b08d3491af75d0707adab3b70c6a6a580217bf81} +x{b53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f} +3 +x{abababababababababababababababababababababababababababababababab} +x{9712c3edd73a209c742b8250759db12549b3eaf43b5ca61376d9f30e2747dbcf842d8b2ac0901d2a093713e20284a7670fcf6954e9ab93de991bb9b313e664785a075fc285806fa5224c82bde146561b446ccfc706a64b8579513cfc4ff1d930} +-1 +test-fast-aggregate-verify + +."Test fast_aggregate_verify_valid_5e745ad0c6199a6c" cr +x{a491d1b0ecd9bb917989f0e74f0dea0422eac4a873e5e2644f368dffb9a6e20fd6e10c1b77654d067c0618f6e5a7f79a} +1 +x{0000000000000000000000000000000000000000000000000000000000000000} +x{b6ed936746e01f8ecf281f020953fbf1f01debd5657c4a383940b020b26507f6076334f91e2366c96e9ab279fb5158090352ea1c5b0c9274504f4f0e7053af24802e51e4568d164fe986834f41e55c8e850ce1f98458c0cfc9ab380b55285a55} +-1 +test-fast-aggregate-verify + +."Test fast_aggregate_verify_valid_652ce62f09290811" cr +x{a491d1b0ecd9bb917989f0e74f0dea0422eac4a873e5e2644f368dffb9a6e20fd6e10c1b77654d067c0618f6e5a7f79a} +x{b301803f8b5ac4a1133581fc676dfedc60d891dd5fa99028805e5ea5b08d3491af75d0707adab3b70c6a6a580217bf81} +2 +x{5656565656565656565656565656565656565656565656565656565656565656} +x{912c3615f69575407db9392eb21fee18fff797eeb2fbe1816366ca2a08ae574d8824dbfafb4c9eaa1cf61b63c6f9b69911f269b664c42947dd1b53ef1081926c1e82bb2a465f927124b08391a5249036146d6f3f1e17ff5f162f779746d830d1} +-1 +test-fast-aggregate-verify + +."Test aggregate_verify_infinity_pubkey" cr +x{a491d1b0ecd9bb917989f0e74f0dea0422eac4a873e5e2644f368dffb9a6e20fd6e10c1b77654d067c0618f6e5a7f79a} +x{0000000000000000000000000000000000000000000000000000000000000000} +x{b301803f8b5ac4a1133581fc676dfedc60d891dd5fa99028805e5ea5b08d3491af75d0707adab3b70c6a6a580217bf81} +x{5656565656565656565656565656565656565656565656565656565656565656} +x{b53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f} +x{abababababababababababababababababababababababababababababababab} +x{c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000} +x{1212121212121212121212121212121212121212121212121212121212121212} +4 +x{9104e74b9dfd3ad502f25d6a5ef57db0ed7d9a0e00f3500586d8ce44231212542fcfaf87840539b398bf07626705cf1105d246ca1062c6c2e1a53029a0f790ed5e3cb1f52f8234dc5144c45fc847c0cd37a92d68e7c5ba7c648a8a339f171244} +0 +test-aggregate-verify + +."Test aggregate_verify_na_pubkeys_and_infinity_signature" cr +0 +x{c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000} +0 +test-aggregate-verify + +."Test aggregate_verify_na_pubkeys_and_na_signature" cr +0 +x{000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000} +0 +test-aggregate-verify + +."Test aggregate_verify_tampered_signature" cr +// Test is modified to make all signatures 96 bytes +x{a491d1b0ecd9bb917989f0e74f0dea0422eac4a873e5e2644f368dffb9a6e20fd6e10c1b77654d067c0618f6e5a7f79a} +x{0000000000000000000000000000000000000000000000000000000000000000} +x{b301803f8b5ac4a1133581fc676dfedc60d891dd5fa99028805e5ea5b08d3491af75d0707adab3b70c6a6a580217bf81} +x{5656565656565656565656565656565656565656565656565656565656565656} +x{b53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f} +x{abababababababababababababababababababababababababababababababab} +3 +x{9104e74bffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff} +0 +test-aggregate-verify + +."Test aggregate_verify_valid" cr +x{a491d1b0ecd9bb917989f0e74f0dea0422eac4a873e5e2644f368dffb9a6e20fd6e10c1b77654d067c0618f6e5a7f79a} +x{0000000000000000000000000000000000000000000000000000000000000000} +x{b301803f8b5ac4a1133581fc676dfedc60d891dd5fa99028805e5ea5b08d3491af75d0707adab3b70c6a6a580217bf81} +x{5656565656565656565656565656565656565656565656565656565656565656} +x{b53d21a4cfd562c469cc81514d4ce5a6b577d8403d32a394dc265dd190b47fa9f829fdd7963afdf972e5e77854051f6f} +x{abababababababababababababababababababababababababababababababab} +3 +x{9104e74b9dfd3ad502f25d6a5ef57db0ed7d9a0e00f3500586d8ce44231212542fcfaf87840539b398bf07626705cf1105d246ca1062c6c2e1a53029a0f790ed5e3cb1f52f8234dc5144c45fc847c0cd37a92d68e7c5ba7c648a8a339f171244} +-1 +test-aggregate-verify + +."Test fast_aggregate_verify with actual Ethereum signatures" cr +x{abd15ed19f6b6f4a199b6c90637e70222a8aadcb34ffdef3c6cc57a824a7410852d209835c91680875794ba287ce7de3} +x{832451ba329e4fe2e81056ae224feea873ed8d6ffc86840437976a421a352282faca08b93e9e3a8068a9e3979e6056b5} +x{b09e63a9a8b80928532e1fe0d4624be4990bbdf4c11293cd72ac635f56262fea853aef1348c26a8263f83d0c3144bdda} +x{98e38f383949a34bc925a381b3e922fafc200201f743001fff15171bdab7a82a3731c54522c98297840ca687e47057bf} +x{af2d7140be9332b536222680c690a54de4eea2644b3b3a84e45b3e425d4a37f07c98eff13e123da39b22c536be230f94} +x{8277ce097d82f3810e28936bb0f200c47be681db6b22a34bae6b39a0616a1dc5eb542dbc0f68adccbf5e9517a03c0736} +x{9663cda398d72bf5bdd4bbc49651bdefb88f5dfb32e305a7f70a968cbd1e6f4ae7cd50e675393e04ec56f7566bbc153c} +x{a9c0360b88f09c0528a0cca8173061eeef7e8cc84064dd3bd894808e1da3536093f9acf7552e3485d037cf5b10a9c036} +x{8259ce226809ebf580488ad7c8d054732eedd21552d8ab4d8652b62524f3858eb10b055f40f7d1752b200bee9a94b8c0} +x{a605c777fb1ce8b67450e0d243305f5a906074dbeaa514d93cb43eae638794c2ef38c60b478f7aabdae5f4e1461d2d69} +x{9376c20d92aae571465aba7286ada5c50733fd5c417fc8a4cc7cf2af8298f35ec6f43b6d9fe1bf8c950abe3499b4dd62} +x{a4ca29cca82a9bbc250556c459d0ac48c90652b7ecaf77aabc14a64e15a6e686e48efced318ba848303908036fd44f86} +x{aa50e833ee57e0b71b52a5f826cb7f131720fb35c828701d0703eeb417613a8f10be0be445b35ad5b9fa7a6da3c3a836} +x{825f98ebe875b1b6ad876153d3969a0fbb088f3fdffecdfda40bdad451fb336a05296f325ed85bbc271eab8dd9d541d1} +x{9895f1615cec63b3a28be707874dc4273168f91c0c5bf687893680f5e4d88964620b2fc91dc0e663c1d9968c25a95a01} +x{a4255953e52e4bf80a8c2c52cd24fbf869bc43c7fc815239d54d841b86b660a3b2841191f9716fc4aad465432b8415c0} +x{b663d3f303a2dcc660f274c6b051e4c2a16271840315148748ba0585f47e02e99827527d365acae65094c39a09c1b065} +x{86753154725dedbc2f0425723b551f4fccb2f69087354ac0c0c0bbbc23e10d07325b87989623c54e58b7b2241c97efbb} +x{b1de760c0271e5ad1422d8b58ddf954fb104ec7b8a74927ddad612833d189fdf6bb776249cde1b48786ad69e015a7b0d} +x{976cd11d84723509780656ca3b92f12eeaa7413b78d9c74824bbfb0269d7ffb63499be7340ca75ede7cfac28024785a8} +x{b2318f9647bc8ce79734177a85877d63f486ce98ab2a247672c54cc4093dabd013ea2d95157a1c5c5d7cbe00e4c33b11} +x{97ec1840d0795bd66dfb2a3ef954f93ac93e05755c2b3d61b714be2d732d37291e5a95a69a02ab6af4ae1441e45e9323} +x{a3fc6160fb647a1d43d94d639704d6d17528b18361e3acbfce051fe10d7b9608e717f6d73f51b784e69b59789c121f2f} +x{82406906b3a6e0baa89d93fb2e1c1f87537e6920f4c1390a1526fb5f490f65dc35432ac6d265418167d7a68c5cd70c74} +x{999a313756535ae0d6816b99c69db91c3792648d45cad753c8f6cea0d9997f2006c508cdb39c3d286e24a607598a8c8a} +x{87dffdc02f4797901553cc7de4a8ffdae4df2b5728a0418923d1d4c01da14a482b21a13332b7227f49b4396f88e776ac} +x{82c9236f69038abc40acbf1b730b3c1d3962ff9567f823c83eb3e984339e77c4f9373f679d30c4ffd0e2aad1d002f4ba} +x{81b989f4a65fce31ba9c63a80ff53d28e75606f5ec563a6aef5f2d76d13f591ad60b974f97fd08155325148df95cfcc1} +x{8f3470b02f8e0a5af092625b3f3812d44d76b86cd1c7af64f1c350f6931eacc38490f10cb9888f1b6cdf68fac06ef9f7} +x{ad77903be9a9c4611d9c152ba31faf49d936be3ca7ace051ee0d8862a3869d61194d990b5b4c86bdcf5219e0ea2c25f9} +x{b6654b4984b1f41517fb3d1f8471f2b3a389329fe304cf315d0c087b7a29f58c8fe61cacd99620ad337fb73e773836a6} +x{819cc51962fafcbe99160959e9bbe66e43bf06b5cae2300fe7cb9aca5739f31c4e8a4c695e77f6b06a881eb6322f4399} +x{a9aadc9694fc9c214875b66f00472ab3bdb9fec2cf07a46845050506c077a029ee75f88a24ea2a1b9c2c0b50810e6254} +x{ab5c69e503a2a7becebb24f17c921e9ec2f4d18f7b1565b0c2a1468271348aee861ee0e9e091a786fbdad10743a7c449} +x{8a1d4d071d08a340f3a3df825d748da28b9dcbc52482d34ce05e6272ebaf93858c8460831ded3c81d2d19352414c7fec} +x{8fe4c6f6aa8951d3f092132c645f4ccf9a50a1c200fa96f27306968b4baab7b5bb09e1f6b8d7bb91dc82a408576d6894} +x{958fa0b9e4a095c7c270d7c3d9a603533e5d91b6e1624ff6dc235d5e96ba7cf3df71b202d96667b4add69c10af948fa3} +x{aa9fbae1a28af5400c5fcee35a6b968422c3015f018d06793d21bd4fbf492c484b15d8a7558d0081ce174d1b648bb057} +x{819a691281ccb2aef76c9523a9e0329e1abf6fa1917227fcc9c3996662de6d4d3623ba2298845b2f82c893b5f3b7f638} +x{b59bc108d9780d1322e318765f0064e93a475f4ef76519a05c76b52b9a640b3d6add5635ff4477b8124e42ada3b95a85} +x{b040da098f2f81cb66270ad142035080da510bbb7c7835d12432ab0f689665052aa226e818a2f16c60e38e2e9dae1ce7} +x{834a7bc5d9eea4c10508dbbf366e1c0c1d4ae87a0832b7d9cc6b635da6b37b70a0ee57215e7ed2eaadc58587792a7341} +x{97c3281b2f7d1893dab3802b620d72e7061af12ad57847ce00af912dbd8c3ab62f8c548764aee8da5ca57d527e41655c} +x{af4241e7dff52b4912d0b6b06f6d59562c77858538f0838f652d91cf13afbafbbb7f16aaa2c53c6159cb1aa8e4c672fe} +x{92a78d4bc61c4cb7783a57e3f423e83402659de44efc167a943604a89cc7afe3c6db64dbc05cac81503f17bef2f3bfcc} +x{907fdbf2864372dfd420acb19dd045f090c088a858bb58e2de82ac6aef9a5efe423c6fcbd1f1810d6bec89f9eab8cea1} +x{96ccd37a8c0755f54d66d380ba56e5520a8adfcb70271d223aa9b42bd6cbbe3a7d186012b49dff85f0d7df3f366b41e3} +x{89e738b1b69710038089a990a7278b2a54d30075998fb1d4540c6e264e57def74fbed1b35cf0f3f236f12887e4df1949} +x{910a6f5d7a02bc8641e179000faf86f92445d63174999b55d6671d05a2253f1b5890f02d2b679a0b3f500faa89f160d2} +x{a6d921cc92c139a4345b7aab309318c1067e5169f179f82a9bc735c08ff61232a63d88ea59157ea354476a9d2b40db49} +x{b3ed37713ef72ce168beee0aa38f89ed3cf76337a198ccc1b30b58d808aea839af97f5c2fe95ffc61ba875f095058cc6} +x{81104459d2f669aa6bd481fead334fb0dd709e6029e7374d61d9cb1a9589255efde16ca1e6ce1c5dbc510dbef512ad4f} +x{9684fa7e1f44fd4e326104ede6348bbb978e68a944448dc311a634cda46bb52086cac4b435eef009dcdcfe04f3413ed0} +x{b93fe3dbbdf63678e0cb0e4e299015e6a4187d5a6039ac6eb0d96ca4baecfe1a0d3672ed716d2b129fbe7b8373750cf4} +x{ac97c56a24b9dbdf87337a87f27a90b1d936c20e68a39e2f8bbc2ae5a9e1ae9490477cbe03e03f180ff42637246b9541} +x{8032d15ee910491ac3e5d228e00ef8ae41e50c0ca2692fbe7f006f3d8b360e556a1e0ad0c67d18a231f2d71ae83914f5} +x{8ad43c8688a00ea88ddacbf8d2aa99997dd425a35bd7294be0fac420af95bc1723919d2d03862dbee6e3564331d26033} +x{ab2aecb129638db624254e7001a725260d2739fd50b374418a1155569eff3581dcd6f25cb0d0b85ae20aa176c30c99e7} +x{a9a420966d5d57d11a2d881fb462c7ed53dabba8e51b78bb852acc08c4f5c86791828e8cc59631c3fb1351e0960f4347} +x{b974b7427651578b63669ef47ec8cde65c9d134e18e9f2e2761b3250fb472a578424fff31614404d0bc23c0cd75d26e2} +x{a2ebd309811f0c33ae45b709bf55e1c0cc0a09cb18a47855d42d5012e6d2e340105dcd85347060c64852a9da589c26fa} +x{af6c39210b9c6d153e141fb34d7739fc7a9fbf5656b30744e9a8440e6049d84863c2f208f14e03ab5fb5fb47a19dc277} +x{a19820316b4136e5b937c0b60812983f7fbf7a53b1758ad8dc0e249cd49dafbfda4c25d9c798a15a9693bc565a11383f} +x{851cf2adc1d7ad56a0ea3a7abff62570b9b4b25ad84e847e278edd8b11b2a6d2ce901bb45604e51ae5d3aa3c13ed4a07} +x{b9fd9b663fa540237e214c7b1e3b6b74a4381120f82e9691d3d9243d871eed4672926059973ade7badbb88961868e5ef} +x{b943affd10c2735dd5ef10376763e44ad0a848796e28fd215cf5c56d124d191f11fe22fd419ead954a4610b0990fcaa4} +x{a579968409343bda51ec7ba8a4870d8bd560c43a313db0564c5e6df8381d7ff818ffb4676c9a06469ab14d00dd26b00b} +x{b0fb5a7cf2df5834f822e44e6efe63211db7a747b196c6bc2d1605933dc1d24f75d8373d219da0210ebb4228d98cc4e9} +x{967512e3287fa5c4c0093dadf799a4a4053bf95574c18688f9ec9896c74978862517d118058590511a332ca303f42efb} +x{80532dfc08ee2b181e9c591d23c8aa53ad6b4dab54b87d57c1530cccfef26bc53a69e6edcdd74a0e072612f9967bc99f} +x{a03974bf0c0cd74db34d19a4a417bc5a4bff8a2a7566728a8b411317b2ae1c85d02eb17e955264a7439153876b41208a} +x{903f9407477a39f67d5bbd9b6d9a353470019d1f3508fe803a13ba69b01fee8713f468db4846b01d4d4e9f955b87f55b} +x{a7a6df5ba7ca5697aa90ee6a4806346d3e2a6b9aac371ac1aa2dc48616c4c24d236a8e8b4900f9382bec9b21907133f6} +x{b98f676715d14957248d0738e81e5d5f5359391181e0f573580539751f3d2311264aa8649eea66bc7016413e69ca7d01} +x{ac77a4b3a57b754a63d7ae75bc916e0750fdbaa01115f038c7c60f621dc6b0fc30d7e5eea1f0d0a533c90615bbe4429d} +x{9561be8f41acdee26b2d35c8812f2265a7e96f2964d90d1c4c26b057381d4f5406891c45c86c27ba1313e2d0938dcefe} +x{a61cf69326c454b9f25b54d47f5f7209649992124966b2decdb201e4fd469200a91ac95c8d606e856b02cc01553d806a} +x{8f9fb79917775751341b7c96d5325df771b9450b9e1a82d05a3674586f8a374fc4eeb7e2f176b96f18144f8ce6104723} +x{8830d4880c14d6ec69305f09d0b7f5d138efc37a1f8fe3d916f51c96feb6c4d6ef51e1a4d574d6f06b0a5c41ddc8b364} +x{ab109b1ae29d1687ebdd2d1d2c59d3196ea2afe222466b31ad4242f94a040c81e06e2175ab6f4f75f4738f8d7083e551} +x{92b58f175d394c5e5ef3bca7c1f63b53e6f5443572194b7122b6484db96605141c0fc8ca0a670f553c61f58af0f58ae3} +x{a3ae85c74796adfa4efdecdd39e014bfa26e69c9f14f69e0383313c7fa99768788037d64bb34b13826ab5a969c43b334} +x{b49f0e0bc429eb0016d2d203fb470016cfdcfddc30fb7c7c4bd6f10b77731f8bb2b91a5dfad9200b96a1201ed0b4754b} +x{a82a80e0e24d94c5cd6c66d0a8a937aa6f447678b2015f56d8123397f0165456e7000ba51501595a34203a9cbdd88077} +x{a52cd7f09adb8f5bf2fe6a3d206f795dd3d2fbf8c50ca54203c5cc04e9af5c78ea72cf0b2f5cc9f40e1ca9e995ade465} +x{b271b9a754c2cab8cadb0c2084c525ae4321d4a4200110152189b6fe9537156cbed646816c5a3b8458c8f879bdc2a2e3} +x{b09b2fa0f1f5ba3857c8be27197eb3349486c6a9b9e878e2cb6036e79ff8c7463983d73d28d8e9b0f119e7a3fb494e73} +x{b9adba35b8a1a01f5f6f6daa26e3bfbe9793b8402eb5e3e2aeccfcbdb597c0636ef8d4bf2c8713f5cdaa1f6432e5dc2b} +x{8de96521d0ec24ec97e69da3996389c60c782461a894a67b09562bd0fe2416e1d3b83c241dff07fba400d0def39dfe0a} +x{991ac0813bdd9a59ce2322a90da2511d511a12855b09df44555cf43b48296fa9869ef9df66f359179c706ea2e2a1977c} +x{978eaadba569a36fa8f5cc0532803b76432d51b26bfb2f3eb1841db325ffbf4f1f72be2d478dead2b79677cffe1331d4} +x{b4e52c231a548b29f496e4589afd554dba83b5db9565091eea0c4c79a10163181be836e15070b88c11ce306e8da2b356} +x{b928dd2f764ff18bd663ee455af3ed5518088a780fd3486535eaffbf5f55979ac115ba2f745fff6438be6623366c5663} +x{ae48e5fb169ac93073b1b36eb20a93fdf5d48f6ad8364a441b3dcbdf5761b5caf84176017e879bd2cc6e0efbe0dece21} +x{a4f81f4f3a77a94d3c24ac297d376418a114ea16771ba1b9abac90d6a048c3ce4ba67a724dbd9fc84d941838ec946aa2} +x{aad763ee3c5bb1272c24449f6458554039e3370234542b7131dc44c0b1f0f9dd18890b7048e690cd208ad5e066544833} +x{b05ef4dd1a820b49d0594240e906bca9e716cd4bf6d45a51d5d571ed205e39b6501930782cd8573d51963d5a0401ea5b} +x{a027e8dc718735f9acb9b2f06fc74052bdfd1f2b7ccbf181de1ebf7080e7bd35b7b8fffc825e7f6fe7aa75a814697d3e} +x{b07c797102e4f17ac8c33dd40081cd2a502ee2e4bce02bcf4eccd073c51d7db0ef4cccf83de1b7adb864cd38f92e8903} +x{98d211310a79d0da3594e88835e5b8cf944c61b1c4199b56af9d703451a21e55c38ba60e9cf566d677437c65d8fb026b} +x{92e5e8bbe9ffaf880adc98a42d3e41249b789c94551ffb380620da4cea22b163ea9ba8cbdb393e7dbbbc9553b85f4445} +x{a89604f3c9920aaaab0a40a059e621e42fec8902fb15f7e3fc24bad435386eaffcda79e94f52daf9b37058d7eb6af895} +x{9372aa0f3f96d683d81ce18233c326ddebca93e6f29a9fce651b1cae20410c1e93e54990af18bb55cc0b97dd3d4ae26d} +x{a9c4b9b1ff3b74ea6de69a396be9511a715c8ff87a9b33e0e74f99cea0d823eb907cd127ec06cdfdada16105cd973163} +x{a10c5483b6877478a90de5cb4fd2f23da77fca45cb6b53a8c2559b4e310116ff4feea80f308bb296d3e8aa5b00f1c40d} +x{8148b79dc931b103ac4a15ba1d613e8ff3b54941708c78bfdb7d8748c6cef01949e09d6a67488f7c12801f0c02b713ed} +x{a2e4657128e1bee011fee679b7fe7515335766ab7f635bea59b820de972a24dd3eb3829c875884143f223c03cd18c8d9} +x{88e8f1442b2f54fb6e882b71168dbfcfae30f73b422215d7cfac80e87ca87da2a235c71f0965d5428c3654a7cfb79235} +x{af992a54c30f72966ff2b5fabb01219812b189d501ad7d5f6e1d89b167f6500b1908b995bb1d96ed27ed74fed800b150} +x{99fd4146e659a523b7f3be89e6225849a51377ea49d8fe12b39b39ab926ba59f8176c6fbb121a6aaf4b47a4f4a9df89c} +x{8561e48d685afd45b2d0f6e7944c9cb87f3d3f7759cf2d40a9a67fa202db25dffe1380ff9028c6692c14b05fca52bd65} +x{93467762e27dd2647b83360aeb0b53b27ad1a49262191a83143e109519414f6e783109362d1f6f6c062f633c4311e2e7} +x{b2e9357b2a7e6dbc0168b7f5035e9c3c5f49f47702dddd3e6b08e235086780119a04006909aba277c54fa33768be7585} +x{b9820304d2fb65fa993e9316d434c21d4660e3b47db4bdcb4bc6e24dc11b1cd6e4c79d5ee66cf12aa9477d9d903434b2} +x{849cde94b4bc6c895ed590db92577a454577fcb2492aec26ca4df48a650b6252a69e7f43ed82ca920a81703e9f2f40a4} +x{8f2139e3eda2efbf6466768d5e16679e9cde6607fb15cdb1b91f437f232d830717cffd60da4062c3b5eb382de8330fd1} +x{abe47bb7af31ec2bb434ac94bdf514e21c8620eed7943545b30f4c1fe851f9413464550f301ca93ceeff925869cbb18a} +x{8130ffad0227ccdf49b6fc98d818baed9032168ad55b2f888af636a7cad52a0411bd8d699c78f63ee3288140759547af} +x{aa1760f885bea6903efa5d557a9d5c9f4cb54ff34129ead9e9faa07c0a41d066b667244229627ba5797feb984537f8df} +x{98403382601e7766acff837ada3342b15b744a8623ba6e5a55cf8df199ba0c2cd5cc5fd1aa319dff980b11bc6a199d6e} +x{b8af58fb6b4f548ff6918c42dd69608b9136a8909edd73fcd2cb994a45a627c6dc26793b7c69262f3e8892d3bf2eefff} +x{b2e385e6ee0bcd709dde65cf095f2d9c36f315ff06e83d9597df934a1d7eea5fcad77de858f3ab094772800d491e3819} +x{8c10a30693b4b1b752ae0aaca5c6dc656c871085701a8cf68c5c7958dbb896e8798b616fa0fbbeb7a2932adbe0f308d4} +x{91b0e15e00aa788d83be05ead90be662bfd17cf16cd5da705d81b3acaa7d478a3bd349552c1429652a64e3967833ae62} +x{8ddfe0d8e41c12332f7c5e4310899d9b1be138c80ccfdeadccaf5e74077e13aad63f4bd508290ded0e2c40c6628ec035} +x{94c43da38a8ed19392a8662d5ef7c8358eadcdd4a566a61f6a4fdc57cc3aac58d8ad2c4e61f283fef9ab5aea324c080c} +x{815fbf4029d334d63bd8f667a6684004714415fee64ca97c60b850b188ea3360d17299d9ff6a332f89ec561c179167e8} +x{89873aedf223606eb09afb13c34b1b0ba65532aa4e9864cb588601fa42d69250546214b2498e5047a6fed4e09a0e36a4} +x{82a2bb33c62a75abd2ed172d14abc0ef5637b14331c41b6f49b42c2297feb277566975a8f2c720b04474c221648dd95b} +x{a46d8964d9854114e82f0d98562e7f3b0b0f38037f9050a47645cece99f75d9cdafb304a8bf7dcc1049ead05a6086eee} +x{aece4f7f8c68a959f88128502115201de816f748f8f4f83d406f73806821aa43d204d2d02b487c44266f1559e7eddc69} +x{a999ffbea55a6ce252fc4661c337adc2b5675f999351071e6198570d5a7e7e5855ce0312685d691427cc4a230e1aa031} +x{b57e85d1390f113c620a772075d6edebf2f2379c7a3bcd469358c128b888f73c947d00f4054df5c2b220082eafd02ae1} +x{994df161084d1ac510ae0996121c74ff8c16cf5bebaecbf21aa1da07eed0f602654adaba025636f0bd338e9bc218d942} +x{9018e9e6318c824de62f2a3278ef214863b0c98e895e9ffbaefefdf56386c3e5c20f6d5236d4ff7709fd015e6abc3e96} +x{a6f41705254ddf479c3d19d60b7800b091561bc7790aedd2d33fa0f41a8d5147c81ebc347f4f012aaf363b3187bff990} +x{8fc374c056f5e726063f59e84f97b066af2e07e599d6e2e2593d8e31bf5665afdf3034be6a529ffc434e2b365b94fb01} +x{b05b312f4dad2ffba9b50d651607cdd19c22f2eba0cc5384d2743faf74f0f081533a50f7230756bd5638a8570fe73d56} +x{a96f25869f617c81188dda1b4d9422dc2a8ded653de0e1879f3e5534b582fd332cdeb13f6d887d9b28f653ab6ac09f88} +x{8f6c0bbc567c089a3b22a381d3e5f5915421f27563b7ac51318b9612515ce0d3224d40b3388983afe2c92d2da3844db3} +x{b2779d398c462262fb1b5ecb4ebc50d00c44337e6ed20210ada4b8b6321b99ef262455e50647a61452aa2678e64def15} +x{85ba9c1a72a420f960a443e7724835e08023581226ee4912a6a1876f40944c7262f92e3bbe66796dcf9c706986917671} +x{af01794226c596db4f502475eca86f399df28b1344af6a54b1faba31158702d56c57f1b35a9909f2458734f58c04ffd4} +x{8918c724fd6a5530e76d3518e5ab73a7144685769b1553861e0441cba0b0c307a68429923cb7c3cdc577e3dd9e0f11de} +x{b6bd93d8442d05d514a5cecd64efd5bc5571221769c1a76177376e2e0d6504c7c8196e300bdbdd38ab8514e0423c1dec} +x{b7a6125a2ec2dfe7fe6731bdb7d4d604c473a9f74abe11866f7cb20ca4fe11157d379ed3577b35d96839e95eff110fbe} +x{85960eb8968b9a267f65f4f7b2ee1e9ed676e5815615aa4b29abb6bccc669c0232401e529a1597b67b0cfd310e6ca5db} +x{a02101be2ec64317b52358933f7b68f0d49df19404f6dc035176d3313eaa5253584d023c43eee1e8e2e627b253fb5634} +x{8baede2de9d880a5e9e4f3c35c181a1fdc68867fb432ffb53b530a29daf52d844389cbe3d7ae2557ee00feba9ee2d7e3} +x{8f1eee175b515d5c4e8784a03c7192efcf93588d476eacbee95d01f3c7cba46be8279ba65e74eb7e8983383908b07cd7} +x{a24b3d111b7ec72a3281402e9d1be91b51f8e71d80d27ae62e6c75e5c8bd3a860f45de65e9bee657d400b78e7a2c5e16} +x{85d13e876e384ba0c55f72bf391ccf65248736d0a707e143d65146d439e945abd0133812319626c9e8092f5fefd54c45} +x{a0deed3a9b3cbd8ea5ea6497ecd3afd02616b953e33cbe8b117a4add6cf8cc4398268eec7e9e0bd91899472410a452ef} +x{a149ae720e47fbcbddde029074ef65969e587d09a8d6cd3aedeb63a46b54bab17b84216feabdd3bb81341c6a83a243c7} +x{85913becf48a087fc75613d24b332ccfda08abb67c0b8bd9bcbf5f0a8ab1bf6285c68d89fa80f6ec18c666316f19088b} +x{85685bbb5bb114db0d337bfe51089039749e7afcb7099b6d04e668e10e52d29256e950532b4684333dda9a094c6fc7e8} +x{a7327cb5024ea8438403f2836ba21bc84460ec2f267d399e0e0fd07ff9344baf689744ecfcb90ca9dfd95b157ffdf29d} +x{a3740c34b3dc44df50ac1fe8916a592e8cc86fea02ae29ef53632671b69bd9ee3c6d8b9aed87d4b1bbcb9457486bbfaa} +x{8d2c4c61467fde9a45d1adbe02a8c7d2e925e8f6f126be1b91fae1d8578311740678ae9e280c79620d08fcd879159ccf} +x{903f21545ef331d5c312801b78b8675fbf093ce8d3e4684da155f80f2153bde840f60436ea506e1d07bf5cf8cbb566b2} +x{a2ab1ea627cf7b2b3e3f4a358719d5696e9917b76898beb0b0519962cc34cf222b200367ecff032052023b0d5326e5a4} +x{b79fb683f4aa467dd1859468efbf53214cc9301f38958daf8c6f313274cc70a532a167cf24d7d023eb66cfeacd6b553d} +x{b202b37b8f801d67896cb60227a72138918488a46034556c90d00cccdc46b86e4d77adf3991e222f4d383f0e0cfec46d} +x{a721aab3a83aa009cc22fde90c0a5f1fcb42712a062a68628ee402399a81c4c3bec1a58ab4d1a37a4d4befbc275848af} +x{8fb335baabc4d561e133b874c98bbeb8b1787bd7b44fafc9dbd02c85d98db6bc03f439d285c7fe17c2c751268e651255} +x{aabc892d3f6b3dbf04b8f1f38215bd58f23f117e6643c47ccff6803fde56f0c950a5885d7e2ce97fdaf01a1ef32ffd9d} +x{87a0549679757add0f06d239aa760274c57ccfbf30e722d8a6b33c76ff5576d354f7a54a53073cc3babce0b0465eef3a} +x{99e84647be315371ee2714fd2524370daedf5e64c343a31cf35d05f0737f73ca97ac1645d17780916c73317cfb7ecce3} +x{ae30f745530dbc0096e0dbdf8b7547499eab8f68c5ffefa05ebf6b735b3f94df7161c09360165459d939f562961a7af0} +x{b3ddcfb849a79096763ba2abe2c51b4b0666d548d2c19be8042da0c02956a5b13fd4166ef3002aaf63e16f18ca27e777} +x{b7cd98db331a01c52012560297ea9b6a208bf7734164c35777db809aa1e987b5c1ae20bf62e68a6f8160f6606db25bb7} +x{a72480094bb69733ed322482458f26732449ec5520b28c70543bfb0e291aa6b2c454b9a67321af82a5aa614f3110a8d8} +x{a52162089b34711d405613a42e715e1f06f410f9e1824d91c10db392db200fae8771480ea75b65e43752f5c38f9325cc} +x{93869a5603e08ed1431cc308a64a094c2bd24f28bf1e8ebdb667e4da4af58488aaae3cb078b0d1e3ea92e1cecd902f93} +x{a6e3a7e4aa049bad205ea2d84f10e1ff096796e82242cbb06b8657bee144d8daa2d5ae23713df4616221a016fd269578} +x{8ef617adab8dda4770b870d42d3f3ce503665cbb55ab603f38e9cec0cad1e2f7f322d60a42dd2416a21c783a280a0e4f} +x{83f5fe871d91c771d8ad09d26ee4617220721cb4f2324d8b24ef21bc9f5f059949a94d2cb15d3011b86178a4eb52e88d} +x{8d528528af05691c4f5085aa3747fde2ba2716e5a79e050765e83c8e52927f32033b994a45b5de5ae310e055dee1dafb} +x{a6ddede616eb02303bbc8569d569082872967c7bc0c97462ebd871cca264cba03f114780e1436674022f11da754ec23f} +x{aae1a58b985833f827df87146e157900943c16c89d15451f464e17c480b8949228f9ee9117768f92042dfca3bc2e2290} +x{b74e17e474fee881d4a2aa803cb0018e5eb94d0d751ec04f2f5235a85c46445c94f21f314fdd6f6b6b862c8387976faa} +x{91d1bc79e3abe5156a7fcef084136622198a8bc4fc5fcb4d384a3c3089bae93ed80349e0220f168ede6df2992674b4ec} +x{95c1d6d602a03b1fe2eb36308cb939f7093bb9065a1441dcfe6cbdb6fd0051f1a58906829334dc4675c96a9835b23ba8} +x{ae07d4db6f10df60bb09d3c187c75953bdd4e7a492b3be74f753074d330b52bfb2f3118ba2f2561d90850afec527306c} +x{af93966afc0ba0acd2b91a254222e672dcf204fae9bf405de574f8c1ead72577016ecc5d45ddd1a5c67fa1bcf0e6f765} +x{a51adfff1f07ec1b2ba7e7352d262f54bc553510e4b6066b683aafc2748958d4b7a2016902ea8482889975ed61d5c85a} +x{a20433f93219eb9976803b7bbae5247c641bcd11821d5eaa7ae3a088faeaf8356f05c9116f28c78ef11cde111f1c1be1} +x{b6e6a8bbcba88ce35a38be0d6bacfa32cda83937cd68d504afb087cf02ebb8882ecc1db14a5a903d1abc3db589963d12} +x{b8adc04ea1f6ab4bacebbe38e5674d3cdee56f097455b0e94e02ca08f667af82379f8798c5f026b8c5e361c9926a3c3b} +x{84be6bbd912f3eb274c9fd12a15dfaa0a908eaef9a849b2aed2a620af2f11ccbf880d419a89e8f67bb81efce710afca0} +x{ac4e4d3ec168e14985f73c7b30469473a6e0c89db0f6649435aaff4267fc1ad1989de930ae843521253909d1032fb6be} +x{a6cb5e28f19b8a235f9d3e2097528650fb629fab591f2a5d32793fe0ca0b26803a4a284aa132478272ce4c393938ddb4} +x{acef262359340b6fc7104d2ad0f67b420f66cb1f8b51956257942a2a5b515db3c5b03a09c09bec9284af2ca4a73507b4} +x{ac3f31a5c3ad602dceb6fe7a1a64e167b268bb8493ede809938e2213b3b1608e679918b37a069499f95596ee1735778c} +x{8b774778623dcc4d48a049d75c9d42ca7b4a66f79583377fa5ae59c2a3719a97911f55c1768ed655fbbbd16540ea6095} +x{a8ad6c5a569b856997ef600b1b1728e4ad9ab8606d90619e2fbc336d4562b9d38498f69556e651b3cba7d61a69e48eb5} +x{abd6f511bf89d9c18e7fa0cf6c8aa0f028c31ff78b5d980136e1c528d593d259f1d26ae4c79576a47af8996396b76094} +x{a4479f00869f56b0b389d79ab9e23ec2cae1460593f6847b3f2bf5415929db91a0b94aefdf093c99870b1f66c6a0ae64} +x{a03f2529e611683bfa7977b08f27c9f5b1eb962acdaaa8c72933ba22376cf461a6fde95e4102a25908ce056f14ddf525} +x{a1b6f7748d7c2f7e767773316c7f4d489bd5b793cb30fd750f2a33a02b6db34f64c7a4180a05ac6fef049392cf87e70e} +x{93528f173f2dafe0114d688d33c4eb6b33753fb53c54f50c3c7b5990d873593dd6faa65906f1df7c7eb1bc1f6ddc21f7} +x{ad8996fb3c231514562b9926628906edf4dc6fdc714d35a8dece3a235c3f49637534166d1ab0715fd7bfc22ec5d17e26} +x{b6e5ed2b40848c59db54850ca64971759de81d0601a56d552060f1a6d98305ee97667cefacead2c3dccbcc58b75b1c87} +x{83fb0b6260971d174b14377e999617ab4d2621982a3d26555c27520ff8439e2b3585d12a61f34e1bbaf93779b89c2907} +x{8b6cdaaf5946847b1e7b7c6d2d5fa8ea1e6c2fc9dd21641b936fbf1b32121fb3bbf65dcba0df83b822e370c9c35a9a5f} +x{872eed4104bddf37251a144cf766ba3bc01bd27904b320e42b6122831087a7da0c5a5cf8dc6c00a4c8631f7a91dd3fa4} +x{919532809f2e0c285fbb90a52dc9e3d5004eacdc71622bcf8144376c0a63b995a2531ca0d18a6e0ca4df50044626f00c} +x{8f1d94624744ba2b92ef07663324c7784081e22911de67b1dd1780942c9bde1dd9667914e749829d76e87a1e86170485} +x{94a257f057123beaab8baacf6d3fbbd932ab05c84540273f3ea69633fa9cefe0cebf778da45661d9dd2f617dce9fb960} +x{8a33dbe5e72280da6e8d781246af15e92ebcdaa0add650bc0c13535537efe9ca4d51a1a083c952427d77f70954a9430c} +x{8d78bcf42dec8ef23bd0fdc515115cef37a89d3d2f5ac0723374c4076a3718600b8505eae1faaf854728ca398dfb1649} +x{952601b96441c611dbccaa960cd3657de437e4ef6a0458e4078c4ca45c90a0c90eab2189c6512691bb615ede1a0d835d} +x{b5e65e21d0b97ee6fbf858e270bc51d3cf2a95e6364b94eba8d9b5a5817dd345aca03b631b9d30c3fa018c482184303d} +x{938af819a7f9662a49dd1353191b43f7c3ed87ccc2b4472dba0f73fd9117177891d6fdd754605a0a98fc9d93b3132d5c} +x{a83a552c4b56d70f98d95c68c022959b1f0688df8d13631ba960fc0052b7db74a58a5e56e7389b41c05e0bc7d2ac9ba9} +x{ab68cb8be7456a2086cd3695e1bda4135da91894d2f9b49560c39a9ae07f7d91ac2d453c4be9497a7b8631ba83aa9cc6} +x{a5e60edcf107dff7d57e51c250862b3eb8c106e40d452a95246f2be08d163a333d626c377d1945cbd514733b5421bfed} +x{a72f4482408250365ca1648b6c9a1a6e0adde5bfc1e47933bb93bb72b4418e7782a13abfa550287aaa9137d44ca1d17e} +x{a3e0045f65097ab4951747733730d233df35935a08b8934148846c4aa57559c0cbd74de20dad1ebfc6fee1b26b94b868} +x{a611d5c5c00db0c6c98c41c74f5f4a6be5523e39a2ec3b172e4b6f142100f5cd700d7eb21d432c4cf3b8dea3fb981cdc} +x{8ed47f35a890e5238ae1445fb519646c17a17187370c10c267723cb3c95613da9bc4c9d3fa5dcd0de052716b2f3773a9} +x{86b93b4cdc3b3186d8ffcf7c75472b41176fccf3845d83c5e0172bcf6d04443ec644f8916c39d07e6a3f3e947212fe7d} +x{8290d28317930e75ce905397d9b203afd6bdb8daac8f654317c7fde3b61f36e67e517ffb43db58cb09fc67f571270119} +x{b34fbdd89c0fe771088ce52e16e1a166cabd4643e8848560f376c56f8265e478998c5dc236fd693509bf1615bff5495d} +x{b7c3cf81dbec8f4b430a6ce91006b90b8e90d04f5c299161e2db4caadede926ca3a082224342d37120ef42254eabb3b3} +x{a072b63854a4c2c6d003d79fd959ff4e6d2fa22d90bd87de6a08b568714caa6b9ee1a43766fb813e01882a9a48086758} +x{b319caf57dd380fbdc537ae09f5f59f9d30f3607d9ef7410b2c2d8d88ff8e64e66154c363299c67a8526b5f07d507e2e} +x{878fb941a76dce335d77daba3bc41cd9d1b145c4df980e5be9868b37d3c60421e3509d46ed0ff3ea01e8f6b717996774} +x{8d2262fa7cfa4fb5c1b5cb6daeadabb220f10171e4c041de13ad9b5aba30599ba9e7f0bf2eb19a446a3a6b0fe6720fcd} +x{b249b90858c33e11fa84aa6f412986a886743f7a8a3b81d59c752baec291ad58d9d592aeeb26a00103b8bac4f882eac8} +x{b02aa595959b64b09da7de5ffedd316a98fa7e913d442e4bcdc08a584302ebef651c22a5c06dfb891504c53ceb15e5ec} +x{9077c6fa2e76347814dbb1c3e9291b4536a5d23441db87ebfcf453b2a07cecdd7a8aed655a71c28112c826ef146a5b85} +x{8368584e6e9fd52e8dd9bfdb58087e380b99e9e8cbc551dd40d6d6e4e5207d9384ed6295e03704c00a874ec280b6cf6f} +x{b34d3cc4d5cd507fbcbabd2c808c068ef41d3b8efaf7e5302ead08301b4f79324f028968dca7e0a56d628718ff164c83} +x{ab9ae1b9172c4d067445552f7da039141ef72ad4fdd82462182d7daad8f6fd3789d2d40c8174dda3fe196f8665c48f64} +x{b95ec7ef32d3b0ec97b28073cfc2eacdad7c4a063d5f31c791169bbff0ec4496822624b67264002b1a2eba048c4fbd38} +x{b2ed832b474cc84c4d9296bcef944a26a5eacbfbb411cf9834010d3e67127c36759a92f667b46078ffb9482fe5ba37bc} +x{95004b20bceeaf744a40289440c1145b4e8d4f7f40460b38cfe272d49b2abcf99880573ea0dd9defd4991a0620d9e528} +x{90a921f83766c423cfd9ef04cc3faf208dad5d52b77af9678842b8db2894c33d2d8962edfe2c6e8b477dd6a89223fb61} +x{986217666c2fe4711465a2da8daa7d6ee44e4cf32b63a09aba26204b371f296b086f31701e114a3233799b8d2052d7f8} +x{88a94b40813d23933c86e6ebbaf36531c7de89fbc345d7593a62c355aedb2ac504ed1a58aa735c2fabf82e73bba87679} +x{8441f6b6db07ceeb3cda0d8649b557e082fdfd610a9760a811d0302bc9da6c3107d8a22a0c752fa20ef82054017c20e1} +x{854ad0cfac2b94faac229ec7792123fc54377e0d285570ef1435aec0e72d072da978c00d453975415550ab52e8284501} +x{b0ad4b51e7c698f77828d3c034fd9b177a877510d7552025fd3cd1c68e405dba23a7a72d921c4798a6cc237887baf967} +x{a59e691f1ae2265788523bb6ac5cae309420caaa54d4feaffa6882c1542fdaba817f497c5a8e30bfeaea3bc3a6ba4b42} +x{841fdbbdfa848144fc7b29520199a00c3398ef3d78ac67491a9c0f933eee22059f9a0446fa599df627336b1fb84b0145} +x{89a8c1e13030ff719ca22c939f4c84b1bdcf0ed11b6d0ff0deb1c3bf9c62640f80c96362e4e88362f7f7d69de831f523} +x{88b306b8323a1ccd933782e31e94ec070117a02a8f8349b445da66b33d1dca77e23256644904416751136bed9adc0d45} +x{a77b375d16d85257c5f4b996f5514ffa8a5ca31454129d2a25b4bfcf4309f27c97a71e6c33ec7fb9cc57914b3b7baca9} +x{a8098c4186848d9d9ba4a4b1af79decdafacdfea7b5cbfd3536aee6ed67653c6129557836252b22421f61b66c84b2e9d} +x{912569e76a70ee20160774d2956b70965689d19c8d64941313fcb0d61f35b10648860f85be7a81e90ef6975730f2f636} +x{b144b4047dd0ece9b3619bc04e7c971aa608d9143608a49c46a293df2bf2c30facfe4d39675fe23f5b7fc274d07f1b17} +x{89489bc8c8ef1224ed29d5f82370c6c3fef80ea23809673414309a1f4b2941acb049a0648c34cf0740f682624a660c3b} +x{b72f1a424b0c3609531bb9f5299ee2120066dd5561e42a3beb526f8dd32e89cfc51937fcc668834414b7c16a1516510d} +x{8a462a5dfe9b78315fb9945fb6717a27849785d98744d3bf798697823e1f8e6acad5581ed337d4d9e212f9dd96d94ae7} +x{a4d046c2d04306fba6c56bf0c9f546ef1a0b5c5762484c5de70e42d1b6a30d39cc80b22f66933bf3534b1d26fb6cb7eb} +x{ad8903641d07eb96663b0a2fbac3115127b24945a138ea6617e14545cec689dfd162422818cc4050a23a8df42cb08ce9} +x{b50cb2de4074397cc88600126c50da4bb2acbb02ef90bb3b370e1828c39fd78c17902818556e1689e5a7bca50237aafb} +x{8f5fb00cd6e476bed5b5562e934383cfc2f1a2b799e9c1f3dc2e0b0ccacee4734a31ff2d9a986e7c0251778e41bb99df} +x{aae8828201155e602880a4a9137d3d44c01d32b2374e4223576e8fa4ec56411aded934c2f3121526fe2d834e37c713d4} +x{8a074eb84f1ae6d049947cd6dc6bbbbada2aa4209ffc167fad467b44ba3926f3b6ddacae7186c9e155e233226c7fd3e0} +x{a85635c3cb2397e961c821cad4a9945bbbbf53c20721f9f1dba9a8c616b2819625767884e22757c29507f0ed73304c43} +x{809ac8f6102f411e5ace89bafee2b73c8a2f5340d7677c0877eb9de7b7d522d77b79499bb27bcdec06a2434ade31d40a} +x{82134713d5873008449747fb04026b02790c89d9656f2d2b3c22fcdb4e9bf1b328e4051200afd86bf2c7ceda424dc447} +x{96a195a8bc8e61dd270c9e1e2b2ac60ffcce8156b11cb2ec45229fa3151a6f8c1a796a752312ecebd5c5bafa2ab4991e} +x{89192f77e8caa08c0a976650701206da423fd3e4adf4c11c0b12cf693966e2d32a06bec27de11ef76cb57b1abee47a12} +x{89e7e76e55b1b3bde410d10c8c32d0f10c63e1284d4636f5ba73a285abcc9def139c3e419ddce83ea07a9f825b441523} +x{97d8551ed66159c461b1679ae0ae0bdc34f306f4cad6cb39b77999f85ad976f2ee2218f777e4e1368133ab31604df540} +x{941b0de89934fde7ad4dd6ee46bb1ac0d1228649469969fa9e016a51fe63f3a687b536f53eb70e867e11b306b66954f9} +x{8ad57d3d490a57d2cbada20ec81812c429daf3ae6ed372276c1c527dce09737ef4ba3bec464c44165594c8e552b95f55} +x{a6cd2543e60f86f7c976df845ca30debc686413418d559162597bf34e44952243906f249939d06d878e1d942500a7c4d} +x{b2e066f07c6cfd5e045f94f39f1e040ca09cae4b1b8e9cae55a57ca85301787e772f51b15000fbbfca19b1d28341eeab} +x{88242a0a598559686228cca4b72b47d503276c6d3f9270d943e43f6652ac600d91a6245ae3ce365b20d8121ffcd2c7eb} +x{b47776fda2429344019841061308ae5cc3e39e55355b4fa2bab80a79be35250cf38127be2d3f93eca196ef8bfa3049f4} +x{929c6bc299a6b479b7a5ea1a2c73cf7575161f184fa1fa536fed71fce16cc0127df765c9c3d02d5df15442b50e4d7c81} +x{87fd7e06cbdc2e1ffc1a0cf328870fa73cc1df6d4562508efbdd0d71d97866d8890f085e02a8845c8be69840e73c0698} +x{855a53160cd2baff4f69b01af39a8e6486aef4c8fba5b23fa958f8c4ae9a4779e3a171c35ba28a23b63d5a746e0def27} +x{b4e1464b63413c94f636d648ec6e9dabd357c23d7a37c9fcc3bedbbde7a94249c6dec2192702ced872856c92cc829da2} +x{aaaaa6549f519e43b3935cc1d9edcac6a32209ec8f9e9eb8598e7ad1448a4504d204175eda00d2a8bbeb736171389f2a} +x{8a6205892d6986bcdc1c3f8deb7f7301cb9be41a131c7c046bd2515e3b36f3253fdfc8609c80478dcc51a7b66b279a45} +x{b20d4402ef47455ba7d8f4f9bcf9a7c0d402396e6aaa14d3b1a6f1f5d0ef6705ba9dca6538d0e9fdacde8a5e252c786c} +x{a30880c55a04b15d14381001ec06340bf8d60f6c952ec50dcdd8222bb393f88f635dc85d99384a597eebde0693424d47} +x{b729dce9e0bf10d615d74931773c3dd81ec38050f597f7d6005def587526d4c587d7767b30f4108d83342dfff313c74f} +x{a1cced77186993b4215c66ddcb187e397e10a8ee9666986c6ef2f41398bedfed12c087430d81f4a608da46d275b9bc04} +x{862144b431ab9de9c4ceaa784858a2c6788b3758866a228b8e2671433bc56ebe9918a08a3be11d05a7ff9fc64ca882e7} +x{b4205cbe870360c5452b2efe87efe9b4aacb4f7edbb88ff2e414931a8613d686608b583fa9bc053ce96bb9930d4a4e8d} +x{8063a97bae4b2025eef8522d73a5e3932140a1afff478765dc1279b2bb8133d4bfeb256861afb213f2ace1d885b514cb} +x{a4f78536213b3cfdf5ce14fc845cb0a06726f80726740319c812c938417ce6346ca2cc055e2735eeae9b8cd1aafdbe9e} +x{a82edc8ff174d91a5c195a778aa265f35a4b7adb596caeddb566d86b15b4ad446ebc4a7c34889d0f69e6b820ae5ee2d1} +x{85ff418bd6582cd00828c5cc5fabbe68927d8cff526f435bdac63071707dc2f6cf0d4f916f2e52f8a341ad8e3725cf67} +x{ace11bad2e6d2ad1cf84b01c3f194c23d5b8e96840e76444f42a063ef357a3f5376e3ddd3b2031c99400c2f82afdaedb} +x{88eb0d46e735ddbdccc6d3763db910420a2c864895f03f3537dfe49aff54784b181d2d02ff37165b29eea18b529d8f82} +x{b904f873c0cb532c4633dace5b389ecca9513ec84562ea714d52e1d359c97c567c0267c4c747f060e5f6e3e2fa0f2c13} +x{b63791ae81e8abf907f7a6a1091d72e37b655cf1830a6a0bc634049c6f9f4afb597c4389b9636c7fe5e8db52867b26fa} +x{b3294d1fc9c45bb0f080db57cb628faf8e2684876c853390a548ee78878697030c932b9e5b5f77ba54cdc4162b439400} +x{a38e9faa508b2feb73b4040a0d79acf3bf39bc5a62ef30bbe43c8073a98f12ff9e286ff1490c8c9aed3faac16957b7aa} +x{ada99e8309ed1d1fa2621693c8c9e5d618218dde92888a3aeaa0692754e7dd94a4268afbb75550c2dfeaf5764adfc0c9} +x{9232ab744f72c5ae7fc674890d07027dc5541eaaf8dc1edc8d8fb4d5c763053a53697d9e5100ad9afa1e9fc888cf1197} +x{8b87270d542706c9d0db167db7302c5e3fa81642f8bc707eae2bbbbf731b3948dad68cf0dd1c1d69f61a911d1f8e87bb} +x{a9664311d8ac973f6ce364f7033c0d710a98df5ac6d7c3d1df70f78a33701bc0a9a3b2da1113e1667b89aad41c7d1b4e} +x{a6d02f30da6b850da8377c718d2d8ca90d2f1d352edc0abc5c0af2435dd180be4d0e1cade63382c990b8ca930dccb1be} +x{824c864ee1af1691d009e4c7d1d201da8a691e93b62523e22aca2e4b8a5d9502b6ff941851f8f46a203589e093539bab} +x{9873aa35cbb7235aba6dfb17cf8e046b84824b4598acb8cf63a20d26cb242087db54aa6d44270e948fa6a4bb3b1a4aa5} +x{929f1a544585afaa2df9f393129bdf3a0ad4e981146e9c9fd2ed89b1ca61e865c88e7347b07121684229e778b10be160} +x{97952ed5cb54d0451e8cf08cfc29a423a33a1ad84f7e02b00cc5de88a27b23f77428e4c3a49a022617229952dd27ba6f} +x{a49a6e47098ce9ed5d432aeed22a981c05dccda0eaeb25ddb2452d2c5c39f53f781b0840a894cf70804ed300e90c95b0} +x{ac3d8d959683d6db8298d6437aa22378375c5b3a4fdb416925b71646e6f7b0087d0c6e65d4f9beb0ae72358e99c1f5fa} +x{aaeae464badaba7da40bc59ac873e1f07fceedbcac915f33a94daaa72688c3c49e5717300abe2571446b010416b2dcd5} +x{a2f6caa19c0a43d3d6a3d1af5e8bc33a0bb3a54b8f5f6d434bd9b3328cd3b6b148168e554a0af10f47d43943dfc6566b} +x{85b48b545c089325b055d2b34666d091a07411272aa6ea074b550f9c9bffe118b1f2f9d4cb903e9caa39fe0cf1ffcfac} +x{8db60a483dbadfd71f08f6460027f9dc608df180ccded4a2677e88aba24e72a80d55b783ce84aa787049dc1ca9a53512} +x{b0888ba78b37c36f1c61064284aae6021722ae38ae8f64e6da2c3e13daa9396eb5520b910433d0ee25f36162a5d69ad6} +x{8e6777565dd463b41aba4c7a63df9d1ae5b6b5b1d7d336f67e86fca8280bfc90c1d7e95ace60c34e6b3bf55d28bb5d2c} +x{99275fbc4baac72896a20267af060f21ee65293b7cf8b27d6495d559886ca71a9572f6699e58b63ff2433d760ef2e31f} +x{85b16af1b7c62f0beb9c801fd36731dde735c9ef6bcf97bd29dfddf2ec438cd5cb568d914c7701080a7434841ccb0f1e} +x{a87a5cf9d7c130418cceb93f2b26747d3f70d8a6eb8ff0cec28ab969bc92addb4fc5dbcf5d18dbe21e150ec78a755e07} +x{abcc8ccb7854b76b80a0e6e4d390df544ed8d09daff0818a71e8adc7f4b6f2267769017db965a404665032c46766040c} +x{b90e7b9ca3237916f211c02cc1575d8b8df43313f54feae4c78944bca535ee95e37b63729910f9fc4dabb907f3e73acf} +x{aa968ee64eedc74b51ef13248e6ad4ed8fb96eb5e499602889b971c17fdb126335f59f98952de0e5243a0b486e2d4e62} +x{ac49aeaf558fcf8086def396c8db61cb80ad416a94af50a1b583c5814bc7d54bcaa72feae09a2e162ecb1d342608db10} +x{87ef03700a561ad0299e32ae473c1d724a391e3523636e64c75d8676eb2856ce8189a9e0279922fdd7a9365c2a483e7f} +x{b949a74107e0c6b93cfbbad5dd0a47f64364e5c7ecc55cca0fad5f85c4bbbac48555144b7f676603a5292fc1712c1bc0} +x{869029064f1a86ed00ea442bbc5d330c7a0ae4a06b5595c9a5b57934f6876f3bb96d9dfaef1e926428d14e450624be57} +x{8379963a7f188739e953a4e9799ab37d5c49e4b8f49dc9bcb686371219a04dfb6c55c441c59b2ad89d7145ab3dc08631} +x{8460a57fed7db8691ed1cc69c66116e1544e164478613ade64eea6b3926c99011d2cf386e0d8bb1681ef9a5d556f3c88} +x{8c9cb7d59ef8fe907074382c959b661665dfe9fe4fa328fc93752cc2cf80d431b8799868eaec777013e67966b37ee0f3} +x{85e7cba0dc3614869ccccee0ccd782554ec3821a8d6056daf10285973bbf548f7c7120627d1f0edab7bb29cccd4fd35a} +x{a2ed49318a58eaf3d2211f3c3573a522605481278c5d5aa06ae610928845f030b3455f8cff3cbe3afe08e8bcb5839b31} +x{898c2de6094e5d40e725faa69caae6ae499f2ce0d13ae0fac39291869852d5f9711d447de374942fe7281030cf6a4cab} +x{b59b4d2f3da26779b0c3c33493fd23d78d956e688eec9df7260e622e2228b6cf30b111088e6f1aec6e690a4ed80f086c} +x{af9916b2d4c305532662819fdafb544f4d4f74a3bf5738cc31cbfc0018310f0848ff749be71d797fa287a7b5e2774e88} +x{a6eb7c89dbc44ea9c7439d0ff2a7b302a5182020743d0e54d84048ea668a012c0a808aab9c7809ed9920f75bc1df04e3} +x{8da5e1e8173da17fb2527d049c746f216013a0cd3034be30fab97314d67f90638efeb394a504052f84dd3314f5a6e1fa} +x{a0a9dfea9829381a8a8be0dd6ae9df703da93546d93e5a8e2a0895d0149894fc424c5735b32517305a575936e6d0bd41} +x{afc39ec1ce487612179154f7e36e485626cdcde18593fabcf3a4fe5743abe57d6de0938d1e5b1f45e3861cbe9cff2107} +x{8182fbea440a49e1f1deda5b1973eb1a6555d8bef74271ea56489508ae847db9f2a2e0064c8f25c6acad4893ea3556a7} +x{b3af977dad139de108294d0fab8dcc817abff1948f6ed746e7f6aae1ca15ff628c4d9ef871481b8a05f3e4eb060f1569} +x{b5b23529ae9ecdad0030248957d7135bb0f2212654f8b06b921f4f2e9f3374cf8314ed08a2020a2b81307db91bd31325} +x{a05aee8260d4e96ffdfc3764c6aeefee513255daf8abe5df1e455ad9dc1ff826d0e57eb59352d6f3ca805f854ced20f9} +x{b3d57b48afdd734921b826b6e165fc7ede42a6298d795cc3df87a659f2920e21f6dcac0fb1ec28e145e415cf1f75f15c} +x{b9f8e0bb3f230f87496e6ba4ac104106426c87e1e6732863c28f4ab8b6d752c00d3d935f5e33669045c38a61adc09b5f} +x{974e00079eec61f2c2dd74a02da965d367ac1f4cc1e128af147fcfd4c6940d2e0cc3babcbf5514b2beae2882e1ee3e14} +x{9539571783ff73f8b5172728c3c5af82a1f36562e06e4e79fa8aff690eb5e4e6cb53f0fdc2a1fc2161e638313d379b59} +x{af64b9701e9073ef4cf2b5345190791b0704dbd28b3f626042e1a0c4fa215fda937e996dfbaa07c65d9ca5f96b8f61fa} +x{a10eeb3b9cfc1761bf9a22bfdeac39fa830cf2c52e2b70b82d99c005467243948821eafec82310ff761171ca4a5eccc4} +x{86a5d807681cb61dead165e7b6dc51fbc4d87a3e1ba203d0850a88fa6abc8c443f6e59d0367375e04166319bbbd70653} +x{b68306c1ee04db2c41a5599e8c65b36f27e2ef2dbf74153f50f3ef5a8566a8522268b7131ed8f6244a8e9285b8cc1363} +x{b168681033750e64fab4b54121fbf6dc64bb214eb9bacc00d199745354517e3e1b110ad882264862a91990400eb0d832} +x{a5a41fa143084456fc9240f4a0b0959c6ab2e4c77a8d0fbacbaebaa7762d19cf25e2bc9bf642c3dc4bd52ef9a125d9b6} +x{aa1044716ac0ce361fe3d7cf79cda8ac7bfd5d75c4c9a6385e54c0f058dc37bba7a98066419340119ea957835e3f5062} +x{8316f76c23a66cac4d13966c641a8b1fddec4a8a1da95e418d3c29a318123dcf62680b3d7d7fe989200f087da31f2d47} +x{b3520e472e5206768de36f16c59537ba7a5f548f1e606ef341be57c61c69b36dd3e9b6fe7fe8a36269e9f6b3b6860612} +x{8e58d534f013ab17fe6602ee4a477187367ef4c48dbabedbf81433243dd5138aad25aa483a34cc96665f6fc2b0c43ce1} +x{a53b58680e5397ace528cc8b2fe5e9bb72c40eb450c3fc5ca787cb95b1da733b6ba759b2df4e9e2b774e0423b57aa0c3} +x{9124081b9f24d896cb79f75a667e58bd97e7150bf245f9f50d3556e1108c47c73bdd153e65d75641b324ca75ef2ac4b8} +x{a6f86b395f56ea59ddb0f2329aaf2fb7ad7779b83e9284f9303dee3b29dfb318196f836f8abddcb8368a8e7c2cdaa462} +x{993bf4eaf832dcbff2f8c1e8009f779189ad123bcb12ea9c46e0db23dc0772fb0cbb89796d9dff8cc3c96a35efe33b6e} +x{b3fc6d1fbf08661dd9912d9f6badde61cfd6ad5a8be87024d9b25724f51d0fdf746b1c7ef08fdccb7453563c32d3b753} +x{875284793bc53f9b3483e59ee05094e45c9aaf49d7a94d9d541e1426fd02a0271f6619c1d5dd2ef2ffbcb1429ac65ffd} +x{87b53aec91540ea4bc0d89fb7f863a095e4fe9644b433810e75316c0015663cd0087b5044bcd6483b6de46052439ec61} +x{926d4637eea543e4d3a97963c9644632bcbce065cabffe5af044c73f43294446dff6f502b0d43978121f8783e789b784} +x{83709d410068ec4da081a18cb1000401741e652d2db38fecfc33c2b124cfbc2497ad490fae22c460905066eea5a0295f} +x{805b9aef8db25a170b664495fe825c92c737c2c2d42c982a0a87161d3e7ded0a75603946fb68e2dc05525a2617c950da} +x{844f8360c42e3472b8e1f0515be2b17c57263569f40e931e96ec456338f89f75f41998eae7d78a32d97f532dedd93ef0} +x{8f8a897dcd7a5a3c7bd68c3e9d7a742c116273337c15c7518264dd403467a26647573837ddf08644165fe87dccc8a263} +x{8b83e1982cf15c9910075224a8009c3363f2c1c914f2d74dc88f2d1acdb17bac75a838a56d123f8a6746b611b1105b18} +x{b27d8d49ee63ecf6f4ced8bd515c4fa27478e8a70344ce098e9cd3e6dc4712f3d24e47f93d23242571444cc726511865} +x{a3252f0ad64cfcdfe330f8939f8c9ffa477ff17cd45929f7fa62ba25a9cb5af0f83a070d141f315cc6e82f4564ec4f64} +x{a6bb5240fd5ab570f4b5bf8836c34241933973f0fcb537a124dff7ec545c1c55d13cc6c23ba1b7452b9aabb2b66dcb37} +x{af83b1ffb771e0698f9b2a7ffef6df235b18e89aa90a5358abe82a2d9b077e08dc1ee11869a53b95519072adff5d26c2} +x{a990df005e123fc73bc97c79ecf34dc029265a66b530687192230689aa6a96d4fd8d71d5d6292566d195b968184cc3b9} +x{b6db2b4d155a26adc7667e84a98ee9e2f57ba6eaa1a0b1d966502890a060ef5acb1b9ac0e0cd488b0d70f356d4ab1575} +x{ad23f7a128230b69b92156de6d12f3d7adca4f17e4fa95ad1e4aba93afeeffa1aec73cfa543b2e9387823c95db18eecc} +x{b021626d7882e973d1dc142833d7b78b6ea7ce6d7c5abc4e5c4232d61d9b8d540a01d0c3726f253702335a80d4873e3d} +x{97d37b022828ccbf20450561fcba5febcbf2627e2e0e61f0259d9aae32b63efd795e83b247da7828ab3a52f40b055c8d} +x{972829ba8a181be19333088f332b80506d5519853ae22d50a1be9fde4eb181f1a00b3e695da8872381d8c663e53ea752} +x{a97af7a23e81bc3b56d245bdfe656ddfbb30b00dd05b358b29c442c6d440c84c46a87b9df9bc7706ad6eb004091783a6} +x{b062a29f4441bd839232e2dde99e329f7e6c043f7f315ee156852fd557987b04ee911ad8130cca4c49cc8e09ceec6e81} +x{a8239d71d680131c1d3857e13d53f11858d81a92217b37aeeb19b634a02441082d966f0b66e7d3f330402f8436ff8424} +x{a1ffe6c1a89d5368bdd0366d4f780288d7856fa6085e85945c382d40931d2748b2db1cd443f574847649376487bdfebc} +x{aeb35f89d305071d9aff9cce962a0320f84e206dbd927a5a9554d5b4e18e65734b1e7c81d02e245b120c1f7566d9fefd} +x{9935f67ac4d2ee6a4c513e46220d67d5bd6bce78e73fc3acd7e4b4ea895f7c99d44c3ca6400b07c925e7c43c97805301} +x{b606626eee2ca8f22eafcc063b961b78ad58e67295f9e81ebebd07834d5fb7b094476d4cf4ba977fdbcee58b8ff91ec2} +x{aa4f50a49daa01f4e2b92fd58f30029c0692fd529cb2db3aa3a6ea0f990df5020ba8c09992cced29d50e88e77b9eef1f} +x{90cb4f0932eafe48081aaeaed46c608fc0d5aaf7cac29a0f28e7a1826882983136f5e91f3d47b147b1f1d252a68cdfa4} +x{8610084a4b1884b49f1a4785c8a485cda8d0719510e0013b31810db813dfd115b31eb1c3e29ac17233e903e102f2f858} +x{84c5434f9c4554d0d5ac44dc84142ff00c3fd20ddcf207c1b899399d71fd2104d408fced0b58d3673bccc56123e7a62b} +x{a186868726ea2fe22049b1dfaa62abdb42ec46ae860d2f60546ece07db5402383b6bf3c7d26781c393a611512c250b38} +x{b2fb64b19612176bc0961538c8ef5d5d167c98f3d43747d95fe2f030449efb61a962fb0562bf4c42e7bca96cca4bcd17} +x{907261678b16023479eac3a3d921374a6a42ea163a09e7a975627b23043011711f7729efa8a39eaa8b97f9d19a336b4c} +x{872246987efe00f8f6a24bd2eeeb6b431204c01f3dcd4ece3b680f2bc164fb65f671551c9611d892a313ffae8ad35bfa} +x{8a35e8cf02a79ebeece284cd83d4499b10e8f3bc31f0bb058dab05fed30c7ec1d790a1dcf22685189853d3c8de81fdba} +x{8c4251af4d9e27118bd59aaaaf2fedbc6a8d5dde289f66345ac64bd566d094e961a6860238f669e2b4170a85022619f1} +x{9798f348d7bdabbbdfb6d2345083ff74ac58db16a6f3d68cd972d8c58380f3f455e589a91beb866d6ee379b8801ca20f} +x{932552f80b6a1a93dfddde98362f1f6f2b753f3fa05d6a21feeb47caa44548c8052143e3f71a21189e22bede3b700fbe} +x{82a0a28b240899e36db8f0ea641a01225c0e148dda443929f1f4dcc7020dc64a56532c0e0c754f458a78292e67b01098} +x{8d131e2eaf0f6ed5b293f22a8523d3c99f7456a5ad866100ad11141148e4825bd041d6870af2db4b6068841ec306181e} +x{a7281362b96c621f04e4f9dad0f09f2fd880171649efb46381ac5846a18376018cc43734971f977d18d1751cfb05c709} +x{a5d0207289aa93655aff791bdd430ea25ab93076d4295a256529411b4a55aa25ed8b3433086512dedd0255ded3ff9a63} +x{8703dcb528278c5c65b410dad0047bf2257f988bac6a11f1fe55e9d9794fc2515ced5f9863262d13daebc487ba4d4cc7} +x{b080c720c764ee177c35cbd1e29096e54feb2c29c68c7b5279916fc32177eae21bdf8328c7477e7a7feab59182f79c8d} +x{a98842b26ac4212e620a9a09018ce4573c02bb64a37c4e6ba91f972744e73b37a5c2d3f84c6cb59bc00ee25915e8a918} +x{aaaff1c584152dbec4fb2b50b91abd0703853a8051cbd3bc536e90aa3b65637ce77ab3dad4e9d6e8f3def0ffc920f9ea} +x{8069968e491d68b934b27e91a355249b2b1999cfa2534b7cafb7adb7a333af7920ec7b628cc5725630c39462b967cb7b} +x{b4da9f8378e711b30c3dbdaea9dd95e0d43420f7498f12181fb5bd3743392d5b5c0f12c1121bc2dd6f325a3d4c268bf4} +x{ad234661b80488143b93ee2efc1ab8eacc50a137aec333ea8141b426e5fd4070c968618d1a875947e08c8428e68c88e5} +x{823f71e9139e4982ce3b5dc7242a61876fbff25b3e9c1b532b82895e8624942f5e57af2c7819a80d716e06e04b921250} +x{b8f32a192438bf2dd454d0d9b831630c54b65a79a79611ab00386dfd99a80e58101df5455084d91a9a9939e108670214} +x{a5f68bd025fe3fb4921de2a081381870d41c85999f1754ff66d270a74794c7ad50d2fc01cf01ae1ef63247528e02aab4} +x{b7bb0c34c3e5da34302e937ca40a53c980de5f43fab50299f98b6d44fde2603355845030a29c1593cf342f85a8955450} +x{83077ae296c08c8aae4bcfc79b2828403433cb6ac48bb5c48eb85d4ce09f61d916947354e49183fe675c0d7f5905fc4f} +x{ade42daf027ed0939428a379dee1d6f52d4d69bd8854a5a5b166b8b1715a5f51b3d7281a0a2a4fa86524775c694deca3} +x{a327e8017149cbc021a583348c07edf1699b26bca4e07b6f79b1bc9dc030b77512f2968ceb47c8f72c81a55dbc293a58} +x{8dfec6255438a25f4076480b80786a58eb3e5d63d76e30f95e39e590a39d82c405b33bae71001ce647a1e3b3822aa2d3} +x{b744e6c66ac37ad90bbdaa8daef65b66a41d6f7c7662e86db742a7d0e52e8b3e6de185c99b53d1d75bd120763d260e1e} +x{9954e4646ef602352b8316de9b9186ebfada10718712d0d7ce646dec72a4840e288f6029aa0c2a0a4cd1749a3fc49760} +x{aeab8fcafa3f35b2b6585c7b573ab001cb1b1eca62839430010d69bd626bd522a1f64eba0a72d7e35e6081f8f24f9ecf} +x{a92ec41af6d290cd87c272d30a9c6afbde28daead0266987cbc24ebb3d68793b3e3030ef1c0fd6162e5d6067189e5b0f} +x{870da719fb854d4e97bce5f84994ca582a7973f85abd172682083bf974e07ab0217c56dbc8668d978510ea6d49ebf0e8} +x{971389abf73b0b18a85a9bf1a5b78ae54af0e9dab2810ff2a235b106ec2093f31c6144b6267e143b79ffcd4c79ee4902} +x{94f929a1834d477597f4d1047d09298676948e7136cd7554de2a5bb89d947c476fb3a9391adb0d804f0304bdefb2c074} +x{b250f0c4310487e162a6e8a8b2362a6fc643772838b4fd0b330502f913b04ea55c88212e83ec5a8103408511905807f1} +x{a3b0ac5a206ed5a9032ed5cd213751fcee66c7d629fe473f09457b9da9142c957ed39c0090c7eef24397669b0d361f72} +x{a151ed4066eafd3ea7eb646def1ffb5d14258a603045e37cf2c8cd7248700e7e1ac3e7f852e2fe7ad5691eccce99252c} +x{9711657a493af933761b8b848b5f09fc872d4ec85e3567c40bb48cb4f922627c0088d173d835b115811aa095dd8e5d82} +x{a5e7f4a06080b860d376871ce0798aa7677e7a4b117a5bd0909f15fee02f28a62388496982c133fef1eba087d8a06005} +x{a821c289f883e5a9bc76076a7cf6f6ee4dcac14d064c76d4a98756f6bfff85cc1fd45f5031b0c491d393869dbf37ab58} +x{b5716cb955ab67df3f9035325e00ddf29855f010fdff09c708726aa08196807ba946cfb70a710dea25ee8209af5cd38e} +x{b370f1a22a538bce3680d8094eed26372201563713ded3308c2fb4998a4b30f92f57616f2d859ef0806dd6da776fb12c} +x{b144c4ec1624abdb6b0b006091ef941f4f785751d114d6bc39269da9431da9991fa23e2b0f333efeb38770a25523e820} +x{919a189511a5d7590ecf34fbe5f36d1f92244db0c06aeaf6fb56569abf30ffb71cfc7c42913a6483a57a39a0ccc90b9b} +x{b9f830b001c45c07a83d1ef9dfab49a9c76fa9c13d62a0678e4336c6031bda6e07223b87a7302decf09e72364121ebeb} +x{af014b3be9c6b92fe8ca07cdcc900c86611e35e32a758ec468c1afc13ce640a7375482d51b8681903372f9c5ecca7687} +x{90293185a10067becf6f10dd5bbcdad97e189504d6a1df630a1028103613c80c15b29e0fb4109337b611c66e09ffb050} +x{b23acf81c4f787dd63e3f8c31ba9dde38b44a455a18b04f7325e09bbd439e5ee2a75b67d3673aea6d2476998bb2adb0c} +x{916efb4510ac05f36693e543384a54c875c5ce811f3bce476a294d00e4d64f27b5ab4d4c4ac7ca9a2c46a2c317326f72} +x{b76b06907a9876f8538e3f58758aa95d23d449dad3caab1e9708ebf51b09b9edc29d04d60ca3d6fda0e2823a4437b0d8} +x{83c0ccbf5f94afa5a7f5e7b3fcf7eb2afd2e656ff327f357f1a4b36f630fa2856ad1f07e51b38ee3a6e92d8075f85fbf} +x{9527d6a088468097aee5e3f0c7cd469aa666f096f054432d2a34df504217dcc5f70b45c7a4923b84b8cc9ca9f4a4b23a} +x{aea834fd07048eb22116cee9cdcc04c0ca900b5ae15e297091ce64c5631cae870ca13323241cad96c81d1ffdee43495e} +x{a9973ccb82ca88c7862fb684abd6814420b5f763416527a4020ed15adb1d5280b71d21e2ef280bfeb3f64104fd0410df} +x{a85b5024a160e7ed3467930b62c1346166e3bc35be18c9a891f19c5f343d2ee3c2702e987a4a1b7c9980dc4e099a7494} +x{a279c5583f81d1b1c1c7f6652bef822f0ccd3364de4dd56132a70fc4dbde3e006c17bbaeba9ca3dff6ea0e0d7dc0bede} +x{929713e325fbc3b9012896497213781dcd7edcf1122ac2e71bf0eeedd4b72090f9b61780f159f89066aeeb29d1b0bbb1} +x{8da3be746b47660268d2f6d74236047f5cc5291b68799ac8bf1bf34e1d035e4c203ecc415e71381a11b3b5884c0f263f} +x{98742a51f3d4581ae8f22b8c8e195d4e87017f502c54862d7b76039958d90bff3c02c07cde47d3e0d6877b4937648bef} +x{aada0601bff5813cec1b849893e2e6db5fff6a1beb605517c77e4dd2722aaa41510e528ccd072a76c10f12744d2ae172} +x{84ea9203ef391535d61ba30fa3a2104acbc59318c2da90d5e6f5df40ae8e234eeee714af1b4715200c471b00682ec3c1} +x{899047e4fb2a2824bf3c612a1040fb6c71c1f7ef0654177144d236455b198ddef003b64bb7c9d3f0da78c32800956e2a} +x{a10af4ddc28d1794167cef4b751022d8f57860a2618ce05a8b5d87c95584f1c6ee2774df7de60768ff81db4f432a3211} +x{93614feda46e422eefb30a724bb519c7c2ca3f743546806d4ba6ed59bf8862fcbc1f3fdae9f17c127255eb6968780b17} +x{a1f5e841c97a4ff52b2fdd00164fb77d6960cba1ddb3477d6b522a05521aac0796dee38c7a43e10d9c1681b38d8892ff} +x{92d976efe2f26ed00fe867c789407dbdeec51921e718b1a18461f795460d3bbe7cd74ea817d1bdea149423f0ad8393b2} +x{a2a3e37af6c9bed27e8f5d7dce5eeb9cfcadd2a942e7e9f84c9d95fed6b1dc7ad7d079f3f5bafe2d44dbbc67d99b0a5f} +x{81ffe874b2efce3625b5cd61e0d276d4f460597907a7f862a0a17a8bdd6778f7884e6a9d80e4c5cda224b2701655c96e} +x{8d2a05160ae509264ae64e056518a0f3ae2c197c2450d14d7f30a85db58bc7c562b5d450fef287d7355e63cbe558f44b} +x{aa3284d0d6e875fdca38b4009f57d7185dd3d8fe58aeda254de4e7df3932d446b80706f754c6d36ac230877a14da2160} +x{85a98b4ee6889d0fcec0fb23025e62a0bd6acc92df9cc2ab0cce430d7b096dc3de22f79a22ed727a5d15ab9257216469} +x{874bf95557e0094df23d3f1cd41155862b755482d3ef1a2707f341a1c66cf5f3a9edf52df5b167940eb1c1bfc565fc67} +x{a8324649bb016038ab07cd91879772df95a952973a19c34cae6af7e4fefbc545a94d12b23a2159e48bd049f57f5e93fb} +x{83b8c1dfeb31c8b61d8ffebf44d54a36af52f75aaa0531322020a6b04df9adb53d9a8099c521844f4e88d6e6aa36ab3c} +x{abbd6ff3a89fb9e14ad17978f088eb4b60af37b02835eb8d3fb7e7ef59342b68ca6ed1fce928446806015951caabcd6b} +x{8e9e5baad181d47ace0c6b5e9d639a2dac88cf667c0a90610267046563afe28b7be6ccd887940b5261579f7dbe6f920d} +x{b3f0b8cf0b5a7c6ebcf0c46c1be6d3c1d0a5a8ff5639188b3854206f2912f384e9f4d13fe1de35da1ec0ff4cc0917c56} +x{88132b949584e6d96043228bd589ca6438c594fdf91b6640f65932f379cc94a02e294e9d27be40e3d889b63d33c388cd} +x{b449cff96f2dfe5bfa879b713b081d48898aebbcae376c5cb32ab1f2bf3a8eb329f3aafcf6c251b55c3622b6a5036a10} +x{b705ff85c991b76587c7ddf9dfa144b25d985aa85b0b79e3c21d2229542388d9e66bfe8aadd511229e64c3c14e06bcc3} +x{a1f5ecdb881f30fab751b45c17326d04aeb3b67db946b37c3ec9ca8c5608d949437612e0d08e96c393d5b8110d9fb619} +x{b53734cdc0454bb8e02789f20b414e3461a73a347aeed2c331ec625f35cf07b44e676d568cdfb1fda7e3bb46fec2a179} +x{a496d69daf6ffeb228106883ddc885a7df0290527b4d9ff902b446df6b220842c0040f9be9fd08d8e2c17731d9d6355f} +x{89532cb6bdaf72d8e430e8baf25a94ea475328edf876659039e71b1804b407964ec043508a81fc23659ac5d225f81752} +x{aadb20736b040c633d4790e0d30748ae047e86efcb5008b586f1bf2dceb99ff4659711b2933f50096ff2af89d9bc187c} +x{b1d65445dad3417235c059d084f23fa5407c2de501f3501729bfe9952e7a03c2699f7115130bf3cd7f3878be3bae963e} +x{89cbb614041988d7f15e4a2cd7552b4c54afa224839dbe5076ddaf4ef8c30e661af4080f64245588fa4ff71314a1ddb2} +x{8dec221ff90b27567f53bea9638293624c5fc64572a32ad93651fa2172b37132195b3d4de8b89df6809d312ddec8e554} +x{89700f819dcd49c6c2c6586333773f3a67043bbc780c9ca4f05dd1a48035138c47560dac51c7e0e58cf238b910bb8001} +x{96abae0439028d53d35dd4c0ec26dc8a4ba1545726e25620f0a5ce3901ac7b185c9315ef46c0a29e10d8f0a0c7c0fac3} +x{9799c21074cd36b98d1e37f66dc05741c13f463b893ed5bce9fd2db604fe95c65f23d07015eb53e2ed9fcece39a2145a} +x{9119782077a9ae2564124374d877b0835e79d9de524c6cea6b1f3be2e1f33417040bf62bf064200c74c086d9dde10084} +x{8075657d35885a8a3bf5ba8087a5cd5b089f894da72ba7150302a0e6adcdac186eac45cb79385445e3785278711f2091} +x{8e2f9aa48d8a3693c73995113ef3d2bb8c2c37f8b528b2e8a9e52104940b1fe3bb6df234eeee099d445c59dab9eefdd4} +x{99107a6ba9b0717141c9c36c8a2f99a293e0909da65f20c44594b7bfa519de73c2c93e3ae651f94ee7e7787d16e39ccb} +x{8239a6c06392fa5854bdb4a3703b7a38a6ee44df66d498e0c3afc7381a61f335012835bef8a0b558630f1e1863a11731} +x{8fdf5c3d7bd9d5d46a0714ba236ebe0f9f8f04db756bcb30f49e87f758a28f7833aef65dfb1bac9963ec945c9ae69314} +x{911419794a5b85e78452a8e156eef297d5f10a0b5488b215e8f7d9a757f88eb7f2febcbc4f2b135181f9a0ec5c8943a7} +x{acfcbcea8262c87fa851c4d6601cbe9cca7a597425f9aa4f002b518153fe6e9a84da7185a67bfef2be7d07de638617ad} +x{b89acb00a4d16e56460ea0bab3f3f5754eb79220d60fa8ea6d5588f20d21a4d16f38107994b2e350d52d5565924c88c5} +x{b0bc0f14093c805453040cabb3a58530b5c7d75dd8ed2499c3c089ae6eaff8cb90fee7f4fb4a9a18ab43567eb06b8bdc} +x{ad3d2da3b07f0d9688fddc6e19ba30dfcebbd3a5af3a3f791e056d89650bf98d931ebabeda4338d8196e86856b3d109a} +x{b501ede6b849057031006a356607df1690206fbd337cd6ef1db764aa257ab09818af900587a751bbfa7d384c0d542c35} +x{abc8caaffa2225d3df4aa9020e517f57561efce8bf1628d1a75ad7e5c1efb4469ef56e2fb44427c740aba293e1896992} +x{a166cccc21273eca781e33dce0769633296101a1e43515a120f027d06e110e5eb7ff1f6a1a324b9c9315bd89e12819e3} +x{81c29db55f0957d889ea8db8e2a15c2d7020d50f1f9f60ccbe74d4a58a9aa5579955d4a73065614d556e443551f1ec51} +x{89024f719d34cc220acc52d4075eb1d69a7d3624d6eeca3c0e6ceea66975c94947f6001189b21f72980f883a9f73eb2d} +x{b53fddc5ad0059ee152c0d096efef63293999e54a66c867432de3dce6043f7811dc58111480e184a99e4c7f360a3de70} +x{a21b66e7db725312ef2929a65d18cf60ad3f595d8d3a76cbbe419430e4e6a83b489343edd714f774ef69b07771ed57cb} +x{b177aede11e8288c8d2c7e274cf1b5d9cd17f0b965455d42d014bc4645b93572467c96f47c8d6a0c98c12730f323170a} +497 +x{30cfebbd980ea1c8f157a34abed82f3a34ecc2594e626265063e565230ae9193} +x{91cdca080f9cd6ae0bd15b427455053e12025843c2f602e0af70fa0e5d18dceb0bc46aa1a7652b54cfd8ea97c7210c5808d7cc72b82be7a2feb57f69d3511e4590a665e45e0e0f9bb43b05d8a0167fe2a91f4aa54c0a095c864be6e2b1e43081} +-1 +test-fast-aggregate-verify \ No newline at end of file diff --git a/crypto/test/fift/bls_ops.fif b/crypto/test/fift/bls_ops.fif new file mode 100644 index 00000000..c7f76d76 --- /dev/null +++ b/crypto/test/fift/bls_ops.fif @@ -0,0 +1,215 @@ +"Asm.fif" include +"FiftExt.fif" include + +{ { drop } depth 1- times } : clear-stack + +// Map to g1 +."G1 Points:" cr +x{7abd13983c76661a98659da83066c71bd6581baf20c82c825b007bf8057a258dc53f7a6d44fb6fdecb63d9586e845d92} +<{ BLS_MAP_TO_G1 }>s 0 runvmx abort"Exitcode != 0" dup ."a1 = " csr. constant a1 +x{7a6990b38d5a7bfc47b38c5adeec60680637e8a5030dddd796e7befbec3585c54c378472daadd7756ce7a52adbea507c} +<{ BLS_MAP_TO_G1 }>s 0 runvmx abort"Exitcode != 0" dup ."a2 = " csr. constant a2 +x{4e51f1317a8d7981f7bb061488b6e6528978209226ded49b02fd45fcb9b5ff8d33c360cd6db9661143a77edb34aac125} +<{ BLS_MAP_TO_G1 }>s 0 runvmx abort"Exitcode != 0" dup ."a3 = " csr. constant a3 +x{0ca4a2a9a055367caa8c41facaae4c1f28360e2bfc70182904ff966011de9c02e6744bad6b0096e7ef3f21bd972386af} +<{ BLS_MAP_TO_G1 }>s 0 runvmx abort"Exitcode != 0" dup ."a4 = " csr. constant a4 +x{1473aa897a1a166ce6c1b1d11e2401ad719b9c03f3a86d8dd63158d389667d66917d3845414a23c69ccef01762ec78d4} +<{ BLS_MAP_TO_G1 }>s 0 runvmx abort"Exitcode != 0" dup ."a5 = " csr. constant a5 + +// Validate points +a1 a2 a3 a4 a5 +<{ { BLS_G1_INGROUP 33 THROWIFNOT } 5 times }>s 0 runvmx abort"Exitcode != 0" + +// Invalid point +x{1d549908b5eb3c16f91174abe436c1a91442a57f922da813cb3dbc55de9e62bd63eac19a664eb8c3ea34b5a5c176d844} +<{ BLS_G1_INGROUP }>s 0 runvmx abort"Exitcode != 0" abort"0 expected" + +// Zero +."Zero:" cr +a1 a2 a3 a4 a5 +<{ { BLS_G1_ISZERO 33 THROWIF } 5 times }>s 0 runvmx abort"Exitcode != 0" +<{ BLS_G1_ZERO }>s 0 runvmx abort"Exitcode != 0" dup csr. constant zero +zero <{ BLS_G1_INGROUP }>s 0 runvmx abort"Exitcode != 0" not abort"-1 expected" +zero <{ BLS_G1_ISZERO }>s 0 runvmx abort"Exitcode != 0" not abort"-1 expected" + +// Addition +."a1 + a2 + a3:" cr +a1 a2 a3 <{ { BLS_G1_ADD } 2 times }>s 0 runvmx abort"Exitcode != 0" csr. +a3 a2 a1 <{ { BLS_G1_ADD } 2 times }>s 0 runvmx abort"Exitcode != 0" csr. +a2 a3 a1 <{ { BLS_G1_ADD } 2 times }>s 0 runvmx abort"Exitcode != 0" csr. +a1 zero a2 a3 zero <{ { BLS_G1_ADD } 4 times }>s 0 runvmx abort"Exitcode != 0" csr. + +// Subtraction +."a1 - a2:" cr +a1 a2 <{ BLS_G1_SUB }>s 0 runvmx abort"Exitcode != 0" csr. +a1 zero a2 <{ BLS_G1_SUB BLS_G1_ADD }>s 0 runvmx abort"Exitcode != 0" csr. + +// Negation +."-a1:" cr +a1 <{ BLS_G1_NEG }>s 0 runvmx abort"Exitcode != 0" csr. +zero a1 <{ BLS_G1_SUB }>s 0 runvmx abort"Exitcode != 0" csr. +."0:" cr +a1 a1 <{ BLS_G1_NEG BLS_G1_ADD }>s 0 runvmx abort"Exitcode != 0" csr. +a1 a1 <{ BLS_G1_SUB }>s 0 runvmx abort"Exitcode != 0" csr. + +// Multiplication: +."a1 * 1:" cr +a1 csr. +a1 1 <{ BLS_G1_MUL }>s 0 runvmx abort"Exitcode != 0" csr. +."a1 * 0:" cr +zero csr. +a1 0 <{ BLS_G1_MUL }>s 0 runvmx abort"Exitcode != 0" csr. +."a1 * (-1):" cr +a1 -1 <{ BLS_G1_MUL }>s 0 runvmx abort"Exitcode != 0" csr. +a1 <{ BLS_G1_NEG }>s 0 runvmx abort"Exitcode != 0" csr. +."a1 * 3:" cr +a1 3 <{ BLS_G1_MUL }>s 0 runvmx abort"Exitcode != 0" csr. +a1 a1 a1 <{ { BLS_G1_ADD } 2 times }>s 0 runvmx abort"Exitcode != 0" csr. +."a1 * 123:" cr +a1 123 <{ BLS_G1_MUL }>s 0 runvmx abort"Exitcode != 0" csr. +<{ a1 SLICE 100 INT BLS_G1_MUL a1 SLICE 23 INT BLS_G1_MUL BLS_G1_ADD }>s 0 runvmx abort"Exitcode != 0" csr. +a1 -123 <{ BLS_G1_MUL BLS_G1_NEG }>s 0 runvmx abort"Exitcode != 0" csr. + +// Multiexp +."a1*111 + a2*222 + a3*(-333) + a4*0 + a5*1:" cr +a1 111 a2 222 a3 -333 a4 0 a5 1 5 <{ BLS_G1_MULTIEXP }>s 0 runvmx abort"Exitcode != 0" csr. +a1 111 a2 222 a3 -333 a5 1 4 <{ BLS_G1_MULTIEXP }>s 0 runvmx abort"Exitcode != 0" csr. +<{ + a1 SLICE 111 INT BLS_G1_MUL + a2 SLICE 222 INT BLS_G1_MUL + a3 SLICE -333 INT BLS_G1_MUL + a5 SLICE + { BLS_G1_ADD } 3 times +}>s 0 runvmx abort"Exitcode != 0" csr. +."0:" cr +zero csr. +0 <{ BLS_G1_MULTIEXP }>s 0 runvmx abort"Exitcode != 0" csr. +a1 0 1 <{ BLS_G1_MULTIEXP }>s 0 runvmx abort"Exitcode != 0" csr. + +// Map to g2 +."G2 Points:" cr +x{cce34c6322b8f3b455617a975aff8b6eaedf04fbae74a8890db6bc3fab0475b94cd8fbde0e1182ce6993afd56ed6e71919cae59c891923b4014ed9e42d9f0e1a779d9a7edb64f5e2fd600012805fc773b5092af5d2f0c6c0946ee9ad8394bf19} +<{ BLS_MAP_TO_G2 }>s 0 runvmx abort"Exitcode != 0" dup ."b1 = " csr. constant b1 +x{2faa65f3431da8f04b8d029f7699b6426eb31feb06b3429b13b99fde35d5c0ab17e67943802313a96b2252a69dfdcc6e56f5671d905984940f4b9ce3b410042457dff7ae5fd4be6a0b73cad5d0390ed379d658cb24e11973d80f98bd7ff64f19} +<{ BLS_MAP_TO_G2 }>s 0 runvmx abort"Exitcode != 0" dup ."b2 = " csr. constant b2 +x{28619564e5cbb27c9e709d80b654f2eb1fd2c3ab435d7b97b4bd80638dbfe5b47e52df0e5be0b2c328357c5ddd8018acc6e739c4d132cc6f2b9797c210051acef9513ae54bb66de2a9ea8d02cbca7e96ce8193be1557d3128906e12f37913887} +<{ BLS_MAP_TO_G2 }>s 0 runvmx abort"Exitcode != 0" dup ."b3 = " csr. constant b3 +x{66f14fc1bb199ece07fde0a7af3cb3d2719acd4bb5186ab4ddda7de6a9f96557df44f3d14264eb0fed79f53d972ddc4517e362a001c5e7c7217169a05d9e3cd82b521236737f5d564f5860139d027018d3b33605d51e48c77b51554bf1d5b24a} +<{ BLS_MAP_TO_G2 }>s 0 runvmx abort"Exitcode != 0" dup ."b4 = " csr. constant b4 +x{a9e68db711778adb0bcee53ae4fd2d31605c1eff02ae38279eebfb45fc319964d33cb45ee32bbcb13663fe2131f79120af2d8ce26400ece9a7fb57ef9666c5b1b6f1856cb121b1c618b2dcfb359ffa63a08989c1f457b355958f589e7314610a} +<{ BLS_MAP_TO_G2 }>s 0 runvmx abort"Exitcode != 0" dup ."b5 = " csr. constant b5 + +// Validate points +b1 b2 b3 b4 b5 +<{ { BLS_G2_INGROUP 33 THROWIFNOT } 5 times }>s 0 runvmx abort"Exitcode != 0" + +// Invalid point +x{090069862cb1b1ac4241c4b1ed5f98edb95413db77f534bba7e85d9cb54d953c61416c0eeb5c65c6f0b494e9f59b2c9dfe8b4a9af75e1114b45ec60f6b5d2327cc05a6d9d6e76d7a9efd947302966d4f357bd48e5c3f950101c88c65b13bd5c7} +<{ BLS_G2_INGROUP }>s 0 runvmx abort"Exitcode != 0" abort"0 expected" + +// Zero +."Zero:" cr +b1 b2 b3 b4 b5 +<{ { BLS_G2_ISZERO 33 THROWIF } 5 times }>s 0 runvmx abort"Exitcode != 0" +<{ BLS_G2_ZERO }>s 0 runvmx abort"Exitcode != 0" dup csr. constant zero +zero <{ BLS_G2_INGROUP }>s 0 runvmx abort"Exitcode != 0" not abort"-1 expected" +zero <{ BLS_G2_ISZERO }>s 0 runvmx abort"Exitcode != 0" not abort"-1 expected" + +// Addition +."b1 + b2 + b3:" cr +b1 b2 b3 <{ { BLS_G2_ADD } 2 times }>s 0 runvmx abort"Exitcode != 0" csr. +b3 b2 b1 <{ { BLS_G2_ADD } 2 times }>s 0 runvmx abort"Exitcode != 0" csr. +b2 b3 b1 <{ { BLS_G2_ADD } 2 times }>s 0 runvmx abort"Exitcode != 0" csr. +b1 zero b2 b3 zero <{ { BLS_G2_ADD } 4 times }>s 0 runvmx abort"Exitcode != 0" csr. + +// Subtraction +."b1 - b2:" cr +b1 b2 <{ BLS_G2_SUB }>s 0 runvmx abort"Exitcode != 0" csr. +b1 zero b2 <{ BLS_G2_SUB BLS_G2_ADD }>s 0 runvmx abort"Exitcode != 0" csr. + +// Negation +."-b1:" cr +b1 <{ BLS_G2_NEG }>s 0 runvmx abort"Exitcode != 0" csr. +zero b1 <{ BLS_G2_SUB }>s 0 runvmx abort"Exitcode != 0" csr. +."0:" cr +b1 b1 <{ BLS_G2_NEG BLS_G2_ADD }>s 0 runvmx abort"Exitcode != 0" csr. +b1 b1 <{ BLS_G2_SUB }>s 0 runvmx abort"Exitcode != 0" csr. + +// Multiplication: +."b1 * 1:" cr +b1 csr. +b1 1 <{ BLS_G2_MUL }>s 0 runvmx abort"Exitcode != 0" csr. +."b1 * 0:" cr +zero csr. +b1 0 <{ BLS_G2_MUL }>s 0 runvmx abort"Exitcode != 0" csr. +."b1 * (-1):" cr +b1 -1 <{ BLS_G2_MUL }>s 0 runvmx abort"Exitcode != 0" csr. +b1 <{ BLS_G2_NEG }>s 0 runvmx abort"Exitcode != 0" csr. +."b1 * 3:" cr +b1 3 <{ BLS_G2_MUL }>s 0 runvmx abort"Exitcode != 0" csr. +b1 b1 b1 <{ { BLS_G2_ADD } 2 times }>s 0 runvmx abort"Exitcode != 0" csr. +."b1 * 123:" cr +b1 123 <{ BLS_G2_MUL }>s 0 runvmx abort"Exitcode != 0" csr. +<{ b1 SLICE 100 INT BLS_G2_MUL b1 SLICE 23 INT BLS_G2_MUL BLS_G2_ADD }>s 0 runvmx abort"Exitcode != 0" csr. +b1 -123 <{ BLS_G2_MUL BLS_G2_NEG }>s 0 runvmx abort"Exitcode != 0" csr. + +// Multiexp +."b1*111 + b2*222 + b3*(-333) + b4*0 + b5*1:" cr +b1 111 b2 222 b3 -333 b4 0 b5 1 5 <{ BLS_G2_MULTIEXP }>s 0 runvmx abort"Exitcode != 0" csr. +b1 111 b2 222 b3 -333 b5 1 4 <{ BLS_G2_MULTIEXP }>s 0 runvmx abort"Exitcode != 0" csr. +<{ + b1 SLICE 111 INT BLS_G2_MUL + b2 SLICE 222 INT BLS_G2_MUL + b3 SLICE -333 INT BLS_G2_MUL + b5 SLICE + { BLS_G2_ADD } 3 times +}>s 0 runvmx abort"Exitcode != 0" csr. +."0:" cr +zero csr. +0 <{ BLS_G2_MULTIEXP }>s 0 runvmx abort"Exitcode != 0" csr. +b1 0 1 <{ BLS_G2_MULTIEXP }>s 0 runvmx abort"Exitcode != 0" csr. + +// r +<{ BLS_PUSHR }>s 0 runvmx abort"Exitcode != 0" cr ."r = " . cr +b1 <{ BLS_PUSHR BLS_G2_MUL }>s 0 runvmx abort"Exitcode != 0" csr. +zero csr. +b1 <{ BLS_PUSHR INC BLS_G2_MUL }>s 0 runvmx abort"Exitcode != 0" csr. +b1 csr. + +// Pairings +{ [[ <{ BLS_G1_MUL }>s ]] 0 runvmx abort"Exitcode != -1" } : bls_g1_mul +{ [[ <{ BLS_G2_MUL }>s ]] 0 runvmx abort"Exitcode != -1" } : bls_g2_mul +75634785643785634785634876232423354534 constant x +."a1*x,b1 a1,b1*(-x) : " +a1 x bls_g1_mul b1 +a1 b1 x negate bls_g2_mul +2 <{ BLS_PAIRING }>s 0 runvmx abort"Exitcode != 0" .s not abort"-1 expected" +."a1*x,b1 a1,b1*(-x-1) : " +a1 x bls_g1_mul b1 +a1 b1 x negate 1 - bls_g2_mul +2 <{ BLS_PAIRING }>s 0 runvmx abort"Exitcode != 0" .s abort"0 expected" + +08036758068232723862526737758751120353935980577994643429668638941492109432519 constant x1 +76720311667788346189068792441910584335982488547394926476426087533015880449318 constant x2 +73698677644295053147826041647629389417255852559045739853199261775689421644183 constant x3 +00651749128863148819911470689106677724299434569675211711456038250594316760143 constant x4 +."a1*x1,b1 a2*x2,b2 a3*x3,b3 a4*x4,b4 a1,b1*(-x1) a2,b2*(-x2) a3,b3*(-x3) a4,b4*(-x4) : " +a1 x1 bls_g1_mul b1 +a2 x2 bls_g1_mul b2 +a3 x3 bls_g1_mul b3 +a4 x4 bls_g1_mul b4 +a1 b1 x1 negate bls_g2_mul +a2 b2 x2 negate bls_g2_mul +a3 b3 x3 negate bls_g2_mul +a4 b4 x4 negate bls_g2_mul +8 <{ BLS_PAIRING }>s 0 runvmx abort"Exitcode != 0" .s not abort"-1 expected" +."a1*x1,b1 a2*x2,b2 a3*x3,b3 a4*x4,b4 a1,b1*(-x1) a2,b2*(-x2) a3,b3*(-x4) a4,b4*(-x3) : " +a1 x1 bls_g1_mul b1 +a2 x2 bls_g1_mul b2 +a3 x3 bls_g1_mul b3 +a4 x4 bls_g1_mul b4 +a1 b1 x1 negate bls_g2_mul +a2 b2 x2 negate bls_g2_mul +a3 b3 x4 negate bls_g2_mul +a4 b4 x3 negate bls_g2_mul +8 <{ BLS_PAIRING }>s 0 runvmx abort"Exitcode != 0" .s abort"0 expected" diff --git a/crypto/test/fift/deep_stack_ops.fif b/crypto/test/fift/deep_stack_ops.fif new file mode 100644 index 00000000..460ecdd7 --- /dev/null +++ b/crypto/test/fift/deep_stack_ops.fif @@ -0,0 +1,27 @@ +"Asm.fif" include + +{ { drop } depth 1- times } : clear-stack + +0 { dup 1+ } 500 times +<{ 400 INT XCHGX 300 INT PICK 450 INT CHKDEPTH }>s 1000000 64 8 + runvmx .s clear-stack + +0 { dup 1+ } 500 times +<{ 400 INT ROLLX }>s 1000000 64 8 + runvmx .s clear-stack + +0 { dup 1+ } 500 times +<{ 400 INT -ROLLX }>s 1000000 64 8 + runvmx .s clear-stack + +0 { dup 1+ } 700 times +<{ 350 INT 300 INT BLKSWX }>s 1000000 64 8 + runvmx .s clear-stack + +0 { dup 1+ } 500 times +<{ 400 INT 5 INT REVX }>s 1000000 64 8 + runvmx .s clear-stack + +0 { dup 1+ } 500 times +<{ 400 INT DROPX }>s 1000000 64 8 + runvmx .s clear-stack + +0 { dup 1+ } 500 times +<{ 400 INT ONLYTOPX }>s 1000000 64 8 + runvmx .s clear-stack + +0 { dup 1+ } 500 times +<{ 400 INT ONLYX }>s 1000000 64 8 + runvmx .s clear-stack diff --git a/crypto/test/fift/disasm.fif b/crypto/test/fift/disasm.fif new file mode 100644 index 00000000..8d9f31ac --- /dev/null +++ b/crypto/test/fift/disasm.fif @@ -0,0 +1,70 @@ +"Disasm.fif" include +"Asm.fif" include + +<{ + IF:<{ + 123456789 PUSHINT + }>ELSE<{ + x{12345} PUSHSLICE + }> + WHILE:<{ ADD }>DO<{ + 10 PUSHINT REPEAT:<{ + CONT:<{ NOP }> + CONT:<{ }> + }> + }> +}>s +disasm cr + +x{007A7} disasm cr // Cannot disassemble: x{7} + +<{ + SWAP + s0 s10 XCHG s0 100 s() XCHG + s5 PUSH s6 POP + s4 s10 PUSH2 + 5 10 BLKSWAP + c4 PUSH c5 POP +}>s dup dup +disasm cr +std-disasm disasm cr +stack-disasm show-vm-code disasm cr +hide-vm-code + +<{ + 1 INT 123456789 INT 123456789123456789123456789 INT + PUSHREF + x{567} PUSHSLICE + 10 ADDCONST + DIV DIVR DIVC + RSHIFTC 10 RSHIFTC# + 20 MODPOW2# + 30 MULRSHIFTR# + LSHIFTDIVC 40 LSHIFT#DIVR +}>s +disasm cr + +PROGRAM{ + 11 DECLMETHOD foo1 + 12 DECLMETHOD foo2 + 13 DECLMETHOD foo3 + DECLPROC main + foo1 PROC:<{ + 123 ADDCONST + }> + foo2 PROC:<{ + OVER + 5 EQINT + IFJMP:<{ + NIP + }> + MUL + }> + foo3 PROC:<{ + ADD + foo2 CALLDICT + }> + main PROC:<{ + }> +}END>s +disasm \ No newline at end of file diff --git a/crypto/test/fift/fift-ext.fif b/crypto/test/fift/fift-ext.fif new file mode 100644 index 00000000..b0b08c12 --- /dev/null +++ b/crypto/test/fift/fift-ext.fif @@ -0,0 +1,107 @@ +"FiftExt.fif" include + +// if then +{ + if{ 0> }then{ "Yes" } +} : run +5 run type cr // Yes +-5 run .s // nothing + +// if then else +{ + if{ 0> }then{ "Yes" }else{ "No" } +} : run +5 run type ." " // Yes +-5 run type cr // No + +// if then elseif else +{ + dup dup if{ 20 > }then{ 2drop "AAA" }elseif{ 10 > }then{ drop "BBB" }elseif{ 0> }then{ "CCC" }else{ "DDD" } +} : run +25 run type ." " // AAA +15 run type ." " // BBB +5 run type ." " // CCC +-5 run type cr // DDD + +// while do +1 +while{ dup 100 < }do{ + dup . + 2 * +} +cr // 1 2 4 8 16 32 64 +drop + +// repeat until +1 +repeat{ + dup . + 3 * +}until{ dup 500 > } +cr // 1 3 9 27 81 243 +drop + +// def +def foo1 { * + } +5 10 20 foo1 . // 205 +6 100 100 foo1 . cr // 10006 + +// defrec +defrec fib { + if{ dup 2 < }then{ + drop 1 + }else{ + dup 1- fib swap 2- fib + + } +} +8 fib . // 34 +9 fib . // 55 +20 fib . cr // 10946 + +// [[ ... ]] +def foo2 { [[ ."Exec" cr 5 5 * ]] + } +."Calling foo2: " +100 foo2 . 200 foo2 . cr + +// Nested blocks +def sqr { dup * } +def power { + 1 -rot + while{ dup 0> }do{ + if{ dup 1 and }then{ + -rot tuck * swap rot + } + 2/ + if{ dup 0> }then{ + swap sqr swap + } + } + 2drop +} + +3 0 power . // 1 +3 1 power . // 3 +3 4 power . // 81 +3 12 power . // 531441 +3 50 power . // 717897987691852588770249 +cr + +0 while{ dup 2 < }do{ + ."A" + 0 while{ dup 2 < }do{ + ."B" + 0 while{ dup 2 < }do{ + ."C" + 0 while{ dup 2 < }do{ + ."D" + 0 while{ dup 2 < }do{ + ."E" + 0 while{ dup 2 < }do{ + ."F" + 1+ } drop + 1+ } drop + 1+ } drop + 1+ } drop + 1+ } drop +1+ } drop +cr // ABCDEFFEFFDEFFEFFCDEFFEFFDEFFEFFBCDEFFEFFDEFFEFFCDEFFEFFDEFFEFFABCDEFFEFFDEFFEFFCDEFFEFFDEFFEFFBCDEFFEFFDEFFEFFCDEFFEFFDEFFEFF \ No newline at end of file diff --git a/crypto/test/fift/hash_ext.fif b/crypto/test/fift/hash_ext.fif new file mode 100644 index 00000000..3f009410 --- /dev/null +++ b/crypto/test/fift/hash_ext.fif @@ -0,0 +1,98 @@ +"Asm.fif" include + +{ { drop } depth 1- times } : clear-stack + +// Compare HASHEXT_SHA256 with SHA256U +<{ + x{0123456789abcdef} PUSHSLICE SHA256U + + x{0123456789abcdef} PUSHSLICE 1 PUSHINT HASHEXT_SHA256 + + x{01} PUSHSLICE + x{2} PUSHSLICE + b{001101} PUSHSLICE NEWC STSLICE + b{0} PUSHSLICE + b{00101} PUSHSLICE NEWC STSLICE + x{6789a} PUSHSLICE + b{1} PUSHSLICE + b{0111100} PUSHSLICE + x{def} PUSHSLICE + 9 PUSHINT HASHEXT_SHA256 + + x{01} PUSHSLICE + x{2} PUSHSLICE + b{001101} PUSHSLICE NEWC STSLICE + b{0} PUSHSLICE + b{00101} PUSHSLICE NEWC STSLICE + x{6789a} PUSHSLICE + b{1} PUSHSLICE + b{0111100} PUSHSLICE + x{def} PUSHSLICE + 9 0 REVERSE + 9 PUSHINT HASHEXTR_SHA256 +}>s +0 runvmx abort"runvmx finished with exitcode != 0" +. cr . cr . cr . cr .s + +// HASHEXTA +<{ + NEWC x{ff} PUSHSLICE STSLICER x{01234567} PUSHSLICE SHA256U 256 STUR ENDC CTOS + NEWC x{ff} PUSHSLICE STSLICER x{0123} PUSHSLICE x{4567} PUSHSLICE 2 PUSHINT HASHEXTA_SHA256 ENDC CTOS + NEWC x{ff} PUSHSLICE STSLICER x{4567} PUSHSLICE x{0123} PUSHSLICE 2 PUSHINT HASHEXTAR_SHA256 ENDC CTOS +}>s +0 runvmx abort"runvmx finished with exitcode != 0" +csr. csr. csr. .s + +// Exceptions on errors +<{ x{001122334455667} PUSHSLICE 1 PUSHINT HASHEXT_SHA256 }>s 0 runvmx .s 9 <> abort"exitcode != 9" clear-stack +<{ x{00} PUSHSLICE x{11} PUSHSLICE 3 PUSHINT HASHEXT_SHA256 }>s 0 runvmx .s 5 <> abort"exitcode != 5" clear-stack +<{ x{00} PUSHSLICE 1 PUSHINT 2 PUSHINT HASHEXT_SHA256 }>s 0 runvmx .s 7 <> abort"exitcode != 7" clear-stack +<{ x{1234} PUSHSLICE 1 PUSHINT 100 HASHEXT }>s 0 runvmx .s 5 <> abort"exitcode != 5" clear-stack + +// Other hash functions + s 0 runvmx .s abort"runvmx finished with exitcode != 0" clear-stack +<{ + str PUSHSLICE 1 PUSHINT 0 HASHEXT + str PUSHSLICE 1 PUSHINT 1 HASHEXT + str PUSHSLICE 1 PUSHINT 2 HASHEXT + str PUSHSLICE 1 PUSHINT 3 HASHEXT + str PUSHSLICE 1 PUSHINT 4 HASHEXT +}>s 0 runvmx .s abort"runvmx finished with exitcode != 0" clear-stack +<{ + NEWC str PUSHSLICE 1 PUSHINT 4 HASHEXTA ENDC CTOS + NEWC str PUSHSLICE 1 PUSHINT 3 HASHEXTA ENDC CTOS + NEWC str PUSHSLICE 1 PUSHINT 2 HASHEXTA ENDC CTOS + NEWC str PUSHSLICE 1 PUSHINT 1 HASHEXTA ENDC CTOS + NEWC str PUSHSLICE 1 PUSHINT 0 HASHEXTA ENDC CTOS +}>s 0 runvmx abort"runvmx finished with exitcode != 0" +{ csr. } 5 times .s + +// Long string +0 { + =: hash-idx + 0 { + dup =: len + s ]] 0 runvmx abort"exitcode != 0" ."Level = " . cr + dup [[ <{ CLEVELMASK }>s ]] 0 runvmx abort"exitcode != 0" ."Level mask = 0b" b. cr + dup dup [[ <{ 0 CHASHI DUP ROT 0 INT CHASHIX EQUAL 55 THROWIFNOT }>s ]] 0 runvmx abort"exitcode != 0" ."Hash_0 = " X. cr + dup dup [[ <{ 1 CHASHI DUP ROT 1 INT CHASHIX EQUAL 55 THROWIFNOT }>s ]] 0 runvmx abort"exitcode != 0" ."Hash_1 = " X. cr + dup dup [[ <{ 2 CHASHI DUP ROT 2 INT CHASHIX EQUAL 55 THROWIFNOT }>s ]] 0 runvmx abort"exitcode != 0" ."Hash_2 = " X. cr + dup dup [[ <{ 3 CHASHI DUP ROT 3 INT CHASHIX EQUAL 55 THROWIFNOT }>s ]] 0 runvmx abort"exitcode != 0" ."Hash_3 = " X. cr + dup dup [[ <{ 0 CDEPTHI DUP ROT 0 INT CDEPTHIX EQUAL 55 THROWIFNOT }>s ]] 0 runvmx abort"exitcode != 0" ."Depth_0 = " . cr + dup dup [[ <{ 1 CDEPTHI DUP ROT 1 INT CDEPTHIX EQUAL 55 THROWIFNOT }>s ]] 0 runvmx abort"exitcode != 0" ."Depth_1 = " . cr + dup dup [[ <{ 2 CDEPTHI DUP ROT 2 INT CDEPTHIX EQUAL 55 THROWIFNOT }>s ]] 0 runvmx abort"exitcode != 0" ."Depth_2 = " . cr + dup dup [[ <{ 3 CDEPTHI DUP ROT 3 INT CDEPTHIX EQUAL 55 THROWIFNOT }>s ]] 0 runvmx abort"exitcode != 0" ."Depth_3 = " . cr + drop + cr +} : print-all + +// Ordinary cell of level 0 + ref, b> ref, + ref, +b> +print-all + +// Prunned branch of level 1 +spec +print-all + +// Prunned branch of level 3 +spec +print-all + +// Prunned branch of level 3, mask 0b101 +spec +print-all + +// Tree with the previous cell inside +spec ref, + b> ref, +b> +print-all diff --git a/crypto/test/fift/namespaces.fif b/crypto/test/fift/namespaces.fif new file mode 100644 index 00000000..3f10f608 --- /dev/null +++ b/crypto/test/fift/namespaces.fif @@ -0,0 +1,29 @@ +namespace My + +"a" constant a +"b" constant b +"c" constant c + +a b c .s { drop } 3 times // "a" "b" "c" + +My definitions +"b-my" constant b +"c-my" constant c +"d-my" constant d + +a b c d .s { drop } 4 times // "a" "b-my" "c-my" "d-my" + +Fift definitions +a b c .s { drop } 3 times // "a" "b-my" "c-my" "d-my" + +My b My c My d .s { drop } 3 times // "b-my" "c-my" "d-my" +a b c .s { drop } 3 times // "a" "b" "c" "d" + +My definitions +a b c d .s { drop } 4 times // "a" "b-my" "c-my" "d-my" +Fift a Fift b Fift c d .s { drop } 4 times // "a" "b" "c" "d-my" + +Fift definitions +cr +My-wordlist @ +{ drop type -1 } hmapforeach drop cr // "b " "d " "c " \ No newline at end of file diff --git a/crypto/test/fift/rist255.fif b/crypto/test/fift/rist255.fif new file mode 100644 index 00000000..ddd9b619 --- /dev/null +++ b/crypto/test/fift/rist255.fif @@ -0,0 +1,92 @@ +// Test data: https://ristretto.group/test_vectors/ristretto255.html +"Asm.fif" include +"FiftExt.fif" include + +."Basepoint multiples:" cr +{ + =: ans =: n + @' n + [[ <{ RIST255_MULBASE DUP RIST255_VALIDATE }>s ]] 0 runvmx abort"Exitcode != 0" + @' n . dup (x.) type cr + @' ans <> abort"Invalid result" + @' n + [[ <{ 1 INT RIST255_MULBASE SWAP RIST255_MUL DUP RIST255_VALIDATE }>s ]] 0 runvmx abort"Exitcode != 0" + @' ans <> abort"Invalid result" +} : test-basepoint + + 0 0x0000000000000000000000000000000000000000000000000000000000000000 test-basepoint + 1 0xe2f2ae0a6abc4e71a884a961c500515f58e30b6aa582dd8db6a65945e08d2d76 test-basepoint + 2 0x6a493210f7499cd17fecb510ae0cea23a110e8d5b901f8acadd3095c73a3b919 test-basepoint + 3 0x94741f5d5d52755ece4f23f044ee27d5d1ea1e2bd196b462166b16152a9d0259 test-basepoint + 4 0xda80862773358b466ffadfe0b3293ab3d9fd53c5ea6c955358f568322daf6a57 test-basepoint + 5 0xe882b131016b52c1d3337080187cf768423efccbb517bb495ab812c4160ff44e test-basepoint + 6 0xf64746d3c92b13050ed8d80236a7f0007c3b3f962f5ba793d19a601ebb1df403 test-basepoint + 7 0x44f53520926ec81fbd5a387845beb7df85a96a24ece18738bdcfa6a7822a176d test-basepoint + 8 0x903293d8f2287ebe10e2374dc1a53e0bc887e592699f02d077d5263cdd55601c test-basepoint + 9 0x02622ace8f7303a31cafc63f8fc48fdc16e1c8c8d234b2f0d6685282a9076031 test-basepoint +10 0x20706fd788b2720a1ed2a5dad4952b01f413bcf0e7564de8cdc816689e2db95f test-basepoint +11 0xbce83f8ba5dd2fa572864c24ba1810f9522bc6004afe95877ac73241cafdab42 test-basepoint +12 0xe4549ee16b9aa03099ca208c67adafcafa4c3f3e4e5303de6026e3ca8ff84460 test-basepoint +13 0xaa52e000df2e16f55fb1032fc33bc42742dad6bd5a8fc0be0167436c5948501f test-basepoint +14 0x46376b80f409b29dc2b5f6f0c52591990896e5716f41477cd30085ab7f10301e test-basepoint +15 0xe0c418f7c8d9c4cdd7395b93ea124f3ad99021bb681dfc3302a9d99a2e53e64e test-basepoint + +cr ."Invalid points:" cr +{ + =: x + @' x (x.) type cr + @' x + [[ <{ RIST255_QVALIDATE }>s ]] 0 runvmx abort"Exitcode != 0" + abort"Invalid result" +} : test-invalid +// These are all bad because they're non-canonical field encodings. +0x00ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff test-invalid +0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff7f test-invalid +0xf3ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff7f test-invalid +0xedffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff7f test-invalid +// These are all bad because they're negative field elements. +0x0100000000000000000000000000000000000000000000000000000000000000 test-invalid +0x01ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff7f test-invalid +0xed57ffd8c914fb201471d1c3d245ce3c746fcbe63a3679d51b6a516ebebe0e20 test-invalid +0xc34c4e1826e5d403b78e246e88aa051c36ccf0aafebffe137d148a2bf9104562 test-invalid +0xc940e5a4404157cfb1628b108db051a8d439e1a421394ec4ebccb9ec92a8ac78 test-invalid +0x47cfc5497c53dc8e61c91d17fd626ffb1c49e2bca94eed052281b510b1117a24 test-invalid +0xf1c6165d33367351b0da8f6e4511010c68174a03b6581212c71c0e1d026c3c72 test-invalid +0x87260f7a2f12495118360f02c26a470f450dadf34a413d21042b43b9d93e1309 test-invalid +// These are all bad because they give a nonsquare x^2. +0x26948d35ca62e643e26a83177332e6b6afeb9d08e4268b650f1f5bbd8d81d371 test-invalid +0x4eac077a713c57b4f4397629a4145982c661f48044dd3f96427d40b147d9742f test-invalid +0xde6a7b00deadc788eb6b6c8d20c0ae96c2f2019078fa604fee5b87d6e989ad7b test-invalid +0xbcab477be20861e01e4a0e295284146a510150d9817763caf1a6f4b422d67042 test-invalid +0x2a292df7e32cababbd9de088d1d1abec9fc0440f637ed2fba145094dc14bea08 test-invalid +0xf4a9e534fc0d216c44b218fa0c42d99635a0127ee2e53c712f70609649fdff22 test-invalid +0x8268436f8c4126196cf64b3c7ddbda90746a378625f9813dd9b8457077256731 test-invalid +0x2810e5cbc2cc4d4eece54f61c6f69758e289aa7ab440b3cbeaa21995c2f4232b test-invalid +// These are all bad because they give a negative xy value. +0x3eb858e78f5a7254d8c9731174a94f76755fd3941c0ac93735c07ba14579630e test-invalid +0xa45fdc55c76448c049a1ab33f17023edfb2be3581e9c7aade8a6125215e04220 test-invalid +0xd483fe813c6ba647ebbfd3ec41adca1c6130c2beeee9d9bf065c8d151c5f396e test-invalid +0x8a2e1d30050198c65a54483123960ccc38aef6848e1ec8f5f780e8523769ba32 test-invalid +0x32888462f8b486c68ad7dd9610be5192bbeaf3b443951ac1a8118419d9fa097b test-invalid +0x227142501b9d4355ccba290404bde41575b037693cef1f438c47f8fbf35d1165 test-invalid +0x5c37cc491da847cfeb9281d407efc41e15144c876e0170b499a96a22ed31e01e test-invalid +0x445425117cb8c90edcbc7c1cc0e74f747f2c1efa5630a967c64f287792a48a4b test-invalid +// This is s = -1, which causes y = 0. +0xecffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff7f test-invalid + +cr ."Hash to point:" cr +{ + =: text =: ans + @' text $>s + [[ <{ 1 INT HASHEXT_SHA512 2 UNTUPLE RIST255_FROMHASH }>s ]] 0 runvmx abort"Exitcode != 0" + @' ans <> abort"Invalid result" + @' ans (x.) type ." " @' text type cr +} : test-hash + +0x3066f82a1a747d45120d1740f14358531a8f04bbffe6a819f86dfe50f44a0a46 "Ristretto is traditionally a short shot of espresso coffee" test-hash +0xf26e5b6f7d362d2d2a94c5d0e7602cb4773c95a2e5c31a64f133189fa76ed61b "made with the normal amount of ground coffee but extracted with" test-hash +0x006ccd2a9e6867e6a2c5cea83d3302cc9de128dd2a9a57dd8ee7b9d7ffe02826 "about half the amount of water in the same amount of time" test-hash +0xf8f0c87cf237953c5890aec3998169005dae3eca1fbb04548c635953c817f92a "by using a finer grind." test-hash +0xae81e7dedf20a497e10c304a765c1767a42d6e06029758d2d7e8ef7cc4c41179 "This produces a concentrated shot of coffee per volume." test-hash +0xe2705652ff9f5e44d3e841bf1c251cf7dddb77d140870d1ab2ed64f1a9ce8628 "Just pulling a normal shot short will produce a weaker shot" test-hash +0x80bd07262511cdde4863f8a7434cef696750681cb9510eea557088f76d9e5065 "and is not a Ristretto as some believe." test-hash \ No newline at end of file diff --git a/crypto/test/fift/secp256k1.fif b/crypto/test/fift/secp256k1.fif new file mode 100644 index 00000000..9a0e5178 --- /dev/null +++ b/crypto/test/fift/secp256k1.fif @@ -0,0 +1,82 @@ +"Asm.fif" include +"FiftExt.fif" include + +{ + =: expected_result + =: tweak + =: pubkey + B{e80fe1639c9ca050e3af1b39c143c63e429cbceb15d940fbb5c5a1f4af57c5e9e80fe1639c9ca050e3af1b39c143c63e429cbceb15d940fbb5c5a1f4af57c5e9} + @' pubkey 256 u>B @' tweak 256 u>B B+ B+ Bhashu =: tweak2 + @' pubkey @' tweak2 + [[ <{ SECP256K1_XONLY_PUBKEY_TWEAK_ADD }>s ]] 0 runvmx + abort"exitcode != 0" -1 <> abort"xonly_pubkey_tweak_add failed" + drop + =: result + ."Pubkey = " @' pubkey x. cr + ."Tweak = " @' tweak x. cr + ."Tweak2 = " @' tweak2 x. cr + @' expected_result @' result <> { ."Expected = " @' expected_result x. cr ."Found = " @' result x. cr -1 abort"result mismatch" } if + ."Result = " @' result x. cr + 4 <> abort"first byte is expected to be 4" + cr +} : run-test + +0x4be5f2020ebe2f37b65fb1bcd8aecf2ab0a333427208a5e6dd3dc691f1fb6ae2 +0x7eeda5cc8b219363ffc29a77d170f923dbb939ea5164b71db862d4a192f2680f +0x5a001d32f9fc59c2965ef36d9ff154469a3ba87f00b95879f3b5198bb557494d +run-test + +0x3ad84c3844e66aa255a121ed46d0e69e7b98f31233941c5197917c97c7aeaec5 +0x3f339347d8b1dde7edade3dc59fc63d18cb21fc2326ffbd30f0711a02d25075a +0xc621ca417b8915474020c0c9471a13918b6a02fd20d48a0e526c896923457fcd +run-test + +0x57bb69f0d446ee2cf9f77a2bdca7a3a462a61d85997a1154f2321a2717951b02 +0x3197d6b03d78ebbe694d2b89945a21a5a671ca78393481d44739b7351767adef +0xcbf0da3bbb498fd575506060d04db426164cb9d1477f07481fc6e3a2f84b01ea +run-test + +0x4af537be1a1ae11770eef23e6087f83ce1019fbee7a5876837107f84929a1d19 +0xcf9bfedc7251c2f8e233ae1e2c8b7a6cbe25d96a46a38b79e45d5684d026b64c +0x19b1aa4551bf08363b2533c146c02fa61e26941336aaa16cdd4393a5440392ea +run-test + +0x4be5f2020ebe2f37b65fb1bcd8aecf2ab0a333427208a5e6dd3dc691f1fb6ae2 +0x1762cbfa935318fb1395b50c64f961baab3ecaed4afad3068ba2f7f3a9d15cec +0x38be4b9791c0cb4952b9fb944eb0fe9256ce0d48be7b92129caafdf2f521248a +run-test + +0x3ad84c3844e66aa255a121ed46d0e69e7b98f31233941c5197917c97c7aeaec5 +0x099ec2e6ee1c40f533a61abca2860733727204c2f31d078297194d5e93e148f7 +0x8e9d15851e9aeb652216378f6bc0b88fb66b491697c9b9588f37c11d0b0fbced +run-test + +0x57bb69f0d446ee2cf9f77a2bdca7a3a462a61d85997a1154f2321a2717951b02 +0x63494cee8217ca2a10076e2031534fdda39f132dbdf91606aa65021709cb3116 +0x5dd1aa62a438469f4934e1ab9ada9ba3945651a2641316ec91d0780ca71891f8 +run-test + +0x4af537be1a1ae11770eef23e6087f83ce1019fbee7a5876837107f84929a1d19 +0x36e26133dc62474040eae511ce89610dae6bb0359aa80738baf812ecfa03d2a1 +0xf9ab8bbbee0ad1e90492c952b362d43d56412debdd1224e9729dc79aff931943 +run-test + +0x4be5f2020ebe2f37b65fb1bcd8aecf2ab0a333427208a5e6dd3dc691f1fb6ae2 +0x94f8fb43dd6001bb8acd2c53d094f9c919766dad596498d3d582cebdb39c63da +0xf7b4c76d79925f62e7969cb0e4af0673808f646add78563f711dc33a2687dc1b +run-test + +0x3ad84c3844e66aa255a121ed46d0e69e7b98f31233941c5197917c97c7aeaec5 +0xa2f55c34c4eee5881584714bd5c7a63c397ef38ff1afedd04ada10fdeb541cef +0x73f9a15b76612ca4254e6c2758508589c0112eb724f42dbb4c65ff8b025d2103 +run-test + +0x57bb69f0d446ee2cf9f77a2bdca7a3a462a61d85997a1154f2321a2717951b02 +0x05130a68853e5e4aff579fa21ff10010410a3be47b94d908e203f69ec9dc7d00 +0x4ce6b5b1bd77b1666d33a0c9fd37b98952078bcc451d6de2d0bff65e8f2b46ed +run-test + +0x4af537be1a1ae11770eef23e6087f83ce1019fbee7a5876837107f84929a1d19 +0x9765369cb4467bebc8a468d44aa60f0154f04ee32fbcf1c8bdf646d4840163d1 +0x118953c642b8f25fea3519bcaab3ae6cea25402088e11a8efdc2e0bd222958ad +run-test \ No newline at end of file diff --git a/crypto/test/fift/testdb.fif b/crypto/test/fift/testdb.fif deleted file mode 100644 index cbaa9817..00000000 --- a/crypto/test/fift/testdb.fif +++ /dev/null @@ -1,102 +0,0 @@ -"Asm.fif" include - -PROGRAM{ - -NEWPROC load_dict -NEWPROC generate_dict -NEWPROC save_dict - -NEWPROC do_get -NEWPROC do_set -NEWPROC do_erase - -main PROC:<{ - DUP 1 INT EQUAL IF:<{ - DROP - do_get CALL - }>ELSE<{ - DUP 2 INT EQUAL IF:<{ - DROP - do_set CALL - }>ELSE<{ - DUP 3 INT EQUAL IF:<{ - DROP - do_erase CALL - }> }> }> - -1 INT -}> - -do_get PROC:<{ - load_dict CALL - 32 INT - DICTIGET -}> - -do_set PROC:<{ - load_dict CALL - 32 INT - DICTISET - save_dict CALL -}> - -do_erase PROC:<{ - load_dict CALL - 32 INT - DICTIDEL - DROP - save_dict CALL -}> - -generate_dict PROC:<{ - 4 INT 100 INT REPEAT:<{ - DUP 2DUP MUL ROT 617 INT ADD 1000 INT MOD - }> - DROP 100 INT - NEWDICT - SWAP REPEAT:<{ - s0 s2 XCHG - NEWC - 16 STU - s0 s2 XCHG - 32 INT - DICTISETB - }> -}> - -load_dict PROC:<{ - PUSHROOT - CTOS DUP SEMPTY IF:<{ - DROP - generate_dict CALL - }> -}> - -save_dict PROC:<{ - NEWC - STSLICE - ENDC - POPROOT -}> - -}END>s constant pmc_prog - -{ 1 2 rot pmc_prog } : task_pmc_get -{ 2 3 rot pmc_prog } : task_pmc_set -{ 3 2 rot pmc_prog } : task_pmc_erase - -{ task_pmc_get dbrunvm 2drop } : pmc_get -{ task_pmc_set dbrunvm 2drop } : pmc_set -{ task_pmc_erase dbrunvm 2drop } : pmc_erase - - PUSHREF c4 POP // Ensure that it does not affect c4, c5, c7 in parent vm + PUSHREF c5 POP + NIL 100 PUSHINT TPUSH 200 PUSHINT TPUSH c7 POP + 123 PUSHINT +}>s +<{ + PUSHREF c4 POP + PUSHREF c5 POP + NIL 5 PUSHINT TPUSH 6 PUSHINT TPUSH c7 POP + 0 RUNVM + c4 PUSH CTOS c5 PUSH CTOS c7 PUSH // Ensure that c4, c5, c7 are unchanged +}>s 1000000 8 runvmx // Show gas usage +.s { drop } depth 1- times // 111 30 1 0 0 0 0 [] 123 0 x{1234} x{5678} [ 5 6 ] 0 1197 + +// Exception +111 10 20 2 +<{ 22 PUSHINT 55 PUSHINT 66 THROWARG }>s +<{ 0 RUNVM }>s 0 runvmx +.s { drop } depth 1- times // 111 55 66 0 + +// Mode +1 - set c3 to code +PROGRAM{ + 22 DECLMETHOD foo + DECLPROC main + foo PROC:<{ + MUL + }> + main PROC:<{ + DUP + foo CALLDICT + INC + }> +}END>s constant prog +<{ + 10 PUSHINT 0 PUSHINT 2 PUSHINT prog PUSHSLICE 1 RUNVM + 10 PUSHINT 0 PUSHINT 2 PUSHINT prog PUSHSLICE 0 RUNVM +}>s 0 runvmx +.s { drop } depth 1- times // 101 0 10 10 22 11 0 + +// Mode +2 - push 0 +<{ 10 PUSHINT 1 PUSHINT prog PUSHSLICE 3 RUNVM }>s 0 runvmx +.s { drop } depth 1- times // 101 0 0 + +// Mode +4 - load and return c4 +0 +<{ + c4 PUSHCTR CTOS // Ensure that this is x{5678} + PUSHREF c4 POPCTR // This should be returned from RUNVM + 1000 PUSHINT +}>s + +<{ + PUSHREF c4 POP // Ensure that this does not change + 4 RUNVM + CTOS + c4 PUSH CTOS // x{1234} +}>s 0 runvmx +.s { drop } depth 1- times // x{5678} 1000 0 x{abcd} x{1234} 0 + +// Mode +16 - load c7 +0 +<{ + c7 PUSH // Ensure that this is [ 10 15 20 ] + NIL 111 PUSHINT TPUSH 222 PUSHINT TPUSH 3333 PUSHINT TPUSH c7 POP + 1000 PUSHINT +}>s +10 15 20 3 tuple +<{ + NIL 1 PUSHINT TPUSH 2 PUSHINT TPUSH 3 PUSHINT TPUSH c7 POP // Ensure that this does not change + 16 RUNVM + c7 PUSH // [ 1 2 3 ] +}>s 0 runvmx +.s { drop } depth 1- times // [ 10 15 20 ] 1000 0 [ 1 2 3 ] 0 + +// Mode +32 - return c5 +0 +<{ + c5 PUSH CTOS SBITREFS // Ensure that this is empty + PUSHREF c5 POP // Ensure that this is returned from RUNVM + 1000 PUSHINT +}>s +<{ + PUSHREF c5 POP // Ensure that this does not change + 32 RUNVM + CTOS + c5 PUSH CTOS // x{1234} +}>s 0 runvmx +.s { drop } depth 1- times // 0 0 1000 0 x{5678} x{1234} 0 + +// c4, c5 with exception +0 +<{ + PUSHREF c4 POP + PUSHREF c5 POP + 55 THROW +}>s + // c4 for RUNVM +<{ + PUSHREF c4 POP // Ensure that this does not change + PUSHREF c5 POP // Ensure that this does not change + 32 4 + RUNVM + c4 PUSH CTOS // x{1234aaaa} + c5 PUSH CTOS // x{1234bbbb} +}>s 0 runvmx +.s { drop } depth 1- times // 0 55 null null x{1234aaaa} x{1234bbbb} 0 + +// c4, c5 with exception and commit +0 +<{ + PUSHREF c4 POP + PUSHREF c5 POP + COMMIT + PUSHREF c4 POP + PUSHREF c5 POP + 55 THROW +}>s + // c4 for RUNVM +<{ + PUSHREF c4 POP // Ensure that this does not change + PUSHREF c5 POP // Ensure that this does not change + 32 4 + RUNVM + CTOS SWAP CTOS SWAP + c4 PUSH CTOS // x{1234aaaa} + c5 PUSH CTOS // x{1234bbbb} +}>s 0 runvmx +.s { drop } depth 1- times // 0 55 x{abcdaaaa} x{abcdbbbb} x{1234aaaa} x{1234bbbb} 0 + +// Mode +8 - gas limit +0 +<{ AGAIN:<{ NOP }> }>s +200 +<{ 8 RUNVM 1234 PUSHINT }>s 0 runvmx +.s { drop } depth 1- times // 215 -14 215 1234 0 + +// Gas limit of parent vm is too low +0 +<{ AGAIN:<{ NOP }> }>s +1000000 +<{ 8 RUNVM 1234 PUSHINT }>s 300 8 runvmx +.s { drop } depth 1- times // 301 -14 301 + +// Mode +64 - hard gas limit +0 <{ AGAIN:<{ NOP }> }>s 200 500 +<{ 8 64 + RUNVM 1234 PUSHINT }>s 0 runvmx +.s { drop } depth 1- times // 215 -14 215 1234 0 +0 <{ ACCEPT AGAIN:<{ NOP }> }>s 200 500 +<{ 8 64 + RUNVM 1234 PUSHINT }>s 0 runvmx +.s { drop } depth 1- times // 517 -14 517 1234 0 + +// 10000 nested VMs (recursively calculating 1+...+10000) +<{ + DUP + 0 EQINT + IFJMP:<{ + DROP DROP + 0 PUSHINT + }> + OVER OVER DEC + 2 PUSHINT + s2 PUSH + 0 RUNVM + 11 THROWIF + ADD + NIP +}>s constant code1 +<{ code1 PUSHSLICE 10000 PUSHINT 2 PUSHINT code1 PUSHSLICE 0 RUNVM }>s 10000000 8 runvmx // Show gas +.s { drop } depth 1- times // 50005000 0 0 2710286 + +// Same thing, but out of gas +<{ code1 PUSHSLICE 10000 PUSHINT 2 PUSHINT code1 PUSHSLICE 0 RUNVM }>s 100000 8 runvmx // Show gas +.s { drop } depth 1- times // 100001 -14 100001 + +// RUNVMX +0 +<{ AGAIN:<{ NOP }> }>s +200 +<{ 8 PUSHINT RUNVMX 1234 PUSHINT }>s 0 runvmx +.s { drop } depth 1- times // 215 -14 215 1234 0 + +// +128 - separate loaded_cells + +<{ DUP CTOS DROP 2 INT <{ CTOS DROP CTOS DROP }>s SLICE 10000 INT 8 RUNVM }>s 1000000 8 runvmx +.s { drop } depth 1- times // 0 202 0 509 + +<{ DUP CTOS DROP 2 INT <{ CTOS DROP CTOS DROP }>s SLICE 10000 INT 8 128 + RUNVM }>s 1000000 8 runvmx +.s { drop } depth 1- times // 0 277 0 584 + +// +256 - fixed number of return values +11 22 33 3 +<{ 1 INT 2 INT 3 INT 4 INT 5 INT }>s +3 +<{ 256 RUNVM }>s 0 runvmx +.s { drop } depth 1- times // 3 4 5 0 0 + +11 22 33 3 +<{ 1 INT 2 INT 3 INT 4 INT 5 INT }>s +0 +<{ 256 RUNVM }>s 0 runvmx +.s { drop } depth 1- times // 0 0 + +11 22 33 3 +<{ 1 INT 2 INT 3 INT 4 INT 5 INT 77 THROW }>s +3 +<{ 256 RUNVM }>s 0 runvmx +.s { drop } depth 1- times // 0 77 0 + +11 22 33 3 +<{ 1 INT 2 INT 3 INT 4 INT 5 INT }>s +20 +<{ 256 RUNVM }>s 0 runvmx +.s { drop } depth 1- times // 0 -3 0 + +// GASCONSUMED +<{ 10 INT 20 INT ADD DROP GASCONSUMED }>s 0 runvmx +.s { drop } depth 1- times // 106 0 +0 <{ 10 INT 20 INT ADD DROP GASCONSUMED }>s +<{ 100 INT 200 INT 300 INT MUL DIV DROP 0 RUNVM GASCONSUMED }>s 0 runvmx +.s { drop } depth 1- times // 106 0 367 0 diff --git a/crypto/test/modbigint.cpp b/crypto/test/modbigint.cpp index b6480d38..75051fa6 100644 --- a/crypto/test/modbigint.cpp +++ b/crypto/test/modbigint.cpp @@ -180,7 +180,7 @@ struct MixedRadix { template const MixedRadix& as_shorter() const { - static_assert(M <= N); + static_assert(M <= N,"error"); return *reinterpret_cast*>(this); } @@ -289,7 +289,7 @@ struct MixedRadix { } explicit operator long long() const { - long long acc = 0.; + unsigned long long acc = 0; for (int i = N - 1; i >= 0; --i) { acc = acc * mod[i] + a[i]; } @@ -458,7 +458,7 @@ struct ModArray { } template ModArray(const ModArray& other) { - static_assert(M >= N); + static_assert(M >= N,"error"); std::copy(other.a, other.a + N, a); } ModArray(const int* p) : a(p) { @@ -819,7 +819,7 @@ struct ModArray { template const ModArray& as_shorter() const { - static_assert(M <= N); + static_assert(M <= N,"error"); return *reinterpret_cast*>(this); } @@ -903,7 +903,7 @@ struct ModArray { } for (; i < size; i++) { pow += 8; - acc = (acc << 8) + arr[i]; + acc = (acc * 256) + arr[i]; if (pow >= 56) { lshift_add(pow, acc); acc = pow = 0; diff --git a/crypto/test/test-bigint.cpp b/crypto/test/test-bigint.cpp index bf85e2ad..a6f6e8d6 100644 --- a/crypto/test/test-bigint.cpp +++ b/crypto/test/test-bigint.cpp @@ -16,12 +16,12 @@ */ #include #include -#include #include #include #include #include #include +#include #include "common/refcnt.hpp" #include "common/bigint.hpp" #include "common/refint.h" @@ -186,7 +186,7 @@ td::RefInt256 make_special_int(int x, BInt* ptr = nullptr, unsigned char bin[64] int acc = b, r = ord; for (int i = 63; i >= 0; --i) { if (r < 8) { - acc += (a << r); + acc += ((unsigned)a << r); r = 1024; } r -= 8; @@ -211,11 +211,11 @@ bool coin() { // returns 0 with probability 1/2, 1 with prob. 1/4, ..., k with prob. 1/2^(k+1) int randexp(int max = 63, int min = 0) { - return min + __builtin_clzll(Random() | (1ULL << (63 - max + min))); + return min + td::count_leading_zeroes64(Random() | (1ULL << (63 - max + min))); } void bin_add_small(unsigned char bin[64], long long val, int shift = 0) { - val <<= shift & 7; + val *= (1 << (shift & 7)); for (int i = 63 - (shift >> 3); i >= 0 && val; --i) { val += bin[i]; bin[i] = (unsigned char)val; @@ -363,7 +363,7 @@ void check_one_int_repr(td::RefInt256 x, int mode, int in_range, const BInt* val if (is_small) { // special check for small (64-bit) values CHECK(x->to_long() == xval); - CHECK((long long)__builtin_bswap64(*(long long*)(bytes + 64 - 8)) == xval); + CHECK((long long)td::bswap64(*(long long*)(bytes + 64 - 8)) == xval); CHECK(in_range); // check sign CHECK(x->sgn() == (xval > 0 ? 1 : (xval < 0 ? -1 : 0))); diff --git a/crypto/test/test-db.cpp b/crypto/test/test-db.cpp index 413d774f..dc7fcf37 100644 --- a/crypto/test/test-db.cpp +++ b/crypto/test/test-db.cpp @@ -54,12 +54,88 @@ #include #include +#include #include #include "openssl/digest.hpp" +#include "vm/dict.h" + +#include +#include +#include +#include +#include namespace vm { +class ThreadExecutor : public DynamicBagOfCellsDb::AsyncExecutor { + public: + explicit ThreadExecutor(size_t threads_n) { + for (size_t i = 0; i < threads_n; ++i) { + threads_.emplace_back([this]() { + while (true) { + auto task = pop_task(); + if (!task) { + break; + } + CHECK(generation_.load() % 2 == 1); + task(); + } + }); + } + } + + ~ThreadExecutor() override { + for (size_t i = 0; i < threads_.size(); ++i) { + push_task({}); + } + for (auto &t : threads_) { + t.join(); + } + } + + void execute_async(std::function f) override { + push_task(std::move(f)); + } + + void execute_sync(std::function f) override { + auto x = generation_.load(); + std::scoped_lock lock(sync_mutex_); + CHECK(x == generation_); + CHECK(generation_.load() % 2 == 1); + f(); + CHECK(generation_.load() % 2 == 1); + } + void inc_generation() { + generation_.fetch_add(1); + } + + private: + std::atomic generation_{0}; + std::queue, size_t>> queue_; + std::mutex queue_mutex_; + std::condition_variable cv_; + std::mutex sync_mutex_; + std::vector threads_; + + std::function pop_task() { + std::unique_lock lock(queue_mutex_); + cv_.wait(lock, [&] { return !queue_.empty(); }); + CHECK(!queue_.empty()); + auto task = std::move(queue_.front()); + queue_.pop(); + CHECK(task.second == generation_); + return task.first; + } + + void push_task(std::function task) { + { + std::scoped_lock lock(queue_mutex_); + queue_.emplace(std::move(task), generation_.load()); + } + cv_.notify_one(); + } +}; std::vector do_get_serialization_modes() { std::vector res; @@ -82,9 +158,23 @@ int get_random_serialization_mode(T &rnd) { return modes[rnd.fast(0, (int)modes.size() - 1)]; } -class BenchSha256 : public td::Benchmark { +class BenchSha : public td::Benchmark { public: + explicit BenchSha(size_t n) : str_(n, 'a') { + } std::string get_description() const override { + return PSTRING() << get_name() << " length=" << str_.size(); + } + + virtual std::string get_name() const = 0; + + protected: + std::string str_; +}; +class BenchSha256 : public BenchSha { + public: + using BenchSha::BenchSha; + std::string get_name() const override { return "SHA256"; } @@ -92,7 +182,7 @@ class BenchSha256 : public td::Benchmark { int res = 0; for (int i = 0; i < n; i++) { digest::SHA256 hasher; - hasher.feed("abcd", 4); + hasher.feed(str_); unsigned char buf[32]; hasher.extract(buf); res += buf[0]; @@ -100,10 +190,12 @@ class BenchSha256 : public td::Benchmark { td::do_not_optimize_away(res); } }; -class BenchSha256Reuse : public td::Benchmark { +class BenchSha256Reuse : public BenchSha { public: - std::string get_description() const override { - return "SHA256 reuse"; + using BenchSha::BenchSha; + + std::string get_name() const override { + return "SHA256 reuse (used in DataCell)"; } void run(int n) override { @@ -111,7 +203,7 @@ class BenchSha256Reuse : public td::Benchmark { digest::SHA256 hasher; for (int i = 0; i < n; i++) { hasher.reset(); - hasher.feed("abcd", 4); + hasher.feed(str_); unsigned char buf[32]; hasher.extract(buf); res += buf[0]; @@ -119,18 +211,28 @@ class BenchSha256Reuse : public td::Benchmark { td::do_not_optimize_away(res); } }; -class BenchSha256Low : public td::Benchmark { +class BenchSha256Low : public BenchSha { public: - std::string get_description() const override { + using BenchSha::BenchSha; + + std::string get_name() const override { return "SHA256 low level"; } +// Use the old method to check for performance degradation +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#elif defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4996) // Disable deprecated warning for MSVC +#endif void run(int n) override { int res = 0; SHA256_CTX ctx; for (int i = 0; i < n; i++) { SHA256_Init(&ctx); - SHA256_Update(&ctx, "abcd", 4); + SHA256_Update(&ctx, str_.data(), str_.size()); unsigned char buf[32]; SHA256_Final(buf, &ctx); res += buf[0]; @@ -138,9 +240,17 @@ class BenchSha256Low : public td::Benchmark { td::do_not_optimize_away(res); } }; -class BenchSha256Tdlib : public td::Benchmark { +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#elif defined(_MSC_VER) +#pragma warning(pop) +#endif + +class BenchSha256Tdlib : public BenchSha { public: - std::string get_description() const override { + using BenchSha::BenchSha; + + std::string get_name() const override { return "SHA256 TDLib"; } @@ -150,7 +260,7 @@ class BenchSha256Tdlib : public td::Benchmark { for (int i = 0; i < n; i++) { td::init_thread_local(ctx); ctx->init(); - ctx->feed("abcd"); + ctx->feed(str_); unsigned char buf[32]; ctx->extract(td::MutableSlice(buf, 32), false); res += buf[0]; @@ -158,11 +268,61 @@ class BenchSha256Tdlib : public td::Benchmark { td::do_not_optimize_away(res); } }; + +template +void bench_threaded(F &&f) { + class Threaded : public td::Benchmark { + public: + explicit Threaded(F &&f) : f_(std::move(f)), base(f_()) { + } + F f_; + std::decay_t base; + + std::string get_description() const override { + return base.get_description() + " threaded"; + } + + void run(int n) override { + std::atomic task_i{0}; + int chunk_size = 1024; + int num_threads = 16; + n *= num_threads; + std::vector threads; + for (int i = 0; i < num_threads; i++) { + threads.emplace_back([&]() mutable { + auto bench = f_(); + while (true) { + i = task_i.fetch_add(chunk_size, std::memory_order_relaxed); + auto i_end = std::min(n, i + chunk_size); + if (i > n) { + break; + } + bench.run(i_end - i); + } + }); + } + for (auto &thread : threads) { + thread.join(); + } + }; + }; + bench(Threaded(std::forward(f))); +} TEST(Cell, sha_benchmark) { - bench(BenchSha256Tdlib()); - bench(BenchSha256Low()); - bench(BenchSha256Reuse()); - bench(BenchSha256()); + for (size_t n : {4, 64, 128}) { + bench(BenchSha256Tdlib(n)); + bench(BenchSha256Low(n)); + bench(BenchSha256Reuse(n)); + bench(BenchSha256(n)); + } +} +TEST(Cell, sha_benchmark_threaded) { + for (size_t n : {4, 64, 128}) { + bench_threaded([n] { return BenchSha256Tdlib(n); }); + bench_threaded([n]() { return BenchSha256Low(n); }); + bench_threaded([n]() { return BenchSha256Reuse(n); }); + bench_threaded([n]() { return BenchSha256(n); }); + } } std::string serialize_boc(Ref cell, int mode = 31) { @@ -762,16 +922,136 @@ TEST(TonDb, BocMultipleRoots) { } }; -TEST(TonDb, DynamicBoc) { +TEST(TonDb, InMemoryDynamicBocSimple) { + auto counter = [] { return td::NamedThreadSafeCounter::get_default().get_counter("DataCell").sum(); }; + auto before = counter(); + SCOPE_EXIT { + LOG_CHECK(before == counter()) << before << " vs " << counter(); + ; + }; td::Random::Xorshift128plus rnd{123}; + auto kv = std::make_shared(); + CellStorer storer(*kv); + + auto boc = DynamicBagOfCellsDb::create_in_memory(kv.get(), {}); + + auto empty_cell = vm::CellBuilder().finalize(); + boc->inc(empty_cell); + boc->prepare_commit().ensure(); + boc->commit(storer).ensure(); + auto got_empty_cell = boc->load_cell(empty_cell->get_hash().as_slice()).move_as_ok(); + ASSERT_EQ(empty_cell->get_hash(), got_empty_cell->get_hash()); + + boc->dec(empty_cell); + + auto one_ref_cell = vm::CellBuilder().store_ref(empty_cell).finalize(); + boc->inc(one_ref_cell); + boc->prepare_commit().ensure(); + boc->commit(storer).ensure(); + auto got_one_ref_cell = boc->load_cell(one_ref_cell->get_hash().as_slice()).move_as_ok(); + ASSERT_EQ(one_ref_cell->get_hash(), got_one_ref_cell->get_hash()); + boc = DynamicBagOfCellsDb::create_in_memory(kv.get(), {}); + + auto random_ref_cell = gen_random_cell(3, rnd); + boc->inc(random_ref_cell); + boc->prepare_commit().ensure(); + boc->commit(storer).ensure(); + auto got_random_ref_cell = boc->load_cell(random_ref_cell->get_hash().as_slice()).move_as_ok(); + ASSERT_EQ(random_ref_cell->get_hash(), got_random_ref_cell->get_hash()); + boc = DynamicBagOfCellsDb::create_in_memory(kv.get(), {}); +} + +int VERBOSITY_NAME(boc) = VERBOSITY_NAME(DEBUG) + 10; + +struct BocOptions { + std::shared_ptr async_executor; + std::optional o_in_memory; + td::uint64 seed{123}; + + auto create_dboc(td::KeyValueReader *kv, std::optional o_root_n) { + if (o_in_memory) { + auto res = DynamicBagOfCellsDb::create_in_memory(kv, *o_in_memory); + auto stats = res->get_stats().move_as_ok(); + if (o_root_n) { + ASSERT_EQ(*o_root_n, stats.roots_total_count); + } + VLOG(boc) << "reset roots_n=" << stats.roots_total_count << " cells_n=" << stats.cells_total_count; + return res; + } + return DynamicBagOfCellsDb::create(); + }; + void prepare_commit(DynamicBagOfCellsDb &dboc) { + if (async_executor) { + async_executor->inc_generation(); + std::latch latch(1); + td::Result res; + async_executor->execute_sync([&] { + dboc.prepare_commit_async(async_executor, [&](auto r) { + res = std::move(r); + latch.count_down(); + }); + }); + latch.wait(); + async_executor->execute_sync([&] {}); + async_executor->inc_generation(); + } else { + dboc.prepare_commit(); + } + } +}; + +template +void with_all_boc_options(F &&f, size_t tests_n = 500) { + LOG(INFO) << "Test dynamic boc"; + auto counter = [] { return td::NamedThreadSafeCounter::get_default().get_counter("DataCell").sum(); }; + auto run = [&](BocOptions options) { + LOG(INFO) << "\t" << (options.o_in_memory ? "in memory" : "on disk") << (options.async_executor ? " async" : ""); + if (options.o_in_memory) { + LOG(INFO) << "\t\tuse_arena=" << options.o_in_memory->use_arena + << " less_memory=" << options.o_in_memory->use_less_memory_during_creation; + } + for (td::uint32 i = 0; i < tests_n; i++) { + auto before = counter(); + options.seed = i == 0 ? 123 : i; + f(options); + auto after = counter(); + LOG_CHECK((options.o_in_memory && options.o_in_memory->use_arena) || before == after) + << before << " vs " << after; + } + }; + run({.async_executor = std::make_shared(4)}); + run({}); + for (auto use_arena : {false, true}) { + for (auto less_memory : {false, true}) { + run({.o_in_memory = + DynamicBagOfCellsDb::CreateInMemoryOptions{.extra_threads = std::thread::hardware_concurrency(), + .verbose = false, + .use_arena = use_arena, + .use_less_memory_during_creation = less_memory}}); + } + } +} + +void test_dynamic_boc(BocOptions options) { + auto counter = [] { return td::NamedThreadSafeCounter::get_default().get_counter("DataCell").sum(); }; + auto before = counter(); + SCOPE_EXIT { + LOG_CHECK((options.o_in_memory && options.o_in_memory->use_arena) || before == counter()) + << before << " vs " << counter(); + }; + td::Random::Xorshift128plus rnd{options.seed}; std::string old_root_hash; std::string old_root_serialization; auto kv = std::make_shared(); - auto dboc = DynamicBagOfCellsDb::create(); + auto create_dboc = [&]() { + auto roots_n = old_root_hash.empty() ? 0 : 1; + return options.create_dboc(kv.get(), roots_n); + }; + auto dboc = create_dboc(); dboc->set_loader(std::make_unique(kv)); for (int t = 1000; t >= 0; t--) { if (rnd() % 10 == 0) { - dboc = DynamicBagOfCellsDb::create(); + dboc = create_dboc(); } dboc->set_loader(std::make_unique(kv)); Ref old_root; @@ -795,29 +1075,41 @@ TEST(TonDb, DynamicBoc) { if (t != 0) { dboc->inc(cell); } - dboc->prepare_commit(); + dboc->prepare_commit().ensure(); { CellStorer cell_storer(*kv); - dboc->commit(cell_storer); + dboc->commit(cell_storer).ensure(); } } ASSERT_EQ(0u, kv->count("").ok()); +} + +TEST(TonDb, DynamicBoc) { + with_all_boc_options(test_dynamic_boc, 1); }; -TEST(TonDb, DynamicBoc2) { - int VERBOSITY_NAME(boc) = VERBOSITY_NAME(DEBUG) + 10; - td::Random::Xorshift128plus rnd{123}; - int total_roots = 10000; - int max_roots = 20; - std::vector root_hashes(max_roots); - std::vector> roots(max_roots); +void test_dynamic_boc2(BocOptions options) { + td::Random::Xorshift128plus rnd{options.seed}; + + int total_roots = rnd.fast(1, !rnd.fast(0, 10) * 100 + 10); + int max_roots = rnd.fast(1, 20); int last_commit_at = 0; int first_root_id = 0; int last_root_id = 0; auto kv = std::make_shared(); - auto dboc = DynamicBagOfCellsDb::create(); + auto create_dboc = [&](td::int64 root_n) { return options.create_dboc(kv.get(), root_n); }; + auto dboc = create_dboc(0); dboc->set_loader(std::make_unique(kv)); + auto counter = [] { return td::NamedThreadSafeCounter::get_default().get_counter("DataCell").sum(); }; + auto before = counter(); + SCOPE_EXIT{ + // LOG_CHECK((options.o_in_memory && options.o_in_memory->use_arena) || before == counter()) + // << before << " vs " << counter(); + }; + + std::vector> roots(max_roots); + std::vector root_hashes(max_roots); auto add_root = [&](Ref root) { dboc->inc(root); root_hashes[last_root_id % max_roots] = (root->get_hash().as_slice().str()); @@ -825,18 +1117,23 @@ TEST(TonDb, DynamicBoc2) { last_root_id++; }; - auto get_root = [&](int root_id) { + auto get_root = [&](int root_id) -> Ref { VLOG(boc) << " from older root #" << root_id; auto from_root = roots[root_id % max_roots]; if (from_root.is_null()) { VLOG(boc) << " from db"; auto from_root_hash = root_hashes[root_id % max_roots]; - from_root = dboc->load_cell(from_root_hash).move_as_ok(); + if (rnd() % 2 == 0) { + from_root = dboc->load_root(from_root_hash).move_as_ok(); + } else { + from_root = dboc->load_cell(from_root_hash).move_as_ok(); + } } else { VLOG(boc) << "FROM MEMORY"; } return from_root; }; + std::map root_cnt; auto new_root = [&] { if (last_root_id == total_roots) { return; @@ -850,13 +1147,16 @@ TEST(TonDb, DynamicBoc2) { from_root = get_root(rnd.fast(first_root_id, last_root_id - 1)); } VLOG(boc) << " ..."; - add_root(gen_random_cell(rnd.fast(1, 20), from_root, rnd)); + auto new_root = gen_random_cell(rnd.fast(1, 20), from_root, rnd); + root_cnt[new_root->get_hash()]++; + add_root(std::move(new_root)); VLOG(boc) << " OK"; }; auto commit = [&] { VLOG(boc) << "commit"; - dboc->prepare_commit(); + //rnd.fast(0, 1); + options.prepare_commit(*dboc); { CellStorer cell_storer(*kv); dboc->commit(cell_storer); @@ -870,7 +1170,7 @@ TEST(TonDb, DynamicBoc2) { auto reset = [&] { VLOG(boc) << "reset"; commit(); - dboc = DynamicBagOfCellsDb::create(); + dboc = create_dboc(td::int64(root_cnt.size())); dboc->set_loader(std::make_unique(kv)); }; @@ -879,7 +1179,15 @@ TEST(TonDb, DynamicBoc2) { if (first_root_id == last_root_id) { return; } - dboc->dec(get_root(first_root_id)); + auto old_root = get_root(first_root_id); + auto it = root_cnt.find(old_root->get_hash()); + it->second--; + CHECK(it->second >= 0); + if (it->second == 0) { + root_cnt.erase(it); + } + + dboc->dec(std::move(old_root)); first_root_id++; VLOG(boc) << " OK"; }; @@ -893,6 +1201,10 @@ TEST(TonDb, DynamicBoc2) { ASSERT_EQ(0u, kv->count("").ok()); } +TEST(TonDb, DynamicBoc2) { + with_all_boc_options(test_dynamic_boc2); +} + template td::Status test_boc_deserializer(std::vector> cells, int mode) { auto total_data_cells_before = vm::DataCell::get_total_data_cells(); @@ -1848,7 +2160,7 @@ TEST(TonDb, CompactArrayOld) { SCOPE_EXIT { ton_db->commit_transaction(std::move(txn)); }; - auto smart = txn->begin_smartcontract(""); + auto smart = txn->begin_smartcontract(); SCOPE_EXIT { txn->commit_smartcontract(std::move(smart)); }; @@ -1875,7 +2187,7 @@ TEST(TonDb, CompactArrayOld) { SCOPE_EXIT { ton_db->commit_transaction(std::move(txn)); }; - auto smart = txn->begin_smartcontract(""); + auto smart = txn->begin_smartcontract(); //smart->validate_meta(); SCOPE_EXIT { txn->commit_smartcontract(std::move(smart)); @@ -1896,7 +2208,7 @@ TEST(TonDb, CompactArrayOld) { SCOPE_EXIT { ton_db->abort_transaction(std::move(txn)); }; - auto smart = txn->begin_smartcontract(""); + auto smart = txn->begin_smartcontract(); SCOPE_EXIT { txn->abort_smartcontract(std::move(smart)); }; @@ -1950,17 +2262,18 @@ TEST(TonDb, BocRespectsUsageCell) { ASSERT_STREQ(serialization, serialization_of_virtualized_cell); } -TEST(TonDb, DynamicBocRespectsUsageCell) { - td::Random::Xorshift128plus rnd(123); +void test_dynamic_boc_respectes_usage_cell(vm::BocOptions options) { + td::Random::Xorshift128plus rnd(options.seed); auto cell = vm::gen_random_cell(20, rnd, true); auto usage_tree = std::make_shared(); auto usage_cell = vm::UsageCell::create(cell, usage_tree->root_ptr()); auto kv = std::make_shared(); - auto dboc = vm::DynamicBagOfCellsDb::create(); + auto dboc = options.create_dboc(kv.get(), {}); dboc->set_loader(std::make_unique(kv)); dboc->inc(usage_cell); { + options.prepare_commit(*dboc); vm::CellStorer cell_storer(*kv); dboc->commit(cell_storer); } @@ -1972,6 +2285,42 @@ TEST(TonDb, DynamicBocRespectsUsageCell) { ASSERT_STREQ(serialization, serialization_of_virtualized_cell); } +TEST(TonDb, DynamicBocRespectsUsageCell) { + vm::with_all_boc_options(test_dynamic_boc_respectes_usage_cell, 20); +} + +TEST(TonDb, LargeBocSerializer) { + td::Random::Xorshift128plus rnd{123}; + size_t n = 1000000; + std::vector data(n); + std::iota(data.begin(), data.end(), 0); + vm::CompactArray arr(data); + auto root = arr.root(); + std::string path = "serialization"; + td::unlink(path).ignore(); + auto fd = td::FileFd::open(path, td::FileFd::Flags::Create | td::FileFd::Flags::Truncate | td::FileFd::Flags::Write) + .move_as_ok(); + std_boc_serialize_to_file(root, fd, 31); + fd.close(); + auto a = td::read_file_str(path).move_as_ok(); + + auto kv = std::make_shared(); + auto dboc = vm::DynamicBagOfCellsDb::create(); + dboc->set_loader(std::make_unique(kv)); + dboc->inc(root); + dboc->prepare_commit(); + vm::CellStorer cell_storer(*kv); + dboc->commit(cell_storer); + dboc->set_loader(std::make_unique(kv)); + td::unlink(path).ignore(); + fd = td::FileFd::open(path, td::FileFd::Flags::Create | td::FileFd::Flags::Truncate | td::FileFd::Flags::Write) + .move_as_ok(); + std_boc_serialize_to_file_large(dboc->get_cell_db_reader(), root->get_hash(), fd, 31); + fd.close(); + auto b = td::read_file_str(path).move_as_ok(); + CHECK(a == b); +} + TEST(TonDb, DoNotMakeListsPrunned) { auto cell = vm::CellBuilder().store_bytes("abc").finalize(); auto is_prunned = [&](const td::Ref &cell) { return true; }; @@ -2020,7 +2369,7 @@ TEST(TonDb, CellStat) { ASSERT_EQ(stat.cells, new_stat.get_stat().cells); ASSERT_EQ(stat.bits, new_stat.get_stat().bits); - CHECK(usage_tree.unique()); + CHECK(usage_tree.use_count() == 1); usage_tree.reset(); td::Ref C, BC, C_proof; std::shared_ptr usage_tree_B; @@ -2057,7 +2406,6 @@ TEST(Ref, AtomicRef) { int threads_n = 10; std::vector nodes(threads_n); std::vector threads(threads_n); - int thread_id = 0; for (auto &thread : threads) { thread = td::thread([&] { for (int i = 0; i < 1000000; i++) { @@ -2072,7 +2420,6 @@ TEST(Ref, AtomicRef) { } } }); - thread_id++; } for (auto &thread : threads) { thread.join(); diff --git a/crypto/test/test-smartcont.cpp b/crypto/test/test-smartcont.cpp index 673bb758..7f512cea 100644 --- a/crypto/test/test-smartcont.cpp +++ b/crypto/test/test-smartcont.cpp @@ -35,6 +35,7 @@ #include "smc-envelope/SmartContract.h" #include "smc-envelope/SmartContractCode.h" #include "smc-envelope/WalletV3.h" +#include "smc-envelope/WalletV4.h" #include "smc-envelope/HighloadWallet.h" #include "smc-envelope/HighloadWalletV2.h" #include "smc-envelope/PaymentChannel.h" @@ -488,7 +489,7 @@ void do_test_wallet(int revision) { auto address = std::move(res.address); auto iwallet = std::move(res.wallet); auto public_key = priv_key.get_public_key().move_as_ok().as_octet_string(); - ; + check_wallet_state(iwallet, 1, 123, public_key); // lets send a lot of messages @@ -526,6 +527,7 @@ void do_test_wallet() { TEST(Tonlib, Wallet) { do_test_wallet(); + do_test_wallet(); do_test_wallet(); do_test_wallet(); do_test_wallet(); @@ -956,7 +958,7 @@ class MapDns { } return; } - if (!actions.category.is_zero()) { + if (actions.category.is_zero()) { entries_.erase(actions.name); LOG(ERROR) << "CLEAR " << actions.name; if (!actions.actions) { @@ -1001,7 +1003,7 @@ class CheckedDns { explicit CheckedDns(bool check_smc = true, bool check_combine = true) { if (check_smc) { key_ = td::Ed25519::generate_private_key().move_as_ok(); - dns_ = ManualDns::create(ManualDns::create_init_data_fast(key_.value().get_public_key().move_as_ok(), 123)); + dns_ = ManualDns::create(ManualDns::create_init_data_fast(key_.value().get_public_key().move_as_ok(), 123), -1); } if (check_combine) { combined_map_dns_ = MapDns(); @@ -1024,7 +1026,7 @@ class CheckedDns { } return action; }); - auto query = dns_->create_update_query(key_.value(), smc_actions).move_as_ok(); + auto query = dns_->create_update_query(key_.value(), smc_actions, query_id_++).move_as_ok(); CHECK(dns_.write().send_external_message(std::move(query)).code == 0); } map_dns_.update(entries); @@ -1079,6 +1081,7 @@ class CheckedDns { using ManualDns = ton::ManualDns; td::optional key_; td::Ref dns_; + td::uint32 query_id_ = 1; // Query id serve as "valid until", but in tests now() == 0 MapDns map_dns_; td::optional combined_map_dns_; @@ -1092,9 +1095,10 @@ class CheckedDns { } }; -static td::Bits256 intToCat(int x) { - td::Bits256 cat = td::Bits256::zero(); - cat.as_slice().copy_from(td::Slice((char*)&x, sizeof(x))); +static td::Bits256 intToCat(td::uint32 x) { + auto y = td::make_refint(x); + td::Bits256 cat; + y->export_bytes(cat.data(), 32, false); return cat; } @@ -1180,7 +1184,7 @@ TEST(Smartcont, DnsManual) { auto key = td::Ed25519::generate_private_key().move_as_ok(); - auto manual = ManualDns::create(ManualDns::create_init_data_fast(key.get_public_key().move_as_ok(), 123)); + auto manual = ManualDns::create(ManualDns::create_init_data_fast(key.get_public_key().move_as_ok(), 123), -1); CHECK(manual->get_wallet_id().move_as_ok() == 123); auto init_query = manual->create_init_query(key).move_as_ok(); LOG(ERROR) << "A"; diff --git a/crypto/test/vm.cpp b/crypto/test/vm.cpp index 3227f8fa..0f1b0442 100644 --- a/crypto/test/vm.cpp +++ b/crypto/test/vm.cpp @@ -28,7 +28,7 @@ #include "td/utils/StringBuilder.h" std::string run_vm(td::Ref cell) { - vm::init_op_cp0(); + vm::init_vm().ensure(); vm::DictionaryBase::get_empty_dictionary(); class Logger : public td::LogInterface { diff --git a/crypto/tl/tlbc-gen-cpp.cpp b/crypto/tl/tlbc-gen-cpp.cpp index dedec15d..5730f169 100644 --- a/crypto/tl/tlbc-gen-cpp.cpp +++ b/crypto/tl/tlbc-gen-cpp.cpp @@ -159,7 +159,6 @@ std::string CppIdentSet::compute_cpp_ident(std::string orig_ident, int count) { } if (!cnt) { os << '_'; - prev_skip = true; } if (count) { os << count; @@ -1317,7 +1316,7 @@ void CppTypeCode::clear_context() { std::string CppTypeCode::new_tmp_var() { char buffer[16]; while (true) { - sprintf(buffer, "t%d", ++tmp_ints); + snprintf(buffer, sizeof(buffer), "t%d", ++tmp_ints); if (tmp_cpp_ids.is_good_ident(buffer) && local_cpp_ids.is_good_ident(buffer)) { break; } @@ -2075,7 +2074,7 @@ void CppTypeCode::generate_skip_field(const Constructor& constr, const Field& fi output_cpp_expr(ss, expr, 100); ss << '.'; } - ss << "validate_skip_ref(ops, cs, weak)" << tail; + ss << "validate_skip_ref(ops, cs, " << (constr.is_special ? "true" : "weak") << ")" << tail; actions += Action{ss.str()}; } diff --git a/crypto/tl/tlbc.cpp b/crypto/tl/tlbc.cpp index 01b8a31a..d3a6edb5 100644 --- a/crypto/tl/tlbc.cpp +++ b/crypto/tl/tlbc.cpp @@ -420,7 +420,7 @@ void AdmissibilityInfo::operator|=(const AdmissibilityInfo& other) { std::size_t i, j, n = info.size(), n1 = other.info.size(); assert(n1 && !(n1 & (n1 - 1))); for (i = j = 0; i < n; i++) { - info[i] = info[i] | other.info[j]; + info[i] = info[i] || other.info[j]; j = (j + 1) & (n1 - 1); } } @@ -1800,9 +1800,6 @@ void Constructor::show(std::ostream& os, int mode) const { } for (int i = 0; i < type_arity; i++) { os << ' '; - if (param_negated.at(i)) { - os << '~'; - } params.at(i)->show(os, this, 100, mode | 1); } if (!(mode & 2)) { @@ -1998,7 +1995,7 @@ TypeExpr* parse_anonymous_constructor(Lexer& lex, Constructor& cs) { if (types[i].parent_type_idx >= 0) { types[i].parent_type_idx = -2; } - delete cs2; + cs2->~Constructor(); return TypeExpr::mk_apply_empty(lex.cur().loc, 0, &types[i]); } } @@ -2252,11 +2249,9 @@ TypeExpr* parse_expr10(Lexer& lex, Constructor& cs, int mode) { } if (op == '>') { std::swap(expr, expr2); - op = '<'; op_name = Less_name; } else if (op == src::_Geq) { std::swap(expr, expr2); - op = src::_Leq; op_name = Leq_name; } auto sym_def = sym::lookup_symbol(op_name, 2); @@ -2513,7 +2508,7 @@ void define_builtins() { Bits_type = define_builtin_type("bits", "#", false, 1023, 0, true, 0); for (int i = 1; i <= 257; i++) { char buff[8]; - sprintf(buff, "uint%d", i); + snprintf(buff, sizeof(buff), "uint%d", i); define_builtin_type(buff + 1, "", false, i, i, true, -1); if (i < 257) { define_builtin_type(buff, "", false, i, i, true, 1); @@ -2521,7 +2516,7 @@ void define_builtins() { } for (int i = 1; i <= 1023; i++) { char buff[12]; - sprintf(buff, "bits%d", i); + snprintf(buff, sizeof(buff), "bits%d", i); define_builtin_type(buff, "", false, i, i, true, 0); } Eq_type = define_builtin_type("=", "##", false, 0, 0, true); diff --git a/crypto/tl/tlblib.cpp b/crypto/tl/tlblib.cpp index 0e0e5626..de5a483c 100644 --- a/crypto/tl/tlblib.cpp +++ b/crypto/tl/tlblib.cpp @@ -45,7 +45,7 @@ bool Bool::print_skip(PrettyPrinter& pp, vm::CellSlice& cs) const { } bool NatWidth::print_skip(PrettyPrinter& pp, vm::CellSlice& cs) const { - long long value = (long long)cs.fetch_ulong(32); + long long value = (long long)cs.fetch_ulong(n); return value >= 0 && pp.out_int(value); } @@ -133,7 +133,13 @@ bool TLB::validate_ref_internal(int* ops, Ref cell_ref, bool weak) con } bool is_special; auto cs = load_cell_slice_special(std::move(cell_ref), is_special); - return always_special() ? is_special : (is_special ? weak : (validate_skip(ops, cs) && cs.empty_ext())); + if (cs.special_type() == vm::Cell::SpecialType::PrunnedBranch && weak) { + return true; + } + if (always_special() != is_special) { + return false; + } + return validate_skip(ops, cs, weak) && cs.empty_ext(); } bool TLB::print_skip(PrettyPrinter& pp, vm::CellSlice& cs) const { @@ -190,6 +196,13 @@ bool TLB::print_ref(std::ostream& os, Ref cell_ref, int indent, int re return pp.fail_unless(print_ref(pp, std::move(cell_ref))); } +bool TLB::print_ref(td::StringBuilder& sb, Ref cell_ref, int indent, int rec_limit) const { + std::ostringstream ss; + auto result = print_ref(ss, std::move(cell_ref), indent, rec_limit); + sb << ss.str(); + return result; +} + std::string TLB::as_string_skip(vm::CellSlice& cs, int indent) const { std::ostringstream os; print_skip(os, cs, indent); diff --git a/crypto/tl/tlblib.hpp b/crypto/tl/tlblib.hpp index a6350ece..c10049a9 100644 --- a/crypto/tl/tlblib.hpp +++ b/crypto/tl/tlblib.hpp @@ -246,7 +246,14 @@ class TLB { bool print(std::ostream& os, Ref cs_ref, int indent = 0, int rec_limit = 0) const { return print(os, *cs_ref, indent, rec_limit); } + bool print(td::StringBuilder& sb, Ref cs_ref, int indent = 0, int rec_limit = 0) const { + std::ostringstream ss; + auto result = print(ss, *cs_ref, indent, rec_limit); + sb << ss.str(); + return result; + } bool print_ref(std::ostream& os, Ref cell_ref, int indent = 0, int rec_limit = 0) const; + bool print_ref(td::StringBuilder& sb, Ref cell_ref, int indent = 0, int rec_limit = 0) const; bool print_ref(int rec_limit, std::ostream& os, Ref cell_ref, int indent = 0) const { return print_ref(os, std::move(cell_ref), indent, rec_limit); } diff --git a/crypto/util/mintless-proof-generator.cpp b/crypto/util/mintless-proof-generator.cpp new file mode 100644 index 00000000..62b5f0d6 --- /dev/null +++ b/crypto/util/mintless-proof-generator.cpp @@ -0,0 +1,395 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ + +#include "block-parse.h" +#include "block.h" +#include "td/actor/core/Actor.h" +#include "td/db/utils/BlobView.h" + +#include +#include "td/utils/OptionParser.h" +#include "td/utils/Time.h" +#include "td/utils/base64.h" +#include "td/utils/filesystem.h" +#include "td/utils/logging.h" +#include "vm/cells/MerkleProof.h" +#include "vm/db/StaticBagOfCellsDb.h" + +#include +#include + +const size_t KEY_LEN = 3 + 8 + 256; + +void print_help() { + std::cerr << "mintless-proof-generator - generates proofs for mintless jettons. Usage:\n\n"; + std::cerr << "mintless-proof-generator generate \n"; + std::cerr << " Generate a full tree for , save boc to .\n"; + std::cerr << " Input format: each line is
.\n\n"; + std::cerr << "mintless-proof-generator make_proof
.\n"; + std::cerr << " Generate a proof for address
from tree , save boc to file .\n\n"; + std::cerr << "mintless-proof-generator parse \n"; + std::cerr << " Read a tree from and output it as text to .\n"; + std::cerr << " Output format: same as input for 'generate'.\n\n"; + std::cerr << "mintless-proof-generator make_all_proofs [--threads ]\n"; + std::cerr << " Read a tree from and output proofs for all accounts to .\n"; + std::cerr << " Output format:
,\n"; + std::cerr << " Default : 1\n"; + exit(2); +} + +void log_mem_stat() { + auto r_stat = td::mem_stat(); + if (r_stat.is_error()) { + LOG(WARNING) << "Memory: " << r_stat.move_as_error(); + return; + } + auto stat = r_stat.move_as_ok(); + LOG(WARNING) << "Memory: " + << "res=" << stat.resident_size_ << " (peak=" << stat.resident_size_peak_ + << ") virt=" << stat.virtual_size_ << " (peak=" << stat.virtual_size_peak_ << ")"; +} + +td::BitArray address_to_key(const block::StdAddress &address) { + // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; + vm::CellBuilder cb; + cb.store_long(0b100, 3); + cb.store_long(address.workchain, 8); + cb.store_bits(address.addr.as_bitslice()); + return cb.data_bits(); +} + +block::StdAddress key_to_address(const td::BitArray &key) { + block::StdAddress addr; + td::ConstBitPtr ptr = key.bits(); + LOG_CHECK(ptr.get_uint(3) == 0b100) << "Invalid address"; + ptr.advance(3); + addr.workchain = (ton::WorkchainId)ptr.get_int(8); + ptr.advance(8); + addr.addr = ptr; + return addr; +} + +struct Entry { + block::StdAddress address; + td::RefInt256 amount; + td::uint64 start_from = 0, expired_at = 0; + + td::BitArray get_key() const { + return address_to_key(address); + } + + td::Ref get_value() const { + // _ amount:Coins start_from:uint48 expired_at:uint48 = AirdropItem; + vm::CellBuilder cb; + bool ok = block::tlb::t_Grams.store_integer_value(cb, *amount) && cb.store_ulong_rchk_bool(start_from, 48) && + cb.store_ulong_rchk_bool(expired_at, 48); + LOG_CHECK(ok) << "Failed to serialize AirdropItem"; + return cb.as_cellslice_ref(); + } + + static Entry parse(const td::BitArray &key, vm::CellSlice value) { + Entry e; + e.address = key_to_address(key); + bool ok = block::tlb::t_Grams.as_integer_skip_to(value, e.amount) && value.fetch_uint_to(48, e.start_from) && + value.fetch_uint_to(48, e.expired_at) && value.empty_ext(); + LOG_CHECK(ok) << "Failed to parse AirdropItem"; + return e; + } +}; + +bool read_entry(std::istream &f, Entry &entry) { + std::string line; + while (std::getline(f, line)) { + std::vector v = td::full_split(line, ' '); + if (v.empty()) { + continue; + } + auto S = [&]() -> td::Status { + if (v.size() != 4) { + return td::Status::Error("Invalid line in input"); + } + TRY_RESULT_PREFIX_ASSIGN(entry.address, block::StdAddress::parse(v[0]), "Invalid address in input: "); + entry.amount = td::string_to_int256(v[1]); + if (entry.amount.is_null() || !entry.amount->is_valid() || entry.amount->sgn() < 0) { + return td::Status::Error(PSTRING() << "Invalid amount in input: " << v[1]); + } + TRY_RESULT_PREFIX_ASSIGN(entry.start_from, td::to_integer_safe(v[2]), + "Invalid start_from in input: "); + TRY_RESULT_PREFIX_ASSIGN(entry.expired_at, td::to_integer_safe(v[3]), + "Invalid expired_at in input: "); + return td::Status::OK(); + }(); + S.ensure(); + return true; + } + return false; +} + +td::Status run_generate(std::string in_filename, std::string out_filename) { + LOG(INFO) << "Generating tree from " << in_filename; + std::ifstream in_file{in_filename}; + LOG_CHECK(in_file.is_open()) << "Cannot open file " << in_filename; + + Entry entry; + vm::Dictionary dict{KEY_LEN}; + td::uint64 count = 0; + td::Timestamp log_at = td::Timestamp::in(5.0); + while (read_entry(in_file, entry)) { + ++count; + bool ok = dict.set(entry.get_key(), entry.get_value(), vm::DictionaryBase::SetMode::Add); + LOG_CHECK(ok) << "Failed to add entry " << entry.address.rserialize() << " (line #" << count << ")"; + if (log_at.is_in_past()) { + LOG(INFO) << "Added " << count << " entries"; + log_at = td::Timestamp::in(5.0); + } + } + LOG_CHECK(in_file.eof()) << "Failed to read file " << in_filename; + in_file.close(); + + LOG_CHECK(count != 0) << "Input is empty"; + td::Ref root = dict.get_root_cell(); + LOG(INFO) << "Total: " << count << " entries, root hash: " << root->get_hash().to_hex(); + vm::BagOfCells boc; + boc.add_root(root); + TRY_STATUS(boc.import_cells()); + LOG(INFO) << "Writing to " << out_filename; + TRY_RESULT(fd, td::FileFd::open(out_filename, td::FileFd::Write | td::FileFd::Truncate | td::FileFd::Create)); + TRY_STATUS(boc.serialize_to_file(fd, 31)); + TRY_STATUS(fd.sync()); + fd.close(); + log_mem_stat(); + return td::Status::OK(); +} + +td::Status run_make_proof(std::string in_filename, std::string s_address, std::string out_filename) { + LOG(INFO) << "Generating proof for " << s_address << ", input file is " << in_filename; + TRY_RESULT(address, block::StdAddress::parse(s_address)); + + TRY_RESULT(blob_view, td::FileBlobView::create(in_filename)); + TRY_RESULT(boc, vm::StaticBagOfCellsDbLazy::create(std::move(blob_view))); + TRY_RESULT(root, boc->get_root_cell(0)); + + vm::MerkleProofBuilder mpb{root}; + vm::Dictionary dict{mpb.root(), KEY_LEN}; + auto key = address_to_key(address); + td::Ref value = dict.lookup(key); + LOG_CHECK(value.not_null()) << "No entry for address " << s_address; + Entry e = Entry::parse(key, *value); + LOG(INFO) << "Entry: address=" << e.address.workchain << ":" << e.address.addr.to_hex() + << " amount=" << e.amount->to_dec_string() << " start_from=" << e.start_from + << " expire_at=" << e.expired_at; + + TRY_RESULT(proof, mpb.extract_proof_boc()); + LOG(INFO) << "Writing proof to " << out_filename << " (" << td::format::as_size(proof.size()) << ")"; + TRY_STATUS(td::write_file(out_filename, proof)); + log_mem_stat(); + return td::Status::OK(); +} + +td::Status run_parse(std::string in_filename, std::string out_filename) { + LOG(INFO) << "Parsing " << in_filename; + std::ofstream out_file{out_filename}; + LOG_CHECK(out_file.is_open()) << "Cannot open file " << out_filename; + + TRY_RESULT(blob_view, td::FileBlobView::create(in_filename)); + TRY_RESULT(boc, vm::StaticBagOfCellsDbLazy::create(std::move(blob_view))); + TRY_RESULT(root, boc->get_root_cell(0)); + LOG(INFO) << "Root hash = " << root->get_hash().to_hex(); + vm::Dictionary dict{root, KEY_LEN}; + td::Timestamp log_at = td::Timestamp::in(5.0); + td::uint64 count = 0; + bool ok = dict.check_for_each([&](td::Ref value, td::ConstBitPtr key, int key_len) { + CHECK(key_len == KEY_LEN); + Entry e = Entry::parse(key, *value); + out_file << e.address.workchain << ":" << e.address.addr.to_hex() << " " << e.amount->to_dec_string() << " " + << e.start_from << " " << e.expired_at << "\n"; + LOG_CHECK(!out_file.fail()) << "Failed to write to " << out_filename; + ++count; + if (log_at.is_in_past()) { + LOG(INFO) << "Parsed " << count << " entries"; + log_at = td::Timestamp::in(5.0); + } + return true; + }); + LOG_CHECK(ok) << "Failed to parse dictionary"; + out_file.close(); + LOG_CHECK(!out_file.fail()) << "Failed to write to " << out_filename; + LOG(INFO) << "Written " << count << " entries to " << out_filename; + log_mem_stat(); + return td::Status::OK(); +} + +class MakeAllProofsActor : public td::actor::core::Actor { + public: + MakeAllProofsActor(std::string in_filename, std::string out_filename, int max_workers) + : in_filename_(in_filename), out_filename_(out_filename), max_workers_(max_workers) { + } + + void start_up() override { + auto S = [&]() -> td::Status { + out_file_.open(out_filename_); + LOG_CHECK(out_file_.is_open()) << "Cannot open file " << out_filename_; + LOG(INFO) << "Reading " << in_filename_; + TRY_RESULT(blob_view, td::FileBlobView::create(in_filename_)); + TRY_RESULT(boc, vm::StaticBagOfCellsDbLazy::create(std::move(blob_view))); + TRY_RESULT(root, boc->get_root_cell(0)); + LOG(INFO) << "Root hash = " << root->get_hash().to_hex(); + dict_ = vm::Dictionary{root, KEY_LEN}; + return td::Status::OK(); + }(); + S.ensure(); + run(); + alarm_timestamp() = td::Timestamp::in(5.0); + } + + void alarm() override { + alarm_timestamp() = td::Timestamp::in(5.0); + LOG(INFO) << "Processed " << written_count_ << " entries"; + } + + void run() { + for (auto it = pending_results_.begin(); it != pending_results_.end() && !it->second.empty();) { + out_file_ << it->second << "\n"; + LOG_CHECK(!out_file_.fail()) << "Failed to write to " << out_filename_; + it = pending_results_.erase(it); + ++written_count_; + } + while (active_workers_ < max_workers_ && !eof_) { + td::Ref value = dict_.lookup_nearest_key(current_key_, true, current_idx_ == 0); + if (value.is_null()) { + eof_ = true; + break; + } + run_worker(current_key_, current_idx_); + ++current_idx_; + ++active_workers_; + } + if (eof_ && active_workers_ == 0) { + out_file_.close(); + LOG_CHECK(!out_file_.fail()) << "Failed to write to " << out_filename_; + LOG(INFO) << "Written " << written_count_ << " entries to " << out_filename_; + stop(); + td::actor::SchedulerContext::get()->stop(); + } + } + + void run_worker(td::BitArray key, td::uint64 idx) { + pending_results_[idx] = ""; + ton::delay_action( + [SelfId = actor_id(this), key, idx, root = dict_.get_root_cell()]() { + vm::MerkleProofBuilder mpb{root}; + CHECK(vm::Dictionary(mpb.root(), KEY_LEN).lookup(key).not_null()); + auto r_proof = mpb.extract_proof_boc(); + r_proof.ensure(); + block::StdAddress addr = key_to_address(key); + std::string result = PSTRING() << addr.workchain << ":" << addr.addr.to_hex() << "," + << td::base64_encode(r_proof.move_as_ok()); + td::actor::send_closure(SelfId, &MakeAllProofsActor::on_result, idx, std::move(result)); + }, + td::Timestamp::now()); + } + + void on_result(td::uint64 idx, std::string result) { + pending_results_[idx] = std::move(result); + --active_workers_; + run(); + } + + private: + std::string in_filename_, out_filename_; + int max_workers_; + + std::ofstream out_file_; + vm::Dictionary dict_{KEY_LEN}; + td::BitArray current_key_ = td::BitArray::zero(); + td::uint64 current_idx_ = 0; + bool eof_ = false; + int active_workers_ = 0; + + std::map pending_results_; + td::uint64 written_count_ = 0; +}; + +td::Status run_make_all_proofs(std::string in_filename, std::string out_filename, int threads) { + td::actor::Scheduler scheduler({(size_t)threads}); + scheduler.run_in_context( + [&] { td::actor::create_actor("proofs", in_filename, out_filename, threads).release(); }); + while (scheduler.run(1)) { + } + log_mem_stat(); + return td::Status::OK(); +} + +int main(int argc, char *argv[]) { + SET_VERBOSITY_LEVEL(verbosity_INFO); + td::set_log_fatal_error_callback([](td::CSlice) { exit(2); }); + if (argc <= 1) { + print_help(); + return 2; + } + + std::string command = argv[1]; + try { + if (command == "generate") { + if (argc != 4) { + print_help(); + } + run_generate(argv[2], argv[3]).ensure(); + return 0; + } + if (command == "make_proof") { + if (argc != 5) { + print_help(); + } + run_make_proof(argv[2], argv[3], argv[4]).ensure(); + return 0; + } + if (command == "parse") { + if (argc != 4) { + print_help(); + } + run_parse(argv[2], argv[3]).ensure(); + return 0; + } + + if (command == "make_all_proofs") { + std::vector args; + int threads = 1; + for (int i = 2; i < argc; ++i) { + if (!strcmp(argv[i], "--threads")) { + ++i; + auto r = td::to_integer_safe(td::as_slice(argv[i])); + LOG_CHECK(r.is_ok() && r.ok() >= 1 && r.ok() <= 127) << " should be in [1..127]"; + threads = r.move_as_ok(); + } else { + args.push_back(argv[i]); + } + } + if (args.size() != 2) { + print_help(); + } + run_make_all_proofs(args[0], args[1], threads).ensure(); + return 0; + } + } catch (vm::VmError &e) { + LOG(FATAL) << "VM error: " << e.get_msg(); + } catch (vm::VmVirtError &e) { + LOG(FATAL) << "VM error: " << e.get_msg(); + } + + LOG(FATAL) << "Unknown command '" << command << "'"; +} diff --git a/crypto/vm/Hasher.cpp b/crypto/vm/Hasher.cpp new file mode 100644 index 00000000..f70988d3 --- /dev/null +++ b/crypto/vm/Hasher.cpp @@ -0,0 +1,148 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "vm/Hasher.h" +#include "vm/excno.hpp" +#include "vm/vm.h" +#include +#include +#include "keccak/keccak.h" + +namespace vm { + +using td::Ref; + +class HasherImplEVP : public Hasher::HasherImpl { + public: + explicit HasherImplEVP(EVP_MD_CTX* ctx) : ctx_(ctx) { + } + + ~HasherImplEVP() override { + EVP_MD_CTX_free(ctx_); + } + + void append(const unsigned char *data, size_t size) override { + CHECK(EVP_DigestUpdate(ctx_, data, size)); + } + + td::BufferSlice finish() override { + td::BufferSlice hash(EVP_MD_CTX_size(ctx_)); + unsigned size; + CHECK(EVP_DigestFinal_ex(ctx_, (unsigned char *)hash.data(), &size) || size != hash.size()); + return hash; + } + + std::unique_ptr make_copy() const override { + EVP_MD_CTX *new_ctx = nullptr; + new_ctx = EVP_MD_CTX_new(); + CHECK(new_ctx != nullptr); + CHECK(EVP_MD_CTX_copy_ex(new_ctx, ctx_)); + return std::make_unique(new_ctx); + } + + private: + EVP_MD_CTX *ctx_; +}; + +class HasherImplKeccak : public Hasher::HasherImpl { + public: + explicit HasherImplKeccak(size_t hash_size) : hash_size_(hash_size) { + CHECK(keccak_init(&state_, hash_size * 2, 24) == 0); + CHECK(state_ != nullptr); + } + + ~HasherImplKeccak() override { + CHECK(keccak_destroy(state_) == 0); + } + + void append(const unsigned char *data, size_t size) override { + CHECK(keccak_absorb(state_, data, size) == 0); + } + + td::BufferSlice finish() override { + td::BufferSlice hash(hash_size_); + CHECK(keccak_digest(state_, (unsigned char*)hash.data(), hash_size_, 1) == 0); + return hash; + } + + std::unique_ptr make_copy() const override { + auto copy = std::make_unique(hash_size_); + CHECK(keccak_copy(state_, copy->state_) == 0); + return copy; + } + + private: + size_t hash_size_; + keccak_state *state_ = nullptr; +}; + +Hasher::Hasher(int hash_id) : id_(hash_id) { + if (hash_id == KECCAK256 || hash_id == KECCAK512) { + impl_ = std::make_unique(hash_id == KECCAK256 ? 32 : 64); + return; + } + + EVP_MD_CTX *ctx = EVP_MD_CTX_new(); + CHECK(ctx != nullptr); + const EVP_MD *evp; + switch (hash_id) { + case SHA256: evp = EVP_sha256(); break; + case SHA512: evp = EVP_sha512(); break; + case BLAKE2B: evp = EVP_blake2b512(); break; + default: + throw VmError{Excno::range_chk, "invalid hash id"}; + } + CHECK(evp != nullptr && EVP_DigestInit_ex(ctx, evp, nullptr)); + impl_ = std::make_unique(ctx); +} + +void Hasher::append(td::ConstBitPtr data, unsigned size) { + if (!impl_) { + throw VmError{Excno::unknown, "can't use finished hasher"}; + } + while (size > 0) { + unsigned cur_size = std::min(size, BUF_SIZE * 8 - buf_ptr_); + td::BitPtr{buf_, (int)buf_ptr_}.copy_from(data, cur_size); + buf_ptr_ += cur_size; + if (buf_ptr_ == BUF_SIZE * 8) { + impl_->append(buf_, BUF_SIZE); + buf_ptr_ = 0; + } + size -= cur_size; + data += cur_size; + } +} + +td::BufferSlice Hasher::finish() { + if (!impl_) { + throw VmError{Excno::unknown, "can't use finished hasher"}; + } + if (buf_ptr_ % 8 != 0) { + throw VmError{Excno::cell_und, "data does not consist of an integer number of bytes"}; + } + impl_->append(buf_, buf_ptr_ / 8); + td::BufferSlice hash = impl_->finish(); + impl_ = nullptr; + return hash; +} + +static const size_t BYTES_PER_GAS_UNIT[5] = {33, 16, 19, 11, 6}; + +size_t Hasher::bytes_per_gas_unit() const { + return BYTES_PER_GAS_UNIT[id_]; +} + +} diff --git a/crypto/vm/Hasher.h b/crypto/vm/Hasher.h new file mode 100644 index 00000000..7e441690 --- /dev/null +++ b/crypto/vm/Hasher.h @@ -0,0 +1,58 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "common/refcnt.hpp" +#include "td/utils/buffer.h" +#include "common/bitstring.h" +#include "vm/cells/Cell.h" +#include + +namespace vm { + +using td::Ref; + +class Hasher { + public: + explicit Hasher(int id); + Hasher(const Hasher&) = delete; + void append(td::ConstBitPtr data, unsigned size); + td::BufferSlice finish(); + size_t bytes_per_gas_unit() const; + + static const int SHA256 = 0; + static const int SHA512 = 1; + static const int BLAKE2B = 2; + static const int KECCAK256 = 3; + static const int KECCAK512 = 4; + + class HasherImpl { + public: + virtual ~HasherImpl() = default; + virtual void append(const unsigned char* data, size_t size) = 0; + virtual td::BufferSlice finish() = 0; + virtual std::unique_ptr make_copy() const = 0; + }; + + private: + int id_ = 0; + static const unsigned BUF_SIZE = 256; + unsigned char buf_[BUF_SIZE]; + unsigned buf_ptr_ = 0; + std::unique_ptr impl_; +}; + +} \ No newline at end of file diff --git a/crypto/vm/arithops.cpp b/crypto/vm/arithops.cpp index 2831944b..fc482fc4 100644 --- a/crypto/vm/arithops.cpp +++ b/crypto/vm/arithops.cpp @@ -38,8 +38,8 @@ int exec_push_tinyint4(VmState* st, unsigned args) { std::string dump_push_tinyint4(CellSlice&, unsigned args) { int x = (int)((args + 5) & 15) - 5; - std::ostringstream os{"PUSHINT "}; - os << x; + std::ostringstream os; + os << "PUSHINT " << x; return os.str(); } @@ -53,8 +53,8 @@ int exec_push_tinyint8(VmState* st, unsigned args) { std::string dump_op_tinyint8(const char* op_prefix, CellSlice&, unsigned args) { int x = (signed char)args; - std::ostringstream os{op_prefix}; - os << x; + std::ostringstream os; + os << op_prefix << x; return os.str(); } @@ -68,8 +68,8 @@ int exec_push_smallint(VmState* st, unsigned args) { std::string dump_push_smallint(CellSlice&, unsigned args) { int x = (short)args; - std::ostringstream os{"PUSHINT "}; - os << x; + std::ostringstream os; + os << "PUSHINT " << x; return os.str(); } @@ -93,8 +93,8 @@ std::string dump_push_int(CellSlice& cs, unsigned args, int pfx_bits) { } cs.advance(pfx_bits); td::RefInt256 x = cs.fetch_int256(3 + l * 8); - std::ostringstream os{"PUSHINT "}; - os << x; + std::ostringstream os; + os << "PUSHINT " << x; return os.str(); } @@ -265,26 +265,45 @@ void register_add_mul_ops(OpcodeTable& cp0) { int exec_divmod(VmState* st, unsigned args, int quiet) { int round_mode = (int)(args & 3) - 1; - if (!(args & 12) || round_mode == 2) { + unsigned d = (args >> 2) & 3; + bool add = false; + if (d == 0 && st->get_global_version() >= 4) { + d = 3; + add = true; + } + if (d == 0 || round_mode == 2) { throw VmError{Excno::inv_opcode}; } Stack& stack = st->get_stack(); VM_LOG(st) << "execute DIV/MOD " << (args & 15); - stack.check_underflow(2); + stack.check_underflow(add ? 3 : 2); auto y = stack.pop_int(); + auto w = add ? stack.pop_int() : td::RefInt256{}; auto x = stack.pop_int(); - switch ((args >> 2) & 3) { - case 1: - stack.push_int_quiet(td::div(std::move(x), std::move(y), round_mode), quiet); - break; - case 2: - stack.push_int_quiet(td::mod(std::move(x), std::move(y), round_mode), quiet); - break; - case 3: { - auto dm = td::divmod(std::move(x), std::move(y), round_mode); - stack.push_int_quiet(std::move(dm.first), quiet); - stack.push_int_quiet(std::move(dm.second), quiet); - break; + if (add) { + CHECK(d == 3); + typename td::BigInt256::DoubleInt tmp{*x}, quot; + tmp += *w; + tmp.mod_div(*y, quot, round_mode); + auto q = td::make_refint(quot), r = td::make_refint(tmp); + q.write().normalize(); + r.write().normalize(); + stack.push_int_quiet(std::move(q), quiet); + stack.push_int_quiet(std::move(r), quiet); + } else { + switch (d) { + case 1: + stack.push_int_quiet(td::div(std::move(x), std::move(y), round_mode), quiet); + break; + case 2: + stack.push_int_quiet(td::mod(std::move(x), std::move(y), round_mode), quiet); + break; + case 3: { + auto dm = td::divmod(std::move(x), std::move(y), round_mode); + stack.push_int_quiet(std::move(dm.first), quiet); + stack.push_int_quiet(std::move(dm.second), quiet); + break; + } } } return 0; @@ -292,17 +311,26 @@ int exec_divmod(VmState* st, unsigned args, int quiet) { std::string dump_divmod(CellSlice&, unsigned args, bool quiet) { int round_mode = (int)(args & 3); - if (!(args & 12) || round_mode == 3) { + unsigned d = (args >> 2) & 3; + bool add = false; + if (d == 0) { + d = 3; + add = true; + } + if (round_mode == 3) { return ""; } - std::string s = (args & 4) ? "DIV" : ""; - if (args & 8) { + std::string s = add ? "ADD" : ""; + if (d & 1) { + s += "DIV"; + } + if (d & 2) { s += "MOD"; } if (quiet) { s = "Q" + s; } - return s + "FRC"[round_mode]; + return round_mode ? s + "FRC"[round_mode] : s; } int exec_shrmod(VmState* st, unsigned args, int mode) { @@ -312,32 +340,50 @@ int exec_shrmod(VmState* st, unsigned args, int mode) { args >>= 8; } int round_mode = (int)(args & 3) - 1; - if (!(args & 12) || round_mode == 2) { + unsigned d = (args >> 2) & 3; + bool add = false; + if (d == 0 && st->get_global_version() >= 4) { + d = 3; + add = true; + } + if (d == 0 || round_mode == 2) { throw VmError{Excno::inv_opcode}; } Stack& stack = st->get_stack(); VM_LOG(st) << "execute SHR/MOD " << (args & 15) << ',' << y; if (!(mode & 2)) { - stack.check_underflow(2); + stack.check_underflow(add ? 3 : 2); y = stack.pop_smallint_range(256); } else { - stack.check_underflow(1); + stack.check_underflow(add ? 2 : 1); } if (!y) { round_mode = -1; } + auto w = add ? stack.pop_int() : td::RefInt256{}; auto x = stack.pop_int(); - switch ((args >> 2) & 3) { - case 1: - stack.push_int_quiet(td::rshift(std::move(x), y, round_mode), mode & 1); - break; - case 3: - stack.push_int_quiet(td::rshift(x, y, round_mode), mode & 1); - // fallthrough - case 2: - x.write().mod_pow2(y, round_mode).normalize(); - stack.push_int_quiet(std::move(x), mode & 1); - break; + if (add) { + CHECK(d == 3); + typename td::BigInt256::DoubleInt tmp{*x}, quot; + tmp += *w; + typename td::BigInt256::DoubleInt tmp2{tmp}; + tmp2.rshift(y, round_mode).normalize(); + stack.push_int_quiet(td::make_refint(tmp2), mode & 1); + tmp.normalize().mod_pow2(y, round_mode).normalize(); + stack.push_int_quiet(td::make_refint(tmp), mode & 1); + } else { + switch (d) { + case 1: + stack.push_int_quiet(td::rshift(std::move(x), y, round_mode), mode & 1); + break; + case 3: + stack.push_int_quiet(td::rshift(x, y, round_mode), mode & 1); + // fallthrough + case 2: + x.write().mod_pow2(y, round_mode).normalize(); + stack.push_int_quiet(std::move(x), mode & 1); + break; + } } return 0; } @@ -349,49 +395,68 @@ std::string dump_shrmod(CellSlice&, unsigned args, int mode) { args >>= 8; } int round_mode = (int)(args & 3); - if (!(args & 12) || round_mode == 3) { + if (round_mode == 3) { return ""; } - std::string s; + std::ostringstream os; + if (mode & 1) { + os << 'Q'; + } + std::string end; switch (args & 12) { case 4: - s = "RSHIFT"; + os << "RSHIFT"; break; case 8: - s = "MODPOW2"; + os << "MODPOW2"; break; case 12: - s = "RSHIFTMOD"; + os << "RSHIFT"; + end = "MOD"; + break; + case 0: + os << "ADDRSHIFT"; + end = "MOD"; break; } - if (mode & 1) { - s = "Q" + s; + if (!(mode & 2)) { + os << end; + } + if (round_mode) { + os << "FRC"[round_mode]; } - s += "FRC"[round_mode]; if (mode & 2) { - char buff[8]; - sprintf(buff, " %d", y); - s += buff; + os << "#" << end << ' ' << y; } - return s; + return os.str(); } int exec_muldivmod(VmState* st, unsigned args, int quiet) { int round_mode = (int)(args & 3) - 1; - if (!(args & 12) || round_mode == 2) { + unsigned d = (args >> 2) & 3; + bool add = false; + if (d == 0 && st->get_global_version() >= 4) { + d = 3; + add = true; + } + if (d == 0 || round_mode == 2) { throw VmError{Excno::inv_opcode}; } Stack& stack = st->get_stack(); VM_LOG(st) << "execute MULDIV/MOD " << (args & 15); - stack.check_underflow(3); + stack.check_underflow(add ? 4 : 3); auto z = stack.pop_int(); + auto w = add ? stack.pop_int() : td::RefInt256{}; auto y = stack.pop_int(); auto x = stack.pop_int(); typename td::BigInt256::DoubleInt tmp{0}, quot; + if (add) { + tmp = *w; + } tmp.add_mul(*x, *y); auto q = td::make_refint(); tmp.mod_div(*z, quot, round_mode); - switch ((args >> 2) & 3) { + switch (d) { case 1: stack.push_int_quiet(td::make_refint(quot.normalize()), quiet); break; @@ -407,17 +472,26 @@ int exec_muldivmod(VmState* st, unsigned args, int quiet) { std::string dump_muldivmod(CellSlice&, unsigned args, bool quiet) { int round_mode = (int)(args & 3); - if (!(args & 12) || round_mode == 3) { + unsigned d = (args >> 2) & 3; + bool add = false; + if (d == 0) { + d = 3; + add = true; + } + if (round_mode == 3) { return ""; } - std::string s = (args & 4) ? "MULDIV" : "MUL"; - if (args & 8) { + std::string s = add ? "MULADD" : "MUL"; + if (d & 1) { + s += "DIV"; + } + if (d & 2) { s += "MOD"; } if (quiet) { s = "Q" + s; } - return s + "FRC"[round_mode]; + return round_mode ? s + "FRC"[round_mode] : s; } int exec_mulshrmod(VmState* st, unsigned args, int mode) { @@ -427,25 +501,35 @@ int exec_mulshrmod(VmState* st, unsigned args, int mode) { args >>= 8; } int round_mode = (int)(args & 3) - 1; - if (!(args & 12) || round_mode == 2) { + unsigned d = (args >> 2) & 3; + bool add = false; + if (d == 0 && st->get_global_version() >= 4) { + d = 3; + add = true; + } + if (d == 0 || round_mode == 2) { throw VmError{Excno::inv_opcode}; } Stack& stack = st->get_stack(); VM_LOG(st) << "execute MULSHR/MOD " << (args & 15) << ',' << z; if (!(mode & 2)) { - stack.check_underflow(3); + stack.check_underflow(add ? 4 : 3); z = stack.pop_smallint_range(256); } else { - stack.check_underflow(2); + stack.check_underflow(add ? 3 : 2); } if (!z) { round_mode = -1; } + auto w = add ? stack.pop_int() : td::RefInt256{}; auto y = stack.pop_int(); auto x = stack.pop_int(); typename td::BigInt256::DoubleInt tmp{0}; - tmp.add_mul(*x, *y); - switch ((args >> 2) & 3) { + if (add) { + tmp = *w; + } + tmp.add_mul(*x, *y).normalize(); + switch (d) { case 1: tmp.rshift(z, round_mode).normalize(); stack.push_int_quiet(td::make_refint(tmp), mode & 1); @@ -471,31 +555,41 @@ std::string dump_mulshrmod(CellSlice&, unsigned args, int mode) { args >>= 8; } int round_mode = (int)(args & 3); - if (!(args & 12) || round_mode == 3) { + if (round_mode == 3) { return ""; } - std::string s; + std::ostringstream os; + if (mode & 1) { + os << 'Q'; + } + std::string end; switch (args & 12) { case 4: - s = "MULRSHIFT"; + os << "MULRSHIFT"; break; case 8: - s = "MULMODPOW2"; + os << "MULMODPOW2"; break; case 12: - s = "MULRSHIFTMOD"; + os << "MULRSHIFT"; + end = "MOD"; + break; + case 0: + os << "MULADDRSHIFT"; + end = "MOD"; break; } - if (mode & 1) { - s = "Q" + s; + if (round_mode) { + os << "FRC"[round_mode]; } - s += "FRC"[round_mode]; if (mode & 2) { - char buff[8]; - sprintf(buff, " %d", y); - s += buff; + os << "#"; } - return s; + os << end; + if (mode & 2) { + os << ' ' << y; + } + return os.str(); } int exec_shldivmod(VmState* st, unsigned args, int mode) { @@ -505,22 +599,32 @@ int exec_shldivmod(VmState* st, unsigned args, int mode) { args >>= 8; } int round_mode = (int)(args & 3) - 1; - if (!(args & 12) || round_mode == 2) { + unsigned d = (args >> 2) & 3; + bool add = false; + if (d == 0 && st->get_global_version() >= 4) { + d = 3; + add = true; + } + if (d == 0 || round_mode == 2) { throw VmError{Excno::inv_opcode}; } Stack& stack = st->get_stack(); VM_LOG(st) << "execute SHLDIV/MOD " << (args & 15) << ',' << y; if (!(mode & 2)) { - stack.check_underflow(3); + stack.check_underflow(add ? 4 : 3); y = stack.pop_smallint_range(256); } else { - stack.check_underflow(2); + stack.check_underflow(add ? 3 : 2); } auto z = stack.pop_int(); + auto w = add ? stack.pop_int() : td::RefInt256{}; auto x = stack.pop_int(); typename td::BigInt256::DoubleInt tmp{*x}, quot; tmp <<= y; - switch ((args >> 2) & 3) { + if (add) { + tmp += *w; + } + switch (d) { case 1: { tmp.mod_div(*z, quot, round_mode); stack.push_int_quiet(td::make_refint(quot.normalize()), mode & 1); @@ -542,19 +646,45 @@ int exec_shldivmod(VmState* st, unsigned args, int mode) { return 0; } -std::string dump_shldivmod(CellSlice&, unsigned args, bool quiet) { +std::string dump_shldivmod(CellSlice&, unsigned args, int mode) { + int y = -1; + if (mode & 2) { + y = (args & 0xff) + 1; + args >>= 8; + } int round_mode = (int)(args & 3); - if (!(args & 12) || round_mode == 3) { + if (round_mode == 3) { return ""; } - std::string s = (args & 4) ? "LSHIFTDIV" : "LSHIFT"; - if (args & 8) { - s += "MOD"; + std::ostringstream os; + if (mode & 1) { + os << "Q"; } - if (quiet) { - s = "Q" + s; + os << "LSHIFT"; + if (mode & 2) { + os << "#"; } - return s + "FRC"[round_mode]; + switch (args & 12) { + case 4: + os << "DIV"; + break; + case 8: + os << "MOD"; + break; + case 12: + os << "DIVMOD"; + break; + case 0: + os << "ADDDIVMOD"; + break; + } + if (round_mode) { + os << "FRC"[round_mode]; + } + if (y >= 0) { + os << ' ' << y; + } + return os.str(); } void register_div_ops(OpcodeTable& cp0) { @@ -849,7 +979,9 @@ int exec_cmp(VmState* st, int mode, bool quiet, const char* name) { auto y = stack.pop_int(); auto x = stack.pop_int(); if (!x->is_valid() || !y->is_valid()) { - stack.push_int_quiet(std::move(x), quiet); + td::RefInt256 r{true}; + r.unique_write().invalidate(); + stack.push_int_quiet(std::move(r), quiet); } else { int z = td::cmp(std::move(x), std::move(y)); stack.push_smallint(((mode >> (4 + z * 4)) & 15) - 8); @@ -937,4 +1069,26 @@ void register_arith_ops(OpcodeTable& cp0) { register_int_cmp_ops(cp0); } +namespace util { + +const td::RefInt256& check_signed_fits(const td::RefInt256& x, int bits) { + if (!x->signed_fits_bits(bits)) { + throw VmError{Excno::int_ov}; + } + return x; +} + +const td::RefInt256& check_unsigned_fits(const td::RefInt256& x, int bits) { + if (!x->unsigned_fits_bits(bits)) { + throw VmError{Excno::int_ov}; + } + return x; +} + +const td::RefInt256& check_finite(const td::RefInt256& x) { + return check_signed_fits(x, 257); +} + +} // namespace util + } // namespace vm diff --git a/crypto/vm/arithops.h b/crypto/vm/arithops.h index 63adcf7c..c9735798 100644 --- a/crypto/vm/arithops.h +++ b/crypto/vm/arithops.h @@ -18,10 +18,20 @@ */ #pragma once +#include "common/refint.h" namespace vm { class OpcodeTable; void register_arith_ops(OpcodeTable& cp0); +namespace util { + +// throw on error +const td::RefInt256& check_signed_fits(const td::RefInt256& x, int bits); +const td::RefInt256& check_unsigned_fits(const td::RefInt256& x, int bits); +const td::RefInt256& check_finite(const td::RefInt256& x); + +} // namespace util + } // namespace vm diff --git a/crypto/vm/atom.cpp b/crypto/vm/atom.cpp index a7d5486a..dbf1b16f 100644 --- a/crypto/vm/atom.cpp +++ b/crypto/vm/atom.cpp @@ -35,7 +35,7 @@ void Atom::print_to(std::ostream& os) const { std::string Atom::make_name() const { char buffer[16]; - sprintf(buffer, "atom#%d", index_); + snprintf(buffer, sizeof(buffer), "atom#%d", index_); return buffer; } diff --git a/crypto/vm/bls.cpp b/crypto/vm/bls.cpp new file mode 100644 index 00000000..ff5179c7 --- /dev/null +++ b/crypto/vm/bls.cpp @@ -0,0 +1,335 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ + +#include "bls.h" +#include "blst.h" +#include "blst.hpp" +#include "excno.hpp" + +namespace vm { +namespace bls { + +static const std::string DST = "BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_"; + +bool verify(const P1 &pub, td::Slice msg, const P2 &sig) { + try { + blst::P1_Affine p1(pub.data(), P1_SIZE); + if (p1.is_inf()) { + return false; + } + blst::P2_Affine p2(sig.data(), P2_SIZE); + // core_verify checks for p1.in_group() and p2.in_group() + return p2.core_verify(p1, true, (const byte *)msg.data(), msg.size(), DST) == BLST_SUCCESS; + } catch (BLST_ERROR) { + return false; + } +} + +P2 aggregate(const std::vector &sig) { + try { + if (sig.empty()) { + throw VmError{Excno::unknown, "no signatures"}; + } + blst::P2 aggregated; + for (size_t i = 0; i < sig.size(); ++i) { + blst::P2_Affine p2(sig[i].data(), P2_SIZE); + if (i == 0) { + aggregated = p2.to_jacobian(); + } else { + aggregated.aggregate(p2); + } + } + P2 result; + aggregated.compress(result.data()); + return result; + } catch (BLST_ERROR e) { + throw VmError{Excno::unknown, PSTRING() << "blst error " << e}; + } +} + +bool fast_aggregate_verify(const std::vector &pubs, td::Slice msg, const P2 &sig) { + try { + if (pubs.empty()) { + return false; + } + blst::P1 p1_aggregated; + for (size_t i = 0; i < pubs.size(); ++i) { + blst::P1_Affine p1(pubs[i].data(), P1_SIZE); + if (p1.is_inf()) { + return false; + } + if (i == 0) { + p1_aggregated = p1.to_jacobian(); + } else { + p1_aggregated.aggregate(p1); + } + } + blst::P2_Affine p2(sig.data(), P2_SIZE); + blst::P1_Affine p1 = p1_aggregated.to_affine(); + // core_verify checks for p1.in_group() and p2.in_group() + return p2.core_verify(p1, true, (const byte *)msg.data(), msg.size(), DST) == BLST_SUCCESS; + } catch (BLST_ERROR) { + return false; + } +} + +bool aggregate_verify(const std::vector> &pubs_msgs, const P2 &sig) { + try { + if (pubs_msgs.empty()) { + return false; + } + std::unique_ptr pairing = std::make_unique(true, DST); + blst::P2_Affine p2_zero; + for (const auto &p : pubs_msgs) { + blst::P1_Affine p1(p.first.data(), P1_SIZE); + if (!p1.in_group() || p1.is_inf()) { + return false; + } + pairing->aggregate(&p1, &p2_zero, (const td::uint8 *)p.second.data(), p.second.size()); + } + pairing->commit(); + blst::P2_Affine p2(sig.data(), P2_SIZE); + if (!p2.in_group()) { + return false; + } + blst::PT pt(p2); + return pairing->finalverify(&pt); + } catch (BLST_ERROR) { + return false; + } +} + +template +static P generic_add(const P &a, const P &b) { + try { + blst_P point(a.data(), a.size() / 8); + point.aggregate(blst_P_Affine(b.data(), b.size() / 8)); + P result; + point.compress(result.data()); + return result; + } catch (BLST_ERROR e) { + throw VmError{Excno::unknown, PSTRING() << "blst error " << e}; + } +} + +template +static P generic_sub(const P &a, const P &b) { + try { + blst_P point(b.data(), b.size() / 8); + point.neg(); + point.aggregate(blst_P_Affine(a.data(), a.size() / 8)); + P result; + point.compress(result.data()); + return result; + } catch (BLST_ERROR e) { + throw VmError{Excno::unknown, PSTRING() << "blst error " << e}; + } +} + +template +static P generic_neg(const P &a) { + try { + blst_P point(a.data(), a.size() / 8); + point.neg(); + P result; + point.compress(result.data()); + return result; + } catch (BLST_ERROR e) { + throw VmError{Excno::unknown, PSTRING() << "blst error " << e}; + } +} + +template +static P generic_zero() { + static P zero = []() -> P { + blst_P point = blst_P(); + P result; + point.compress(result.data()); + return result; + }(); + return zero; +} + +template +static P generic_mul(const P &p, const td::RefInt256 &x) { + CHECK(x.not_null() && x->is_valid()); + if (x->sgn() == 0) { + return generic_zero(); + } + td::uint8 x_bytes[32]; + CHECK((x % get_r())->export_bytes(x_bytes, 32, false)); + try { + blst_P point(p.data(), p.size() / 8); + blst::Scalar scalar; + scalar.from_bendian(x_bytes, 32); + point.mult(scalar); + P result; + point.compress(result.data()); + return result; + } catch (BLST_ERROR e) { + throw VmError{Excno::unknown, PSTRING() << "blst error " << e}; + } +} + +template +static P generic_multiexp(const std::vector> &ps) { + if (ps.size() == 1) { + return generic_mul(ps[0].first, ps[0].second); + } + try { + std::vector points(ps.size()); + std::vector scalars(ps.size()); + std::vector scalar_ptrs(ps.size()); + for (size_t i = 0; i < ps.size(); ++i) { + points[i] = blst_P_Affine(ps[i].first.data(), ps[i].first.size() / 8); + CHECK(ps[i].second.not_null() && ps[i].second->is_valid()); + CHECK((ps[i].second % get_r())->export_bytes_lsb(scalars[i].data(), 32)); + scalar_ptrs[i] = (const byte *)&scalars[i]; + } + blst_P point = + ps.empty() ? blst_P() : blst_P_Affines::mult_pippenger(points.data(), points.size(), scalar_ptrs.data(), 256); + P result; + point.compress(result.data()); + return result; + } catch (BLST_ERROR e) { + throw VmError{Excno::unknown, PSTRING() << "blst error " << e}; + } +} + +template +static bool generic_in_group(const P &a) { + try { + blst_P point = blst_P(a.data(), a.size() / 8); + return point.in_group(); + } catch (BLST_ERROR e) { + return false; + } +} + +template +static bool generic_is_zero(const P &a) { + return a == generic_zero(); +} + +P1 g1_add(const P1 &a, const P1 &b) { + return generic_add(a, b); +} + +P1 g1_sub(const P1 &a, const P1 &b) { + return generic_sub(a, b); +} + +P1 g1_neg(const P1 &a) { + return generic_neg(a); +} + +P1 g1_mul(const P1 &p, const td::RefInt256 &x) { + return generic_mul(p, x); +} + +P1 g1_multiexp(const std::vector> &ps) { + return generic_multiexp(ps); +} + +P1 g1_zero() { + return generic_zero(); +} + +P1 map_to_g1(const FP &a) { + blst_fp fp; + blst_fp_from_bendian(&fp, a.data()); + blst_p1 point; + blst_map_to_g1(&point, &fp, nullptr); + P1 result; + blst_p1_compress(result.data(), &point); + return result; +} + +bool g1_in_group(const P1 &a) { + return generic_in_group(a); +} + +bool g1_is_zero(const P1 &a) { + return generic_is_zero(a); +} + +P2 g2_add(const P2 &a, const P2 &b) { + return generic_add(a, b); +} + +P2 g2_sub(const P2 &a, const P2 &b) { + return generic_sub(a, b); +} + +P2 g2_neg(const P2 &a) { + return generic_neg(a); +} + +P2 g2_mul(const P2 &p, const td::RefInt256 &x) { + return generic_mul(p, x); +} + +P2 g2_multiexp(const std::vector> &ps) { + return generic_multiexp(ps); +} + +P2 g2_zero() { + return generic_zero(); +} + +P2 map_to_g2(const FP2 &a) { + blst_fp2 fp2; + blst_fp_from_bendian(&fp2.fp[0], a.data()); + blst_fp_from_bendian(&fp2.fp[1], a.data() + FP_SIZE); + blst_p2 point; + blst_map_to_g2(&point, &fp2, nullptr); + P2 result; + blst_p2_compress(result.data(), &point); + return result; +} + +bool g2_in_group(const P2 &a) { + return generic_in_group(a); +} + +bool g2_is_zero(const P2 &a) { + return generic_is_zero(a); +} + +bool pairing(const std::vector> &ps) { + try { + std::unique_ptr pairing = std::make_unique(true, DST); + for (const auto &p : ps) { + blst::P1_Affine point1(p.first.data(), P1_SIZE); + blst::P2_Affine point2(p.second.data(), P2_SIZE); + pairing->raw_aggregate(&point2, &point1); + } + pairing->commit(); + return pairing->finalverify(); + } catch (BLST_ERROR e) { + throw VmError{Excno::unknown, PSTRING() << "blst error " << e}; + } +} + +td::RefInt256 get_r() { + static td::RefInt256 r = td::dec_string_to_int256( + td::Slice{"52435875175126190479447740508185965837690552500527637822603658699938581184513"}); + return r; +} + +} // namespace bls +} // namespace vm diff --git a/crypto/vm/bls.h b/crypto/vm/bls.h new file mode 100644 index 00000000..b7ffc136 --- /dev/null +++ b/crypto/vm/bls.h @@ -0,0 +1,65 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ + +#include +#include "td/utils/buffer.h" +#include "common/bitstring.h" +#include "common/refint.h" + +namespace vm { +namespace bls { + +const size_t P1_SIZE = 48; +const size_t P2_SIZE = 96; +const size_t FP_SIZE = 48; + +using P1 = td::BitArray; +using P2 = td::BitArray; +using FP = td::BitArray; +using FP2 = td::BitArray; + +bool verify(const P1 &pub, td::Slice msg, const P2 &sig); +P2 aggregate(const std::vector &sig); +bool fast_aggregate_verify(const std::vector &pubs, td::Slice msg, const P2 &sig); +bool aggregate_verify(const std::vector> &pubs_msgs, const P2 &sig); + +P1 g1_add(const P1 &a, const P1 &b); +P1 g1_sub(const P1 &a, const P1 &b); +P1 g1_neg(const P1 &a); +P1 g1_mul(const P1 &p, const td::RefInt256 &x); +P1 g1_multiexp(const std::vector> &ps); +P1 g1_zero(); +P1 map_to_g1(const FP &a); +bool g1_in_group(const P1 &a); +bool g1_is_zero(const P1 &a); + +P2 g2_add(const P2 &a, const P2 &b); +P2 g2_sub(const P2 &a, const P2 &b); +P2 g2_neg(const P2 &a); +P2 g2_mul(const P2 &p, const td::RefInt256 &x); +P2 g2_multiexp(const std::vector> &ps); +P2 g2_zero(); +P2 map_to_g2(const FP2 &a); +bool g2_in_group(const P2 &a); +bool g2_is_zero(const P2 &a); + +bool pairing(const std::vector> &ps); + +td::RefInt256 get_r(); + +} // namespace bls +} // namespace vm diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index f438d480..72afb998 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -183,6 +183,9 @@ int BagOfCells::add_root(td::Ref add_root) { // Changes in this function may require corresponding changes in crypto/vm/large-boc-serializer.cpp td::Status BagOfCells::import_cells() { + if (logger_ptr_) { + logger_ptr_->start_stage("import_cells"); + } cells_clear(); for (auto& root : roots) { auto res = import_cell(root.cell, 0); @@ -196,6 +199,9 @@ td::Status BagOfCells::import_cells() { //LOG(INFO) << "[cells: " << cell_count << ", refs: " << int_refs << ", bytes: " << data_bytes //<< ", internal hashes: " << int_hashes << ", top hashes: " << top_hashes << "]"; CHECK(cell_count != 0); + if (logger_ptr_) { + logger_ptr_->finish_stage(PSLICE() << cell_count << " cells"); + } return td::Status::OK(); } @@ -207,6 +213,9 @@ td::Result BagOfCells::import_cell(td::Ref cell, int depth) { if (cell.is_null()) { return td::Status::Error("error while importing a cell into a bag of cells: cell is null"); } + if (logger_ptr_) { + TRY_STATUS(logger_ptr_->on_cell_processed()); + } auto it = cells.find(cell->get_hash()); if (it != cells.end()) { auto pos = it->second; @@ -436,17 +445,19 @@ std::size_t BagOfCells::estimate_serialized_size(int mode) { return res.ok(); } -BagOfCells& BagOfCells::serialize(int mode) { +td::Status BagOfCells::serialize(int mode) { std::size_t size_est = estimate_serialized_size(mode); if (!size_est) { serialized.clear(); - return *this; + return td::Status::OK(); } serialized.resize(size_est); - if (serialize_to(const_cast(serialized.data()), serialized.size(), mode) != size_est) { + TRY_RESULT(size, serialize_to(const_cast(serialized.data()), serialized.size(), mode)); + if (size != size_est) { serialized.clear(); + return td::Status::Error("serialization failed"); } - return *this; + return td::Status::OK(); } std::string BagOfCells::serialize_to_string(int mode) { @@ -456,8 +467,8 @@ std::string BagOfCells::serialize_to_string(int mode) { } std::string res; res.resize(size_est, 0); - if (serialize_to(const_cast(reinterpret_cast(res.data())), res.size(), mode) == - res.size()) { + if (serialize_to(const_cast(reinterpret_cast(res.data())), res.size(), mode) + .move_as_ok() == res.size()) { return res; } else { return {}; @@ -470,8 +481,9 @@ td::Result BagOfCells::serialize_to_slice(int mode) { return td::Status::Error("no cells to serialize to this bag of cells"); } td::BufferSlice res(size_est); - if (serialize_to(const_cast(reinterpret_cast(res.data())), res.size(), mode) == - res.size()) { + TRY_RESULT(size, serialize_to(const_cast(reinterpret_cast(res.data())), + res.size(), mode)); + if (size == res.size()) { return std::move(res); } else { return td::Status::Error("error while serializing a bag of cells: actual serialized size differs from estimated"); @@ -494,14 +506,10 @@ std::string BagOfCells::extract_string() const { // cell_data:(tot_cells_size * [ uint8 ]) // = BagOfCells; // Changes in this function may require corresponding changes in crypto/vm/large-boc-serializer.cpp -template -std::size_t BagOfCells::serialize_to_impl(WriterT& writer, int mode) { - auto store_ref = [&](unsigned long long value) { - writer.store_uint(value, info.ref_byte_size); - }; - auto store_offset = [&](unsigned long long value) { - writer.store_uint(value, info.offset_byte_size); - }; +template +td::Result BagOfCells::serialize_to_impl(WriterT& writer, int mode) { + auto store_ref = [&](unsigned long long value) { writer.store_uint(value, info.ref_byte_size); }; + auto store_offset = [&](unsigned long long value) { writer.store_uint(value, info.offset_byte_size); }; writer.store_uint(info.magic, 4); @@ -536,6 +544,9 @@ std::size_t BagOfCells::serialize_to_impl(WriterT& writer, int mode) { DCHECK((unsigned)cell_count == cell_list_.size()); if (info.has_index) { std::size_t offs = 0; + if (logger_ptr_) { + logger_ptr_->start_stage("generate_index"); + } for (int i = cell_count - 1; i >= 0; --i) { const Ref& dc = cell_list_[i].dc_ref; bool with_hash = (mode & Mode::WithIntHashes) && !cell_list_[i].wt; @@ -548,11 +559,20 @@ std::size_t BagOfCells::serialize_to_impl(WriterT& writer, int mode) { fixed_offset = offs * 2 + cell_list_[i].should_cache; } store_offset(fixed_offset); + if (logger_ptr_) { + TRY_STATUS(logger_ptr_->on_cell_processed()); + } + } + if (logger_ptr_) { + logger_ptr_->finish_stage(""); } DCHECK(offs == info.data_size); } DCHECK(writer.position() == info.data_offset); size_t keep_position = writer.position(); + if (logger_ptr_) { + logger_ptr_->start_stage("serialize"); + } for (int i = 0; i < cell_count; ++i) { const auto& dc_info = cell_list_[cell_count - 1 - i]; const Ref& dc = dc_info.dc_ref; @@ -572,6 +592,9 @@ std::size_t BagOfCells::serialize_to_impl(WriterT& writer, int mode) { // std::cerr << ' ' << k; } // std::cerr << std::endl; + if (logger_ptr_) { + TRY_STATUS(logger_ptr_->on_cell_processed()); + } } writer.chk(); DCHECK(writer.position() - keep_position == info.data_size); @@ -580,11 +603,14 @@ std::size_t BagOfCells::serialize_to_impl(WriterT& writer, int mode) { unsigned crc = writer.get_crc32(); writer.store_uint(td::bswap32(crc), 4); } + if (logger_ptr_) { + logger_ptr_->finish_stage(PSLICE() << cell_count << " cells, " << writer.position() << " bytes"); + } DCHECK(writer.empty()); return writer.position(); } -std::size_t BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_size, int mode) { +td::Result BagOfCells::serialize_to(unsigned char* buffer, std::size_t buff_size, int mode) { std::size_t size_est = estimate_serialized_size(mode); if (!size_est || size_est > buff_size) { return 0; @@ -599,7 +625,7 @@ td::Status BagOfCells::serialize_to_file(td::FileFd& fd, int mode) { return td::Status::Error("no cells to serialize to this bag of cells"); } boc_writers::FileWriter writer{fd, size_est}; - size_t s = serialize_to_impl(writer, mode); + TRY_RESULT(s, serialize_to_impl(writer, mode)); TRY_STATUS(writer.finalize()); if (s != size_est) { return td::Status::Error("error while serializing a bag of cells: actual serialized size differs from estimated"); @@ -930,7 +956,7 @@ unsigned long long BagOfCells::get_idx_entry_raw(int index) { * */ -td::Result> std_boc_deserialize(td::Slice data, bool can_be_empty) { +td::Result> std_boc_deserialize(td::Slice data, bool can_be_empty, bool allow_nonzero_level) { if (data.empty() && can_be_empty) { return Ref(); } @@ -946,7 +972,7 @@ td::Result> std_boc_deserialize(td::Slice data, bool can_be_empty) { if (root.is_null()) { return td::Status::Error("bag of cells has null root cell (?)"); } - if (root->get_level() != 0) { + if (!allow_nonzero_level && root->get_level() != 0) { return td::Status::Error("bag of cells has a root with non-zero level"); } return std::move(root); @@ -1001,6 +1027,21 @@ td::Result std_boc_serialize_multi(std::vector> roots } return boc.serialize_to_slice(mode); } +td::Status std_boc_serialize_to_file(Ref root, td::FileFd& fd, int mode, + td::CancellationToken cancellation_token) { + if (root.is_null()) { + return td::Status::Error("cannot serialize a null cell reference into a bag of cells"); + } + td::Timer timer; + BagOfCellsLogger logger(std::move(cancellation_token)); + BagOfCells boc; + boc.set_logger(&logger); + boc.add_root(std::move(root)); + TRY_STATUS(boc.import_cells()); + TRY_STATUS(boc.serialize_to_file(fd, mode)); + LOG(ERROR) << "serialization took " << timer.elapsed() << "s"; + return td::Status::OK(); +} /* * @@ -1008,27 +1049,40 @@ td::Result std_boc_serialize_multi(std::vector> roots * */ -bool CellStorageStat::compute_used_storage(Ref cs_ref, bool kill_dup, unsigned skip_count_root) { +td::Result CellStorageStat::compute_used_storage(Ref cs_ref, bool kill_dup, + unsigned skip_count_root) { clear(); - return add_used_storage(std::move(cs_ref), kill_dup, skip_count_root) && clear_seen(); + TRY_RESULT(res, add_used_storage(std::move(cs_ref), kill_dup, skip_count_root)); + clear_seen(); + return res; } -bool CellStorageStat::compute_used_storage(const CellSlice& cs, bool kill_dup, unsigned skip_count_root) { +td::Result CellStorageStat::compute_used_storage(const CellSlice& cs, bool kill_dup, + unsigned skip_count_root) { clear(); - return add_used_storage(cs, kill_dup, skip_count_root) && clear_seen(); + TRY_RESULT(res, add_used_storage(cs, kill_dup, skip_count_root)); + clear_seen(); + return res; } -bool CellStorageStat::compute_used_storage(CellSlice&& cs, bool kill_dup, unsigned skip_count_root) { +td::Result CellStorageStat::compute_used_storage(CellSlice&& cs, bool kill_dup, + unsigned skip_count_root) { clear(); - return add_used_storage(std::move(cs), kill_dup, skip_count_root) && clear_seen(); + TRY_RESULT(res, add_used_storage(std::move(cs), kill_dup, skip_count_root)); + clear_seen(); + return res; } -bool CellStorageStat::compute_used_storage(Ref cell, bool kill_dup, unsigned skip_count_root) { +td::Result CellStorageStat::compute_used_storage(Ref cell, bool kill_dup, + unsigned skip_count_root) { clear(); - return add_used_storage(std::move(cell), kill_dup, skip_count_root) && clear_seen(); + TRY_RESULT(res, add_used_storage(std::move(cell), kill_dup, skip_count_root)); + clear_seen(); + return res; } -bool CellStorageStat::add_used_storage(Ref cs_ref, bool kill_dup, unsigned skip_count_root) { +td::Result CellStorageStat::add_used_storage(Ref cs_ref, bool kill_dup, + unsigned skip_count_root) { if (cs_ref->is_unique()) { return add_used_storage(std::move(cs_ref.unique_write()), kill_dup, skip_count_root); } else { @@ -1036,60 +1090,75 @@ bool CellStorageStat::add_used_storage(Ref cs_ref, bool kill_dup, } } -bool CellStorageStat::add_used_storage(const CellSlice& cs, bool kill_dup, unsigned skip_count_root) { +td::Result CellStorageStat::add_used_storage(const CellSlice& cs, bool kill_dup, + unsigned skip_count_root) { if (!(skip_count_root & 1)) { ++cells; if (cells > limit_cells) { - return false; + return td::Status::Error("too many cells"); } } if (!(skip_count_root & 2)) { bits += cs.size(); if (bits > limit_bits) { - return false; + return td::Status::Error("too many bits"); } } + CellInfo res; for (unsigned i = 0; i < cs.size_refs(); i++) { - if (!add_used_storage(cs.prefetch_ref(i), kill_dup)) { - return false; - } + TRY_RESULT(child, add_used_storage(cs.prefetch_ref(i), kill_dup)); + res.max_merkle_depth = std::max(res.max_merkle_depth, child.max_merkle_depth); } - return true; + if (cs.special_type() == CellTraits::SpecialType::MerkleProof || + cs.special_type() == CellTraits::SpecialType::MerkleUpdate) { + ++res.max_merkle_depth; + } + return res; } -bool CellStorageStat::add_used_storage(CellSlice&& cs, bool kill_dup, unsigned skip_count_root) { +td::Result CellStorageStat::add_used_storage(CellSlice&& cs, bool kill_dup, + unsigned skip_count_root) { if (!(skip_count_root & 1)) { ++cells; if (cells > limit_cells) { - return false; + return td::Status::Error("too many cells"); } } if (!(skip_count_root & 2)) { bits += cs.size(); if (bits > limit_bits) { - return false; + return td::Status::Error("too many bits"); } } + CellInfo res; while (cs.size_refs()) { - if (!add_used_storage(cs.fetch_ref(), kill_dup)) { - return false; - } + TRY_RESULT(child, add_used_storage(cs.fetch_ref(), kill_dup)); + res.max_merkle_depth = std::max(res.max_merkle_depth, child.max_merkle_depth); } - return true; + if (cs.special_type() == CellTraits::SpecialType::MerkleProof || + cs.special_type() == CellTraits::SpecialType::MerkleUpdate) { + ++res.max_merkle_depth; + } + return res; } -bool CellStorageStat::add_used_storage(Ref cell, bool kill_dup, unsigned skip_count_root) { +td::Result CellStorageStat::add_used_storage(Ref cell, bool kill_dup, + unsigned skip_count_root) { if (cell.is_null()) { - return false; + return td::Status::Error("cell is null"); } if (kill_dup) { - auto ins = seen.insert(cell->get_hash()); + auto ins = seen.emplace(cell->get_hash(), CellInfo{}); if (!ins.second) { - return true; + return ins.first->second; } } - vm::CellSlice cs{vm::NoVm{}, std::move(cell)}; - return add_used_storage(std::move(cs), kill_dup, skip_count_root); + vm::CellSlice cs{vm::NoVm{}, cell}; + TRY_RESULT(res, add_used_storage(std::move(cs), kill_dup, skip_count_root)); + if (kill_dup) { + seen[cell->get_hash()] = res; + } + return res; } void NewCellStorageStat::add_cell(Ref cell) { @@ -1190,4 +1259,35 @@ bool VmStorageStat::add_storage(const CellSlice& cs) { return true; } +static td::uint64 estimate_prunned_size() { + return 41; +} + +static td::uint64 estimate_serialized_size(const Ref& cell) { + return cell->get_serialized_size() + cell->size_refs() * 3 + 3; +} + +void ProofStorageStat::add_cell(const Ref& cell) { + auto& status = cells_[cell->get_hash()]; + if (status == c_loaded) { + return; + } + if (status == c_prunned) { + proof_size_ -= estimate_prunned_size(); + } + status = c_loaded; + proof_size_ += estimate_serialized_size(cell); + for (unsigned i = 0; i < cell->size_refs(); ++i) { + auto& child_status = cells_[cell->get_ref(i)->get_hash()]; + if (child_status == c_none) { + child_status = c_prunned; + proof_size_ += estimate_prunned_size(); + } + } +} + +td::uint64 ProofStorageStat::estimate_proof_size() const { + return proof_size_; +} + } // namespace vm diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index dd74a6d1..17e7eb69 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -17,13 +17,18 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "td/utils/CancellationToken.h" + #include +#include #include "vm/db/DynamicBagOfCellsDb.h" #include "vm/cells.h" #include "td/utils/Status.h" #include "td/utils/buffer.h" #include "td/utils/HashMap.h" #include "td/utils/HashSet.h" +#include "td/utils/Time.h" +#include "td/utils/Timer.h" #include "td/utils/port/FileFd.h" namespace vm { @@ -51,6 +56,7 @@ class NewCellStorageStat { bool operator==(const Stat& other) const { return key() == other.key(); } + Stat(const Stat& other) = default; Stat& operator=(const Stat& other) = default; Stat& operator+=(const Stat& other) { cells += other.cells; @@ -95,9 +101,9 @@ class NewCellStorageStat { private: const CellUsageTree* usage_tree_; - std::set seen_; + td::HashSet seen_; Stat stat_; - std::set proof_seen_; + td::HashSet proof_seen_; Stat proof_stat_; const NewCellStorageStat* parent_{nullptr}; @@ -108,12 +114,17 @@ struct CellStorageStat { unsigned long long cells; unsigned long long bits; unsigned long long public_cells; - std::set seen; + struct CellInfo { + td::uint32 max_merkle_depth = 0; + }; + td::HashMap seen; CellStorageStat() : cells(0), bits(0), public_cells(0) { } - bool clear_seen() { + explicit CellStorageStat(unsigned long long limit_cells) + : cells(0), bits(0), public_cells(0), limit_cells(limit_cells) { + } + void clear_seen() { seen.clear(); - return true; } void clear() { cells = bits = public_cells = 0; @@ -124,15 +135,16 @@ struct CellStorageStat { limit_cells = std::numeric_limits::max(); limit_bits = std::numeric_limits::max(); } - bool compute_used_storage(Ref cs_ref, bool kill_dup = true, unsigned skip_count_root = 0); - bool compute_used_storage(const CellSlice& cs, bool kill_dup = true, unsigned skip_count_root = 0); - bool compute_used_storage(CellSlice&& cs, bool kill_dup = true, unsigned skip_count_root = 0); - bool compute_used_storage(Ref cell, bool kill_dup = true, unsigned skip_count_root = 0); + td::Result compute_used_storage(Ref cs_ref, bool kill_dup = true, + unsigned skip_count_root = 0); + td::Result compute_used_storage(const CellSlice& cs, bool kill_dup = true, unsigned skip_count_root = 0); + td::Result compute_used_storage(CellSlice&& cs, bool kill_dup = true, unsigned skip_count_root = 0); + td::Result compute_used_storage(Ref cell, bool kill_dup = true, unsigned skip_count_root = 0); - bool add_used_storage(Ref cs_ref, bool kill_dup = true, unsigned skip_count_root = 0); - bool add_used_storage(const CellSlice& cs, bool kill_dup = true, unsigned skip_count_root = 0); - bool add_used_storage(CellSlice&& cs, bool kill_dup = true, unsigned skip_count_root = 0); - bool add_used_storage(Ref cell, bool kill_dup = true, unsigned skip_count_root = 0); + td::Result add_used_storage(Ref cs_ref, bool kill_dup = true, unsigned skip_count_root = 0); + td::Result add_used_storage(const CellSlice& cs, bool kill_dup = true, unsigned skip_count_root = 0); + td::Result add_used_storage(CellSlice&& cs, bool kill_dup = true, unsigned skip_count_root = 0); + td::Result add_used_storage(Ref cell, bool kill_dup = true, unsigned skip_count_root = 0); unsigned long long limit_cells = std::numeric_limits::max(); unsigned long long limit_bits = std::numeric_limits::max(); @@ -153,6 +165,18 @@ struct VmStorageStat { } }; +class ProofStorageStat { + public: + void add_cell(const Ref& cell); + td::uint64 estimate_proof_size() const; + private: + enum CellStatus { + c_none = 0, c_prunned = 1, c_loaded = 2 + }; + td::HashMap cells_; + td::uint64 proof_size_ = 0; +}; + struct CellSerializationInfo { bool special; Cell::LevelMask level_mask; @@ -177,6 +201,43 @@ struct CellSerializationInfo { td::Result> create_data_cell(td::Slice data, td::Span> refs) const; }; +class BagOfCellsLogger { + public: + BagOfCellsLogger() = default; + explicit BagOfCellsLogger(td::CancellationToken cancellation_token) + : cancellation_token_(std::move(cancellation_token)) { + } + + void start_stage(std::string stage) { + log_speed_at_ = td::Timestamp::in(LOG_SPEED_PERIOD); + processed_cells_ = 0; + timer_ = {}; + stage_ = std::move(stage); + } + void finish_stage(td::Slice desc) { + LOG(ERROR) << "serializer: " << stage_ << " took " << timer_.elapsed() << "s, " << desc; + } + td::Status on_cell_processed() { + ++processed_cells_; + if (processed_cells_ % 1000 == 0) { + TRY_STATUS(cancellation_token_.check()); + } + if (log_speed_at_.is_in_past()) { + log_speed_at_ += LOG_SPEED_PERIOD; + LOG(WARNING) << "serializer: " << stage_ << " " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s"; + processed_cells_ = 0; + } + return td::Status::OK(); + } + + private: + std::string stage_; + td::Timer timer_; + td::CancellationToken cancellation_token_; + td::Timestamp log_speed_at_; + size_t processed_cells_ = 0; + static constexpr double LOG_SPEED_PERIOD = 120.0; +}; class BagOfCells { public: enum { hash_bytes = vm::Cell::hash_bytes, default_max_roots = 16384 }; @@ -261,6 +322,7 @@ class BagOfCells { const unsigned char* index_ptr{nullptr}; const unsigned char* data_ptr{nullptr}; std::vector custom_index; + BagOfCellsLogger* logger_ptr_{nullptr}; public: void clear(); @@ -270,14 +332,17 @@ class BagOfCells { int add_root(td::Ref add_root); td::Status import_cells() TD_WARN_UNUSED_RESULT; BagOfCells() = default; + void set_logger(BagOfCellsLogger* logger_ptr) { + logger_ptr_ = logger_ptr; + } std::size_t estimate_serialized_size(int mode = 0); - BagOfCells& serialize(int mode = 0); - std::string serialize_to_string(int mode = 0); + td::Status serialize(int mode = 0); + td::string serialize_to_string(int mode = 0); td::Result serialize_to_slice(int mode = 0); - std::size_t serialize_to(unsigned char* buffer, std::size_t buff_size, int mode = 0); + td::Result serialize_to(unsigned char* buffer, std::size_t buff_size, int mode = 0); td::Status serialize_to_file(td::FileFd& fd, int mode = 0); - template - std::size_t serialize_to_impl(WriterT& writer, int mode = 0); + template + td::Result serialize_to_impl(WriterT& writer, int mode = 0); std::string extract_string() const; td::Result deserialize(const td::Slice& data, int max_roots = default_max_roots); @@ -316,14 +381,16 @@ class BagOfCells { std::vector* cell_should_cache); }; -td::Result> std_boc_deserialize(td::Slice data, bool can_be_empty = false); +td::Result> std_boc_deserialize(td::Slice data, bool can_be_empty = false, bool allow_nonzero_level = false); td::Result std_boc_serialize(Ref root, int mode = 0); td::Result>> std_boc_deserialize_multi(td::Slice data, int max_roots = BagOfCells::default_max_roots); td::Result std_boc_serialize_multi(std::vector> root, int mode = 0); -td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, - td::FileFd& fd, int mode = 0); +td::Status std_boc_serialize_to_file(Ref root, td::FileFd& fd, int mode = 0, + td::CancellationToken cancellation_token = {}); +td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, + int mode = 0, td::CancellationToken cancellation_token = {}); } // namespace vm diff --git a/crypto/vm/box.hpp b/crypto/vm/box.hpp index 68b131ce..0d676cae 100644 --- a/crypto/vm/box.hpp +++ b/crypto/vm/box.hpp @@ -30,7 +30,7 @@ class Box : public td::CntObject { Box(const Box&) = default; Box(Box&&) = default; template - Box(Args... args) : data_{std::move(args...)} { + Box(Args&&... args) : data_{std::forward(args)...} { } ~Box() override = default; Box(const StackEntry& data) : data_(data) { diff --git a/crypto/vm/cellops.cpp b/crypto/vm/cellops.cpp index 1dfb69fb..61ffe5c5 100644 --- a/crypto/vm/cellops.cpp +++ b/crypto/vm/cellops.cpp @@ -103,7 +103,8 @@ std::string dump_push_slice_common(CellSlice& cs, unsigned data_bits, unsigned r cs.advance(pfx_bits); auto slice = cs.fetch_subslice(data_bits, refs); slice.unique_write().remove_trailing(); - std::ostringstream os{name}; + std::ostringstream os; + os << name; slice->dump_hex(os, 1, false); return os.str(); } @@ -188,7 +189,8 @@ std::string dump_push_cont(CellSlice& cs, unsigned args, int pfx_bits) { } cs.advance(pfx_bits); auto slice = cs.fetch_subslice(data_bits, refs); - std::ostringstream os{"PUSHCONT "}; + std::ostringstream os; + os << "PUSHCONT "; slice->dump_hex(os, 1, false); return os.str(); } @@ -219,7 +221,8 @@ std::string dump_push_cont_simple(CellSlice& cs, unsigned args, int pfx_bits) { } cs.advance(pfx_bits); auto slice = cs.fetch_subslice(data_bits); - std::ostringstream os{"PUSHCONT "}; + std::ostringstream os; + os << "PUSHCONT "; slice->dump_hex(os, 1, false); return os.str(); } @@ -889,6 +892,40 @@ int exec_load_special_cell(VmState* st, bool quiet) { Stack& stack = st->get_stack(); VM_LOG(st) << "execute XLOAD" << (quiet ? "Q" : ""); auto cell = stack.pop_cell(); + if (st->get_global_version() >= 5) { + st->register_cell_load(cell->get_hash()); + auto r_loaded_cell = cell->load_cell(); + if (r_loaded_cell.is_error()) { + if (quiet) { + stack.push_bool(false); + return 0; + } else { + throw VmError{Excno::cell_und, "failed to load cell"}; + } + } + auto loaded_cell = r_loaded_cell.move_as_ok(); + if (loaded_cell.data_cell->is_special()) { + if (loaded_cell.data_cell->special_type() != CellTraits::SpecialType::Library) { + if (quiet) { + stack.push_bool(false); + return 0; + } else { + throw VmError{Excno::cell_und, "unexpected special cell"}; + } + } + CellSlice cs(std::move(loaded_cell)); + DCHECK(cs.size() == Cell::hash_bits + 8); + cell = st->load_library(cs.data_bits() + 8); + if (cell.is_null()) { + if (quiet) { + stack.push_bool(false); + return 0; + } else { + throw VmError{Excno::cell_und, "failed to load library cell"}; + } + } + } + } stack.push_cell(cell); if (quiet) { stack.push_bool(true); @@ -1060,8 +1097,8 @@ int exec_load_int_fixed2(VmState* st, unsigned args) { } std::string dump_load_int_fixed2(CellSlice&, unsigned args) { - std::ostringstream os{args & 0x200 ? "PLD" : "LD"}; - os << (args & 0x100 ? 'U' : 'I'); + std::ostringstream os; + os << (args & 0x200 ? "PLD" : "LD") << (args & 0x100 ? 'U' : 'I'); if (args & 0x400) { os << 'Q'; } @@ -1081,9 +1118,9 @@ int exec_preload_uint_fixed_0e(VmState* st, unsigned args) { } std::string dump_preload_uint_fixed_0e(CellSlice&, unsigned args) { - std::ostringstream os{"PLDUZ "}; + std::ostringstream os; unsigned bits = ((args & 7) + 1) << 5; - os << bits; + os << "PLDUZ " << bits; return os.str(); } @@ -1108,7 +1145,8 @@ int exec_load_slice_fixed2(VmState* st, unsigned args) { std::string dump_load_slice_fixed2(CellSlice&, unsigned args) { unsigned bits = (args & 0xff) + 1; - std::ostringstream os{args & 0x100 ? "PLDSLICE" : "LDSLICE"}; + std::ostringstream os; + os << (args & 0x100 ? "PLDSLICE" : "LDSLICE"); if (args & 0x200) { os << 'Q'; } @@ -1353,6 +1391,55 @@ int exec_slice_depth(VmState* st) { return 0; } +int exec_cell_level(VmState* st) { + Stack& stack = st->get_stack(); + VM_LOG(st) << "execute CLEVEL"; + auto cell = stack.pop_cell(); + stack.push_smallint(cell->get_level()); + return 0; +} + +int exec_cell_level_mask(VmState* st) { + Stack& stack = st->get_stack(); + VM_LOG(st) << "execute CLEVELMASK"; + auto cell = stack.pop_cell(); + stack.push_smallint(cell->get_level_mask().get_mask()); + return 0; +} + +int exec_cell_hash_i(VmState* st, unsigned args, bool var) { + unsigned i; + Stack& stack = st->get_stack(); + if (var) { + VM_LOG(st) << "execute CHASHIX"; + i = stack.pop_smallint_range(3); + } else { + i = args & 3; + VM_LOG(st) << "execute CHASHI " << i; + } + auto cell = stack.pop_cell(); + std::array hash = cell->get_hash(i).as_array(); + td::RefInt256 res{true}; + CHECK(res.write().import_bytes(hash.data(), hash.size(), false)); + stack.push_int(std::move(res)); + return 0; +} + +int exec_cell_depth_i(VmState* st, unsigned args, bool var) { + unsigned i; + Stack& stack = st->get_stack(); + if (var) { + VM_LOG(st) << "execute CDEPTHIX"; + i = stack.pop_smallint_range(3); + } else { + i = args & 3; + VM_LOG(st) << "execute CDEPTHI " << i; + } + auto cell = stack.pop_cell(); + stack.push_smallint(cell->get_depth(i)); + return 0; +} + void register_cell_deserialize_ops(OpcodeTable& cp0) { using namespace std::placeholders; cp0.insert(OpcodeInstr::mksimple(0xd0, 8, "CTOS", exec_cell_to_slice)) @@ -1441,7 +1528,13 @@ void register_cell_deserialize_ops(OpcodeTable& cp0) { .insert(OpcodeInstr::mksimple(0xd761, 16, "LDONES", std::bind(exec_load_same, _1, "LDONES", 1))) .insert(OpcodeInstr::mksimple(0xd762, 16, "LDSAME", std::bind(exec_load_same, _1, "LDSAME", -1))) .insert(OpcodeInstr::mksimple(0xd764, 16, "SDEPTH", exec_slice_depth)) - .insert(OpcodeInstr::mksimple(0xd765, 16, "CDEPTH", exec_cell_depth)); + .insert(OpcodeInstr::mksimple(0xd765, 16, "CDEPTH", exec_cell_depth)) + .insert(OpcodeInstr::mksimple(0xd766, 16, "CLEVEL", exec_cell_level)->require_version(6)) + .insert(OpcodeInstr::mksimple(0xd767, 16, "CLEVELMASK", exec_cell_level_mask)->require_version(6)) + .insert(OpcodeInstr::mkfixed(0xd768 >> 2, 14, 2, instr::dump_1c_and(3, "CHASHI "), std::bind(exec_cell_hash_i, _1, _2, false))->require_version(6)) + .insert(OpcodeInstr::mkfixed(0xd76c >> 2, 14, 2, instr::dump_1c_and(3, "CDEPTHI "), std::bind(exec_cell_depth_i, _1, _2, false))->require_version(6)) + .insert(OpcodeInstr::mksimple(0xd770, 16, "CHASHIX ", std::bind(exec_cell_hash_i, _1, 0, true))->require_version(6)) + .insert(OpcodeInstr::mksimple(0xd771, 16, "CDEPTHIX ", std::bind(exec_cell_depth_i, _1, 0, true))->require_version(6)); } void register_cell_ops(OpcodeTable& cp0) { @@ -1451,4 +1544,193 @@ void register_cell_ops(OpcodeTable& cp0) { register_cell_deserialize_ops(cp0); } +namespace util { + +bool load_int256_q(CellSlice& cs, td::RefInt256& res, int len, bool sgnd, bool quiet) { + if (!cs.fetch_int256_to(len, res, sgnd)) { + if (quiet) { + return false; + } + throw VmError{Excno::cell_und}; + } + return true; +} +bool load_long_q(CellSlice& cs, td::int64& res, int len, bool quiet) { + CHECK(0 <= len && len <= 64); + if (!cs.have(len)) { + if (quiet) { + return false; + } + throw VmError{Excno::cell_und}; + } + res = cs.fetch_long(len); + return true; +} +bool load_ulong_q(CellSlice& cs, td::uint64& res, int len, bool quiet) { + CHECK(0 <= len && len <= 64); + if (!cs.have(len)) { + if (quiet) { + return false; + } + throw VmError{Excno::cell_und}; + } + res = cs.fetch_ulong(len); + return true; +} +bool load_ref_q(CellSlice& cs, td::Ref& res, bool quiet) { + if (!cs.have_refs(1)) { + if (quiet) { + return false; + } + throw VmError{Excno::cell_und}; + } + res = cs.fetch_ref(); + return true; +} +bool load_maybe_ref_q(CellSlice& cs, td::Ref& res, bool quiet) { + if (!cs.fetch_maybe_ref(res)) { + if (quiet) { + return false; + } + throw VmError{Excno::cell_und}; + } + return true; +} +bool skip_bits_q(CellSlice& cs, int bits, bool quiet) { + if (!cs.skip_first(bits)) { + if (quiet) { + return false; + } + throw VmError{Excno::cell_und}; + } + return true; +} + +td::RefInt256 load_int256(CellSlice& cs, int len, bool sgnd) { + td::RefInt256 x; + load_int256_q(cs, x, len, sgnd, false); + return x; +} +td::int64 load_long(CellSlice& cs, int len) { + td::int64 x; + load_long_q(cs, x, len, false); + return x; +} +td::uint64 load_ulong(CellSlice& cs, int len) { + td::uint64 x; + load_ulong_q(cs, x, len, false); + return x; +} +td::Ref load_ref(CellSlice& cs) { + td::Ref x; + load_ref_q(cs, x, false); + return x; +} +td::Ref load_maybe_ref(CellSlice& cs) { + td::Ref x; + load_maybe_ref_q(cs, x, false); + return x; +} +void check_have_bits(const CellSlice& cs, int bits) { + if (!cs.have(bits)) { + throw VmError{Excno::cell_und}; + } +} +void skip_bits(CellSlice& cs, int bits) { + skip_bits_q(cs, bits, false); +} +void end_parse(CellSlice& cs) { + if (cs.size() || cs.size_refs()) { + throw VmError{Excno::cell_und, "extra data remaining in deserialized cell"}; + } +} + +bool store_int256(CellBuilder& cb, const td::RefInt256& x, int len, bool sgnd, bool quiet) { + if (!cb.can_extend_by(len)) { + if (quiet) { + return false; + } + throw VmError{Excno::cell_ov}; + } + if (!x->fits_bits(len, sgnd)) { + if (quiet) { + return false; + } + throw VmError{Excno::range_chk}; + } + cb.store_int256(*x, len, sgnd); + return true; +} +bool store_long(CellBuilder& cb, td::int64 x, int len, bool quiet) { + CHECK(len > 0); + if (!cb.can_extend_by(len)) { + if (quiet) { + return false; + } + throw VmError{Excno::cell_ov}; + } + if (len < 64 && (x < td::int64(std::numeric_limits::max() << (len - 1)) || x >= (1LL << (len - 1)))) { + if (quiet) { + return false; + } + throw VmError{Excno::range_chk}; + } + if (len > 64) { + cb.store_bits_same(len - 64, x < 0); + len = 64; + } + cb.store_long(x, len); + return true; +} +bool store_ulong(CellBuilder& cb, td::uint64 x, int len, bool quiet) { + CHECK(len > 0); + if (!cb.can_extend_by(len)) { + if (quiet) { + return false; + } + throw VmError{Excno::cell_ov}; + } + if (len < 64 && x >= (1ULL << len)) { + if (quiet) { + return false; + } + throw VmError{Excno::range_chk}; + } + if (len > 64) { + cb.store_zeroes(len - 64); + len = 64; + } + cb.store_long(x, len); + return true; +} +bool store_ref(CellBuilder& cb, td::Ref x, bool quiet) { + if (!cb.store_ref_bool(std::move(x))) { + if (quiet) { + return false; + } + throw VmError{Excno::cell_ov}; + } + return true; +} +bool store_maybe_ref(CellBuilder& cb, td::Ref x, bool quiet) { + if (!cb.store_maybe_ref(std::move(x))) { + if (quiet) { + return false; + } + throw VmError{Excno::cell_ov}; + } + return true; +} +bool store_slice(CellBuilder& cb, const CellSlice& cs, bool quiet) { + if (!cell_builder_add_slice_bool(cb, cs)) { + if (quiet) { + return false; + } + throw VmError{Excno::cell_ov}; + } + return true; +} + +} // namespace util + } // namespace vm diff --git a/crypto/vm/cellops.h b/crypto/vm/cellops.h index 1c08d8e3..0fd62854 100644 --- a/crypto/vm/cellops.h +++ b/crypto/vm/cellops.h @@ -23,12 +23,42 @@ namespace vm { class OpcodeTable; -void register_cell_ops(OpcodeTable &cp0); +void register_cell_ops(OpcodeTable& cp0); -std::string dump_push_ref(CellSlice &cs, unsigned args, int pfx_bits, std::string name); -int compute_len_push_ref(const CellSlice &cs, unsigned args, int pfx_bits); +std::string dump_push_ref(CellSlice& cs, unsigned args, int pfx_bits, std::string name); +int compute_len_push_ref(const CellSlice& cs, unsigned args, int pfx_bits); -std::string dump_push_ref2(CellSlice &cs, unsigned args, int pfx_bits, std::string name); -int compute_len_push_ref2(const CellSlice &cs, unsigned args, int pfx_bits); +std::string dump_push_ref2(CellSlice& cs, unsigned args, int pfx_bits, std::string name); +int compute_len_push_ref2(const CellSlice& cs, unsigned args, int pfx_bits); + +namespace util { + +// "_q" functions throw on error if not quiet, return false if quiet (leaving cs unchanged) +bool load_int256_q(CellSlice& cs, td::RefInt256& res, int len, bool sgnd, bool quiet); +bool load_long_q(CellSlice& cs, td::int64& res, int len, bool quiet); +bool load_ulong_q(CellSlice& cs, td::uint64& res, int len, bool quiet); +bool load_ref_q(CellSlice& cs, td::Ref& res, bool quiet); +bool load_maybe_ref_q(CellSlice& cs, td::Ref& res, bool quiet); +bool skip_bits_q(CellSlice& cs, int bits, bool quiet); + +// Non-"_q" functions throw on error +td::RefInt256 load_int256(CellSlice& cs, int len, bool sgnd); +td::int64 load_long(CellSlice& cs, int len); +td::uint64 load_ulong(CellSlice& cs, int len); +td::Ref load_ref(CellSlice& cs); +td::Ref load_maybe_ref(CellSlice& cs); +void check_have_bits(const CellSlice& cs, int bits); +void skip_bits(CellSlice& cs, int bits); +void end_parse(CellSlice& cs); + +// store_... functions throw on error if not quiet, return false if quiet (leaving cb unchanged) +bool store_int256(CellBuilder& cb, const td::RefInt256& x, int len, bool sgnd, bool quiet = false); +bool store_long(CellBuilder& cb, td::int64 x, int len, bool quiet = false); +bool store_ulong(CellBuilder& cb, td::uint64 x, int len, bool quiet = false); +bool store_ref(CellBuilder& cb, td::Ref x, bool quiet = false); +bool store_maybe_ref(CellBuilder& cb, td::Ref x, bool quiet = false); +bool store_slice(CellBuilder& cb, const CellSlice& cs, bool quiet = false); + +} // namespace util } // namespace vm diff --git a/crypto/vm/cells/Cell.h b/crypto/vm/cells/Cell.h index 03f73b14..a75371db 100644 --- a/crypto/vm/cells/Cell.h +++ b/crypto/vm/cells/Cell.h @@ -19,6 +19,7 @@ #pragma once #include "common/refcnt.hpp" #include "common/bitstring.h" +#include "td/utils/HashSet.h" #include "vm/cells/CellHash.h" #include "vm/cells/CellTraits.h" @@ -86,4 +87,31 @@ class Cell : public CellTraits { }; std::ostream& operator<<(std::ostream& os, const Cell& c); + +using is_transparent = void; // Pred to use +inline vm::CellHash as_cell_hash(const Ref& cell) { + return cell->get_hash(); +} +inline vm::CellHash as_cell_hash(td::Slice hash) { + return vm::CellHash::from_slice(hash); +} +inline vm::CellHash as_cell_hash(vm::CellHash hash) { + return hash; +} +struct CellEqF { + using is_transparent = void; // Pred to use + template + bool operator()(const A& a, const B& b) const { + return as_cell_hash(a) == as_cell_hash(b); + } +}; +struct CellHashF { + using is_transparent = void; // Pred to use + using transparent_key_equal = CellEqF; + template + size_t operator()(const T& value) const { + return cell_hash_slice_hash(as_cell_hash(value).as_slice()); + } +}; +using CellHashSet = td::HashSet, CellHashF, CellEqF>; } // namespace vm diff --git a/crypto/vm/cells/CellBuilder.cpp b/crypto/vm/cells/CellBuilder.cpp index 43058531..772b5f6b 100644 --- a/crypto/vm/cells/CellBuilder.cpp +++ b/crypto/vm/cells/CellBuilder.cpp @@ -617,7 +617,7 @@ std::string CellBuilder::to_hex() const { int len = serialize(buff, sizeof(buff)); char hex_buff[Cell::max_serialized_bytes * 2 + 1]; for (int i = 0; i < len; i++) { - sprintf(hex_buff + 2 * i, "%02x", buff[i]); + snprintf(hex_buff + 2 * i, sizeof(hex_buff) - 2 * i, "%02x", buff[i]); } return hex_buff; } diff --git a/crypto/vm/cells/CellHash.h b/crypto/vm/cells/CellHash.h index 2cbc0e8b..9435675f 100644 --- a/crypto/vm/cells/CellHash.h +++ b/crypto/vm/cells/CellHash.h @@ -74,13 +74,17 @@ struct CellHash { }; } // namespace vm +inline size_t cell_hash_slice_hash(td::Slice hash) { + // use offset 8, because in db keys are grouped by first bytes. + return td::as(hash.substr(8, 8).ubegin()); +} namespace std { template <> struct hash { typedef vm::CellHash argument_type; typedef std::size_t result_type; result_type operator()(argument_type const& s) const noexcept { - return td::as(s.as_slice().ubegin()); + return cell_hash_slice_hash(s.as_slice()); } }; } // namespace std diff --git a/crypto/vm/cells/CellSlice.cpp b/crypto/vm/cells/CellSlice.cpp index e1df5759..9cd3e931 100644 --- a/crypto/vm/cells/CellSlice.cpp +++ b/crypto/vm/cells/CellSlice.cpp @@ -264,7 +264,7 @@ bool CellSlice::advance_ext(unsigned bits, unsigned refs) { } bool CellSlice::advance_ext(unsigned bits_refs) { - return advance_ext(bits_refs >> 16, bits_refs & 0xffff); + return advance_ext(bits_refs & 0xffff, bits_refs >> 16); } // (PRIVATE) @@ -595,7 +595,7 @@ td::RefInt256 CellSlice::fetch_int256(unsigned bits, bool sgnd) { if (!have(bits)) { return {}; } else if (bits < td::BigInt256::word_shift) { - return td::make_refint(sgnd ? fetch_long(bits) : fetch_ulong(bits)); + return td::make_refint(td::int64(sgnd ? fetch_long(bits) : fetch_ulong(bits))); } else { td::RefInt256 res{true}; res.unique_write().import_bits(data_bits(), bits, sgnd); @@ -608,7 +608,7 @@ td::RefInt256 CellSlice::prefetch_int256(unsigned bits, bool sgnd) const { if (!have(bits)) { return {}; } else if (bits < td::BigInt256::word_shift) { - return td::make_refint(sgnd ? prefetch_long(bits) : prefetch_ulong(bits)); + return td::make_refint(td::int64(sgnd ? prefetch_long(bits) : prefetch_ulong(bits))); } else { td::RefInt256 res{true}; res.unique_write().import_bits(data_bits(), bits, sgnd); @@ -976,7 +976,7 @@ void CellSlice::dump(std::ostream& os, int level, bool endl) const { os << "; refs: " << refs_st << ".." << refs_en; if (level > 2) { char tmp[64]; - std::sprintf(tmp, "; ptr=data+%ld; z=%016llx", + std::snprintf(tmp, sizeof(tmp), "; ptr=data+%ld; z=%016llx", static_cast(ptr && cell.not_null() ? ptr - cell->get_data() : -1), static_cast(z)); os << tmp << " (have " << size() << " bits; " << zd << " preloaded)"; } @@ -1026,6 +1026,13 @@ bool CellSlice::print_rec(std::ostream& os, int indent) const { return print_rec(os, &limit, indent); } +bool CellSlice::print_rec(td::StringBuilder& sb, int indent) const { + std::ostringstream ss; + auto result = print_rec(ss, indent); + sb << ss.str(); + return result; +} + bool CellSlice::print_rec(int limit, std::ostream& os, int indent) const { return print_rec(os, &limit, indent); } @@ -1056,9 +1063,10 @@ std::ostream& operator<<(std::ostream& os, Ref cs_ref) { // If can_be_special is not null, then it is allowed to load special cell // Flag whether loaded cell is actually special will be stored into can_be_special VirtualCell::LoadedCell load_cell_slice_impl(Ref cell, bool* can_be_special) { + auto* vm_state_interface = VmStateInterface::get(); + bool library_loaded = false; while (true) { - auto* vm_state_interface = VmStateInterface::get(); - if (vm_state_interface) { + if (vm_state_interface && !library_loaded) { vm_state_interface->register_cell_load(cell->get_hash()); } auto r_loaded_cell = cell->load_cell(); @@ -1077,6 +1085,12 @@ VirtualCell::LoadedCell load_cell_slice_impl(Ref cell, bool* can_be_specia } else if (loaded_cell.data_cell->is_special()) { if (loaded_cell.data_cell->special_type() == DataCell::SpecialType::Library) { if (vm_state_interface) { + if (vm_state_interface->get_global_version() >= 5) { + if (library_loaded) { + throw VmError{Excno::cell_und, "failed to load library cell: recursive library cells are not allowed"}; + } + library_loaded = true; + } CellSlice cs(std::move(loaded_cell)); DCHECK(cs.size() == Cell::hash_bits + 8); auto library_cell = vm_state_interface->load_library(cs.data_bits() + 8); diff --git a/crypto/vm/cells/CellSlice.h b/crypto/vm/cells/CellSlice.h index 33fad741..ecce30f5 100644 --- a/crypto/vm/cells/CellSlice.h +++ b/crypto/vm/cells/CellSlice.h @@ -257,6 +257,7 @@ class CellSlice : public td::CntObject { void dump(std::ostream& os, int level = 0, bool endl = true) const; void dump_hex(std::ostream& os, int mode = 0, bool endl = false) const; bool print_rec(std::ostream& os, int indent = 0) const; + bool print_rec(td::StringBuilder& sb, int indent = 0) const; bool print_rec(std::ostream& os, int* limit, int indent = 0) const; bool print_rec(int limit, std::ostream& os, int indent = 0) const; void error() const { diff --git a/crypto/vm/cells/CellString.cpp b/crypto/vm/cells/CellString.cpp index b4738f88..474bc797 100644 --- a/crypto/vm/cells/CellString.cpp +++ b/crypto/vm/cells/CellString.cpp @@ -142,28 +142,57 @@ td::Ref CellText::do_store(td::BitSlice slice) { } template -void CellText::for_each(F &&f, CellSlice cs) { +td::Status CellText::for_each(F &&f, CellSlice cs) { + if (!cs.have(8)) { + return td::Status::Error("Cell underflow"); + } auto depth = cs.fetch_ulong(8); + if (depth > max_chain_length) { + return td::Status::Error("Too deep string"); + } for (td::uint32 i = 0; i < depth; i++) { - auto size = cs.fetch_ulong(8); - f(cs.fetch_bits(td::narrow_cast(size) * 8)); + if (!cs.have(8)) { + return td::Status::Error("Cell underflow"); + } + auto size = td::narrow_cast(cs.fetch_ulong(8)); + if (!cs.have(size * 8)) { + return td::Status::Error("Cell underflow"); + } + TRY_STATUS(f(cs.fetch_bits(size * 8))); if (i + 1 < depth) { + if (!cs.have_refs()) { + return td::Status::Error("Cell underflow"); + } cs = vm::load_cell_slice(cs.prefetch_ref()); } } + return td::Status::OK(); } td::Result CellText::load(CellSlice &cs) { unsigned int size = 0; - for_each([&](auto slice) { size += slice.size(); }, cs); + TRY_STATUS(for_each( + [&](auto slice) { + size += slice.size(); + if (size > max_bytes * 8) { + return td::Status::Error("String is too long"); + } + return td::Status::OK(); + }, + cs)); if (size % 8 != 0) { return td::Status::Error("Size is not divisible by 8"); } std::string res(size / 8, 0); td::BitPtr to(td::MutableSlice(res).ubegin()); - for_each([&](auto slice) { to.concat(slice); }, cs); + TRY_STATUS(for_each( + [&](auto slice) { + to.concat(slice); + return td::Status::OK(); + }, + cs)); CHECK(to.offs == (int)size); return res; } diff --git a/crypto/vm/cells/CellString.h b/crypto/vm/cells/CellString.h index 78b63f35..10bd89aa 100644 --- a/crypto/vm/cells/CellString.h +++ b/crypto/vm/cells/CellString.h @@ -52,7 +52,7 @@ class CellText { private: template - static void for_each(F &&f, CellSlice cs); + static td::Status for_each(F &&f, CellSlice cs); static td::Ref do_store(td::BitSlice slice); }; diff --git a/crypto/vm/cells/CellUsageTree.cpp b/crypto/vm/cells/CellUsageTree.cpp index 3f43ec6b..410b3fcd 100644 --- a/crypto/vm/cells/CellUsageTree.cpp +++ b/crypto/vm/cells/CellUsageTree.cpp @@ -22,12 +22,12 @@ namespace vm { // // CellUsageTree::NodePtr // -bool CellUsageTree::NodePtr::on_load() const { +bool CellUsageTree::NodePtr::on_load(const td::Ref& cell) const { auto tree = tree_weak_.lock(); if (!tree) { return false; } - tree->on_load(node_id_); + tree->on_load(node_id_, cell); return true; } @@ -111,8 +111,14 @@ void CellUsageTree::set_use_mark_for_is_loaded(bool use_mark) { use_mark_ = use_mark; } -void CellUsageTree::on_load(NodeId node_id) { +void CellUsageTree::on_load(NodeId node_id, const td::Ref& cell) { + if (nodes_[node_id].is_loaded) { + return; + } nodes_[node_id].is_loaded = true; + if (cell_load_callback_) { + cell_load_callback_(cell); + } } CellUsageTree::NodeId CellUsageTree::create_child(NodeId node_id, unsigned ref_id) { diff --git a/crypto/vm/cells/CellUsageTree.h b/crypto/vm/cells/CellUsageTree.h index 150dd2bd..af0f21f5 100644 --- a/crypto/vm/cells/CellUsageTree.h +++ b/crypto/vm/cells/CellUsageTree.h @@ -22,8 +22,12 @@ #include "td/utils/int_types.h" #include "td/utils/logging.h" +#include namespace vm { + +class DataCell; + class CellUsageTree : public std::enable_shared_from_this { public: using NodeId = td::uint32; @@ -38,7 +42,7 @@ class CellUsageTree : public std::enable_shared_from_this { return node_id_ == 0 || tree_weak_.expired(); } - bool on_load() const; + bool on_load(const td::Ref& cell) const; NodePtr create_child(unsigned ref_id) const; bool mark_path(CellUsageTree* master_tree) const; bool is_from_tree(const CellUsageTree* master_tree) const; @@ -59,6 +63,10 @@ class CellUsageTree : public std::enable_shared_from_this { void set_use_mark_for_is_loaded(bool use_mark = true); NodeId create_child(NodeId node_id, unsigned ref_id); + void set_cell_load_callback(std::function&)> f) { + cell_load_callback_ = std::move(f); + } + private: struct Node { bool is_loaded{false}; @@ -68,8 +76,9 @@ class CellUsageTree : public std::enable_shared_from_this { }; bool use_mark_{false}; std::vector nodes_{2}; + std::function&)> cell_load_callback_; - void on_load(NodeId node_id); + void on_load(NodeId node_id, const td::Ref& cell); NodeId create_node(NodeId parent); }; } // namespace vm diff --git a/crypto/vm/cells/CellWithStorage.h b/crypto/vm/cells/CellWithStorage.h index 1830c37d..f3fedfc3 100644 --- a/crypto/vm/cells/CellWithStorage.h +++ b/crypto/vm/cells/CellWithStorage.h @@ -20,6 +20,15 @@ namespace vm { namespace detail { + +template +struct DefaultAllocator { + template + std::unique_ptr make_unique(ArgsT&&... args) { + return std::make_unique(std::forward(args)...); + } +}; + template class CellWithArrayStorage : public CellT { public: @@ -29,14 +38,14 @@ class CellWithArrayStorage : public CellT { ~CellWithArrayStorage() { CellT::destroy_storage(get_storage()); } - template - static std::unique_ptr create(size_t storage_size, ArgsT&&... args) { + template + static auto create(Allocator allocator, size_t storage_size, ArgsT&&... args) { static_assert(CellT::max_storage_size <= 40 * 8, ""); //size = 128 + 32 + 8; auto size = (storage_size + 7) / 8; #define CASE(size) \ case (size): \ - return std::make_unique>(std::forward(args)...); + return allocator. template make_unique>(std::forward(args)...); #define CASE2(offset) CASE(offset) CASE(offset + 1) #define CASE8(offset) CASE2(offset) CASE2(offset + 2) CASE2(offset + 4) CASE2(offset + 6) #define CASE32(offset) CASE8(offset) CASE8(offset + 8) CASE8(offset + 16) CASE8(offset + 24) @@ -48,6 +57,10 @@ class CellWithArrayStorage : public CellT { LOG(FATAL) << "TOO BIG " << storage_size; UNREACHABLE(); } + template + static std::unique_ptr create(size_t storage_size, ArgsT&&... args) { + return create(DefaultAllocator{}, storage_size, std::forward(args)...); + } private: alignas(alignof(void*)) char storage_[Size]; diff --git a/crypto/vm/cells/DataCell.cpp b/crypto/vm/cells/DataCell.cpp index cccb11dc..4dd30161 100644 --- a/crypto/vm/cells/DataCell.cpp +++ b/crypto/vm/cells/DataCell.cpp @@ -25,7 +25,44 @@ #include "vm/cells/CellWithStorage.h" namespace vm { +thread_local bool DataCell::use_arena = false; + +namespace { +template +struct ArenaAllocator { + template + std::unique_ptr make_unique(ArgsT&&... args) { + auto* ptr = fast_alloc(sizeof(T)); + T* obj = new (ptr) T(std::forward(args)...); + return std::unique_ptr(obj); + } +private: + td::MutableSlice alloc_batch() { + size_t batch_size = 1 << 20; + auto batch = std::make_unique(batch_size); + return td::MutableSlice(batch.release(), batch_size); + } + char* fast_alloc(size_t size) { + thread_local td::MutableSlice batch; + auto aligned_size = (size + 7) / 8 * 8; + if (batch.size() < size) { + batch = alloc_batch(); + } + auto res = batch.begin(); + batch.remove_prefix(aligned_size); + return res; + } +}; +} std::unique_ptr DataCell::create_empty_data_cell(Info info) { + if (use_arena) { + ArenaAllocator allocator; + auto res = detail::CellWithArrayStorage::create(allocator, info.get_storage_size(), info); + // this is dangerous + Ref(res.get()).release(); + return res; + } + return detail::CellWithUniquePtrStorage::create(info.get_storage_size(), info); } @@ -113,6 +150,9 @@ td::Result> DataCell::create(td::ConstBitPtr data, unsigned bits, if (bits != 8 + hash_bytes * 8) { return td::Status::Error("Not enouch data for a Library special cell"); } + if (!refs.empty()) { + return td::Status::Error("Library special cell has a cell reference"); + } break; } @@ -356,7 +396,7 @@ std::string DataCell::to_hex() const { int len = serialize(buff, sizeof(buff)); char hex_buff[max_serialized_bytes * 2 + 1]; for (int i = 0; i < len; i++) { - sprintf(hex_buff + 2 * i, "%02x", buff[i]); + snprintf(hex_buff + 2 * i, sizeof(hex_buff) - 2 * i, "%02x", buff[i]); } return hex_buff; } diff --git a/crypto/vm/cells/DataCell.h b/crypto/vm/cells/DataCell.h index 933ff04a..6d3c845f 100644 --- a/crypto/vm/cells/DataCell.h +++ b/crypto/vm/cells/DataCell.h @@ -27,6 +27,9 @@ namespace vm { class DataCell : public Cell { public: + // NB: cells created with use_arena=true are never freed + static thread_local bool use_arena; + DataCell(const DataCell& other) = delete; ~DataCell() override; @@ -121,10 +124,6 @@ class DataCell : public Cell { void destroy_storage(char* storage); explicit DataCell(Info info); - Cell* get_ref_raw_ptr(unsigned idx) const { - DCHECK(idx < get_refs_cnt()); - return info_.get_refs(get_storage())[idx]; - } public: td::Result load_cell() const override { @@ -152,6 +151,20 @@ class DataCell : public Cell { return Ref(get_ref_raw_ptr(idx)); } + Cell* get_ref_raw_ptr(unsigned idx) const { + DCHECK(idx < get_refs_cnt()); + return info_.get_refs(get_storage())[idx]; + } + + Ref reset_ref_unsafe(unsigned idx, Ref ref, bool check_hash = true) { + CHECK(idx < get_refs_cnt()); + auto refs = info_.get_refs(get_storage()); + CHECK(!check_hash || refs[idx]->get_hash() == ref->get_hash()); + auto res = Ref(refs[idx], Ref::acquire_t{}); // call destructor + refs[idx] = ref.release(); + return res; + } + td::uint32 get_virtualization() const override { return info_.virtualization_; } @@ -173,6 +186,9 @@ class DataCell : public Cell { return ((get_bits() + 23) >> 3) + (with_hashes ? get_level_mask().get_hashes_count() * (hash_bytes + depth_bytes) : 0); } + size_t get_storage_size() const { + return info_.get_storage_size(); + } int serialize(unsigned char* buff, int buff_size, bool with_hashes = false) const; std::string serialize() const; std::string to_hex() const; @@ -207,6 +223,9 @@ class DataCell : public Cell { }; std::ostream& operator<<(std::ostream& os, const DataCell& c); +inline CellHash as_cell_hash(const Ref& cell) { + return cell->get_hash(); +} } // namespace vm diff --git a/crypto/vm/cells/MerkleProof.cpp b/crypto/vm/cells/MerkleProof.cpp index aee34367..26dff787 100644 --- a/crypto/vm/cells/MerkleProof.cpp +++ b/crypto/vm/cells/MerkleProof.cpp @@ -39,7 +39,13 @@ class MerkleProofImpl { dfs_usage_tree(cell, usage_tree_->root_id()); is_prunned_ = [this](const Ref &cell) { return visited_cells_.count(cell->get_hash()) == 0; }; } - return dfs(cell, cell->get_level()); + try { + return dfs(cell, cell->get_level()); + } catch (CellBuilder::CellWriteError &) { + return {}; + } catch (CellBuilder::CellCreateError &) { + return {}; + } } private: @@ -119,6 +125,9 @@ Ref MerkleProof::generate(Ref cell, CellUsageTree *usage_tree) { return {}; } auto raw = generate_raw(std::move(cell), usage_tree); + if (raw.is_null()) { + return {}; + } return CellBuilder::create_merkle_proof(std::move(raw)); } @@ -384,21 +393,29 @@ bool MerkleProofBuilder::clear() { return true; } -Ref MerkleProofBuilder::extract_proof() const { - return MerkleProof::generate(orig_root, usage_tree.get()); +td::Result> MerkleProofBuilder::extract_proof() const { + Ref proof = MerkleProof::generate(orig_root, usage_tree.get()); + if (proof.is_null()) { + return td::Status::Error("cannot create Merkle proof"); + } + return proof; } bool MerkleProofBuilder::extract_proof_to(Ref &proof_root) const { - return orig_root.not_null() && (proof_root = extract_proof()).not_null(); + if (orig_root.is_null()) { + return false; + } + auto R = extract_proof(); + if (R.is_error()) { + return false; + } + proof_root = R.move_as_ok(); + return true; } td::Result MerkleProofBuilder::extract_proof_boc() const { - Ref proof_root = extract_proof(); - if (proof_root.is_null()) { - return td::Status::Error("cannot create Merkle proof"); - } else { - return std_boc_serialize(std::move(proof_root)); - } + TRY_RESULT(proof_root, extract_proof()); + return std_boc_serialize(std::move(proof_root)); } } // namespace vm diff --git a/crypto/vm/cells/MerkleProof.h b/crypto/vm/cells/MerkleProof.h index 4f2add6c..fc2cb6eb 100644 --- a/crypto/vm/cells/MerkleProof.h +++ b/crypto/vm/cells/MerkleProof.h @@ -63,9 +63,13 @@ class MerkleProofBuilder { Ref root() const { return usage_root; } - Ref extract_proof() const; + td::Result> extract_proof() const; bool extract_proof_to(Ref &proof_root) const; td::Result extract_proof_boc() const; + + void set_cell_load_callback(std::function&)> f) { + usage_tree->set_cell_load_callback(std::move(f)); + } }; } // namespace vm diff --git a/crypto/vm/cells/MerkleUpdate.cpp b/crypto/vm/cells/MerkleUpdate.cpp index 3cc656c0..894612bd 100644 --- a/crypto/vm/cells/MerkleUpdate.cpp +++ b/crypto/vm/cells/MerkleUpdate.cpp @@ -221,6 +221,9 @@ Ref MerkleUpdate::generate(Ref from, Ref to, CellUsageTree *us return {}; } auto res = generate_raw(std::move(from), std::move(to), usage_tree); + if (res.first.is_null() || res.second.is_null()) { + return {}; + } return CellBuilder::create_merkle_update(res.first, res.second); } diff --git a/crypto/vm/cells/PrunnedCell.h b/crypto/vm/cells/PrunnedCell.h index e8434ae4..a58b245c 100644 --- a/crypto/vm/cells/PrunnedCell.h +++ b/crypto/vm/cells/PrunnedCell.h @@ -30,18 +30,27 @@ struct PrunnedCellInfo { template class PrunnedCell : public Cell { public: + ExtraT& get_extra() { + return extra_; + } const ExtraT& get_extra() const { return extra_; } static td::Result>> create(const PrunnedCellInfo& prunned_cell_info, ExtraT&& extra) { + return create(detail::DefaultAllocator>(), prunned_cell_info, std::forward(extra)); + } + + template + static td::Result>> create(AllocatorT allocator, const PrunnedCellInfo& prunned_cell_info, + ExtraT&& extra) { auto level_mask = prunned_cell_info.level_mask; if (level_mask.get_level() > max_level) { return td::Status::Error("Level is too big"); } Info info(level_mask); auto prunned_cell = - detail::CellWithUniquePtrStorage>::create(info.get_storage_size(), info, std::move(extra)); + detail::CellWithArrayStorage>::create(allocator, info.get_storage_size(), info, std::move(extra)); TRY_STATUS(prunned_cell->init(prunned_cell_info)); return Ref>(prunned_cell.release(), typename Ref>::acquire_t{}); } @@ -51,6 +60,7 @@ class PrunnedCell : public Cell { } protected: + static constexpr auto max_storage_size = (max_level + 1) * (hash_bytes + sizeof(td::uint16)); struct Info { Info(LevelMask level_mask) { level_mask_ = level_mask.get_mask() & 7; diff --git a/crypto/vm/cells/UsageCell.h b/crypto/vm/cells/UsageCell.h index bf15bb56..3e6e8898 100644 --- a/crypto/vm/cells/UsageCell.h +++ b/crypto/vm/cells/UsageCell.h @@ -39,7 +39,7 @@ class UsageCell : public Cell { // load interface td::Result load_cell() const override { TRY_RESULT(loaded_cell, cell_->load_cell()); - if (tree_node_.on_load()) { + if (tree_node_.on_load(loaded_cell.data_cell)) { CHECK(loaded_cell.tree_node.empty()); loaded_cell.tree_node = tree_node_; } diff --git a/crypto/vm/continuation.cpp b/crypto/vm/continuation.cpp index 94d20125..bc67c6d2 100644 --- a/crypto/vm/continuation.cpp +++ b/crypto/vm/continuation.cpp @@ -22,11 +22,13 @@ #include "vm/log.h" #include "vm/vm.h" #include "vm/vmstate.h" +#include "vm/boc.h" +#include "td/utils/misc.h" namespace vm { -int Continuation::jump_w(VmState* st) & { - return static_cast(this)->jump(st); +td::Ref Continuation::jump_w(VmState* st, int& exitcode) & { + return static_cast(this)->jump(st, exitcode); } bool Continuation::has_c0() const { @@ -254,6 +256,17 @@ bool Continuation::deserialize_to(Ref cell, Ref& cont, int m return deserialize_to(cs, cont, mode & ~0x1000) && cs.empty_ext(); } +std::ostream& operator<<(std::ostream& os, const Continuation& cont) { + CellBuilder cb; + if (cont.serialize(cb)) { + auto boc = vm::std_boc_serialize(cb.finalize()); + if (boc.is_ok()) { + os << td::buffer_to_hex(boc.move_as_ok().as_slice()); + } + } + return os; +} + bool QuitCont::serialize(CellBuilder& cb) const { // vmc_quit$1000 exit_code:int32 = VmCont; return cb.store_long_bool(8, 4) && cb.store_long_bool(exit_code, 32); @@ -269,7 +282,11 @@ Ref QuitCont::deserialize(CellSlice& cs, int mode) { } } -int ExcQuitCont::jump(VmState* st) const & { +std::string QuitCont::type() const { + return "vmc_quit"; +} + +td::Ref ExcQuitCont::jump(VmState* st, int& exitcode) const& { int n = 0; try { n = st->get_stack().pop_smallint_range(0xffff); @@ -277,7 +294,12 @@ int ExcQuitCont::jump(VmState* st) const & { n = vme.get_errno(); } VM_LOG(st) << "default exception handler, terminating vm with exit code " << n; - return ~n; + exitcode = ~n; + return {}; +} + +std::string ExcQuitCont::type() const { + return "vmc_quit_exc"; } bool ExcQuitCont::serialize(CellBuilder& cb) const { @@ -290,16 +312,20 @@ Ref ExcQuitCont::deserialize(CellSlice& cs, int mode) { return cs.fetch_ulong(4) == 9 ? Ref{true} : Ref{}; } -int PushIntCont::jump(VmState* st) const & { +td::Ref PushIntCont::jump(VmState* st, int& exitcode) const& { VM_LOG(st) << "execute implicit PUSH " << push_val << " (slow)"; st->get_stack().push_smallint(push_val); - return st->jump(next); + return next; } -int PushIntCont::jump_w(VmState* st) & { +td::Ref PushIntCont::jump_w(VmState* st, int& exitcode) & { VM_LOG(st) << "execute implicit PUSH " << push_val; st->get_stack().push_smallint(push_val); - return st->jump(std::move(next)); + return std::move(next); +} + +std::string PushIntCont::type() const { + return "vmc_pushint"; } bool PushIntCont::serialize(CellBuilder& cb) const { @@ -320,20 +346,20 @@ Ref PushIntCont::deserialize(CellSlice& cs, int mode) { } } -int ArgContExt::jump(VmState* st) const & { +td::Ref ArgContExt::jump(VmState* st, int& exitcode) const& { st->adjust_cr(data.save); if (data.cp != -1) { st->force_cp(data.cp); } - return ext->jump(st); + return ext; } -int ArgContExt::jump_w(VmState* st) & { +td::Ref ArgContExt::jump_w(VmState* st, int& exitcode) & { st->adjust_cr(std::move(data.save)); if (data.cp != -1) { st->force_cp(data.cp); } - return st->jump_to(std::move(ext)); + return std::move(ext); } bool ArgContExt::serialize(CellBuilder& cb) const { @@ -353,32 +379,36 @@ Ref ArgContExt::deserialize(CellSlice& cs, int mode) { : Ref{}; } -int RepeatCont::jump(VmState* st) const & { - VM_LOG(st) << "repeat " << count << " more times (slow)\n"; - if (count <= 0) { - return st->jump(after); - } - if (body->has_c0()) { - return st->jump(body); - } - st->set_c0(Ref{true, body, after, count - 1}); - return st->jump(body); +std::string ArgContExt::type() const { + return "vmc_envelope"; } -int RepeatCont::jump_w(VmState* st) & { +td::Ref RepeatCont::jump(VmState* st, int& exitcode) const& { + VM_LOG(st) << "repeat " << count << " more times (slow)\n"; + if (count <= 0) { + return after; + } + if (body->has_c0()) { + return body; + } + st->set_c0(Ref{true, body, after, count - 1}); + return body; +} + +td::Ref RepeatCont::jump_w(VmState* st, int& exitcode) & { VM_LOG(st) << "repeat " << count << " more times\n"; if (count <= 0) { body.clear(); - return st->jump(std::move(after)); + return std::move(after); } if (body->has_c0()) { after.clear(); - return st->jump(std::move(body)); + return std::move(body); } // optimization: since this is unique, reuse *this instead of creating new object --count; st->set_c0(Ref{this}); - return st->jump(body); + return body; } bool RepeatCont::serialize(CellBuilder& cb) const { @@ -401,6 +431,10 @@ Ref RepeatCont::deserialize(CellSlice& cs, int mode) { } } +std::string RepeatCont::type() const { + return "vmc_repeat"; +} + int VmState::repeat(Ref body, Ref after, long long count) { if (count <= 0) { body.clear(); @@ -410,21 +444,21 @@ int VmState::repeat(Ref body, Ref after, long long c } } -int AgainCont::jump(VmState* st) const & { +td::Ref AgainCont::jump(VmState* st, int& exitcode) const& { VM_LOG(st) << "again an infinite loop iteration (slow)\n"; if (!body->has_c0()) { st->set_c0(Ref{this}); } - return st->jump(body); + return body; } -int AgainCont::jump_w(VmState* st) & { +td::Ref AgainCont::jump_w(VmState* st, int& exitcode) & { VM_LOG(st) << "again an infinite loop iteration\n"; if (!body->has_c0()) { st->set_c0(Ref{this}); - return st->jump(body); + return body; } else { - return st->jump(std::move(body)); + return std::move(body); } } @@ -444,35 +478,39 @@ Ref AgainCont::deserialize(CellSlice& cs, int mode) { } } +std::string AgainCont::type() const { + return "vmc_again"; +} + int VmState::again(Ref body) { return jump(Ref{true, std::move(body)}); } -int UntilCont::jump(VmState* st) const & { +td::Ref UntilCont::jump(VmState* st, int& exitcode) const& { VM_LOG(st) << "until loop body end (slow)\n"; if (st->get_stack().pop_bool()) { VM_LOG(st) << "until loop terminated\n"; - return st->jump(after); + return after; } if (!body->has_c0()) { st->set_c0(Ref{this}); } - return st->jump(body); + return body; } -int UntilCont::jump_w(VmState* st) & { +td::Ref UntilCont::jump_w(VmState* st, int& exitcode) & { VM_LOG(st) << "until loop body end\n"; if (st->get_stack().pop_bool()) { VM_LOG(st) << "until loop terminated\n"; body.clear(); - return st->jump(std::move(after)); + return std::move(after); } if (!body->has_c0()) { st->set_c0(Ref{this}); - return st->jump(body); + return body; } else { after.clear(); - return st->jump(std::move(body)); + return std::move(body); } } @@ -493,6 +531,10 @@ Ref UntilCont::deserialize(CellSlice& cs, int mode) { } } +std::string UntilCont::type() const { + return "vmc_until"; +} + int VmState::until(Ref body, Ref after) { if (!body->has_c0()) { set_c0(Ref{true, body, std::move(after)}); @@ -500,54 +542,54 @@ int VmState::until(Ref body, Ref after) { return jump(std::move(body)); } -int WhileCont::jump(VmState* st) const & { +td::Ref WhileCont::jump(VmState* st, int& exitcode) const& { if (chkcond) { VM_LOG(st) << "while loop condition end (slow)\n"; if (!st->get_stack().pop_bool()) { VM_LOG(st) << "while loop terminated\n"; - return st->jump(after); + return after; } if (!body->has_c0()) { st->set_c0(Ref{true, cond, body, after, false}); } - return st->jump(body); + return body; } else { VM_LOG(st) << "while loop body end (slow)\n"; if (!cond->has_c0()) { st->set_c0(Ref{true, cond, body, after, true}); } - return st->jump(cond); + return cond; } } -int WhileCont::jump_w(VmState* st) & { +td::Ref WhileCont::jump_w(VmState* st, int& exitcode) & { if (chkcond) { VM_LOG(st) << "while loop condition end\n"; if (!st->get_stack().pop_bool()) { VM_LOG(st) << "while loop terminated\n"; cond.clear(); body.clear(); - return st->jump(std::move(after)); + return std::move(after); } if (!body->has_c0()) { chkcond = false; // re-use current object since we hold the unique pointer to it st->set_c0(Ref{this}); - return st->jump(body); + return body; } else { cond.clear(); after.clear(); - return st->jump(std::move(body)); + return std::move(body); } } else { VM_LOG(st) << "while loop body end\n"; if (!cond->has_c0()) { chkcond = true; // re-use current object st->set_c0(Ref{this}); - return st->jump(cond); + return cond; } else { body.clear(); after.clear(); - return st->jump(std::move(cond)); + return std::move(cond); } } } @@ -575,6 +617,10 @@ Ref WhileCont::deserialize(CellSlice& cs, int mode) { } } +std::string WhileCont::type() const { + return chkcond ? "vmc_while_cond" : "vmc_while_body"; +} + int VmState::loop_while(Ref cond, Ref body, Ref after) { if (!cond->has_c0()) { set_c0(Ref{true, cond, std::move(body), std::move(after), true}); @@ -582,16 +628,16 @@ int VmState::loop_while(Ref cond, Ref body, Ref OrdCont::jump(VmState* st, int& exitcode) const& { st->adjust_cr(data.save); st->set_code(code, data.cp); - return 0; + return {}; } -int OrdCont::jump_w(VmState* st) & { +td::Ref OrdCont::jump_w(VmState* st, int& exitcode) & { st->adjust_cr(std::move(data.save)); st->set_code(std::move(code), data.cp); - return 0; + return {}; } bool OrdCont::serialize(CellBuilder& cb) const { @@ -610,4 +656,8 @@ Ref OrdCont::deserialize(CellSlice& cs, int mode) { : Ref{}; } +std::string OrdCont::type() const { + return "vmc_std"; +} + } // namespace vm diff --git a/crypto/vm/continuation.h b/crypto/vm/continuation.h index 37abe869..0c758c92 100644 --- a/crypto/vm/continuation.h +++ b/crypto/vm/continuation.h @@ -161,8 +161,8 @@ struct ControlData { class Continuation : public td::CntObject { public: - virtual int jump(VmState* st) const & = 0; - virtual int jump_w(VmState* st) &; + virtual td::Ref jump(VmState* st, int& exitcode) const& = 0; + virtual td::Ref jump_w(VmState* st, int& exitcode) &; virtual ControlData* get_cdata() { return 0; } @@ -191,8 +191,11 @@ class Continuation : public td::CntObject { return (cont = deserialize(cs, mode)).not_null(); } static bool deserialize_to(Ref cell, Ref& cont, int mode = 0); + virtual std::string type() const = 0; }; +std::ostream& operator<<(std::ostream& os, const Continuation& cont); + class QuitCont : public Continuation { int exit_code; @@ -200,20 +203,23 @@ class QuitCont : public Continuation { QuitCont(int _code = 0) : exit_code(_code) { } ~QuitCont() override = default; - int jump(VmState* st) const & override { - return ~exit_code; + td::Ref jump(VmState* st, int& exitcode) const& override { + exitcode = ~exit_code; + return {}; } bool serialize(CellBuilder& cb) const override; static Ref deserialize(CellSlice& cs, int mode = 0); + std::string type() const override; }; class ExcQuitCont : public Continuation { public: ExcQuitCont() = default; ~ExcQuitCont() override = default; - int jump(VmState* st) const & override; + td::Ref jump(VmState* st, int& exitcode) const& override; bool serialize(CellBuilder& cb) const override; static Ref deserialize(CellSlice& cs, int mode = 0); + std::string type() const override; }; class PushIntCont : public Continuation { @@ -224,10 +230,11 @@ class PushIntCont : public Continuation { PushIntCont(int val, Ref _next) : push_val(val), next(_next) { } ~PushIntCont() override = default; - int jump(VmState* st) const & override; - int jump_w(VmState* st) & override; + td::Ref jump(VmState* st, int& exitcode) const& override; + td::Ref jump_w(VmState* st, int& exitcode) & override; bool serialize(CellBuilder& cb) const override; static Ref deserialize(CellSlice& cs, int mode = 0); + std::string type() const override; }; class RepeatCont : public Continuation { @@ -239,10 +246,11 @@ class RepeatCont : public Continuation { : body(std::move(_body)), after(std::move(_after)), count(_count) { } ~RepeatCont() override = default; - int jump(VmState* st) const & override; - int jump_w(VmState* st) & override; + td::Ref jump(VmState* st, int& exitcode) const& override; + td::Ref jump_w(VmState* st, int& exitcode) & override; bool serialize(CellBuilder& cb) const override; static Ref deserialize(CellSlice& cs, int mode = 0); + std::string type() const override; }; class AgainCont : public Continuation { @@ -252,10 +260,11 @@ class AgainCont : public Continuation { AgainCont(Ref _body) : body(std::move(_body)) { } ~AgainCont() override = default; - int jump(VmState* st) const & override; - int jump_w(VmState* st) & override; + td::Ref jump(VmState* st, int& exitcode) const& override; + td::Ref jump_w(VmState* st, int& exitcode) & override; bool serialize(CellBuilder& cb) const override; static Ref deserialize(CellSlice& cs, int mode = 0); + std::string type() const override; }; class UntilCont : public Continuation { @@ -265,10 +274,11 @@ class UntilCont : public Continuation { UntilCont(Ref _body, Ref _after) : body(std::move(_body)), after(std::move(_after)) { } ~UntilCont() override = default; - int jump(VmState* st) const & override; - int jump_w(VmState* st) & override; + td::Ref jump(VmState* st, int& exitcode) const& override; + td::Ref jump_w(VmState* st, int& exitcode) & override; bool serialize(CellBuilder& cb) const override; static Ref deserialize(CellSlice& cs, int mode = 0); + std::string type() const override; }; class WhileCont : public Continuation { @@ -280,10 +290,11 @@ class WhileCont : public Continuation { : cond(std::move(_cond)), body(std::move(_body)), after(std::move(_after)), chkcond(_chk) { } ~WhileCont() override = default; - int jump(VmState* st) const & override; - int jump_w(VmState* st) & override; + td::Ref jump(VmState* st, int& exitcode) const& override; + td::Ref jump_w(VmState* st, int& exitcode) & override; bool serialize(CellBuilder& cb) const override; static Ref deserialize(CellSlice& cs, int mode = 0); + std::string type() const override; }; class ArgContExt : public Continuation { @@ -302,8 +313,8 @@ class ArgContExt : public Continuation { ArgContExt(const ArgContExt&) = default; ArgContExt(ArgContExt&&) = default; ~ArgContExt() override = default; - int jump(VmState* st) const & override; - int jump_w(VmState* st) & override; + td::Ref jump(VmState* st, int& exitcode) const& override; + td::Ref jump_w(VmState* st, int& exitcode) & override; ControlData* get_cdata() override { return &data; } @@ -315,6 +326,7 @@ class ArgContExt : public Continuation { } bool serialize(CellBuilder& cb) const override; static Ref deserialize(CellSlice& cs, int mode = 0); + std::string type() const override; }; class OrdCont : public Continuation { @@ -343,8 +355,8 @@ class OrdCont : public Continuation { td::CntObject* make_copy() const override { return new OrdCont{*this}; } - int jump(VmState* st) const & override; - int jump_w(VmState* st) & override; + td::Ref jump(VmState* st, int& exitcode) const& override; + td::Ref jump_w(VmState* st, int& exitcode) & override; ControlData* get_cdata() override { return &data; @@ -369,6 +381,7 @@ class OrdCont : public Continuation { } bool serialize(CellBuilder& cb) const override; static Ref deserialize(CellSlice& cs, int mode = 0); + std::string type() const override; }; ControlData* force_cdata(Ref& cont); diff --git a/crypto/vm/contops.cpp b/crypto/vm/contops.cpp index c3cb36bb..1ccf53da 100644 --- a/crypto/vm/contops.cpp +++ b/crypto/vm/contops.cpp @@ -212,6 +212,77 @@ int exec_ret_data(VmState* st) { return st->ret(); } +// Mode: +// +1 = same_c3 (set c3 to code) +// +2 = push_0 (push an implicit 0 before running the code) +// +4 = load c4 (persistent data) from stack and return its final value +// +8 = load gas limit from stack and return consumed gas +// +16 = load c7 (smart-contract context) +// +32 = return c5 (actions) +// +64 = pop hard gas limit (enabled by ACCEPT) from stack as well +// +128 = isolated gas consumption (separate set of visited cells, reset chksgn counter) +// +256 = pop number N, return exactly N values from stack (only if res=0 or 1; if not enough then res=stk_und) +int exec_runvm_common(VmState* st, unsigned mode) { + if (mode >= 512) { + throw VmError{Excno::range_chk, "invalid flags"}; + } + st->consume_gas(VmState::runvm_gas_price); + Stack& stack = st->get_stack(); + bool with_data = mode & 4; + Ref c7; + Ref data, actions; + long long gas_max = (mode & 64) ? stack.pop_long_range(vm::GasLimits::infty) : vm::GasLimits::infty; + long long gas_limit = (mode & 8) ? stack.pop_long_range(vm::GasLimits::infty) : vm::GasLimits::infty; + if (!(mode & 64)) { + gas_max = gas_limit; + } else { + gas_max = std::max(gas_max, gas_limit); + } + if (mode & 16) { + c7 = stack.pop_tuple(); + } + if (with_data) { + data = stack.pop_cell(); + } + int ret_vals = -1; + if (mode & 256) { + ret_vals = stack.pop_smallint_range(1 << 30); + } + auto code = stack.pop_cellslice(); + int stack_size = stack.pop_smallint_range(stack.depth() - 1); + std::vector new_stack_entries(stack_size); + for (int i = 0; i < stack_size; ++i) { + new_stack_entries[stack_size - 1 - i] = stack.pop(); + } + td::Ref new_stack{true, std::move(new_stack_entries)}; + st->consume_stack_gas(new_stack); + gas_max = std::min(gas_max, st->get_gas_limits().gas_remaining); + gas_limit = std::min(gas_limit, st->get_gas_limits().gas_remaining); + vm::GasLimits gas{gas_limit, gas_max}; + + VmStateInterface::Guard guard{nullptr}; // Don't consume gas for creating/loading cells during VM init + VmState new_state{ + std::move(code), st->get_global_version(), std::move(new_stack), gas, (int)mode & 3, std::move(data), + VmLog{}, std::vector>{}, std::move(c7)}; + new_state.set_chksig_always_succeed(st->get_chksig_always_succeed()); + st->run_child_vm(std::move(new_state), with_data, mode & 32, mode & 8, mode & 128, ret_vals); + return 0; +} + +int exec_runvm(VmState* st, unsigned args) { + VM_LOG(st) << "execute RUNVM " << (args & 4095) << "\n"; + return exec_runvm_common(st, args & 4095); +} + +int exec_runvmx(VmState* st) { + VM_LOG(st) << "execute RUNVMX\n"; + return exec_runvm_common(st, st->get_stack().pop_smallint_range(4095)); +} + +std::string dump_runvm(CellSlice&, unsigned args) { + return PSTRING() << "RUNVM " << (args & 4095); +} + void register_continuation_jump_ops(OpcodeTable& cp0) { using namespace std::placeholders; cp0.insert(OpcodeInstr::mksimple(0xd8, 8, "EXECUTE", exec_execute)) @@ -246,7 +317,9 @@ void register_continuation_jump_ops(OpcodeTable& cp0) { }, "JMPREFDATA"), compute_len_push_ref)) - .insert(OpcodeInstr::mksimple(0xdb3f, 16, "RETDATA", exec_ret_data)); + .insert(OpcodeInstr::mksimple(0xdb3f, 16, "RETDATA", exec_ret_data)) + .insert(OpcodeInstr::mkfixed(0xdb4, 12, 12, dump_runvm, exec_runvm)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xdb50, 16, "RUNVMX ", exec_runvmx)->require_version(4)); } int exec_if(VmState* st) { @@ -378,8 +451,8 @@ int exec_if_bit_jmp(VmState* st, unsigned args) { } std::string dump_if_bit_jmp(CellSlice& cs, unsigned args) { - std::ostringstream os{args & 0x20 ? "IFN" : " IF"}; - os << "BITJMP " << (args & 0x1f); + std::ostringstream os; + os << "IF" << (args & 0x20 ? "N" : "") << "BITJMP " << (args & 0x1f); return os.str(); } @@ -408,8 +481,8 @@ std::string dump_if_bit_jmpref(CellSlice& cs, unsigned args, int pfx_bits) { } cs.advance(pfx_bits); cs.advance_refs(1); - std::ostringstream os{args & 0x20 ? "IFN" : " IF"}; - os << "BITJMPREF " << (args & 0x1f); + std::ostringstream os; + os << "IF" << (args & 0x20 ? "N" : "") << "BITJMPREF " << (args & 0x1f); return os.str(); } @@ -597,8 +670,8 @@ int exec_setcontargs(VmState* st, unsigned args) { std::string dump_setcontargs(CellSlice& cs, unsigned args, const char* name) { int copy = (args >> 4) & 15, more = ((args + 1) & 15) - 1; - std::ostringstream os{name}; - os << ' ' << copy << ',' << more; + std::ostringstream os; + os << name << ' ' << copy << ',' << more; return os.str(); } @@ -850,6 +923,41 @@ int exec_setcont_ctr_var(VmState* st) { return 0; } +int exec_setcont_ctr_many(VmState* st, unsigned args) { + unsigned mask = args & 255; + VM_LOG(st) << "execute SETCONTCTRMANY " << mask; + if (mask & (1 << 6)) { + throw VmError{Excno::range_chk, "no control register c6"}; + } + Stack& stack = st->get_stack(); + auto cont = stack.pop_cont(); + for (int i = 0; i < 8; ++i) { + if (mask & (1 << i)) { + throw_typechk(force_cregs(cont)->define(i, st->get(i))); + } + } + st->get_stack().push_cont(std::move(cont)); + return 0; +} + +int exec_setcont_ctr_many_var(VmState* st) { + VM_LOG(st) << "execute SETCONTCTRMANYX"; + Stack& stack = st->get_stack(); + stack.check_underflow(2); + int mask = stack.pop_smallint_range(255); + if (mask & (1 << 6)) { + throw VmError{Excno::range_chk, "no control register c6"}; + } + auto cont = stack.pop_cont(); + for (int i = 0; i < 8; ++i) { + if (mask & (1 << i)) { + throw_typechk(force_cregs(cont)->define(i, st->get(i))); + } + } + st->get_stack().push_cont(std::move(cont)); + return 0; +} + int exec_compos(VmState* st, unsigned mask, const char* name) { Stack& stack = st->get_stack(); VM_LOG(st) << "execute " << name; @@ -964,6 +1072,8 @@ void register_continuation_change_ops(OpcodeTable& cp0) { cp0.insert(OpcodeInstr::mksimple(0xede0, 16, "PUSHCTRX", exec_push_ctr_var)) .insert(OpcodeInstr::mksimple(0xede1, 16, "POPCTRX", exec_pop_ctr_var)) .insert(OpcodeInstr::mksimple(0xede2, 16, "SETCONTCTRX", exec_setcont_ctr_var)) + .insert(OpcodeInstr::mkfixed(0xede3, 16, 8, instr::dump_1c_l_add(1, "SETCONTCTRMANY "), exec_setcont_ctr_many)->require_version(9)) + .insert(OpcodeInstr::mksimple(0xede4, 16, "SETCONTCTRMANYX", exec_setcont_ctr_many_var)->require_version(9)) .insert(OpcodeInstr::mksimple(0xedf0, 16, "BOOLAND", std::bind(exec_compos, _1, 1, "BOOLAND"))) .insert(OpcodeInstr::mksimple(0xedf1, 16, "BOOLOR", std::bind(exec_compos, _1, 2, "BOOLOR"))) .insert(OpcodeInstr::mksimple(0xedf2, 16, "COMPOSBOTH", std::bind(exec_compos, _1, 3, "COMPOSBOTH"))) @@ -1065,8 +1175,8 @@ std::string dump_throw_any(CellSlice& cs, unsigned args) { bool has_param = args & 1; bool has_cond = args & 6; bool throw_cond = args & 2; - std::ostringstream os{has_param ? "THROWARG" : "THROW"}; - os << "ANY"; + std::ostringstream os; + os << "THROW" << (has_param ? "ARG" : "") << "ANY"; if (has_cond) { os << (throw_cond ? "IF" : "IFNOT"); } diff --git a/crypto/vm/db/CellHashTable.h b/crypto/vm/db/CellHashTable.h index 7d0308b7..522c987b 100644 --- a/crypto/vm/db/CellHashTable.h +++ b/crypto/vm/db/CellHashTable.h @@ -19,7 +19,7 @@ #pragma once #include "td/utils/Slice.h" - +#include "td/utils/HashSet.h" #include namespace vm { @@ -43,7 +43,7 @@ class CellHashTable { template void for_each(F &&f) { for (auto &info : set_) { - f(info); + f(const_cast(info)); } } template @@ -73,6 +73,6 @@ class CellHashTable { } private: - std::set> set_; + td::NodeHashSet set_; }; } // namespace vm diff --git a/crypto/vm/db/CellStorage.cpp b/crypto/vm/db/CellStorage.cpp index a1b7365b..06df461e 100644 --- a/crypto/vm/db/CellStorage.cpp +++ b/crypto/vm/db/CellStorage.cpp @@ -27,16 +27,26 @@ namespace vm { namespace { class RefcntCellStorer { public: - RefcntCellStorer(td::int32 refcnt, const DataCell &cell) : refcnt_(refcnt), cell_(cell) { + RefcntCellStorer(td::int32 refcnt, const td::Ref &cell, bool as_boc) + : refcnt_(refcnt), cell_(cell), as_boc_(as_boc) { } template void store(StorerT &storer) const { + TD_PERF_COUNTER(cell_store); using td::store; + if (as_boc_) { + td::int32 tag = -1; + store(tag, storer); + store(refcnt_, storer); + td::BufferSlice data = vm::std_boc_serialize(cell_).move_as_ok(); + storer.store_slice(data); + return; + } store(refcnt_, storer); - store(cell_, storer); - for (unsigned i = 0; i < cell_.size_refs(); i++) { - auto cell = cell_.get_ref(i); + store(*cell_, storer); + for (unsigned i = 0; i < cell_->size_refs(); i++) { + auto cell = cell_->get_ref(i); auto level_mask = cell->get_level_mask(); auto level = level_mask.get_level(); td::uint8 x = static_cast(level_mask.get_mask()); @@ -60,7 +70,8 @@ class RefcntCellStorer { private: td::int32 refcnt_; - const DataCell &cell_; + td::Ref cell_; + bool as_boc_; }; class RefcntCellParser { @@ -69,11 +80,17 @@ class RefcntCellParser { } td::int32 refcnt; Ref cell; + bool stored_boc_; template void parse(ParserT &parser, ExtCellCreator &ext_cell_creator) { using ::td::parse; parse(refcnt, parser); + stored_boc_ = false; + if (refcnt == -1) { + stored_boc_ = true; + parse(refcnt, parser); + } if (!need_data_) { return; } @@ -81,6 +98,12 @@ class RefcntCellParser { TRY_STATUS(parser.get_status()); auto size = parser.get_left_len(); td::Slice data = parser.template fetch_string_raw(size); + if (stored_boc_) { + TRY_RESULT(boc, vm::std_boc_deserialize(data, false, true)); + TRY_RESULT(loaded_cell, boc->load_cell()); + cell = std::move(loaded_cell.data_cell); + return td::Status::OK(); + } CellSerializationInfo info; auto cell_data = data; TRY_STATUS(info.init(cell_data, 0 /*ref_byte_size*/)); @@ -122,12 +145,46 @@ class RefcntCellParser { }; } // namespace -CellLoader::CellLoader(std::shared_ptr reader) : reader_(std::move(reader)) { +CellLoader::CellLoader(std::shared_ptr reader, std::function on_load_callback) + : reader_(std::move(reader)), on_load_callback_(std::move(on_load_callback)) { CHECK(reader_); } td::Result CellLoader::load(td::Slice hash, bool need_data, ExtCellCreator &ext_cell_creator) { //LOG(ERROR) << "Storage: load cell " << hash.size() << " " << td::base64_encode(hash); + TD_PERF_COUNTER(cell_load); + std::string serialized; + TRY_RESULT(get_status, reader_->get(hash, serialized)); + if (get_status != KeyValue::GetStatus::Ok) { + DCHECK(get_status == KeyValue::GetStatus::NotFound); + return LoadResult{}; + } + TRY_RESULT(res, load(hash, serialized, need_data, ext_cell_creator)); + if (on_load_callback_) { + on_load_callback_(res); + } + return res; +} + +td::Result CellLoader::load(td::Slice hash, td::Slice value, bool need_data, + ExtCellCreator &ext_cell_creator) { + LoadResult res; + res.status = LoadResult::Ok; + + RefcntCellParser refcnt_cell(need_data); + td::TlParser parser(value); + refcnt_cell.parse(parser, ext_cell_creator); + TRY_STATUS(parser.get_status()); + + res.refcnt_ = refcnt_cell.refcnt; + res.cell_ = std::move(refcnt_cell.cell); + res.stored_boc_ = refcnt_cell.stored_boc_; + //CHECK(res.cell_->get_hash() == hash); + + return res; +} + +td::Result CellLoader::load_refcnt(td::Slice hash) { LoadResult res; std::string serialized; TRY_RESULT(get_status, reader_->get(hash, serialized)); @@ -135,18 +192,13 @@ td::Result CellLoader::load(td::Slice hash, bool need_da DCHECK(get_status == KeyValue::GetStatus::NotFound); return res; } - res.status = LoadResult::Ok; - - RefcntCellParser refcnt_cell(need_data); td::TlParser parser(serialized); - refcnt_cell.parse(parser, ext_cell_creator); + td::parse(res.refcnt_, parser); + if (res.refcnt_ == -1) { + parse(res.refcnt_, parser); + } TRY_STATUS(parser.get_status()); - - res.refcnt_ = refcnt_cell.refcnt; - res.cell_ = std::move(refcnt_cell.cell); - //CHECK(res.cell_->get_hash() == hash); - return res; } @@ -157,7 +209,11 @@ td::Status CellStorer::erase(td::Slice hash) { return kv_.erase(hash); } -td::Status CellStorer::set(td::int32 refcnt, const DataCell &cell) { - return kv_.set(cell.get_hash().as_slice(), td::serialize(RefcntCellStorer(refcnt, cell))); +std::string CellStorer::serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc) { + return td::serialize(RefcntCellStorer(refcnt, cell, as_boc)); +} + +td::Status CellStorer::set(td::int32 refcnt, const td::Ref &cell, bool as_boc) { + return kv_.set(cell->get_hash().as_slice(), serialize_value(refcnt, cell, as_boc)); } } // namespace vm diff --git a/crypto/vm/db/CellStorage.h b/crypto/vm/db/CellStorage.h index b705b531..cabd7fdc 100644 --- a/crypto/vm/db/CellStorage.h +++ b/crypto/vm/db/CellStorage.h @@ -45,19 +45,24 @@ class CellLoader { Ref cell_; td::int32 refcnt_{0}; + bool stored_boc_{false}; }; - CellLoader(std::shared_ptr reader); + CellLoader(std::shared_ptr reader, std::function on_load_callback = {}); td::Result load(td::Slice hash, bool need_data, ExtCellCreator &ext_cell_creator); + static td::Result load(td::Slice hash, td::Slice value, bool need_data, ExtCellCreator &ext_cell_creator); + td::Result load_refcnt(td::Slice hash); // This only loads refcnt_, cell_ == null private: std::shared_ptr reader_; + std::function on_load_callback_; }; class CellStorer { public: CellStorer(KeyValue &kv); td::Status erase(td::Slice hash); - td::Status set(td::int32 refcnt, const DataCell &cell); + td::Status set(td::int32 refcnt, const td::Ref &cell, bool as_boc); + static std::string serialize_value(td::int32 refcnt, const td::Ref &cell, bool as_boc); private: KeyValue &kv_; diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index 5441feea..09303758 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -27,6 +27,9 @@ #include "td/utils/ThreadSafeCounter.h" #include "vm/cellslice.h" +#include +#include "td/actor/actor.h" +#include "common/delay.h" namespace vm { namespace { @@ -60,6 +63,20 @@ struct CellInfo { bool operator<(const CellInfo &other) const { return key() < other.key(); } + + struct Eq { + using is_transparent = void; // Pred to use + bool operator()(const CellInfo &info, const CellInfo &other_info) const { return info.key() == other_info.key();} + bool operator()(const CellInfo &info, td::Slice hash) const { return info.key().as_slice() == hash;} + bool operator()(td::Slice hash, const CellInfo &info) const { return info.key().as_slice() == hash;} + + }; + struct Hash { + using is_transparent = void; // Pred to use + using transparent_key_equal = Eq; + size_t operator()(td::Slice hash) const { return cell_hash_slice_hash(hash); } + size_t operator()(const CellInfo &info) const { return cell_hash_slice_hash(info.key().as_slice());} + }; }; bool operator<(const CellInfo &a, td::Slice b) { @@ -83,19 +100,37 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat return get_cell_info_lazy(level_mask, hash, depth).cell; } td::Result> load_cell(td::Slice hash) override { - TRY_RESULT(loaded_cell, get_cell_info_force(hash).cell->load_cell()); - return std::move(loaded_cell.data_cell); + auto info = hash_table_.get_if_exists(hash); + if (info && info->sync_with_db) { + TRY_RESULT(loaded_cell, info->cell->load_cell()); + return std::move(loaded_cell.data_cell); + } + TRY_RESULT(res, loader_->load(hash, true, *this)); + if (res.status != CellLoader::LoadResult::Ok) { + return td::Status::Error("cell not found"); + } + Ref cell = res.cell(); + hash_table_.apply(hash, [&](CellInfo &info) { update_cell_info_loaded(info, hash, std::move(res)); }); + return cell; + } + td::Result> load_root(td::Slice hash) override { + return load_cell(hash); + } + td::Result> load_root_thread_safe(td::Slice hash) const override { + return td::Status::Error("Not implemented"); } void load_cell_async(td::Slice hash, std::shared_ptr executor, td::Promise> promise) override { + auto promise_ptr = std::make_shared>>(std::move(promise)); auto info = hash_table_.get_if_exists(hash); if (info && info->sync_with_db) { - TRY_RESULT_PROMISE(promise, loaded_cell, info->cell->load_cell()); - promise.set_result(loaded_cell.data_cell); + executor->execute_async([promise = std::move(promise_ptr), cell = info->cell]() mutable { + TRY_RESULT_PROMISE((*promise), loaded_cell, cell->load_cell()); + promise->set_result(loaded_cell.data_cell); + }); return; } SimpleExtCellCreator ext_cell_creator(cell_db_reader_); - auto promise_ptr = std::make_shared>>(std::move(promise)); executor->execute_async( [executor, loader = *loader_, hash = CellHash::from_slice(hash), db = this, ext_cell_creator = std::move(ext_cell_creator), promise = std::move(promise_ptr)]() mutable { @@ -120,9 +155,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat promise->set_result(std::move(cell)); }); } - CellInfo &get_cell_info_force(td::Slice hash) { - return hash_table_.apply(hash, [&](CellInfo &info) { update_cell_info_force(info, hash); }); - } CellInfo &get_cell_info_lazy(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) { return hash_table_.apply(hash.substr(hash.size() - Cell::hash_bytes), [&](CellInfo &info) { update_cell_info_lazy(info, level_mask, hash, depth); }); @@ -138,8 +170,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat if (cell->get_virtualization() != 0) { return; } - //LOG(ERROR) << "INC"; - //CellSlice(cell, nullptr).print_rec(std::cout); to_inc_.push_back(cell); } void dec(const Ref &cell) override { @@ -149,8 +179,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat if (cell->get_virtualization() != 0) { return; } - //LOG(ERROR) << "DEC"; - //CellSlice(cell, nullptr).print_rec(std::cout); to_dec_.push_back(cell); } @@ -164,28 +192,26 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat } td::Status prepare_commit() override { + if (pca_state_) { + return td::Status::Error("prepare_commit_async is not finished"); + } if (is_prepared_for_commit()) { return td::Status::OK(); } - //LOG(ERROR) << "dfs_new_cells_in_db"; for (auto &new_cell : to_inc_) { auto &new_cell_info = get_cell_info(new_cell); dfs_new_cells_in_db(new_cell_info); } - //return td::Status::OK(); - //LOG(ERROR) << "dfs_new_cells"; for (auto &new_cell : to_inc_) { auto &new_cell_info = get_cell_info(new_cell); dfs_new_cells(new_cell_info); } - //LOG(ERROR) << "dfs_old_cells"; for (auto &old_cell : to_dec_) { auto &old_cell_info = get_cell_info(old_cell); dfs_old_cells(old_cell_info); } - //LOG(ERROR) << "save_diff_prepare"; save_diff_prepare(); to_inc_.clear(); @@ -219,6 +245,14 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat return td::Status::OK(); } + void set_celldb_compress_depth(td::uint32 value) override { + celldb_compress_depth_ = value; + } + + vm::ExtCellCreator& as_ext_cell_creator() override { + return *this; + } + private: std::unique_ptr loader_; std::vector> to_inc_; @@ -226,6 +260,7 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat CellHashTable hash_table_; std::vector visited_; Stats stats_diff_; + td::uint32 celldb_compress_depth_{0}; static td::NamedThreadSafeCounter::CounterRef get_thread_safe_counter() { static auto res = td::NamedThreadSafeCounter::get_default().get_counter("DynamicBagOfCellsDb"); @@ -363,7 +398,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat info.was = true; visited_.push_back(&info); } - //LOG(ERROR) << "dfs new " << td::format::escaped(info.cell->hash()); if (info.was_dfs_new_cells) { return; @@ -384,7 +418,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat info.was = true; visited_.push_back(&info); } - //LOG(ERROR) << "dfs old " << td::format::escaped(info.cell->hash()); load_cell(info); @@ -405,7 +438,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat } void save_diff(CellStorer &storer) { - //LOG(ERROR) << hash_table_.size(); for (auto info_ptr : visited_) { save_cell(*info_ptr, storer); } @@ -414,7 +446,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat void save_cell_prepare(CellInfo &info) { if (info.refcnt_diff == 0) { - //CellSlice(info.cell, nullptr).print_rec(std::cout); return; } load_cell(info); @@ -450,17 +481,14 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat if (info.db_refcnt == 0) { CHECK(info.in_db); - //LOG(ERROR) << "ERASE"; - //CellSlice(NoVm(), info.cell).print_rec(std::cout); storer.erase(info.cell->get_hash().as_slice()); info.in_db = false; hash_table_.erase(info.cell->get_hash().as_slice()); guard.dismiss(); } else { - //LOG(ERROR) << "SAVE " << info.db_refcnt; - //CellSlice(NoVm(), info.cell).print_rec(std::cout); auto loaded_cell = info.cell->load_cell().move_as_ok(); - storer.set(info.db_refcnt, *loaded_cell.data_cell); + storer.set(info.db_refcnt, loaded_cell.data_cell, + loaded_cell.data_cell->get_depth() == celldb_compress_depth_ && celldb_compress_depth_ != 0); info.in_db = true; } } @@ -482,7 +510,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat CHECK(cell->is_loaded()); vm::CellSlice cs(vm::NoVm{}, cell); // FIXME for (unsigned i = 0; i < cs.size_refs(); i++) { - //LOG(ERROR) << "---> " << td::format::escaped(cell->ref(i)->hash()); f(get_cell_info(cs.prefetch_ref(i))); } } @@ -573,6 +600,221 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat DynamicBocExtCellExtra{cell_db_reader_})); return std::move(res); } + + struct PrepareCommitAsyncState { + size_t remaining_ = 0; + std::shared_ptr executor_; + td::Promise promise_; + + struct CellInfo2 { + CellInfo *info{}; + std::vector parents; + unsigned remaining_children = 0; + Cell::Hash key() const { + return info->key(); + } + bool operator<(const CellInfo2 &other) const { + return key() < other.key(); + } + + friend bool operator<(const CellInfo2 &a, td::Slice b) { + return a.key().as_slice() < b; + } + + friend bool operator<(td::Slice a, const CellInfo2 &b) { + return a < b.key().as_slice(); + } + + struct Eq { + using is_transparent = void; // Pred to use + bool operator()(const CellInfo2 &info, const CellInfo2 &other_info) const { + return info.key() == other_info.key(); + } + bool operator()(const CellInfo2 &info, td::Slice hash) const { + return info.key().as_slice() == hash; + } + bool operator()(td::Slice hash, const CellInfo2 &info) const { + return info.key().as_slice() == hash; + } + }; + struct Hash { + using is_transparent = void; // Pred to use + using transparent_key_equal = Eq; + size_t operator()(td::Slice hash) const { + return cell_hash_slice_hash(hash); + } + size_t operator()(const CellInfo2 &info) const { + return cell_hash_slice_hash(info.key().as_slice()); + } + }; + }; + + CellHashTable cells_; + + std::queue load_queue_; + td::uint32 active_load_ = 0; + td::uint32 max_parallel_load_ = 4; + }; + std::unique_ptr pca_state_; + + void prepare_commit_async(std::shared_ptr executor, td::Promise promise) override { + hash_table_ = {}; + if (pca_state_) { + promise.set_error(td::Status::Error("Other prepare_commit_async is not finished")); + return; + } + if (is_prepared_for_commit()) { + promise.set_result(td::Unit()); + return; + } + pca_state_ = std::make_unique(); + pca_state_->executor_ = std::move(executor); + pca_state_->promise_ = std::move(promise); + for (auto &new_cell : to_inc_) { + dfs_new_cells_in_db_async(new_cell); + } + pca_state_->cells_.for_each([&](PrepareCommitAsyncState::CellInfo2 &info) { + ++pca_state_->remaining_; + if (info.remaining_children == 0) { + pca_load_from_db(&info); + } + }); + if (pca_state_->remaining_ == 0) { + prepare_commit_async_cont(); + } + } + + void dfs_new_cells_in_db_async(const td::Ref &cell, PrepareCommitAsyncState::CellInfo2 *parent = nullptr) { + bool exists = true; + pca_state_->cells_.apply(cell->get_hash().as_slice(), [&](PrepareCommitAsyncState::CellInfo2 &info) { + if (info.info == nullptr) { + exists = false; + info.info = &get_cell_info(cell); + } + }); + auto info = pca_state_->cells_.get_if_exists(cell->get_hash().as_slice()); + if (parent) { + info->parents.push_back(parent); + ++parent->remaining_children; + } + if (exists) { + return; + } + if (cell->is_loaded()) { + vm::CellSlice cs(vm::NoVm{}, cell); + for (unsigned i = 0; i < cs.size_refs(); i++) { + dfs_new_cells_in_db_async(cs.prefetch_ref(i), info); + } + } + } + + void pca_load_from_db(PrepareCommitAsyncState::CellInfo2 *info) { + if (pca_state_->active_load_ >= pca_state_->max_parallel_load_) { + pca_state_->load_queue_.push(info); + return; + } + ++pca_state_->active_load_; + pca_state_->executor_->execute_async( + [db = this, info, executor = pca_state_->executor_, loader = *loader_]() mutable { + auto res = loader.load_refcnt(info->info->cell->get_hash().as_slice()).move_as_ok(); + executor->execute_sync([db, info, res = std::move(res)]() { + --db->pca_state_->active_load_; + db->pca_process_load_queue(); + db->pca_set_in_db(info, std::move(res)); + }); + }); + } + + void pca_process_load_queue() { + while (pca_state_->active_load_ < pca_state_->max_parallel_load_ && !pca_state_->load_queue_.empty()) { + PrepareCommitAsyncState::CellInfo2 *info = pca_state_->load_queue_.front(); + pca_state_->load_queue_.pop(); + pca_load_from_db(info); + } + } + + void pca_set_in_db(PrepareCommitAsyncState::CellInfo2 *info, CellLoader::LoadResult result) { + info->info->sync_with_db = true; + if (result.status == CellLoader::LoadResult::Ok) { + info->info->in_db = true; + info->info->db_refcnt = result.refcnt(); + } else { + info->info->in_db = false; + } + for (PrepareCommitAsyncState::CellInfo2 *parent_info : info->parents) { + if (parent_info->info->sync_with_db) { + continue; + } + if (!info->info->in_db) { + pca_set_in_db(parent_info, {}); + } else if (--parent_info->remaining_children == 0) { + pca_load_from_db(parent_info); + } + } + CHECK(pca_state_->remaining_ != 0); + if (--pca_state_->remaining_ == 0) { + prepare_commit_async_cont(); + } + } + + void prepare_commit_async_cont() { + for (auto &new_cell : to_inc_) { + auto &new_cell_info = get_cell_info(new_cell); + dfs_new_cells(new_cell_info); + } + + CHECK(pca_state_->remaining_ == 0); + for (auto &old_cell : to_dec_) { + auto &old_cell_info = get_cell_info(old_cell); + dfs_old_cells_async(old_cell_info); + } + if (pca_state_->remaining_ == 0) { + prepare_commit_async_cont2(); + } + } + + void dfs_old_cells_async(CellInfo &info) { + if (!info.was) { + info.was = true; + visited_.push_back(&info); + if (!info.sync_with_db) { + ++pca_state_->remaining_; + load_cell_async( + info.cell->get_hash().as_slice(), pca_state_->executor_, + [executor = pca_state_->executor_, db = this, info = &info](td::Result> R) { + R.ensure(); + executor->execute_sync([db, info]() { + CHECK(info->sync_with_db); + db->dfs_old_cells_async(*info); + if (--db->pca_state_->remaining_ == 0) { + db->prepare_commit_async_cont2(); + } + }); + }); + return; + } + } + info.refcnt_diff--; + if (!info.sync_with_db) { + return; + } + auto new_refcnt = info.refcnt_diff + info.db_refcnt; + CHECK(new_refcnt >= 0); + if (new_refcnt != 0) { + return; + } + + for_each(info, [this](auto &child_info) { dfs_old_cells_async(child_info); }); + } + + void prepare_commit_async_cont2() { + save_diff_prepare(); + to_inc_.clear(); + to_dec_.clear(); + pca_state_->promise_.set_result(td::Unit()); + pca_state_ = {}; + } + }; } // namespace diff --git a/crypto/vm/db/DynamicBagOfCellsDb.h b/crypto/vm/db/DynamicBagOfCellsDb.h index 3569208c..62864ad9 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.h +++ b/crypto/vm/db/DynamicBagOfCellsDb.h @@ -23,6 +23,11 @@ #include "td/utils/Status.h" #include "td/actor/PromiseFuture.h" +#include + +namespace td { +class KeyValueReader; +} namespace vm { class CellLoader; class CellStorer; @@ -45,12 +50,20 @@ class DynamicBagOfCellsDb { public: virtual ~DynamicBagOfCellsDb() = default; virtual td::Result> load_cell(td::Slice hash) = 0; + virtual td::Result> load_root(td::Slice hash) = 0; + virtual td::Result> load_root_thread_safe(td::Slice hash) const = 0; struct Stats { + td::int64 roots_total_count{0}; td::int64 cells_total_count{0}; td::int64 cells_total_size{0}; - void apply_diff(Stats diff) { + std::vector> custom_stats; + void apply_diff(const Stats &diff) { + roots_total_count += diff.roots_total_count; cells_total_count += diff.cells_total_count; cells_total_size += diff.cells_total_size; + CHECK(roots_total_count >= 0); + CHECK(cells_total_count >= 0); + CHECK(cells_total_size >= 0); } }; virtual void inc(const Ref &old_root) = 0; @@ -58,23 +71,41 @@ class DynamicBagOfCellsDb { virtual td::Status prepare_commit() = 0; virtual Stats get_stats_diff() = 0; + virtual td::Result get_stats() { + return td::Status::Error("Not implemented"); + } virtual td::Status commit(CellStorer &) = 0; virtual std::shared_ptr get_cell_db_reader() = 0; // restart with new loader will also reset stats_diff virtual td::Status set_loader(std::unique_ptr loader) = 0; + virtual void set_celldb_compress_depth(td::uint32 value) = 0; + virtual vm::ExtCellCreator &as_ext_cell_creator() = 0; + static std::unique_ptr create(); + struct CreateInMemoryOptions { + size_t extra_threads{std::thread::hardware_concurrency()}; + bool verbose{true}; + // Allocated DataCels will never be deleted + bool use_arena{false}; + // Almost no overhead in memory during creation, but will scan database twice + bool use_less_memory_during_creation{true}; + }; + static std::unique_ptr create_in_memory(td::KeyValueReader *kv, CreateInMemoryOptions options); + class AsyncExecutor { public: - virtual ~AsyncExecutor() {} + virtual ~AsyncExecutor() { + } virtual void execute_async(std::function f) = 0; virtual void execute_sync(std::function f) = 0; }; virtual void load_cell_async(td::Slice hash, std::shared_ptr executor, td::Promise> promise) = 0; + virtual void prepare_commit_async(std::shared_ptr executor, td::Promise promise) = 0; }; } // namespace vm diff --git a/crypto/vm/db/InMemoryBagOfCellsDb.cpp b/crypto/vm/db/InMemoryBagOfCellsDb.cpp new file mode 100644 index 00000000..03cad093 --- /dev/null +++ b/crypto/vm/db/InMemoryBagOfCellsDb.cpp @@ -0,0 +1,988 @@ +#include "CellStorage.h" +#include "DynamicBagOfCellsDb.h" +#include "td/utils/Timer.h" +#include "td/utils/base64.h" +#include "td/utils/format.h" +#include "td/utils/int_types.h" +#include "td/utils/misc.h" +#include "td/utils/port/Stat.h" +#include "vm/cells/CellHash.h" +#include "vm/cells/CellSlice.h" +#include "vm/cells/DataCell.h" +#include "vm/cells/ExtCell.h" + +#include "td/utils/HashMap.h" +#include "td/utils/HashSet.h" + +#include + +#if TD_PORT_POSIX +#include +#include +#endif + +namespace vm { +namespace { +constexpr bool use_dense_hash_map = true; + +template +void parallel_run(size_t n, F &&run_task, size_t extra_threads_n) { + std::atomic next_task_id{0}; + auto loop = [&] { + while (true) { + auto task_id = next_task_id++; + if (task_id >= n) { + break; + } + run_task(task_id); + } + }; + + // NB: it could be important that td::thread is used, not std::thread + std::vector threads; + for (size_t i = 0; i < extra_threads_n; i++) { + threads.emplace_back(loop); + } + loop(); + for (auto &thread : threads) { + thread.join(); + } + threads.clear(); +} + +struct UniqueAccess { + struct Release { + void operator()(UniqueAccess *access) const { + if (access) { + access->release(); + } + } + }; + using Lock = std::unique_ptr; + Lock lock() { + CHECK(!locked_.exchange(true)); + return Lock(this); + } + + private: + std::atomic locked_{false}; + void release() { + locked_ = false; + } +}; +class DefaultPrunnedCellCreator : public ExtCellCreator { + public: + td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { + TRY_RESULT(cell, PrunnedCell::create(PrunnedCellInfo{level_mask, hash, depth}, td::Unit{})); + return cell; + } +}; + +class ArenaPrunnedCellCreator : public ExtCellCreator { + struct ArenaAllocator { + ArenaAllocator() { + // only one instance ever + static UniqueAccess unique_access; + [[maybe_unused]] auto ptr = unique_access.lock().release(); + } + std::mutex mutex; + struct Deleter { + static constexpr size_t batch_size = 1 << 24; +#if TD_PORT_POSIX + static std::unique_ptr alloc() { + char *ptr = reinterpret_cast( + mmap(NULL, batch_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); + CHECK(ptr != nullptr); + return std::unique_ptr(ptr); + } + void operator()(char *ptr) const { + munmap(ptr, batch_size); + } +#else + static std::unique_ptr alloc() { + auto ptr = reinterpret_cast(malloc(batch_size)); + CHECK(ptr != nullptr); + return std::unique_ptr(ptr); + } + void operator()(char *ptr) const { + free(ptr); + } +#endif + }; + std::vector> arena; + td::uint64 arena_generation{0}; + + td::MutableSlice alloc_batch() { + auto batch = Deleter::alloc(); + auto res = td::MutableSlice(batch.get(), Deleter::batch_size); + std::lock_guard guard(mutex); + arena.emplace_back(std::move(batch)); + return res; + } + + char *alloc(size_t size) { + thread_local td::MutableSlice batch; + thread_local td::uint64 batch_generation{0}; + auto aligned_size = (size + 7) / 8 * 8; + if (batch.size() < size || batch_generation != arena_generation) { + batch = alloc_batch(); + batch_generation = arena_generation; + } + auto res = batch.begin(); + batch.remove_prefix(aligned_size); + return res; + } + void clear() { + std::lock_guard guard(mutex); + arena_generation++; + td::reset_to_empty(arena); + } + }; + static ArenaAllocator arena_; + static td::ThreadSafeCounter cells_count_; + + public: + struct Counter { + Counter() { + cells_count_.add(1); + } + Counter(Counter &&other) { + cells_count_.add(1); + } + Counter(const Counter &other) { + cells_count_.add(1); + } + ~Counter() { + cells_count_.add(-1); + } + }; + + struct Allocator { + template + std::unique_ptr> make_unique(ArgsT &&...args) { + auto *ptr = arena_.alloc(sizeof(T)); + T *obj = new (ptr) T(std::forward(args)...); + return std::unique_ptr(obj); + } + }; + td::Result> ext_cell(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) override { + Allocator allocator; + TRY_RESULT(cell, PrunnedCell::create(allocator, PrunnedCellInfo{level_mask, hash, depth}, Counter())); + return cell; + } + static td::int64 count() { + return cells_count_.sum(); + } + static void clear_arena() { + LOG_CHECK(cells_count_.sum() == 0) << cells_count_.sum(); + arena_.clear(); + } +}; +td::ThreadSafeCounter ArenaPrunnedCellCreator::cells_count_; +ArenaPrunnedCellCreator::ArenaAllocator ArenaPrunnedCellCreator::arena_; + +struct CellInfo { + mutable td::int32 db_refcnt{0}; + Ref cell; +}; +static_assert(sizeof(CellInfo) == 16); + +CellHash as_cell_hash(const CellInfo &info) { + return info.cell->get_hash(); +} + +struct CellInfoHashTableBaseline { + td::HashSet ht_; + const CellInfo *find(CellHash hash) const { + if (auto it = ht_.find(hash); it != ht_.end()) { + return &*it; + } + return nullptr; + } + void erase(CellHash hash) { + auto it = ht_.find(hash); + CHECK(it != ht_.end()); + ht_.erase(it); + } + void insert(CellInfo info) { + ht_.insert(std::move(info)); + } + template + void init_from(Iterator begin, Iterator end) { + ht_ = td::HashSet(begin, end); + } + size_t size() const { + return ht_.size(); + } + auto begin() const { + return ht_.begin(); + } + auto end() const { + return ht_.end(); + } + size_t bucket_count() const { + return ht_.bucket_count(); + } + template + auto for_each(F &&f) { + for (auto &it : ht_) { + f(it); + } + } +}; + +struct CellInfoHashTableDense { + size_t dense_ht_size_{0}; + size_t dense_ht_buckets_{1}; + std::vector dense_ht_offsets_{1}; + std::vector dense_ht_values_; + td::HashSet new_ht_; + size_t dense_choose_bucket(const CellHash &hash) const { + return cell_hash_slice_hash(hash.as_slice()) % dense_ht_buckets_; + } + const CellInfo *dense_find(CellHash hash) const { + auto bucket_i = dense_choose_bucket(hash); + auto begin = dense_ht_values_.begin() + dense_ht_offsets_[bucket_i]; + auto end = dense_ht_values_.begin() + dense_ht_offsets_[bucket_i + 1]; + for (auto it = begin; it != end; ++it) { + if (it->cell.not_null() && it->cell->get_hash() == hash) { + return &*it; + } + } + return nullptr; + } + CellInfo *dense_find_empty(CellHash hash) { + auto bucket_i = dense_choose_bucket(hash); + auto begin = dense_ht_values_.begin() + dense_ht_offsets_[bucket_i]; + auto end = dense_ht_values_.begin() + dense_ht_offsets_[bucket_i + 1]; + for (auto it = begin; it != end; ++it) { + if (it->cell.is_null()) { + return &*it; + } + } + return nullptr; + } + const CellInfo *find(CellHash hash) const { + if (auto it = new_ht_.find(hash); it != new_ht_.end()) { + return &*it; + } + if (auto it = dense_find(hash)) { + return it; + } + return nullptr; + } + void erase(CellHash hash) { + if (auto it = new_ht_.find(hash); it != new_ht_.end()) { + new_ht_.erase(it); + return; + } + auto info = dense_find(hash); + CHECK(info && info->db_refcnt > 0); + info->db_refcnt = 0; + const_cast(info)->cell = {}; + CHECK(dense_ht_size_ > 0); + dense_ht_size_--; + } + + void insert(CellInfo info) { + if (auto dest = dense_find_empty(info.cell->get_hash())) { + *dest = std::move(info); + dense_ht_size_++; + return; + } + new_ht_.insert(std::move(info)); + } + template + void init_from(Iterator begin, Iterator end) { + auto size = td::narrow_cast(std::distance(begin, end)); + dense_ht_buckets_ = std::max(size_t(1), size_t(size / 8)); + + std::vector offsets(dense_ht_buckets_ + 2); + for (auto it = begin; it != end; ++it) { + auto bucket_i = dense_choose_bucket(it->cell->get_hash()); + offsets[bucket_i + 2]++; + } + for (size_t i = 1; i < offsets.size(); i++) { + offsets[i] += offsets[i - 1]; + } + dense_ht_values_.resize(size); + for (auto it = begin; it != end; ++it) { + auto bucket_i = dense_choose_bucket(it->cell->get_hash()); + dense_ht_values_[offsets[bucket_i + 1]++] = std::move(*it); + } + CHECK(offsets[0] == 0); + CHECK(offsets[offsets.size() - 1] == size); + CHECK(offsets[offsets.size() - 2] == size); + dense_ht_offsets_ = std::move(offsets); + dense_ht_size_ = size; + } + size_t size() const { + return dense_ht_size_ + new_ht_.size(); + } + template + auto for_each(F &&f) { + for (auto &it : dense_ht_values_) { + if (it.cell.not_null()) { + f(it); + } + } + for (auto &it : new_ht_) { + f(it); + } + } + size_t bucket_count() const { + return new_ht_.bucket_count() + dense_ht_values_.size(); + } +}; + +using CellInfoHashTable = std::conditional_t; + +class CellStorage { + struct PrivateTag {}; + struct CellBucket; + struct None { + void operator()(CellBucket *bucket) { + } + }; + struct CellBucketRef { + UniqueAccess::Lock lock; + std::unique_ptr bucket; + CellBucket &operator*() { + return *bucket; + } + CellBucket *operator->() { + return bucket.get(); + } + }; + struct CellBucket { + mutable UniqueAccess access_; + CellInfoHashTable infos_; + std::vector cells_; + std::vector> roots_; + size_t boc_count_{0}; + [[maybe_unused]] char pad3[TD_CONCURRENCY_PAD]; + + void clear() { + td::reset_to_empty(infos_); + td::reset_to_empty(cells_); + td::reset_to_empty(roots_); + } + + CellBucketRef unique_access() const { + auto lock = access_.lock(); + return CellBucketRef{.lock = std::move(lock), + .bucket = std::unique_ptr(const_cast(this))}; + } + }; + std::array buckets_{}; + bool inited_{false}; + + const CellBucket &get_bucket(size_t i) const { + return buckets_.at(i); + } + const CellBucket &get_bucket(const CellHash &hash) const { + return get_bucket(hash.as_array()[0]); + } + + mutable UniqueAccess local_access_; + td::HashSet, CellHashF, CellEqF> local_roots_; + DynamicBagOfCellsDb::Stats stats_; + + mutable std::mutex root_mutex_; + td::HashSet, CellHashF, CellEqF> roots_; + + public: + std::optional get_info(const CellHash &hash) const { + auto lock = local_access_.lock(); + auto &bucket = get_bucket(hash); + if (auto info_ptr = bucket.infos_.find(hash)) { + return *info_ptr; + } + return {}; + } + + DynamicBagOfCellsDb::Stats get_stats() { + auto unique_access = local_access_.lock(); + auto stats = stats_; + auto add_stat = [&stats](auto key, auto value) { + stats.custom_stats.emplace_back(std::move(key), PSTRING() << value); + }; + if constexpr (use_dense_hash_map) { + size_t dense_ht_capacity = 0; + size_t new_ht_capacity = 0; + size_t dense_ht_size = 0; + size_t new_ht_size = 0; + for_each_bucket(0, [&](auto bucket_id, CellBucket &bucket) { + dense_ht_capacity += bucket.infos_.dense_ht_values_.size(); + dense_ht_size += bucket.infos_.dense_ht_size_; + new_ht_capacity += bucket.infos_.new_ht_.bucket_count(); + new_ht_size += bucket.infos_.new_ht_.size(); + }); + auto size = new_ht_size + dense_ht_size; + auto capacity = new_ht_capacity + dense_ht_capacity; + add_stat("ht.capacity", capacity); + add_stat("ht.size", size); + add_stat("ht.load", double(size) / std::max(1.0, double(capacity))); + add_stat("ht.dense_ht_capacity", dense_ht_capacity); + add_stat("ht.dense_ht_size", dense_ht_size); + add_stat("ht.dense_ht_load", double(dense_ht_size) / std::max(1.0, double(dense_ht_capacity))); + add_stat("ht.new_ht_capacity", new_ht_capacity); + add_stat("ht.new_ht_size", new_ht_size); + add_stat("ht.new_ht_load", double(new_ht_size) / std::max(1.0, double(new_ht_capacity))); + } else { + size_t capacity = 0; + size_t size = 0; + for_each_bucket(0, [&](auto bucket_id, CellBucket &bucket) { + capacity += bucket.infos_.bucket_count(); + size += bucket.infos_.size(); + }); + add_stat("ht.capacity", capacity); + add_stat("ht.size", size); + add_stat("ht.load", double(size) / std::max(1.0, double(capacity))); + } + CHECK(td::narrow_cast(stats.roots_total_count) == local_roots_.size()); + return stats; + } + void apply_stats_diff(DynamicBagOfCellsDb::Stats diff) { + auto unique_access = local_access_.lock(); + stats_.apply_diff(diff); + CHECK(td::narrow_cast(stats_.roots_total_count) == local_roots_.size()); + size_t cells_count{0}; + for_each_bucket(0, [&](size_t bucket_id, auto &bucket) { cells_count += bucket.infos_.size(); }); + CHECK(td::narrow_cast(stats_.cells_total_count) == cells_count); + } + + td::Result> load_cell(const CellHash &hash) const { + auto lock = local_access_.lock(); + auto &bucket = get_bucket(hash); + if (auto info_ptr = bucket.infos_.find(hash)) { + return info_ptr->cell; + } + return td::Status::Error("not found"); + } + + td::Result> load_root_local(const CellHash &hash) const { + auto lock = local_access_.lock(); + if (auto it = local_roots_.find(hash); it != local_roots_.end()) { + return *it; + } + return td::Status::Error("not found"); + } + td::Result> load_root_shared(const CellHash &hash) const { + std::lock_guard lock(root_mutex_); + if (auto it = roots_.find(hash); it != roots_.end()) { + return *it; + } + return td::Status::Error("not found"); + } + + void erase(const CellHash &hash) { + auto lock = local_access_.lock(); + auto bucket = get_bucket(hash).unique_access(); + bucket->infos_.erase(hash); + if (auto local_it = local_roots_.find(hash); local_it != local_roots_.end()) { + local_roots_.erase(local_it); + std::lock_guard root_lock(root_mutex_); + auto shared_it = roots_.find(hash); + CHECK(shared_it != roots_.end()); + roots_.erase(shared_it); + CHECK(stats_.roots_total_count > 0); + stats_.roots_total_count--; + } + } + + void add_new_root(Ref cell) { + auto lock = local_access_.lock(); + if (local_roots_.insert(cell).second) { + std::lock_guard lock(root_mutex_); + roots_.insert(std::move(cell)); + stats_.roots_total_count++; + } + } + + void set(td::int32 refcnt, Ref cell) { + auto lock = local_access_.lock(); + //LOG(ERROR) << "setting refcnt to " << refcnt << ", cell " << td::base64_encode(cell->get_hash().as_slice()); + auto hash = cell->get_hash(); + auto bucket = get_bucket(hash).unique_access(); + if (auto info_ptr = bucket->infos_.find(hash)) { + CHECK(info_ptr->cell.get() == cell.get()); + info_ptr->db_refcnt = refcnt; + } else { + bucket->infos_.insert({.db_refcnt = refcnt, .cell = std::move(cell)}); + } + } + + template + static td::unique_ptr build(DynamicBagOfCellsDb::CreateInMemoryOptions options, + F &¶llel_scan_cells) { + auto storage = td::make_unique(PrivateTag{}); + storage->do_build(options, parallel_scan_cells); + return storage; + } + + ~CellStorage() { + clear(); + } + CellStorage() = delete; + explicit CellStorage(PrivateTag) { + } + + private: + template + void do_build(DynamicBagOfCellsDb::CreateInMemoryOptions options, F &¶llel_scan_cells) { + auto verbose = options.verbose; + td::Slice P = "loading in-memory cell database: "; + LOG_IF(WARNING, verbose) << P << "start with options use_arena=" << options.use_arena + << " use_less_memory_during_creation=" << options.use_less_memory_during_creation + << " use_dense_hash_map=" << use_dense_hash_map; + auto full_timer = td::Timer(); + auto lock = local_access_.lock(); + CHECK(ArenaPrunnedCellCreator::count() == 0); + ArenaPrunnedCellCreator arena_pc_creator; + DefaultPrunnedCellCreator default_pc_creator; + + auto timer = td::Timer(); + td::int64 cell_count{0}; + td::int64 desc_count{0}; + if (options.use_less_memory_during_creation) { + auto [new_cell_count, new_desc_count] = parallel_scan_cells( + default_pc_creator, options.use_arena, + [&](td::int32 refcnt, Ref cell) { initial_set_without_refs(refcnt, std::move(cell)); }); + cell_count = new_cell_count; + desc_count = new_desc_count; + } else { + auto [new_cell_count, new_desc_count] = + parallel_scan_cells(arena_pc_creator, options.use_arena, + [&](td::int32 refcnt, Ref cell) { initial_set(refcnt, std::move(cell)); }); + cell_count = new_cell_count; + desc_count = new_desc_count; + } + LOG_IF(WARNING, verbose) << P << "cells loaded in " << timer.elapsed() << "s, cells_count= " << cell_count + << " prunned_cells_count=" << ArenaPrunnedCellCreator::count(); + + timer = td::Timer(); + for_each_bucket(options.extra_threads, [&](size_t bucket_id, auto &bucket) { build_hashtable(bucket); }); + + size_t ht_capacity = 0; + size_t ht_size = 0; + for_each_bucket(0, [&](size_t bucket_id, auto &bucket) { + ht_size += bucket.infos_.size(); + ht_capacity += bucket.infos_.bucket_count(); + }); + double load_factor = double(ht_size) / std::max(double(ht_capacity), 1.0); + LOG_IF(WARNING, verbose) << P << "hashtable created in " << timer.elapsed() + << "s, hashtables_expected_size=" << td::format::as_size(ht_capacity * sizeof(CellInfo)) + << " load_factor=" << load_factor; + + timer = td::Timer(); + if (options.use_less_memory_during_creation) { + auto [new_cell_count, new_desc_count] = + parallel_scan_cells(default_pc_creator, false, + [&](td::int32 refcnt, Ref cell) { secondary_set(refcnt, std::move(cell)); }); + CHECK(new_cell_count == cell_count); + CHECK(new_desc_count == desc_count); + } else { + for_each_bucket(options.extra_threads, [&](size_t bucket_id, auto &bucket) { reset_refs(bucket); }); + } + LOG_IF(WARNING, verbose) << P << "refs rearranged in " << timer.elapsed() << "s"; + + timer = td::Timer(); + using Stats = DynamicBagOfCellsDb::Stats; + std::vector bucket_stats(buckets_.size()); + std::atomic boc_count{0}; + for_each_bucket(options.extra_threads, [&](size_t bucket_id, auto &bucket) { + bucket_stats[bucket_id] = validate_bucket_a(bucket, options.use_arena); + boc_count += bucket.boc_count_; + }); + for_each_bucket(options.extra_threads, [&](size_t bucket_id, auto &bucket) { validate_bucket_b(bucket); }); + stats_ = {}; + for (auto &bucket_stat : bucket_stats) { + stats_.apply_diff(bucket_stat); + } + LOG_IF(WARNING, verbose) << P << "refcnt validated in " << timer.elapsed() << "s"; + + timer = td::Timer(); + build_roots(); + LOG_IF(WARNING, verbose) << P << "roots hashtable built in " << timer.elapsed() << "s"; + ArenaPrunnedCellCreator::clear_arena(); + LOG_IF(WARNING, verbose) << P << "arena cleared in " << timer.elapsed(); + + lock.reset(); + auto r_mem_stat = td::mem_stat(); + td::MemStat mem_stat; + if (r_mem_stat.is_ok()) { + mem_stat = r_mem_stat.move_as_ok(); + } + auto stats = get_stats(); + td::StringBuilder sb; + for (auto &[key, value] : stats.custom_stats) { + sb << "\n\t" << key << "=" << value; + } + LOG_IF(ERROR, desc_count != 0 && desc_count != stats.roots_total_count + 1) + << "desc<> keys count is " << desc_count << " wich is different from roots count " << stats.roots_total_count; + LOG_IF(WARNING, verbose) + << P << "done in " << full_timer.elapsed() << "\n\troots_count=" << stats.roots_total_count << "\n\t" + << desc_count << "\n\tcells_count=" << stats.cells_total_count + << "\n\tcells_size=" << td::format::as_size(stats.cells_total_size) << "\n\tboc_count=" << boc_count.load() + << sb.as_cslice() << "\n\tdata_cells_size=" << td::format::as_size(sizeof(DataCell) * stats.cells_total_count) + << "\n\tdata_cell_size=" << sizeof(DataCell) << "\n\texpected_memory_used=" + << td::format::as_size(stats.cells_total_count * (sizeof(DataCell) + sizeof(CellInfo) * 3 / 2) + + stats.cells_total_size) + << "\n\tbest_possible_memory_used" + << td::format::as_size(stats.cells_total_count * (sizeof(DataCell) + sizeof(CellInfo)) + stats.cells_total_size) + << "\n\tmemory_used=" << td::format::as_size(mem_stat.resident_size_) + << "\n\tpeak_memory_used=" << td::format::as_size(mem_stat.resident_size_peak_); + + inited_ = true; + } + + template + void for_each_bucket(size_t extra_threads, F &&f) { + parallel_run( + buckets_.size(), [&](auto task_id) { f(task_id, *get_bucket(task_id).unique_access()); }, extra_threads); + } + + void clear() { + auto unique_access = local_access_.lock(); + for_each_bucket(td::thread::hardware_concurrency(), [&](size_t bucket_id, auto &bucket) { bucket.clear(); }); + local_roots_.clear(); + { + auto lock = std::lock_guard(root_mutex_); + roots_.clear(); + } + } + + void initial_set(td::int32 refcnt, Ref cell) { + CHECK(!inited_); + auto bucket = get_bucket(cell->get_hash()).unique_access(); + bucket->cells_.push_back({.db_refcnt = refcnt, .cell = std::move(cell)}); + } + + void initial_set_without_refs(td::int32 refcnt, Ref cell_ref) { + CHECK(!inited_); + auto bucket = get_bucket(cell_ref->get_hash()).unique_access(); + auto &cell = const_cast(*cell_ref); + for (unsigned i = 0; i < cell.size_refs(); i++) { + auto to_destroy = cell.reset_ref_unsafe(i, Ref(), false); + if (to_destroy->is_loaded()) { + bucket->boc_count_++; + } + } + bucket->cells_.push_back({.db_refcnt = refcnt, .cell = std::move(cell_ref)}); + } + + void secondary_set(td::int32 refcnt, Ref cell_copy) { + CHECK(!inited_); + auto bucket = get_bucket(cell_copy->get_hash()).unique_access(); + auto info = bucket->infos_.find(cell_copy->get_hash()); + CHECK(info); + CellSlice cs(NoVm{}, std::move(cell_copy)); + auto &cell = const_cast(*info->cell); + CHECK(cs.size_refs() == cell.size_refs()); + for (unsigned i = 0; i < cell.size_refs(); i++) { + auto prunned_cell_hash = cs.fetch_ref()->get_hash(); + auto &prunned_cell_bucket = get_bucket(prunned_cell_hash); + auto full_cell_ptr = prunned_cell_bucket.infos_.find(prunned_cell_hash); + CHECK(full_cell_ptr); + auto full_cell = full_cell_ptr->cell; + auto to_destroy = cell.reset_ref_unsafe(i, std::move(full_cell), false); + CHECK(to_destroy.is_null()); + } + } + + void build_hashtable(CellBucket &bucket) { + bucket.infos_.init_from(bucket.cells_.begin(), bucket.cells_.end()); + LOG_CHECK(bucket.infos_.size() == bucket.cells_.size()) << bucket.infos_.size() << " vs " << bucket.cells_.size(); + td::reset_to_empty(bucket.cells_); + LOG_CHECK(bucket.cells_.capacity() == 0) << bucket.cells_.capacity(); + } + + void reset_refs(CellBucket &bucket) { + bucket.infos_.for_each([&](auto &it) { + // This is generally very dangerous, but should be safe here + auto &cell = const_cast(*it.cell); + for (unsigned i = 0; i < cell.size_refs(); i++) { + auto prunned_cell = cell.get_ref_raw_ptr(i); + auto prunned_cell_hash = prunned_cell->get_hash(); + auto &prunned_cell_bucket = get_bucket(prunned_cell_hash); + auto full_cell_ptr = prunned_cell_bucket.infos_.find(prunned_cell_hash); + CHECK(full_cell_ptr); + auto full_cell = full_cell_ptr->cell; + auto to_destroy = cell.reset_ref_unsafe(i, std::move(full_cell)); + if (!to_destroy->is_loaded()) { + Ref> x(std::move(to_destroy)); + x->~PrunnedCell(); + x.release(); + } else { + bucket.boc_count_++; + } + } + }); + } + + DynamicBagOfCellsDb::Stats validate_bucket_a(CellBucket &bucket, bool use_arena) { + DynamicBagOfCellsDb::Stats stats; + bucket.infos_.for_each([&](auto &it) { + int cell_ref_cnt = it.cell->get_refcnt(); + CHECK(it.db_refcnt + 1 + use_arena >= cell_ref_cnt); + auto extra_refcnt = it.db_refcnt + 1 + use_arena - cell_ref_cnt; + if (extra_refcnt != 0) { + bucket.roots_.push_back(it.cell); + stats.roots_total_count++; + } + stats.cells_total_count++; + stats.cells_total_size += static_cast(it.cell->get_storage_size()); + }); + return stats; + } + void validate_bucket_b(CellBucket &bucket) { + // sanity check + bucket.infos_.for_each([&](auto &it) { + CellSlice cs(NoVm{}, it.cell); + while (cs.have_refs()) { + CHECK(cs.fetch_ref().not_null()); + } + }); + } + void build_roots() { + for (auto &it : buckets_) { + for (auto &root : it.roots_) { + local_roots_.insert(std::move(root)); + } + td::reset_to_empty(it.roots_); + } + auto lock = std::lock_guard(root_mutex_); + roots_ = local_roots_; + } +}; + +class InMemoryBagOfCellsDb : public DynamicBagOfCellsDb { + public: + explicit InMemoryBagOfCellsDb(td::unique_ptr storage) : storage_(std::move(storage)) { + } + + td::Result> load_cell(td::Slice hash) override { + return storage_->load_cell(CellHash::from_slice(hash)); + } + + td::Result> load_root(td::Slice hash) override { + return storage_->load_root_local(CellHash::from_slice(hash)); + } + td::Result> load_root_thread_safe(td::Slice hash) const override { + return storage_->load_root_shared(CellHash::from_slice(hash)); + } + + void inc(const Ref &cell) override { + if (cell.is_null()) { + return; + } + if (cell->get_virtualization() != 0) { + return; + } + to_inc_.push_back(cell); + } + + void dec(const Ref &cell) override { + if (cell.is_null()) { + return; + } + if (cell->get_virtualization() != 0) { + return; + } + to_dec_.push_back(cell); + } + + td::Status commit(CellStorer &cell_storer) override { + if (!to_inc_.empty() || !to_dec_.empty()) { + TRY_STATUS(prepare_commit()); + } + + Stats diff; + CHECK(to_dec_.empty()); + for (auto &it : info_) { + auto &info = it.second; + if (info.diff_refcnt == 0) { + continue; + } + auto refcnt = td::narrow_cast(static_cast(info.db_refcnt) + info.diff_refcnt); + CHECK(refcnt >= 0); + if (refcnt > 0) { + cell_storer.set(refcnt, info.cell, false); + storage_->set(refcnt, info.cell); + if (info.db_refcnt == 0) { + diff.cells_total_count++; + diff.cells_total_size += static_cast(info.cell->get_storage_size()); + } + } else { + cell_storer.erase(info.cell->get_hash().as_slice()); + storage_->erase(info.cell->get_hash()); + diff.cells_total_count--; + diff.cells_total_size -= static_cast(info.cell->get_storage_size()); + } + } + storage_->apply_stats_diff(diff); + info_ = {}; + return td::Status::OK(); + } + + td::Result get_stats() override { + return storage_->get_stats(); + } + + // Not implemented or trivial or deprecated methods + td::Status set_loader(std::unique_ptr loader) override { + return td::Status::OK(); + } + + td::Status prepare_commit() override { + CHECK(info_.empty()); + for (auto &to_inc : to_inc_) { + auto new_root = do_inc(to_inc); + storage_->add_new_root(std::move(new_root)); + } + for (auto &to_dec : to_dec_) { + do_dec(to_dec); + } + to_dec_ = {}; + to_inc_ = {}; + return td::Status::OK(); + } + void prepare_commit_async(std::shared_ptr executor, td::Promise promise) override { + TRY_STATUS_PROMISE(promise, prepare_commit()); + promise.set_value(td::Unit()); + } + Stats get_stats_diff() override { + LOG(FATAL) << "Not implemented"; + return {}; + } + std::shared_ptr get_cell_db_reader() override { + return {}; + } + void set_celldb_compress_depth(td::uint32 value) override { + LOG(FATAL) << "Not implemented"; + } + ExtCellCreator &as_ext_cell_creator() override { + UNREACHABLE(); + } + void load_cell_async(td::Slice hash, std::shared_ptr executor, + td::Promise> promise) override { + LOG(FATAL) << "Not implemented"; + } + + private: + td::unique_ptr storage_; + + struct Info { + td::int32 db_refcnt{0}; + td::int32 diff_refcnt{0}; + Ref cell; + }; + td::HashMap info_; + + std::unique_ptr loader_; + std::vector> to_inc_; + std::vector> to_dec_; + + Ref do_inc(Ref cell) { + auto cell_hash = cell->get_hash(); + if (auto it = info_.find(cell_hash); it != info_.end()) { + CHECK(it->second.diff_refcnt != std::numeric_limits::max()); + it->second.diff_refcnt++; + return it->second.cell; + } + if (auto o_info = storage_->get_info(cell_hash)) { + info_.emplace(cell_hash, Info{.db_refcnt = o_info->db_refcnt, .diff_refcnt = 1, .cell = o_info->cell}); + return std::move(o_info->cell); + } + + CellSlice cs(NoVm{}, std::move(cell)); + CellBuilder cb; + cb.store_bits(cs.data(), cs.size()); + while (cs.have_refs()) { + auto ref = do_inc(cs.fetch_ref()); + cb.store_ref(std::move(ref)); + } + auto res = cb.finalize(cs.is_special()); + CHECK(res->get_hash() == cell_hash); + info_.emplace(cell_hash, Info{.db_refcnt = 0, .diff_refcnt = 1, .cell = res}); + return res; + } + + void do_dec(Ref cell) { + auto cell_hash = cell->get_hash(); + auto it = info_.find(cell_hash); + if (it != info_.end()) { + CHECK(it->second.diff_refcnt != std::numeric_limits::min()); + --it->second.diff_refcnt; + } else { + auto info = *storage_->get_info(cell_hash); + it = info_.emplace(cell_hash, Info{.db_refcnt = info.db_refcnt, .diff_refcnt = -1, .cell = info.cell}).first; + } + if (it->second.diff_refcnt + it->second.db_refcnt != 0) { + return; + } + CellSlice cs(NoVm{}, std::move(cell)); + while (cs.have_refs()) { + do_dec(cs.fetch_ref()); + } + } +}; + +} // namespace + +std::unique_ptr DynamicBagOfCellsDb::create_in_memory(td::KeyValueReader *kv, + CreateInMemoryOptions options) { + if (kv == nullptr) { + LOG_IF(WARNING, options.verbose) << "Create empty in-memory cells database (no key value is given)"; + auto storage = CellStorage::build(options, [](auto, auto, auto) { return std::make_pair(0, 0); }); + return std::make_unique(std::move(storage)); + } + + std::vector keys; + keys.emplace_back(""); + for (td::uint32 c = 1; c <= 0xff; c++) { + keys.emplace_back(1, static_cast(c)); + } + keys.emplace_back(33, static_cast(0xff)); + + auto parallel_scan_cells = [&](ExtCellCreator &pc_creator, bool use_arena, + auto &&f) -> std::pair { + std::atomic cell_count{0}; + std::atomic desc_count{0}; + parallel_run( + keys.size() - 1, + [&](auto task_id) { + td::int64 local_cell_count = 0; + td::int64 local_desc_count = 0; + CHECK(!DataCell::use_arena); + DataCell::use_arena = use_arena; + kv->for_each_in_range(keys.at(task_id), keys.at(task_id + 1), [&](td::Slice key, td::Slice value) { + if (td::begins_with(key, "desc") && key.size() != 32) { + local_desc_count++; + return td::Status::OK(); + } + auto r_res = CellLoader::load(key, value.str(), true, pc_creator); + if (r_res.is_error()) { + LOG(ERROR) << r_res.error() << " at " << td::format::escaped(key); + return td::Status::OK(); + } + CHECK(key.size() == 32); + CHECK(key.ubegin()[0] == task_id); + auto res = r_res.move_as_ok(); + f(res.refcnt(), res.cell()); + local_cell_count++; + return td::Status::OK(); + }).ensure(); + DataCell::use_arena = false; + cell_count += local_cell_count; + desc_count += local_desc_count; + }, + options.extra_threads); + return std::make_pair(cell_count.load(), desc_count.load()); + }; + + auto storage = CellStorage::build(options, parallel_scan_cells); + return std::make_unique(std::move(storage)); +} +} // namespace vm diff --git a/crypto/vm/db/StaticBagOfCellsDb.cpp b/crypto/vm/db/StaticBagOfCellsDb.cpp index 9c00a98c..80dbfbf0 100644 --- a/crypto/vm/db/StaticBagOfCellsDb.cpp +++ b/crypto/vm/db/StaticBagOfCellsDb.cpp @@ -167,7 +167,7 @@ td::Result> StaticBagOfCellsDb::create_ext_cell(Cell::LevelMask level_ // class StaticBagOfCellsDbBaselineImpl : public StaticBagOfCellsDb { public: - StaticBagOfCellsDbBaselineImpl(std::vector> roots) : roots_(std::move(roots)) { + explicit StaticBagOfCellsDbBaselineImpl(std::vector> roots) : roots_(std::move(roots)) { } td::Result get_root_count() override { return roots_.size(); @@ -233,7 +233,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { return create_root_cell(std::move(data_cell)); }; - ~StaticBagOfCellsDbLazyImpl() { + ~StaticBagOfCellsDbLazyImpl() override { //LOG(ERROR) << deserialize_cell_cnt_ << " " << deserialize_cell_hash_cnt_; get_thread_safe_counter().add(-1); } @@ -309,31 +309,39 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { return 0; } td::Slice offset_view; - CHECK(info_.offset_byte_size <= 8); + if (info_.offset_byte_size > 8) { + return td::Status::Error(PSTRING() << "bag-of-cell error: invalid offset_byte_size " << info_.offset_byte_size); + } char arr[8]; td::RwMutex::ReadLock guard; if (info_.has_index) { TRY_RESULT(new_offset_view, data_.view(td::MutableSlice(arr, info_.offset_byte_size), - info_.index_offset + idx * info_.offset_byte_size)); + info_.index_offset + (td::int64)idx * info_.offset_byte_size)); offset_view = new_offset_view; } else { guard = index_data_rw_mutex_.lock_read().move_as_ok(); - offset_view = td::Slice(index_data_).substr(idx * info_.offset_byte_size, info_.offset_byte_size); + offset_view = td::Slice(index_data_).substr((td::int64)idx * info_.offset_byte_size, info_.offset_byte_size); } - CHECK(offset_view.size() == (size_t)info_.offset_byte_size); + if (offset_view.size() != (size_t)info_.offset_byte_size) { + return td::Status::Error(PSTRING() << "bag-of-cell error: invalid offset view size" << offset_view.size()); + } return td::narrow_cast(info_.read_offset(offset_view.ubegin())); } td::Result load_root_idx(int root_i) { - CHECK(root_i >= 0 && root_i < info_.root_count); + if (root_i < 0 || root_i >= info_.root_count) { + return td::Status::Error(PSTRING() << "bag-of-cell error: invalid root index " << root_i); + } if (!info_.has_roots) { return 0; } char arr[8]; TRY_RESULT(idx_view, data_.view(td::MutableSlice(arr, info_.ref_byte_size), - info_.roots_offset + root_i * info_.ref_byte_size)); - CHECK(idx_view.size() == (size_t)info_.ref_byte_size); + info_.roots_offset + (td::int64)root_i * info_.ref_byte_size)); + if (idx_view.size() != (size_t)info_.ref_byte_size) { + return td::Status::Error(PSTRING() << "bag-of-cell error: invalid idx_view size" << idx_view.size()); + } return info_.read_ref(idx_view.ubegin()); } @@ -343,8 +351,9 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { bool should_cache; }; td::Result get_cell_location(int idx) { - CHECK(idx >= 0); - CHECK(idx < info_.cell_count); + if (idx < 0 || idx >= info_.cell_count) { + return td::Status::Error(PSTRING() << "bag-of-cell error: invalid cell index " << idx); + } TRY_STATUS(preload_index(idx)); TRY_RESULT(from, load_idx_offset(idx - 1)); TRY_RESULT(till, load_idx_offset(idx)); @@ -357,10 +366,15 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { res.should_cache = res.end % 2 == 1; res.end /= 2; } - CHECK(std::numeric_limits::max() - res.begin >= info_.data_offset); - CHECK(std::numeric_limits::max() - res.end >= info_.data_offset); + if (std::numeric_limits::max() - res.begin < info_.data_offset || + std::numeric_limits::max() - res.end < info_.data_offset) { + return td::Status::Error(PSTRING() << "bag-of-cell error: invalid cell location (1) " << res.begin << ":" << res.end); + } res.begin += static_cast(info_.data_offset); res.end += static_cast(info_.data_offset); + if (res.begin > res.end) { + return td::Status::Error(PSTRING() << "bag-of-cell error: invalid cell location (2) " << res.begin << ":" << res.end); + } return res; } @@ -396,8 +410,6 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { if (info_.has_index) { return td::Status::OK(); } - - CHECK(idx < info_.cell_count); if (index_i_.load(std::memory_order_relaxed) > idx) { return td::Status::OK(); } @@ -407,12 +419,17 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { auto buf_slice = td::MutableSlice(buf.data(), buf.size()); for (; index_i_ <= idx; index_i_++) { auto offset = td::narrow_cast(info_.data_offset + index_offset_); - CHECK(data_.size() >= offset); + if (data_.size() < offset) { + return td::Status::Error(PSLICE() << "bag-of-cells error: invalid offset " << offset + << " (size=" << data_.size() << ")"); + } TRY_RESULT(cell, data_.view(buf_slice.copy().truncate(data_.size() - offset), offset)); CellSerializationInfo cell_info; TRY_STATUS(cell_info.init(cell, info_.ref_byte_size)); index_offset_ += cell_info.end_offset; - LOG_CHECK((unsigned)info_.offset_byte_size <= 8) << info_.offset_byte_size; + if ((unsigned)info_.offset_byte_size > 8) { + return td::Status::Error(PSTRING() << "bag-of-cell error: invalid offset_byte_size " << info_.offset_byte_size); + } td::uint8 tmp[8]; info_.write_offset(tmp, index_offset_); auto guard = index_data_rw_mutex_.lock_write(); @@ -488,7 +505,10 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { bool should_cache) { deserialize_cell_cnt_.add(1); Ref refs[4]; - CHECK(cell_info.refs_cnt <= 4); + if (cell_info.refs_cnt > 4) { + return td::Status::Error(PSLICE() << "invalid bag-of-cells cell #" << idx << " has " << cell_info.refs_cnt + << " refs"); + } auto* ref_ptr = cell_slice.ubegin() + cell_info.refs_offset; for (int k = 0; k < cell_info.refs_cnt; k++, ref_ptr += info_.ref_byte_size) { int ref_idx = td::narrow_cast(info_.read_ref(ref_ptr)); diff --git a/crypto/vm/db/TonDb.h b/crypto/vm/db/TonDb.h index dfce3110..6cd8aa4f 100644 --- a/crypto/vm/db/TonDb.h +++ b/crypto/vm/db/TonDb.h @@ -113,7 +113,8 @@ class TonDbTransactionImpl; using TonDbTransaction = std::unique_ptr; class TonDbTransactionImpl { public: - SmartContractDb begin_smartcontract(td::Slice hash = {}); + + SmartContractDb begin_smartcontract(td::Slice hash = std::string(32, '\0')); void commit_smartcontract(SmartContractDb txn); void commit_smartcontract(SmartContractDiff txn); @@ -142,6 +143,20 @@ class TonDbTransactionImpl { friend bool operator<(td::Slice hash, const SmartContractInfo &info) { return hash < info.hash; } + + struct Eq { + using is_transparent = void; // Pred to use + bool operator()(const SmartContractInfo &info, const SmartContractInfo &other_info) const { return info.hash == other_info.hash;} + bool operator()(const SmartContractInfo &info, td::Slice hash) const { return info.hash == hash;} + bool operator()(td::Slice hash, const SmartContractInfo &info) const { return info.hash == hash;} + + }; + struct Hash { + using is_transparent = void; // Pred to use + using transparent_key_equal = Eq; + size_t operator()(td::Slice hash) const { return cell_hash_slice_hash(hash); } + size_t operator()(const SmartContractInfo &info) const { return cell_hash_slice_hash(info.hash);} + }; }; CellHashTable contracts_; diff --git a/crypto/vm/dict.cpp b/crypto/vm/dict.cpp index ac32b38f..41f9c339 100644 --- a/crypto/vm/dict.cpp +++ b/crypto/vm/dict.cpp @@ -21,6 +21,7 @@ #include "vm/cellslice.h" #include "vm/stack.hpp" #include "common/bitstring.h" +#include "td/utils/Random.h" #include "td/utils/bits.h" @@ -1778,7 +1779,7 @@ Ref DictionaryFixed::dict_combine_with(Ref dict1, Ref dict2, t int mode, int skip1, int skip2) const { if (dict1.is_null()) { assert(!skip2); - if ((mode & 1) && dict2.is_null()) { + if ((mode & 1) && dict2.not_null()) { throw CombineError{}; } return dict2; @@ -1853,11 +1854,11 @@ Ref DictionaryFixed::dict_combine_with(Ref dict1, Ref dict2, t key_buffer[-1] = 0; // combine left subtrees auto c1 = dict_combine_with(label1.remainder->prefetch_ref(0), label2.remainder->prefetch_ref(0), key_buffer, - n - c - 1, total_key_len, combine_func); + n - c - 1, total_key_len, combine_func, mode); key_buffer[-1] = 1; // combine right subtrees auto c2 = dict_combine_with(label1.remainder->prefetch_ref(1), label2.remainder->prefetch_ref(1), key_buffer, - n - c - 1, total_key_len, combine_func); + n - c - 1, total_key_len, combine_func, mode); label1.remainder.clear(); label2.remainder.clear(); // c1 and c2 are merged left and right children of dict1 and dict2 @@ -2007,7 +2008,7 @@ bool DictionaryFixed::combine_with(DictionaryFixed& dict2) { bool DictionaryFixed::dict_check_for_each(Ref dict, td::BitPtr key_buffer, int n, int total_key_len, const DictionaryFixed::foreach_func_t& foreach_func, - bool invert_first) const { + bool invert_first, bool shuffle) const { if (dict.is_null()) { return true; } @@ -2026,26 +2027,29 @@ bool DictionaryFixed::dict_check_for_each(Ref dict, td::BitPtr key_buffer, key_buffer += l + 1; if (l) { invert_first = false; - } else if (invert_first) { + } + bool invert = shuffle ? td::Random::fast(0, 1) == 1: invert_first; + if (invert) { std::swap(c1, c2); } - key_buffer[-1] = invert_first; + key_buffer[-1] = invert; // recursive check_foreach applied to both children - if (!dict_check_for_each(std::move(c1), key_buffer, n - l - 1, total_key_len, foreach_func)) { + if (!dict_check_for_each(std::move(c1), key_buffer, n - l - 1, total_key_len, foreach_func, false, shuffle)) { return false; } - key_buffer[-1] = !invert_first; - return dict_check_for_each(std::move(c2), key_buffer, n - l - 1, total_key_len, foreach_func); + key_buffer[-1] = !invert; + return dict_check_for_each(std::move(c2), key_buffer, n - l - 1, total_key_len, foreach_func, false, shuffle); } -bool DictionaryFixed::check_for_each(const foreach_func_t& foreach_func, bool invert_first) { +bool DictionaryFixed::check_for_each(const foreach_func_t& foreach_func, bool invert_first, bool shuffle) { force_validate(); if (is_empty()) { return true; } int key_len = get_key_bits(); unsigned char key_buffer[max_key_bytes]; - return dict_check_for_each(get_root_cell(), td::BitPtr{key_buffer}, key_len, key_len, foreach_func, invert_first); + return dict_check_for_each(get_root_cell(), td::BitPtr{key_buffer}, key_len, key_len, foreach_func, invert_first, + shuffle); } static inline bool set_bit(td::BitPtr ptr, bool value = true) { diff --git a/crypto/vm/dict.h b/crypto/vm/dict.h index 978f4d53..c4044963 100644 --- a/crypto/vm/dict.h +++ b/crypto/vm/dict.h @@ -223,7 +223,7 @@ class DictionaryFixed : public DictionaryBase { int get_common_prefix(td::BitPtr buffer, unsigned buffer_len); bool cut_prefix_subdict(td::ConstBitPtr prefix, int prefix_len, bool remove_prefix = false); Ref extract_prefix_subdict_root(td::ConstBitPtr prefix, int prefix_len, bool remove_prefix = false); - bool check_for_each(const foreach_func_t& foreach_func, bool invert_first = false); + bool check_for_each(const foreach_func_t& foreach_func, bool invert_first = false, bool shuffle = false); int filter(filter_func_t check); bool combine_with(DictionaryFixed& dict2, const combine_func_t& combine_func, int mode = 0); bool combine_with(DictionaryFixed& dict2, const simple_combine_func_t& simple_combine_func, int mode = 0); @@ -292,7 +292,7 @@ class DictionaryFixed : public DictionaryBase { std::pair, bool> extract_prefix_subdict_internal(Ref dict, td::ConstBitPtr prefix, int prefix_len, bool remove_prefix = false) const; bool dict_check_for_each(Ref dict, td::BitPtr key_buffer, int n, int total_key_len, - const foreach_func_t& foreach_func, bool invert_first = false) const; + const foreach_func_t& foreach_func, bool invert_first = false, bool shuffle = false) const; std::pair, int> dict_filter(Ref dict, td::BitPtr key, int n, const filter_func_t& check_leaf, int& skip_rest) const; Ref dict_combine_with(Ref dict1, Ref dict2, td::BitPtr key_buffer, int n, int total_key_len, diff --git a/crypto/vm/dictops.cpp b/crypto/vm/dictops.cpp index c798b333..02f26fdd 100644 --- a/crypto/vm/dictops.cpp +++ b/crypto/vm/dictops.cpp @@ -172,7 +172,8 @@ int exec_load_dict(VmState* st, unsigned args) { } std::string dump_dictop(unsigned args, const char* name) { - std::ostringstream os{"DICT"}; + std::ostringstream os; + os << "DICT"; if (args & 4) { os << (args & 2 ? 'U' : 'I'); } @@ -184,7 +185,8 @@ std::string dump_dictop(unsigned args, const char* name) { } std::string dump_dictop2(unsigned args, const char* name) { - std::ostringstream os{"DICT"}; + std::ostringstream os; + os << "DICT"; if (args & 2) { os << (args & 1 ? 'U' : 'I'); } @@ -193,7 +195,8 @@ std::string dump_dictop2(unsigned args, const char* name) { } std::string dump_subdictop2(unsigned args, const char* name) { - std::ostringstream os{"SUBDICT"}; + std::ostringstream os; + os << "SUBDICT"; if (args & 2) { os << (args & 1 ? 'U' : 'I'); } @@ -508,7 +511,8 @@ int exec_dict_getmin(VmState* st, unsigned args) { } std::string dump_dictop_getnear(CellSlice& cs, unsigned args) { - std::ostringstream os{"DICT"}; + std::ostringstream os; + os << "DICT"; if (args & 8) { os << (args & 4 ? 'U' : 'I'); } @@ -562,7 +566,7 @@ int exec_dict_getnear(VmState* st, unsigned args) { int exec_pfx_dict_set(VmState* st, Dictionary::SetMode mode, const char* name) { Stack& stack = st->get_stack(); VM_LOG(st) << "execute PFXDICT" << name; - stack.check_underflow(3); + stack.check_underflow(st->get_global_version() >= 9 ? 4 : 3); int n = stack.pop_smallint_range(PrefixDictionary::max_key_bits); PrefixDictionary dict{stack.pop_maybe_cell(), n}; auto key_slice = stack.pop_cellslice(); @@ -576,7 +580,7 @@ int exec_pfx_dict_set(VmState* st, Dictionary::SetMode mode, const char* name) { int exec_pfx_dict_delete(VmState* st) { Stack& stack = st->get_stack(); VM_LOG(st) << "execute PFXDICTDEL\n"; - stack.check_underflow(2); + stack.check_underflow(st->get_global_version() >= 9 ? 3 : 2); int n = stack.pop_smallint_range(PrefixDictionary::max_key_bits); PrefixDictionary dict{stack.pop_maybe_cell(), n}; auto key_slice = stack.pop_cellslice(); @@ -637,8 +641,8 @@ std::string dump_push_const_dict(CellSlice& cs, int pfx_bits, const char* name) cs.advance(pfx_bits - 11); auto slice = cs.fetch_subslice(1, 1); int n = (int)cs.fetch_ulong(10); - std::ostringstream os{name}; - os << ' ' << n << " ("; + std::ostringstream os; + os << name << ' ' << n << " ("; slice->dump_hex(os, false); os << ')'; return os.str(); diff --git a/crypto/vm/large-boc-serializer.cpp b/crypto/vm/large-boc-serializer.cpp index fe16b767..d209c88e 100644 --- a/crypto/vm/large-boc-serializer.cpp +++ b/crypto/vm/large-boc-serializer.cpp @@ -14,6 +14,9 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ +#include "td/utils/Time.h" +#include "td/utils/Timer.h" + #include #include "vm/boc.h" #include "vm/boc-writers.h" @@ -30,8 +33,12 @@ class LargeBocSerializer { public: using Hash = Cell::Hash; - explicit LargeBocSerializer(std::shared_ptr reader) : reader(std::move(reader)) {} + explicit LargeBocSerializer(std::shared_ptr reader) : reader(std::move(reader)) { + } + void set_logger(BagOfCellsLogger* logger_ptr) { + logger_ptr_ = logger_ptr; + } void add_root(Hash root); td::Status import_cells(); td::Status serialize(td::FileFd& fd, int mode); @@ -39,6 +46,7 @@ class LargeBocSerializer { private: std::shared_ptr reader; struct CellInfo { + Cell::Hash hash; std::array ref_idx; int idx; unsigned short serialized_size; @@ -62,10 +70,11 @@ class LargeBocSerializer { return 4; } }; - std::map cells; + td::NodeHashMap cells; std::vector*> cell_list; struct RootInfo { - RootInfo(Hash hash, int idx) : hash(hash), idx(idx) {} + RootInfo(Hash hash, int idx) : hash(hash), idx(idx) { + } Hash hash; int idx; }; @@ -78,6 +87,8 @@ class LargeBocSerializer { void reorder_cells(); int revisit(int cell_idx, int force = 0); td::uint64 compute_sizes(int mode, int& r_size, int& o_size); + + BagOfCellsLogger* logger_ptr_{}; }; void LargeBocSerializer::add_root(Hash root) { @@ -85,12 +96,18 @@ void LargeBocSerializer::add_root(Hash root) { } td::Status LargeBocSerializer::import_cells() { + if (logger_ptr_) { + logger_ptr_->start_stage("import_cells"); + } for (auto& root : roots) { TRY_RESULT(idx, import_cell(root.hash)); root.idx = idx; } reorder_cells(); CHECK(!cell_list.empty()); + if (logger_ptr_) { + logger_ptr_->finish_stage(PSLICE() << cell_count << " cells"); + } return td::Status::OK(); } @@ -98,6 +115,9 @@ td::Result LargeBocSerializer::import_cell(Hash hash, int depth) { if (depth > Cell::max_depth) { return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large"); } + if (logger_ptr_) { + TRY_STATUS(logger_ptr_->on_cell_processed()); + } auto it = cells.find(hash); if (it != cells.end()) { it->second.should_cache = true; @@ -313,13 +333,9 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { return td::Status::Error("bag of cells is too large"); } - boc_writers::FileWriter writer{fd, (size_t) info.total_size}; - auto store_ref = [&](unsigned long long value) { - writer.store_uint(value, info.ref_byte_size); - }; - auto store_offset = [&](unsigned long long value) { - writer.store_uint(value, info.offset_byte_size); - }; + boc_writers::FileWriter writer{fd, (size_t)info.total_size}; + auto store_ref = [&](unsigned long long value) { writer.store_uint(value, info.ref_byte_size); }; + auto store_offset = [&](unsigned long long value) { writer.store_uint(value, info.offset_byte_size); }; writer.store_uint(info.magic, 4); @@ -349,6 +365,9 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { DCHECK(writer.position() == info.index_offset); DCHECK((unsigned)cell_count == cell_list.size()); if (info.has_index) { + if (logger_ptr_) { + logger_ptr_->start_stage("generate_index"); + } std::size_t offs = 0; for (int i = cell_count - 1; i >= 0; --i) { const auto& dc_info = cell_list[i]->second; @@ -366,11 +385,20 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { fixed_offset = offs * 2 + dc_info.should_cache; } store_offset(fixed_offset); + if (logger_ptr_) { + TRY_STATUS(logger_ptr_->on_cell_processed()); + } } DCHECK(offs == info.data_size); + if (logger_ptr_) { + logger_ptr_->finish_stage(""); + } } DCHECK(writer.position() == info.data_offset); size_t keep_position = writer.position(); + if (logger_ptr_) { + logger_ptr_->start_stage("serialize"); + } for (int i = 0; i < cell_count; ++i) { auto hash = cell_list[cell_count - 1 - i]->first; const auto& dc_info = cell_list[cell_count - 1 - i]->second; @@ -389,6 +417,9 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { DCHECK(k > i && k < cell_count); store_ref(k); } + if (logger_ptr_) { + TRY_STATUS(logger_ptr_->on_cell_processed()); + } } DCHECK(writer.position() - keep_position == info.data_size); if (info.has_crc32c) { @@ -396,17 +427,26 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { writer.store_uint(td::bswap32(crc), 4); } DCHECK(writer.empty()); - return writer.finalize(); -} + TRY_STATUS(writer.finalize()); + if (logger_ptr_) { + logger_ptr_->finish_stage(PSLICE() << cell_count << " cells, " << writer.position() << " bytes"); + } + return td::Status::OK(); } +} // namespace -td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, - td::FileFd& fd, int mode) { +td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, + int mode, td::CancellationToken cancellation_token) { + td::Timer timer; CHECK(reader != nullptr) LargeBocSerializer serializer(reader); + BagOfCellsLogger logger(std::move(cancellation_token)); + serializer.set_logger(&logger); serializer.add_root(root_hash); TRY_STATUS(serializer.import_cells()); - return serializer.serialize(fd, mode); + TRY_STATUS(serializer.serialize(fd, mode)); + LOG(ERROR) << "serialization took " << timer.elapsed() << "s"; + return td::Status::OK(); } -} +} // namespace vm diff --git a/crypto/vm/log.h b/crypto/vm/log.h index 30ace2c4..c8486324 100644 --- a/crypto/vm/log.h +++ b/crypto/vm/log.h @@ -31,7 +31,7 @@ namespace vm { struct VmLog { td::LogInterface *log_interface{td::log_interface}; td::LogOptions log_options{td::log_options}; - enum { DumpStack = 2 }; + enum { DumpStack = 2, ExecLocation = 4, GasRemaining = 8, DumpStackVerbose = 16, DumpC5 = 32 }; int log_mask{1}; static VmLog Null() { VmLog res; diff --git a/crypto/vm/memo.cpp b/crypto/vm/memo.cpp index 576f28f1..015bab6d 100644 --- a/crypto/vm/memo.cpp +++ b/crypto/vm/memo.cpp @@ -18,6 +18,7 @@ */ #include "vm/memo.h" #include "vm/excno.hpp" +#include "vm/vm.h" namespace vm { using td::Ref; @@ -30,4 +31,18 @@ bool FakeVmStateLimits::register_op(int op_units) { return ok; } +Ref DummyVmState::load_library(td::ConstBitPtr hash) { + std::unique_ptr tmp_ctx; + // install temporary dummy vm state interface to prevent charging for cell load operations during library lookup + VmStateInterface::Guard guard{global_version >= 4 ? tmp_ctx.get() : VmStateInterface::get()}; + for (const auto& lib_collection : libraries) { + auto lib = lookup_library_in(hash, lib_collection); + if (lib.not_null()) { + return lib; + } + } + missing_library = td::Bits256{hash}; + return {}; +} + } // namespace vm diff --git a/crypto/vm/memo.h b/crypto/vm/memo.h index 3aee39a6..db6dc8e2 100644 --- a/crypto/vm/memo.h +++ b/crypto/vm/memo.h @@ -20,6 +20,7 @@ #include "common/refcnt.hpp" #include "vm/cells.h" #include "vm/vmstate.h" +#include "td/utils/optional.h" namespace vm { using td::Ref; @@ -34,4 +35,23 @@ class FakeVmStateLimits : public VmStateInterface { bool register_op(int op_units = 1) override; }; +class DummyVmState : public VmStateInterface { + public: + explicit DummyVmState(std::vector> libraries, int global_version = ton::SUPPORTED_VERSION) + : libraries(std::move(libraries)), global_version(global_version) { + } + Ref load_library(td::ConstBitPtr hash) override; + int get_global_version() const override { + return global_version; + } + td::optional get_missing_library() const { + return missing_library; + } + + private: + std::vector> libraries; + int global_version; + td::optional missing_library; +}; + } // namespace vm diff --git a/crypto/vm/opctable.cpp b/crypto/vm/opctable.cpp index d4f0f3e9..0521a763 100644 --- a/crypto/vm/opctable.cpp +++ b/crypto/vm/opctable.cpp @@ -357,32 +357,32 @@ using namespace std::placeholders; dump_arg_instr_func_t dump_1sr(std::string prefix, std::string suffix) { return [prefix, suffix](CellSlice&, unsigned args) -> std::string { - std::ostringstream os{prefix}; - os << 's' << (args & 15) << suffix; + std::ostringstream os; + os << prefix << 's' << (args & 15) << suffix; return os.str(); }; } dump_arg_instr_func_t dump_1sr_l(std::string prefix, std::string suffix) { return [prefix, suffix](CellSlice&, unsigned args) -> std::string { - std::ostringstream os{prefix}; - os << 's' << (args & 255) << suffix; + std::ostringstream os; + os << prefix << 's' << (args & 255) << suffix; return os.str(); }; } dump_arg_instr_func_t dump_2sr(std::string prefix, std::string suffix) { return [prefix, suffix](CellSlice&, unsigned args) -> std::string { - std::ostringstream os{prefix}; - os << 's' << ((args >> 4) & 15) << ",s" << (args & 15) << suffix; + std::ostringstream os; + os << prefix << 's' << ((args >> 4) & 15) << ",s" << (args & 15) << suffix; return os.str(); }; } dump_arg_instr_func_t dump_2sr_adj(unsigned adj, std::string prefix, std::string suffix) { return [adj, prefix, suffix](CellSlice&, unsigned args) -> std::string { - std::ostringstream os{prefix}; - os << 's' << (int)((args >> 4) & 15) - (int)((adj >> 4) & 15) << ",s" << (int)(args & 15) - (int)(adj & 15) + std::ostringstream os; + os << prefix << 's' << (int)((args >> 4) & 15) - (int)((adj >> 4) & 15) << ",s" << (int)(args & 15) - (int)(adj & 15) << suffix; return os.str(); }; @@ -390,16 +390,16 @@ dump_arg_instr_func_t dump_2sr_adj(unsigned adj, std::string prefix, std::string dump_arg_instr_func_t dump_3sr(std::string prefix, std::string suffix) { return [prefix, suffix](CellSlice&, unsigned args) -> std::string { - std::ostringstream os{prefix}; - os << 's' << ((args >> 8) & 15) << ",s" << ((args >> 4) & 15) << ",s" << (args & 15) << suffix; + std::ostringstream os; + os << prefix << 's' << ((args >> 8) & 15) << ",s" << ((args >> 4) & 15) << ",s" << (args & 15) << suffix; return os.str(); }; } dump_arg_instr_func_t dump_3sr_adj(unsigned adj, std::string prefix, std::string suffix) { return [adj, prefix, suffix](CellSlice&, unsigned args) -> std::string { - std::ostringstream os{prefix}; - os << 's' << (int)((args >> 8) & 15) - (int)((adj >> 8) & 15) << ",s" + std::ostringstream os; + os << prefix << 's' << (int)((args >> 8) & 15) - (int)((adj >> 8) & 15) << ",s" << (int)((args >> 4) & 15) - (int)((adj >> 4) & 15) << ",s" << (int)(args & 15) - (int)(adj & 15) << suffix; return os.str(); }; @@ -407,44 +407,64 @@ dump_arg_instr_func_t dump_3sr_adj(unsigned adj, std::string prefix, std::string dump_arg_instr_func_t dump_1c(std::string prefix, std::string suffix) { return [prefix, suffix](CellSlice&, unsigned args) -> std::string { - std::ostringstream os{prefix}; - os << (args & 15) << suffix; + std::ostringstream os; + os << prefix << (args & 15) << suffix; return os.str(); }; } dump_arg_instr_func_t dump_1c_l_add(int adj, std::string prefix, std::string suffix) { return [adj, prefix, suffix](CellSlice&, unsigned args) -> std::string { - std::ostringstream os{prefix}; - os << (int)(args & 255) + adj << suffix; + std::ostringstream os; + os << prefix << (int)(args & 255) + adj << suffix; return os.str(); }; } dump_arg_instr_func_t dump_1c_and(unsigned mask, std::string prefix, std::string suffix) { return [mask, prefix, suffix](CellSlice&, unsigned args) -> std::string { - std::ostringstream os{prefix}; - os << (args & mask) << suffix; + std::ostringstream os; + os << prefix << (args & mask) << suffix; return os.str(); }; } dump_arg_instr_func_t dump_2c(std::string prefix, std::string interfix, std::string suffix) { return [prefix, interfix, suffix](CellSlice&, unsigned args) -> std::string { - std::ostringstream os{prefix}; - os << ((args >> 4) & 15) << interfix << (args & 15) << suffix; + std::ostringstream os; + os << prefix << ((args >> 4) & 15) << interfix << (args & 15) << suffix; return os.str(); }; } dump_arg_instr_func_t dump_2c_add(unsigned add, std::string prefix, std::string interfix, std::string suffix) { return [add, prefix, interfix, suffix](CellSlice&, unsigned args) -> std::string { - std::ostringstream os{prefix}; - os << ((args >> 4) & 15) + ((add >> 4) & 15) << interfix << (args & 15) + (add & 15) << suffix; + std::ostringstream os; + os << prefix << ((args >> 4) & 15) + ((add >> 4) & 15) << interfix << (args & 15) + (add & 15) << suffix; return os.str(); }; } } // namespace instr +OpcodeInstr* OpcodeInstr::require_version(int required_version) { + return new OpcodeInstrWithVersion(this, required_version); +} + +int OpcodeInstrWithVersion::dispatch(VmState* st, CellSlice& cs, unsigned opcode, unsigned bits) const { + if (st->get_global_version() < required_version) { + st->consume_gas(gas_per_instr); + throw VmError{Excno::inv_opcode, "invalid opcode", opcode}; + } + return instr->dispatch(st, cs, opcode, bits); +} + +std::string OpcodeInstrWithVersion::dump(CellSlice& cs, unsigned opcode, unsigned bits) const { + return instr->dump(cs, opcode, bits); +} + +int OpcodeInstrWithVersion::instr_len(const CellSlice& cs, unsigned opcode, unsigned bits) const { + return instr->instr_len(cs, opcode, bits); +} + } // namespace vm diff --git a/crypto/vm/opctable.h b/crypto/vm/opctable.h index f9ea7023..34d2ef0a 100644 --- a/crypto/vm/opctable.h +++ b/crypto/vm/opctable.h @@ -57,6 +57,9 @@ class OpcodeInstr { std::pair get_opcode_range() const { return {min_opcode, max_opcode}; } + + OpcodeInstr* require_version(int required_version); + //static OpcodeInstr* mksimple(unsigned opcode, unsigned opc_bits, std::string _name, exec_instr_func_t exec); static OpcodeInstr* mksimple(unsigned opcode, unsigned opc_bits, std::string _name, exec_simple_instr_func_t exec); static OpcodeInstr* mkfixed(unsigned opcode, unsigned opc_bits, unsigned arg_bits, dump_arg_instr_func_t dump, @@ -188,4 +191,19 @@ class OpcodeInstrExt : public OpcodeInstr { int instr_len(const CellSlice& cs, unsigned opcode, unsigned bits) const override; }; +class OpcodeInstrWithVersion : public OpcodeInstr { + public: + OpcodeInstrWithVersion() = delete; + OpcodeInstrWithVersion(OpcodeInstr* instr, int required_version) : + OpcodeInstr(instr->get_opcode_min(), instr->get_opcode_max()), instr(instr), required_version(required_version) { + } + ~OpcodeInstrWithVersion() override = default; + int dispatch(VmState* st, CellSlice& cs, unsigned opcode, unsigned bits) const override; + std::string dump(CellSlice& cs, unsigned opcode, unsigned bits) const override; + int instr_len(const CellSlice& cs, unsigned opcode, unsigned bits) const override; + private: + OpcodeInstr* instr; + int required_version; +}; + } // namespace vm diff --git a/crypto/vm/stack.cpp b/crypto/vm/stack.cpp index 9a17c8e2..aac1b6b5 100644 --- a/crypto/vm/stack.cpp +++ b/crypto/vm/stack.cpp @@ -21,6 +21,8 @@ #include "vm/box.hpp" #include "vm/atom.h" #include "vm/vmstate.h" +#include "vm/boc.h" +#include "td/utils/misc.h" namespace td { template class td::Cnt; @@ -81,7 +83,15 @@ std::string StackEntry::to_lisp_string() const { return std::move(os).str(); } -void StackEntry::dump(std::ostream& os) const { +static std::string cell_to_hex(const td::Ref &cell) { + auto boc = vm::std_boc_serialize(cell); + if (boc.is_ok()) { + return td::buffer_to_hex(boc.move_as_ok().as_slice()); + } + return "???"; +} + +void StackEntry::dump(std::ostream& os, bool verbose) const { switch (tp) { case t_null: os << "(null)"; @@ -91,14 +101,23 @@ void StackEntry::dump(std::ostream& os) const { break; case t_cell: if (ref.not_null()) { - os << "C{" << static_cast>(ref)->get_hash().to_hex() << "}"; + if (verbose) { + os << "C{" << cell_to_hex(as_cell()) << "}"; + } else { + os << "C{" << *as_cell() << "}"; + } } else { os << "C{null}"; } break; case t_builder: if (ref.not_null()) { - os << "BC{" << static_cast>(ref)->to_hex() << "}"; + if (verbose) { + Ref cb = as_builder(); + os << "BC{" << cell_to_hex(cb.write().finalize_novm()) << "}"; + } else { + os << "BC{" << *as_builder() << "}"; + } } else { os << "BC{null}"; } @@ -106,7 +125,13 @@ void StackEntry::dump(std::ostream& os) const { case t_slice: { if (ref.not_null()) { os << "CS{"; - static_cast>(ref)->dump(os, 1, false); + if (verbose) { + CellBuilder cb; + cb.append_cellslice(as_slice()); + os << cell_to_hex(cb.finalize_novm()); + } else { + static_cast>(ref)->dump(os, 1, false); + } os << '}'; } else { os << "CS{null}"; @@ -149,12 +174,24 @@ void StackEntry::dump(std::ostream& os) const { os << "Object{" << (const void*)&*ref << "}"; break; } + case t_vmcont: { + if (ref.not_null()) { + if (verbose) { + os << "Cont{" << *as_cont() << "}"; + } else { + os << "Cont{" << as_cont()->type() << "}"; + } + } else { + os << "Cont{null}"; + } + break; + } default: os << "???"; } } -void StackEntry::print_list(std::ostream& os) const { +void StackEntry::print_list(std::ostream& os, bool verbose) const { switch (tp) { case t_null: os << "()"; @@ -163,7 +200,7 @@ void StackEntry::print_list(std::ostream& os) const { const auto& tuple = *static_cast>(ref); if (is_list()) { os << '('; - tuple[0].print_list(os); + tuple[0].print_list(os, verbose); print_list_tail(os, &tuple[1]); break; } @@ -172,7 +209,7 @@ void StackEntry::print_list(std::ostream& os) const { os << "[]"; } else if (n == 1) { os << "["; - tuple[0].print_list(os); + tuple[0].print_list(os, verbose); os << "]"; } else { os << "["; @@ -181,14 +218,14 @@ void StackEntry::print_list(std::ostream& os) const { if (c++) { os << " "; } - entry.print_list(os); + entry.print_list(os, verbose); } os << ']'; } break; } default: - dump(os); + dump(os, verbose); } } @@ -326,7 +363,7 @@ void StackEntry::for_each_scalar(const std::function& f } } -const StackEntry& tuple_index(const Tuple& tup, unsigned idx) { +const StackEntry& tuple_index(const Ref& tup, unsigned idx) { if (idx >= tup->size()) { throw VmError{Excno::range_chk, "tuple index out of range"}; } @@ -687,12 +724,12 @@ void Stack::dump(std::ostream& os, int mode) const { os << " [ "; if (mode & 2) { for (const auto& x : stack) { - x.print_list(os); + x.print_list(os, mode & 4); os << ' '; } } else { for (const auto& x : stack) { - x.dump(os); + x.dump(os, mode & 4); os << ' '; } } @@ -908,23 +945,29 @@ bool Stack::serialize(vm::CellBuilder& cb, int mode) const { if (vsi && !vsi->register_op()) { return false; } - // vm_stack#_ depth:(## 24) stack:(VmStackList depth) = VmStack; - unsigned n = depth(); - if (!cb.store_ulong_rchk_bool(n, 24)) { // vm_stack#_ depth:(## 24) - return false; - } - if (!n) { - return true; - } - vm::CellBuilder cb2; - Ref rest = cb2.finalize(); // vm_stk_nil#_ = VmStackList 0; - for (unsigned i = 0; i < n - 1; i++) { - // vm_stk_cons#_ {n:#} rest:^(VmStackList n) tos:VmStackValue = VmStackList (n + 1); - if (!(cb2.store_ref_bool(std::move(rest)) && stack[i].serialize(cb2, mode) && cb2.finalize_to(rest))) { + try { + // vm_stack#_ depth:(## 24) stack:(VmStackList depth) = VmStack; + unsigned n = depth(); + if (!cb.store_ulong_rchk_bool(n, 24)) { // vm_stack#_ depth:(## 24) return false; } + if (!n) { + return true; + } + vm::CellBuilder cb2; + Ref rest = cb2.finalize(); // vm_stk_nil#_ = VmStackList 0; + for (unsigned i = 0; i < n - 1; i++) { + // vm_stk_cons#_ {n:#} rest:^(VmStackList n) tos:VmStackValue = VmStackList (n + 1); + if (!(cb2.store_ref_bool(std::move(rest)) && stack[i].serialize(cb2, mode) && cb2.finalize_to(rest))) { + return false; + } + } + return cb.store_ref_bool(std::move(rest)) && stack[n - 1].serialize(cb, mode); + } catch (CellBuilder::CellCreateError) { + return false; + } catch (CellBuilder::CellWriteError) { + return false; } - return cb.store_ref_bool(std::move(rest)) && stack[n - 1].serialize(cb, mode); } bool Stack::deserialize(vm::CellSlice& cs, int mode) { diff --git a/crypto/vm/stack.hpp b/crypto/vm/stack.hpp index 2de7c1e4..6a52e4a2 100644 --- a/crypto/vm/stack.hpp +++ b/crypto/vm/stack.hpp @@ -103,6 +103,9 @@ class StackEntry { } StackEntry(td::RefInt256 int_ref) : ref(std::move(int_ref)), tp(t_int) { } + StackEntry(Ref> str_ref, bool bytes = false) + : ref(std::move(str_ref)), tp(bytes ? t_bytes : t_string) { + } StackEntry(std::string str, bool bytes = false) : ref(), tp(bytes ? t_bytes : t_string) { ref = Ref>{true, std::move(str)}; } @@ -289,8 +292,8 @@ class StackEntry { } bool for_each_scalar(const std::function& func) const; void for_each_scalar(const std::function& func) const; - void dump(std::ostream& os) const; - void print_list(std::ostream& os) const; + void dump(std::ostream& os, bool verbose = false) const; + void print_list(std::ostream& os, bool verbose = false) const; std::string to_string() const; std::string to_lisp_string() const; @@ -302,7 +305,7 @@ inline void swap(StackEntry& se1, StackEntry& se2) { se1.swap(se2); } -const StackEntry& tuple_index(const Tuple& tup, unsigned idx); +const StackEntry& tuple_index(const Ref& tup, unsigned idx); StackEntry tuple_extend_index(const Ref& tup, unsigned idx); unsigned tuple_extend_set_index(Ref& tup, unsigned idx, StackEntry&& value, bool force = false); @@ -555,7 +558,7 @@ class Stack : public td::CntObject { } bool for_each_scalar(const std::function& func) const; void for_each_scalar(const std::function& func) const; - // mode: +1 = add eoln, +2 = Lisp-style lists + // mode: +1 = add eoln, +2 = Lisp-style lists, +4 = serialized bocs void dump(std::ostream& os, int mode = 1) const; bool serialize(vm::CellBuilder& cb, int mode = 0) const; bool deserialize(vm::CellSlice& cs, int mode = 0); diff --git a/crypto/vm/stackops.cpp b/crypto/vm/stackops.cpp index 0f5be6d4..c8180f1a 100644 --- a/crypto/vm/stackops.cpp +++ b/crypto/vm/stackops.cpp @@ -75,8 +75,8 @@ std::string dump_xchg(CellSlice&, unsigned args) { if (!x || x >= y) { return ""; } - std::ostringstream os{"XCHG s"}; - os << x << ",s" << y; + std::ostringstream os; + os << "XCHG s" << x << ",s" << y; return os.str(); } @@ -92,7 +92,7 @@ int exec_xchg1(VmState* st, unsigned args) { int exec_dup(VmState* st) { Stack& stack = st->get_stack(); - VM_LOG(st) << "execute DUP\n"; + VM_LOG(st) << "execute DUP"; stack.check_underflow(1); stack.push(stack.fetch(0)); return 0; @@ -100,7 +100,7 @@ int exec_dup(VmState* st) { int exec_over(VmState* st) { Stack& stack = st->get_stack(); - VM_LOG(st) << "execute OVER\n"; + VM_LOG(st) << "execute OVER"; stack.check_underflow(2); stack.push(stack.fetch(1)); return 0; @@ -126,7 +126,7 @@ int exec_push_l(VmState* st, unsigned args) { int exec_drop(VmState* st) { Stack& stack = st->get_stack(); - VM_LOG(st) << "execute DROP\n"; + VM_LOG(st) << "execute DROP"; stack.check_underflow(1); stack.pop(); return 0; @@ -134,7 +134,7 @@ int exec_drop(VmState* st) { int exec_nip(VmState* st) { Stack& stack = st->get_stack(); - VM_LOG(st) << "execute NIP\n"; + VM_LOG(st) << "execute NIP"; stack.check_underflow(2); stack.pop(stack[1]); return 0; @@ -301,9 +301,7 @@ int exec_blkswap(VmState* st, unsigned args) { Stack& stack = st->get_stack(); VM_LOG(st) << "execute BLKSWAP " << x << ',' << y; stack.check_underflow(x + y); - std::reverse(stack.from_top(x + y), stack.from_top(y)); - std::reverse(stack.from_top(y), stack.top()); - std::reverse(stack.from_top(x + y), stack.top()); + std::rotate(stack.from_top(x + y), stack.from_top(y), stack.top()); return 0; } @@ -403,7 +401,7 @@ int exec_pick(VmState* st) { Stack& stack = st->get_stack(); VM_LOG(st) << "execute PICK\n"; stack.check_underflow(1); - int x = stack.pop_smallint_range(255); + int x = stack.pop_smallint_range(st->get_global_version() >= 4 ? (1 << 30) - 1 : 255); stack.check_underflow_p(x); stack.push(stack.fetch(x)); return 0; @@ -413,8 +411,9 @@ int exec_roll(VmState* st) { Stack& stack = st->get_stack(); VM_LOG(st) << "execute ROLL\n"; stack.check_underflow(1); - int x = stack.pop_smallint_range(255); + int x = stack.pop_smallint_range(st->get_global_version() >= 4 ? (1 << 30) - 1 : 255); stack.check_underflow_p(x); + st->consume_gas(std::max(x - 255, 0)); while (--x >= 0) { swap(stack[x], stack[x + 1]); } @@ -425,8 +424,9 @@ int exec_rollrev(VmState* st) { Stack& stack = st->get_stack(); VM_LOG(st) << "execute ROLLREV\n"; stack.check_underflow(1); - int x = stack.pop_smallint_range(255); + int x = stack.pop_smallint_range(st->get_global_version() >= 4 ? (1 << 30) - 1 : 255); stack.check_underflow_p(x); + st->consume_gas(std::max(x - 255, 0)); for (int i = 0; i < x; i++) { swap(stack[i], stack[i + 1]); } @@ -437,13 +437,14 @@ int exec_blkswap_x(VmState* st) { Stack& stack = st->get_stack(); VM_LOG(st) << "execute BLKSWX\n"; stack.check_underflow(2); - int y = stack.pop_smallint_range(255); - int x = stack.pop_smallint_range(255); + int y = stack.pop_smallint_range(st->get_global_version() >= 4 ? (1 << 30) - 1 : 255); + int x = stack.pop_smallint_range(st->get_global_version() >= 4 ? (1 << 30) - 1 : 255); stack.check_underflow(x + y); if (x > 0 && y > 0) { - std::reverse(stack.from_top(x + y), stack.from_top(y)); - std::reverse(stack.from_top(y), stack.top()); - std::reverse(stack.from_top(x + y), stack.top()); + if (st->get_global_version() >= 4) { + st->consume_gas(std::max(x + y - 255, 0)); + } + std::rotate(stack.from_top(x + y), stack.from_top(y), stack.top()); } return 0; } @@ -452,9 +453,10 @@ int exec_reverse_x(VmState* st) { Stack& stack = st->get_stack(); VM_LOG(st) << "execute REVX\n"; stack.check_underflow(2); - int y = stack.pop_smallint_range(255); - int x = stack.pop_smallint_range(255); + int y = stack.pop_smallint_range(st->get_global_version() >= 4 ? (1 << 30) - 1 : 255); + int x = stack.pop_smallint_range(st->get_global_version() >= 4 ? (1 << 30) - 1 : 255); stack.check_underflow(x + y); + st->consume_gas(std::max(x - 255, 0)); std::reverse(stack.from_top(x + y), stack.from_top(y)); return 0; } @@ -463,7 +465,7 @@ int exec_drop_x(VmState* st) { Stack& stack = st->get_stack(); VM_LOG(st) << "execute DROPX\n"; stack.check_underflow(1); - int x = stack.pop_smallint_range(255); + int x = stack.pop_smallint_range(st->get_global_version() >= 4 ? (1 << 30) - 1 : 255); stack.check_underflow(x); stack.pop_many(x); return 0; @@ -482,7 +484,7 @@ int exec_xchg_x(VmState* st) { Stack& stack = st->get_stack(); VM_LOG(st) << "execute XCHGX\n"; stack.check_underflow(1); - int x = stack.pop_smallint_range(255); + int x = stack.pop_smallint_range(st->get_global_version() >= 4 ? (1 << 30) - 1 : 255); stack.check_underflow_p(x); swap(stack[0], stack[x]); return 0; @@ -499,7 +501,7 @@ int exec_chkdepth(VmState* st) { Stack& stack = st->get_stack(); VM_LOG(st) << "execute CHKDEPTH\n"; stack.check_underflow(1); - int x = stack.pop_smallint_range(255); + int x = stack.pop_smallint_range(st->get_global_version() >= 4 ? (1 << 30) - 1 : 255); stack.check_underflow(x); return 0; } @@ -508,10 +510,11 @@ int exec_onlytop_x(VmState* st) { Stack& stack = st->get_stack(); VM_LOG(st) << "execute ONLYTOPX\n"; stack.check_underflow(1); - int x = stack.pop_smallint_range(255); + int x = stack.pop_smallint_range(st->get_global_version() >= 4 ? (1 << 30) - 1 : 255); stack.check_underflow(x); int n = stack.depth(), d = n - x; if (d > 0) { + st->consume_gas(std::max(x - 255, 0)); for (int i = n - 1; i >= d; i--) { stack[i] = std::move(stack[i - d]); } @@ -524,7 +527,7 @@ int exec_only_x(VmState* st) { Stack& stack = st->get_stack(); VM_LOG(st) << "execute ONLYX\n"; stack.check_underflow(1); - int x = stack.pop_smallint_range(255); + int x = stack.pop_smallint_range(st->get_global_version() >= 4 ? (1 << 30) - 1 : 255); stack.check_underflow(x); stack.pop_many(stack.depth() - x); return 0; diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index a164df83..aab1711f 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -26,8 +26,16 @@ #include "vm/dict.h" #include "vm/boc.h" #include "Ed25519.h" +#include "vm/Hasher.h" +#include "block/block-auto.h" +#include "block/block-parse.h" +#include "crypto/ellcurve/secp256k1.h" +#include "crypto/ellcurve/p256.h" #include "openssl/digest.hpp" +#include +#include "bls.h" +#include "mc-config.h" namespace vm { @@ -60,6 +68,10 @@ int exec_set_gas_generic(VmState* st, long long new_gas_limit) { throw VmNoGas{}; } st->change_gas_limit(new_gas_limit); + if (st->get_stop_on_accept_message()) { + VM_LOG(st) << "External message is accepted, stopping TVM"; + return st->jump(td::Ref{true, 0}); + } return 0; } @@ -78,6 +90,12 @@ int exec_set_gas_limit(VmState* st) { return exec_set_gas_generic(st, gas); } +int exec_gas_consumed(VmState* st) { + VM_LOG(st) << "execute GASCONSUMED"; + st->get_stack().push_smallint(st->gas_consumed()); + return 0; +} + int exec_commit(VmState* st) { VM_LOG(st) << "execute COMMIT"; st->force_commit(); @@ -88,6 +106,7 @@ void register_basic_gas_ops(OpcodeTable& cp0) { using namespace std::placeholders; cp0.insert(OpcodeInstr::mksimple(0xf800, 16, "ACCEPT", exec_accept)) .insert(OpcodeInstr::mksimple(0xf801, 16, "SETGASLIMIT", exec_set_gas_limit)) + .insert(OpcodeInstr::mksimple(0xf807, 16, "GASCONSUMED", exec_gas_consumed)->require_version(4)) .insert(OpcodeInstr::mksimple(0xf80f, 16, "COMMIT", exec_commit)); } @@ -95,17 +114,35 @@ void register_ton_gas_ops(OpcodeTable& cp0) { using namespace std::placeholders; } +static const StackEntry& get_param(VmState* st, unsigned idx) { + auto tuple = st->get_c7(); + auto t1 = tuple_index(tuple, 0).as_tuple_range(255); + if (t1.is_null()) { + throw VmError{Excno::type_chk, "intermediate value is not a tuple"}; + } + return tuple_index(t1, idx); +} + +// ConfigParams: 18 (only one entry), 19, 20, 21, 24, 25, 43 +static td::Ref get_unpacked_config_tuple(VmState* st) { + auto tuple = st->get_c7(); + auto t1 = tuple_index(tuple, 0).as_tuple_range(255); + if (t1.is_null()) { + throw VmError{Excno::type_chk, "intermediate value is not a tuple"}; + } + auto t2 = tuple_index(t1, 14).as_tuple_range(255); + if (t2.is_null()) { + throw VmError{Excno::type_chk, "intermediate value is not a tuple"}; + } + return t2; +} + int exec_get_param(VmState* st, unsigned idx, const char* name) { if (name) { VM_LOG(st) << "execute " << name; } Stack& stack = st->get_stack(); - auto tuple = st->get_c7(); - auto t1 = tuple_index(*tuple, 0).as_tuple_range(255); - if (t1.is_null()) { - throw VmError{Excno::type_chk, "intermediate value is not a tuple"}; - } - stack.push(tuple_index(*t1, idx)); + stack.push(get_param(st, idx)); return 0; } @@ -192,6 +229,135 @@ int exec_set_global_var(VmState* st) { return exec_set_global_common(st, args); } +int exec_get_prev_blocks_info(VmState* st, unsigned idx, const char* name) { + idx &= 3; + VM_LOG(st) << "execute " << name; + Stack& stack = st->get_stack(); + auto tuple = st->get_c7(); + auto t1 = tuple_index(tuple, 0).as_tuple_range(255); + if (t1.is_null()) { + throw VmError{Excno::type_chk, "intermediate value is not a tuple"}; + } + auto t2 = tuple_index(t1, 13).as_tuple_range(255); + if (t2.is_null()) { + throw VmError{Excno::type_chk, "intermediate value is not a tuple"}; + } + stack.push(tuple_index(t2, idx)); + return 0; +} + +int exec_get_global_id(VmState* st) { + VM_LOG(st) << "execute GLOBALID"; + if (st->get_global_version() >= 6) { + Ref cs = tuple_index(get_unpacked_config_tuple(st), 1).as_slice(); + if (cs.is_null()) { + throw VmError{Excno::type_chk, "intermediate value is not a slice"}; + } + if (cs->size() < 32) { + throw VmError{Excno::cell_und, "invalid global-id config"}; + } + st->get_stack().push_smallint(cs->prefetch_long(32)); + } else { + Ref config = get_param(st, 19).as_cell(); + if (config.is_null()) { + throw VmError{Excno::type_chk, "intermediate value is not a cell"}; + } + Dictionary config_dict{std::move(config), 32}; + Ref cell = config_dict.lookup_ref(td::BitArray<32>{19}); + if (cell.is_null()) { + throw VmError{Excno::unknown, "invalid global-id config"}; + } + CellSlice cs = load_cell_slice(cell); + if (cs.size() < 32) { + throw VmError{Excno::unknown, "invalid global-id config"}; + } + st->get_stack().push_smallint(cs.fetch_long(32)); + } + return 0; +} + +int exec_get_gas_fee(VmState* st) { + VM_LOG(st) << "execute GETGASFEE"; + Stack& stack = st->get_stack(); + stack.check_underflow(st->get_global_version() >= 9 ? 2 : 0); + bool is_masterchain = stack.pop_bool(); + td::uint64 gas = stack.pop_long_range(std::numeric_limits::max(), 0); + block::GasLimitsPrices prices = util::get_gas_prices(get_unpacked_config_tuple(st), is_masterchain); + stack.push_int(prices.compute_gas_price(gas)); + return 0; +} + +int exec_get_storage_fee(VmState* st) { + VM_LOG(st) << "execute GETSTORAGEFEE"; + Stack& stack = st->get_stack(); + stack.check_underflow(st->get_global_version() >= 9 ? 4 : 0); + bool is_masterchain = stack.pop_bool(); + td::int64 delta = stack.pop_long_range(std::numeric_limits::max(), 0); + td::uint64 bits = stack.pop_long_range(std::numeric_limits::max(), 0); + td::uint64 cells = stack.pop_long_range(std::numeric_limits::max(), 0); + td::optional maybe_prices = + util::get_storage_prices(get_unpacked_config_tuple(st)); + stack.push_int(util::calculate_storage_fee(maybe_prices, is_masterchain, delta, bits, cells)); + return 0; +} + +int exec_get_forward_fee(VmState* st) { + VM_LOG(st) << "execute GETFORWARDFEE"; + Stack& stack = st->get_stack(); + stack.check_underflow(st->get_global_version() >= 9 ? 3 : 0); + bool is_masterchain = stack.pop_bool(); + td::uint64 bits = stack.pop_long_range(std::numeric_limits::max(), 0); + td::uint64 cells = stack.pop_long_range(std::numeric_limits::max(), 0); + block::MsgPrices prices = util::get_msg_prices(get_unpacked_config_tuple(st), is_masterchain); + stack.push_int(prices.compute_fwd_fees256(cells, bits)); + return 0; +} + +int exec_get_precompiled_gas(VmState* st) { + VM_LOG(st) << "execute GETPRECOMPILEDGAS"; + Stack& stack = st->get_stack(); + stack.push(get_param(st, 16)); + return 0; +} + +int exec_get_original_fwd_fee(VmState* st) { + VM_LOG(st) << "execute GETORIGINALFWDFEE"; + Stack& stack = st->get_stack(); + stack.check_underflow(st->get_global_version() >= 9 ? 2 : 0); + bool is_masterchain = stack.pop_bool(); + td::RefInt256 fwd_fee = stack.pop_int_finite(); + if (fwd_fee->sgn() < 0) { + throw VmError{Excno::range_chk, "fwd_fee is negative"}; + } + block::MsgPrices prices = util::get_msg_prices(get_unpacked_config_tuple(st), is_masterchain); + stack.push_int(td::muldiv(fwd_fee, td::make_refint(1 << 16), td::make_refint((1 << 16) - prices.first_frac))); + return 0; +} + +int exec_get_gas_fee_simple(VmState* st) { + VM_LOG(st) << "execute GETGASFEESIMPLE"; + Stack& stack = st->get_stack(); + stack.check_underflow(st->get_global_version() >= 9 ? 2 : 0); + bool is_masterchain = stack.pop_bool(); + td::uint64 gas = stack.pop_long_range(std::numeric_limits::max(), 0); + block::GasLimitsPrices prices = util::get_gas_prices(get_unpacked_config_tuple(st), is_masterchain); + stack.push_int(td::rshift(td::make_refint(prices.gas_price) * gas, 16, 1)); + return 0; +} + +int exec_get_forward_fee_simple(VmState* st) { + VM_LOG(st) << "execute GETFORWARDFEESIMPLE"; + Stack& stack = st->get_stack(); + stack.check_underflow(st->get_global_version() >= 9 ? 3 : 0); + bool is_masterchain = stack.pop_bool(); + td::uint64 bits = stack.pop_long_range(std::numeric_limits::max(), 0); + td::uint64 cells = stack.pop_long_range(std::numeric_limits::max(), 0); + block::MsgPrices prices = util::get_msg_prices(get_unpacked_config_tuple(st), is_masterchain); + stack.push_int(td::rshift(td::make_refint(prices.bit_price) * bits + td::make_refint(prices.cell_price) * cells, 16, + 1)); // divide by 2^16 with ceil rounding + return 0; +} + void register_ton_config_ops(OpcodeTable& cp0) { using namespace std::placeholders; cp0.insert(OpcodeInstr::mkfixedrange(0xf820, 0xf823, 16, 4, instr::dump_1c("GETPARAM "), exec_get_var_param)) @@ -202,10 +368,26 @@ void register_ton_config_ops(OpcodeTable& cp0) { .insert(OpcodeInstr::mksimple(0xf827, 16, "BALANCE", std::bind(exec_get_param, _1, 7, "BALANCE"))) .insert(OpcodeInstr::mksimple(0xf828, 16, "MYADDR", std::bind(exec_get_param, _1, 8, "MYADDR"))) .insert(OpcodeInstr::mksimple(0xf829, 16, "CONFIGROOT", std::bind(exec_get_param, _1, 9, "CONFIGROOT"))) - .insert(OpcodeInstr::mkfixedrange(0xf82a, 0xf830, 16, 4, instr::dump_1c("GETPARAM "), exec_get_var_param)) + .insert(OpcodeInstr::mksimple(0xf82a, 16, "MYCODE", std::bind(exec_get_param, _1, 10, "MYCODE"))) + .insert(OpcodeInstr::mksimple(0xf82b, 16, "INCOMINGVALUE", std::bind(exec_get_param, _1, 11, "INCOMINGVALUE"))) + .insert(OpcodeInstr::mksimple(0xf82c, 16, "STORAGEFEES", std::bind(exec_get_param, _1, 12, "STORAGEFEES"))) + .insert(OpcodeInstr::mksimple(0xf82d, 16, "PREVBLOCKSINFOTUPLE", std::bind(exec_get_param, _1, 13, "PREVBLOCKSINFOTUPLE"))) + .insert(OpcodeInstr::mksimple(0xf82e, 16, "UNPACKEDCONFIGTUPLE", std::bind(exec_get_param, _1, 14, "UNPACKEDCONFIGTUPLE"))) + .insert(OpcodeInstr::mksimple(0xf82f, 16, "DUEPAYMENT", std::bind(exec_get_param, _1, 15, "DUEPAYMENT"))) .insert(OpcodeInstr::mksimple(0xf830, 16, "CONFIGDICT", exec_get_config_dict)) .insert(OpcodeInstr::mksimple(0xf832, 16, "CONFIGPARAM", std::bind(exec_get_config_param, _1, false))) .insert(OpcodeInstr::mksimple(0xf833, 16, "CONFIGOPTPARAM", std::bind(exec_get_config_param, _1, true))) + .insert(OpcodeInstr::mksimple(0xf83400, 24, "PREVMCBLOCKS", std::bind(exec_get_prev_blocks_info, _1, 0, "PREVMCBLOCKS"))->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf83401, 24, "PREVKEYBLOCK", std::bind(exec_get_prev_blocks_info, _1, 1, "PREVKEYBLOCK"))->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf83402, 24, "PREVMCBLOCKS_100", std::bind(exec_get_prev_blocks_info, _1, 2, "PREVMCBLOCKS_100"))->require_version(9)) + .insert(OpcodeInstr::mksimple(0xf835, 16, "GLOBALID", exec_get_global_id)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf836, 16, "GETGASFEE", exec_get_gas_fee)->require_version(6)) + .insert(OpcodeInstr::mksimple(0xf837, 16, "GETSTORAGEFEE", exec_get_storage_fee)->require_version(6)) + .insert(OpcodeInstr::mksimple(0xf838, 16, "GETFORWARDFEE", exec_get_forward_fee)->require_version(6)) + .insert(OpcodeInstr::mksimple(0xf839, 16, "GETPRECOMPILEDGAS", exec_get_precompiled_gas)->require_version(6)) + .insert(OpcodeInstr::mksimple(0xf83a, 16, "GETORIGINALFWDFEE", exec_get_original_fwd_fee)->require_version(6)) + .insert(OpcodeInstr::mksimple(0xf83b, 16, "GETGASFEESIMPLE", exec_get_gas_fee_simple)->require_version(6)) + .insert(OpcodeInstr::mksimple(0xf83c, 16, "GETFORWARDFEESIMPLE", exec_get_forward_fee_simple)->require_version(6)) .insert(OpcodeInstr::mksimple(0xf840, 16, "GETGLOBVAR", exec_get_global_var)) .insert(OpcodeInstr::mkfixedrange(0xf841, 0xf860, 16, 5, instr::dump_1c_and(31, "GETGLOB "), exec_get_global)) .insert(OpcodeInstr::mksimple(0xf860, 16, "SETGLOBVAR", exec_set_global_var)) @@ -216,11 +398,11 @@ static constexpr int randseed_idx = 6; td::RefInt256 generate_randu256(VmState* st) { auto tuple = st->get_c7(); - auto t1 = tuple_index(*tuple, 0).as_tuple_range(255); + auto t1 = tuple_index(tuple, 0).as_tuple_range(255); if (t1.is_null()) { throw VmError{Excno::type_chk, "intermediate value is not a tuple"}; } - auto seedv = tuple_index(*t1, randseed_idx).as_int(); + auto seedv = tuple_index(t1, randseed_idx).as_int(); if (seedv.is_null()) { throw VmError{Excno::type_chk, "random seed is not an integer"}; } @@ -276,12 +458,12 @@ int exec_set_rand(VmState* st, bool mix) { throw VmError{Excno::range_chk, "new random seed out of range"}; } auto tuple = st->get_c7(); - auto t1 = tuple_index(*tuple, 0).as_tuple_range(255); + auto t1 = tuple_index(tuple, 0).as_tuple_range(255); if (t1.is_null()) { throw VmError{Excno::type_chk, "intermediate value is not a tuple"}; } if (mix) { - auto seedv = tuple_index(*t1, randseed_idx).as_int(); + auto seedv = tuple_index(t1, randseed_idx).as_int(); if (seedv.is_null()) { throw VmError{Excno::type_chk, "random seed is not an integer"}; } @@ -356,6 +538,78 @@ int exec_compute_sha256(VmState* st) { return 0; } +int exec_hash_ext(VmState* st, unsigned args) { + bool rev = (args >> 8) & 1; + bool append = (args >> 9) & 1; + int hash_id = args & 255; + VM_LOG(st) << "execute HASHEXT" << (append ? "A" : "") << (rev ? "R" : "") << " " << (hash_id == 255 ? -1 : hash_id); + Stack& stack = st->get_stack(); + if (hash_id == 255) { + stack.check_underflow(st->get_global_version() >= 9 ? 2 : 0); + hash_id = stack.pop_smallint_range(254); + } + int cnt = stack.pop_smallint_range(stack.depth() - 1 - (st->get_global_version() >= 9 ? (int)append : 0)); + Hasher hasher{hash_id}; + size_t total_bits = 0; + long long gas_consumed = 0; + for (int i = 0; i < cnt; ++i) { + td::ConstBitPtr data{nullptr}; + unsigned size; + int idx = rev ? i : cnt - 1 - i; + auto slice = stack[idx].as_slice(); + if (slice.not_null()) { + data = slice->data_bits(); + size = slice->size(); + } else { + auto builder = stack[idx].as_builder(); + if (builder.not_null()) { + data = builder->data_bits(); + size = builder->size(); + } else { + stack.pop_many(cnt); + throw VmError{Excno::type_chk, "expected slice or builder"}; + } + } + total_bits += size; + long long gas_total = (i + 1) * VmState::hash_ext_entry_gas_price + total_bits / 8 / hasher.bytes_per_gas_unit(); + st->consume_gas(gas_total - gas_consumed); + gas_consumed = gas_total; + hasher.append(data, size); + } + stack.pop_many(cnt); + td::BufferSlice hash = hasher.finish(); + if (append) { + Ref builder = stack.pop_builder(); + if (!builder->can_extend_by(hash.size() * 8)) { + throw VmError{Excno::cell_ov}; + } + builder.write().store_bytes(hash.as_slice()); + stack.push_builder(std::move(builder)); + } else { + if (hash.size() <= 32) { + td::RefInt256 res{true}; + CHECK(res.write().import_bytes((unsigned char*)hash.data(), hash.size(), false)); + stack.push_int(std::move(res)); + } else { + std::vector res; + for (size_t i = 0; i < hash.size(); i += 32) { + td::RefInt256 x{true}; + CHECK(x.write().import_bytes((unsigned char*)hash.data() + i, std::min(hash.size() - i, 32), false)); + res.push_back(std::move(x)); + } + stack.push_tuple(std::move(res)); + } + } + return 0; +} + +std::string dump_hash_ext(CellSlice& cs, unsigned args) { + bool rev = (args >> 8) & 1; + bool append = (args >> 9) & 1; + int hash_id = args & 255; + return PSTRING() << "HASHEXT" << (append ? "A" : "") << (rev ? "R" : "") << " " << (hash_id == 255 ? -1 : hash_id); +} + int exec_ed25519_check_signature(VmState* st, bool from_slice) { VM_LOG(st) << "execute CHKSIGN" << (from_slice ? 'S' : 'U'); Stack& stack = st->get_stack(); @@ -385,23 +639,669 @@ int exec_ed25519_check_signature(VmState* st, bool from_slice) { if (!key_int->export_bytes(key, 32, false)) { throw VmError{Excno::range_chk, "Ed25519 public key must fit in an unsigned 256-bit integer"}; } + st->register_chksgn_call(); td::Ed25519::PublicKey pub_key{td::SecureString(td::Slice{key, 32})}; auto res = pub_key.verify_signature(td::Slice{data, data_len}, td::Slice{signature, 64}); stack.push_bool(res.is_ok() || st->get_chksig_always_succeed()); return 0; } +int exec_ecrecover(VmState* st) { + VM_LOG(st) << "execute ECRECOVER"; + Stack& stack = st->get_stack(); + stack.check_underflow(4); + auto s = stack.pop_int(); + auto r = stack.pop_int(); + auto v = (td::uint8)stack.pop_smallint_range(255); + auto hash = stack.pop_int(); + + unsigned char signature[65]; + if (!r->export_bytes(signature, 32, false)) { + throw VmError{Excno::range_chk, "r must fit in an unsigned 256-bit integer"}; + } + if (!s->export_bytes(signature + 32, 32, false)) { + throw VmError{Excno::range_chk, "s must fit in an unsigned 256-bit integer"}; + } + signature[64] = v; + unsigned char hash_bytes[32]; + if (!hash->export_bytes(hash_bytes, 32, false)) { + throw VmError{Excno::range_chk, "data hash must fit in an unsigned 256-bit integer"}; + } + st->consume_gas(VmState::ecrecover_gas_price); + unsigned char public_key[65]; + if (td::secp256k1::ecrecover(hash_bytes, signature, public_key)) { + td::uint8 h = public_key[0]; + td::RefInt256 x1{true}, x2{true}; + CHECK(x1.write().import_bytes(public_key + 1, 32, false)); + CHECK(x2.write().import_bytes(public_key + 33, 32, false)); + stack.push_smallint(h); + stack.push_int(std::move(x1)); + stack.push_int(std::move(x2)); + stack.push_bool(true); + } else { + stack.push_bool(false); + } + return 0; +} + +int exec_secp256k1_xonly_pubkey_tweak_add(VmState* st) { + VM_LOG(st) << "execute SECP256K1_XONLY_PUBKEY_TWEAK_ADD"; + Stack& stack = st->get_stack(); + stack.check_underflow(2); + auto tweak_int = stack.pop_int(); + auto key_int = stack.pop_int(); + + unsigned char key[32], tweak[32]; + if (!key_int->export_bytes(key, 32, false)) { + throw VmError{Excno::range_chk, "key must fit in an unsigned 256-bit integer"}; + } + if (!tweak_int->export_bytes(tweak, 32, false)) { + throw VmError{Excno::range_chk, "tweak must fit in an unsigned 256-bit integer"}; + } + st->consume_gas(VmState::secp256k1_xonly_pubkey_tweak_add_gas_price); + unsigned char public_key[65]; + if (td::secp256k1::xonly_pubkey_tweak_add(key, tweak, public_key)) { + td::uint8 h = public_key[0]; + td::RefInt256 x1{true}, x2{true}; + CHECK(x1.write().import_bytes(public_key + 1, 32, false)); + CHECK(x2.write().import_bytes(public_key + 33, 32, false)); + stack.push_smallint(h); + stack.push_int(std::move(x1)); + stack.push_int(std::move(x2)); + stack.push_bool(true); + } else { + stack.push_bool(false); + } + return 0; +} + +int exec_p256_chksign(VmState* st, bool from_slice) { + VM_LOG(st) << "execute P256_CHKSIGN" << (from_slice ? 'S' : 'U'); + Stack& stack = st->get_stack(); + stack.check_underflow(3); + auto key_cs = stack.pop_cellslice(); + auto signature_cs = stack.pop_cellslice(); + unsigned char data[128], key[33], signature[64]; + unsigned data_len; + if (from_slice) { + auto cs = stack.pop_cellslice(); + if (cs->size() & 7) { + throw VmError{Excno::cell_und, "Slice does not consist of an integer number of bytes"}; + } + data_len = (cs->size() >> 3); + CHECK(data_len <= sizeof(data)); + CHECK(cs->prefetch_bytes(data, data_len)); + } else { + auto hash_int = stack.pop_int(); + data_len = 32; + if (!hash_int->export_bytes(data, data_len, false)) { + throw VmError{Excno::range_chk, "data hash must fit in an unsigned 256-bit integer"}; + } + } + if (!signature_cs->prefetch_bytes(signature, 64)) { + throw VmError{Excno::cell_und, "P256 signature must contain at least 512 data bits"}; + } + if (!key_cs->prefetch_bytes(key, 33)) { + throw VmError{Excno::cell_und, "P256 public key must contain at least 33 data bytes"}; + } + st->consume_gas(VmState::p256_chksgn_gas_price); + auto res = td::p256_check_signature(td::Slice{data, data_len}, td::Slice{key, 33}, td::Slice{signature, 64}); + if (res.is_error()) { + VM_LOG(st) << "P256_CHKSIGN: " << res.error().message(); + } + stack.push_bool(res.is_ok() || st->get_chksig_always_succeed()); + return 0; +} + +static_assert(crypto_scalarmult_ristretto255_BYTES == 32, "Unexpected value of ristretto255 constant"); +static_assert(crypto_scalarmult_ristretto255_SCALARBYTES == 32, "Unexpected value of ristretto255 constant"); +static_assert(crypto_core_ristretto255_BYTES == 32, "Unexpected value of ristretto255 constant"); +static_assert(crypto_core_ristretto255_HASHBYTES == 64, "Unexpected value of ristretto255 constant"); +static_assert(crypto_core_ristretto255_SCALARBYTES == 32, "Unexpected value of ristretto255 constant"); +static_assert(crypto_core_ristretto255_NONREDUCEDSCALARBYTES == 64, "Unexpected value of ristretto255 constant"); + +int exec_ristretto255_from_hash(VmState* st) { + VM_LOG(st) << "execute RIST255_FROMHASH"; + Stack& stack = st->get_stack(); + stack.check_underflow(2); + auto x2 = stack.pop_int(); + auto x1 = stack.pop_int(); + st->consume_gas(VmState::rist255_fromhash_gas_price); + unsigned char xb[64], rb[32]; + if (!x1->export_bytes(xb, 32, false)) { + throw VmError{Excno::range_chk, "x1 must fit in an unsigned 256-bit integer"}; + } + if (!x2->export_bytes(xb + 32, 32, false)) { + throw VmError{Excno::range_chk, "x2 must fit in an unsigned 256-bit integer"}; + } + crypto_core_ristretto255_from_hash(rb, xb); + td::RefInt256 r{true}; + CHECK(r.write().import_bytes(rb, 32, false)); + stack.push_int(std::move(r)); + return 0; +} + +int exec_ristretto255_validate(VmState* st, bool quiet) { + VM_LOG(st) << "execute RIST255_VALIDATE"; + Stack& stack = st->get_stack(); + auto x = stack.pop_int(); + st->consume_gas(VmState::rist255_validate_gas_price); + unsigned char xb[32]; + if (!x->export_bytes(xb, 32, false) || !crypto_core_ristretto255_is_valid_point(xb)) { + if (quiet) { + stack.push_bool(false); + return 0; + } + throw VmError{Excno::range_chk, "x is not a valid encoded element"}; + } + if (quiet) { + stack.push_bool(true); + } + return 0; +} + +int exec_ristretto255_add(VmState* st, bool quiet) { + VM_LOG(st) << "execute RIST255_ADD"; + Stack& stack = st->get_stack(); + stack.check_underflow(2); + auto y = stack.pop_int(); + auto x = stack.pop_int(); + st->consume_gas(VmState::rist255_add_gas_price); + unsigned char xb[32], yb[32], rb[32]; + if (!x->export_bytes(xb, 32, false) || !y->export_bytes(yb, 32, false) || crypto_core_ristretto255_add(rb, xb, yb)) { + if (quiet) { + stack.push_bool(false); + return 0; + } + throw VmError{Excno::range_chk, "x and/or y are not valid encoded elements"}; + } + td::RefInt256 r{true}; + CHECK(r.write().import_bytes(rb, 32, false)); + stack.push_int(std::move(r)); + if (quiet) { + stack.push_bool(true); + } + return 0; +} + +int exec_ristretto255_sub(VmState* st, bool quiet) { + VM_LOG(st) << "execute RIST255_SUB"; + Stack& stack = st->get_stack(); + stack.check_underflow(2); + auto y = stack.pop_int(); + auto x = stack.pop_int(); + st->consume_gas(VmState::rist255_add_gas_price); + unsigned char xb[32], yb[32], rb[32]; + if (!x->export_bytes(xb, 32, false) || !y->export_bytes(yb, 32, false) || crypto_core_ristretto255_sub(rb, xb, yb)) { + if (quiet) { + stack.push_bool(false); + return 0; + } + throw VmError{Excno::range_chk, "x and/or y are not valid encoded elements"}; + } + td::RefInt256 r{true}; + CHECK(r.write().import_bytes(rb, 32, false)); + stack.push_int(std::move(r)); + if (quiet) { + stack.push_bool(true); + } + return 0; +} + +static bool export_bytes_little(const td::RefInt256& n, unsigned char* nb) { + if (!n->export_bytes(nb, 32, false)) { + return false; + } + std::reverse(nb, nb + 32); + return true; +} + +static td::RefInt256 get_ristretto256_l() { + static td::RefInt256 l = + (td::make_refint(1) << 252) + td::dec_string_to_int256(td::Slice("27742317777372353535851937790883648493")); + return l; +} + +int exec_ristretto255_mul(VmState* st, bool quiet) { + VM_LOG(st) << "execute RIST255_MUL"; + Stack& stack = st->get_stack(); + stack.check_underflow(2); + auto n = stack.pop_int() % get_ristretto256_l(); + auto x = stack.pop_int(); + st->consume_gas(VmState::rist255_mul_gas_price); + if (n->sgn() == 0) { + stack.push_smallint(0); + if (quiet) { + stack.push_bool(true); + } + return 0; + } + unsigned char xb[32], nb[32], rb[32]; + if (!x->export_bytes(xb, 32, false) || !export_bytes_little(n, nb) || crypto_scalarmult_ristretto255(rb, nb, xb)) { + if (quiet) { + stack.push_bool(false); + return 0; + } + throw VmError{Excno::range_chk, "invalid x or n"}; + } + td::RefInt256 r{true}; + CHECK(r.write().import_bytes(rb, 32, false)); + stack.push_int(std::move(r)); + if (quiet) { + stack.push_bool(true); + } + return 0; +} + +int exec_ristretto255_mul_base(VmState* st, bool quiet) { + VM_LOG(st) << "execute RIST255_MULBASE"; + Stack& stack = st->get_stack(); + auto n = stack.pop_int() % get_ristretto256_l(); + st->consume_gas(VmState::rist255_mulbase_gas_price); + unsigned char nb[32], rb[32]; + memset(rb, 255, sizeof(rb)); + if (!export_bytes_little(n, nb) || crypto_scalarmult_ristretto255_base(rb, nb)) { + if (std::all_of(rb, rb + 32, [](unsigned char c) { return c == 255; })) { + if (quiet) { + stack.push_bool(false); + return 0; + } + throw VmError{Excno::range_chk, "invalid n"}; + } + } + td::RefInt256 r{true}; + CHECK(r.write().import_bytes(rb, 32, false)); + stack.push_int(std::move(r)); + if (quiet) { + stack.push_bool(true); + } + return 0; +} + +int exec_ristretto255_push_l(VmState* st) { + VM_LOG(st) << "execute RIST255_PUSHL"; + Stack& stack = st->get_stack(); + stack.push_int(get_ristretto256_l()); + return 0; +} + +static bls::P1 slice_to_bls_p1(const CellSlice& cs) { + bls::P1 p1; + if (!cs.prefetch_bytes(p1.as_slice())) { + throw VmError{Excno::cell_und, PSTRING() << "slice must contain at least " << bls::P1_SIZE << " bytes"}; + } + return p1; +} + +static bls::P2 slice_to_bls_p2(const CellSlice& cs) { + bls::P2 p2; + if (!cs.prefetch_bytes(p2.as_slice())) { + throw VmError{Excno::cell_und, PSTRING() << "slice must contain at least " << bls::P2_SIZE << " bytes"}; + } + return p2; +} + +static bls::FP slice_to_bls_fp(const CellSlice& cs) { + bls::FP fp; + if (!cs.prefetch_bytes(fp.as_slice())) { + throw VmError{Excno::cell_und, PSTRING() << "slice must contain at least " << bls::FP_SIZE << " bytes"}; + } + return fp; +} + +static bls::FP2 slice_to_bls_fp2(const CellSlice& cs) { + bls::FP2 fp2; + if (!cs.prefetch_bytes(fp2.as_slice())) { + throw VmError{Excno::cell_und, PSTRING() << "slice must contain at least " << bls::FP_SIZE * 2 << " bytes"}; + } + return fp2; +} + +static td::BufferSlice slice_to_bls_msg(const CellSlice& cs) { + if (cs.size() % 8 != 0) { + throw VmError{Excno::cell_und, "message does not consist of an integer number of bytes"}; + } + size_t msg_size = cs.size() / 8; + td::BufferSlice s(msg_size); + cs.prefetch_bytes((td::uint8*)s.data(), (int)msg_size); + return s; +} + +static Ref bls_to_slice(td::Slice s) { + VmStateInterface::Guard guard{nullptr}; // Don't consume gas for finalize and load_cell_slice + CellBuilder cb; + return load_cell_slice_ref(cb.store_bytes(s).finalize()); +} + +static long long bls_calculate_multiexp_gas(int n, long long base, long long coef1, long long coef2) { + int l = 4; + while ((1LL << (l + 1)) <= n) { + ++l; + } + return base + n * coef1 + n * coef2 / l; +} + +int exec_bls_verify(VmState* st) { + VM_LOG(st) << "execute BLS_VERIFY"; + Stack& stack = st->get_stack(); + stack.check_underflow(3); + st->consume_gas(VmState::bls_verify_gas_price); + bls::P2 sig = slice_to_bls_p2(*stack.pop_cellslice()); + td::BufferSlice msg = slice_to_bls_msg(*stack.pop_cellslice()); + bls::P1 pub = slice_to_bls_p1(*stack.pop_cellslice()); + stack.push_bool(bls::verify(pub, msg, sig)); + return 0; +} + +int exec_bls_aggregate(VmState* st) { + VM_LOG(st) << "execute BLS_AGGREGATE"; + Stack& stack = st->get_stack(); + int n = stack.pop_smallint_range(stack.depth() - 1, 1); + st->consume_gas(VmState::bls_aggregate_base_gas_price + (long long)n * VmState::bls_aggregate_element_gas_price); + std::vector sigs(n); + for (int i = n - 1; i >= 0; --i) { + sigs[i] = slice_to_bls_p2(*stack.pop_cellslice()); + } + bls::P2 aggregated = bls::aggregate(sigs); + stack.push_cellslice(bls_to_slice(aggregated.as_slice())); + return 0; +} + +int exec_bls_fast_aggregate_verify(VmState* st) { + VM_LOG(st) << "execute BLS_FASTAGGREGATEVERIFY"; + Stack& stack = st->get_stack(); + stack.check_underflow(3); + Ref sig = stack.pop_cellslice(); + Ref msg = stack.pop_cellslice(); + int n = stack.pop_smallint_range(stack.depth() - 1); + st->consume_gas(VmState::bls_fast_aggregate_verify_base_gas_price + + (long long)n * VmState::bls_fast_aggregate_verify_element_gas_price); + std::vector pubs(n); + for (int i = n - 1; i >= 0; --i) { + pubs[i] = slice_to_bls_p1(*stack.pop_cellslice()); + } + stack.push_bool(bls::fast_aggregate_verify(pubs, slice_to_bls_msg(*msg), slice_to_bls_p2(*sig))); + return 0; +} + +int exec_bls_aggregate_verify(VmState* st) { + VM_LOG(st) << "execute BLS_AGGREGATEVERIFY"; + Stack& stack = st->get_stack(); + stack.check_underflow(2); + Ref sig = stack.pop_cellslice(); + int n = stack.pop_smallint_range((stack.depth() - 1) / 2); + st->consume_gas(VmState::bls_aggregate_verify_base_gas_price + + (long long)n * VmState::bls_aggregate_verify_element_gas_price); + std::vector> vec(n); + for (int i = n - 1; i >= 0; --i) { + vec[i].second = slice_to_bls_msg(*stack.pop_cellslice()); + vec[i].first = slice_to_bls_p1(*stack.pop_cellslice()); + } + stack.push_bool(bls::aggregate_verify(vec, slice_to_bls_p2(*sig))); + return 0; +} + +int exec_bls_g1_add(VmState* st) { + VM_LOG(st) << "execute BLS_G1_ADD"; + Stack& stack = st->get_stack(); + stack.check_underflow(2); + st->consume_gas(VmState::bls_g1_add_sub_gas_price); + bls::P1 b = slice_to_bls_p1(*stack.pop_cellslice()); + bls::P1 a = slice_to_bls_p1(*stack.pop_cellslice()); + stack.push_cellslice(bls_to_slice(bls::g1_add(a, b).as_slice())); + return 0; +} + +int exec_bls_g1_sub(VmState* st) { + VM_LOG(st) << "execute BLS_G1_SUB"; + Stack& stack = st->get_stack(); + stack.check_underflow(2); + st->consume_gas(VmState::bls_g1_add_sub_gas_price); + bls::P1 b = slice_to_bls_p1(*stack.pop_cellslice()); + bls::P1 a = slice_to_bls_p1(*stack.pop_cellslice()); + stack.push_cellslice(bls_to_slice(bls::g1_sub(a, b).as_slice())); + return 0; +} + +int exec_bls_g1_neg(VmState* st) { + VM_LOG(st) << "execute BLS_G1_NEG"; + Stack& stack = st->get_stack(); + st->consume_gas(VmState::bls_g1_neg_gas_price); + bls::P1 a = slice_to_bls_p1(*stack.pop_cellslice()); + stack.push_cellslice(bls_to_slice(bls::g1_neg(a).as_slice())); + return 0; +} + +int exec_bls_g1_mul(VmState* st) { + VM_LOG(st) << "execute BLS_G1_MUL"; + Stack& stack = st->get_stack(); + stack.check_underflow(2); + st->consume_gas(VmState::bls_g1_mul_gas_price); + td::RefInt256 x = stack.pop_int_finite(); + bls::P1 p = slice_to_bls_p1(*stack.pop_cellslice()); + stack.push_cellslice(bls_to_slice(bls::g1_mul(p, x).as_slice())); + return 0; +} + +int exec_bls_g1_multiexp(VmState* st) { + VM_LOG(st) << "execute BLS_G1_MULTIEXP"; + Stack& stack = st->get_stack(); + int n = stack.pop_smallint_range((stack.depth() - 1) / 2); + st->consume_gas(bls_calculate_multiexp_gas(n, VmState::bls_g1_multiexp_base_gas_price, + VmState::bls_g1_multiexp_coef1_gas_price, + VmState::bls_g1_multiexp_coef2_gas_price)); + std::vector> ps(n); + for (int i = n - 1; i >= 0; --i) { + ps[i].second = stack.pop_int_finite(); + ps[i].first = slice_to_bls_p1(*stack.pop_cellslice()); + } + stack.push_cellslice(bls_to_slice(bls::g1_multiexp(ps).as_slice())); + return 0; +} + +int exec_bls_g1_zero(VmState* st) { + VM_LOG(st) << "execute BLS_G1_ZERO"; + Stack& stack = st->get_stack(); + stack.push_cellslice(bls_to_slice(bls::g1_zero().as_slice())); + return 0; +} + +int exec_bls_map_to_g1(VmState* st) { + VM_LOG(st) << "execute BLS_MAP_TO_G1"; + Stack& stack = st->get_stack(); + st->consume_gas(VmState::bls_map_to_g1_gas_price); + bls::FP a = slice_to_bls_fp(*stack.pop_cellslice()); + stack.push_cellslice(bls_to_slice(bls::map_to_g1(a).as_slice())); + return 0; +} + +int exec_bls_g1_in_group(VmState* st) { + VM_LOG(st) << "execute BLS_G1_INGROUP"; + Stack& stack = st->get_stack(); + st->consume_gas(VmState::bls_g1_in_group_gas_price); + bls::P1 a = slice_to_bls_p1(*stack.pop_cellslice()); + stack.push_bool(bls::g1_in_group(a)); + return 0; +} + +int exec_bls_g1_is_zero(VmState* st) { + VM_LOG(st) << "execute BLS_G1_ISZERO"; + Stack& stack = st->get_stack(); + bls::P1 a = slice_to_bls_p1(*stack.pop_cellslice()); + stack.push_bool(bls::g1_is_zero(a)); + return 0; +} + +int exec_bls_g2_add(VmState* st) { + VM_LOG(st) << "execute BLS_G2_ADD"; + Stack& stack = st->get_stack(); + stack.check_underflow(2); + st->consume_gas(VmState::bls_g2_add_sub_gas_price); + bls::P2 b = slice_to_bls_p2(*stack.pop_cellslice()); + bls::P2 a = slice_to_bls_p2(*stack.pop_cellslice()); + stack.push_cellslice(bls_to_slice(bls::g2_add(a, b).as_slice())); + return 0; +} + +int exec_bls_g2_sub(VmState* st) { + VM_LOG(st) << "execute BLS_G2_SUB"; + Stack& stack = st->get_stack(); + stack.check_underflow(2); + st->consume_gas(VmState::bls_g2_add_sub_gas_price); + bls::P2 b = slice_to_bls_p2(*stack.pop_cellslice()); + bls::P2 a = slice_to_bls_p2(*stack.pop_cellslice()); + stack.push_cellslice(bls_to_slice(bls::g2_sub(a, b).as_slice())); + return 0; +} + +int exec_bls_g2_neg(VmState* st) { + VM_LOG(st) << "execute BLS_G2_NEG"; + Stack& stack = st->get_stack(); + st->consume_gas(VmState::bls_g2_neg_gas_price); + bls::P2 a = slice_to_bls_p2(*stack.pop_cellslice()); + stack.push_cellslice(bls_to_slice(bls::g2_neg(a).as_slice())); + return 0; +} + +int exec_bls_g2_mul(VmState* st) { + VM_LOG(st) << "execute BLS_G2_MUL"; + Stack& stack = st->get_stack(); + stack.check_underflow(2); + st->consume_gas(VmState::bls_g2_mul_gas_price); + td::RefInt256 x = stack.pop_int_finite(); + bls::P2 p = slice_to_bls_p2(*stack.pop_cellslice()); + stack.push_cellslice(bls_to_slice(bls::g2_mul(p, x).as_slice())); + return 0; +} + +int exec_bls_g2_multiexp(VmState* st) { + VM_LOG(st) << "execute BLS_G2_MULTIEXP"; + Stack& stack = st->get_stack(); + int n = stack.pop_smallint_range((stack.depth() - 1) / 2); + st->consume_gas(bls_calculate_multiexp_gas(n, VmState::bls_g2_multiexp_base_gas_price, + VmState::bls_g2_multiexp_coef1_gas_price, + VmState::bls_g2_multiexp_coef2_gas_price)); + std::vector> ps(n); + for (int i = n - 1; i >= 0; --i) { + ps[i].second = stack.pop_int_finite(); + ps[i].first = slice_to_bls_p2(*stack.pop_cellslice()); + } + stack.push_cellslice(bls_to_slice(bls::g2_multiexp(ps).as_slice())); + return 0; +} + +int exec_bls_g2_zero(VmState* st) { + VM_LOG(st) << "execute BLS_G2_ZERO"; + Stack& stack = st->get_stack(); + stack.push_cellslice(bls_to_slice(bls::g2_zero().as_slice())); + return 0; +} + +int exec_bls_map_to_g2(VmState* st) { + VM_LOG(st) << "execute BLS_MAP_TO_G2"; + Stack& stack = st->get_stack(); + st->consume_gas(VmState::bls_map_to_g2_gas_price); + bls::FP2 a = slice_to_bls_fp2(*stack.pop_cellslice()); + stack.push_cellslice(bls_to_slice(bls::map_to_g2(a).as_slice())); + return 0; +} + +int exec_bls_g2_in_group(VmState* st) { + VM_LOG(st) << "execute BLS_G2_INGROUP"; + Stack& stack = st->get_stack(); + st->consume_gas(VmState::bls_g2_in_group_gas_price); + bls::P2 a = slice_to_bls_p2(*stack.pop_cellslice()); + stack.push_bool(bls::g2_in_group(a)); + return 0; +} + +int exec_bls_g2_is_zero(VmState* st) { + VM_LOG(st) << "execute BLS_G2_ISZERO"; + Stack& stack = st->get_stack(); + bls::P2 a = slice_to_bls_p2(*stack.pop_cellslice()); + stack.push_bool(bls::g2_is_zero(a)); + return 0; +} + +int exec_bls_pairing(VmState* st) { + VM_LOG(st) << "execute BLS_PAIRING"; + Stack& stack = st->get_stack(); + int n = stack.pop_smallint_range((stack.depth() - 1) / 2); + st->consume_gas(VmState::bls_pairing_base_gas_price + (long long)n * VmState::bls_pairing_element_gas_price); + std::vector> ps(n); + for (int i = n - 1; i >= 0; --i) { + ps[i].second = slice_to_bls_p2(*stack.pop_cellslice()); + ps[i].first = slice_to_bls_p1(*stack.pop_cellslice()); + } + stack.push_bool(bls::pairing(ps)); + return 0; +} + +int exec_bls_push_r(VmState* st) { + VM_LOG(st) << "execute BLS_PUSHR"; + Stack& stack = st->get_stack(); + stack.push_int(bls::get_r()); + return 0; +} + void register_ton_crypto_ops(OpcodeTable& cp0) { using namespace std::placeholders; cp0.insert(OpcodeInstr::mksimple(0xf900, 16, "HASHCU", std::bind(exec_compute_hash, _1, 0))) .insert(OpcodeInstr::mksimple(0xf901, 16, "HASHSU", std::bind(exec_compute_hash, _1, 1))) .insert(OpcodeInstr::mksimple(0xf902, 16, "SHA256U", exec_compute_sha256)) + .insert(OpcodeInstr::mkfixed(0xf904 >> 2, 14, 10, dump_hash_ext, exec_hash_ext)->require_version(4)) .insert(OpcodeInstr::mksimple(0xf910, 16, "CHKSIGNU", std::bind(exec_ed25519_check_signature, _1, false))) - .insert(OpcodeInstr::mksimple(0xf911, 16, "CHKSIGNS", std::bind(exec_ed25519_check_signature, _1, true))); + .insert(OpcodeInstr::mksimple(0xf911, 16, "CHKSIGNS", std::bind(exec_ed25519_check_signature, _1, true))) + .insert(OpcodeInstr::mksimple(0xf912, 16, "ECRECOVER", exec_ecrecover)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf913, 16, "SECP256K1_XONLY_PUBKEY_TWEAK_ADD", exec_secp256k1_xonly_pubkey_tweak_add)->require_version(9)) + .insert(OpcodeInstr::mksimple(0xf914, 16, "P256_CHKSIGNU", std::bind(exec_p256_chksign, _1, false))->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf915, 16, "P256_CHKSIGNS", std::bind(exec_p256_chksign, _1, true))->require_version(4)) + + .insert(OpcodeInstr::mksimple(0xf920, 16, "RIST255_FROMHASH", exec_ristretto255_from_hash)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf921, 16, "RIST255_VALIDATE", std::bind(exec_ristretto255_validate, _1, false))->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf922, 16, "RIST255_ADD", std::bind(exec_ristretto255_add, _1, false))->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf923, 16, "RIST255_SUB", std::bind(exec_ristretto255_sub, _1, false))->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf924, 16, "RIST255_MUL", std::bind(exec_ristretto255_mul, _1, false))->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf925, 16, "RIST255_MULBASE", std::bind(exec_ristretto255_mul_base, _1, false))->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf926, 16, "RIST255_PUSHL", exec_ristretto255_push_l)->require_version(4)) + + .insert(OpcodeInstr::mksimple(0xb7f921, 24, "RIST255_QVALIDATE", std::bind(exec_ristretto255_validate, _1, true))->require_version(4)) + .insert(OpcodeInstr::mksimple(0xb7f922, 24, "RIST255_QADD", std::bind(exec_ristretto255_add, _1, true))->require_version(4)) + .insert(OpcodeInstr::mksimple(0xb7f923, 24, "RIST255_QSUB", std::bind(exec_ristretto255_sub, _1, true))->require_version(4)) + .insert(OpcodeInstr::mksimple(0xb7f924, 24, "RIST255_QMUL", std::bind(exec_ristretto255_mul, _1, true))->require_version(4)) + .insert(OpcodeInstr::mksimple(0xb7f925, 24, "RIST255_QMULBASE", std::bind(exec_ristretto255_mul_base, _1, true))->require_version(4)) + + .insert(OpcodeInstr::mksimple(0xf93000, 24, "BLS_VERIFY", exec_bls_verify)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf93001, 24, "BLS_AGGREGATE", exec_bls_aggregate)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf93002, 24, "BLS_FASTAGGREGATEVERIFY", exec_bls_fast_aggregate_verify)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf93003, 24, "BLS_AGGREGATEVERIFY", exec_bls_aggregate_verify)->require_version(4)) + + .insert(OpcodeInstr::mksimple(0xf93010, 24, "BLS_G1_ADD", exec_bls_g1_add)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf93011, 24, "BLS_G1_SUB", exec_bls_g1_sub)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf93012, 24, "BLS_G1_NEG", exec_bls_g1_neg)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf93013, 24, "BLS_G1_MUL", exec_bls_g1_mul)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf93014, 24, "BLS_G1_MULTIEXP", exec_bls_g1_multiexp)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf93015, 24, "BLS_G1_ZERO", exec_bls_g1_zero)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf93016, 24, "BLS_MAP_TO_G1", exec_bls_map_to_g1)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf93017, 24, "BLS_G1_INGROUP", exec_bls_g1_in_group)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf93018, 24, "BLS_G1_ISZERO", exec_bls_g1_is_zero)->require_version(4)) + + .insert(OpcodeInstr::mksimple(0xf93020, 24, "BLS_G2_ADD", exec_bls_g2_add)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf93021, 24, "BLS_G2_SUB", exec_bls_g2_sub)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf93022, 24, "BLS_G2_NEG", exec_bls_g2_neg)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf93023, 24, "BLS_G2_MUL", exec_bls_g2_mul)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf93024, 24, "BLS_G2_MULTIEXP", exec_bls_g2_multiexp)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf93025, 24, "BLS_G2_ZERO", exec_bls_g2_zero)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf93026, 24, "BLS_MAP_TO_G2", exec_bls_map_to_g2)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf93027, 24, "BLS_G2_INGROUP", exec_bls_g2_in_group)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf93028, 24, "BLS_G2_ISZERO", exec_bls_g2_is_zero)->require_version(4)) + + .insert(OpcodeInstr::mksimple(0xf93030, 24, "BLS_PAIRING", exec_bls_pairing)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf93031, 24, "BLS_PUSHR", exec_bls_push_r)->require_version(4)); } int exec_compute_data_size(VmState* st, int mode) { - VM_LOG(st) << (mode & 2 ? 'S' : 'C') << "DATASIZE" << (mode & 1 ? "Q" : ""); + VM_LOG(st) << "execute " << (mode & 2 ? 'S' : 'C') << "DATASIZE" << (mode & 1 ? "Q" : ""); Stack& stack = st->get_stack(); stack.check_underflow(2); auto bound = stack.pop_int(); @@ -447,19 +1347,14 @@ int exec_load_var_integer(VmState* st, int len_bits, bool sgnd, bool quiet) { Stack& stack = st->get_stack(); auto csr = stack.pop_cellslice(); td::RefInt256 x; - int len; - if (!(csr.write().fetch_uint_to(len_bits, len) && csr.unique_write().fetch_int256_to(len * 8, x, sgnd))) { - if (quiet) { - stack.push_bool(false); - } else { - throw VmError{Excno::cell_und, "cannot deserialize a variable-length integer"}; - } - } else { + if (util::load_var_integer_q(csr.write(), x, len_bits, sgnd, quiet)) { stack.push_int(std::move(x)); stack.push_cellslice(std::move(csr)); if (quiet) { stack.push_bool(true); } + } else { + stack.push_bool(false); } return 0; } @@ -474,21 +1369,13 @@ int exec_store_var_integer(VmState* st, int len_bits, bool sgnd, bool quiet) { stack.check_underflow(2); auto x = stack.pop_int(); auto cbr = stack.pop_builder(); - unsigned len = (((unsigned)x->bit_size(sgnd) + 7) >> 3); - if (len >= (1u << len_bits)) { - throw VmError{Excno::range_chk}; - } - if (!(cbr.write().store_long_bool(len, len_bits) && cbr.unique_write().store_int256_bool(*x, len * 8, sgnd))) { - if (quiet) { - stack.push_bool(false); - } else { - throw VmError{Excno::cell_ov, "cannot serialize a variable-length integer"}; - } - } else { + if (util::store_var_integer(cbr.write(), x, len_bits, sgnd, quiet)) { stack.push_builder(std::move(cbr)); if (quiet) { stack.push_bool(true); } + } else { + stack.push_bool(false); } return 0; } @@ -531,22 +1418,17 @@ bool skip_message_addr(CellSlice& cs) { int exec_load_message_addr(VmState* st, bool quiet) { VM_LOG(st) << "execute LDMSGADDR" << (quiet ? "Q" : ""); Stack& stack = st->get_stack(); - auto csr = stack.pop_cellslice(), csr_copy = csr; - auto& cs = csr.write(); - if (!(skip_message_addr(cs) && csr_copy.write().cut_tail(cs))) { - csr.clear(); - if (quiet) { - stack.push_cellslice(std::move(csr_copy)); - stack.push_bool(false); - } else { - throw VmError{Excno::cell_und, "cannot load a MsgAddress"}; - } - } else { - stack.push_cellslice(std::move(csr_copy)); + auto csr = stack.pop_cellslice(); + td::Ref addr{true}; + if (util::load_msg_addr_q(csr.write(), addr.write(), quiet)) { + stack.push_cellslice(std::move(addr)); stack.push_cellslice(std::move(csr)); if (quiet) { stack.push_bool(true); } + } else { + stack.push_cellslice(std::move(csr)); + stack.push_bool(false); } return 0; } @@ -769,6 +1651,250 @@ int exec_send_raw_message(VmState* st) { return install_output_action(st, cb.finalize()); } +int parse_addr_workchain(CellSlice cs) { + // anycast_info$_ depth:(#<= 30) { depth >= 1 } rewrite_pfx:(bits depth) = Anycast; + // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; + // addr_var$11 anycast:(Maybe Anycast) addr_len:(## 9) workchain_id:int32 address:(bits addr_len) = MsgAddressInt; + if (cs.fetch_ulong(1) != 1) { + throw VmError{Excno::range_chk, "not an internal MsgAddress"}; + } + bool is_var = cs.fetch_ulong(1); + if (cs.fetch_ulong(1) == 1) { // Anycast + unsigned depth; + cs.fetch_uint_leq(30, depth); + cs.skip_first(depth); + } + + if (is_var) { + cs.skip_first(9); + return (int)cs.fetch_long(32); + } else { + return (int)cs.fetch_long(8); + } +} + +int exec_send_message(VmState* st) { + VM_LOG(st) << "execute SENDMSG"; + Stack& stack = st->get_stack(); + stack.check_underflow(2); + int mode = stack.pop_smallint_range(2047); + bool send = !(mode & 1024); + mode &= ~1024; + if (mode >= 256) { + throw VmError{Excno::range_chk}; + } + Ref msg_cell = stack.pop_cell(); + + block::gen::MessageRelaxed::Record msg; + if (!tlb::type_unpack_cell(msg_cell, block::gen::t_MessageRelaxed_Any, msg)) { + throw VmError{Excno::unknown, "invalid message"}; + } + + Ref my_addr = get_param(st, 8).as_slice(); + if (my_addr.is_null()) { + throw VmError{Excno::type_chk, "invalid param MYADDR"}; + } + bool ihr_disabled; + Ref dest; + td::RefInt256 value; + td::RefInt256 user_fwd_fee, user_ihr_fee; + bool have_extra_currencies = false; + bool ext_msg = msg.info->prefetch_ulong(1); + if (ext_msg) { // External message + block::gen::CommonMsgInfoRelaxed::Record_ext_out_msg_info info; + if (!tlb::csr_unpack(msg.info, info)) { + throw VmError{Excno::unknown, "invalid message"}; + } + ihr_disabled = true; + dest = std::move(info.dest); + value = user_fwd_fee = user_ihr_fee = td::zero_refint(); + } else { // Internal message + block::gen::CommonMsgInfoRelaxed::Record_int_msg_info info; + if (!tlb::csr_unpack(msg.info, info)) { + throw VmError{Excno::unknown, "invalid message"}; + } + ihr_disabled = info.ihr_disabled; + dest = std::move(info.dest); + Ref extra; + if (!block::tlb::t_CurrencyCollection.unpack_special(info.value.write(), value, extra)) { + throw VmError{Excno::unknown, "invalid message"}; + } + have_extra_currencies = !extra.is_null(); + user_fwd_fee = block::tlb::t_Grams.as_integer(info.fwd_fee); + user_ihr_fee = block::tlb::t_Grams.as_integer(info.ihr_fee); + } + + bool is_masterchain = parse_addr_workchain(*my_addr) == -1 || (!ext_msg && parse_addr_workchain(*dest) == -1); + td::Ref prices_cs; + if (st->get_global_version() >= 6) { + prices_cs = tuple_index(get_unpacked_config_tuple(st), is_masterchain ? 4 : 5).as_slice(); + } else { + Ref config_dict = get_param(st, 9).as_cell(); + Dictionary config{config_dict, 32}; + Ref prices_cell = config.lookup_ref(td::BitArray<32>{is_masterchain ? 24 : 25}); + if (prices_cell.not_null()) { + prices_cs = load_cell_slice_ref(prices_cell); + } + } + if (prices_cs.is_null()) { + throw VmError{Excno::unknown, "invalid prices config"}; + } + auto r_prices = block::Config::do_get_msg_prices(*prices_cs, is_masterchain ? 24 : 25); + if (r_prices.is_error()) { + throw VmError{Excno::cell_und, PSTRING() << "cannot parse config: " << r_prices.error().message()}; + } + block::MsgPrices prices = r_prices.move_as_ok(); + + // msg_fwd_fees = (lump_price + ceil((bit_price * msg.bits + cell_price * msg.cells)/2^16)) nanograms + // bits in the root cell of a message are not included in msg.bits (lump_price pays for them) + td::uint64 max_cells; + if (st->get_global_version() >= 6) { + auto r_size_limits_config = + block::Config::do_get_size_limits_config(tuple_index(get_unpacked_config_tuple(st), 6).as_slice()); + if (r_size_limits_config.is_error()) { + throw VmError{Excno::cell_und, PSTRING() << "cannot parse config: " << r_size_limits_config.error().message()}; + } + max_cells = r_size_limits_config.ok().max_msg_cells; + } else { + max_cells = 1 << 13; + } + vm::VmStorageStat stat(max_cells); + CellSlice cs = load_cell_slice(msg_cell); + cs.skip_first(cs.size()); + if (st->get_global_version() >= 10 && have_extra_currencies) { + // Skip extra currency dict + cs.advance_refs(1); + } + stat.add_storage(cs); + + if (!ext_msg) { + if (mode & 128) { // value is balance of the contract + Ref balance = get_param(st, 7).as_tuple(); + if (balance.is_null()) { + throw VmError{Excno::type_chk, "invalid param BALANCE"}; + } + value = tuple_index(balance, 0).as_int(); + if (value.is_null()) { + throw VmError{Excno::type_chk, "invalid param BALANCE"}; + } + if (st->get_global_version() < 10) { + have_extra_currencies |= !tuple_index(balance, 1).as_cell().is_null(); + } + } else if (mode & 64) { // value += value of incoming message + Ref balance = get_param(st, 11).as_tuple(); + if (balance.is_null()) { + throw VmError{Excno::type_chk, "invalid param INCOMINGVALUE"}; + } + td::RefInt256 balance_grams = tuple_index(balance, 0).as_int(); + if (balance_grams.is_null()) { + throw VmError{Excno::type_chk, "invalid param INCOMINGVALUE"}; + } + value += balance_grams; + if (st->get_global_version() < 10) { + have_extra_currencies |= !tuple_index(balance, 1).as_cell().is_null(); + } + } + } + + bool have_init = msg.init->bit_at(0); + bool init_ref = have_init && msg.init->bit_at(1); + bool body_ref = msg.body->bit_at(0); + + td::RefInt256 fwd_fee, ihr_fee; + td::uint64 cells = stat.cells; + td::uint64 bits = stat.bits; + auto compute_fees = [&]() { + td::uint64 fwd_fee_short = prices.lump_price + td::uint128(prices.bit_price) + .mult(bits) + .add(td::uint128(prices.cell_price).mult(cells)) + .add(td::uint128(0xffffu)) + .shr(16) + .lo(); + td::uint64 ihr_fee_short; + if (ihr_disabled) { + ihr_fee_short = 0; + } else { + ihr_fee_short = td::uint128(fwd_fee_short).mult(prices.ihr_factor).shr(16).lo(); + } + fwd_fee = td::RefInt256{true, fwd_fee_short}; + ihr_fee = td::RefInt256{true, ihr_fee_short}; + fwd_fee = std::max(fwd_fee, user_fwd_fee); + if (!ihr_disabled) { + ihr_fee = std::max(ihr_fee, user_ihr_fee); + } + }; + compute_fees(); + + auto stored_grams_len = [](td::RefInt256 const& x) -> unsigned { + unsigned bits = x->bit_size(false); + return 4 + ((bits + 7) & ~7); + }; + + auto msg_root_bits = [&]() -> unsigned { + unsigned bits; + // CommonMsgInfo + if (ext_msg) { + bits = 2 + my_addr->size() + dest->size() + 32 + 64; + } else { + bits = 4 + my_addr->size() + dest->size() + stored_grams_len(value) + 1 + 32 + 64; + td::RefInt256 fwd_fee_first = (fwd_fee * prices.first_frac) >> 16; + bits += stored_grams_len(fwd_fee - fwd_fee_first); + bits += stored_grams_len(ihr_fee); + } + // init + bits++; + if (have_init) { + bits += 1 + (init_ref ? 0 : msg.init->size() - 2); + } + // body + bits++; + bits += (body_ref ? 0 : msg.body->size() - 1); + return bits; + }; + auto msg_root_refs = [&]() -> unsigned { + unsigned refs; + // CommonMsgInfo + if (ext_msg) { + refs = 0; + } else { + refs = have_extra_currencies; + } + // init + if (have_init) { + refs += (init_ref ? 1 : msg.init->size_refs()); + } + // body + refs += (body_ref ? 1 : msg.body->size_refs()); + return refs; + }; + + if (have_init && !init_ref && (msg_root_bits() > Cell::max_bits || msg_root_refs() > Cell::max_refs)) { + init_ref = true; + cells += 1; + bits += msg.init->size() - 2; + compute_fees(); + } + if (!body_ref && (msg_root_bits() > Cell::max_bits || msg_root_refs() > Cell::max_refs)) { + body_ref = true; + cells += 1; + bits += msg.body->size() - 1; + compute_fees(); + } + stack.push_int(fwd_fee + ihr_fee); + + if (send) { + CellBuilder cb; + if (!(cb.store_ref_bool(get_actions(st)) // out_list$_ {n:#} prev:^(OutList n) + && cb.store_long_bool(0x0ec3c86d, 32) // action_send_msg#0ec3c86d + && cb.store_long_bool(mode, 8) // mode:(## 8) + && cb.store_ref_bool(std::move(msg_cell)))) { + throw VmError{Excno::cell_ov, "cannot serialize raw output message into an output action cell"}; + } + return install_output_action(st, cb.finalize()); + } + return 0; +} + bool store_grams(CellBuilder& cb, td::RefInt256 value) { int k = value->bit_size(false); return k <= 15 * 8 && cb.store_long_bool((k + 7) >> 3, 4) && cb.store_int256_bool(*value, (k + 7) & -8, false); @@ -778,7 +1904,7 @@ int exec_reserve_raw(VmState* st, int mode) { VM_LOG(st) << "execute RAWRESERVE" << (mode & 1 ? "X" : ""); Stack& stack = st->get_stack(); stack.check_underflow(2 + (mode & 1)); - int f = stack.pop_smallint_range(15); + int f = stack.pop_smallint_range(st->get_global_version() >= 4 ? 31 : 15); Ref y; if (mode & 1) { y = stack.pop_maybe_cell(); @@ -814,12 +1940,20 @@ int exec_set_lib_code(VmState* st) { VM_LOG(st) << "execute SETLIBCODE"; Stack& stack = st->get_stack(); stack.check_underflow(2); - int mode = stack.pop_smallint_range(2); + int mode; + if (st->get_global_version() >= 4) { + mode = stack.pop_smallint_range(31); + if ((mode & ~16) > 2) { + throw VmError{Excno::range_chk}; + } + } else { + mode = stack.pop_smallint_range(2); + } auto code = stack.pop_cell(); CellBuilder cb; if (!(cb.store_ref_bool(get_actions(st)) // out_list$_ {n:#} prev:^(OutList n) && cb.store_long_bool(0x26fa1dd4, 32) // action_change_library#26fa1dd4 - && cb.store_long_bool(mode * 2 + 1, 8) // mode:(## 7) { mode <= 2 } + && cb.store_long_bool(mode * 2 + 1, 8) // mode:(## 7) && cb.store_ref_bool(std::move(code)))) { // libref:LibRef = OutAction; throw VmError{Excno::cell_ov, "cannot serialize new library code into an output action cell"}; } @@ -830,7 +1964,15 @@ int exec_change_lib(VmState* st) { VM_LOG(st) << "execute CHANGELIB"; Stack& stack = st->get_stack(); stack.check_underflow(2); - int mode = stack.pop_smallint_range(2); + int mode; + if (st->get_global_version() >= 4) { + mode = stack.pop_smallint_range(31); + if ((mode & ~16) > 2) { + throw VmError{Excno::range_chk}; + } + } else { + mode = stack.pop_smallint_range(2); + } auto hash = stack.pop_int_finite(); if (!hash->unsigned_fits_bits(256)) { throw VmError{Excno::range_chk, "library hash must be non-negative"}; @@ -852,7 +1994,8 @@ void register_ton_message_ops(OpcodeTable& cp0) { .insert(OpcodeInstr::mksimple(0xfb03, 16, "RAWRESERVEX", std::bind(exec_reserve_raw, _1, 1))) .insert(OpcodeInstr::mksimple(0xfb04, 16, "SETCODE", exec_set_code)) .insert(OpcodeInstr::mksimple(0xfb06, 16, "SETLIBCODE", exec_set_lib_code)) - .insert(OpcodeInstr::mksimple(0xfb07, 16, "CHANGELIB", exec_change_lib)); + .insert(OpcodeInstr::mksimple(0xfb07, 16, "CHANGELIB", exec_change_lib)) + .insert(OpcodeInstr::mksimple(0xfb08, 16, "SENDMSG", exec_send_message)->require_version(4)); } void register_ton_ops(OpcodeTable& cp0) { @@ -866,4 +2009,158 @@ void register_ton_ops(OpcodeTable& cp0) { register_ton_message_ops(cp0); } +namespace util { + +bool load_var_integer_q(CellSlice& cs, td::RefInt256& res, int len_bits, bool sgnd, bool quiet) { + CellSlice cs0 = cs; + int len; + if (!(cs.fetch_uint_to(len_bits, len) && cs.fetch_int256_to(len * 8, res, sgnd))) { + cs = std::move(cs0); + if (quiet) { + return false; + } + throw VmError{Excno::cell_und, "cannot deserialize a variable-length integer"}; + } + return true; +} +bool load_coins_q(CellSlice& cs, td::RefInt256& res, bool quiet) { + return load_var_integer_q(cs, res, 4, false, quiet); +} +bool load_msg_addr_q(CellSlice& cs, CellSlice& res, bool quiet) { + res = cs; + if (!skip_message_addr(cs)) { + cs = res; + if (quiet) { + return false; + } + throw VmError{Excno::cell_und, "cannot load a MsgAddress"}; + } + res.cut_tail(cs); + return true; +} +bool parse_std_addr_q(CellSlice cs, ton::WorkchainId& res_wc, ton::StdSmcAddress& res_addr, bool quiet) { + // Like exec_rewrite_message_addr, but for std address case + std::vector tuple; + if (!(parse_message_addr(cs, tuple) && cs.empty_ext())) { + if (quiet) { + return false; + } + throw VmError{Excno::cell_und, "cannot parse a MsgAddress"}; + } + int t = (int)std::move(tuple[0]).as_int()->to_long(); + if (t != 2 && t != 3) { + if (quiet) { + return false; + } + throw VmError{Excno::cell_und, "cannot parse a MsgAddressInt"}; + } + auto addr = std::move(tuple[3]).as_slice(); + auto prefix = std::move(tuple[1]).as_slice(); + if (addr->size() != 256) { + if (quiet) { + return false; + } + throw VmError{Excno::cell_und, "MsgAddressInt is not a standard 256-bit address"}; + } + res_wc = (int)tuple[2].as_int()->to_long(); + CHECK(addr->prefetch_bits_to(res_addr) && + (prefix.is_null() || prefix->prefetch_bits_to(res_addr.bits(), prefix->size()))); + return true; +} + +td::RefInt256 load_var_integer(CellSlice& cs, int len_bits, bool sgnd) { + td::RefInt256 x; + load_var_integer_q(cs, x, len_bits, sgnd, false); + return x; +} +td::RefInt256 load_coins(CellSlice& cs) { + return load_var_integer(cs, 4, false); +} +CellSlice load_msg_addr(CellSlice& cs) { + CellSlice addr; + load_msg_addr_q(cs, addr, false); + return addr; +} +std::pair parse_std_addr(CellSlice cs) { + std::pair res; + parse_std_addr_q(std::move(cs), res.first, res.second, false); + return res; +} + +bool store_var_integer(CellBuilder& cb, const td::RefInt256& x, int len_bits, bool sgnd, bool quiet) { + unsigned len = (((unsigned)x->bit_size(sgnd) + 7) >> 3); + if (len >= (1u << len_bits)) { + throw VmError{Excno::range_chk}; // throw even if quiet + } + if (!cb.can_extend_by(len_bits + len * 8)) { + if (quiet) { + return false; + } + throw VmError{Excno::cell_ov, "cannot serialize a variable-length integer"}; + } + CHECK(cb.store_long_bool(len, len_bits) && cb.store_int256_bool(*x, len * 8, sgnd)); + return true; +} +bool store_coins(CellBuilder& cb, const td::RefInt256& x, bool quiet) { + return store_var_integer(cb, x, 4, false, quiet); +} + +block::GasLimitsPrices get_gas_prices(const Ref& unpacked_config, bool is_masterchain) { + Ref cs = tuple_index(unpacked_config, is_masterchain ? 2 : 3).as_slice(); + if (cs.is_null()) { + throw VmError{Excno::type_chk, "intermediate value is not a slice"}; + } + auto r_prices = block::Config::do_get_gas_limits_prices(*cs, is_masterchain ? 20 : 21); + if (r_prices.is_error()) { + throw VmError{Excno::cell_und, PSTRING() << "cannot parse config: " << r_prices.error().message()}; + } + return r_prices.move_as_ok(); +} + +block::MsgPrices get_msg_prices(const Ref& unpacked_config, bool is_masterchain) { + Ref cs = tuple_index(unpacked_config, is_masterchain ? 4 : 5).as_slice(); + if (cs.is_null()) { + throw VmError{Excno::type_chk, "intermediate value is not a slice"}; + } + auto r_prices = block::Config::do_get_msg_prices(*cs, is_masterchain ? 24 : 25); + if (r_prices.is_error()) { + throw VmError{Excno::cell_und, PSTRING() << "cannot parse config: " << r_prices.error().message()}; + } + return r_prices.move_as_ok(); +} + +td::optional get_storage_prices(const Ref& unpacked_config) { + Ref cs = tuple_index(unpacked_config, 0).as_slice(); + if (cs.is_null()) { + // null means tat no StoragePrices is active, so the price is 0 + return {}; + } + auto r_prices = block::Config::do_get_one_storage_prices(*cs); + if (r_prices.is_error()) { + throw VmError{Excno::cell_und, PSTRING() << "cannot parse config: " << r_prices.error().message()}; + } + return r_prices.move_as_ok(); +} + +td::RefInt256 calculate_storage_fee(const td::optional& maybe_prices, bool is_masterchain, + td::uint64 delta, td::uint64 bits, td::uint64 cells) { + if (!maybe_prices) { + // no StoragePrices is active, so the price is 0 + return td::zero_refint(); + } + const block::StoragePrices& prices = maybe_prices.value(); + td::RefInt256 total; + if (is_masterchain) { + total = td::make_refint(cells) * prices.mc_cell_price; + total += td::make_refint(bits) * prices.mc_bit_price; + } else { + total = td::make_refint(cells) * prices.cell_price; + total += td::make_refint(bits) * prices.bit_price; + } + total *= delta; + return td::rshift(total, 16, 1); +} + +} // namespace util + } // namespace vm diff --git a/crypto/vm/tonops.h b/crypto/vm/tonops.h index 5b3e14d9..bbac078f 100644 --- a/crypto/vm/tonops.h +++ b/crypto/vm/tonops.h @@ -17,6 +17,9 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "vm/vm.h" +#include "ton/ton-types.h" +#include "mc-config.h" namespace vm { @@ -24,4 +27,30 @@ class OpcodeTable; void register_ton_ops(OpcodeTable& cp0); +namespace util { + +// "_q" functions throw on error if not quiet, return false if quiet (leaving cs unchanged) +bool load_var_integer_q(CellSlice& cs, td::RefInt256& res, int len_bits, bool sgnd, bool quiet); +bool load_coins_q(CellSlice& cs, td::RefInt256& res, bool quiet); +bool load_msg_addr_q(CellSlice& cs, CellSlice& res, bool quiet); +bool parse_std_addr_q(CellSlice cs, ton::WorkchainId& res_wc, ton::StdSmcAddress& res_addr, bool quiet); + +// Non-"_q" functions throw on error +td::RefInt256 load_var_integer(CellSlice& cs, int len_bits, bool sgnd); +td::RefInt256 load_coins(CellSlice& cs); +CellSlice load_msg_addr(CellSlice& cs); +std::pair parse_std_addr(CellSlice cs); + +// store_... functions throw on error if not quiet, return false if quiet (leaving cb unchanged) +bool store_var_integer(CellBuilder& cb, const td::RefInt256& x, int len_bits, bool sgnd, bool quiet = false); +bool store_coins(CellBuilder& cb, const td::RefInt256& x, bool quiet = false); + +block::GasLimitsPrices get_gas_prices(const td::Ref& unpacked_config, bool is_masterchain); +block::MsgPrices get_msg_prices(const td::Ref& unpacked_config, bool is_masterchain); +td::optional get_storage_prices(const td::Ref& unpacked_config); +td::RefInt256 calculate_storage_fee(const td::optional& maybe_prices, bool is_masterchain, + td::uint64 delta, td::uint64 bits, td::uint64 cells); + +} // namespace util + } // namespace vm diff --git a/crypto/vm/tupleops.cpp b/crypto/vm/tupleops.cpp index ef906f6a..f4be2c76 100644 --- a/crypto/vm/tupleops.cpp +++ b/crypto/vm/tupleops.cpp @@ -99,7 +99,7 @@ int exec_mktuple_var(VmState* st) { int exec_tuple_index_common(Stack& stack, unsigned n) { auto tuple = stack.pop_tuple_range(255); - stack.push(tuple_index(*tuple, n)); + stack.push(tuple_index(tuple, n)); return 0; } @@ -322,11 +322,11 @@ int exec_tuple_index2(VmState* st, unsigned args) { VM_LOG(st) << "execute INDEX2 " << i << "," << j; Stack& stack = st->get_stack(); auto tuple = stack.pop_tuple_range(255); - auto t1 = tuple_index(*tuple, i).as_tuple_range(255); + auto t1 = tuple_index(tuple, i).as_tuple_range(255); if (t1.is_null()) { throw VmError{Excno::type_chk, "intermediate value is not a tuple"}; } - stack.push(tuple_index(*t1, j)); + stack.push(tuple_index(t1, j)); return 0; } @@ -342,15 +342,15 @@ int exec_tuple_index3(VmState* st, unsigned args) { VM_LOG(st) << "execute INDEX3 " << i << "," << j << "," << k; Stack& stack = st->get_stack(); auto tuple = stack.pop_tuple_range(255); - auto t1 = tuple_index(*tuple, i).as_tuple_range(255); + auto t1 = tuple_index(tuple, i).as_tuple_range(255); if (t1.is_null()) { throw VmError{Excno::type_chk, "intermediate value is not a tuple"}; } - auto t2 = tuple_index(*t1, j).as_tuple_range(255); + auto t2 = tuple_index(t1, j).as_tuple_range(255); if (t2.is_null()) { throw VmError{Excno::type_chk, "intermediate value is not a tuple"}; } - stack.push(tuple_index(*t2, k)); + stack.push(tuple_index(t2, k)); return 0; } diff --git a/crypto/vm/utils.cpp b/crypto/vm/utils.cpp index 783bf132..52bfb0d4 100644 --- a/crypto/vm/utils.cpp +++ b/crypto/vm/utils.cpp @@ -96,10 +96,10 @@ td::Result convert_stack_entry(td::Slice str) { } if (l >= 3 && (str[0] == 'x' || str[0] == 'b') && str[1] == '{' && str.back() == '}') { unsigned char buff[128]; - int bits = - (str[0] == 'x') - ? (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.begin() + 2, str.end() - 1) - : (int)td::bitstring::parse_bitstring_binary_literal(buff, sizeof(buff), str.begin() + 2, str.end() - 1); + int bits = (str[0] == 'x') + ? (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.begin() + 2, str.end() - 1) + : (int)td::bitstring::parse_bitstring_binary_literal(buff, sizeof(buff) * 8, str.begin() + 2, + str.end() - 1); if (bits < 0) { return td::Status::Error("failed to parse raw b{...}/x{...} number"); } diff --git a/crypto/vm/vm.cpp b/crypto/vm/vm.cpp index 6668c6d5..3c1118c6 100644 --- a/crypto/vm/vm.cpp +++ b/crypto/vm/vm.cpp @@ -21,6 +21,10 @@ #include "vm/dict.h" #include "vm/log.h" #include "vm/vm.h" +#include "cp0.h" +#include "memo.h" + +#include namespace vm { @@ -29,33 +33,8 @@ VmState::VmState() : cp(-1), dispatch(&dummy_dispatch_table), quit0(true, 0), qu init_cregs(); } -VmState::VmState(Ref _code) - : code(std::move(_code)), cp(-1), dispatch(&dummy_dispatch_table), quit0(true, 0), quit1(true, 1) { - ensure_throw(init_cp(0)); - init_cregs(); -} - -VmState::VmState(Ref _code, Ref _stack, int flags, Ref _data, VmLog log, - std::vector> _libraries, Ref init_c7) - : code(std::move(_code)) - , stack(std::move(_stack)) - , cp(-1) - , dispatch(&dummy_dispatch_table) - , quit0(true, 0) - , quit1(true, 1) - , log(log) - , libraries(std::move(_libraries)) - , stack_trace((flags >> 2) & 1) { - ensure_throw(init_cp(0)); - set_c4(std::move(_data)); - if (init_c7.not_null()) { - set_c7(std::move(init_c7)); - } - init_cregs(flags & 1, flags & 2); -} - -VmState::VmState(Ref _code, Ref _stack, const GasLimits& gas, int flags, Ref _data, VmLog log, - std::vector> _libraries, Ref init_c7) +VmState::VmState(Ref _code, int global_version, Ref _stack, const GasLimits& gas, int flags, + Ref _data, VmLog log, std::vector> _libraries, Ref init_c7) : code(std::move(_code)) , stack(std::move(_stack)) , cp(-1) @@ -65,7 +44,8 @@ VmState::VmState(Ref _code, Ref _stack, const GasLimits& gas, , log(log) , gas(gas) , libraries(std::move(_libraries)) - , stack_trace((flags >> 2) & 1) { + , stack_trace((flags >> 2) & 1) + , global_version(global_version) { ensure_throw(init_cp(0)); set_c4(std::move(_data)); if (init_c7.not_null()) { @@ -100,12 +80,24 @@ void VmState::init_cregs(bool same_c3, bool push_0) { } } -Ref VmState::convert_code_cell(Ref code_cell) { +Ref VmState::convert_code_cell(Ref code_cell, int global_version, + const std::vector>& libraries) { if (code_cell.is_null()) { return {}; } - Ref csr{true, NoVmOrd(), code_cell}; - if (csr->is_valid()) { + Ref csr; + if (global_version >= 9) { + // Use DummyVmState instead of this to avoid consuming gas for cell loading + DummyVmState dummy{libraries, global_version}; + Guard guard(&dummy); + try { + csr = load_cell_slice_ref(code_cell); + } catch (VmError&) { // NOLINT(*-empty-catch) + } + } else { + csr = td::Ref{true, NoVmOrd(), code_cell}; + } + if (csr.not_null() && csr->is_valid()) { return csr; } return load_cell_slice_ref(CellBuilder{}.store_ref(std::move(code_cell)).finalize()); @@ -255,6 +247,11 @@ int VmState::jump(Ref cont) { // general jump to continuation cont int VmState::jump(Ref cont, int pass_args) { + cont = adjust_jump_cont(std::move(cont), pass_args); + return jump_to(std::move(cont)); +} + +Ref VmState::adjust_jump_cont(Ref cont, int pass_args) { const ControlData* cont_data = cont->get_cdata(); if (cont_data) { // first do the checks @@ -295,7 +292,7 @@ int VmState::jump(Ref cont, int pass_args) { consume_stack_gas(copy); } } - return jump_to(std::move(cont)); + return cont; } else { // have no continuation data, situation is somewhat simpler if (pass_args >= 0) { @@ -307,7 +304,7 @@ int VmState::jump(Ref cont, int pass_args) { consume_stack_gas(pass_args); } } - return jump_to(std::move(cont)); + return cont; } } @@ -389,7 +386,7 @@ int VmState::throw_exception(int excno) { stack_ref.push_smallint(0); stack_ref.push_smallint(excno); code.clear(); - gas.consume_chk(exception_gas_price); + consume_gas_chk(exception_gas_price); return jump(get_c2()); } @@ -399,7 +396,7 @@ int VmState::throw_exception(int excno, StackEntry&& arg) { stack_ref.push(std::move(arg)); stack_ref.push_smallint(excno); code.clear(); - gas.consume_chk(exception_gas_price); + consume_gas_chk(exception_gas_price); return jump(get_c2()); } @@ -433,63 +430,67 @@ void VmState::change_gas_limit(long long new_limit) { int VmState::step() { CHECK(code.not_null() && stack.not_null()); - //VM_LOG(st) << "stack:"; stack->dump(VM_LOG(st)); - //VM_LOG(st) << "; cr0.refcnt = " << get_c0()->get_refcnt() - 1 << std::endl; + if (log.log_mask & vm::VmLog::DumpStack) { + std::stringstream ss; + int mode = 3; + if (log.log_mask & vm::VmLog::DumpStackVerbose) { + mode += 4; + } + std::unique_ptr tmp_ctx; + // install temporary dummy vm state interface to prevent charging for cell load operations during dump + VmStateInterface::Guard guard(tmp_ctx.get()); + stack->dump(ss, mode); + VM_LOG(this) << "stack:" << ss.str(); + } if (stack_trace) { + std::unique_ptr tmp_ctx; + // install temporary dummy vm state interface to prevent charging for cell load operations during dump + VmStateInterface::Guard guard(tmp_ctx.get()); stack->dump(std::cerr, 3); } ++steps; if (code->size()) { + VM_LOG_MASK(this, vm::VmLog::ExecLocation) << "code cell hash: " << code->get_base_cell()->get_hash().to_hex() << " offset: " << code->cur_pos(); return dispatch->dispatch(this, code.write()); } else if (code->size_refs()) { VM_LOG(this) << "execute implicit JMPREF"; - gas.consume_chk(implicit_jmpref_gas_price); - Ref cont = Ref{true, load_cell_slice_ref(code->prefetch_ref()), get_cp()}; + auto ref_cell = code->prefetch_ref(); + VM_LOG_MASK(this, vm::VmLog::ExecLocation) << "code cell hash: " << ref_cell->get_hash().to_hex() << " offset: 0"; + consume_gas_chk(implicit_jmpref_gas_price); + Ref cont = Ref{true, load_cell_slice_ref(std::move(ref_cell)), get_cp()}; return jump(std::move(cont)); } else { VM_LOG(this) << "execute implicit RET"; - gas.consume_chk(implicit_ret_gas_price); + consume_gas_chk(implicit_ret_gas_price); return ret(); } } -int VmState::run() { - if (code.is_null() || stack.is_null()) { - // throw VmError{Excno::fatal, "cannot run an uninitialized VM"}; - return (int)Excno::fatal; // no ~ for unhandled exceptions - } +int VmState::run_inner() { int res; Guard guard(this); do { try { try { - try { - res = step(); - gas.check(); - } catch (vm::CellBuilder::CellWriteError) { - throw VmError{Excno::cell_ov}; - } catch (vm::CellBuilder::CellCreateError) { - throw VmError{Excno::cell_ov}; - } catch (vm::CellSlice::CellReadError) { - throw VmError{Excno::cell_und}; - } - } catch (const VmError& vme) { - VM_LOG(this) << "handling exception code " << vme.get_errno() << ": " << vme.get_msg(); - try { - ++steps; - res = throw_exception(vme.get_errno()); - } catch (const VmError& vme2) { - VM_LOG(this) << "exception " << vme2.get_errno() << " while handling exception: " << vme.get_msg(); - return ~vme2.get_errno(); - } + res = step(); + VM_LOG_MASK(this, vm::VmLog::GasRemaining) << "gas remaining: " << gas.gas_remaining; + gas.check(); + } catch (vm::CellBuilder::CellWriteError) { + throw VmError{Excno::cell_ov}; + } catch (vm::CellBuilder::CellCreateError) { + throw VmError{Excno::cell_ov}; + } catch (vm::CellSlice::CellReadError) { + throw VmError{Excno::cell_und}; + } + } catch (const VmError& vme) { + VM_LOG(this) << "handling exception code " << vme.get_errno() << ": " << vme.get_msg(); + try { + ++steps; + res = throw_exception(vme.get_errno()); + } catch (const VmError& vme2) { + VM_LOG(this) << "exception " << vme2.get_errno() << " while handling exception: " << vme.get_msg(); + return ~vme2.get_errno(); } - } catch (VmNoGas vmoog) { - ++steps; - VM_LOG(this) << "unhandled out-of-gas exception: gas consumed=" << gas.gas_consumed() - << ", limit=" << gas.gas_limit; - get_stack().clear(); - get_stack().push_smallint(gas.gas_consumed()); - return vmoog.get_errno(); // no ~ for unhandled exceptions (to make their faking impossible) } } while (!res); if ((res | 1) == -1 && !try_commit()) { @@ -501,6 +502,41 @@ int VmState::run() { return res; } +int VmState::run() { + if (code.is_null() || stack.is_null()) { + // throw VmError{Excno::fatal, "cannot run an uninitialized VM"}; + return (int)Excno::fatal; // no ~ for unhandled exceptions + } + int res = 0; + bool restore_parent = false; + while (true) { + try { + if (restore_parent) { + restore_parent_vm(~res); + } + res = run_inner(); + } catch (VmNoGas &vmoog) { + ++steps; + VM_LOG(this) << "unhandled out-of-gas exception: gas consumed=" << gas.gas_consumed() + << ", limit=" << gas.gas_limit; + get_stack().clear(); + get_stack().push_smallint(gas.gas_consumed()); + res = vmoog.get_errno(); // no ~ for unhandled exceptions (to make their faking impossible) + } + if (!parent) { + if ((log.log_mask & VmLog::DumpC5) && cstate.committed) { + std::stringstream ss; + ss << "final c5: "; + StackEntry::maybe(cstate.c5).dump(ss, true); + ss << "\n"; + VM_LOG(this) << ss.str(); + } + return res; + } + restore_parent = true; + } +} + bool VmState::try_commit() { if (cr.d[0].not_null() && cr.d[1].not_null() && cr.d[0]->get_depth() <= max_data_depth && cr.d[1]->get_depth() <= max_data_depth && cr.d[0]->get_level() == 0 && cr.d[1]->get_level() == 0) { @@ -533,8 +569,10 @@ ControlRegs* force_cregs(Ref& cont) { } int run_vm_code(Ref code, Ref& stack, int flags, Ref* data_ptr, VmLog log, long long* steps, - GasLimits* gas_limits, std::vector> libraries, Ref init_c7, Ref* actions_ptr) { + GasLimits* gas_limits, std::vector> libraries, Ref init_c7, Ref* actions_ptr, + int global_version) { VmState vm{code, + global_version, std::move(stack), gas_limits ? *gas_limits : GasLimits{}, flags, @@ -571,12 +609,13 @@ int run_vm_code(Ref code, Ref& stack, int flags, Ref* da } int run_vm_code(Ref code, Stack& stack, int flags, Ref* data_ptr, VmLog log, long long* steps, - GasLimits* gas_limits, std::vector> libraries, Ref init_c7, Ref* actions_ptr) { + GasLimits* gas_limits, std::vector> libraries, Ref init_c7, Ref* actions_ptr, + int global_version) { Ref stk{true}; stk.unique_write().set_contents(std::move(stack)); stack.clear(); int res = run_vm_code(code, stk, flags, data_ptr, log, steps, gas_limits, std::move(libraries), std::move(init_c7), - actions_ptr); + actions_ptr, global_version); CHECK(stack.is_unique()); if (stk.is_null()) { stack.clear(); @@ -597,14 +636,14 @@ int run_vm_code(Ref code, Stack& stack, int flags, Ref* data_pt Ref VmState::load_library(td::ConstBitPtr hash) { std::unique_ptr tmp_ctx; // install temporary dummy vm state interface to prevent charging for cell load operations during library lookup - VmStateInterface::Guard(tmp_ctx.get()); + VmStateInterface::Guard guard{global_version >= 4 ? tmp_ctx.get() : VmStateInterface::get()}; for (const auto& lib_collection : libraries) { auto lib = lookup_library_in(hash, lib_collection); if (lib.not_null()) { return lib; } } - missing_library = hash; + missing_library = td::Bits256{hash}; return {}; } @@ -621,9 +660,6 @@ void VmState::register_cell_load(const CellHash& cell_hash) { consume_gas(cell_load_gas_price); } else { auto ok = loaded_cells.insert(cell_hash); // check whether this is the first time this cell is loaded - if (ok.second) { - loaded_cells_count++; - } consume_gas(ok.second ? cell_load_gas_price : cell_reload_gas_price); } } @@ -670,4 +706,89 @@ Ref lookup_library_in(td::ConstBitPtr key, Ref lib_root) { return lookup_library_in(key, dict); } +void VmState::run_child_vm(VmState&& new_state, bool return_data, bool return_actions, bool return_gas, + bool isolate_gas, int ret_vals) { + new_state.log = std::move(log); + new_state.libraries = std::move(libraries); + new_state.stack_trace = stack_trace; + new_state.max_data_depth = max_data_depth; + if (!isolate_gas) { + new_state.loaded_cells = std::move(loaded_cells); + } else { + consume_gas(std::min(chksgn_counter, chksgn_free_count) * chksgn_gas_price); + chksgn_counter = 0; + } + new_state.chksgn_counter = chksgn_counter; + + auto new_parent = std::make_unique(); + new_parent->return_data = return_data; + new_parent->return_actions = return_actions; + new_parent->return_gas = return_gas; + new_parent->isolate_gas = isolate_gas; + new_parent->ret_vals = ret_vals; + new_parent->state = std::move(*this); + new_state.parent = std::move(new_parent); + *this = std::move(new_state); +} + +void VmState::restore_parent_vm(int res) { + auto parent = std::move(this->parent); + CHECK(parent); + VmState child_state = std::move(*this); + *this = std::move(parent->state); + log = std::move(child_state.log); + libraries = std::move(child_state.libraries); + steps += child_state.steps; + if (!parent->isolate_gas) { + loaded_cells = std::move(child_state.loaded_cells); + } + chksgn_counter = child_state.chksgn_counter; + VM_LOG(this) << "Child VM finished. res: " << res << ", steps: " << child_state.steps + << ", gas: " << child_state.gas_consumed(); + + consume_gas(std::min(child_state.gas_consumed(), child_state.gas.gas_limit + 1)); + Stack& cur_stack = get_stack(); + int ret_cnt; + if (res == 0 || res == 1) { + if (parent->ret_vals >= 0) { + if (child_state.stack->depth() >= parent->ret_vals) { + ret_cnt = parent->ret_vals; + } else { + ret_cnt = 0; + res = ~(int)Excno::stk_und; + cur_stack.push(td::zero_refint()); + } + } else { + ret_cnt = child_state.stack->depth(); + } + } else { + ret_cnt = std::min(child_state.stack->depth(), 1); + } + consume_stack_gas(ret_cnt); + for (int i = ret_cnt - 1; i >= 0; --i) { + cur_stack.push(std::move(child_state.stack->at(i))); + } + cur_stack.push_smallint(res); + if (parent->return_data) { + cur_stack.push_cell(child_state.get_committed_state().c4); + } + if (parent->return_actions) { + cur_stack.push_cell(child_state.get_committed_state().c5); + } + if (parent->return_gas) { + cur_stack.push_smallint(child_state.gas.gas_consumed()); + } +} + +td::Status init_vm(bool enable_debug) { + if (!init_op_cp0(enable_debug)) { + return td::Status::Error("Failed to init TVM: failed to init cp0"); + } + auto code = sodium_init(); + if (code < 0) { + return td::Status::Error(PSTRING() << "Failed to init TVM: sodium_init, code=" << code); + } + return td::Status::OK(); +} + } // namespace vm diff --git a/crypto/vm/vm.h b/crypto/vm/vm.h index 17ce1aa3..a171ef27 100644 --- a/crypto/vm/vm.h +++ b/crypto/vm/vm.h @@ -25,6 +25,7 @@ #include "vm/log.h" #include "vm/continuation.h" #include "td/utils/HashSet.h" +#include "td/utils/optional.h" namespace vm { @@ -80,6 +81,8 @@ struct CommittedState { bool committed{false}; }; +struct ParentVmState; + class VmState final : public VmStateInterface { Ref code; Ref stack; @@ -93,11 +96,14 @@ class VmState final : public VmStateInterface { GasLimits gas; std::vector> libraries; td::HashSet loaded_cells; - td::int64 loaded_cells_count{0}; int stack_trace{0}, debug_off{0}; bool chksig_always_succeed{false}; - td::ConstBitPtr missing_library{0}; + bool stop_on_accept_message{false}; + td::optional missing_library; td::uint16 max_data_depth = 512; // Default value + int global_version{0}; + size_t chksgn_counter = 0; + std::unique_ptr parent = nullptr; public: enum { @@ -109,22 +115,66 @@ class VmState final : public VmStateInterface { implicit_jmpref_gas_price = 10, implicit_ret_gas_price = 5, free_stack_depth = 32, - stack_entry_gas_price = 1 + stack_entry_gas_price = 1, + runvm_gas_price = 40, + hash_ext_entry_gas_price = 1, + free_nested_cont_jump = 8, + + rist255_mul_gas_price = 2000, + rist255_mulbase_gas_price = 750, + rist255_add_gas_price = 600, + rist255_fromhash_gas_price = 600, + rist255_validate_gas_price = 200, + + ecrecover_gas_price = 1500, + secp256k1_xonly_pubkey_tweak_add_gas_price = 1250, + chksgn_free_count = 10, + chksgn_gas_price = 4000, + p256_chksgn_gas_price = 3500, + + bls_verify_gas_price = 61000, + bls_aggregate_base_gas_price = -2650, + bls_aggregate_element_gas_price = 4350, + bls_fast_aggregate_verify_base_gas_price = 58000, + bls_fast_aggregate_verify_element_gas_price = 3000, + bls_aggregate_verify_base_gas_price = 38500, + bls_aggregate_verify_element_gas_price = 22500, + + bls_g1_add_sub_gas_price = 3900, + bls_g1_neg_gas_price = 750, + bls_g1_mul_gas_price = 5200, + bls_map_to_g1_gas_price = 2350, + bls_g1_in_group_gas_price = 2950, + + bls_g2_add_sub_gas_price = 6100, + bls_g2_neg_gas_price = 1550, + bls_g2_mul_gas_price = 10550, + bls_map_to_g2_gas_price = 7950, + bls_g2_in_group_gas_price = 4250, + + // multiexp gas = base + n * coef1 + n/floor(max(log2(n), 4)) * coef2 + bls_g1_multiexp_base_gas_price = 11375, + bls_g1_multiexp_coef1_gas_price = 630, + bls_g1_multiexp_coef2_gas_price = 8820, + bls_g2_multiexp_base_gas_price = 30388, + bls_g2_multiexp_coef1_gas_price = 1280, + bls_g2_multiexp_coef2_gas_price = 22840, + + bls_pairing_base_gas_price = 20000, + bls_pairing_element_gas_price = 11800 }; VmState(); - VmState(Ref _code); - VmState(Ref _code, Ref _stack, int flags = 0, Ref _data = {}, VmLog log = {}, - std::vector> _libraries = {}, Ref init_c7 = {}); - VmState(Ref _code, Ref _stack, const GasLimits& _gas, int flags = 0, Ref _data = {}, + VmState(Ref _code, int global_version, Ref _stack, const GasLimits& _gas, int flags = 0, Ref _data = {}, VmLog log = {}, std::vector> _libraries = {}, Ref init_c7 = {}); - template - VmState(Ref code_cell, Args&&... args) - : VmState(convert_code_cell(std::move(code_cell)), std::forward(args)...) { + VmState(Ref _code, int global_version, Ref _stack, const GasLimits& _gas, int flags = 0, + Ref _data = {}, VmLog log = {}, std::vector> _libraries = {}, Ref init_c7 = {}) + : VmState(convert_code_cell(std::move(_code), global_version, _libraries), global_version, std::move(_stack), + _gas, flags, std::move(_data), std::move(log), _libraries, std::move(init_c7)) { } VmState(const VmState&) = delete; - VmState(VmState&&) = delete; + VmState(VmState&&) = default; VmState& operator=(const VmState&) = delete; - VmState& operator=(VmState&&) = delete; + VmState& operator=(VmState&&) = default; bool set_gas_limits(long long _max, long long _limit, long long _credit = 0); bool final_gas_ok() const { return gas.final_ok(); @@ -138,8 +188,15 @@ class VmState final : public VmStateInterface { const CommittedState& get_committed_state() const { return cstate; } + void consume_gas_chk(long long amount) { + gas.consume_chk(amount); + } void consume_gas(long long amount) { - gas.consume(amount); + if (global_version >= 4) { + gas.consume_chk(amount); + } else { + gas.consume(amount); + } } void consume_tuple_gas(unsigned tuple_len) { consume_gas(tuple_len * tuple_entry_gas_price); @@ -283,10 +340,14 @@ class VmState final : public VmStateInterface { void preclear_cr(const ControlRegs& save) { cr &= save; } + int get_global_version() const override { + return global_version; + } int call(Ref cont); int call(Ref cont, int pass_args, int ret_args = -1); int jump(Ref cont); int jump(Ref cont, int pass_args); + Ref adjust_jump_cont(Ref cont, int pass_args); int ret(); int ret(int ret_args); int ret_alt(); @@ -303,13 +364,29 @@ class VmState final : public VmStateInterface { return cond ? c1_envelope(std::move(cont), save) : std::move(cont); } void c1_save_set(bool save = true); - void fatal(void) const { + void fatal() const { throw VmFatal{}; } int jump_to(Ref cont) { - return cont->is_unique() ? cont.unique_write().jump_w(this) : cont->jump(this); + int res = 0, cnt = 0; + while (cont.not_null()) { + cont = cont->is_unique() ? cont.unique_write().jump_w(this, res) : cont->jump(this, res); + cnt++; + if (cnt > free_nested_cont_jump && global_version >= 9) { + consume_gas(1); + } + if (cont.not_null() && global_version >= 9) { + const ControlData* cont_data = cont->get_cdata(); + if (cont_data && (cont_data->stack.not_null() || cont_data->nargs >= 0)) { + // if cont has non-empty stack or expects fixed number of arguments, jump is not simple + cont = adjust_jump_cont(std::move(cont), -1); + } + } + } + return res; } - static Ref convert_code_cell(Ref code_cell); + static Ref convert_code_cell(Ref code_cell, int global_version, + const std::vector>& libraries); bool try_commit(); void force_commit(); @@ -319,27 +396,54 @@ class VmState final : public VmStateInterface { bool get_chksig_always_succeed() const { return chksig_always_succeed; } + void set_stop_on_accept_message(bool flag) { + stop_on_accept_message = flag; + } + bool get_stop_on_accept_message() const { + return stop_on_accept_message; + } Ref ref_to_cont(Ref cell) const { return td::make_ref(load_cell_slice_ref(std::move(cell)), get_cp()); } - td::ConstBitPtr get_missing_library() const { + td::optional get_missing_library() const { return missing_library; } void set_max_data_depth(td::uint16 depth) { max_data_depth = depth; } + void run_child_vm(VmState&& new_state, bool return_data, bool return_actions, bool return_gas, bool isolate_gas, + int ret_vals); + void restore_parent_vm(int res); + + void register_chksgn_call() { + if (global_version >= 4) { + ++chksgn_counter; + if (chksgn_counter > chksgn_free_count) { + consume_gas(chksgn_gas_price); + } + } + } private: void init_cregs(bool same_c3 = false, bool push_0 = true); + int run_inner(); +}; + +struct ParentVmState { + VmState state; + bool return_data, return_actions, return_gas, isolate_gas; + int ret_vals; }; int run_vm_code(Ref _code, Ref& _stack, int flags = 0, Ref* data_ptr = nullptr, VmLog log = {}, long long* steps = nullptr, GasLimits* gas_limits = nullptr, std::vector> libraries = {}, - Ref init_c7 = {}, Ref* actions_ptr = nullptr); + Ref init_c7 = {}, Ref* actions_ptr = nullptr, int global_version = 0); int run_vm_code(Ref _code, Stack& _stack, int flags = 0, Ref* data_ptr = nullptr, VmLog log = {}, long long* steps = nullptr, GasLimits* gas_limits = nullptr, std::vector> libraries = {}, - Ref init_c7 = {}, Ref* actions_ptr = nullptr); + Ref init_c7 = {}, Ref* actions_ptr = nullptr, int global_version = 0); Ref lookup_library_in(td::ConstBitPtr key, Ref lib_root); +td::Status init_vm(bool enable_debug = false); + } // namespace vm diff --git a/crypto/vm/vmstate.h b/crypto/vm/vmstate.h index a81a4e78..0d6c3fdf 100644 --- a/crypto/vm/vmstate.h +++ b/crypto/vm/vmstate.h @@ -19,6 +19,7 @@ #pragma once #include "common/refcnt.hpp" #include "vm/cells.h" +#include "common/global-version.h" #include "td/utils/Context.h" @@ -38,6 +39,9 @@ class VmStateInterface : public td::Context { virtual bool register_op(int op_units = 1) { return true; }; + virtual int get_global_version() const { + return ton::SUPPORTED_VERSION; + } }; } // namespace vm diff --git a/dht-server/CMakeLists.txt b/dht-server/CMakeLists.txt index 889b3f30..6daac033 100644 --- a/dht-server/CMakeLists.txt +++ b/dht-server/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index 212a6fae..fa9fad13 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -54,7 +54,7 @@ Config::Config() { out_port = 3278; } -Config::Config(ton::ton_api::engine_validator_config &config) { +Config::Config(const ton::ton_api::engine_validator_config &config) { out_port = static_cast(config.out_port_); if (!out_port) { out_port = 3278; @@ -162,6 +162,7 @@ ton::tl_object_ptr Config::tl() const { control_vec.push_back(ton::create_tl_object(x.second.key.tl(), x.first, std::move(control_proc_vec))); } + std::vector> shard_vec; auto gc_vec = ton::create_tl_object(std::vector{}); for (auto &id : gc) { @@ -170,7 +171,7 @@ ton::tl_object_ptr Config::tl() const { return ton::create_tl_object( out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), - std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); + nullptr, nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(shard_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, @@ -572,6 +573,12 @@ void DhtServer::load_config(td::Promise promise) { config_file_ = db_root_ + "/config.json"; } auto conf_data_R = td::read_file(config_file_); + if (conf_data_R.is_error()) { + conf_data_R = td::read_file(temp_config_file()); + if (conf_data_R.is_ok()) { + td::rename(temp_config_file(), config_file_).ensure(); + } + } if (conf_data_R.is_error()) { auto P = td::PromiseCreator::lambda( [name = local_config_, new_name = config_file_, promise = std::move(promise)](td::Result R) { @@ -620,12 +627,15 @@ void DhtServer::load_config(td::Promise promise) { void DhtServer::write_config(td::Promise promise) { auto s = td::json_encode(td::ToJson(*config_.tl().get()), true); - auto S = td::write_file(config_file_, s); - if (S.is_ok()) { - promise.set_value(td::Unit()); - } else { + auto S = td::write_file(temp_config_file(), s); + if (S.is_error()) { + td::unlink(temp_config_file()).ignore(); promise.set_error(std::move(S)); + return; } + td::unlink(config_file_).ignore(); + TRY_STATUS_PROMISE(promise, td::rename(temp_config_file(), config_file_)); + promise.set_value(td::Unit()); } td::Promise DhtServer::get_key_promise(td::MultiPromise::InitGuard &ig) { @@ -1222,15 +1232,19 @@ int main(int argc, char *argv[]) { }); td::uint32 threads = 7; p.add_checked_option( - 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice fname) { + 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice arg) { td::int32 v; try { - v = std::stoi(fname.str()); + v = std::stoi(arg.str()); } catch (...) { return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: not a number"); } - if (v < 1 || v > 256) { - return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: should be in range [1..256]"); + if (v <= 0) { + return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: should be > 0"); + } + if (v > 127) { + LOG(WARNING) << "`--threads " << v << "` is too big, effective value will be 127"; + v = 127; } threads = v; return td::Status::OK(); diff --git a/dht-server/dht-server.hpp b/dht-server/dht-server.hpp index bf24d621..7c9e5619 100644 --- a/dht-server/dht-server.hpp +++ b/dht-server/dht-server.hpp @@ -94,7 +94,7 @@ struct Config { ton::tl_object_ptr tl() const; Config(); - Config(ton::ton_api::engine_validator_config &config); + Config(const ton::ton_api::engine_validator_config &config); }; class DhtServer : public td::actor::Actor { @@ -109,6 +109,9 @@ class DhtServer : public td::actor::Actor { std::string local_config_ = ""; std::string global_config_ = "ton-global.config"; std::string config_file_; + std::string temp_config_file() const { + return config_file_ + ".tmp"; + } std::string db_root_ = "/var/ton-work/db/"; diff --git a/dht/CMakeLists.txt b/dht/CMakeLists.txt index e50a7497..95ee7069 100644 --- a/dht/CMakeLists.txt +++ b/dht/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) diff --git a/dht/dht-in.hpp b/dht/dht-in.hpp index 59ce2184..0d668d43 100644 --- a/dht/dht-in.hpp +++ b/dht/dht-in.hpp @@ -155,10 +155,7 @@ class DhtMemberImpl : public DhtMember { } } - void add_full_node(DhtKeyId id, DhtNode node) override { - add_full_node_impl(id, std::move(node)); - } - void add_full_node_impl(DhtKeyId id, DhtNode node, bool set_active = false); + void add_full_node(DhtKeyId id, DhtNode node, bool set_active) override; adnl::AdnlNodeIdShort get_id() const override { return id_; @@ -182,6 +179,7 @@ class DhtMemberImpl : public DhtMember { void get_value(DhtKey key, td::Promise result) override { get_value_in(key.compute_key_id(), std::move(result)); } + void get_value_many(DhtKey key, std::function callback, td::Promise promise) override; void alarm() override { alarm_timestamp() = td::Timestamp::in(1.0); diff --git a/dht/dht-query.cpp b/dht/dht-query.cpp index bc61242d..3d43b106 100644 --- a/dht/dht-query.cpp +++ b/dht/dht-query.cpp @@ -34,24 +34,33 @@ namespace ton { namespace dht { void DhtQuery::send_queries() { + while (pending_queries_.size() > k_ * 2) { + pending_queries_.erase(--pending_queries_.end()); + } VLOG(DHT_EXTRA_DEBUG) << this << ": sending new queries. active=" << active_queries_ << " max_active=" << a_; - while (pending_ids_.size() > 0 && active_queries_ < a_) { + while (pending_queries_.size() > 0 && active_queries_ < a_) { + auto id_xor = *pending_queries_.begin(); + if (result_list_.size() == k_ && *result_list_.rbegin() < id_xor) { + break; + } active_queries_++; - auto id_xor = *pending_ids_.begin(); auto id = id_xor ^ key_; VLOG(DHT_EXTRA_DEBUG) << this << ": sending " << get_name() << " query to " << id; - pending_ids_.erase(id_xor); + pending_queries_.erase(id_xor); - auto it = list_.find(id_xor); - CHECK(it != list_.end()); - td::actor::send_closure(adnl_, &adnl::Adnl::add_peer, get_src(), it->second.adnl_id(), it->second.addr_list()); + auto it = nodes_.find(id_xor); + CHECK(it != nodes_.end()); + td::actor::send_closure(adnl_, &adnl::Adnl::add_peer, get_src(), it->second.node.adnl_id(), + it->second.node.addr_list()); send_one_query(id.to_adnl()); } if (active_queries_ == 0) { - CHECK(pending_ids_.size() == 0); + pending_queries_.clear(); DhtNodesList list; - for (auto &node : list_) { - list.push_back(std::move(node.second)); + for (auto id_xor : result_list_) { + auto it = nodes_.find(id_xor); + CHECK(it != nodes_.end()); + list.push_back(it->second.node.clone()); } CHECK(list.size() <= k_); VLOG(DHT_EXTRA_DEBUG) << this << ": finalizing " << get_name() << " query. List size=" << list.size(); @@ -65,30 +74,32 @@ void DhtQuery::add_nodes(DhtNodesList list) { for (auto &node : list.list()) { auto id = node.get_key(); auto id_xor = key_ ^ id; - if (list_.find(id_xor) != list_.end()) { + if (nodes_.find(id_xor) != nodes_.end()) { continue; } - td::actor::send_closure(node_, &DhtMember::add_full_node, id, node.clone()); + VLOG(DHT_EXTRA_DEBUG) << this << ": " << get_name() << " query: adding " << id << " key"; + td::actor::send_closure(node_, &DhtMember::add_full_node, id, node.clone(), false); + nodes_[id_xor].node = std::move(node); + pending_queries_.insert(id_xor); + } +} - DhtKeyId last_id_xor; - if (list_.size() > 0) { - last_id_xor = list_.rbegin()->first; +void DhtQuery::finish_query(adnl::AdnlNodeIdShort id, bool success) { + active_queries_--; + CHECK(active_queries_ <= k_); + auto id_xor = key_ ^ DhtKeyId(id); + if (success) { + result_list_.insert(id_xor); + if (result_list_.size() > k_) { + result_list_.erase(--result_list_.end()); } - - if (list_.size() < k_ || id_xor < last_id_xor) { - list_[id_xor] = std::move(node); - pending_ids_.insert(id_xor); - if (list_.size() > k_) { - CHECK(id_xor != last_id_xor); - VLOG(DHT_EXTRA_DEBUG) << this << ": " << get_name() << " query: replacing " << (last_id_xor ^ key_) - << " key with " << id; - pending_ids_.erase(last_id_xor); - list_.erase(last_id_xor); - } else { - VLOG(DHT_EXTRA_DEBUG) << this << ": " << get_name() << " query: adding " << id << " key"; - } + } else { + NodeInfo &info = nodes_[id_xor]; + if (++info.failed_attempts < MAX_ATTEMPTS) { + pending_queries_.insert(id_xor); } } + send_queries(); } void DhtQueryFindNodes::send_one_query(adnl::AdnlNodeIdShort id) { @@ -111,7 +122,7 @@ void DhtQueryFindNodes::send_one_query(adnl::AdnlNodeIdShort id) { void DhtQueryFindNodes::on_result(td::Result R, adnl::AdnlNodeIdShort dst) { if (R.is_error()) { VLOG(DHT_INFO) << this << ": failed find nodes query " << get_src() << "->" << dst << ": " << R.move_as_error(); - finish_query(); + finish_query(dst, false); return; } @@ -122,7 +133,7 @@ void DhtQueryFindNodes::on_result(td::Result R, adnl::AdnlNodeI } else { add_nodes(DhtNodesList{Res.move_as_ok(), our_network_id()}); } - finish_query(); + finish_query(dst); } void DhtQueryFindNodes::finish(DhtNodesList list) { @@ -166,14 +177,14 @@ void DhtQueryFindValue::send_one_query_nodes(adnl::AdnlNodeIdShort id) { void DhtQueryFindValue::on_result(td::Result R, adnl::AdnlNodeIdShort dst) { if (R.is_error()) { VLOG(DHT_INFO) << this << ": failed find value query " << get_src() << "->" << dst << ": " << R.move_as_error(); - finish_query(); + finish_query(dst, false); return; } auto Res = fetch_tl_object(R.move_as_ok(), true); if (Res.is_error()) { VLOG(DHT_WARNING) << this << ": dropping incorrect answer on dht.findValue query from " << dst << ": " << Res.move_as_error(); - finish_query(); + finish_query(dst, false); return; } @@ -199,8 +210,11 @@ void DhtQueryFindValue::on_result(td::Result R, adnl::AdnlNodeI send_get_nodes = true; return; } - promise_.set_value(std::move(value)); - need_stop = true; + if (on_value_found(std::move(value))) { + send_get_nodes = true; + } else { + need_stop = true; + } }, [&](ton_api::dht_valueNotFound &v) { add_nodes(DhtNodesList{std::move(v.nodes_), our_network_id()}); @@ -210,30 +224,55 @@ void DhtQueryFindValue::on_result(td::Result R, adnl::AdnlNodeI } else if (send_get_nodes) { send_one_query_nodes(dst); } else { - finish_query(); + finish_query(dst); } } void DhtQueryFindValue::on_result_nodes(td::Result R, adnl::AdnlNodeIdShort dst) { if (R.is_error()) { VLOG(DHT_INFO) << this << ": failed find nodes query " << get_src() << "->" << dst << ": " << R.move_as_error(); - finish_query(); + finish_query(dst, false); return; } auto Res = fetch_tl_object(R.move_as_ok(), true); if (Res.is_error()) { VLOG(DHT_WARNING) << this << ": dropping incorrect answer on dht.findNodes query from " << dst << ": " << Res.move_as_error(); - finish_query(); + finish_query(dst, false); return; } auto r = Res.move_as_ok(); add_nodes(DhtNodesList{create_tl_object(std::move(r->nodes_)), our_network_id()}); - finish_query(); + finish_query(dst); } void DhtQueryFindValue::finish(DhtNodesList list) { - promise_.set_error(td::Status::Error(ErrorCode::notready, "dht key not found")); +} + +bool DhtQueryFindValueSingle::on_value_found(DhtValue value) { + promise_.set_value(std::move(value)); + found_ = true; + return false; +} + +void DhtQueryFindValueSingle::tear_down() { + if (!found_) { + promise_.set_error(td::Status::Error(ErrorCode::notready, "dht key not found")); + } +} + +bool DhtQueryFindValueMany::on_value_found(DhtValue value) { + callback_(std::move(value)); + found_ = true; + return true; +} + +void DhtQueryFindValueMany::tear_down() { + if (found_) { + promise_.set_value(td::Unit()); + } else { + promise_.set_error(td::Status::Error(ErrorCode::notready, "dht key not found")); + } } DhtQueryStore::DhtQueryStore(DhtValue key_value, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, @@ -422,14 +461,14 @@ void DhtQueryRequestReversePing::send_one_query(adnl::AdnlNodeIdShort id) { void DhtQueryRequestReversePing::on_result(td::Result R, adnl::AdnlNodeIdShort dst) { if (R.is_error()) { VLOG(DHT_INFO) << this << ": failed reverse ping query " << get_src() << "->" << dst << ": " << R.move_as_error(); - finish_query(); + finish_query(dst, false); return; } auto Res = fetch_tl_object(R.move_as_ok(), true); if (Res.is_error()) { VLOG(DHT_WARNING) << this << ": dropping incorrect answer on dht.requestReversePing query from " << dst << ": " << Res.move_as_error(); - finish_query(); + finish_query(dst, false); return; } @@ -441,7 +480,7 @@ void DhtQueryRequestReversePing::on_result(td::Result R, adnl:: }, [&](ton_api::dht_clientNotFound &v) { add_nodes(DhtNodesList{std::move(v.nodes_), our_network_id()}); - finish_query(); + finish_query(dst); })); } diff --git a/dht/dht-query.hpp b/dht/dht-query.hpp index cf085e25..e305f107 100644 --- a/dht/dht-query.hpp +++ b/dht/dht-query.hpp @@ -44,7 +44,7 @@ class DhtQuery : public td::actor::Actor { bool client_only_; public: - DhtQuery(DhtKeyId key, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, DhtNodesList list, td::uint32 k, + DhtQuery(DhtKeyId key, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, td::uint32 k, td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only, td::actor::ActorId node, td::actor::ActorId adnl) : key_(key) @@ -57,18 +57,13 @@ class DhtQuery : public td::actor::Actor { , our_network_id_(our_network_id) , node_(node) , adnl_(adnl) { - add_nodes(std::move(list)); } DhtMember::PrintId print_id() const { return print_id_; } void send_queries(); void add_nodes(DhtNodesList list); - void finish_query() { - active_queries_--; - CHECK(active_queries_ <= k_); - send_queries(); - } + void finish_query(adnl::AdnlNodeIdShort id, bool success = true); DhtKeyId get_key() const { return key_; } @@ -89,16 +84,22 @@ class DhtQuery : public td::actor::Actor { virtual std::string get_name() const = 0; private: + struct NodeInfo { + DhtNode node; + int failed_attempts = 0; + }; DhtMember::PrintId print_id_; adnl::AdnlNodeIdShort src_; - std::map list_; - std::set pending_ids_; + std::map nodes_; + std::set result_list_, pending_queries_; td::uint32 k_; td::uint32 a_; td::int32 our_network_id_; td::actor::ActorId node_; td::uint32 active_queries_ = 0; + static const int MAX_ATTEMPTS = 1; + protected: td::actor::ActorId adnl_; }; @@ -112,8 +113,9 @@ class DhtQueryFindNodes : public DhtQuery { td::uint32 k, td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only, td::actor::ActorId node, td::actor::ActorId adnl, td::Promise promise) - : DhtQuery(key, print_id, src, std::move(list), k, a, our_network_id, std::move(self), client_only, node, adnl) + : DhtQuery(key, print_id, src, k, a, our_network_id, std::move(self), client_only, node, adnl) , promise_(std::move(promise)) { + add_nodes(std::move(list)); } void send_one_query(adnl::AdnlNodeIdShort id) override; void on_result(td::Result R, adnl::AdnlNodeIdShort dst); @@ -124,16 +126,12 @@ class DhtQueryFindNodes : public DhtQuery { }; class DhtQueryFindValue : public DhtQuery { - private: - td::Promise promise_; - public: DhtQueryFindValue(DhtKeyId key, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, DhtNodesList list, td::uint32 k, td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only, - td::actor::ActorId node, td::actor::ActorId adnl, - td::Promise promise) - : DhtQuery(key, print_id, src, std::move(list), k, a, our_network_id, std::move(self), client_only, node, adnl) - , promise_(std::move(promise)) { + td::actor::ActorId node, td::actor::ActorId adnl) + : DhtQuery(key, print_id, src, k, a, our_network_id, std::move(self), client_only, node, adnl) { + add_nodes(std::move(list)); } void send_one_query(adnl::AdnlNodeIdShort id) override; void send_one_query_nodes(adnl::AdnlNodeIdShort id); @@ -143,6 +141,48 @@ class DhtQueryFindValue : public DhtQuery { std::string get_name() const override { return "find value"; } + + virtual bool on_value_found(DhtValue value) = 0; +}; + +class DhtQueryFindValueSingle : public DhtQueryFindValue { + public: + DhtQueryFindValueSingle(DhtKeyId key, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, DhtNodesList list, + td::uint32 k, td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only, + td::actor::ActorId node, td::actor::ActorId adnl, + td::Promise promise) + : DhtQueryFindValue(key, print_id, src, std::move(list), k, a, our_network_id, std::move(self), client_only, node, + adnl) + , promise_(std::move(promise)) { + add_nodes(std::move(list)); + } + bool on_value_found(DhtValue value) override; + void tear_down() override; + + private: + td::Promise promise_; + bool found_ = false; +}; + +class DhtQueryFindValueMany : public DhtQueryFindValue { + public: + DhtQueryFindValueMany(DhtKeyId key, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, DhtNodesList list, + td::uint32 k, td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only, + td::actor::ActorId node, td::actor::ActorId adnl, + std::function callback, td::Promise promise) + : DhtQueryFindValue(key, print_id, src, std::move(list), k, a, our_network_id, std::move(self), client_only, node, + adnl) + , callback_(std::move(callback)) + , promise_(std::move(promise)) { + add_nodes(std::move(list)); + } + bool on_value_found(DhtValue value) override; + void tear_down() override; + + private: + std::function callback_; + td::Promise promise_; + bool found_ = false; }; class DhtQueryStore : public td::actor::Actor { @@ -219,11 +259,12 @@ class DhtQueryRequestReversePing : public DhtQuery { td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only, td::actor::ActorId node, td::actor::ActorId adnl, td::Promise promise) - : DhtQuery(DhtMember::get_reverse_connection_key(client).compute_key_id(), print_id, src, std::move(list), k, a, - our_network_id, std::move(self), client_only, node, adnl) + : DhtQuery(DhtMember::get_reverse_connection_key(client).compute_key_id(), print_id, src, k, a, our_network_id, + std::move(self), client_only, node, adnl) , promise_(std::move(promise)) , query_(create_serialize_tl_object(target.tl(), std::move(signature), client.bits256_value(), k)) { + add_nodes(std::move(list)); } void send_one_query(adnl::AdnlNodeIdShort id) override; void on_result(td::Result R, adnl::AdnlNodeIdShort dst); diff --git a/dht/dht-remote-node.cpp b/dht/dht-remote-node.cpp index 653de256..1273750c 100644 --- a/dht/dht-remote-node.cpp +++ b/dht/dht-remote-node.cpp @@ -140,13 +140,7 @@ adnl::AdnlNodeIdFull DhtRemoteNode::get_full_id() const { td::Result> DhtRemoteNode::create(DhtNode node, td::uint32 max_missed_pings, td::int32 our_network_id) { - TRY_RESULT(enc, node.adnl_id().pubkey().create_encryptor()); - auto tl = node.tl(); - auto sig = std::move(tl->signature_); - - TRY_STATUS_PREFIX(enc->check_signature(serialize_tl_object(tl, true).as_slice(), sig.as_slice()), - "bad node signature: "); - + TRY_STATUS(node.check_signature()); return std::make_unique(std::move(node), max_missed_pings, our_network_id); } diff --git a/dht/dht.cpp b/dht/dht.cpp index e1e20d45..b774b5f9 100644 --- a/dht/dht.cpp +++ b/dht/dht.cpp @@ -57,7 +57,7 @@ td::Result> Dht::create(adnl::AdnlNodeIdShort id, std:: for (auto &node : nodes.list()) { auto key = node.get_key(); - td::actor::send_closure(D, &DhtMember::add_full_node, key, node.clone()); + td::actor::send_closure(D, &DhtMember::add_full_node, key, node.clone(), true); } return std::move(D); } @@ -74,7 +74,7 @@ td::Result> Dht::create_client(adnl::AdnlNodeIdShort id for (auto &node : nodes.list()) { auto key = node.get_key(); - td::actor::send_closure(D, &DhtMember::add_full_node, key, node.clone()); + td::actor::send_closure(D, &DhtMember::add_full_node, key, node.clone(), true); } return std::move(D); } @@ -368,7 +368,7 @@ void DhtMemberImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice dat auto node = N.move_as_ok(); if (node.adnl_id().compute_short_id() == src) { auto key = node.get_key(); - add_full_node_impl(key, std::move(node), true); + add_full_node(key, std::move(node), true); } else { VLOG(DHT_WARNING) << this << ": dropping bad node: unexpected adnl id"; } @@ -398,7 +398,7 @@ void DhtMemberImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice dat ton_api::downcast_call(*Q, [&](auto &object) { this->process_query(src, object, std::move(promise)); }); } -void DhtMemberImpl::add_full_node_impl(DhtKeyId key, DhtNode node, bool set_active) { +void DhtMemberImpl::add_full_node(DhtKeyId key, DhtNode node, bool set_active) { VLOG(DHT_EXTRA_DEBUG) << this << ": adding full node " << key; auto eid = key ^ key_; @@ -466,11 +466,11 @@ void DhtMemberImpl::set_value(DhtValue value, td::Promise promise) { void DhtMemberImpl::get_value_in(DhtKeyId key, td::Promise result) { auto P = td::PromiseCreator::lambda([key, promise = std::move(result), SelfId = actor_id(this), print_id = print_id(), - adnl = adnl_, list = get_nearest_nodes(key, k_), k = k_, a = a_, + adnl = adnl_, list = get_nearest_nodes(key, k_ * 2), k = k_, a = a_, network_id = network_id_, id = id_, client_only = client_only_](td::Result R) mutable { R.ensure(); - td::actor::create_actor("FindValueQuery", key, print_id, id, std::move(list), k, a, network_id, + td::actor::create_actor("FindValueQuery", key, print_id, id, std::move(list), k, a, network_id, R.move_as_ok(), client_only, SelfId, adnl, std::move(promise)) .release(); }); @@ -478,6 +478,21 @@ void DhtMemberImpl::get_value_in(DhtKeyId key, td::Promise result) { get_self_node(std::move(P)); } +void DhtMemberImpl::get_value_many(DhtKey key, std::function callback, td::Promise promise) { + DhtKeyId key_id = key.compute_key_id(); + auto P = td::PromiseCreator::lambda( + [key = key_id, callback = std::move(callback), promise = std::move(promise), SelfId = actor_id(this), + print_id = print_id(), adnl = adnl_, list = get_nearest_nodes(key_id, k_ * 2), k = k_, a = a_, + network_id = network_id_, id = id_, client_only = client_only_](td::Result R) mutable { + R.ensure(); + td::actor::create_actor("FindValueManyQuery", key, print_id, id, std::move(list), k, a, + network_id, R.move_as_ok(), client_only, SelfId, adnl, + std::move(callback), std::move(promise)) + .release(); + }); + get_self_node(std::move(P)); +} + void DhtMemberImpl::register_reverse_connection(adnl::AdnlNodeIdFull client, td::Promise promise) { auto client_short = client.compute_short_id(); td::uint32 ttl = (td::uint32)td::Clocks::system() + 300; @@ -485,7 +500,7 @@ void DhtMemberImpl::register_reverse_connection(adnl::AdnlNodeIdFull client, td: auto key_id = get_reverse_connection_key(client_short).compute_key_id(); td::actor::send_closure(keyring_, &keyring::Keyring::sign_message, client_short.pubkey_hash(), register_reverse_connection_to_sign(client_short, id_, ttl), - [=, print_id = print_id(), list = get_nearest_nodes(key_id, k_), SelfId = actor_id(this), + [=, print_id = print_id(), list = get_nearest_nodes(key_id, k_ * 2), SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { TRY_RESULT_PROMISE_PREFIX(promise, signature, std::move(R), "Failed to sign: "); td::actor::send_closure(SelfId, &DhtMemberImpl::get_self_node, @@ -532,7 +547,7 @@ void DhtMemberImpl::request_reverse_ping_cont(adnl::AdnlNode target, td::BufferS } auto key_id = get_reverse_connection_key(client).compute_key_id(); get_self_node([=, target = std::move(target), signature = std::move(signature), promise = std::move(promise), - SelfId = actor_id(this), print_id = print_id(), list = get_nearest_nodes(key_id, k_), + SelfId = actor_id(this), print_id = print_id(), list = get_nearest_nodes(key_id, k_ * 2), client_only = client_only_](td::Result R) mutable { R.ensure(); td::actor::create_actor( @@ -651,8 +666,8 @@ void DhtMemberImpl::check() { DhtKeyId key{x}; auto P = td::PromiseCreator::lambda([key, promise = std::move(promise), SelfId = actor_id(this), - print_id = print_id(), adnl = adnl_, list = get_nearest_nodes(key, k_), k = k_, - a = a_, network_id = network_id_, id = id_, + print_id = print_id(), adnl = adnl_, list = get_nearest_nodes(key, k_ * 2), + k = k_, a = a_, network_id = network_id_, id = id_, client_only = client_only_](td::Result R) mutable { R.ensure(); td::actor::create_actor("FindNodesQuery", key, print_id, id, std::move(list), k, a, network_id, @@ -677,8 +692,8 @@ void DhtMemberImpl::send_store(DhtValue value, td::Promise promise) { auto key_id = value.key_id(); auto P = td::PromiseCreator::lambda([value = std::move(value), print_id = print_id(), id = id_, - client_only = client_only_, list = get_nearest_nodes(key_id, k_), k = k_, a = a_, - network_id = network_id_, SelfId = actor_id(this), adnl = adnl_, + client_only = client_only_, list = get_nearest_nodes(key_id, k_ * 2), k = k_, + a = a_, network_id = network_id_, SelfId = actor_id(this), adnl = adnl_, promise = std::move(promise)](td::Result R) mutable { R.ensure(); td::actor::create_actor("StoreQuery", std::move(value), print_id, id, std::move(list), k, a, diff --git a/dht/dht.h b/dht/dht.h index b9c65c8a..5abff94a 100644 --- a/dht/dht.h +++ b/dht/dht.h @@ -53,6 +53,7 @@ class Dht : public td::actor::Actor { virtual void set_value(DhtValue key_value, td::Promise result) = 0; virtual void get_value(DhtKey key, td::Promise result) = 0; + virtual void get_value_many(DhtKey key, std::function callback, td::Promise promise) = 0; virtual void register_reverse_connection(adnl::AdnlNodeIdFull client, td::Promise promise) = 0; virtual void request_reverse_ping(adnl::AdnlNode target, adnl::AdnlNodeIdShort client, diff --git a/dht/dht.hpp b/dht/dht.hpp index 0b46d635..9fb05e08 100644 --- a/dht/dht.hpp +++ b/dht/dht.hpp @@ -95,7 +95,7 @@ class DhtMember : public Dht { //virtual void update_addr_list(tl_object_ptr addr_list) = 0; //virtual void add_node(adnl::AdnlNodeIdShort id) = 0; - virtual void add_full_node(DhtKeyId id, DhtNode node) = 0; + virtual void add_full_node(DhtKeyId id, DhtNode node, bool set_active) = 0; virtual void receive_ping(DhtKeyId id, DhtNode result) = 0; diff --git a/doc/ConfigParam-HOWTO b/doc/ConfigParam-HOWTO index 8432657a..639014bb 100644 --- a/doc/ConfigParam-HOWTO +++ b/doc/ConfigParam-HOWTO @@ -13,7 +13,7 @@ All configuration parameters are combined into a *configuration dictionary* - a We see that, apart from the configuration dictionary, `ConfigParams` contains `config_addr` -- 256-bit address of the configuration smart contract in the masterchain. More details on the configuration smart contract will be provided later. -The configuration dictionary containing the active values of all configuration parameters is available via special TVM register *c7* to all smart contracts when their code is executed in a transaction. More precisely, when a smart contract is executed, *c7* is initialized by a Tuple, the only element of which is a Tuple with several "context" values useful for the execution of the smart contract, such as the current Unix time (as registered in the block header). The tenth entry of this Tuple (i.e., the one with zero-based index 9) contains a Cell representing the configuration dictionary. Therefore, it can be accesses by means of TVM instructions "PUSH c7; FIRST; INDEX 9", or by equivalent instruction "CONFIGROOT". In fact, special TVM instructions "CONFIGPARAM" and "CONFIGOPTPARAM" combine the previous actions with a dictionary lookup, returning any configuration parameter by its index. We refer to the TVM documentation for more details on these instructions. What is relevant here is that all configuration parameters are easily accessible from all smart contracts (masterchain or shardchain), and smart contracts may inspect them and use them to perform specific checks. For instance, a smart contract might extract workchain data storage prices from a configuration parameter to compute the price for storing a chunk of user-provided data. +The configuration dictionary containing the active values of all configuration parameters is available via special TVM register *c7* to all smart contracts when their code is executed in a transaction. More precisely, when a smart contract is executed, *c7* is initialized by a Tuple, the only element of which is a Tuple with several "context" values useful for the execution of the smart contract, such as the current Unix time (as registered in the block header). The tenth entry of this Tuple (i.e., the one with zero-based index 9) contains a Cell representing the configuration dictionary. Therefore, it can be accessed by means of TVM instructions "PUSH c7; FIRST; INDEX 9", or by equivalent instruction "CONFIGROOT". In fact, special TVM instructions "CONFIGPARAM" and "CONFIGOPTPARAM" combine the previous actions with a dictionary lookup, returning any configuration parameter by its index. We refer to the TVM documentation for more details on these instructions. What is relevant here is that all configuration parameters are easily accessible from all smart contracts (masterchain or shardchain), and smart contracts may inspect them and use them to perform specific checks. For instance, a smart contract might extract workchain data storage prices from a configuration parameter to compute the price for storing a chunk of user-provided data. The values of configuration parameters are not arbitrary. In fact, if the configuration parameter index *i* is non-negative, then the value of this parameter must be a valid value of TL-B type (ConfigParam i). This restriction is enforced by the validators, which will not accept changes to configuration parameters with non-negative indices unless they are valid values of the corresponding TL-B type. diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md new file mode 100644 index 00000000..77963e95 --- /dev/null +++ b/doc/GlobalVersions.md @@ -0,0 +1,158 @@ +# Global versions +Global version is a parameter specified in `ConfigParam 8` ([block.tlb](https://github.com/ton-blockchain/ton/blob/master/crypto/block/block.tlb#L595)). +Various features are enabled depending on the global version. + +## Version 4 +New features of version 4 are desctibed in detail in [the documentation](https://docs.ton.org/v3/documentation/tvm/changelog/tvm-upgrade-2023-07). + +### New TVM instructions +* `PREVMCBLOCKS`, `PREVKEYBLOCK` +* `GLOBALID` +* `HASHEXT(A)(R)` +* `ECRECOVER` +* `SENDMSG` +* `RUNVM`, `RUNVMX` +* `GASCONSUMED` +* `RIST255_...` instructions +* `BLS_...` instructions +* `P256_CHKSIGNS`, `P256_CHKSIGNU` + +### Division +[Division instruction](https://ton.org/docs/learn/tvm-instructions/instructions#52-division) can add a number to the +intermediate value before division (e.g. `(xy+w)/z`). + +### Stack operations +* Arguments of `PICK`, `ROLL`, `ROLLREV`, `BLKSWX`, `REVX`, `DROPX`, `XCHGX`, `CHKDEPTH`, `ONLYTOPX`, `ONLYX` are now unlimited. +* `ROLL`, `ROLLREV`, `BLKSWX`, `REVX`, `ONLYTOPX` consume more gas when arguments are big. + +### c7 tuple +**c7** tuple extended from 10 to 14 elements: +* **10**: code of the smart contract. +* **11**: value of the incoming message. +* **12**: fees collected in the storage phase. +* **13**: information about previous blocks. + +### Action phase +* If "send message" action fails, the account is required to pay for processing cells of the message. +* Flag +16 in actions "Send message", "Reserve", "Change library" causes bounce if action fails. + +### Storage phase +* Unpaid storage fee is now saved to `due_payment` + +## Version 5 + +### Gas limits +Version 5 enables higher gas limits for special contracts. + +* Gas limit for all transactions on special contracts is set to `special_gas_limit` from `ConfigParam 20` (which is 35M at the moment of writing). +Previously only ticktock transactions had this limit, while ordinary transactions had a default limit of `gas_limit` gas (1M). +* Gas usage of special contracts is not taken into account when checking block limits. This allows keeping masterchain block limits low +while having high gas limits for elector. +* Gas limit on `EQD_v9j1rlsuHHw2FIhcsCFFSD367ldfDdCKcsNmNpIRzUlu` is increased to 70M (`special_gas_limit * 2`) until 2024-02-29. +See [this post](https://t.me/tonstatus/88) for details. + +### Loading libraries +* Loading "nested libraries" (i.e. a library cell that points to another library cell) throws an exception. +* Loading a library consumes gas for cell load only once (for the library cell), not twice (both for the library cell and the cell in the library). +* `XLOAD` now works differently. When it takes a library cell, it returns the cell that it points to. This allows loading "nested libraries", if needed. + +## Version 6 + +### c7 tuple +**c7** tuple extended from 14 to 17 elements: +* **14**: tuple that contains some config parameters as cell slices. If the parameter is absent from the config, the value is null. Asm opcode: `UNPACKEDCONFIGTUPLE`. + * **0**: `StoragePrices` from `ConfigParam 18`. Not the whole dict, but only the one StoragePrices entry (one which corresponds to the current time). + * **1**: `ConfigParam 19` (global id). + * **2**: `ConfigParam 20` (mc gas prices). + * **3**: `ConfigParam 21` (gas prices). + * **4**: `ConfigParam 24` (mc fwd fees). + * **5**: `ConfigParam 25` (fwd fees). + * **6**: `ConfigParam 43` (size limits). +* **15**: "[due payment](https://github.com/ton-blockchain/ton/blob/8a9ff339927b22b72819c5125428b70c406da631/crypto/block/block.tlb#L237)" - current debt for storage fee (nanotons). Asm opcode: `DUEPAYMENT`. +* **16**: "precompiled gas usage" - gas usage for the current contract if it is precompiled (see `ConfigParam 45`), `null` otherwise. Asm opcode: `GETPRECOMPILEDGAS`. + +### New TVM instructions + +#### Fee calculation +* `GETGASFEE` (`gas_used is_mc - price`) - calculates gas fee. +* `GETSTORAGEFEE` (`cells bits seconds is_mc - price`) - calculates storage fees (only current StoragePrices entry is used). +* `GETFORWARDFEE` (`cells bits is_mc - price`) - calculates forward fee. +* `GETPRECOMPILEDGAS` (`- x`) - returns gas usage for the current contract if it is precompiled, `null` otherwise. +* `GETORIGINALFWDFEE` (`fwd_fee is_mc - orig_fwd_fee`) - calculate `fwd_fee * 2^16 / first_frac`. Can be used to get the original `fwd_fee` of the message. +* `GETGASFEESIMPLE` (`gas_used is_mc - price`) - same as `GETGASFEE`, but without flat price (just `(gas_used * price) / 2^16`). +* `GETFORWARDFEESIMPLE` (`cells bits is_mc - price`) - same as `GETFORWARDFEE`, but without lump price (just `(bits*bit_price + cells*cell_price) / 2^16`). + +`gas_used`, `cells`, `bits`, `time_delta` are integers in range `0..2^63-1`. + +#### Cell operations +Operations for working with Merkle proofs, where cells can have non-zero level and multiple hashes. +* `CLEVEL` (`cell - level`) - returns level of the cell. +* `CLEVELMASK` (`cell - level_mask`) - returns level mask of the cell. +* `i CHASHI` (`cell - hash`) - returns `i`th hash of the cell. +* `i CDEPTHI` (`cell - depth`) - returns `i`th depth of the cell. +* `CHASHIX` (`cell i - hash`) - returns `i`th hash of the cell. +* `CDEPTHIX` (`cell i - depth`) - returns `i`th depth of the cell. + +`i` is in range `0..3`. + +### Other changes +* `GLOBALID` gets `ConfigParam 19` from the tuple, not from the config dict. This decreases gas usage. +* `SENDMSG` gets `ConfigParam 24/25` (message prices) from the tuple, not from the config dict, and also uses `ConfigParam 43` to get max_msg_cells. + + +## Version 7 + +[Explicitly nullify](https://github.com/ton-blockchain/ton/pull/957/files) `due_payment` after due reimbursment. + +## Version 8 + +- Check mode on invalid `action_send_msg`. Ignore action if `IGNORE_ERROR` (+2) bit is set, bounce if `BOUNCE_ON_FAIL` (+16) bit is set. +- Slightly change random seed generation to fix mix of `addr_rewrite` and `addr`. +- Fill in `skipped_actions` for both invalid and valid messages with `IGNORE_ERROR` mode that can't be sent. +- Allow unfreeze through external messages. +- Don't use user-provided `fwd_fee` and `ihr_fee` for internal messages. + +## Version 9 + +### c7 tuple +c7 tuple parameter number **13** (previous blocks info tuple) now has the third element. It contains ids of the 16 last masterchain blocks with seqno divisible by 100. +Example: if the last masterchain block seqno is `19071` then the list contains block ids with seqnos `19000`, `18900`, ..., `17500`. + +### New TVM instructions +- `SECP256K1_XONLY_PUBKEY_TWEAK_ADD` (`key tweak - 0 or f x y -1`) - performs [`secp256k1_xonly_pubkey_tweak_add`](https://github.com/bitcoin-core/secp256k1/blob/master/include/secp256k1_extrakeys.h#L120). +`key` and `tweak` are 256-bit unsigned integers. 65-byte public key is returned as `uint8 f`, `uint256 x, y` (as in `ECRECOVER`). Gas cost: `1276`. +- `mask SETCONTCTRMANY` (`cont - cont'`) - takes continuation, performs the equivalent of `c[i] PUSHCTR SWAP c[i] SETCONTCNR` for each `i` that is set in `mask` (mask is in `0..255`). +- `SETCONTCTRMANYX` (`cont mask - cont'`) - same as `SETCONTCTRMANY`, but takes `mask` from stack. +- `PREVMCBLOCKS_100` returns the third element of the previous block info tuple (see above). + +### Other changes +- Fix `RAWRESERVE` action with flag `4` (use original balance of the account) by explicitly setting `original_balance` to `balance - msg_balance_remaining`. + - Previously it did not work if storage fee was greater than the original balance. +- Jumps to nested continuations of depth more than 8 consume 1 gas for eact subsequent continuation (this does not affect most of TVM code). +- Support extra currencies in reserve action with `+2` mode. +- Fix exception code in some TVM instructions: now `stk_und` has priority over other error codes. + - `PFXDICTADD`, `PFXDICTSET`, `PFXDICTREPLACE`, `PFXDICTDEL`, `GETGASFEE`, `GETSTORAGEFEE`, `GETFORWARDFEE`, `GETORIGINALFWDFEE`, `GETGASFEESIMPLE`, `GETFORWARDFEESIMPLE`, `HASHEXT` +- Now setting the contract code to a library cell does not consume additional gas on execution of the code. +- Temporary increase gas limit for some accounts (see [this post](https://t.me/tondev_news/129) for details, `override_gas_limit` in `transaction.cpp` for the list of accounts). +- Fix recursive jump to continuations with non-null control data. + +## Version 10 + +### Extra currencies +- Internal messages cannot carry more than 2 different extra currencies. The limit can be changed in size limits config (`ConfigParam 43`). +- Amount of an extra currency in an output action "send message" can be zero. + - In action phase zero values are automatically deleted from the dictionary before sending. + - However, the size of the extra currency dictionary in the "send message" action should not be greater than 2 (or the value in size limits config). +- Extra currency dictionary is not counted in message size and does not affect message fees. +- Message mode `+64` (carry all remaining message balance) is now considered as "carry all remaining TONs from message balance". +- Message mode `+128` (carry all remaining account balance) is now considered as "carry all remaining TONs from account balance". +- Message mode `+32` (delete account if balance is zero) deletes account if it has zero TONs, regardless of extra currencies. + - Deleted accounts with extra currencies become `account_uninit`, extra currencies remain on the account. +- `SENDMSG` in TVM calculates message size and fees without extra currencies, uses new `+64` and `+128` mode behavior. + - `SENDMSG` does not check the number of extra currencies. +- Extra currency dictionary is not counted in the account size and does not affect storage fees. + - Accounts with already existing extra currencies will get their sizes recomputed without EC only after modifying `AccountState`. + +### TVM changes +- `SENDMSG` calculates messages size and fees without extra currencies, uses new +64 and +128 mode behavior. + - `SENDMSG` does not check the number of extra currencies. diff --git a/doc/Tests.md b/doc/Tests.md new file mode 100644 index 00000000..c883731a --- /dev/null +++ b/doc/Tests.md @@ -0,0 +1,24 @@ +# Tests execution +TON contains multiple unit-tests, that facilitate detection of erroneous blockchain behaviour on each commit. +## Build tests +Go inside the build directory and, if you use ninja, build the tests using the following command: + +```ninja test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state``` + +For more details on how to build TON artifacts, please refer to any of Github actions. + +For cmake use: + +```cmake --build . --target test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state``` + +## Run tests +Go inside the build directory and with ninja execute: + +```ninja test``` + +with ctest: + +```ctest``` + +## Integration of tests into CI +Most relevant GitHub actions include the step ```Run tests``` that executes the tests. If any of tests fails, the action will be interrupted and no artifacts will be provided. \ No newline at end of file diff --git a/doc/catchain-dos.md b/doc/catchain-dos.md index 028cb95a..d256b157 100644 --- a/doc/catchain-dos.md +++ b/doc/catchain-dos.md @@ -13,8 +13,8 @@ For catchain protocol version higher or equal to 2 hash covers catchain block de ### Catchain block size All catchain messages, except `REJECT` message, have (and had) a limited size. After update `REJECT` message will be limited to 1024 bytes. At the same time one block contains at most number of block candidates per round messages. That way, 16kb limit per catchain block should be enough to prevent DoS. ### Limiting block height -Another potential DoS is related to a situation when a malbehaviouring node sends too many catchain blocks. Note that limiting of maximal number of blocks per second is not a good solution since it will hinder synchronization after node disconnect. -At the same time, catchain groups exist for quite short period of time (around a few hunderd seconds), while number of blocks production is determined by "natural block production speed" on the one hand and number of blocks generated to decrease dependencies size on the other. In any case, total number of blocks is limited by `catchain_lifetime * natural_block_production_speed * (1 + number_of_catchain_participants / max_dependencies_size)`. +Another potential DoS is related to a situation when a misbehaviouring node sends too many catchain blocks. Note that limiting of maximal number of blocks per second is not a good solution since it will hinder synchronization after node disconnect. +At the same time, catchain groups exist for quite short period of time (around a few hundred seconds), while number of blocks production is determined by "natural block production speed" on the one hand and number of blocks generated to decrease dependencies size on the other. In any case, total number of blocks is limited by `catchain_lifetime * natural_block_production_speed * (1 + number_of_catchain_participants / max_dependencies_size)`. To prevent DoS attack we limit maximal height of the block which will be processed by node by `catchain_lifetime * natural_block_production_speed * (1 + number_of_catchain_participants / max_dependencies_size)`, where `catchain_lifetime` is set by `ConfigParam 28` (`CatchainConfig`), `natural_block_production_speed` and `max_dependencies_size` are set by `ConfigParam 29` (`ConsensusConfig`) (`natural_block_production_speed` is calculated as `catchain_max_blocks_coeff / 1000`) and `number_of_catchain_participants` is set from catchain group configuration. By default, `catchain_max_blocks_coeff` is set to zero: special value which means that there is no limitation on catchain block height. It is recommended to set `catchain_max_blocks_coeff` to `10000`: we expect that natural production rate is about one block per 3 seconds, so we set the coefficient to allow 30 times higher block production than expected. At the same time, this number is low enough to prevent resource-intensive attacks. diff --git a/doc/catchain.tex b/doc/catchain.tex index b3cae642..b1a27164 100644 --- a/doc/catchain.tex +++ b/doc/catchain.tex @@ -243,7 +243,7 @@ Notice that an external punishment for creating catchain forks may be used in th Once a fork (created by~$i$) is detected (by another process~$j$), i.e.\ $j$ learns about two different messages $m_{i,s}$ and $m'_{i,s}$ created by $i$ and having same height $s$ (usually this happens while recursively downloading dependencies of some other messages), $j$ starts ignoring~$i$ and all of its subsequent messages. They are not accepted and not broadcast further. However, messages created by~$i$ prior to the fork detection may be still downloaded if they are referred to in messages (blocks) created by processes that did not see this fork before referring to such messages created by~$i$. \nxsubpoint\emb{Accepting messages from a ``bad'' process is bad}\label{sp:no.bad.accept} -Furthermore, if process $i$ learns about a fork created by process $j$, then $i$ shows this to its neighbors by creating a new service broadcast message that contains the corresponding fork proof (cf.~\ptref{sp:fork.proofs}). Afterwards, this and all subsequent messages of $j$ cannot directly depend on any messages by the known ``bad'' producer $i$ (but they still can refer to messages from another party $k$ that directly or indirectly refer to messages of~$i$ if no fork by~$i$ was known to $k$ at the time when the referring message was created). If $j$ violates this restriction and creates messages with such invalid references, these messages will be discarded by all honest processes in the group. +Furthermore, if process $i$ learns about a fork created by process $j$, then $i$ shows this to its neighbors by creating a new service broadcast message that contains the corresponding fork proof (cf.~\ptref{sp:fork.proofs}). Afterwards, this and all subsequent messages of $i$ cannot directly depend on any messages by the known ``bad'' producer $j$ (but they still can refer to messages from another party $k$ that directly or indirectly refer to messages of~$i$ if no fork by~$i$ was known to $k$ at the time when the referring message was created). If $i$ violates this restriction and creates messages with such invalid references, these messages will be discarded by all honest processes in the group. \nxsubpoint\emb{The set of ``bad'' group members is a part of the intrinsic state}\label{sp:bad.proc.set} Each process~$i$ keeps its own copy of the set of known ``bad'' processes in the group, i.e., those processes that have created at least one fork or have violated \ptref{sp:no.bad.accept}. This set is updated by adding~$j$ into it as soon as $i$ learns about a fork created by~$j$ (or about a violation of~\ptref{sp:no.bad.accept} by $j$); after that, a callback provided by the higher-level protocol is invoked. This set is used when a new broadcast message arrives: if the sender is bad, then the message is ignored and discarded. diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 7536a492..00000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -FROM ubuntu:20.04 as builder -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang-6.0 openssl libssl-dev zlib1g-dev gperf wget git ninja-build && \ - rm -rf /var/lib/apt/lists/* -ENV CC clang-6.0 -ENV CXX clang++-6.0 -ENV CCACHE_DISABLE 1 -WORKDIR / -RUN git clone --recursive https://github.com/ton-blockchain/ton -WORKDIR /ton - -RUN mkdir build && \ - cd build && \ - cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. && \ - ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console generate-random-id dht-server lite-client - -FROM ubuntu:20.04 -RUN apt-get update && \ - apt-get install -y openssl wget libatomic1 && \ - rm -rf /var/lib/apt/lists/* - -RUN mkdir -p /var/ton-work/db && \ - mkdir -p /var/ton-work/db/static - -COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon /usr/local/bin/ -COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon-cli /usr/local/bin/ -COPY --from=builder /ton/build/lite-client/lite-client /usr/local/bin/ -COPY --from=builder /ton/build/validator-engine/validator-engine /usr/local/bin/ -COPY --from=builder /ton/build/validator-engine-console/validator-engine-console /usr/local/bin/ -COPY --from=builder /ton/build/utils/generate-random-id /usr/local/bin/ - -WORKDIR /var/ton-work/db -COPY init.sh control.template ./ -RUN chmod +x init.sh - -ENTRYPOINT ["/var/ton-work/db/init.sh"] \ No newline at end of file diff --git a/docker/README.md b/docker/README.md index fd98374b..47e109db 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,28 +1,525 @@ -# The Open Network Node -Dockerfile for The Open Network Node +# Official TON Docker image + +1. [Dockerfile](#docker) +2. [Kubernetes deployment on-premises](#deploy-on-premises-with-metallb-load-balancer-) +3. [Kubernetes deployment on AWS](#deploy-on-aws-cloud-amazon-web-services) +4. [Kubernetes deployment on GCP](#deploy-on-gcp-google-cloud-platform) +5. [Kubernetes deployment on AliCloud](#deploy-on-ali-cloud) +6. [Troubleshooting](#troubleshooting) +## Prerequisites + +The TON node, whether it is validator or fullnode, requires a public IP address. +If your server is within an internal network or kubernetes you have to make sure that the required ports are available from the outside. + +Also pay attention at [hardware requirements](https://docs.ton.org/participate/run-nodes/full-node) for TON fullnodes and validators. Pods and StatefulSets in this guide imply these requirements. + +It is recommended to everyone to read Docker chapter first in order to get a better understanding about TON Docker image and its parameters. + +## Docker + +### Installation +```docker pull ghcr.io/ton-blockchain/ton:latest``` + +### Configuration +TON validator-engine supports number of command line parameters, +these parameters can be handed over to the container via environment variables. +Below is the list of supported arguments and their default values: + +| Argument | Description | Mandatory? | Default value | +|:------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:----------:|:-------------------------------------------------------:| +| PUBLIC_IP | This will be a public IP address of your TON node. Normally it is the same IP address as your server's external IP. This also can be your proxy server or load balancer IP address. | yes | | +| GLOBAL_CONFIG_URL | TON global configuration file. Mainnet - https://ton.org/global-config.json, Testnet - https://ton.org/testnet-global.config.json | no | https://api.tontech.io/ton/wallet-mainnet.autoconf.json | +| DUMP_URL | URL to TON dump. Specify dump from https://dump.ton.org. If you are using testnet dump, make sure to download global config for testnet. | no | | +| VALIDATOR_PORT | UDP port that must be available from the outside. Used for communication with other nodes. | no | 30001 | +| CONSOLE_PORT | This TCP port is used to access validator's console. Not necessarily to be opened for external access. | no | 30002 | +| LITE_PORT | Lite-server's TCP port. Used by lite-client. | no | 30003 | +| LITESERVER | true or false. Set to true if you want up and running lite-server. | no | false | +| STATE_TTL | Node's state will be gc'd after this time (in seconds). | no | 86400 | +| ARCHIVE_TTL | Node's archived blocks will be deleted after this time (in seconds). | no | 86400 | +| THREADS | Number of threads used by validator-engine. | no | 8 | +| VERBOSITY | Verbosity level. | no | 3 | +| CUSTOM_ARG | validator-engine might have some undocumented arguments. This is reserved for the test purposes.
For example you can pass **--logname /var/ton-work/log** in order to have log files. | no | | + +### Run the node - the quick way +The below command runs docker container with a TON node, that will start synchronization process. + +Notice **--network host** option, means that the Docker container will use the network namespace of the host machine. +In this case there is no need to map ports between the host and the container. The container will use the same IP address and ports as the host. +This approach simplifies networking configuration for the container, and usually is used on the dedicated server with assigned public IP. + +Keep in mind that this option can also introduce security concerns because the container has access to the host's network interfaces directly, which might not be desirable in a multi-tenant environment. + +Check your firewall configuration and make sure that at least UDP port 43677 is publicly available. +Find out your PUBLIC_IP: +``` +curl -4 ifconfig.me +``` +and replace it in the command below: +``` +docker run -d --name ton-node -v /data/db:/var/ton-work/db \ +-e "PUBLIC_IP=" \ +-e "LITESERVER=true" \ +-e "DUMP_URL=https://dump.ton.org/dumps/latest.tar.lz" \ +--network host \ +-it ghcr.io/ton-blockchain/ton +``` +If you don't need Lite-server, then remove -e "LITESERVER=true". + +### Run the node - isolated way +In production environments it is recommended to use **Port mapping** feature of Docker's default bridge network. +When you use port mapping, Docker allocates a specific port on the host to forward traffic to a port inside the container. +This is ideal for running multiple containers with isolated networks on the same host. +``` +docker run -d --name ton-node -v /data/db:/var/ton-work/db \ +-e "PUBLIC_IP=" \ +-e "DUMP_URL=https://dump.ton.org/dumps/latest.tar.lz" \ +-e "VALIDATOR_PORT=443" \ +-e "CONSOLE_PORT=88" \ +-e "LITE_PORT=443" \ +-e "LITESERVER=true" \ +-p 443:443/udp \ +-p 88:88/tcp \ +-p 443:443/tcp \ +-it ghcr.io/ton-blockchain/ton +``` +Adjust ports per your need. +Check your firewall configuration and make sure that customized ports (443/udp, 88/tcp and 443/tcp in this example) are publicly available. + +### Verify if TON node is operating correctly +After executing above command check the log files: + +```docker logs ton-node``` + +This is totally fine if in the log output for some time (up to 15 minutes) you see messages like: + +```log +failed to download proof link: [Error : 651 : no nodes] +``` + +After some time you should be able to see multiple messages similar to these below: +```log +failed to download key blocks: [Error : 652 : adnl query timeout] +last key block is [ w=-1 s=9223372036854775808 seq=34879845 rcEsfLF3E80PqQPWesW+rlOY2EpXd5UDrW32SzRWgus= C1Hs+q2Vew+WxbGL6PU1P6R2iYUJVJs4032CTS/DQzI= ] +getnextkey: [Error : 651 : not inited] +downloading state (-1,8000000000000000,38585739):9E86E166AE7E24BAA22762766381440C625F47E2B11D72967BB58CE8C90F7EBA:5BFFF759380097DF178325A7151E9C0571C4E452A621441A03A0CECAED970F57: total=1442840576 (71MB/s)downloading state (-1,8000000000000000,38585739):9E86E166AE7E24BAA22762766381440C625F47E2B11D72967BB58CE8C90F7EBA:5BFFF759380097DF178325A7151E9C0571C4E452A621441A03A0CECAED970F57: total=1442840576 (71MB/s) +finished downloading state (-1,8000000000000000,38585739):9E86E166AE7E24BAA22762766381440C625F47E2B11D72967BB58CE8C90F7EBA:5BFFF759380097DF178325A7151E9C0571C4E452A621441A03A0CECAED970F57: total=4520747390 +getnextkey: [Error : 651 : not inited] +getnextkey: [Error : 651 : not inited] +``` +As you noticed we have mounted docker volume to a local folder **/data/db**. +Go inside this folder on your server and check if its size is growing (```sudo du -h .*```) + +Now connect to the running container: +``` +docker exec -ti ton-node /bin/bash +``` +and try to connect and execute **getconfig** command via validator-engine-console: +``` +validator-engine-console -k client -p server.pub -a localhost:$(jq .control[].port <<< cat /var/ton-work/db/config.json) -c getconfig +``` +if you see a json output that means that validator-engine is up, now execute **last** command with a lite-client: +``` +lite-client -a localhost:$(jq .liteservers[].port <<< cat /var/ton-work/db/config.json) -p liteserver.pub -c last +``` +if you see the following output: +``` +conn ready +failed query: [Error : 652 : adnl query timeout] +cannot get server version and time (server too old?) +server version is too old (at least 1.1 with capabilities 1 required), some queries are unavailable +fatal error executing command-line queries, skipping the rest +``` +it means that the lite-server is up, but the node is not synchronized yet. +Once the node is synchronized, the output of **last** command will be similar to this one: + +``` +conn ready +server version is 1.1, capabilities 7 +server time is 1719306580 (delta 0) +last masterchain block is (-1,8000000000000000,20435927):47A517265B25CE4F2C8B3058D46343C070A4B31C5C37745390CE916C7D1CE1C5:279F9AA88C8146257E6C9B537905238C26E37DC2E627F2B6F1D558CB29A6EC82 +server time is 1719306580 (delta 0) +zerostate id set to -1:823F81F306FF02694F935CF5021548E3CE2B86B529812AF6A12148879E95A128:67E20AC184B9E039A62667ACC3F9C00F90F359A76738233379EFA47604980CE8 +``` +If you can't make it working, refer to the [Troubleshooting](#troubleshooting) section below. +### Use validator-engine-console +```docker exec -ti ton-node /bin/bash``` + +```validator-engine-console -k client -p server.pub -a 127.0.0.1:$(jq .control[].port <<< cat /var/ton-work/db/config.json)``` + +### Use lite-client +```docker exec -ti ton-node /bin/bash``` + +```lite-client -p liteserver.pub -a 127.0.0.1:$(jq .liteservers[].port <<< cat /var/ton-work/db/config.json)``` + +If you use lite-client outside the Docker container, copy the **liteserver.pub** from the container: + +```docker cp ton-node:/var/ton-work/db/liteserver.pub /your/path``` + +```lite-client -p /your/path/liteserver.pub -a :``` + +### Stop TON docker container +``` +docker stop ton-node +``` + +## Kubernetes +### Deploy in a quick way (without load balancer) +If the nodes within your kubernetes cluster have external IPs, +make sure that the PUBLIC_IP used for validator-engine matches the node's external IP. +If all Kubernetes nodes are inside DMZ - skip this section. + +#### Prepare +If you are using **flannel** network driver you can find node's IP this way: +```yaml +kubectl get nodes +kubectl describe node | grep public-ip +``` +for **calico** driver use: +```yaml +kubectl describe node | grep IPv4Address +``` +Double check if your Kubernetes node's external IP coincides with the host's IP address: +``` +kubectl run --image=ghcr.io/ton-blockchain/ton:latest validator-engine-pod --env="HOST_IP=1.1.1.1" --env="PUBLIC_IP=1.1.1.1" +kubectl exec -it validator-engine-pod -- curl -4 ifconfig.me +kubectl delete pod validator-engine-pod +``` +If IPs do not match, refer to the sections where load balancers are used. + +Now do the following: +* Add a label to this particular node. +* By this label our pod will know where to be deployed and what storage to use: +``` +kubectl label nodes node_type=ton-validator +``` +* Replace **** (and ports if needed) in file [ton-node-port.yaml](ton-node-port.yaml). +* Replace **** with a real path on host for Persistent Volume. +* If you change the ports, make sure you specify appropriate env vars in Pod section. +* If you want to use dynamic storage provisioning via volumeClaimTemplates, feel free to create own StorageClass. #### Install -```docker pull ghcr.io/ton-blockchain/ton:latest``` -#### Create volume -```docker volume create ton-db``` -#### Run -```docker run -d --name ton-node --mount source=ton-db,target=/var/ton-work/db --network host -e "PUBLIC_IP=" -e "CONSOLE_PORT=" -e "LITESERVER=true" -e "LITE_PORT=" -it ghcr.io/ton-blockchain/ton``` +```yaml +kubectl apply -f ton-node-port.yaml +``` + +this deployment uses host's network stack (**hostNetwork: true**) option and service of **NodePort** type. +Actually you can also use service of type **LoadBalancer**. +This way the service will get public IP assigned to the endpoints. + +#### Verify installation +See if service endpoints were correctly created: + +```yaml +kubectl get endpoints + +NAME ENDPOINTS +validator-engine-srv :30002,:30001,:30003 +``` +Check the logs for the deployment status: +```yaml +kubectl logs validator-engine-pod +``` +or go inside the pod and check if blockchain size is growing: +```yaml +kubectl exec --stdin --tty validator-engine-pod -- /bin/bash +du -h . +``` +### Deploy on-premises with metalLB load balancer + +Often Kubernetes cluster is located in DMZ, is behind corporate firewall and access is controlled via proxy configuration. +In this case we can't use host's network stack (**hostNetwork: true**) within a Pod and must manually proxy the access to the pod. + +A **LoadBalancer** service type automatically provisions an external load balancer (such as those provided by cloud providers like AWS, GCP, Azure) and assigns a public IP address to your service. In a non-cloud environment or in a DMZ setup, you need to manually configure the load balancer. + +If you are running your Kubernetes cluster on-premises or in an environment where an external load balancer is not automatically provided, you can use a load balancer implementation like MetalLB. + +#### Prepare +Select the node where persistent storage will be located for TON validator. +* Add a label to this particular node. By this label our pod will know where to be deployed: +``` +kubectl label nodes node_type=ton-validator +``` +* Replace **** (and ports if needed) in file [ton-metal-lb.yaml](ton-metal-lb.yaml). +* Replace **** with a real path on host for Persistent Volume. +* If you change the ports, make sure you specify appropriate env vars in Pod section. +* If you want to use dynamic storage provisioning via volumeClaimTemplates, feel free to create own StorageClass. + +* Install MetalLB +```yaml +kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.14.5/config/manifests/metallb-native.yaml +``` + +* Configure MetalLB +Create a configuration map to define the IP address range that MetalLB can use for external load balancer services. +```yaml +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: first-pool + namespace: metallb-system +spec: + addresses: + - 10.244.1.0/24 <-- your CIDR address +``` +apply configuration +```yaml +kubectl apply -f metallb-config.yaml +``` +#### Install + +```yaml +kubectl apply -f ton-metal-lb.yaml +``` +We do not use Pod Node Affinity here, since the Pod will remember the host with local storage it was bound to. + +#### Verify installation +Assume your network CIDR (**--pod-network-cidr**) within cluster is 10.244.1.0/24, then you can compare the output with the one below: +```yaml +kubectl get service + +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +kubernetes ClusterIP 443/TCP 28h +validator-engine-srv LoadBalancer 10.244.1.1 30001:30001/UDP,30002:30002/TCP,30003:30003/TCP 60m +``` +you can see that endpoints are pointing to metal-LB subnet: +``` +kubectl get endpoints + +NAME ENDPOINTS +kubernetes :6443 +validator-engine-srv 10.244.1.10:30002,10.244.1.10:30001,10.244.1.10:30003 +``` +and metal-LB itself operates with the right endpoint: +``` +kubectl describe service metallb-webhook-service -n metallb-system + +Name: metallb-webhook-service +Namespace: metallb-system +Selector: component=controller +Type: ClusterIP +IP: +IPs: +Port: 443/TCP +TargetPort: 9443/TCP +Endpoints: 10.244.2.3:9443 <-- CIDR +``` + +Use the commands from the previous chapter to see if node operates properly. + +### Deploy on AWS cloud (Amazon Web Services) + +#### Prepare +* AWS EKS is configured with worker nodes with selected add-ons: + * CoreDNS - Enable service discovery within your cluster. + * kube-proxy - Enable service networking within your cluster. + * Amazon VPC CNI - Enable pod networking within your cluster. +* Allocate Elastic IP. +* Replace **** with the newly created Elastic IP in [ton-aws.yaml](ton-aws.yaml) +* Replace **** with Elastic IP allocation ID (see in AWS console). +* Adjust StorageClass name. Make sure you are providing fast storage. + +#### Install + +```kubectl apply -f ton-aws.yaml``` + +#### Verify installation +Use instructions from the previous sections. + +### Deploy on GCP (Google Cloud Platform) + +#### Prepare +* Kubernetes cluster of type Standard (not Autopilot). +* Premium static IP address. +* Adjust firewall rules and security groups to allow ports 30001/udp, 30002/tcp and 30003/tcp (default ones). +* Replace **** (and ports if needed) in file [ton-gcp.yaml](ton-gcp.yaml). +* Adjust StorageClass name. Make sure you are providing fast storage. + +* Load Balancer will be created automatically according to Kubernetes service in yaml file. + +#### Install +```kubectl apply -f ton-gcp.yaml``` + +#### Verify installation +Use instructions from the previous sections. + +### Deploy on Ali Cloud + +#### Prepare +* AliCloud kubernetes cluster. +* Elastic IP. +* Replace **** with Elastic IP allocation ID (see in AliCloud console). +* Replace **** (and ports if needed) in file [ton-ali.yaml](ton-ali.yaml) with the elastic IP attached to your CLB. +* Adjust StorageClass name. Make sure you are providing fast storage. + +#### Install +```kubectl apply -f ton-ali.yaml``` + +As a result CLB (classic internal Load Balancer) will be created automatically with assigned external IP. + +#### Verify installation +Use instructions from the previous sections. + +## Troubleshooting +## Docker +### TON node cannot synchronize, constantly see messages [Error : 651 : no nodes] in the log + +Start the new container without starting validator-engine: + +``` +docker run -it -v /data/db:/var/ton-work/db \ +-e "HOST_IP=" \ +-e "PUBLIC_IP=" \ +-e "LITESERVER=true" \ +-p 43677:43677/udp \ +-p 43678:43678/tcp \ +-p 43679:43679/tcp \ +--entrypoint /bin/bash \ +ghcr.io/ton-blockchain/ton +``` +identify your PUBLIC_IP: +``` +curl -4 ifconfig.me +``` +compare if resulted IP coincides with your . +If it doesn't, exit container and launch it with the correct public IP. +Then open UDP port (inside the container) you plan to allocate for TON node using netcat utility: +``` +nc -ul 30001 +``` +and from any **other** linux machine check if you can reach this UDP port by sending a test message to that port: +``` +echo "test" | nc -u 30001 +``` +as a result inside the container you have to receive the "test" message. + +If you don't get the message inside the docker container, that means that either your firewall, LoadBalancer, NAT or proxy is blocking it. +Ask your system administrator for assistance. + +In the same way you can check if TCP port is available: + +Execute inside the container ```nc -l 30003``` and test connection from another server +```nc -vz 30003``` + +### Can't connect to lite-server +* check if lite-server was enabled on start by passing **"LITESERVER=true"** argument; +* check if TCP port (LITE_PORT) is available from the outside. From any other linux machine execute: + ``` +nc -vz +``` +### How to see what traffic is generated inside the TON docker container? +There is available a traffic monitoring utility inside the container, just execute: +``` +iptraf-ng +``` +Other tools like **tcpdump**, **nc**, **wget**, **curl**, **ifconfig**, **pv**, **plzip**, **jq** and **netstat** are also available. + +### How to build TON docker image from sources? +``` +git clone --recursive https://github.com/ton-blockchain/ton.git +cd ton +docker build . +``` + +## Kubernetes +### AWS +#### After installing AWS LB, load balancer is still not available (pending): +``` +kubectl get deployment -n kube-system aws-load-balancer-controller +``` +Solution: + +Try to install AWS LoadBalancer using ```Helm``` way. + +--- + +#### After installing AWS LB and running ton node, service shows error: + +```k describe service validator-engine-srv``` + +```log +Failed build model due to unable to resolve at least one subnet (0 match VPC and tags: [kubernetes.io/role/elb]) +``` +Solution: + +You haven't labeled the AWS subnets with the correct resource tags. + +* Public Subnets should be resource tagged with: kubernetes.io/role/elb: 1 +* Private Subnets should be tagged with: kubernetes.io/role/internal-elb: 1 +* Both private and public subnets should be tagged with: kubernetes.io/cluster/${your-cluster-name}: owned +* or if the subnets are also used by non-EKS resources kubernetes.io/cluster/${your-cluster-name}: shared + +So create tags for at least one subnet: +``` +kubernetes.io/role/elb: 1 +kubernetes.io/cluster/: owner +``` +--- +#### AWS Load Balancer works, but I still see ```[no nodes]``` in validator's log +It is required to add the security group for the EC2 instances to the load balancer along with the default security group. +It's a misleading that the default security group has "everything open." + +Add security group (default name is usually something like 'launch-wizard-1'). +And make sure you allow the ports you specified or default ports 30001/udp, 30002/tcp and 30003/tcp. + +You can also set inbound and outbound rules of new security group to allow ALL ports and for ALL protocols and for source CIDR 0.0.0.0/0 for testing purposes. + +--- + +#### Pending PersistentVolumeClaim ```Waiting for a volume to be created either by the external provisioner 'ebs.csi.aws.com' or manually by the system administrator.``` + +Solution: + +Configure Amazon EBS CSI driver for working PersistentVolumes in EKS. + +1. Enable IAM OIDC provider +``` +eksctl utils associate-iam-oidc-provider --region=us-west-2 --cluster=k8s-my --approve +``` + +2. Create Amazon EBS CSI driver IAM role +``` +eksctl create iamserviceaccount \ +--region us-west-2 \ +--name ebs-csi-controller-sa \ +--namespace kube-system \ +--cluster k8s-my \ +--attach-policy-arn arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy \ +--approve \ +--role-only \ +--role-name AmazonEKS_EBS_CSI_DriverRole +``` + +3. Add the Amazon EBS CSI add-on +```yaml +eksctl create addon --name aws-ebs-csi-driver --cluster k8s-my --service-account-role-arn arn:aws:iam::$(aws sts get-caller-identity --query Account --output text):role/AmazonEKS_EBS_CSI_DriverRole --force +``` +### Google Cloud +#### Load Balancer cannot obtain external IP (pending) + +``` +kubectl describe service validator-engine-srv + +Events: +Type Reason Age From Message + ---- ------ ---- ---- ------- +Warning LoadBalancerMixedProtocolNotSupported 7m8s g-cloudprovider LoadBalancers with multiple protocols are not supported. +Normal EnsuringLoadBalancer 113s (x7 over 7m8s) service-controller Ensuring load balancer +Warning SyncLoadBalancerFailed 113s (x7 over 7m8s) service-controller Error syncing load balancer: failed to ensure load balancer: mixed protocol is not supported for LoadBalancer +``` +Solution: + +Create static IP address of type Premium in GCP console and use it as a value for field ```loadBalancerIP``` in Kubernetes service. + +### Ali Cloud + +#### Validator logs always show +``` +Client got error [PosixError : Connection reset by peer : 104 : Error on [fd:45]] +[!NetworkManager][&ADNL_WARNING] [networkmanager]: received too small proxy packet of size 21 +``` +Solution: + +The node is sychnronizing, but very slow though. +Try to use Network Load Balancer (NLB) instead of default CLB. -If you don't need Liteserver, then remove -e "LITESERVER=true". - -#### Use -```docker exec -ti /bin/bash``` - -```./validator-engine-console -k client -p server.pub -a :``` - -IP:PORT is shown at start of container. - -#### Lite-client -To use lite-client you need to get liteserver.pub from container. - -```docker cp :/var/ton-work/db/liteserver.pub /your/path``` - -Then you can connect to it, but be sure you use right port, it's different from fullnode console port. - -```lite-client -a : -p liteserver.pub``` diff --git a/docker/control.template b/docker/control.template index 857bcebc..13b9b6b7 100644 --- a/docker/control.template +++ b/docker/control.template @@ -6,4 +6,4 @@ "permissions" : 15 } ] - } \ No newline at end of file + } diff --git a/docker/init.sh b/docker/init.sh index 1fe4f5e1..2d64690c 100644 --- a/docker/init.sh +++ b/docker/init.sh @@ -1,30 +1,93 @@ #!/usr/bin/env bash -# global config -if [ ! -z "$GCONFURL" ]; then +if [ ! -z "$TEST" ]; then + echo -e "Running simple validator-engine test..." + validator-engine -h + test $? -eq 2 || { echo "simple validator-engine test failed"; exit 1; } + exit 0; +fi + +# global config +if [ ! -z "$GLOBAL_CONFIG_URL" ]; then echo -e "\e[1;32m[+]\e[0m Downloading provided global config." - wget -q $GCONFURL -O /var/ton-work/db/ton-global.config + wget -q $GLOBAL_CONFIG_URL -O /var/ton-work/db/ton-global.config else - echo -e "\e[1;33m[=]\e[0m No global config provided, downloading default." + echo -e "\e[1;33m[=]\e[0m No global config provided, downloading mainnet default." wget -q https://api.tontech.io/ton/wallet-mainnet.autoconf.json -O /var/ton-work/db/ton-global.config fi +if [ -z "$VALIDATOR_PORT" ]; then + VALIDATOR_PORT=30001 + echo -e "\e[1;33m[=]\e[0m Using default VALIDATOR_PORT $VALIDATOR_PORT udp" +else + echo -e "\e[1;33m[=]\e[0m Using VALIDATOR_PORT $VALIDATOR_PORT udp" +fi + # Init local config with IP:PORT if [ ! -z "$PUBLIC_IP" ]; then - if [ -z "$CONSOLE_PORT" ]; then - CONSOLE_PORT="43678" - fi - echo -e "\e[1;32m[+]\e[0m Using provided IP: $PUBLIC_IP:$CONSOLE_PORT" - validator-engine -C /var/ton-work/db/ton-global.config --db /var/ton-work/db --ip "$PUBLIC_IP:$CONSOLE_PORT" + echo -e "\e[1;32m[+]\e[0m Using provided IP: $PUBLIC_IP:$VALIDATOR_PORT" else - echo -e "\e[1;31m[!]\e[0m No IP:PORT provided, exiting" + echo -e "\e[1;31m[!]\e[0m No PUBLIC_IP provided, exiting..." exit 1 fi +if [ ! -f "/var/ton-work/db/config.json" ]; then + echo -e "\e[1;32m[+]\e[0m Initializing validator-engine:" + echo validator-engine -C /var/ton-work/db/ton-global.config --db /var/ton-work/db --ip "$PUBLIC_IP:$VALIDATOR_PORT" + validator-engine -C /var/ton-work/db/ton-global.config --db /var/ton-work/db --ip "$PUBLIC_IP:$VALIDATOR_PORT" + test $? -eq 0 || { echo "Cannot initialize validator-engine"; exit 2; } +fi + +if [ ! -z "$DUMP_URL" ]; then + echo -e "\e[1;32m[+]\e[0m Using provided dump $DUMP_URL" + if [ ! -f "dump_downloaded" ]; then + echo -e "\e[1;32m[+]\e[0m Downloading dump..." + curl --retry 10 --retry-delay 30 -Ls $DUMP_URL | pv | plzip -d -n8 | tar -xC /var/ton-work/db + touch dump_downloaded + else + echo -e "\e[1;32m[+]\e[0m Dump has been already used." + fi +fi + +if [ -z "$STATE_TTL" ]; then + STATE_TTL=86400 + echo -e "\e[1;33m[=]\e[0m Using default STATE_TTL $STATE_TTL" +else + echo -e "\e[1;33m[=]\e[0m Using STATE_TTL $STATE_TTL" +fi + +if [ -z "$ARCHIVE_TTL" ]; then + ARCHIVE_TTL=86400 + echo -e "\e[1;33m[=]\e[0m Using default ARCHIVE_TTL $ARCHIVE_TTL" +else + echo -e "\e[1;33m[=]\e[0m Using ARCHIVE_TTL $ARCHIVE_TTL" +fi + +if [ -z "$THREADS" ]; then + THREADS=8 + echo -e "\e[1;33m[=]\e[0m Using default THREADS $THREADS" +else + echo -e "\e[1;33m[=]\e[0m Using THREADS $THREADS" +fi + +if [ -z "$VERBOSITY" ]; then + VERBOSITY=3 + echo -e "\e[1;33m[=]\e[0m Using default VERBOSITY $VERBOSITY" +else + echo -e "\e[1;33m[=]\e[0m Using VERBOSITY $VERBOSITY" +fi + +if [ -z "$CONSOLE_PORT" ]; then + CONSOLE_PORT=30002 + echo -e "\e[1;33m[=]\e[0m Using default CONSOLE_PORT $CONSOLE_PORT tcp" +else + echo -e "\e[1;33m[=]\e[0m Using CONSOLE_PORT $CONSOLE_PORT tcp" +fi + # Generating server certificate if [ -f "./server" ]; then echo -e "\e[1;33m[=]\e[0m Found existing server certificate, skipping" -else +else echo -e "\e[1;32m[+]\e[0m Generating and installing server certificate for remote control" read -r SERVER_ID1 SERVER_ID2 <<< $(generate-random-id -m keys -n server) echo "Server IDs: $SERVER_ID1 $SERVER_ID2" @@ -32,16 +95,16 @@ else fi # Generating client certificate -if [ -f "./client" ]; then +if [ -f "./client" ]; then echo -e "\e[1;33m[=]\e[0m Found existing client certificate, skipping" else read -r CLIENT_ID1 CLIENT_ID2 <<< $(generate-random-id -m keys -n client) echo -e "\e[1;32m[+]\e[0m Generated client private certificate $CLIENT_ID1 $CLIENT_ID2" echo -e "\e[1;32m[+]\e[0m Generated client public certificate" # Adding client permissions - sed -e "s/CONSOLE-PORT/\"$(printf "%q" $CONSOLE_PORT)\"/g" -e "s~SERVER-ID~\"$(printf "%q" $SERVER_ID2)\"~g" -e "s~CLIENT-ID~\"$(printf "%q" $CLIENT_ID2)\"~g" control.template > control.new - sed -e "s~\"control\"\ \:\ \[~$(printf "%q" $(cat control.new))~g" config.json > config.json.new - mv config.json.new config.json + sed -e "s/CONSOLE-PORT/\"$(printf "%q" $CONSOLE_PORT)\"/g" -e "s~SERVER-ID~\"$(printf "%q" $SERVER_ID2)\"~g" -e "s~CLIENT-ID~\"$(printf "%q" $CLIENT_ID2)\"~g" /var/ton-work/scripts/control.template > control.new + sed -e "s~\"control\"\ \:\ \[~$(printf "%q" $(cat control.new))~g" /var/ton-work/db/config.json > config.json.new + mv config.json.new /var/ton-work/db/config.json fi # Liteserver @@ -50,20 +113,25 @@ if [ -z "$LITESERVER" ]; then else if [ -f "./liteserver" ]; then echo -e "\e[1;33m[=]\e[0m Found existing liteserver certificate, skipping" - else + else echo -e "\e[1;32m[+]\e[0m Generating and installing liteserver certificate for remote control" read -r LITESERVER_ID1 LITESERVER_ID2 <<< $(generate-random-id -m keys -n liteserver) echo "Liteserver IDs: $LITESERVER_ID1 $LITESERVER_ID2" cp liteserver /var/ton-work/db/keyring/$LITESERVER_ID1 + if [ -z "$LITE_PORT" ]; then - LITE_PORT="43679" + LITE_PORT=30003 + echo -e "\e[1;33m[=]\e[0m Using default LITE_PORT $LITE_PORT tcp" + else + echo -e "\e[1;33m[=]\e[0m Using LITE_PORT $LITE_PORT tcp" fi + LITESERVERS=$(printf "%q" "\"liteservers\":[{\"id\":\"$LITESERVER_ID2\",\"port\":\"$LITE_PORT\"}") - sed -e "s~\"liteservers\"\ \:\ \[~$LITESERVERS~g" config.json > config.json.liteservers - mv config.json.liteservers config.json + sed -e "s~\"liteservers\"\ \:\ \[~$LITESERVERS~g" /var/ton-work/db/config.json > config.json.liteservers + mv config.json.liteservers /var/ton-work/db/config.json fi fi -echo -e "\e[1;32m[+]\e[0m Running validator-engine" - -exec validator-engine -c /var/ton-work/db/config.json -C /var/ton-work/db/ton-global.config --db /var/ton-work/db \ No newline at end of file +echo -e "\e[1;32m[+]\e[0m Starting validator-engine:" +echo validator-engine -c /var/ton-work/db/config.json -C /var/ton-work/db/ton-global.config --db /var/ton-work/db --state-ttl $STATE_TTL --archive-ttl $ARCHIVE_TTL --threads $THREADS --verbosity $VERBOSITY $CUSTOM_ARG +exec validator-engine -c /var/ton-work/db/config.json -C /var/ton-work/db/ton-global.config --db /var/ton-work/db --state-ttl $STATE_TTL --archive-ttl $ARCHIVE_TTL --threads $THREADS --verbosity $VERBOSITY $CUSTOM_ARG diff --git a/docker/ton-ali.yaml b/docker/ton-ali.yaml new file mode 100644 index 00000000..2dd5daf6 --- /dev/null +++ b/docker/ton-ali.yaml @@ -0,0 +1,121 @@ +apiVersion: "apps/v1" +kind: StatefulSet +metadata: + name: validator-engine-pod + labels: + name: validator-engine-pod +spec: + volumeClaimTemplates: + - metadata: + name: validator-engine-pvc + spec: + storageClassName: alicloud-disk-ssd + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 800Gi + serviceName: validator-engine-srv-headless + replicas: 1 + selector: + matchLabels: + name: validator-engine-pod + template: + metadata: + labels: + name: validator-engine-pod + spec: + containers: + - name: validator-engine-container + image: ghcr.io/ton-blockchain/ton:latest + env: + - name: PUBLIC_IP + value: "" + - name: GLOBAL_CONFIG_URL + value: "https://api.tontech.io/ton/wallet-mainnet.autoconf.json" + - name: DUMP_URL + value: "https://dump.ton.org/dumps/latest.tar.lz" + - name: LITESERVER + value: "true" + - name: VALIDATOR_PORT + value: "30001" + - name: CONSOLE_PORT + value: "30002" + - name: LITE_PORT + value: "30003" + - name: STATE_TTL + value: "86400" + - name: ARCHIVE_TTL + value: "86400" + - name: THREADS + value: "8" + - name: VERBOSITY + value: "3" + ports: + - containerPort: 30001 + protocol: UDP + - containerPort: 30002 + protocol: TCP + - containerPort: 30003 + protocol: TCP + volumeMounts: + - mountPath: "/var/ton-work/db" + name: validator-engine-pvc + resources: + requests: + memory: "64Gi" + cpu: "16" + limits: + memory: "128Gi" + cpu: "32" +--- +kind: Service +apiVersion: v1 +metadata: + name: validator-engine-srv + annotations: + service.beta.kubernetes.io/alibaba-cloud-loadbalancer-eip-ids: "" + service.beta.kubernetes.io/alibaba-cloud-loadbalancer-address-type: "intranet" +spec: + type: LoadBalancer + externalTrafficPolicy: Local + ports: + - name: validator-udp + nodePort: 30001 + port: 30001 + targetPort: 30001 + protocol: UDP + - name: console-tcp + nodePort: 30002 + port: 30002 + targetPort: 30002 + protocol: TCP + - name: ls-tcp + nodePort: 30003 + port: 30003 + targetPort: 30003 + protocol: TCP + selector: + name: validator-engine-pod +--- +apiVersion: v1 +kind: Service +metadata: + name: validator-engine-srv-headless +spec: + clusterIP: None + ports: + - name: validator-udp + port: 30001 + targetPort: 30001 + protocol: UDP + - name: console-tcp + port: 30002 + targetPort: 30002 + protocol: TCP + - name: ls-tcp + port: 30003 + targetPort: 30003 + protocol: TCP + selector: + name: validator-engine-pod diff --git a/docker/ton-aws.yaml b/docker/ton-aws.yaml new file mode 100644 index 00000000..4cab1b55 --- /dev/null +++ b/docker/ton-aws.yaml @@ -0,0 +1,122 @@ +apiVersion: "apps/v1" +kind: StatefulSet +metadata: + name: validator-engine-pod + labels: + name: validator-engine-pod +spec: + volumeClaimTemplates: + - metadata: + name: validator-engine-pvc + spec: + storageClassName: gp2 + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 800Gi + serviceName: validator-engine-srv-headless + replicas: 1 + selector: + matchLabels: + name: validator-engine-pod + template: + metadata: + labels: + name: validator-engine-pod + spec: + containers: + - name: validator-engine-container + image: ghcr.io/ton-blockchain/ton:latest + env: + - name: PUBLIC_IP + value: "" + - name: GLOBAL_CONFIG_URL + value: "https://api.tontech.io/ton/wallet-mainnet.autoconf.json" + - name: DUMP_URL + value: "https://dump.ton.org/dumps/latest.tar.lz" + - name: LITESERVER + value: "true" + - name: VALIDATOR_PORT + value: "30001" + - name: CONSOLE_PORT + value: "30002" + - name: LITE_PORT + value: "30003" + - name: STATE_TTL + value: "86400" + - name: ARCHIVE_TTL + value: "86400" + - name: THREADS + value: "8" + - name: VERBOSITY + value: "3" + ports: + - containerPort: 30001 + protocol: UDP + - containerPort: 30002 + protocol: TCP + - containerPort: 30003 + protocol: TCP + volumeMounts: + - mountPath: "/var/ton-work/db" + name: validator-engine-pvc + resources: + requests: + memory: "64Gi" + cpu: "16" + limits: + memory: "128Gi" + cpu: "32" +--- +kind: Service +apiVersion: v1 +metadata: + name: validator-engine-srv + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: external + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing + service.beta.kubernetes.io/aws-load-balancer-eip-allocations: "" # Replace with your EIP allocation ID +spec: + type: LoadBalancer + ports: + - name: validator-udp + nodePort: 30001 + port: 30001 + targetPort: 30001 + protocol: UDP + - name: console-tcp + nodePort: 30002 + port: 30002 + targetPort: 30002 + protocol: TCP + - name: ls-tcp + nodePort: 30003 + port: 30003 + targetPort: 30003 + protocol: TCP + selector: + name: validator-engine-pod +--- +apiVersion: v1 +kind: Service +metadata: + name: validator-engine-srv-headless +spec: + clusterIP: None + ports: + - name: validator-udp + port: 30001 + targetPort: 30001 + protocol: UDP + - name: console-tcp + port: 30002 + targetPort: 30002 + protocol: TCP + - name: ls-tcp + port: 30003 + targetPort: 30003 + protocol: TCP + selector: + name: validator-engine-pod diff --git a/docker/ton-gcp.yaml b/docker/ton-gcp.yaml new file mode 100644 index 00000000..e3d89a06 --- /dev/null +++ b/docker/ton-gcp.yaml @@ -0,0 +1,134 @@ +apiVersion: "apps/v1" +kind: StatefulSet +metadata: + name: validator-engine-pod + labels: + name: validator-engine-pod +spec: + volumeClaimTemplates: + - metadata: + name: validator-engine-pvc + spec: + storageClassName: standard-rwo + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 800Gi + serviceName: validator-engine-srv-headless + replicas: 1 + selector: + matchLabels: + name: validator-engine-pod + template: + metadata: + labels: + name: validator-engine-pod + spec: + containers: + - name: validator-engine-container + image: ghcr.io/ton-blockchain/ton:latest + env: + - name: PUBLIC_IP + value: "" + - name: GLOBAL_CONFIG_URL + value: "https://api.tontech.io/ton/wallet-mainnet.autoconf.json" + - name: DUMP_URL + value: "https://dump.ton.org/dumps/latest.tar.lz" + - name: LITESERVER + value: "true" + - name: VALIDATOR_PORT + value: "30001" + - name: CONSOLE_PORT + value: "30002" + - name: LITE_PORT + value: "30003" + - name: STATE_TTL + value: "86400" + - name: ARCHIVE_TTL + value: "86400" + - name: THREADS + value: "8" + - name: VERBOSITY + value: "3" + ports: + - containerPort: 30001 + protocol: UDP + - containerPort: 30002 + protocol: TCP + - containerPort: 30003 + protocol: TCP + volumeMounts: + - mountPath: "/var/ton-work/db" + name: validator-engine-pvc + resources: + requests: + memory: "64Gi" + cpu: "16" + limits: + memory: "128Gi" + cpu: "32" +--- +kind: Service +apiVersion: v1 +metadata: + name: validator-engine-srv +spec: + type: LoadBalancer + loadBalancerIP: + ports: + - port: 30001 + targetPort: 30001 + protocol: UDP + selector: + name: validator-engine-pod +--- +kind: Service +apiVersion: v1 +metadata: + name: validator-engine-console-srv +spec: + type: LoadBalancer + loadBalancerIP: + ports: + - port: 30002 + targetPort: 30002 + protocol: TCP + selector: + name: validator-engine-pod +--- +kind: Service +apiVersion: v1 +metadata: + name: lite-server-srv +spec: + type: LoadBalancer + loadBalancerIP: + ports: + - port: 30003 + targetPort: 30003 + protocol: TCP + selector: + name: validator-engine-pod +--- +apiVersion: v1 +kind: Service +metadata: + name: validator-engine-srv-headless +spec: + clusterIP: None + ports: + - name: validator-udp + port: 30001 + targetPort: 30001 + protocol: UDP + - name: console-tcp + port: 30002 + targetPort: 30002 + protocol: TCP + - name: ls-tcp + port: 30003 + targetPort: 30003 + protocol: TCP + selector: + name: validator-engine-pod diff --git a/docker/ton-metal-lb.yaml b/docker/ton-metal-lb.yaml new file mode 100644 index 00000000..1e62a45a --- /dev/null +++ b/docker/ton-metal-lb.yaml @@ -0,0 +1,118 @@ +apiVersion: v1 +kind: Pod +metadata: + name: validator-engine-pod + labels: + name: validator-engine-pod +spec: + volumes: + - name: validator-engine-pv + persistentVolumeClaim: + claimName: validator-engine-pvc + containers: + - name: validator-engine-container + image: ghcr.io/ton-blockchain/ton:latest + env: + - name: PUBLIC_IP + value: "" + - name: GLOBAL_CONFIG_URL + value: "https://api.tontech.io/ton/wallet-mainnet.autoconf.json" + - name: DUMP_URL + value: "https://dump.ton.org/dumps/latest.tar.lz" + - name: LITESERVER + value: "true" + - name: VALIDATOR_PORT + value: "30001" + - name: CONSOLE_PORT + value: "30002" + - name: LITE_PORT + value: "30003" + - name: STATE_TTL + value: "86400" + - name: ARCHIVE_TTL + value: "86400" + - name: THREADS + value: "8" + - name: VERBOSITY + value: "3" + volumeMounts: + - mountPath: "/var/ton-work/db" + name: validator-engine-pv + resources: + requests: + memory: "64Gi" + cpu: "16" + limits: + memory: "128Gi" + cpu: "32" +--- +kind: Service +apiVersion: v1 +metadata: + name: validator-engine-srv + annotations: + metallb.universe.tf/address-pool: first-pool +spec: + type: LoadBalancer + ports: + - name: validator-engine-public-udp-port + nodePort: 30001 + port: 30001 + targetPort: 30001 + protocol: UDP + - name: validator-console-tcp-port + nodePort: 30002 + port: 30002 + targetPort: 30002 + protocol: TCP + - name: lite-server-tcp-port + nodePort: 30003 + port: 30003 + targetPort: 30003 + protocol: TCP + selector: + name: validator-engine-pod +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: local-storage +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: validator-engine-pv + labels: + type: local +spec: + storageClassName: local-storage + capacity: + storage: 800Gi + accessModes: + - ReadWriteOnce + - ReadOnlyMany + persistentVolumeReclaimPolicy: Retain + local: + path: + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: node_type + operator: In + values: + - ton-validator +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: validator-engine-pvc +spec: + storageClassName: local-storage + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 800Gi diff --git a/docker/ton-node-port.yaml b/docker/ton-node-port.yaml new file mode 100644 index 00000000..a9f1fe5c --- /dev/null +++ b/docker/ton-node-port.yaml @@ -0,0 +1,126 @@ +apiVersion: v1 +kind: Pod +metadata: + name: validator-engine-pod + labels: + name: validator-engine-pod +spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node_type + operator: In + values: + - ton-validator + hostNetwork: true + volumes: + - name: validator-engine-pv + persistentVolumeClaim: + claimName: validator-engine-pvc + containers: + - name: validator-engine-container + image: ghcr.io/ton-blockchain/ton:latest + env: + - name: PUBLIC_IP + value: "" + - name: GLOBAL_CONFIG_URL + value: "https://api.tontech.io/ton/wallet-mainnet.autoconf.json" + - name: DUMP_URL + value: "https://dump.ton.org/dumps/latest.tar.lz" + - name: LITESERVER + value: "true" + - name: VALIDATOR_PORT + value: "30001" + - name: CONSOLE_PORT + value: "30002" + - name: LITE_PORT + value: "30003" + - name: STATE_TTL + value: "86400" + - name: ARCHIVE_TTL + value: "86400" + - name: THREADS + value: "8" + - name: VERBOSITY + value: "3" + volumeMounts: + - mountPath: "/var/ton-work/db" + name: validator-engine-pv + resources: + requests: + memory: "64Gi" + cpu: "16" + limits: + memory: "128Gi" + cpu: "32" +--- +kind: Service +apiVersion: v1 +metadata: + name: validator-engine-srv +spec: + type: NodePort + ports: + - name: validator-engine-public-udp-port + nodePort: 30001 + port: 30001 + targetPort: 30001 + protocol: UDP + - name: validator-console-tcp-port + nodePort: 30002 + port: 30002 + targetPort: 30002 + protocol: TCP + - name: lite-server-tcp-port + nodePort: 30003 + port: 30003 + targetPort: 30003 + protocol: TCP + selector: + name: validator-engine-pod +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: local-storage +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: validator-engine-pv + labels: + type: local +spec: + storageClassName: local-storage + capacity: + storage: 800Gi + accessModes: + - ReadWriteOnce + - ReadOnlyMany + persistentVolumeReclaimPolicy: Retain + local: + path: + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: node_type + operator: In + values: + - ton-validator +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: validator-engine-pvc +spec: + storageClassName: local-storage + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 800Gi diff --git a/emulator/CMakeLists.txt b/emulator/CMakeLists.txt new file mode 100644 index 00000000..663c8fd2 --- /dev/null +++ b/emulator/CMakeLists.txt @@ -0,0 +1,67 @@ +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) + +option(EMULATOR_STATIC "Build emulator as static library" OFF) + +set(EMULATOR_STATIC_SOURCE + transaction-emulator.cpp + tvm-emulator.hpp +) + +set(EMULATOR_SOURCE + emulator-extern.cpp +) + +set(EMULATOR_EMSCRIPTEN_SOURCE + emulator-emscripten.cpp +) + +include(GenerateExportHeader) + +add_library(emulator_static STATIC ${EMULATOR_STATIC_SOURCE}) +target_link_libraries(emulator_static PUBLIC ton_crypto smc-envelope) + +if (EMULATOR_STATIC OR USE_EMSCRIPTEN) + add_library(emulator STATIC ${EMULATOR_SOURCE}) +else() + add_library(emulator SHARED ${EMULATOR_SOURCE}) +endif() + +if (PORTABLE AND NOT APPLE) + target_link_libraries(emulator PUBLIC emulator_static git -static-libgcc -static-libstdc++) +else() + target_link_libraries(emulator PUBLIC emulator_static git) +endif() + +generate_export_header(emulator EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/emulator_export.h) +if (EMULATOR_STATIC OR USE_EMSCRIPTEN) + target_compile_definitions(emulator PUBLIC EMULATOR_STATIC_DEFINE) +endif() +target_include_directories(emulator PUBLIC + $ + $) +if (APPLE) + set_target_properties(emulator PROPERTIES LINK_FLAGS "-Wl,-exported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/emulator_export_list") +endif() + +if (USE_EMSCRIPTEN) + add_executable(emulator-emscripten ${EMULATOR_EMSCRIPTEN_SOURCE}) + target_link_libraries(emulator-emscripten PUBLIC emulator) + target_link_options(emulator-emscripten PRIVATE -sEXPORTED_RUNTIME_METHODS=UTF8ToString,stringToUTF8,allocate,ALLOC_NORMAL,lengthBytesUTF8) + target_link_options(emulator-emscripten PRIVATE -sEXPORTED_FUNCTIONS=_emulate,_free,_malloc,_run_get_method,_create_emulator,_destroy_emulator,_emulate_with_emulator,_version) + target_link_options(emulator-emscripten PRIVATE -sEXPORT_NAME=EmulatorModule) + target_link_options(emulator-emscripten PRIVATE -sERROR_ON_UNDEFINED_SYMBOLS=0) + target_link_options(emulator-emscripten PRIVATE -Oz) + target_link_options(emulator-emscripten PRIVATE -sIGNORE_MISSING_MAIN=1) + target_link_options(emulator-emscripten PRIVATE -sAUTO_NATIVE_LIBRARIES=0) + target_link_options(emulator-emscripten PRIVATE -sMODULARIZE=1) + target_link_options(emulator-emscripten PRIVATE -sENVIRONMENT=web) + target_link_options(emulator-emscripten PRIVATE -sFILESYSTEM=0) + target_link_options(emulator-emscripten PRIVATE -sALLOW_MEMORY_GROWTH=1) + target_link_options(emulator-emscripten PRIVATE -fexceptions) + if (USE_EMSCRIPTEN_NO_WASM) + target_link_options(emulator-emscripten PRIVATE -sWASM=0) + endif() + target_compile_options(emulator-emscripten PRIVATE -fexceptions) +endif() + +install(TARGETS emulator ARCHIVE DESTINATION lib LIBRARY DESTINATION lib) diff --git a/emulator/README.md b/emulator/README.md new file mode 100644 index 00000000..d4e2244e --- /dev/null +++ b/emulator/README.md @@ -0,0 +1,32 @@ +# Emulator + +Emulator is a shared library containing the following functionality: +- Emulating blockchain transactions +- Emulating TVM - get methods and sending external and internal messages. + +## Transaction Emulator + +To emulate transaction you need the following data: + +- Account state of type *ShardAccount*. +- Global config of type *(Hashmap 32 ^Cell)*. +- Inbound message of type *MessageAny*. + +Optionally you can set emulation parameters: +- *ignore_chksig* - whether CHKSIG instructions are set to always succeed. Default: *false* +- *lt* - logical time of emulation. Default: next block's lt after the account's last transaction block. +- *unixtime* - unix time of emulation. Default: current system time +- *rand_seed* - random seed. Default: generated randomly +- *libs* - shared libraries. If your smart contract uses shared libraries (located in masterchain), you should set this parameter. + +Emulator output contains: +- Transaction object (*Transaction*) +- New account state (*ShardAccount*) +- Actions cell (*OutList n*) +- TVM log + +## TVM Emulator + +TVM emulator is intended to run get methods or emulate sending message on TVM level. It is initialized with smart contract code and data cells. +- To run get method you pass *initial stack* and *method id* (as integer). +- To emulate sending message you pass *message body* and in case of internal message *amount* in nanograms. diff --git a/emulator/StringLog.h b/emulator/StringLog.h new file mode 100644 index 00000000..724182c4 --- /dev/null +++ b/emulator/StringLog.h @@ -0,0 +1,27 @@ +#ifndef TON_STRINGLOG_H +#define TON_STRINGLOG_H + +#include "td/utils/logging.h" +#include + +class StringLog : public td::LogInterface { + public: + StringLog() { + } + + void append(td::CSlice new_slice, int log_level) override { + str.append(new_slice.str()); + } + + void rotate() override { + } + + std::string get_string() const { + return str; + } + + private: + std::string str; +}; + +#endif //TON_STRINGLOG_H \ No newline at end of file diff --git a/emulator/emulator-emscripten.cpp b/emulator/emulator-emscripten.cpp new file mode 100644 index 00000000..efb14eff --- /dev/null +++ b/emulator/emulator-emscripten.cpp @@ -0,0 +1,288 @@ +#include "emulator-extern.h" +#include "td/utils/logging.h" +#include "td/utils/JsonBuilder.h" +#include "td/utils/misc.h" +#include "td/utils/optional.h" +#include "StringLog.h" +#include +#include "crypto/common/bitstring.h" + +struct TransactionEmulationParams { + uint32_t utime; + uint64_t lt; + td::optional rand_seed_hex; + bool ignore_chksig; + bool is_tick_tock; + bool is_tock; + bool debug_enabled; +}; + +td::Result decode_transaction_emulation_params(const char* json) { + TransactionEmulationParams params; + + std::string json_str(json); + TRY_RESULT(input_json, td::json_decode(td::MutableSlice(json_str))); + auto &obj = input_json.get_object(); + + TRY_RESULT(utime_field, td::get_json_object_field(obj, "utime", td::JsonValue::Type::Number, false)); + TRY_RESULT(utime, td::to_integer_safe(utime_field.get_number())); + params.utime = utime; + + TRY_RESULT(lt_field, td::get_json_object_field(obj, "lt", td::JsonValue::Type::String, false)); + TRY_RESULT(lt, td::to_integer_safe(lt_field.get_string())); + params.lt = lt; + + TRY_RESULT(rand_seed_str, td::get_json_object_string_field(obj, "rand_seed", true)); + if (rand_seed_str.size() > 0) { + params.rand_seed_hex = rand_seed_str; + } + + TRY_RESULT(ignore_chksig, td::get_json_object_bool_field(obj, "ignore_chksig", false)); + params.ignore_chksig = ignore_chksig; + + TRY_RESULT(debug_enabled, td::get_json_object_bool_field(obj, "debug_enabled", false)); + params.debug_enabled = debug_enabled; + + TRY_RESULT(is_tick_tock, td::get_json_object_bool_field(obj, "is_tick_tock", true, false)); + params.is_tick_tock = is_tick_tock; + + TRY_RESULT(is_tock, td::get_json_object_bool_field(obj, "is_tock", true, false)); + params.is_tock = is_tock; + + if (is_tock && !is_tick_tock) { + return td::Status::Error("Inconsistent parameters is_tick_tock=false, is_tock=true"); + } + + return params; +} + +struct GetMethodParams { + std::string code; + std::string data; + int verbosity; + td::optional libs; + td::optional prev_blocks_info; + std::string address; + uint32_t unixtime; + uint64_t balance; + std::string extra_currencies; + std::string rand_seed_hex; + int64_t gas_limit; + int method_id; + bool debug_enabled; +}; + +td::Result decode_get_method_params(const char* json) { + GetMethodParams params; + + std::string json_str(json); + TRY_RESULT(input_json, td::json_decode(td::MutableSlice(json_str))); + auto &obj = input_json.get_object(); + + TRY_RESULT(code, td::get_json_object_string_field(obj, "code", false)); + params.code = code; + + TRY_RESULT(data, td::get_json_object_string_field(obj, "data", false)); + params.data = data; + + TRY_RESULT(verbosity, td::get_json_object_int_field(obj, "verbosity", false)); + params.verbosity = verbosity; + + TRY_RESULT(libs, td::get_json_object_string_field(obj, "libs", true)); + if (libs.size() > 0) { + params.libs = libs; + } + + TRY_RESULT(prev_blocks_info, td::get_json_object_string_field(obj, "prev_blocks_info", true)); + if (prev_blocks_info.size() > 0) { + params.prev_blocks_info = prev_blocks_info; + } + + TRY_RESULT(address, td::get_json_object_string_field(obj, "address", false)); + params.address = address; + + TRY_RESULT(unixtime_field, td::get_json_object_field(obj, "unixtime", td::JsonValue::Type::Number, false)); + TRY_RESULT(unixtime, td::to_integer_safe(unixtime_field.get_number())); + params.unixtime = unixtime; + + TRY_RESULT(balance_field, td::get_json_object_field(obj, "balance", td::JsonValue::Type::String, false)); + TRY_RESULT(balance, td::to_integer_safe(balance_field.get_string())); + params.balance = balance; + + TRY_RESULT(ec_field, td::get_json_object_field(obj, "extra_currencies", td::JsonValue::Type::Object, true)); + if (ec_field.type() != td::JsonValue::Type::Null) { + if (ec_field.type() != td::JsonValue::Type::Object) { + return td::Status::Error("EC must be of type Object"); + } + td::StringBuilder ec_builder; + auto& ec_obj = ec_field.get_object(); + bool is_first = true; + for (auto &field_value : ec_obj) { + auto currency_id = field_value.first; + if (field_value.second.type() != td::JsonValue::Type::String) { + return td::Status::Error(PSLICE() << "EC amount must be of type String"); + } + auto amount = field_value.second.get_string(); + if (!is_first) { + ec_builder << " "; + is_first = false; + } + ec_builder << currency_id << "=" << amount; + } + if (ec_builder.is_error()) { + return td::Status::Error(PSLICE() << "Error building extra currencies string"); + } + params.extra_currencies = ec_builder.as_cslice().str(); + } + + TRY_RESULT(rand_seed_str, td::get_json_object_string_field(obj, "rand_seed", false)); + params.rand_seed_hex = rand_seed_str; + + TRY_RESULT(gas_limit_field, td::get_json_object_field(obj, "gas_limit", td::JsonValue::Type::String, false)); + TRY_RESULT(gas_limit, td::to_integer_safe(gas_limit_field.get_string())); + params.gas_limit = gas_limit; + + TRY_RESULT(method_id, td::get_json_object_int_field(obj, "method_id", false)); + params.method_id = method_id; + + TRY_RESULT(debug_enabled, td::get_json_object_bool_field(obj, "debug_enabled", false)); + params.debug_enabled = debug_enabled; + + return params; +} + +class NoopLog : public td::LogInterface { + public: + NoopLog() { + } + + void append(td::CSlice new_slice, int log_level) override { + } + + void rotate() override { + } +}; + +extern "C" { + +void* create_emulator(const char *config, int verbosity) { + NoopLog logger; + + td::log_interface = &logger; + + SET_VERBOSITY_LEVEL(verbosity_NEVER); + return transaction_emulator_create(config, verbosity); +} + +void destroy_emulator(void* em) { + NoopLog logger; + + td::log_interface = &logger; + + SET_VERBOSITY_LEVEL(verbosity_NEVER); + transaction_emulator_destroy(em); +} + +const char *emulate_with_emulator(void* em, const char* libs, const char* account, const char* message, const char* params) { + StringLog logger; + + td::log_interface = &logger; + SET_VERBOSITY_LEVEL(verbosity_DEBUG); + + auto decoded_params_res = decode_transaction_emulation_params(params); + if (decoded_params_res.is_error()) { + return strdup(R"({"fail":true,"message":"Can't decode other params"})"); + } + auto decoded_params = decoded_params_res.move_as_ok(); + + bool rand_seed_set = true; + if (decoded_params.rand_seed_hex) { + rand_seed_set = transaction_emulator_set_rand_seed(em, decoded_params.rand_seed_hex.unwrap().c_str()); + } + + if (!transaction_emulator_set_libs(em, libs) || + !transaction_emulator_set_lt(em, decoded_params.lt) || + !transaction_emulator_set_unixtime(em, decoded_params.utime) || + !transaction_emulator_set_ignore_chksig(em, decoded_params.ignore_chksig) || + !transaction_emulator_set_debug_enabled(em, decoded_params.debug_enabled) || + !rand_seed_set) { + transaction_emulator_destroy(em); + return strdup(R"({"fail":true,"message":"Can't set params"})"); + } + + const char *result; + if (decoded_params.is_tick_tock) { + result = transaction_emulator_emulate_tick_tock_transaction(em, account, decoded_params.is_tock); + } else { + result = transaction_emulator_emulate_transaction(em, account, message); + } + + const char* output = nullptr; + { + td::JsonBuilder jb; + auto json_obj = jb.enter_object(); + json_obj("output", td::JsonRaw(td::Slice(result))); + json_obj("logs", logger.get_string()); + json_obj.leave(); + output = strdup(jb.string_builder().as_cslice().c_str()); + } + free((void*) result); + + return output; +} + +const char *emulate(const char *config, const char* libs, int verbosity, const char* account, const char* message, const char* params) { + auto em = transaction_emulator_create(config, verbosity); + auto result = emulate_with_emulator(em, libs, account, message, params); + transaction_emulator_destroy(em); + return result; +} + +const char *run_get_method(const char *params, const char* stack, const char* config) { + StringLog logger; + + td::log_interface = &logger; + SET_VERBOSITY_LEVEL(verbosity_DEBUG); + + auto decoded_params_res = decode_get_method_params(params); + if (decoded_params_res.is_error()) { + return strdup(R"({"fail":true,"message":"Can't decode params"})"); + } + auto decoded_params = decoded_params_res.move_as_ok(); + + auto tvm = tvm_emulator_create(decoded_params.code.c_str(), decoded_params.data.c_str(), decoded_params.verbosity); + + if ((decoded_params.libs && !tvm_emulator_set_libraries(tvm, decoded_params.libs.value().c_str())) || + !tvm_emulator_set_c7(tvm, decoded_params.address.c_str(), decoded_params.unixtime, decoded_params.balance, + decoded_params.rand_seed_hex.c_str(), config) || + (decoded_params.extra_currencies.size() > 0 && !tvm_emulator_set_extra_currencies(tvm, decoded_params.extra_currencies.c_str())) || + (decoded_params.prev_blocks_info && !tvm_emulator_set_prev_blocks_info(tvm, decoded_params.prev_blocks_info.value().c_str())) || + (decoded_params.gas_limit > 0 && !tvm_emulator_set_gas_limit(tvm, decoded_params.gas_limit)) || + !tvm_emulator_set_debug_enabled(tvm, decoded_params.debug_enabled)) { + tvm_emulator_destroy(tvm); + return strdup(R"({"fail":true,"message":"Can't set params"})"); + } + + auto res = tvm_emulator_run_get_method(tvm, decoded_params.method_id, stack); + + tvm_emulator_destroy(tvm); + + const char* output = nullptr; + { + td::JsonBuilder jb; + auto json_obj = jb.enter_object(); + json_obj("output", td::JsonRaw(td::Slice(res))); + json_obj("logs", logger.get_string()); + json_obj.leave(); + output = strdup(jb.string_builder().as_cslice().c_str()); + } + free((void*) res); + + return output; +} + +const char *version() { + return emulator_version(); +} + +} \ No newline at end of file diff --git a/emulator/emulator-extern.cpp b/emulator/emulator-extern.cpp new file mode 100644 index 00000000..eb5ff9f9 --- /dev/null +++ b/emulator/emulator-extern.cpp @@ -0,0 +1,782 @@ +#include "emulator-extern.h" +#include "td/utils/base64.h" +#include "td/utils/Status.h" +#include "td/utils/JsonBuilder.h" +#include "td/utils/logging.h" +#include "td/utils/Variant.h" +#include "td/utils/overloaded.h" +#include "transaction-emulator.h" +#include "tvm-emulator.hpp" +#include "crypto/vm/stack.hpp" +#include "crypto/vm/memo.h" +#include "git.h" + +td::Result> boc_b64_to_cell(const char *boc) { + TRY_RESULT_PREFIX(boc_decoded, td::base64_decode(td::Slice(boc)), "Can't decode base64 boc: "); + return vm::std_boc_deserialize(boc_decoded); +} + +td::Result cell_to_boc_b64(td::Ref cell) { + TRY_RESULT_PREFIX(boc, vm::std_boc_serialize(std::move(cell), vm::BagOfCells::Mode::WithCRC32C), "Can't serialize cell: "); + return td::base64_encode(boc.as_slice()); +} + +const char *success_response(std::string&& transaction, std::string&& new_shard_account, std::string&& vm_log, + td::optional&& actions, double elapsed_time) { + td::JsonBuilder jb; + auto json_obj = jb.enter_object(); + json_obj("success", td::JsonTrue()); + json_obj("transaction", std::move(transaction)); + json_obj("shard_account", std::move(new_shard_account)); + json_obj("vm_log", std::move(vm_log)); + if (actions) { + json_obj("actions", actions.unwrap()); + } else { + json_obj("actions", td::JsonNull()); + } + json_obj("elapsed_time", elapsed_time); + json_obj.leave(); + return strdup(jb.string_builder().as_cslice().c_str()); +} + +const char *error_response(std::string&& error) { + td::JsonBuilder jb; + auto json_obj = jb.enter_object(); + json_obj("success", td::JsonFalse()); + json_obj("error", std::move(error)); + json_obj("external_not_accepted", td::JsonFalse()); + json_obj.leave(); + return strdup(jb.string_builder().as_cslice().c_str()); +} + +const char *external_not_accepted_response(std::string&& vm_log, int vm_exit_code, double elapsed_time) { + td::JsonBuilder jb; + auto json_obj = jb.enter_object(); + json_obj("success", td::JsonFalse()); + json_obj("error", "External message not accepted by smart contract"); + json_obj("external_not_accepted", td::JsonTrue()); + json_obj("vm_log", std::move(vm_log)); + json_obj("vm_exit_code", vm_exit_code); + json_obj("elapsed_time", elapsed_time); + json_obj.leave(); + return strdup(jb.string_builder().as_cslice().c_str()); +} + +#define ERROR_RESPONSE(error) return error_response(error) + +td::Result decode_config(const char* config_boc) { + TRY_RESULT_PREFIX(config_params_cell, boc_b64_to_cell(config_boc), "Can't deserialize config params boc: "); + auto config_dict = std::make_unique(config_params_cell, 32); + auto config_addr_cell = config_dict->lookup_ref(td::BitArray<32>::zero()); + if (config_addr_cell.is_null()) { + return td::Status::Error("Can't find config address (param 0) is missing in config params"); + } + auto config_addr_cs = vm::load_cell_slice(std::move(config_addr_cell)); + if (config_addr_cs.size() != 0x100) { + return td::Status::Error(PSLICE() << "configuration parameter 0 with config address has wrong size"); + } + ton::StdSmcAddress config_addr; + config_addr_cs.fetch_bits_to(config_addr); + auto global_config = block::Config(config_params_cell, std::move(config_addr), block::Config::needWorkchainInfo | block::Config::needSpecialSmc | block::Config::needCapabilities); + TRY_STATUS_PREFIX(global_config.unpack(), "Can't unpack config params: "); + return global_config; +} + +void *transaction_emulator_create(const char *config_params_boc, int vm_log_verbosity) { + auto global_config_res = decode_config(config_params_boc); + if (global_config_res.is_error()) { + LOG(ERROR) << global_config_res.move_as_error().message(); + return nullptr; + } + auto global_config = std::make_shared(global_config_res.move_as_ok()); + return new emulator::TransactionEmulator(std::move(global_config), vm_log_verbosity); +} + +void *emulator_config_create(const char *config_params_boc) { + auto config = decode_config(config_params_boc); + if (config.is_error()) { + LOG(ERROR) << "Error decoding config: " << config.move_as_error(); + return nullptr; + } + return new block::Config(config.move_as_ok()); +} + +const char *transaction_emulator_emulate_transaction(void *transaction_emulator, const char *shard_account_boc, const char *message_boc) { + auto emulator = static_cast(transaction_emulator); + + auto message_cell_r = boc_b64_to_cell(message_boc); + if (message_cell_r.is_error()) { + ERROR_RESPONSE(PSTRING() << "Can't deserialize message boc: " << message_cell_r.move_as_error()); + } + auto message_cell = message_cell_r.move_as_ok(); + auto message_cs = vm::load_cell_slice(message_cell); + int msg_tag = block::gen::t_CommonMsgInfo.get_tag(message_cs); + + auto shard_account_cell = boc_b64_to_cell(shard_account_boc); + if (shard_account_cell.is_error()) { + ERROR_RESPONSE(PSTRING() << "Can't deserialize shard account boc: " << shard_account_cell.move_as_error()); + } + auto shard_account_slice = vm::load_cell_slice(shard_account_cell.ok_ref()); + block::gen::ShardAccount::Record shard_account; + if (!tlb::unpack(shard_account_slice, shard_account)) { + ERROR_RESPONSE(PSTRING() << "Can't unpack shard account cell"); + } + + td::Ref addr_slice; + auto account_slice = vm::load_cell_slice(shard_account.account); + bool account_exists = block::gen::t_Account.get_tag(account_slice) == block::gen::Account::account; + if (block::gen::t_Account.get_tag(account_slice) == block::gen::Account::account_none) { + if (msg_tag == block::gen::CommonMsgInfo::ext_in_msg_info) { + block::gen::CommonMsgInfo::Record_ext_in_msg_info info; + if (!tlb::unpack(message_cs, info)) { + ERROR_RESPONSE(PSTRING() << "Can't unpack inbound external message"); + } + addr_slice = std::move(info.dest); + } + else if (msg_tag == block::gen::CommonMsgInfo::int_msg_info) { + block::gen::CommonMsgInfo::Record_int_msg_info info; + if (!tlb::unpack(message_cs, info)) { + ERROR_RESPONSE(PSTRING() << "Can't unpack inbound internal message"); + } + addr_slice = std::move(info.dest); + } else { + ERROR_RESPONSE(PSTRING() << "Only ext in and int message are supported"); + } + } else if (block::gen::t_Account.get_tag(account_slice) == block::gen::Account::account) { + block::gen::Account::Record_account account_record; + if (!tlb::unpack(account_slice, account_record)) { + ERROR_RESPONSE(PSTRING() << "Can't unpack account cell"); + } + addr_slice = std::move(account_record.addr); + } else { + ERROR_RESPONSE(PSTRING() << "Can't parse account cell"); + } + ton::WorkchainId wc; + ton::StdSmcAddress addr; + if (!block::tlb::t_MsgAddressInt.extract_std_address(addr_slice, wc, addr)) { + ERROR_RESPONSE(PSTRING() << "Can't extract account address"); + } + + auto account = block::Account(wc, addr.bits()); + ton::UnixTime now = emulator->get_unixtime(); + if (!now) { + now = (unsigned)std::time(nullptr); + } + bool is_special = wc == ton::masterchainId && emulator->get_config().is_special_smartcontract(addr); + if (account_exists) { + if (!account.unpack(vm::load_cell_slice_ref(shard_account_cell.move_as_ok()), now, is_special)) { + ERROR_RESPONSE(PSTRING() << "Can't unpack shard account"); + } + } else { + if (!account.init_new(now)) { + ERROR_RESPONSE(PSTRING() << "Can't init new account"); + } + account.last_trans_lt_ = shard_account.last_trans_lt; + account.last_trans_hash_ = shard_account.last_trans_hash; + } + + auto result = emulator->emulate_transaction(std::move(account), message_cell, now, 0, block::transaction::Transaction::tr_ord); + if (result.is_error()) { + ERROR_RESPONSE(PSTRING() << "Emulate transaction failed: " << result.move_as_error()); + } + auto emulation_result = result.move_as_ok(); + + auto external_not_accepted = dynamic_cast(emulation_result.get()); + if (external_not_accepted) { + return external_not_accepted_response(std::move(external_not_accepted->vm_log), external_not_accepted->vm_exit_code, + external_not_accepted->elapsed_time); + } + + auto emulation_success = dynamic_cast(*emulation_result); + auto trans_boc_b64 = cell_to_boc_b64(std::move(emulation_success.transaction)); + if (trans_boc_b64.is_error()) { + ERROR_RESPONSE(PSTRING() << "Can't serialize Transaction to boc " << trans_boc_b64.move_as_error()); + } + + auto new_shard_account_cell = vm::CellBuilder().store_ref(emulation_success.account.total_state) + .store_bits(emulation_success.account.last_trans_hash_.as_bitslice()) + .store_long(emulation_success.account.last_trans_lt_).finalize(); + auto new_shard_account_boc_b64 = cell_to_boc_b64(std::move(new_shard_account_cell)); + if (new_shard_account_boc_b64.is_error()) { + ERROR_RESPONSE(PSTRING() << "Can't serialize ShardAccount to boc " << new_shard_account_boc_b64.move_as_error()); + } + + td::optional actions_boc_b64; + if (emulation_success.actions.not_null()) { + auto actions_boc_b64_result = cell_to_boc_b64(std::move(emulation_success.actions)); + if (actions_boc_b64_result.is_error()) { + ERROR_RESPONSE(PSTRING() << "Can't serialize actions list cell to boc " << actions_boc_b64_result.move_as_error()); + } + actions_boc_b64 = actions_boc_b64_result.move_as_ok(); + } + + return success_response(trans_boc_b64.move_as_ok(), new_shard_account_boc_b64.move_as_ok(), std::move(emulation_success.vm_log), + std::move(actions_boc_b64), emulation_success.elapsed_time); +} + +const char *transaction_emulator_emulate_tick_tock_transaction(void *transaction_emulator, const char *shard_account_boc, bool is_tock) { + auto emulator = static_cast(transaction_emulator); + + auto shard_account_cell = boc_b64_to_cell(shard_account_boc); + if (shard_account_cell.is_error()) { + ERROR_RESPONSE(PSTRING() << "Can't deserialize shard account boc: " << shard_account_cell.move_as_error()); + } + auto shard_account_slice = vm::load_cell_slice(shard_account_cell.ok_ref()); + block::gen::ShardAccount::Record shard_account; + if (!tlb::unpack(shard_account_slice, shard_account)) { + ERROR_RESPONSE(PSTRING() << "Can't unpack shard account cell"); + } + + td::Ref addr_slice; + auto account_slice = vm::load_cell_slice(shard_account.account); + if (block::gen::t_Account.get_tag(account_slice) == block::gen::Account::account_none) { + ERROR_RESPONSE(PSTRING() << "Can't run tick/tock transaction on account_none"); + } + block::gen::Account::Record_account account_record; + if (!tlb::unpack(account_slice, account_record)) { + ERROR_RESPONSE(PSTRING() << "Can't unpack account cell"); + } + addr_slice = std::move(account_record.addr); + ton::WorkchainId wc; + ton::StdSmcAddress addr; + if (!block::tlb::t_MsgAddressInt.extract_std_address(addr_slice, wc, addr)) { + ERROR_RESPONSE(PSTRING() << "Can't extract account address"); + } + + auto account = block::Account(wc, addr.bits()); + ton::UnixTime now = emulator->get_unixtime(); + if (!now) { + now = (unsigned)std::time(nullptr); + } + bool is_special = wc == ton::masterchainId && emulator->get_config().is_special_smartcontract(addr); + if (!account.unpack(vm::load_cell_slice_ref(shard_account_cell.move_as_ok()), now, is_special)) { + ERROR_RESPONSE(PSTRING() << "Can't unpack shard account"); + } + + auto trans_type = is_tock ? block::transaction::Transaction::tr_tock : block::transaction::Transaction::tr_tick; + auto result = emulator->emulate_transaction(std::move(account), {}, now, 0, trans_type); + if (result.is_error()) { + ERROR_RESPONSE(PSTRING() << "Emulate transaction failed: " << result.move_as_error()); + } + auto emulation_result = result.move_as_ok(); + + auto emulation_success = dynamic_cast(*emulation_result); + auto trans_boc_b64 = cell_to_boc_b64(std::move(emulation_success.transaction)); + if (trans_boc_b64.is_error()) { + ERROR_RESPONSE(PSTRING() << "Can't serialize Transaction to boc " << trans_boc_b64.move_as_error()); + } + + auto new_shard_account_cell = vm::CellBuilder().store_ref(emulation_success.account.total_state) + .store_bits(emulation_success.account.last_trans_hash_.as_bitslice()) + .store_long(emulation_success.account.last_trans_lt_).finalize(); + auto new_shard_account_boc_b64 = cell_to_boc_b64(std::move(new_shard_account_cell)); + if (new_shard_account_boc_b64.is_error()) { + ERROR_RESPONSE(PSTRING() << "Can't serialize ShardAccount to boc " << new_shard_account_boc_b64.move_as_error()); + } + + td::optional actions_boc_b64; + if (emulation_success.actions.not_null()) { + auto actions_boc_b64_result = cell_to_boc_b64(std::move(emulation_success.actions)); + if (actions_boc_b64_result.is_error()) { + ERROR_RESPONSE(PSTRING() << "Can't serialize actions list cell to boc " << actions_boc_b64_result.move_as_error()); + } + actions_boc_b64 = actions_boc_b64_result.move_as_ok(); + } + + return success_response(trans_boc_b64.move_as_ok(), new_shard_account_boc_b64.move_as_ok(), std::move(emulation_success.vm_log), + std::move(actions_boc_b64), emulation_success.elapsed_time); +} + +bool transaction_emulator_set_unixtime(void *transaction_emulator, uint32_t unixtime) { + auto emulator = static_cast(transaction_emulator); + + emulator->set_unixtime(unixtime); + + return true; +} + +bool transaction_emulator_set_lt(void *transaction_emulator, uint64_t lt) { + auto emulator = static_cast(transaction_emulator); + + emulator->set_lt(lt); + + return true; +} + +bool transaction_emulator_set_rand_seed(void *transaction_emulator, const char* rand_seed_hex) { + auto emulator = static_cast(transaction_emulator); + + auto rand_seed_hex_slice = td::Slice(rand_seed_hex); + if (rand_seed_hex_slice.size() != 64) { + LOG(ERROR) << "Rand seed expected as 64 characters hex string"; + return false; + } + auto rand_seed_bytes = td::hex_decode(rand_seed_hex_slice); + if (rand_seed_bytes.is_error()) { + LOG(ERROR) << "Can't decode hex rand seed"; + return false; + } + td::BitArray<256> rand_seed; + rand_seed.as_slice().copy_from(rand_seed_bytes.move_as_ok()); + + emulator->set_rand_seed(rand_seed); + return true; +} + +bool transaction_emulator_set_ignore_chksig(void *transaction_emulator, bool ignore_chksig) { + auto emulator = static_cast(transaction_emulator); + + emulator->set_ignore_chksig(ignore_chksig); + + return true; +} + +bool transaction_emulator_set_config(void *transaction_emulator, const char* config_boc) { + auto emulator = static_cast(transaction_emulator); + + auto global_config_res = decode_config(config_boc); + if (global_config_res.is_error()) { + LOG(ERROR) << global_config_res.move_as_error().message(); + return false; + } + + emulator->set_config(std::make_shared(global_config_res.move_as_ok())); + + return true; +} + +void config_deleter(block::Config* ptr) { + // We do not delete the config object, since ownership management is delegated to the caller +} + +bool transaction_emulator_set_config_object(void *transaction_emulator, void* config) { + auto emulator = static_cast(transaction_emulator); + + std::shared_ptr config_ptr(static_cast(config), config_deleter); + + emulator->set_config(config_ptr); + + return true; +} + +bool transaction_emulator_set_libs(void *transaction_emulator, const char* shardchain_libs_boc) { + auto emulator = static_cast(transaction_emulator); + + if (shardchain_libs_boc != nullptr) { + auto shardchain_libs_cell = boc_b64_to_cell(shardchain_libs_boc); + if (shardchain_libs_cell.is_error()) { + LOG(ERROR) << "Can't deserialize shardchain libraries boc: " << shardchain_libs_cell.move_as_error(); + return false; + } + emulator->set_libs(vm::Dictionary(shardchain_libs_cell.move_as_ok(), 256)); + } + + return true; +} + +bool transaction_emulator_set_debug_enabled(void *transaction_emulator, bool debug_enabled) { + auto emulator = static_cast(transaction_emulator); + + emulator->set_debug_enabled(debug_enabled); + + return true; +} + +bool transaction_emulator_set_prev_blocks_info(void *transaction_emulator, const char* info_boc) { + auto emulator = static_cast(transaction_emulator); + + if (info_boc != nullptr) { + auto info_cell = boc_b64_to_cell(info_boc); + if (info_cell.is_error()) { + LOG(ERROR) << "Can't deserialize previous blocks boc: " << info_cell.move_as_error(); + return false; + } + vm::StackEntry info_value; + if (!info_value.deserialize(info_cell.move_as_ok())) { + LOG(ERROR) << "Can't deserialize previous blocks tuple"; + return false; + } + if (info_value.is_null()) { + emulator->set_prev_blocks_info({}); + } else if (info_value.is_tuple()) { + emulator->set_prev_blocks_info(info_value.as_tuple()); + } else { + LOG(ERROR) << "Can't set previous blocks tuple: not a tuple"; + return false; + } + } + + return true; +} + +void transaction_emulator_destroy(void *transaction_emulator) { + delete static_cast(transaction_emulator); +} + +bool emulator_set_verbosity_level(int verbosity_level) { + if (0 <= verbosity_level && verbosity_level <= VERBOSITY_NAME(NEVER)) { + SET_VERBOSITY_LEVEL(VERBOSITY_NAME(FATAL) + verbosity_level); + return true; + } + return false; +} + +void *tvm_emulator_create(const char *code, const char *data, int vm_log_verbosity) { + auto code_cell = boc_b64_to_cell(code); + if (code_cell.is_error()) { + LOG(ERROR) << "Can't deserialize code boc: " << code_cell.move_as_error(); + return nullptr; + } + auto data_cell = boc_b64_to_cell(data); + if (data_cell.is_error()) { + LOG(ERROR) << "Can't deserialize code boc: " << data_cell.move_as_error(); + return nullptr; + } + + auto emulator = new emulator::TvmEmulator(code_cell.move_as_ok(), data_cell.move_as_ok()); + emulator->set_vm_verbosity_level(vm_log_verbosity); + return emulator; +} + +bool tvm_emulator_set_libraries(void *tvm_emulator, const char *libs_boc) { + vm::Dictionary libs{256}; + auto libs_cell = boc_b64_to_cell(libs_boc); + if (libs_cell.is_error()) { + LOG(ERROR) << "Can't deserialize libraries boc: " << libs_cell.move_as_error(); + return false; + } + libs = vm::Dictionary(libs_cell.move_as_ok(), 256); + + auto emulator = static_cast(tvm_emulator); + emulator->set_libraries(std::move(libs)); + + return true; +} + +bool tvm_emulator_set_c7(void *tvm_emulator, const char *address, uint32_t unixtime, uint64_t balance, const char *rand_seed_hex, const char *config_boc) { + auto emulator = static_cast(tvm_emulator); + auto std_address = block::StdAddress::parse(td::Slice(address)); + if (std_address.is_error()) { + LOG(ERROR) << "Can't parse address: " << std_address.move_as_error(); + return false; + } + + std::shared_ptr global_config; + if (config_boc != nullptr) { + auto config_params_cell = boc_b64_to_cell(config_boc); + if (config_params_cell.is_error()) { + LOG(ERROR) << "Can't deserialize config params boc: " << config_params_cell.move_as_error(); + return false; + } + global_config = std::make_shared( + config_params_cell.move_as_ok(), td::Bits256::zero(), + block::Config::needWorkchainInfo | block::Config::needSpecialSmc | block::Config::needCapabilities); + auto unpack_res = global_config->unpack(); + if (unpack_res.is_error()) { + LOG(ERROR) << "Can't unpack config params"; + return false; + } + } + + auto rand_seed_hex_slice = td::Slice(rand_seed_hex); + if (rand_seed_hex_slice.size() != 64) { + LOG(ERROR) << "Rand seed expected as 64 characters hex string"; + return false; + } + auto rand_seed_bytes = td::hex_decode(rand_seed_hex_slice); + if (rand_seed_bytes.is_error()) { + LOG(ERROR) << "Can't decode hex rand seed"; + return false; + } + td::BitArray<256> rand_seed; + rand_seed.as_slice().copy_from(rand_seed_bytes.move_as_ok()); + + emulator->set_c7(std_address.move_as_ok(), unixtime, balance, rand_seed, std::const_pointer_cast(global_config)); + + return true; +} + +bool tvm_emulator_set_extra_currencies(void *tvm_emulator, const char *extra_currencies) { + auto emulator = static_cast(tvm_emulator); + vm::Dictionary dict{32}; + td::Slice extra_currencies_str{extra_currencies}; + while (true) { + auto next_space_pos = extra_currencies_str.find(' '); + auto currency_id_amount = next_space_pos == td::Slice::npos ? + extra_currencies_str.substr(0) : extra_currencies_str.substr(0, next_space_pos); + + if (!currency_id_amount.empty()) { + auto delim_pos = currency_id_amount.find('='); + if (delim_pos == td::Slice::npos) { + LOG(ERROR) << "Invalid extra currency format, missing '='"; + return false; + } + + auto currency_id_str = currency_id_amount.substr(0, delim_pos); + auto amount_str = currency_id_amount.substr(delim_pos + 1); + + auto currency_id = td::to_integer_safe(currency_id_str); + if (currency_id.is_error()) { + LOG(ERROR) << "Invalid extra currency id: " << currency_id_str; + return false; + } + auto amount = td::dec_string_to_int256(amount_str); + if (amount.is_null()) { + LOG(ERROR) << "Invalid extra currency amount: " << amount_str; + return false; + } + if (amount == 0) { + continue; + } + if (amount < 0) { + LOG(ERROR) << "Negative extra currency amount: " << amount_str; + return false; + } + + vm::CellBuilder cb; + block::tlb::t_VarUInteger_32.store_integer_value(cb, *amount); + if (!dict.set_builder(td::BitArray<32>(currency_id.ok()), cb, vm::DictionaryBase::SetMode::Add)) { + LOG(ERROR) << "Duplicate extra currency id"; + return false; + } + } + if (next_space_pos == td::Slice::npos) { + break; + } + extra_currencies_str.remove_prefix(next_space_pos + 1); + } + emulator->set_extra_currencies(std::move(dict).extract_root_cell()); + return true; +} + +bool tvm_emulator_set_config_object(void* tvm_emulator, void* config) { + auto emulator = static_cast(tvm_emulator); + auto global_config = std::shared_ptr(static_cast(config), config_deleter); + emulator->set_config(global_config); + return true; +} + +bool tvm_emulator_set_prev_blocks_info(void *tvm_emulator, const char* info_boc) { + auto emulator = static_cast(tvm_emulator); + + if (info_boc != nullptr) { + auto info_cell = boc_b64_to_cell(info_boc); + if (info_cell.is_error()) { + LOG(ERROR) << "Can't deserialize previous blocks boc: " << info_cell.move_as_error(); + return false; + } + vm::StackEntry info_value; + if (!info_value.deserialize(info_cell.move_as_ok())) { + LOG(ERROR) << "Can't deserialize previous blocks tuple"; + return false; + } + if (info_value.is_null()) { + emulator->set_prev_blocks_info({}); + } else if (info_value.is_tuple()) { + emulator->set_prev_blocks_info(info_value.as_tuple()); + } else { + LOG(ERROR) << "Can't set previous blocks tuple: not a tuple"; + return false; + } + } + + return true; +} + +bool tvm_emulator_set_gas_limit(void *tvm_emulator, int64_t gas_limit) { + auto emulator = static_cast(tvm_emulator); + emulator->set_gas_limit(gas_limit); + return true; +} + +bool tvm_emulator_set_debug_enabled(void *tvm_emulator, bool debug_enabled) { + auto emulator = static_cast(tvm_emulator); + emulator->set_debug_enabled(debug_enabled); + return true; +} + +const char *tvm_emulator_run_get_method(void *tvm_emulator, int method_id, const char *stack_boc) { + auto stack_cell = boc_b64_to_cell(stack_boc); + if (stack_cell.is_error()) { + ERROR_RESPONSE(PSTRING() << "Couldn't deserialize stack cell: " << stack_cell.move_as_error().to_string()); + } + auto stack_cs = vm::load_cell_slice(stack_cell.move_as_ok()); + td::Ref stack; + if (!vm::Stack::deserialize_to(stack_cs, stack)) { + ERROR_RESPONSE(PSTRING() << "Couldn't deserialize stack"); + } + + auto emulator = static_cast(tvm_emulator); + auto result = emulator->run_get_method(method_id, stack); + + vm::FakeVmStateLimits fstate(3500); // limit recursive (de)serialization calls + vm::VmStateInterface::Guard guard(&fstate); + + vm::CellBuilder stack_cb; + if (!result.stack->serialize(stack_cb)) { + ERROR_RESPONSE(PSTRING() << "Couldn't serialize stack"); + } + auto result_stack_boc = cell_to_boc_b64(stack_cb.finalize()); + if (result_stack_boc.is_error()) { + ERROR_RESPONSE(PSTRING() << "Couldn't serialize stack cell: " << result_stack_boc.move_as_error().to_string()); + } + + td::JsonBuilder jb; + auto json_obj = jb.enter_object(); + json_obj("success", td::JsonTrue()); + json_obj("stack", result_stack_boc.move_as_ok()); + json_obj("gas_used", std::to_string(result.gas_used)); + json_obj("vm_exit_code", result.code); + json_obj("vm_log", result.vm_log); + if (!result.missing_library) { + json_obj("missing_library", td::JsonNull()); + } else { + json_obj("missing_library", result.missing_library.value().to_hex()); + } + json_obj.leave(); + + return strdup(jb.string_builder().as_cslice().c_str()); +} + +const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc, int64_t gas_limit) { + auto params_cell = vm::std_boc_deserialize(td::Slice(params_boc, len)); + if (params_cell.is_error()) { + return nullptr; + } + auto params_cs = vm::load_cell_slice(params_cell.move_as_ok()); + auto code = params_cs.fetch_ref(); + auto data = params_cs.fetch_ref(); + + auto stack_cs = vm::load_cell_slice(params_cs.fetch_ref()); + auto params = vm::load_cell_slice(params_cs.fetch_ref()); + auto c7_cs = vm::load_cell_slice(params.fetch_ref()); + auto libs = vm::Dictionary(params.fetch_ref(), 256); + + auto method_id = params_cs.fetch_long(32); + + td::Ref stack; + if (!vm::Stack::deserialize_to(stack_cs, stack)) { + return nullptr; + } + + td::Ref c7; + if (!vm::Stack::deserialize_to(c7_cs, c7)) { + return nullptr; + } + + auto emulator = new emulator::TvmEmulator(code, data); + emulator->set_vm_verbosity_level(0); + emulator->set_gas_limit(gas_limit); + emulator->set_c7_raw(c7->fetch(0).as_tuple()); + if (!libs.is_empty()) { + emulator->set_libraries(std::move(libs)); + } + auto result = emulator->run_get_method(int(method_id), stack); + delete emulator; + + vm::CellBuilder stack_cb; + if (!result.stack->serialize(stack_cb)) { + return nullptr; + } + + vm::CellBuilder cb; + cb.store_long(result.code, 32); + cb.store_long(result.gas_used, 64); + cb.store_ref(stack_cb.finalize()); + + auto ser = vm::std_boc_serialize(cb.finalize()); + if (!ser.is_ok()) { + return nullptr; + } + auto sok = ser.move_as_ok(); + + auto sz = uint32_t(sok.size()); + char* rn = (char*)malloc(sz + 4); + memcpy(rn, &sz, 4); + memcpy(rn+4, sok.data(), sz); + + return rn; +} + +const char *tvm_emulator_send_external_message(void *tvm_emulator, const char *message_body_boc) { + auto message_body_cell = boc_b64_to_cell(message_body_boc); + if (message_body_cell.is_error()) { + ERROR_RESPONSE(PSTRING() << "Can't deserialize message body boc: " << message_body_cell.move_as_error()); + } + + auto emulator = static_cast(tvm_emulator); + auto result = emulator->send_external_message(message_body_cell.move_as_ok()); + + td::JsonBuilder jb; + auto json_obj = jb.enter_object(); + json_obj("success", td::JsonTrue()); + json_obj("gas_used", std::to_string(result.gas_used)); + json_obj("vm_exit_code", result.code); + json_obj("accepted", td::JsonBool(result.accepted)); + json_obj("vm_log", result.vm_log); + if (!result.missing_library) { + json_obj("missing_library", td::JsonNull()); + } else { + json_obj("missing_library", result.missing_library.value().to_hex()); + } + if (result.actions.is_null()) { + json_obj("actions", td::JsonNull()); + } else { + json_obj("actions", cell_to_boc_b64(result.actions).move_as_ok()); + } + json_obj("new_code", cell_to_boc_b64(result.new_state.code).move_as_ok()); + json_obj("new_data", cell_to_boc_b64(result.new_state.data).move_as_ok()); + json_obj.leave(); + + return strdup(jb.string_builder().as_cslice().c_str()); +} + +const char *tvm_emulator_send_internal_message(void *tvm_emulator, const char *message_body_boc, uint64_t amount) { + auto message_body_cell = boc_b64_to_cell(message_body_boc); + if (message_body_cell.is_error()) { + ERROR_RESPONSE(PSTRING() << "Can't deserialize message body boc: " << message_body_cell.move_as_error()); + } + + auto emulator = static_cast(tvm_emulator); + auto result = emulator->send_internal_message(message_body_cell.move_as_ok(), amount); + + td::JsonBuilder jb; + auto json_obj = jb.enter_object(); + json_obj("success", td::JsonTrue()); + json_obj("gas_used", std::to_string(result.gas_used)); + json_obj("vm_exit_code", result.code); + json_obj("accepted", td::JsonBool(result.accepted)); + json_obj("vm_log", result.vm_log); + if (!result.missing_library) { + json_obj("missing_library", td::JsonNull()); + } else { + json_obj("missing_library", result.missing_library.value().to_hex()); + } + if (result.actions.is_null()) { + json_obj("actions", td::JsonNull()); + } else { + json_obj("actions", cell_to_boc_b64(result.actions).move_as_ok()); + } + json_obj("new_code", cell_to_boc_b64(result.new_state.code).move_as_ok()); + json_obj("new_data", cell_to_boc_b64(result.new_state.data).move_as_ok()); + json_obj.leave(); + + return strdup(jb.string_builder().as_cslice().c_str()); +} + +void tvm_emulator_destroy(void *tvm_emulator) { + delete static_cast(tvm_emulator); +} + +void emulator_config_destroy(void *config) { + delete static_cast(config); +} + +const char* emulator_version() { + auto version_json = td::JsonBuilder(); + auto obj = version_json.enter_object(); + obj("emulatorLibCommitHash", GitMetadata::CommitSHA1()); + obj("emulatorLibCommitDate", GitMetadata::CommitDate()); + obj.leave(); + return strdup(version_json.string_builder().as_cslice().c_str()); +} diff --git a/emulator/emulator-extern.h b/emulator/emulator-extern.h new file mode 100644 index 00000000..14879e1e --- /dev/null +++ b/emulator/emulator-extern.h @@ -0,0 +1,325 @@ +#pragma once + +#include +#include "emulator_export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief Creates TransactionEmulator object + * @param config_params_boc Base64 encoded BoC serialized Config dictionary (Hashmap 32 ^Cell) + * @param vm_log_verbosity Verbosity level of VM log. 0 - log truncated to last 256 characters. 1 - unlimited length log. + * 2 - for each command prints its cell hash and offset. 3 - for each command log prints all stack values. + * @return Pointer to TransactionEmulator or nullptr in case of error + */ +EMULATOR_EXPORT void *transaction_emulator_create(const char *config_params_boc, int vm_log_verbosity); + +/** + * @brief Creates Config object from base64 encoded BoC + * @param config_params_boc Base64 encoded BoC serialized Config dictionary (Hashmap 32 ^Cell) + * @return Pointer to Config object or nullptr in case of error + */ +EMULATOR_EXPORT void *emulator_config_create(const char *config_params_boc); + +/** + * @brief Set unixtime for emulation + * @param transaction_emulator Pointer to TransactionEmulator object + * @param unixtime Unix timestamp + * @return true in case of success, false in case of error + */ +EMULATOR_EXPORT bool transaction_emulator_set_unixtime(void *transaction_emulator, uint32_t unixtime); + +/** + * @brief Set lt for emulation + * @param transaction_emulator Pointer to TransactionEmulator object + * @param lt Logical time + * @return true in case of success, false in case of error + */ +EMULATOR_EXPORT bool transaction_emulator_set_lt(void *transaction_emulator, uint64_t lt); + +/** + * @brief Set rand seed for emulation + * @param transaction_emulator Pointer to TransactionEmulator object + * @param rand_seed_hex Hex string of length 64 + * @return true in case of success, false in case of error + */ +EMULATOR_EXPORT bool transaction_emulator_set_rand_seed(void *transaction_emulator, const char* rand_seed_hex); + +/** + * @brief Set ignore_chksig flag for emulation + * @param transaction_emulator Pointer to TransactionEmulator object + * @param ignore_chksig Whether emulation should always succeed on CHKSIG operation + * @return true in case of success, false in case of error + */ +EMULATOR_EXPORT bool transaction_emulator_set_ignore_chksig(void *transaction_emulator, bool ignore_chksig); + +/** + * @brief Set config for emulation + * @param transaction_emulator Pointer to TransactionEmulator object + * @param config_boc Base64 encoded BoC serialized Config dictionary (Hashmap 32 ^Cell) + * @return true in case of success, false in case of error + */ +EMULATOR_EXPORT bool transaction_emulator_set_config(void *transaction_emulator, const char* config_boc); + +/** + * @brief Set config for emulation + * @param transaction_emulator Pointer to TransactionEmulator object + * @param config Pointer to Config object + * @return true in case of success, false in case of error + */ +EMULATOR_EXPORT bool transaction_emulator_set_config_object(void *transaction_emulator, void* config); + +/** + * @brief Set libraries for emulation + * @param transaction_emulator Pointer to TransactionEmulator object + * @param libs_boc Base64 encoded BoC serialized shared libraries dictionary (HashmapE 256 ^Cell). + * @return true in case of success, false in case of error + */ +EMULATOR_EXPORT bool transaction_emulator_set_libs(void *transaction_emulator, const char* libs_boc); + +/** + * @brief Enable or disable TVM debug primitives + * @param transaction_emulator Pointer to TransactionEmulator object + * @param debug_enabled Whether debug primitives should be enabled or not + * @return true in case of success, false in case of error + */ +EMULATOR_EXPORT bool transaction_emulator_set_debug_enabled(void *transaction_emulator, bool debug_enabled); + +/** + * @brief Set tuple of previous blocks (13th element of c7) + * @param transaction_emulator Pointer to TransactionEmulator object + * @param info_boc Base64 encoded BoC serialized TVM tuple (VmStackValue). + * @return true in case of success, false in case of error + */ +EMULATOR_EXPORT bool transaction_emulator_set_prev_blocks_info(void *transaction_emulator, const char* info_boc); + +/** + * @brief Emulate transaction + * @param transaction_emulator Pointer to TransactionEmulator object + * @param shard_account_boc Base64 encoded BoC serialized ShardAccount + * @param message_boc Base64 encoded BoC serialized inbound Message (internal or external) + * @return Json object with error: + * { + * "success": false, + * "error": "Error description", + * "external_not_accepted": false, + * // and optional fields "vm_exit_code", "vm_log", "elapsed_time" in case external message was not accepted. + * } + * Or success: + * { + * "success": true, + * "transaction": "Base64 encoded Transaction boc", + * "shard_account": "Base64 encoded new ShardAccount boc", + * "vm_log": "execute DUP...", + * "actions": "Base64 encoded compute phase actions boc (OutList n)", + * "elapsed_time": 0.02 + * } + */ +EMULATOR_EXPORT const char *transaction_emulator_emulate_transaction(void *transaction_emulator, const char *shard_account_boc, const char *message_boc); + +/** + * @brief Emulate tick tock transaction + * @param transaction_emulator Pointer to TransactionEmulator object + * @param shard_account_boc Base64 encoded BoC serialized ShardAccount of special account + * @param is_tock True for tock transactions, false for tick + * @return Json object with error: + * { + * "success": false, + * "error": "Error description", + * "external_not_accepted": false + * } + * Or success: + * { + * "success": true, + * "transaction": "Base64 encoded Transaction boc", + * "shard_account": "Base64 encoded new ShardAccount boc", + * "vm_log": "execute DUP...", + * "actions": "Base64 encoded compute phase actions boc (OutList n)", + * "elapsed_time": 0.02 + * } + */ +EMULATOR_EXPORT const char *transaction_emulator_emulate_tick_tock_transaction(void *transaction_emulator, const char *shard_account_boc, bool is_tock); + +/** + * @brief Destroy TransactionEmulator object + * @param transaction_emulator Pointer to TransactionEmulator object + */ +EMULATOR_EXPORT void transaction_emulator_destroy(void *transaction_emulator); + +/** + * @brief Set global verbosity level of the library + * @param verbosity_level New verbosity level (0 - never, 1 - error, 2 - warning, 3 - info, 4 - debug) + */ +EMULATOR_EXPORT bool emulator_set_verbosity_level(int verbosity_level); + +/** + * @brief Create TVM emulator + * @param code_boc Base64 encoded BoC serialized smart contract code cell + * @param data_boc Base64 encoded BoC serialized smart contract data cell + * @param vm_log_verbosity Verbosity level of VM log + * @return Pointer to TVM emulator object + */ +EMULATOR_EXPORT void *tvm_emulator_create(const char *code_boc, const char *data_boc, int vm_log_verbosity); + +/** + * @brief Set libraries for TVM emulator + * @param libs_boc Base64 encoded BoC serialized libraries dictionary (HashmapE 256 ^Cell). + * @return true in case of success, false in case of error + */ +EMULATOR_EXPORT bool tvm_emulator_set_libraries(void *tvm_emulator, const char *libs_boc); + +/** + * @brief Set c7 parameters + * @param tvm_emulator Pointer to TVM emulator + * @param address Adress of smart contract + * @param unixtime Unix timestamp + * @param balance Smart contract balance + * @param rand_seed_hex Random seed as hex string of length 64 + * @param config Base64 encoded BoC serialized Config dictionary (Hashmap 32 ^Cell). Optional. + * @return true in case of success, false in case of error + */ +EMULATOR_EXPORT bool tvm_emulator_set_c7(void *tvm_emulator, const char *address, uint32_t unixtime, uint64_t balance, const char *rand_seed_hex, const char *config); + +/** + * @brief Set extra currencies balance + * @param tvm_emulator Pointer to TVM emulator + * @param extra_currencies String with extra currencies balance in format "currency_id1=balance1 currency_id2=balance2 ..." + * @return true in case of success, false in case of error + */ +EMULATOR_EXPORT bool tvm_emulator_set_extra_currencies(void *tvm_emulator, const char *extra_currencies); + +/** + * @brief Set config for TVM emulator + * @param tvm_emulator Pointer to TVM emulator + * @param config Pointer to Config object + * @return true in case of success, false in case of error + */ +EMULATOR_EXPORT bool tvm_emulator_set_config_object(void* tvm_emulator, void* config); + +/** + * @brief Set tuple of previous blocks (13th element of c7) + * @param tvm_emulator Pointer to TVM emulator + * @param info_boc Base64 encoded BoC serialized TVM tuple (VmStackValue). + * @return true in case of success, false in case of error + */ +EMULATOR_EXPORT bool tvm_emulator_set_prev_blocks_info(void *tvm_emulator, const char* info_boc); + +/** + * @brief Set TVM gas limit + * @param tvm_emulator Pointer to TVM emulator + * @param gas_limit Gas limit + * @return true in case of success, false in case of error + */ +EMULATOR_EXPORT bool tvm_emulator_set_gas_limit(void *tvm_emulator, int64_t gas_limit); + +/** + * @brief Enable or disable TVM debug primitives + * @param tvm_emulator Pointer to TVM emulator + * @param debug_enabled Whether debug primitives should be enabled or not + * @return true in case of success, false in case of error + */ +EMULATOR_EXPORT bool tvm_emulator_set_debug_enabled(void *tvm_emulator, bool debug_enabled); + +/** + * @brief Run get method + * @param tvm_emulator Pointer to TVM emulator + * @param method_id Integer method id + * @param stack_boc Base64 encoded BoC serialized stack (VmStack) + * @return Json object with error: + * { + * "success": false, + * "error": "Error description" + * } + * Or success: + * { + * "success": true + * "vm_log": "...", + * "vm_exit_code": 0, + * "stack": "Base64 encoded BoC serialized stack (VmStack)", + * "missing_library": null, + * "gas_used": 1212 + * } + */ +EMULATOR_EXPORT const char *tvm_emulator_run_get_method(void *tvm_emulator, int method_id, const char *stack_boc); + +/** + * @brief Optimized version of "run get method" with all passed parameters in a single call + * @param len Length of params_boc buffer + * @param params_boc BoC serialized parameters, scheme: request$_ code:^Cell data:^Cell stack:^VmStack params:^[c7:^VmStack libs:^Cell] method_id:(## 32) + * @param gas_limit Gas limit + * @return Char* with first 4 bytes defining length, and the rest BoC serialized result + * Scheme: result$_ exit_code:(## 32) gas_used:(## 32) stack:^VmStack + */ +EMULATOR_EXPORT const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc, int64_t gas_limit); + +/** + * @brief Send external message + * @param tvm_emulator Pointer to TVM emulator + * @param message_body_boc Base64 encoded BoC serialized message body cell. + * @return Json object with error: + * { + * "success": false, + * "error": "Error description" + * } + * Or success: + * { + * "success": true, + * "new_code": "Base64 boc decoded new code cell", + * "new_data": "Base64 boc decoded new data cell", + * "accepted": true, + * "vm_exit_code": 0, + * "vm_log": "...", + * "missing_library": null, + * "gas_used": 1212, + * "actions": "Base64 boc decoded actions cell of type (OutList n)" + * } + */ +EMULATOR_EXPORT const char *tvm_emulator_send_external_message(void *tvm_emulator, const char *message_body_boc); + +/** + * @brief Send internal message + * @param tvm_emulator Pointer to TVM emulator + * @param message_body_boc Base64 encoded BoC serialized message body cell. + * @param amount Amount of nanograms attached with internal message. + * @return Json object with error: + * { + * "success": false, + * "error": "Error description" + * } + * Or success: + * { + * "success": true, + * "new_code": "Base64 boc decoded new code cell", + * "new_data": "Base64 boc decoded new data cell", + * "accepted": true, + * "vm_exit_code": 0, + * "vm_log": "...", + * "missing_library": null, + * "gas_used": 1212, + * "actions": "Base64 boc decoded actions cell of type (OutList n)" + * } + */ +EMULATOR_EXPORT const char *tvm_emulator_send_internal_message(void *tvm_emulator, const char *message_body_boc, uint64_t amount); + +/** + * @brief Destroy TVM emulator object + * @param tvm_emulator Pointer to TVM emulator object + */ +EMULATOR_EXPORT void tvm_emulator_destroy(void *tvm_emulator); + +/** + * @brief Destroy Config object + * @param tvm_emulator Pointer to Config object + */ +EMULATOR_EXPORT void emulator_config_destroy(void *config); + +/** + * @brief Get git commit hash and date of the library + */ +EMULATOR_EXPORT const char* emulator_version(); + +#ifdef __cplusplus +} // extern "C" +#endif diff --git a/emulator/emulator_export_list b/emulator/emulator_export_list new file mode 100644 index 00000000..bd991cd7 --- /dev/null +++ b/emulator/emulator_export_list @@ -0,0 +1,30 @@ +_transaction_emulator_create +_transaction_emulator_set_unixtime +_transaction_emulator_set_lt +_transaction_emulator_set_rand_seed +_transaction_emulator_set_ignore_chksig +_transaction_emulator_set_config +_transaction_emulator_set_config_object +_transaction_emulator_set_libs +_transaction_emulator_set_debug_enabled +_transaction_emulator_set_prev_blocks_info +_transaction_emulator_emulate_transaction +_transaction_emulator_emulate_tick_tock_transaction +_transaction_emulator_destroy +_emulator_set_verbosity_level +_emulator_config_create +_emulator_config_destroy +_tvm_emulator_create +_tvm_emulator_set_libraries +_tvm_emulator_set_c7 +_tvm_emulator_set_extra_currencies +_tvm_emulator_set_config_object +_tvm_emulator_set_prev_blocks_info +_tvm_emulator_set_gas_limit +_tvm_emulator_set_debug_enabled +_tvm_emulator_run_get_method +_tvm_emulator_send_external_message +_tvm_emulator_send_internal_message +_tvm_emulator_destroy +_tvm_emulator_emulate_run_method +_emulator_version diff --git a/emulator/test/emulator-tests.cpp b/emulator/test/emulator-tests.cpp new file mode 100644 index 00000000..ae273ddf --- /dev/null +++ b/emulator/test/emulator-tests.cpp @@ -0,0 +1,457 @@ +#include "td/utils/tests.h" + +#include "block/block-auto.h" +#include "block/block.h" +#include "block/block-parse.h" + +#include "crypto/vm/boc.h" + +#include "td/utils/base64.h" +#include "td/utils/crypto.h" +#include "td/utils/JsonBuilder.h" + +#include "smc-envelope/WalletV3.h" + +#include "emulator/emulator-extern.h" + +// testnet config as of 27.06.24 +const char *config_boc = "te6cckICAl8AAQAANecAAAIBIAABAAICAtgAAwAEAgL1AA0ADgIBIAAFAAYCAUgCPgI/AgEgAAcACAIBSAAJAAoCASAAHgAfAgEgAGUAZgIBSAALAAwCAWoA0gDTAQFI" + "AJIBAUgAsgEDpDMADwIBbgAQABEAQDPAueB1cC0DTaIjG28I/scJsoxoIScEE9LNtuiQoYa2AgOuIAASABMBA7LwABoBASAAFAEBIAAYAQHAABUCAWoAFgAXAIm/VzGV" + "o387z8N7BhdH91LBHMMhBLu7nv21jwo9wtTSXQIBABvI0aFLnw2QbZgjMPCLRdtRHxhUyinQudg6sdiohIwgwCAAQ79oJ47o6vzJDO5wV60LQESEyBcI3zuSSKtFQIlz" + "hk86tAMBg+mbgbrrZVY0qEWL8HxF+gYzy9t5jLO50+QkJ2DWbWFHj0Qaw5TPlNDYOnY0A2VNeAnS9bZ98W8X7FTvgVqStlmABAAZAIOgCYiOTH0TnIIa0oSKjkT3CsgH" + "NUU1Iy/5E472ortANeCAAAAAAAAAAAAAAAAROiXXYZuWf8AAi5Oy+xV/i+2JL9ABA6BgABsCASAAHAAdAFur4AAAAAAHGv1JjQAAEeDul1fav9HZ8+939/IsLGZ46E5h" + "3qjR13yIrB8mcfbBAFur/////8AHGv1JjQAAEeDul1fav9HZ8+939/IsLGZ46E5h3qjR13yIrB8mcfbBAgEgACAAIQIBIAAzADQCASAAIgAjAgEgACkAKgIBIAAkACUB" + "AUgAKAEBIAAmAQEgACcAQFVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVAEAzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMwBAAQEBAQEBAQEBAQEB" + "AQEBAQEBAQEBAQEBAQEBAQEBAQECASAAKwAsAQFYAC8BASAALQEBIAAuAEDv5x0Thgr6pq6ur2NvkWhIf4DxAxsL+Nk5rknT6n99oABTAf//////////////////////" + "////////////////////gAAAAIAAAAFAAQHAADACASAAMQAyABW+AAADvLNnDcFVUAAVv////7y9GpSiABACASAANQA2AgEgADcAOAIBIABCAEMCASAATgBPAgEgADkA" + "OgIBIAA+AD8BASAAOwEBIAA9AQHAADwAt9BTLudOzwABAnAAKtiftocOhhpk4QsHt8jHSWwV/O7nxvFyZKUf75zoqiN3Bfb/JZk7D9mvTw7EDHU5BlaNBz2ml2s54kRz" + "l0iBoQAAAAAP////+AAAAAAAAAAEABMaQ7msoAEBIB9IAQEgAEABASAAQQAUa0ZVPxAEO5rKAAAgAAAcIAAACWAAAAC0AAADhAEBIABEAQEgAEUAGsQAAAAGAAAAAAAA" + "AC4CA81AAEYARwIBIABVAEgAA6igAgEgAEkASgIBIABLAEwCASAATQBdAgEgAFsAXgIBIABbAFsCAUgAYQBhAQEgAFABASAAYgIBIABRAFICAtkAUwBUAgm3///wYABf" + "AGACASAAVQBWAgFiAFwAXQIBIABgAFcCAc4AYQBhAgEgAFgAWQIBIABaAF4CASAAXgBbAAFYAgEgAGEAYQIBIABeAF4AAdQAAUgAAfwCAdQAYQBhAAEgAgKRAGMAZAAq" + "NgIGAgUAD0JAAJiWgAAAAAEAAAH0ACo2BAcDBQBMS0ABMS0AAAAAAgAAA+gCASAAZwBoAgEgAHoAewIBIABpAGoCASAAcABxAgEgAGsAbAEBSABvAQEgAG0BASAAbgAM" + "AB4AHgADADFgkYTnKgAHEcN5N+CAAGteYg9IAAAB4AAIAE3QZgAAAAAAAAAAAAAAAIAAAAAAAAD6AAAAAAAAAfQAAAAAAAPQkEACASAAcgBzAgEgAHYAdwEBIAB0AQEg" + "AHUAlNEAAAAAAAAAZAAAAAAAD0JA3gAAAAAnEAAAAAAAAAAPQkAAAAAAAhYOwAAAAAAAACcQAAAAAAAmJaAAAAAABfXhAAAAAAA7msoAAJTRAAAAAAAAAGQAAAAAAACc" + "QN4AAAAAAZAAAAAAAAAAD0JAAAAAAAAPQkAAAAAAAAAnEAAAAAAAmJaAAAAAAAX14QAAAAAAO5rKAAEBIAB4AQEgAHkAUF3DAAIAAAAIAAAAEAAAwwAATiAAAYagAAJJ" + "8MMAAAPoAAATiAAAJxAAUF3DAAIAAAAIAAAAEAAAwwAehIAAmJaAATEtAMMAAABkAAATiAAAJxACAUgAfAB9AgEgAIAAgQEBIAB+AQEgAH8AQuoAAAAAAJiWgAAAAAAn" + "EAAAAAAAD0JAAAAAAYAAVVVVVQBC6gAAAAAABhqAAAAAAAGQAAAAAAAAnEAAAAABgABVVVVVAgEgAIIAgwEBWACGAQEgAIQBASAAhQAkwgEAAAD6AAAA+gAAA+gAAAAP" + "AErZAQMAAAfQAAA+gAAAAAMAAAAIAAAABAAgAAAAIAAAAAQAACcQAQHAAIcCASAAiACJAgFIAIoAiwIBagCQAJEAA9+wAgFYAIwAjQIBIACOAI8AQb7c3f6FapnFy4B4" + "QZnAdwvqMfKODXM49zeESA3vRM2QFABBvrMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM4AEG+tWede5qpBXVOzaq9SvpqBpwzTJ067Hk01rWZxT5wQ7gAQb8a" + "Yme1MOiTF+EsYXWNG8wYLwlq/ZXmR6g2PgSXaPOEegBBvzSTEofK4j4twU1E7XMbFoxvESypy3LTYwDOK8PDTfsWASsSZn08y2Z9WOsAEAAQD/////////3AAJMCAswA" + "lACVAgEgAJYAlwIBIACkAKUCASAAmACZAgEgAJ4AnwIBIACaAJsCASAAnACdAJsc46BJ4rulpzksHMZaJjfdtBExV1HRdikp9U7VlmJllrEaW2TYAFmAXnBlZIRH4Sqp" + "CbKkE6v60jyawOEYfVWJDgHg5kDaLMWq7kWQy6AAmxzjoEniuRloX7kgG9FNmRyw/AB/KERuToZdY5v8AHv9JJ8bCIKAWYBecGVkhEt/mk7tOEXbKUWuqIz/1NliY9sm" + "KNHFQimyb79WXudTIACbHOOgSeK0/SaSD6j2aEnWfmW/B7LOQBq2QiiBlnaLIzfq+J2HM0BZgF5wZWSEWPYUSh0McOyjsLL8prcsF5RNab+7jLN/5bOme1r98c8gAJsc" + "46BJ4rT4ptGRb52wRyHzhe/A8y/IQOC/W5R5aC6/l1IM4f/EgFmAXnBlZIRmDW7+WN70SpQsfX5DetODFOpW6zjCBx7cDf6E+rEipKACASAAoAChAgEgAKIAowCbHOOg" + "SeKqqZCAjJ16vfAa2GI9Dcp/I9zBTG2CwPqbx22lq00uLoBZgF5wZWSETeqWp7jqIGPuCYnPZSlQ1fMuSS4e1gF/i9uIeD8GEkNgAJsc46BJ4rugeQAFCtwRUJhvWRbx" + "smlpXTdXCio8SJSBdH/6VPCkAFmAXnBlZIRQPeE6JpjzEwkPI2mvCM1sDTcny96f2dhZ2DcBQmmywCAAmxzjoEnimDpTGClVkh/V+/mJmKVKEpdp4MvFgP5onw6saJRD" + "QApAWYBecGVkhElWAHSIgIhlXt+lUyQjmndd50temeILBd7WJwjjWBeIIACbHOOgSeKtcjPEr2gq3gMraY11K9Ikv1SPcVaj3veDWrY1o4nxKcBZgF5wZWSEabqKQLtX" + "PIkaYDaKvupB8EOxFDWpuMaJJVqafjw4h4sgAgEgAKYApwIBIACsAK0CASAAqACpAgEgAKoAqwCbHOOgSeK8POt5lMj96a3WrXWw7peFtWWh5oi9wsZqXRsrnHM4eoBZ" + "gF5wZWSEXlJk0ILG3LG9zsmxXf+r2OTayqr9FSKLBt9LJAow+aBgAJsc46BJ4qjb23m1w/0EvFl179XCQUUMk32z0kjSh+t6V2jnnqeFwFmAXnBlZIR2KWk8cqZgC06K" + "AphhfzE3VceQWtppAGEbybk06szO9KAAmxzjoEnihVEG74vb19K1l5o8WtWa0dH/gTPfytoA1LsVXR3ztfgAWYBecGVkhEVHN0AzKnDpKLX5P7Tnay/Ogc4rxeoks/yh" + "U3aWhEnGIACbHOOgSeKNl8PpsnZjGIy1CTzi01K8MhvQAEhGlzUDwj2ACC/yFUALGRulQuFOdHw2ulDcYktF860U0mFOYFaQPC7MVNbEeSsk45C9tSPgAgEgAK4ArwIB" + "IACwALEAmxzjoEnivAzuiTw+hkcXtw4XyJGYavfPayk6ehceV8FqrxrzKbQACMou1fGNuRpwF6ilPaS03+BSsz0YID1gpIkGozQp7gRFcQsyZFvVYACbHOOgSeKsoYF9" + "T9f0ArrtFxbViCRmpw2DsDzrllY35uHzP9DEosAICQwVUUQOx01jZ84Uy8ccqQ90Ml6tj5Sw14wOK055ds2sYSPy532gAJsc46BJ4piyhqkrUrk/KUOony6llV0S+DnZ" + "xDLdccZzKJ7bV+XiAAeBJKPSjdajMGMdZwRvewwnwsyc/7uHN718Pd8cHn7VQG1i9BJSeaAAmxzjoEnihY8aTVKeJnW4JHbfVPfkJwElQXxxqG94pNWmN6n9I5jABA51" + "90xtZChBtmQcmPHlOmtU6aLeZ+HBY7/jW6AMz26cNcymYyIuIAErEmZ9WOtmfXULABAAEA/////////3wACzAgLMALQAtQIBIAC2ALcCASAAxADFAgEgALgAuQIBIAC+" + "AL8CASAAugC7AgEgALwAvQCbHOOgSeK5Nyl3TF7AOD2UwhNOh+y3h9P5e0emd2zjffbNatQR1EBS4qdSDsPAZjIVSudNcsvyCAIbiOyNPYmj/MJG5lMjVLkYt4TIEDCg" + "AJsc46BJ4q0qr9PzfnnT+A41FG5Owo+9L+LsuT6PrQkuoR7XsLMzgFLioMqMr4sLf5pO7ThF2ylFrqiM/9TZYmPbJijRxUIpsm+/Vl7nUyAAmxzjoEnisgCK09re8agW" + "Ee8S6q329jm1WbZoHBHjO9oP0q3qItiAUuKgyoyviwfhKqkJsqQTq/rSPJrA4Rh9VYkOAeDmQNosxaruRZDLoACbHOOgSeKeKPVNUBZ96hhTOP8lp1kiAm2wfuT0HIxn" + "lw/0cyISP8BS4qDKjK+LGPYUSh0McOyjsLL8prcsF5RNab+7jLN/5bOme1r98c8gAgEgAMAAwQIBIADCAMMAmxzjoEnip+PTCe8vsapzyPHm88uO5qKBwt9yvn+S6aJW" + "OlcBqeDAUuKgyoyviyYNbv5Y3vRKlCx9fkN604MU6lbrOMIHHtwN/oT6sSKkoACbHOOgSeKwOTDV9phg7jYWvy7bbTD8N773bX9y1P7lxC7vtvdbvsBS4qDKjK+LDeqW" + "p7jqIGPuCYnPZSlQ1fMuSS4e1gF/i9uIeD8GEkNgAJsc46BJ4opGGis7tEqqLAW2742I2ugw5S5lFxeYpc4D9f/qbOMhwFLioMqMr4sQPeE6JpjzEwkPI2mvCM1sDTcn" + "y96f2dhZ2DcBQmmywCAAmxzjoEniqGUvGQXdvzVXTq/g3DpDkom5aqVipETXzq2o+FZdGDfAUuKgyoyviwlWAHSIgIhlXt+lUyQjmndd50temeILBd7WJwjjWBeIIAIB" + "IADGAMcCASAAzADNAgEgAMgAyQIBIADKAMsAmxzjoEnihA6ouVC73YehzpHoNBKL8q3Gp4YbwxOBhJdxpNWePHwAUuKgyoyviym6ikC7VzyJGmA2ir7qQfBDsRQ1qbjG" + "iSVamn48OIeLIACbHOOgSeKr2ACjLl9IlajrtDqvMLD+lfOMRQvmZAaL2NVDooVPYQBS4qDKjK+LHlJk0ILG3LG9zsmxXf+r2OTayqr9FSKLBt9LJAow+aBgAJsc46BJ" + "4oohDH+XJf2EoPKNkp+gv/WG2UonjUWXV+B/IvWUldUuQFLioMqMr4s2KWk8cqZgC06KAphhfzE3VceQWtppAGEbybk06szO9KAAmxzjoEnilP2IvoMbkK7LwTeBBX8u" + "dYI608SRo4nDIg7XUWQf2CYAUuKgyoyviwVHN0AzKnDpKLX5P7Tnay/Ogc4rxeoks/yhU3aWhEnGIAIBIADOAM8CASAA0ADRAJsc46BJ4qS3beCYCuu47Ohag9xU5wk6" + "/1uLtI/5NZ+VaqSyKsGdAApHFgZLFGK0fDa6UNxiS0XzrRTSYU5gVpA8LsxU1sR5KyTjkL21I+AAmxzjoEnivJI7eg6kFGx7dvMX7Xzoog/s5cwHxrcfec5z8/aP/8kA" + "CFtq86KYH4dNY2fOFMvHHKkPdDJerY+UsNeMDitOeXbNrGEj8ud9oACbHOOgSeKlwkl68jfkl6kGCq/tElh6bM85sFBPnt7exnkRJq68iQAG+mnlyjEXYzBjHWcEb3sM" + "J8LMnP+7hze9fD3fHB5+1UBtYvQSUnmgAJsc46BJ4oYswn2e5gWf+Va6NJ+K8sfz4qIHmVG2ryktqCkE9P8hQAPDhRot06toQbZkHJjx5TprVOmi3mfhwWO/41ugDM9u" + "nDXMpmMiLiABASAA1AEBIAD6AQsAtb0+sEAA1QIBIADWANcCA8H4ANgA2QID4fgA+AD5AgEgAPwA/QIBIADaANsCASAA3ADdAgEgAbgBuQIBIAGQAZECASAA3gDfAgEg" + "AOAA4QIBIADqAOsAQb7edpH5xbuqiZNqTG9H7flTOIfNiYtDxI5AH4T6G4tcVAIBIADiAOMAQb6U4RvTn2B6e+8nmlEv/eZoRz1YKr3qyDudETjcrMFgKAIBIADkAOUC" + "ASAA5gDnAgEgAOgA6QBBvgukN4cHaqlFuawJv/TGaxhU3HU2B5iu8cZPVMOseQOgAEG+K7U1xAKEqaBEZoqjpyAnvSx8Z9jfPTeAR/anR5axvmAAQb4tEpbKJaulevOY" + "XQPqlmgiMgHDU6C6X7KRxpFyzPf0YABBvjbzLj0Z1oudyhyW/QhJ0OUxRj9zEM8Y1YUI9Py3ga6gAgFqAOwA7QIBIADuAO8AQb4JmTypqySHVMVJMHWspb3xrs2Lrdy4" + "eJ+M7QxpbS4cIABBvgOb8O+4IZEUWqtnRGQ8JpMkMBocpZyk/do3d/9MYnVgAgEgAPAA8QBBvqQeZ13QP0lszxNKt380fCWuaV94vwC/bfuqmrlg1/fIAgEgAPIA8wIB" + "IAD0APUAQb4G2ph6AS/mD/+cIv4aIYm1z5jAgCW/TTDEr72ygXOP4ABBvhBZkdUWyc1zdg9Fhp9QSsWD+LSyXChKLJOiMF3rVNqgAgEgAPYA9wBBvhsYuojZc90oYnM2" + "WQ+c6cHdiTDRBD2UgxkJlbkZa+mgAEG9wBVbqgGsx1Pog5dkmDyUl4VIe1ZME2BEDY6zMNoQYsAAQb3R4obtqmXfb1H2NxdElqeDuWD4d+Y73ozNJ7dE4jGfQAIBIAHw" + "AfECASACGAIZAQPAwAD7AFWgESjR4FjxyuEAXHMvOQot+HG+D9TtSQavwKbeV09n3G92AAAAAAAAAH0QAgEgAP4A/wIBIAEcAR0CASABAAEBAgEgAR4BHwIBIAECAQMC" + "ASABEAERAgEgAQQBBQIBIAEIAQkCAWIBBgEHAEG+tp/96j2CYcuIRGkfljl5uv/Pilfg3KwCY8xwdr1JdqgAA97wAEG99o5GkuI7pwd5/g4Lt+avHh31l5WoNTndbJgd" + "dTJBicACAUgBCgELAgEgAQwBDQBBvgIKjJdXg0pHrRIfDgYLQ20dIU6mEbDa1FxtUXy9B6rgAEG+Cev2EcR/qY3lMYZ3tIojHR5s+wWySfwNg7XZgP23waACASABDgEP" + "AEG+fZGfOd+cHGx01cd8+xQAwUjfI/VrANsfVPw1jZFJhTAAQb4y2lPdHZUPm695Z+bh0Z1dcta4xXX7fl6dlc2SXOliIABBvhfW5EoZl/I8jARohetHRk6pp1y3mrXR" + "28rFYjHHtJCgAgFqARIBEwIBIAEUARUAQb4zE+Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU+KeaYLIABBvgPcWeL0jqPxd5IiX7AAYESGqFqZ7o60BjQZJwpPQP1gAgEg" + "ARYBFwBBvofANH7PG2eeTdX5Vr2ZUebxCfwJyzBCE4oriUVRU3jIAgEgARgBGQIBIAEaARsAQb4btDCZEGRAOXaB6WwVqFzYTd1zZgyp15BIuy9n029k4ABBvimf97Kd" + "WV/siLZ3qM/+nVRE+t0X0XdLsOK51DJ6WSPgAEG+CQrglDQDcC3b6lTaIr2tVPRR4RlxVAwxYNcF+6BkvaAAQb4mML93xvUT+iBDJrOfhiRGSs3vOczEy9DJAbuCb7aU" + "4AIBIAFAAUECASABYAFhAgEgASABIQIBIAE0ATUCASABIgEjAgFYAS4BLwIBIAEkASUAQb6L1UE7T5lmGOuEiyPgykuqAW0ENCaxjsi4fdzZq2D0GAICcAEmAScCASAB" + "KAEpAD+9QolK/7nMhu3MO9bzK31P7DqSFoQkLyeYP3RWz5f3KwA/vVaiOV3iXF+2BW0R7uGwqmnXP7y0cjEHibQT6v4MssECASABKgErAgV/rWABLAEtAEG96YUi7d3r" + "hTwVGwv/pocif6dNQ6DcZ3JVzvqdhFltQ0AAQb3zT7C1dlWQlR1QmfrLfaGi5Sj94Guq/gLQXakuFmoVwAA/u8n6yK+GpbUUdG9dja4DHHLGGEu5ZXb6rUHFOFMS7kAA" + "P7v3dUiUhgaZGC+mdUGyJEzagm0IMNe3d2Q1lCRBTK5AAEG+co6LJmQv3h46OSV3KsT2gWyv6MLPKOrfIXFt86dsXVACASABMAExAEG+KQF+kzAAZybpH/1z1zYof09W" + "YAAY6MbQHDj3AO9dCGACASABMgEzAEG9xJZFhUbajV1FgRPu0X8LSHY3DIBRmI4wC6uLpNG5lkAAQb3/+UXNzozn7Eb1PsCLs8NaD2VhG+9qBBlvLJG76KkTQAIBIAE2" + "ATcCASABPgE/AgEgATgBOQIBYgE8AT0AQb5l6UC6/ZmwRTHlWwthzsJcYx+8Vj2vmom9/nu617FmkAIBIAE6ATsAQb4J64Df7Vfb8/jmlGnsZByGAdCsEWA/FfWXyVEU" + "5d6CoABBvhv0Q/VEAfHxjnYRJRxb6xtGetqoO1OgjstzC/3Ok41gAEG964EWqVOQS0JWHUcxnAz6STWs7+BsROmocJCo+xmqe0AAQb3vR9oRALXcwLQPRb70F/gP7SAV" + "WqyMgCIasOqw+b47wABBvpbvxWd5+q2vJUVqR9AlbEIfdFysLR0PXGgVlBf8x5hYAEG+j9bgcxjKxRmfMrJEC6BbHTCQ+WNXqC3H+z591gZw0AgCASABQgFDAgEgAUgB" + "SQIBSAFEAUUAQb7KkreZXaSZXSPGxbgwuJddzpWJly3MFNYwALkyQcIdDABBvnLW0BTZocy0D6h48ehPtgqA0XqNxrqB86bTTks9uvuQAgEgAUYBRwBBvjYzcOXWIfyk" + "HqSDt3m92Hacz/XRoWD5F4yy0AQ/E0ogAEG+AShOVhiiJZ6Itzjs8O75CiiF+eXloz74MSVsHpPAMiACASABSgFLAgEgAVABUQIDeuABTAFNAgFYAU4BTwA/vVuDIbt9" + "1w2Z2FpLSOsyAUPo2ovei28SxaHKDSUdRz0AP71qm4D4evL40x1qJi6AGLh6oOBtxFr5bgc8Xr8jaeWRAEG+HzK7ymUhDh5PL//pLHqwaYidq3sym7hIWC32Rqol+mAA" + "Qb41DOvSox2jnjN40ZFtUSQhSJMCyEWhBRdRERRSltibIAIBIAFSAVMCASABWAFZAgFYAVQBVQIBIAFWAVcAQb3cHJ+brtBSsROnSioWNJqFxZ+5hIGX7ta5KuhleBFn" + "wABBvf/lQA5TJrGDmv6EqacNl5j6ktTzbQOEGqpl45xcekNAAEG+Nve9GdRJhn/t0fgYe7d1pkTBxa2AfiXcWeRYqE1K3yAAQb4jrXHoxDyh1ZYGBdBoQgLaScxW6pZR" + "1hEhJC8BqF+5IAIBIAFaAVsCAVgBXgFfAEG+CdErMSfFYmEK9J9XimJDXyszQjtVELtHIXQt7AvQjKACAUgBXAFdAEC9ivFB4bA7PAP0VXnTs784TO/4CoWLb1QqRdyr" + "0orLAgBAvb5z8xm2yt/HlB1G9TB2Qna4rVgzGxI/n4z3UYr3a7gAQb3f0PQO3/nU5ypuXD5/SaZboj2RhZjd5z47o7VM8AjDwABBvfGIqWXxgi7mCltWrYf4pQa2aRZP" + "FvMA8LBV1hmpauDAAgEgAWIBYwIBIAGAAYECASABZAFlAgEgAXIBcwIBIAFmAWcCAVgBcAFxAgFIAWgBaQIBIAFqAWsAQb33dj2qlHUSOf2DkiVrVwhcqy3SkE9YbBfn" + "zU07vK+uwABBvdxiQ8Yt/Lb9BztkNe9dyXuUyTOcKJRlF9BteI2LK99AAgEgAWwBbQBBvjxAsXZAtTQoMwJV27nrzNCyFum1aU1fbygeFMFuYX9gAgFIAW4BbwBBvdro" + "odCnIayUb5VXYFh23qJGAE4Oed7iqqU/L0iFAPpAAD+9QlUpU0rFnXRmWi3ZnIsFtIIm3JDSdtVPEGqGefBt/wA/vWGl+1GrGASEj3GaAizvMOXDl69yZpcU2YUtCHfG" + "jLUAQb4d/oR88TrfAGcKrMn44T3wBnbh3TWVQWr8rVq0bYTnYABBvhpY6fA3+apwMQXdpEMu8s8uFXf+625mtfciMt0dh4LgAgEgAXQBdQIBIAF4AXkAQb5d0CvPvsyC" + "ZxuTbUe5O2PtTudCwtgc3Ou4DMuX2WizEAIBSAF2AXcAQb3BrlEdo+Hw0uZZJxCgCdxWs/njs6bTHuprY7HtqNl0QABBvcSsc0L20So00ByQZ2oo0aUWf4BlreuHcpYk" + "R/C5Av7AAgEgAXoBewIBIAF+AX8CASABfAF9AEG+ErNElODwkPB+KvEKqCtCz8CS5HCcsC8/VoJGV5f0+uAAQb3FCW/Cy20jtvAS0j4k9eQvRg9tcpaQgFnHc5cB7Fdv" + "wABBvc5nMn9h2c6FeqzonvA74SwaTxZXTgLEXOKOIFOki9BAAEG+NkNRDvICKDQNaqBlpx1LnSn5qpShA00BPg8Tfv+LHaAAQb4+0zsN9j+Lxs1EvbGG0fMwbeeqbWlx" + "TzyjV4LE+0uJYAIBIAGCAYMCAUgBigGLAgEgAYQBhQIBIAGGAYcAQb5O+6O6Y7dWb4HOnMBK4fZ7QNo9woEzBIeKd5+K08xlkABBvlwlLor18dZ5/O3AomXxI5hxYM4o" + "J1Xrrx0JChLVxHpQAgFYAYgBiQBBvn9hAM+g43TTR8vOvZfnhX3kPBCgPp3T0+YF+Ai6RFHwAEG99KmZCgwzysLzIR2TNaJdbyX4lKduOMlCmhCp4L9gJEAAQb3Ntnmm" + "W4yzmAdiAYg7sNjoD8sCiWIvgvkpuYpTXcyiQAIBZgGMAY0CAW4BjgGPAEC9hzviVxD170gIZfsWPGFKfbOB6LCP5YhH7I7fWz7wdwBAvaey9kbu3gkPDYYEraB8b3sF" + "UrCgg4ask3C+O8UJ1mkAQL2wAL6FGQaCTbDdEwGUJ82TDpVMLoNr4ZGZWxcofghZAEC9lqzgehIXoMRj58vAWaHnNAi6UXEU5Ce942dJqf4HawIBIAGSAZMCASABqAGp" + "AgEgAZQBlQIBIAGkAaUCASABlgGXAgEgAZwBnQIBagGYAZkCASABmgGbAEC9syAieemf3vF3umY0lCaQxLhwvbTFuL8eQxPYrpeZ8ABAvbl6reyIsCKH2fq2I8+oEnkS" + "4xYy3RUH/7ka152WrisAQb4CJHgAcs+wQzgf/9IPKdknw/ej0Z+Q+n3BtSEKi0hIoABBvgqovnD/owP5nsA4G62765H5klOyA1TV+7jriGf2CtjgAgFYAZ4BnwIBIAGg" + "AaEAQb3dAG8Nta3/iYiTymgGxV0CfKQlN6UlidHeNgbvtMT9wABBve7An2cFgShRoZx3xA7hUDRtwbcLae0x4dPQQlAH8o3AAEG+HDeG9ZNvkzq3wDDpGt0cb5cHHFQ0" + "itHD3s5R2YHy8eACAWIBogGjAD+9ewqjet2JVaCzHa8NXfnW3ZtLEzEASpk9eicyztCrvwA/vXDzaFNMjF1BnqMojulsIHfT2Dj1ltCTVvoe8wu+GKcCASABpgGnAEG+" + "un2oV7CbmRhYGc7tLiCXj/L40+4ZlzvlmEnZPxyuQrgAQb5ElmikSUchX0lT+0ASVhwF0OBnUB8X4TD4m4/v2Dfl0ABBvlBR7mcUQO8IfN+DkkDYHF1reSJZhv08w6k+" + "JIA6ITiwAgEgAaoBqwIBIAG0AbUCAVgBrAGtAgEgAbIBswBBvhX0m4apMW/GEDxtnd+z0ug75voHd+OibSQbA2+tUPigAgEgAa4BrwIBWAGwAbEAQb3WKikPb9a/J2ti" + "V6yOhNUW5BivimV3gM+EI3VAxst6QAA/vUeSH4ZL+7V8eQBEF/0lm/ouIJ+wQs5QTzBpsSHSXLcAP71t4YT+jYHLpx5Gv3HFoOzL5rhg0Ukud8G3adF8AYlRAEG+Zf0n" + "TrwaPPTPlLjegNsGkoz7UV5wz7oYQet9+SNmRfAAQb5m0tqyXFYp4ntucDLTwJV1gxwoh6JoJL1Y0rfwfLQhUABBvqSCHVak+jIc9ANutTAfHpZNM3YdGky7yaDzsTrg" + "0WhIAgN9eAG2AbcAP70AGCAXHtaQJNqiST0rNTs8mUZSo5H6vM7gvA+3q7+iAD+9FgzFlOZUrfRtonCQzjDSFzrRv4l/94TFs9oi+RQ6kgIBIAG6AbsCASAB1gHXAgEg" + "AbwBvQIBIAHKAcsCASABvgG/AgEgAcQBxQBBvqg93lUVxmlCEks5kL8jTFcqg8lElfAi8dSee8j2jFDIAgEgAcABwQICcwHCAcMAQb5gqEQiOqBKE6++9fJCR6LRVtNC" + "cE9MFknXFlF0leXQMAA/vWDgwPyHRVDvZl2iYgjJ3nWePRW2wjoUWAxrbgzB5a8AP71vi5ua8R9Xas7ZJOxnHw9u9q/5yyOmKiac4YXhpzZdAEG+s1A7ERdFjokIunFC" + "SgeOxki+V8FwbGaF2nFzHDuF3TgCASABxgHHAEG+VoZmB1FqSlGFLPm5r9LBLAX67F6BFQLDlwahNArjz1ACAnIByAHJAD+9QiJtY3MezTL7KB0xvFikeKH4EL/XSXL0" + "b7P1FoVCXwA/vWinW8a2SNxgyMi+e0ML00BiBRy4kZh/JQrAHMZZ3Y0CASABzAHNAgEgAdIB0wIBWAHOAc8CBX+rYAHQAdEAQb4MUGwt25IQd3/yHjI03F71G8Kp2GMa" + "MEv2TiWoTKbs4ABBvjfgYNaJyJijra4RuhLyyPeGUpRcBZhwzdStzQ2MIyDgAD+8XsswC94XkGKDsoUR3B73WxXRX2LdrWSok77uwX/c8AA/vF/xbT+aFbepxFKzgZQ9" + "HbF9uy1KEVspm2/20klhldAAQb6ORoMEHrkmcAR+9ntDkAj0Hq6gLGUT0ceglU8Tm9jfuAIBIAHUAdUAQb5A/TMaqnaKx2BBvcxafTpwUxZYRXcKXTAZj80OapRScABB" + "vm8iGJqmHDhbx34EGjoh2YHhU4mpC/HVkmnz7NBQA0LwAgEgAdgB2QIBIAHmAecCASAB2gHbAgEgAd4B3wIDeqAB3AHdAEG+rC9orZ39Jto92k4zrR5989Z4qySyANXA" + "U8TLG5+0zfgAP71bgmShTXyEATbw0sECEmtwNtuzKI+S3DHEAPCPRhvTAD+9YC74p2ZuEIcz5A4sE69a7MTFuARvrmQnzUDgc7Mo3QIBIAHgAeECA3jgAeQB5QBBvlnO" + "v0cNQ7XgFJEwo9boghCVUHzfZ+urQtJh6esRW5xQAgFqAeIB4wBAvYY1sTf2ZnuWrkRZ+aijWbaH+q5ZMHkghn/Ys+tCZhoAQL2mLfoqMZw77ln7oAn0Cna+Bkp/snNw" + "xHgR2MTl/uqVAD+9XiSecyAvpnbNK3Z28HAfLhXvbXN59PmK+A7M2VDdAwA/vVcEpETq6AblfmVHtN91B7GNEyGglVc2447ooPciTZMCAUgB6AHpAgEgAe4B7wIBIAHq" + "AesAQb5J79ZyWgm+nqrXs6x0I4wkPiKQBH28C7RWNfPTqAfu8ABBvga7i8W/V7fCfyaKf+LLs48ld6A5hMVDltkVnlrlk+IgAgFYAewB7QBAvZIZkLzw7YHDbLe+Scl6" + "3uhdXfRwOUa0JHwJvuhGG3kAQL2a+QtRGkljjF6hjiME0j7LnnMjJkDh6mYBahv3SgufAEG+q3Z1cONnEXUOq6coX7x0RaK8l2WJj/QViIJee2G6qcgAQb6p4a4p479A" + "eC04K9HUR0x8B9TDrIBoSgVyWXe7xEjGWAIBIAHyAfMCASACBAIFAgEgAfQB9QIBIAH6AfsCAUgB9gH3AEG/JvWFCk64ubdT7k9fADlAADZW2oUeE0F//hNAx5vmQ24C" + "ASAB+AH5AEG+ortA8RL/qsRfVCCcmhh9yV+abEsHsmRmSDIyM5jiKZgAQb52rnetuJmLxwetwRXlQ8SwkzMrIHn9f1t+3vxypn8ikABBvlRRrWQUSUCo75+dTtj6fP1U" + "VTmV5DEujv1TIAc3ZLZQAgFYAfwB/QIBIAH+Af8AQb6OgDPbFGfKzqixWPD2Hmgt4G6KWUdQTJBPH3A9K+TZ6ABBvoMGKypw006AeRYqimLjmY2Ufp+SHk8C0ZJBNgVB" + "lzw4AgFqAgACAQIBWAICAgMAQb4FNJ5NJO4+0QwlVAWckUZXdk+PfYDexDZ1+ju9SxhF4ABBvjxQpfN455vPpJ/T+t2rtlKCE9X6KviHFRV802gCPe5gAEG+eMP12XnW" + "n0wTl6XmbgClnjYFM2JY2UAZYhUaknKJf3AAQb5WLKPfVeykQ1NoeXCT+51aWRbOsYTKmyd3AQSzEZ39EAIBIAIGAgcCASACDAINAgFYAggCCQIBIAIKAgsAQb68pxxy" + "oAcWOvpflv3VjfgrRk9v44uazdxMziPqfc1hGABBvqK0CHqoBidcEUJHx4naV3TtgmUv1oEhGpt3DFLGnncoAEG+xnddXOiUNI6DJEK4qY1Cxoa8Hl6iQkWXMWUwTPTo" + "H6wAQb72G1Ke4q6X03mCI87z+qVMO/gd+xvXv6SSwdWpfbnvjAIBIAIOAg8AQb8B8+e/xOcnn+D3yL8SGkEf/SXAx3pRSH/Lf3UDC6zxGgIBIAIQAhEAQb7an34AE4Mg" + "4PeqZAW6F6j/JbgFl8egPBFDGYC5dIgrvABBvpMd78gzSiVsK0zz0AHtEja8x1UoB/NDZMjn+l86NQK4AgFYAhICEwIBIAIUAhUAQb4zj6RBc4mQ6p3ng7mGJ7tp7Mbz" + "ERhe7obkM9A0wnCCIABBvcdlWZEG0Xj7uGgLfagzT4G4zmtS/JDEdPQBzOA0r99AAgEgAhYCFwBAvYD00VNmocZyrS8LPuogdwJgYw9wWC7QCKaicnWos7IAQL2UR4JV" + "cHfZibOIOqdJm+OTPN6Z1z0bykKu09Up+xc/AgEgAhoCGwIBIAIoAikCASACHAIdAgEgAiYCJwIBWAIeAh8CASACJAIlAEG+pJiW3Qo4nq8pKjVzzfs3/0uJxMmWXYyD" + "sduLHtuy8ggCASACIAIhAEG+VOzUzgqzn6yjJdPd2lOP2LQqiZF7O2/LbcmLzMf+hfACAnICIgIjAD+9bmuGAYNACsk0M2FDu866cYUghqLilNK52oLflBoKXQA/vU+c" + "jkDnrb+NojfOEJpwm2m9hlmHmr3HOWwyl4LEIcEAQb7xrpmUHCzHHfaaDbiK66LDRKeKblhi4QoTVRthJ2OzbABBvu6d/bOGE/iiKiKq5AGCvcetA3Izw45ihY196+ey" + "/BbcAEG/IPVJM6fGP9OC+PczMUdiKPNfwkUrt4eslgzXXEY0qCIAQb8FwRfn4LbYMTzpLsSBuEI3vAaLitADflpdxp+M5JVWtgIBIAIqAisCASACNgI3AEG/OXz/ktGT" + "HClb8arzLt3XEjlJTw9LEYxjGvSJNff79loCASACLAItAgFIAi4CLwIBIAIwAjEAQb5bNqQnT8GAdHDnixf9NzTB5VYvmnvaYs6m53KwbxMzsABBvlGslmQWFAphVxFA" + "GGIJvfuk/oBpngdzy0sJ8WxmWNSQAgN+ugIyAjMCAW4CNAI1AD+84Hccb00HqhGM3lRQZIZ3QmOuWlRDBQ9+uXRKu1L+hAA/vOLc2o+R4+ofOAQzeQiU06F6MN1nTGWW" + "J0eurH869zQAQb36Q2nDRQfZx/XsGJ+z0zYtk4S6OXPZcUASOm420y1FQABBvd9bukINCpKmNEXeA+ve7Mnhp8WSt+MPJFDCUYjDLZ1AAgEgAjgCOQBBvzD0lLSsv1Pi" + "WQ0jVDajeXFbJ/TkSakvdy+g0TPR27KGAgFYAjoCOwIBWAI8Aj0AQb53taVCRMwrV1sky/EE45BOJoTTJ0d6vkLZIb6j4k+G0ABBvlKuPPc+sdv9ffRS/Kj+bSQKZFE7" + "fT/jbtog/5dYYCCQAEG+ZZdBcxF7VCWJS+ti78o7J2qY+aXyKipCl2P0CfXeUhAAQb5gdZIvzW7H8KDz4y1oKMiuAzlXY+TF7PGVAwUvGCn0UAIBIAJAAkEBA6DAAkwB" + "AfwCQgIBIAJDAkQBwbnpmKopRu2n8DHZCDhXCHvJdckI7xw0kBvbb0npdd7jjldXaYBVRMxJsrwBE0/IJ4amdSKh5/Ec0+nZhJr583uAAAAAAAAAAAAAAABtiv/XlkR5" + "bE7cmy0osGrcZKJHU0ACRwEB1AJFAQH0AkYBwcaYme1MOiTF+EsYXWNG8wYLwlq/ZXmR6g2PgSXaPOEeN1Z517mqkFdU7Nqr1K+moGnDNMnTrseTTWtZnFPnBDuAAAAA" + "AAAAAAAAAABtiv/XlkR5bE7cmy0osGrcZKJHU0ACRwLFAaUkEAuNdJLBIqJ50rOuJIeLHBBTEnUHFMTTlSvkBfBlTSx/ArBlJBChmMwsWi3fU4ek+WJDvjF7AhFPUcNX" + "4kaAAAAAAAAAAAAAAAAAJ37Hglt9pn14Z9Vgj9pE3L7fXbBAAkcCTgIBIAJIAkkCASACSgJLAIO/z+IwR9x5RqPSfAzguJqFxanKeUhZQgFsmKwj4GuAK2WAAAAAAAAA" + "AAAAAAB7G3oHXwv9lQmh8vd3TonVSERFqMAAgr+jPzrhTYloKgTCsGgEFNx7OdH+sJ98etJnwrIVSsFxHwAAAAAAAAAAAAAAAOsF4basDVdO8s8p/fAcwLo9j5vxAIK/" + "n8LJGSxLhg32E0QLb7fZPphHZGiLJJFDrBMD8NcM15MAAAAAAAAAAAAAAADlTNYxyXvgdnFyrRaQRoiWLQnS/gLFAbUl61s8X25tzWBr7nugeg7IMDUhKEm34FWUmcD2" + "utVNIR8VdL9iPRR4dwjF/dVl4ymiWr+kkJXphEJvGbzwSXSAAAAAAAAAAAAAAAAAWZG0lbam3LV4+pciTNFehvbNeeLAAk0CTgIBIAJPAlAAMEO5rKAEO5rKADehIAPk" + "4cBAX14QA5iWgAIBIAJRAlIAg7/T7quzPdTpPcCght7xTpoi+g9Sw7gtkYDSyaOh0qHc0AAAAAAAAAAAAAAAADavGw+/CvXTnyDIJ6fZU+llAiixQAIBIAJTAlQCASAC" + "WwJcAgEgAlUCVgCBv1wad2ywThLttxU0gcwWuSJSuLNadPm8j3J85ggRzjkGAAAAAAAAAAAAAAAB1xLrLNteGQzkOClxdvv3E/l3M5UAgb8JuDCFQxifbIdTfjd1x7Mq" + "S+Z7dzIUkHtIdVjcVeFT2AAAAAAAAAAAAAAAAiwal03Yl9B7p2fVDSCtlYsZX6m+AgEgAlcCWAIBIAJZAloAgb7jxvbib0yb3DKvQBDcHL/hdg7NjCuqjUQ09t8hgmhV" + "oAAAAAAAAAAAAAAABEGpMZGoNId5F80sBzWgnjo+AP2UAIG+sE8ccijAbmkaBJVfyfgqY5pf4QSO+c5IFGVC9WwlY/AAAAAAAAAAAAAAAAeg08QveVui23B9QhrdMd7a" + "nx/sGACBvqxwYOyAk+H0YGBc70gZFJc6oqUvcHywU+yJNBfSNh+AAAAAAAAAAAAAAAADFU5kDFbQI6mIkEJqJNGncvWjiygCASACXQJeAIG/acxhhr+dznhtppGVCg+k" + "FqjL65rOddHn1mwyRj1rYgQAAAAAAAAAAAAAAACRfpTwfZ9v81WVbRpRYN+1/m9YhwCBvw9fhTm/NqURBT4FuwJczZWe39F575hmpFtt8KVniCwIAAAAAAAAAAAAAAAB" + "DkxuMKeNKjBZpVAjNVjJ/URzwhoAgb8RuD3rFDyNUpuXtBAnWTykKVAuY7UKLrye419st2b25AAAAAAAAAAAAAAAAlUrmS7Amiwb/77tvRUhnpfLLMXeL4vIgQ=="; + + +constexpr td::int64 Ton = 1000000000; + +TEST(Emulator, wallet_int_and_ext_msg) { + td::Ed25519::PrivateKey priv_key = td::Ed25519::generate_private_key().move_as_ok(); + auto pub_key = priv_key.get_public_key().move_as_ok(); + ton::WalletV3::InitData init_data; + init_data.public_key = pub_key.as_octet_string(); + init_data.wallet_id = 239; + auto wallet = ton::WalletV3::create(init_data, 2); + + auto address = wallet->get_address(); + + void *emulator = transaction_emulator_create(config_boc, 3); + const uint64_t lt = 42000000000; + CHECK(transaction_emulator_set_lt(emulator, lt)); + const uint32_t utime = 1337; + transaction_emulator_set_unixtime(emulator, utime); + + std::string shard_account_after_boc_b64; + + // emulate internal message with init state on uninit account + { + td::Ref account_root; + block::gen::Account().cell_pack_account_none(account_root); + auto none_shard_account_cell = vm::CellBuilder().store_ref(account_root).store_bits(td::Bits256::zero().as_bitslice()).store_long(0).finalize(); + auto none_shard_account_boc = td::base64_encode(std_boc_serialize(none_shard_account_cell).move_as_ok()); + + td::Ref int_msg; + { + block::gen::Message::Record message; + block::gen::CommonMsgInfo::Record_int_msg_info msg_info; + msg_info.ihr_disabled = true; + msg_info.bounce = false; + msg_info.bounced = false; + { + block::gen::MsgAddressInt::Record_addr_std src; + src.anycast = vm::CellBuilder().store_zeroes(1).as_cellslice_ref(); + src.workchain_id = 0; + src.address = td::Bits256();; + tlb::csr_pack(msg_info.src, src); + } + { + block::gen::MsgAddressInt::Record_addr_std dest; + dest.anycast = vm::CellBuilder().store_zeroes(1).as_cellslice_ref(); + dest.workchain_id = address.workchain; + dest.address = address.addr; + tlb::csr_pack(msg_info.dest, dest); + } + { + block::CurrencyCollection cc{10 * Ton}; + cc.pack_to(msg_info.value); + } + { + vm::CellBuilder cb; + block::tlb::t_Grams.store_integer_value(cb, td::BigInt256(int(0.03 * Ton))); + msg_info.fwd_fee = cb.as_cellslice_ref(); + } + { + vm::CellBuilder cb; + block::tlb::t_Grams.store_integer_value(cb, td::BigInt256(0)); + msg_info.ihr_fee = cb.as_cellslice_ref(); + } + msg_info.created_lt = 0; + msg_info.created_at = static_cast(utime); + tlb::csr_pack(message.info, msg_info); + message.init = vm::CellBuilder() + .store_ones(1) + .store_zeroes(1) + .append_cellslice(vm::load_cell_slice(ton::GenericAccount::get_init_state(wallet->get_state()))) + .as_cellslice_ref(); + message.body = vm::CellBuilder().store_zeroes(1).as_cellslice_ref(); + + tlb::type_pack_cell(int_msg, block::gen::t_Message_Any, message); + } + + CHECK(int_msg.not_null()); + + auto int_msg_boc = td::base64_encode(std_boc_serialize(int_msg).move_as_ok()); + + std::string int_emu_res = transaction_emulator_emulate_transaction(emulator, none_shard_account_boc.c_str(), int_msg_boc.c_str()); + LOG(ERROR) << "int_emu_res = " << int_emu_res; + + auto int_result_json = td::json_decode(td::MutableSlice(int_emu_res)); + CHECK(int_result_json.is_ok()); + auto int_result_value = int_result_json.move_as_ok(); + auto& int_result_obj = int_result_value.get_object(); + + auto success_field = td::get_json_object_field(int_result_obj, "success", td::JsonValue::Type::Boolean, false); + CHECK(success_field.is_ok()); + auto success = success_field.move_as_ok().get_boolean(); + CHECK(success); + + auto transaction_field = td::get_json_object_field(int_result_obj, "transaction", td::JsonValue::Type::String, false); + CHECK(transaction_field.is_ok()); + auto transaction_boc_b64 = transaction_field.move_as_ok().get_string(); + auto transaction_boc = td::base64_decode(transaction_boc_b64); + CHECK(transaction_boc.is_ok()); + auto trans_cell = vm::std_boc_deserialize(transaction_boc.move_as_ok()); + CHECK(trans_cell.is_ok()); + td::Bits256 trans_hash = trans_cell.ok()->get_hash().bits(); + block::gen::Transaction::Record trans; + block::gen::TransactionDescr::Record_trans_ord trans_descr; + CHECK(tlb::unpack_cell(trans_cell.move_as_ok(), trans) && tlb::unpack_cell(trans.description, trans_descr)); + CHECK(trans.outmsg_cnt == 0); + CHECK(trans.account_addr == wallet->get_address().addr); + CHECK(trans_descr.aborted == false); + CHECK(trans_descr.destroyed == false); + CHECK(trans.lt == lt); + CHECK(trans.now == utime); + + auto shard_account_field = td::get_json_object_field(int_result_obj, "shard_account", td::JsonValue::Type::String, false); + CHECK(shard_account_field.is_ok()); + auto shard_account_boc_b64 = shard_account_field.move_as_ok().get_string(); + shard_account_after_boc_b64 = shard_account_boc_b64.str(); + auto shard_account_boc = td::base64_decode(shard_account_boc_b64); + CHECK(shard_account_boc.is_ok()); + auto shard_account_cell = vm::std_boc_deserialize(shard_account_boc.move_as_ok()); + CHECK(shard_account_cell.is_ok()); + block::gen::ShardAccount::Record shard_account; + block::gen::Account::Record_account account; + CHECK(tlb::unpack_cell(shard_account_cell.move_as_ok(), shard_account) && tlb::unpack_cell(shard_account.account, account)); + CHECK(shard_account.last_trans_hash == trans_hash); + CHECK(shard_account.last_trans_lt == lt); + ton::WorkchainId wc; + ton::StdSmcAddress addr; + CHECK(block::tlb::t_MsgAddressInt.extract_std_address(account.addr, wc, addr)); + CHECK(address.workchain == wc); + CHECK(address.addr == addr); + } + + // emulate external message + { + auto ext_body = wallet->make_a_gift_message(priv_key, utime + 60, {ton::WalletV3::Gift{block::StdAddress(0, ton::StdSmcAddress()), 1 * Ton}}); + CHECK(ext_body.is_ok()); + auto ext_msg = ton::GenericAccount::create_ext_message(address, {}, ext_body.move_as_ok()); + auto ext_msg_boc = td::base64_encode(std_boc_serialize(ext_msg).move_as_ok()); + std::string ext_emu_res = transaction_emulator_emulate_transaction(emulator, shard_account_after_boc_b64.c_str(), ext_msg_boc.c_str()); + LOG(ERROR) << "ext_emu_res = " << ext_emu_res; + + auto ext_result_json = td::json_decode(td::MutableSlice(ext_emu_res)); + CHECK(ext_result_json.is_ok()); + auto ext_result = ext_result_json.move_as_ok(); + auto &ext_result_obj = ext_result.get_object(); + auto ext_success_field = td::get_json_object_field(ext_result_obj, "success", td::JsonValue::Type::Boolean, false); + CHECK(ext_success_field.is_ok()); + auto ext_success = ext_success_field.move_as_ok().get_boolean(); + CHECK(ext_success); + + auto ext_transaction_field = td::get_json_object_field(ext_result_obj, "transaction", td::JsonValue::Type::String, false); + CHECK(ext_transaction_field.is_ok()); + auto ext_transaction_boc_b64 = ext_transaction_field.move_as_ok().get_string(); + auto ext_transaction_boc = td::base64_decode(ext_transaction_boc_b64); + CHECK(ext_transaction_boc.is_ok()); + auto ext_trans_cell = vm::std_boc_deserialize(ext_transaction_boc.move_as_ok()); + CHECK(ext_trans_cell.is_ok()); + td::Bits256 ext_trans_hash = ext_trans_cell.ok()->get_hash().bits(); + block::gen::Transaction::Record ext_trans; + block::gen::TransactionDescr::Record_trans_ord ext_trans_descr; + CHECK(tlb::unpack_cell(ext_trans_cell.move_as_ok(), ext_trans) && tlb::unpack_cell(ext_trans.description, ext_trans_descr)); + CHECK(ext_trans.outmsg_cnt == 1); + CHECK(ext_trans.account_addr == wallet->get_address().addr); + CHECK(ext_trans_descr.aborted == false); + CHECK(ext_trans_descr.destroyed == false); + + auto ext_shard_account_field = td::get_json_object_field(ext_result_obj, "shard_account", td::JsonValue::Type::String, false); + CHECK(ext_shard_account_field.is_ok()); + auto ext_shard_account_boc_b64 = ext_shard_account_field.move_as_ok().get_string(); + auto ext_shard_account_boc = td::base64_decode(ext_shard_account_boc_b64); + CHECK(ext_shard_account_boc.is_ok()); + auto ext_shard_account_cell = vm::std_boc_deserialize(ext_shard_account_boc.move_as_ok()); + CHECK(ext_shard_account_cell.is_ok()); + block::gen::ShardAccount::Record ext_shard_account; + block::gen::Account::Record_account ext_account; + CHECK(tlb::unpack_cell(ext_shard_account_cell.move_as_ok(), ext_shard_account) && tlb::unpack_cell(ext_shard_account.account, ext_account)); + CHECK(ext_shard_account.last_trans_hash == ext_trans_hash); + CHECK(ext_shard_account.last_trans_lt == ext_trans.lt); + ton::WorkchainId wc; + ton::StdSmcAddress addr; + CHECK(block::tlb::t_MsgAddressInt.extract_std_address(ext_account.addr, wc, addr)); + CHECK(address.workchain == wc); + CHECK(address.addr == addr); + } +} + +TEST(Emulator, tvm_emulator) { + td::Ed25519::PrivateKey priv_key = td::Ed25519::generate_private_key().move_as_ok(); + auto pub_key = priv_key.get_public_key().move_as_ok(); + ton::WalletV3::InitData init_data; + init_data.public_key = pub_key.as_octet_string(); + init_data.wallet_id = 239; + init_data.seqno = 1337; + auto wallet = ton::WalletV3::create(init_data, 2); + + auto code = ton::SmartContractCode::get_code(ton::SmartContractCode::Type::WalletV3, 2); + auto code_boc_b64 = td::base64_encode(std_boc_serialize(code).move_as_ok()); + auto data = ton::WalletV3::get_init_data(init_data); + auto data_boc_b64 = td::base64_encode(std_boc_serialize(data).move_as_ok()); + + void *tvm_emulator = tvm_emulator_create(code_boc_b64.c_str(), data_boc_b64.c_str(), 1); + unsigned method_crc = td::crc16("seqno"); + unsigned method_id = (method_crc & 0xffff) | 0x10000; + auto stack = td::make_ref(); + vm::CellBuilder stack_cb; + CHECK(stack->serialize(stack_cb)); + auto stack_cell = stack_cb.finalize(); + auto stack_boc = td::base64_encode(std_boc_serialize(stack_cell).move_as_ok()); + + char addr_buffer[49] = {0}; + CHECK(wallet->get_address().rserialize_to(addr_buffer)); + + auto rand_seed = std::string(64, 'F'); + CHECK(tvm_emulator_set_c7(tvm_emulator, addr_buffer, 1337, 10 * Ton, rand_seed.c_str(), config_boc)); + std::string tvm_res = tvm_emulator_run_get_method(tvm_emulator, method_id, stack_boc.c_str()); + LOG(ERROR) << "tvm_res = " << tvm_res; + + auto result_json = td::json_decode(td::MutableSlice(tvm_res)); + CHECK(result_json.is_ok()); + auto result = result_json.move_as_ok(); + auto& result_obj = result.get_object(); + + auto success_field = td::get_json_object_field(result_obj, "success", td::JsonValue::Type::Boolean, false); + CHECK(success_field.is_ok()); + auto success = success_field.move_as_ok().get_boolean(); + CHECK(success); + + auto stack_field = td::get_json_object_field(result_obj, "stack", td::JsonValue::Type::String, false); + CHECK(stack_field.is_ok()); + auto stack_val = stack_field.move_as_ok(); + auto& stack_obj = stack_val.get_string(); + auto stack_res_boc = td::base64_decode(stack_obj); + CHECK(stack_res_boc.is_ok()); + auto stack_res_cell = vm::std_boc_deserialize(stack_res_boc.move_as_ok()); + CHECK(stack_res_cell.is_ok()); + td::Ref stack_res; + auto stack_res_cs = vm::load_cell_slice(stack_res_cell.move_as_ok()); + CHECK(vm::Stack::deserialize_to(stack_res_cs, stack_res)); + CHECK(stack_res->depth() == 1); + CHECK(stack_res.write().pop_int()->to_long() == init_data.seqno); +} + +TEST(Emulator, tvm_emulator_extra_currencies) { + void *tvm_emulator = tvm_emulator_create("te6cckEBBAEAHgABFP8A9KQT9LzyyAsBAgFiAgMABtBfBAAJofpP8E8XmGlj", "te6cckEBAQEAAgAAAEysuc0=", 1); + std::string addr = "0:" + std::string(64, 'F'); + tvm_emulator_set_c7(tvm_emulator, addr.c_str(), 1337, 1000, std::string(64, 'F').c_str(), nullptr); + CHECK(tvm_emulator_set_extra_currencies(tvm_emulator, "100=20000 200=1")); + unsigned method_crc = td::crc16("get_balance"); + unsigned method_id = (method_crc & 0xffff) | 0x10000; + + auto stack = td::make_ref(); + vm::CellBuilder stack_cb; + CHECK(stack->serialize(stack_cb)); + auto stack_cell = stack_cb.finalize(); + auto stack_boc = td::base64_encode(std_boc_serialize(stack_cell).move_as_ok()); + + std::string tvm_res = tvm_emulator_run_get_method(tvm_emulator, method_id, stack_boc.c_str()); + + auto result_json = td::json_decode(td::MutableSlice(tvm_res)); + auto result = result_json.move_as_ok(); + auto& result_obj = result.get_object(); + + auto success_field = td::get_json_object_field(result_obj, "success", td::JsonValue::Type::Boolean, false); + auto success = success_field.move_as_ok().get_boolean(); + CHECK(success); + + auto stack_field = td::get_json_object_field(result_obj, "stack", td::JsonValue::Type::String, false); + auto stack_val = stack_field.move_as_ok(); + auto& stack_obj = stack_val.get_string(); + auto stack_res_boc = td::base64_decode(stack_obj); + auto stack_res_cell = vm::std_boc_deserialize(stack_res_boc.move_as_ok()); + td::Ref stack_res; + auto stack_res_cs = vm::load_cell_slice(stack_res_cell.move_as_ok()); + CHECK(vm::Stack::deserialize_to(stack_res_cs, stack_res)); + CHECK(stack_res->depth() == 1); + auto tuple = stack_res.write().pop_tuple(); + CHECK(tuple->size() == 2); + + auto ton_balance = tuple->at(0).as_int(); + CHECK(ton_balance == 1000); + + auto cell = tuple->at(1).as_cell(); + auto dict = vm::Dictionary{cell, 32}; + auto it = dict.begin(); + std::map ec_balance; + while (!it.eof()) { + auto id = static_cast(td::BitArray<32>(it.cur_pos()).to_ulong()); + auto value_cs = it.cur_value(); + auto value = block::tlb::t_VarUInteger_32.as_integer(value_cs); + ec_balance[id] = value; + ++it; + } + CHECK(ec_balance.size() == 2); + CHECK(ec_balance[100] == 20000); + CHECK(ec_balance[200] == 1); +} diff --git a/emulator/transaction-emulator.cpp b/emulator/transaction-emulator.cpp new file mode 100644 index 00000000..6267f9bd --- /dev/null +++ b/emulator/transaction-emulator.cpp @@ -0,0 +1,281 @@ +#include +#include "transaction-emulator.h" +#include "crypto/common/refcnt.hpp" +#include "vm/vm.h" +#include "tdutils/td/utils/Time.h" + +using td::Ref; +using namespace std::string_literals; + +namespace emulator { +td::Result> TransactionEmulator::emulate_transaction( + block::Account&& account, td::Ref msg_root, ton::UnixTime utime, ton::LogicalTime lt, int trans_type) { + + td::Ref old_mparams; + std::vector storage_prices; + block::StoragePhaseConfig storage_phase_cfg{&storage_prices}; + block::ComputePhaseConfig compute_phase_cfg; + block::ActionPhaseConfig action_phase_cfg; + block::SerializeConfig serialize_config; + td::RefInt256 masterchain_create_fee, basechain_create_fee; + + if (!utime) { + utime = unixtime_; + } + if (!utime) { + utime = (unsigned)std::time(nullptr); + } + + auto fetch_res = block::FetchConfigParams::fetch_config_params( + *config_, prev_blocks_info_, &old_mparams, &storage_prices, &storage_phase_cfg, &rand_seed_, &compute_phase_cfg, + &action_phase_cfg, &serialize_config, &masterchain_create_fee, &basechain_create_fee, account.workchain, utime); + if(fetch_res.is_error()) { + return fetch_res.move_as_error_prefix("cannot fetch config params "); + } + + TRY_STATUS(vm::init_vm(debug_enabled_)); + + if (!lt) { + lt = lt_; + } + if (!lt) { + lt = (account.last_trans_lt_ / block::ConfigInfo::get_lt_align() + 1) * block::ConfigInfo::get_lt_align(); // next block after account_.last_trans_lt_ + } + account.block_lt = lt - lt % block::ConfigInfo::get_lt_align(); + + compute_phase_cfg.libraries = std::make_unique(libraries_); + compute_phase_cfg.ignore_chksig = ignore_chksig_; + compute_phase_cfg.with_vm_log = true; + compute_phase_cfg.vm_log_verbosity = vm_log_verbosity_; + + double start_time = td::Time::now(); + auto res = create_transaction(msg_root, &account, utime, lt, trans_type, + &storage_phase_cfg, &compute_phase_cfg, + &action_phase_cfg); + double elapsed = td::Time::now() - start_time; + + if(res.is_error()) { + return res.move_as_error_prefix("cannot run message on account "); + } + std::unique_ptr trans = res.move_as_ok(); + + if (!trans->compute_phase->accepted && trans->in_msg_extern) { + auto vm_log = trans->compute_phase->vm_log; + auto vm_exit_code = trans->compute_phase->exit_code; + return std::make_unique(std::move(vm_log), vm_exit_code, elapsed); + } + + if (!trans->serialize(serialize_config)) { + return td::Status::Error(-669,"cannot serialize new transaction for smart contract "s + trans->account.addr.to_hex()); + } + + auto trans_root = trans->commit(account); + if (trans_root.is_null()) { + return td::Status::Error(PSLICE() << "cannot commit new transaction for smart contract"); + } + + return std::make_unique(std::move(trans_root), std::move(account), + std::move(trans->compute_phase->vm_log), std::move(trans->compute_phase->actions), elapsed); +} + +td::Result TransactionEmulator::emulate_transaction(block::Account&& account, td::Ref original_trans) { + + block::gen::Transaction::Record record_trans; + if (!tlb::unpack_cell(original_trans, record_trans)) { + return td::Status::Error("Failed to unpack Transaction"); + } + + ton::LogicalTime lt = record_trans.lt; + ton::UnixTime utime = record_trans.now; + account.now_ = utime; + account.block_lt = record_trans.lt - record_trans.lt % block::ConfigInfo::get_lt_align(); + td::Ref msg_root = record_trans.r1.in_msg->prefetch_ref(); + int tag = block::gen::t_TransactionDescr.get_tag(vm::load_cell_slice(record_trans.description)); + + int trans_type = block::transaction::Transaction::tr_none; + switch (tag) { + case block::gen::TransactionDescr::trans_ord: { + trans_type = block::transaction::Transaction::tr_ord; + break; + } + case block::gen::TransactionDescr::trans_storage: { + trans_type = block::transaction::Transaction::tr_storage; + break; + } + case block::gen::TransactionDescr::trans_tick_tock: { + block::gen::TransactionDescr::Record_trans_tick_tock tick_tock; + if (!tlb::unpack_cell(record_trans.description, tick_tock)) { + return td::Status::Error("Failed to unpack tick tock transaction description"); + } + trans_type = tick_tock.is_tock ? block::transaction::Transaction::tr_tock : block::transaction::Transaction::tr_tick; + break; + } + case block::gen::TransactionDescr::trans_split_prepare: { + trans_type = block::transaction::Transaction::tr_split_prepare; + break; + } + case block::gen::TransactionDescr::trans_split_install: { + trans_type = block::transaction::Transaction::tr_split_install; + break; + } + case block::gen::TransactionDescr::trans_merge_prepare: { + trans_type = block::transaction::Transaction::tr_merge_prepare; + break; + } + case block::gen::TransactionDescr::trans_merge_install: { + trans_type = block::transaction::Transaction::tr_merge_install; + break; + } + } + + TRY_RESULT(emulation, emulate_transaction(std::move(account), msg_root, utime, lt, trans_type)); + + if (auto emulation_result_ptr = dynamic_cast(emulation.get())) { + auto& emulation_result = *emulation_result_ptr; + + if (td::Bits256(emulation_result.transaction->get_hash().bits()) != td::Bits256(original_trans->get_hash().bits())) { + return td::Status::Error("transaction hash mismatch"); + } + + if (!check_state_update(emulation_result.account, record_trans)) { + return td::Status::Error("account hash mismatch"); + } + + return emulation_result; + + } else if (auto emulation_not_accepted_ptr = dynamic_cast(emulation.get())) { + return td::Status::Error( PSTRING() + << "VM Log: " << emulation_not_accepted_ptr->vm_log + << ", VM Exit Code: " << emulation_not_accepted_ptr->vm_exit_code + << ", Elapsed Time: " << emulation_not_accepted_ptr->elapsed_time); + } else { + return td::Status::Error("emulation failed"); + } +} + +td::Result TransactionEmulator::emulate_transactions_chain(block::Account&& account, std::vector>&& original_transactions) { + + std::vector> emulated_transactions; + for (const auto& original_trans : original_transactions) { + if (original_trans.is_null()) { + continue; + } + + TRY_RESULT(emulation_result, emulate_transaction(std::move(account), original_trans)); + emulated_transactions.push_back(std::move(emulation_result.transaction)); + account = std::move(emulation_result.account); + } + + return TransactionEmulator::EmulationChain{ std::move(emulated_transactions), std::move(account) }; +} + +bool TransactionEmulator::check_state_update(const block::Account& account, const block::gen::Transaction::Record& trans) { + block::gen::HASH_UPDATE::Record hash_update; + return tlb::type_unpack_cell(trans.state_update, block::gen::t_HASH_UPDATE_Account, hash_update) && + hash_update.new_hash == account.total_state->get_hash().bits(); +} + +td::Result> TransactionEmulator::create_transaction( + td::Ref msg_root, block::Account* acc, + ton::UnixTime utime, ton::LogicalTime lt, int trans_type, + block::StoragePhaseConfig* storage_phase_cfg, + block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg) { + bool external{false}, ihr_delivered{false}, need_credit_phase{false}; + + if (msg_root.not_null()) { + auto cs = vm::load_cell_slice(msg_root); + external = block::gen::t_CommonMsgInfo.get_tag(cs); + } + + if (trans_type == block::transaction::Transaction::tr_ord) { + need_credit_phase = !external; + } else if (trans_type == block::transaction::Transaction::tr_merge_install) { + need_credit_phase = true; + } + + std::unique_ptr trans = + std::make_unique(*acc, trans_type, lt, utime, msg_root); + + if (msg_root.not_null() && !trans->unpack_input_msg(ihr_delivered, action_phase_cfg)) { + if (external) { + // inbound external message was not accepted + return td::Status::Error(-701,"inbound external message rejected by account "s + acc->addr.to_hex() + + " before smart-contract execution"); + } + return td::Status::Error(-669,"cannot unpack input message for a new transaction"); + } + + if (trans->bounce_enabled) { + if (!trans->prepare_storage_phase(*storage_phase_cfg, true)) { + return td::Status::Error(-669,"cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (need_credit_phase && !trans->prepare_credit_phase()) { + return td::Status::Error(-669,"cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + } else { + if (need_credit_phase && !trans->prepare_credit_phase()) { + return td::Status::Error(-669,"cannot create credit phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + if (!trans->prepare_storage_phase(*storage_phase_cfg, true, need_credit_phase)) { + return td::Status::Error(-669,"cannot create storage phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + } + + if (!trans->prepare_compute_phase(*compute_phase_cfg)) { + return td::Status::Error(-669,"cannot create compute phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + + if (!trans->compute_phase->accepted) { + if (!external && trans->compute_phase->skip_reason == block::ComputePhase::sk_none) { + return td::Status::Error(-669,"new ordinary transaction for smart contract "s + acc->addr.to_hex() + + " has not been accepted by the smart contract (?)"); + } + } + + if (trans->compute_phase->success && !trans->prepare_action_phase(*action_phase_cfg)) { + return td::Status::Error(-669,"cannot create action phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + + if (trans->bounce_enabled + && (!trans->compute_phase->success || trans->action_phase->state_exceeds_limits || trans->action_phase->bounce) + && !trans->prepare_bounce_phase(*action_phase_cfg)) { + return td::Status::Error(-669,"cannot create bounce phase of a new transaction for smart contract "s + acc->addr.to_hex()); + } + + return trans; +} + +void TransactionEmulator::set_unixtime(ton::UnixTime unixtime) { + unixtime_ = unixtime; +} + +void TransactionEmulator::set_lt(ton::LogicalTime lt) { + lt_ = lt; +} + +void TransactionEmulator::set_rand_seed(td::BitArray<256>& rand_seed) { + rand_seed_ = rand_seed; +} + +void TransactionEmulator::set_ignore_chksig(bool ignore_chksig) { + ignore_chksig_ = ignore_chksig; +} + +void TransactionEmulator::set_config(std::shared_ptr config) { + config_ = std::move(config); +} + +void TransactionEmulator::set_libs(vm::Dictionary &&libs) { + libraries_ = std::forward(libs); +} + +void TransactionEmulator::set_debug_enabled(bool debug_enabled) { + debug_enabled_ = debug_enabled; +} + +void TransactionEmulator::set_prev_blocks_info(td::Ref prev_blocks_info) { + prev_blocks_info_ = std::move(prev_blocks_info); +} + +} // namespace emulator diff --git a/emulator/transaction-emulator.h b/emulator/transaction-emulator.h new file mode 100644 index 00000000..eae109f4 --- /dev/null +++ b/emulator/transaction-emulator.h @@ -0,0 +1,92 @@ +#pragma once +#include "crypto/common/refcnt.hpp" +#include "ton/ton-types.h" +#include "crypto/vm/cells.h" +#include "block/transaction.h" +#include "block/block-auto.h" +#include "block/block-parse.h" +#include "block/mc-config.h" + +namespace emulator { +class TransactionEmulator { + std::shared_ptr config_; + vm::Dictionary libraries_; + int vm_log_verbosity_; + ton::UnixTime unixtime_; + ton::LogicalTime lt_; + td::BitArray<256> rand_seed_; + bool ignore_chksig_; + bool debug_enabled_; + td::Ref prev_blocks_info_; + +public: + TransactionEmulator(std::shared_ptr config, int vm_log_verbosity = 0) : + config_(std::move(config)), libraries_(256), vm_log_verbosity_(vm_log_verbosity), + unixtime_(0), lt_(0), rand_seed_(td::BitArray<256>::zero()), ignore_chksig_(false), debug_enabled_(false) { + } + + struct EmulationResult { + std::string vm_log; + double elapsed_time; + + EmulationResult(std::string vm_log_, double elapsed_time_) : vm_log(vm_log_), elapsed_time(elapsed_time_) {} + virtual ~EmulationResult() = default; + }; + + struct EmulationSuccess: EmulationResult { + td::Ref transaction; + block::Account account; + td::Ref actions; + + EmulationSuccess(td::Ref transaction_, block::Account account_, std::string vm_log_, td::Ref actions_, double elapsed_time_) : + EmulationResult(vm_log_, elapsed_time_), transaction(transaction_), account(account_) , actions(actions_) + {} + }; + + struct EmulationExternalNotAccepted: EmulationResult { + int vm_exit_code; + + EmulationExternalNotAccepted(std::string vm_log_, int vm_exit_code_, double elapsed_time_) : + EmulationResult(vm_log_, elapsed_time_), vm_exit_code(vm_exit_code_) + {} + }; + + struct EmulationChain { + std::vector> transactions; + block::Account account; + }; + + const block::Config& get_config() { + return *config_; + } + + ton::UnixTime get_unixtime() { + return unixtime_; + } + + td::Result> emulate_transaction( + block::Account&& account, td::Ref msg_root, ton::UnixTime utime, ton::LogicalTime lt, int trans_type); + + td::Result emulate_transaction(block::Account&& account, td::Ref original_trans); + td::Result emulate_transactions_chain(block::Account&& account, std::vector>&& original_transactions); + + void set_unixtime(ton::UnixTime unixtime); + void set_lt(ton::LogicalTime lt); + void set_rand_seed(td::BitArray<256>& rand_seed); + void set_ignore_chksig(bool ignore_chksig); + void set_config(std::shared_ptr config); + void set_libs(vm::Dictionary &&libs); + void set_debug_enabled(bool debug_enabled); + void set_prev_blocks_info(td::Ref prev_blocks_info); + +private: + bool check_state_update(const block::Account& account, const block::gen::Transaction::Record& trans); + + td::Result> create_transaction( + td::Ref msg_root, block::Account* acc, + ton::UnixTime utime, ton::LogicalTime lt, int trans_type, + block::StoragePhaseConfig* storage_phase_cfg, + block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg); +}; +} // namespace emulator diff --git a/emulator/tvm-emulator.hpp b/emulator/tvm-emulator.hpp new file mode 100644 index 00000000..acc13627 --- /dev/null +++ b/emulator/tvm-emulator.hpp @@ -0,0 +1,70 @@ +#pragma once +#include "smc-envelope/SmartContract.h" + +namespace emulator { +class TvmEmulator { + ton::SmartContract smc_; + ton::SmartContract::Args args_; +public: + using Answer = ton::SmartContract::Answer; + + TvmEmulator(td::Ref code, td::Ref data): smc_({code, data}) { + } + + void set_vm_verbosity_level(int vm_log_verbosity) { + args_.set_vm_verbosity_level(vm_log_verbosity); + } + + void set_libraries(vm::Dictionary&& libraries) { + args_.set_libraries(libraries); + } + + void set_gas_limit(int64_t limit) { + args_.set_limits(vm::GasLimits(limit)); + } + + void set_c7(block::StdAddress address, uint32_t unixtime, uint64_t balance, td::BitArray<256> rand_seed, std::shared_ptr config) { + args_.set_address(std::move(address)); + args_.set_now(unixtime); + args_.set_balance(balance); + args_.set_rand_seed(std::move(rand_seed)); + if (config) { + args_.set_config(std::move(config)); + } + } + + void set_extra_currencies(td::Ref extra_currencies) { + args_.set_extra_currencies(std::move(extra_currencies)); + } + + void set_c7_raw(td::Ref c7) { + args_.set_c7(std::move(c7)); + } + + void set_config(std::shared_ptr config) { + args_.set_config(std::move(config)); + } + + void set_prev_blocks_info(td::Ref tuple) { + args_.set_prev_blocks_info(std::move(tuple)); + } + + void set_debug_enabled(bool debug_enabled) { + args_.set_debug_enabled(debug_enabled); + } + + Answer run_get_method(int method_id, td::Ref stack) { + ton::SmartContract::Args args = args_; + return smc_.run_get_method(args.set_stack(stack).set_method_id(method_id)); + } + + Answer send_external_message(td::Ref message_body) { + return smc_.send_external_message(message_body, args_); + } + + Answer send_internal_message(td::Ref message_body, uint64_t amount) { + ton::SmartContract::Args args = args_; + return smc_.send_internal_message(message_body, args.set_amount(amount)); + } +}; +} \ No newline at end of file diff --git a/example/android/CMakeLists.txt b/example/android/CMakeLists.txt index 55eda220..0101ab64 100644 --- a/example/android/CMakeLists.txt +++ b/example/android/CMakeLists.txt @@ -3,7 +3,9 @@ # Sets the minimum version of CMake required to build the native library. -cmake_minimum_required(VERSION 3.4.1 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) + +project(TON_ANDROID VERSION 0.5 LANGUAGES C CXX) option(TONLIB_ENABLE_JNI "Enable JNI-compatible TonLib API" ON) @@ -34,12 +36,9 @@ add_library( # Sets the name of the library. list(APPEND CMAKE_FIND_ROOT_PATH "${CMAKE_CURRENT_SOURCE_DIR}/third_party/crypto/${ANDROID_ARCH_NAME}") set(TON_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../..) add_subdirectory(${TON_DIR} ton EXCLUDE_FROM_ALL) -target_link_libraries(native-lib tonlibjson_static) +target_link_libraries(native-lib tonlibjson) target_link_libraries(native-lib tonlib) -#target_sources(native-lib PRIVATE ${ALL_TEST_SOURCE}) -#target_link_libraries(native-lib all_tests) - set(TONLIB_API_JAVA_PACKAGE "drinkless/org/ton") target_compile_definitions(native-lib PRIVATE PACKAGE_NAME="${TONLIB_API_JAVA_PACKAGE}") diff --git a/example/android/README.md b/example/android/README.md index 0d10ff1e..cf12ba30 100644 --- a/example/android/README.md +++ b/example/android/README.md @@ -6,23 +6,9 @@ Prerequisite: installed Java and set environment variable JAVA_HOME. ```bash git clone --recursive https://github.com/ton-blockchain/ton.git cd ton -wget https://dl.google.com/android/repository/android-ndk-r25b-linux.zip -unzip android-ndk-r25b-linux.zip -export JAVA_AWT_LIBRARY=NotNeeded -export JAVA_JVM_LIBRARY=NotNeeded -export JAVA_INCLUDE_PATH=${JAVA_HOME}/include -export JAVA_AWT_INCLUDE_PATH=${JAVA_HOME}/include -export JAVA_INCLUDE_PATH2=${JAVA_HOME}/include/linux - -export ANDROID_NDK_ROOT=$(pwd)/android-ndk-r25b -export OPENSSL_DIR=$(pwd)/example/android/third_party/crypto - -rm -rf example/android/src/drinkless/org/ton/TonApi.java -cd example/android/ -cmake -GNinja -DTON_ONLY_TONLIB=ON . -ninja prepare_cross_compiling -rm CMakeCache.txt -./build-all.sh +cp assembly/android/build-android-tonlib.sh . +chmod +x build-android-tonlib.sh +sudo -E ./build-android-tonlib.sh ``` # Generation of Tonlib libraries for iOS in Xcode diff --git a/example/android/build-all.sh b/example/android/build-all.sh index e1abc469..6f97dec0 100755 --- a/example/android/build-all.sh +++ b/example/android/build-all.sh @@ -1,5 +1,18 @@ #!/bin/bash +echo ANDROID_NDK_ROOT = $ANDROID_NDK_ROOT + +echo Building tonlib for x86... +echo ARCH="x86" ./build.sh || exit 1 + +echo Building tonlib for x86_64... +echo ARCH="x86_64" ./build.sh || exit 1 + +echo Building tonlib for arm... +echo ARCH="arm" ./build.sh || exit 1 + +echo Building tonlib for arm64... +echo ARCH="arm64" ./build.sh || exit 1 diff --git a/example/android/build.sh b/example/android/build.sh index b16a9df9..dc0618e9 100755 --- a/example/android/build.sh +++ b/example/android/build.sh @@ -1,43 +1,77 @@ #!/bin/bash + pushd . -# ANDROID_TOOLCHAIN -# ANDROID_ABI -# ANDROID_PLATFORM -# ANDROID_STL -# ANDROID_PIE -# ANDROID_CPP_FEATURES -# ANDROID_ALLOW_UNDEFINED_SYMBOLS -# ANDROID_ARM_MODE -# ANDROID_ARM_NEON -# ANDROID_DISABLE_FORMAT_STRING_CHECKS -# ANDROID_CCACHE + +SECP256K1_INCLUDE_DIR=$(pwd)/third_party/secp256k1/include +OPENSSL_DIR=$(pwd)/third_party/crypto/ +LZ4_INCLUDE_DIR=$(pwd)/third_party/lz4/include if [ $ARCH == "arm" ] then ABI="armeabi-v7a" + SODIUM_INCLUDE_DIR=$(pwd)/third_party/libsodium/libsodium-android-armv7-a/include + SODIUM_LIBRARY_RELEASE=$(pwd)/third_party/libsodium/libsodium-android-armv7-a/lib/libsodium.a + SECP256K1_LIBRARY=$(pwd)/third_party/secp256k1/armv7/libsecp256k1.a + BLST_LIBRARY=$(pwd)/third_party/blst/armv7/libblst.a + LZ4_LIBRARY=$(pwd)/third_party/lz4/armv7/liblz4.a elif [ $ARCH == "x86" ] then ABI=$ARCH + SODIUM_INCLUDE_DIR=$(pwd)/third_party/libsodium/libsodium-android-i686/include + SODIUM_LIBRARY_RELEASE=$(pwd)/third_party/libsodium/libsodium-android-i686/lib/libsodium.a + SECP256K1_LIBRARY=$(pwd)/third_party/secp256k1/i686/libsecp256k1.a + BLST_LIBRARY=$(pwd)/third_party/blst/i686/libblst.a + LZ4_LIBRARY=$(pwd)/third_party/lz4/i686/liblz4.a + TARGET=i686-linux-android21 elif [ $ARCH == "x86_64" ] then ABI=$ARCH + SODIUM_INCLUDE_DIR=$(pwd)/third_party/libsodium/libsodium-android-westmere/include + SODIUM_LIBRARY_RELEASE=$(pwd)/third_party/libsodium/libsodium-android-westmere/lib/libsodium.a + SECP256K1_LIBRARY=$(pwd)/third_party/secp256k1/x86-64/libsecp256k1.a + BLST_LIBRARY=$(pwd)/third_party/blst/x86-64/libblst.a + LZ4_LIBRARY=$(pwd)/third_party/lz4/x86-64/liblz4.a elif [ $ARCH == "arm64" ] then ABI="arm64-v8a" + SODIUM_INCLUDE_DIR=$(pwd)/third_party/libsodium/libsodium-android-armv8-a/include + SODIUM_LIBRARY_RELEASE=$(pwd)/third_party/libsodium/libsodium-android-armv8-a/lib/libsodium.a + SECP256K1_LIBRARY=$(pwd)/third_party/secp256k1/armv8/libsecp256k1.a + BLST_LIBRARY=$(pwd)/third_party/blst/armv8/libblst.a + LZ4_LIBRARY=$(pwd)/third_party/lz4/armv8/liblz4.a fi +ORIG_ARCH=$ARCH ARCH=$ABI -echo $ABI mkdir -p build-$ARCH cd build-$ARCH +cmake .. -GNinja \ +-DTON_ONLY_TONLIB=ON \ +-DTON_ARCH="" \ +-DANDROID_ABI=x86 \ +-DANDROID_PLATFORM=android-32 \ +-DANDROID_NDK=${ANDROID_NDK_ROOT} \ +-DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake \ +-DCMAKE_BUILD_TYPE=Release \ +-DANDROID_ABI=${ABI} \ +-DOPENSSL_ROOT_DIR=${OPENSSL_DIR}/${ORIG_ARCH} \ +-DSECP256K1_INCLUDE_DIR=${SECP256K1_INCLUDE_DIR} \ +-DSECP256K1_LIBRARY=${SECP256K1_LIBRARY} \ +-DLZ4_FOUND=1 \ +-DLZ4_INCLUDE_DIRS=${LZ4_INCLUDE_DIR} \ +-DLZ4_LIBRARIES=${LZ4_LIBRARY} \ +-DSODIUM_FOUND=1 \ +-DSODIUM_INCLUDE_DIR=${SODIUM_INCLUDE_DIR} \ +-DSODIUM_LIBRARY_RELEASE=${SODIUM_LIBRARY_RELEASE} \ +-DSODIUM_USE_STATIC_LIBS=1 \ +-DBLST_LIB=${BLST_LIBRARY} || exit 1 -cmake .. -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake -DCMAKE_BUILD_TYPE=Release -GNinja -DANDROID_ABI=${ABI} -DOPENSSL_ROOT_DIR=${OPENSSL_DIR}/${ARCH} -DTON_ARCH="" -DTON_ONLY_TONLIB=ON || exit 1 ninja native-lib || exit 1 popd +$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip build-$ARCH/libnative-lib.so + mkdir -p libs/$ARCH/ cp build-$ARCH/libnative-lib.so* libs/$ARCH/ - - diff --git a/example/android/export.sh b/example/android/export.sh index 26367b9c..8f4e9162 100755 --- a/example/android/export.sh +++ b/example/android/export.sh @@ -2,7 +2,7 @@ pushd . mkdir -p build_native cd build_native -cmake -DTON_ONLY_TONLIB=ON .. || exit 1 +cmake -DTON_ONLY_TONLIB=ON -DBUILD_SHARED_LIBS=OFF .. || exit 1 cmake --build . --target prepare_cross_compiling || exit 2 #cmake --build . --target tl_generate_java || exit 1 popd diff --git a/example/android/third_party/blst/armv7/libblst.a b/example/android/third_party/blst/armv7/libblst.a new file mode 100644 index 00000000..d9e8ff8d Binary files /dev/null and b/example/android/third_party/blst/armv7/libblst.a differ diff --git a/example/android/third_party/blst/armv8/libblst.a b/example/android/third_party/blst/armv8/libblst.a new file mode 100644 index 00000000..351f7f65 Binary files /dev/null and b/example/android/third_party/blst/armv8/libblst.a differ diff --git a/example/android/third_party/blst/i686/libblst.a b/example/android/third_party/blst/i686/libblst.a new file mode 100644 index 00000000..666954b2 Binary files /dev/null and b/example/android/third_party/blst/i686/libblst.a differ diff --git a/example/android/third_party/blst/x86-64/libblst.a b/example/android/third_party/blst/x86-64/libblst.a new file mode 100644 index 00000000..4046ed9e Binary files /dev/null and b/example/android/third_party/blst/x86-64/libblst.a differ diff --git a/example/android/third_party/libsodium/build.sh b/example/android/third_party/libsodium/build.sh new file mode 100644 index 00000000..4eca044c --- /dev/null +++ b/example/android/third_party/libsodium/build.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +rm -rf libsodium-1.0.18 libsodium-1.0.18.tar.gz + +export ANDROID_NDK_ROOT=../../../../android-ndk-r25b +export NDK_PLATFORM="android-32" +export OPENSSL_DIR=../crypto +export LIBSODIUM_FULL_BUILD=1 +export CC= +export CXX= + +wget https://download.libsodium.org/libsodium/releases/libsodium-1.0.18.tar.gz +tar -xvf libsodium-1.0.18.tar.gz + +cd libsodium-1.0.18 +#./autogen.sh -s + +./dist-build/android-x86.sh +cp -R libsodium-android-i686 .. + +./dist-build/android-x86_64.sh +cp -R libsodium-android-westmere .. + +./dist-build/android-armv7-a.sh +cp -R libsodium-android-armv7-a .. + +./dist-build/android-armv8-a.sh +cp -R libsodium-android-armv8-a .. + +#./dist-build/android-aar.sh + diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium.h new file mode 100644 index 00000000..295f911c --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium.h @@ -0,0 +1,69 @@ + +#ifndef sodium_H +#define sodium_H + +#include "sodium/version.h" + +#include "sodium/core.h" +#include "sodium/crypto_aead_aes256gcm.h" +#include "sodium/crypto_aead_chacha20poly1305.h" +#include "sodium/crypto_aead_xchacha20poly1305.h" +#include "sodium/crypto_auth.h" +#include "sodium/crypto_auth_hmacsha256.h" +#include "sodium/crypto_auth_hmacsha512.h" +#include "sodium/crypto_auth_hmacsha512256.h" +#include "sodium/crypto_box.h" +#include "sodium/crypto_box_curve25519xsalsa20poly1305.h" +#include "sodium/crypto_core_hsalsa20.h" +#include "sodium/crypto_core_hchacha20.h" +#include "sodium/crypto_core_salsa20.h" +#include "sodium/crypto_core_salsa2012.h" +#include "sodium/crypto_core_salsa208.h" +#include "sodium/crypto_generichash.h" +#include "sodium/crypto_generichash_blake2b.h" +#include "sodium/crypto_hash.h" +#include "sodium/crypto_hash_sha256.h" +#include "sodium/crypto_hash_sha512.h" +#include "sodium/crypto_kdf.h" +#include "sodium/crypto_kdf_blake2b.h" +#include "sodium/crypto_kx.h" +#include "sodium/crypto_onetimeauth.h" +#include "sodium/crypto_onetimeauth_poly1305.h" +#include "sodium/crypto_pwhash.h" +#include "sodium/crypto_pwhash_argon2i.h" +#include "sodium/crypto_scalarmult.h" +#include "sodium/crypto_scalarmult_curve25519.h" +#include "sodium/crypto_secretbox.h" +#include "sodium/crypto_secretbox_xsalsa20poly1305.h" +#include "sodium/crypto_secretstream_xchacha20poly1305.h" +#include "sodium/crypto_shorthash.h" +#include "sodium/crypto_shorthash_siphash24.h" +#include "sodium/crypto_sign.h" +#include "sodium/crypto_sign_ed25519.h" +#include "sodium/crypto_stream.h" +#include "sodium/crypto_stream_chacha20.h" +#include "sodium/crypto_stream_salsa20.h" +#include "sodium/crypto_stream_xsalsa20.h" +#include "sodium/crypto_verify_16.h" +#include "sodium/crypto_verify_32.h" +#include "sodium/crypto_verify_64.h" +#include "sodium/randombytes.h" +#include "sodium/randombytes_internal_random.h" +#include "sodium/randombytes_sysrandom.h" +#include "sodium/runtime.h" +#include "sodium/utils.h" + +#ifndef SODIUM_LIBRARY_MINIMAL +# include "sodium/crypto_box_curve25519xchacha20poly1305.h" +# include "sodium/crypto_core_ed25519.h" +# include "sodium/crypto_core_ristretto255.h" +# include "sodium/crypto_scalarmult_ed25519.h" +# include "sodium/crypto_scalarmult_ristretto255.h" +# include "sodium/crypto_secretbox_xchacha20poly1305.h" +# include "sodium/crypto_pwhash_scryptsalsa208sha256.h" +# include "sodium/crypto_stream_salsa2012.h" +# include "sodium/crypto_stream_salsa208.h" +# include "sodium/crypto_stream_xchacha20.h" +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/core.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/core.h new file mode 100644 index 00000000..dd088d2c --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/core.h @@ -0,0 +1,28 @@ + +#ifndef sodium_core_H +#define sodium_core_H + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +SODIUM_EXPORT +int sodium_init(void) + __attribute__ ((warn_unused_result)); + +/* ---- */ + +SODIUM_EXPORT +int sodium_set_misuse_handler(void (*handler)(void)); + +SODIUM_EXPORT +void sodium_misuse(void) + __attribute__ ((noreturn)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_aead_aes256gcm.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_aead_aes256gcm.h new file mode 100644 index 00000000..9baeb3f1 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_aead_aes256gcm.h @@ -0,0 +1,179 @@ +#ifndef crypto_aead_aes256gcm_H +#define crypto_aead_aes256gcm_H + +/* + * WARNING: Despite being the most popular AEAD construction due to its + * use in TLS, safely using AES-GCM in a different context is tricky. + * + * No more than ~ 350 GB of input data should be encrypted with a given key. + * This is for ~ 16 KB messages -- Actual figures vary according to + * message sizes. + * + * In addition, nonces are short and repeated nonces would totally destroy + * the security of this scheme. + * + * Nonces should thus come from atomic counters, which can be difficult to + * set up in a distributed environment. + * + * Unless you absolutely need AES-GCM, use crypto_aead_xchacha20poly1305_ietf_*() + * instead. It doesn't have any of these limitations. + * Or, if you don't need to authenticate additional data, just stick to + * crypto_secretbox(). + */ + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +SODIUM_EXPORT +int crypto_aead_aes256gcm_is_available(void); + +#define crypto_aead_aes256gcm_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_keybytes(void); + +#define crypto_aead_aes256gcm_NSECBYTES 0U +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_nsecbytes(void); + +#define crypto_aead_aes256gcm_NPUBBYTES 12U +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_npubbytes(void); + +#define crypto_aead_aes256gcm_ABYTES 16U +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_abytes(void); + +#define crypto_aead_aes256gcm_MESSAGEBYTES_MAX \ + SODIUM_MIN(SODIUM_SIZE_MAX - crypto_aead_aes256gcm_ABYTES, \ + (16ULL * ((1ULL << 32) - 2ULL))) +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_messagebytes_max(void); + +typedef struct CRYPTO_ALIGN(16) crypto_aead_aes256gcm_state_ { + unsigned char opaque[512]; +} crypto_aead_aes256gcm_state; + +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_statebytes(void); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_encrypt(unsigned char *c, + unsigned long long *clen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_decrypt(unsigned char *m, + unsigned long long *mlen_p, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(4, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_encrypt_detached(unsigned char *c, + unsigned char *mac, + unsigned long long *maclen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 9, 10))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_decrypt_detached(unsigned char *m, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *mac, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5, 8, 9))); + +/* -- Precomputation interface -- */ + +SODIUM_EXPORT +int crypto_aead_aes256gcm_beforenm(crypto_aead_aes256gcm_state *ctx_, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_encrypt_afternm(unsigned char *c, + unsigned long long *clen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const crypto_aead_aes256gcm_state *ctx_) + __attribute__ ((nonnull(1, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_decrypt_afternm(unsigned char *m, + unsigned long long *mlen_p, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const crypto_aead_aes256gcm_state *ctx_) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(4, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_encrypt_detached_afternm(unsigned char *c, + unsigned char *mac, + unsigned long long *maclen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const crypto_aead_aes256gcm_state *ctx_) + __attribute__ ((nonnull(1, 2, 9, 10))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_decrypt_detached_afternm(unsigned char *m, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *mac, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const crypto_aead_aes256gcm_state *ctx_) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5, 8, 9))); + +SODIUM_EXPORT +void crypto_aead_aes256gcm_keygen(unsigned char k[crypto_aead_aes256gcm_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_aead_chacha20poly1305.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_aead_chacha20poly1305.h new file mode 100644 index 00000000..5d671df1 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_aead_chacha20poly1305.h @@ -0,0 +1,180 @@ +#ifndef crypto_aead_chacha20poly1305_H +#define crypto_aead_chacha20poly1305_H + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +/* -- IETF ChaCha20-Poly1305 construction with a 96-bit nonce and a 32-bit internal counter -- */ + +#define crypto_aead_chacha20poly1305_ietf_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_ietf_keybytes(void); + +#define crypto_aead_chacha20poly1305_ietf_NSECBYTES 0U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_ietf_nsecbytes(void); + +#define crypto_aead_chacha20poly1305_ietf_NPUBBYTES 12U + +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_ietf_npubbytes(void); + +#define crypto_aead_chacha20poly1305_ietf_ABYTES 16U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_ietf_abytes(void); + +#define crypto_aead_chacha20poly1305_ietf_MESSAGEBYTES_MAX \ + SODIUM_MIN(SODIUM_SIZE_MAX - crypto_aead_chacha20poly1305_ietf_ABYTES, \ + (64ULL * ((1ULL << 32) - 1ULL))) +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_ietf_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_ietf_encrypt(unsigned char *c, + unsigned long long *clen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_ietf_decrypt(unsigned char *m, + unsigned long long *mlen_p, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(4, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_ietf_encrypt_detached(unsigned char *c, + unsigned char *mac, + unsigned long long *maclen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 9, 10))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_ietf_decrypt_detached(unsigned char *m, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *mac, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5, 8, 9))); + +SODIUM_EXPORT +void crypto_aead_chacha20poly1305_ietf_keygen(unsigned char k[crypto_aead_chacha20poly1305_ietf_KEYBYTES]) + __attribute__ ((nonnull)); + +/* -- Original ChaCha20-Poly1305 construction with a 64-bit nonce and a 64-bit internal counter -- */ + +#define crypto_aead_chacha20poly1305_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_keybytes(void); + +#define crypto_aead_chacha20poly1305_NSECBYTES 0U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_nsecbytes(void); + +#define crypto_aead_chacha20poly1305_NPUBBYTES 8U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_npubbytes(void); + +#define crypto_aead_chacha20poly1305_ABYTES 16U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_abytes(void); + +#define crypto_aead_chacha20poly1305_MESSAGEBYTES_MAX \ + (SODIUM_SIZE_MAX - crypto_aead_chacha20poly1305_ABYTES) +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_encrypt(unsigned char *c, + unsigned long long *clen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_decrypt(unsigned char *m, + unsigned long long *mlen_p, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(4, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_encrypt_detached(unsigned char *c, + unsigned char *mac, + unsigned long long *maclen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 9, 10))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_decrypt_detached(unsigned char *m, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *mac, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5, 8, 9))); + +SODIUM_EXPORT +void crypto_aead_chacha20poly1305_keygen(unsigned char k[crypto_aead_chacha20poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +/* Aliases */ + +#define crypto_aead_chacha20poly1305_IETF_KEYBYTES crypto_aead_chacha20poly1305_ietf_KEYBYTES +#define crypto_aead_chacha20poly1305_IETF_NSECBYTES crypto_aead_chacha20poly1305_ietf_NSECBYTES +#define crypto_aead_chacha20poly1305_IETF_NPUBBYTES crypto_aead_chacha20poly1305_ietf_NPUBBYTES +#define crypto_aead_chacha20poly1305_IETF_ABYTES crypto_aead_chacha20poly1305_ietf_ABYTES +#define crypto_aead_chacha20poly1305_IETF_MESSAGEBYTES_MAX crypto_aead_chacha20poly1305_ietf_MESSAGEBYTES_MAX + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_aead_xchacha20poly1305.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_aead_xchacha20poly1305.h new file mode 100644 index 00000000..6643b0cb --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_aead_xchacha20poly1305.h @@ -0,0 +1,100 @@ +#ifndef crypto_aead_xchacha20poly1305_H +#define crypto_aead_xchacha20poly1305_H + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_aead_xchacha20poly1305_ietf_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_aead_xchacha20poly1305_ietf_keybytes(void); + +#define crypto_aead_xchacha20poly1305_ietf_NSECBYTES 0U +SODIUM_EXPORT +size_t crypto_aead_xchacha20poly1305_ietf_nsecbytes(void); + +#define crypto_aead_xchacha20poly1305_ietf_NPUBBYTES 24U +SODIUM_EXPORT +size_t crypto_aead_xchacha20poly1305_ietf_npubbytes(void); + +#define crypto_aead_xchacha20poly1305_ietf_ABYTES 16U +SODIUM_EXPORT +size_t crypto_aead_xchacha20poly1305_ietf_abytes(void); + +#define crypto_aead_xchacha20poly1305_ietf_MESSAGEBYTES_MAX \ + (SODIUM_SIZE_MAX - crypto_aead_xchacha20poly1305_ietf_ABYTES) +SODIUM_EXPORT +size_t crypto_aead_xchacha20poly1305_ietf_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_aead_xchacha20poly1305_ietf_encrypt(unsigned char *c, + unsigned long long *clen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_xchacha20poly1305_ietf_decrypt(unsigned char *m, + unsigned long long *mlen_p, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(4, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_xchacha20poly1305_ietf_encrypt_detached(unsigned char *c, + unsigned char *mac, + unsigned long long *maclen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 9, 10))); + +SODIUM_EXPORT +int crypto_aead_xchacha20poly1305_ietf_decrypt_detached(unsigned char *m, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *mac, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5, 8, 9))); + +SODIUM_EXPORT +void crypto_aead_xchacha20poly1305_ietf_keygen(unsigned char k[crypto_aead_xchacha20poly1305_ietf_KEYBYTES]) + __attribute__ ((nonnull)); + +/* Aliases */ + +#define crypto_aead_xchacha20poly1305_IETF_KEYBYTES crypto_aead_xchacha20poly1305_ietf_KEYBYTES +#define crypto_aead_xchacha20poly1305_IETF_NSECBYTES crypto_aead_xchacha20poly1305_ietf_NSECBYTES +#define crypto_aead_xchacha20poly1305_IETF_NPUBBYTES crypto_aead_xchacha20poly1305_ietf_NPUBBYTES +#define crypto_aead_xchacha20poly1305_IETF_ABYTES crypto_aead_xchacha20poly1305_ietf_ABYTES +#define crypto_aead_xchacha20poly1305_IETF_MESSAGEBYTES_MAX crypto_aead_xchacha20poly1305_ietf_MESSAGEBYTES_MAX + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_auth.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_auth.h new file mode 100644 index 00000000..540aee0e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_auth.h @@ -0,0 +1,46 @@ +#ifndef crypto_auth_H +#define crypto_auth_H + +#include + +#include "crypto_auth_hmacsha512256.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_auth_BYTES crypto_auth_hmacsha512256_BYTES +SODIUM_EXPORT +size_t crypto_auth_bytes(void); + +#define crypto_auth_KEYBYTES crypto_auth_hmacsha512256_KEYBYTES +SODIUM_EXPORT +size_t crypto_auth_keybytes(void); + +#define crypto_auth_PRIMITIVE "hmacsha512256" +SODIUM_EXPORT +const char *crypto_auth_primitive(void); + +SODIUM_EXPORT +int crypto_auth(unsigned char *out, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_auth_verify(const unsigned char *h, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +void crypto_auth_keygen(unsigned char k[crypto_auth_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_auth_hmacsha256.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_auth_hmacsha256.h new file mode 100644 index 00000000..3da864c7 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_auth_hmacsha256.h @@ -0,0 +1,70 @@ +#ifndef crypto_auth_hmacsha256_H +#define crypto_auth_hmacsha256_H + +#include +#include "crypto_hash_sha256.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_auth_hmacsha256_BYTES 32U +SODIUM_EXPORT +size_t crypto_auth_hmacsha256_bytes(void); + +#define crypto_auth_hmacsha256_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_auth_hmacsha256_keybytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha256(unsigned char *out, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_auth_hmacsha256_verify(const unsigned char *h, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +/* ------------------------------------------------------------------------- */ + +typedef struct crypto_auth_hmacsha256_state { + crypto_hash_sha256_state ictx; + crypto_hash_sha256_state octx; +} crypto_auth_hmacsha256_state; + +SODIUM_EXPORT +size_t crypto_auth_hmacsha256_statebytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha256_init(crypto_auth_hmacsha256_state *state, + const unsigned char *key, + size_t keylen) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_auth_hmacsha256_update(crypto_auth_hmacsha256_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_auth_hmacsha256_final(crypto_auth_hmacsha256_state *state, + unsigned char *out) __attribute__ ((nonnull)); + + +SODIUM_EXPORT +void crypto_auth_hmacsha256_keygen(unsigned char k[crypto_auth_hmacsha256_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_auth_hmacsha512.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_auth_hmacsha512.h new file mode 100644 index 00000000..d992cb81 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_auth_hmacsha512.h @@ -0,0 +1,68 @@ +#ifndef crypto_auth_hmacsha512_H +#define crypto_auth_hmacsha512_H + +#include +#include "crypto_hash_sha512.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_auth_hmacsha512_BYTES 64U +SODIUM_EXPORT +size_t crypto_auth_hmacsha512_bytes(void); + +#define crypto_auth_hmacsha512_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_auth_hmacsha512_keybytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha512(unsigned char *out, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_auth_hmacsha512_verify(const unsigned char *h, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +/* ------------------------------------------------------------------------- */ + +typedef struct crypto_auth_hmacsha512_state { + crypto_hash_sha512_state ictx; + crypto_hash_sha512_state octx; +} crypto_auth_hmacsha512_state; + +SODIUM_EXPORT +size_t crypto_auth_hmacsha512_statebytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha512_init(crypto_auth_hmacsha512_state *state, + const unsigned char *key, + size_t keylen) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_auth_hmacsha512_update(crypto_auth_hmacsha512_state *state, + const unsigned char *in, + unsigned long long inlen) __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_auth_hmacsha512_final(crypto_auth_hmacsha512_state *state, + unsigned char *out) __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_auth_hmacsha512_keygen(unsigned char k[crypto_auth_hmacsha512_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_auth_hmacsha512256.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_auth_hmacsha512256.h new file mode 100644 index 00000000..3fb52638 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_auth_hmacsha512256.h @@ -0,0 +1,65 @@ +#ifndef crypto_auth_hmacsha512256_H +#define crypto_auth_hmacsha512256_H + +#include +#include "crypto_auth_hmacsha512.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_auth_hmacsha512256_BYTES 32U +SODIUM_EXPORT +size_t crypto_auth_hmacsha512256_bytes(void); + +#define crypto_auth_hmacsha512256_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_auth_hmacsha512256_keybytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha512256(unsigned char *out, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_auth_hmacsha512256_verify(const unsigned char *h, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +/* ------------------------------------------------------------------------- */ + +typedef crypto_auth_hmacsha512_state crypto_auth_hmacsha512256_state; + +SODIUM_EXPORT +size_t crypto_auth_hmacsha512256_statebytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha512256_init(crypto_auth_hmacsha512256_state *state, + const unsigned char *key, + size_t keylen) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_auth_hmacsha512256_update(crypto_auth_hmacsha512256_state *state, + const unsigned char *in, + unsigned long long inlen) __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_auth_hmacsha512256_final(crypto_auth_hmacsha512256_state *state, + unsigned char *out) __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_auth_hmacsha512256_keygen(unsigned char k[crypto_auth_hmacsha512256_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_box.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_box.h new file mode 100644 index 00000000..e060dd29 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_box.h @@ -0,0 +1,177 @@ +#ifndef crypto_box_H +#define crypto_box_H + +/* + * THREAD SAFETY: crypto_box_keypair() is thread-safe, + * provided that sodium_init() was called before. + * + * Other functions are always thread-safe. + */ + +#include + +#include "crypto_box_curve25519xsalsa20poly1305.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_box_SEEDBYTES crypto_box_curve25519xsalsa20poly1305_SEEDBYTES +SODIUM_EXPORT +size_t crypto_box_seedbytes(void); + +#define crypto_box_PUBLICKEYBYTES crypto_box_curve25519xsalsa20poly1305_PUBLICKEYBYTES +SODIUM_EXPORT +size_t crypto_box_publickeybytes(void); + +#define crypto_box_SECRETKEYBYTES crypto_box_curve25519xsalsa20poly1305_SECRETKEYBYTES +SODIUM_EXPORT +size_t crypto_box_secretkeybytes(void); + +#define crypto_box_NONCEBYTES crypto_box_curve25519xsalsa20poly1305_NONCEBYTES +SODIUM_EXPORT +size_t crypto_box_noncebytes(void); + +#define crypto_box_MACBYTES crypto_box_curve25519xsalsa20poly1305_MACBYTES +SODIUM_EXPORT +size_t crypto_box_macbytes(void); + +#define crypto_box_MESSAGEBYTES_MAX crypto_box_curve25519xsalsa20poly1305_MESSAGEBYTES_MAX +SODIUM_EXPORT +size_t crypto_box_messagebytes_max(void); + +#define crypto_box_PRIMITIVE "curve25519xsalsa20poly1305" +SODIUM_EXPORT +const char *crypto_box_primitive(void); + +SODIUM_EXPORT +int crypto_box_seed_keypair(unsigned char *pk, unsigned char *sk, + const unsigned char *seed) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_keypair(unsigned char *pk, unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_easy(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *pk, const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_open_easy(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *pk, const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_detached(unsigned char *c, unsigned char *mac, + const unsigned char *m, unsigned long long mlen, + const unsigned char *n, const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 2, 5, 6, 7))); + +SODIUM_EXPORT +int crypto_box_open_detached(unsigned char *m, const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6, 7))); + +/* -- Precomputation interface -- */ + +#define crypto_box_BEFORENMBYTES crypto_box_curve25519xsalsa20poly1305_BEFORENMBYTES +SODIUM_EXPORT +size_t crypto_box_beforenmbytes(void); + +SODIUM_EXPORT +int crypto_box_beforenm(unsigned char *k, const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_easy_afternm(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_box_open_easy_afternm(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +SODIUM_EXPORT +int crypto_box_detached_afternm(unsigned char *c, unsigned char *mac, + const unsigned char *m, unsigned long long mlen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull(1, 2, 5, 6))); + +SODIUM_EXPORT +int crypto_box_open_detached_afternm(unsigned char *m, const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6))); + +/* -- Ephemeral SK interface -- */ + +#define crypto_box_SEALBYTES (crypto_box_PUBLICKEYBYTES + crypto_box_MACBYTES) +SODIUM_EXPORT +size_t crypto_box_sealbytes(void); + +SODIUM_EXPORT +int crypto_box_seal(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *pk) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_box_seal_open(unsigned char *m, const unsigned char *c, + unsigned long long clen, + const unsigned char *pk, const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +/* -- NaCl compatibility interface ; Requires padding -- */ + +#define crypto_box_ZEROBYTES crypto_box_curve25519xsalsa20poly1305_ZEROBYTES +SODIUM_EXPORT +size_t crypto_box_zerobytes(void); + +#define crypto_box_BOXZEROBYTES crypto_box_curve25519xsalsa20poly1305_BOXZEROBYTES +SODIUM_EXPORT +size_t crypto_box_boxzerobytes(void); + +SODIUM_EXPORT +int crypto_box(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *pk, const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_open(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *pk, const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_afternm(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_box_open_afternm(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_box_curve25519xchacha20poly1305.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_box_curve25519xchacha20poly1305.h new file mode 100644 index 00000000..26a3d31e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_box_curve25519xchacha20poly1305.h @@ -0,0 +1,164 @@ + +#ifndef crypto_box_curve25519xchacha20poly1305_H +#define crypto_box_curve25519xchacha20poly1305_H + +#include +#include "crypto_stream_xchacha20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_box_curve25519xchacha20poly1305_SEEDBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_seedbytes(void); + +#define crypto_box_curve25519xchacha20poly1305_PUBLICKEYBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_publickeybytes(void); + +#define crypto_box_curve25519xchacha20poly1305_SECRETKEYBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_secretkeybytes(void); + +#define crypto_box_curve25519xchacha20poly1305_BEFORENMBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_beforenmbytes(void); + +#define crypto_box_curve25519xchacha20poly1305_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_noncebytes(void); + +#define crypto_box_curve25519xchacha20poly1305_MACBYTES 16U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_macbytes(void); + +#define crypto_box_curve25519xchacha20poly1305_MESSAGEBYTES_MAX \ + (crypto_stream_xchacha20_MESSAGEBYTES_MAX - crypto_box_curve25519xchacha20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_seed_keypair(unsigned char *pk, + unsigned char *sk, + const unsigned char *seed) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_keypair(unsigned char *pk, + unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_easy(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_open_easy(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_detached(unsigned char *c, + unsigned char *mac, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 2, 5, 6, 7))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_open_detached(unsigned char *m, + const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6, 7))); + +/* -- Precomputation interface -- */ + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_beforenm(unsigned char *k, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_easy_afternm(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_open_easy_afternm(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_detached_afternm(unsigned char *c, + unsigned char *mac, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 5, 6))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_open_detached_afternm(unsigned char *m, + const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6))); + +/* -- Ephemeral SK interface -- */ + +#define crypto_box_curve25519xchacha20poly1305_SEALBYTES \ + (crypto_box_curve25519xchacha20poly1305_PUBLICKEYBYTES + \ + crypto_box_curve25519xchacha20poly1305_MACBYTES) + +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_sealbytes(void); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_seal(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *pk) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_seal_open(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_box_curve25519xsalsa20poly1305.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_box_curve25519xsalsa20poly1305.h new file mode 100644 index 00000000..e733f499 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_box_curve25519xsalsa20poly1305.h @@ -0,0 +1,112 @@ +#ifndef crypto_box_curve25519xsalsa20poly1305_H +#define crypto_box_curve25519xsalsa20poly1305_H + +#include +#include "crypto_stream_xsalsa20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_box_curve25519xsalsa20poly1305_SEEDBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_seedbytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_PUBLICKEYBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_publickeybytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_SECRETKEYBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_secretkeybytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_BEFORENMBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_beforenmbytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_noncebytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_MACBYTES 16U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_macbytes(void); + +/* Only for the libsodium API - The NaCl compatibility API would require BOXZEROBYTES extra bytes */ +#define crypto_box_curve25519xsalsa20poly1305_MESSAGEBYTES_MAX \ + (crypto_stream_xsalsa20_MESSAGEBYTES_MAX - crypto_box_curve25519xsalsa20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_seed_keypair(unsigned char *pk, + unsigned char *sk, + const unsigned char *seed) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_keypair(unsigned char *pk, + unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_beforenm(unsigned char *k, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +/* -- NaCl compatibility interface ; Requires padding -- */ + +#define crypto_box_curve25519xsalsa20poly1305_BOXZEROBYTES 16U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_boxzerobytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_ZEROBYTES \ + (crypto_box_curve25519xsalsa20poly1305_BOXZEROBYTES + \ + crypto_box_curve25519xsalsa20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_zerobytes(void); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_open(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_afternm(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_open_afternm(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_core_ed25519.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_core_ed25519.h new file mode 100644 index 00000000..3eae00c4 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_core_ed25519.h @@ -0,0 +1,100 @@ +#ifndef crypto_core_ed25519_H +#define crypto_core_ed25519_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_ed25519_BYTES 32 +SODIUM_EXPORT +size_t crypto_core_ed25519_bytes(void); + +#define crypto_core_ed25519_UNIFORMBYTES 32 +SODIUM_EXPORT +size_t crypto_core_ed25519_uniformbytes(void); + +#define crypto_core_ed25519_HASHBYTES 64 +SODIUM_EXPORT +size_t crypto_core_ed25519_hashbytes(void); + +#define crypto_core_ed25519_SCALARBYTES 32 +SODIUM_EXPORT +size_t crypto_core_ed25519_scalarbytes(void); + +#define crypto_core_ed25519_NONREDUCEDSCALARBYTES 64 +SODIUM_EXPORT +size_t crypto_core_ed25519_nonreducedscalarbytes(void); + +SODIUM_EXPORT +int crypto_core_ed25519_is_valid_point(const unsigned char *p) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ed25519_add(unsigned char *r, + const unsigned char *p, const unsigned char *q) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ed25519_sub(unsigned char *r, + const unsigned char *p, const unsigned char *q) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ed25519_from_uniform(unsigned char *p, const unsigned char *r) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ed25519_from_hash(unsigned char *p, const unsigned char *h) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_random(unsigned char *p) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_random(unsigned char *r) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ed25519_scalar_invert(unsigned char *recip, const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_negate(unsigned char *neg, const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_complement(unsigned char *comp, const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_add(unsigned char *z, const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_sub(unsigned char *z, const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_mul(unsigned char *z, const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +/* + * The interval `s` is sampled from should be at least 317 bits to ensure almost + * uniformity of `r` over `L`. + */ +SODIUM_EXPORT +void crypto_core_ed25519_scalar_reduce(unsigned char *r, const unsigned char *s) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_core_hchacha20.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_core_hchacha20.h new file mode 100644 index 00000000..ece141b0 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_core_hchacha20.h @@ -0,0 +1,36 @@ +#ifndef crypto_core_hchacha20_H +#define crypto_core_hchacha20_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_hchacha20_OUTPUTBYTES 32U +SODIUM_EXPORT +size_t crypto_core_hchacha20_outputbytes(void); + +#define crypto_core_hchacha20_INPUTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_hchacha20_inputbytes(void); + +#define crypto_core_hchacha20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_core_hchacha20_keybytes(void); + +#define crypto_core_hchacha20_CONSTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_hchacha20_constbytes(void); + +SODIUM_EXPORT +int crypto_core_hchacha20(unsigned char *out, const unsigned char *in, + const unsigned char *k, const unsigned char *c) + __attribute__ ((nonnull(1, 2, 3))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_core_hsalsa20.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_core_hsalsa20.h new file mode 100644 index 00000000..4bf7a487 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_core_hsalsa20.h @@ -0,0 +1,36 @@ +#ifndef crypto_core_hsalsa20_H +#define crypto_core_hsalsa20_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_hsalsa20_OUTPUTBYTES 32U +SODIUM_EXPORT +size_t crypto_core_hsalsa20_outputbytes(void); + +#define crypto_core_hsalsa20_INPUTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_hsalsa20_inputbytes(void); + +#define crypto_core_hsalsa20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_core_hsalsa20_keybytes(void); + +#define crypto_core_hsalsa20_CONSTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_hsalsa20_constbytes(void); + +SODIUM_EXPORT +int crypto_core_hsalsa20(unsigned char *out, const unsigned char *in, + const unsigned char *k, const unsigned char *c) + __attribute__ ((nonnull(1, 2, 3))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_core_ristretto255.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_core_ristretto255.h new file mode 100644 index 00000000..f2820e55 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_core_ristretto255.h @@ -0,0 +1,100 @@ +#ifndef crypto_core_ristretto255_H +#define crypto_core_ristretto255_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_ristretto255_BYTES 32 +SODIUM_EXPORT +size_t crypto_core_ristretto255_bytes(void); + +#define crypto_core_ristretto255_HASHBYTES 64 +SODIUM_EXPORT +size_t crypto_core_ristretto255_hashbytes(void); + +#define crypto_core_ristretto255_SCALARBYTES 32 +SODIUM_EXPORT +size_t crypto_core_ristretto255_scalarbytes(void); + +#define crypto_core_ristretto255_NONREDUCEDSCALARBYTES 64 +SODIUM_EXPORT +size_t crypto_core_ristretto255_nonreducedscalarbytes(void); + +SODIUM_EXPORT +int crypto_core_ristretto255_is_valid_point(const unsigned char *p) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ristretto255_add(unsigned char *r, + const unsigned char *p, const unsigned char *q) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ristretto255_sub(unsigned char *r, + const unsigned char *p, const unsigned char *q) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ristretto255_from_hash(unsigned char *p, + const unsigned char *r) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_random(unsigned char *p) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_random(unsigned char *r) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ristretto255_scalar_invert(unsigned char *recip, + const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_negate(unsigned char *neg, + const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_complement(unsigned char *comp, + const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_add(unsigned char *z, + const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_sub(unsigned char *z, + const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_mul(unsigned char *z, + const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +/* + * The interval `s` is sampled from should be at least 317 bits to ensure almost + * uniformity of `r` over `L`. + */ +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_reduce(unsigned char *r, + const unsigned char *s) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_core_salsa20.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_core_salsa20.h new file mode 100644 index 00000000..bd79fd9f --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_core_salsa20.h @@ -0,0 +1,36 @@ +#ifndef crypto_core_salsa20_H +#define crypto_core_salsa20_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_salsa20_OUTPUTBYTES 64U +SODIUM_EXPORT +size_t crypto_core_salsa20_outputbytes(void); + +#define crypto_core_salsa20_INPUTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa20_inputbytes(void); + +#define crypto_core_salsa20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_core_salsa20_keybytes(void); + +#define crypto_core_salsa20_CONSTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa20_constbytes(void); + +SODIUM_EXPORT +int crypto_core_salsa20(unsigned char *out, const unsigned char *in, + const unsigned char *k, const unsigned char *c) + __attribute__ ((nonnull(1, 2, 3))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_core_salsa2012.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_core_salsa2012.h new file mode 100644 index 00000000..05957591 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_core_salsa2012.h @@ -0,0 +1,36 @@ +#ifndef crypto_core_salsa2012_H +#define crypto_core_salsa2012_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_salsa2012_OUTPUTBYTES 64U +SODIUM_EXPORT +size_t crypto_core_salsa2012_outputbytes(void); + +#define crypto_core_salsa2012_INPUTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa2012_inputbytes(void); + +#define crypto_core_salsa2012_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_core_salsa2012_keybytes(void); + +#define crypto_core_salsa2012_CONSTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa2012_constbytes(void); + +SODIUM_EXPORT +int crypto_core_salsa2012(unsigned char *out, const unsigned char *in, + const unsigned char *k, const unsigned char *c) + __attribute__ ((nonnull(1, 2, 3))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_core_salsa208.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_core_salsa208.h new file mode 100644 index 00000000..d2f216af --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_core_salsa208.h @@ -0,0 +1,40 @@ +#ifndef crypto_core_salsa208_H +#define crypto_core_salsa208_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_salsa208_OUTPUTBYTES 64U +SODIUM_EXPORT +size_t crypto_core_salsa208_outputbytes(void) + __attribute__ ((deprecated)); + +#define crypto_core_salsa208_INPUTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa208_inputbytes(void) + __attribute__ ((deprecated)); + +#define crypto_core_salsa208_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_core_salsa208_keybytes(void) + __attribute__ ((deprecated)); + +#define crypto_core_salsa208_CONSTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa208_constbytes(void) + __attribute__ ((deprecated)); + +SODIUM_EXPORT +int crypto_core_salsa208(unsigned char *out, const unsigned char *in, + const unsigned char *k, const unsigned char *c) + __attribute__ ((nonnull(1, 2, 3))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_generichash.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_generichash.h new file mode 100644 index 00000000..d897e5d2 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_generichash.h @@ -0,0 +1,84 @@ +#ifndef crypto_generichash_H +#define crypto_generichash_H + +#include + +#include "crypto_generichash_blake2b.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_generichash_BYTES_MIN crypto_generichash_blake2b_BYTES_MIN +SODIUM_EXPORT +size_t crypto_generichash_bytes_min(void); + +#define crypto_generichash_BYTES_MAX crypto_generichash_blake2b_BYTES_MAX +SODIUM_EXPORT +size_t crypto_generichash_bytes_max(void); + +#define crypto_generichash_BYTES crypto_generichash_blake2b_BYTES +SODIUM_EXPORT +size_t crypto_generichash_bytes(void); + +#define crypto_generichash_KEYBYTES_MIN crypto_generichash_blake2b_KEYBYTES_MIN +SODIUM_EXPORT +size_t crypto_generichash_keybytes_min(void); + +#define crypto_generichash_KEYBYTES_MAX crypto_generichash_blake2b_KEYBYTES_MAX +SODIUM_EXPORT +size_t crypto_generichash_keybytes_max(void); + +#define crypto_generichash_KEYBYTES crypto_generichash_blake2b_KEYBYTES +SODIUM_EXPORT +size_t crypto_generichash_keybytes(void); + +#define crypto_generichash_PRIMITIVE "blake2b" +SODIUM_EXPORT +const char *crypto_generichash_primitive(void); + +/* + * Important when writing bindings for other programming languages: + * the state address should be 64-bytes aligned. + */ +typedef crypto_generichash_blake2b_state crypto_generichash_state; + +SODIUM_EXPORT +size_t crypto_generichash_statebytes(void); + +SODIUM_EXPORT +int crypto_generichash(unsigned char *out, size_t outlen, + const unsigned char *in, unsigned long long inlen, + const unsigned char *key, size_t keylen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_init(crypto_generichash_state *state, + const unsigned char *key, + const size_t keylen, const size_t outlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_update(crypto_generichash_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_final(crypto_generichash_state *state, + unsigned char *out, const size_t outlen) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_generichash_keygen(unsigned char k[crypto_generichash_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_generichash_blake2b.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_generichash_blake2b.h new file mode 100644 index 00000000..fee9d8ad --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_generichash_blake2b.h @@ -0,0 +1,118 @@ +#ifndef crypto_generichash_blake2b_H +#define crypto_generichash_blake2b_H + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#if defined(__IBMC__) || defined(__SUNPRO_C) || defined(__SUNPRO_CC) +# pragma pack(1) +#else +# pragma pack(push, 1) +#endif + +typedef struct CRYPTO_ALIGN(64) crypto_generichash_blake2b_state { + unsigned char opaque[384]; +} crypto_generichash_blake2b_state; + +#if defined(__IBMC__) || defined(__SUNPRO_C) || defined(__SUNPRO_CC) +# pragma pack() +#else +# pragma pack(pop) +#endif + +#define crypto_generichash_blake2b_BYTES_MIN 16U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_bytes_min(void); + +#define crypto_generichash_blake2b_BYTES_MAX 64U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_bytes_max(void); + +#define crypto_generichash_blake2b_BYTES 32U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_bytes(void); + +#define crypto_generichash_blake2b_KEYBYTES_MIN 16U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_keybytes_min(void); + +#define crypto_generichash_blake2b_KEYBYTES_MAX 64U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_keybytes_max(void); + +#define crypto_generichash_blake2b_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_keybytes(void); + +#define crypto_generichash_blake2b_SALTBYTES 16U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_saltbytes(void); + +#define crypto_generichash_blake2b_PERSONALBYTES 16U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_personalbytes(void); + +SODIUM_EXPORT +size_t crypto_generichash_blake2b_statebytes(void); + +SODIUM_EXPORT +int crypto_generichash_blake2b(unsigned char *out, size_t outlen, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *key, size_t keylen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_blake2b_salt_personal(unsigned char *out, size_t outlen, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *key, + size_t keylen, + const unsigned char *salt, + const unsigned char *personal) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_blake2b_init(crypto_generichash_blake2b_state *state, + const unsigned char *key, + const size_t keylen, const size_t outlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_blake2b_init_salt_personal(crypto_generichash_blake2b_state *state, + const unsigned char *key, + const size_t keylen, const size_t outlen, + const unsigned char *salt, + const unsigned char *personal) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_blake2b_update(crypto_generichash_blake2b_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_blake2b_final(crypto_generichash_blake2b_state *state, + unsigned char *out, + const size_t outlen) __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_generichash_blake2b_keygen(unsigned char k[crypto_generichash_blake2b_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_hash.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_hash.h new file mode 100644 index 00000000..8752f9ca --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_hash.h @@ -0,0 +1,40 @@ +#ifndef crypto_hash_H +#define crypto_hash_H + +/* + * WARNING: Unless you absolutely need to use SHA512 for interoperatibility, + * purposes, you might want to consider crypto_generichash() instead. + * Unlike SHA512, crypto_generichash() is not vulnerable to length + * extension attacks. + */ + +#include + +#include "crypto_hash_sha512.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_hash_BYTES crypto_hash_sha512_BYTES +SODIUM_EXPORT +size_t crypto_hash_bytes(void); + +SODIUM_EXPORT +int crypto_hash(unsigned char *out, const unsigned char *in, + unsigned long long inlen) __attribute__ ((nonnull(1))); + +#define crypto_hash_PRIMITIVE "sha512" +SODIUM_EXPORT +const char *crypto_hash_primitive(void) + __attribute__ ((warn_unused_result)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_hash_sha256.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_hash_sha256.h new file mode 100644 index 00000000..b18217e1 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_hash_sha256.h @@ -0,0 +1,60 @@ +#ifndef crypto_hash_sha256_H +#define crypto_hash_sha256_H + +/* + * WARNING: Unless you absolutely need to use SHA256 for interoperatibility, + * purposes, you might want to consider crypto_generichash() instead. + * Unlike SHA256, crypto_generichash() is not vulnerable to length + * extension attacks. + */ + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef struct crypto_hash_sha256_state { + uint32_t state[8]; + uint64_t count; + uint8_t buf[64]; +} crypto_hash_sha256_state; + +SODIUM_EXPORT +size_t crypto_hash_sha256_statebytes(void); + +#define crypto_hash_sha256_BYTES 32U +SODIUM_EXPORT +size_t crypto_hash_sha256_bytes(void); + +SODIUM_EXPORT +int crypto_hash_sha256(unsigned char *out, const unsigned char *in, + unsigned long long inlen) __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_hash_sha256_init(crypto_hash_sha256_state *state) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_hash_sha256_update(crypto_hash_sha256_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_hash_sha256_final(crypto_hash_sha256_state *state, + unsigned char *out) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_hash_sha512.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_hash_sha512.h new file mode 100644 index 00000000..8efa7193 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_hash_sha512.h @@ -0,0 +1,60 @@ +#ifndef crypto_hash_sha512_H +#define crypto_hash_sha512_H + +/* + * WARNING: Unless you absolutely need to use SHA512 for interoperatibility, + * purposes, you might want to consider crypto_generichash() instead. + * Unlike SHA512, crypto_generichash() is not vulnerable to length + * extension attacks. + */ + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef struct crypto_hash_sha512_state { + uint64_t state[8]; + uint64_t count[2]; + uint8_t buf[128]; +} crypto_hash_sha512_state; + +SODIUM_EXPORT +size_t crypto_hash_sha512_statebytes(void); + +#define crypto_hash_sha512_BYTES 64U +SODIUM_EXPORT +size_t crypto_hash_sha512_bytes(void); + +SODIUM_EXPORT +int crypto_hash_sha512(unsigned char *out, const unsigned char *in, + unsigned long long inlen) __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_hash_sha512_init(crypto_hash_sha512_state *state) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_hash_sha512_update(crypto_hash_sha512_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_hash_sha512_final(crypto_hash_sha512_state *state, + unsigned char *out) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_kdf.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_kdf.h new file mode 100644 index 00000000..ac2fc618 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_kdf.h @@ -0,0 +1,53 @@ +#ifndef crypto_kdf_H +#define crypto_kdf_H + +#include +#include + +#include "crypto_kdf_blake2b.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_kdf_BYTES_MIN crypto_kdf_blake2b_BYTES_MIN +SODIUM_EXPORT +size_t crypto_kdf_bytes_min(void); + +#define crypto_kdf_BYTES_MAX crypto_kdf_blake2b_BYTES_MAX +SODIUM_EXPORT +size_t crypto_kdf_bytes_max(void); + +#define crypto_kdf_CONTEXTBYTES crypto_kdf_blake2b_CONTEXTBYTES +SODIUM_EXPORT +size_t crypto_kdf_contextbytes(void); + +#define crypto_kdf_KEYBYTES crypto_kdf_blake2b_KEYBYTES +SODIUM_EXPORT +size_t crypto_kdf_keybytes(void); + +#define crypto_kdf_PRIMITIVE "blake2b" +SODIUM_EXPORT +const char *crypto_kdf_primitive(void) + __attribute__ ((warn_unused_result)); + +SODIUM_EXPORT +int crypto_kdf_derive_from_key(unsigned char *subkey, size_t subkey_len, + uint64_t subkey_id, + const char ctx[crypto_kdf_CONTEXTBYTES], + const unsigned char key[crypto_kdf_KEYBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_kdf_keygen(unsigned char k[crypto_kdf_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_kdf_blake2b.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_kdf_blake2b.h new file mode 100644 index 00000000..3ae47dd3 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_kdf_blake2b.h @@ -0,0 +1,44 @@ +#ifndef crypto_kdf_blake2b_H +#define crypto_kdf_blake2b_H + +#include +#include + +#include "crypto_kdf_blake2b.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_kdf_blake2b_BYTES_MIN 16 +SODIUM_EXPORT +size_t crypto_kdf_blake2b_bytes_min(void); + +#define crypto_kdf_blake2b_BYTES_MAX 64 +SODIUM_EXPORT +size_t crypto_kdf_blake2b_bytes_max(void); + +#define crypto_kdf_blake2b_CONTEXTBYTES 8 +SODIUM_EXPORT +size_t crypto_kdf_blake2b_contextbytes(void); + +#define crypto_kdf_blake2b_KEYBYTES 32 +SODIUM_EXPORT +size_t crypto_kdf_blake2b_keybytes(void); + +SODIUM_EXPORT +int crypto_kdf_blake2b_derive_from_key(unsigned char *subkey, size_t subkey_len, + uint64_t subkey_id, + const char ctx[crypto_kdf_blake2b_CONTEXTBYTES], + const unsigned char key[crypto_kdf_blake2b_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_kx.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_kx.h new file mode 100644 index 00000000..347132c3 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_kx.h @@ -0,0 +1,66 @@ +#ifndef crypto_kx_H +#define crypto_kx_H + +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_kx_PUBLICKEYBYTES 32 +SODIUM_EXPORT +size_t crypto_kx_publickeybytes(void); + +#define crypto_kx_SECRETKEYBYTES 32 +SODIUM_EXPORT +size_t crypto_kx_secretkeybytes(void); + +#define crypto_kx_SEEDBYTES 32 +SODIUM_EXPORT +size_t crypto_kx_seedbytes(void); + +#define crypto_kx_SESSIONKEYBYTES 32 +SODIUM_EXPORT +size_t crypto_kx_sessionkeybytes(void); + +#define crypto_kx_PRIMITIVE "x25519blake2b" +SODIUM_EXPORT +const char *crypto_kx_primitive(void); + +SODIUM_EXPORT +int crypto_kx_seed_keypair(unsigned char pk[crypto_kx_PUBLICKEYBYTES], + unsigned char sk[crypto_kx_SECRETKEYBYTES], + const unsigned char seed[crypto_kx_SEEDBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_kx_keypair(unsigned char pk[crypto_kx_PUBLICKEYBYTES], + unsigned char sk[crypto_kx_SECRETKEYBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_kx_client_session_keys(unsigned char rx[crypto_kx_SESSIONKEYBYTES], + unsigned char tx[crypto_kx_SESSIONKEYBYTES], + const unsigned char client_pk[crypto_kx_PUBLICKEYBYTES], + const unsigned char client_sk[crypto_kx_SECRETKEYBYTES], + const unsigned char server_pk[crypto_kx_PUBLICKEYBYTES]) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 4, 5))); + +SODIUM_EXPORT +int crypto_kx_server_session_keys(unsigned char rx[crypto_kx_SESSIONKEYBYTES], + unsigned char tx[crypto_kx_SESSIONKEYBYTES], + const unsigned char server_pk[crypto_kx_PUBLICKEYBYTES], + const unsigned char server_sk[crypto_kx_SECRETKEYBYTES], + const unsigned char client_pk[crypto_kx_PUBLICKEYBYTES]) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 4, 5))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_onetimeauth.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_onetimeauth.h new file mode 100644 index 00000000..7cd7b070 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_onetimeauth.h @@ -0,0 +1,65 @@ +#ifndef crypto_onetimeauth_H +#define crypto_onetimeauth_H + +#include + +#include "crypto_onetimeauth_poly1305.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef crypto_onetimeauth_poly1305_state crypto_onetimeauth_state; + +SODIUM_EXPORT +size_t crypto_onetimeauth_statebytes(void); + +#define crypto_onetimeauth_BYTES crypto_onetimeauth_poly1305_BYTES +SODIUM_EXPORT +size_t crypto_onetimeauth_bytes(void); + +#define crypto_onetimeauth_KEYBYTES crypto_onetimeauth_poly1305_KEYBYTES +SODIUM_EXPORT +size_t crypto_onetimeauth_keybytes(void); + +#define crypto_onetimeauth_PRIMITIVE "poly1305" +SODIUM_EXPORT +const char *crypto_onetimeauth_primitive(void); + +SODIUM_EXPORT +int crypto_onetimeauth(unsigned char *out, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_onetimeauth_verify(const unsigned char *h, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_onetimeauth_init(crypto_onetimeauth_state *state, + const unsigned char *key) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_onetimeauth_update(crypto_onetimeauth_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_onetimeauth_final(crypto_onetimeauth_state *state, + unsigned char *out) __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_onetimeauth_keygen(unsigned char k[crypto_onetimeauth_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_onetimeauth_poly1305.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_onetimeauth_poly1305.h new file mode 100644 index 00000000..f3e34d86 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_onetimeauth_poly1305.h @@ -0,0 +1,72 @@ +#ifndef crypto_onetimeauth_poly1305_H +#define crypto_onetimeauth_poly1305_H + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#include +#include +#include + +#include + +#include "export.h" + +typedef struct CRYPTO_ALIGN(16) crypto_onetimeauth_poly1305_state { + unsigned char opaque[256]; +} crypto_onetimeauth_poly1305_state; + +SODIUM_EXPORT +size_t crypto_onetimeauth_poly1305_statebytes(void); + +#define crypto_onetimeauth_poly1305_BYTES 16U +SODIUM_EXPORT +size_t crypto_onetimeauth_poly1305_bytes(void); + +#define crypto_onetimeauth_poly1305_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_onetimeauth_poly1305_keybytes(void); + +SODIUM_EXPORT +int crypto_onetimeauth_poly1305(unsigned char *out, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_onetimeauth_poly1305_verify(const unsigned char *h, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_onetimeauth_poly1305_init(crypto_onetimeauth_poly1305_state *state, + const unsigned char *key) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_onetimeauth_poly1305_update(crypto_onetimeauth_poly1305_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_onetimeauth_poly1305_final(crypto_onetimeauth_poly1305_state *state, + unsigned char *out) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_onetimeauth_poly1305_keygen(unsigned char k[crypto_onetimeauth_poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_pwhash.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_pwhash.h new file mode 100644 index 00000000..585a993e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_pwhash.h @@ -0,0 +1,147 @@ +#ifndef crypto_pwhash_H +#define crypto_pwhash_H + +#include + +#include "crypto_pwhash_argon2i.h" +#include "crypto_pwhash_argon2id.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_pwhash_ALG_ARGON2I13 crypto_pwhash_argon2i_ALG_ARGON2I13 +SODIUM_EXPORT +int crypto_pwhash_alg_argon2i13(void); + +#define crypto_pwhash_ALG_ARGON2ID13 crypto_pwhash_argon2id_ALG_ARGON2ID13 +SODIUM_EXPORT +int crypto_pwhash_alg_argon2id13(void); + +#define crypto_pwhash_ALG_DEFAULT crypto_pwhash_ALG_ARGON2ID13 +SODIUM_EXPORT +int crypto_pwhash_alg_default(void); + +#define crypto_pwhash_BYTES_MIN crypto_pwhash_argon2id_BYTES_MIN +SODIUM_EXPORT +size_t crypto_pwhash_bytes_min(void); + +#define crypto_pwhash_BYTES_MAX crypto_pwhash_argon2id_BYTES_MAX +SODIUM_EXPORT +size_t crypto_pwhash_bytes_max(void); + +#define crypto_pwhash_PASSWD_MIN crypto_pwhash_argon2id_PASSWD_MIN +SODIUM_EXPORT +size_t crypto_pwhash_passwd_min(void); + +#define crypto_pwhash_PASSWD_MAX crypto_pwhash_argon2id_PASSWD_MAX +SODIUM_EXPORT +size_t crypto_pwhash_passwd_max(void); + +#define crypto_pwhash_SALTBYTES crypto_pwhash_argon2id_SALTBYTES +SODIUM_EXPORT +size_t crypto_pwhash_saltbytes(void); + +#define crypto_pwhash_STRBYTES crypto_pwhash_argon2id_STRBYTES +SODIUM_EXPORT +size_t crypto_pwhash_strbytes(void); + +#define crypto_pwhash_STRPREFIX crypto_pwhash_argon2id_STRPREFIX +SODIUM_EXPORT +const char *crypto_pwhash_strprefix(void); + +#define crypto_pwhash_OPSLIMIT_MIN crypto_pwhash_argon2id_OPSLIMIT_MIN +SODIUM_EXPORT +size_t crypto_pwhash_opslimit_min(void); + +#define crypto_pwhash_OPSLIMIT_MAX crypto_pwhash_argon2id_OPSLIMIT_MAX +SODIUM_EXPORT +size_t crypto_pwhash_opslimit_max(void); + +#define crypto_pwhash_MEMLIMIT_MIN crypto_pwhash_argon2id_MEMLIMIT_MIN +SODIUM_EXPORT +size_t crypto_pwhash_memlimit_min(void); + +#define crypto_pwhash_MEMLIMIT_MAX crypto_pwhash_argon2id_MEMLIMIT_MAX +SODIUM_EXPORT +size_t crypto_pwhash_memlimit_max(void); + +#define crypto_pwhash_OPSLIMIT_INTERACTIVE crypto_pwhash_argon2id_OPSLIMIT_INTERACTIVE +SODIUM_EXPORT +size_t crypto_pwhash_opslimit_interactive(void); + +#define crypto_pwhash_MEMLIMIT_INTERACTIVE crypto_pwhash_argon2id_MEMLIMIT_INTERACTIVE +SODIUM_EXPORT +size_t crypto_pwhash_memlimit_interactive(void); + +#define crypto_pwhash_OPSLIMIT_MODERATE crypto_pwhash_argon2id_OPSLIMIT_MODERATE +SODIUM_EXPORT +size_t crypto_pwhash_opslimit_moderate(void); + +#define crypto_pwhash_MEMLIMIT_MODERATE crypto_pwhash_argon2id_MEMLIMIT_MODERATE +SODIUM_EXPORT +size_t crypto_pwhash_memlimit_moderate(void); + +#define crypto_pwhash_OPSLIMIT_SENSITIVE crypto_pwhash_argon2id_OPSLIMIT_SENSITIVE +SODIUM_EXPORT +size_t crypto_pwhash_opslimit_sensitive(void); + +#define crypto_pwhash_MEMLIMIT_SENSITIVE crypto_pwhash_argon2id_MEMLIMIT_SENSITIVE +SODIUM_EXPORT +size_t crypto_pwhash_memlimit_sensitive(void); + +/* + * With this function, do not forget to store all parameters, including the + * algorithm identifier in order to produce deterministic output. + * The crypto_pwhash_* definitions, including crypto_pwhash_ALG_DEFAULT, + * may change. + */ +SODIUM_EXPORT +int crypto_pwhash(unsigned char * const out, unsigned long long outlen, + const char * const passwd, unsigned long long passwdlen, + const unsigned char * const salt, + unsigned long long opslimit, size_t memlimit, int alg) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +/* + * The output string already includes all the required parameters, including + * the algorithm identifier. The string is all that has to be stored in + * order to verify a password. + */ +SODIUM_EXPORT +int crypto_pwhash_str(char out[crypto_pwhash_STRBYTES], + const char * const passwd, unsigned long long passwdlen, + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_str_alg(char out[crypto_pwhash_STRBYTES], + const char * const passwd, unsigned long long passwdlen, + unsigned long long opslimit, size_t memlimit, int alg) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_str_verify(const char str[crypto_pwhash_STRBYTES], + const char * const passwd, + unsigned long long passwdlen) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_str_needs_rehash(const char str[crypto_pwhash_STRBYTES], + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#define crypto_pwhash_PRIMITIVE "argon2i" +SODIUM_EXPORT +const char *crypto_pwhash_primitive(void) + __attribute__ ((warn_unused_result)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_pwhash_argon2i.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_pwhash_argon2i.h new file mode 100644 index 00000000..88ff6221 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_pwhash_argon2i.h @@ -0,0 +1,122 @@ +#ifndef crypto_pwhash_argon2i_H +#define crypto_pwhash_argon2i_H + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_pwhash_argon2i_ALG_ARGON2I13 1 +SODIUM_EXPORT +int crypto_pwhash_argon2i_alg_argon2i13(void); + +#define crypto_pwhash_argon2i_BYTES_MIN 16U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_bytes_min(void); + +#define crypto_pwhash_argon2i_BYTES_MAX SODIUM_MIN(SODIUM_SIZE_MAX, 4294967295U) +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_bytes_max(void); + +#define crypto_pwhash_argon2i_PASSWD_MIN 0U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_passwd_min(void); + +#define crypto_pwhash_argon2i_PASSWD_MAX 4294967295U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_passwd_max(void); + +#define crypto_pwhash_argon2i_SALTBYTES 16U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_saltbytes(void); + +#define crypto_pwhash_argon2i_STRBYTES 128U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_strbytes(void); + +#define crypto_pwhash_argon2i_STRPREFIX "$argon2i$" +SODIUM_EXPORT +const char *crypto_pwhash_argon2i_strprefix(void); + +#define crypto_pwhash_argon2i_OPSLIMIT_MIN 3U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_opslimit_min(void); + +#define crypto_pwhash_argon2i_OPSLIMIT_MAX 4294967295U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_opslimit_max(void); + +#define crypto_pwhash_argon2i_MEMLIMIT_MIN 8192U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_memlimit_min(void); + +#define crypto_pwhash_argon2i_MEMLIMIT_MAX \ + ((SIZE_MAX >= 4398046510080U) ? 4398046510080U : (SIZE_MAX >= 2147483648U) ? 2147483648U : 32768U) +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_memlimit_max(void); + +#define crypto_pwhash_argon2i_OPSLIMIT_INTERACTIVE 4U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_opslimit_interactive(void); + +#define crypto_pwhash_argon2i_MEMLIMIT_INTERACTIVE 33554432U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_memlimit_interactive(void); + +#define crypto_pwhash_argon2i_OPSLIMIT_MODERATE 6U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_opslimit_moderate(void); + +#define crypto_pwhash_argon2i_MEMLIMIT_MODERATE 134217728U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_memlimit_moderate(void); + +#define crypto_pwhash_argon2i_OPSLIMIT_SENSITIVE 8U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_opslimit_sensitive(void); + +#define crypto_pwhash_argon2i_MEMLIMIT_SENSITIVE 536870912U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_memlimit_sensitive(void); + +SODIUM_EXPORT +int crypto_pwhash_argon2i(unsigned char * const out, + unsigned long long outlen, + const char * const passwd, + unsigned long long passwdlen, + const unsigned char * const salt, + unsigned long long opslimit, size_t memlimit, + int alg) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2i_str(char out[crypto_pwhash_argon2i_STRBYTES], + const char * const passwd, + unsigned long long passwdlen, + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2i_str_verify(const char str[crypto_pwhash_argon2i_STRBYTES], + const char * const passwd, + unsigned long long passwdlen) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2i_str_needs_rehash(const char str[crypto_pwhash_argon2i_STRBYTES], + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_pwhash_argon2id.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_pwhash_argon2id.h new file mode 100644 index 00000000..7183abd1 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_pwhash_argon2id.h @@ -0,0 +1,122 @@ +#ifndef crypto_pwhash_argon2id_H +#define crypto_pwhash_argon2id_H + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_pwhash_argon2id_ALG_ARGON2ID13 2 +SODIUM_EXPORT +int crypto_pwhash_argon2id_alg_argon2id13(void); + +#define crypto_pwhash_argon2id_BYTES_MIN 16U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_bytes_min(void); + +#define crypto_pwhash_argon2id_BYTES_MAX SODIUM_MIN(SODIUM_SIZE_MAX, 4294967295U) +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_bytes_max(void); + +#define crypto_pwhash_argon2id_PASSWD_MIN 0U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_passwd_min(void); + +#define crypto_pwhash_argon2id_PASSWD_MAX 4294967295U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_passwd_max(void); + +#define crypto_pwhash_argon2id_SALTBYTES 16U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_saltbytes(void); + +#define crypto_pwhash_argon2id_STRBYTES 128U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_strbytes(void); + +#define crypto_pwhash_argon2id_STRPREFIX "$argon2id$" +SODIUM_EXPORT +const char *crypto_pwhash_argon2id_strprefix(void); + +#define crypto_pwhash_argon2id_OPSLIMIT_MIN 1U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_opslimit_min(void); + +#define crypto_pwhash_argon2id_OPSLIMIT_MAX 4294967295U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_opslimit_max(void); + +#define crypto_pwhash_argon2id_MEMLIMIT_MIN 8192U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_memlimit_min(void); + +#define crypto_pwhash_argon2id_MEMLIMIT_MAX \ + ((SIZE_MAX >= 4398046510080U) ? 4398046510080U : (SIZE_MAX >= 2147483648U) ? 2147483648U : 32768U) +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_memlimit_max(void); + +#define crypto_pwhash_argon2id_OPSLIMIT_INTERACTIVE 2U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_opslimit_interactive(void); + +#define crypto_pwhash_argon2id_MEMLIMIT_INTERACTIVE 67108864U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_memlimit_interactive(void); + +#define crypto_pwhash_argon2id_OPSLIMIT_MODERATE 3U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_opslimit_moderate(void); + +#define crypto_pwhash_argon2id_MEMLIMIT_MODERATE 268435456U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_memlimit_moderate(void); + +#define crypto_pwhash_argon2id_OPSLIMIT_SENSITIVE 4U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_opslimit_sensitive(void); + +#define crypto_pwhash_argon2id_MEMLIMIT_SENSITIVE 1073741824U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_memlimit_sensitive(void); + +SODIUM_EXPORT +int crypto_pwhash_argon2id(unsigned char * const out, + unsigned long long outlen, + const char * const passwd, + unsigned long long passwdlen, + const unsigned char * const salt, + unsigned long long opslimit, size_t memlimit, + int alg) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2id_str(char out[crypto_pwhash_argon2id_STRBYTES], + const char * const passwd, + unsigned long long passwdlen, + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2id_str_verify(const char str[crypto_pwhash_argon2id_STRBYTES], + const char * const passwd, + unsigned long long passwdlen) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2id_str_needs_rehash(const char str[crypto_pwhash_argon2id_STRBYTES], + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_pwhash_scryptsalsa208sha256.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_pwhash_scryptsalsa208sha256.h new file mode 100644 index 00000000..5c0bf7d3 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_pwhash_scryptsalsa208sha256.h @@ -0,0 +1,120 @@ +#ifndef crypto_pwhash_scryptsalsa208sha256_H +#define crypto_pwhash_scryptsalsa208sha256_H + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_pwhash_scryptsalsa208sha256_BYTES_MIN 16U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_bytes_min(void); + +#define crypto_pwhash_scryptsalsa208sha256_BYTES_MAX \ + SODIUM_MIN(SODIUM_SIZE_MAX, 0x1fffffffe0ULL) +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_bytes_max(void); + +#define crypto_pwhash_scryptsalsa208sha256_PASSWD_MIN 0U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_passwd_min(void); + +#define crypto_pwhash_scryptsalsa208sha256_PASSWD_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_passwd_max(void); + +#define crypto_pwhash_scryptsalsa208sha256_SALTBYTES 32U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_saltbytes(void); + +#define crypto_pwhash_scryptsalsa208sha256_STRBYTES 102U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_strbytes(void); + +#define crypto_pwhash_scryptsalsa208sha256_STRPREFIX "$7$" +SODIUM_EXPORT +const char *crypto_pwhash_scryptsalsa208sha256_strprefix(void); + +#define crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_MIN 32768U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_opslimit_min(void); + +#define crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_MAX 4294967295U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_opslimit_max(void); + +#define crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_MIN 16777216U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_memlimit_min(void); + +#define crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_MAX \ + SODIUM_MIN(SIZE_MAX, 68719476736ULL) +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_memlimit_max(void); + +#define crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_INTERACTIVE 524288U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_opslimit_interactive(void); + +#define crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_INTERACTIVE 16777216U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_memlimit_interactive(void); + +#define crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_SENSITIVE 33554432U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_opslimit_sensitive(void); + +#define crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_SENSITIVE 1073741824U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_memlimit_sensitive(void); + +SODIUM_EXPORT +int crypto_pwhash_scryptsalsa208sha256(unsigned char * const out, + unsigned long long outlen, + const char * const passwd, + unsigned long long passwdlen, + const unsigned char * const salt, + unsigned long long opslimit, + size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_scryptsalsa208sha256_str(char out[crypto_pwhash_scryptsalsa208sha256_STRBYTES], + const char * const passwd, + unsigned long long passwdlen, + unsigned long long opslimit, + size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_scryptsalsa208sha256_str_verify(const char str[crypto_pwhash_scryptsalsa208sha256_STRBYTES], + const char * const passwd, + unsigned long long passwdlen) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_scryptsalsa208sha256_ll(const uint8_t * passwd, size_t passwdlen, + const uint8_t * salt, size_t saltlen, + uint64_t N, uint32_t r, uint32_t p, + uint8_t * buf, size_t buflen) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_scryptsalsa208sha256_str_needs_rehash(const char str[crypto_pwhash_scryptsalsa208sha256_STRBYTES], + unsigned long long opslimit, + size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_scalarmult.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_scalarmult.h new file mode 100644 index 00000000..1c685853 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_scalarmult.h @@ -0,0 +1,46 @@ +#ifndef crypto_scalarmult_H +#define crypto_scalarmult_H + +#include + +#include "crypto_scalarmult_curve25519.h" +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_scalarmult_BYTES crypto_scalarmult_curve25519_BYTES +SODIUM_EXPORT +size_t crypto_scalarmult_bytes(void); + +#define crypto_scalarmult_SCALARBYTES crypto_scalarmult_curve25519_SCALARBYTES +SODIUM_EXPORT +size_t crypto_scalarmult_scalarbytes(void); + +#define crypto_scalarmult_PRIMITIVE "curve25519" +SODIUM_EXPORT +const char *crypto_scalarmult_primitive(void); + +SODIUM_EXPORT +int crypto_scalarmult_base(unsigned char *q, const unsigned char *n) + __attribute__ ((nonnull)); + +/* + * NOTE: Do not use the result of this function directly for key exchange. + * + * Hash the result with the public keys in order to compute a shared + * secret key: H(q || client_pk || server_pk) + * + * Or unless this is not an option, use the crypto_kx() API instead. + */ +SODIUM_EXPORT +int crypto_scalarmult(unsigned char *q, const unsigned char *n, + const unsigned char *p) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_scalarmult_curve25519.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_scalarmult_curve25519.h new file mode 100644 index 00000000..60e9d0c5 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_scalarmult_curve25519.h @@ -0,0 +1,42 @@ +#ifndef crypto_scalarmult_curve25519_H +#define crypto_scalarmult_curve25519_H + +#include + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_scalarmult_curve25519_BYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_curve25519_bytes(void); + +#define crypto_scalarmult_curve25519_SCALARBYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_curve25519_scalarbytes(void); + +/* + * NOTE: Do not use the result of this function directly for key exchange. + * + * Hash the result with the public keys in order to compute a shared + * secret key: H(q || client_pk || server_pk) + * + * Or unless this is not an option, use the crypto_kx() API instead. + */ +SODIUM_EXPORT +int crypto_scalarmult_curve25519(unsigned char *q, const unsigned char *n, + const unsigned char *p) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_scalarmult_curve25519_base(unsigned char *q, + const unsigned char *n) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_scalarmult_ed25519.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_scalarmult_ed25519.h new file mode 100644 index 00000000..2dfa4d70 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_scalarmult_ed25519.h @@ -0,0 +1,51 @@ + +#ifndef crypto_scalarmult_ed25519_H +#define crypto_scalarmult_ed25519_H + +#include + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_scalarmult_ed25519_BYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_ed25519_bytes(void); + +#define crypto_scalarmult_ed25519_SCALARBYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_ed25519_scalarbytes(void); + +/* + * NOTE: Do not use the result of this function directly for key exchange. + * + * Hash the result with the public keys in order to compute a shared + * secret key: H(q || client_pk || server_pk) + * + * Or unless this is not an option, use the crypto_kx() API instead. + */ +SODIUM_EXPORT +int crypto_scalarmult_ed25519(unsigned char *q, const unsigned char *n, + const unsigned char *p) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_scalarmult_ed25519_noclamp(unsigned char *q, const unsigned char *n, + const unsigned char *p) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_scalarmult_ed25519_base(unsigned char *q, const unsigned char *n) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_scalarmult_ed25519_base_noclamp(unsigned char *q, const unsigned char *n) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_scalarmult_ristretto255.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_scalarmult_ristretto255.h new file mode 100644 index 00000000..40a45cce --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_scalarmult_ristretto255.h @@ -0,0 +1,43 @@ + +#ifndef crypto_scalarmult_ristretto255_H +#define crypto_scalarmult_ristretto255_H + +#include + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_scalarmult_ristretto255_BYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_ristretto255_bytes(void); + +#define crypto_scalarmult_ristretto255_SCALARBYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_ristretto255_scalarbytes(void); + +/* + * NOTE: Do not use the result of this function directly for key exchange. + * + * Hash the result with the public keys in order to compute a shared + * secret key: H(q || client_pk || server_pk) + * + * Or unless this is not an option, use the crypto_kx() API instead. + */ +SODIUM_EXPORT +int crypto_scalarmult_ristretto255(unsigned char *q, const unsigned char *n, + const unsigned char *p) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_scalarmult_ristretto255_base(unsigned char *q, + const unsigned char *n) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_secretbox.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_secretbox.h new file mode 100644 index 00000000..1d3709db --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_secretbox.h @@ -0,0 +1,93 @@ +#ifndef crypto_secretbox_H +#define crypto_secretbox_H + +#include + +#include "crypto_secretbox_xsalsa20poly1305.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_secretbox_KEYBYTES crypto_secretbox_xsalsa20poly1305_KEYBYTES +SODIUM_EXPORT +size_t crypto_secretbox_keybytes(void); + +#define crypto_secretbox_NONCEBYTES crypto_secretbox_xsalsa20poly1305_NONCEBYTES +SODIUM_EXPORT +size_t crypto_secretbox_noncebytes(void); + +#define crypto_secretbox_MACBYTES crypto_secretbox_xsalsa20poly1305_MACBYTES +SODIUM_EXPORT +size_t crypto_secretbox_macbytes(void); + +#define crypto_secretbox_PRIMITIVE "xsalsa20poly1305" +SODIUM_EXPORT +const char *crypto_secretbox_primitive(void); + +#define crypto_secretbox_MESSAGEBYTES_MAX crypto_secretbox_xsalsa20poly1305_MESSAGEBYTES_MAX +SODIUM_EXPORT +size_t crypto_secretbox_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_secretbox_easy(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_open_easy(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_detached(unsigned char *c, unsigned char *mac, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 5, 6))); + +SODIUM_EXPORT +int crypto_secretbox_open_detached(unsigned char *m, + const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6))); + +SODIUM_EXPORT +void crypto_secretbox_keygen(unsigned char k[crypto_secretbox_KEYBYTES]) + __attribute__ ((nonnull)); + +/* -- NaCl compatibility interface ; Requires padding -- */ + +#define crypto_secretbox_ZEROBYTES crypto_secretbox_xsalsa20poly1305_ZEROBYTES +SODIUM_EXPORT +size_t crypto_secretbox_zerobytes(void); + +#define crypto_secretbox_BOXZEROBYTES crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES +SODIUM_EXPORT +size_t crypto_secretbox_boxzerobytes(void); + +SODIUM_EXPORT +int crypto_secretbox(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_open(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_secretbox_xchacha20poly1305.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_secretbox_xchacha20poly1305.h new file mode 100644 index 00000000..6ec674e3 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_secretbox_xchacha20poly1305.h @@ -0,0 +1,70 @@ +#ifndef crypto_secretbox_xchacha20poly1305_H +#define crypto_secretbox_xchacha20poly1305_H + +#include +#include "crypto_stream_xchacha20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_secretbox_xchacha20poly1305_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_secretbox_xchacha20poly1305_keybytes(void); + +#define crypto_secretbox_xchacha20poly1305_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_secretbox_xchacha20poly1305_noncebytes(void); + +#define crypto_secretbox_xchacha20poly1305_MACBYTES 16U +SODIUM_EXPORT +size_t crypto_secretbox_xchacha20poly1305_macbytes(void); + +#define crypto_secretbox_xchacha20poly1305_MESSAGEBYTES_MAX \ + (crypto_stream_xchacha20_MESSAGEBYTES_MAX - crypto_secretbox_xchacha20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_secretbox_xchacha20poly1305_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_secretbox_xchacha20poly1305_easy(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_xchacha20poly1305_open_easy(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_xchacha20poly1305_detached(unsigned char *c, + unsigned char *mac, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 5, 6))); + +SODIUM_EXPORT +int crypto_secretbox_xchacha20poly1305_open_detached(unsigned char *m, + const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_secretbox_xsalsa20poly1305.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_secretbox_xsalsa20poly1305.h new file mode 100644 index 00000000..be0874cb --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_secretbox_xsalsa20poly1305.h @@ -0,0 +1,69 @@ +#ifndef crypto_secretbox_xsalsa20poly1305_H +#define crypto_secretbox_xsalsa20poly1305_H + +#include +#include "crypto_stream_xsalsa20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_secretbox_xsalsa20poly1305_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_keybytes(void); + +#define crypto_secretbox_xsalsa20poly1305_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_noncebytes(void); + +#define crypto_secretbox_xsalsa20poly1305_MACBYTES 16U +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_macbytes(void); + +/* Only for the libsodium API - The NaCl compatibility API would require BOXZEROBYTES extra bytes */ +#define crypto_secretbox_xsalsa20poly1305_MESSAGEBYTES_MAX \ + (crypto_stream_xsalsa20_MESSAGEBYTES_MAX - crypto_secretbox_xsalsa20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_secretbox_xsalsa20poly1305(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_xsalsa20poly1305_open(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +SODIUM_EXPORT +void crypto_secretbox_xsalsa20poly1305_keygen(unsigned char k[crypto_secretbox_xsalsa20poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +/* -- NaCl compatibility interface ; Requires padding -- */ + +#define crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES 16U +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_boxzerobytes(void); + +#define crypto_secretbox_xsalsa20poly1305_ZEROBYTES \ + (crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES + \ + crypto_secretbox_xsalsa20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_zerobytes(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_secretstream_xchacha20poly1305.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_secretstream_xchacha20poly1305.h new file mode 100644 index 00000000..b22e4e93 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_secretstream_xchacha20poly1305.h @@ -0,0 +1,108 @@ +#ifndef crypto_secretstream_xchacha20poly1305_H +#define crypto_secretstream_xchacha20poly1305_H + +#include + +#include "crypto_aead_xchacha20poly1305.h" +#include "crypto_stream_chacha20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_secretstream_xchacha20poly1305_ABYTES \ + (1U + crypto_aead_xchacha20poly1305_ietf_ABYTES) +SODIUM_EXPORT +size_t crypto_secretstream_xchacha20poly1305_abytes(void); + +#define crypto_secretstream_xchacha20poly1305_HEADERBYTES \ + crypto_aead_xchacha20poly1305_ietf_NPUBBYTES +SODIUM_EXPORT +size_t crypto_secretstream_xchacha20poly1305_headerbytes(void); + +#define crypto_secretstream_xchacha20poly1305_KEYBYTES \ + crypto_aead_xchacha20poly1305_ietf_KEYBYTES +SODIUM_EXPORT +size_t crypto_secretstream_xchacha20poly1305_keybytes(void); + +#define crypto_secretstream_xchacha20poly1305_MESSAGEBYTES_MAX \ + SODIUM_MIN(SODIUM_SIZE_MAX - crypto_secretstream_xchacha20poly1305_ABYTES, \ + (64ULL * ((1ULL << 32) - 2ULL))) +SODIUM_EXPORT +size_t crypto_secretstream_xchacha20poly1305_messagebytes_max(void); + +#define crypto_secretstream_xchacha20poly1305_TAG_MESSAGE 0x00 +SODIUM_EXPORT +unsigned char crypto_secretstream_xchacha20poly1305_tag_message(void); + +#define crypto_secretstream_xchacha20poly1305_TAG_PUSH 0x01 +SODIUM_EXPORT +unsigned char crypto_secretstream_xchacha20poly1305_tag_push(void); + +#define crypto_secretstream_xchacha20poly1305_TAG_REKEY 0x02 +SODIUM_EXPORT +unsigned char crypto_secretstream_xchacha20poly1305_tag_rekey(void); + +#define crypto_secretstream_xchacha20poly1305_TAG_FINAL \ + (crypto_secretstream_xchacha20poly1305_TAG_PUSH | \ + crypto_secretstream_xchacha20poly1305_TAG_REKEY) +SODIUM_EXPORT +unsigned char crypto_secretstream_xchacha20poly1305_tag_final(void); + +typedef struct crypto_secretstream_xchacha20poly1305_state { + unsigned char k[crypto_stream_chacha20_ietf_KEYBYTES]; + unsigned char nonce[crypto_stream_chacha20_ietf_NONCEBYTES]; + unsigned char _pad[8]; +} crypto_secretstream_xchacha20poly1305_state; + +SODIUM_EXPORT +size_t crypto_secretstream_xchacha20poly1305_statebytes(void); + +SODIUM_EXPORT +void crypto_secretstream_xchacha20poly1305_keygen + (unsigned char k[crypto_secretstream_xchacha20poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_secretstream_xchacha20poly1305_init_push + (crypto_secretstream_xchacha20poly1305_state *state, + unsigned char header[crypto_secretstream_xchacha20poly1305_HEADERBYTES], + const unsigned char k[crypto_secretstream_xchacha20poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_secretstream_xchacha20poly1305_push + (crypto_secretstream_xchacha20poly1305_state *state, + unsigned char *c, unsigned long long *clen_p, + const unsigned char *m, unsigned long long mlen, + const unsigned char *ad, unsigned long long adlen, unsigned char tag) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_secretstream_xchacha20poly1305_init_pull + (crypto_secretstream_xchacha20poly1305_state *state, + const unsigned char header[crypto_secretstream_xchacha20poly1305_HEADERBYTES], + const unsigned char k[crypto_secretstream_xchacha20poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_secretstream_xchacha20poly1305_pull + (crypto_secretstream_xchacha20poly1305_state *state, + unsigned char *m, unsigned long long *mlen_p, unsigned char *tag_p, + const unsigned char *c, unsigned long long clen, + const unsigned char *ad, unsigned long long adlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +void crypto_secretstream_xchacha20poly1305_rekey + (crypto_secretstream_xchacha20poly1305_state *state); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_shorthash.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_shorthash.h new file mode 100644 index 00000000..fecaa88b --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_shorthash.h @@ -0,0 +1,41 @@ +#ifndef crypto_shorthash_H +#define crypto_shorthash_H + +#include + +#include "crypto_shorthash_siphash24.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_shorthash_BYTES crypto_shorthash_siphash24_BYTES +SODIUM_EXPORT +size_t crypto_shorthash_bytes(void); + +#define crypto_shorthash_KEYBYTES crypto_shorthash_siphash24_KEYBYTES +SODIUM_EXPORT +size_t crypto_shorthash_keybytes(void); + +#define crypto_shorthash_PRIMITIVE "siphash24" +SODIUM_EXPORT +const char *crypto_shorthash_primitive(void); + +SODIUM_EXPORT +int crypto_shorthash(unsigned char *out, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +void crypto_shorthash_keygen(unsigned char k[crypto_shorthash_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_shorthash_siphash24.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_shorthash_siphash24.h new file mode 100644 index 00000000..1e6f72a6 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_shorthash_siphash24.h @@ -0,0 +1,50 @@ +#ifndef crypto_shorthash_siphash24_H +#define crypto_shorthash_siphash24_H + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +/* -- 64-bit output -- */ + +#define crypto_shorthash_siphash24_BYTES 8U +SODIUM_EXPORT +size_t crypto_shorthash_siphash24_bytes(void); + +#define crypto_shorthash_siphash24_KEYBYTES 16U +SODIUM_EXPORT +size_t crypto_shorthash_siphash24_keybytes(void); + +SODIUM_EXPORT +int crypto_shorthash_siphash24(unsigned char *out, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((nonnull(1, 4))); + +#ifndef SODIUM_LIBRARY_MINIMAL +/* -- 128-bit output -- */ + +#define crypto_shorthash_siphashx24_BYTES 16U +SODIUM_EXPORT +size_t crypto_shorthash_siphashx24_bytes(void); + +#define crypto_shorthash_siphashx24_KEYBYTES 16U +SODIUM_EXPORT +size_t crypto_shorthash_siphashx24_keybytes(void); + +SODIUM_EXPORT +int crypto_shorthash_siphashx24(unsigned char *out, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((nonnull(1, 4))); +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_sign.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_sign.h new file mode 100644 index 00000000..f5fafb12 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_sign.h @@ -0,0 +1,107 @@ +#ifndef crypto_sign_H +#define crypto_sign_H + +/* + * THREAD SAFETY: crypto_sign_keypair() is thread-safe, + * provided that sodium_init() was called before. + * + * Other functions, including crypto_sign_seed_keypair() are always thread-safe. + */ + +#include + +#include "crypto_sign_ed25519.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef crypto_sign_ed25519ph_state crypto_sign_state; + +SODIUM_EXPORT +size_t crypto_sign_statebytes(void); + +#define crypto_sign_BYTES crypto_sign_ed25519_BYTES +SODIUM_EXPORT +size_t crypto_sign_bytes(void); + +#define crypto_sign_SEEDBYTES crypto_sign_ed25519_SEEDBYTES +SODIUM_EXPORT +size_t crypto_sign_seedbytes(void); + +#define crypto_sign_PUBLICKEYBYTES crypto_sign_ed25519_PUBLICKEYBYTES +SODIUM_EXPORT +size_t crypto_sign_publickeybytes(void); + +#define crypto_sign_SECRETKEYBYTES crypto_sign_ed25519_SECRETKEYBYTES +SODIUM_EXPORT +size_t crypto_sign_secretkeybytes(void); + +#define crypto_sign_MESSAGEBYTES_MAX crypto_sign_ed25519_MESSAGEBYTES_MAX +SODIUM_EXPORT +size_t crypto_sign_messagebytes_max(void); + +#define crypto_sign_PRIMITIVE "ed25519" +SODIUM_EXPORT +const char *crypto_sign_primitive(void); + +SODIUM_EXPORT +int crypto_sign_seed_keypair(unsigned char *pk, unsigned char *sk, + const unsigned char *seed) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_keypair(unsigned char *pk, unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign(unsigned char *sm, unsigned long long *smlen_p, + const unsigned char *m, unsigned long long mlen, + const unsigned char *sk) __attribute__ ((nonnull(1, 5))); + +SODIUM_EXPORT +int crypto_sign_open(unsigned char *m, unsigned long long *mlen_p, + const unsigned char *sm, unsigned long long smlen, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5))); + +SODIUM_EXPORT +int crypto_sign_detached(unsigned char *sig, unsigned long long *siglen_p, + const unsigned char *m, unsigned long long mlen, + const unsigned char *sk) __attribute__ ((nonnull(1, 5))); + +SODIUM_EXPORT +int crypto_sign_verify_detached(const unsigned char *sig, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_sign_init(crypto_sign_state *state); + +SODIUM_EXPORT +int crypto_sign_update(crypto_sign_state *state, + const unsigned char *m, unsigned long long mlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_sign_final_create(crypto_sign_state *state, unsigned char *sig, + unsigned long long *siglen_p, + const unsigned char *sk) + __attribute__ ((nonnull(1, 2, 4))); + +SODIUM_EXPORT +int crypto_sign_final_verify(crypto_sign_state *state, const unsigned char *sig, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_sign_ed25519.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_sign_ed25519.h new file mode 100644 index 00000000..0fdac42d --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_sign_ed25519.h @@ -0,0 +1,124 @@ +#ifndef crypto_sign_ed25519_H +#define crypto_sign_ed25519_H + +#include +#include "crypto_hash_sha512.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef struct crypto_sign_ed25519ph_state { + crypto_hash_sha512_state hs; +} crypto_sign_ed25519ph_state; + +SODIUM_EXPORT +size_t crypto_sign_ed25519ph_statebytes(void); + +#define crypto_sign_ed25519_BYTES 64U +SODIUM_EXPORT +size_t crypto_sign_ed25519_bytes(void); + +#define crypto_sign_ed25519_SEEDBYTES 32U +SODIUM_EXPORT +size_t crypto_sign_ed25519_seedbytes(void); + +#define crypto_sign_ed25519_PUBLICKEYBYTES 32U +SODIUM_EXPORT +size_t crypto_sign_ed25519_publickeybytes(void); + +#define crypto_sign_ed25519_SECRETKEYBYTES (32U + 32U) +SODIUM_EXPORT +size_t crypto_sign_ed25519_secretkeybytes(void); + +#define crypto_sign_ed25519_MESSAGEBYTES_MAX (SODIUM_SIZE_MAX - crypto_sign_ed25519_BYTES) +SODIUM_EXPORT +size_t crypto_sign_ed25519_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_sign_ed25519(unsigned char *sm, unsigned long long *smlen_p, + const unsigned char *m, unsigned long long mlen, + const unsigned char *sk) + __attribute__ ((nonnull(1, 5))); + +SODIUM_EXPORT +int crypto_sign_ed25519_open(unsigned char *m, unsigned long long *mlen_p, + const unsigned char *sm, unsigned long long smlen, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5))); + +SODIUM_EXPORT +int crypto_sign_ed25519_detached(unsigned char *sig, + unsigned long long *siglen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *sk) + __attribute__ ((nonnull(1, 5))); + +SODIUM_EXPORT +int crypto_sign_ed25519_verify_detached(const unsigned char *sig, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_sign_ed25519_keypair(unsigned char *pk, unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519_seed_keypair(unsigned char *pk, unsigned char *sk, + const unsigned char *seed) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519_pk_to_curve25519(unsigned char *curve25519_pk, + const unsigned char *ed25519_pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519_sk_to_curve25519(unsigned char *curve25519_sk, + const unsigned char *ed25519_sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519_sk_to_seed(unsigned char *seed, + const unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519_sk_to_pk(unsigned char *pk, const unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519ph_init(crypto_sign_ed25519ph_state *state) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519ph_update(crypto_sign_ed25519ph_state *state, + const unsigned char *m, + unsigned long long mlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_sign_ed25519ph_final_create(crypto_sign_ed25519ph_state *state, + unsigned char *sig, + unsigned long long *siglen_p, + const unsigned char *sk) + __attribute__ ((nonnull(1, 2, 4))); + +SODIUM_EXPORT +int crypto_sign_ed25519ph_final_verify(crypto_sign_ed25519ph_state *state, + const unsigned char *sig, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_sign_edwards25519sha512batch.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_sign_edwards25519sha512batch.h new file mode 100644 index 00000000..eed158aa --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_sign_edwards25519sha512batch.h @@ -0,0 +1,55 @@ +#ifndef crypto_sign_edwards25519sha512batch_H +#define crypto_sign_edwards25519sha512batch_H + +/* + * WARNING: This construction was a prototype, which should not be used + * any more in new projects. + * + * crypto_sign_edwards25519sha512batch is provided for applications + * initially built with NaCl, but as recommended by the author of this + * construction, new applications should use ed25519 instead. + * + * In Sodium, you should use the high-level crypto_sign_*() functions instead. + */ + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_sign_edwards25519sha512batch_BYTES 64U +#define crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES 32U +#define crypto_sign_edwards25519sha512batch_SECRETKEYBYTES (32U + 32U) +#define crypto_sign_edwards25519sha512batch_MESSAGEBYTES_MAX (SODIUM_SIZE_MAX - crypto_sign_edwards25519sha512batch_BYTES) + +SODIUM_EXPORT +int crypto_sign_edwards25519sha512batch(unsigned char *sm, + unsigned long long *smlen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *sk) + __attribute__ ((deprecated)) __attribute__ ((nonnull(1, 5))); + +SODIUM_EXPORT +int crypto_sign_edwards25519sha512batch_open(unsigned char *m, + unsigned long long *mlen_p, + const unsigned char *sm, + unsigned long long smlen, + const unsigned char *pk) + __attribute__ ((deprecated)) __attribute__ ((nonnull(3, 5))); + +SODIUM_EXPORT +int crypto_sign_edwards25519sha512batch_keypair(unsigned char *pk, + unsigned char *sk) + __attribute__ ((deprecated)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_stream.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_stream.h new file mode 100644 index 00000000..88dab5f6 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_stream.h @@ -0,0 +1,59 @@ +#ifndef crypto_stream_H +#define crypto_stream_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include + +#include "crypto_stream_xsalsa20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_KEYBYTES crypto_stream_xsalsa20_KEYBYTES +SODIUM_EXPORT +size_t crypto_stream_keybytes(void); + +#define crypto_stream_NONCEBYTES crypto_stream_xsalsa20_NONCEBYTES +SODIUM_EXPORT +size_t crypto_stream_noncebytes(void); + +#define crypto_stream_MESSAGEBYTES_MAX crypto_stream_xsalsa20_MESSAGEBYTES_MAX +SODIUM_EXPORT +size_t crypto_stream_messagebytes_max(void); + +#define crypto_stream_PRIMITIVE "xsalsa20" +SODIUM_EXPORT +const char *crypto_stream_primitive(void); + +SODIUM_EXPORT +int crypto_stream(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_keygen(unsigned char k[crypto_stream_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_stream_chacha20.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_stream_chacha20.h new file mode 100644 index 00000000..40889755 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_stream_chacha20.h @@ -0,0 +1,106 @@ +#ifndef crypto_stream_chacha20_H +#define crypto_stream_chacha20_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_chacha20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_chacha20_keybytes(void); + +#define crypto_stream_chacha20_NONCEBYTES 8U +SODIUM_EXPORT +size_t crypto_stream_chacha20_noncebytes(void); + +#define crypto_stream_chacha20_MESSAGEBYTES_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_stream_chacha20_messagebytes_max(void); + +/* ChaCha20 with a 64-bit nonce and a 64-bit counter, as originally designed */ + +SODIUM_EXPORT +int crypto_stream_chacha20(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_chacha20_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_chacha20_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, uint64_t ic, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_chacha20_keygen(unsigned char k[crypto_stream_chacha20_KEYBYTES]) + __attribute__ ((nonnull)); + +/* ChaCha20 with a 96-bit nonce and a 32-bit counter (IETF) */ + +#define crypto_stream_chacha20_ietf_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_chacha20_ietf_keybytes(void); + +#define crypto_stream_chacha20_ietf_NONCEBYTES 12U +SODIUM_EXPORT +size_t crypto_stream_chacha20_ietf_noncebytes(void); + +#define crypto_stream_chacha20_ietf_MESSAGEBYTES_MAX \ + SODIUM_MIN(SODIUM_SIZE_MAX, 64ULL * (1ULL << 32)) +SODIUM_EXPORT +size_t crypto_stream_chacha20_ietf_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_stream_chacha20_ietf(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_chacha20_ietf_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_chacha20_ietf_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, uint32_t ic, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_chacha20_ietf_keygen(unsigned char k[crypto_stream_chacha20_ietf_KEYBYTES]) + __attribute__ ((nonnull)); + +/* Aliases */ + +#define crypto_stream_chacha20_IETF_KEYBYTES crypto_stream_chacha20_ietf_KEYBYTES +#define crypto_stream_chacha20_IETF_NONCEBYTES crypto_stream_chacha20_ietf_NONCEBYTES +#define crypto_stream_chacha20_IETF_MESSAGEBYTES_MAX crypto_stream_chacha20_ietf_MESSAGEBYTES_MAX + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_stream_salsa20.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_stream_salsa20.h new file mode 100644 index 00000000..45b3b3e3 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_stream_salsa20.h @@ -0,0 +1,61 @@ +#ifndef crypto_stream_salsa20_H +#define crypto_stream_salsa20_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_salsa20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_salsa20_keybytes(void); + +#define crypto_stream_salsa20_NONCEBYTES 8U +SODIUM_EXPORT +size_t crypto_stream_salsa20_noncebytes(void); + +#define crypto_stream_salsa20_MESSAGEBYTES_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_stream_salsa20_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_stream_salsa20(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_salsa20_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_salsa20_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, uint64_t ic, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_salsa20_keygen(unsigned char k[crypto_stream_salsa20_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_stream_salsa2012.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_stream_salsa2012.h new file mode 100644 index 00000000..6c5d303c --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_stream_salsa2012.h @@ -0,0 +1,53 @@ +#ifndef crypto_stream_salsa2012_H +#define crypto_stream_salsa2012_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_salsa2012_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_salsa2012_keybytes(void); + +#define crypto_stream_salsa2012_NONCEBYTES 8U +SODIUM_EXPORT +size_t crypto_stream_salsa2012_noncebytes(void); + +#define crypto_stream_salsa2012_MESSAGEBYTES_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_stream_salsa2012_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_stream_salsa2012(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_salsa2012_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_salsa2012_keygen(unsigned char k[crypto_stream_salsa2012_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_stream_salsa208.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_stream_salsa208.h new file mode 100644 index 00000000..d574f304 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_stream_salsa208.h @@ -0,0 +1,56 @@ +#ifndef crypto_stream_salsa208_H +#define crypto_stream_salsa208_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_salsa208_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_salsa208_keybytes(void) + __attribute__ ((deprecated)); + +#define crypto_stream_salsa208_NONCEBYTES 8U +SODIUM_EXPORT +size_t crypto_stream_salsa208_noncebytes(void) + __attribute__ ((deprecated)); + +#define crypto_stream_salsa208_MESSAGEBYTES_MAX SODIUM_SIZE_MAX + SODIUM_EXPORT +size_t crypto_stream_salsa208_messagebytes_max(void) + __attribute__ ((deprecated)); + +SODIUM_EXPORT +int crypto_stream_salsa208(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((deprecated)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_salsa208_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((deprecated)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_salsa208_keygen(unsigned char k[crypto_stream_salsa208_KEYBYTES]) + __attribute__ ((deprecated)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_stream_xchacha20.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_stream_xchacha20.h new file mode 100644 index 00000000..c4002db0 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_stream_xchacha20.h @@ -0,0 +1,61 @@ +#ifndef crypto_stream_xchacha20_H +#define crypto_stream_xchacha20_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_xchacha20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_xchacha20_keybytes(void); + +#define crypto_stream_xchacha20_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_stream_xchacha20_noncebytes(void); + +#define crypto_stream_xchacha20_MESSAGEBYTES_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_stream_xchacha20_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_stream_xchacha20(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_xchacha20_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_xchacha20_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, uint64_t ic, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_xchacha20_keygen(unsigned char k[crypto_stream_xchacha20_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_stream_xsalsa20.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_stream_xsalsa20.h new file mode 100644 index 00000000..20034e34 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_stream_xsalsa20.h @@ -0,0 +1,61 @@ +#ifndef crypto_stream_xsalsa20_H +#define crypto_stream_xsalsa20_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_xsalsa20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_xsalsa20_keybytes(void); + +#define crypto_stream_xsalsa20_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_stream_xsalsa20_noncebytes(void); + +#define crypto_stream_xsalsa20_MESSAGEBYTES_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_stream_xsalsa20_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_stream_xsalsa20(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_xsalsa20_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_xsalsa20_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, uint64_t ic, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_xsalsa20_keygen(unsigned char k[crypto_stream_xsalsa20_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_verify_16.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_verify_16.h new file mode 100644 index 00000000..7b9c8077 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_verify_16.h @@ -0,0 +1,23 @@ +#ifndef crypto_verify_16_H +#define crypto_verify_16_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_verify_16_BYTES 16U +SODIUM_EXPORT +size_t crypto_verify_16_bytes(void); + +SODIUM_EXPORT +int crypto_verify_16(const unsigned char *x, const unsigned char *y) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_verify_32.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_verify_32.h new file mode 100644 index 00000000..9b0f4529 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_verify_32.h @@ -0,0 +1,23 @@ +#ifndef crypto_verify_32_H +#define crypto_verify_32_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_verify_32_BYTES 32U +SODIUM_EXPORT +size_t crypto_verify_32_bytes(void); + +SODIUM_EXPORT +int crypto_verify_32(const unsigned char *x, const unsigned char *y) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_verify_64.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_verify_64.h new file mode 100644 index 00000000..c83b7302 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/crypto_verify_64.h @@ -0,0 +1,23 @@ +#ifndef crypto_verify_64_H +#define crypto_verify_64_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_verify_64_BYTES 64U +SODIUM_EXPORT +size_t crypto_verify_64_bytes(void); + +SODIUM_EXPORT +int crypto_verify_64(const unsigned char *x, const unsigned char *y) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/export.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/export.h new file mode 100644 index 00000000..a0074fc9 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/export.h @@ -0,0 +1,57 @@ + +#ifndef sodium_export_H +#define sodium_export_H + +#include +#include +#include + +#if !defined(__clang__) && !defined(__GNUC__) +# ifdef __attribute__ +# undef __attribute__ +# endif +# define __attribute__(a) +#endif + +#ifdef SODIUM_STATIC +# define SODIUM_EXPORT +# define SODIUM_EXPORT_WEAK +#else +# if defined(_MSC_VER) +# ifdef SODIUM_DLL_EXPORT +# define SODIUM_EXPORT __declspec(dllexport) +# else +# define SODIUM_EXPORT __declspec(dllimport) +# endif +# else +# if defined(__SUNPRO_C) +# ifndef __GNU_C__ +# define SODIUM_EXPORT __attribute__ (visibility(__global)) +# else +# define SODIUM_EXPORT __attribute__ __global +# endif +# elif defined(_MSG_VER) +# define SODIUM_EXPORT extern __declspec(dllexport) +# else +# define SODIUM_EXPORT __attribute__ ((visibility ("default"))) +# endif +# endif +# if defined(__ELF__) && !defined(SODIUM_DISABLE_WEAK_FUNCTIONS) +# define SODIUM_EXPORT_WEAK SODIUM_EXPORT __attribute__((weak)) +# else +# define SODIUM_EXPORT_WEAK SODIUM_EXPORT +# endif +#endif + +#ifndef CRYPTO_ALIGN +# if defined(__INTEL_COMPILER) || defined(_MSC_VER) +# define CRYPTO_ALIGN(x) __declspec(align(x)) +# else +# define CRYPTO_ALIGN(x) __attribute__ ((aligned(x))) +# endif +#endif + +#define SODIUM_MIN(A, B) ((A) < (B) ? (A) : (B)) +#define SODIUM_SIZE_MAX SODIUM_MIN(UINT64_MAX, SIZE_MAX) + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/randombytes.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/randombytes.h new file mode 100644 index 00000000..a03cc657 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/randombytes.h @@ -0,0 +1,72 @@ + +#ifndef randombytes_H +#define randombytes_H + +#include +#include + +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef struct randombytes_implementation { + const char *(*implementation_name)(void); /* required */ + uint32_t (*random)(void); /* required */ + void (*stir)(void); /* optional */ + uint32_t (*uniform)(const uint32_t upper_bound); /* optional, a default implementation will be used if NULL */ + void (*buf)(void * const buf, const size_t size); /* required */ + int (*close)(void); /* optional */ +} randombytes_implementation; + +#define randombytes_BYTES_MAX SODIUM_MIN(SODIUM_SIZE_MAX, 0xffffffffUL) + +#define randombytes_SEEDBYTES 32U +SODIUM_EXPORT +size_t randombytes_seedbytes(void); + +SODIUM_EXPORT +void randombytes_buf(void * const buf, const size_t size) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void randombytes_buf_deterministic(void * const buf, const size_t size, + const unsigned char seed[randombytes_SEEDBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +uint32_t randombytes_random(void); + +SODIUM_EXPORT +uint32_t randombytes_uniform(const uint32_t upper_bound); + +SODIUM_EXPORT +void randombytes_stir(void); + +SODIUM_EXPORT +int randombytes_close(void); + +SODIUM_EXPORT +int randombytes_set_implementation(randombytes_implementation *impl) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +const char *randombytes_implementation_name(void); + +/* -- NaCl compatibility interface -- */ + +SODIUM_EXPORT +void randombytes(unsigned char * const buf, const unsigned long long buf_len) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/randombytes_internal_random.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/randombytes_internal_random.h new file mode 100644 index 00000000..2b2b7d6e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/randombytes_internal_random.h @@ -0,0 +1,22 @@ + +#ifndef randombytes_internal_random_H +#define randombytes_internal_random_H + +#include "export.h" +#include "randombytes.h" + +#ifdef __cplusplus +extern "C" { +#endif + +SODIUM_EXPORT +extern struct randombytes_implementation randombytes_internal_implementation; + +/* Backwards compatibility with libsodium < 1.0.18 */ +#define randombytes_salsa20_implementation randombytes_internal_implementation + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/randombytes_sysrandom.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/randombytes_sysrandom.h new file mode 100644 index 00000000..9e27b674 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/randombytes_sysrandom.h @@ -0,0 +1,19 @@ + +#ifndef randombytes_sysrandom_H +#define randombytes_sysrandom_H + +#include "export.h" +#include "randombytes.h" + +#ifdef __cplusplus +extern "C" { +#endif + +SODIUM_EXPORT +extern struct randombytes_implementation randombytes_sysrandom_implementation; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/runtime.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/runtime.h new file mode 100644 index 00000000..7f15d58e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/runtime.h @@ -0,0 +1,52 @@ + +#ifndef sodium_runtime_H +#define sodium_runtime_H + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_neon(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_sse2(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_sse3(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_ssse3(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_sse41(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_avx(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_avx2(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_avx512f(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_pclmul(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_aesni(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_rdrand(void); + +/* ------------------------------------------------------------------------- */ + +int _sodium_runtime_get_cpu_features(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/utils.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/utils.h new file mode 100644 index 00000000..ac801512 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/utils.h @@ -0,0 +1,179 @@ + +#ifndef sodium_utils_H +#define sodium_utils_H + +#include + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef SODIUM_C99 +# if defined(__cplusplus) || !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L +# define SODIUM_C99(X) +# else +# define SODIUM_C99(X) X +# endif +#endif + +SODIUM_EXPORT +void sodium_memzero(void * const pnt, const size_t len); + +SODIUM_EXPORT +void sodium_stackzero(const size_t len); + +/* + * WARNING: sodium_memcmp() must be used to verify if two secret keys + * are equal, in constant time. + * It returns 0 if the keys are equal, and -1 if they differ. + * This function is not designed for lexicographical comparisons. + */ +SODIUM_EXPORT +int sodium_memcmp(const void * const b1_, const void * const b2_, size_t len) + __attribute__ ((warn_unused_result)); + +/* + * sodium_compare() returns -1 if b1_ < b2_, 1 if b1_ > b2_ and 0 if b1_ == b2_ + * It is suitable for lexicographical comparisons, or to compare nonces + * and counters stored in little-endian format. + * However, it is slower than sodium_memcmp(). + */ +SODIUM_EXPORT +int sodium_compare(const unsigned char *b1_, const unsigned char *b2_, + size_t len) __attribute__ ((warn_unused_result)); + +SODIUM_EXPORT +int sodium_is_zero(const unsigned char *n, const size_t nlen); + +SODIUM_EXPORT +void sodium_increment(unsigned char *n, const size_t nlen); + +SODIUM_EXPORT +void sodium_add(unsigned char *a, const unsigned char *b, const size_t len); + +SODIUM_EXPORT +void sodium_sub(unsigned char *a, const unsigned char *b, const size_t len); + +SODIUM_EXPORT +char *sodium_bin2hex(char * const hex, const size_t hex_maxlen, + const unsigned char * const bin, const size_t bin_len) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int sodium_hex2bin(unsigned char * const bin, const size_t bin_maxlen, + const char * const hex, const size_t hex_len, + const char * const ignore, size_t * const bin_len, + const char ** const hex_end) + __attribute__ ((nonnull(1))); + +#define sodium_base64_VARIANT_ORIGINAL 1 +#define sodium_base64_VARIANT_ORIGINAL_NO_PADDING 3 +#define sodium_base64_VARIANT_URLSAFE 5 +#define sodium_base64_VARIANT_URLSAFE_NO_PADDING 7 + +/* + * Computes the required length to encode BIN_LEN bytes as a base64 string + * using the given variant. The computed length includes a trailing \0. + */ +#define sodium_base64_ENCODED_LEN(BIN_LEN, VARIANT) \ + (((BIN_LEN) / 3U) * 4U + \ + ((((BIN_LEN) - ((BIN_LEN) / 3U) * 3U) | (((BIN_LEN) - ((BIN_LEN) / 3U) * 3U) >> 1)) & 1U) * \ + (4U - (~((((VARIANT) & 2U) >> 1) - 1U) & (3U - ((BIN_LEN) - ((BIN_LEN) / 3U) * 3U)))) + 1U) + +SODIUM_EXPORT +size_t sodium_base64_encoded_len(const size_t bin_len, const int variant); + +SODIUM_EXPORT +char *sodium_bin2base64(char * const b64, const size_t b64_maxlen, + const unsigned char * const bin, const size_t bin_len, + const int variant) __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int sodium_base642bin(unsigned char * const bin, const size_t bin_maxlen, + const char * const b64, const size_t b64_len, + const char * const ignore, size_t * const bin_len, + const char ** const b64_end, const int variant) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int sodium_mlock(void * const addr, const size_t len) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int sodium_munlock(void * const addr, const size_t len) + __attribute__ ((nonnull)); + +/* WARNING: sodium_malloc() and sodium_allocarray() are not general-purpose + * allocation functions. + * + * They return a pointer to a region filled with 0xd0 bytes, immediately + * followed by a guard page. + * As a result, accessing a single byte after the requested allocation size + * will intentionally trigger a segmentation fault. + * + * A canary and an additional guard page placed before the beginning of the + * region may also kill the process if a buffer underflow is detected. + * + * The memory layout is: + * [unprotected region size (read only)][guard page (no access)][unprotected pages (read/write)][guard page (no access)] + * With the layout of the unprotected pages being: + * [optional padding][16-bytes canary][user region] + * + * However: + * - These functions are significantly slower than standard functions + * - Each allocation requires 3 or 4 additional pages + * - The returned address will not be aligned if the allocation size is not + * a multiple of the required alignment. For this reason, these functions + * are designed to store data, such as secret keys and messages. + * + * sodium_malloc() can be used to allocate any libsodium data structure. + * + * The crypto_generichash_state structure is packed and its length is + * either 357 or 361 bytes. For this reason, when using sodium_malloc() to + * allocate a crypto_generichash_state structure, padding must be added in + * order to ensure proper alignment. crypto_generichash_statebytes() + * returns the rounded up structure size, and should be prefered to sizeof(): + * state = sodium_malloc(crypto_generichash_statebytes()); + */ + +SODIUM_EXPORT +void *sodium_malloc(const size_t size) + __attribute__ ((malloc)); + +SODIUM_EXPORT +void *sodium_allocarray(size_t count, size_t size) + __attribute__ ((malloc)); + +SODIUM_EXPORT +void sodium_free(void *ptr); + +SODIUM_EXPORT +int sodium_mprotect_noaccess(void *ptr) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int sodium_mprotect_readonly(void *ptr) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int sodium_mprotect_readwrite(void *ptr) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int sodium_pad(size_t *padded_buflen_p, unsigned char *buf, + size_t unpadded_buflen, size_t blocksize, size_t max_buflen) + __attribute__ ((nonnull(2))); + +SODIUM_EXPORT +int sodium_unpad(size_t *unpadded_buflen_p, const unsigned char *buf, + size_t padded_buflen, size_t blocksize) + __attribute__ ((nonnull(2))); + +/* -------- */ + +int _sodium_alloc_init(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/version.h b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/version.h new file mode 100644 index 00000000..201a290e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/include/sodium/version.h @@ -0,0 +1,33 @@ + +#ifndef sodium_version_H +#define sodium_version_H + +#include "export.h" + +#define SODIUM_VERSION_STRING "1.0.18" + +#define SODIUM_LIBRARY_VERSION_MAJOR 10 +#define SODIUM_LIBRARY_VERSION_MINOR 3 + + +#ifdef __cplusplus +extern "C" { +#endif + +SODIUM_EXPORT +const char *sodium_version_string(void); + +SODIUM_EXPORT +int sodium_library_version_major(void); + +SODIUM_EXPORT +int sodium_library_version_minor(void); + +SODIUM_EXPORT +int sodium_library_minimal(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/lib/libsodium.a b/example/android/third_party/libsodium/libsodium-android-armv7-a/lib/libsodium.a new file mode 100644 index 00000000..981e22ca Binary files /dev/null and b/example/android/third_party/libsodium/libsodium-android-armv7-a/lib/libsodium.a differ diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/lib/libsodium.la b/example/android/third_party/libsodium/libsodium-android-armv7-a/lib/libsodium.la new file mode 100644 index 00000000..0a5a3478 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/lib/libsodium.la @@ -0,0 +1,41 @@ +# libsodium.la - a libtool library file +# Generated by libtool (GNU libtool) 2.4.6 +# +# Please DO NOT delete this file! +# It is necessary for linking the library. + +# The name that we can dlopen(3). +dlname='libsodium.so' + +# Names of this library. +library_names='libsodium.so' + +# The name of the static archive. +old_library='libsodium.a' + +# Linker flags that cannot go in dependency_libs. +inherited_linker_flags=' -pthread' + +# Libraries that this one depends upon. +dependency_libs='' + +# Names of additional weak libraries provided by this library +weak_library_names='' + +# Version information for libsodium. +current=0 +age=0 +revision=0 + +# Is this an already installed library? +installed=yes + +# Should we warn about portability when linking against -modules? +shouldnotlink=no + +# Files to dlopen/dlpreopen +dlopen='' +dlpreopen='' + +# Directory that this library needs to be installed in: +libdir='/home/alex/magnet/example/android/third_party/libsodium/libsodium-1.0.18/libsodium-android-armv7-a/lib' diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/lib/libsodium.so b/example/android/third_party/libsodium/libsodium-android-armv7-a/lib/libsodium.so new file mode 100644 index 00000000..58caf164 Binary files /dev/null and b/example/android/third_party/libsodium/libsodium-android-armv7-a/lib/libsodium.so differ diff --git a/example/android/third_party/libsodium/libsodium-android-armv7-a/lib/pkgconfig/libsodium.pc b/example/android/third_party/libsodium/libsodium-android-armv7-a/lib/pkgconfig/libsodium.pc new file mode 100644 index 00000000..8ef4197f --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv7-a/lib/pkgconfig/libsodium.pc @@ -0,0 +1,12 @@ +prefix=/home/alex/magnet/example/android/third_party/libsodium/libsodium-1.0.18/libsodium-android-armv7-a +exec_prefix=${prefix} +libdir=${exec_prefix}/lib +includedir=${prefix}/include + +Name: libsodium +Version: 1.0.18 +Description: A modern and easy-to-use crypto library + +Libs: -L${libdir} -lsodium +Libs.private: -pthread +Cflags: -I${includedir} diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium.h new file mode 100644 index 00000000..295f911c --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium.h @@ -0,0 +1,69 @@ + +#ifndef sodium_H +#define sodium_H + +#include "sodium/version.h" + +#include "sodium/core.h" +#include "sodium/crypto_aead_aes256gcm.h" +#include "sodium/crypto_aead_chacha20poly1305.h" +#include "sodium/crypto_aead_xchacha20poly1305.h" +#include "sodium/crypto_auth.h" +#include "sodium/crypto_auth_hmacsha256.h" +#include "sodium/crypto_auth_hmacsha512.h" +#include "sodium/crypto_auth_hmacsha512256.h" +#include "sodium/crypto_box.h" +#include "sodium/crypto_box_curve25519xsalsa20poly1305.h" +#include "sodium/crypto_core_hsalsa20.h" +#include "sodium/crypto_core_hchacha20.h" +#include "sodium/crypto_core_salsa20.h" +#include "sodium/crypto_core_salsa2012.h" +#include "sodium/crypto_core_salsa208.h" +#include "sodium/crypto_generichash.h" +#include "sodium/crypto_generichash_blake2b.h" +#include "sodium/crypto_hash.h" +#include "sodium/crypto_hash_sha256.h" +#include "sodium/crypto_hash_sha512.h" +#include "sodium/crypto_kdf.h" +#include "sodium/crypto_kdf_blake2b.h" +#include "sodium/crypto_kx.h" +#include "sodium/crypto_onetimeauth.h" +#include "sodium/crypto_onetimeauth_poly1305.h" +#include "sodium/crypto_pwhash.h" +#include "sodium/crypto_pwhash_argon2i.h" +#include "sodium/crypto_scalarmult.h" +#include "sodium/crypto_scalarmult_curve25519.h" +#include "sodium/crypto_secretbox.h" +#include "sodium/crypto_secretbox_xsalsa20poly1305.h" +#include "sodium/crypto_secretstream_xchacha20poly1305.h" +#include "sodium/crypto_shorthash.h" +#include "sodium/crypto_shorthash_siphash24.h" +#include "sodium/crypto_sign.h" +#include "sodium/crypto_sign_ed25519.h" +#include "sodium/crypto_stream.h" +#include "sodium/crypto_stream_chacha20.h" +#include "sodium/crypto_stream_salsa20.h" +#include "sodium/crypto_stream_xsalsa20.h" +#include "sodium/crypto_verify_16.h" +#include "sodium/crypto_verify_32.h" +#include "sodium/crypto_verify_64.h" +#include "sodium/randombytes.h" +#include "sodium/randombytes_internal_random.h" +#include "sodium/randombytes_sysrandom.h" +#include "sodium/runtime.h" +#include "sodium/utils.h" + +#ifndef SODIUM_LIBRARY_MINIMAL +# include "sodium/crypto_box_curve25519xchacha20poly1305.h" +# include "sodium/crypto_core_ed25519.h" +# include "sodium/crypto_core_ristretto255.h" +# include "sodium/crypto_scalarmult_ed25519.h" +# include "sodium/crypto_scalarmult_ristretto255.h" +# include "sodium/crypto_secretbox_xchacha20poly1305.h" +# include "sodium/crypto_pwhash_scryptsalsa208sha256.h" +# include "sodium/crypto_stream_salsa2012.h" +# include "sodium/crypto_stream_salsa208.h" +# include "sodium/crypto_stream_xchacha20.h" +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/core.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/core.h new file mode 100644 index 00000000..dd088d2c --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/core.h @@ -0,0 +1,28 @@ + +#ifndef sodium_core_H +#define sodium_core_H + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +SODIUM_EXPORT +int sodium_init(void) + __attribute__ ((warn_unused_result)); + +/* ---- */ + +SODIUM_EXPORT +int sodium_set_misuse_handler(void (*handler)(void)); + +SODIUM_EXPORT +void sodium_misuse(void) + __attribute__ ((noreturn)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_aead_aes256gcm.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_aead_aes256gcm.h new file mode 100644 index 00000000..9baeb3f1 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_aead_aes256gcm.h @@ -0,0 +1,179 @@ +#ifndef crypto_aead_aes256gcm_H +#define crypto_aead_aes256gcm_H + +/* + * WARNING: Despite being the most popular AEAD construction due to its + * use in TLS, safely using AES-GCM in a different context is tricky. + * + * No more than ~ 350 GB of input data should be encrypted with a given key. + * This is for ~ 16 KB messages -- Actual figures vary according to + * message sizes. + * + * In addition, nonces are short and repeated nonces would totally destroy + * the security of this scheme. + * + * Nonces should thus come from atomic counters, which can be difficult to + * set up in a distributed environment. + * + * Unless you absolutely need AES-GCM, use crypto_aead_xchacha20poly1305_ietf_*() + * instead. It doesn't have any of these limitations. + * Or, if you don't need to authenticate additional data, just stick to + * crypto_secretbox(). + */ + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +SODIUM_EXPORT +int crypto_aead_aes256gcm_is_available(void); + +#define crypto_aead_aes256gcm_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_keybytes(void); + +#define crypto_aead_aes256gcm_NSECBYTES 0U +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_nsecbytes(void); + +#define crypto_aead_aes256gcm_NPUBBYTES 12U +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_npubbytes(void); + +#define crypto_aead_aes256gcm_ABYTES 16U +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_abytes(void); + +#define crypto_aead_aes256gcm_MESSAGEBYTES_MAX \ + SODIUM_MIN(SODIUM_SIZE_MAX - crypto_aead_aes256gcm_ABYTES, \ + (16ULL * ((1ULL << 32) - 2ULL))) +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_messagebytes_max(void); + +typedef struct CRYPTO_ALIGN(16) crypto_aead_aes256gcm_state_ { + unsigned char opaque[512]; +} crypto_aead_aes256gcm_state; + +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_statebytes(void); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_encrypt(unsigned char *c, + unsigned long long *clen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_decrypt(unsigned char *m, + unsigned long long *mlen_p, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(4, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_encrypt_detached(unsigned char *c, + unsigned char *mac, + unsigned long long *maclen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 9, 10))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_decrypt_detached(unsigned char *m, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *mac, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5, 8, 9))); + +/* -- Precomputation interface -- */ + +SODIUM_EXPORT +int crypto_aead_aes256gcm_beforenm(crypto_aead_aes256gcm_state *ctx_, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_encrypt_afternm(unsigned char *c, + unsigned long long *clen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const crypto_aead_aes256gcm_state *ctx_) + __attribute__ ((nonnull(1, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_decrypt_afternm(unsigned char *m, + unsigned long long *mlen_p, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const crypto_aead_aes256gcm_state *ctx_) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(4, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_encrypt_detached_afternm(unsigned char *c, + unsigned char *mac, + unsigned long long *maclen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const crypto_aead_aes256gcm_state *ctx_) + __attribute__ ((nonnull(1, 2, 9, 10))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_decrypt_detached_afternm(unsigned char *m, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *mac, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const crypto_aead_aes256gcm_state *ctx_) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5, 8, 9))); + +SODIUM_EXPORT +void crypto_aead_aes256gcm_keygen(unsigned char k[crypto_aead_aes256gcm_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_aead_chacha20poly1305.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_aead_chacha20poly1305.h new file mode 100644 index 00000000..5d671df1 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_aead_chacha20poly1305.h @@ -0,0 +1,180 @@ +#ifndef crypto_aead_chacha20poly1305_H +#define crypto_aead_chacha20poly1305_H + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +/* -- IETF ChaCha20-Poly1305 construction with a 96-bit nonce and a 32-bit internal counter -- */ + +#define crypto_aead_chacha20poly1305_ietf_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_ietf_keybytes(void); + +#define crypto_aead_chacha20poly1305_ietf_NSECBYTES 0U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_ietf_nsecbytes(void); + +#define crypto_aead_chacha20poly1305_ietf_NPUBBYTES 12U + +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_ietf_npubbytes(void); + +#define crypto_aead_chacha20poly1305_ietf_ABYTES 16U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_ietf_abytes(void); + +#define crypto_aead_chacha20poly1305_ietf_MESSAGEBYTES_MAX \ + SODIUM_MIN(SODIUM_SIZE_MAX - crypto_aead_chacha20poly1305_ietf_ABYTES, \ + (64ULL * ((1ULL << 32) - 1ULL))) +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_ietf_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_ietf_encrypt(unsigned char *c, + unsigned long long *clen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_ietf_decrypt(unsigned char *m, + unsigned long long *mlen_p, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(4, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_ietf_encrypt_detached(unsigned char *c, + unsigned char *mac, + unsigned long long *maclen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 9, 10))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_ietf_decrypt_detached(unsigned char *m, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *mac, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5, 8, 9))); + +SODIUM_EXPORT +void crypto_aead_chacha20poly1305_ietf_keygen(unsigned char k[crypto_aead_chacha20poly1305_ietf_KEYBYTES]) + __attribute__ ((nonnull)); + +/* -- Original ChaCha20-Poly1305 construction with a 64-bit nonce and a 64-bit internal counter -- */ + +#define crypto_aead_chacha20poly1305_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_keybytes(void); + +#define crypto_aead_chacha20poly1305_NSECBYTES 0U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_nsecbytes(void); + +#define crypto_aead_chacha20poly1305_NPUBBYTES 8U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_npubbytes(void); + +#define crypto_aead_chacha20poly1305_ABYTES 16U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_abytes(void); + +#define crypto_aead_chacha20poly1305_MESSAGEBYTES_MAX \ + (SODIUM_SIZE_MAX - crypto_aead_chacha20poly1305_ABYTES) +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_encrypt(unsigned char *c, + unsigned long long *clen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_decrypt(unsigned char *m, + unsigned long long *mlen_p, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(4, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_encrypt_detached(unsigned char *c, + unsigned char *mac, + unsigned long long *maclen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 9, 10))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_decrypt_detached(unsigned char *m, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *mac, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5, 8, 9))); + +SODIUM_EXPORT +void crypto_aead_chacha20poly1305_keygen(unsigned char k[crypto_aead_chacha20poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +/* Aliases */ + +#define crypto_aead_chacha20poly1305_IETF_KEYBYTES crypto_aead_chacha20poly1305_ietf_KEYBYTES +#define crypto_aead_chacha20poly1305_IETF_NSECBYTES crypto_aead_chacha20poly1305_ietf_NSECBYTES +#define crypto_aead_chacha20poly1305_IETF_NPUBBYTES crypto_aead_chacha20poly1305_ietf_NPUBBYTES +#define crypto_aead_chacha20poly1305_IETF_ABYTES crypto_aead_chacha20poly1305_ietf_ABYTES +#define crypto_aead_chacha20poly1305_IETF_MESSAGEBYTES_MAX crypto_aead_chacha20poly1305_ietf_MESSAGEBYTES_MAX + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_aead_xchacha20poly1305.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_aead_xchacha20poly1305.h new file mode 100644 index 00000000..6643b0cb --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_aead_xchacha20poly1305.h @@ -0,0 +1,100 @@ +#ifndef crypto_aead_xchacha20poly1305_H +#define crypto_aead_xchacha20poly1305_H + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_aead_xchacha20poly1305_ietf_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_aead_xchacha20poly1305_ietf_keybytes(void); + +#define crypto_aead_xchacha20poly1305_ietf_NSECBYTES 0U +SODIUM_EXPORT +size_t crypto_aead_xchacha20poly1305_ietf_nsecbytes(void); + +#define crypto_aead_xchacha20poly1305_ietf_NPUBBYTES 24U +SODIUM_EXPORT +size_t crypto_aead_xchacha20poly1305_ietf_npubbytes(void); + +#define crypto_aead_xchacha20poly1305_ietf_ABYTES 16U +SODIUM_EXPORT +size_t crypto_aead_xchacha20poly1305_ietf_abytes(void); + +#define crypto_aead_xchacha20poly1305_ietf_MESSAGEBYTES_MAX \ + (SODIUM_SIZE_MAX - crypto_aead_xchacha20poly1305_ietf_ABYTES) +SODIUM_EXPORT +size_t crypto_aead_xchacha20poly1305_ietf_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_aead_xchacha20poly1305_ietf_encrypt(unsigned char *c, + unsigned long long *clen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_xchacha20poly1305_ietf_decrypt(unsigned char *m, + unsigned long long *mlen_p, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(4, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_xchacha20poly1305_ietf_encrypt_detached(unsigned char *c, + unsigned char *mac, + unsigned long long *maclen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 9, 10))); + +SODIUM_EXPORT +int crypto_aead_xchacha20poly1305_ietf_decrypt_detached(unsigned char *m, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *mac, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5, 8, 9))); + +SODIUM_EXPORT +void crypto_aead_xchacha20poly1305_ietf_keygen(unsigned char k[crypto_aead_xchacha20poly1305_ietf_KEYBYTES]) + __attribute__ ((nonnull)); + +/* Aliases */ + +#define crypto_aead_xchacha20poly1305_IETF_KEYBYTES crypto_aead_xchacha20poly1305_ietf_KEYBYTES +#define crypto_aead_xchacha20poly1305_IETF_NSECBYTES crypto_aead_xchacha20poly1305_ietf_NSECBYTES +#define crypto_aead_xchacha20poly1305_IETF_NPUBBYTES crypto_aead_xchacha20poly1305_ietf_NPUBBYTES +#define crypto_aead_xchacha20poly1305_IETF_ABYTES crypto_aead_xchacha20poly1305_ietf_ABYTES +#define crypto_aead_xchacha20poly1305_IETF_MESSAGEBYTES_MAX crypto_aead_xchacha20poly1305_ietf_MESSAGEBYTES_MAX + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_auth.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_auth.h new file mode 100644 index 00000000..540aee0e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_auth.h @@ -0,0 +1,46 @@ +#ifndef crypto_auth_H +#define crypto_auth_H + +#include + +#include "crypto_auth_hmacsha512256.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_auth_BYTES crypto_auth_hmacsha512256_BYTES +SODIUM_EXPORT +size_t crypto_auth_bytes(void); + +#define crypto_auth_KEYBYTES crypto_auth_hmacsha512256_KEYBYTES +SODIUM_EXPORT +size_t crypto_auth_keybytes(void); + +#define crypto_auth_PRIMITIVE "hmacsha512256" +SODIUM_EXPORT +const char *crypto_auth_primitive(void); + +SODIUM_EXPORT +int crypto_auth(unsigned char *out, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_auth_verify(const unsigned char *h, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +void crypto_auth_keygen(unsigned char k[crypto_auth_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_auth_hmacsha256.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_auth_hmacsha256.h new file mode 100644 index 00000000..3da864c7 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_auth_hmacsha256.h @@ -0,0 +1,70 @@ +#ifndef crypto_auth_hmacsha256_H +#define crypto_auth_hmacsha256_H + +#include +#include "crypto_hash_sha256.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_auth_hmacsha256_BYTES 32U +SODIUM_EXPORT +size_t crypto_auth_hmacsha256_bytes(void); + +#define crypto_auth_hmacsha256_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_auth_hmacsha256_keybytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha256(unsigned char *out, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_auth_hmacsha256_verify(const unsigned char *h, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +/* ------------------------------------------------------------------------- */ + +typedef struct crypto_auth_hmacsha256_state { + crypto_hash_sha256_state ictx; + crypto_hash_sha256_state octx; +} crypto_auth_hmacsha256_state; + +SODIUM_EXPORT +size_t crypto_auth_hmacsha256_statebytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha256_init(crypto_auth_hmacsha256_state *state, + const unsigned char *key, + size_t keylen) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_auth_hmacsha256_update(crypto_auth_hmacsha256_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_auth_hmacsha256_final(crypto_auth_hmacsha256_state *state, + unsigned char *out) __attribute__ ((nonnull)); + + +SODIUM_EXPORT +void crypto_auth_hmacsha256_keygen(unsigned char k[crypto_auth_hmacsha256_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_auth_hmacsha512.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_auth_hmacsha512.h new file mode 100644 index 00000000..d992cb81 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_auth_hmacsha512.h @@ -0,0 +1,68 @@ +#ifndef crypto_auth_hmacsha512_H +#define crypto_auth_hmacsha512_H + +#include +#include "crypto_hash_sha512.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_auth_hmacsha512_BYTES 64U +SODIUM_EXPORT +size_t crypto_auth_hmacsha512_bytes(void); + +#define crypto_auth_hmacsha512_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_auth_hmacsha512_keybytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha512(unsigned char *out, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_auth_hmacsha512_verify(const unsigned char *h, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +/* ------------------------------------------------------------------------- */ + +typedef struct crypto_auth_hmacsha512_state { + crypto_hash_sha512_state ictx; + crypto_hash_sha512_state octx; +} crypto_auth_hmacsha512_state; + +SODIUM_EXPORT +size_t crypto_auth_hmacsha512_statebytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha512_init(crypto_auth_hmacsha512_state *state, + const unsigned char *key, + size_t keylen) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_auth_hmacsha512_update(crypto_auth_hmacsha512_state *state, + const unsigned char *in, + unsigned long long inlen) __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_auth_hmacsha512_final(crypto_auth_hmacsha512_state *state, + unsigned char *out) __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_auth_hmacsha512_keygen(unsigned char k[crypto_auth_hmacsha512_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_auth_hmacsha512256.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_auth_hmacsha512256.h new file mode 100644 index 00000000..3fb52638 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_auth_hmacsha512256.h @@ -0,0 +1,65 @@ +#ifndef crypto_auth_hmacsha512256_H +#define crypto_auth_hmacsha512256_H + +#include +#include "crypto_auth_hmacsha512.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_auth_hmacsha512256_BYTES 32U +SODIUM_EXPORT +size_t crypto_auth_hmacsha512256_bytes(void); + +#define crypto_auth_hmacsha512256_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_auth_hmacsha512256_keybytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha512256(unsigned char *out, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_auth_hmacsha512256_verify(const unsigned char *h, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +/* ------------------------------------------------------------------------- */ + +typedef crypto_auth_hmacsha512_state crypto_auth_hmacsha512256_state; + +SODIUM_EXPORT +size_t crypto_auth_hmacsha512256_statebytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha512256_init(crypto_auth_hmacsha512256_state *state, + const unsigned char *key, + size_t keylen) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_auth_hmacsha512256_update(crypto_auth_hmacsha512256_state *state, + const unsigned char *in, + unsigned long long inlen) __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_auth_hmacsha512256_final(crypto_auth_hmacsha512256_state *state, + unsigned char *out) __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_auth_hmacsha512256_keygen(unsigned char k[crypto_auth_hmacsha512256_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_box.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_box.h new file mode 100644 index 00000000..e060dd29 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_box.h @@ -0,0 +1,177 @@ +#ifndef crypto_box_H +#define crypto_box_H + +/* + * THREAD SAFETY: crypto_box_keypair() is thread-safe, + * provided that sodium_init() was called before. + * + * Other functions are always thread-safe. + */ + +#include + +#include "crypto_box_curve25519xsalsa20poly1305.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_box_SEEDBYTES crypto_box_curve25519xsalsa20poly1305_SEEDBYTES +SODIUM_EXPORT +size_t crypto_box_seedbytes(void); + +#define crypto_box_PUBLICKEYBYTES crypto_box_curve25519xsalsa20poly1305_PUBLICKEYBYTES +SODIUM_EXPORT +size_t crypto_box_publickeybytes(void); + +#define crypto_box_SECRETKEYBYTES crypto_box_curve25519xsalsa20poly1305_SECRETKEYBYTES +SODIUM_EXPORT +size_t crypto_box_secretkeybytes(void); + +#define crypto_box_NONCEBYTES crypto_box_curve25519xsalsa20poly1305_NONCEBYTES +SODIUM_EXPORT +size_t crypto_box_noncebytes(void); + +#define crypto_box_MACBYTES crypto_box_curve25519xsalsa20poly1305_MACBYTES +SODIUM_EXPORT +size_t crypto_box_macbytes(void); + +#define crypto_box_MESSAGEBYTES_MAX crypto_box_curve25519xsalsa20poly1305_MESSAGEBYTES_MAX +SODIUM_EXPORT +size_t crypto_box_messagebytes_max(void); + +#define crypto_box_PRIMITIVE "curve25519xsalsa20poly1305" +SODIUM_EXPORT +const char *crypto_box_primitive(void); + +SODIUM_EXPORT +int crypto_box_seed_keypair(unsigned char *pk, unsigned char *sk, + const unsigned char *seed) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_keypair(unsigned char *pk, unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_easy(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *pk, const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_open_easy(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *pk, const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_detached(unsigned char *c, unsigned char *mac, + const unsigned char *m, unsigned long long mlen, + const unsigned char *n, const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 2, 5, 6, 7))); + +SODIUM_EXPORT +int crypto_box_open_detached(unsigned char *m, const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6, 7))); + +/* -- Precomputation interface -- */ + +#define crypto_box_BEFORENMBYTES crypto_box_curve25519xsalsa20poly1305_BEFORENMBYTES +SODIUM_EXPORT +size_t crypto_box_beforenmbytes(void); + +SODIUM_EXPORT +int crypto_box_beforenm(unsigned char *k, const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_easy_afternm(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_box_open_easy_afternm(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +SODIUM_EXPORT +int crypto_box_detached_afternm(unsigned char *c, unsigned char *mac, + const unsigned char *m, unsigned long long mlen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull(1, 2, 5, 6))); + +SODIUM_EXPORT +int crypto_box_open_detached_afternm(unsigned char *m, const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6))); + +/* -- Ephemeral SK interface -- */ + +#define crypto_box_SEALBYTES (crypto_box_PUBLICKEYBYTES + crypto_box_MACBYTES) +SODIUM_EXPORT +size_t crypto_box_sealbytes(void); + +SODIUM_EXPORT +int crypto_box_seal(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *pk) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_box_seal_open(unsigned char *m, const unsigned char *c, + unsigned long long clen, + const unsigned char *pk, const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +/* -- NaCl compatibility interface ; Requires padding -- */ + +#define crypto_box_ZEROBYTES crypto_box_curve25519xsalsa20poly1305_ZEROBYTES +SODIUM_EXPORT +size_t crypto_box_zerobytes(void); + +#define crypto_box_BOXZEROBYTES crypto_box_curve25519xsalsa20poly1305_BOXZEROBYTES +SODIUM_EXPORT +size_t crypto_box_boxzerobytes(void); + +SODIUM_EXPORT +int crypto_box(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *pk, const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_open(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *pk, const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_afternm(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_box_open_afternm(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_box_curve25519xchacha20poly1305.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_box_curve25519xchacha20poly1305.h new file mode 100644 index 00000000..26a3d31e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_box_curve25519xchacha20poly1305.h @@ -0,0 +1,164 @@ + +#ifndef crypto_box_curve25519xchacha20poly1305_H +#define crypto_box_curve25519xchacha20poly1305_H + +#include +#include "crypto_stream_xchacha20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_box_curve25519xchacha20poly1305_SEEDBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_seedbytes(void); + +#define crypto_box_curve25519xchacha20poly1305_PUBLICKEYBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_publickeybytes(void); + +#define crypto_box_curve25519xchacha20poly1305_SECRETKEYBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_secretkeybytes(void); + +#define crypto_box_curve25519xchacha20poly1305_BEFORENMBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_beforenmbytes(void); + +#define crypto_box_curve25519xchacha20poly1305_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_noncebytes(void); + +#define crypto_box_curve25519xchacha20poly1305_MACBYTES 16U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_macbytes(void); + +#define crypto_box_curve25519xchacha20poly1305_MESSAGEBYTES_MAX \ + (crypto_stream_xchacha20_MESSAGEBYTES_MAX - crypto_box_curve25519xchacha20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_seed_keypair(unsigned char *pk, + unsigned char *sk, + const unsigned char *seed) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_keypair(unsigned char *pk, + unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_easy(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_open_easy(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_detached(unsigned char *c, + unsigned char *mac, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 2, 5, 6, 7))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_open_detached(unsigned char *m, + const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6, 7))); + +/* -- Precomputation interface -- */ + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_beforenm(unsigned char *k, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_easy_afternm(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_open_easy_afternm(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_detached_afternm(unsigned char *c, + unsigned char *mac, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 5, 6))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_open_detached_afternm(unsigned char *m, + const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6))); + +/* -- Ephemeral SK interface -- */ + +#define crypto_box_curve25519xchacha20poly1305_SEALBYTES \ + (crypto_box_curve25519xchacha20poly1305_PUBLICKEYBYTES + \ + crypto_box_curve25519xchacha20poly1305_MACBYTES) + +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_sealbytes(void); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_seal(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *pk) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_seal_open(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_box_curve25519xsalsa20poly1305.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_box_curve25519xsalsa20poly1305.h new file mode 100644 index 00000000..e733f499 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_box_curve25519xsalsa20poly1305.h @@ -0,0 +1,112 @@ +#ifndef crypto_box_curve25519xsalsa20poly1305_H +#define crypto_box_curve25519xsalsa20poly1305_H + +#include +#include "crypto_stream_xsalsa20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_box_curve25519xsalsa20poly1305_SEEDBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_seedbytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_PUBLICKEYBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_publickeybytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_SECRETKEYBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_secretkeybytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_BEFORENMBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_beforenmbytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_noncebytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_MACBYTES 16U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_macbytes(void); + +/* Only for the libsodium API - The NaCl compatibility API would require BOXZEROBYTES extra bytes */ +#define crypto_box_curve25519xsalsa20poly1305_MESSAGEBYTES_MAX \ + (crypto_stream_xsalsa20_MESSAGEBYTES_MAX - crypto_box_curve25519xsalsa20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_seed_keypair(unsigned char *pk, + unsigned char *sk, + const unsigned char *seed) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_keypair(unsigned char *pk, + unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_beforenm(unsigned char *k, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +/* -- NaCl compatibility interface ; Requires padding -- */ + +#define crypto_box_curve25519xsalsa20poly1305_BOXZEROBYTES 16U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_boxzerobytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_ZEROBYTES \ + (crypto_box_curve25519xsalsa20poly1305_BOXZEROBYTES + \ + crypto_box_curve25519xsalsa20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_zerobytes(void); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_open(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_afternm(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_open_afternm(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_core_ed25519.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_core_ed25519.h new file mode 100644 index 00000000..3eae00c4 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_core_ed25519.h @@ -0,0 +1,100 @@ +#ifndef crypto_core_ed25519_H +#define crypto_core_ed25519_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_ed25519_BYTES 32 +SODIUM_EXPORT +size_t crypto_core_ed25519_bytes(void); + +#define crypto_core_ed25519_UNIFORMBYTES 32 +SODIUM_EXPORT +size_t crypto_core_ed25519_uniformbytes(void); + +#define crypto_core_ed25519_HASHBYTES 64 +SODIUM_EXPORT +size_t crypto_core_ed25519_hashbytes(void); + +#define crypto_core_ed25519_SCALARBYTES 32 +SODIUM_EXPORT +size_t crypto_core_ed25519_scalarbytes(void); + +#define crypto_core_ed25519_NONREDUCEDSCALARBYTES 64 +SODIUM_EXPORT +size_t crypto_core_ed25519_nonreducedscalarbytes(void); + +SODIUM_EXPORT +int crypto_core_ed25519_is_valid_point(const unsigned char *p) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ed25519_add(unsigned char *r, + const unsigned char *p, const unsigned char *q) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ed25519_sub(unsigned char *r, + const unsigned char *p, const unsigned char *q) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ed25519_from_uniform(unsigned char *p, const unsigned char *r) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ed25519_from_hash(unsigned char *p, const unsigned char *h) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_random(unsigned char *p) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_random(unsigned char *r) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ed25519_scalar_invert(unsigned char *recip, const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_negate(unsigned char *neg, const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_complement(unsigned char *comp, const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_add(unsigned char *z, const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_sub(unsigned char *z, const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_mul(unsigned char *z, const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +/* + * The interval `s` is sampled from should be at least 317 bits to ensure almost + * uniformity of `r` over `L`. + */ +SODIUM_EXPORT +void crypto_core_ed25519_scalar_reduce(unsigned char *r, const unsigned char *s) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_core_hchacha20.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_core_hchacha20.h new file mode 100644 index 00000000..ece141b0 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_core_hchacha20.h @@ -0,0 +1,36 @@ +#ifndef crypto_core_hchacha20_H +#define crypto_core_hchacha20_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_hchacha20_OUTPUTBYTES 32U +SODIUM_EXPORT +size_t crypto_core_hchacha20_outputbytes(void); + +#define crypto_core_hchacha20_INPUTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_hchacha20_inputbytes(void); + +#define crypto_core_hchacha20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_core_hchacha20_keybytes(void); + +#define crypto_core_hchacha20_CONSTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_hchacha20_constbytes(void); + +SODIUM_EXPORT +int crypto_core_hchacha20(unsigned char *out, const unsigned char *in, + const unsigned char *k, const unsigned char *c) + __attribute__ ((nonnull(1, 2, 3))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_core_hsalsa20.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_core_hsalsa20.h new file mode 100644 index 00000000..4bf7a487 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_core_hsalsa20.h @@ -0,0 +1,36 @@ +#ifndef crypto_core_hsalsa20_H +#define crypto_core_hsalsa20_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_hsalsa20_OUTPUTBYTES 32U +SODIUM_EXPORT +size_t crypto_core_hsalsa20_outputbytes(void); + +#define crypto_core_hsalsa20_INPUTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_hsalsa20_inputbytes(void); + +#define crypto_core_hsalsa20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_core_hsalsa20_keybytes(void); + +#define crypto_core_hsalsa20_CONSTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_hsalsa20_constbytes(void); + +SODIUM_EXPORT +int crypto_core_hsalsa20(unsigned char *out, const unsigned char *in, + const unsigned char *k, const unsigned char *c) + __attribute__ ((nonnull(1, 2, 3))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_core_ristretto255.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_core_ristretto255.h new file mode 100644 index 00000000..f2820e55 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_core_ristretto255.h @@ -0,0 +1,100 @@ +#ifndef crypto_core_ristretto255_H +#define crypto_core_ristretto255_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_ristretto255_BYTES 32 +SODIUM_EXPORT +size_t crypto_core_ristretto255_bytes(void); + +#define crypto_core_ristretto255_HASHBYTES 64 +SODIUM_EXPORT +size_t crypto_core_ristretto255_hashbytes(void); + +#define crypto_core_ristretto255_SCALARBYTES 32 +SODIUM_EXPORT +size_t crypto_core_ristretto255_scalarbytes(void); + +#define crypto_core_ristretto255_NONREDUCEDSCALARBYTES 64 +SODIUM_EXPORT +size_t crypto_core_ristretto255_nonreducedscalarbytes(void); + +SODIUM_EXPORT +int crypto_core_ristretto255_is_valid_point(const unsigned char *p) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ristretto255_add(unsigned char *r, + const unsigned char *p, const unsigned char *q) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ristretto255_sub(unsigned char *r, + const unsigned char *p, const unsigned char *q) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ristretto255_from_hash(unsigned char *p, + const unsigned char *r) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_random(unsigned char *p) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_random(unsigned char *r) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ristretto255_scalar_invert(unsigned char *recip, + const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_negate(unsigned char *neg, + const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_complement(unsigned char *comp, + const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_add(unsigned char *z, + const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_sub(unsigned char *z, + const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_mul(unsigned char *z, + const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +/* + * The interval `s` is sampled from should be at least 317 bits to ensure almost + * uniformity of `r` over `L`. + */ +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_reduce(unsigned char *r, + const unsigned char *s) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_core_salsa20.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_core_salsa20.h new file mode 100644 index 00000000..bd79fd9f --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_core_salsa20.h @@ -0,0 +1,36 @@ +#ifndef crypto_core_salsa20_H +#define crypto_core_salsa20_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_salsa20_OUTPUTBYTES 64U +SODIUM_EXPORT +size_t crypto_core_salsa20_outputbytes(void); + +#define crypto_core_salsa20_INPUTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa20_inputbytes(void); + +#define crypto_core_salsa20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_core_salsa20_keybytes(void); + +#define crypto_core_salsa20_CONSTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa20_constbytes(void); + +SODIUM_EXPORT +int crypto_core_salsa20(unsigned char *out, const unsigned char *in, + const unsigned char *k, const unsigned char *c) + __attribute__ ((nonnull(1, 2, 3))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_core_salsa2012.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_core_salsa2012.h new file mode 100644 index 00000000..05957591 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_core_salsa2012.h @@ -0,0 +1,36 @@ +#ifndef crypto_core_salsa2012_H +#define crypto_core_salsa2012_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_salsa2012_OUTPUTBYTES 64U +SODIUM_EXPORT +size_t crypto_core_salsa2012_outputbytes(void); + +#define crypto_core_salsa2012_INPUTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa2012_inputbytes(void); + +#define crypto_core_salsa2012_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_core_salsa2012_keybytes(void); + +#define crypto_core_salsa2012_CONSTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa2012_constbytes(void); + +SODIUM_EXPORT +int crypto_core_salsa2012(unsigned char *out, const unsigned char *in, + const unsigned char *k, const unsigned char *c) + __attribute__ ((nonnull(1, 2, 3))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_core_salsa208.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_core_salsa208.h new file mode 100644 index 00000000..d2f216af --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_core_salsa208.h @@ -0,0 +1,40 @@ +#ifndef crypto_core_salsa208_H +#define crypto_core_salsa208_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_salsa208_OUTPUTBYTES 64U +SODIUM_EXPORT +size_t crypto_core_salsa208_outputbytes(void) + __attribute__ ((deprecated)); + +#define crypto_core_salsa208_INPUTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa208_inputbytes(void) + __attribute__ ((deprecated)); + +#define crypto_core_salsa208_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_core_salsa208_keybytes(void) + __attribute__ ((deprecated)); + +#define crypto_core_salsa208_CONSTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa208_constbytes(void) + __attribute__ ((deprecated)); + +SODIUM_EXPORT +int crypto_core_salsa208(unsigned char *out, const unsigned char *in, + const unsigned char *k, const unsigned char *c) + __attribute__ ((nonnull(1, 2, 3))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_generichash.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_generichash.h new file mode 100644 index 00000000..d897e5d2 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_generichash.h @@ -0,0 +1,84 @@ +#ifndef crypto_generichash_H +#define crypto_generichash_H + +#include + +#include "crypto_generichash_blake2b.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_generichash_BYTES_MIN crypto_generichash_blake2b_BYTES_MIN +SODIUM_EXPORT +size_t crypto_generichash_bytes_min(void); + +#define crypto_generichash_BYTES_MAX crypto_generichash_blake2b_BYTES_MAX +SODIUM_EXPORT +size_t crypto_generichash_bytes_max(void); + +#define crypto_generichash_BYTES crypto_generichash_blake2b_BYTES +SODIUM_EXPORT +size_t crypto_generichash_bytes(void); + +#define crypto_generichash_KEYBYTES_MIN crypto_generichash_blake2b_KEYBYTES_MIN +SODIUM_EXPORT +size_t crypto_generichash_keybytes_min(void); + +#define crypto_generichash_KEYBYTES_MAX crypto_generichash_blake2b_KEYBYTES_MAX +SODIUM_EXPORT +size_t crypto_generichash_keybytes_max(void); + +#define crypto_generichash_KEYBYTES crypto_generichash_blake2b_KEYBYTES +SODIUM_EXPORT +size_t crypto_generichash_keybytes(void); + +#define crypto_generichash_PRIMITIVE "blake2b" +SODIUM_EXPORT +const char *crypto_generichash_primitive(void); + +/* + * Important when writing bindings for other programming languages: + * the state address should be 64-bytes aligned. + */ +typedef crypto_generichash_blake2b_state crypto_generichash_state; + +SODIUM_EXPORT +size_t crypto_generichash_statebytes(void); + +SODIUM_EXPORT +int crypto_generichash(unsigned char *out, size_t outlen, + const unsigned char *in, unsigned long long inlen, + const unsigned char *key, size_t keylen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_init(crypto_generichash_state *state, + const unsigned char *key, + const size_t keylen, const size_t outlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_update(crypto_generichash_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_final(crypto_generichash_state *state, + unsigned char *out, const size_t outlen) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_generichash_keygen(unsigned char k[crypto_generichash_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_generichash_blake2b.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_generichash_blake2b.h new file mode 100644 index 00000000..fee9d8ad --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_generichash_blake2b.h @@ -0,0 +1,118 @@ +#ifndef crypto_generichash_blake2b_H +#define crypto_generichash_blake2b_H + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#if defined(__IBMC__) || defined(__SUNPRO_C) || defined(__SUNPRO_CC) +# pragma pack(1) +#else +# pragma pack(push, 1) +#endif + +typedef struct CRYPTO_ALIGN(64) crypto_generichash_blake2b_state { + unsigned char opaque[384]; +} crypto_generichash_blake2b_state; + +#if defined(__IBMC__) || defined(__SUNPRO_C) || defined(__SUNPRO_CC) +# pragma pack() +#else +# pragma pack(pop) +#endif + +#define crypto_generichash_blake2b_BYTES_MIN 16U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_bytes_min(void); + +#define crypto_generichash_blake2b_BYTES_MAX 64U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_bytes_max(void); + +#define crypto_generichash_blake2b_BYTES 32U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_bytes(void); + +#define crypto_generichash_blake2b_KEYBYTES_MIN 16U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_keybytes_min(void); + +#define crypto_generichash_blake2b_KEYBYTES_MAX 64U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_keybytes_max(void); + +#define crypto_generichash_blake2b_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_keybytes(void); + +#define crypto_generichash_blake2b_SALTBYTES 16U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_saltbytes(void); + +#define crypto_generichash_blake2b_PERSONALBYTES 16U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_personalbytes(void); + +SODIUM_EXPORT +size_t crypto_generichash_blake2b_statebytes(void); + +SODIUM_EXPORT +int crypto_generichash_blake2b(unsigned char *out, size_t outlen, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *key, size_t keylen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_blake2b_salt_personal(unsigned char *out, size_t outlen, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *key, + size_t keylen, + const unsigned char *salt, + const unsigned char *personal) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_blake2b_init(crypto_generichash_blake2b_state *state, + const unsigned char *key, + const size_t keylen, const size_t outlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_blake2b_init_salt_personal(crypto_generichash_blake2b_state *state, + const unsigned char *key, + const size_t keylen, const size_t outlen, + const unsigned char *salt, + const unsigned char *personal) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_blake2b_update(crypto_generichash_blake2b_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_blake2b_final(crypto_generichash_blake2b_state *state, + unsigned char *out, + const size_t outlen) __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_generichash_blake2b_keygen(unsigned char k[crypto_generichash_blake2b_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_hash.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_hash.h new file mode 100644 index 00000000..8752f9ca --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_hash.h @@ -0,0 +1,40 @@ +#ifndef crypto_hash_H +#define crypto_hash_H + +/* + * WARNING: Unless you absolutely need to use SHA512 for interoperatibility, + * purposes, you might want to consider crypto_generichash() instead. + * Unlike SHA512, crypto_generichash() is not vulnerable to length + * extension attacks. + */ + +#include + +#include "crypto_hash_sha512.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_hash_BYTES crypto_hash_sha512_BYTES +SODIUM_EXPORT +size_t crypto_hash_bytes(void); + +SODIUM_EXPORT +int crypto_hash(unsigned char *out, const unsigned char *in, + unsigned long long inlen) __attribute__ ((nonnull(1))); + +#define crypto_hash_PRIMITIVE "sha512" +SODIUM_EXPORT +const char *crypto_hash_primitive(void) + __attribute__ ((warn_unused_result)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_hash_sha256.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_hash_sha256.h new file mode 100644 index 00000000..b18217e1 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_hash_sha256.h @@ -0,0 +1,60 @@ +#ifndef crypto_hash_sha256_H +#define crypto_hash_sha256_H + +/* + * WARNING: Unless you absolutely need to use SHA256 for interoperatibility, + * purposes, you might want to consider crypto_generichash() instead. + * Unlike SHA256, crypto_generichash() is not vulnerable to length + * extension attacks. + */ + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef struct crypto_hash_sha256_state { + uint32_t state[8]; + uint64_t count; + uint8_t buf[64]; +} crypto_hash_sha256_state; + +SODIUM_EXPORT +size_t crypto_hash_sha256_statebytes(void); + +#define crypto_hash_sha256_BYTES 32U +SODIUM_EXPORT +size_t crypto_hash_sha256_bytes(void); + +SODIUM_EXPORT +int crypto_hash_sha256(unsigned char *out, const unsigned char *in, + unsigned long long inlen) __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_hash_sha256_init(crypto_hash_sha256_state *state) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_hash_sha256_update(crypto_hash_sha256_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_hash_sha256_final(crypto_hash_sha256_state *state, + unsigned char *out) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_hash_sha512.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_hash_sha512.h new file mode 100644 index 00000000..8efa7193 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_hash_sha512.h @@ -0,0 +1,60 @@ +#ifndef crypto_hash_sha512_H +#define crypto_hash_sha512_H + +/* + * WARNING: Unless you absolutely need to use SHA512 for interoperatibility, + * purposes, you might want to consider crypto_generichash() instead. + * Unlike SHA512, crypto_generichash() is not vulnerable to length + * extension attacks. + */ + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef struct crypto_hash_sha512_state { + uint64_t state[8]; + uint64_t count[2]; + uint8_t buf[128]; +} crypto_hash_sha512_state; + +SODIUM_EXPORT +size_t crypto_hash_sha512_statebytes(void); + +#define crypto_hash_sha512_BYTES 64U +SODIUM_EXPORT +size_t crypto_hash_sha512_bytes(void); + +SODIUM_EXPORT +int crypto_hash_sha512(unsigned char *out, const unsigned char *in, + unsigned long long inlen) __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_hash_sha512_init(crypto_hash_sha512_state *state) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_hash_sha512_update(crypto_hash_sha512_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_hash_sha512_final(crypto_hash_sha512_state *state, + unsigned char *out) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_kdf.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_kdf.h new file mode 100644 index 00000000..ac2fc618 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_kdf.h @@ -0,0 +1,53 @@ +#ifndef crypto_kdf_H +#define crypto_kdf_H + +#include +#include + +#include "crypto_kdf_blake2b.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_kdf_BYTES_MIN crypto_kdf_blake2b_BYTES_MIN +SODIUM_EXPORT +size_t crypto_kdf_bytes_min(void); + +#define crypto_kdf_BYTES_MAX crypto_kdf_blake2b_BYTES_MAX +SODIUM_EXPORT +size_t crypto_kdf_bytes_max(void); + +#define crypto_kdf_CONTEXTBYTES crypto_kdf_blake2b_CONTEXTBYTES +SODIUM_EXPORT +size_t crypto_kdf_contextbytes(void); + +#define crypto_kdf_KEYBYTES crypto_kdf_blake2b_KEYBYTES +SODIUM_EXPORT +size_t crypto_kdf_keybytes(void); + +#define crypto_kdf_PRIMITIVE "blake2b" +SODIUM_EXPORT +const char *crypto_kdf_primitive(void) + __attribute__ ((warn_unused_result)); + +SODIUM_EXPORT +int crypto_kdf_derive_from_key(unsigned char *subkey, size_t subkey_len, + uint64_t subkey_id, + const char ctx[crypto_kdf_CONTEXTBYTES], + const unsigned char key[crypto_kdf_KEYBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_kdf_keygen(unsigned char k[crypto_kdf_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_kdf_blake2b.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_kdf_blake2b.h new file mode 100644 index 00000000..3ae47dd3 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_kdf_blake2b.h @@ -0,0 +1,44 @@ +#ifndef crypto_kdf_blake2b_H +#define crypto_kdf_blake2b_H + +#include +#include + +#include "crypto_kdf_blake2b.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_kdf_blake2b_BYTES_MIN 16 +SODIUM_EXPORT +size_t crypto_kdf_blake2b_bytes_min(void); + +#define crypto_kdf_blake2b_BYTES_MAX 64 +SODIUM_EXPORT +size_t crypto_kdf_blake2b_bytes_max(void); + +#define crypto_kdf_blake2b_CONTEXTBYTES 8 +SODIUM_EXPORT +size_t crypto_kdf_blake2b_contextbytes(void); + +#define crypto_kdf_blake2b_KEYBYTES 32 +SODIUM_EXPORT +size_t crypto_kdf_blake2b_keybytes(void); + +SODIUM_EXPORT +int crypto_kdf_blake2b_derive_from_key(unsigned char *subkey, size_t subkey_len, + uint64_t subkey_id, + const char ctx[crypto_kdf_blake2b_CONTEXTBYTES], + const unsigned char key[crypto_kdf_blake2b_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_kx.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_kx.h new file mode 100644 index 00000000..347132c3 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_kx.h @@ -0,0 +1,66 @@ +#ifndef crypto_kx_H +#define crypto_kx_H + +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_kx_PUBLICKEYBYTES 32 +SODIUM_EXPORT +size_t crypto_kx_publickeybytes(void); + +#define crypto_kx_SECRETKEYBYTES 32 +SODIUM_EXPORT +size_t crypto_kx_secretkeybytes(void); + +#define crypto_kx_SEEDBYTES 32 +SODIUM_EXPORT +size_t crypto_kx_seedbytes(void); + +#define crypto_kx_SESSIONKEYBYTES 32 +SODIUM_EXPORT +size_t crypto_kx_sessionkeybytes(void); + +#define crypto_kx_PRIMITIVE "x25519blake2b" +SODIUM_EXPORT +const char *crypto_kx_primitive(void); + +SODIUM_EXPORT +int crypto_kx_seed_keypair(unsigned char pk[crypto_kx_PUBLICKEYBYTES], + unsigned char sk[crypto_kx_SECRETKEYBYTES], + const unsigned char seed[crypto_kx_SEEDBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_kx_keypair(unsigned char pk[crypto_kx_PUBLICKEYBYTES], + unsigned char sk[crypto_kx_SECRETKEYBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_kx_client_session_keys(unsigned char rx[crypto_kx_SESSIONKEYBYTES], + unsigned char tx[crypto_kx_SESSIONKEYBYTES], + const unsigned char client_pk[crypto_kx_PUBLICKEYBYTES], + const unsigned char client_sk[crypto_kx_SECRETKEYBYTES], + const unsigned char server_pk[crypto_kx_PUBLICKEYBYTES]) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 4, 5))); + +SODIUM_EXPORT +int crypto_kx_server_session_keys(unsigned char rx[crypto_kx_SESSIONKEYBYTES], + unsigned char tx[crypto_kx_SESSIONKEYBYTES], + const unsigned char server_pk[crypto_kx_PUBLICKEYBYTES], + const unsigned char server_sk[crypto_kx_SECRETKEYBYTES], + const unsigned char client_pk[crypto_kx_PUBLICKEYBYTES]) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 4, 5))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_onetimeauth.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_onetimeauth.h new file mode 100644 index 00000000..7cd7b070 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_onetimeauth.h @@ -0,0 +1,65 @@ +#ifndef crypto_onetimeauth_H +#define crypto_onetimeauth_H + +#include + +#include "crypto_onetimeauth_poly1305.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef crypto_onetimeauth_poly1305_state crypto_onetimeauth_state; + +SODIUM_EXPORT +size_t crypto_onetimeauth_statebytes(void); + +#define crypto_onetimeauth_BYTES crypto_onetimeauth_poly1305_BYTES +SODIUM_EXPORT +size_t crypto_onetimeauth_bytes(void); + +#define crypto_onetimeauth_KEYBYTES crypto_onetimeauth_poly1305_KEYBYTES +SODIUM_EXPORT +size_t crypto_onetimeauth_keybytes(void); + +#define crypto_onetimeauth_PRIMITIVE "poly1305" +SODIUM_EXPORT +const char *crypto_onetimeauth_primitive(void); + +SODIUM_EXPORT +int crypto_onetimeauth(unsigned char *out, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_onetimeauth_verify(const unsigned char *h, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_onetimeauth_init(crypto_onetimeauth_state *state, + const unsigned char *key) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_onetimeauth_update(crypto_onetimeauth_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_onetimeauth_final(crypto_onetimeauth_state *state, + unsigned char *out) __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_onetimeauth_keygen(unsigned char k[crypto_onetimeauth_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_onetimeauth_poly1305.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_onetimeauth_poly1305.h new file mode 100644 index 00000000..f3e34d86 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_onetimeauth_poly1305.h @@ -0,0 +1,72 @@ +#ifndef crypto_onetimeauth_poly1305_H +#define crypto_onetimeauth_poly1305_H + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#include +#include +#include + +#include + +#include "export.h" + +typedef struct CRYPTO_ALIGN(16) crypto_onetimeauth_poly1305_state { + unsigned char opaque[256]; +} crypto_onetimeauth_poly1305_state; + +SODIUM_EXPORT +size_t crypto_onetimeauth_poly1305_statebytes(void); + +#define crypto_onetimeauth_poly1305_BYTES 16U +SODIUM_EXPORT +size_t crypto_onetimeauth_poly1305_bytes(void); + +#define crypto_onetimeauth_poly1305_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_onetimeauth_poly1305_keybytes(void); + +SODIUM_EXPORT +int crypto_onetimeauth_poly1305(unsigned char *out, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_onetimeauth_poly1305_verify(const unsigned char *h, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_onetimeauth_poly1305_init(crypto_onetimeauth_poly1305_state *state, + const unsigned char *key) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_onetimeauth_poly1305_update(crypto_onetimeauth_poly1305_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_onetimeauth_poly1305_final(crypto_onetimeauth_poly1305_state *state, + unsigned char *out) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_onetimeauth_poly1305_keygen(unsigned char k[crypto_onetimeauth_poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_pwhash.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_pwhash.h new file mode 100644 index 00000000..585a993e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_pwhash.h @@ -0,0 +1,147 @@ +#ifndef crypto_pwhash_H +#define crypto_pwhash_H + +#include + +#include "crypto_pwhash_argon2i.h" +#include "crypto_pwhash_argon2id.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_pwhash_ALG_ARGON2I13 crypto_pwhash_argon2i_ALG_ARGON2I13 +SODIUM_EXPORT +int crypto_pwhash_alg_argon2i13(void); + +#define crypto_pwhash_ALG_ARGON2ID13 crypto_pwhash_argon2id_ALG_ARGON2ID13 +SODIUM_EXPORT +int crypto_pwhash_alg_argon2id13(void); + +#define crypto_pwhash_ALG_DEFAULT crypto_pwhash_ALG_ARGON2ID13 +SODIUM_EXPORT +int crypto_pwhash_alg_default(void); + +#define crypto_pwhash_BYTES_MIN crypto_pwhash_argon2id_BYTES_MIN +SODIUM_EXPORT +size_t crypto_pwhash_bytes_min(void); + +#define crypto_pwhash_BYTES_MAX crypto_pwhash_argon2id_BYTES_MAX +SODIUM_EXPORT +size_t crypto_pwhash_bytes_max(void); + +#define crypto_pwhash_PASSWD_MIN crypto_pwhash_argon2id_PASSWD_MIN +SODIUM_EXPORT +size_t crypto_pwhash_passwd_min(void); + +#define crypto_pwhash_PASSWD_MAX crypto_pwhash_argon2id_PASSWD_MAX +SODIUM_EXPORT +size_t crypto_pwhash_passwd_max(void); + +#define crypto_pwhash_SALTBYTES crypto_pwhash_argon2id_SALTBYTES +SODIUM_EXPORT +size_t crypto_pwhash_saltbytes(void); + +#define crypto_pwhash_STRBYTES crypto_pwhash_argon2id_STRBYTES +SODIUM_EXPORT +size_t crypto_pwhash_strbytes(void); + +#define crypto_pwhash_STRPREFIX crypto_pwhash_argon2id_STRPREFIX +SODIUM_EXPORT +const char *crypto_pwhash_strprefix(void); + +#define crypto_pwhash_OPSLIMIT_MIN crypto_pwhash_argon2id_OPSLIMIT_MIN +SODIUM_EXPORT +size_t crypto_pwhash_opslimit_min(void); + +#define crypto_pwhash_OPSLIMIT_MAX crypto_pwhash_argon2id_OPSLIMIT_MAX +SODIUM_EXPORT +size_t crypto_pwhash_opslimit_max(void); + +#define crypto_pwhash_MEMLIMIT_MIN crypto_pwhash_argon2id_MEMLIMIT_MIN +SODIUM_EXPORT +size_t crypto_pwhash_memlimit_min(void); + +#define crypto_pwhash_MEMLIMIT_MAX crypto_pwhash_argon2id_MEMLIMIT_MAX +SODIUM_EXPORT +size_t crypto_pwhash_memlimit_max(void); + +#define crypto_pwhash_OPSLIMIT_INTERACTIVE crypto_pwhash_argon2id_OPSLIMIT_INTERACTIVE +SODIUM_EXPORT +size_t crypto_pwhash_opslimit_interactive(void); + +#define crypto_pwhash_MEMLIMIT_INTERACTIVE crypto_pwhash_argon2id_MEMLIMIT_INTERACTIVE +SODIUM_EXPORT +size_t crypto_pwhash_memlimit_interactive(void); + +#define crypto_pwhash_OPSLIMIT_MODERATE crypto_pwhash_argon2id_OPSLIMIT_MODERATE +SODIUM_EXPORT +size_t crypto_pwhash_opslimit_moderate(void); + +#define crypto_pwhash_MEMLIMIT_MODERATE crypto_pwhash_argon2id_MEMLIMIT_MODERATE +SODIUM_EXPORT +size_t crypto_pwhash_memlimit_moderate(void); + +#define crypto_pwhash_OPSLIMIT_SENSITIVE crypto_pwhash_argon2id_OPSLIMIT_SENSITIVE +SODIUM_EXPORT +size_t crypto_pwhash_opslimit_sensitive(void); + +#define crypto_pwhash_MEMLIMIT_SENSITIVE crypto_pwhash_argon2id_MEMLIMIT_SENSITIVE +SODIUM_EXPORT +size_t crypto_pwhash_memlimit_sensitive(void); + +/* + * With this function, do not forget to store all parameters, including the + * algorithm identifier in order to produce deterministic output. + * The crypto_pwhash_* definitions, including crypto_pwhash_ALG_DEFAULT, + * may change. + */ +SODIUM_EXPORT +int crypto_pwhash(unsigned char * const out, unsigned long long outlen, + const char * const passwd, unsigned long long passwdlen, + const unsigned char * const salt, + unsigned long long opslimit, size_t memlimit, int alg) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +/* + * The output string already includes all the required parameters, including + * the algorithm identifier. The string is all that has to be stored in + * order to verify a password. + */ +SODIUM_EXPORT +int crypto_pwhash_str(char out[crypto_pwhash_STRBYTES], + const char * const passwd, unsigned long long passwdlen, + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_str_alg(char out[crypto_pwhash_STRBYTES], + const char * const passwd, unsigned long long passwdlen, + unsigned long long opslimit, size_t memlimit, int alg) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_str_verify(const char str[crypto_pwhash_STRBYTES], + const char * const passwd, + unsigned long long passwdlen) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_str_needs_rehash(const char str[crypto_pwhash_STRBYTES], + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#define crypto_pwhash_PRIMITIVE "argon2i" +SODIUM_EXPORT +const char *crypto_pwhash_primitive(void) + __attribute__ ((warn_unused_result)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_pwhash_argon2i.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_pwhash_argon2i.h new file mode 100644 index 00000000..88ff6221 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_pwhash_argon2i.h @@ -0,0 +1,122 @@ +#ifndef crypto_pwhash_argon2i_H +#define crypto_pwhash_argon2i_H + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_pwhash_argon2i_ALG_ARGON2I13 1 +SODIUM_EXPORT +int crypto_pwhash_argon2i_alg_argon2i13(void); + +#define crypto_pwhash_argon2i_BYTES_MIN 16U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_bytes_min(void); + +#define crypto_pwhash_argon2i_BYTES_MAX SODIUM_MIN(SODIUM_SIZE_MAX, 4294967295U) +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_bytes_max(void); + +#define crypto_pwhash_argon2i_PASSWD_MIN 0U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_passwd_min(void); + +#define crypto_pwhash_argon2i_PASSWD_MAX 4294967295U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_passwd_max(void); + +#define crypto_pwhash_argon2i_SALTBYTES 16U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_saltbytes(void); + +#define crypto_pwhash_argon2i_STRBYTES 128U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_strbytes(void); + +#define crypto_pwhash_argon2i_STRPREFIX "$argon2i$" +SODIUM_EXPORT +const char *crypto_pwhash_argon2i_strprefix(void); + +#define crypto_pwhash_argon2i_OPSLIMIT_MIN 3U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_opslimit_min(void); + +#define crypto_pwhash_argon2i_OPSLIMIT_MAX 4294967295U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_opslimit_max(void); + +#define crypto_pwhash_argon2i_MEMLIMIT_MIN 8192U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_memlimit_min(void); + +#define crypto_pwhash_argon2i_MEMLIMIT_MAX \ + ((SIZE_MAX >= 4398046510080U) ? 4398046510080U : (SIZE_MAX >= 2147483648U) ? 2147483648U : 32768U) +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_memlimit_max(void); + +#define crypto_pwhash_argon2i_OPSLIMIT_INTERACTIVE 4U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_opslimit_interactive(void); + +#define crypto_pwhash_argon2i_MEMLIMIT_INTERACTIVE 33554432U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_memlimit_interactive(void); + +#define crypto_pwhash_argon2i_OPSLIMIT_MODERATE 6U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_opslimit_moderate(void); + +#define crypto_pwhash_argon2i_MEMLIMIT_MODERATE 134217728U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_memlimit_moderate(void); + +#define crypto_pwhash_argon2i_OPSLIMIT_SENSITIVE 8U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_opslimit_sensitive(void); + +#define crypto_pwhash_argon2i_MEMLIMIT_SENSITIVE 536870912U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_memlimit_sensitive(void); + +SODIUM_EXPORT +int crypto_pwhash_argon2i(unsigned char * const out, + unsigned long long outlen, + const char * const passwd, + unsigned long long passwdlen, + const unsigned char * const salt, + unsigned long long opslimit, size_t memlimit, + int alg) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2i_str(char out[crypto_pwhash_argon2i_STRBYTES], + const char * const passwd, + unsigned long long passwdlen, + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2i_str_verify(const char str[crypto_pwhash_argon2i_STRBYTES], + const char * const passwd, + unsigned long long passwdlen) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2i_str_needs_rehash(const char str[crypto_pwhash_argon2i_STRBYTES], + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_pwhash_argon2id.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_pwhash_argon2id.h new file mode 100644 index 00000000..7183abd1 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_pwhash_argon2id.h @@ -0,0 +1,122 @@ +#ifndef crypto_pwhash_argon2id_H +#define crypto_pwhash_argon2id_H + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_pwhash_argon2id_ALG_ARGON2ID13 2 +SODIUM_EXPORT +int crypto_pwhash_argon2id_alg_argon2id13(void); + +#define crypto_pwhash_argon2id_BYTES_MIN 16U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_bytes_min(void); + +#define crypto_pwhash_argon2id_BYTES_MAX SODIUM_MIN(SODIUM_SIZE_MAX, 4294967295U) +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_bytes_max(void); + +#define crypto_pwhash_argon2id_PASSWD_MIN 0U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_passwd_min(void); + +#define crypto_pwhash_argon2id_PASSWD_MAX 4294967295U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_passwd_max(void); + +#define crypto_pwhash_argon2id_SALTBYTES 16U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_saltbytes(void); + +#define crypto_pwhash_argon2id_STRBYTES 128U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_strbytes(void); + +#define crypto_pwhash_argon2id_STRPREFIX "$argon2id$" +SODIUM_EXPORT +const char *crypto_pwhash_argon2id_strprefix(void); + +#define crypto_pwhash_argon2id_OPSLIMIT_MIN 1U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_opslimit_min(void); + +#define crypto_pwhash_argon2id_OPSLIMIT_MAX 4294967295U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_opslimit_max(void); + +#define crypto_pwhash_argon2id_MEMLIMIT_MIN 8192U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_memlimit_min(void); + +#define crypto_pwhash_argon2id_MEMLIMIT_MAX \ + ((SIZE_MAX >= 4398046510080U) ? 4398046510080U : (SIZE_MAX >= 2147483648U) ? 2147483648U : 32768U) +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_memlimit_max(void); + +#define crypto_pwhash_argon2id_OPSLIMIT_INTERACTIVE 2U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_opslimit_interactive(void); + +#define crypto_pwhash_argon2id_MEMLIMIT_INTERACTIVE 67108864U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_memlimit_interactive(void); + +#define crypto_pwhash_argon2id_OPSLIMIT_MODERATE 3U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_opslimit_moderate(void); + +#define crypto_pwhash_argon2id_MEMLIMIT_MODERATE 268435456U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_memlimit_moderate(void); + +#define crypto_pwhash_argon2id_OPSLIMIT_SENSITIVE 4U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_opslimit_sensitive(void); + +#define crypto_pwhash_argon2id_MEMLIMIT_SENSITIVE 1073741824U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_memlimit_sensitive(void); + +SODIUM_EXPORT +int crypto_pwhash_argon2id(unsigned char * const out, + unsigned long long outlen, + const char * const passwd, + unsigned long long passwdlen, + const unsigned char * const salt, + unsigned long long opslimit, size_t memlimit, + int alg) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2id_str(char out[crypto_pwhash_argon2id_STRBYTES], + const char * const passwd, + unsigned long long passwdlen, + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2id_str_verify(const char str[crypto_pwhash_argon2id_STRBYTES], + const char * const passwd, + unsigned long long passwdlen) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2id_str_needs_rehash(const char str[crypto_pwhash_argon2id_STRBYTES], + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_pwhash_scryptsalsa208sha256.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_pwhash_scryptsalsa208sha256.h new file mode 100644 index 00000000..5c0bf7d3 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_pwhash_scryptsalsa208sha256.h @@ -0,0 +1,120 @@ +#ifndef crypto_pwhash_scryptsalsa208sha256_H +#define crypto_pwhash_scryptsalsa208sha256_H + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_pwhash_scryptsalsa208sha256_BYTES_MIN 16U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_bytes_min(void); + +#define crypto_pwhash_scryptsalsa208sha256_BYTES_MAX \ + SODIUM_MIN(SODIUM_SIZE_MAX, 0x1fffffffe0ULL) +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_bytes_max(void); + +#define crypto_pwhash_scryptsalsa208sha256_PASSWD_MIN 0U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_passwd_min(void); + +#define crypto_pwhash_scryptsalsa208sha256_PASSWD_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_passwd_max(void); + +#define crypto_pwhash_scryptsalsa208sha256_SALTBYTES 32U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_saltbytes(void); + +#define crypto_pwhash_scryptsalsa208sha256_STRBYTES 102U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_strbytes(void); + +#define crypto_pwhash_scryptsalsa208sha256_STRPREFIX "$7$" +SODIUM_EXPORT +const char *crypto_pwhash_scryptsalsa208sha256_strprefix(void); + +#define crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_MIN 32768U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_opslimit_min(void); + +#define crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_MAX 4294967295U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_opslimit_max(void); + +#define crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_MIN 16777216U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_memlimit_min(void); + +#define crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_MAX \ + SODIUM_MIN(SIZE_MAX, 68719476736ULL) +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_memlimit_max(void); + +#define crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_INTERACTIVE 524288U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_opslimit_interactive(void); + +#define crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_INTERACTIVE 16777216U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_memlimit_interactive(void); + +#define crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_SENSITIVE 33554432U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_opslimit_sensitive(void); + +#define crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_SENSITIVE 1073741824U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_memlimit_sensitive(void); + +SODIUM_EXPORT +int crypto_pwhash_scryptsalsa208sha256(unsigned char * const out, + unsigned long long outlen, + const char * const passwd, + unsigned long long passwdlen, + const unsigned char * const salt, + unsigned long long opslimit, + size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_scryptsalsa208sha256_str(char out[crypto_pwhash_scryptsalsa208sha256_STRBYTES], + const char * const passwd, + unsigned long long passwdlen, + unsigned long long opslimit, + size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_scryptsalsa208sha256_str_verify(const char str[crypto_pwhash_scryptsalsa208sha256_STRBYTES], + const char * const passwd, + unsigned long long passwdlen) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_scryptsalsa208sha256_ll(const uint8_t * passwd, size_t passwdlen, + const uint8_t * salt, size_t saltlen, + uint64_t N, uint32_t r, uint32_t p, + uint8_t * buf, size_t buflen) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_scryptsalsa208sha256_str_needs_rehash(const char str[crypto_pwhash_scryptsalsa208sha256_STRBYTES], + unsigned long long opslimit, + size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_scalarmult.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_scalarmult.h new file mode 100644 index 00000000..1c685853 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_scalarmult.h @@ -0,0 +1,46 @@ +#ifndef crypto_scalarmult_H +#define crypto_scalarmult_H + +#include + +#include "crypto_scalarmult_curve25519.h" +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_scalarmult_BYTES crypto_scalarmult_curve25519_BYTES +SODIUM_EXPORT +size_t crypto_scalarmult_bytes(void); + +#define crypto_scalarmult_SCALARBYTES crypto_scalarmult_curve25519_SCALARBYTES +SODIUM_EXPORT +size_t crypto_scalarmult_scalarbytes(void); + +#define crypto_scalarmult_PRIMITIVE "curve25519" +SODIUM_EXPORT +const char *crypto_scalarmult_primitive(void); + +SODIUM_EXPORT +int crypto_scalarmult_base(unsigned char *q, const unsigned char *n) + __attribute__ ((nonnull)); + +/* + * NOTE: Do not use the result of this function directly for key exchange. + * + * Hash the result with the public keys in order to compute a shared + * secret key: H(q || client_pk || server_pk) + * + * Or unless this is not an option, use the crypto_kx() API instead. + */ +SODIUM_EXPORT +int crypto_scalarmult(unsigned char *q, const unsigned char *n, + const unsigned char *p) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_scalarmult_curve25519.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_scalarmult_curve25519.h new file mode 100644 index 00000000..60e9d0c5 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_scalarmult_curve25519.h @@ -0,0 +1,42 @@ +#ifndef crypto_scalarmult_curve25519_H +#define crypto_scalarmult_curve25519_H + +#include + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_scalarmult_curve25519_BYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_curve25519_bytes(void); + +#define crypto_scalarmult_curve25519_SCALARBYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_curve25519_scalarbytes(void); + +/* + * NOTE: Do not use the result of this function directly for key exchange. + * + * Hash the result with the public keys in order to compute a shared + * secret key: H(q || client_pk || server_pk) + * + * Or unless this is not an option, use the crypto_kx() API instead. + */ +SODIUM_EXPORT +int crypto_scalarmult_curve25519(unsigned char *q, const unsigned char *n, + const unsigned char *p) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_scalarmult_curve25519_base(unsigned char *q, + const unsigned char *n) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_scalarmult_ed25519.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_scalarmult_ed25519.h new file mode 100644 index 00000000..2dfa4d70 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_scalarmult_ed25519.h @@ -0,0 +1,51 @@ + +#ifndef crypto_scalarmult_ed25519_H +#define crypto_scalarmult_ed25519_H + +#include + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_scalarmult_ed25519_BYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_ed25519_bytes(void); + +#define crypto_scalarmult_ed25519_SCALARBYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_ed25519_scalarbytes(void); + +/* + * NOTE: Do not use the result of this function directly for key exchange. + * + * Hash the result with the public keys in order to compute a shared + * secret key: H(q || client_pk || server_pk) + * + * Or unless this is not an option, use the crypto_kx() API instead. + */ +SODIUM_EXPORT +int crypto_scalarmult_ed25519(unsigned char *q, const unsigned char *n, + const unsigned char *p) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_scalarmult_ed25519_noclamp(unsigned char *q, const unsigned char *n, + const unsigned char *p) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_scalarmult_ed25519_base(unsigned char *q, const unsigned char *n) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_scalarmult_ed25519_base_noclamp(unsigned char *q, const unsigned char *n) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_scalarmult_ristretto255.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_scalarmult_ristretto255.h new file mode 100644 index 00000000..40a45cce --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_scalarmult_ristretto255.h @@ -0,0 +1,43 @@ + +#ifndef crypto_scalarmult_ristretto255_H +#define crypto_scalarmult_ristretto255_H + +#include + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_scalarmult_ristretto255_BYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_ristretto255_bytes(void); + +#define crypto_scalarmult_ristretto255_SCALARBYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_ristretto255_scalarbytes(void); + +/* + * NOTE: Do not use the result of this function directly for key exchange. + * + * Hash the result with the public keys in order to compute a shared + * secret key: H(q || client_pk || server_pk) + * + * Or unless this is not an option, use the crypto_kx() API instead. + */ +SODIUM_EXPORT +int crypto_scalarmult_ristretto255(unsigned char *q, const unsigned char *n, + const unsigned char *p) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_scalarmult_ristretto255_base(unsigned char *q, + const unsigned char *n) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_secretbox.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_secretbox.h new file mode 100644 index 00000000..1d3709db --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_secretbox.h @@ -0,0 +1,93 @@ +#ifndef crypto_secretbox_H +#define crypto_secretbox_H + +#include + +#include "crypto_secretbox_xsalsa20poly1305.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_secretbox_KEYBYTES crypto_secretbox_xsalsa20poly1305_KEYBYTES +SODIUM_EXPORT +size_t crypto_secretbox_keybytes(void); + +#define crypto_secretbox_NONCEBYTES crypto_secretbox_xsalsa20poly1305_NONCEBYTES +SODIUM_EXPORT +size_t crypto_secretbox_noncebytes(void); + +#define crypto_secretbox_MACBYTES crypto_secretbox_xsalsa20poly1305_MACBYTES +SODIUM_EXPORT +size_t crypto_secretbox_macbytes(void); + +#define crypto_secretbox_PRIMITIVE "xsalsa20poly1305" +SODIUM_EXPORT +const char *crypto_secretbox_primitive(void); + +#define crypto_secretbox_MESSAGEBYTES_MAX crypto_secretbox_xsalsa20poly1305_MESSAGEBYTES_MAX +SODIUM_EXPORT +size_t crypto_secretbox_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_secretbox_easy(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_open_easy(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_detached(unsigned char *c, unsigned char *mac, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 5, 6))); + +SODIUM_EXPORT +int crypto_secretbox_open_detached(unsigned char *m, + const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6))); + +SODIUM_EXPORT +void crypto_secretbox_keygen(unsigned char k[crypto_secretbox_KEYBYTES]) + __attribute__ ((nonnull)); + +/* -- NaCl compatibility interface ; Requires padding -- */ + +#define crypto_secretbox_ZEROBYTES crypto_secretbox_xsalsa20poly1305_ZEROBYTES +SODIUM_EXPORT +size_t crypto_secretbox_zerobytes(void); + +#define crypto_secretbox_BOXZEROBYTES crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES +SODIUM_EXPORT +size_t crypto_secretbox_boxzerobytes(void); + +SODIUM_EXPORT +int crypto_secretbox(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_open(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_secretbox_xchacha20poly1305.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_secretbox_xchacha20poly1305.h new file mode 100644 index 00000000..6ec674e3 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_secretbox_xchacha20poly1305.h @@ -0,0 +1,70 @@ +#ifndef crypto_secretbox_xchacha20poly1305_H +#define crypto_secretbox_xchacha20poly1305_H + +#include +#include "crypto_stream_xchacha20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_secretbox_xchacha20poly1305_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_secretbox_xchacha20poly1305_keybytes(void); + +#define crypto_secretbox_xchacha20poly1305_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_secretbox_xchacha20poly1305_noncebytes(void); + +#define crypto_secretbox_xchacha20poly1305_MACBYTES 16U +SODIUM_EXPORT +size_t crypto_secretbox_xchacha20poly1305_macbytes(void); + +#define crypto_secretbox_xchacha20poly1305_MESSAGEBYTES_MAX \ + (crypto_stream_xchacha20_MESSAGEBYTES_MAX - crypto_secretbox_xchacha20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_secretbox_xchacha20poly1305_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_secretbox_xchacha20poly1305_easy(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_xchacha20poly1305_open_easy(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_xchacha20poly1305_detached(unsigned char *c, + unsigned char *mac, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 5, 6))); + +SODIUM_EXPORT +int crypto_secretbox_xchacha20poly1305_open_detached(unsigned char *m, + const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_secretbox_xsalsa20poly1305.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_secretbox_xsalsa20poly1305.h new file mode 100644 index 00000000..be0874cb --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_secretbox_xsalsa20poly1305.h @@ -0,0 +1,69 @@ +#ifndef crypto_secretbox_xsalsa20poly1305_H +#define crypto_secretbox_xsalsa20poly1305_H + +#include +#include "crypto_stream_xsalsa20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_secretbox_xsalsa20poly1305_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_keybytes(void); + +#define crypto_secretbox_xsalsa20poly1305_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_noncebytes(void); + +#define crypto_secretbox_xsalsa20poly1305_MACBYTES 16U +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_macbytes(void); + +/* Only for the libsodium API - The NaCl compatibility API would require BOXZEROBYTES extra bytes */ +#define crypto_secretbox_xsalsa20poly1305_MESSAGEBYTES_MAX \ + (crypto_stream_xsalsa20_MESSAGEBYTES_MAX - crypto_secretbox_xsalsa20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_secretbox_xsalsa20poly1305(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_xsalsa20poly1305_open(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +SODIUM_EXPORT +void crypto_secretbox_xsalsa20poly1305_keygen(unsigned char k[crypto_secretbox_xsalsa20poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +/* -- NaCl compatibility interface ; Requires padding -- */ + +#define crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES 16U +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_boxzerobytes(void); + +#define crypto_secretbox_xsalsa20poly1305_ZEROBYTES \ + (crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES + \ + crypto_secretbox_xsalsa20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_zerobytes(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_secretstream_xchacha20poly1305.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_secretstream_xchacha20poly1305.h new file mode 100644 index 00000000..b22e4e93 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_secretstream_xchacha20poly1305.h @@ -0,0 +1,108 @@ +#ifndef crypto_secretstream_xchacha20poly1305_H +#define crypto_secretstream_xchacha20poly1305_H + +#include + +#include "crypto_aead_xchacha20poly1305.h" +#include "crypto_stream_chacha20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_secretstream_xchacha20poly1305_ABYTES \ + (1U + crypto_aead_xchacha20poly1305_ietf_ABYTES) +SODIUM_EXPORT +size_t crypto_secretstream_xchacha20poly1305_abytes(void); + +#define crypto_secretstream_xchacha20poly1305_HEADERBYTES \ + crypto_aead_xchacha20poly1305_ietf_NPUBBYTES +SODIUM_EXPORT +size_t crypto_secretstream_xchacha20poly1305_headerbytes(void); + +#define crypto_secretstream_xchacha20poly1305_KEYBYTES \ + crypto_aead_xchacha20poly1305_ietf_KEYBYTES +SODIUM_EXPORT +size_t crypto_secretstream_xchacha20poly1305_keybytes(void); + +#define crypto_secretstream_xchacha20poly1305_MESSAGEBYTES_MAX \ + SODIUM_MIN(SODIUM_SIZE_MAX - crypto_secretstream_xchacha20poly1305_ABYTES, \ + (64ULL * ((1ULL << 32) - 2ULL))) +SODIUM_EXPORT +size_t crypto_secretstream_xchacha20poly1305_messagebytes_max(void); + +#define crypto_secretstream_xchacha20poly1305_TAG_MESSAGE 0x00 +SODIUM_EXPORT +unsigned char crypto_secretstream_xchacha20poly1305_tag_message(void); + +#define crypto_secretstream_xchacha20poly1305_TAG_PUSH 0x01 +SODIUM_EXPORT +unsigned char crypto_secretstream_xchacha20poly1305_tag_push(void); + +#define crypto_secretstream_xchacha20poly1305_TAG_REKEY 0x02 +SODIUM_EXPORT +unsigned char crypto_secretstream_xchacha20poly1305_tag_rekey(void); + +#define crypto_secretstream_xchacha20poly1305_TAG_FINAL \ + (crypto_secretstream_xchacha20poly1305_TAG_PUSH | \ + crypto_secretstream_xchacha20poly1305_TAG_REKEY) +SODIUM_EXPORT +unsigned char crypto_secretstream_xchacha20poly1305_tag_final(void); + +typedef struct crypto_secretstream_xchacha20poly1305_state { + unsigned char k[crypto_stream_chacha20_ietf_KEYBYTES]; + unsigned char nonce[crypto_stream_chacha20_ietf_NONCEBYTES]; + unsigned char _pad[8]; +} crypto_secretstream_xchacha20poly1305_state; + +SODIUM_EXPORT +size_t crypto_secretstream_xchacha20poly1305_statebytes(void); + +SODIUM_EXPORT +void crypto_secretstream_xchacha20poly1305_keygen + (unsigned char k[crypto_secretstream_xchacha20poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_secretstream_xchacha20poly1305_init_push + (crypto_secretstream_xchacha20poly1305_state *state, + unsigned char header[crypto_secretstream_xchacha20poly1305_HEADERBYTES], + const unsigned char k[crypto_secretstream_xchacha20poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_secretstream_xchacha20poly1305_push + (crypto_secretstream_xchacha20poly1305_state *state, + unsigned char *c, unsigned long long *clen_p, + const unsigned char *m, unsigned long long mlen, + const unsigned char *ad, unsigned long long adlen, unsigned char tag) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_secretstream_xchacha20poly1305_init_pull + (crypto_secretstream_xchacha20poly1305_state *state, + const unsigned char header[crypto_secretstream_xchacha20poly1305_HEADERBYTES], + const unsigned char k[crypto_secretstream_xchacha20poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_secretstream_xchacha20poly1305_pull + (crypto_secretstream_xchacha20poly1305_state *state, + unsigned char *m, unsigned long long *mlen_p, unsigned char *tag_p, + const unsigned char *c, unsigned long long clen, + const unsigned char *ad, unsigned long long adlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +void crypto_secretstream_xchacha20poly1305_rekey + (crypto_secretstream_xchacha20poly1305_state *state); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_shorthash.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_shorthash.h new file mode 100644 index 00000000..fecaa88b --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_shorthash.h @@ -0,0 +1,41 @@ +#ifndef crypto_shorthash_H +#define crypto_shorthash_H + +#include + +#include "crypto_shorthash_siphash24.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_shorthash_BYTES crypto_shorthash_siphash24_BYTES +SODIUM_EXPORT +size_t crypto_shorthash_bytes(void); + +#define crypto_shorthash_KEYBYTES crypto_shorthash_siphash24_KEYBYTES +SODIUM_EXPORT +size_t crypto_shorthash_keybytes(void); + +#define crypto_shorthash_PRIMITIVE "siphash24" +SODIUM_EXPORT +const char *crypto_shorthash_primitive(void); + +SODIUM_EXPORT +int crypto_shorthash(unsigned char *out, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +void crypto_shorthash_keygen(unsigned char k[crypto_shorthash_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_shorthash_siphash24.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_shorthash_siphash24.h new file mode 100644 index 00000000..1e6f72a6 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_shorthash_siphash24.h @@ -0,0 +1,50 @@ +#ifndef crypto_shorthash_siphash24_H +#define crypto_shorthash_siphash24_H + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +/* -- 64-bit output -- */ + +#define crypto_shorthash_siphash24_BYTES 8U +SODIUM_EXPORT +size_t crypto_shorthash_siphash24_bytes(void); + +#define crypto_shorthash_siphash24_KEYBYTES 16U +SODIUM_EXPORT +size_t crypto_shorthash_siphash24_keybytes(void); + +SODIUM_EXPORT +int crypto_shorthash_siphash24(unsigned char *out, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((nonnull(1, 4))); + +#ifndef SODIUM_LIBRARY_MINIMAL +/* -- 128-bit output -- */ + +#define crypto_shorthash_siphashx24_BYTES 16U +SODIUM_EXPORT +size_t crypto_shorthash_siphashx24_bytes(void); + +#define crypto_shorthash_siphashx24_KEYBYTES 16U +SODIUM_EXPORT +size_t crypto_shorthash_siphashx24_keybytes(void); + +SODIUM_EXPORT +int crypto_shorthash_siphashx24(unsigned char *out, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((nonnull(1, 4))); +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_sign.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_sign.h new file mode 100644 index 00000000..f5fafb12 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_sign.h @@ -0,0 +1,107 @@ +#ifndef crypto_sign_H +#define crypto_sign_H + +/* + * THREAD SAFETY: crypto_sign_keypair() is thread-safe, + * provided that sodium_init() was called before. + * + * Other functions, including crypto_sign_seed_keypair() are always thread-safe. + */ + +#include + +#include "crypto_sign_ed25519.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef crypto_sign_ed25519ph_state crypto_sign_state; + +SODIUM_EXPORT +size_t crypto_sign_statebytes(void); + +#define crypto_sign_BYTES crypto_sign_ed25519_BYTES +SODIUM_EXPORT +size_t crypto_sign_bytes(void); + +#define crypto_sign_SEEDBYTES crypto_sign_ed25519_SEEDBYTES +SODIUM_EXPORT +size_t crypto_sign_seedbytes(void); + +#define crypto_sign_PUBLICKEYBYTES crypto_sign_ed25519_PUBLICKEYBYTES +SODIUM_EXPORT +size_t crypto_sign_publickeybytes(void); + +#define crypto_sign_SECRETKEYBYTES crypto_sign_ed25519_SECRETKEYBYTES +SODIUM_EXPORT +size_t crypto_sign_secretkeybytes(void); + +#define crypto_sign_MESSAGEBYTES_MAX crypto_sign_ed25519_MESSAGEBYTES_MAX +SODIUM_EXPORT +size_t crypto_sign_messagebytes_max(void); + +#define crypto_sign_PRIMITIVE "ed25519" +SODIUM_EXPORT +const char *crypto_sign_primitive(void); + +SODIUM_EXPORT +int crypto_sign_seed_keypair(unsigned char *pk, unsigned char *sk, + const unsigned char *seed) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_keypair(unsigned char *pk, unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign(unsigned char *sm, unsigned long long *smlen_p, + const unsigned char *m, unsigned long long mlen, + const unsigned char *sk) __attribute__ ((nonnull(1, 5))); + +SODIUM_EXPORT +int crypto_sign_open(unsigned char *m, unsigned long long *mlen_p, + const unsigned char *sm, unsigned long long smlen, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5))); + +SODIUM_EXPORT +int crypto_sign_detached(unsigned char *sig, unsigned long long *siglen_p, + const unsigned char *m, unsigned long long mlen, + const unsigned char *sk) __attribute__ ((nonnull(1, 5))); + +SODIUM_EXPORT +int crypto_sign_verify_detached(const unsigned char *sig, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_sign_init(crypto_sign_state *state); + +SODIUM_EXPORT +int crypto_sign_update(crypto_sign_state *state, + const unsigned char *m, unsigned long long mlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_sign_final_create(crypto_sign_state *state, unsigned char *sig, + unsigned long long *siglen_p, + const unsigned char *sk) + __attribute__ ((nonnull(1, 2, 4))); + +SODIUM_EXPORT +int crypto_sign_final_verify(crypto_sign_state *state, const unsigned char *sig, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_sign_ed25519.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_sign_ed25519.h new file mode 100644 index 00000000..0fdac42d --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_sign_ed25519.h @@ -0,0 +1,124 @@ +#ifndef crypto_sign_ed25519_H +#define crypto_sign_ed25519_H + +#include +#include "crypto_hash_sha512.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef struct crypto_sign_ed25519ph_state { + crypto_hash_sha512_state hs; +} crypto_sign_ed25519ph_state; + +SODIUM_EXPORT +size_t crypto_sign_ed25519ph_statebytes(void); + +#define crypto_sign_ed25519_BYTES 64U +SODIUM_EXPORT +size_t crypto_sign_ed25519_bytes(void); + +#define crypto_sign_ed25519_SEEDBYTES 32U +SODIUM_EXPORT +size_t crypto_sign_ed25519_seedbytes(void); + +#define crypto_sign_ed25519_PUBLICKEYBYTES 32U +SODIUM_EXPORT +size_t crypto_sign_ed25519_publickeybytes(void); + +#define crypto_sign_ed25519_SECRETKEYBYTES (32U + 32U) +SODIUM_EXPORT +size_t crypto_sign_ed25519_secretkeybytes(void); + +#define crypto_sign_ed25519_MESSAGEBYTES_MAX (SODIUM_SIZE_MAX - crypto_sign_ed25519_BYTES) +SODIUM_EXPORT +size_t crypto_sign_ed25519_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_sign_ed25519(unsigned char *sm, unsigned long long *smlen_p, + const unsigned char *m, unsigned long long mlen, + const unsigned char *sk) + __attribute__ ((nonnull(1, 5))); + +SODIUM_EXPORT +int crypto_sign_ed25519_open(unsigned char *m, unsigned long long *mlen_p, + const unsigned char *sm, unsigned long long smlen, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5))); + +SODIUM_EXPORT +int crypto_sign_ed25519_detached(unsigned char *sig, + unsigned long long *siglen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *sk) + __attribute__ ((nonnull(1, 5))); + +SODIUM_EXPORT +int crypto_sign_ed25519_verify_detached(const unsigned char *sig, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_sign_ed25519_keypair(unsigned char *pk, unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519_seed_keypair(unsigned char *pk, unsigned char *sk, + const unsigned char *seed) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519_pk_to_curve25519(unsigned char *curve25519_pk, + const unsigned char *ed25519_pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519_sk_to_curve25519(unsigned char *curve25519_sk, + const unsigned char *ed25519_sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519_sk_to_seed(unsigned char *seed, + const unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519_sk_to_pk(unsigned char *pk, const unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519ph_init(crypto_sign_ed25519ph_state *state) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519ph_update(crypto_sign_ed25519ph_state *state, + const unsigned char *m, + unsigned long long mlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_sign_ed25519ph_final_create(crypto_sign_ed25519ph_state *state, + unsigned char *sig, + unsigned long long *siglen_p, + const unsigned char *sk) + __attribute__ ((nonnull(1, 2, 4))); + +SODIUM_EXPORT +int crypto_sign_ed25519ph_final_verify(crypto_sign_ed25519ph_state *state, + const unsigned char *sig, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_sign_edwards25519sha512batch.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_sign_edwards25519sha512batch.h new file mode 100644 index 00000000..eed158aa --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_sign_edwards25519sha512batch.h @@ -0,0 +1,55 @@ +#ifndef crypto_sign_edwards25519sha512batch_H +#define crypto_sign_edwards25519sha512batch_H + +/* + * WARNING: This construction was a prototype, which should not be used + * any more in new projects. + * + * crypto_sign_edwards25519sha512batch is provided for applications + * initially built with NaCl, but as recommended by the author of this + * construction, new applications should use ed25519 instead. + * + * In Sodium, you should use the high-level crypto_sign_*() functions instead. + */ + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_sign_edwards25519sha512batch_BYTES 64U +#define crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES 32U +#define crypto_sign_edwards25519sha512batch_SECRETKEYBYTES (32U + 32U) +#define crypto_sign_edwards25519sha512batch_MESSAGEBYTES_MAX (SODIUM_SIZE_MAX - crypto_sign_edwards25519sha512batch_BYTES) + +SODIUM_EXPORT +int crypto_sign_edwards25519sha512batch(unsigned char *sm, + unsigned long long *smlen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *sk) + __attribute__ ((deprecated)) __attribute__ ((nonnull(1, 5))); + +SODIUM_EXPORT +int crypto_sign_edwards25519sha512batch_open(unsigned char *m, + unsigned long long *mlen_p, + const unsigned char *sm, + unsigned long long smlen, + const unsigned char *pk) + __attribute__ ((deprecated)) __attribute__ ((nonnull(3, 5))); + +SODIUM_EXPORT +int crypto_sign_edwards25519sha512batch_keypair(unsigned char *pk, + unsigned char *sk) + __attribute__ ((deprecated)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_stream.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_stream.h new file mode 100644 index 00000000..88dab5f6 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_stream.h @@ -0,0 +1,59 @@ +#ifndef crypto_stream_H +#define crypto_stream_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include + +#include "crypto_stream_xsalsa20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_KEYBYTES crypto_stream_xsalsa20_KEYBYTES +SODIUM_EXPORT +size_t crypto_stream_keybytes(void); + +#define crypto_stream_NONCEBYTES crypto_stream_xsalsa20_NONCEBYTES +SODIUM_EXPORT +size_t crypto_stream_noncebytes(void); + +#define crypto_stream_MESSAGEBYTES_MAX crypto_stream_xsalsa20_MESSAGEBYTES_MAX +SODIUM_EXPORT +size_t crypto_stream_messagebytes_max(void); + +#define crypto_stream_PRIMITIVE "xsalsa20" +SODIUM_EXPORT +const char *crypto_stream_primitive(void); + +SODIUM_EXPORT +int crypto_stream(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_keygen(unsigned char k[crypto_stream_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_stream_chacha20.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_stream_chacha20.h new file mode 100644 index 00000000..40889755 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_stream_chacha20.h @@ -0,0 +1,106 @@ +#ifndef crypto_stream_chacha20_H +#define crypto_stream_chacha20_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_chacha20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_chacha20_keybytes(void); + +#define crypto_stream_chacha20_NONCEBYTES 8U +SODIUM_EXPORT +size_t crypto_stream_chacha20_noncebytes(void); + +#define crypto_stream_chacha20_MESSAGEBYTES_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_stream_chacha20_messagebytes_max(void); + +/* ChaCha20 with a 64-bit nonce and a 64-bit counter, as originally designed */ + +SODIUM_EXPORT +int crypto_stream_chacha20(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_chacha20_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_chacha20_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, uint64_t ic, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_chacha20_keygen(unsigned char k[crypto_stream_chacha20_KEYBYTES]) + __attribute__ ((nonnull)); + +/* ChaCha20 with a 96-bit nonce and a 32-bit counter (IETF) */ + +#define crypto_stream_chacha20_ietf_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_chacha20_ietf_keybytes(void); + +#define crypto_stream_chacha20_ietf_NONCEBYTES 12U +SODIUM_EXPORT +size_t crypto_stream_chacha20_ietf_noncebytes(void); + +#define crypto_stream_chacha20_ietf_MESSAGEBYTES_MAX \ + SODIUM_MIN(SODIUM_SIZE_MAX, 64ULL * (1ULL << 32)) +SODIUM_EXPORT +size_t crypto_stream_chacha20_ietf_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_stream_chacha20_ietf(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_chacha20_ietf_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_chacha20_ietf_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, uint32_t ic, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_chacha20_ietf_keygen(unsigned char k[crypto_stream_chacha20_ietf_KEYBYTES]) + __attribute__ ((nonnull)); + +/* Aliases */ + +#define crypto_stream_chacha20_IETF_KEYBYTES crypto_stream_chacha20_ietf_KEYBYTES +#define crypto_stream_chacha20_IETF_NONCEBYTES crypto_stream_chacha20_ietf_NONCEBYTES +#define crypto_stream_chacha20_IETF_MESSAGEBYTES_MAX crypto_stream_chacha20_ietf_MESSAGEBYTES_MAX + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_stream_salsa20.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_stream_salsa20.h new file mode 100644 index 00000000..45b3b3e3 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_stream_salsa20.h @@ -0,0 +1,61 @@ +#ifndef crypto_stream_salsa20_H +#define crypto_stream_salsa20_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_salsa20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_salsa20_keybytes(void); + +#define crypto_stream_salsa20_NONCEBYTES 8U +SODIUM_EXPORT +size_t crypto_stream_salsa20_noncebytes(void); + +#define crypto_stream_salsa20_MESSAGEBYTES_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_stream_salsa20_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_stream_salsa20(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_salsa20_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_salsa20_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, uint64_t ic, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_salsa20_keygen(unsigned char k[crypto_stream_salsa20_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_stream_salsa2012.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_stream_salsa2012.h new file mode 100644 index 00000000..6c5d303c --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_stream_salsa2012.h @@ -0,0 +1,53 @@ +#ifndef crypto_stream_salsa2012_H +#define crypto_stream_salsa2012_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_salsa2012_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_salsa2012_keybytes(void); + +#define crypto_stream_salsa2012_NONCEBYTES 8U +SODIUM_EXPORT +size_t crypto_stream_salsa2012_noncebytes(void); + +#define crypto_stream_salsa2012_MESSAGEBYTES_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_stream_salsa2012_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_stream_salsa2012(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_salsa2012_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_salsa2012_keygen(unsigned char k[crypto_stream_salsa2012_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_stream_salsa208.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_stream_salsa208.h new file mode 100644 index 00000000..d574f304 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_stream_salsa208.h @@ -0,0 +1,56 @@ +#ifndef crypto_stream_salsa208_H +#define crypto_stream_salsa208_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_salsa208_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_salsa208_keybytes(void) + __attribute__ ((deprecated)); + +#define crypto_stream_salsa208_NONCEBYTES 8U +SODIUM_EXPORT +size_t crypto_stream_salsa208_noncebytes(void) + __attribute__ ((deprecated)); + +#define crypto_stream_salsa208_MESSAGEBYTES_MAX SODIUM_SIZE_MAX + SODIUM_EXPORT +size_t crypto_stream_salsa208_messagebytes_max(void) + __attribute__ ((deprecated)); + +SODIUM_EXPORT +int crypto_stream_salsa208(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((deprecated)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_salsa208_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((deprecated)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_salsa208_keygen(unsigned char k[crypto_stream_salsa208_KEYBYTES]) + __attribute__ ((deprecated)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_stream_xchacha20.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_stream_xchacha20.h new file mode 100644 index 00000000..c4002db0 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_stream_xchacha20.h @@ -0,0 +1,61 @@ +#ifndef crypto_stream_xchacha20_H +#define crypto_stream_xchacha20_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_xchacha20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_xchacha20_keybytes(void); + +#define crypto_stream_xchacha20_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_stream_xchacha20_noncebytes(void); + +#define crypto_stream_xchacha20_MESSAGEBYTES_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_stream_xchacha20_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_stream_xchacha20(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_xchacha20_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_xchacha20_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, uint64_t ic, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_xchacha20_keygen(unsigned char k[crypto_stream_xchacha20_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_stream_xsalsa20.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_stream_xsalsa20.h new file mode 100644 index 00000000..20034e34 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_stream_xsalsa20.h @@ -0,0 +1,61 @@ +#ifndef crypto_stream_xsalsa20_H +#define crypto_stream_xsalsa20_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_xsalsa20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_xsalsa20_keybytes(void); + +#define crypto_stream_xsalsa20_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_stream_xsalsa20_noncebytes(void); + +#define crypto_stream_xsalsa20_MESSAGEBYTES_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_stream_xsalsa20_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_stream_xsalsa20(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_xsalsa20_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_xsalsa20_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, uint64_t ic, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_xsalsa20_keygen(unsigned char k[crypto_stream_xsalsa20_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_verify_16.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_verify_16.h new file mode 100644 index 00000000..7b9c8077 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_verify_16.h @@ -0,0 +1,23 @@ +#ifndef crypto_verify_16_H +#define crypto_verify_16_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_verify_16_BYTES 16U +SODIUM_EXPORT +size_t crypto_verify_16_bytes(void); + +SODIUM_EXPORT +int crypto_verify_16(const unsigned char *x, const unsigned char *y) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_verify_32.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_verify_32.h new file mode 100644 index 00000000..9b0f4529 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_verify_32.h @@ -0,0 +1,23 @@ +#ifndef crypto_verify_32_H +#define crypto_verify_32_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_verify_32_BYTES 32U +SODIUM_EXPORT +size_t crypto_verify_32_bytes(void); + +SODIUM_EXPORT +int crypto_verify_32(const unsigned char *x, const unsigned char *y) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_verify_64.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_verify_64.h new file mode 100644 index 00000000..c83b7302 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/crypto_verify_64.h @@ -0,0 +1,23 @@ +#ifndef crypto_verify_64_H +#define crypto_verify_64_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_verify_64_BYTES 64U +SODIUM_EXPORT +size_t crypto_verify_64_bytes(void); + +SODIUM_EXPORT +int crypto_verify_64(const unsigned char *x, const unsigned char *y) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/export.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/export.h new file mode 100644 index 00000000..a0074fc9 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/export.h @@ -0,0 +1,57 @@ + +#ifndef sodium_export_H +#define sodium_export_H + +#include +#include +#include + +#if !defined(__clang__) && !defined(__GNUC__) +# ifdef __attribute__ +# undef __attribute__ +# endif +# define __attribute__(a) +#endif + +#ifdef SODIUM_STATIC +# define SODIUM_EXPORT +# define SODIUM_EXPORT_WEAK +#else +# if defined(_MSC_VER) +# ifdef SODIUM_DLL_EXPORT +# define SODIUM_EXPORT __declspec(dllexport) +# else +# define SODIUM_EXPORT __declspec(dllimport) +# endif +# else +# if defined(__SUNPRO_C) +# ifndef __GNU_C__ +# define SODIUM_EXPORT __attribute__ (visibility(__global)) +# else +# define SODIUM_EXPORT __attribute__ __global +# endif +# elif defined(_MSG_VER) +# define SODIUM_EXPORT extern __declspec(dllexport) +# else +# define SODIUM_EXPORT __attribute__ ((visibility ("default"))) +# endif +# endif +# if defined(__ELF__) && !defined(SODIUM_DISABLE_WEAK_FUNCTIONS) +# define SODIUM_EXPORT_WEAK SODIUM_EXPORT __attribute__((weak)) +# else +# define SODIUM_EXPORT_WEAK SODIUM_EXPORT +# endif +#endif + +#ifndef CRYPTO_ALIGN +# if defined(__INTEL_COMPILER) || defined(_MSC_VER) +# define CRYPTO_ALIGN(x) __declspec(align(x)) +# else +# define CRYPTO_ALIGN(x) __attribute__ ((aligned(x))) +# endif +#endif + +#define SODIUM_MIN(A, B) ((A) < (B) ? (A) : (B)) +#define SODIUM_SIZE_MAX SODIUM_MIN(UINT64_MAX, SIZE_MAX) + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/randombytes.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/randombytes.h new file mode 100644 index 00000000..a03cc657 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/randombytes.h @@ -0,0 +1,72 @@ + +#ifndef randombytes_H +#define randombytes_H + +#include +#include + +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef struct randombytes_implementation { + const char *(*implementation_name)(void); /* required */ + uint32_t (*random)(void); /* required */ + void (*stir)(void); /* optional */ + uint32_t (*uniform)(const uint32_t upper_bound); /* optional, a default implementation will be used if NULL */ + void (*buf)(void * const buf, const size_t size); /* required */ + int (*close)(void); /* optional */ +} randombytes_implementation; + +#define randombytes_BYTES_MAX SODIUM_MIN(SODIUM_SIZE_MAX, 0xffffffffUL) + +#define randombytes_SEEDBYTES 32U +SODIUM_EXPORT +size_t randombytes_seedbytes(void); + +SODIUM_EXPORT +void randombytes_buf(void * const buf, const size_t size) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void randombytes_buf_deterministic(void * const buf, const size_t size, + const unsigned char seed[randombytes_SEEDBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +uint32_t randombytes_random(void); + +SODIUM_EXPORT +uint32_t randombytes_uniform(const uint32_t upper_bound); + +SODIUM_EXPORT +void randombytes_stir(void); + +SODIUM_EXPORT +int randombytes_close(void); + +SODIUM_EXPORT +int randombytes_set_implementation(randombytes_implementation *impl) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +const char *randombytes_implementation_name(void); + +/* -- NaCl compatibility interface -- */ + +SODIUM_EXPORT +void randombytes(unsigned char * const buf, const unsigned long long buf_len) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/randombytes_internal_random.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/randombytes_internal_random.h new file mode 100644 index 00000000..2b2b7d6e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/randombytes_internal_random.h @@ -0,0 +1,22 @@ + +#ifndef randombytes_internal_random_H +#define randombytes_internal_random_H + +#include "export.h" +#include "randombytes.h" + +#ifdef __cplusplus +extern "C" { +#endif + +SODIUM_EXPORT +extern struct randombytes_implementation randombytes_internal_implementation; + +/* Backwards compatibility with libsodium < 1.0.18 */ +#define randombytes_salsa20_implementation randombytes_internal_implementation + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/randombytes_sysrandom.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/randombytes_sysrandom.h new file mode 100644 index 00000000..9e27b674 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/randombytes_sysrandom.h @@ -0,0 +1,19 @@ + +#ifndef randombytes_sysrandom_H +#define randombytes_sysrandom_H + +#include "export.h" +#include "randombytes.h" + +#ifdef __cplusplus +extern "C" { +#endif + +SODIUM_EXPORT +extern struct randombytes_implementation randombytes_sysrandom_implementation; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/runtime.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/runtime.h new file mode 100644 index 00000000..7f15d58e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/runtime.h @@ -0,0 +1,52 @@ + +#ifndef sodium_runtime_H +#define sodium_runtime_H + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_neon(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_sse2(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_sse3(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_ssse3(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_sse41(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_avx(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_avx2(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_avx512f(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_pclmul(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_aesni(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_rdrand(void); + +/* ------------------------------------------------------------------------- */ + +int _sodium_runtime_get_cpu_features(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/utils.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/utils.h new file mode 100644 index 00000000..ac801512 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/utils.h @@ -0,0 +1,179 @@ + +#ifndef sodium_utils_H +#define sodium_utils_H + +#include + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef SODIUM_C99 +# if defined(__cplusplus) || !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L +# define SODIUM_C99(X) +# else +# define SODIUM_C99(X) X +# endif +#endif + +SODIUM_EXPORT +void sodium_memzero(void * const pnt, const size_t len); + +SODIUM_EXPORT +void sodium_stackzero(const size_t len); + +/* + * WARNING: sodium_memcmp() must be used to verify if two secret keys + * are equal, in constant time. + * It returns 0 if the keys are equal, and -1 if they differ. + * This function is not designed for lexicographical comparisons. + */ +SODIUM_EXPORT +int sodium_memcmp(const void * const b1_, const void * const b2_, size_t len) + __attribute__ ((warn_unused_result)); + +/* + * sodium_compare() returns -1 if b1_ < b2_, 1 if b1_ > b2_ and 0 if b1_ == b2_ + * It is suitable for lexicographical comparisons, or to compare nonces + * and counters stored in little-endian format. + * However, it is slower than sodium_memcmp(). + */ +SODIUM_EXPORT +int sodium_compare(const unsigned char *b1_, const unsigned char *b2_, + size_t len) __attribute__ ((warn_unused_result)); + +SODIUM_EXPORT +int sodium_is_zero(const unsigned char *n, const size_t nlen); + +SODIUM_EXPORT +void sodium_increment(unsigned char *n, const size_t nlen); + +SODIUM_EXPORT +void sodium_add(unsigned char *a, const unsigned char *b, const size_t len); + +SODIUM_EXPORT +void sodium_sub(unsigned char *a, const unsigned char *b, const size_t len); + +SODIUM_EXPORT +char *sodium_bin2hex(char * const hex, const size_t hex_maxlen, + const unsigned char * const bin, const size_t bin_len) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int sodium_hex2bin(unsigned char * const bin, const size_t bin_maxlen, + const char * const hex, const size_t hex_len, + const char * const ignore, size_t * const bin_len, + const char ** const hex_end) + __attribute__ ((nonnull(1))); + +#define sodium_base64_VARIANT_ORIGINAL 1 +#define sodium_base64_VARIANT_ORIGINAL_NO_PADDING 3 +#define sodium_base64_VARIANT_URLSAFE 5 +#define sodium_base64_VARIANT_URLSAFE_NO_PADDING 7 + +/* + * Computes the required length to encode BIN_LEN bytes as a base64 string + * using the given variant. The computed length includes a trailing \0. + */ +#define sodium_base64_ENCODED_LEN(BIN_LEN, VARIANT) \ + (((BIN_LEN) / 3U) * 4U + \ + ((((BIN_LEN) - ((BIN_LEN) / 3U) * 3U) | (((BIN_LEN) - ((BIN_LEN) / 3U) * 3U) >> 1)) & 1U) * \ + (4U - (~((((VARIANT) & 2U) >> 1) - 1U) & (3U - ((BIN_LEN) - ((BIN_LEN) / 3U) * 3U)))) + 1U) + +SODIUM_EXPORT +size_t sodium_base64_encoded_len(const size_t bin_len, const int variant); + +SODIUM_EXPORT +char *sodium_bin2base64(char * const b64, const size_t b64_maxlen, + const unsigned char * const bin, const size_t bin_len, + const int variant) __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int sodium_base642bin(unsigned char * const bin, const size_t bin_maxlen, + const char * const b64, const size_t b64_len, + const char * const ignore, size_t * const bin_len, + const char ** const b64_end, const int variant) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int sodium_mlock(void * const addr, const size_t len) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int sodium_munlock(void * const addr, const size_t len) + __attribute__ ((nonnull)); + +/* WARNING: sodium_malloc() and sodium_allocarray() are not general-purpose + * allocation functions. + * + * They return a pointer to a region filled with 0xd0 bytes, immediately + * followed by a guard page. + * As a result, accessing a single byte after the requested allocation size + * will intentionally trigger a segmentation fault. + * + * A canary and an additional guard page placed before the beginning of the + * region may also kill the process if a buffer underflow is detected. + * + * The memory layout is: + * [unprotected region size (read only)][guard page (no access)][unprotected pages (read/write)][guard page (no access)] + * With the layout of the unprotected pages being: + * [optional padding][16-bytes canary][user region] + * + * However: + * - These functions are significantly slower than standard functions + * - Each allocation requires 3 or 4 additional pages + * - The returned address will not be aligned if the allocation size is not + * a multiple of the required alignment. For this reason, these functions + * are designed to store data, such as secret keys and messages. + * + * sodium_malloc() can be used to allocate any libsodium data structure. + * + * The crypto_generichash_state structure is packed and its length is + * either 357 or 361 bytes. For this reason, when using sodium_malloc() to + * allocate a crypto_generichash_state structure, padding must be added in + * order to ensure proper alignment. crypto_generichash_statebytes() + * returns the rounded up structure size, and should be prefered to sizeof(): + * state = sodium_malloc(crypto_generichash_statebytes()); + */ + +SODIUM_EXPORT +void *sodium_malloc(const size_t size) + __attribute__ ((malloc)); + +SODIUM_EXPORT +void *sodium_allocarray(size_t count, size_t size) + __attribute__ ((malloc)); + +SODIUM_EXPORT +void sodium_free(void *ptr); + +SODIUM_EXPORT +int sodium_mprotect_noaccess(void *ptr) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int sodium_mprotect_readonly(void *ptr) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int sodium_mprotect_readwrite(void *ptr) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int sodium_pad(size_t *padded_buflen_p, unsigned char *buf, + size_t unpadded_buflen, size_t blocksize, size_t max_buflen) + __attribute__ ((nonnull(2))); + +SODIUM_EXPORT +int sodium_unpad(size_t *unpadded_buflen_p, const unsigned char *buf, + size_t padded_buflen, size_t blocksize) + __attribute__ ((nonnull(2))); + +/* -------- */ + +int _sodium_alloc_init(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/version.h b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/version.h new file mode 100644 index 00000000..201a290e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/include/sodium/version.h @@ -0,0 +1,33 @@ + +#ifndef sodium_version_H +#define sodium_version_H + +#include "export.h" + +#define SODIUM_VERSION_STRING "1.0.18" + +#define SODIUM_LIBRARY_VERSION_MAJOR 10 +#define SODIUM_LIBRARY_VERSION_MINOR 3 + + +#ifdef __cplusplus +extern "C" { +#endif + +SODIUM_EXPORT +const char *sodium_version_string(void); + +SODIUM_EXPORT +int sodium_library_version_major(void); + +SODIUM_EXPORT +int sodium_library_version_minor(void); + +SODIUM_EXPORT +int sodium_library_minimal(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/lib/libsodium.a b/example/android/third_party/libsodium/libsodium-android-armv8-a/lib/libsodium.a new file mode 100644 index 00000000..865ac560 Binary files /dev/null and b/example/android/third_party/libsodium/libsodium-android-armv8-a/lib/libsodium.a differ diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/lib/libsodium.la b/example/android/third_party/libsodium/libsodium-android-armv8-a/lib/libsodium.la new file mode 100644 index 00000000..4248f421 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/lib/libsodium.la @@ -0,0 +1,41 @@ +# libsodium.la - a libtool library file +# Generated by libtool (GNU libtool) 2.4.6 +# +# Please DO NOT delete this file! +# It is necessary for linking the library. + +# The name that we can dlopen(3). +dlname='libsodium.so' + +# Names of this library. +library_names='libsodium.so' + +# The name of the static archive. +old_library='libsodium.a' + +# Linker flags that cannot go in dependency_libs. +inherited_linker_flags=' -pthread' + +# Libraries that this one depends upon. +dependency_libs='' + +# Names of additional weak libraries provided by this library +weak_library_names='' + +# Version information for libsodium. +current=0 +age=0 +revision=0 + +# Is this an already installed library? +installed=yes + +# Should we warn about portability when linking against -modules? +shouldnotlink=no + +# Files to dlopen/dlpreopen +dlopen='' +dlpreopen='' + +# Directory that this library needs to be installed in: +libdir='/home/alex/magnet/example/android/third_party/libsodium/libsodium-1.0.18/libsodium-android-armv8-a/lib' diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/lib/libsodium.so b/example/android/third_party/libsodium/libsodium-android-armv8-a/lib/libsodium.so new file mode 100644 index 00000000..b5fe5034 Binary files /dev/null and b/example/android/third_party/libsodium/libsodium-android-armv8-a/lib/libsodium.so differ diff --git a/example/android/third_party/libsodium/libsodium-android-armv8-a/lib/pkgconfig/libsodium.pc b/example/android/third_party/libsodium/libsodium-android-armv8-a/lib/pkgconfig/libsodium.pc new file mode 100644 index 00000000..e4e15745 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-armv8-a/lib/pkgconfig/libsodium.pc @@ -0,0 +1,12 @@ +prefix=/home/alex/magnet/example/android/third_party/libsodium/libsodium-1.0.18/libsodium-android-armv8-a +exec_prefix=${prefix} +libdir=${exec_prefix}/lib +includedir=${prefix}/include + +Name: libsodium +Version: 1.0.18 +Description: A modern and easy-to-use crypto library + +Libs: -L${libdir} -lsodium +Libs.private: -pthread +Cflags: -I${includedir} diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium.h new file mode 100644 index 00000000..295f911c --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium.h @@ -0,0 +1,69 @@ + +#ifndef sodium_H +#define sodium_H + +#include "sodium/version.h" + +#include "sodium/core.h" +#include "sodium/crypto_aead_aes256gcm.h" +#include "sodium/crypto_aead_chacha20poly1305.h" +#include "sodium/crypto_aead_xchacha20poly1305.h" +#include "sodium/crypto_auth.h" +#include "sodium/crypto_auth_hmacsha256.h" +#include "sodium/crypto_auth_hmacsha512.h" +#include "sodium/crypto_auth_hmacsha512256.h" +#include "sodium/crypto_box.h" +#include "sodium/crypto_box_curve25519xsalsa20poly1305.h" +#include "sodium/crypto_core_hsalsa20.h" +#include "sodium/crypto_core_hchacha20.h" +#include "sodium/crypto_core_salsa20.h" +#include "sodium/crypto_core_salsa2012.h" +#include "sodium/crypto_core_salsa208.h" +#include "sodium/crypto_generichash.h" +#include "sodium/crypto_generichash_blake2b.h" +#include "sodium/crypto_hash.h" +#include "sodium/crypto_hash_sha256.h" +#include "sodium/crypto_hash_sha512.h" +#include "sodium/crypto_kdf.h" +#include "sodium/crypto_kdf_blake2b.h" +#include "sodium/crypto_kx.h" +#include "sodium/crypto_onetimeauth.h" +#include "sodium/crypto_onetimeauth_poly1305.h" +#include "sodium/crypto_pwhash.h" +#include "sodium/crypto_pwhash_argon2i.h" +#include "sodium/crypto_scalarmult.h" +#include "sodium/crypto_scalarmult_curve25519.h" +#include "sodium/crypto_secretbox.h" +#include "sodium/crypto_secretbox_xsalsa20poly1305.h" +#include "sodium/crypto_secretstream_xchacha20poly1305.h" +#include "sodium/crypto_shorthash.h" +#include "sodium/crypto_shorthash_siphash24.h" +#include "sodium/crypto_sign.h" +#include "sodium/crypto_sign_ed25519.h" +#include "sodium/crypto_stream.h" +#include "sodium/crypto_stream_chacha20.h" +#include "sodium/crypto_stream_salsa20.h" +#include "sodium/crypto_stream_xsalsa20.h" +#include "sodium/crypto_verify_16.h" +#include "sodium/crypto_verify_32.h" +#include "sodium/crypto_verify_64.h" +#include "sodium/randombytes.h" +#include "sodium/randombytes_internal_random.h" +#include "sodium/randombytes_sysrandom.h" +#include "sodium/runtime.h" +#include "sodium/utils.h" + +#ifndef SODIUM_LIBRARY_MINIMAL +# include "sodium/crypto_box_curve25519xchacha20poly1305.h" +# include "sodium/crypto_core_ed25519.h" +# include "sodium/crypto_core_ristretto255.h" +# include "sodium/crypto_scalarmult_ed25519.h" +# include "sodium/crypto_scalarmult_ristretto255.h" +# include "sodium/crypto_secretbox_xchacha20poly1305.h" +# include "sodium/crypto_pwhash_scryptsalsa208sha256.h" +# include "sodium/crypto_stream_salsa2012.h" +# include "sodium/crypto_stream_salsa208.h" +# include "sodium/crypto_stream_xchacha20.h" +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/core.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/core.h new file mode 100644 index 00000000..dd088d2c --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/core.h @@ -0,0 +1,28 @@ + +#ifndef sodium_core_H +#define sodium_core_H + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +SODIUM_EXPORT +int sodium_init(void) + __attribute__ ((warn_unused_result)); + +/* ---- */ + +SODIUM_EXPORT +int sodium_set_misuse_handler(void (*handler)(void)); + +SODIUM_EXPORT +void sodium_misuse(void) + __attribute__ ((noreturn)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_aead_aes256gcm.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_aead_aes256gcm.h new file mode 100644 index 00000000..9baeb3f1 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_aead_aes256gcm.h @@ -0,0 +1,179 @@ +#ifndef crypto_aead_aes256gcm_H +#define crypto_aead_aes256gcm_H + +/* + * WARNING: Despite being the most popular AEAD construction due to its + * use in TLS, safely using AES-GCM in a different context is tricky. + * + * No more than ~ 350 GB of input data should be encrypted with a given key. + * This is for ~ 16 KB messages -- Actual figures vary according to + * message sizes. + * + * In addition, nonces are short and repeated nonces would totally destroy + * the security of this scheme. + * + * Nonces should thus come from atomic counters, which can be difficult to + * set up in a distributed environment. + * + * Unless you absolutely need AES-GCM, use crypto_aead_xchacha20poly1305_ietf_*() + * instead. It doesn't have any of these limitations. + * Or, if you don't need to authenticate additional data, just stick to + * crypto_secretbox(). + */ + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +SODIUM_EXPORT +int crypto_aead_aes256gcm_is_available(void); + +#define crypto_aead_aes256gcm_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_keybytes(void); + +#define crypto_aead_aes256gcm_NSECBYTES 0U +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_nsecbytes(void); + +#define crypto_aead_aes256gcm_NPUBBYTES 12U +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_npubbytes(void); + +#define crypto_aead_aes256gcm_ABYTES 16U +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_abytes(void); + +#define crypto_aead_aes256gcm_MESSAGEBYTES_MAX \ + SODIUM_MIN(SODIUM_SIZE_MAX - crypto_aead_aes256gcm_ABYTES, \ + (16ULL * ((1ULL << 32) - 2ULL))) +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_messagebytes_max(void); + +typedef struct CRYPTO_ALIGN(16) crypto_aead_aes256gcm_state_ { + unsigned char opaque[512]; +} crypto_aead_aes256gcm_state; + +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_statebytes(void); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_encrypt(unsigned char *c, + unsigned long long *clen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_decrypt(unsigned char *m, + unsigned long long *mlen_p, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(4, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_encrypt_detached(unsigned char *c, + unsigned char *mac, + unsigned long long *maclen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 9, 10))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_decrypt_detached(unsigned char *m, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *mac, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5, 8, 9))); + +/* -- Precomputation interface -- */ + +SODIUM_EXPORT +int crypto_aead_aes256gcm_beforenm(crypto_aead_aes256gcm_state *ctx_, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_encrypt_afternm(unsigned char *c, + unsigned long long *clen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const crypto_aead_aes256gcm_state *ctx_) + __attribute__ ((nonnull(1, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_decrypt_afternm(unsigned char *m, + unsigned long long *mlen_p, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const crypto_aead_aes256gcm_state *ctx_) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(4, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_encrypt_detached_afternm(unsigned char *c, + unsigned char *mac, + unsigned long long *maclen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const crypto_aead_aes256gcm_state *ctx_) + __attribute__ ((nonnull(1, 2, 9, 10))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_decrypt_detached_afternm(unsigned char *m, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *mac, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const crypto_aead_aes256gcm_state *ctx_) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5, 8, 9))); + +SODIUM_EXPORT +void crypto_aead_aes256gcm_keygen(unsigned char k[crypto_aead_aes256gcm_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_aead_chacha20poly1305.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_aead_chacha20poly1305.h new file mode 100644 index 00000000..5d671df1 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_aead_chacha20poly1305.h @@ -0,0 +1,180 @@ +#ifndef crypto_aead_chacha20poly1305_H +#define crypto_aead_chacha20poly1305_H + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +/* -- IETF ChaCha20-Poly1305 construction with a 96-bit nonce and a 32-bit internal counter -- */ + +#define crypto_aead_chacha20poly1305_ietf_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_ietf_keybytes(void); + +#define crypto_aead_chacha20poly1305_ietf_NSECBYTES 0U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_ietf_nsecbytes(void); + +#define crypto_aead_chacha20poly1305_ietf_NPUBBYTES 12U + +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_ietf_npubbytes(void); + +#define crypto_aead_chacha20poly1305_ietf_ABYTES 16U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_ietf_abytes(void); + +#define crypto_aead_chacha20poly1305_ietf_MESSAGEBYTES_MAX \ + SODIUM_MIN(SODIUM_SIZE_MAX - crypto_aead_chacha20poly1305_ietf_ABYTES, \ + (64ULL * ((1ULL << 32) - 1ULL))) +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_ietf_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_ietf_encrypt(unsigned char *c, + unsigned long long *clen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_ietf_decrypt(unsigned char *m, + unsigned long long *mlen_p, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(4, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_ietf_encrypt_detached(unsigned char *c, + unsigned char *mac, + unsigned long long *maclen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 9, 10))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_ietf_decrypt_detached(unsigned char *m, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *mac, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5, 8, 9))); + +SODIUM_EXPORT +void crypto_aead_chacha20poly1305_ietf_keygen(unsigned char k[crypto_aead_chacha20poly1305_ietf_KEYBYTES]) + __attribute__ ((nonnull)); + +/* -- Original ChaCha20-Poly1305 construction with a 64-bit nonce and a 64-bit internal counter -- */ + +#define crypto_aead_chacha20poly1305_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_keybytes(void); + +#define crypto_aead_chacha20poly1305_NSECBYTES 0U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_nsecbytes(void); + +#define crypto_aead_chacha20poly1305_NPUBBYTES 8U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_npubbytes(void); + +#define crypto_aead_chacha20poly1305_ABYTES 16U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_abytes(void); + +#define crypto_aead_chacha20poly1305_MESSAGEBYTES_MAX \ + (SODIUM_SIZE_MAX - crypto_aead_chacha20poly1305_ABYTES) +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_encrypt(unsigned char *c, + unsigned long long *clen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_decrypt(unsigned char *m, + unsigned long long *mlen_p, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(4, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_encrypt_detached(unsigned char *c, + unsigned char *mac, + unsigned long long *maclen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 9, 10))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_decrypt_detached(unsigned char *m, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *mac, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5, 8, 9))); + +SODIUM_EXPORT +void crypto_aead_chacha20poly1305_keygen(unsigned char k[crypto_aead_chacha20poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +/* Aliases */ + +#define crypto_aead_chacha20poly1305_IETF_KEYBYTES crypto_aead_chacha20poly1305_ietf_KEYBYTES +#define crypto_aead_chacha20poly1305_IETF_NSECBYTES crypto_aead_chacha20poly1305_ietf_NSECBYTES +#define crypto_aead_chacha20poly1305_IETF_NPUBBYTES crypto_aead_chacha20poly1305_ietf_NPUBBYTES +#define crypto_aead_chacha20poly1305_IETF_ABYTES crypto_aead_chacha20poly1305_ietf_ABYTES +#define crypto_aead_chacha20poly1305_IETF_MESSAGEBYTES_MAX crypto_aead_chacha20poly1305_ietf_MESSAGEBYTES_MAX + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_aead_xchacha20poly1305.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_aead_xchacha20poly1305.h new file mode 100644 index 00000000..6643b0cb --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_aead_xchacha20poly1305.h @@ -0,0 +1,100 @@ +#ifndef crypto_aead_xchacha20poly1305_H +#define crypto_aead_xchacha20poly1305_H + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_aead_xchacha20poly1305_ietf_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_aead_xchacha20poly1305_ietf_keybytes(void); + +#define crypto_aead_xchacha20poly1305_ietf_NSECBYTES 0U +SODIUM_EXPORT +size_t crypto_aead_xchacha20poly1305_ietf_nsecbytes(void); + +#define crypto_aead_xchacha20poly1305_ietf_NPUBBYTES 24U +SODIUM_EXPORT +size_t crypto_aead_xchacha20poly1305_ietf_npubbytes(void); + +#define crypto_aead_xchacha20poly1305_ietf_ABYTES 16U +SODIUM_EXPORT +size_t crypto_aead_xchacha20poly1305_ietf_abytes(void); + +#define crypto_aead_xchacha20poly1305_ietf_MESSAGEBYTES_MAX \ + (SODIUM_SIZE_MAX - crypto_aead_xchacha20poly1305_ietf_ABYTES) +SODIUM_EXPORT +size_t crypto_aead_xchacha20poly1305_ietf_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_aead_xchacha20poly1305_ietf_encrypt(unsigned char *c, + unsigned long long *clen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_xchacha20poly1305_ietf_decrypt(unsigned char *m, + unsigned long long *mlen_p, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(4, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_xchacha20poly1305_ietf_encrypt_detached(unsigned char *c, + unsigned char *mac, + unsigned long long *maclen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 9, 10))); + +SODIUM_EXPORT +int crypto_aead_xchacha20poly1305_ietf_decrypt_detached(unsigned char *m, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *mac, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5, 8, 9))); + +SODIUM_EXPORT +void crypto_aead_xchacha20poly1305_ietf_keygen(unsigned char k[crypto_aead_xchacha20poly1305_ietf_KEYBYTES]) + __attribute__ ((nonnull)); + +/* Aliases */ + +#define crypto_aead_xchacha20poly1305_IETF_KEYBYTES crypto_aead_xchacha20poly1305_ietf_KEYBYTES +#define crypto_aead_xchacha20poly1305_IETF_NSECBYTES crypto_aead_xchacha20poly1305_ietf_NSECBYTES +#define crypto_aead_xchacha20poly1305_IETF_NPUBBYTES crypto_aead_xchacha20poly1305_ietf_NPUBBYTES +#define crypto_aead_xchacha20poly1305_IETF_ABYTES crypto_aead_xchacha20poly1305_ietf_ABYTES +#define crypto_aead_xchacha20poly1305_IETF_MESSAGEBYTES_MAX crypto_aead_xchacha20poly1305_ietf_MESSAGEBYTES_MAX + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_auth.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_auth.h new file mode 100644 index 00000000..540aee0e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_auth.h @@ -0,0 +1,46 @@ +#ifndef crypto_auth_H +#define crypto_auth_H + +#include + +#include "crypto_auth_hmacsha512256.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_auth_BYTES crypto_auth_hmacsha512256_BYTES +SODIUM_EXPORT +size_t crypto_auth_bytes(void); + +#define crypto_auth_KEYBYTES crypto_auth_hmacsha512256_KEYBYTES +SODIUM_EXPORT +size_t crypto_auth_keybytes(void); + +#define crypto_auth_PRIMITIVE "hmacsha512256" +SODIUM_EXPORT +const char *crypto_auth_primitive(void); + +SODIUM_EXPORT +int crypto_auth(unsigned char *out, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_auth_verify(const unsigned char *h, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +void crypto_auth_keygen(unsigned char k[crypto_auth_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_auth_hmacsha256.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_auth_hmacsha256.h new file mode 100644 index 00000000..3da864c7 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_auth_hmacsha256.h @@ -0,0 +1,70 @@ +#ifndef crypto_auth_hmacsha256_H +#define crypto_auth_hmacsha256_H + +#include +#include "crypto_hash_sha256.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_auth_hmacsha256_BYTES 32U +SODIUM_EXPORT +size_t crypto_auth_hmacsha256_bytes(void); + +#define crypto_auth_hmacsha256_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_auth_hmacsha256_keybytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha256(unsigned char *out, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_auth_hmacsha256_verify(const unsigned char *h, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +/* ------------------------------------------------------------------------- */ + +typedef struct crypto_auth_hmacsha256_state { + crypto_hash_sha256_state ictx; + crypto_hash_sha256_state octx; +} crypto_auth_hmacsha256_state; + +SODIUM_EXPORT +size_t crypto_auth_hmacsha256_statebytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha256_init(crypto_auth_hmacsha256_state *state, + const unsigned char *key, + size_t keylen) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_auth_hmacsha256_update(crypto_auth_hmacsha256_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_auth_hmacsha256_final(crypto_auth_hmacsha256_state *state, + unsigned char *out) __attribute__ ((nonnull)); + + +SODIUM_EXPORT +void crypto_auth_hmacsha256_keygen(unsigned char k[crypto_auth_hmacsha256_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_auth_hmacsha512.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_auth_hmacsha512.h new file mode 100644 index 00000000..d992cb81 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_auth_hmacsha512.h @@ -0,0 +1,68 @@ +#ifndef crypto_auth_hmacsha512_H +#define crypto_auth_hmacsha512_H + +#include +#include "crypto_hash_sha512.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_auth_hmacsha512_BYTES 64U +SODIUM_EXPORT +size_t crypto_auth_hmacsha512_bytes(void); + +#define crypto_auth_hmacsha512_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_auth_hmacsha512_keybytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha512(unsigned char *out, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_auth_hmacsha512_verify(const unsigned char *h, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +/* ------------------------------------------------------------------------- */ + +typedef struct crypto_auth_hmacsha512_state { + crypto_hash_sha512_state ictx; + crypto_hash_sha512_state octx; +} crypto_auth_hmacsha512_state; + +SODIUM_EXPORT +size_t crypto_auth_hmacsha512_statebytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha512_init(crypto_auth_hmacsha512_state *state, + const unsigned char *key, + size_t keylen) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_auth_hmacsha512_update(crypto_auth_hmacsha512_state *state, + const unsigned char *in, + unsigned long long inlen) __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_auth_hmacsha512_final(crypto_auth_hmacsha512_state *state, + unsigned char *out) __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_auth_hmacsha512_keygen(unsigned char k[crypto_auth_hmacsha512_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_auth_hmacsha512256.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_auth_hmacsha512256.h new file mode 100644 index 00000000..3fb52638 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_auth_hmacsha512256.h @@ -0,0 +1,65 @@ +#ifndef crypto_auth_hmacsha512256_H +#define crypto_auth_hmacsha512256_H + +#include +#include "crypto_auth_hmacsha512.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_auth_hmacsha512256_BYTES 32U +SODIUM_EXPORT +size_t crypto_auth_hmacsha512256_bytes(void); + +#define crypto_auth_hmacsha512256_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_auth_hmacsha512256_keybytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha512256(unsigned char *out, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_auth_hmacsha512256_verify(const unsigned char *h, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +/* ------------------------------------------------------------------------- */ + +typedef crypto_auth_hmacsha512_state crypto_auth_hmacsha512256_state; + +SODIUM_EXPORT +size_t crypto_auth_hmacsha512256_statebytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha512256_init(crypto_auth_hmacsha512256_state *state, + const unsigned char *key, + size_t keylen) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_auth_hmacsha512256_update(crypto_auth_hmacsha512256_state *state, + const unsigned char *in, + unsigned long long inlen) __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_auth_hmacsha512256_final(crypto_auth_hmacsha512256_state *state, + unsigned char *out) __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_auth_hmacsha512256_keygen(unsigned char k[crypto_auth_hmacsha512256_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_box.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_box.h new file mode 100644 index 00000000..e060dd29 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_box.h @@ -0,0 +1,177 @@ +#ifndef crypto_box_H +#define crypto_box_H + +/* + * THREAD SAFETY: crypto_box_keypair() is thread-safe, + * provided that sodium_init() was called before. + * + * Other functions are always thread-safe. + */ + +#include + +#include "crypto_box_curve25519xsalsa20poly1305.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_box_SEEDBYTES crypto_box_curve25519xsalsa20poly1305_SEEDBYTES +SODIUM_EXPORT +size_t crypto_box_seedbytes(void); + +#define crypto_box_PUBLICKEYBYTES crypto_box_curve25519xsalsa20poly1305_PUBLICKEYBYTES +SODIUM_EXPORT +size_t crypto_box_publickeybytes(void); + +#define crypto_box_SECRETKEYBYTES crypto_box_curve25519xsalsa20poly1305_SECRETKEYBYTES +SODIUM_EXPORT +size_t crypto_box_secretkeybytes(void); + +#define crypto_box_NONCEBYTES crypto_box_curve25519xsalsa20poly1305_NONCEBYTES +SODIUM_EXPORT +size_t crypto_box_noncebytes(void); + +#define crypto_box_MACBYTES crypto_box_curve25519xsalsa20poly1305_MACBYTES +SODIUM_EXPORT +size_t crypto_box_macbytes(void); + +#define crypto_box_MESSAGEBYTES_MAX crypto_box_curve25519xsalsa20poly1305_MESSAGEBYTES_MAX +SODIUM_EXPORT +size_t crypto_box_messagebytes_max(void); + +#define crypto_box_PRIMITIVE "curve25519xsalsa20poly1305" +SODIUM_EXPORT +const char *crypto_box_primitive(void); + +SODIUM_EXPORT +int crypto_box_seed_keypair(unsigned char *pk, unsigned char *sk, + const unsigned char *seed) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_keypair(unsigned char *pk, unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_easy(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *pk, const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_open_easy(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *pk, const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_detached(unsigned char *c, unsigned char *mac, + const unsigned char *m, unsigned long long mlen, + const unsigned char *n, const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 2, 5, 6, 7))); + +SODIUM_EXPORT +int crypto_box_open_detached(unsigned char *m, const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6, 7))); + +/* -- Precomputation interface -- */ + +#define crypto_box_BEFORENMBYTES crypto_box_curve25519xsalsa20poly1305_BEFORENMBYTES +SODIUM_EXPORT +size_t crypto_box_beforenmbytes(void); + +SODIUM_EXPORT +int crypto_box_beforenm(unsigned char *k, const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_easy_afternm(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_box_open_easy_afternm(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +SODIUM_EXPORT +int crypto_box_detached_afternm(unsigned char *c, unsigned char *mac, + const unsigned char *m, unsigned long long mlen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull(1, 2, 5, 6))); + +SODIUM_EXPORT +int crypto_box_open_detached_afternm(unsigned char *m, const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6))); + +/* -- Ephemeral SK interface -- */ + +#define crypto_box_SEALBYTES (crypto_box_PUBLICKEYBYTES + crypto_box_MACBYTES) +SODIUM_EXPORT +size_t crypto_box_sealbytes(void); + +SODIUM_EXPORT +int crypto_box_seal(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *pk) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_box_seal_open(unsigned char *m, const unsigned char *c, + unsigned long long clen, + const unsigned char *pk, const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +/* -- NaCl compatibility interface ; Requires padding -- */ + +#define crypto_box_ZEROBYTES crypto_box_curve25519xsalsa20poly1305_ZEROBYTES +SODIUM_EXPORT +size_t crypto_box_zerobytes(void); + +#define crypto_box_BOXZEROBYTES crypto_box_curve25519xsalsa20poly1305_BOXZEROBYTES +SODIUM_EXPORT +size_t crypto_box_boxzerobytes(void); + +SODIUM_EXPORT +int crypto_box(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *pk, const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_open(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *pk, const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_afternm(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_box_open_afternm(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_box_curve25519xchacha20poly1305.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_box_curve25519xchacha20poly1305.h new file mode 100644 index 00000000..26a3d31e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_box_curve25519xchacha20poly1305.h @@ -0,0 +1,164 @@ + +#ifndef crypto_box_curve25519xchacha20poly1305_H +#define crypto_box_curve25519xchacha20poly1305_H + +#include +#include "crypto_stream_xchacha20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_box_curve25519xchacha20poly1305_SEEDBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_seedbytes(void); + +#define crypto_box_curve25519xchacha20poly1305_PUBLICKEYBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_publickeybytes(void); + +#define crypto_box_curve25519xchacha20poly1305_SECRETKEYBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_secretkeybytes(void); + +#define crypto_box_curve25519xchacha20poly1305_BEFORENMBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_beforenmbytes(void); + +#define crypto_box_curve25519xchacha20poly1305_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_noncebytes(void); + +#define crypto_box_curve25519xchacha20poly1305_MACBYTES 16U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_macbytes(void); + +#define crypto_box_curve25519xchacha20poly1305_MESSAGEBYTES_MAX \ + (crypto_stream_xchacha20_MESSAGEBYTES_MAX - crypto_box_curve25519xchacha20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_seed_keypair(unsigned char *pk, + unsigned char *sk, + const unsigned char *seed) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_keypair(unsigned char *pk, + unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_easy(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_open_easy(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_detached(unsigned char *c, + unsigned char *mac, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 2, 5, 6, 7))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_open_detached(unsigned char *m, + const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6, 7))); + +/* -- Precomputation interface -- */ + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_beforenm(unsigned char *k, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_easy_afternm(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_open_easy_afternm(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_detached_afternm(unsigned char *c, + unsigned char *mac, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 5, 6))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_open_detached_afternm(unsigned char *m, + const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6))); + +/* -- Ephemeral SK interface -- */ + +#define crypto_box_curve25519xchacha20poly1305_SEALBYTES \ + (crypto_box_curve25519xchacha20poly1305_PUBLICKEYBYTES + \ + crypto_box_curve25519xchacha20poly1305_MACBYTES) + +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_sealbytes(void); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_seal(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *pk) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_seal_open(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_box_curve25519xsalsa20poly1305.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_box_curve25519xsalsa20poly1305.h new file mode 100644 index 00000000..e733f499 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_box_curve25519xsalsa20poly1305.h @@ -0,0 +1,112 @@ +#ifndef crypto_box_curve25519xsalsa20poly1305_H +#define crypto_box_curve25519xsalsa20poly1305_H + +#include +#include "crypto_stream_xsalsa20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_box_curve25519xsalsa20poly1305_SEEDBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_seedbytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_PUBLICKEYBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_publickeybytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_SECRETKEYBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_secretkeybytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_BEFORENMBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_beforenmbytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_noncebytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_MACBYTES 16U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_macbytes(void); + +/* Only for the libsodium API - The NaCl compatibility API would require BOXZEROBYTES extra bytes */ +#define crypto_box_curve25519xsalsa20poly1305_MESSAGEBYTES_MAX \ + (crypto_stream_xsalsa20_MESSAGEBYTES_MAX - crypto_box_curve25519xsalsa20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_seed_keypair(unsigned char *pk, + unsigned char *sk, + const unsigned char *seed) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_keypair(unsigned char *pk, + unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_beforenm(unsigned char *k, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +/* -- NaCl compatibility interface ; Requires padding -- */ + +#define crypto_box_curve25519xsalsa20poly1305_BOXZEROBYTES 16U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_boxzerobytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_ZEROBYTES \ + (crypto_box_curve25519xsalsa20poly1305_BOXZEROBYTES + \ + crypto_box_curve25519xsalsa20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_zerobytes(void); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_open(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_afternm(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_open_afternm(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_core_ed25519.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_core_ed25519.h new file mode 100644 index 00000000..3eae00c4 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_core_ed25519.h @@ -0,0 +1,100 @@ +#ifndef crypto_core_ed25519_H +#define crypto_core_ed25519_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_ed25519_BYTES 32 +SODIUM_EXPORT +size_t crypto_core_ed25519_bytes(void); + +#define crypto_core_ed25519_UNIFORMBYTES 32 +SODIUM_EXPORT +size_t crypto_core_ed25519_uniformbytes(void); + +#define crypto_core_ed25519_HASHBYTES 64 +SODIUM_EXPORT +size_t crypto_core_ed25519_hashbytes(void); + +#define crypto_core_ed25519_SCALARBYTES 32 +SODIUM_EXPORT +size_t crypto_core_ed25519_scalarbytes(void); + +#define crypto_core_ed25519_NONREDUCEDSCALARBYTES 64 +SODIUM_EXPORT +size_t crypto_core_ed25519_nonreducedscalarbytes(void); + +SODIUM_EXPORT +int crypto_core_ed25519_is_valid_point(const unsigned char *p) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ed25519_add(unsigned char *r, + const unsigned char *p, const unsigned char *q) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ed25519_sub(unsigned char *r, + const unsigned char *p, const unsigned char *q) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ed25519_from_uniform(unsigned char *p, const unsigned char *r) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ed25519_from_hash(unsigned char *p, const unsigned char *h) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_random(unsigned char *p) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_random(unsigned char *r) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ed25519_scalar_invert(unsigned char *recip, const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_negate(unsigned char *neg, const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_complement(unsigned char *comp, const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_add(unsigned char *z, const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_sub(unsigned char *z, const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_mul(unsigned char *z, const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +/* + * The interval `s` is sampled from should be at least 317 bits to ensure almost + * uniformity of `r` over `L`. + */ +SODIUM_EXPORT +void crypto_core_ed25519_scalar_reduce(unsigned char *r, const unsigned char *s) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_core_hchacha20.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_core_hchacha20.h new file mode 100644 index 00000000..ece141b0 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_core_hchacha20.h @@ -0,0 +1,36 @@ +#ifndef crypto_core_hchacha20_H +#define crypto_core_hchacha20_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_hchacha20_OUTPUTBYTES 32U +SODIUM_EXPORT +size_t crypto_core_hchacha20_outputbytes(void); + +#define crypto_core_hchacha20_INPUTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_hchacha20_inputbytes(void); + +#define crypto_core_hchacha20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_core_hchacha20_keybytes(void); + +#define crypto_core_hchacha20_CONSTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_hchacha20_constbytes(void); + +SODIUM_EXPORT +int crypto_core_hchacha20(unsigned char *out, const unsigned char *in, + const unsigned char *k, const unsigned char *c) + __attribute__ ((nonnull(1, 2, 3))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_core_hsalsa20.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_core_hsalsa20.h new file mode 100644 index 00000000..4bf7a487 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_core_hsalsa20.h @@ -0,0 +1,36 @@ +#ifndef crypto_core_hsalsa20_H +#define crypto_core_hsalsa20_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_hsalsa20_OUTPUTBYTES 32U +SODIUM_EXPORT +size_t crypto_core_hsalsa20_outputbytes(void); + +#define crypto_core_hsalsa20_INPUTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_hsalsa20_inputbytes(void); + +#define crypto_core_hsalsa20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_core_hsalsa20_keybytes(void); + +#define crypto_core_hsalsa20_CONSTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_hsalsa20_constbytes(void); + +SODIUM_EXPORT +int crypto_core_hsalsa20(unsigned char *out, const unsigned char *in, + const unsigned char *k, const unsigned char *c) + __attribute__ ((nonnull(1, 2, 3))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_core_ristretto255.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_core_ristretto255.h new file mode 100644 index 00000000..f2820e55 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_core_ristretto255.h @@ -0,0 +1,100 @@ +#ifndef crypto_core_ristretto255_H +#define crypto_core_ristretto255_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_ristretto255_BYTES 32 +SODIUM_EXPORT +size_t crypto_core_ristretto255_bytes(void); + +#define crypto_core_ristretto255_HASHBYTES 64 +SODIUM_EXPORT +size_t crypto_core_ristretto255_hashbytes(void); + +#define crypto_core_ristretto255_SCALARBYTES 32 +SODIUM_EXPORT +size_t crypto_core_ristretto255_scalarbytes(void); + +#define crypto_core_ristretto255_NONREDUCEDSCALARBYTES 64 +SODIUM_EXPORT +size_t crypto_core_ristretto255_nonreducedscalarbytes(void); + +SODIUM_EXPORT +int crypto_core_ristretto255_is_valid_point(const unsigned char *p) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ristretto255_add(unsigned char *r, + const unsigned char *p, const unsigned char *q) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ristretto255_sub(unsigned char *r, + const unsigned char *p, const unsigned char *q) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ristretto255_from_hash(unsigned char *p, + const unsigned char *r) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_random(unsigned char *p) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_random(unsigned char *r) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ristretto255_scalar_invert(unsigned char *recip, + const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_negate(unsigned char *neg, + const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_complement(unsigned char *comp, + const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_add(unsigned char *z, + const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_sub(unsigned char *z, + const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_mul(unsigned char *z, + const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +/* + * The interval `s` is sampled from should be at least 317 bits to ensure almost + * uniformity of `r` over `L`. + */ +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_reduce(unsigned char *r, + const unsigned char *s) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_core_salsa20.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_core_salsa20.h new file mode 100644 index 00000000..bd79fd9f --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_core_salsa20.h @@ -0,0 +1,36 @@ +#ifndef crypto_core_salsa20_H +#define crypto_core_salsa20_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_salsa20_OUTPUTBYTES 64U +SODIUM_EXPORT +size_t crypto_core_salsa20_outputbytes(void); + +#define crypto_core_salsa20_INPUTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa20_inputbytes(void); + +#define crypto_core_salsa20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_core_salsa20_keybytes(void); + +#define crypto_core_salsa20_CONSTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa20_constbytes(void); + +SODIUM_EXPORT +int crypto_core_salsa20(unsigned char *out, const unsigned char *in, + const unsigned char *k, const unsigned char *c) + __attribute__ ((nonnull(1, 2, 3))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_core_salsa2012.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_core_salsa2012.h new file mode 100644 index 00000000..05957591 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_core_salsa2012.h @@ -0,0 +1,36 @@ +#ifndef crypto_core_salsa2012_H +#define crypto_core_salsa2012_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_salsa2012_OUTPUTBYTES 64U +SODIUM_EXPORT +size_t crypto_core_salsa2012_outputbytes(void); + +#define crypto_core_salsa2012_INPUTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa2012_inputbytes(void); + +#define crypto_core_salsa2012_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_core_salsa2012_keybytes(void); + +#define crypto_core_salsa2012_CONSTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa2012_constbytes(void); + +SODIUM_EXPORT +int crypto_core_salsa2012(unsigned char *out, const unsigned char *in, + const unsigned char *k, const unsigned char *c) + __attribute__ ((nonnull(1, 2, 3))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_core_salsa208.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_core_salsa208.h new file mode 100644 index 00000000..d2f216af --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_core_salsa208.h @@ -0,0 +1,40 @@ +#ifndef crypto_core_salsa208_H +#define crypto_core_salsa208_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_salsa208_OUTPUTBYTES 64U +SODIUM_EXPORT +size_t crypto_core_salsa208_outputbytes(void) + __attribute__ ((deprecated)); + +#define crypto_core_salsa208_INPUTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa208_inputbytes(void) + __attribute__ ((deprecated)); + +#define crypto_core_salsa208_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_core_salsa208_keybytes(void) + __attribute__ ((deprecated)); + +#define crypto_core_salsa208_CONSTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa208_constbytes(void) + __attribute__ ((deprecated)); + +SODIUM_EXPORT +int crypto_core_salsa208(unsigned char *out, const unsigned char *in, + const unsigned char *k, const unsigned char *c) + __attribute__ ((nonnull(1, 2, 3))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_generichash.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_generichash.h new file mode 100644 index 00000000..d897e5d2 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_generichash.h @@ -0,0 +1,84 @@ +#ifndef crypto_generichash_H +#define crypto_generichash_H + +#include + +#include "crypto_generichash_blake2b.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_generichash_BYTES_MIN crypto_generichash_blake2b_BYTES_MIN +SODIUM_EXPORT +size_t crypto_generichash_bytes_min(void); + +#define crypto_generichash_BYTES_MAX crypto_generichash_blake2b_BYTES_MAX +SODIUM_EXPORT +size_t crypto_generichash_bytes_max(void); + +#define crypto_generichash_BYTES crypto_generichash_blake2b_BYTES +SODIUM_EXPORT +size_t crypto_generichash_bytes(void); + +#define crypto_generichash_KEYBYTES_MIN crypto_generichash_blake2b_KEYBYTES_MIN +SODIUM_EXPORT +size_t crypto_generichash_keybytes_min(void); + +#define crypto_generichash_KEYBYTES_MAX crypto_generichash_blake2b_KEYBYTES_MAX +SODIUM_EXPORT +size_t crypto_generichash_keybytes_max(void); + +#define crypto_generichash_KEYBYTES crypto_generichash_blake2b_KEYBYTES +SODIUM_EXPORT +size_t crypto_generichash_keybytes(void); + +#define crypto_generichash_PRIMITIVE "blake2b" +SODIUM_EXPORT +const char *crypto_generichash_primitive(void); + +/* + * Important when writing bindings for other programming languages: + * the state address should be 64-bytes aligned. + */ +typedef crypto_generichash_blake2b_state crypto_generichash_state; + +SODIUM_EXPORT +size_t crypto_generichash_statebytes(void); + +SODIUM_EXPORT +int crypto_generichash(unsigned char *out, size_t outlen, + const unsigned char *in, unsigned long long inlen, + const unsigned char *key, size_t keylen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_init(crypto_generichash_state *state, + const unsigned char *key, + const size_t keylen, const size_t outlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_update(crypto_generichash_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_final(crypto_generichash_state *state, + unsigned char *out, const size_t outlen) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_generichash_keygen(unsigned char k[crypto_generichash_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_generichash_blake2b.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_generichash_blake2b.h new file mode 100644 index 00000000..fee9d8ad --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_generichash_blake2b.h @@ -0,0 +1,118 @@ +#ifndef crypto_generichash_blake2b_H +#define crypto_generichash_blake2b_H + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#if defined(__IBMC__) || defined(__SUNPRO_C) || defined(__SUNPRO_CC) +# pragma pack(1) +#else +# pragma pack(push, 1) +#endif + +typedef struct CRYPTO_ALIGN(64) crypto_generichash_blake2b_state { + unsigned char opaque[384]; +} crypto_generichash_blake2b_state; + +#if defined(__IBMC__) || defined(__SUNPRO_C) || defined(__SUNPRO_CC) +# pragma pack() +#else +# pragma pack(pop) +#endif + +#define crypto_generichash_blake2b_BYTES_MIN 16U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_bytes_min(void); + +#define crypto_generichash_blake2b_BYTES_MAX 64U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_bytes_max(void); + +#define crypto_generichash_blake2b_BYTES 32U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_bytes(void); + +#define crypto_generichash_blake2b_KEYBYTES_MIN 16U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_keybytes_min(void); + +#define crypto_generichash_blake2b_KEYBYTES_MAX 64U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_keybytes_max(void); + +#define crypto_generichash_blake2b_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_keybytes(void); + +#define crypto_generichash_blake2b_SALTBYTES 16U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_saltbytes(void); + +#define crypto_generichash_blake2b_PERSONALBYTES 16U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_personalbytes(void); + +SODIUM_EXPORT +size_t crypto_generichash_blake2b_statebytes(void); + +SODIUM_EXPORT +int crypto_generichash_blake2b(unsigned char *out, size_t outlen, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *key, size_t keylen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_blake2b_salt_personal(unsigned char *out, size_t outlen, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *key, + size_t keylen, + const unsigned char *salt, + const unsigned char *personal) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_blake2b_init(crypto_generichash_blake2b_state *state, + const unsigned char *key, + const size_t keylen, const size_t outlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_blake2b_init_salt_personal(crypto_generichash_blake2b_state *state, + const unsigned char *key, + const size_t keylen, const size_t outlen, + const unsigned char *salt, + const unsigned char *personal) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_blake2b_update(crypto_generichash_blake2b_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_blake2b_final(crypto_generichash_blake2b_state *state, + unsigned char *out, + const size_t outlen) __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_generichash_blake2b_keygen(unsigned char k[crypto_generichash_blake2b_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_hash.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_hash.h new file mode 100644 index 00000000..8752f9ca --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_hash.h @@ -0,0 +1,40 @@ +#ifndef crypto_hash_H +#define crypto_hash_H + +/* + * WARNING: Unless you absolutely need to use SHA512 for interoperatibility, + * purposes, you might want to consider crypto_generichash() instead. + * Unlike SHA512, crypto_generichash() is not vulnerable to length + * extension attacks. + */ + +#include + +#include "crypto_hash_sha512.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_hash_BYTES crypto_hash_sha512_BYTES +SODIUM_EXPORT +size_t crypto_hash_bytes(void); + +SODIUM_EXPORT +int crypto_hash(unsigned char *out, const unsigned char *in, + unsigned long long inlen) __attribute__ ((nonnull(1))); + +#define crypto_hash_PRIMITIVE "sha512" +SODIUM_EXPORT +const char *crypto_hash_primitive(void) + __attribute__ ((warn_unused_result)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_hash_sha256.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_hash_sha256.h new file mode 100644 index 00000000..b18217e1 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_hash_sha256.h @@ -0,0 +1,60 @@ +#ifndef crypto_hash_sha256_H +#define crypto_hash_sha256_H + +/* + * WARNING: Unless you absolutely need to use SHA256 for interoperatibility, + * purposes, you might want to consider crypto_generichash() instead. + * Unlike SHA256, crypto_generichash() is not vulnerable to length + * extension attacks. + */ + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef struct crypto_hash_sha256_state { + uint32_t state[8]; + uint64_t count; + uint8_t buf[64]; +} crypto_hash_sha256_state; + +SODIUM_EXPORT +size_t crypto_hash_sha256_statebytes(void); + +#define crypto_hash_sha256_BYTES 32U +SODIUM_EXPORT +size_t crypto_hash_sha256_bytes(void); + +SODIUM_EXPORT +int crypto_hash_sha256(unsigned char *out, const unsigned char *in, + unsigned long long inlen) __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_hash_sha256_init(crypto_hash_sha256_state *state) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_hash_sha256_update(crypto_hash_sha256_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_hash_sha256_final(crypto_hash_sha256_state *state, + unsigned char *out) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_hash_sha512.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_hash_sha512.h new file mode 100644 index 00000000..8efa7193 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_hash_sha512.h @@ -0,0 +1,60 @@ +#ifndef crypto_hash_sha512_H +#define crypto_hash_sha512_H + +/* + * WARNING: Unless you absolutely need to use SHA512 for interoperatibility, + * purposes, you might want to consider crypto_generichash() instead. + * Unlike SHA512, crypto_generichash() is not vulnerable to length + * extension attacks. + */ + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef struct crypto_hash_sha512_state { + uint64_t state[8]; + uint64_t count[2]; + uint8_t buf[128]; +} crypto_hash_sha512_state; + +SODIUM_EXPORT +size_t crypto_hash_sha512_statebytes(void); + +#define crypto_hash_sha512_BYTES 64U +SODIUM_EXPORT +size_t crypto_hash_sha512_bytes(void); + +SODIUM_EXPORT +int crypto_hash_sha512(unsigned char *out, const unsigned char *in, + unsigned long long inlen) __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_hash_sha512_init(crypto_hash_sha512_state *state) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_hash_sha512_update(crypto_hash_sha512_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_hash_sha512_final(crypto_hash_sha512_state *state, + unsigned char *out) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_kdf.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_kdf.h new file mode 100644 index 00000000..ac2fc618 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_kdf.h @@ -0,0 +1,53 @@ +#ifndef crypto_kdf_H +#define crypto_kdf_H + +#include +#include + +#include "crypto_kdf_blake2b.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_kdf_BYTES_MIN crypto_kdf_blake2b_BYTES_MIN +SODIUM_EXPORT +size_t crypto_kdf_bytes_min(void); + +#define crypto_kdf_BYTES_MAX crypto_kdf_blake2b_BYTES_MAX +SODIUM_EXPORT +size_t crypto_kdf_bytes_max(void); + +#define crypto_kdf_CONTEXTBYTES crypto_kdf_blake2b_CONTEXTBYTES +SODIUM_EXPORT +size_t crypto_kdf_contextbytes(void); + +#define crypto_kdf_KEYBYTES crypto_kdf_blake2b_KEYBYTES +SODIUM_EXPORT +size_t crypto_kdf_keybytes(void); + +#define crypto_kdf_PRIMITIVE "blake2b" +SODIUM_EXPORT +const char *crypto_kdf_primitive(void) + __attribute__ ((warn_unused_result)); + +SODIUM_EXPORT +int crypto_kdf_derive_from_key(unsigned char *subkey, size_t subkey_len, + uint64_t subkey_id, + const char ctx[crypto_kdf_CONTEXTBYTES], + const unsigned char key[crypto_kdf_KEYBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_kdf_keygen(unsigned char k[crypto_kdf_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_kdf_blake2b.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_kdf_blake2b.h new file mode 100644 index 00000000..3ae47dd3 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_kdf_blake2b.h @@ -0,0 +1,44 @@ +#ifndef crypto_kdf_blake2b_H +#define crypto_kdf_blake2b_H + +#include +#include + +#include "crypto_kdf_blake2b.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_kdf_blake2b_BYTES_MIN 16 +SODIUM_EXPORT +size_t crypto_kdf_blake2b_bytes_min(void); + +#define crypto_kdf_blake2b_BYTES_MAX 64 +SODIUM_EXPORT +size_t crypto_kdf_blake2b_bytes_max(void); + +#define crypto_kdf_blake2b_CONTEXTBYTES 8 +SODIUM_EXPORT +size_t crypto_kdf_blake2b_contextbytes(void); + +#define crypto_kdf_blake2b_KEYBYTES 32 +SODIUM_EXPORT +size_t crypto_kdf_blake2b_keybytes(void); + +SODIUM_EXPORT +int crypto_kdf_blake2b_derive_from_key(unsigned char *subkey, size_t subkey_len, + uint64_t subkey_id, + const char ctx[crypto_kdf_blake2b_CONTEXTBYTES], + const unsigned char key[crypto_kdf_blake2b_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_kx.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_kx.h new file mode 100644 index 00000000..347132c3 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_kx.h @@ -0,0 +1,66 @@ +#ifndef crypto_kx_H +#define crypto_kx_H + +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_kx_PUBLICKEYBYTES 32 +SODIUM_EXPORT +size_t crypto_kx_publickeybytes(void); + +#define crypto_kx_SECRETKEYBYTES 32 +SODIUM_EXPORT +size_t crypto_kx_secretkeybytes(void); + +#define crypto_kx_SEEDBYTES 32 +SODIUM_EXPORT +size_t crypto_kx_seedbytes(void); + +#define crypto_kx_SESSIONKEYBYTES 32 +SODIUM_EXPORT +size_t crypto_kx_sessionkeybytes(void); + +#define crypto_kx_PRIMITIVE "x25519blake2b" +SODIUM_EXPORT +const char *crypto_kx_primitive(void); + +SODIUM_EXPORT +int crypto_kx_seed_keypair(unsigned char pk[crypto_kx_PUBLICKEYBYTES], + unsigned char sk[crypto_kx_SECRETKEYBYTES], + const unsigned char seed[crypto_kx_SEEDBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_kx_keypair(unsigned char pk[crypto_kx_PUBLICKEYBYTES], + unsigned char sk[crypto_kx_SECRETKEYBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_kx_client_session_keys(unsigned char rx[crypto_kx_SESSIONKEYBYTES], + unsigned char tx[crypto_kx_SESSIONKEYBYTES], + const unsigned char client_pk[crypto_kx_PUBLICKEYBYTES], + const unsigned char client_sk[crypto_kx_SECRETKEYBYTES], + const unsigned char server_pk[crypto_kx_PUBLICKEYBYTES]) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 4, 5))); + +SODIUM_EXPORT +int crypto_kx_server_session_keys(unsigned char rx[crypto_kx_SESSIONKEYBYTES], + unsigned char tx[crypto_kx_SESSIONKEYBYTES], + const unsigned char server_pk[crypto_kx_PUBLICKEYBYTES], + const unsigned char server_sk[crypto_kx_SECRETKEYBYTES], + const unsigned char client_pk[crypto_kx_PUBLICKEYBYTES]) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 4, 5))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_onetimeauth.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_onetimeauth.h new file mode 100644 index 00000000..7cd7b070 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_onetimeauth.h @@ -0,0 +1,65 @@ +#ifndef crypto_onetimeauth_H +#define crypto_onetimeauth_H + +#include + +#include "crypto_onetimeauth_poly1305.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef crypto_onetimeauth_poly1305_state crypto_onetimeauth_state; + +SODIUM_EXPORT +size_t crypto_onetimeauth_statebytes(void); + +#define crypto_onetimeauth_BYTES crypto_onetimeauth_poly1305_BYTES +SODIUM_EXPORT +size_t crypto_onetimeauth_bytes(void); + +#define crypto_onetimeauth_KEYBYTES crypto_onetimeauth_poly1305_KEYBYTES +SODIUM_EXPORT +size_t crypto_onetimeauth_keybytes(void); + +#define crypto_onetimeauth_PRIMITIVE "poly1305" +SODIUM_EXPORT +const char *crypto_onetimeauth_primitive(void); + +SODIUM_EXPORT +int crypto_onetimeauth(unsigned char *out, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_onetimeauth_verify(const unsigned char *h, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_onetimeauth_init(crypto_onetimeauth_state *state, + const unsigned char *key) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_onetimeauth_update(crypto_onetimeauth_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_onetimeauth_final(crypto_onetimeauth_state *state, + unsigned char *out) __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_onetimeauth_keygen(unsigned char k[crypto_onetimeauth_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_onetimeauth_poly1305.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_onetimeauth_poly1305.h new file mode 100644 index 00000000..f3e34d86 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_onetimeauth_poly1305.h @@ -0,0 +1,72 @@ +#ifndef crypto_onetimeauth_poly1305_H +#define crypto_onetimeauth_poly1305_H + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#include +#include +#include + +#include + +#include "export.h" + +typedef struct CRYPTO_ALIGN(16) crypto_onetimeauth_poly1305_state { + unsigned char opaque[256]; +} crypto_onetimeauth_poly1305_state; + +SODIUM_EXPORT +size_t crypto_onetimeauth_poly1305_statebytes(void); + +#define crypto_onetimeauth_poly1305_BYTES 16U +SODIUM_EXPORT +size_t crypto_onetimeauth_poly1305_bytes(void); + +#define crypto_onetimeauth_poly1305_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_onetimeauth_poly1305_keybytes(void); + +SODIUM_EXPORT +int crypto_onetimeauth_poly1305(unsigned char *out, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_onetimeauth_poly1305_verify(const unsigned char *h, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_onetimeauth_poly1305_init(crypto_onetimeauth_poly1305_state *state, + const unsigned char *key) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_onetimeauth_poly1305_update(crypto_onetimeauth_poly1305_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_onetimeauth_poly1305_final(crypto_onetimeauth_poly1305_state *state, + unsigned char *out) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_onetimeauth_poly1305_keygen(unsigned char k[crypto_onetimeauth_poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_pwhash.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_pwhash.h new file mode 100644 index 00000000..585a993e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_pwhash.h @@ -0,0 +1,147 @@ +#ifndef crypto_pwhash_H +#define crypto_pwhash_H + +#include + +#include "crypto_pwhash_argon2i.h" +#include "crypto_pwhash_argon2id.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_pwhash_ALG_ARGON2I13 crypto_pwhash_argon2i_ALG_ARGON2I13 +SODIUM_EXPORT +int crypto_pwhash_alg_argon2i13(void); + +#define crypto_pwhash_ALG_ARGON2ID13 crypto_pwhash_argon2id_ALG_ARGON2ID13 +SODIUM_EXPORT +int crypto_pwhash_alg_argon2id13(void); + +#define crypto_pwhash_ALG_DEFAULT crypto_pwhash_ALG_ARGON2ID13 +SODIUM_EXPORT +int crypto_pwhash_alg_default(void); + +#define crypto_pwhash_BYTES_MIN crypto_pwhash_argon2id_BYTES_MIN +SODIUM_EXPORT +size_t crypto_pwhash_bytes_min(void); + +#define crypto_pwhash_BYTES_MAX crypto_pwhash_argon2id_BYTES_MAX +SODIUM_EXPORT +size_t crypto_pwhash_bytes_max(void); + +#define crypto_pwhash_PASSWD_MIN crypto_pwhash_argon2id_PASSWD_MIN +SODIUM_EXPORT +size_t crypto_pwhash_passwd_min(void); + +#define crypto_pwhash_PASSWD_MAX crypto_pwhash_argon2id_PASSWD_MAX +SODIUM_EXPORT +size_t crypto_pwhash_passwd_max(void); + +#define crypto_pwhash_SALTBYTES crypto_pwhash_argon2id_SALTBYTES +SODIUM_EXPORT +size_t crypto_pwhash_saltbytes(void); + +#define crypto_pwhash_STRBYTES crypto_pwhash_argon2id_STRBYTES +SODIUM_EXPORT +size_t crypto_pwhash_strbytes(void); + +#define crypto_pwhash_STRPREFIX crypto_pwhash_argon2id_STRPREFIX +SODIUM_EXPORT +const char *crypto_pwhash_strprefix(void); + +#define crypto_pwhash_OPSLIMIT_MIN crypto_pwhash_argon2id_OPSLIMIT_MIN +SODIUM_EXPORT +size_t crypto_pwhash_opslimit_min(void); + +#define crypto_pwhash_OPSLIMIT_MAX crypto_pwhash_argon2id_OPSLIMIT_MAX +SODIUM_EXPORT +size_t crypto_pwhash_opslimit_max(void); + +#define crypto_pwhash_MEMLIMIT_MIN crypto_pwhash_argon2id_MEMLIMIT_MIN +SODIUM_EXPORT +size_t crypto_pwhash_memlimit_min(void); + +#define crypto_pwhash_MEMLIMIT_MAX crypto_pwhash_argon2id_MEMLIMIT_MAX +SODIUM_EXPORT +size_t crypto_pwhash_memlimit_max(void); + +#define crypto_pwhash_OPSLIMIT_INTERACTIVE crypto_pwhash_argon2id_OPSLIMIT_INTERACTIVE +SODIUM_EXPORT +size_t crypto_pwhash_opslimit_interactive(void); + +#define crypto_pwhash_MEMLIMIT_INTERACTIVE crypto_pwhash_argon2id_MEMLIMIT_INTERACTIVE +SODIUM_EXPORT +size_t crypto_pwhash_memlimit_interactive(void); + +#define crypto_pwhash_OPSLIMIT_MODERATE crypto_pwhash_argon2id_OPSLIMIT_MODERATE +SODIUM_EXPORT +size_t crypto_pwhash_opslimit_moderate(void); + +#define crypto_pwhash_MEMLIMIT_MODERATE crypto_pwhash_argon2id_MEMLIMIT_MODERATE +SODIUM_EXPORT +size_t crypto_pwhash_memlimit_moderate(void); + +#define crypto_pwhash_OPSLIMIT_SENSITIVE crypto_pwhash_argon2id_OPSLIMIT_SENSITIVE +SODIUM_EXPORT +size_t crypto_pwhash_opslimit_sensitive(void); + +#define crypto_pwhash_MEMLIMIT_SENSITIVE crypto_pwhash_argon2id_MEMLIMIT_SENSITIVE +SODIUM_EXPORT +size_t crypto_pwhash_memlimit_sensitive(void); + +/* + * With this function, do not forget to store all parameters, including the + * algorithm identifier in order to produce deterministic output. + * The crypto_pwhash_* definitions, including crypto_pwhash_ALG_DEFAULT, + * may change. + */ +SODIUM_EXPORT +int crypto_pwhash(unsigned char * const out, unsigned long long outlen, + const char * const passwd, unsigned long long passwdlen, + const unsigned char * const salt, + unsigned long long opslimit, size_t memlimit, int alg) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +/* + * The output string already includes all the required parameters, including + * the algorithm identifier. The string is all that has to be stored in + * order to verify a password. + */ +SODIUM_EXPORT +int crypto_pwhash_str(char out[crypto_pwhash_STRBYTES], + const char * const passwd, unsigned long long passwdlen, + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_str_alg(char out[crypto_pwhash_STRBYTES], + const char * const passwd, unsigned long long passwdlen, + unsigned long long opslimit, size_t memlimit, int alg) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_str_verify(const char str[crypto_pwhash_STRBYTES], + const char * const passwd, + unsigned long long passwdlen) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_str_needs_rehash(const char str[crypto_pwhash_STRBYTES], + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#define crypto_pwhash_PRIMITIVE "argon2i" +SODIUM_EXPORT +const char *crypto_pwhash_primitive(void) + __attribute__ ((warn_unused_result)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_pwhash_argon2i.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_pwhash_argon2i.h new file mode 100644 index 00000000..88ff6221 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_pwhash_argon2i.h @@ -0,0 +1,122 @@ +#ifndef crypto_pwhash_argon2i_H +#define crypto_pwhash_argon2i_H + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_pwhash_argon2i_ALG_ARGON2I13 1 +SODIUM_EXPORT +int crypto_pwhash_argon2i_alg_argon2i13(void); + +#define crypto_pwhash_argon2i_BYTES_MIN 16U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_bytes_min(void); + +#define crypto_pwhash_argon2i_BYTES_MAX SODIUM_MIN(SODIUM_SIZE_MAX, 4294967295U) +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_bytes_max(void); + +#define crypto_pwhash_argon2i_PASSWD_MIN 0U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_passwd_min(void); + +#define crypto_pwhash_argon2i_PASSWD_MAX 4294967295U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_passwd_max(void); + +#define crypto_pwhash_argon2i_SALTBYTES 16U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_saltbytes(void); + +#define crypto_pwhash_argon2i_STRBYTES 128U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_strbytes(void); + +#define crypto_pwhash_argon2i_STRPREFIX "$argon2i$" +SODIUM_EXPORT +const char *crypto_pwhash_argon2i_strprefix(void); + +#define crypto_pwhash_argon2i_OPSLIMIT_MIN 3U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_opslimit_min(void); + +#define crypto_pwhash_argon2i_OPSLIMIT_MAX 4294967295U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_opslimit_max(void); + +#define crypto_pwhash_argon2i_MEMLIMIT_MIN 8192U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_memlimit_min(void); + +#define crypto_pwhash_argon2i_MEMLIMIT_MAX \ + ((SIZE_MAX >= 4398046510080U) ? 4398046510080U : (SIZE_MAX >= 2147483648U) ? 2147483648U : 32768U) +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_memlimit_max(void); + +#define crypto_pwhash_argon2i_OPSLIMIT_INTERACTIVE 4U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_opslimit_interactive(void); + +#define crypto_pwhash_argon2i_MEMLIMIT_INTERACTIVE 33554432U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_memlimit_interactive(void); + +#define crypto_pwhash_argon2i_OPSLIMIT_MODERATE 6U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_opslimit_moderate(void); + +#define crypto_pwhash_argon2i_MEMLIMIT_MODERATE 134217728U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_memlimit_moderate(void); + +#define crypto_pwhash_argon2i_OPSLIMIT_SENSITIVE 8U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_opslimit_sensitive(void); + +#define crypto_pwhash_argon2i_MEMLIMIT_SENSITIVE 536870912U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_memlimit_sensitive(void); + +SODIUM_EXPORT +int crypto_pwhash_argon2i(unsigned char * const out, + unsigned long long outlen, + const char * const passwd, + unsigned long long passwdlen, + const unsigned char * const salt, + unsigned long long opslimit, size_t memlimit, + int alg) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2i_str(char out[crypto_pwhash_argon2i_STRBYTES], + const char * const passwd, + unsigned long long passwdlen, + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2i_str_verify(const char str[crypto_pwhash_argon2i_STRBYTES], + const char * const passwd, + unsigned long long passwdlen) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2i_str_needs_rehash(const char str[crypto_pwhash_argon2i_STRBYTES], + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_pwhash_argon2id.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_pwhash_argon2id.h new file mode 100644 index 00000000..7183abd1 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_pwhash_argon2id.h @@ -0,0 +1,122 @@ +#ifndef crypto_pwhash_argon2id_H +#define crypto_pwhash_argon2id_H + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_pwhash_argon2id_ALG_ARGON2ID13 2 +SODIUM_EXPORT +int crypto_pwhash_argon2id_alg_argon2id13(void); + +#define crypto_pwhash_argon2id_BYTES_MIN 16U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_bytes_min(void); + +#define crypto_pwhash_argon2id_BYTES_MAX SODIUM_MIN(SODIUM_SIZE_MAX, 4294967295U) +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_bytes_max(void); + +#define crypto_pwhash_argon2id_PASSWD_MIN 0U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_passwd_min(void); + +#define crypto_pwhash_argon2id_PASSWD_MAX 4294967295U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_passwd_max(void); + +#define crypto_pwhash_argon2id_SALTBYTES 16U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_saltbytes(void); + +#define crypto_pwhash_argon2id_STRBYTES 128U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_strbytes(void); + +#define crypto_pwhash_argon2id_STRPREFIX "$argon2id$" +SODIUM_EXPORT +const char *crypto_pwhash_argon2id_strprefix(void); + +#define crypto_pwhash_argon2id_OPSLIMIT_MIN 1U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_opslimit_min(void); + +#define crypto_pwhash_argon2id_OPSLIMIT_MAX 4294967295U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_opslimit_max(void); + +#define crypto_pwhash_argon2id_MEMLIMIT_MIN 8192U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_memlimit_min(void); + +#define crypto_pwhash_argon2id_MEMLIMIT_MAX \ + ((SIZE_MAX >= 4398046510080U) ? 4398046510080U : (SIZE_MAX >= 2147483648U) ? 2147483648U : 32768U) +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_memlimit_max(void); + +#define crypto_pwhash_argon2id_OPSLIMIT_INTERACTIVE 2U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_opslimit_interactive(void); + +#define crypto_pwhash_argon2id_MEMLIMIT_INTERACTIVE 67108864U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_memlimit_interactive(void); + +#define crypto_pwhash_argon2id_OPSLIMIT_MODERATE 3U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_opslimit_moderate(void); + +#define crypto_pwhash_argon2id_MEMLIMIT_MODERATE 268435456U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_memlimit_moderate(void); + +#define crypto_pwhash_argon2id_OPSLIMIT_SENSITIVE 4U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_opslimit_sensitive(void); + +#define crypto_pwhash_argon2id_MEMLIMIT_SENSITIVE 1073741824U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_memlimit_sensitive(void); + +SODIUM_EXPORT +int crypto_pwhash_argon2id(unsigned char * const out, + unsigned long long outlen, + const char * const passwd, + unsigned long long passwdlen, + const unsigned char * const salt, + unsigned long long opslimit, size_t memlimit, + int alg) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2id_str(char out[crypto_pwhash_argon2id_STRBYTES], + const char * const passwd, + unsigned long long passwdlen, + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2id_str_verify(const char str[crypto_pwhash_argon2id_STRBYTES], + const char * const passwd, + unsigned long long passwdlen) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2id_str_needs_rehash(const char str[crypto_pwhash_argon2id_STRBYTES], + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_pwhash_scryptsalsa208sha256.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_pwhash_scryptsalsa208sha256.h new file mode 100644 index 00000000..5c0bf7d3 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_pwhash_scryptsalsa208sha256.h @@ -0,0 +1,120 @@ +#ifndef crypto_pwhash_scryptsalsa208sha256_H +#define crypto_pwhash_scryptsalsa208sha256_H + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_pwhash_scryptsalsa208sha256_BYTES_MIN 16U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_bytes_min(void); + +#define crypto_pwhash_scryptsalsa208sha256_BYTES_MAX \ + SODIUM_MIN(SODIUM_SIZE_MAX, 0x1fffffffe0ULL) +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_bytes_max(void); + +#define crypto_pwhash_scryptsalsa208sha256_PASSWD_MIN 0U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_passwd_min(void); + +#define crypto_pwhash_scryptsalsa208sha256_PASSWD_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_passwd_max(void); + +#define crypto_pwhash_scryptsalsa208sha256_SALTBYTES 32U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_saltbytes(void); + +#define crypto_pwhash_scryptsalsa208sha256_STRBYTES 102U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_strbytes(void); + +#define crypto_pwhash_scryptsalsa208sha256_STRPREFIX "$7$" +SODIUM_EXPORT +const char *crypto_pwhash_scryptsalsa208sha256_strprefix(void); + +#define crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_MIN 32768U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_opslimit_min(void); + +#define crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_MAX 4294967295U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_opslimit_max(void); + +#define crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_MIN 16777216U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_memlimit_min(void); + +#define crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_MAX \ + SODIUM_MIN(SIZE_MAX, 68719476736ULL) +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_memlimit_max(void); + +#define crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_INTERACTIVE 524288U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_opslimit_interactive(void); + +#define crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_INTERACTIVE 16777216U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_memlimit_interactive(void); + +#define crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_SENSITIVE 33554432U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_opslimit_sensitive(void); + +#define crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_SENSITIVE 1073741824U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_memlimit_sensitive(void); + +SODIUM_EXPORT +int crypto_pwhash_scryptsalsa208sha256(unsigned char * const out, + unsigned long long outlen, + const char * const passwd, + unsigned long long passwdlen, + const unsigned char * const salt, + unsigned long long opslimit, + size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_scryptsalsa208sha256_str(char out[crypto_pwhash_scryptsalsa208sha256_STRBYTES], + const char * const passwd, + unsigned long long passwdlen, + unsigned long long opslimit, + size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_scryptsalsa208sha256_str_verify(const char str[crypto_pwhash_scryptsalsa208sha256_STRBYTES], + const char * const passwd, + unsigned long long passwdlen) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_scryptsalsa208sha256_ll(const uint8_t * passwd, size_t passwdlen, + const uint8_t * salt, size_t saltlen, + uint64_t N, uint32_t r, uint32_t p, + uint8_t * buf, size_t buflen) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_scryptsalsa208sha256_str_needs_rehash(const char str[crypto_pwhash_scryptsalsa208sha256_STRBYTES], + unsigned long long opslimit, + size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_scalarmult.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_scalarmult.h new file mode 100644 index 00000000..1c685853 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_scalarmult.h @@ -0,0 +1,46 @@ +#ifndef crypto_scalarmult_H +#define crypto_scalarmult_H + +#include + +#include "crypto_scalarmult_curve25519.h" +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_scalarmult_BYTES crypto_scalarmult_curve25519_BYTES +SODIUM_EXPORT +size_t crypto_scalarmult_bytes(void); + +#define crypto_scalarmult_SCALARBYTES crypto_scalarmult_curve25519_SCALARBYTES +SODIUM_EXPORT +size_t crypto_scalarmult_scalarbytes(void); + +#define crypto_scalarmult_PRIMITIVE "curve25519" +SODIUM_EXPORT +const char *crypto_scalarmult_primitive(void); + +SODIUM_EXPORT +int crypto_scalarmult_base(unsigned char *q, const unsigned char *n) + __attribute__ ((nonnull)); + +/* + * NOTE: Do not use the result of this function directly for key exchange. + * + * Hash the result with the public keys in order to compute a shared + * secret key: H(q || client_pk || server_pk) + * + * Or unless this is not an option, use the crypto_kx() API instead. + */ +SODIUM_EXPORT +int crypto_scalarmult(unsigned char *q, const unsigned char *n, + const unsigned char *p) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_scalarmult_curve25519.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_scalarmult_curve25519.h new file mode 100644 index 00000000..60e9d0c5 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_scalarmult_curve25519.h @@ -0,0 +1,42 @@ +#ifndef crypto_scalarmult_curve25519_H +#define crypto_scalarmult_curve25519_H + +#include + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_scalarmult_curve25519_BYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_curve25519_bytes(void); + +#define crypto_scalarmult_curve25519_SCALARBYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_curve25519_scalarbytes(void); + +/* + * NOTE: Do not use the result of this function directly for key exchange. + * + * Hash the result with the public keys in order to compute a shared + * secret key: H(q || client_pk || server_pk) + * + * Or unless this is not an option, use the crypto_kx() API instead. + */ +SODIUM_EXPORT +int crypto_scalarmult_curve25519(unsigned char *q, const unsigned char *n, + const unsigned char *p) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_scalarmult_curve25519_base(unsigned char *q, + const unsigned char *n) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_scalarmult_ed25519.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_scalarmult_ed25519.h new file mode 100644 index 00000000..2dfa4d70 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_scalarmult_ed25519.h @@ -0,0 +1,51 @@ + +#ifndef crypto_scalarmult_ed25519_H +#define crypto_scalarmult_ed25519_H + +#include + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_scalarmult_ed25519_BYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_ed25519_bytes(void); + +#define crypto_scalarmult_ed25519_SCALARBYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_ed25519_scalarbytes(void); + +/* + * NOTE: Do not use the result of this function directly for key exchange. + * + * Hash the result with the public keys in order to compute a shared + * secret key: H(q || client_pk || server_pk) + * + * Or unless this is not an option, use the crypto_kx() API instead. + */ +SODIUM_EXPORT +int crypto_scalarmult_ed25519(unsigned char *q, const unsigned char *n, + const unsigned char *p) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_scalarmult_ed25519_noclamp(unsigned char *q, const unsigned char *n, + const unsigned char *p) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_scalarmult_ed25519_base(unsigned char *q, const unsigned char *n) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_scalarmult_ed25519_base_noclamp(unsigned char *q, const unsigned char *n) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_scalarmult_ristretto255.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_scalarmult_ristretto255.h new file mode 100644 index 00000000..40a45cce --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_scalarmult_ristretto255.h @@ -0,0 +1,43 @@ + +#ifndef crypto_scalarmult_ristretto255_H +#define crypto_scalarmult_ristretto255_H + +#include + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_scalarmult_ristretto255_BYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_ristretto255_bytes(void); + +#define crypto_scalarmult_ristretto255_SCALARBYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_ristretto255_scalarbytes(void); + +/* + * NOTE: Do not use the result of this function directly for key exchange. + * + * Hash the result with the public keys in order to compute a shared + * secret key: H(q || client_pk || server_pk) + * + * Or unless this is not an option, use the crypto_kx() API instead. + */ +SODIUM_EXPORT +int crypto_scalarmult_ristretto255(unsigned char *q, const unsigned char *n, + const unsigned char *p) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_scalarmult_ristretto255_base(unsigned char *q, + const unsigned char *n) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_secretbox.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_secretbox.h new file mode 100644 index 00000000..1d3709db --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_secretbox.h @@ -0,0 +1,93 @@ +#ifndef crypto_secretbox_H +#define crypto_secretbox_H + +#include + +#include "crypto_secretbox_xsalsa20poly1305.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_secretbox_KEYBYTES crypto_secretbox_xsalsa20poly1305_KEYBYTES +SODIUM_EXPORT +size_t crypto_secretbox_keybytes(void); + +#define crypto_secretbox_NONCEBYTES crypto_secretbox_xsalsa20poly1305_NONCEBYTES +SODIUM_EXPORT +size_t crypto_secretbox_noncebytes(void); + +#define crypto_secretbox_MACBYTES crypto_secretbox_xsalsa20poly1305_MACBYTES +SODIUM_EXPORT +size_t crypto_secretbox_macbytes(void); + +#define crypto_secretbox_PRIMITIVE "xsalsa20poly1305" +SODIUM_EXPORT +const char *crypto_secretbox_primitive(void); + +#define crypto_secretbox_MESSAGEBYTES_MAX crypto_secretbox_xsalsa20poly1305_MESSAGEBYTES_MAX +SODIUM_EXPORT +size_t crypto_secretbox_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_secretbox_easy(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_open_easy(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_detached(unsigned char *c, unsigned char *mac, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 5, 6))); + +SODIUM_EXPORT +int crypto_secretbox_open_detached(unsigned char *m, + const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6))); + +SODIUM_EXPORT +void crypto_secretbox_keygen(unsigned char k[crypto_secretbox_KEYBYTES]) + __attribute__ ((nonnull)); + +/* -- NaCl compatibility interface ; Requires padding -- */ + +#define crypto_secretbox_ZEROBYTES crypto_secretbox_xsalsa20poly1305_ZEROBYTES +SODIUM_EXPORT +size_t crypto_secretbox_zerobytes(void); + +#define crypto_secretbox_BOXZEROBYTES crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES +SODIUM_EXPORT +size_t crypto_secretbox_boxzerobytes(void); + +SODIUM_EXPORT +int crypto_secretbox(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_open(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_secretbox_xchacha20poly1305.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_secretbox_xchacha20poly1305.h new file mode 100644 index 00000000..6ec674e3 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_secretbox_xchacha20poly1305.h @@ -0,0 +1,70 @@ +#ifndef crypto_secretbox_xchacha20poly1305_H +#define crypto_secretbox_xchacha20poly1305_H + +#include +#include "crypto_stream_xchacha20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_secretbox_xchacha20poly1305_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_secretbox_xchacha20poly1305_keybytes(void); + +#define crypto_secretbox_xchacha20poly1305_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_secretbox_xchacha20poly1305_noncebytes(void); + +#define crypto_secretbox_xchacha20poly1305_MACBYTES 16U +SODIUM_EXPORT +size_t crypto_secretbox_xchacha20poly1305_macbytes(void); + +#define crypto_secretbox_xchacha20poly1305_MESSAGEBYTES_MAX \ + (crypto_stream_xchacha20_MESSAGEBYTES_MAX - crypto_secretbox_xchacha20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_secretbox_xchacha20poly1305_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_secretbox_xchacha20poly1305_easy(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_xchacha20poly1305_open_easy(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_xchacha20poly1305_detached(unsigned char *c, + unsigned char *mac, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 5, 6))); + +SODIUM_EXPORT +int crypto_secretbox_xchacha20poly1305_open_detached(unsigned char *m, + const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_secretbox_xsalsa20poly1305.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_secretbox_xsalsa20poly1305.h new file mode 100644 index 00000000..be0874cb --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_secretbox_xsalsa20poly1305.h @@ -0,0 +1,69 @@ +#ifndef crypto_secretbox_xsalsa20poly1305_H +#define crypto_secretbox_xsalsa20poly1305_H + +#include +#include "crypto_stream_xsalsa20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_secretbox_xsalsa20poly1305_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_keybytes(void); + +#define crypto_secretbox_xsalsa20poly1305_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_noncebytes(void); + +#define crypto_secretbox_xsalsa20poly1305_MACBYTES 16U +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_macbytes(void); + +/* Only for the libsodium API - The NaCl compatibility API would require BOXZEROBYTES extra bytes */ +#define crypto_secretbox_xsalsa20poly1305_MESSAGEBYTES_MAX \ + (crypto_stream_xsalsa20_MESSAGEBYTES_MAX - crypto_secretbox_xsalsa20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_secretbox_xsalsa20poly1305(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_xsalsa20poly1305_open(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +SODIUM_EXPORT +void crypto_secretbox_xsalsa20poly1305_keygen(unsigned char k[crypto_secretbox_xsalsa20poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +/* -- NaCl compatibility interface ; Requires padding -- */ + +#define crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES 16U +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_boxzerobytes(void); + +#define crypto_secretbox_xsalsa20poly1305_ZEROBYTES \ + (crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES + \ + crypto_secretbox_xsalsa20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_zerobytes(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_secretstream_xchacha20poly1305.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_secretstream_xchacha20poly1305.h new file mode 100644 index 00000000..b22e4e93 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_secretstream_xchacha20poly1305.h @@ -0,0 +1,108 @@ +#ifndef crypto_secretstream_xchacha20poly1305_H +#define crypto_secretstream_xchacha20poly1305_H + +#include + +#include "crypto_aead_xchacha20poly1305.h" +#include "crypto_stream_chacha20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_secretstream_xchacha20poly1305_ABYTES \ + (1U + crypto_aead_xchacha20poly1305_ietf_ABYTES) +SODIUM_EXPORT +size_t crypto_secretstream_xchacha20poly1305_abytes(void); + +#define crypto_secretstream_xchacha20poly1305_HEADERBYTES \ + crypto_aead_xchacha20poly1305_ietf_NPUBBYTES +SODIUM_EXPORT +size_t crypto_secretstream_xchacha20poly1305_headerbytes(void); + +#define crypto_secretstream_xchacha20poly1305_KEYBYTES \ + crypto_aead_xchacha20poly1305_ietf_KEYBYTES +SODIUM_EXPORT +size_t crypto_secretstream_xchacha20poly1305_keybytes(void); + +#define crypto_secretstream_xchacha20poly1305_MESSAGEBYTES_MAX \ + SODIUM_MIN(SODIUM_SIZE_MAX - crypto_secretstream_xchacha20poly1305_ABYTES, \ + (64ULL * ((1ULL << 32) - 2ULL))) +SODIUM_EXPORT +size_t crypto_secretstream_xchacha20poly1305_messagebytes_max(void); + +#define crypto_secretstream_xchacha20poly1305_TAG_MESSAGE 0x00 +SODIUM_EXPORT +unsigned char crypto_secretstream_xchacha20poly1305_tag_message(void); + +#define crypto_secretstream_xchacha20poly1305_TAG_PUSH 0x01 +SODIUM_EXPORT +unsigned char crypto_secretstream_xchacha20poly1305_tag_push(void); + +#define crypto_secretstream_xchacha20poly1305_TAG_REKEY 0x02 +SODIUM_EXPORT +unsigned char crypto_secretstream_xchacha20poly1305_tag_rekey(void); + +#define crypto_secretstream_xchacha20poly1305_TAG_FINAL \ + (crypto_secretstream_xchacha20poly1305_TAG_PUSH | \ + crypto_secretstream_xchacha20poly1305_TAG_REKEY) +SODIUM_EXPORT +unsigned char crypto_secretstream_xchacha20poly1305_tag_final(void); + +typedef struct crypto_secretstream_xchacha20poly1305_state { + unsigned char k[crypto_stream_chacha20_ietf_KEYBYTES]; + unsigned char nonce[crypto_stream_chacha20_ietf_NONCEBYTES]; + unsigned char _pad[8]; +} crypto_secretstream_xchacha20poly1305_state; + +SODIUM_EXPORT +size_t crypto_secretstream_xchacha20poly1305_statebytes(void); + +SODIUM_EXPORT +void crypto_secretstream_xchacha20poly1305_keygen + (unsigned char k[crypto_secretstream_xchacha20poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_secretstream_xchacha20poly1305_init_push + (crypto_secretstream_xchacha20poly1305_state *state, + unsigned char header[crypto_secretstream_xchacha20poly1305_HEADERBYTES], + const unsigned char k[crypto_secretstream_xchacha20poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_secretstream_xchacha20poly1305_push + (crypto_secretstream_xchacha20poly1305_state *state, + unsigned char *c, unsigned long long *clen_p, + const unsigned char *m, unsigned long long mlen, + const unsigned char *ad, unsigned long long adlen, unsigned char tag) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_secretstream_xchacha20poly1305_init_pull + (crypto_secretstream_xchacha20poly1305_state *state, + const unsigned char header[crypto_secretstream_xchacha20poly1305_HEADERBYTES], + const unsigned char k[crypto_secretstream_xchacha20poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_secretstream_xchacha20poly1305_pull + (crypto_secretstream_xchacha20poly1305_state *state, + unsigned char *m, unsigned long long *mlen_p, unsigned char *tag_p, + const unsigned char *c, unsigned long long clen, + const unsigned char *ad, unsigned long long adlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +void crypto_secretstream_xchacha20poly1305_rekey + (crypto_secretstream_xchacha20poly1305_state *state); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_shorthash.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_shorthash.h new file mode 100644 index 00000000..fecaa88b --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_shorthash.h @@ -0,0 +1,41 @@ +#ifndef crypto_shorthash_H +#define crypto_shorthash_H + +#include + +#include "crypto_shorthash_siphash24.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_shorthash_BYTES crypto_shorthash_siphash24_BYTES +SODIUM_EXPORT +size_t crypto_shorthash_bytes(void); + +#define crypto_shorthash_KEYBYTES crypto_shorthash_siphash24_KEYBYTES +SODIUM_EXPORT +size_t crypto_shorthash_keybytes(void); + +#define crypto_shorthash_PRIMITIVE "siphash24" +SODIUM_EXPORT +const char *crypto_shorthash_primitive(void); + +SODIUM_EXPORT +int crypto_shorthash(unsigned char *out, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +void crypto_shorthash_keygen(unsigned char k[crypto_shorthash_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_shorthash_siphash24.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_shorthash_siphash24.h new file mode 100644 index 00000000..1e6f72a6 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_shorthash_siphash24.h @@ -0,0 +1,50 @@ +#ifndef crypto_shorthash_siphash24_H +#define crypto_shorthash_siphash24_H + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +/* -- 64-bit output -- */ + +#define crypto_shorthash_siphash24_BYTES 8U +SODIUM_EXPORT +size_t crypto_shorthash_siphash24_bytes(void); + +#define crypto_shorthash_siphash24_KEYBYTES 16U +SODIUM_EXPORT +size_t crypto_shorthash_siphash24_keybytes(void); + +SODIUM_EXPORT +int crypto_shorthash_siphash24(unsigned char *out, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((nonnull(1, 4))); + +#ifndef SODIUM_LIBRARY_MINIMAL +/* -- 128-bit output -- */ + +#define crypto_shorthash_siphashx24_BYTES 16U +SODIUM_EXPORT +size_t crypto_shorthash_siphashx24_bytes(void); + +#define crypto_shorthash_siphashx24_KEYBYTES 16U +SODIUM_EXPORT +size_t crypto_shorthash_siphashx24_keybytes(void); + +SODIUM_EXPORT +int crypto_shorthash_siphashx24(unsigned char *out, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((nonnull(1, 4))); +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_sign.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_sign.h new file mode 100644 index 00000000..f5fafb12 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_sign.h @@ -0,0 +1,107 @@ +#ifndef crypto_sign_H +#define crypto_sign_H + +/* + * THREAD SAFETY: crypto_sign_keypair() is thread-safe, + * provided that sodium_init() was called before. + * + * Other functions, including crypto_sign_seed_keypair() are always thread-safe. + */ + +#include + +#include "crypto_sign_ed25519.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef crypto_sign_ed25519ph_state crypto_sign_state; + +SODIUM_EXPORT +size_t crypto_sign_statebytes(void); + +#define crypto_sign_BYTES crypto_sign_ed25519_BYTES +SODIUM_EXPORT +size_t crypto_sign_bytes(void); + +#define crypto_sign_SEEDBYTES crypto_sign_ed25519_SEEDBYTES +SODIUM_EXPORT +size_t crypto_sign_seedbytes(void); + +#define crypto_sign_PUBLICKEYBYTES crypto_sign_ed25519_PUBLICKEYBYTES +SODIUM_EXPORT +size_t crypto_sign_publickeybytes(void); + +#define crypto_sign_SECRETKEYBYTES crypto_sign_ed25519_SECRETKEYBYTES +SODIUM_EXPORT +size_t crypto_sign_secretkeybytes(void); + +#define crypto_sign_MESSAGEBYTES_MAX crypto_sign_ed25519_MESSAGEBYTES_MAX +SODIUM_EXPORT +size_t crypto_sign_messagebytes_max(void); + +#define crypto_sign_PRIMITIVE "ed25519" +SODIUM_EXPORT +const char *crypto_sign_primitive(void); + +SODIUM_EXPORT +int crypto_sign_seed_keypair(unsigned char *pk, unsigned char *sk, + const unsigned char *seed) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_keypair(unsigned char *pk, unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign(unsigned char *sm, unsigned long long *smlen_p, + const unsigned char *m, unsigned long long mlen, + const unsigned char *sk) __attribute__ ((nonnull(1, 5))); + +SODIUM_EXPORT +int crypto_sign_open(unsigned char *m, unsigned long long *mlen_p, + const unsigned char *sm, unsigned long long smlen, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5))); + +SODIUM_EXPORT +int crypto_sign_detached(unsigned char *sig, unsigned long long *siglen_p, + const unsigned char *m, unsigned long long mlen, + const unsigned char *sk) __attribute__ ((nonnull(1, 5))); + +SODIUM_EXPORT +int crypto_sign_verify_detached(const unsigned char *sig, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_sign_init(crypto_sign_state *state); + +SODIUM_EXPORT +int crypto_sign_update(crypto_sign_state *state, + const unsigned char *m, unsigned long long mlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_sign_final_create(crypto_sign_state *state, unsigned char *sig, + unsigned long long *siglen_p, + const unsigned char *sk) + __attribute__ ((nonnull(1, 2, 4))); + +SODIUM_EXPORT +int crypto_sign_final_verify(crypto_sign_state *state, const unsigned char *sig, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_sign_ed25519.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_sign_ed25519.h new file mode 100644 index 00000000..0fdac42d --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_sign_ed25519.h @@ -0,0 +1,124 @@ +#ifndef crypto_sign_ed25519_H +#define crypto_sign_ed25519_H + +#include +#include "crypto_hash_sha512.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef struct crypto_sign_ed25519ph_state { + crypto_hash_sha512_state hs; +} crypto_sign_ed25519ph_state; + +SODIUM_EXPORT +size_t crypto_sign_ed25519ph_statebytes(void); + +#define crypto_sign_ed25519_BYTES 64U +SODIUM_EXPORT +size_t crypto_sign_ed25519_bytes(void); + +#define crypto_sign_ed25519_SEEDBYTES 32U +SODIUM_EXPORT +size_t crypto_sign_ed25519_seedbytes(void); + +#define crypto_sign_ed25519_PUBLICKEYBYTES 32U +SODIUM_EXPORT +size_t crypto_sign_ed25519_publickeybytes(void); + +#define crypto_sign_ed25519_SECRETKEYBYTES (32U + 32U) +SODIUM_EXPORT +size_t crypto_sign_ed25519_secretkeybytes(void); + +#define crypto_sign_ed25519_MESSAGEBYTES_MAX (SODIUM_SIZE_MAX - crypto_sign_ed25519_BYTES) +SODIUM_EXPORT +size_t crypto_sign_ed25519_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_sign_ed25519(unsigned char *sm, unsigned long long *smlen_p, + const unsigned char *m, unsigned long long mlen, + const unsigned char *sk) + __attribute__ ((nonnull(1, 5))); + +SODIUM_EXPORT +int crypto_sign_ed25519_open(unsigned char *m, unsigned long long *mlen_p, + const unsigned char *sm, unsigned long long smlen, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5))); + +SODIUM_EXPORT +int crypto_sign_ed25519_detached(unsigned char *sig, + unsigned long long *siglen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *sk) + __attribute__ ((nonnull(1, 5))); + +SODIUM_EXPORT +int crypto_sign_ed25519_verify_detached(const unsigned char *sig, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_sign_ed25519_keypair(unsigned char *pk, unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519_seed_keypair(unsigned char *pk, unsigned char *sk, + const unsigned char *seed) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519_pk_to_curve25519(unsigned char *curve25519_pk, + const unsigned char *ed25519_pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519_sk_to_curve25519(unsigned char *curve25519_sk, + const unsigned char *ed25519_sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519_sk_to_seed(unsigned char *seed, + const unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519_sk_to_pk(unsigned char *pk, const unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519ph_init(crypto_sign_ed25519ph_state *state) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519ph_update(crypto_sign_ed25519ph_state *state, + const unsigned char *m, + unsigned long long mlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_sign_ed25519ph_final_create(crypto_sign_ed25519ph_state *state, + unsigned char *sig, + unsigned long long *siglen_p, + const unsigned char *sk) + __attribute__ ((nonnull(1, 2, 4))); + +SODIUM_EXPORT +int crypto_sign_ed25519ph_final_verify(crypto_sign_ed25519ph_state *state, + const unsigned char *sig, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_sign_edwards25519sha512batch.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_sign_edwards25519sha512batch.h new file mode 100644 index 00000000..eed158aa --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_sign_edwards25519sha512batch.h @@ -0,0 +1,55 @@ +#ifndef crypto_sign_edwards25519sha512batch_H +#define crypto_sign_edwards25519sha512batch_H + +/* + * WARNING: This construction was a prototype, which should not be used + * any more in new projects. + * + * crypto_sign_edwards25519sha512batch is provided for applications + * initially built with NaCl, but as recommended by the author of this + * construction, new applications should use ed25519 instead. + * + * In Sodium, you should use the high-level crypto_sign_*() functions instead. + */ + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_sign_edwards25519sha512batch_BYTES 64U +#define crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES 32U +#define crypto_sign_edwards25519sha512batch_SECRETKEYBYTES (32U + 32U) +#define crypto_sign_edwards25519sha512batch_MESSAGEBYTES_MAX (SODIUM_SIZE_MAX - crypto_sign_edwards25519sha512batch_BYTES) + +SODIUM_EXPORT +int crypto_sign_edwards25519sha512batch(unsigned char *sm, + unsigned long long *smlen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *sk) + __attribute__ ((deprecated)) __attribute__ ((nonnull(1, 5))); + +SODIUM_EXPORT +int crypto_sign_edwards25519sha512batch_open(unsigned char *m, + unsigned long long *mlen_p, + const unsigned char *sm, + unsigned long long smlen, + const unsigned char *pk) + __attribute__ ((deprecated)) __attribute__ ((nonnull(3, 5))); + +SODIUM_EXPORT +int crypto_sign_edwards25519sha512batch_keypair(unsigned char *pk, + unsigned char *sk) + __attribute__ ((deprecated)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_stream.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_stream.h new file mode 100644 index 00000000..88dab5f6 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_stream.h @@ -0,0 +1,59 @@ +#ifndef crypto_stream_H +#define crypto_stream_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include + +#include "crypto_stream_xsalsa20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_KEYBYTES crypto_stream_xsalsa20_KEYBYTES +SODIUM_EXPORT +size_t crypto_stream_keybytes(void); + +#define crypto_stream_NONCEBYTES crypto_stream_xsalsa20_NONCEBYTES +SODIUM_EXPORT +size_t crypto_stream_noncebytes(void); + +#define crypto_stream_MESSAGEBYTES_MAX crypto_stream_xsalsa20_MESSAGEBYTES_MAX +SODIUM_EXPORT +size_t crypto_stream_messagebytes_max(void); + +#define crypto_stream_PRIMITIVE "xsalsa20" +SODIUM_EXPORT +const char *crypto_stream_primitive(void); + +SODIUM_EXPORT +int crypto_stream(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_keygen(unsigned char k[crypto_stream_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_stream_chacha20.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_stream_chacha20.h new file mode 100644 index 00000000..40889755 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_stream_chacha20.h @@ -0,0 +1,106 @@ +#ifndef crypto_stream_chacha20_H +#define crypto_stream_chacha20_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_chacha20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_chacha20_keybytes(void); + +#define crypto_stream_chacha20_NONCEBYTES 8U +SODIUM_EXPORT +size_t crypto_stream_chacha20_noncebytes(void); + +#define crypto_stream_chacha20_MESSAGEBYTES_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_stream_chacha20_messagebytes_max(void); + +/* ChaCha20 with a 64-bit nonce and a 64-bit counter, as originally designed */ + +SODIUM_EXPORT +int crypto_stream_chacha20(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_chacha20_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_chacha20_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, uint64_t ic, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_chacha20_keygen(unsigned char k[crypto_stream_chacha20_KEYBYTES]) + __attribute__ ((nonnull)); + +/* ChaCha20 with a 96-bit nonce and a 32-bit counter (IETF) */ + +#define crypto_stream_chacha20_ietf_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_chacha20_ietf_keybytes(void); + +#define crypto_stream_chacha20_ietf_NONCEBYTES 12U +SODIUM_EXPORT +size_t crypto_stream_chacha20_ietf_noncebytes(void); + +#define crypto_stream_chacha20_ietf_MESSAGEBYTES_MAX \ + SODIUM_MIN(SODIUM_SIZE_MAX, 64ULL * (1ULL << 32)) +SODIUM_EXPORT +size_t crypto_stream_chacha20_ietf_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_stream_chacha20_ietf(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_chacha20_ietf_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_chacha20_ietf_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, uint32_t ic, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_chacha20_ietf_keygen(unsigned char k[crypto_stream_chacha20_ietf_KEYBYTES]) + __attribute__ ((nonnull)); + +/* Aliases */ + +#define crypto_stream_chacha20_IETF_KEYBYTES crypto_stream_chacha20_ietf_KEYBYTES +#define crypto_stream_chacha20_IETF_NONCEBYTES crypto_stream_chacha20_ietf_NONCEBYTES +#define crypto_stream_chacha20_IETF_MESSAGEBYTES_MAX crypto_stream_chacha20_ietf_MESSAGEBYTES_MAX + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_stream_salsa20.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_stream_salsa20.h new file mode 100644 index 00000000..45b3b3e3 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_stream_salsa20.h @@ -0,0 +1,61 @@ +#ifndef crypto_stream_salsa20_H +#define crypto_stream_salsa20_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_salsa20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_salsa20_keybytes(void); + +#define crypto_stream_salsa20_NONCEBYTES 8U +SODIUM_EXPORT +size_t crypto_stream_salsa20_noncebytes(void); + +#define crypto_stream_salsa20_MESSAGEBYTES_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_stream_salsa20_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_stream_salsa20(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_salsa20_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_salsa20_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, uint64_t ic, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_salsa20_keygen(unsigned char k[crypto_stream_salsa20_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_stream_salsa2012.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_stream_salsa2012.h new file mode 100644 index 00000000..6c5d303c --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_stream_salsa2012.h @@ -0,0 +1,53 @@ +#ifndef crypto_stream_salsa2012_H +#define crypto_stream_salsa2012_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_salsa2012_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_salsa2012_keybytes(void); + +#define crypto_stream_salsa2012_NONCEBYTES 8U +SODIUM_EXPORT +size_t crypto_stream_salsa2012_noncebytes(void); + +#define crypto_stream_salsa2012_MESSAGEBYTES_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_stream_salsa2012_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_stream_salsa2012(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_salsa2012_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_salsa2012_keygen(unsigned char k[crypto_stream_salsa2012_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_stream_salsa208.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_stream_salsa208.h new file mode 100644 index 00000000..d574f304 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_stream_salsa208.h @@ -0,0 +1,56 @@ +#ifndef crypto_stream_salsa208_H +#define crypto_stream_salsa208_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_salsa208_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_salsa208_keybytes(void) + __attribute__ ((deprecated)); + +#define crypto_stream_salsa208_NONCEBYTES 8U +SODIUM_EXPORT +size_t crypto_stream_salsa208_noncebytes(void) + __attribute__ ((deprecated)); + +#define crypto_stream_salsa208_MESSAGEBYTES_MAX SODIUM_SIZE_MAX + SODIUM_EXPORT +size_t crypto_stream_salsa208_messagebytes_max(void) + __attribute__ ((deprecated)); + +SODIUM_EXPORT +int crypto_stream_salsa208(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((deprecated)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_salsa208_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((deprecated)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_salsa208_keygen(unsigned char k[crypto_stream_salsa208_KEYBYTES]) + __attribute__ ((deprecated)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_stream_xchacha20.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_stream_xchacha20.h new file mode 100644 index 00000000..c4002db0 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_stream_xchacha20.h @@ -0,0 +1,61 @@ +#ifndef crypto_stream_xchacha20_H +#define crypto_stream_xchacha20_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_xchacha20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_xchacha20_keybytes(void); + +#define crypto_stream_xchacha20_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_stream_xchacha20_noncebytes(void); + +#define crypto_stream_xchacha20_MESSAGEBYTES_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_stream_xchacha20_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_stream_xchacha20(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_xchacha20_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_xchacha20_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, uint64_t ic, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_xchacha20_keygen(unsigned char k[crypto_stream_xchacha20_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_stream_xsalsa20.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_stream_xsalsa20.h new file mode 100644 index 00000000..20034e34 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_stream_xsalsa20.h @@ -0,0 +1,61 @@ +#ifndef crypto_stream_xsalsa20_H +#define crypto_stream_xsalsa20_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_xsalsa20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_xsalsa20_keybytes(void); + +#define crypto_stream_xsalsa20_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_stream_xsalsa20_noncebytes(void); + +#define crypto_stream_xsalsa20_MESSAGEBYTES_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_stream_xsalsa20_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_stream_xsalsa20(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_xsalsa20_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_xsalsa20_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, uint64_t ic, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_xsalsa20_keygen(unsigned char k[crypto_stream_xsalsa20_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_verify_16.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_verify_16.h new file mode 100644 index 00000000..7b9c8077 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_verify_16.h @@ -0,0 +1,23 @@ +#ifndef crypto_verify_16_H +#define crypto_verify_16_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_verify_16_BYTES 16U +SODIUM_EXPORT +size_t crypto_verify_16_bytes(void); + +SODIUM_EXPORT +int crypto_verify_16(const unsigned char *x, const unsigned char *y) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_verify_32.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_verify_32.h new file mode 100644 index 00000000..9b0f4529 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_verify_32.h @@ -0,0 +1,23 @@ +#ifndef crypto_verify_32_H +#define crypto_verify_32_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_verify_32_BYTES 32U +SODIUM_EXPORT +size_t crypto_verify_32_bytes(void); + +SODIUM_EXPORT +int crypto_verify_32(const unsigned char *x, const unsigned char *y) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_verify_64.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_verify_64.h new file mode 100644 index 00000000..c83b7302 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/crypto_verify_64.h @@ -0,0 +1,23 @@ +#ifndef crypto_verify_64_H +#define crypto_verify_64_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_verify_64_BYTES 64U +SODIUM_EXPORT +size_t crypto_verify_64_bytes(void); + +SODIUM_EXPORT +int crypto_verify_64(const unsigned char *x, const unsigned char *y) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/export.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/export.h new file mode 100644 index 00000000..a0074fc9 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/export.h @@ -0,0 +1,57 @@ + +#ifndef sodium_export_H +#define sodium_export_H + +#include +#include +#include + +#if !defined(__clang__) && !defined(__GNUC__) +# ifdef __attribute__ +# undef __attribute__ +# endif +# define __attribute__(a) +#endif + +#ifdef SODIUM_STATIC +# define SODIUM_EXPORT +# define SODIUM_EXPORT_WEAK +#else +# if defined(_MSC_VER) +# ifdef SODIUM_DLL_EXPORT +# define SODIUM_EXPORT __declspec(dllexport) +# else +# define SODIUM_EXPORT __declspec(dllimport) +# endif +# else +# if defined(__SUNPRO_C) +# ifndef __GNU_C__ +# define SODIUM_EXPORT __attribute__ (visibility(__global)) +# else +# define SODIUM_EXPORT __attribute__ __global +# endif +# elif defined(_MSG_VER) +# define SODIUM_EXPORT extern __declspec(dllexport) +# else +# define SODIUM_EXPORT __attribute__ ((visibility ("default"))) +# endif +# endif +# if defined(__ELF__) && !defined(SODIUM_DISABLE_WEAK_FUNCTIONS) +# define SODIUM_EXPORT_WEAK SODIUM_EXPORT __attribute__((weak)) +# else +# define SODIUM_EXPORT_WEAK SODIUM_EXPORT +# endif +#endif + +#ifndef CRYPTO_ALIGN +# if defined(__INTEL_COMPILER) || defined(_MSC_VER) +# define CRYPTO_ALIGN(x) __declspec(align(x)) +# else +# define CRYPTO_ALIGN(x) __attribute__ ((aligned(x))) +# endif +#endif + +#define SODIUM_MIN(A, B) ((A) < (B) ? (A) : (B)) +#define SODIUM_SIZE_MAX SODIUM_MIN(UINT64_MAX, SIZE_MAX) + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/randombytes.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/randombytes.h new file mode 100644 index 00000000..a03cc657 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/randombytes.h @@ -0,0 +1,72 @@ + +#ifndef randombytes_H +#define randombytes_H + +#include +#include + +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef struct randombytes_implementation { + const char *(*implementation_name)(void); /* required */ + uint32_t (*random)(void); /* required */ + void (*stir)(void); /* optional */ + uint32_t (*uniform)(const uint32_t upper_bound); /* optional, a default implementation will be used if NULL */ + void (*buf)(void * const buf, const size_t size); /* required */ + int (*close)(void); /* optional */ +} randombytes_implementation; + +#define randombytes_BYTES_MAX SODIUM_MIN(SODIUM_SIZE_MAX, 0xffffffffUL) + +#define randombytes_SEEDBYTES 32U +SODIUM_EXPORT +size_t randombytes_seedbytes(void); + +SODIUM_EXPORT +void randombytes_buf(void * const buf, const size_t size) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void randombytes_buf_deterministic(void * const buf, const size_t size, + const unsigned char seed[randombytes_SEEDBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +uint32_t randombytes_random(void); + +SODIUM_EXPORT +uint32_t randombytes_uniform(const uint32_t upper_bound); + +SODIUM_EXPORT +void randombytes_stir(void); + +SODIUM_EXPORT +int randombytes_close(void); + +SODIUM_EXPORT +int randombytes_set_implementation(randombytes_implementation *impl) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +const char *randombytes_implementation_name(void); + +/* -- NaCl compatibility interface -- */ + +SODIUM_EXPORT +void randombytes(unsigned char * const buf, const unsigned long long buf_len) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/randombytes_internal_random.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/randombytes_internal_random.h new file mode 100644 index 00000000..2b2b7d6e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/randombytes_internal_random.h @@ -0,0 +1,22 @@ + +#ifndef randombytes_internal_random_H +#define randombytes_internal_random_H + +#include "export.h" +#include "randombytes.h" + +#ifdef __cplusplus +extern "C" { +#endif + +SODIUM_EXPORT +extern struct randombytes_implementation randombytes_internal_implementation; + +/* Backwards compatibility with libsodium < 1.0.18 */ +#define randombytes_salsa20_implementation randombytes_internal_implementation + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/randombytes_sysrandom.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/randombytes_sysrandom.h new file mode 100644 index 00000000..9e27b674 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/randombytes_sysrandom.h @@ -0,0 +1,19 @@ + +#ifndef randombytes_sysrandom_H +#define randombytes_sysrandom_H + +#include "export.h" +#include "randombytes.h" + +#ifdef __cplusplus +extern "C" { +#endif + +SODIUM_EXPORT +extern struct randombytes_implementation randombytes_sysrandom_implementation; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/runtime.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/runtime.h new file mode 100644 index 00000000..7f15d58e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/runtime.h @@ -0,0 +1,52 @@ + +#ifndef sodium_runtime_H +#define sodium_runtime_H + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_neon(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_sse2(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_sse3(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_ssse3(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_sse41(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_avx(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_avx2(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_avx512f(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_pclmul(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_aesni(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_rdrand(void); + +/* ------------------------------------------------------------------------- */ + +int _sodium_runtime_get_cpu_features(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/utils.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/utils.h new file mode 100644 index 00000000..ac801512 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/utils.h @@ -0,0 +1,179 @@ + +#ifndef sodium_utils_H +#define sodium_utils_H + +#include + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef SODIUM_C99 +# if defined(__cplusplus) || !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L +# define SODIUM_C99(X) +# else +# define SODIUM_C99(X) X +# endif +#endif + +SODIUM_EXPORT +void sodium_memzero(void * const pnt, const size_t len); + +SODIUM_EXPORT +void sodium_stackzero(const size_t len); + +/* + * WARNING: sodium_memcmp() must be used to verify if two secret keys + * are equal, in constant time. + * It returns 0 if the keys are equal, and -1 if they differ. + * This function is not designed for lexicographical comparisons. + */ +SODIUM_EXPORT +int sodium_memcmp(const void * const b1_, const void * const b2_, size_t len) + __attribute__ ((warn_unused_result)); + +/* + * sodium_compare() returns -1 if b1_ < b2_, 1 if b1_ > b2_ and 0 if b1_ == b2_ + * It is suitable for lexicographical comparisons, or to compare nonces + * and counters stored in little-endian format. + * However, it is slower than sodium_memcmp(). + */ +SODIUM_EXPORT +int sodium_compare(const unsigned char *b1_, const unsigned char *b2_, + size_t len) __attribute__ ((warn_unused_result)); + +SODIUM_EXPORT +int sodium_is_zero(const unsigned char *n, const size_t nlen); + +SODIUM_EXPORT +void sodium_increment(unsigned char *n, const size_t nlen); + +SODIUM_EXPORT +void sodium_add(unsigned char *a, const unsigned char *b, const size_t len); + +SODIUM_EXPORT +void sodium_sub(unsigned char *a, const unsigned char *b, const size_t len); + +SODIUM_EXPORT +char *sodium_bin2hex(char * const hex, const size_t hex_maxlen, + const unsigned char * const bin, const size_t bin_len) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int sodium_hex2bin(unsigned char * const bin, const size_t bin_maxlen, + const char * const hex, const size_t hex_len, + const char * const ignore, size_t * const bin_len, + const char ** const hex_end) + __attribute__ ((nonnull(1))); + +#define sodium_base64_VARIANT_ORIGINAL 1 +#define sodium_base64_VARIANT_ORIGINAL_NO_PADDING 3 +#define sodium_base64_VARIANT_URLSAFE 5 +#define sodium_base64_VARIANT_URLSAFE_NO_PADDING 7 + +/* + * Computes the required length to encode BIN_LEN bytes as a base64 string + * using the given variant. The computed length includes a trailing \0. + */ +#define sodium_base64_ENCODED_LEN(BIN_LEN, VARIANT) \ + (((BIN_LEN) / 3U) * 4U + \ + ((((BIN_LEN) - ((BIN_LEN) / 3U) * 3U) | (((BIN_LEN) - ((BIN_LEN) / 3U) * 3U) >> 1)) & 1U) * \ + (4U - (~((((VARIANT) & 2U) >> 1) - 1U) & (3U - ((BIN_LEN) - ((BIN_LEN) / 3U) * 3U)))) + 1U) + +SODIUM_EXPORT +size_t sodium_base64_encoded_len(const size_t bin_len, const int variant); + +SODIUM_EXPORT +char *sodium_bin2base64(char * const b64, const size_t b64_maxlen, + const unsigned char * const bin, const size_t bin_len, + const int variant) __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int sodium_base642bin(unsigned char * const bin, const size_t bin_maxlen, + const char * const b64, const size_t b64_len, + const char * const ignore, size_t * const bin_len, + const char ** const b64_end, const int variant) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int sodium_mlock(void * const addr, const size_t len) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int sodium_munlock(void * const addr, const size_t len) + __attribute__ ((nonnull)); + +/* WARNING: sodium_malloc() and sodium_allocarray() are not general-purpose + * allocation functions. + * + * They return a pointer to a region filled with 0xd0 bytes, immediately + * followed by a guard page. + * As a result, accessing a single byte after the requested allocation size + * will intentionally trigger a segmentation fault. + * + * A canary and an additional guard page placed before the beginning of the + * region may also kill the process if a buffer underflow is detected. + * + * The memory layout is: + * [unprotected region size (read only)][guard page (no access)][unprotected pages (read/write)][guard page (no access)] + * With the layout of the unprotected pages being: + * [optional padding][16-bytes canary][user region] + * + * However: + * - These functions are significantly slower than standard functions + * - Each allocation requires 3 or 4 additional pages + * - The returned address will not be aligned if the allocation size is not + * a multiple of the required alignment. For this reason, these functions + * are designed to store data, such as secret keys and messages. + * + * sodium_malloc() can be used to allocate any libsodium data structure. + * + * The crypto_generichash_state structure is packed and its length is + * either 357 or 361 bytes. For this reason, when using sodium_malloc() to + * allocate a crypto_generichash_state structure, padding must be added in + * order to ensure proper alignment. crypto_generichash_statebytes() + * returns the rounded up structure size, and should be prefered to sizeof(): + * state = sodium_malloc(crypto_generichash_statebytes()); + */ + +SODIUM_EXPORT +void *sodium_malloc(const size_t size) + __attribute__ ((malloc)); + +SODIUM_EXPORT +void *sodium_allocarray(size_t count, size_t size) + __attribute__ ((malloc)); + +SODIUM_EXPORT +void sodium_free(void *ptr); + +SODIUM_EXPORT +int sodium_mprotect_noaccess(void *ptr) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int sodium_mprotect_readonly(void *ptr) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int sodium_mprotect_readwrite(void *ptr) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int sodium_pad(size_t *padded_buflen_p, unsigned char *buf, + size_t unpadded_buflen, size_t blocksize, size_t max_buflen) + __attribute__ ((nonnull(2))); + +SODIUM_EXPORT +int sodium_unpad(size_t *unpadded_buflen_p, const unsigned char *buf, + size_t padded_buflen, size_t blocksize) + __attribute__ ((nonnull(2))); + +/* -------- */ + +int _sodium_alloc_init(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/version.h b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/version.h new file mode 100644 index 00000000..201a290e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/include/sodium/version.h @@ -0,0 +1,33 @@ + +#ifndef sodium_version_H +#define sodium_version_H + +#include "export.h" + +#define SODIUM_VERSION_STRING "1.0.18" + +#define SODIUM_LIBRARY_VERSION_MAJOR 10 +#define SODIUM_LIBRARY_VERSION_MINOR 3 + + +#ifdef __cplusplus +extern "C" { +#endif + +SODIUM_EXPORT +const char *sodium_version_string(void); + +SODIUM_EXPORT +int sodium_library_version_major(void); + +SODIUM_EXPORT +int sodium_library_version_minor(void); + +SODIUM_EXPORT +int sodium_library_minimal(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-i686/lib/libsodium.a b/example/android/third_party/libsodium/libsodium-android-i686/lib/libsodium.a new file mode 100644 index 00000000..8d5be2e7 Binary files /dev/null and b/example/android/third_party/libsodium/libsodium-android-i686/lib/libsodium.a differ diff --git a/example/android/third_party/libsodium/libsodium-android-i686/lib/libsodium.la b/example/android/third_party/libsodium/libsodium-android-i686/lib/libsodium.la new file mode 100644 index 00000000..1a9c26bf --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/lib/libsodium.la @@ -0,0 +1,41 @@ +# libsodium.la - a libtool library file +# Generated by libtool (GNU libtool) 2.4.6 +# +# Please DO NOT delete this file! +# It is necessary for linking the library. + +# The name that we can dlopen(3). +dlname='libsodium.so' + +# Names of this library. +library_names='libsodium.so' + +# The name of the static archive. +old_library='libsodium.a' + +# Linker flags that cannot go in dependency_libs. +inherited_linker_flags=' -pthread' + +# Libraries that this one depends upon. +dependency_libs='' + +# Names of additional weak libraries provided by this library +weak_library_names='' + +# Version information for libsodium. +current=0 +age=0 +revision=0 + +# Is this an already installed library? +installed=yes + +# Should we warn about portability when linking against -modules? +shouldnotlink=no + +# Files to dlopen/dlpreopen +dlopen='' +dlpreopen='' + +# Directory that this library needs to be installed in: +libdir='/home/alex/magnet/example/android/third_party/libsodium/libsodium-1.0.18/libsodium-android-i686/lib' diff --git a/example/android/third_party/libsodium/libsodium-android-i686/lib/libsodium.so b/example/android/third_party/libsodium/libsodium-android-i686/lib/libsodium.so new file mode 100644 index 00000000..1bb07497 Binary files /dev/null and b/example/android/third_party/libsodium/libsodium-android-i686/lib/libsodium.so differ diff --git a/example/android/third_party/libsodium/libsodium-android-i686/lib/pkgconfig/libsodium.pc b/example/android/third_party/libsodium/libsodium-android-i686/lib/pkgconfig/libsodium.pc new file mode 100644 index 00000000..0bed9f77 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-i686/lib/pkgconfig/libsodium.pc @@ -0,0 +1,12 @@ +prefix=/home/alex/magnet/example/android/third_party/libsodium/libsodium-1.0.18/libsodium-android-i686 +exec_prefix=${prefix} +libdir=${exec_prefix}/lib +includedir=${prefix}/include + +Name: libsodium +Version: 1.0.18 +Description: A modern and easy-to-use crypto library + +Libs: -L${libdir} -lsodium +Libs.private: -pthread +Cflags: -I${includedir} diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium.h new file mode 100644 index 00000000..295f911c --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium.h @@ -0,0 +1,69 @@ + +#ifndef sodium_H +#define sodium_H + +#include "sodium/version.h" + +#include "sodium/core.h" +#include "sodium/crypto_aead_aes256gcm.h" +#include "sodium/crypto_aead_chacha20poly1305.h" +#include "sodium/crypto_aead_xchacha20poly1305.h" +#include "sodium/crypto_auth.h" +#include "sodium/crypto_auth_hmacsha256.h" +#include "sodium/crypto_auth_hmacsha512.h" +#include "sodium/crypto_auth_hmacsha512256.h" +#include "sodium/crypto_box.h" +#include "sodium/crypto_box_curve25519xsalsa20poly1305.h" +#include "sodium/crypto_core_hsalsa20.h" +#include "sodium/crypto_core_hchacha20.h" +#include "sodium/crypto_core_salsa20.h" +#include "sodium/crypto_core_salsa2012.h" +#include "sodium/crypto_core_salsa208.h" +#include "sodium/crypto_generichash.h" +#include "sodium/crypto_generichash_blake2b.h" +#include "sodium/crypto_hash.h" +#include "sodium/crypto_hash_sha256.h" +#include "sodium/crypto_hash_sha512.h" +#include "sodium/crypto_kdf.h" +#include "sodium/crypto_kdf_blake2b.h" +#include "sodium/crypto_kx.h" +#include "sodium/crypto_onetimeauth.h" +#include "sodium/crypto_onetimeauth_poly1305.h" +#include "sodium/crypto_pwhash.h" +#include "sodium/crypto_pwhash_argon2i.h" +#include "sodium/crypto_scalarmult.h" +#include "sodium/crypto_scalarmult_curve25519.h" +#include "sodium/crypto_secretbox.h" +#include "sodium/crypto_secretbox_xsalsa20poly1305.h" +#include "sodium/crypto_secretstream_xchacha20poly1305.h" +#include "sodium/crypto_shorthash.h" +#include "sodium/crypto_shorthash_siphash24.h" +#include "sodium/crypto_sign.h" +#include "sodium/crypto_sign_ed25519.h" +#include "sodium/crypto_stream.h" +#include "sodium/crypto_stream_chacha20.h" +#include "sodium/crypto_stream_salsa20.h" +#include "sodium/crypto_stream_xsalsa20.h" +#include "sodium/crypto_verify_16.h" +#include "sodium/crypto_verify_32.h" +#include "sodium/crypto_verify_64.h" +#include "sodium/randombytes.h" +#include "sodium/randombytes_internal_random.h" +#include "sodium/randombytes_sysrandom.h" +#include "sodium/runtime.h" +#include "sodium/utils.h" + +#ifndef SODIUM_LIBRARY_MINIMAL +# include "sodium/crypto_box_curve25519xchacha20poly1305.h" +# include "sodium/crypto_core_ed25519.h" +# include "sodium/crypto_core_ristretto255.h" +# include "sodium/crypto_scalarmult_ed25519.h" +# include "sodium/crypto_scalarmult_ristretto255.h" +# include "sodium/crypto_secretbox_xchacha20poly1305.h" +# include "sodium/crypto_pwhash_scryptsalsa208sha256.h" +# include "sodium/crypto_stream_salsa2012.h" +# include "sodium/crypto_stream_salsa208.h" +# include "sodium/crypto_stream_xchacha20.h" +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/core.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/core.h new file mode 100644 index 00000000..dd088d2c --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/core.h @@ -0,0 +1,28 @@ + +#ifndef sodium_core_H +#define sodium_core_H + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +SODIUM_EXPORT +int sodium_init(void) + __attribute__ ((warn_unused_result)); + +/* ---- */ + +SODIUM_EXPORT +int sodium_set_misuse_handler(void (*handler)(void)); + +SODIUM_EXPORT +void sodium_misuse(void) + __attribute__ ((noreturn)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_aead_aes256gcm.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_aead_aes256gcm.h new file mode 100644 index 00000000..9baeb3f1 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_aead_aes256gcm.h @@ -0,0 +1,179 @@ +#ifndef crypto_aead_aes256gcm_H +#define crypto_aead_aes256gcm_H + +/* + * WARNING: Despite being the most popular AEAD construction due to its + * use in TLS, safely using AES-GCM in a different context is tricky. + * + * No more than ~ 350 GB of input data should be encrypted with a given key. + * This is for ~ 16 KB messages -- Actual figures vary according to + * message sizes. + * + * In addition, nonces are short and repeated nonces would totally destroy + * the security of this scheme. + * + * Nonces should thus come from atomic counters, which can be difficult to + * set up in a distributed environment. + * + * Unless you absolutely need AES-GCM, use crypto_aead_xchacha20poly1305_ietf_*() + * instead. It doesn't have any of these limitations. + * Or, if you don't need to authenticate additional data, just stick to + * crypto_secretbox(). + */ + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +SODIUM_EXPORT +int crypto_aead_aes256gcm_is_available(void); + +#define crypto_aead_aes256gcm_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_keybytes(void); + +#define crypto_aead_aes256gcm_NSECBYTES 0U +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_nsecbytes(void); + +#define crypto_aead_aes256gcm_NPUBBYTES 12U +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_npubbytes(void); + +#define crypto_aead_aes256gcm_ABYTES 16U +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_abytes(void); + +#define crypto_aead_aes256gcm_MESSAGEBYTES_MAX \ + SODIUM_MIN(SODIUM_SIZE_MAX - crypto_aead_aes256gcm_ABYTES, \ + (16ULL * ((1ULL << 32) - 2ULL))) +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_messagebytes_max(void); + +typedef struct CRYPTO_ALIGN(16) crypto_aead_aes256gcm_state_ { + unsigned char opaque[512]; +} crypto_aead_aes256gcm_state; + +SODIUM_EXPORT +size_t crypto_aead_aes256gcm_statebytes(void); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_encrypt(unsigned char *c, + unsigned long long *clen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_decrypt(unsigned char *m, + unsigned long long *mlen_p, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(4, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_encrypt_detached(unsigned char *c, + unsigned char *mac, + unsigned long long *maclen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 9, 10))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_decrypt_detached(unsigned char *m, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *mac, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5, 8, 9))); + +/* -- Precomputation interface -- */ + +SODIUM_EXPORT +int crypto_aead_aes256gcm_beforenm(crypto_aead_aes256gcm_state *ctx_, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_encrypt_afternm(unsigned char *c, + unsigned long long *clen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const crypto_aead_aes256gcm_state *ctx_) + __attribute__ ((nonnull(1, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_decrypt_afternm(unsigned char *m, + unsigned long long *mlen_p, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const crypto_aead_aes256gcm_state *ctx_) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(4, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_encrypt_detached_afternm(unsigned char *c, + unsigned char *mac, + unsigned long long *maclen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const crypto_aead_aes256gcm_state *ctx_) + __attribute__ ((nonnull(1, 2, 9, 10))); + +SODIUM_EXPORT +int crypto_aead_aes256gcm_decrypt_detached_afternm(unsigned char *m, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *mac, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const crypto_aead_aes256gcm_state *ctx_) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5, 8, 9))); + +SODIUM_EXPORT +void crypto_aead_aes256gcm_keygen(unsigned char k[crypto_aead_aes256gcm_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_aead_chacha20poly1305.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_aead_chacha20poly1305.h new file mode 100644 index 00000000..5d671df1 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_aead_chacha20poly1305.h @@ -0,0 +1,180 @@ +#ifndef crypto_aead_chacha20poly1305_H +#define crypto_aead_chacha20poly1305_H + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +/* -- IETF ChaCha20-Poly1305 construction with a 96-bit nonce and a 32-bit internal counter -- */ + +#define crypto_aead_chacha20poly1305_ietf_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_ietf_keybytes(void); + +#define crypto_aead_chacha20poly1305_ietf_NSECBYTES 0U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_ietf_nsecbytes(void); + +#define crypto_aead_chacha20poly1305_ietf_NPUBBYTES 12U + +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_ietf_npubbytes(void); + +#define crypto_aead_chacha20poly1305_ietf_ABYTES 16U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_ietf_abytes(void); + +#define crypto_aead_chacha20poly1305_ietf_MESSAGEBYTES_MAX \ + SODIUM_MIN(SODIUM_SIZE_MAX - crypto_aead_chacha20poly1305_ietf_ABYTES, \ + (64ULL * ((1ULL << 32) - 1ULL))) +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_ietf_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_ietf_encrypt(unsigned char *c, + unsigned long long *clen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_ietf_decrypt(unsigned char *m, + unsigned long long *mlen_p, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(4, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_ietf_encrypt_detached(unsigned char *c, + unsigned char *mac, + unsigned long long *maclen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 9, 10))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_ietf_decrypt_detached(unsigned char *m, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *mac, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5, 8, 9))); + +SODIUM_EXPORT +void crypto_aead_chacha20poly1305_ietf_keygen(unsigned char k[crypto_aead_chacha20poly1305_ietf_KEYBYTES]) + __attribute__ ((nonnull)); + +/* -- Original ChaCha20-Poly1305 construction with a 64-bit nonce and a 64-bit internal counter -- */ + +#define crypto_aead_chacha20poly1305_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_keybytes(void); + +#define crypto_aead_chacha20poly1305_NSECBYTES 0U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_nsecbytes(void); + +#define crypto_aead_chacha20poly1305_NPUBBYTES 8U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_npubbytes(void); + +#define crypto_aead_chacha20poly1305_ABYTES 16U +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_abytes(void); + +#define crypto_aead_chacha20poly1305_MESSAGEBYTES_MAX \ + (SODIUM_SIZE_MAX - crypto_aead_chacha20poly1305_ABYTES) +SODIUM_EXPORT +size_t crypto_aead_chacha20poly1305_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_encrypt(unsigned char *c, + unsigned long long *clen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_decrypt(unsigned char *m, + unsigned long long *mlen_p, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(4, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_encrypt_detached(unsigned char *c, + unsigned char *mac, + unsigned long long *maclen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 9, 10))); + +SODIUM_EXPORT +int crypto_aead_chacha20poly1305_decrypt_detached(unsigned char *m, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *mac, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5, 8, 9))); + +SODIUM_EXPORT +void crypto_aead_chacha20poly1305_keygen(unsigned char k[crypto_aead_chacha20poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +/* Aliases */ + +#define crypto_aead_chacha20poly1305_IETF_KEYBYTES crypto_aead_chacha20poly1305_ietf_KEYBYTES +#define crypto_aead_chacha20poly1305_IETF_NSECBYTES crypto_aead_chacha20poly1305_ietf_NSECBYTES +#define crypto_aead_chacha20poly1305_IETF_NPUBBYTES crypto_aead_chacha20poly1305_ietf_NPUBBYTES +#define crypto_aead_chacha20poly1305_IETF_ABYTES crypto_aead_chacha20poly1305_ietf_ABYTES +#define crypto_aead_chacha20poly1305_IETF_MESSAGEBYTES_MAX crypto_aead_chacha20poly1305_ietf_MESSAGEBYTES_MAX + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_aead_xchacha20poly1305.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_aead_xchacha20poly1305.h new file mode 100644 index 00000000..6643b0cb --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_aead_xchacha20poly1305.h @@ -0,0 +1,100 @@ +#ifndef crypto_aead_xchacha20poly1305_H +#define crypto_aead_xchacha20poly1305_H + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_aead_xchacha20poly1305_ietf_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_aead_xchacha20poly1305_ietf_keybytes(void); + +#define crypto_aead_xchacha20poly1305_ietf_NSECBYTES 0U +SODIUM_EXPORT +size_t crypto_aead_xchacha20poly1305_ietf_nsecbytes(void); + +#define crypto_aead_xchacha20poly1305_ietf_NPUBBYTES 24U +SODIUM_EXPORT +size_t crypto_aead_xchacha20poly1305_ietf_npubbytes(void); + +#define crypto_aead_xchacha20poly1305_ietf_ABYTES 16U +SODIUM_EXPORT +size_t crypto_aead_xchacha20poly1305_ietf_abytes(void); + +#define crypto_aead_xchacha20poly1305_ietf_MESSAGEBYTES_MAX \ + (SODIUM_SIZE_MAX - crypto_aead_xchacha20poly1305_ietf_ABYTES) +SODIUM_EXPORT +size_t crypto_aead_xchacha20poly1305_ietf_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_aead_xchacha20poly1305_ietf_encrypt(unsigned char *c, + unsigned long long *clen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_xchacha20poly1305_ietf_decrypt(unsigned char *m, + unsigned long long *mlen_p, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(4, 8, 9))); + +SODIUM_EXPORT +int crypto_aead_xchacha20poly1305_ietf_encrypt_detached(unsigned char *c, + unsigned char *mac, + unsigned long long *maclen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *nsec, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 9, 10))); + +SODIUM_EXPORT +int crypto_aead_xchacha20poly1305_ietf_decrypt_detached(unsigned char *m, + unsigned char *nsec, + const unsigned char *c, + unsigned long long clen, + const unsigned char *mac, + const unsigned char *ad, + unsigned long long adlen, + const unsigned char *npub, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5, 8, 9))); + +SODIUM_EXPORT +void crypto_aead_xchacha20poly1305_ietf_keygen(unsigned char k[crypto_aead_xchacha20poly1305_ietf_KEYBYTES]) + __attribute__ ((nonnull)); + +/* Aliases */ + +#define crypto_aead_xchacha20poly1305_IETF_KEYBYTES crypto_aead_xchacha20poly1305_ietf_KEYBYTES +#define crypto_aead_xchacha20poly1305_IETF_NSECBYTES crypto_aead_xchacha20poly1305_ietf_NSECBYTES +#define crypto_aead_xchacha20poly1305_IETF_NPUBBYTES crypto_aead_xchacha20poly1305_ietf_NPUBBYTES +#define crypto_aead_xchacha20poly1305_IETF_ABYTES crypto_aead_xchacha20poly1305_ietf_ABYTES +#define crypto_aead_xchacha20poly1305_IETF_MESSAGEBYTES_MAX crypto_aead_xchacha20poly1305_ietf_MESSAGEBYTES_MAX + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_auth.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_auth.h new file mode 100644 index 00000000..540aee0e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_auth.h @@ -0,0 +1,46 @@ +#ifndef crypto_auth_H +#define crypto_auth_H + +#include + +#include "crypto_auth_hmacsha512256.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_auth_BYTES crypto_auth_hmacsha512256_BYTES +SODIUM_EXPORT +size_t crypto_auth_bytes(void); + +#define crypto_auth_KEYBYTES crypto_auth_hmacsha512256_KEYBYTES +SODIUM_EXPORT +size_t crypto_auth_keybytes(void); + +#define crypto_auth_PRIMITIVE "hmacsha512256" +SODIUM_EXPORT +const char *crypto_auth_primitive(void); + +SODIUM_EXPORT +int crypto_auth(unsigned char *out, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_auth_verify(const unsigned char *h, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +void crypto_auth_keygen(unsigned char k[crypto_auth_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_auth_hmacsha256.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_auth_hmacsha256.h new file mode 100644 index 00000000..3da864c7 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_auth_hmacsha256.h @@ -0,0 +1,70 @@ +#ifndef crypto_auth_hmacsha256_H +#define crypto_auth_hmacsha256_H + +#include +#include "crypto_hash_sha256.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_auth_hmacsha256_BYTES 32U +SODIUM_EXPORT +size_t crypto_auth_hmacsha256_bytes(void); + +#define crypto_auth_hmacsha256_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_auth_hmacsha256_keybytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha256(unsigned char *out, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_auth_hmacsha256_verify(const unsigned char *h, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +/* ------------------------------------------------------------------------- */ + +typedef struct crypto_auth_hmacsha256_state { + crypto_hash_sha256_state ictx; + crypto_hash_sha256_state octx; +} crypto_auth_hmacsha256_state; + +SODIUM_EXPORT +size_t crypto_auth_hmacsha256_statebytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha256_init(crypto_auth_hmacsha256_state *state, + const unsigned char *key, + size_t keylen) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_auth_hmacsha256_update(crypto_auth_hmacsha256_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_auth_hmacsha256_final(crypto_auth_hmacsha256_state *state, + unsigned char *out) __attribute__ ((nonnull)); + + +SODIUM_EXPORT +void crypto_auth_hmacsha256_keygen(unsigned char k[crypto_auth_hmacsha256_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_auth_hmacsha512.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_auth_hmacsha512.h new file mode 100644 index 00000000..d992cb81 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_auth_hmacsha512.h @@ -0,0 +1,68 @@ +#ifndef crypto_auth_hmacsha512_H +#define crypto_auth_hmacsha512_H + +#include +#include "crypto_hash_sha512.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_auth_hmacsha512_BYTES 64U +SODIUM_EXPORT +size_t crypto_auth_hmacsha512_bytes(void); + +#define crypto_auth_hmacsha512_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_auth_hmacsha512_keybytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha512(unsigned char *out, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_auth_hmacsha512_verify(const unsigned char *h, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +/* ------------------------------------------------------------------------- */ + +typedef struct crypto_auth_hmacsha512_state { + crypto_hash_sha512_state ictx; + crypto_hash_sha512_state octx; +} crypto_auth_hmacsha512_state; + +SODIUM_EXPORT +size_t crypto_auth_hmacsha512_statebytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha512_init(crypto_auth_hmacsha512_state *state, + const unsigned char *key, + size_t keylen) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_auth_hmacsha512_update(crypto_auth_hmacsha512_state *state, + const unsigned char *in, + unsigned long long inlen) __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_auth_hmacsha512_final(crypto_auth_hmacsha512_state *state, + unsigned char *out) __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_auth_hmacsha512_keygen(unsigned char k[crypto_auth_hmacsha512_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_auth_hmacsha512256.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_auth_hmacsha512256.h new file mode 100644 index 00000000..3fb52638 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_auth_hmacsha512256.h @@ -0,0 +1,65 @@ +#ifndef crypto_auth_hmacsha512256_H +#define crypto_auth_hmacsha512256_H + +#include +#include "crypto_auth_hmacsha512.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_auth_hmacsha512256_BYTES 32U +SODIUM_EXPORT +size_t crypto_auth_hmacsha512256_bytes(void); + +#define crypto_auth_hmacsha512256_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_auth_hmacsha512256_keybytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha512256(unsigned char *out, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_auth_hmacsha512256_verify(const unsigned char *h, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +/* ------------------------------------------------------------------------- */ + +typedef crypto_auth_hmacsha512_state crypto_auth_hmacsha512256_state; + +SODIUM_EXPORT +size_t crypto_auth_hmacsha512256_statebytes(void); + +SODIUM_EXPORT +int crypto_auth_hmacsha512256_init(crypto_auth_hmacsha512256_state *state, + const unsigned char *key, + size_t keylen) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_auth_hmacsha512256_update(crypto_auth_hmacsha512256_state *state, + const unsigned char *in, + unsigned long long inlen) __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_auth_hmacsha512256_final(crypto_auth_hmacsha512256_state *state, + unsigned char *out) __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_auth_hmacsha512256_keygen(unsigned char k[crypto_auth_hmacsha512256_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_box.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_box.h new file mode 100644 index 00000000..e060dd29 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_box.h @@ -0,0 +1,177 @@ +#ifndef crypto_box_H +#define crypto_box_H + +/* + * THREAD SAFETY: crypto_box_keypair() is thread-safe, + * provided that sodium_init() was called before. + * + * Other functions are always thread-safe. + */ + +#include + +#include "crypto_box_curve25519xsalsa20poly1305.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_box_SEEDBYTES crypto_box_curve25519xsalsa20poly1305_SEEDBYTES +SODIUM_EXPORT +size_t crypto_box_seedbytes(void); + +#define crypto_box_PUBLICKEYBYTES crypto_box_curve25519xsalsa20poly1305_PUBLICKEYBYTES +SODIUM_EXPORT +size_t crypto_box_publickeybytes(void); + +#define crypto_box_SECRETKEYBYTES crypto_box_curve25519xsalsa20poly1305_SECRETKEYBYTES +SODIUM_EXPORT +size_t crypto_box_secretkeybytes(void); + +#define crypto_box_NONCEBYTES crypto_box_curve25519xsalsa20poly1305_NONCEBYTES +SODIUM_EXPORT +size_t crypto_box_noncebytes(void); + +#define crypto_box_MACBYTES crypto_box_curve25519xsalsa20poly1305_MACBYTES +SODIUM_EXPORT +size_t crypto_box_macbytes(void); + +#define crypto_box_MESSAGEBYTES_MAX crypto_box_curve25519xsalsa20poly1305_MESSAGEBYTES_MAX +SODIUM_EXPORT +size_t crypto_box_messagebytes_max(void); + +#define crypto_box_PRIMITIVE "curve25519xsalsa20poly1305" +SODIUM_EXPORT +const char *crypto_box_primitive(void); + +SODIUM_EXPORT +int crypto_box_seed_keypair(unsigned char *pk, unsigned char *sk, + const unsigned char *seed) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_keypair(unsigned char *pk, unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_easy(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *pk, const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_open_easy(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *pk, const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_detached(unsigned char *c, unsigned char *mac, + const unsigned char *m, unsigned long long mlen, + const unsigned char *n, const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 2, 5, 6, 7))); + +SODIUM_EXPORT +int crypto_box_open_detached(unsigned char *m, const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6, 7))); + +/* -- Precomputation interface -- */ + +#define crypto_box_BEFORENMBYTES crypto_box_curve25519xsalsa20poly1305_BEFORENMBYTES +SODIUM_EXPORT +size_t crypto_box_beforenmbytes(void); + +SODIUM_EXPORT +int crypto_box_beforenm(unsigned char *k, const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_easy_afternm(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_box_open_easy_afternm(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +SODIUM_EXPORT +int crypto_box_detached_afternm(unsigned char *c, unsigned char *mac, + const unsigned char *m, unsigned long long mlen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull(1, 2, 5, 6))); + +SODIUM_EXPORT +int crypto_box_open_detached_afternm(unsigned char *m, const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6))); + +/* -- Ephemeral SK interface -- */ + +#define crypto_box_SEALBYTES (crypto_box_PUBLICKEYBYTES + crypto_box_MACBYTES) +SODIUM_EXPORT +size_t crypto_box_sealbytes(void); + +SODIUM_EXPORT +int crypto_box_seal(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *pk) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_box_seal_open(unsigned char *m, const unsigned char *c, + unsigned long long clen, + const unsigned char *pk, const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +/* -- NaCl compatibility interface ; Requires padding -- */ + +#define crypto_box_ZEROBYTES crypto_box_curve25519xsalsa20poly1305_ZEROBYTES +SODIUM_EXPORT +size_t crypto_box_zerobytes(void); + +#define crypto_box_BOXZEROBYTES crypto_box_curve25519xsalsa20poly1305_BOXZEROBYTES +SODIUM_EXPORT +size_t crypto_box_boxzerobytes(void); + +SODIUM_EXPORT +int crypto_box(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *pk, const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_open(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *pk, const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_afternm(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_box_open_afternm(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_box_curve25519xchacha20poly1305.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_box_curve25519xchacha20poly1305.h new file mode 100644 index 00000000..26a3d31e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_box_curve25519xchacha20poly1305.h @@ -0,0 +1,164 @@ + +#ifndef crypto_box_curve25519xchacha20poly1305_H +#define crypto_box_curve25519xchacha20poly1305_H + +#include +#include "crypto_stream_xchacha20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_box_curve25519xchacha20poly1305_SEEDBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_seedbytes(void); + +#define crypto_box_curve25519xchacha20poly1305_PUBLICKEYBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_publickeybytes(void); + +#define crypto_box_curve25519xchacha20poly1305_SECRETKEYBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_secretkeybytes(void); + +#define crypto_box_curve25519xchacha20poly1305_BEFORENMBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_beforenmbytes(void); + +#define crypto_box_curve25519xchacha20poly1305_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_noncebytes(void); + +#define crypto_box_curve25519xchacha20poly1305_MACBYTES 16U +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_macbytes(void); + +#define crypto_box_curve25519xchacha20poly1305_MESSAGEBYTES_MAX \ + (crypto_stream_xchacha20_MESSAGEBYTES_MAX - crypto_box_curve25519xchacha20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_seed_keypair(unsigned char *pk, + unsigned char *sk, + const unsigned char *seed) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_keypair(unsigned char *pk, + unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_easy(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_open_easy(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_detached(unsigned char *c, + unsigned char *mac, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 2, 5, 6, 7))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_open_detached(unsigned char *m, + const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6, 7))); + +/* -- Precomputation interface -- */ + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_beforenm(unsigned char *k, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_easy_afternm(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_open_easy_afternm(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_detached_afternm(unsigned char *c, + unsigned char *mac, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 5, 6))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_open_detached_afternm(unsigned char *m, + const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6))); + +/* -- Ephemeral SK interface -- */ + +#define crypto_box_curve25519xchacha20poly1305_SEALBYTES \ + (crypto_box_curve25519xchacha20poly1305_PUBLICKEYBYTES + \ + crypto_box_curve25519xchacha20poly1305_MACBYTES) + +SODIUM_EXPORT +size_t crypto_box_curve25519xchacha20poly1305_sealbytes(void); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_seal(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *pk) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_box_curve25519xchacha20poly1305_seal_open(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_box_curve25519xsalsa20poly1305.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_box_curve25519xsalsa20poly1305.h new file mode 100644 index 00000000..e733f499 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_box_curve25519xsalsa20poly1305.h @@ -0,0 +1,112 @@ +#ifndef crypto_box_curve25519xsalsa20poly1305_H +#define crypto_box_curve25519xsalsa20poly1305_H + +#include +#include "crypto_stream_xsalsa20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_box_curve25519xsalsa20poly1305_SEEDBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_seedbytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_PUBLICKEYBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_publickeybytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_SECRETKEYBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_secretkeybytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_BEFORENMBYTES 32U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_beforenmbytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_noncebytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_MACBYTES 16U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_macbytes(void); + +/* Only for the libsodium API - The NaCl compatibility API would require BOXZEROBYTES extra bytes */ +#define crypto_box_curve25519xsalsa20poly1305_MESSAGEBYTES_MAX \ + (crypto_stream_xsalsa20_MESSAGEBYTES_MAX - crypto_box_curve25519xsalsa20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_seed_keypair(unsigned char *pk, + unsigned char *sk, + const unsigned char *seed) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_keypair(unsigned char *pk, + unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_beforenm(unsigned char *k, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +/* -- NaCl compatibility interface ; Requires padding -- */ + +#define crypto_box_curve25519xsalsa20poly1305_BOXZEROBYTES 16U +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_boxzerobytes(void); + +#define crypto_box_curve25519xsalsa20poly1305_ZEROBYTES \ + (crypto_box_curve25519xsalsa20poly1305_BOXZEROBYTES + \ + crypto_box_curve25519xsalsa20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_box_curve25519xsalsa20poly1305_zerobytes(void); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_open(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *pk, + const unsigned char *sk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5, 6))); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_afternm(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_box_curve25519xsalsa20poly1305_open_afternm(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_core_ed25519.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_core_ed25519.h new file mode 100644 index 00000000..3eae00c4 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_core_ed25519.h @@ -0,0 +1,100 @@ +#ifndef crypto_core_ed25519_H +#define crypto_core_ed25519_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_ed25519_BYTES 32 +SODIUM_EXPORT +size_t crypto_core_ed25519_bytes(void); + +#define crypto_core_ed25519_UNIFORMBYTES 32 +SODIUM_EXPORT +size_t crypto_core_ed25519_uniformbytes(void); + +#define crypto_core_ed25519_HASHBYTES 64 +SODIUM_EXPORT +size_t crypto_core_ed25519_hashbytes(void); + +#define crypto_core_ed25519_SCALARBYTES 32 +SODIUM_EXPORT +size_t crypto_core_ed25519_scalarbytes(void); + +#define crypto_core_ed25519_NONREDUCEDSCALARBYTES 64 +SODIUM_EXPORT +size_t crypto_core_ed25519_nonreducedscalarbytes(void); + +SODIUM_EXPORT +int crypto_core_ed25519_is_valid_point(const unsigned char *p) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ed25519_add(unsigned char *r, + const unsigned char *p, const unsigned char *q) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ed25519_sub(unsigned char *r, + const unsigned char *p, const unsigned char *q) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ed25519_from_uniform(unsigned char *p, const unsigned char *r) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ed25519_from_hash(unsigned char *p, const unsigned char *h) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_random(unsigned char *p) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_random(unsigned char *r) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ed25519_scalar_invert(unsigned char *recip, const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_negate(unsigned char *neg, const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_complement(unsigned char *comp, const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_add(unsigned char *z, const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_sub(unsigned char *z, const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ed25519_scalar_mul(unsigned char *z, const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +/* + * The interval `s` is sampled from should be at least 317 bits to ensure almost + * uniformity of `r` over `L`. + */ +SODIUM_EXPORT +void crypto_core_ed25519_scalar_reduce(unsigned char *r, const unsigned char *s) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_core_hchacha20.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_core_hchacha20.h new file mode 100644 index 00000000..ece141b0 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_core_hchacha20.h @@ -0,0 +1,36 @@ +#ifndef crypto_core_hchacha20_H +#define crypto_core_hchacha20_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_hchacha20_OUTPUTBYTES 32U +SODIUM_EXPORT +size_t crypto_core_hchacha20_outputbytes(void); + +#define crypto_core_hchacha20_INPUTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_hchacha20_inputbytes(void); + +#define crypto_core_hchacha20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_core_hchacha20_keybytes(void); + +#define crypto_core_hchacha20_CONSTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_hchacha20_constbytes(void); + +SODIUM_EXPORT +int crypto_core_hchacha20(unsigned char *out, const unsigned char *in, + const unsigned char *k, const unsigned char *c) + __attribute__ ((nonnull(1, 2, 3))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_core_hsalsa20.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_core_hsalsa20.h new file mode 100644 index 00000000..4bf7a487 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_core_hsalsa20.h @@ -0,0 +1,36 @@ +#ifndef crypto_core_hsalsa20_H +#define crypto_core_hsalsa20_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_hsalsa20_OUTPUTBYTES 32U +SODIUM_EXPORT +size_t crypto_core_hsalsa20_outputbytes(void); + +#define crypto_core_hsalsa20_INPUTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_hsalsa20_inputbytes(void); + +#define crypto_core_hsalsa20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_core_hsalsa20_keybytes(void); + +#define crypto_core_hsalsa20_CONSTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_hsalsa20_constbytes(void); + +SODIUM_EXPORT +int crypto_core_hsalsa20(unsigned char *out, const unsigned char *in, + const unsigned char *k, const unsigned char *c) + __attribute__ ((nonnull(1, 2, 3))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_core_ristretto255.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_core_ristretto255.h new file mode 100644 index 00000000..f2820e55 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_core_ristretto255.h @@ -0,0 +1,100 @@ +#ifndef crypto_core_ristretto255_H +#define crypto_core_ristretto255_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_ristretto255_BYTES 32 +SODIUM_EXPORT +size_t crypto_core_ristretto255_bytes(void); + +#define crypto_core_ristretto255_HASHBYTES 64 +SODIUM_EXPORT +size_t crypto_core_ristretto255_hashbytes(void); + +#define crypto_core_ristretto255_SCALARBYTES 32 +SODIUM_EXPORT +size_t crypto_core_ristretto255_scalarbytes(void); + +#define crypto_core_ristretto255_NONREDUCEDSCALARBYTES 64 +SODIUM_EXPORT +size_t crypto_core_ristretto255_nonreducedscalarbytes(void); + +SODIUM_EXPORT +int crypto_core_ristretto255_is_valid_point(const unsigned char *p) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ristretto255_add(unsigned char *r, + const unsigned char *p, const unsigned char *q) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ristretto255_sub(unsigned char *r, + const unsigned char *p, const unsigned char *q) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ristretto255_from_hash(unsigned char *p, + const unsigned char *r) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_random(unsigned char *p) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_random(unsigned char *r) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_core_ristretto255_scalar_invert(unsigned char *recip, + const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_negate(unsigned char *neg, + const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_complement(unsigned char *comp, + const unsigned char *s) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_add(unsigned char *z, + const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_sub(unsigned char *z, + const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_mul(unsigned char *z, + const unsigned char *x, + const unsigned char *y) + __attribute__ ((nonnull)); + +/* + * The interval `s` is sampled from should be at least 317 bits to ensure almost + * uniformity of `r` over `L`. + */ +SODIUM_EXPORT +void crypto_core_ristretto255_scalar_reduce(unsigned char *r, + const unsigned char *s) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_core_salsa20.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_core_salsa20.h new file mode 100644 index 00000000..bd79fd9f --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_core_salsa20.h @@ -0,0 +1,36 @@ +#ifndef crypto_core_salsa20_H +#define crypto_core_salsa20_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_salsa20_OUTPUTBYTES 64U +SODIUM_EXPORT +size_t crypto_core_salsa20_outputbytes(void); + +#define crypto_core_salsa20_INPUTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa20_inputbytes(void); + +#define crypto_core_salsa20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_core_salsa20_keybytes(void); + +#define crypto_core_salsa20_CONSTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa20_constbytes(void); + +SODIUM_EXPORT +int crypto_core_salsa20(unsigned char *out, const unsigned char *in, + const unsigned char *k, const unsigned char *c) + __attribute__ ((nonnull(1, 2, 3))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_core_salsa2012.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_core_salsa2012.h new file mode 100644 index 00000000..05957591 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_core_salsa2012.h @@ -0,0 +1,36 @@ +#ifndef crypto_core_salsa2012_H +#define crypto_core_salsa2012_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_salsa2012_OUTPUTBYTES 64U +SODIUM_EXPORT +size_t crypto_core_salsa2012_outputbytes(void); + +#define crypto_core_salsa2012_INPUTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa2012_inputbytes(void); + +#define crypto_core_salsa2012_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_core_salsa2012_keybytes(void); + +#define crypto_core_salsa2012_CONSTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa2012_constbytes(void); + +SODIUM_EXPORT +int crypto_core_salsa2012(unsigned char *out, const unsigned char *in, + const unsigned char *k, const unsigned char *c) + __attribute__ ((nonnull(1, 2, 3))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_core_salsa208.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_core_salsa208.h new file mode 100644 index 00000000..d2f216af --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_core_salsa208.h @@ -0,0 +1,40 @@ +#ifndef crypto_core_salsa208_H +#define crypto_core_salsa208_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_core_salsa208_OUTPUTBYTES 64U +SODIUM_EXPORT +size_t crypto_core_salsa208_outputbytes(void) + __attribute__ ((deprecated)); + +#define crypto_core_salsa208_INPUTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa208_inputbytes(void) + __attribute__ ((deprecated)); + +#define crypto_core_salsa208_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_core_salsa208_keybytes(void) + __attribute__ ((deprecated)); + +#define crypto_core_salsa208_CONSTBYTES 16U +SODIUM_EXPORT +size_t crypto_core_salsa208_constbytes(void) + __attribute__ ((deprecated)); + +SODIUM_EXPORT +int crypto_core_salsa208(unsigned char *out, const unsigned char *in, + const unsigned char *k, const unsigned char *c) + __attribute__ ((nonnull(1, 2, 3))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_generichash.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_generichash.h new file mode 100644 index 00000000..d897e5d2 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_generichash.h @@ -0,0 +1,84 @@ +#ifndef crypto_generichash_H +#define crypto_generichash_H + +#include + +#include "crypto_generichash_blake2b.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_generichash_BYTES_MIN crypto_generichash_blake2b_BYTES_MIN +SODIUM_EXPORT +size_t crypto_generichash_bytes_min(void); + +#define crypto_generichash_BYTES_MAX crypto_generichash_blake2b_BYTES_MAX +SODIUM_EXPORT +size_t crypto_generichash_bytes_max(void); + +#define crypto_generichash_BYTES crypto_generichash_blake2b_BYTES +SODIUM_EXPORT +size_t crypto_generichash_bytes(void); + +#define crypto_generichash_KEYBYTES_MIN crypto_generichash_blake2b_KEYBYTES_MIN +SODIUM_EXPORT +size_t crypto_generichash_keybytes_min(void); + +#define crypto_generichash_KEYBYTES_MAX crypto_generichash_blake2b_KEYBYTES_MAX +SODIUM_EXPORT +size_t crypto_generichash_keybytes_max(void); + +#define crypto_generichash_KEYBYTES crypto_generichash_blake2b_KEYBYTES +SODIUM_EXPORT +size_t crypto_generichash_keybytes(void); + +#define crypto_generichash_PRIMITIVE "blake2b" +SODIUM_EXPORT +const char *crypto_generichash_primitive(void); + +/* + * Important when writing bindings for other programming languages: + * the state address should be 64-bytes aligned. + */ +typedef crypto_generichash_blake2b_state crypto_generichash_state; + +SODIUM_EXPORT +size_t crypto_generichash_statebytes(void); + +SODIUM_EXPORT +int crypto_generichash(unsigned char *out, size_t outlen, + const unsigned char *in, unsigned long long inlen, + const unsigned char *key, size_t keylen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_init(crypto_generichash_state *state, + const unsigned char *key, + const size_t keylen, const size_t outlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_update(crypto_generichash_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_final(crypto_generichash_state *state, + unsigned char *out, const size_t outlen) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_generichash_keygen(unsigned char k[crypto_generichash_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_generichash_blake2b.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_generichash_blake2b.h new file mode 100644 index 00000000..fee9d8ad --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_generichash_blake2b.h @@ -0,0 +1,118 @@ +#ifndef crypto_generichash_blake2b_H +#define crypto_generichash_blake2b_H + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#if defined(__IBMC__) || defined(__SUNPRO_C) || defined(__SUNPRO_CC) +# pragma pack(1) +#else +# pragma pack(push, 1) +#endif + +typedef struct CRYPTO_ALIGN(64) crypto_generichash_blake2b_state { + unsigned char opaque[384]; +} crypto_generichash_blake2b_state; + +#if defined(__IBMC__) || defined(__SUNPRO_C) || defined(__SUNPRO_CC) +# pragma pack() +#else +# pragma pack(pop) +#endif + +#define crypto_generichash_blake2b_BYTES_MIN 16U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_bytes_min(void); + +#define crypto_generichash_blake2b_BYTES_MAX 64U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_bytes_max(void); + +#define crypto_generichash_blake2b_BYTES 32U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_bytes(void); + +#define crypto_generichash_blake2b_KEYBYTES_MIN 16U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_keybytes_min(void); + +#define crypto_generichash_blake2b_KEYBYTES_MAX 64U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_keybytes_max(void); + +#define crypto_generichash_blake2b_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_keybytes(void); + +#define crypto_generichash_blake2b_SALTBYTES 16U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_saltbytes(void); + +#define crypto_generichash_blake2b_PERSONALBYTES 16U +SODIUM_EXPORT +size_t crypto_generichash_blake2b_personalbytes(void); + +SODIUM_EXPORT +size_t crypto_generichash_blake2b_statebytes(void); + +SODIUM_EXPORT +int crypto_generichash_blake2b(unsigned char *out, size_t outlen, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *key, size_t keylen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_blake2b_salt_personal(unsigned char *out, size_t outlen, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *key, + size_t keylen, + const unsigned char *salt, + const unsigned char *personal) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_blake2b_init(crypto_generichash_blake2b_state *state, + const unsigned char *key, + const size_t keylen, const size_t outlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_blake2b_init_salt_personal(crypto_generichash_blake2b_state *state, + const unsigned char *key, + const size_t keylen, const size_t outlen, + const unsigned char *salt, + const unsigned char *personal) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_blake2b_update(crypto_generichash_blake2b_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_generichash_blake2b_final(crypto_generichash_blake2b_state *state, + unsigned char *out, + const size_t outlen) __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_generichash_blake2b_keygen(unsigned char k[crypto_generichash_blake2b_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_hash.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_hash.h new file mode 100644 index 00000000..8752f9ca --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_hash.h @@ -0,0 +1,40 @@ +#ifndef crypto_hash_H +#define crypto_hash_H + +/* + * WARNING: Unless you absolutely need to use SHA512 for interoperatibility, + * purposes, you might want to consider crypto_generichash() instead. + * Unlike SHA512, crypto_generichash() is not vulnerable to length + * extension attacks. + */ + +#include + +#include "crypto_hash_sha512.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_hash_BYTES crypto_hash_sha512_BYTES +SODIUM_EXPORT +size_t crypto_hash_bytes(void); + +SODIUM_EXPORT +int crypto_hash(unsigned char *out, const unsigned char *in, + unsigned long long inlen) __attribute__ ((nonnull(1))); + +#define crypto_hash_PRIMITIVE "sha512" +SODIUM_EXPORT +const char *crypto_hash_primitive(void) + __attribute__ ((warn_unused_result)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_hash_sha256.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_hash_sha256.h new file mode 100644 index 00000000..b18217e1 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_hash_sha256.h @@ -0,0 +1,60 @@ +#ifndef crypto_hash_sha256_H +#define crypto_hash_sha256_H + +/* + * WARNING: Unless you absolutely need to use SHA256 for interoperatibility, + * purposes, you might want to consider crypto_generichash() instead. + * Unlike SHA256, crypto_generichash() is not vulnerable to length + * extension attacks. + */ + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef struct crypto_hash_sha256_state { + uint32_t state[8]; + uint64_t count; + uint8_t buf[64]; +} crypto_hash_sha256_state; + +SODIUM_EXPORT +size_t crypto_hash_sha256_statebytes(void); + +#define crypto_hash_sha256_BYTES 32U +SODIUM_EXPORT +size_t crypto_hash_sha256_bytes(void); + +SODIUM_EXPORT +int crypto_hash_sha256(unsigned char *out, const unsigned char *in, + unsigned long long inlen) __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_hash_sha256_init(crypto_hash_sha256_state *state) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_hash_sha256_update(crypto_hash_sha256_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_hash_sha256_final(crypto_hash_sha256_state *state, + unsigned char *out) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_hash_sha512.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_hash_sha512.h new file mode 100644 index 00000000..8efa7193 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_hash_sha512.h @@ -0,0 +1,60 @@ +#ifndef crypto_hash_sha512_H +#define crypto_hash_sha512_H + +/* + * WARNING: Unless you absolutely need to use SHA512 for interoperatibility, + * purposes, you might want to consider crypto_generichash() instead. + * Unlike SHA512, crypto_generichash() is not vulnerable to length + * extension attacks. + */ + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef struct crypto_hash_sha512_state { + uint64_t state[8]; + uint64_t count[2]; + uint8_t buf[128]; +} crypto_hash_sha512_state; + +SODIUM_EXPORT +size_t crypto_hash_sha512_statebytes(void); + +#define crypto_hash_sha512_BYTES 64U +SODIUM_EXPORT +size_t crypto_hash_sha512_bytes(void); + +SODIUM_EXPORT +int crypto_hash_sha512(unsigned char *out, const unsigned char *in, + unsigned long long inlen) __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_hash_sha512_init(crypto_hash_sha512_state *state) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_hash_sha512_update(crypto_hash_sha512_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_hash_sha512_final(crypto_hash_sha512_state *state, + unsigned char *out) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_kdf.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_kdf.h new file mode 100644 index 00000000..ac2fc618 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_kdf.h @@ -0,0 +1,53 @@ +#ifndef crypto_kdf_H +#define crypto_kdf_H + +#include +#include + +#include "crypto_kdf_blake2b.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_kdf_BYTES_MIN crypto_kdf_blake2b_BYTES_MIN +SODIUM_EXPORT +size_t crypto_kdf_bytes_min(void); + +#define crypto_kdf_BYTES_MAX crypto_kdf_blake2b_BYTES_MAX +SODIUM_EXPORT +size_t crypto_kdf_bytes_max(void); + +#define crypto_kdf_CONTEXTBYTES crypto_kdf_blake2b_CONTEXTBYTES +SODIUM_EXPORT +size_t crypto_kdf_contextbytes(void); + +#define crypto_kdf_KEYBYTES crypto_kdf_blake2b_KEYBYTES +SODIUM_EXPORT +size_t crypto_kdf_keybytes(void); + +#define crypto_kdf_PRIMITIVE "blake2b" +SODIUM_EXPORT +const char *crypto_kdf_primitive(void) + __attribute__ ((warn_unused_result)); + +SODIUM_EXPORT +int crypto_kdf_derive_from_key(unsigned char *subkey, size_t subkey_len, + uint64_t subkey_id, + const char ctx[crypto_kdf_CONTEXTBYTES], + const unsigned char key[crypto_kdf_KEYBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_kdf_keygen(unsigned char k[crypto_kdf_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_kdf_blake2b.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_kdf_blake2b.h new file mode 100644 index 00000000..3ae47dd3 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_kdf_blake2b.h @@ -0,0 +1,44 @@ +#ifndef crypto_kdf_blake2b_H +#define crypto_kdf_blake2b_H + +#include +#include + +#include "crypto_kdf_blake2b.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_kdf_blake2b_BYTES_MIN 16 +SODIUM_EXPORT +size_t crypto_kdf_blake2b_bytes_min(void); + +#define crypto_kdf_blake2b_BYTES_MAX 64 +SODIUM_EXPORT +size_t crypto_kdf_blake2b_bytes_max(void); + +#define crypto_kdf_blake2b_CONTEXTBYTES 8 +SODIUM_EXPORT +size_t crypto_kdf_blake2b_contextbytes(void); + +#define crypto_kdf_blake2b_KEYBYTES 32 +SODIUM_EXPORT +size_t crypto_kdf_blake2b_keybytes(void); + +SODIUM_EXPORT +int crypto_kdf_blake2b_derive_from_key(unsigned char *subkey, size_t subkey_len, + uint64_t subkey_id, + const char ctx[crypto_kdf_blake2b_CONTEXTBYTES], + const unsigned char key[crypto_kdf_blake2b_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_kx.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_kx.h new file mode 100644 index 00000000..347132c3 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_kx.h @@ -0,0 +1,66 @@ +#ifndef crypto_kx_H +#define crypto_kx_H + +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_kx_PUBLICKEYBYTES 32 +SODIUM_EXPORT +size_t crypto_kx_publickeybytes(void); + +#define crypto_kx_SECRETKEYBYTES 32 +SODIUM_EXPORT +size_t crypto_kx_secretkeybytes(void); + +#define crypto_kx_SEEDBYTES 32 +SODIUM_EXPORT +size_t crypto_kx_seedbytes(void); + +#define crypto_kx_SESSIONKEYBYTES 32 +SODIUM_EXPORT +size_t crypto_kx_sessionkeybytes(void); + +#define crypto_kx_PRIMITIVE "x25519blake2b" +SODIUM_EXPORT +const char *crypto_kx_primitive(void); + +SODIUM_EXPORT +int crypto_kx_seed_keypair(unsigned char pk[crypto_kx_PUBLICKEYBYTES], + unsigned char sk[crypto_kx_SECRETKEYBYTES], + const unsigned char seed[crypto_kx_SEEDBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_kx_keypair(unsigned char pk[crypto_kx_PUBLICKEYBYTES], + unsigned char sk[crypto_kx_SECRETKEYBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_kx_client_session_keys(unsigned char rx[crypto_kx_SESSIONKEYBYTES], + unsigned char tx[crypto_kx_SESSIONKEYBYTES], + const unsigned char client_pk[crypto_kx_PUBLICKEYBYTES], + const unsigned char client_sk[crypto_kx_SECRETKEYBYTES], + const unsigned char server_pk[crypto_kx_PUBLICKEYBYTES]) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 4, 5))); + +SODIUM_EXPORT +int crypto_kx_server_session_keys(unsigned char rx[crypto_kx_SESSIONKEYBYTES], + unsigned char tx[crypto_kx_SESSIONKEYBYTES], + const unsigned char server_pk[crypto_kx_PUBLICKEYBYTES], + const unsigned char server_sk[crypto_kx_SECRETKEYBYTES], + const unsigned char client_pk[crypto_kx_PUBLICKEYBYTES]) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 4, 5))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_onetimeauth.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_onetimeauth.h new file mode 100644 index 00000000..7cd7b070 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_onetimeauth.h @@ -0,0 +1,65 @@ +#ifndef crypto_onetimeauth_H +#define crypto_onetimeauth_H + +#include + +#include "crypto_onetimeauth_poly1305.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef crypto_onetimeauth_poly1305_state crypto_onetimeauth_state; + +SODIUM_EXPORT +size_t crypto_onetimeauth_statebytes(void); + +#define crypto_onetimeauth_BYTES crypto_onetimeauth_poly1305_BYTES +SODIUM_EXPORT +size_t crypto_onetimeauth_bytes(void); + +#define crypto_onetimeauth_KEYBYTES crypto_onetimeauth_poly1305_KEYBYTES +SODIUM_EXPORT +size_t crypto_onetimeauth_keybytes(void); + +#define crypto_onetimeauth_PRIMITIVE "poly1305" +SODIUM_EXPORT +const char *crypto_onetimeauth_primitive(void); + +SODIUM_EXPORT +int crypto_onetimeauth(unsigned char *out, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_onetimeauth_verify(const unsigned char *h, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_onetimeauth_init(crypto_onetimeauth_state *state, + const unsigned char *key) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_onetimeauth_update(crypto_onetimeauth_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_onetimeauth_final(crypto_onetimeauth_state *state, + unsigned char *out) __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_onetimeauth_keygen(unsigned char k[crypto_onetimeauth_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_onetimeauth_poly1305.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_onetimeauth_poly1305.h new file mode 100644 index 00000000..f3e34d86 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_onetimeauth_poly1305.h @@ -0,0 +1,72 @@ +#ifndef crypto_onetimeauth_poly1305_H +#define crypto_onetimeauth_poly1305_H + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#include +#include +#include + +#include + +#include "export.h" + +typedef struct CRYPTO_ALIGN(16) crypto_onetimeauth_poly1305_state { + unsigned char opaque[256]; +} crypto_onetimeauth_poly1305_state; + +SODIUM_EXPORT +size_t crypto_onetimeauth_poly1305_statebytes(void); + +#define crypto_onetimeauth_poly1305_BYTES 16U +SODIUM_EXPORT +size_t crypto_onetimeauth_poly1305_bytes(void); + +#define crypto_onetimeauth_poly1305_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_onetimeauth_poly1305_keybytes(void); + +SODIUM_EXPORT +int crypto_onetimeauth_poly1305(unsigned char *out, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_onetimeauth_poly1305_verify(const unsigned char *h, + const unsigned char *in, + unsigned long long inlen, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_onetimeauth_poly1305_init(crypto_onetimeauth_poly1305_state *state, + const unsigned char *key) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_onetimeauth_poly1305_update(crypto_onetimeauth_poly1305_state *state, + const unsigned char *in, + unsigned long long inlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_onetimeauth_poly1305_final(crypto_onetimeauth_poly1305_state *state, + unsigned char *out) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_onetimeauth_poly1305_keygen(unsigned char k[crypto_onetimeauth_poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_pwhash.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_pwhash.h new file mode 100644 index 00000000..585a993e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_pwhash.h @@ -0,0 +1,147 @@ +#ifndef crypto_pwhash_H +#define crypto_pwhash_H + +#include + +#include "crypto_pwhash_argon2i.h" +#include "crypto_pwhash_argon2id.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_pwhash_ALG_ARGON2I13 crypto_pwhash_argon2i_ALG_ARGON2I13 +SODIUM_EXPORT +int crypto_pwhash_alg_argon2i13(void); + +#define crypto_pwhash_ALG_ARGON2ID13 crypto_pwhash_argon2id_ALG_ARGON2ID13 +SODIUM_EXPORT +int crypto_pwhash_alg_argon2id13(void); + +#define crypto_pwhash_ALG_DEFAULT crypto_pwhash_ALG_ARGON2ID13 +SODIUM_EXPORT +int crypto_pwhash_alg_default(void); + +#define crypto_pwhash_BYTES_MIN crypto_pwhash_argon2id_BYTES_MIN +SODIUM_EXPORT +size_t crypto_pwhash_bytes_min(void); + +#define crypto_pwhash_BYTES_MAX crypto_pwhash_argon2id_BYTES_MAX +SODIUM_EXPORT +size_t crypto_pwhash_bytes_max(void); + +#define crypto_pwhash_PASSWD_MIN crypto_pwhash_argon2id_PASSWD_MIN +SODIUM_EXPORT +size_t crypto_pwhash_passwd_min(void); + +#define crypto_pwhash_PASSWD_MAX crypto_pwhash_argon2id_PASSWD_MAX +SODIUM_EXPORT +size_t crypto_pwhash_passwd_max(void); + +#define crypto_pwhash_SALTBYTES crypto_pwhash_argon2id_SALTBYTES +SODIUM_EXPORT +size_t crypto_pwhash_saltbytes(void); + +#define crypto_pwhash_STRBYTES crypto_pwhash_argon2id_STRBYTES +SODIUM_EXPORT +size_t crypto_pwhash_strbytes(void); + +#define crypto_pwhash_STRPREFIX crypto_pwhash_argon2id_STRPREFIX +SODIUM_EXPORT +const char *crypto_pwhash_strprefix(void); + +#define crypto_pwhash_OPSLIMIT_MIN crypto_pwhash_argon2id_OPSLIMIT_MIN +SODIUM_EXPORT +size_t crypto_pwhash_opslimit_min(void); + +#define crypto_pwhash_OPSLIMIT_MAX crypto_pwhash_argon2id_OPSLIMIT_MAX +SODIUM_EXPORT +size_t crypto_pwhash_opslimit_max(void); + +#define crypto_pwhash_MEMLIMIT_MIN crypto_pwhash_argon2id_MEMLIMIT_MIN +SODIUM_EXPORT +size_t crypto_pwhash_memlimit_min(void); + +#define crypto_pwhash_MEMLIMIT_MAX crypto_pwhash_argon2id_MEMLIMIT_MAX +SODIUM_EXPORT +size_t crypto_pwhash_memlimit_max(void); + +#define crypto_pwhash_OPSLIMIT_INTERACTIVE crypto_pwhash_argon2id_OPSLIMIT_INTERACTIVE +SODIUM_EXPORT +size_t crypto_pwhash_opslimit_interactive(void); + +#define crypto_pwhash_MEMLIMIT_INTERACTIVE crypto_pwhash_argon2id_MEMLIMIT_INTERACTIVE +SODIUM_EXPORT +size_t crypto_pwhash_memlimit_interactive(void); + +#define crypto_pwhash_OPSLIMIT_MODERATE crypto_pwhash_argon2id_OPSLIMIT_MODERATE +SODIUM_EXPORT +size_t crypto_pwhash_opslimit_moderate(void); + +#define crypto_pwhash_MEMLIMIT_MODERATE crypto_pwhash_argon2id_MEMLIMIT_MODERATE +SODIUM_EXPORT +size_t crypto_pwhash_memlimit_moderate(void); + +#define crypto_pwhash_OPSLIMIT_SENSITIVE crypto_pwhash_argon2id_OPSLIMIT_SENSITIVE +SODIUM_EXPORT +size_t crypto_pwhash_opslimit_sensitive(void); + +#define crypto_pwhash_MEMLIMIT_SENSITIVE crypto_pwhash_argon2id_MEMLIMIT_SENSITIVE +SODIUM_EXPORT +size_t crypto_pwhash_memlimit_sensitive(void); + +/* + * With this function, do not forget to store all parameters, including the + * algorithm identifier in order to produce deterministic output. + * The crypto_pwhash_* definitions, including crypto_pwhash_ALG_DEFAULT, + * may change. + */ +SODIUM_EXPORT +int crypto_pwhash(unsigned char * const out, unsigned long long outlen, + const char * const passwd, unsigned long long passwdlen, + const unsigned char * const salt, + unsigned long long opslimit, size_t memlimit, int alg) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +/* + * The output string already includes all the required parameters, including + * the algorithm identifier. The string is all that has to be stored in + * order to verify a password. + */ +SODIUM_EXPORT +int crypto_pwhash_str(char out[crypto_pwhash_STRBYTES], + const char * const passwd, unsigned long long passwdlen, + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_str_alg(char out[crypto_pwhash_STRBYTES], + const char * const passwd, unsigned long long passwdlen, + unsigned long long opslimit, size_t memlimit, int alg) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_str_verify(const char str[crypto_pwhash_STRBYTES], + const char * const passwd, + unsigned long long passwdlen) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_str_needs_rehash(const char str[crypto_pwhash_STRBYTES], + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#define crypto_pwhash_PRIMITIVE "argon2i" +SODIUM_EXPORT +const char *crypto_pwhash_primitive(void) + __attribute__ ((warn_unused_result)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_pwhash_argon2i.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_pwhash_argon2i.h new file mode 100644 index 00000000..88ff6221 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_pwhash_argon2i.h @@ -0,0 +1,122 @@ +#ifndef crypto_pwhash_argon2i_H +#define crypto_pwhash_argon2i_H + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_pwhash_argon2i_ALG_ARGON2I13 1 +SODIUM_EXPORT +int crypto_pwhash_argon2i_alg_argon2i13(void); + +#define crypto_pwhash_argon2i_BYTES_MIN 16U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_bytes_min(void); + +#define crypto_pwhash_argon2i_BYTES_MAX SODIUM_MIN(SODIUM_SIZE_MAX, 4294967295U) +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_bytes_max(void); + +#define crypto_pwhash_argon2i_PASSWD_MIN 0U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_passwd_min(void); + +#define crypto_pwhash_argon2i_PASSWD_MAX 4294967295U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_passwd_max(void); + +#define crypto_pwhash_argon2i_SALTBYTES 16U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_saltbytes(void); + +#define crypto_pwhash_argon2i_STRBYTES 128U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_strbytes(void); + +#define crypto_pwhash_argon2i_STRPREFIX "$argon2i$" +SODIUM_EXPORT +const char *crypto_pwhash_argon2i_strprefix(void); + +#define crypto_pwhash_argon2i_OPSLIMIT_MIN 3U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_opslimit_min(void); + +#define crypto_pwhash_argon2i_OPSLIMIT_MAX 4294967295U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_opslimit_max(void); + +#define crypto_pwhash_argon2i_MEMLIMIT_MIN 8192U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_memlimit_min(void); + +#define crypto_pwhash_argon2i_MEMLIMIT_MAX \ + ((SIZE_MAX >= 4398046510080U) ? 4398046510080U : (SIZE_MAX >= 2147483648U) ? 2147483648U : 32768U) +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_memlimit_max(void); + +#define crypto_pwhash_argon2i_OPSLIMIT_INTERACTIVE 4U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_opslimit_interactive(void); + +#define crypto_pwhash_argon2i_MEMLIMIT_INTERACTIVE 33554432U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_memlimit_interactive(void); + +#define crypto_pwhash_argon2i_OPSLIMIT_MODERATE 6U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_opslimit_moderate(void); + +#define crypto_pwhash_argon2i_MEMLIMIT_MODERATE 134217728U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_memlimit_moderate(void); + +#define crypto_pwhash_argon2i_OPSLIMIT_SENSITIVE 8U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_opslimit_sensitive(void); + +#define crypto_pwhash_argon2i_MEMLIMIT_SENSITIVE 536870912U +SODIUM_EXPORT +size_t crypto_pwhash_argon2i_memlimit_sensitive(void); + +SODIUM_EXPORT +int crypto_pwhash_argon2i(unsigned char * const out, + unsigned long long outlen, + const char * const passwd, + unsigned long long passwdlen, + const unsigned char * const salt, + unsigned long long opslimit, size_t memlimit, + int alg) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2i_str(char out[crypto_pwhash_argon2i_STRBYTES], + const char * const passwd, + unsigned long long passwdlen, + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2i_str_verify(const char str[crypto_pwhash_argon2i_STRBYTES], + const char * const passwd, + unsigned long long passwdlen) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2i_str_needs_rehash(const char str[crypto_pwhash_argon2i_STRBYTES], + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_pwhash_argon2id.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_pwhash_argon2id.h new file mode 100644 index 00000000..7183abd1 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_pwhash_argon2id.h @@ -0,0 +1,122 @@ +#ifndef crypto_pwhash_argon2id_H +#define crypto_pwhash_argon2id_H + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_pwhash_argon2id_ALG_ARGON2ID13 2 +SODIUM_EXPORT +int crypto_pwhash_argon2id_alg_argon2id13(void); + +#define crypto_pwhash_argon2id_BYTES_MIN 16U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_bytes_min(void); + +#define crypto_pwhash_argon2id_BYTES_MAX SODIUM_MIN(SODIUM_SIZE_MAX, 4294967295U) +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_bytes_max(void); + +#define crypto_pwhash_argon2id_PASSWD_MIN 0U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_passwd_min(void); + +#define crypto_pwhash_argon2id_PASSWD_MAX 4294967295U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_passwd_max(void); + +#define crypto_pwhash_argon2id_SALTBYTES 16U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_saltbytes(void); + +#define crypto_pwhash_argon2id_STRBYTES 128U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_strbytes(void); + +#define crypto_pwhash_argon2id_STRPREFIX "$argon2id$" +SODIUM_EXPORT +const char *crypto_pwhash_argon2id_strprefix(void); + +#define crypto_pwhash_argon2id_OPSLIMIT_MIN 1U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_opslimit_min(void); + +#define crypto_pwhash_argon2id_OPSLIMIT_MAX 4294967295U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_opslimit_max(void); + +#define crypto_pwhash_argon2id_MEMLIMIT_MIN 8192U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_memlimit_min(void); + +#define crypto_pwhash_argon2id_MEMLIMIT_MAX \ + ((SIZE_MAX >= 4398046510080U) ? 4398046510080U : (SIZE_MAX >= 2147483648U) ? 2147483648U : 32768U) +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_memlimit_max(void); + +#define crypto_pwhash_argon2id_OPSLIMIT_INTERACTIVE 2U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_opslimit_interactive(void); + +#define crypto_pwhash_argon2id_MEMLIMIT_INTERACTIVE 67108864U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_memlimit_interactive(void); + +#define crypto_pwhash_argon2id_OPSLIMIT_MODERATE 3U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_opslimit_moderate(void); + +#define crypto_pwhash_argon2id_MEMLIMIT_MODERATE 268435456U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_memlimit_moderate(void); + +#define crypto_pwhash_argon2id_OPSLIMIT_SENSITIVE 4U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_opslimit_sensitive(void); + +#define crypto_pwhash_argon2id_MEMLIMIT_SENSITIVE 1073741824U +SODIUM_EXPORT +size_t crypto_pwhash_argon2id_memlimit_sensitive(void); + +SODIUM_EXPORT +int crypto_pwhash_argon2id(unsigned char * const out, + unsigned long long outlen, + const char * const passwd, + unsigned long long passwdlen, + const unsigned char * const salt, + unsigned long long opslimit, size_t memlimit, + int alg) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2id_str(char out[crypto_pwhash_argon2id_STRBYTES], + const char * const passwd, + unsigned long long passwdlen, + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2id_str_verify(const char str[crypto_pwhash_argon2id_STRBYTES], + const char * const passwd, + unsigned long long passwdlen) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_argon2id_str_needs_rehash(const char str[crypto_pwhash_argon2id_STRBYTES], + unsigned long long opslimit, size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_pwhash_scryptsalsa208sha256.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_pwhash_scryptsalsa208sha256.h new file mode 100644 index 00000000..5c0bf7d3 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_pwhash_scryptsalsa208sha256.h @@ -0,0 +1,120 @@ +#ifndef crypto_pwhash_scryptsalsa208sha256_H +#define crypto_pwhash_scryptsalsa208sha256_H + +#include +#include +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_pwhash_scryptsalsa208sha256_BYTES_MIN 16U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_bytes_min(void); + +#define crypto_pwhash_scryptsalsa208sha256_BYTES_MAX \ + SODIUM_MIN(SODIUM_SIZE_MAX, 0x1fffffffe0ULL) +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_bytes_max(void); + +#define crypto_pwhash_scryptsalsa208sha256_PASSWD_MIN 0U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_passwd_min(void); + +#define crypto_pwhash_scryptsalsa208sha256_PASSWD_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_passwd_max(void); + +#define crypto_pwhash_scryptsalsa208sha256_SALTBYTES 32U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_saltbytes(void); + +#define crypto_pwhash_scryptsalsa208sha256_STRBYTES 102U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_strbytes(void); + +#define crypto_pwhash_scryptsalsa208sha256_STRPREFIX "$7$" +SODIUM_EXPORT +const char *crypto_pwhash_scryptsalsa208sha256_strprefix(void); + +#define crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_MIN 32768U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_opslimit_min(void); + +#define crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_MAX 4294967295U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_opslimit_max(void); + +#define crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_MIN 16777216U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_memlimit_min(void); + +#define crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_MAX \ + SODIUM_MIN(SIZE_MAX, 68719476736ULL) +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_memlimit_max(void); + +#define crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_INTERACTIVE 524288U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_opslimit_interactive(void); + +#define crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_INTERACTIVE 16777216U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_memlimit_interactive(void); + +#define crypto_pwhash_scryptsalsa208sha256_OPSLIMIT_SENSITIVE 33554432U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_opslimit_sensitive(void); + +#define crypto_pwhash_scryptsalsa208sha256_MEMLIMIT_SENSITIVE 1073741824U +SODIUM_EXPORT +size_t crypto_pwhash_scryptsalsa208sha256_memlimit_sensitive(void); + +SODIUM_EXPORT +int crypto_pwhash_scryptsalsa208sha256(unsigned char * const out, + unsigned long long outlen, + const char * const passwd, + unsigned long long passwdlen, + const unsigned char * const salt, + unsigned long long opslimit, + size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_scryptsalsa208sha256_str(char out[crypto_pwhash_scryptsalsa208sha256_STRBYTES], + const char * const passwd, + unsigned long long passwdlen, + unsigned long long opslimit, + size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_scryptsalsa208sha256_str_verify(const char str[crypto_pwhash_scryptsalsa208sha256_STRBYTES], + const char * const passwd, + unsigned long long passwdlen) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_scryptsalsa208sha256_ll(const uint8_t * passwd, size_t passwdlen, + const uint8_t * salt, size_t saltlen, + uint64_t N, uint32_t r, uint32_t p, + uint8_t * buf, size_t buflen) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_pwhash_scryptsalsa208sha256_str_needs_rehash(const char str[crypto_pwhash_scryptsalsa208sha256_STRBYTES], + unsigned long long opslimit, + size_t memlimit) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_scalarmult.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_scalarmult.h new file mode 100644 index 00000000..1c685853 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_scalarmult.h @@ -0,0 +1,46 @@ +#ifndef crypto_scalarmult_H +#define crypto_scalarmult_H + +#include + +#include "crypto_scalarmult_curve25519.h" +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_scalarmult_BYTES crypto_scalarmult_curve25519_BYTES +SODIUM_EXPORT +size_t crypto_scalarmult_bytes(void); + +#define crypto_scalarmult_SCALARBYTES crypto_scalarmult_curve25519_SCALARBYTES +SODIUM_EXPORT +size_t crypto_scalarmult_scalarbytes(void); + +#define crypto_scalarmult_PRIMITIVE "curve25519" +SODIUM_EXPORT +const char *crypto_scalarmult_primitive(void); + +SODIUM_EXPORT +int crypto_scalarmult_base(unsigned char *q, const unsigned char *n) + __attribute__ ((nonnull)); + +/* + * NOTE: Do not use the result of this function directly for key exchange. + * + * Hash the result with the public keys in order to compute a shared + * secret key: H(q || client_pk || server_pk) + * + * Or unless this is not an option, use the crypto_kx() API instead. + */ +SODIUM_EXPORT +int crypto_scalarmult(unsigned char *q, const unsigned char *n, + const unsigned char *p) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_scalarmult_curve25519.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_scalarmult_curve25519.h new file mode 100644 index 00000000..60e9d0c5 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_scalarmult_curve25519.h @@ -0,0 +1,42 @@ +#ifndef crypto_scalarmult_curve25519_H +#define crypto_scalarmult_curve25519_H + +#include + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_scalarmult_curve25519_BYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_curve25519_bytes(void); + +#define crypto_scalarmult_curve25519_SCALARBYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_curve25519_scalarbytes(void); + +/* + * NOTE: Do not use the result of this function directly for key exchange. + * + * Hash the result with the public keys in order to compute a shared + * secret key: H(q || client_pk || server_pk) + * + * Or unless this is not an option, use the crypto_kx() API instead. + */ +SODIUM_EXPORT +int crypto_scalarmult_curve25519(unsigned char *q, const unsigned char *n, + const unsigned char *p) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_scalarmult_curve25519_base(unsigned char *q, + const unsigned char *n) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_scalarmult_ed25519.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_scalarmult_ed25519.h new file mode 100644 index 00000000..2dfa4d70 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_scalarmult_ed25519.h @@ -0,0 +1,51 @@ + +#ifndef crypto_scalarmult_ed25519_H +#define crypto_scalarmult_ed25519_H + +#include + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_scalarmult_ed25519_BYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_ed25519_bytes(void); + +#define crypto_scalarmult_ed25519_SCALARBYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_ed25519_scalarbytes(void); + +/* + * NOTE: Do not use the result of this function directly for key exchange. + * + * Hash the result with the public keys in order to compute a shared + * secret key: H(q || client_pk || server_pk) + * + * Or unless this is not an option, use the crypto_kx() API instead. + */ +SODIUM_EXPORT +int crypto_scalarmult_ed25519(unsigned char *q, const unsigned char *n, + const unsigned char *p) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_scalarmult_ed25519_noclamp(unsigned char *q, const unsigned char *n, + const unsigned char *p) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_scalarmult_ed25519_base(unsigned char *q, const unsigned char *n) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_scalarmult_ed25519_base_noclamp(unsigned char *q, const unsigned char *n) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_scalarmult_ristretto255.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_scalarmult_ristretto255.h new file mode 100644 index 00000000..40a45cce --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_scalarmult_ristretto255.h @@ -0,0 +1,43 @@ + +#ifndef crypto_scalarmult_ristretto255_H +#define crypto_scalarmult_ristretto255_H + +#include + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_scalarmult_ristretto255_BYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_ristretto255_bytes(void); + +#define crypto_scalarmult_ristretto255_SCALARBYTES 32U +SODIUM_EXPORT +size_t crypto_scalarmult_ristretto255_scalarbytes(void); + +/* + * NOTE: Do not use the result of this function directly for key exchange. + * + * Hash the result with the public keys in order to compute a shared + * secret key: H(q || client_pk || server_pk) + * + * Or unless this is not an option, use the crypto_kx() API instead. + */ +SODIUM_EXPORT +int crypto_scalarmult_ristretto255(unsigned char *q, const unsigned char *n, + const unsigned char *p) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_scalarmult_ristretto255_base(unsigned char *q, + const unsigned char *n) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_secretbox.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_secretbox.h new file mode 100644 index 00000000..1d3709db --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_secretbox.h @@ -0,0 +1,93 @@ +#ifndef crypto_secretbox_H +#define crypto_secretbox_H + +#include + +#include "crypto_secretbox_xsalsa20poly1305.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_secretbox_KEYBYTES crypto_secretbox_xsalsa20poly1305_KEYBYTES +SODIUM_EXPORT +size_t crypto_secretbox_keybytes(void); + +#define crypto_secretbox_NONCEBYTES crypto_secretbox_xsalsa20poly1305_NONCEBYTES +SODIUM_EXPORT +size_t crypto_secretbox_noncebytes(void); + +#define crypto_secretbox_MACBYTES crypto_secretbox_xsalsa20poly1305_MACBYTES +SODIUM_EXPORT +size_t crypto_secretbox_macbytes(void); + +#define crypto_secretbox_PRIMITIVE "xsalsa20poly1305" +SODIUM_EXPORT +const char *crypto_secretbox_primitive(void); + +#define crypto_secretbox_MESSAGEBYTES_MAX crypto_secretbox_xsalsa20poly1305_MESSAGEBYTES_MAX +SODIUM_EXPORT +size_t crypto_secretbox_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_secretbox_easy(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_open_easy(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_detached(unsigned char *c, unsigned char *mac, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 5, 6))); + +SODIUM_EXPORT +int crypto_secretbox_open_detached(unsigned char *m, + const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6))); + +SODIUM_EXPORT +void crypto_secretbox_keygen(unsigned char k[crypto_secretbox_KEYBYTES]) + __attribute__ ((nonnull)); + +/* -- NaCl compatibility interface ; Requires padding -- */ + +#define crypto_secretbox_ZEROBYTES crypto_secretbox_xsalsa20poly1305_ZEROBYTES +SODIUM_EXPORT +size_t crypto_secretbox_zerobytes(void); + +#define crypto_secretbox_BOXZEROBYTES crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES +SODIUM_EXPORT +size_t crypto_secretbox_boxzerobytes(void); + +SODIUM_EXPORT +int crypto_secretbox(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_open(unsigned char *m, const unsigned char *c, + unsigned long long clen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_secretbox_xchacha20poly1305.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_secretbox_xchacha20poly1305.h new file mode 100644 index 00000000..6ec674e3 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_secretbox_xchacha20poly1305.h @@ -0,0 +1,70 @@ +#ifndef crypto_secretbox_xchacha20poly1305_H +#define crypto_secretbox_xchacha20poly1305_H + +#include +#include "crypto_stream_xchacha20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_secretbox_xchacha20poly1305_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_secretbox_xchacha20poly1305_keybytes(void); + +#define crypto_secretbox_xchacha20poly1305_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_secretbox_xchacha20poly1305_noncebytes(void); + +#define crypto_secretbox_xchacha20poly1305_MACBYTES 16U +SODIUM_EXPORT +size_t crypto_secretbox_xchacha20poly1305_macbytes(void); + +#define crypto_secretbox_xchacha20poly1305_MESSAGEBYTES_MAX \ + (crypto_stream_xchacha20_MESSAGEBYTES_MAX - crypto_secretbox_xchacha20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_secretbox_xchacha20poly1305_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_secretbox_xchacha20poly1305_easy(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_xchacha20poly1305_open_easy(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_xchacha20poly1305_detached(unsigned char *c, + unsigned char *mac, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 2, 5, 6))); + +SODIUM_EXPORT +int crypto_secretbox_xchacha20poly1305_open_detached(unsigned char *m, + const unsigned char *c, + const unsigned char *mac, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 3, 5, 6))); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_secretbox_xsalsa20poly1305.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_secretbox_xsalsa20poly1305.h new file mode 100644 index 00000000..be0874cb --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_secretbox_xsalsa20poly1305.h @@ -0,0 +1,69 @@ +#ifndef crypto_secretbox_xsalsa20poly1305_H +#define crypto_secretbox_xsalsa20poly1305_H + +#include +#include "crypto_stream_xsalsa20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_secretbox_xsalsa20poly1305_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_keybytes(void); + +#define crypto_secretbox_xsalsa20poly1305_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_noncebytes(void); + +#define crypto_secretbox_xsalsa20poly1305_MACBYTES 16U +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_macbytes(void); + +/* Only for the libsodium API - The NaCl compatibility API would require BOXZEROBYTES extra bytes */ +#define crypto_secretbox_xsalsa20poly1305_MESSAGEBYTES_MAX \ + (crypto_stream_xsalsa20_MESSAGEBYTES_MAX - crypto_secretbox_xsalsa20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_secretbox_xsalsa20poly1305(unsigned char *c, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull(1, 4, 5))); + +SODIUM_EXPORT +int crypto_secretbox_xsalsa20poly1305_open(unsigned char *m, + const unsigned char *c, + unsigned long long clen, + const unsigned char *n, + const unsigned char *k) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(2, 4, 5))); + +SODIUM_EXPORT +void crypto_secretbox_xsalsa20poly1305_keygen(unsigned char k[crypto_secretbox_xsalsa20poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +/* -- NaCl compatibility interface ; Requires padding -- */ + +#define crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES 16U +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_boxzerobytes(void); + +#define crypto_secretbox_xsalsa20poly1305_ZEROBYTES \ + (crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES + \ + crypto_secretbox_xsalsa20poly1305_MACBYTES) +SODIUM_EXPORT +size_t crypto_secretbox_xsalsa20poly1305_zerobytes(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_secretstream_xchacha20poly1305.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_secretstream_xchacha20poly1305.h new file mode 100644 index 00000000..b22e4e93 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_secretstream_xchacha20poly1305.h @@ -0,0 +1,108 @@ +#ifndef crypto_secretstream_xchacha20poly1305_H +#define crypto_secretstream_xchacha20poly1305_H + +#include + +#include "crypto_aead_xchacha20poly1305.h" +#include "crypto_stream_chacha20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_secretstream_xchacha20poly1305_ABYTES \ + (1U + crypto_aead_xchacha20poly1305_ietf_ABYTES) +SODIUM_EXPORT +size_t crypto_secretstream_xchacha20poly1305_abytes(void); + +#define crypto_secretstream_xchacha20poly1305_HEADERBYTES \ + crypto_aead_xchacha20poly1305_ietf_NPUBBYTES +SODIUM_EXPORT +size_t crypto_secretstream_xchacha20poly1305_headerbytes(void); + +#define crypto_secretstream_xchacha20poly1305_KEYBYTES \ + crypto_aead_xchacha20poly1305_ietf_KEYBYTES +SODIUM_EXPORT +size_t crypto_secretstream_xchacha20poly1305_keybytes(void); + +#define crypto_secretstream_xchacha20poly1305_MESSAGEBYTES_MAX \ + SODIUM_MIN(SODIUM_SIZE_MAX - crypto_secretstream_xchacha20poly1305_ABYTES, \ + (64ULL * ((1ULL << 32) - 2ULL))) +SODIUM_EXPORT +size_t crypto_secretstream_xchacha20poly1305_messagebytes_max(void); + +#define crypto_secretstream_xchacha20poly1305_TAG_MESSAGE 0x00 +SODIUM_EXPORT +unsigned char crypto_secretstream_xchacha20poly1305_tag_message(void); + +#define crypto_secretstream_xchacha20poly1305_TAG_PUSH 0x01 +SODIUM_EXPORT +unsigned char crypto_secretstream_xchacha20poly1305_tag_push(void); + +#define crypto_secretstream_xchacha20poly1305_TAG_REKEY 0x02 +SODIUM_EXPORT +unsigned char crypto_secretstream_xchacha20poly1305_tag_rekey(void); + +#define crypto_secretstream_xchacha20poly1305_TAG_FINAL \ + (crypto_secretstream_xchacha20poly1305_TAG_PUSH | \ + crypto_secretstream_xchacha20poly1305_TAG_REKEY) +SODIUM_EXPORT +unsigned char crypto_secretstream_xchacha20poly1305_tag_final(void); + +typedef struct crypto_secretstream_xchacha20poly1305_state { + unsigned char k[crypto_stream_chacha20_ietf_KEYBYTES]; + unsigned char nonce[crypto_stream_chacha20_ietf_NONCEBYTES]; + unsigned char _pad[8]; +} crypto_secretstream_xchacha20poly1305_state; + +SODIUM_EXPORT +size_t crypto_secretstream_xchacha20poly1305_statebytes(void); + +SODIUM_EXPORT +void crypto_secretstream_xchacha20poly1305_keygen + (unsigned char k[crypto_secretstream_xchacha20poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_secretstream_xchacha20poly1305_init_push + (crypto_secretstream_xchacha20poly1305_state *state, + unsigned char header[crypto_secretstream_xchacha20poly1305_HEADERBYTES], + const unsigned char k[crypto_secretstream_xchacha20poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_secretstream_xchacha20poly1305_push + (crypto_secretstream_xchacha20poly1305_state *state, + unsigned char *c, unsigned long long *clen_p, + const unsigned char *m, unsigned long long mlen, + const unsigned char *ad, unsigned long long adlen, unsigned char tag) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_secretstream_xchacha20poly1305_init_pull + (crypto_secretstream_xchacha20poly1305_state *state, + const unsigned char header[crypto_secretstream_xchacha20poly1305_HEADERBYTES], + const unsigned char k[crypto_secretstream_xchacha20poly1305_KEYBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_secretstream_xchacha20poly1305_pull + (crypto_secretstream_xchacha20poly1305_state *state, + unsigned char *m, unsigned long long *mlen_p, unsigned char *tag_p, + const unsigned char *c, unsigned long long clen, + const unsigned char *ad, unsigned long long adlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +void crypto_secretstream_xchacha20poly1305_rekey + (crypto_secretstream_xchacha20poly1305_state *state); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_shorthash.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_shorthash.h new file mode 100644 index 00000000..fecaa88b --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_shorthash.h @@ -0,0 +1,41 @@ +#ifndef crypto_shorthash_H +#define crypto_shorthash_H + +#include + +#include "crypto_shorthash_siphash24.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_shorthash_BYTES crypto_shorthash_siphash24_BYTES +SODIUM_EXPORT +size_t crypto_shorthash_bytes(void); + +#define crypto_shorthash_KEYBYTES crypto_shorthash_siphash24_KEYBYTES +SODIUM_EXPORT +size_t crypto_shorthash_keybytes(void); + +#define crypto_shorthash_PRIMITIVE "siphash24" +SODIUM_EXPORT +const char *crypto_shorthash_primitive(void); + +SODIUM_EXPORT +int crypto_shorthash(unsigned char *out, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +void crypto_shorthash_keygen(unsigned char k[crypto_shorthash_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_shorthash_siphash24.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_shorthash_siphash24.h new file mode 100644 index 00000000..1e6f72a6 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_shorthash_siphash24.h @@ -0,0 +1,50 @@ +#ifndef crypto_shorthash_siphash24_H +#define crypto_shorthash_siphash24_H + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +/* -- 64-bit output -- */ + +#define crypto_shorthash_siphash24_BYTES 8U +SODIUM_EXPORT +size_t crypto_shorthash_siphash24_bytes(void); + +#define crypto_shorthash_siphash24_KEYBYTES 16U +SODIUM_EXPORT +size_t crypto_shorthash_siphash24_keybytes(void); + +SODIUM_EXPORT +int crypto_shorthash_siphash24(unsigned char *out, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((nonnull(1, 4))); + +#ifndef SODIUM_LIBRARY_MINIMAL +/* -- 128-bit output -- */ + +#define crypto_shorthash_siphashx24_BYTES 16U +SODIUM_EXPORT +size_t crypto_shorthash_siphashx24_bytes(void); + +#define crypto_shorthash_siphashx24_KEYBYTES 16U +SODIUM_EXPORT +size_t crypto_shorthash_siphashx24_keybytes(void); + +SODIUM_EXPORT +int crypto_shorthash_siphashx24(unsigned char *out, const unsigned char *in, + unsigned long long inlen, const unsigned char *k) + __attribute__ ((nonnull(1, 4))); +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_sign.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_sign.h new file mode 100644 index 00000000..f5fafb12 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_sign.h @@ -0,0 +1,107 @@ +#ifndef crypto_sign_H +#define crypto_sign_H + +/* + * THREAD SAFETY: crypto_sign_keypair() is thread-safe, + * provided that sodium_init() was called before. + * + * Other functions, including crypto_sign_seed_keypair() are always thread-safe. + */ + +#include + +#include "crypto_sign_ed25519.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef crypto_sign_ed25519ph_state crypto_sign_state; + +SODIUM_EXPORT +size_t crypto_sign_statebytes(void); + +#define crypto_sign_BYTES crypto_sign_ed25519_BYTES +SODIUM_EXPORT +size_t crypto_sign_bytes(void); + +#define crypto_sign_SEEDBYTES crypto_sign_ed25519_SEEDBYTES +SODIUM_EXPORT +size_t crypto_sign_seedbytes(void); + +#define crypto_sign_PUBLICKEYBYTES crypto_sign_ed25519_PUBLICKEYBYTES +SODIUM_EXPORT +size_t crypto_sign_publickeybytes(void); + +#define crypto_sign_SECRETKEYBYTES crypto_sign_ed25519_SECRETKEYBYTES +SODIUM_EXPORT +size_t crypto_sign_secretkeybytes(void); + +#define crypto_sign_MESSAGEBYTES_MAX crypto_sign_ed25519_MESSAGEBYTES_MAX +SODIUM_EXPORT +size_t crypto_sign_messagebytes_max(void); + +#define crypto_sign_PRIMITIVE "ed25519" +SODIUM_EXPORT +const char *crypto_sign_primitive(void); + +SODIUM_EXPORT +int crypto_sign_seed_keypair(unsigned char *pk, unsigned char *sk, + const unsigned char *seed) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_keypair(unsigned char *pk, unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign(unsigned char *sm, unsigned long long *smlen_p, + const unsigned char *m, unsigned long long mlen, + const unsigned char *sk) __attribute__ ((nonnull(1, 5))); + +SODIUM_EXPORT +int crypto_sign_open(unsigned char *m, unsigned long long *mlen_p, + const unsigned char *sm, unsigned long long smlen, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5))); + +SODIUM_EXPORT +int crypto_sign_detached(unsigned char *sig, unsigned long long *siglen_p, + const unsigned char *m, unsigned long long mlen, + const unsigned char *sk) __attribute__ ((nonnull(1, 5))); + +SODIUM_EXPORT +int crypto_sign_verify_detached(const unsigned char *sig, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_sign_init(crypto_sign_state *state); + +SODIUM_EXPORT +int crypto_sign_update(crypto_sign_state *state, + const unsigned char *m, unsigned long long mlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_sign_final_create(crypto_sign_state *state, unsigned char *sig, + unsigned long long *siglen_p, + const unsigned char *sk) + __attribute__ ((nonnull(1, 2, 4))); + +SODIUM_EXPORT +int crypto_sign_final_verify(crypto_sign_state *state, const unsigned char *sig, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_sign_ed25519.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_sign_ed25519.h new file mode 100644 index 00000000..0fdac42d --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_sign_ed25519.h @@ -0,0 +1,124 @@ +#ifndef crypto_sign_ed25519_H +#define crypto_sign_ed25519_H + +#include +#include "crypto_hash_sha512.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef struct crypto_sign_ed25519ph_state { + crypto_hash_sha512_state hs; +} crypto_sign_ed25519ph_state; + +SODIUM_EXPORT +size_t crypto_sign_ed25519ph_statebytes(void); + +#define crypto_sign_ed25519_BYTES 64U +SODIUM_EXPORT +size_t crypto_sign_ed25519_bytes(void); + +#define crypto_sign_ed25519_SEEDBYTES 32U +SODIUM_EXPORT +size_t crypto_sign_ed25519_seedbytes(void); + +#define crypto_sign_ed25519_PUBLICKEYBYTES 32U +SODIUM_EXPORT +size_t crypto_sign_ed25519_publickeybytes(void); + +#define crypto_sign_ed25519_SECRETKEYBYTES (32U + 32U) +SODIUM_EXPORT +size_t crypto_sign_ed25519_secretkeybytes(void); + +#define crypto_sign_ed25519_MESSAGEBYTES_MAX (SODIUM_SIZE_MAX - crypto_sign_ed25519_BYTES) +SODIUM_EXPORT +size_t crypto_sign_ed25519_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_sign_ed25519(unsigned char *sm, unsigned long long *smlen_p, + const unsigned char *m, unsigned long long mlen, + const unsigned char *sk) + __attribute__ ((nonnull(1, 5))); + +SODIUM_EXPORT +int crypto_sign_ed25519_open(unsigned char *m, unsigned long long *mlen_p, + const unsigned char *sm, unsigned long long smlen, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(3, 5))); + +SODIUM_EXPORT +int crypto_sign_ed25519_detached(unsigned char *sig, + unsigned long long *siglen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *sk) + __attribute__ ((nonnull(1, 5))); + +SODIUM_EXPORT +int crypto_sign_ed25519_verify_detached(const unsigned char *sig, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull(1, 4))); + +SODIUM_EXPORT +int crypto_sign_ed25519_keypair(unsigned char *pk, unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519_seed_keypair(unsigned char *pk, unsigned char *sk, + const unsigned char *seed) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519_pk_to_curve25519(unsigned char *curve25519_pk, + const unsigned char *ed25519_pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519_sk_to_curve25519(unsigned char *curve25519_sk, + const unsigned char *ed25519_sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519_sk_to_seed(unsigned char *seed, + const unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519_sk_to_pk(unsigned char *pk, const unsigned char *sk) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519ph_init(crypto_sign_ed25519ph_state *state) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_sign_ed25519ph_update(crypto_sign_ed25519ph_state *state, + const unsigned char *m, + unsigned long long mlen) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int crypto_sign_ed25519ph_final_create(crypto_sign_ed25519ph_state *state, + unsigned char *sig, + unsigned long long *siglen_p, + const unsigned char *sk) + __attribute__ ((nonnull(1, 2, 4))); + +SODIUM_EXPORT +int crypto_sign_ed25519ph_final_verify(crypto_sign_ed25519ph_state *state, + const unsigned char *sig, + const unsigned char *pk) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_sign_edwards25519sha512batch.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_sign_edwards25519sha512batch.h new file mode 100644 index 00000000..eed158aa --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_sign_edwards25519sha512batch.h @@ -0,0 +1,55 @@ +#ifndef crypto_sign_edwards25519sha512batch_H +#define crypto_sign_edwards25519sha512batch_H + +/* + * WARNING: This construction was a prototype, which should not be used + * any more in new projects. + * + * crypto_sign_edwards25519sha512batch is provided for applications + * initially built with NaCl, but as recommended by the author of this + * construction, new applications should use ed25519 instead. + * + * In Sodium, you should use the high-level crypto_sign_*() functions instead. + */ + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_sign_edwards25519sha512batch_BYTES 64U +#define crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES 32U +#define crypto_sign_edwards25519sha512batch_SECRETKEYBYTES (32U + 32U) +#define crypto_sign_edwards25519sha512batch_MESSAGEBYTES_MAX (SODIUM_SIZE_MAX - crypto_sign_edwards25519sha512batch_BYTES) + +SODIUM_EXPORT +int crypto_sign_edwards25519sha512batch(unsigned char *sm, + unsigned long long *smlen_p, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *sk) + __attribute__ ((deprecated)) __attribute__ ((nonnull(1, 5))); + +SODIUM_EXPORT +int crypto_sign_edwards25519sha512batch_open(unsigned char *m, + unsigned long long *mlen_p, + const unsigned char *sm, + unsigned long long smlen, + const unsigned char *pk) + __attribute__ ((deprecated)) __attribute__ ((nonnull(3, 5))); + +SODIUM_EXPORT +int crypto_sign_edwards25519sha512batch_keypair(unsigned char *pk, + unsigned char *sk) + __attribute__ ((deprecated)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_stream.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_stream.h new file mode 100644 index 00000000..88dab5f6 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_stream.h @@ -0,0 +1,59 @@ +#ifndef crypto_stream_H +#define crypto_stream_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include + +#include "crypto_stream_xsalsa20.h" +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_KEYBYTES crypto_stream_xsalsa20_KEYBYTES +SODIUM_EXPORT +size_t crypto_stream_keybytes(void); + +#define crypto_stream_NONCEBYTES crypto_stream_xsalsa20_NONCEBYTES +SODIUM_EXPORT +size_t crypto_stream_noncebytes(void); + +#define crypto_stream_MESSAGEBYTES_MAX crypto_stream_xsalsa20_MESSAGEBYTES_MAX +SODIUM_EXPORT +size_t crypto_stream_messagebytes_max(void); + +#define crypto_stream_PRIMITIVE "xsalsa20" +SODIUM_EXPORT +const char *crypto_stream_primitive(void); + +SODIUM_EXPORT +int crypto_stream(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_keygen(unsigned char k[crypto_stream_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_stream_chacha20.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_stream_chacha20.h new file mode 100644 index 00000000..40889755 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_stream_chacha20.h @@ -0,0 +1,106 @@ +#ifndef crypto_stream_chacha20_H +#define crypto_stream_chacha20_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_chacha20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_chacha20_keybytes(void); + +#define crypto_stream_chacha20_NONCEBYTES 8U +SODIUM_EXPORT +size_t crypto_stream_chacha20_noncebytes(void); + +#define crypto_stream_chacha20_MESSAGEBYTES_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_stream_chacha20_messagebytes_max(void); + +/* ChaCha20 with a 64-bit nonce and a 64-bit counter, as originally designed */ + +SODIUM_EXPORT +int crypto_stream_chacha20(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_chacha20_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_chacha20_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, uint64_t ic, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_chacha20_keygen(unsigned char k[crypto_stream_chacha20_KEYBYTES]) + __attribute__ ((nonnull)); + +/* ChaCha20 with a 96-bit nonce and a 32-bit counter (IETF) */ + +#define crypto_stream_chacha20_ietf_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_chacha20_ietf_keybytes(void); + +#define crypto_stream_chacha20_ietf_NONCEBYTES 12U +SODIUM_EXPORT +size_t crypto_stream_chacha20_ietf_noncebytes(void); + +#define crypto_stream_chacha20_ietf_MESSAGEBYTES_MAX \ + SODIUM_MIN(SODIUM_SIZE_MAX, 64ULL * (1ULL << 32)) +SODIUM_EXPORT +size_t crypto_stream_chacha20_ietf_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_stream_chacha20_ietf(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_chacha20_ietf_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_chacha20_ietf_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, uint32_t ic, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_chacha20_ietf_keygen(unsigned char k[crypto_stream_chacha20_ietf_KEYBYTES]) + __attribute__ ((nonnull)); + +/* Aliases */ + +#define crypto_stream_chacha20_IETF_KEYBYTES crypto_stream_chacha20_ietf_KEYBYTES +#define crypto_stream_chacha20_IETF_NONCEBYTES crypto_stream_chacha20_ietf_NONCEBYTES +#define crypto_stream_chacha20_IETF_MESSAGEBYTES_MAX crypto_stream_chacha20_ietf_MESSAGEBYTES_MAX + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_stream_salsa20.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_stream_salsa20.h new file mode 100644 index 00000000..45b3b3e3 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_stream_salsa20.h @@ -0,0 +1,61 @@ +#ifndef crypto_stream_salsa20_H +#define crypto_stream_salsa20_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_salsa20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_salsa20_keybytes(void); + +#define crypto_stream_salsa20_NONCEBYTES 8U +SODIUM_EXPORT +size_t crypto_stream_salsa20_noncebytes(void); + +#define crypto_stream_salsa20_MESSAGEBYTES_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_stream_salsa20_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_stream_salsa20(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_salsa20_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_salsa20_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, uint64_t ic, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_salsa20_keygen(unsigned char k[crypto_stream_salsa20_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_stream_salsa2012.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_stream_salsa2012.h new file mode 100644 index 00000000..6c5d303c --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_stream_salsa2012.h @@ -0,0 +1,53 @@ +#ifndef crypto_stream_salsa2012_H +#define crypto_stream_salsa2012_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_salsa2012_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_salsa2012_keybytes(void); + +#define crypto_stream_salsa2012_NONCEBYTES 8U +SODIUM_EXPORT +size_t crypto_stream_salsa2012_noncebytes(void); + +#define crypto_stream_salsa2012_MESSAGEBYTES_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_stream_salsa2012_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_stream_salsa2012(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_salsa2012_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_salsa2012_keygen(unsigned char k[crypto_stream_salsa2012_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_stream_salsa208.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_stream_salsa208.h new file mode 100644 index 00000000..d574f304 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_stream_salsa208.h @@ -0,0 +1,56 @@ +#ifndef crypto_stream_salsa208_H +#define crypto_stream_salsa208_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_salsa208_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_salsa208_keybytes(void) + __attribute__ ((deprecated)); + +#define crypto_stream_salsa208_NONCEBYTES 8U +SODIUM_EXPORT +size_t crypto_stream_salsa208_noncebytes(void) + __attribute__ ((deprecated)); + +#define crypto_stream_salsa208_MESSAGEBYTES_MAX SODIUM_SIZE_MAX + SODIUM_EXPORT +size_t crypto_stream_salsa208_messagebytes_max(void) + __attribute__ ((deprecated)); + +SODIUM_EXPORT +int crypto_stream_salsa208(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((deprecated)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_salsa208_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((deprecated)) __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_salsa208_keygen(unsigned char k[crypto_stream_salsa208_KEYBYTES]) + __attribute__ ((deprecated)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_stream_xchacha20.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_stream_xchacha20.h new file mode 100644 index 00000000..c4002db0 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_stream_xchacha20.h @@ -0,0 +1,61 @@ +#ifndef crypto_stream_xchacha20_H +#define crypto_stream_xchacha20_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_xchacha20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_xchacha20_keybytes(void); + +#define crypto_stream_xchacha20_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_stream_xchacha20_noncebytes(void); + +#define crypto_stream_xchacha20_MESSAGEBYTES_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_stream_xchacha20_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_stream_xchacha20(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_xchacha20_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_xchacha20_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, uint64_t ic, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_xchacha20_keygen(unsigned char k[crypto_stream_xchacha20_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_stream_xsalsa20.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_stream_xsalsa20.h new file mode 100644 index 00000000..20034e34 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_stream_xsalsa20.h @@ -0,0 +1,61 @@ +#ifndef crypto_stream_xsalsa20_H +#define crypto_stream_xsalsa20_H + +/* + * WARNING: This is just a stream cipher. It is NOT authenticated encryption. + * While it provides some protection against eavesdropping, it does NOT + * provide any security against active attacks. + * Unless you know what you're doing, what you are looking for is probably + * the crypto_box functions. + */ + +#include +#include +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +#define crypto_stream_xsalsa20_KEYBYTES 32U +SODIUM_EXPORT +size_t crypto_stream_xsalsa20_keybytes(void); + +#define crypto_stream_xsalsa20_NONCEBYTES 24U +SODIUM_EXPORT +size_t crypto_stream_xsalsa20_noncebytes(void); + +#define crypto_stream_xsalsa20_MESSAGEBYTES_MAX SODIUM_SIZE_MAX +SODIUM_EXPORT +size_t crypto_stream_xsalsa20_messagebytes_max(void); + +SODIUM_EXPORT +int crypto_stream_xsalsa20(unsigned char *c, unsigned long long clen, + const unsigned char *n, const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_xsalsa20_xor(unsigned char *c, const unsigned char *m, + unsigned long long mlen, const unsigned char *n, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int crypto_stream_xsalsa20_xor_ic(unsigned char *c, const unsigned char *m, + unsigned long long mlen, + const unsigned char *n, uint64_t ic, + const unsigned char *k) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void crypto_stream_xsalsa20_keygen(unsigned char k[crypto_stream_xsalsa20_KEYBYTES]) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_verify_16.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_verify_16.h new file mode 100644 index 00000000..7b9c8077 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_verify_16.h @@ -0,0 +1,23 @@ +#ifndef crypto_verify_16_H +#define crypto_verify_16_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_verify_16_BYTES 16U +SODIUM_EXPORT +size_t crypto_verify_16_bytes(void); + +SODIUM_EXPORT +int crypto_verify_16(const unsigned char *x, const unsigned char *y) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_verify_32.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_verify_32.h new file mode 100644 index 00000000..9b0f4529 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_verify_32.h @@ -0,0 +1,23 @@ +#ifndef crypto_verify_32_H +#define crypto_verify_32_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_verify_32_BYTES 32U +SODIUM_EXPORT +size_t crypto_verify_32_bytes(void); + +SODIUM_EXPORT +int crypto_verify_32(const unsigned char *x, const unsigned char *y) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_verify_64.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_verify_64.h new file mode 100644 index 00000000..c83b7302 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/crypto_verify_64.h @@ -0,0 +1,23 @@ +#ifndef crypto_verify_64_H +#define crypto_verify_64_H + +#include +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define crypto_verify_64_BYTES 64U +SODIUM_EXPORT +size_t crypto_verify_64_bytes(void); + +SODIUM_EXPORT +int crypto_verify_64(const unsigned char *x, const unsigned char *y) + __attribute__ ((warn_unused_result)) __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/export.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/export.h new file mode 100644 index 00000000..a0074fc9 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/export.h @@ -0,0 +1,57 @@ + +#ifndef sodium_export_H +#define sodium_export_H + +#include +#include +#include + +#if !defined(__clang__) && !defined(__GNUC__) +# ifdef __attribute__ +# undef __attribute__ +# endif +# define __attribute__(a) +#endif + +#ifdef SODIUM_STATIC +# define SODIUM_EXPORT +# define SODIUM_EXPORT_WEAK +#else +# if defined(_MSC_VER) +# ifdef SODIUM_DLL_EXPORT +# define SODIUM_EXPORT __declspec(dllexport) +# else +# define SODIUM_EXPORT __declspec(dllimport) +# endif +# else +# if defined(__SUNPRO_C) +# ifndef __GNU_C__ +# define SODIUM_EXPORT __attribute__ (visibility(__global)) +# else +# define SODIUM_EXPORT __attribute__ __global +# endif +# elif defined(_MSG_VER) +# define SODIUM_EXPORT extern __declspec(dllexport) +# else +# define SODIUM_EXPORT __attribute__ ((visibility ("default"))) +# endif +# endif +# if defined(__ELF__) && !defined(SODIUM_DISABLE_WEAK_FUNCTIONS) +# define SODIUM_EXPORT_WEAK SODIUM_EXPORT __attribute__((weak)) +# else +# define SODIUM_EXPORT_WEAK SODIUM_EXPORT +# endif +#endif + +#ifndef CRYPTO_ALIGN +# if defined(__INTEL_COMPILER) || defined(_MSC_VER) +# define CRYPTO_ALIGN(x) __declspec(align(x)) +# else +# define CRYPTO_ALIGN(x) __attribute__ ((aligned(x))) +# endif +#endif + +#define SODIUM_MIN(A, B) ((A) < (B) ? (A) : (B)) +#define SODIUM_SIZE_MAX SODIUM_MIN(UINT64_MAX, SIZE_MAX) + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/randombytes.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/randombytes.h new file mode 100644 index 00000000..a03cc657 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/randombytes.h @@ -0,0 +1,72 @@ + +#ifndef randombytes_H +#define randombytes_H + +#include +#include + +#include + +#include "export.h" + +#ifdef __cplusplus +# ifdef __GNUC__ +# pragma GCC diagnostic ignored "-Wlong-long" +# endif +extern "C" { +#endif + +typedef struct randombytes_implementation { + const char *(*implementation_name)(void); /* required */ + uint32_t (*random)(void); /* required */ + void (*stir)(void); /* optional */ + uint32_t (*uniform)(const uint32_t upper_bound); /* optional, a default implementation will be used if NULL */ + void (*buf)(void * const buf, const size_t size); /* required */ + int (*close)(void); /* optional */ +} randombytes_implementation; + +#define randombytes_BYTES_MAX SODIUM_MIN(SODIUM_SIZE_MAX, 0xffffffffUL) + +#define randombytes_SEEDBYTES 32U +SODIUM_EXPORT +size_t randombytes_seedbytes(void); + +SODIUM_EXPORT +void randombytes_buf(void * const buf, const size_t size) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +void randombytes_buf_deterministic(void * const buf, const size_t size, + const unsigned char seed[randombytes_SEEDBYTES]) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +uint32_t randombytes_random(void); + +SODIUM_EXPORT +uint32_t randombytes_uniform(const uint32_t upper_bound); + +SODIUM_EXPORT +void randombytes_stir(void); + +SODIUM_EXPORT +int randombytes_close(void); + +SODIUM_EXPORT +int randombytes_set_implementation(randombytes_implementation *impl) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +const char *randombytes_implementation_name(void); + +/* -- NaCl compatibility interface -- */ + +SODIUM_EXPORT +void randombytes(unsigned char * const buf, const unsigned long long buf_len) + __attribute__ ((nonnull)); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/randombytes_internal_random.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/randombytes_internal_random.h new file mode 100644 index 00000000..2b2b7d6e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/randombytes_internal_random.h @@ -0,0 +1,22 @@ + +#ifndef randombytes_internal_random_H +#define randombytes_internal_random_H + +#include "export.h" +#include "randombytes.h" + +#ifdef __cplusplus +extern "C" { +#endif + +SODIUM_EXPORT +extern struct randombytes_implementation randombytes_internal_implementation; + +/* Backwards compatibility with libsodium < 1.0.18 */ +#define randombytes_salsa20_implementation randombytes_internal_implementation + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/randombytes_sysrandom.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/randombytes_sysrandom.h new file mode 100644 index 00000000..9e27b674 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/randombytes_sysrandom.h @@ -0,0 +1,19 @@ + +#ifndef randombytes_sysrandom_H +#define randombytes_sysrandom_H + +#include "export.h" +#include "randombytes.h" + +#ifdef __cplusplus +extern "C" { +#endif + +SODIUM_EXPORT +extern struct randombytes_implementation randombytes_sysrandom_implementation; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/runtime.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/runtime.h new file mode 100644 index 00000000..7f15d58e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/runtime.h @@ -0,0 +1,52 @@ + +#ifndef sodium_runtime_H +#define sodium_runtime_H + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_neon(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_sse2(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_sse3(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_ssse3(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_sse41(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_avx(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_avx2(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_avx512f(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_pclmul(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_aesni(void); + +SODIUM_EXPORT_WEAK +int sodium_runtime_has_rdrand(void); + +/* ------------------------------------------------------------------------- */ + +int _sodium_runtime_get_cpu_features(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/utils.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/utils.h new file mode 100644 index 00000000..ac801512 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/utils.h @@ -0,0 +1,179 @@ + +#ifndef sodium_utils_H +#define sodium_utils_H + +#include + +#include "export.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef SODIUM_C99 +# if defined(__cplusplus) || !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L +# define SODIUM_C99(X) +# else +# define SODIUM_C99(X) X +# endif +#endif + +SODIUM_EXPORT +void sodium_memzero(void * const pnt, const size_t len); + +SODIUM_EXPORT +void sodium_stackzero(const size_t len); + +/* + * WARNING: sodium_memcmp() must be used to verify if two secret keys + * are equal, in constant time. + * It returns 0 if the keys are equal, and -1 if they differ. + * This function is not designed for lexicographical comparisons. + */ +SODIUM_EXPORT +int sodium_memcmp(const void * const b1_, const void * const b2_, size_t len) + __attribute__ ((warn_unused_result)); + +/* + * sodium_compare() returns -1 if b1_ < b2_, 1 if b1_ > b2_ and 0 if b1_ == b2_ + * It is suitable for lexicographical comparisons, or to compare nonces + * and counters stored in little-endian format. + * However, it is slower than sodium_memcmp(). + */ +SODIUM_EXPORT +int sodium_compare(const unsigned char *b1_, const unsigned char *b2_, + size_t len) __attribute__ ((warn_unused_result)); + +SODIUM_EXPORT +int sodium_is_zero(const unsigned char *n, const size_t nlen); + +SODIUM_EXPORT +void sodium_increment(unsigned char *n, const size_t nlen); + +SODIUM_EXPORT +void sodium_add(unsigned char *a, const unsigned char *b, const size_t len); + +SODIUM_EXPORT +void sodium_sub(unsigned char *a, const unsigned char *b, const size_t len); + +SODIUM_EXPORT +char *sodium_bin2hex(char * const hex, const size_t hex_maxlen, + const unsigned char * const bin, const size_t bin_len) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int sodium_hex2bin(unsigned char * const bin, const size_t bin_maxlen, + const char * const hex, const size_t hex_len, + const char * const ignore, size_t * const bin_len, + const char ** const hex_end) + __attribute__ ((nonnull(1))); + +#define sodium_base64_VARIANT_ORIGINAL 1 +#define sodium_base64_VARIANT_ORIGINAL_NO_PADDING 3 +#define sodium_base64_VARIANT_URLSAFE 5 +#define sodium_base64_VARIANT_URLSAFE_NO_PADDING 7 + +/* + * Computes the required length to encode BIN_LEN bytes as a base64 string + * using the given variant. The computed length includes a trailing \0. + */ +#define sodium_base64_ENCODED_LEN(BIN_LEN, VARIANT) \ + (((BIN_LEN) / 3U) * 4U + \ + ((((BIN_LEN) - ((BIN_LEN) / 3U) * 3U) | (((BIN_LEN) - ((BIN_LEN) / 3U) * 3U) >> 1)) & 1U) * \ + (4U - (~((((VARIANT) & 2U) >> 1) - 1U) & (3U - ((BIN_LEN) - ((BIN_LEN) / 3U) * 3U)))) + 1U) + +SODIUM_EXPORT +size_t sodium_base64_encoded_len(const size_t bin_len, const int variant); + +SODIUM_EXPORT +char *sodium_bin2base64(char * const b64, const size_t b64_maxlen, + const unsigned char * const bin, const size_t bin_len, + const int variant) __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int sodium_base642bin(unsigned char * const bin, const size_t bin_maxlen, + const char * const b64, const size_t b64_len, + const char * const ignore, size_t * const bin_len, + const char ** const b64_end, const int variant) + __attribute__ ((nonnull(1))); + +SODIUM_EXPORT +int sodium_mlock(void * const addr, const size_t len) + __attribute__ ((nonnull)); + +SODIUM_EXPORT +int sodium_munlock(void * const addr, const size_t len) + __attribute__ ((nonnull)); + +/* WARNING: sodium_malloc() and sodium_allocarray() are not general-purpose + * allocation functions. + * + * They return a pointer to a region filled with 0xd0 bytes, immediately + * followed by a guard page. + * As a result, accessing a single byte after the requested allocation size + * will intentionally trigger a segmentation fault. + * + * A canary and an additional guard page placed before the beginning of the + * region may also kill the process if a buffer underflow is detected. + * + * The memory layout is: + * [unprotected region size (read only)][guard page (no access)][unprotected pages (read/write)][guard page (no access)] + * With the layout of the unprotected pages being: + * [optional padding][16-bytes canary][user region] + * + * However: + * - These functions are significantly slower than standard functions + * - Each allocation requires 3 or 4 additional pages + * - The returned address will not be aligned if the allocation size is not + * a multiple of the required alignment. For this reason, these functions + * are designed to store data, such as secret keys and messages. + * + * sodium_malloc() can be used to allocate any libsodium data structure. + * + * The crypto_generichash_state structure is packed and its length is + * either 357 or 361 bytes. For this reason, when using sodium_malloc() to + * allocate a crypto_generichash_state structure, padding must be added in + * order to ensure proper alignment. crypto_generichash_statebytes() + * returns the rounded up structure size, and should be prefered to sizeof(): + * state = sodium_malloc(crypto_generichash_statebytes()); + */ + +SODIUM_EXPORT +void *sodium_malloc(const size_t size) + __attribute__ ((malloc)); + +SODIUM_EXPORT +void *sodium_allocarray(size_t count, size_t size) + __attribute__ ((malloc)); + +SODIUM_EXPORT +void sodium_free(void *ptr); + +SODIUM_EXPORT +int sodium_mprotect_noaccess(void *ptr) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int sodium_mprotect_readonly(void *ptr) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int sodium_mprotect_readwrite(void *ptr) __attribute__ ((nonnull)); + +SODIUM_EXPORT +int sodium_pad(size_t *padded_buflen_p, unsigned char *buf, + size_t unpadded_buflen, size_t blocksize, size_t max_buflen) + __attribute__ ((nonnull(2))); + +SODIUM_EXPORT +int sodium_unpad(size_t *unpadded_buflen_p, const unsigned char *buf, + size_t padded_buflen, size_t blocksize) + __attribute__ ((nonnull(2))); + +/* -------- */ + +int _sodium_alloc_init(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/version.h b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/version.h new file mode 100644 index 00000000..201a290e --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/include/sodium/version.h @@ -0,0 +1,33 @@ + +#ifndef sodium_version_H +#define sodium_version_H + +#include "export.h" + +#define SODIUM_VERSION_STRING "1.0.18" + +#define SODIUM_LIBRARY_VERSION_MAJOR 10 +#define SODIUM_LIBRARY_VERSION_MINOR 3 + + +#ifdef __cplusplus +extern "C" { +#endif + +SODIUM_EXPORT +const char *sodium_version_string(void); + +SODIUM_EXPORT +int sodium_library_version_major(void); + +SODIUM_EXPORT +int sodium_library_version_minor(void); + +SODIUM_EXPORT +int sodium_library_minimal(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/lib/libsodium.a b/example/android/third_party/libsodium/libsodium-android-westmere/lib/libsodium.a new file mode 100644 index 00000000..b46cb5dc Binary files /dev/null and b/example/android/third_party/libsodium/libsodium-android-westmere/lib/libsodium.a differ diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/lib/libsodium.la b/example/android/third_party/libsodium/libsodium-android-westmere/lib/libsodium.la new file mode 100644 index 00000000..4bd720f4 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/lib/libsodium.la @@ -0,0 +1,41 @@ +# libsodium.la - a libtool library file +# Generated by libtool (GNU libtool) 2.4.6 +# +# Please DO NOT delete this file! +# It is necessary for linking the library. + +# The name that we can dlopen(3). +dlname='libsodium.so' + +# Names of this library. +library_names='libsodium.so' + +# The name of the static archive. +old_library='libsodium.a' + +# Linker flags that cannot go in dependency_libs. +inherited_linker_flags=' -pthread' + +# Libraries that this one depends upon. +dependency_libs='' + +# Names of additional weak libraries provided by this library +weak_library_names='' + +# Version information for libsodium. +current=0 +age=0 +revision=0 + +# Is this an already installed library? +installed=yes + +# Should we warn about portability when linking against -modules? +shouldnotlink=no + +# Files to dlopen/dlpreopen +dlopen='' +dlpreopen='' + +# Directory that this library needs to be installed in: +libdir='/home/alex/magnet/example/android/third_party/libsodium/libsodium-1.0.18/libsodium-android-westmere/lib' diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/lib/libsodium.so b/example/android/third_party/libsodium/libsodium-android-westmere/lib/libsodium.so new file mode 100644 index 00000000..24be50ca Binary files /dev/null and b/example/android/third_party/libsodium/libsodium-android-westmere/lib/libsodium.so differ diff --git a/example/android/third_party/libsodium/libsodium-android-westmere/lib/pkgconfig/libsodium.pc b/example/android/third_party/libsodium/libsodium-android-westmere/lib/pkgconfig/libsodium.pc new file mode 100644 index 00000000..406498f4 --- /dev/null +++ b/example/android/third_party/libsodium/libsodium-android-westmere/lib/pkgconfig/libsodium.pc @@ -0,0 +1,12 @@ +prefix=/home/alex/magnet/example/android/third_party/libsodium/libsodium-1.0.18/libsodium-android-westmere +exec_prefix=${prefix} +libdir=${exec_prefix}/lib +includedir=${prefix}/include + +Name: libsodium +Version: 1.0.18 +Description: A modern and easy-to-use crypto library + +Libs: -L${libdir} -lsodium +Libs.private: -pthread +Cflags: -I${includedir} diff --git a/example/android/third_party/lz4/armv7/liblz4.a b/example/android/third_party/lz4/armv7/liblz4.a new file mode 100644 index 00000000..311fbca4 Binary files /dev/null and b/example/android/third_party/lz4/armv7/liblz4.a differ diff --git a/example/android/third_party/lz4/armv7/liblz4.so b/example/android/third_party/lz4/armv7/liblz4.so new file mode 100644 index 00000000..f98cffe8 Binary files /dev/null and b/example/android/third_party/lz4/armv7/liblz4.so differ diff --git a/example/android/third_party/lz4/armv8/liblz4.a b/example/android/third_party/lz4/armv8/liblz4.a new file mode 100644 index 00000000..6db4a5d4 Binary files /dev/null and b/example/android/third_party/lz4/armv8/liblz4.a differ diff --git a/example/android/third_party/lz4/armv8/liblz4.so b/example/android/third_party/lz4/armv8/liblz4.so new file mode 100644 index 00000000..f22c7945 Binary files /dev/null and b/example/android/third_party/lz4/armv8/liblz4.so differ diff --git a/example/android/third_party/lz4/i686/liblz4.a b/example/android/third_party/lz4/i686/liblz4.a new file mode 100644 index 00000000..2d86f7ef Binary files /dev/null and b/example/android/third_party/lz4/i686/liblz4.a differ diff --git a/example/android/third_party/lz4/i686/liblz4.so b/example/android/third_party/lz4/i686/liblz4.so new file mode 100644 index 00000000..bed09629 Binary files /dev/null and b/example/android/third_party/lz4/i686/liblz4.so differ diff --git a/example/android/third_party/lz4/include/lz4.c b/example/android/third_party/lz4/include/lz4.c new file mode 100644 index 00000000..b7dd32a7 --- /dev/null +++ b/example/android/third_party/lz4/include/lz4.c @@ -0,0 +1,2826 @@ +/* + LZ4 - Fast LZ compression algorithm + Copyright (C) 2011-2023, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 +*/ + +/*-************************************ +* Tuning parameters +**************************************/ +/* + * LZ4_HEAPMODE : + * Select how stateless compression functions like `LZ4_compress_default()` + * allocate memory for their hash table, + * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). + */ +#ifndef LZ4_HEAPMODE +# define LZ4_HEAPMODE 0 +#endif + +/* + * LZ4_ACCELERATION_DEFAULT : + * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 + */ +#define LZ4_ACCELERATION_DEFAULT 1 +/* + * LZ4_ACCELERATION_MAX : + * Any "acceleration" value higher than this threshold + * get treated as LZ4_ACCELERATION_MAX instead (fix #876) + */ +#define LZ4_ACCELERATION_MAX 65537 + + +/*-************************************ +* CPU Feature Detection +**************************************/ +/* LZ4_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets which assembly generation depends on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */ +# if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define LZ4_FORCE_MEMORY_ACCESS 2 +# elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__) || defined(_MSC_VER) +# define LZ4_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +/* + * LZ4_FORCE_SW_BITCOUNT + * Define this parameter if your target system or compiler does not support hardware bit count + */ +#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for WinCE doesn't support Hardware bit count */ +# undef LZ4_FORCE_SW_BITCOUNT /* avoid double def */ +# define LZ4_FORCE_SW_BITCOUNT +#endif + + + +/*-************************************ +* Dependency +**************************************/ +/* + * LZ4_SRC_INCLUDED: + * Amalgamation flag, whether lz4.c is included + */ +#ifndef LZ4_SRC_INCLUDED +# define LZ4_SRC_INCLUDED 1 +#endif + +#ifndef LZ4_DISABLE_DEPRECATE_WARNINGS +# define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */ +#endif + +#ifndef LZ4_STATIC_LINKING_ONLY +# define LZ4_STATIC_LINKING_ONLY +#endif +#include "lz4.h" +/* see also "memory routines" below */ + + +/*-************************************ +* Compiler Options +**************************************/ +#if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Visual Studio 2005+ */ +# include /* only present in VS2005+ */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 6237) /* disable: C6237: conditional expression is always 0 */ +#endif /* _MSC_VER */ + +#ifndef LZ4_FORCE_INLINE +# if defined (_MSC_VER) && !defined (__clang__) /* MSVC */ +# define LZ4_FORCE_INLINE static __forceinline +# else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# if defined (__GNUC__) || defined (__clang__) +# define LZ4_FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define LZ4_FORCE_INLINE static inline +# endif +# else +# define LZ4_FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +# endif /* _MSC_VER */ +#endif /* LZ4_FORCE_INLINE */ + +/* LZ4_FORCE_O2 and LZ4_FORCE_INLINE + * gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy8, + * together with a simple 8-byte copy loop as a fall-back path. + * However, this optimization hurts the decompression speed by >30%, + * because the execution does not go to the optimized loop + * for typical compressible data, and all of the preamble checks + * before going to the fall-back path become useless overhead. + * This optimization happens only with the -O3 flag, and -O2 generates + * a simple 8-byte copy loop. + * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy8 + * functions are annotated with __attribute__((optimize("O2"))), + * and also LZ4_wildCopy8 is forcibly inlined, so that the O2 attribute + * of LZ4_wildCopy8 does not affect the compression speed. + */ +#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) && !defined(__clang__) +# define LZ4_FORCE_O2 __attribute__((optimize("O2"))) +# undef LZ4_FORCE_INLINE +# define LZ4_FORCE_INLINE static __inline __attribute__((optimize("O2"),always_inline)) +#else +# define LZ4_FORCE_O2 +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif + +#ifndef likely +#define likely(expr) expect((expr) != 0, 1) +#endif +#ifndef unlikely +#define unlikely(expr) expect((expr) != 0, 0) +#endif + +/* Should the alignment test prove unreliable, for some reason, + * it can be disabled by setting LZ4_ALIGN_TEST to 0 */ +#ifndef LZ4_ALIGN_TEST /* can be externally provided */ +# define LZ4_ALIGN_TEST 1 +#endif + + +/*-************************************ +* Memory routines +**************************************/ + +/*! LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION : + * Disable relatively high-level LZ4/HC functions that use dynamic memory + * allocation functions (malloc(), calloc(), free()). + * + * Note that this is a compile-time switch. And since it disables + * public/stable LZ4 v1 API functions, we don't recommend using this + * symbol to generate a library for distribution. + * + * The following public functions are removed when this symbol is defined. + * - lz4 : LZ4_createStream, LZ4_freeStream, + * LZ4_createStreamDecode, LZ4_freeStreamDecode, LZ4_create (deprecated) + * - lz4hc : LZ4_createStreamHC, LZ4_freeStreamHC, + * LZ4_createHC (deprecated), LZ4_freeHC (deprecated) + * - lz4frame, lz4file : All LZ4F_* functions + */ +#if defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +# define ALLOC(s) lz4_error_memory_allocation_is_disabled +# define ALLOC_AND_ZERO(s) lz4_error_memory_allocation_is_disabled +# define FREEMEM(p) lz4_error_memory_allocation_is_disabled +#elif defined(LZ4_USER_MEMORY_FUNCTIONS) +/* memory management functions can be customized by user project. + * Below functions must exist somewhere in the Project + * and be available at link time */ +void* LZ4_malloc(size_t s); +void* LZ4_calloc(size_t n, size_t s); +void LZ4_free(void* p); +# define ALLOC(s) LZ4_malloc(s) +# define ALLOC_AND_ZERO(s) LZ4_calloc(1,s) +# define FREEMEM(p) LZ4_free(p) +#else +# include /* malloc, calloc, free */ +# define ALLOC(s) malloc(s) +# define ALLOC_AND_ZERO(s) calloc(1,s) +# define FREEMEM(p) free(p) +#endif + +#if ! LZ4_FREESTANDING +# include /* memset, memcpy */ +#endif +#if !defined(LZ4_memset) +# define LZ4_memset(p,v,s) memset((p),(v),(s)) +#endif +#define MEM_INIT(p,v,s) LZ4_memset((p),(v),(s)) + + +/*-************************************ +* Common Constants +**************************************/ +#define MINMATCH 4 + +#define WILDCOPYLENGTH 8 +#define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */ +#define FASTLOOP_SAFE_DISTANCE 64 +static const int LZ4_minLength = (MFLIMIT+1); + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define LZ4_DISTANCE_ABSOLUTE_MAX 65535 +#if (LZ4_DISTANCE_MAX > LZ4_DISTANCE_ABSOLUTE_MAX) /* max supported by LZ4 format */ +# error "LZ4_DISTANCE_MAX is too big : must be <= 65535" +#endif + +#define ML_BITS 4 +#define ML_MASK ((1U<=1) +# include +#else +# ifndef assert +# define assert(condition) ((void)0) +# endif +#endif + +#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */ + +#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) +# include + static int g_debuglog_enable = 1; +# define DEBUGLOG(l, ...) { \ + if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ + fprintf(stderr, __FILE__ " %i: ", __LINE__); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, " \n"); \ + } } +#else +# define DEBUGLOG(l, ...) {} /* disabled */ +#endif + +static int LZ4_isAligned(const void* ptr, size_t alignment) +{ + return ((size_t)ptr & (alignment -1)) == 0; +} + + +/*-************************************ +* Types +**************************************/ +#include +#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef uintptr_t uptrval; +#else +# if UINT_MAX != 4294967295UL +# error "LZ4 code (when not C++ or C99) assumes that sizeof(int) == 4" +# endif + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef size_t uptrval; /* generally true, except OpenVMS-64 */ +#endif + +#if defined(__x86_64__) + typedef U64 reg_t; /* 64-bits in x32 mode */ +#else + typedef size_t reg_t; /* 32-bits in x32 mode */ +#endif + +typedef enum { + notLimited = 0, + limitedOutput = 1, + fillOutput = 2 +} limitedOutput_directive; + + +/*-************************************ +* Reading and writing into memory +**************************************/ + +/** + * LZ4 relies on memcpy with a constant size being inlined. In freestanding + * environments, the compiler can't assume the implementation of memcpy() is + * standard compliant, so it can't apply its specialized memcpy() inlining + * logic. When possible, use __builtin_memcpy() to tell the compiler to analyze + * memcpy() as if it were standard compliant, so it can inline it in freestanding + * environments. This is needed when decompressing the Linux Kernel, for example. + */ +#if !defined(LZ4_memcpy) +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4_memcpy(dst, src, size) __builtin_memcpy(dst, src, size) +# else +# define LZ4_memcpy(dst, src, size) memcpy(dst, src, size) +# endif +#endif + +#if !defined(LZ4_memmove) +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4_memmove __builtin_memmove +# else +# define LZ4_memmove memmove +# endif +#endif + +static unsigned LZ4_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + +#if defined(__GNUC__) || defined(__INTEL_COMPILER) +#define LZ4_PACK( __Declaration__ ) __Declaration__ __attribute__((__packed__)) +#elif defined(_MSC_VER) +#define LZ4_PACK( __Declaration__ ) __pragma( pack(push, 1) ) __Declaration__ __pragma( pack(pop)) +#endif + +#if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2) +/* lie to the compiler about data alignment; use with caution */ + +static U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; } +static U32 LZ4_read32(const void* memPtr) { return *(const U32*) memPtr; } +static reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*) memPtr; } + +static void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } + +#elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==1) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +LZ4_PACK(typedef struct { U16 u16; }) LZ4_unalign16; +LZ4_PACK(typedef struct { U32 u32; }) LZ4_unalign32; +LZ4_PACK(typedef struct { reg_t uArch; }) LZ4_unalignST; + +static U16 LZ4_read16(const void* ptr) { return ((const LZ4_unalign16*)ptr)->u16; } +static U32 LZ4_read32(const void* ptr) { return ((const LZ4_unalign32*)ptr)->u32; } +static reg_t LZ4_read_ARCH(const void* ptr) { return ((const LZ4_unalignST*)ptr)->uArch; } + +static void LZ4_write16(void* memPtr, U16 value) { ((LZ4_unalign16*)memPtr)->u16 = value; } +static void LZ4_write32(void* memPtr, U32 value) { ((LZ4_unalign32*)memPtr)->u32 = value; } + +#else /* safe and portable access using memcpy() */ + +static U16 LZ4_read16(const void* memPtr) +{ + U16 val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; +} + +static U32 LZ4_read32(const void* memPtr) +{ + U32 val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; +} + +static reg_t LZ4_read_ARCH(const void* memPtr) +{ + reg_t val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; +} + +static void LZ4_write16(void* memPtr, U16 value) +{ + LZ4_memcpy(memPtr, &value, sizeof(value)); +} + +static void LZ4_write32(void* memPtr, U32 value) +{ + LZ4_memcpy(memPtr, &value, sizeof(value)); +} + +#endif /* LZ4_FORCE_MEMORY_ACCESS */ + + +static U16 LZ4_readLE16(const void* memPtr) +{ + if (LZ4_isLittleEndian()) { + return LZ4_read16(memPtr); + } else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)((U16)p[0] + (p[1]<<8)); + } +} + +#ifdef LZ4_STATIC_LINKING_ONLY_ENDIANNESS_INDEPENDENT_OUTPUT +static U32 LZ4_readLE32(const void* memPtr) +{ + if (LZ4_isLittleEndian()) { + return LZ4_read32(memPtr); + } else { + const BYTE* p = (const BYTE*)memPtr; + return (U32)p[0] + (p[1]<<8) + (p[2]<<16) + (p[3]<<24); + } +} +#endif + +static void LZ4_writeLE16(void* memPtr, U16 value) +{ + if (LZ4_isLittleEndian()) { + LZ4_write16(memPtr, value); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE) value; + p[1] = (BYTE)(value>>8); + } +} + +/* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ +LZ4_FORCE_INLINE +void LZ4_wildCopy8(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; + + do { LZ4_memcpy(d,s,8); d+=8; s+=8; } while (d= 16. */ +LZ4_FORCE_INLINE void +LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; + + do { LZ4_memcpy(d,s,16); LZ4_memcpy(d+16,s+16,16); d+=32; s+=32; } while (d= dstPtr + MINMATCH + * - there is at least 8 bytes available to write after dstEnd */ +LZ4_FORCE_INLINE void +LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset) +{ + BYTE v[8]; + + assert(dstEnd >= dstPtr + MINMATCH); + + switch(offset) { + case 1: + MEM_INIT(v, *srcPtr, 8); + break; + case 2: + LZ4_memcpy(v, srcPtr, 2); + LZ4_memcpy(&v[2], srcPtr, 2); +#if defined(_MSC_VER) && (_MSC_VER <= 1937) /* MSVC 2022 ver 17.7 or earlier */ +# pragma warning(push) +# pragma warning(disable : 6385) /* warning C6385: Reading invalid data from 'v'. */ +#endif + LZ4_memcpy(&v[4], v, 4); +#if defined(_MSC_VER) && (_MSC_VER <= 1937) /* MSVC 2022 ver 17.7 or earlier */ +# pragma warning(pop) +#endif + break; + case 4: + LZ4_memcpy(v, srcPtr, 4); + LZ4_memcpy(&v[4], srcPtr, 4); + break; + default: + LZ4_memcpy_using_offset_base(dstPtr, srcPtr, dstEnd, offset); + return; + } + + LZ4_memcpy(dstPtr, v, 8); + dstPtr += 8; + while (dstPtr < dstEnd) { + LZ4_memcpy(dstPtr, v, 8); + dstPtr += 8; + } +} +#endif + + +/*-************************************ +* Common functions +**************************************/ +static unsigned LZ4_NbCommonBytes (reg_t val) +{ + assert(val != 0); + if (LZ4_isLittleEndian()) { + if (sizeof(val) == 8) { +# if defined(_MSC_VER) && (_MSC_VER >= 1800) && (defined(_M_AMD64) && !defined(_M_ARM64EC)) && !defined(LZ4_FORCE_SW_BITCOUNT) +/*-************************************************************************************************* +* ARM64EC is a Microsoft-designed ARM64 ABI compatible with AMD64 applications on ARM64 Windows 11. +* The ARM64EC ABI does not support AVX/AVX2/AVX512 instructions, nor their relevant intrinsics +* including _tzcnt_u64. Therefore, we need to neuter the _tzcnt_u64 code path for ARM64EC. +****************************************************************************************************/ +# if defined(__clang__) && (__clang_major__ < 10) + /* Avoid undefined clang-cl intrinsics issue. + * See https://github.com/lz4/lz4/pull/1017 for details. */ + return (unsigned)__builtin_ia32_tzcnt_u64(val) >> 3; +# else + /* x64 CPUS without BMI support interpret `TZCNT` as `REP BSF` */ + return (unsigned)_tzcnt_u64(val) >> 3; +# endif +# elif defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanForward64(&r, (U64)val); + return (unsigned)r >> 3; +# elif (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_ctzll((U64)val) >> 3; +# else + const U64 m = 0x0101010101010101ULL; + val ^= val - 1; + return (unsigned)(((U64)((val & (m - 1)) * m)) >> 56); +# endif + } else /* 32 bits */ { +# if defined(_MSC_VER) && (_MSC_VER >= 1400) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward(&r, (U32)val); + return (unsigned)r >> 3; +# elif (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(__TINYC__) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_ctz((U32)val) >> 3; +# else + const U32 m = 0x01010101; + return (unsigned)((((val - 1) ^ val) & (m - 1)) * m) >> 24; +# endif + } + } else /* Big Endian CPU */ { + if (sizeof(val)==8) { +# if (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(__TINYC__) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_clzll((U64)val) >> 3; +# else +#if 1 + /* this method is probably faster, + * but adds a 128 bytes lookup table */ + static const unsigned char ctz7_tab[128] = { + 7, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + }; + U64 const mask = 0x0101010101010101ULL; + U64 const t = (((val >> 8) - mask) | val) & mask; + return ctz7_tab[(t * 0x0080402010080402ULL) >> 57]; +#else + /* this method doesn't consume memory space like the previous one, + * but it contains several branches, + * that may end up slowing execution */ + static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits. + Just to avoid some static analyzer complaining about shift by 32 on 32-bits target. + Note that this code path is never triggered in 32-bits mode. */ + unsigned r; + if (!(val>>by32)) { r=4; } else { r=0; val>>=by32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +#endif +# endif + } else /* 32 bits */ { +# if (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_clz((U32)val) >> 3; +# else + val >>= 8; + val = ((((val + 0x00FFFF00) | 0x00FFFFFF) + val) | + (val + 0x00FF0000)) >> 24; + return (unsigned)val ^ 3; +# endif + } + } +} + + +#define STEPSIZE sizeof(reg_t) +LZ4_FORCE_INLINE +unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) +{ + const BYTE* const pStart = pIn; + + if (likely(pIn < pInLimit-(STEPSIZE-1))) { + reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); + if (!diff) { + pIn+=STEPSIZE; pMatch+=STEPSIZE; + } else { + return LZ4_NbCommonBytes(diff); + } } + + while (likely(pIn < pInLimit-(STEPSIZE-1))) { + reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); + if (!diff) { pIn+=STEPSIZE; pMatch+=STEPSIZE; continue; } + pIn += LZ4_NbCommonBytes(diff); + return (unsigned)(pIn - pStart); + } + + if ((STEPSIZE==8) && (pIn<(pInLimit-3)) && (LZ4_read32(pMatch) == LZ4_read32(pIn))) { pIn+=4; pMatch+=4; } + if ((pIn<(pInLimit-1)) && (LZ4_read16(pMatch) == LZ4_read16(pIn))) { pIn+=2; pMatch+=2; } + if ((pIn compression run slower on incompressible data */ + + +/*-************************************ +* Local Structures and types +**************************************/ +typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t; + +/** + * This enum distinguishes several different modes of accessing previous + * content in the stream. + * + * - noDict : There is no preceding content. + * - withPrefix64k : Table entries up to ctx->dictSize before the current blob + * blob being compressed are valid and refer to the preceding + * content (of length ctx->dictSize), which is available + * contiguously preceding in memory the content currently + * being compressed. + * - usingExtDict : Like withPrefix64k, but the preceding content is somewhere + * else in memory, starting at ctx->dictionary with length + * ctx->dictSize. + * - usingDictCtx : Everything concerning the preceding content is + * in a separate context, pointed to by ctx->dictCtx. + * ctx->dictionary, ctx->dictSize, and table entries + * in the current context that refer to positions + * preceding the beginning of the current compression are + * ignored. Instead, ctx->dictCtx->dictionary and ctx->dictCtx + * ->dictSize describe the location and size of the preceding + * content, and matches are found by looking in the ctx + * ->dictCtx->hashTable. + */ +typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingDictCtx } dict_directive; +typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; + + +/*-************************************ +* Local Utils +**************************************/ +int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } +const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } +int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } +int LZ4_sizeofState(void) { return sizeof(LZ4_stream_t); } + + +/*-**************************************** +* Internal Definitions, used only in Tests +*******************************************/ +#if defined (__cplusplus) +extern "C" { +#endif + +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize); + +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, + int compressedSize, int maxOutputSize, + const void* dictStart, size_t dictSize); +int LZ4_decompress_safe_partial_forceExtDict(const char* source, char* dest, + int compressedSize, int targetOutputSize, int dstCapacity, + const void* dictStart, size_t dictSize); +#if defined (__cplusplus) +} +#endif + +/*-****************************** +* Compression functions +********************************/ +LZ4_FORCE_INLINE U32 LZ4_hash4(U32 sequence, tableType_t const tableType) +{ + if (tableType == byU16) + return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); + else + return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); +} + +LZ4_FORCE_INLINE U32 LZ4_hash5(U64 sequence, tableType_t const tableType) +{ + const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; + if (LZ4_isLittleEndian()) { + const U64 prime5bytes = 889523592379ULL; + return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); + } else { + const U64 prime8bytes = 11400714785074694791ULL; + return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); + } +} + +LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) +{ + if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType); + +#ifdef LZ4_STATIC_LINKING_ONLY_ENDIANNESS_INDEPENDENT_OUTPUT + return LZ4_hash4(LZ4_readLE32(p), tableType); +#else + return LZ4_hash4(LZ4_read32(p), tableType); +#endif +} + +LZ4_FORCE_INLINE void LZ4_clearHash(U32 h, void* tableBase, tableType_t const tableType) +{ + switch (tableType) + { + default: /* fallthrough */ + case clearedTable: { /* illegal! */ assert(0); return; } + case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = NULL; return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = 0; return; } + case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = 0; return; } + } +} + +LZ4_FORCE_INLINE void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType) +{ + switch (tableType) + { + default: /* fallthrough */ + case clearedTable: /* fallthrough */ + case byPtr: { /* illegal! */ assert(0); return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = idx; return; } + case byU16: { U16* hashTable = (U16*) tableBase; assert(idx < 65536); hashTable[h] = (U16)idx; return; } + } +} + +/* LZ4_putPosition*() : only used in byPtr mode */ +LZ4_FORCE_INLINE void LZ4_putPositionOnHash(const BYTE* p, U32 h, + void* tableBase, tableType_t const tableType) +{ + const BYTE** const hashTable = (const BYTE**)tableBase; + assert(tableType == byPtr); (void)tableType; + hashTable[h] = p; +} + +LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType) +{ + U32 const h = LZ4_hashPosition(p, tableType); + LZ4_putPositionOnHash(p, h, tableBase, tableType); +} + +/* LZ4_getIndexOnHash() : + * Index of match position registered in hash table. + * hash position must be calculated by using base+index, or dictBase+index. + * Assumption 1 : only valid if tableType == byU32 or byU16. + * Assumption 2 : h is presumed valid (within limits of hash table) + */ +LZ4_FORCE_INLINE U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType) +{ + LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2); + if (tableType == byU32) { + const U32* const hashTable = (const U32*) tableBase; + assert(h < (1U << (LZ4_MEMORY_USAGE-2))); + return hashTable[h]; + } + if (tableType == byU16) { + const U16* const hashTable = (const U16*) tableBase; + assert(h < (1U << (LZ4_MEMORY_USAGE-1))); + return hashTable[h]; + } + assert(0); return 0; /* forbidden case */ +} + +static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType_t tableType) +{ + assert(tableType == byPtr); (void)tableType; + { const BYTE* const* hashTable = (const BYTE* const*) tableBase; return hashTable[h]; } +} + +LZ4_FORCE_INLINE const BYTE* +LZ4_getPosition(const BYTE* p, + const void* tableBase, tableType_t tableType) +{ + U32 const h = LZ4_hashPosition(p, tableType); + return LZ4_getPositionOnHash(h, tableBase, tableType); +} + +LZ4_FORCE_INLINE void +LZ4_prepareTable(LZ4_stream_t_internal* const cctx, + const int inputSize, + const tableType_t tableType) { + /* If the table hasn't been used, it's guaranteed to be zeroed out, and is + * therefore safe to use no matter what mode we're in. Otherwise, we figure + * out if it's safe to leave as is or whether it needs to be reset. + */ + if ((tableType_t)cctx->tableType != clearedTable) { + assert(inputSize >= 0); + if ((tableType_t)cctx->tableType != tableType + || ((tableType == byU16) && cctx->currentOffset + (unsigned)inputSize >= 0xFFFFU) + || ((tableType == byU32) && cctx->currentOffset > 1 GB) + || tableType == byPtr + || inputSize >= 4 KB) + { + DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", cctx); + MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE); + cctx->currentOffset = 0; + cctx->tableType = (U32)clearedTable; + } else { + DEBUGLOG(4, "LZ4_prepareTable: Re-use hash table (no reset)"); + } + } + + /* Adding a gap, so all previous entries are > LZ4_DISTANCE_MAX back, + * is faster than compressing without a gap. + * However, compressing with currentOffset == 0 is faster still, + * so we preserve that case. + */ + if (cctx->currentOffset != 0 && tableType == byU32) { + DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset"); + cctx->currentOffset += 64 KB; + } + + /* Finally, clear history */ + cctx->dictCtx = NULL; + cctx->dictionary = NULL; + cctx->dictSize = 0; +} + +/** LZ4_compress_generic_validated() : + * inlined, to ensure branches are decided at compilation time. + * The following conditions are presumed already validated: + * - source != NULL + * - inputSize > 0 + */ +LZ4_FORCE_INLINE int LZ4_compress_generic_validated( + LZ4_stream_t_internal* const cctx, + const char* const source, + char* const dest, + const int inputSize, + int* inputConsumed, /* only written when outputDirective == fillOutput */ + const int maxOutputSize, + const limitedOutput_directive outputDirective, + const tableType_t tableType, + const dict_directive dictDirective, + const dictIssue_directive dictIssue, + const int acceleration) +{ + int result; + const BYTE* ip = (const BYTE*)source; + + U32 const startIndex = cctx->currentOffset; + const BYTE* base = (const BYTE*)source - startIndex; + const BYTE* lowLimit; + + const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx; + const BYTE* const dictionary = + dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary; + const U32 dictSize = + dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize; + const U32 dictDelta = + (dictDirective == usingDictCtx) ? startIndex - dictCtx->currentOffset : 0; /* make indexes in dictCtx comparable with indexes in current context */ + + int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx); + U32 const prefixIdxLimit = startIndex - dictSize; /* used when dictDirective == dictSmall */ + const BYTE* const dictEnd = dictionary ? dictionary + dictSize : dictionary; + const BYTE* anchor = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimitPlusOne = iend - MFLIMIT + 1; + const BYTE* const matchlimit = iend - LASTLITERALS; + + /* the dictCtx currentOffset is indexed on the start of the dictionary, + * while a dictionary in the current context precedes the currentOffset */ + const BYTE* dictBase = (dictionary == NULL) ? NULL : + (dictDirective == usingDictCtx) ? + dictionary + dictSize - dictCtx->currentOffset : + dictionary + dictSize - startIndex; + + BYTE* op = (BYTE*) dest; + BYTE* const olimit = op + maxOutputSize; + + U32 offset = 0; + U32 forwardH; + + DEBUGLOG(5, "LZ4_compress_generic_validated: srcSize=%i, tableType=%u", inputSize, tableType); + assert(ip != NULL); + if (tableType == byU16) assert(inputSize= 1); + + lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0); + + /* Update context state */ + if (dictDirective == usingDictCtx) { + /* Subsequent linked blocks can't use the dictionary. */ + /* Instead, they use the block we just compressed. */ + cctx->dictCtx = NULL; + cctx->dictSize = (U32)inputSize; + } else { + cctx->dictSize += (U32)inputSize; + } + cctx->currentOffset += (U32)inputSize; + cctx->tableType = (U32)tableType; + + if (inputSizehashTable, byPtr); + } else { + LZ4_putIndexOnHash(startIndex, h, cctx->hashTable, tableType); + } } + ip++; forwardH = LZ4_hashPosition(ip, tableType); + + /* Main Loop */ + for ( ; ; ) { + const BYTE* match; + BYTE* token; + const BYTE* filledIp; + + /* Find a match */ + if (tableType == byPtr) { + const BYTE* forwardIp = ip; + int step = 1; + int searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; + assert(ip < mflimitPlusOne); + + match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType); + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType); + + } while ( (match+LZ4_DISTANCE_MAX < ip) + || (LZ4_read32(match) != LZ4_read32(ip)) ); + + } else { /* byU32, byU16 */ + + const BYTE* forwardIp = ip; + int step = 1; + int searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + U32 const current = (U32)(forwardIp - base); + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + assert(matchIndex <= current); + assert(forwardIp - base < (ptrdiff_t)(2 GB - 1)); + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; + assert(ip < mflimitPlusOne); + + if (dictDirective == usingDictCtx) { + if (matchIndex < startIndex) { + /* there was no match, try the dictionary */ + assert(tableType == byU32); + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; + matchIndex += dictDelta; /* make dictCtx index comparable with current context */ + lowLimit = dictionary; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; + } + } else if (dictDirective == usingExtDict) { + if (matchIndex < startIndex) { + DEBUGLOG(7, "extDict candidate: matchIndex=%5u < startIndex=%5u", matchIndex, startIndex); + assert(startIndex - matchIndex >= MINMATCH); + assert(dictBase); + match = dictBase + matchIndex; + lowLimit = dictionary; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; + } + } else { /* single continuous memory segment */ + match = base + matchIndex; + } + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); + + DEBUGLOG(7, "candidate at pos=%u (offset=%u \n", matchIndex, current - matchIndex); + if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) { continue; } /* match outside of valid area */ + assert(matchIndex < current); + if ( ((tableType != byU16) || (LZ4_DISTANCE_MAX < LZ4_DISTANCE_ABSOLUTE_MAX)) + && (matchIndex+LZ4_DISTANCE_MAX < current)) { + continue; + } /* too far */ + assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* match now expected within distance */ + + if (LZ4_read32(match) == LZ4_read32(ip)) { + if (maybe_extMem) offset = current - matchIndex; + break; /* match found */ + } + + } while(1); + } + + /* Catch up */ + filledIp = ip; + assert(ip > anchor); /* this is always true as ip has been advanced before entering the main loop */ + if ((match > lowLimit) && unlikely(ip[-1] == match[-1])) { + do { ip--; match--; } while (((ip > anchor) & (match > lowLimit)) && (unlikely(ip[-1] == match[-1]))); + } + + /* Encode Literals */ + { unsigned const litLength = (unsigned)(ip - anchor); + token = op++; + if ((outputDirective == limitedOutput) && /* Check output buffer overflow */ + (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)) ) { + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + if ((outputDirective == fillOutput) && + (unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) { + op--; + goto _last_literals; + } + if (litLength >= RUN_MASK) { + int len = (int)(litLength - RUN_MASK); + *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength< olimit)) { + /* the match was too close to the end, rewind and go to last literals */ + op = token; + goto _last_literals; + } + + /* Encode Offset */ + if (maybe_extMem) { /* static test */ + DEBUGLOG(6, " with offset=%u (ext if > %i)", offset, (int)(ip - (const BYTE*)source)); + assert(offset <= LZ4_DISTANCE_MAX && offset > 0); + LZ4_writeLE16(op, (U16)offset); op+=2; + } else { + DEBUGLOG(6, " with offset=%u (same segment)", (U32)(ip - match)); + assert(ip-match <= LZ4_DISTANCE_MAX); + LZ4_writeLE16(op, (U16)(ip - match)); op+=2; + } + + /* Encode MatchLength */ + { unsigned matchCode; + + if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx) + && (lowLimit==dictionary) /* match within extDict */ ) { + const BYTE* limit = ip + (dictEnd-match); + assert(dictEnd > match); + if (limit > matchlimit) limit = matchlimit; + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); + ip += (size_t)matchCode + MINMATCH; + if (ip==limit) { + unsigned const more = LZ4_count(limit, (const BYTE*)source, matchlimit); + matchCode += more; + ip += more; + } + DEBUGLOG(6, " with matchLength=%u starting in extDict", matchCode+MINMATCH); + } else { + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); + ip += (size_t)matchCode + MINMATCH; + DEBUGLOG(6, " with matchLength=%u", matchCode+MINMATCH); + } + + if ((outputDirective) && /* Check output buffer overflow */ + (unlikely(op + (1 + LASTLITERALS) + (matchCode+240)/255 > olimit)) ) { + if (outputDirective == fillOutput) { + /* Match description too long : reduce it */ + U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 1 - LASTLITERALS) * 255; + ip -= matchCode - newMatchCode; + assert(newMatchCode < matchCode); + matchCode = newMatchCode; + if (unlikely(ip <= filledIp)) { + /* We have already filled up to filledIp so if ip ends up less than filledIp + * we have positions in the hash table beyond the current position. This is + * a problem if we reuse the hash table. So we have to remove these positions + * from the hash table. + */ + const BYTE* ptr; + DEBUGLOG(5, "Clearing %u positions", (U32)(filledIp - ip)); + for (ptr = ip; ptr <= filledIp; ++ptr) { + U32 const h = LZ4_hashPosition(ptr, tableType); + LZ4_clearHash(h, cctx->hashTable, tableType); + } + } + } else { + assert(outputDirective == limitedOutput); + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + } + if (matchCode >= ML_MASK) { + *token += ML_MASK; + matchCode -= ML_MASK; + LZ4_write32(op, 0xFFFFFFFF); + while (matchCode >= 4*255) { + op+=4; + LZ4_write32(op, 0xFFFFFFFF); + matchCode -= 4*255; + } + op += matchCode / 255; + *op++ = (BYTE)(matchCode % 255); + } else + *token += (BYTE)(matchCode); + } + /* Ensure we have enough space for the last literals. */ + assert(!(outputDirective == fillOutput && op + 1 + LASTLITERALS > olimit)); + + anchor = ip; + + /* Test end of chunk */ + if (ip >= mflimitPlusOne) break; + + /* Fill table */ + { U32 const h = LZ4_hashPosition(ip-2, tableType); + if (tableType == byPtr) { + LZ4_putPositionOnHash(ip-2, h, cctx->hashTable, byPtr); + } else { + U32 const idx = (U32)((ip-2) - base); + LZ4_putIndexOnHash(idx, h, cctx->hashTable, tableType); + } } + + /* Test next position */ + if (tableType == byPtr) { + + match = LZ4_getPosition(ip, cctx->hashTable, tableType); + LZ4_putPosition(ip, cctx->hashTable, tableType); + if ( (match+LZ4_DISTANCE_MAX >= ip) + && (LZ4_read32(match) == LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + } else { /* byU32, byU16 */ + + U32 const h = LZ4_hashPosition(ip, tableType); + U32 const current = (U32)(ip-base); + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + assert(matchIndex < current); + if (dictDirective == usingDictCtx) { + if (matchIndex < startIndex) { + /* there was no match, try the dictionary */ + assert(tableType == byU32); + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; + lowLimit = dictionary; /* required for match length counter */ + matchIndex += dictDelta; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; /* required for match length counter */ + } + } else if (dictDirective==usingExtDict) { + if (matchIndex < startIndex) { + assert(dictBase); + match = dictBase + matchIndex; + lowLimit = dictionary; /* required for match length counter */ + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; /* required for match length counter */ + } + } else { /* single memory segment */ + match = base + matchIndex; + } + LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); + assert(matchIndex < current); + if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1) + && (((tableType==byU16) && (LZ4_DISTANCE_MAX == LZ4_DISTANCE_ABSOLUTE_MAX)) ? 1 : (matchIndex+LZ4_DISTANCE_MAX >= current)) + && (LZ4_read32(match) == LZ4_read32(ip)) ) { + token=op++; + *token=0; + if (maybe_extMem) offset = current - matchIndex; + DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", + (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source)); + goto _next_match; + } + } + + /* Prepare next loop */ + forwardH = LZ4_hashPosition(++ip, tableType); + + } + +_last_literals: + /* Encode Last Literals */ + { size_t lastRun = (size_t)(iend - anchor); + if ( (outputDirective) && /* Check output buffer overflow */ + (op + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > olimit)) { + if (outputDirective == fillOutput) { + /* adapt lastRun to fill 'dst' */ + assert(olimit >= op); + lastRun = (size_t)(olimit-op) - 1/*token*/; + lastRun -= (lastRun + 256 - RUN_MASK) / 256; /*additional length tokens*/ + } else { + assert(outputDirective == limitedOutput); + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + } + DEBUGLOG(6, "Final literal run : %i literals", (int)lastRun); + if (lastRun >= RUN_MASK) { + size_t accumulator = lastRun - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRun< 0); + DEBUGLOG(5, "LZ4_compress_generic: compressed %i bytes into %i bytes", inputSize, result); + return result; +} + +/** LZ4_compress_generic() : + * inlined, to ensure branches are decided at compilation time; + * takes care of src == (NULL, 0) + * and forward the rest to LZ4_compress_generic_validated */ +LZ4_FORCE_INLINE int LZ4_compress_generic( + LZ4_stream_t_internal* const cctx, + const char* const src, + char* const dst, + const int srcSize, + int *inputConsumed, /* only written when outputDirective == fillOutput */ + const int dstCapacity, + const limitedOutput_directive outputDirective, + const tableType_t tableType, + const dict_directive dictDirective, + const dictIssue_directive dictIssue, + const int acceleration) +{ + DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, dstCapacity=%i", + srcSize, dstCapacity); + + if ((U32)srcSize > (U32)LZ4_MAX_INPUT_SIZE) { return 0; } /* Unsupported srcSize, too large (or negative) */ + if (srcSize == 0) { /* src == NULL supported if srcSize == 0 */ + if (outputDirective != notLimited && dstCapacity <= 0) return 0; /* no output, can't write anything */ + DEBUGLOG(5, "Generating an empty block"); + assert(outputDirective == notLimited || dstCapacity >= 1); + assert(dst != NULL); + dst[0] = 0; + if (outputDirective == fillOutput) { + assert (inputConsumed != NULL); + *inputConsumed = 0; + } + return 1; + } + assert(src != NULL); + + return LZ4_compress_generic_validated(cctx, src, dst, srcSize, + inputConsumed, /* only written into if outputDirective == fillOutput */ + dstCapacity, outputDirective, + tableType, dictDirective, dictIssue, acceleration); +} + + +int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + LZ4_stream_t_internal* const ctx = & LZ4_initStream(state, sizeof(LZ4_stream_t)) -> internal_donotuse; + assert(ctx != NULL); + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; + if (maxOutputSize >= LZ4_compressBound(inputSize)) { + if (inputSize < LZ4_64Klimit) { + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } + } else { + if (inputSize < LZ4_64Klimit) { + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } + } +} + +/** + * LZ4_compress_fast_extState_fastReset() : + * A variant of LZ4_compress_fast_extState(). + * + * Using this variant avoids an expensive initialization step. It is only safe + * to call if the state buffer is known to be correctly initialized already + * (see comment in lz4.h on LZ4_resetStream_fast() for a definition of + * "correctly initialized"). + */ +int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration) +{ + LZ4_stream_t_internal* const ctx = &((LZ4_stream_t*)state)->internal_donotuse; + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; + assert(ctx != NULL); + + if (dstCapacity >= LZ4_compressBound(srcSize)) { + if (srcSize < LZ4_64Klimit) { + const tableType_t tableType = byU16; + LZ4_prepareTable(ctx, srcSize, tableType); + if (ctx->currentOffset) { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, dictSmall, acceleration); + } else { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } + } else { + if (srcSize < LZ4_64Klimit) { + const tableType_t tableType = byU16; + LZ4_prepareTable(ctx, srcSize, tableType); + if (ctx->currentOffset) { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration); + } else { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } + } +} + + +int LZ4_compress_fast(const char* src, char* dest, int srcSize, int dstCapacity, int acceleration) +{ + int result; +#if (LZ4_HEAPMODE) + LZ4_stream_t* const ctxPtr = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + if (ctxPtr == NULL) return 0; +#else + LZ4_stream_t ctx; + LZ4_stream_t* const ctxPtr = &ctx; +#endif + result = LZ4_compress_fast_extState(ctxPtr, src, dest, srcSize, dstCapacity, acceleration); + +#if (LZ4_HEAPMODE) + FREEMEM(ctxPtr); +#endif + return result; +} + + +int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity) +{ + return LZ4_compress_fast(src, dst, srcSize, dstCapacity, 1); +} + + +/* Note!: This function leaves the stream in an unclean/broken state! + * It is not safe to subsequently use the same state with a _fastReset() or + * _continue() call without resetting it. */ +static int LZ4_compress_destSize_extState_internal(LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize, int acceleration) +{ + void* const s = LZ4_initStream(state, sizeof (*state)); + assert(s != NULL); (void)s; + + if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ + return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, acceleration); + } else { + if (*srcSizePtr < LZ4_64Klimit) { + return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, byU16, noDict, noDictIssue, acceleration); + } else { + tableType_t const addrMode = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, addrMode, noDict, noDictIssue, acceleration); + } } +} + +int LZ4_compress_destSize_extState(void* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize, int acceleration) +{ + int const r = LZ4_compress_destSize_extState_internal((LZ4_stream_t*)state, src, dst, srcSizePtr, targetDstSize, acceleration); + /* clean the state on exit */ + LZ4_initStream(state, sizeof (LZ4_stream_t)); + return r; +} + + +int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) +{ +#if (LZ4_HEAPMODE) + LZ4_stream_t* const ctx = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + if (ctx == NULL) return 0; +#else + LZ4_stream_t ctxBody; + LZ4_stream_t* const ctx = &ctxBody; +#endif + + int result = LZ4_compress_destSize_extState_internal(ctx, src, dst, srcSizePtr, targetDstSize, 1); + +#if (LZ4_HEAPMODE) + FREEMEM(ctx); +#endif + return result; +} + + + +/*-****************************** +* Streaming functions +********************************/ + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4_stream_t* LZ4_createStream(void) +{ + LZ4_stream_t* const lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); + LZ4_STATIC_ASSERT(sizeof(LZ4_stream_t) >= sizeof(LZ4_stream_t_internal)); + DEBUGLOG(4, "LZ4_createStream %p", lz4s); + if (lz4s == NULL) return NULL; + LZ4_initStream(lz4s, sizeof(*lz4s)); + return lz4s; +} +#endif + +static size_t LZ4_stream_t_alignment(void) +{ +#if LZ4_ALIGN_TEST + typedef struct { char c; LZ4_stream_t t; } t_a; + return sizeof(t_a) - sizeof(LZ4_stream_t); +#else + return 1; /* effectively disabled */ +#endif +} + +LZ4_stream_t* LZ4_initStream (void* buffer, size_t size) +{ + DEBUGLOG(5, "LZ4_initStream"); + if (buffer == NULL) { return NULL; } + if (size < sizeof(LZ4_stream_t)) { return NULL; } + if (!LZ4_isAligned(buffer, LZ4_stream_t_alignment())) return NULL; + MEM_INIT(buffer, 0, sizeof(LZ4_stream_t_internal)); + return (LZ4_stream_t*)buffer; +} + +/* resetStream is now deprecated, + * prefer initStream() which is more general */ +void LZ4_resetStream (LZ4_stream_t* LZ4_stream) +{ + DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", LZ4_stream); + MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t_internal)); +} + +void LZ4_resetStream_fast(LZ4_stream_t* ctx) { + LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32); +} + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +int LZ4_freeStream (LZ4_stream_t* LZ4_stream) +{ + if (!LZ4_stream) return 0; /* support free on NULL */ + DEBUGLOG(5, "LZ4_freeStream %p", LZ4_stream); + FREEMEM(LZ4_stream); + return (0); +} +#endif + + +typedef enum { _ld_fast, _ld_slow } LoadDict_mode_e; +#define HASH_UNIT sizeof(reg_t) +int LZ4_loadDict_internal(LZ4_stream_t* LZ4_dict, + const char* dictionary, int dictSize, + LoadDict_mode_e _ld) +{ + LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; + const tableType_t tableType = byU32; + const BYTE* p = (const BYTE*)dictionary; + const BYTE* const dictEnd = p + dictSize; + U32 idx32; + + DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, dictionary, LZ4_dict); + + /* It's necessary to reset the context, + * and not just continue it with prepareTable() + * to avoid any risk of generating overflowing matchIndex + * when compressing using this dictionary */ + LZ4_resetStream(LZ4_dict); + + /* We always increment the offset by 64 KB, since, if the dict is longer, + * we truncate it to the last 64k, and if it's shorter, we still want to + * advance by a whole window length so we can provide the guarantee that + * there are only valid offsets in the window, which allows an optimization + * in LZ4_compress_fast_continue() where it uses noDictIssue even when the + * dictionary isn't a full 64k. */ + dict->currentOffset += 64 KB; + + if (dictSize < (int)HASH_UNIT) { + return 0; + } + + if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; + dict->dictionary = p; + dict->dictSize = (U32)(dictEnd - p); + dict->tableType = (U32)tableType; + idx32 = dict->currentOffset - dict->dictSize; + + while (p <= dictEnd-HASH_UNIT) { + U32 const h = LZ4_hashPosition(p, tableType); + /* Note: overwriting => favors positions end of dictionary */ + LZ4_putIndexOnHash(idx32, h, dict->hashTable, tableType); + p+=3; idx32+=3; + } + + if (_ld == _ld_slow) { + /* Fill hash table with additional references, to improve compression capability */ + p = dict->dictionary; + idx32 = dict->currentOffset - dict->dictSize; + while (p <= dictEnd-HASH_UNIT) { + U32 const h = LZ4_hashPosition(p, tableType); + U32 const limit = dict->currentOffset - 64 KB; + if (LZ4_getIndexOnHash(h, dict->hashTable, tableType) <= limit) { + /* Note: not overwriting => favors positions beginning of dictionary */ + LZ4_putIndexOnHash(idx32, h, dict->hashTable, tableType); + } + p++; idx32++; + } + } + + return (int)dict->dictSize; +} + +int LZ4_loadDict(LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) +{ + return LZ4_loadDict_internal(LZ4_dict, dictionary, dictSize, _ld_fast); +} + +int LZ4_loadDictSlow(LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) +{ + return LZ4_loadDict_internal(LZ4_dict, dictionary, dictSize, _ld_slow); +} + +void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) +{ + const LZ4_stream_t_internal* dictCtx = (dictionaryStream == NULL) ? NULL : + &(dictionaryStream->internal_donotuse); + + DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %u)", + workingStream, dictionaryStream, + dictCtx != NULL ? dictCtx->dictSize : 0); + + if (dictCtx != NULL) { + /* If the current offset is zero, we will never look in the + * external dictionary context, since there is no value a table + * entry can take that indicate a miss. In that case, we need + * to bump the offset to something non-zero. + */ + if (workingStream->internal_donotuse.currentOffset == 0) { + workingStream->internal_donotuse.currentOffset = 64 KB; + } + + /* Don't actually attach an empty dictionary. + */ + if (dictCtx->dictSize == 0) { + dictCtx = NULL; + } + } + workingStream->internal_donotuse.dictCtx = dictCtx; +} + + +static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, int nextSize) +{ + assert(nextSize >= 0); + if (LZ4_dict->currentOffset + (unsigned)nextSize > 0x80000000) { /* potential ptrdiff_t overflow (32-bits mode) */ + /* rescale hash table */ + U32 const delta = LZ4_dict->currentOffset - 64 KB; + const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; + int i; + DEBUGLOG(4, "LZ4_renormDictT"); + for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; + else LZ4_dict->hashTable[i] -= delta; + } + LZ4_dict->currentOffset = 64 KB; + if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; + LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; + } +} + + +int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, + const char* source, char* dest, + int inputSize, int maxOutputSize, + int acceleration) +{ + const tableType_t tableType = byU32; + LZ4_stream_t_internal* const streamPtr = &LZ4_stream->internal_donotuse; + const char* dictEnd = streamPtr->dictSize ? (const char*)streamPtr->dictionary + streamPtr->dictSize : NULL; + + DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i, dictSize=%u)", inputSize, streamPtr->dictSize); + + LZ4_renormDictT(streamPtr, inputSize); /* fix index overflow */ + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; + + /* invalidate tiny dictionaries */ + if ( (streamPtr->dictSize < 4) /* tiny dictionary : not enough for a hash */ + && (dictEnd != source) /* prefix mode */ + && (inputSize > 0) /* tolerance : don't lose history, in case next invocation would use prefix mode */ + && (streamPtr->dictCtx == NULL) /* usingDictCtx */ + ) { + DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, streamPtr->dictionary); + /* remove dictionary existence from history, to employ faster prefix mode */ + streamPtr->dictSize = 0; + streamPtr->dictionary = (const BYTE*)source; + dictEnd = source; + } + + /* Check overlapping input/dictionary space */ + { const char* const sourceEnd = source + inputSize; + if ((sourceEnd > (const char*)streamPtr->dictionary) && (sourceEnd < dictEnd)) { + streamPtr->dictSize = (U32)(dictEnd - sourceEnd); + if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; + if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; + streamPtr->dictionary = (const BYTE*)dictEnd - streamPtr->dictSize; + } + } + + /* prefix mode : source data follows dictionary */ + if (dictEnd == source) { + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration); + else + return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, noDictIssue, acceleration); + } + + /* external dictionary mode */ + { int result; + if (streamPtr->dictCtx) { + /* We depend here on the fact that dictCtx'es (produced by + * LZ4_loadDict) guarantee that their tables contain no references + * to offsets between dictCtx->currentOffset - 64 KB and + * dictCtx->currentOffset - dictCtx->dictSize. This makes it safe + * to use noDictIssue even when the dict isn't a full 64 KB. + */ + if (inputSize > 4 KB) { + /* For compressing large blobs, it is faster to pay the setup + * cost to copy the dictionary's tables into the active context, + * so that the compression loop is only looking into one table. + */ + LZ4_memcpy(streamPtr, streamPtr->dictCtx, sizeof(*streamPtr)); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration); + } + } else { /* small data <= 4 KB */ + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); + } + } + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)inputSize; + return result; + } +} + + +/* Hidden debug function, to force-test external dictionary mode */ +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize) +{ + LZ4_stream_t_internal* const streamPtr = &LZ4_dict->internal_donotuse; + int result; + + LZ4_renormDictT(streamPtr, srcSize); + + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { + result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, dictSmall, 1); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); + } + + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)srcSize; + + return result; +} + + +/*! LZ4_saveDict() : + * If previously compressed data block is not guaranteed to remain available at its memory location, + * save it into a safer place (char* safeBuffer). + * Note : no need to call LZ4_loadDict() afterwards, dictionary is immediately usable, + * one can therefore call LZ4_compress_fast_continue() right after. + * @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. + */ +int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) +{ + LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; + + DEBUGLOG(5, "LZ4_saveDict : dictSize=%i, safeBuffer=%p", dictSize, safeBuffer); + + if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */ + if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; } + + if (safeBuffer == NULL) assert(dictSize == 0); + if (dictSize > 0) { + const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; + assert(dict->dictionary); + LZ4_memmove(safeBuffer, previousDictEnd - dictSize, (size_t)dictSize); + } + + dict->dictionary = (const BYTE*)safeBuffer; + dict->dictSize = (U32)dictSize; + + return dictSize; +} + + + +/*-******************************* + * Decompression functions + ********************************/ + +typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive; + +#undef MIN +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + + +/* variant for decompress_unsafe() + * does not know end of input + * presumes input is well formed + * note : will consume at least one byte */ +static size_t read_long_length_no_check(const BYTE** pp) +{ + size_t b, l = 0; + do { b = **pp; (*pp)++; l += b; } while (b==255); + DEBUGLOG(6, "read_long_length_no_check: +length=%zu using %zu input bytes", l, l/255 + 1) + return l; +} + +/* core decoder variant for LZ4_decompress_fast*() + * for legacy support only : these entry points are deprecated. + * - Presumes input is correctly formed (no defense vs malformed inputs) + * - Does not know input size (presume input buffer is "large enough") + * - Decompress a full block (only) + * @return : nb of bytes read from input. + * Note : this variant is not optimized for speed, just for maintenance. + * the goal is to remove support of decompress_fast*() variants by v2.0 +**/ +LZ4_FORCE_INLINE int +LZ4_decompress_unsafe_generic( + const BYTE* const istart, + BYTE* const ostart, + int decompressedSize, + + size_t prefixSize, + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note: =0 if dictStart==NULL */ + ) +{ + const BYTE* ip = istart; + BYTE* op = (BYTE*)ostart; + BYTE* const oend = ostart + decompressedSize; + const BYTE* const prefixStart = ostart - prefixSize; + + DEBUGLOG(5, "LZ4_decompress_unsafe_generic"); + if (dictStart == NULL) assert(dictSize == 0); + + while (1) { + /* start new sequence */ + unsigned token = *ip++; + + /* literals */ + { size_t ll = token >> ML_BITS; + if (ll==15) { + /* long literal length */ + ll += read_long_length_no_check(&ip); + } + if ((size_t)(oend-op) < ll) return -1; /* output buffer overflow */ + LZ4_memmove(op, ip, ll); /* support in-place decompression */ + op += ll; + ip += ll; + if ((size_t)(oend-op) < MFLIMIT) { + if (op==oend) break; /* end of block */ + DEBUGLOG(5, "invalid: literals end at distance %zi from end of block", oend-op); + /* incorrect end of block : + * last match must start at least MFLIMIT==12 bytes before end of output block */ + return -1; + } } + + /* match */ + { size_t ml = token & 15; + size_t const offset = LZ4_readLE16(ip); + ip+=2; + + if (ml==15) { + /* long literal length */ + ml += read_long_length_no_check(&ip); + } + ml += MINMATCH; + + if ((size_t)(oend-op) < ml) return -1; /* output buffer overflow */ + + { const BYTE* match = op - offset; + + /* out of range */ + if (offset > (size_t)(op - prefixStart) + dictSize) { + DEBUGLOG(6, "offset out of range"); + return -1; + } + + /* check special case : extDict */ + if (offset > (size_t)(op - prefixStart)) { + /* extDict scenario */ + const BYTE* const dictEnd = dictStart + dictSize; + const BYTE* extMatch = dictEnd - (offset - (size_t)(op-prefixStart)); + size_t const extml = (size_t)(dictEnd - extMatch); + if (extml > ml) { + /* match entirely within extDict */ + LZ4_memmove(op, extMatch, ml); + op += ml; + ml = 0; + } else { + /* match split between extDict & prefix */ + LZ4_memmove(op, extMatch, extml); + op += extml; + ml -= extml; + } + match = prefixStart; + } + + /* match copy - slow variant, supporting overlap copy */ + { size_t u; + for (u=0; u= ipmax before start of loop. Returns initial_error if so. + * @error (output) - error code. Must be set to 0 before call. +**/ +typedef size_t Rvl_t; +static const Rvl_t rvl_error = (Rvl_t)(-1); +LZ4_FORCE_INLINE Rvl_t +read_variable_length(const BYTE** ip, const BYTE* ilimit, + int initial_check) +{ + Rvl_t s, length = 0; + assert(ip != NULL); + assert(*ip != NULL); + assert(ilimit != NULL); + if (initial_check && unlikely((*ip) >= ilimit)) { /* read limit reached */ + return rvl_error; + } + s = **ip; + (*ip)++; + length += s; + if (unlikely((*ip) > ilimit)) { /* read limit reached */ + return rvl_error; + } + /* accumulator overflow detection (32-bit mode only) */ + if ((sizeof(length) < 8) && unlikely(length > ((Rvl_t)(-1)/2)) ) { + return rvl_error; + } + if (likely(s != 255)) return length; + do { + s = **ip; + (*ip)++; + length += s; + if (unlikely((*ip) > ilimit)) { /* read limit reached */ + return rvl_error; + } + /* accumulator overflow detection (32-bit mode only) */ + if ((sizeof(length) < 8) && unlikely(length > ((Rvl_t)(-1)/2)) ) { + return rvl_error; + } + } while (s == 255); + + return length; +} + +/*! LZ4_decompress_generic() : + * This generic decompression function covers all use cases. + * It shall be instantiated several times, using different sets of directives. + * Note that it is important for performance that this function really get inlined, + * in order to remove useless branches during compilation optimization. + */ +LZ4_FORCE_INLINE int +LZ4_decompress_generic( + const char* const src, + char* const dst, + int srcSize, + int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */ + + earlyEnd_directive partialDecoding, /* full, partial */ + dict_directive dict, /* noDict, withPrefix64k, usingExtDict */ + const BYTE* const lowPrefix, /* always <= dst, == dst when no prefix */ + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note : = 0 if noDict */ + ) +{ + if ((src == NULL) || (outputSize < 0)) { return -1; } + + { const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + + BYTE* op = (BYTE*) dst; + BYTE* const oend = op + outputSize; + BYTE* cpy; + + const BYTE* const dictEnd = (dictStart == NULL) ? NULL : dictStart + dictSize; + + const int checkOffset = (dictSize < (int)(64 KB)); + + + /* Set up the "end" pointers for the shortcut. */ + const BYTE* const shortiend = iend - 14 /*maxLL*/ - 2 /*offset*/; + const BYTE* const shortoend = oend - 14 /*maxLL*/ - 18 /*maxML*/; + + const BYTE* match; + size_t offset; + unsigned token; + size_t length; + + + DEBUGLOG(5, "LZ4_decompress_generic (srcSize:%i, dstSize:%i)", srcSize, outputSize); + + /* Special cases */ + assert(lowPrefix <= op); + if (unlikely(outputSize==0)) { + /* Empty output buffer */ + if (partialDecoding) return 0; + return ((srcSize==1) && (*ip==0)) ? 0 : -1; + } + if (unlikely(srcSize==0)) { return -1; } + + /* LZ4_FAST_DEC_LOOP: + * designed for modern OoO performance cpus, + * where copying reliably 32-bytes is preferable to an unpredictable branch. + * note : fast loop may show a regression for some client arm chips. */ +#if LZ4_FAST_DEC_LOOP + if ((oend - op) < FASTLOOP_SAFE_DISTANCE) { + DEBUGLOG(6, "move to safe decode loop"); + goto safe_decode; + } + + /* Fast loop : decode sequences as long as output < oend-FASTLOOP_SAFE_DISTANCE */ + DEBUGLOG(6, "using fast decode loop"); + while (1) { + /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */ + assert(oend - op >= FASTLOOP_SAFE_DISTANCE); + assert(ip < iend); + token = *ip++; + length = token >> ML_BITS; /* literal length */ + DEBUGLOG(7, "blockPos%6u: litLength token = %u", (unsigned)(op-(BYTE*)dst), (unsigned)length); + + /* decode literal length */ + if (length == RUN_MASK) { + size_t const addl = read_variable_length(&ip, iend-RUN_MASK, 1); + if (addl == rvl_error) { + DEBUGLOG(6, "error reading long literal length"); + goto _output_error; + } + length += addl; + if (unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if (unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ + + /* copy literals */ + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if ((op+length>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; } + LZ4_wildCopy32(op, ip, op+length); + ip += length; op += length; + } else if (ip <= iend-(16 + 1/*max lit + offset + nextToken*/)) { + /* We don't need to check oend, since we check it once for each loop below */ + DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length); + /* Literals can only be <= 14, but hope compilers optimize better when copy by a register size */ + LZ4_memcpy(op, ip, 16); + ip += length; op += length; + } else { + goto safe_literal_copy; + } + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + DEBUGLOG(6, "blockPos%6u: offset = %u", (unsigned)(op-(BYTE*)dst), (unsigned)offset); + match = op - offset; + assert(match <= op); /* overflow check */ + + /* get matchlength */ + length = token & ML_MASK; + DEBUGLOG(7, " match length token = %u (len==%u)", (unsigned)length, (unsigned)length+MINMATCH); + + if (length == ML_MASK) { + size_t const addl = read_variable_length(&ip, iend - LASTLITERALS + 1, 0); + if (addl == rvl_error) { + DEBUGLOG(5, "error reading long match length"); + goto _output_error; + } + length += addl; + length += MINMATCH; + DEBUGLOG(7, " long match length == %u", (unsigned)length); + if (unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */ + if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { + goto safe_match_copy; + } + } else { + length += MINMATCH; + if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { + DEBUGLOG(7, "moving to safe_match_copy (ml==%u)", (unsigned)length); + goto safe_match_copy; + } + + /* Fastpath check: skip LZ4_wildCopy32 when true */ + if ((dict == withPrefix64k) || (match >= lowPrefix)) { + if (offset >= 8) { + assert(match >= lowPrefix); + assert(match <= op); + assert(op + 18 <= oend); + + LZ4_memcpy(op, match, 8); + LZ4_memcpy(op+8, match+8, 8); + LZ4_memcpy(op+16, match+16, 2); + op += length; + continue; + } } } + + if ( checkOffset && (unlikely(match + dictSize < lowPrefix)) ) { + DEBUGLOG(5, "Error : pos=%zi, offset=%zi => outside buffers", op-lowPrefix, op-match); + goto _output_error; + } + /* match starting within external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + assert(dictEnd != NULL); + if (unlikely(op+length > oend-LASTLITERALS)) { + if (partialDecoding) { + DEBUGLOG(7, "partialDecoding: dictionary match, close to dstEnd"); + length = MIN(length, (size_t)(oend-op)); + } else { + DEBUGLOG(6, "end-of-block condition violated") + goto _output_error; + } } + + if (length <= (size_t)(lowPrefix-match)) { + /* match fits entirely within external dictionary : just copy */ + LZ4_memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); + size_t const restSize = length - copySize; + LZ4_memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) { *op++ = *copyFrom++; } + } else { + LZ4_memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } + + /* copy match within block */ + cpy = op + length; + + assert((op <= oend) && (oend-op >= 32)); + if (unlikely(offset<16)) { + LZ4_memcpy_using_offset(op, match, cpy, offset); + } else { + LZ4_wildCopy32(op, match, cpy); + } + + op = cpy; /* wildcopy correction */ + } + safe_decode: +#endif + + /* Main Loop : decode remaining sequences where output < FASTLOOP_SAFE_DISTANCE */ + DEBUGLOG(6, "using safe decode loop"); + while (1) { + assert(ip < iend); + token = *ip++; + length = token >> ML_BITS; /* literal length */ + DEBUGLOG(7, "blockPos%6u: litLength token = %u", (unsigned)(op-(BYTE*)dst), (unsigned)length); + + /* A two-stage shortcut for the most common case: + * 1) If the literal length is 0..14, and there is enough space, + * enter the shortcut and copy 16 bytes on behalf of the literals + * (in the fast mode, only 8 bytes can be safely copied this way). + * 2) Further if the match length is 4..18, copy 18 bytes in a similar + * manner; but we ensure that there's enough space in the output for + * those 18 bytes earlier, upon entering the shortcut (in other words, + * there is a combined check for both stages). + */ + if ( (length != RUN_MASK) + /* strictly "less than" on input, to re-enter the loop with at least one byte */ + && likely((ip < shortiend) & (op <= shortoend)) ) { + /* Copy the literals */ + LZ4_memcpy(op, ip, 16); + op += length; ip += length; + + /* The second stage: prepare for match copying, decode full info. + * If it doesn't work out, the info won't be wasted. */ + length = token & ML_MASK; /* match length */ + DEBUGLOG(7, "blockPos%6u: matchLength token = %u (len=%u)", (unsigned)(op-(BYTE*)dst), (unsigned)length, (unsigned)length + 4); + offset = LZ4_readLE16(ip); ip += 2; + match = op - offset; + assert(match <= op); /* check overflow */ + + /* Do not deal with overlapping matches. */ + if ( (length != ML_MASK) + && (offset >= 8) + && (dict==withPrefix64k || match >= lowPrefix) ) { + /* Copy the match. */ + LZ4_memcpy(op + 0, match + 0, 8); + LZ4_memcpy(op + 8, match + 8, 8); + LZ4_memcpy(op +16, match +16, 2); + op += length + MINMATCH; + /* Both stages worked, load the next token. */ + continue; + } + + /* The second stage didn't work out, but the info is ready. + * Propel it right to the point of match copying. */ + goto _copy_match; + } + + /* decode literal length */ + if (length == RUN_MASK) { + size_t const addl = read_variable_length(&ip, iend-RUN_MASK, 1); + if (addl == rvl_error) { goto _output_error; } + length += addl; + if (unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if (unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ + } + +#if LZ4_FAST_DEC_LOOP + safe_literal_copy: +#endif + /* copy literals */ + cpy = op+length; + + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) { + /* We've either hit the input parsing restriction or the output parsing restriction. + * In the normal scenario, decoding a full block, it must be the last sequence, + * otherwise it's an error (invalid input or dimensions). + * In partialDecoding scenario, it's necessary to ensure there is no buffer overflow. + */ + if (partialDecoding) { + /* Since we are partial decoding we may be in this block because of the output parsing + * restriction, which is not valid since the output buffer is allowed to be undersized. + */ + DEBUGLOG(7, "partialDecoding: copying literals, close to input or output end") + DEBUGLOG(7, "partialDecoding: literal length = %u", (unsigned)length); + DEBUGLOG(7, "partialDecoding: remaining space in dstBuffer : %i", (int)(oend - op)); + DEBUGLOG(7, "partialDecoding: remaining space in srcBuffer : %i", (int)(iend - ip)); + /* Finishing in the middle of a literals segment, + * due to lack of input. + */ + if (ip+length > iend) { + length = (size_t)(iend-ip); + cpy = op + length; + } + /* Finishing in the middle of a literals segment, + * due to lack of output space. + */ + if (cpy > oend) { + cpy = oend; + assert(op<=oend); + length = (size_t)(oend-op); + } + } else { + /* We must be on the last sequence (or invalid) because of the parsing limitations + * so check that we exactly consume the input and don't overrun the output buffer. + */ + if ((ip+length != iend) || (cpy > oend)) { + DEBUGLOG(5, "should have been last run of literals") + DEBUGLOG(5, "ip(%p) + length(%i) = %p != iend (%p)", ip, (int)length, ip+length, iend); + DEBUGLOG(5, "or cpy(%p) > (oend-MFLIMIT)(%p)", cpy, oend-MFLIMIT); + DEBUGLOG(5, "after writing %u bytes / %i bytes available", (unsigned)(op-(BYTE*)dst), outputSize); + goto _output_error; + } + } + LZ4_memmove(op, ip, length); /* supports overlapping memory regions, for in-place decompression scenarios */ + ip += length; + op += length; + /* Necessarily EOF when !partialDecoding. + * When partialDecoding, it is EOF if we've either + * filled the output buffer or + * can't proceed with reading an offset for following match. + */ + if (!partialDecoding || (cpy == oend) || (ip >= (iend-2))) { + break; + } + } else { + LZ4_wildCopy8(op, ip, cpy); /* can overwrite up to 8 bytes beyond cpy */ + ip += length; op = cpy; + } + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; + + /* get matchlength */ + length = token & ML_MASK; + DEBUGLOG(7, "blockPos%6u: matchLength token = %u", (unsigned)(op-(BYTE*)dst), (unsigned)length); + + _copy_match: + if (length == ML_MASK) { + size_t const addl = read_variable_length(&ip, iend - LASTLITERALS + 1, 0); + if (addl == rvl_error) { goto _output_error; } + length += addl; + if (unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + } + length += MINMATCH; + +#if LZ4_FAST_DEC_LOOP + safe_match_copy: +#endif + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ + /* match starting within external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + assert(dictEnd != NULL); + if (unlikely(op+length > oend-LASTLITERALS)) { + if (partialDecoding) length = MIN(length, (size_t)(oend-op)); + else goto _output_error; /* doesn't respect parsing restriction */ + } + + if (length <= (size_t)(lowPrefix-match)) { + /* match fits entirely within external dictionary : just copy */ + LZ4_memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); + size_t const restSize = length - copySize; + LZ4_memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } else { + LZ4_memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } + assert(match >= lowPrefix); + + /* copy match within block */ + cpy = op + length; + + /* partialDecoding : may end anywhere within the block */ + assert(op<=oend); + if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + size_t const mlen = MIN(length, (size_t)(oend-op)); + const BYTE* const matchEnd = match + mlen; + BYTE* const copyEnd = op + mlen; + if (matchEnd > op) { /* overlap copy */ + while (op < copyEnd) { *op++ = *match++; } + } else { + LZ4_memcpy(op, match, mlen); + } + op = copyEnd; + if (op == oend) { break; } + continue; + } + + if (unlikely(offset<8)) { + LZ4_write32(op, 0); /* silence msan warning when offset==0 */ + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += inc32table[offset]; + LZ4_memcpy(op+4, match, 4); + match -= dec64table[offset]; + } else { + LZ4_memcpy(op, match, 8); + match += 8; + } + op += 8; + + if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1); + if (cpy > oend-LASTLITERALS) { goto _output_error; } /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ + if (op < oCopyLimit) { + LZ4_wildCopy8(op, match, oCopyLimit); + match += oCopyLimit - op; + op = oCopyLimit; + } + while (op < cpy) { *op++ = *match++; } + } else { + LZ4_memcpy(op, match, 8); + if (length > 16) { LZ4_wildCopy8(op+8, match+8, cpy); } + } + op = cpy; /* wildcopy correction */ + } + + /* end of decoding */ + DEBUGLOG(5, "decoded %i bytes", (int) (((char*)op)-dst)); + return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ + + /* Overflow error detected */ + _output_error: + return (int) (-(((const char*)ip)-src))-1; + } +} + + +/*===== Instantiate the API decoding functions. =====*/ + +LZ4_FORCE_O2 +int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, + decode_full_block, noDict, + (BYTE*)dest, NULL, 0); +} + +LZ4_FORCE_O2 +int LZ4_decompress_safe_partial(const char* src, char* dst, int compressedSize, int targetOutputSize, int dstCapacity) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(src, dst, compressedSize, dstCapacity, + partial_decode, + noDict, (BYTE*)dst, NULL, 0); +} + +LZ4_FORCE_O2 +int LZ4_decompress_fast(const char* source, char* dest, int originalSize) +{ + DEBUGLOG(5, "LZ4_decompress_fast"); + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + 0, NULL, 0); +} + +/*===== Instantiate a few more decoding cases, used more than once. =====*/ + +LZ4_FORCE_O2 /* Exported, an obsolete API function. */ +int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + decode_full_block, withPrefix64k, + (BYTE*)dest - 64 KB, NULL, 0); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_safe_partial_withPrefix64k(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity, + partial_decode, withPrefix64k, + (BYTE*)dest - 64 KB, NULL, 0); +} + +/* Another obsolete API function, paired with the previous one. */ +int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) +{ + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + 64 KB, NULL, 0); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, int compressedSize, int maxOutputSize, + size_t prefixSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + decode_full_block, noDict, + (BYTE*)dest-prefixSize, NULL, 0); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_safe_partial_withSmallPrefix(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity, + size_t prefixSize) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity, + partial_decode, noDict, + (BYTE*)dest-prefixSize, NULL, 0); +} + +LZ4_FORCE_O2 +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, + int compressedSize, int maxOutputSize, + const void* dictStart, size_t dictSize) +{ + DEBUGLOG(5, "LZ4_decompress_safe_forceExtDict"); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + decode_full_block, usingExtDict, + (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +LZ4_FORCE_O2 +int LZ4_decompress_safe_partial_forceExtDict(const char* source, char* dest, + int compressedSize, int targetOutputSize, int dstCapacity, + const void* dictStart, size_t dictSize) +{ + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(source, dest, compressedSize, dstCapacity, + partial_decode, usingExtDict, + (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_fast_extDict(const char* source, char* dest, int originalSize, + const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + 0, (const BYTE*)dictStart, dictSize); +} + +/* The "double dictionary" mode, for use with e.g. ring buffers: the first part + * of the dictionary is passed as prefix, and the second via dictStart + dictSize. + * These routines are used only once, in LZ4_decompress_*_continue(). + */ +LZ4_FORCE_INLINE +int LZ4_decompress_safe_doubleDict(const char* source, char* dest, int compressedSize, int maxOutputSize, + size_t prefixSize, const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + decode_full_block, usingExtDict, + (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize); +} + +/*===== streaming decompression functions =====*/ + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4_streamDecode_t* LZ4_createStreamDecode(void) +{ + LZ4_STATIC_ASSERT(sizeof(LZ4_streamDecode_t) >= sizeof(LZ4_streamDecode_t_internal)); + return (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t)); +} + +int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) +{ + if (LZ4_stream == NULL) { return 0; } /* support free on NULL */ + FREEMEM(LZ4_stream); + return 0; +} +#endif + +/*! LZ4_setStreamDecode() : + * Use this function to instruct where to find the dictionary. + * This function is not necessary if previous data is still available where it was decoded. + * Loading a size of 0 is allowed (same effect as no dictionary). + * @return : 1 if OK, 0 if error + */ +int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) +{ + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + lz4sd->prefixSize = (size_t)dictSize; + if (dictSize) { + assert(dictionary != NULL); + lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; + } else { + lz4sd->prefixEnd = (const BYTE*) dictionary; + } + lz4sd->externalDict = NULL; + lz4sd->extDictSize = 0; + return 1; +} + +/*! LZ4_decoderRingBufferSize() : + * when setting a ring buffer for streaming decompression (optional scenario), + * provides the minimum size of this ring buffer + * to be compatible with any source respecting maxBlockSize condition. + * Note : in a ring buffer scenario, + * blocks are presumed decompressed next to each other. + * When not enough space remains for next block (remainingSize < maxBlockSize), + * decoding resumes from beginning of ring buffer. + * @return : minimum ring buffer size, + * or 0 if there is an error (invalid maxBlockSize). + */ +int LZ4_decoderRingBufferSize(int maxBlockSize) +{ + if (maxBlockSize < 0) return 0; + if (maxBlockSize > LZ4_MAX_INPUT_SIZE) return 0; + if (maxBlockSize < 16) maxBlockSize = 16; + return LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize); +} + +/* +*_continue() : + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks must still be available at the memory position where they were decoded. + If it's not possible, save the relevant part of decoded data into a safe buffer, + and indicate where it stands using LZ4_setStreamDecode() +*/ +LZ4_FORCE_O2 +int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; + int result; + + if (lz4sd->prefixSize == 0) { + /* The first call, no dictionary yet. */ + assert(lz4sd->extDictSize == 0); + result = LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)result; + lz4sd->prefixEnd = (BYTE*)dest + result; + } else if (lz4sd->prefixEnd == (BYTE*)dest) { + /* They're rolling the current segment. */ + if (lz4sd->prefixSize >= 64 KB - 1) + result = LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); + else if (lz4sd->extDictSize == 0) + result = LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, + lz4sd->prefixSize); + else + result = LZ4_decompress_safe_doubleDict(source, dest, compressedSize, maxOutputSize, + lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += (size_t)result; + lz4sd->prefixEnd += result; + } else { + /* The buffer wraps around, or they're switching to another buffer. */ + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, + lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)result; + lz4sd->prefixEnd = (BYTE*)dest + result; + } + + return result; +} + +LZ4_FORCE_O2 int +LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, + const char* source, char* dest, int originalSize) +{ + LZ4_streamDecode_t_internal* const lz4sd = + (assert(LZ4_streamDecode!=NULL), &LZ4_streamDecode->internal_donotuse); + int result; + + DEBUGLOG(5, "LZ4_decompress_fast_continue (toDecodeSize=%i)", originalSize); + assert(originalSize >= 0); + + if (lz4sd->prefixSize == 0) { + DEBUGLOG(5, "first invocation : no prefix nor extDict"); + assert(lz4sd->extDictSize == 0); + result = LZ4_decompress_fast(source, dest, originalSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; + } else if (lz4sd->prefixEnd == (BYTE*)dest) { + DEBUGLOG(5, "continue using existing prefix"); + result = LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + lz4sd->prefixSize, + lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += (size_t)originalSize; + lz4sd->prefixEnd += originalSize; + } else { + DEBUGLOG(5, "prefix becomes extDict"); + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_fast_extDict(source, dest, originalSize, + lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = (size_t)originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; + } + + return result; +} + + +/* +Advanced decoding functions : +*_usingDict() : + These decoding functions work the same as "_continue" ones, + the dictionary must be explicitly provided within parameters +*/ + +int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + if (dictSize==0) + return LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); + if (dictStart+dictSize == dest) { + if (dictSize >= 64 KB - 1) { + return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, (size_t)dictSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, (size_t)dictSize); +} + +int LZ4_decompress_safe_partial_usingDict(const char* source, char* dest, int compressedSize, int targetOutputSize, int dstCapacity, const char* dictStart, int dictSize) +{ + if (dictSize==0) + return LZ4_decompress_safe_partial(source, dest, compressedSize, targetOutputSize, dstCapacity); + if (dictStart+dictSize == dest) { + if (dictSize >= 64 KB - 1) { + return LZ4_decompress_safe_partial_withPrefix64k(source, dest, compressedSize, targetOutputSize, dstCapacity); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_partial_withSmallPrefix(source, dest, compressedSize, targetOutputSize, dstCapacity, (size_t)dictSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_partial_forceExtDict(source, dest, compressedSize, targetOutputSize, dstCapacity, dictStart, (size_t)dictSize); +} + +int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) +{ + if (dictSize==0 || dictStart+dictSize == dest) + return LZ4_decompress_unsafe_generic( + (const BYTE*)source, (BYTE*)dest, originalSize, + (size_t)dictSize, NULL, 0); + assert(dictSize >= 0); + return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, (size_t)dictSize); +} + + +/*=************************************************* +* Obsolete Functions +***************************************************/ +/* obsolete compression functions */ +int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4_compress_default(source, dest, inputSize, maxOutputSize); +} +int LZ4_compress(const char* src, char* dest, int srcSize) +{ + return LZ4_compress_default(src, dest, srcSize, LZ4_compressBound(srcSize)); +} +int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) +{ + return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); +} +int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) +{ + return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); +} +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int dstCapacity) +{ + return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, dstCapacity, 1); +} +int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) +{ + return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); +} + +/* +These decompression functions are deprecated and should no longer be used. +They are only provided here for compatibility with older user programs. +- LZ4_uncompress is totally equivalent to LZ4_decompress_fast +- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe +*/ +int LZ4_uncompress (const char* source, char* dest, int outputSize) +{ + return LZ4_decompress_fast(source, dest, outputSize); +} +int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) +{ + return LZ4_decompress_safe(source, dest, isize, maxOutputSize); +} + +/* Obsolete Streaming functions */ + +int LZ4_sizeofStreamState(void) { return sizeof(LZ4_stream_t); } + +int LZ4_resetStreamState(void* state, char* inputBuffer) +{ + (void)inputBuffer; + LZ4_resetStream((LZ4_stream_t*)state); + return 0; +} + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +void* LZ4_create (char* inputBuffer) +{ + (void)inputBuffer; + return LZ4_createStream(); +} +#endif + +char* LZ4_slideInputBuffer (void* state) +{ + /* avoid const char * -> char * conversion warning */ + return (char *)(uptrval)((LZ4_stream_t*)state)->internal_donotuse.dictionary; +} + +#endif /* LZ4_COMMONDEFS_ONLY */ diff --git a/example/android/third_party/lz4/include/lz4.h b/example/android/third_party/lz4/include/lz4.h new file mode 100644 index 00000000..5c799723 --- /dev/null +++ b/example/android/third_party/lz4/include/lz4.h @@ -0,0 +1,879 @@ +/* + * LZ4 - Fast LZ compression algorithm + * Header File + * Copyright (C) 2011-2023, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 homepage : http://www.lz4.org + - LZ4 source repository : https://github.com/lz4/lz4 +*/ +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef LZ4_H_2983827168210 +#define LZ4_H_2983827168210 + +/* --- Dependency --- */ +#include /* size_t */ + + +/** + Introduction + + LZ4 is lossless compression algorithm, providing compression speed >500 MB/s per core, + scalable with multi-cores CPU. It features an extremely fast decoder, with speed in + multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. + + The LZ4 compression library provides in-memory compression and decompression functions. + It gives full buffer control to user. + Compression can be done in: + - a single step (described as Simple Functions) + - a single step, reusing a context (described in Advanced Functions) + - unbounded multiple steps (described as Streaming compression) + + lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md). + Decompressing such a compressed block requires additional metadata. + Exact metadata depends on exact decompression function. + For the typical case of LZ4_decompress_safe(), + metadata includes block's compressed size, and maximum bound of decompressed size. + Each application is free to encode and pass such metadata in whichever way it wants. + + lz4.h only handle blocks, it can not generate Frames. + + Blocks are different from Frames (doc/lz4_Frame_format.md). + Frames bundle both blocks and metadata in a specified manner. + Embedding metadata is required for compressed data to be self-contained and portable. + Frame format is delivered through a companion API, declared in lz4frame.h. + The `lz4` CLI can only manage frames. +*/ + +/*^*************************************************************** +* Export parameters +*****************************************************************/ +/* +* LZ4_DLL_EXPORT : +* Enable exporting of functions when building a Windows DLL +* LZ4LIB_VISIBILITY : +* Control library symbols visibility. +*/ +#ifndef LZ4LIB_VISIBILITY +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4LIB_VISIBILITY __attribute__ ((visibility ("default"))) +# else +# define LZ4LIB_VISIBILITY +# endif +#endif +#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) +# define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY +#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) +# define LZ4LIB_API __declspec(dllimport) LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define LZ4LIB_API LZ4LIB_VISIBILITY +#endif + +/*! LZ4_FREESTANDING : + * When this macro is set to 1, it enables "freestanding mode" that is + * suitable for typical freestanding environment which doesn't support + * standard C library. + * + * - LZ4_FREESTANDING is a compile-time switch. + * - It requires the following macros to be defined: + * LZ4_memcpy, LZ4_memmove, LZ4_memset. + * - It only enables LZ4/HC functions which don't use heap. + * All LZ4F_* functions are not supported. + * - See tests/freestanding.c to check its basic setup. + */ +#if defined(LZ4_FREESTANDING) && (LZ4_FREESTANDING == 1) +# define LZ4_HEAPMODE 0 +# define LZ4HC_HEAPMODE 0 +# define LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION 1 +# if !defined(LZ4_memcpy) +# error "LZ4_FREESTANDING requires macro 'LZ4_memcpy'." +# endif +# if !defined(LZ4_memset) +# error "LZ4_FREESTANDING requires macro 'LZ4_memset'." +# endif +# if !defined(LZ4_memmove) +# error "LZ4_FREESTANDING requires macro 'LZ4_memmove'." +# endif +#elif ! defined(LZ4_FREESTANDING) +# define LZ4_FREESTANDING 0 +#endif + + +/*------ Version ------*/ +#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ +#define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */ +#define LZ4_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */ + +#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) + +#define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE +#define LZ4_QUOTE(str) #str +#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) +#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) /* requires v1.7.3+ */ + +LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version; requires v1.3.0+ */ +LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version; requires v1.7.5+ */ + + +/*-************************************ +* Tuning memory usage +**************************************/ +/*! + * LZ4_MEMORY_USAGE : + * Can be selected at compile time, by setting LZ4_MEMORY_USAGE. + * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB) + * Increasing memory usage improves compression ratio, generally at the cost of speed. + * Reduced memory usage may improve speed at the cost of ratio, thanks to better cache locality. + * Default value is 14, for 16KB, which nicely fits into most L1 caches. + */ +#ifndef LZ4_MEMORY_USAGE +# define LZ4_MEMORY_USAGE LZ4_MEMORY_USAGE_DEFAULT +#endif + +/* These are absolute limits, they should not be changed by users */ +#define LZ4_MEMORY_USAGE_MIN 10 +#define LZ4_MEMORY_USAGE_DEFAULT 14 +#define LZ4_MEMORY_USAGE_MAX 20 + +#if (LZ4_MEMORY_USAGE < LZ4_MEMORY_USAGE_MIN) +# error "LZ4_MEMORY_USAGE is too small !" +#endif + +#if (LZ4_MEMORY_USAGE > LZ4_MEMORY_USAGE_MAX) +# error "LZ4_MEMORY_USAGE is too large !" +#endif + +/*-************************************ +* Simple Functions +**************************************/ +/*! LZ4_compress_default() : + * Compresses 'srcSize' bytes from buffer 'src' + * into already allocated 'dst' buffer of size 'dstCapacity'. + * Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). + * It also runs faster, so it's a recommended setting. + * If the function cannot compress 'src' into a more limited 'dst' budget, + * compression stops *immediately*, and the function result is zero. + * In which case, 'dst' content is undefined (invalid). + * srcSize : max supported value is LZ4_MAX_INPUT_SIZE. + * dstCapacity : size of buffer 'dst' (which must be already allocated) + * @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) + * or 0 if compression fails + * Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). + */ +LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); + +/*! LZ4_decompress_safe() : + * @compressedSize : is the exact complete size of the compressed block. + * @dstCapacity : is the size of destination buffer (which must be already allocated), + * presumed an upper bound of decompressed size. + * @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + * If destination buffer is not large enough, decoding will stop and output an error code (negative value). + * If the source stream is detected malformed, the function will stop decoding and return a negative result. + * Note 1 : This function is protected against malicious data packets : + * it will never writes outside 'dst' buffer, nor read outside 'source' buffer, + * even if the compressed block is maliciously modified to order the decoder to do these actions. + * In such case, the decoder stops immediately, and considers the compressed block malformed. + * Note 2 : compressedSize and dstCapacity must be provided to the function, the compressed block does not contain them. + * The implementation is free to send / store / derive this information in whichever way is most beneficial. + * If there is a need for a different format which bundles together both compressed data and its metadata, consider looking at lz4frame.h instead. + */ +LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); + + +/*-************************************ +* Advanced Functions +**************************************/ +#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ +#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) + +/*! LZ4_compressBound() : + Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) + This function is primarily useful for memory allocation purposes (destination buffer size). + Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). + Note that LZ4_compress_default() compresses faster when dstCapacity is >= LZ4_compressBound(srcSize) + inputSize : max supported value is LZ4_MAX_INPUT_SIZE + return : maximum output size in a "worst case" scenario + or 0, if input size is incorrect (too large or negative) +*/ +LZ4LIB_API int LZ4_compressBound(int inputSize); + +/*! LZ4_compress_fast() : + Same as LZ4_compress_default(), but allows selection of "acceleration" factor. + The larger the acceleration value, the faster the algorithm, but also the lesser the compression. + It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. + An acceleration value of "1" is the same as regular LZ4_compress_default() + Values <= 0 will be replaced by LZ4_ACCELERATION_DEFAULT (currently == 1, see lz4.c). + Values > LZ4_ACCELERATION_MAX will be replaced by LZ4_ACCELERATION_MAX (currently == 65537, see lz4.c). +*/ +LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + + +/*! LZ4_compress_fast_extState() : + * Same as LZ4_compress_fast(), using an externally allocated memory space for its state. + * Use LZ4_sizeofState() to know how much memory must be allocated, + * and allocate it on 8-bytes boundaries (using `malloc()` typically). + * Then, provide this buffer as `void* state` to compression function. + */ +LZ4LIB_API int LZ4_sizeofState(void); +LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + +/*! LZ4_compress_destSize() : + * Reverse the logic : compresses as much data as possible from 'src' buffer + * into already allocated buffer 'dst', of size >= 'dstCapacity'. + * This function either compresses the entire 'src' content into 'dst' if it's large enough, + * or fill 'dst' buffer completely with as much data as possible from 'src'. + * note: acceleration parameter is fixed to "default". + * + * *srcSizePtr : in+out parameter. Initially contains size of input. + * Will be modified to indicate how many bytes where read from 'src' to fill 'dst'. + * New value is necessarily <= input value. + * @return : Nb bytes written into 'dst' (necessarily <= dstCapacity) + * or 0 if compression fails. + * + * Note : from v1.8.2 to v1.9.1, this function had a bug (fixed in v1.9.2+): + * the produced compressed content could, in specific circumstances, + * require to be decompressed into a destination buffer larger + * by at least 1 byte than the content to decompress. + * If an application uses `LZ4_compress_destSize()`, + * it's highly recommended to update liblz4 to v1.9.2 or better. + * If this can't be done or ensured, + * the receiving decompression function should provide + * a dstCapacity which is > decompressedSize, by at least 1 byte. + * See https://github.com/lz4/lz4/issues/859 for details + */ +LZ4LIB_API int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize); + +/*! LZ4_decompress_safe_partial() : + * Decompress an LZ4 compressed block, of size 'srcSize' at position 'src', + * into destination buffer 'dst' of size 'dstCapacity'. + * Up to 'targetOutputSize' bytes will be decoded. + * The function stops decoding on reaching this objective. + * This can be useful to boost performance + * whenever only the beginning of a block is required. + * + * @return : the number of bytes decoded in `dst` (necessarily <= targetOutputSize) + * If source stream is detected malformed, function returns a negative result. + * + * Note 1 : @return can be < targetOutputSize, if compressed block contains less data. + * + * Note 2 : targetOutputSize must be <= dstCapacity + * + * Note 3 : this function effectively stops decoding on reaching targetOutputSize, + * so dstCapacity is kind of redundant. + * This is because in older versions of this function, + * decoding operation would still write complete sequences. + * Therefore, there was no guarantee that it would stop writing at exactly targetOutputSize, + * it could write more bytes, though only up to dstCapacity. + * Some "margin" used to be required for this operation to work properly. + * Thankfully, this is no longer necessary. + * The function nonetheless keeps the same signature, in an effort to preserve API compatibility. + * + * Note 4 : If srcSize is the exact size of the block, + * then targetOutputSize can be any value, + * including larger than the block's decompressed size. + * The function will, at most, generate block's decompressed size. + * + * Note 5 : If srcSize is _larger_ than block's compressed size, + * then targetOutputSize **MUST** be <= block's decompressed size. + * Otherwise, *silent corruption will occur*. + */ +LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity); + + +/*-********************************************* +* Streaming Compression Functions +***********************************************/ +typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ + +/*! + Note about RC_INVOKED + + - RC_INVOKED is predefined symbol of rc.exe (the resource compiler which is part of MSVC/Visual Studio). + https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros + + - Since rc.exe is a legacy compiler, it truncates long symbol (> 30 chars) + and reports warning "RC4011: identifier truncated". + + - To eliminate the warning, we surround long preprocessor symbol with + "#if !defined(RC_INVOKED) ... #endif" block that means + "skip this block when rc.exe is trying to read it". +*/ +#if !defined(RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */ +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); +LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); +#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */ +#endif + +/*! LZ4_resetStream_fast() : v1.9.0+ + * Use this to prepare an LZ4_stream_t for a new chain of dependent blocks + * (e.g., LZ4_compress_fast_continue()). + * + * An LZ4_stream_t must be initialized once before usage. + * This is automatically done when created by LZ4_createStream(). + * However, should the LZ4_stream_t be simply declared on stack (for example), + * it's necessary to initialize it first, using LZ4_initStream(). + * + * After init, start any new stream with LZ4_resetStream_fast(). + * A same LZ4_stream_t can be re-used multiple times consecutively + * and compress multiple streams, + * provided that it starts each new stream with LZ4_resetStream_fast(). + * + * LZ4_resetStream_fast() is much faster than LZ4_initStream(), + * but is not compatible with memory regions containing garbage data. + * + * Note: it's only useful to call LZ4_resetStream_fast() + * in the context of streaming compression. + * The *extState* functions perform their own resets. + * Invoking LZ4_resetStream_fast() before is redundant, and even counterproductive. + */ +LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr); + +/*! LZ4_loadDict() : + * Use this function to reference a static dictionary into LZ4_stream_t. + * The dictionary must remain available during compression. + * LZ4_loadDict() triggers a reset, so any previous data will be forgotten. + * The same dictionary will have to be loaded on decompression side for successful decoding. + * Dictionary are useful for better compression of small data (KB range). + * While LZ4 itself accepts any input as dictionary, dictionary efficiency is also a topic. + * When in doubt, employ the Zstandard's Dictionary Builder. + * Loading a size of 0 is allowed, and is the same as reset. + * @return : loaded dictionary size, in bytes (note: only the last 64 KB are loaded) + */ +LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); + +/*! LZ4_loadDictSlow() : v1.9.5+ + * Same as LZ4_loadDict(), + * but uses a bit more cpu to reference the dictionary content more thoroughly. + * This is expected to slightly improve compression ratio. + * The extra-cpu cost is likely worth it if the dictionary is re-used across multiple sessions. + * @return : loaded dictionary size, in bytes (note: only the last 64 KB are loaded) + */ +LZ4LIB_API int LZ4_loadDictSlow(LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); + +/*! LZ4_compress_fast_continue() : + * Compress 'src' content using data from previously compressed blocks, for better compression ratio. + * 'dst' buffer must be already allocated. + * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. + * + * @return : size of compressed block + * or 0 if there is an error (typically, cannot fit into 'dst'). + * + * Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block. + * Each block has precise boundaries. + * Each block must be decompressed separately, calling LZ4_decompress_*() with relevant metadata. + * It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together. + * + * Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory ! + * + * Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB. + * Make sure that buffers are separated, by at least one byte. + * This construction ensures that each block only depends on previous block. + * + * Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. + * + * Note 5 : After an error, the stream status is undefined (invalid), it can only be reset or freed. + */ +LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + +/*! LZ4_saveDict() : + * If last 64KB data cannot be guaranteed to remain available at its current memory location, + * save it into a safer place (char* safeBuffer). + * This is schematically equivalent to a memcpy() followed by LZ4_loadDict(), + * but is much faster, because LZ4_saveDict() doesn't need to rebuild tables. + * @return : saved dictionary size in bytes (necessarily <= maxDictSize), or 0 if error. + */ +LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int maxDictSize); + + +/*-********************************************** +* Streaming Decompression Functions +* Bufferless synchronous API +************************************************/ +typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* tracking context */ + +/*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() : + * creation / destruction of streaming decompression tracking context. + * A tracking context can be re-used multiple times. + */ +#if !defined(RC_INVOKED) /* https://docs.microsoft.com/en-us/windows/win32/menurc/predefined-macros */ +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); +LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); +#endif /* !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) */ +#endif + +/*! LZ4_setStreamDecode() : + * An LZ4_streamDecode_t context can be allocated once and re-used multiple times. + * Use this function to start decompression of a new stream of blocks. + * A dictionary can optionally be set. Use NULL or size 0 for a reset order. + * Dictionary is presumed stable : it must remain accessible and unmodified during next decompression. + * @return : 1 if OK, 0 if error + */ +LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); + +/*! LZ4_decoderRingBufferSize() : v1.8.2+ + * Note : in a ring buffer scenario (optional), + * blocks are presumed decompressed next to each other + * up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize), + * at which stage it resumes from beginning of ring buffer. + * When setting such a ring buffer for streaming decompression, + * provides the minimum size of this ring buffer + * to be compatible with any source respecting maxBlockSize condition. + * @return : minimum ring buffer size, + * or 0 if there is an error (invalid maxBlockSize). + */ +LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize); +#define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 14 + (maxBlockSize)) /* for static allocation; maxBlockSize presumed valid */ + +/*! LZ4_decompress_safe_continue() : + * This decoding function allows decompression of consecutive blocks in "streaming" mode. + * The difference with the usual independent blocks is that + * new blocks are allowed to find references into former blocks. + * A block is an unsplittable entity, and must be presented entirely to the decompression function. + * LZ4_decompress_safe_continue() only accepts one block at a time. + * It's modeled after `LZ4_decompress_safe()` and behaves similarly. + * + * @LZ4_streamDecode : decompression state, tracking the position in memory of past data + * @compressedSize : exact complete size of one compressed block. + * @dstCapacity : size of destination buffer (which must be already allocated), + * must be an upper bound of decompressed size. + * @return : number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + * If destination buffer is not large enough, decoding will stop and output an error code (negative value). + * If the source stream is detected malformed, the function will stop decoding and return a negative result. + * + * The last 64KB of previously decoded data *must* remain available and unmodified + * at the memory position where they were previously decoded. + * If less than 64KB of data has been decoded, all the data must be present. + * + * Special : if decompression side sets a ring buffer, it must respect one of the following conditions : + * - Decompression buffer size is _at least_ LZ4_decoderRingBufferSize(maxBlockSize). + * maxBlockSize is the maximum size of any single block. It can have any value > 16 bytes. + * In which case, encoding and decoding buffers do not need to be synchronized. + * Actually, data can be produced by any source compliant with LZ4 format specification, and respecting maxBlockSize. + * - Synchronized mode : + * Decompression buffer size is _exactly_ the same as compression buffer size, + * and follows exactly same update rule (block boundaries at same positions), + * and decoding function is provided with exact decompressed size of each block (exception for last block of the stream), + * _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB). + * - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes. + * In which case, encoding and decoding buffers do not need to be synchronized, + * and encoding ring buffer can have any size, including small ones ( < 64 KB). + * + * Whenever these conditions are not possible, + * save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression, + * then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block. +*/ +LZ4LIB_API int +LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, + const char* src, char* dst, + int srcSize, int dstCapacity); + + +/*! LZ4_decompress_safe_usingDict() : + * Works the same as + * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_safe_continue() + * However, it's stateless: it doesn't need any LZ4_streamDecode_t state. + * Dictionary is presumed stable : it must remain accessible and unmodified during decompression. + * Performance tip : Decompression speed can be substantially increased + * when dst == dictStart + dictSize. + */ +LZ4LIB_API int +LZ4_decompress_safe_usingDict(const char* src, char* dst, + int srcSize, int dstCapacity, + const char* dictStart, int dictSize); + +/*! LZ4_decompress_safe_partial_usingDict() : + * Behaves the same as LZ4_decompress_safe_partial() + * with the added ability to specify a memory segment for past data. + * Performance tip : Decompression speed can be substantially increased + * when dst == dictStart + dictSize. + */ +LZ4LIB_API int +LZ4_decompress_safe_partial_usingDict(const char* src, char* dst, + int compressedSize, + int targetOutputSize, int maxOutputSize, + const char* dictStart, int dictSize); + +#endif /* LZ4_H_2983827168210 */ + + +/*^************************************* + * !!!!!! STATIC LINKING ONLY !!!!!! + ***************************************/ + +/*-**************************************************************************** + * Experimental section + * + * Symbols declared in this section must be considered unstable. Their + * signatures or semantics may change, or they may be removed altogether in the + * future. They are therefore only safe to depend on when the caller is + * statically linked against the library. + * + * To protect against unsafe usage, not only are the declarations guarded, + * the definitions are hidden by default + * when building LZ4 as a shared/dynamic library. + * + * In order to access these declarations, + * define LZ4_STATIC_LINKING_ONLY in your application + * before including LZ4's headers. + * + * In order to make their implementations accessible dynamically, you must + * define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library. + ******************************************************************************/ + +#ifdef LZ4_STATIC_LINKING_ONLY + +#ifndef LZ4_STATIC_3504398509 +#define LZ4_STATIC_3504398509 + +#ifdef LZ4_PUBLISH_STATIC_FUNCTIONS +# define LZ4LIB_STATIC_API LZ4LIB_API +#else +# define LZ4LIB_STATIC_API +#endif + + +/*! LZ4_compress_fast_extState_fastReset() : + * A variant of LZ4_compress_fast_extState(). + * + * Using this variant avoids an expensive initialization step. + * It is only safe to call if the state buffer is known to be correctly initialized already + * (see above comment on LZ4_resetStream_fast() for a definition of "correctly initialized"). + * From a high level, the difference is that + * this function initializes the provided state with a call to something like LZ4_resetStream_fast() + * while LZ4_compress_fast_extState() starts with a call to LZ4_resetStream(). + */ +LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + +/*! LZ4_compress_destSize_extState() : + * Same as LZ4_compress_destSize(), but using an externally allocated state. + * Also: exposes @acceleration + */ +int LZ4_compress_destSize_extState(void* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize, int acceleration); + +/*! LZ4_attach_dictionary() : + * This is an experimental API that allows + * efficient use of a static dictionary many times. + * + * Rather than re-loading the dictionary buffer into a working context before + * each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a + * working LZ4_stream_t, this function introduces a no-copy setup mechanism, + * in which the working stream references the dictionary stream in-place. + * + * Several assumptions are made about the state of the dictionary stream. + * Currently, only streams which have been prepared by LZ4_loadDict() should + * be expected to work. + * + * Alternatively, the provided dictionaryStream may be NULL, + * in which case any existing dictionary stream is unset. + * + * If a dictionary is provided, it replaces any pre-existing stream history. + * The dictionary contents are the only history that can be referenced and + * logically immediately precede the data compressed in the first subsequent + * compression call. + * + * The dictionary will only remain attached to the working stream through the + * first compression call, at the end of which it is cleared. The dictionary + * stream (and source buffer) must remain in-place / accessible / unchanged + * through the completion of the first compression call on the stream. + */ +LZ4LIB_STATIC_API void +LZ4_attach_dictionary(LZ4_stream_t* workingStream, + const LZ4_stream_t* dictionaryStream); + + +/*! In-place compression and decompression + * + * It's possible to have input and output sharing the same buffer, + * for highly constrained memory environments. + * In both cases, it requires input to lay at the end of the buffer, + * and decompression to start at beginning of the buffer. + * Buffer size must feature some margin, hence be larger than final size. + * + * |<------------------------buffer--------------------------------->| + * |<-----------compressed data--------->| + * |<-----------decompressed size------------------>| + * |<----margin---->| + * + * This technique is more useful for decompression, + * since decompressed size is typically larger, + * and margin is short. + * + * In-place decompression will work inside any buffer + * which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize). + * This presumes that decompressedSize > compressedSize. + * Otherwise, it means compression actually expanded data, + * and it would be more efficient to store such data with a flag indicating it's not compressed. + * This can happen when data is not compressible (already compressed, or encrypted). + * + * For in-place compression, margin is larger, as it must be able to cope with both + * history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX, + * and data expansion, which can happen when input is not compressible. + * As a consequence, buffer size requirements are much higher, + * and memory savings offered by in-place compression are more limited. + * + * There are ways to limit this cost for compression : + * - Reduce history size, by modifying LZ4_DISTANCE_MAX. + * Note that it is a compile-time constant, so all compressions will apply this limit. + * Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX, + * so it's a reasonable trick when inputs are known to be small. + * - Require the compressor to deliver a "maximum compressed size". + * This is the `dstCapacity` parameter in `LZ4_compress*()`. + * When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail, + * in which case, the return code will be 0 (zero). + * The caller must be ready for these cases to happen, + * and typically design a backup scheme to send data uncompressed. + * The combination of both techniques can significantly reduce + * the amount of margin required for in-place compression. + * + * In-place compression can work in any buffer + * which size is >= (maxCompressedSize) + * with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success. + * LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX, + * so it's possible to reduce memory requirements by playing with them. + */ + +#define LZ4_DECOMPRESS_INPLACE_MARGIN(compressedSize) (((compressedSize) >> 8) + 32) +#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize) ((decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize)) /**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */ + +#ifndef LZ4_DISTANCE_MAX /* history window size; can be user-defined at compile time */ +# define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */ +#endif + +#define LZ4_COMPRESS_INPLACE_MARGIN (LZ4_DISTANCE_MAX + 32) /* LZ4_DISTANCE_MAX can be safely replaced by srcSize when it's smaller */ +#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize) ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN) /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */ + +#endif /* LZ4_STATIC_3504398509 */ +#endif /* LZ4_STATIC_LINKING_ONLY */ + + + +#ifndef LZ4_H_98237428734687 +#define LZ4_H_98237428734687 + +/*-************************************************************ + * Private Definitions + ************************************************************** + * Do not use these definitions directly. + * They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. + * Accessing members will expose user code to API and/or ABI break in future versions of the library. + **************************************************************/ +#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) +#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) +#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ + +#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# include + typedef int8_t LZ4_i8; + typedef uint8_t LZ4_byte; + typedef uint16_t LZ4_u16; + typedef uint32_t LZ4_u32; +#else + typedef signed char LZ4_i8; + typedef unsigned char LZ4_byte; + typedef unsigned short LZ4_u16; + typedef unsigned int LZ4_u32; +#endif + +/*! LZ4_stream_t : + * Never ever use below internal definitions directly ! + * These definitions are not API/ABI safe, and may change in future versions. + * If you need static allocation, declare or allocate an LZ4_stream_t object. +**/ + +typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; +struct LZ4_stream_t_internal { + LZ4_u32 hashTable[LZ4_HASH_SIZE_U32]; + const LZ4_byte* dictionary; + const LZ4_stream_t_internal* dictCtx; + LZ4_u32 currentOffset; + LZ4_u32 tableType; + LZ4_u32 dictSize; + /* Implicit padding to ensure structure is aligned */ +}; + +#define LZ4_STREAM_MINSIZE ((1UL << (LZ4_MEMORY_USAGE)) + 32) /* static size, for inter-version compatibility */ +union LZ4_stream_u { + char minStateSize[LZ4_STREAM_MINSIZE]; + LZ4_stream_t_internal internal_donotuse; +}; /* previously typedef'd to LZ4_stream_t */ + + +/*! LZ4_initStream() : v1.9.0+ + * An LZ4_stream_t structure must be initialized at least once. + * This is automatically done when invoking LZ4_createStream(), + * but it's not when the structure is simply declared on stack (for example). + * + * Use LZ4_initStream() to properly initialize a newly declared LZ4_stream_t. + * It can also initialize any arbitrary buffer of sufficient size, + * and will @return a pointer of proper type upon initialization. + * + * Note : initialization fails if size and alignment conditions are not respected. + * In which case, the function will @return NULL. + * Note2: An LZ4_stream_t structure guarantees correct alignment and size. + * Note3: Before v1.9.0, use LZ4_resetStream() instead +**/ +LZ4LIB_API LZ4_stream_t* LZ4_initStream (void* stateBuffer, size_t size); + + +/*! LZ4_streamDecode_t : + * Never ever use below internal definitions directly ! + * These definitions are not API/ABI safe, and may change in future versions. + * If you need static allocation, declare or allocate an LZ4_streamDecode_t object. +**/ +typedef struct { + const LZ4_byte* externalDict; + const LZ4_byte* prefixEnd; + size_t extDictSize; + size_t prefixSize; +} LZ4_streamDecode_t_internal; + +#define LZ4_STREAMDECODE_MINSIZE 32 +union LZ4_streamDecode_u { + char minStateSize[LZ4_STREAMDECODE_MINSIZE]; + LZ4_streamDecode_t_internal internal_donotuse; +} ; /* previously typedef'd to LZ4_streamDecode_t */ + + + +/*-************************************ +* Obsolete Functions +**************************************/ + +/*! Deprecation warnings + * + * Deprecated functions make the compiler generate a warning when invoked. + * This is meant to invite users to update their source code. + * Should deprecation warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc + * or _CRT_SECURE_NO_WARNINGS in Visual. + * + * Another method is to define LZ4_DISABLE_DEPRECATE_WARNINGS + * before including the header file. + */ +#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS +# define LZ4_DEPRECATED(message) /* disable deprecation warnings */ +#else +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define LZ4_DEPRECATED(message) [[deprecated(message)]] +# elif defined(_MSC_VER) +# define LZ4_DEPRECATED(message) __declspec(deprecated(message)) +# elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 45)) +# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 31) +# define LZ4_DEPRECATED(message) __attribute__((deprecated)) +# else +# pragma message("WARNING: LZ4_DEPRECATED needs custom implementation for this compiler") +# define LZ4_DEPRECATED(message) /* disabled */ +# endif +#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ + +/*! Obsolete compression functions (since v1.7.3) */ +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); + +/*! Obsolete decompression functions (since v1.8.0) */ +LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize); +LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); + +/* Obsolete streaming functions (since v1.7.0) + * degraded functionality; do not use! + * + * In order to perform streaming compression, these functions depended on data + * that is no longer tracked in the state. They have been preserved as well as + * possible: using them will still produce a correct output. However, they don't + * actually retain any history between compression calls. The compression ratio + * achieved will therefore be no better than compressing each chunk + * independently. + */ +LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer); +LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStreamState(void); +LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer); +LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state); + +/*! Obsolete streaming decoding functions (since v1.7.0) */ +LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); +LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); + +/*! Obsolete LZ4_decompress_fast variants (since v1.9.0) : + * These functions used to be faster than LZ4_decompress_safe(), + * but this is no longer the case. They are now slower. + * This is because LZ4_decompress_fast() doesn't know the input size, + * and therefore must progress more cautiously into the input buffer to not read beyond the end of block. + * On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability. + * As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated. + * + * The last remaining LZ4_decompress_fast() specificity is that + * it can decompress a block without knowing its compressed size. + * Such functionality can be achieved in a more secure manner + * by employing LZ4_decompress_safe_partial(). + * + * Parameters: + * originalSize : is the uncompressed size to regenerate. + * `dst` must be already allocated, its size must be >= 'originalSize' bytes. + * @return : number of bytes read from source buffer (== compressed size). + * The function expects to finish at block's end exactly. + * If the source stream is detected malformed, the function stops decoding and returns a negative result. + * note : LZ4_decompress_fast*() requires originalSize. Thanks to this information, it never writes past the output buffer. + * However, since it doesn't know its 'src' size, it may read an unknown amount of input, past input buffer bounds. + * Also, since match offsets are not validated, match reads from 'src' may underflow too. + * These issues never happen if input (compressed) data is correct. + * But they may happen if input data is invalid (error or intentional tampering). + * As a consequence, use these functions in trusted environments with trusted data **only**. + */ +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_partial() instead") +LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize); +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider migrating towards LZ4_decompress_safe_continue() instead. " + "Note that the contract will change (requires block's compressed size, instead of decompressed size)") +LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize); +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_partial_usingDict() instead") +LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize); + +/*! LZ4_resetStream() : + * An LZ4_stream_t structure must be initialized at least once. + * This is done with LZ4_initStream(), or LZ4_resetStream(). + * Consider switching to LZ4_initStream(), + * invoking LZ4_resetStream() will trigger deprecation warnings in the future. + */ +LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); + + +#endif /* LZ4_H_98237428734687 */ + + +#if defined (__cplusplus) +} +#endif diff --git a/example/android/third_party/lz4/include/lz4file.c b/example/android/third_party/lz4/include/lz4file.c new file mode 100644 index 00000000..a4197ea8 --- /dev/null +++ b/example/android/third_party/lz4/include/lz4file.c @@ -0,0 +1,341 @@ +/* + * LZ4 file library + * Copyright (C) 2022, Xiaomi Inc. + * + * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at : + * - LZ4 homepage : http://www.lz4.org + * - LZ4 source repository : https://github.com/lz4/lz4 + */ +#include /* malloc, free */ +#include +#include +#include "lz4.h" +#include "lz4file.h" + +static LZ4F_errorCode_t returnErrorCode(LZ4F_errorCodes code) +{ + return (LZ4F_errorCode_t)-(ptrdiff_t)code; +} +#undef RETURN_ERROR +#define RETURN_ERROR(e) return returnErrorCode(LZ4F_ERROR_ ## e) + +/* ===== read API ===== */ + +struct LZ4_readFile_s { + LZ4F_dctx* dctxPtr; + FILE* fp; + LZ4_byte* srcBuf; + size_t srcBufNext; + size_t srcBufSize; + size_t srcBufMaxSize; +}; + +static void LZ4F_freeReadFile(LZ4_readFile_t* lz4fRead) +{ + if (lz4fRead==NULL) return; + LZ4F_freeDecompressionContext(lz4fRead->dctxPtr); + free(lz4fRead->srcBuf); + free(lz4fRead); +} + +static void LZ4F_freeAndNullReadFile(LZ4_readFile_t** statePtr) +{ + assert(statePtr != NULL); + LZ4F_freeReadFile(*statePtr); + *statePtr = NULL; +} + +LZ4F_errorCode_t LZ4F_readOpen(LZ4_readFile_t** lz4fRead, FILE* fp) +{ + char buf[LZ4F_HEADER_SIZE_MAX]; + size_t consumedSize; + LZ4F_errorCode_t ret; + + if (fp == NULL || lz4fRead == NULL) { + RETURN_ERROR(parameter_null); + } + + *lz4fRead = (LZ4_readFile_t*)calloc(1, sizeof(LZ4_readFile_t)); + if (*lz4fRead == NULL) { + RETURN_ERROR(allocation_failed); + } + + ret = LZ4F_createDecompressionContext(&(*lz4fRead)->dctxPtr, LZ4F_VERSION); + if (LZ4F_isError(ret)) { + LZ4F_freeAndNullReadFile(lz4fRead); + return ret; + } + + (*lz4fRead)->fp = fp; + consumedSize = fread(buf, 1, sizeof(buf), (*lz4fRead)->fp); + if (consumedSize != sizeof(buf)) { + LZ4F_freeAndNullReadFile(lz4fRead); + RETURN_ERROR(io_read); + } + + { LZ4F_frameInfo_t info; + LZ4F_errorCode_t const r = LZ4F_getFrameInfo((*lz4fRead)->dctxPtr, &info, buf, &consumedSize); + if (LZ4F_isError(r)) { + LZ4F_freeAndNullReadFile(lz4fRead); + return r; + } + + switch (info.blockSizeID) { + case LZ4F_default : + case LZ4F_max64KB : + (*lz4fRead)->srcBufMaxSize = 64 * 1024; + break; + case LZ4F_max256KB: + (*lz4fRead)->srcBufMaxSize = 256 * 1024; + break; + case LZ4F_max1MB: + (*lz4fRead)->srcBufMaxSize = 1 * 1024 * 1024; + break; + case LZ4F_max4MB: + (*lz4fRead)->srcBufMaxSize = 4 * 1024 * 1024; + break; + default: + LZ4F_freeAndNullReadFile(lz4fRead); + RETURN_ERROR(maxBlockSize_invalid); + } + } + + (*lz4fRead)->srcBuf = (LZ4_byte*)malloc((*lz4fRead)->srcBufMaxSize); + if ((*lz4fRead)->srcBuf == NULL) { + LZ4F_freeAndNullReadFile(lz4fRead); + RETURN_ERROR(allocation_failed); + } + + (*lz4fRead)->srcBufSize = sizeof(buf) - consumedSize; + memcpy((*lz4fRead)->srcBuf, buf + consumedSize, (*lz4fRead)->srcBufSize); + + return ret; +} + +size_t LZ4F_read(LZ4_readFile_t* lz4fRead, void* buf, size_t size) +{ + LZ4_byte* p = (LZ4_byte*)buf; + size_t next = 0; + + if (lz4fRead == NULL || buf == NULL) + RETURN_ERROR(parameter_null); + + while (next < size) { + size_t srcsize = lz4fRead->srcBufSize - lz4fRead->srcBufNext; + size_t dstsize = size - next; + size_t ret; + + if (srcsize == 0) { + ret = fread(lz4fRead->srcBuf, 1, lz4fRead->srcBufMaxSize, lz4fRead->fp); + if (ret > 0) { + lz4fRead->srcBufSize = ret; + srcsize = lz4fRead->srcBufSize; + lz4fRead->srcBufNext = 0; + } else if (ret == 0) { + break; + } else { + RETURN_ERROR(io_read); + } + } + + ret = LZ4F_decompress(lz4fRead->dctxPtr, + p, &dstsize, + lz4fRead->srcBuf + lz4fRead->srcBufNext, + &srcsize, + NULL); + if (LZ4F_isError(ret)) { + return ret; + } + + lz4fRead->srcBufNext += srcsize; + next += dstsize; + p += dstsize; + } + + return next; +} + +LZ4F_errorCode_t LZ4F_readClose(LZ4_readFile_t* lz4fRead) +{ + if (lz4fRead == NULL) + RETURN_ERROR(parameter_null); + LZ4F_freeReadFile(lz4fRead); + return LZ4F_OK_NoError; +} + +/* ===== write API ===== */ + +struct LZ4_writeFile_s { + LZ4F_cctx* cctxPtr; + FILE* fp; + LZ4_byte* dstBuf; + size_t maxWriteSize; + size_t dstBufMaxSize; + LZ4F_errorCode_t errCode; +}; + +static void LZ4F_freeWriteFile(LZ4_writeFile_t* state) +{ + if (state == NULL) return; + LZ4F_freeCompressionContext(state->cctxPtr); + free(state->dstBuf); + free(state); +} + +static void LZ4F_freeAndNullWriteFile(LZ4_writeFile_t** statePtr) +{ + assert(statePtr != NULL); + LZ4F_freeWriteFile(*statePtr); + *statePtr = NULL; +} + +LZ4F_errorCode_t LZ4F_writeOpen(LZ4_writeFile_t** lz4fWrite, FILE* fp, const LZ4F_preferences_t* prefsPtr) +{ + LZ4_byte buf[LZ4F_HEADER_SIZE_MAX]; + size_t ret; + + if (fp == NULL || lz4fWrite == NULL) + RETURN_ERROR(parameter_null); + + *lz4fWrite = (LZ4_writeFile_t*)calloc(1, sizeof(LZ4_writeFile_t)); + if (*lz4fWrite == NULL) { + RETURN_ERROR(allocation_failed); + } + if (prefsPtr != NULL) { + switch (prefsPtr->frameInfo.blockSizeID) { + case LZ4F_default : + case LZ4F_max64KB : + (*lz4fWrite)->maxWriteSize = 64 * 1024; + break; + case LZ4F_max256KB: + (*lz4fWrite)->maxWriteSize = 256 * 1024; + break; + case LZ4F_max1MB: + (*lz4fWrite)->maxWriteSize = 1 * 1024 * 1024; + break; + case LZ4F_max4MB: + (*lz4fWrite)->maxWriteSize = 4 * 1024 * 1024; + break; + default: + LZ4F_freeAndNullWriteFile(lz4fWrite); + RETURN_ERROR(maxBlockSize_invalid); + } + } else { + (*lz4fWrite)->maxWriteSize = 64 * 1024; + } + + (*lz4fWrite)->dstBufMaxSize = LZ4F_compressBound((*lz4fWrite)->maxWriteSize, prefsPtr); + (*lz4fWrite)->dstBuf = (LZ4_byte*)malloc((*lz4fWrite)->dstBufMaxSize); + if ((*lz4fWrite)->dstBuf == NULL) { + LZ4F_freeAndNullWriteFile(lz4fWrite); + RETURN_ERROR(allocation_failed); + } + + ret = LZ4F_createCompressionContext(&(*lz4fWrite)->cctxPtr, LZ4F_VERSION); + if (LZ4F_isError(ret)) { + LZ4F_freeAndNullWriteFile(lz4fWrite); + return ret; + } + + ret = LZ4F_compressBegin((*lz4fWrite)->cctxPtr, buf, LZ4F_HEADER_SIZE_MAX, prefsPtr); + if (LZ4F_isError(ret)) { + LZ4F_freeAndNullWriteFile(lz4fWrite); + return ret; + } + + if (ret != fwrite(buf, 1, ret, fp)) { + LZ4F_freeAndNullWriteFile(lz4fWrite); + RETURN_ERROR(io_write); + } + + (*lz4fWrite)->fp = fp; + (*lz4fWrite)->errCode = LZ4F_OK_NoError; + return LZ4F_OK_NoError; +} + +size_t LZ4F_write(LZ4_writeFile_t* lz4fWrite, const void* buf, size_t size) +{ + const LZ4_byte* p = (const LZ4_byte*)buf; + size_t remain = size; + size_t chunk; + size_t ret; + + if (lz4fWrite == NULL || buf == NULL) + RETURN_ERROR(parameter_null); + while (remain) { + if (remain > lz4fWrite->maxWriteSize) + chunk = lz4fWrite->maxWriteSize; + else + chunk = remain; + + ret = LZ4F_compressUpdate(lz4fWrite->cctxPtr, + lz4fWrite->dstBuf, lz4fWrite->dstBufMaxSize, + p, chunk, + NULL); + if (LZ4F_isError(ret)) { + lz4fWrite->errCode = ret; + return ret; + } + + if (ret != fwrite(lz4fWrite->dstBuf, 1, ret, lz4fWrite->fp)) { + lz4fWrite->errCode = returnErrorCode(LZ4F_ERROR_io_write); + RETURN_ERROR(io_write); + } + + p += chunk; + remain -= chunk; + } + + return size; +} + +LZ4F_errorCode_t LZ4F_writeClose(LZ4_writeFile_t* lz4fWrite) +{ + LZ4F_errorCode_t ret = LZ4F_OK_NoError; + + if (lz4fWrite == NULL) { + RETURN_ERROR(parameter_null); + } + + if (lz4fWrite->errCode == LZ4F_OK_NoError) { + ret = LZ4F_compressEnd(lz4fWrite->cctxPtr, + lz4fWrite->dstBuf, lz4fWrite->dstBufMaxSize, + NULL); + if (LZ4F_isError(ret)) { + goto out; + } + + if (ret != fwrite(lz4fWrite->dstBuf, 1, ret, lz4fWrite->fp)) { + ret = returnErrorCode(LZ4F_ERROR_io_write); + } + } + +out: + LZ4F_freeWriteFile(lz4fWrite); + return ret; +} diff --git a/example/android/third_party/lz4/include/lz4file.h b/example/android/third_party/lz4/include/lz4file.h new file mode 100644 index 00000000..598ad705 --- /dev/null +++ b/example/android/third_party/lz4/include/lz4file.h @@ -0,0 +1,93 @@ +/* + LZ4 file library + Header File + Copyright (C) 2022, Xiaomi Inc. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/lz4/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef LZ4FILE_H +#define LZ4FILE_H + +#include /* FILE* */ +#include "lz4frame_static.h" + +typedef struct LZ4_readFile_s LZ4_readFile_t; +typedef struct LZ4_writeFile_s LZ4_writeFile_t; + +/*! LZ4F_readOpen() : + * Set read lz4file handle. + * `lz4f` will set a lz4file handle. + * `fp` must be the return value of the lz4 file opened by fopen. + */ +LZ4FLIB_STATIC_API LZ4F_errorCode_t LZ4F_readOpen(LZ4_readFile_t** lz4fRead, FILE* fp); + +/*! LZ4F_read() : + * Read lz4file content to buffer. + * `lz4f` must use LZ4_readOpen to set first. + * `buf` read data buffer. + * `size` read data buffer size. + */ +LZ4FLIB_STATIC_API size_t LZ4F_read(LZ4_readFile_t* lz4fRead, void* buf, size_t size); + +/*! LZ4F_readClose() : + * Close lz4file handle. + * `lz4f` must use LZ4_readOpen to set first. + */ +LZ4FLIB_STATIC_API LZ4F_errorCode_t LZ4F_readClose(LZ4_readFile_t* lz4fRead); + +/*! LZ4F_writeOpen() : + * Set write lz4file handle. + * `lz4f` will set a lz4file handle. + * `fp` must be the return value of the lz4 file opened by fopen. + */ +LZ4FLIB_STATIC_API LZ4F_errorCode_t LZ4F_writeOpen(LZ4_writeFile_t** lz4fWrite, FILE* fp, const LZ4F_preferences_t* prefsPtr); + +/*! LZ4F_write() : + * Write buffer to lz4file. + * `lz4f` must use LZ4F_writeOpen to set first. + * `buf` write data buffer. + * `size` write data buffer size. + */ +LZ4FLIB_STATIC_API size_t LZ4F_write(LZ4_writeFile_t* lz4fWrite, const void* buf, size_t size); + +/*! LZ4F_writeClose() : + * Close lz4file handle. + * `lz4f` must use LZ4F_writeOpen to set first. + */ +LZ4FLIB_STATIC_API LZ4F_errorCode_t LZ4F_writeClose(LZ4_writeFile_t* lz4fWrite); + +#endif /* LZ4FILE_H */ + +#if defined (__cplusplus) +} +#endif diff --git a/example/android/third_party/lz4/include/lz4frame.c b/example/android/third_party/lz4/include/lz4frame.c new file mode 100644 index 00000000..4eb9713f --- /dev/null +++ b/example/android/third_party/lz4/include/lz4frame.c @@ -0,0 +1,2134 @@ +/* + * LZ4 auto-framing library + * Copyright (C) 2011-2016, Yann Collet. + * + * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at : + * - LZ4 homepage : http://www.lz4.org + * - LZ4 source repository : https://github.com/lz4/lz4 + */ + +/* LZ4F is a stand-alone API to create LZ4-compressed Frames + * in full conformance with specification v1.6.1 . + * This library rely upon memory management capabilities (malloc, free) + * provided either by , + * or redirected towards another library of user's choice + * (see Memory Routines below). + */ + + +/*-************************************ +* Compiler Options +**************************************/ +#include +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + +/*-************************************ +* Tuning parameters +**************************************/ +/* + * LZ4F_HEAPMODE : + * Control how LZ4F_compressFrame allocates the Compression State, + * either on stack (0:default, fastest), or in memory heap (1:requires malloc()). + */ +#ifndef LZ4F_HEAPMODE +# define LZ4F_HEAPMODE 0 +#endif + + +/*-************************************ +* Library declarations +**************************************/ +#define LZ4F_STATIC_LINKING_ONLY +#include "lz4frame.h" +#define LZ4_STATIC_LINKING_ONLY +#include "lz4.h" +#define LZ4_HC_STATIC_LINKING_ONLY +#include "lz4hc.h" +#define XXH_STATIC_LINKING_ONLY +#include "xxhash.h" + + +/*-************************************ +* Memory routines +**************************************/ +/* + * User may redirect invocations of + * malloc(), calloc() and free() + * towards another library or solution of their choice + * by modifying below section. +**/ + +#include /* memset, memcpy, memmove */ +#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */ +# define MEM_INIT(p,v,s) memset((p),(v),(s)) +#endif + +#ifndef LZ4_SRC_INCLUDED /* avoid redefinition when sources are coalesced */ +# include /* malloc, calloc, free */ +# define ALLOC(s) malloc(s) +# define ALLOC_AND_ZERO(s) calloc(1,(s)) +# define FREEMEM(p) free(p) +#endif + +static void* LZ4F_calloc(size_t s, LZ4F_CustomMem cmem) +{ + /* custom calloc defined : use it */ + if (cmem.customCalloc != NULL) { + return cmem.customCalloc(cmem.opaqueState, s); + } + /* nothing defined : use default 's calloc() */ + if (cmem.customAlloc == NULL) { + return ALLOC_AND_ZERO(s); + } + /* only custom alloc defined : use it, and combine it with memset() */ + { void* const p = cmem.customAlloc(cmem.opaqueState, s); + if (p != NULL) MEM_INIT(p, 0, s); + return p; +} } + +static void* LZ4F_malloc(size_t s, LZ4F_CustomMem cmem) +{ + /* custom malloc defined : use it */ + if (cmem.customAlloc != NULL) { + return cmem.customAlloc(cmem.opaqueState, s); + } + /* nothing defined : use default 's malloc() */ + return ALLOC(s); +} + +static void LZ4F_free(void* p, LZ4F_CustomMem cmem) +{ + /* custom malloc defined : use it */ + if (cmem.customFree != NULL) { + cmem.customFree(cmem.opaqueState, p); + return; + } + /* nothing defined : use default 's free() */ + FREEMEM(p); +} + + +/*-************************************ +* Debug +**************************************/ +#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=1) +# include +#else +# ifndef assert +# define assert(condition) ((void)0) +# endif +#endif + +#define LZ4F_STATIC_ASSERT(c) { enum { LZ4F_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + +#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) && !defined(DEBUGLOG) +# include +static int g_debuglog_enable = 1; +# define DEBUGLOG(l, ...) { \ + if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ + fprintf(stderr, __FILE__ " (%i): ", __LINE__ ); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, " \n"); \ + } } +#else +# define DEBUGLOG(l, ...) {} /* disabled */ +#endif + + +/*-************************************ +* Basic Types +**************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + + +/* unoptimized version; solves endianness & alignment issues */ +static U32 LZ4F_readLE32 (const void* src) +{ + const BYTE* const srcPtr = (const BYTE*)src; + U32 value32 = srcPtr[0]; + value32 += ((U32)srcPtr[1])<< 8; + value32 += ((U32)srcPtr[2])<<16; + value32 += ((U32)srcPtr[3])<<24; + return value32; +} + +static void LZ4F_writeLE32 (void* dst, U32 value32) +{ + BYTE* const dstPtr = (BYTE*)dst; + dstPtr[0] = (BYTE)value32; + dstPtr[1] = (BYTE)(value32 >> 8); + dstPtr[2] = (BYTE)(value32 >> 16); + dstPtr[3] = (BYTE)(value32 >> 24); +} + +static U64 LZ4F_readLE64 (const void* src) +{ + const BYTE* const srcPtr = (const BYTE*)src; + U64 value64 = srcPtr[0]; + value64 += ((U64)srcPtr[1]<<8); + value64 += ((U64)srcPtr[2]<<16); + value64 += ((U64)srcPtr[3]<<24); + value64 += ((U64)srcPtr[4]<<32); + value64 += ((U64)srcPtr[5]<<40); + value64 += ((U64)srcPtr[6]<<48); + value64 += ((U64)srcPtr[7]<<56); + return value64; +} + +static void LZ4F_writeLE64 (void* dst, U64 value64) +{ + BYTE* const dstPtr = (BYTE*)dst; + dstPtr[0] = (BYTE)value64; + dstPtr[1] = (BYTE)(value64 >> 8); + dstPtr[2] = (BYTE)(value64 >> 16); + dstPtr[3] = (BYTE)(value64 >> 24); + dstPtr[4] = (BYTE)(value64 >> 32); + dstPtr[5] = (BYTE)(value64 >> 40); + dstPtr[6] = (BYTE)(value64 >> 48); + dstPtr[7] = (BYTE)(value64 >> 56); +} + + +/*-************************************ +* Constants +**************************************/ +#ifndef LZ4_SRC_INCLUDED /* avoid double definition */ +# define KB *(1<<10) +# define MB *(1<<20) +# define GB *(1<<30) +#endif + +#define _1BIT 0x01 +#define _2BITS 0x03 +#define _3BITS 0x07 +#define _4BITS 0x0F +#define _8BITS 0xFF + +#define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U +#define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB + +static const size_t minFHSize = LZ4F_HEADER_SIZE_MIN; /* 7 */ +static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 19 */ +static const size_t BHSize = LZ4F_BLOCK_HEADER_SIZE; /* block header : size, and compress flag */ +static const size_t BFSize = LZ4F_BLOCK_CHECKSUM_SIZE; /* block footer : checksum (optional) */ + + +/*-************************************ +* Structures and local types +**************************************/ + +typedef enum { LZ4B_COMPRESSED, LZ4B_UNCOMPRESSED} LZ4F_BlockCompressMode_e; +typedef enum { ctxNone, ctxFast, ctxHC } LZ4F_CtxType_e; + +typedef struct LZ4F_cctx_s +{ + LZ4F_CustomMem cmem; + LZ4F_preferences_t prefs; + U32 version; + U32 cStage; /* 0 : compression uninitialized ; 1 : initialized, can compress */ + const LZ4F_CDict* cdict; + size_t maxBlockSize; + size_t maxBufferSize; + BYTE* tmpBuff; /* internal buffer, for streaming */ + BYTE* tmpIn; /* starting position of data compress within internal buffer (>= tmpBuff) */ + size_t tmpInSize; /* amount of data to compress after tmpIn */ + U64 totalInSize; + XXH32_state_t xxh; + void* lz4CtxPtr; + U16 lz4CtxAlloc; /* sized for: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */ + U16 lz4CtxType; /* in use as: 0 = none, 1 = lz4 ctx, 2 = lz4hc ctx */ + LZ4F_BlockCompressMode_e blockCompressMode; +} LZ4F_cctx_t; + + +/*-************************************ +* Error management +**************************************/ +#define LZ4F_GENERATE_STRING(STRING) #STRING, +static const char* LZ4F_errorStrings[] = { LZ4F_LIST_ERRORS(LZ4F_GENERATE_STRING) }; + + +unsigned LZ4F_isError(LZ4F_errorCode_t code) +{ + return (code > (LZ4F_errorCode_t)(-LZ4F_ERROR_maxCode)); +} + +const char* LZ4F_getErrorName(LZ4F_errorCode_t code) +{ + static const char* codeError = "Unspecified error code"; + if (LZ4F_isError(code)) return LZ4F_errorStrings[-(int)(code)]; + return codeError; +} + +LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult) +{ + if (!LZ4F_isError(functionResult)) return LZ4F_OK_NoError; + return (LZ4F_errorCodes)(-(ptrdiff_t)functionResult); +} + +static LZ4F_errorCode_t LZ4F_returnErrorCode(LZ4F_errorCodes code) +{ + /* A compilation error here means sizeof(ptrdiff_t) is not large enough */ + LZ4F_STATIC_ASSERT(sizeof(ptrdiff_t) >= sizeof(size_t)); + return (LZ4F_errorCode_t)-(ptrdiff_t)code; +} + +#define RETURN_ERROR(e) return LZ4F_returnErrorCode(LZ4F_ERROR_ ## e) + +#define RETURN_ERROR_IF(c,e) do { \ + if (c) { \ + DEBUGLOG(3, "Error: " #c); \ + RETURN_ERROR(e); \ + } \ + } while (0) + +#define FORWARD_IF_ERROR(r) do { if (LZ4F_isError(r)) return (r); } while (0) + +unsigned LZ4F_getVersion(void) { return LZ4F_VERSION; } + +int LZ4F_compressionLevel_max(void) { return LZ4HC_CLEVEL_MAX; } + +size_t LZ4F_getBlockSize(LZ4F_blockSizeID_t blockSizeID) +{ + static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB }; + + if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; + if (blockSizeID < LZ4F_max64KB || blockSizeID > LZ4F_max4MB) + RETURN_ERROR(maxBlockSize_invalid); + { int const blockSizeIdx = (int)blockSizeID - (int)LZ4F_max64KB; + return blockSizes[blockSizeIdx]; +} } + +/*-************************************ +* Private functions +**************************************/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + +static BYTE LZ4F_headerChecksum (const void* header, size_t length) +{ + U32 const xxh = XXH32(header, length, 0); + return (BYTE)(xxh >> 8); +} + + +/*-************************************ +* Simple-pass compression functions +**************************************/ +static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID, + const size_t srcSize) +{ + LZ4F_blockSizeID_t proposedBSID = LZ4F_max64KB; + size_t maxBlockSize = 64 KB; + while (requestedBSID > proposedBSID) { + if (srcSize <= maxBlockSize) + return proposedBSID; + proposedBSID = (LZ4F_blockSizeID_t)((int)proposedBSID + 1); + maxBlockSize <<= 2; + } + return requestedBSID; +} + +/*! LZ4F_compressBound_internal() : + * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. + * prefsPtr is optional : if NULL is provided, preferences will be set to cover worst case scenario. + * @return is always the same for a srcSize and prefsPtr, so it can be relied upon to size reusable buffers. + * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. + */ +static size_t LZ4F_compressBound_internal(size_t srcSize, + const LZ4F_preferences_t* preferencesPtr, + size_t alreadyBuffered) +{ + LZ4F_preferences_t prefsNull = LZ4F_INIT_PREFERENCES; + prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ + prefsNull.frameInfo.blockChecksumFlag = LZ4F_blockChecksumEnabled; /* worst case */ + { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; + U32 const flush = prefsPtr->autoFlush | (srcSize==0); + LZ4F_blockSizeID_t const blockID = prefsPtr->frameInfo.blockSizeID; + size_t const blockSize = LZ4F_getBlockSize(blockID); + size_t const maxBuffered = blockSize - 1; + size_t const bufferedSize = MIN(alreadyBuffered, maxBuffered); + size_t const maxSrcSize = srcSize + bufferedSize; + unsigned const nbFullBlocks = (unsigned)(maxSrcSize / blockSize); + size_t const partialBlockSize = maxSrcSize & (blockSize-1); + size_t const lastBlockSize = flush ? partialBlockSize : 0; + unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0); + + size_t const blockCRCSize = BFSize * prefsPtr->frameInfo.blockChecksumFlag; + size_t const frameEnd = BHSize + (prefsPtr->frameInfo.contentChecksumFlag*BFSize); + + return ((BHSize + blockCRCSize) * nbBlocks) + + (blockSize * nbFullBlocks) + lastBlockSize + frameEnd; + } +} + +size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) +{ + LZ4F_preferences_t prefs; + size_t const headerSize = maxFHSize; /* max header size, including optional fields */ + + if (preferencesPtr!=NULL) prefs = *preferencesPtr; + else MEM_INIT(&prefs, 0, sizeof(prefs)); + prefs.autoFlush = 1; + + return headerSize + LZ4F_compressBound_internal(srcSize, &prefs, 0);; +} + + +/*! LZ4F_compressFrame_usingCDict() : + * Compress srcBuffer using a dictionary, in a single step. + * cdict can be NULL, in which case, no dictionary is used. + * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + * The LZ4F_preferences_t structure is optional : you may provide NULL as argument, + * however, it's the only way to provide a dictID, so it's not recommended. + * @return : number of bytes written into dstBuffer, + * or an error code if it fails (can be tested using LZ4F_isError()) + */ +size_t LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* preferencesPtr) +{ + LZ4F_preferences_t prefs; + LZ4F_compressOptions_t options; + BYTE* const dstStart = (BYTE*) dstBuffer; + BYTE* dstPtr = dstStart; + BYTE* const dstEnd = dstStart + dstCapacity; + + DEBUGLOG(4, "LZ4F_compressFrame_usingCDict (srcSize=%u)", (unsigned)srcSize); + if (preferencesPtr!=NULL) + prefs = *preferencesPtr; + else + MEM_INIT(&prefs, 0, sizeof(prefs)); + if (prefs.frameInfo.contentSize != 0) + prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */ + + prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); + prefs.autoFlush = 1; + if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID)) + prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* only one block => no need for inter-block link */ + + MEM_INIT(&options, 0, sizeof(options)); + options.stableSrc = 1; + + RETURN_ERROR_IF(dstCapacity < LZ4F_compressFrameBound(srcSize, &prefs), dstMaxSize_tooSmall); + + { size_t const headerSize = LZ4F_compressBegin_usingCDict(cctx, dstBuffer, dstCapacity, cdict, &prefs); /* write header */ + FORWARD_IF_ERROR(headerSize); + dstPtr += headerSize; /* header size */ } + + assert(dstEnd >= dstPtr); + { size_t const cSize = LZ4F_compressUpdate(cctx, dstPtr, (size_t)(dstEnd-dstPtr), srcBuffer, srcSize, &options); + FORWARD_IF_ERROR(cSize); + dstPtr += cSize; } + + assert(dstEnd >= dstPtr); + { size_t const tailSize = LZ4F_compressEnd(cctx, dstPtr, (size_t)(dstEnd-dstPtr), &options); /* flush last block, and generate suffix */ + FORWARD_IF_ERROR(tailSize); + dstPtr += tailSize; } + + assert(dstEnd >= dstStart); + return (size_t)(dstPtr - dstStart); +} + + +/*! LZ4F_compressFrame() : + * Compress an entire srcBuffer into a valid LZ4 frame, in a single step. + * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. + * @return : number of bytes written into dstBuffer. + * or an error code if it fails (can be tested using LZ4F_isError()) + */ +size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_preferences_t* preferencesPtr) +{ + size_t result; +#if (LZ4F_HEAPMODE) + LZ4F_cctx_t* cctxPtr; + result = LZ4F_createCompressionContext(&cctxPtr, LZ4F_VERSION); + FORWARD_IF_ERROR(result); +#else + LZ4F_cctx_t cctx; + LZ4_stream_t lz4ctx; + LZ4F_cctx_t* const cctxPtr = &cctx; + + MEM_INIT(&cctx, 0, sizeof(cctx)); + cctx.version = LZ4F_VERSION; + cctx.maxBufferSize = 5 MB; /* mess with real buffer size to prevent dynamic allocation; works only because autoflush==1 & stableSrc==1 */ + if ( preferencesPtr == NULL + || preferencesPtr->compressionLevel < LZ4HC_CLEVEL_MIN ) { + LZ4_initStream(&lz4ctx, sizeof(lz4ctx)); + cctxPtr->lz4CtxPtr = &lz4ctx; + cctxPtr->lz4CtxAlloc = 1; + cctxPtr->lz4CtxType = ctxFast; + } +#endif + DEBUGLOG(4, "LZ4F_compressFrame"); + + result = LZ4F_compressFrame_usingCDict(cctxPtr, dstBuffer, dstCapacity, + srcBuffer, srcSize, + NULL, preferencesPtr); + +#if (LZ4F_HEAPMODE) + LZ4F_freeCompressionContext(cctxPtr); +#else + if ( preferencesPtr != NULL + && preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN ) { + LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem); + } +#endif + return result; +} + + +/*-*************************************************** +* Dictionary compression +*****************************************************/ + +struct LZ4F_CDict_s { + LZ4F_CustomMem cmem; + void* dictContent; + LZ4_stream_t* fastCtx; + LZ4_streamHC_t* HCCtx; +}; /* typedef'd to LZ4F_CDict within lz4frame_static.h */ + +LZ4F_CDict* +LZ4F_createCDict_advanced(LZ4F_CustomMem cmem, const void* dictBuffer, size_t dictSize) +{ + const char* dictStart = (const char*)dictBuffer; + LZ4F_CDict* const cdict = (LZ4F_CDict*)LZ4F_malloc(sizeof(*cdict), cmem); + DEBUGLOG(4, "LZ4F_createCDict_advanced"); + if (!cdict) return NULL; + cdict->cmem = cmem; + if (dictSize > 64 KB) { + dictStart += dictSize - 64 KB; + dictSize = 64 KB; + } + cdict->dictContent = LZ4F_malloc(dictSize, cmem); + cdict->fastCtx = (LZ4_stream_t*)LZ4F_malloc(sizeof(LZ4_stream_t), cmem); + if (cdict->fastCtx) + LZ4_initStream(cdict->fastCtx, sizeof(LZ4_stream_t)); + cdict->HCCtx = (LZ4_streamHC_t*)LZ4F_malloc(sizeof(LZ4_streamHC_t), cmem); + if (cdict->HCCtx) + LZ4_initStreamHC(cdict->HCCtx, sizeof(LZ4_streamHC_t)); + if (!cdict->dictContent || !cdict->fastCtx || !cdict->HCCtx) { + LZ4F_freeCDict(cdict); + return NULL; + } + memcpy(cdict->dictContent, dictStart, dictSize); + LZ4_loadDictSlow(cdict->fastCtx, (const char*)cdict->dictContent, (int)dictSize); + LZ4_setCompressionLevel(cdict->HCCtx, LZ4HC_CLEVEL_DEFAULT); + LZ4_loadDictHC(cdict->HCCtx, (const char*)cdict->dictContent, (int)dictSize); + return cdict; +} + +/*! LZ4F_createCDict() : + * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. + * LZ4F_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. + * LZ4F_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * @dictBuffer can be released after LZ4F_CDict creation, since its content is copied within CDict + * @return : digested dictionary for compression, or NULL if failed */ +LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize) +{ + DEBUGLOG(4, "LZ4F_createCDict"); + return LZ4F_createCDict_advanced(LZ4F_defaultCMem, dictBuffer, dictSize); +} + +void LZ4F_freeCDict(LZ4F_CDict* cdict) +{ + if (cdict==NULL) return; /* support free on NULL */ + LZ4F_free(cdict->dictContent, cdict->cmem); + LZ4F_free(cdict->fastCtx, cdict->cmem); + LZ4F_free(cdict->HCCtx, cdict->cmem); + LZ4F_free(cdict, cdict->cmem); +} + + +/*-********************************* +* Advanced compression functions +***********************************/ + +LZ4F_cctx* +LZ4F_createCompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version) +{ + LZ4F_cctx* const cctxPtr = + (LZ4F_cctx*)LZ4F_calloc(sizeof(LZ4F_cctx), customMem); + if (cctxPtr==NULL) return NULL; + + cctxPtr->cmem = customMem; + cctxPtr->version = version; + cctxPtr->cStage = 0; /* Uninitialized. Next stage : init cctx */ + + return cctxPtr; +} + +/*! LZ4F_createCompressionContext() : + * The first thing to do is to create a compressionContext object, which will be used in all compression operations. + * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. + * The version provided MUST be LZ4F_VERSION. It is intended to track potential incompatible differences between different binaries. + * The function will provide a pointer to an allocated LZ4F_compressionContext_t object. + * If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. + * Object can release its memory using LZ4F_freeCompressionContext(); +**/ +LZ4F_errorCode_t +LZ4F_createCompressionContext(LZ4F_cctx** LZ4F_compressionContextPtr, unsigned version) +{ + assert(LZ4F_compressionContextPtr != NULL); /* considered a violation of narrow contract */ + /* in case it nonetheless happen in production */ + RETURN_ERROR_IF(LZ4F_compressionContextPtr == NULL, parameter_null); + + *LZ4F_compressionContextPtr = LZ4F_createCompressionContext_advanced(LZ4F_defaultCMem, version); + RETURN_ERROR_IF(*LZ4F_compressionContextPtr==NULL, allocation_failed); + return LZ4F_OK_NoError; +} + +LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctxPtr) +{ + if (cctxPtr != NULL) { /* support free on NULL */ + LZ4F_free(cctxPtr->lz4CtxPtr, cctxPtr->cmem); /* note: LZ4_streamHC_t and LZ4_stream_t are simple POD types */ + LZ4F_free(cctxPtr->tmpBuff, cctxPtr->cmem); + LZ4F_free(cctxPtr, cctxPtr->cmem); + } + return LZ4F_OK_NoError; +} + + +/** + * This function prepares the internal LZ4(HC) stream for a new compression, + * resetting the context and attaching the dictionary, if there is one. + * + * It needs to be called at the beginning of each independent compression + * stream (i.e., at the beginning of a frame in blockLinked mode, or at the + * beginning of each block in blockIndependent mode). + */ +static void LZ4F_initStream(void* ctx, + const LZ4F_CDict* cdict, + int level, + LZ4F_blockMode_t blockMode) { + if (level < LZ4HC_CLEVEL_MIN) { + if (cdict || blockMode == LZ4F_blockLinked) { + /* In these cases, we will call LZ4_compress_fast_continue(), + * which needs an already reset context. Otherwise, we'll call a + * one-shot API. The non-continued APIs internally perform their own + * resets at the beginning of their calls, where they know what + * tableType they need the context to be in. So in that case this + * would be misguided / wasted work. */ + LZ4_resetStream_fast((LZ4_stream_t*)ctx); + if (cdict) + LZ4_attach_dictionary((LZ4_stream_t*)ctx, cdict->fastCtx); + } + /* In these cases, we'll call a one-shot API. + * The non-continued APIs internally perform their own resets + * at the beginning of their calls, where they know + * which tableType they need the context to be in. + * Therefore, a reset here would be wasted work. */ + } else { + LZ4_resetStreamHC_fast((LZ4_streamHC_t*)ctx, level); + if (cdict) + LZ4_attach_HC_dictionary((LZ4_streamHC_t*)ctx, cdict->HCCtx); + } +} + +static int ctxTypeID_to_size(int ctxTypeID) { + switch(ctxTypeID) { + case 1: + return LZ4_sizeofState(); + case 2: + return LZ4_sizeofStateHC(); + default: + return 0; + } +} + +/* LZ4F_compressBegin_internal() + * Note: only accepts @cdict _or_ @dictBuffer as non NULL. + */ +size_t LZ4F_compressBegin_internal(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const void* dictBuffer, size_t dictSize, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* preferencesPtr) +{ + LZ4F_preferences_t const prefNull = LZ4F_INIT_PREFERENCES; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + + RETURN_ERROR_IF(dstCapacity < maxFHSize, dstMaxSize_tooSmall); + if (preferencesPtr == NULL) preferencesPtr = &prefNull; + cctx->prefs = *preferencesPtr; + + /* cctx Management */ + { U16 const ctxTypeID = (cctx->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2; + int requiredSize = ctxTypeID_to_size(ctxTypeID); + int allocatedSize = ctxTypeID_to_size(cctx->lz4CtxAlloc); + if (allocatedSize < requiredSize) { + /* not enough space allocated */ + LZ4F_free(cctx->lz4CtxPtr, cctx->cmem); + if (cctx->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { + /* must take ownership of memory allocation, + * in order to respect custom allocator contract */ + cctx->lz4CtxPtr = LZ4F_malloc(sizeof(LZ4_stream_t), cctx->cmem); + if (cctx->lz4CtxPtr) + LZ4_initStream(cctx->lz4CtxPtr, sizeof(LZ4_stream_t)); + } else { + cctx->lz4CtxPtr = LZ4F_malloc(sizeof(LZ4_streamHC_t), cctx->cmem); + if (cctx->lz4CtxPtr) + LZ4_initStreamHC(cctx->lz4CtxPtr, sizeof(LZ4_streamHC_t)); + } + RETURN_ERROR_IF(cctx->lz4CtxPtr == NULL, allocation_failed); + cctx->lz4CtxAlloc = ctxTypeID; + cctx->lz4CtxType = ctxTypeID; + } else if (cctx->lz4CtxType != ctxTypeID) { + /* otherwise, a sufficient buffer is already allocated, + * but we need to reset it to the correct context type */ + if (cctx->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) { + LZ4_initStream((LZ4_stream_t*)cctx->lz4CtxPtr, sizeof(LZ4_stream_t)); + } else { + LZ4_initStreamHC((LZ4_streamHC_t*)cctx->lz4CtxPtr, sizeof(LZ4_streamHC_t)); + LZ4_setCompressionLevel((LZ4_streamHC_t*)cctx->lz4CtxPtr, cctx->prefs.compressionLevel); + } + cctx->lz4CtxType = ctxTypeID; + } } + + /* Buffer Management */ + if (cctx->prefs.frameInfo.blockSizeID == 0) + cctx->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; + cctx->maxBlockSize = LZ4F_getBlockSize(cctx->prefs.frameInfo.blockSizeID); + + { size_t const requiredBuffSize = preferencesPtr->autoFlush ? + ((cctx->prefs.frameInfo.blockMode == LZ4F_blockLinked) ? 64 KB : 0) : /* only needs past data up to window size */ + cctx->maxBlockSize + ((cctx->prefs.frameInfo.blockMode == LZ4F_blockLinked) ? 128 KB : 0); + + if (cctx->maxBufferSize < requiredBuffSize) { + cctx->maxBufferSize = 0; + LZ4F_free(cctx->tmpBuff, cctx->cmem); + cctx->tmpBuff = (BYTE*)LZ4F_malloc(requiredBuffSize, cctx->cmem); + RETURN_ERROR_IF(cctx->tmpBuff == NULL, allocation_failed); + cctx->maxBufferSize = requiredBuffSize; + } } + cctx->tmpIn = cctx->tmpBuff; + cctx->tmpInSize = 0; + (void)XXH32_reset(&(cctx->xxh), 0); + + /* context init */ + cctx->cdict = cdict; + if (cctx->prefs.frameInfo.blockMode == LZ4F_blockLinked) { + /* frame init only for blockLinked : blockIndependent will be init at each block */ + LZ4F_initStream(cctx->lz4CtxPtr, cdict, cctx->prefs.compressionLevel, LZ4F_blockLinked); + } + if (preferencesPtr->compressionLevel >= LZ4HC_CLEVEL_MIN) { + LZ4_favorDecompressionSpeed((LZ4_streamHC_t*)cctx->lz4CtxPtr, (int)preferencesPtr->favorDecSpeed); + } + if (dictBuffer) { + assert(cdict == NULL); + RETURN_ERROR_IF(dictSize > INT_MAX, parameter_invalid); + if (cctx->lz4CtxType == ctxFast) { + /* lz4 fast*/ + LZ4_loadDict((LZ4_stream_t*)cctx->lz4CtxPtr, (const char*)dictBuffer, (int)dictSize); + } else { + /* lz4hc */ + assert(cctx->lz4CtxType == ctxHC); + LZ4_loadDictHC((LZ4_streamHC_t*)cctx->lz4CtxPtr, (const char*)dictBuffer, (int)dictSize); + } + } + + /* Stage 2 : Write Frame Header */ + + /* Magic Number */ + LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER); + dstPtr += 4; + { BYTE* const headerStart = dstPtr; + + /* FLG Byte */ + *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */ + + ((cctx->prefs.frameInfo.blockMode & _1BIT ) << 5) + + ((cctx->prefs.frameInfo.blockChecksumFlag & _1BIT ) << 4) + + ((unsigned)(cctx->prefs.frameInfo.contentSize > 0) << 3) + + ((cctx->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) + + (cctx->prefs.frameInfo.dictID > 0) ); + /* BD Byte */ + *dstPtr++ = (BYTE)((cctx->prefs.frameInfo.blockSizeID & _3BITS) << 4); + /* Optional Frame content size field */ + if (cctx->prefs.frameInfo.contentSize) { + LZ4F_writeLE64(dstPtr, cctx->prefs.frameInfo.contentSize); + dstPtr += 8; + cctx->totalInSize = 0; + } + /* Optional dictionary ID field */ + if (cctx->prefs.frameInfo.dictID) { + LZ4F_writeLE32(dstPtr, cctx->prefs.frameInfo.dictID); + dstPtr += 4; + } + /* Header CRC Byte */ + *dstPtr = LZ4F_headerChecksum(headerStart, (size_t)(dstPtr - headerStart)); + dstPtr++; + } + + cctx->cStage = 1; /* header written, now request input data block */ + return (size_t)(dstPtr - dstStart); +} + +size_t LZ4F_compressBegin(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_preferences_t* preferencesPtr) +{ + return LZ4F_compressBegin_internal(cctx, dstBuffer, dstCapacity, + NULL, 0, + NULL, preferencesPtr); +} + +/* LZ4F_compressBegin_usingDictOnce: + * Hidden implementation, + * employed for multi-threaded compression + * when frame defines linked blocks */ +size_t LZ4F_compressBegin_usingDictOnce(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const void* dict, size_t dictSize, + const LZ4F_preferences_t* preferencesPtr) +{ + return LZ4F_compressBegin_internal(cctx, dstBuffer, dstCapacity, + dict, dictSize, + NULL, preferencesPtr); +} + +size_t LZ4F_compressBegin_usingDict(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const void* dict, size_t dictSize, + const LZ4F_preferences_t* preferencesPtr) +{ + /* note : incorrect implementation : + * this will only use the dictionary once, + * instead of once *per* block when frames defines independent blocks */ + return LZ4F_compressBegin_usingDictOnce(cctx, dstBuffer, dstCapacity, + dict, dictSize, + preferencesPtr); +} + +size_t LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* preferencesPtr) +{ + return LZ4F_compressBegin_internal(cctx, dstBuffer, dstCapacity, + NULL, 0, + cdict, preferencesPtr); +} + + +/* LZ4F_compressBound() : + * @return minimum capacity of dstBuffer for a given srcSize to handle worst case scenario. + * LZ4F_preferences_t structure is optional : if NULL, preferences will be set to cover worst case scenario. + * This function cannot fail. + */ +size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) +{ + if (preferencesPtr && preferencesPtr->autoFlush) { + return LZ4F_compressBound_internal(srcSize, preferencesPtr, 0); + } + return LZ4F_compressBound_internal(srcSize, preferencesPtr, (size_t)-1); +} + + +typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level, const LZ4F_CDict* cdict); + + +/*! LZ4F_makeBlock(): + * compress a single block, add header and optional checksum. + * assumption : dst buffer capacity is >= BHSize + srcSize + crcSize + */ +static size_t LZ4F_makeBlock(void* dst, + const void* src, size_t srcSize, + compressFunc_t compress, void* lz4ctx, int level, + const LZ4F_CDict* cdict, + LZ4F_blockChecksum_t crcFlag) +{ + BYTE* const cSizePtr = (BYTE*)dst; + U32 cSize; + assert(compress != NULL); + cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize), + (int)(srcSize), (int)(srcSize-1), + level, cdict); + + if (cSize == 0 || cSize >= srcSize) { + cSize = (U32)srcSize; + LZ4F_writeLE32(cSizePtr, cSize | LZ4F_BLOCKUNCOMPRESSED_FLAG); + memcpy(cSizePtr+BHSize, src, srcSize); + } else { + LZ4F_writeLE32(cSizePtr, cSize); + } + if (crcFlag) { + U32 const crc32 = XXH32(cSizePtr+BHSize, cSize, 0); /* checksum of compressed data */ + LZ4F_writeLE32(cSizePtr+BHSize+cSize, crc32); + } + return BHSize + cSize + ((U32)crcFlag)*BFSize; +} + + +static int LZ4F_compressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) +{ + int const acceleration = (level < 0) ? -level + 1 : 1; + DEBUGLOG(5, "LZ4F_compressBlock (srcSize=%i)", srcSize); + LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent); + if (cdict) { + return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration); + } else { + return LZ4_compress_fast_extState_fastReset(ctx, src, dst, srcSize, dstCapacity, acceleration); + } +} + +static int LZ4F_compressBlock_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) +{ + int const acceleration = (level < 0) ? -level + 1 : 1; + (void)cdict; /* init once at beginning of frame */ + DEBUGLOG(5, "LZ4F_compressBlock_continue (srcSize=%i)", srcSize); + return LZ4_compress_fast_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstCapacity, acceleration); +} + +static int LZ4F_compressBlockHC(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) +{ + LZ4F_initStream(ctx, cdict, level, LZ4F_blockIndependent); + if (cdict) { + return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity); + } + return LZ4_compress_HC_extStateHC_fastReset(ctx, src, dst, srcSize, dstCapacity, level); +} + +static int LZ4F_compressBlockHC_continue(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) +{ + (void)level; (void)cdict; /* init once at beginning of frame */ + return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstCapacity); +} + +static int LZ4F_doNotCompressBlock(void* ctx, const char* src, char* dst, int srcSize, int dstCapacity, int level, const LZ4F_CDict* cdict) +{ + (void)ctx; (void)src; (void)dst; (void)srcSize; (void)dstCapacity; (void)level; (void)cdict; + return 0; +} + +static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level, LZ4F_BlockCompressMode_e compressMode) +{ + if (compressMode == LZ4B_UNCOMPRESSED) + return LZ4F_doNotCompressBlock; + if (level < LZ4HC_CLEVEL_MIN) { + if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlock; + return LZ4F_compressBlock_continue; + } + if (blockMode == LZ4F_blockIndependent) return LZ4F_compressBlockHC; + return LZ4F_compressBlockHC_continue; +} + +/* Save history (up to 64KB) into @tmpBuff */ +static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) +{ + if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) + return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); + return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); +} + +typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; + +static const LZ4F_compressOptions_t k_cOptionsNull = { 0, { 0, 0, 0 } }; + + + /*! LZ4F_compressUpdateImpl() : + * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. + * When successful, the function always entirely consumes @srcBuffer. + * src data is either buffered or compressed into @dstBuffer. + * If the block compression does not match the compression of the previous block, the old data is flushed + * and operations continue with the new compression mode. + * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr) when block compression is turned on. + * @compressOptionsPtr is optional : provide NULL to mean "default". + * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered. + * or an error code if it fails (which can be tested using LZ4F_isError()) + * After an error, the state is left in a UB state, and must be re-initialized. + */ +static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* compressOptionsPtr, + LZ4F_BlockCompressMode_e blockCompression) + { + size_t const blockSize = cctxPtr->maxBlockSize; + const BYTE* srcPtr = (const BYTE*)srcBuffer; + const BYTE* const srcEnd = srcPtr + srcSize; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + LZ4F_lastBlockStatus lastBlockCompressed = notDone; + compressFunc_t const compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel, blockCompression); + size_t bytesWritten; + DEBUGLOG(4, "LZ4F_compressUpdate (srcSize=%zu)", srcSize); + + RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized); /* state must be initialized and waiting for next block */ + if (dstCapacity < LZ4F_compressBound_internal(srcSize, &(cctxPtr->prefs), cctxPtr->tmpInSize)) + RETURN_ERROR(dstMaxSize_tooSmall); + + if (blockCompression == LZ4B_UNCOMPRESSED && dstCapacity < srcSize) + RETURN_ERROR(dstMaxSize_tooSmall); + + /* flush currently written block, to continue with new block compression */ + if (cctxPtr->blockCompressMode != blockCompression) { + bytesWritten = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr); + dstPtr += bytesWritten; + cctxPtr->blockCompressMode = blockCompression; + } + + if (compressOptionsPtr == NULL) compressOptionsPtr = &k_cOptionsNull; + + /* complete tmp buffer */ + if (cctxPtr->tmpInSize > 0) { /* some data already within tmp buffer */ + size_t const sizeToCopy = blockSize - cctxPtr->tmpInSize; + assert(blockSize > cctxPtr->tmpInSize); + if (sizeToCopy > srcSize) { + /* add src to tmpIn buffer */ + memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize); + srcPtr = srcEnd; + cctxPtr->tmpInSize += srcSize; + /* still needs some CRC */ + } else { + /* complete tmpIn block and then compress it */ + lastBlockCompressed = fromTmpBuffer; + memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy); + srcPtr += sizeToCopy; + + dstPtr += LZ4F_makeBlock(dstPtr, + cctxPtr->tmpIn, blockSize, + compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, + cctxPtr->cdict, + cctxPtr->prefs.frameInfo.blockChecksumFlag); + if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize; + cctxPtr->tmpInSize = 0; + } } + + while ((size_t)(srcEnd - srcPtr) >= blockSize) { + /* compress full blocks */ + lastBlockCompressed = fromSrcBuffer; + dstPtr += LZ4F_makeBlock(dstPtr, + srcPtr, blockSize, + compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, + cctxPtr->cdict, + cctxPtr->prefs.frameInfo.blockChecksumFlag); + srcPtr += blockSize; + } + + if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) { + /* autoFlush : remaining input (< blockSize) is compressed */ + lastBlockCompressed = fromSrcBuffer; + dstPtr += LZ4F_makeBlock(dstPtr, + srcPtr, (size_t)(srcEnd - srcPtr), + compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, + cctxPtr->cdict, + cctxPtr->prefs.frameInfo.blockChecksumFlag); + srcPtr = srcEnd; + } + + /* preserve dictionary within @tmpBuff whenever necessary */ + if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) { + /* linked blocks are only supported in compressed mode, see LZ4F_uncompressedUpdate */ + assert(blockCompression == LZ4B_COMPRESSED); + if (compressOptionsPtr->stableSrc) { + cctxPtr->tmpIn = cctxPtr->tmpBuff; /* src is stable : dictionary remains in src across invocations */ + } else { + int const realDictSize = LZ4F_localSaveDict(cctxPtr); + assert(0 <= realDictSize && realDictSize <= 64 KB); + cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; + } + } + + /* keep tmpIn within limits */ + if (!(cctxPtr->prefs.autoFlush) /* no autoflush : there may be some data left within internal buffer */ + && (cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) ) /* not enough room to store next block */ + { + /* only preserve 64KB within internal buffer. Ensures there is enough room for next block. + * note: this situation necessarily implies lastBlockCompressed==fromTmpBuffer */ + int const realDictSize = LZ4F_localSaveDict(cctxPtr); + cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; + assert((cctxPtr->tmpIn + blockSize) <= (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)); + } + + /* some input data left, necessarily < blockSize */ + if (srcPtr < srcEnd) { + /* fill tmp buffer */ + size_t const sizeToCopy = (size_t)(srcEnd - srcPtr); + memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy); + cctxPtr->tmpInSize = sizeToCopy; + } + + if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) + (void)XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize); + + cctxPtr->totalInSize += srcSize; + return (size_t)(dstPtr - dstStart); +} + +/*! LZ4F_compressUpdate() : + * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. + * When successful, the function always entirely consumes @srcBuffer. + * src data is either buffered or compressed into @dstBuffer. + * If previously an uncompressed block was written, buffered data is flushed + * before appending compressed data is continued. + * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr). + * @compressOptionsPtr is optional : provide NULL to mean "default". + * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered. + * or an error code if it fails (which can be tested using LZ4F_isError()) + * After an error, the state is left in a UB state, and must be re-initialized. + */ +size_t LZ4F_compressUpdate(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* compressOptionsPtr) +{ + return LZ4F_compressUpdateImpl(cctxPtr, + dstBuffer, dstCapacity, + srcBuffer, srcSize, + compressOptionsPtr, LZ4B_COMPRESSED); +} + +/*! LZ4F_uncompressedUpdate() : + * Same as LZ4F_compressUpdate(), but requests blocks to be sent uncompressed. + * This symbol is only supported when LZ4F_blockIndependent is used + * @dstCapacity MUST be >= LZ4F_compressBound(srcSize, preferencesPtr). + * @compressOptionsPtr is optional : provide NULL to mean "default". + * @return : the number of bytes written into dstBuffer. It can be zero, meaning input data was just buffered. + * or an error code if it fails (which can be tested using LZ4F_isError()) + * After an error, the state is left in a UB state, and must be re-initialized. + */ +size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* compressOptionsPtr) +{ + return LZ4F_compressUpdateImpl(cctxPtr, + dstBuffer, dstCapacity, + srcBuffer, srcSize, + compressOptionsPtr, LZ4B_UNCOMPRESSED); +} + + +/*! LZ4F_flush() : + * When compressed data must be sent immediately, without waiting for a block to be filled, + * invoke LZ4_flush(), which will immediately compress any remaining data stored within LZ4F_cctx. + * The result of the function is the number of bytes written into dstBuffer. + * It can be zero, this means there was no data left within LZ4F_cctx. + * The function outputs an error code if it fails (can be tested using LZ4F_isError()) + * LZ4F_compressOptions_t* is optional. NULL is a valid argument. + */ +size_t LZ4F_flush(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const LZ4F_compressOptions_t* compressOptionsPtr) +{ + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + compressFunc_t compress; + + if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ + RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized); + RETURN_ERROR_IF(dstCapacity < (cctxPtr->tmpInSize + BHSize + BFSize), dstMaxSize_tooSmall); + (void)compressOptionsPtr; /* not useful (yet) */ + + /* select compression function */ + compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel, cctxPtr->blockCompressMode); + + /* compress tmp buffer */ + dstPtr += LZ4F_makeBlock(dstPtr, + cctxPtr->tmpIn, cctxPtr->tmpInSize, + compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel, + cctxPtr->cdict, + cctxPtr->prefs.frameInfo.blockChecksumFlag); + assert(((void)"flush overflows dstBuffer!", (size_t)(dstPtr - dstStart) <= dstCapacity)); + + if (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) + cctxPtr->tmpIn += cctxPtr->tmpInSize; + cctxPtr->tmpInSize = 0; + + /* keep tmpIn within limits */ + if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) { /* necessarily LZ4F_blockLinked */ + int const realDictSize = LZ4F_localSaveDict(cctxPtr); + cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; + } + + return (size_t)(dstPtr - dstStart); +} + + +/*! LZ4F_compressEnd() : + * When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). + * It will flush whatever data remained within compressionContext (like LZ4_flush()) + * but also properly finalize the frame, with an endMark and an (optional) checksum. + * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + * @return: the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) + * or an error code if it fails (can be tested using LZ4F_isError()) + * The context can then be used again to compress a new frame, starting with LZ4F_compressBegin(). + */ +size_t LZ4F_compressEnd(LZ4F_cctx* cctxPtr, + void* dstBuffer, size_t dstCapacity, + const LZ4F_compressOptions_t* compressOptionsPtr) +{ + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + + size_t const flushSize = LZ4F_flush(cctxPtr, dstBuffer, dstCapacity, compressOptionsPtr); + DEBUGLOG(5,"LZ4F_compressEnd: dstCapacity=%u", (unsigned)dstCapacity); + FORWARD_IF_ERROR(flushSize); + dstPtr += flushSize; + + assert(flushSize <= dstCapacity); + dstCapacity -= flushSize; + + RETURN_ERROR_IF(dstCapacity < 4, dstMaxSize_tooSmall); + LZ4F_writeLE32(dstPtr, 0); + dstPtr += 4; /* endMark */ + + if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) { + U32 const xxh = XXH32_digest(&(cctxPtr->xxh)); + RETURN_ERROR_IF(dstCapacity < 8, dstMaxSize_tooSmall); + DEBUGLOG(5,"Writing 32-bit content checksum (0x%0X)", xxh); + LZ4F_writeLE32(dstPtr, xxh); + dstPtr+=4; /* content Checksum */ + } + + cctxPtr->cStage = 0; /* state is now re-usable (with identical preferences) */ + + if (cctxPtr->prefs.frameInfo.contentSize) { + if (cctxPtr->prefs.frameInfo.contentSize != cctxPtr->totalInSize) + RETURN_ERROR(frameSize_wrong); + } + + return (size_t)(dstPtr - dstStart); +} + + +/*-*************************************************** +* Frame Decompression +*****************************************************/ + +typedef enum { + dstage_getFrameHeader=0, dstage_storeFrameHeader, + dstage_init, + dstage_getBlockHeader, dstage_storeBlockHeader, + dstage_copyDirect, dstage_getBlockChecksum, + dstage_getCBlock, dstage_storeCBlock, + dstage_flushOut, + dstage_getSuffix, dstage_storeSuffix, + dstage_getSFrameSize, dstage_storeSFrameSize, + dstage_skipSkippable +} dStage_t; + +struct LZ4F_dctx_s { + LZ4F_CustomMem cmem; + LZ4F_frameInfo_t frameInfo; + U32 version; + dStage_t dStage; + U64 frameRemainingSize; + size_t maxBlockSize; + size_t maxBufferSize; + BYTE* tmpIn; + size_t tmpInSize; + size_t tmpInTarget; + BYTE* tmpOutBuffer; + const BYTE* dict; + size_t dictSize; + BYTE* tmpOut; + size_t tmpOutSize; + size_t tmpOutStart; + XXH32_state_t xxh; + XXH32_state_t blockChecksum; + int skipChecksum; + BYTE header[LZ4F_HEADER_SIZE_MAX]; +}; /* typedef'd to LZ4F_dctx in lz4frame.h */ + + +LZ4F_dctx* LZ4F_createDecompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version) +{ + LZ4F_dctx* const dctx = (LZ4F_dctx*)LZ4F_calloc(sizeof(LZ4F_dctx), customMem); + if (dctx == NULL) return NULL; + + dctx->cmem = customMem; + dctx->version = version; + return dctx; +} + +/*! LZ4F_createDecompressionContext() : + * Create a decompressionContext object, which will track all decompression operations. + * Provides a pointer to a fully allocated and initialized LZ4F_decompressionContext object. + * Object can later be released using LZ4F_freeDecompressionContext(). + * @return : if != 0, there was an error during context creation. + */ +LZ4F_errorCode_t +LZ4F_createDecompressionContext(LZ4F_dctx** LZ4F_decompressionContextPtr, unsigned versionNumber) +{ + assert(LZ4F_decompressionContextPtr != NULL); /* violation of narrow contract */ + RETURN_ERROR_IF(LZ4F_decompressionContextPtr == NULL, parameter_null); /* in case it nonetheless happen in production */ + + *LZ4F_decompressionContextPtr = LZ4F_createDecompressionContext_advanced(LZ4F_defaultCMem, versionNumber); + if (*LZ4F_decompressionContextPtr == NULL) { /* failed allocation */ + RETURN_ERROR(allocation_failed); + } + return LZ4F_OK_NoError; +} + +LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx) +{ + LZ4F_errorCode_t result = LZ4F_OK_NoError; + if (dctx != NULL) { /* can accept NULL input, like free() */ + result = (LZ4F_errorCode_t)dctx->dStage; + LZ4F_free(dctx->tmpIn, dctx->cmem); + LZ4F_free(dctx->tmpOutBuffer, dctx->cmem); + LZ4F_free(dctx, dctx->cmem); + } + return result; +} + + +/*==--- Streaming Decompression operations ---==*/ +void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx) +{ + DEBUGLOG(5, "LZ4F_resetDecompressionContext"); + dctx->dStage = dstage_getFrameHeader; + dctx->dict = NULL; + dctx->dictSize = 0; + dctx->skipChecksum = 0; + dctx->frameRemainingSize = 0; +} + + +/*! LZ4F_decodeHeader() : + * input : `src` points at the **beginning of the frame** + * output : set internal values of dctx, such as + * dctx->frameInfo and dctx->dStage. + * Also allocates internal buffers. + * @return : nb Bytes read from src (necessarily <= srcSize) + * or an error code (testable with LZ4F_isError()) + */ +static size_t LZ4F_decodeHeader(LZ4F_dctx* dctx, const void* src, size_t srcSize) +{ + unsigned blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, dictIDFlag, blockSizeID; + size_t frameHeaderSize; + const BYTE* srcPtr = (const BYTE*)src; + + DEBUGLOG(5, "LZ4F_decodeHeader"); + /* need to decode header to get frameInfo */ + RETURN_ERROR_IF(srcSize < minFHSize, frameHeader_incomplete); /* minimal frame header size */ + MEM_INIT(&(dctx->frameInfo), 0, sizeof(dctx->frameInfo)); + + /* special case : skippable frames */ + if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) { + dctx->frameInfo.frameType = LZ4F_skippableFrame; + if (src == (void*)(dctx->header)) { + dctx->tmpInSize = srcSize; + dctx->tmpInTarget = 8; + dctx->dStage = dstage_storeSFrameSize; + return srcSize; + } else { + dctx->dStage = dstage_getSFrameSize; + return 4; + } } + + /* control magic number */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) { + DEBUGLOG(4, "frame header error : unknown magic number"); + RETURN_ERROR(frameType_unknown); + } +#endif + dctx->frameInfo.frameType = LZ4F_frame; + + /* Flags */ + { U32 const FLG = srcPtr[4]; + U32 const version = (FLG>>6) & _2BITS; + blockChecksumFlag = (FLG>>4) & _1BIT; + blockMode = (FLG>>5) & _1BIT; + contentSizeFlag = (FLG>>3) & _1BIT; + contentChecksumFlag = (FLG>>2) & _1BIT; + dictIDFlag = FLG & _1BIT; + /* validate */ + if (((FLG>>1)&_1BIT) != 0) RETURN_ERROR(reservedFlag_set); /* Reserved bit */ + if (version != 1) RETURN_ERROR(headerVersion_wrong); /* Version Number, only supported value */ + } + DEBUGLOG(6, "contentSizeFlag: %u", contentSizeFlag); + + /* Frame Header Size */ + frameHeaderSize = minFHSize + (contentSizeFlag?8:0) + (dictIDFlag?4:0); + + if (srcSize < frameHeaderSize) { + /* not enough input to fully decode frame header */ + if (srcPtr != dctx->header) + memcpy(dctx->header, srcPtr, srcSize); + dctx->tmpInSize = srcSize; + dctx->tmpInTarget = frameHeaderSize; + dctx->dStage = dstage_storeFrameHeader; + return srcSize; + } + + { U32 const BD = srcPtr[5]; + blockSizeID = (BD>>4) & _3BITS; + /* validate */ + if (((BD>>7)&_1BIT) != 0) RETURN_ERROR(reservedFlag_set); /* Reserved bit */ + if (blockSizeID < 4) RETURN_ERROR(maxBlockSize_invalid); /* 4-7 only supported values for the time being */ + if (((BD>>0)&_4BITS) != 0) RETURN_ERROR(reservedFlag_set); /* Reserved bits */ + } + + /* check header */ + assert(frameHeaderSize > 5); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + { BYTE const HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5); + RETURN_ERROR_IF(HC != srcPtr[frameHeaderSize-1], headerChecksum_invalid); + } +#endif + + /* save */ + dctx->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode; + dctx->frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)blockChecksumFlag; + dctx->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag; + dctx->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID; + dctx->maxBlockSize = LZ4F_getBlockSize((LZ4F_blockSizeID_t)blockSizeID); + if (contentSizeFlag) { + dctx->frameRemainingSize = dctx->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6); + } + if (dictIDFlag) + dctx->frameInfo.dictID = LZ4F_readLE32(srcPtr + frameHeaderSize - 5); + + dctx->dStage = dstage_init; + + return frameHeaderSize; +} + + +/*! LZ4F_headerSize() : + * @return : size of frame header + * or an error code, which can be tested using LZ4F_isError() + */ +size_t LZ4F_headerSize(const void* src, size_t srcSize) +{ + RETURN_ERROR_IF(src == NULL, srcPtr_wrong); + + /* minimal srcSize to determine header size */ + if (srcSize < LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH) + RETURN_ERROR(frameHeader_incomplete); + + /* special case : skippable frames */ + if ((LZ4F_readLE32(src) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) + return 8; + + /* control magic number */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + if (LZ4F_readLE32(src) != LZ4F_MAGICNUMBER) + RETURN_ERROR(frameType_unknown); +#endif + + /* Frame Header Size */ + { BYTE const FLG = ((const BYTE*)src)[4]; + U32 const contentSizeFlag = (FLG>>3) & _1BIT; + U32 const dictIDFlag = FLG & _1BIT; + return minFHSize + (contentSizeFlag?8:0) + (dictIDFlag?4:0); + } +} + +/*! LZ4F_getFrameInfo() : + * This function extracts frame parameters (max blockSize, frame checksum, etc.). + * Usage is optional. Objective is to provide relevant information for allocation purposes. + * This function works in 2 situations : + * - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process. + * Amount of input data provided must be large enough to successfully decode the frame header. + * A header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. It's possible to provide more input data than this minimum. + * - After decoding has been started. In which case, no input is read, frame parameters are extracted from dctx. + * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). + * Decompression must resume from (srcBuffer + *srcSizePtr). + * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call, + * or an error code which can be tested using LZ4F_isError() + * note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped. + * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. + */ +LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, + LZ4F_frameInfo_t* frameInfoPtr, + const void* srcBuffer, size_t* srcSizePtr) +{ + LZ4F_STATIC_ASSERT(dstage_getFrameHeader < dstage_storeFrameHeader); + if (dctx->dStage > dstage_storeFrameHeader) { + /* frameInfo already decoded */ + size_t o=0, i=0; + *srcSizePtr = 0; + *frameInfoPtr = dctx->frameInfo; + /* returns : recommended nb of bytes for LZ4F_decompress() */ + return LZ4F_decompress(dctx, NULL, &o, NULL, &i, NULL); + } else { + if (dctx->dStage == dstage_storeFrameHeader) { + /* frame decoding already started, in the middle of header => automatic fail */ + *srcSizePtr = 0; + RETURN_ERROR(frameDecoding_alreadyStarted); + } else { + size_t const hSize = LZ4F_headerSize(srcBuffer, *srcSizePtr); + if (LZ4F_isError(hSize)) { *srcSizePtr=0; return hSize; } + if (*srcSizePtr < hSize) { + *srcSizePtr=0; + RETURN_ERROR(frameHeader_incomplete); + } + + { size_t decodeResult = LZ4F_decodeHeader(dctx, srcBuffer, hSize); + if (LZ4F_isError(decodeResult)) { + *srcSizePtr = 0; + } else { + *srcSizePtr = decodeResult; + decodeResult = BHSize; /* block header size */ + } + *frameInfoPtr = dctx->frameInfo; + return decodeResult; + } } } +} + + +/* LZ4F_updateDict() : + * only used for LZ4F_blockLinked mode + * Condition : @dstPtr != NULL + */ +static void LZ4F_updateDict(LZ4F_dctx* dctx, + const BYTE* dstPtr, size_t dstSize, const BYTE* dstBufferStart, + unsigned withinTmp) +{ + assert(dstPtr != NULL); + if (dctx->dictSize==0) dctx->dict = (const BYTE*)dstPtr; /* will lead to prefix mode */ + assert(dctx->dict != NULL); + + if (dctx->dict + dctx->dictSize == dstPtr) { /* prefix mode, everything within dstBuffer */ + dctx->dictSize += dstSize; + return; + } + + assert(dstPtr >= dstBufferStart); + if ((size_t)(dstPtr - dstBufferStart) + dstSize >= 64 KB) { /* history in dstBuffer becomes large enough to become dictionary */ + dctx->dict = (const BYTE*)dstBufferStart; + dctx->dictSize = (size_t)(dstPtr - dstBufferStart) + dstSize; + return; + } + + assert(dstSize < 64 KB); /* if dstSize >= 64 KB, dictionary would be set into dstBuffer directly */ + + /* dstBuffer does not contain whole useful history (64 KB), so it must be saved within tmpOutBuffer */ + assert(dctx->tmpOutBuffer != NULL); + + if (withinTmp && (dctx->dict == dctx->tmpOutBuffer)) { /* continue history within tmpOutBuffer */ + /* withinTmp expectation : content of [dstPtr,dstSize] is same as [dict+dictSize,dstSize], so we just extend it */ + assert(dctx->dict + dctx->dictSize == dctx->tmpOut + dctx->tmpOutStart); + dctx->dictSize += dstSize; + return; + } + + if (withinTmp) { /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */ + size_t const preserveSize = (size_t)(dctx->tmpOut - dctx->tmpOutBuffer); + size_t copySize = 64 KB - dctx->tmpOutSize; + const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart; + if (dctx->tmpOutSize > 64 KB) copySize = 0; + if (copySize > preserveSize) copySize = preserveSize; + + memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); + + dctx->dict = dctx->tmpOutBuffer; + dctx->dictSize = preserveSize + dctx->tmpOutStart + dstSize; + return; + } + + if (dctx->dict == dctx->tmpOutBuffer) { /* copy dst into tmp to complete dict */ + if (dctx->dictSize + dstSize > dctx->maxBufferSize) { /* tmp buffer not large enough */ + size_t const preserveSize = 64 KB - dstSize; + memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize); + dctx->dictSize = preserveSize; + } + memcpy(dctx->tmpOutBuffer + dctx->dictSize, dstPtr, dstSize); + dctx->dictSize += dstSize; + return; + } + + /* join dict & dest into tmp */ + { size_t preserveSize = 64 KB - dstSize; + if (preserveSize > dctx->dictSize) preserveSize = dctx->dictSize; + memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - preserveSize, preserveSize); + memcpy(dctx->tmpOutBuffer + preserveSize, dstPtr, dstSize); + dctx->dict = dctx->tmpOutBuffer; + dctx->dictSize = preserveSize + dstSize; + } +} + + +/*! LZ4F_decompress() : + * Call this function repetitively to regenerate compressed data in srcBuffer. + * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer + * into dstBuffer of capacity *dstSizePtr. + * + * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). + * + * The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). + * If number of bytes read is < number of bytes provided, then decompression operation is not complete. + * Remaining data will have to be presented again in a subsequent invocation. + * + * The function result is an hint of the better srcSize to use for next call to LZ4F_decompress. + * Schematically, it's the size of the current (or remaining) compressed block + header of next block. + * Respecting the hint provides a small boost to performance, since it allows less buffer shuffling. + * Note that this is just a hint, and it's always possible to any srcSize value. + * When a frame is fully decoded, @return will be 0. + * If decompression failed, @return is an error code which can be tested using LZ4F_isError(). + */ +size_t LZ4F_decompress(LZ4F_dctx* dctx, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const LZ4F_decompressOptions_t* decompressOptionsPtr) +{ + LZ4F_decompressOptions_t optionsNull; + const BYTE* const srcStart = (const BYTE*)srcBuffer; + const BYTE* const srcEnd = srcStart + *srcSizePtr; + const BYTE* srcPtr = srcStart; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* const dstEnd = dstStart ? dstStart + *dstSizePtr : NULL; + BYTE* dstPtr = dstStart; + const BYTE* selectedIn = NULL; + unsigned doAnotherStage = 1; + size_t nextSrcSizeHint = 1; + + + DEBUGLOG(5, "LZ4F_decompress: src[%p](%u) => dst[%p](%u)", + srcBuffer, (unsigned)*srcSizePtr, dstBuffer, (unsigned)*dstSizePtr); + if (dstBuffer == NULL) assert(*dstSizePtr == 0); + MEM_INIT(&optionsNull, 0, sizeof(optionsNull)); + if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull; + *srcSizePtr = 0; + *dstSizePtr = 0; + assert(dctx != NULL); + dctx->skipChecksum |= (decompressOptionsPtr->skipChecksums != 0); /* once set, disable for the remainder of the frame */ + + /* behaves as a state machine */ + + while (doAnotherStage) { + + switch(dctx->dStage) + { + + case dstage_getFrameHeader: + DEBUGLOG(6, "dstage_getFrameHeader"); + if ((size_t)(srcEnd-srcPtr) >= maxFHSize) { /* enough to decode - shortcut */ + size_t const hSize = LZ4F_decodeHeader(dctx, srcPtr, (size_t)(srcEnd-srcPtr)); /* will update dStage appropriately */ + FORWARD_IF_ERROR(hSize); + srcPtr += hSize; + break; + } + dctx->tmpInSize = 0; + if (srcEnd-srcPtr == 0) return minFHSize; /* 0-size input */ + dctx->tmpInTarget = minFHSize; /* minimum size to decode header */ + dctx->dStage = dstage_storeFrameHeader; + /* fall-through */ + + case dstage_storeFrameHeader: + DEBUGLOG(6, "dstage_storeFrameHeader"); + { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, (size_t)(srcEnd - srcPtr)); + memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy); + dctx->tmpInSize += sizeToCopy; + srcPtr += sizeToCopy; + } + if (dctx->tmpInSize < dctx->tmpInTarget) { + nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */ + doAnotherStage = 0; /* not enough src data, ask for some more */ + break; + } + FORWARD_IF_ERROR( LZ4F_decodeHeader(dctx, dctx->header, dctx->tmpInTarget) ); /* will update dStage appropriately */ + break; + + case dstage_init: + DEBUGLOG(6, "dstage_init"); + if (dctx->frameInfo.contentChecksumFlag) (void)XXH32_reset(&(dctx->xxh), 0); + /* internal buffers allocation */ + { size_t const bufferNeeded = dctx->maxBlockSize + + ((dctx->frameInfo.blockMode==LZ4F_blockLinked) ? 128 KB : 0); + if (bufferNeeded > dctx->maxBufferSize) { /* tmp buffers too small */ + dctx->maxBufferSize = 0; /* ensure allocation will be re-attempted on next entry*/ + LZ4F_free(dctx->tmpIn, dctx->cmem); + dctx->tmpIn = (BYTE*)LZ4F_malloc(dctx->maxBlockSize + BFSize /* block checksum */, dctx->cmem); + RETURN_ERROR_IF(dctx->tmpIn == NULL, allocation_failed); + LZ4F_free(dctx->tmpOutBuffer, dctx->cmem); + dctx->tmpOutBuffer= (BYTE*)LZ4F_malloc(bufferNeeded, dctx->cmem); + RETURN_ERROR_IF(dctx->tmpOutBuffer== NULL, allocation_failed); + dctx->maxBufferSize = bufferNeeded; + } } + dctx->tmpInSize = 0; + dctx->tmpInTarget = 0; + dctx->tmpOut = dctx->tmpOutBuffer; + dctx->tmpOutStart = 0; + dctx->tmpOutSize = 0; + + dctx->dStage = dstage_getBlockHeader; + /* fall-through */ + + case dstage_getBlockHeader: + if ((size_t)(srcEnd - srcPtr) >= BHSize) { + selectedIn = srcPtr; + srcPtr += BHSize; + } else { + /* not enough input to read cBlockSize field */ + dctx->tmpInSize = 0; + dctx->dStage = dstage_storeBlockHeader; + } + + if (dctx->dStage == dstage_storeBlockHeader) /* can be skipped */ + case dstage_storeBlockHeader: + { size_t const remainingInput = (size_t)(srcEnd - srcPtr); + size_t const wantedData = BHSize - dctx->tmpInSize; + size_t const sizeToCopy = MIN(wantedData, remainingInput); + memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy); + srcPtr += sizeToCopy; + dctx->tmpInSize += sizeToCopy; + + if (dctx->tmpInSize < BHSize) { /* not enough input for cBlockSize */ + nextSrcSizeHint = BHSize - dctx->tmpInSize; + doAnotherStage = 0; + break; + } + selectedIn = dctx->tmpIn; + } /* if (dctx->dStage == dstage_storeBlockHeader) */ + + /* decode block header */ + { U32 const blockHeader = LZ4F_readLE32(selectedIn); + size_t const nextCBlockSize = blockHeader & 0x7FFFFFFFU; + size_t const crcSize = dctx->frameInfo.blockChecksumFlag * BFSize; + if (blockHeader==0) { /* frameEnd signal, no more block */ + DEBUGLOG(5, "end of frame"); + dctx->dStage = dstage_getSuffix; + break; + } + if (nextCBlockSize > dctx->maxBlockSize) { + RETURN_ERROR(maxBlockSize_invalid); + } + if (blockHeader & LZ4F_BLOCKUNCOMPRESSED_FLAG) { + /* next block is uncompressed */ + dctx->tmpInTarget = nextCBlockSize; + DEBUGLOG(5, "next block is uncompressed (size %u)", (U32)nextCBlockSize); + if (dctx->frameInfo.blockChecksumFlag) { + (void)XXH32_reset(&dctx->blockChecksum, 0); + } + dctx->dStage = dstage_copyDirect; + break; + } + /* next block is a compressed block */ + dctx->tmpInTarget = nextCBlockSize + crcSize; + dctx->dStage = dstage_getCBlock; + if (dstPtr==dstEnd || srcPtr==srcEnd) { + nextSrcSizeHint = BHSize + nextCBlockSize + crcSize; + doAnotherStage = 0; + } + break; + } + + case dstage_copyDirect: /* uncompressed block */ + DEBUGLOG(6, "dstage_copyDirect"); + { size_t sizeToCopy; + if (dstPtr == NULL) { + sizeToCopy = 0; + } else { + size_t const minBuffSize = MIN((size_t)(srcEnd-srcPtr), (size_t)(dstEnd-dstPtr)); + sizeToCopy = MIN(dctx->tmpInTarget, minBuffSize); + memcpy(dstPtr, srcPtr, sizeToCopy); + if (!dctx->skipChecksum) { + if (dctx->frameInfo.blockChecksumFlag) { + (void)XXH32_update(&dctx->blockChecksum, srcPtr, sizeToCopy); + } + if (dctx->frameInfo.contentChecksumFlag) + (void)XXH32_update(&dctx->xxh, srcPtr, sizeToCopy); + } + if (dctx->frameInfo.contentSize) + dctx->frameRemainingSize -= sizeToCopy; + + /* history management (linked blocks only)*/ + if (dctx->frameInfo.blockMode == LZ4F_blockLinked) { + LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 0); + } + srcPtr += sizeToCopy; + dstPtr += sizeToCopy; + } + if (sizeToCopy == dctx->tmpInTarget) { /* all done */ + if (dctx->frameInfo.blockChecksumFlag) { + dctx->tmpInSize = 0; + dctx->dStage = dstage_getBlockChecksum; + } else + dctx->dStage = dstage_getBlockHeader; /* new block */ + break; + } + dctx->tmpInTarget -= sizeToCopy; /* need to copy more */ + } + nextSrcSizeHint = dctx->tmpInTarget + + +(dctx->frameInfo.blockChecksumFlag ? BFSize : 0) + + BHSize /* next header size */; + doAnotherStage = 0; + break; + + /* check block checksum for recently transferred uncompressed block */ + case dstage_getBlockChecksum: + DEBUGLOG(6, "dstage_getBlockChecksum"); + { const void* crcSrc; + if ((srcEnd-srcPtr >= 4) && (dctx->tmpInSize==0)) { + crcSrc = srcPtr; + srcPtr += 4; + } else { + size_t const stillToCopy = 4 - dctx->tmpInSize; + size_t const sizeToCopy = MIN(stillToCopy, (size_t)(srcEnd-srcPtr)); + memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy); + dctx->tmpInSize += sizeToCopy; + srcPtr += sizeToCopy; + if (dctx->tmpInSize < 4) { /* all input consumed */ + doAnotherStage = 0; + break; + } + crcSrc = dctx->header; + } + if (!dctx->skipChecksum) { + U32 const readCRC = LZ4F_readLE32(crcSrc); + U32 const calcCRC = XXH32_digest(&dctx->blockChecksum); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + DEBUGLOG(6, "compare block checksum"); + if (readCRC != calcCRC) { + DEBUGLOG(4, "incorrect block checksum: %08X != %08X", + readCRC, calcCRC); + RETURN_ERROR(blockChecksum_invalid); + } +#else + (void)readCRC; + (void)calcCRC; +#endif + } } + dctx->dStage = dstage_getBlockHeader; /* new block */ + break; + + case dstage_getCBlock: + DEBUGLOG(6, "dstage_getCBlock"); + if ((size_t)(srcEnd-srcPtr) < dctx->tmpInTarget) { + dctx->tmpInSize = 0; + dctx->dStage = dstage_storeCBlock; + break; + } + /* input large enough to read full block directly */ + selectedIn = srcPtr; + srcPtr += dctx->tmpInTarget; + + if (0) /* always jump over next block */ + case dstage_storeCBlock: + { size_t const wantedData = dctx->tmpInTarget - dctx->tmpInSize; + size_t const inputLeft = (size_t)(srcEnd-srcPtr); + size_t const sizeToCopy = MIN(wantedData, inputLeft); + memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy); + dctx->tmpInSize += sizeToCopy; + srcPtr += sizeToCopy; + if (dctx->tmpInSize < dctx->tmpInTarget) { /* need more input */ + nextSrcSizeHint = (dctx->tmpInTarget - dctx->tmpInSize) + + (dctx->frameInfo.blockChecksumFlag ? BFSize : 0) + + BHSize /* next header size */; + doAnotherStage = 0; + break; + } + selectedIn = dctx->tmpIn; + } + + /* At this stage, input is large enough to decode a block */ + + /* First, decode and control block checksum if it exists */ + if (dctx->frameInfo.blockChecksumFlag) { + assert(dctx->tmpInTarget >= 4); + dctx->tmpInTarget -= 4; + assert(selectedIn != NULL); /* selectedIn is defined at this stage (either srcPtr, or dctx->tmpIn) */ + { U32 const readBlockCrc = LZ4F_readLE32(selectedIn + dctx->tmpInTarget); + U32 const calcBlockCrc = XXH32(selectedIn, dctx->tmpInTarget, 0); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + RETURN_ERROR_IF(readBlockCrc != calcBlockCrc, blockChecksum_invalid); +#else + (void)readBlockCrc; + (void)calcBlockCrc; +#endif + } } + + /* decode directly into destination buffer if there is enough room */ + if ( ((size_t)(dstEnd-dstPtr) >= dctx->maxBlockSize) + /* unless the dictionary is stored in tmpOut: + * in which case it's faster to decode within tmpOut + * to benefit from prefix speedup */ + && !(dctx->dict!= NULL && (const BYTE*)dctx->dict + dctx->dictSize == dctx->tmpOut) ) + { + const char* dict = (const char*)dctx->dict; + size_t dictSize = dctx->dictSize; + int decodedSize; + assert(dstPtr != NULL); + if (dict && dictSize > 1 GB) { + /* overflow control : dctx->dictSize is an int, avoid truncation / sign issues */ + dict += dictSize - 64 KB; + dictSize = 64 KB; + } + decodedSize = LZ4_decompress_safe_usingDict( + (const char*)selectedIn, (char*)dstPtr, + (int)dctx->tmpInTarget, (int)dctx->maxBlockSize, + dict, (int)dictSize); + RETURN_ERROR_IF(decodedSize < 0, decompressionFailed); + if ((dctx->frameInfo.contentChecksumFlag) && (!dctx->skipChecksum)) + XXH32_update(&(dctx->xxh), dstPtr, (size_t)decodedSize); + if (dctx->frameInfo.contentSize) + dctx->frameRemainingSize -= (size_t)decodedSize; + + /* dictionary management */ + if (dctx->frameInfo.blockMode==LZ4F_blockLinked) { + LZ4F_updateDict(dctx, dstPtr, (size_t)decodedSize, dstStart, 0); + } + + dstPtr += decodedSize; + dctx->dStage = dstage_getBlockHeader; /* end of block, let's get another one */ + break; + } + + /* not enough place into dst : decode into tmpOut */ + + /* manage dictionary */ + if (dctx->frameInfo.blockMode == LZ4F_blockLinked) { + if (dctx->dict == dctx->tmpOutBuffer) { + /* truncate dictionary to 64 KB if too big */ + if (dctx->dictSize > 128 KB) { + memcpy(dctx->tmpOutBuffer, dctx->dict + dctx->dictSize - 64 KB, 64 KB); + dctx->dictSize = 64 KB; + } + dctx->tmpOut = dctx->tmpOutBuffer + dctx->dictSize; + } else { /* dict not within tmpOut */ + size_t const reservedDictSpace = MIN(dctx->dictSize, 64 KB); + dctx->tmpOut = dctx->tmpOutBuffer + reservedDictSpace; + } } + + /* Decode block into tmpOut */ + { const char* dict = (const char*)dctx->dict; + size_t dictSize = dctx->dictSize; + int decodedSize; + if (dict && dictSize > 1 GB) { + /* the dictSize param is an int, avoid truncation / sign issues */ + dict += dictSize - 64 KB; + dictSize = 64 KB; + } + decodedSize = LZ4_decompress_safe_usingDict( + (const char*)selectedIn, (char*)dctx->tmpOut, + (int)dctx->tmpInTarget, (int)dctx->maxBlockSize, + dict, (int)dictSize); + RETURN_ERROR_IF(decodedSize < 0, decompressionFailed); + if (dctx->frameInfo.contentChecksumFlag && !dctx->skipChecksum) + XXH32_update(&(dctx->xxh), dctx->tmpOut, (size_t)decodedSize); + if (dctx->frameInfo.contentSize) + dctx->frameRemainingSize -= (size_t)decodedSize; + dctx->tmpOutSize = (size_t)decodedSize; + dctx->tmpOutStart = 0; + dctx->dStage = dstage_flushOut; + } + /* fall-through */ + + case dstage_flushOut: /* flush decoded data from tmpOut to dstBuffer */ + DEBUGLOG(6, "dstage_flushOut"); + if (dstPtr != NULL) { + size_t const sizeToCopy = MIN(dctx->tmpOutSize - dctx->tmpOutStart, (size_t)(dstEnd-dstPtr)); + memcpy(dstPtr, dctx->tmpOut + dctx->tmpOutStart, sizeToCopy); + + /* dictionary management */ + if (dctx->frameInfo.blockMode == LZ4F_blockLinked) + LZ4F_updateDict(dctx, dstPtr, sizeToCopy, dstStart, 1 /*withinTmp*/); + + dctx->tmpOutStart += sizeToCopy; + dstPtr += sizeToCopy; + } + if (dctx->tmpOutStart == dctx->tmpOutSize) { /* all flushed */ + dctx->dStage = dstage_getBlockHeader; /* get next block */ + break; + } + /* could not flush everything : stop there, just request a block header */ + doAnotherStage = 0; + nextSrcSizeHint = BHSize; + break; + + case dstage_getSuffix: + RETURN_ERROR_IF(dctx->frameRemainingSize, frameSize_wrong); /* incorrect frame size decoded */ + if (!dctx->frameInfo.contentChecksumFlag) { /* no checksum, frame is completed */ + nextSrcSizeHint = 0; + LZ4F_resetDecompressionContext(dctx); + doAnotherStage = 0; + break; + } + if ((srcEnd - srcPtr) < 4) { /* not enough size for entire CRC */ + dctx->tmpInSize = 0; + dctx->dStage = dstage_storeSuffix; + } else { + selectedIn = srcPtr; + srcPtr += 4; + } + + if (dctx->dStage == dstage_storeSuffix) /* can be skipped */ + case dstage_storeSuffix: + { size_t const remainingInput = (size_t)(srcEnd - srcPtr); + size_t const wantedData = 4 - dctx->tmpInSize; + size_t const sizeToCopy = MIN(wantedData, remainingInput); + memcpy(dctx->tmpIn + dctx->tmpInSize, srcPtr, sizeToCopy); + srcPtr += sizeToCopy; + dctx->tmpInSize += sizeToCopy; + if (dctx->tmpInSize < 4) { /* not enough input to read complete suffix */ + nextSrcSizeHint = 4 - dctx->tmpInSize; + doAnotherStage=0; + break; + } + selectedIn = dctx->tmpIn; + } /* if (dctx->dStage == dstage_storeSuffix) */ + + /* case dstage_checkSuffix: */ /* no direct entry, avoid initialization risks */ + if (!dctx->skipChecksum) { + U32 const readCRC = LZ4F_readLE32(selectedIn); + U32 const resultCRC = XXH32_digest(&(dctx->xxh)); + DEBUGLOG(4, "frame checksum: stored 0x%0X vs 0x%0X processed", readCRC, resultCRC); +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + RETURN_ERROR_IF(readCRC != resultCRC, contentChecksum_invalid); +#else + (void)readCRC; + (void)resultCRC; +#endif + } + nextSrcSizeHint = 0; + LZ4F_resetDecompressionContext(dctx); + doAnotherStage = 0; + break; + + case dstage_getSFrameSize: + if ((srcEnd - srcPtr) >= 4) { + selectedIn = srcPtr; + srcPtr += 4; + } else { + /* not enough input to read cBlockSize field */ + dctx->tmpInSize = 4; + dctx->tmpInTarget = 8; + dctx->dStage = dstage_storeSFrameSize; + } + + if (dctx->dStage == dstage_storeSFrameSize) + case dstage_storeSFrameSize: + { size_t const sizeToCopy = MIN(dctx->tmpInTarget - dctx->tmpInSize, + (size_t)(srcEnd - srcPtr) ); + memcpy(dctx->header + dctx->tmpInSize, srcPtr, sizeToCopy); + srcPtr += sizeToCopy; + dctx->tmpInSize += sizeToCopy; + if (dctx->tmpInSize < dctx->tmpInTarget) { + /* not enough input to get full sBlockSize; wait for more */ + nextSrcSizeHint = dctx->tmpInTarget - dctx->tmpInSize; + doAnotherStage = 0; + break; + } + selectedIn = dctx->header + 4; + } /* if (dctx->dStage == dstage_storeSFrameSize) */ + + /* case dstage_decodeSFrameSize: */ /* no direct entry */ + { size_t const SFrameSize = LZ4F_readLE32(selectedIn); + dctx->frameInfo.contentSize = SFrameSize; + dctx->tmpInTarget = SFrameSize; + dctx->dStage = dstage_skipSkippable; + break; + } + + case dstage_skipSkippable: + { size_t const skipSize = MIN(dctx->tmpInTarget, (size_t)(srcEnd-srcPtr)); + srcPtr += skipSize; + dctx->tmpInTarget -= skipSize; + doAnotherStage = 0; + nextSrcSizeHint = dctx->tmpInTarget; + if (nextSrcSizeHint) break; /* still more to skip */ + /* frame fully skipped : prepare context for a new frame */ + LZ4F_resetDecompressionContext(dctx); + break; + } + } /* switch (dctx->dStage) */ + } /* while (doAnotherStage) */ + + /* preserve history within tmpOut whenever necessary */ + LZ4F_STATIC_ASSERT((unsigned)dstage_init == 2); + if ( (dctx->frameInfo.blockMode==LZ4F_blockLinked) /* next block will use up to 64KB from previous ones */ + && (dctx->dict != dctx->tmpOutBuffer) /* dictionary is not already within tmp */ + && (dctx->dict != NULL) /* dictionary exists */ + && (!decompressOptionsPtr->stableDst) /* cannot rely on dst data to remain there for next call */ + && ((unsigned)(dctx->dStage)-2 < (unsigned)(dstage_getSuffix)-2) ) /* valid stages : [init ... getSuffix[ */ + { + if (dctx->dStage == dstage_flushOut) { + size_t const preserveSize = (size_t)(dctx->tmpOut - dctx->tmpOutBuffer); + size_t copySize = 64 KB - dctx->tmpOutSize; + const BYTE* oldDictEnd = dctx->dict + dctx->dictSize - dctx->tmpOutStart; + if (dctx->tmpOutSize > 64 KB) copySize = 0; + if (copySize > preserveSize) copySize = preserveSize; + assert(dctx->tmpOutBuffer != NULL); + + memcpy(dctx->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); + + dctx->dict = dctx->tmpOutBuffer; + dctx->dictSize = preserveSize + dctx->tmpOutStart; + } else { + const BYTE* const oldDictEnd = dctx->dict + dctx->dictSize; + size_t const newDictSize = MIN(dctx->dictSize, 64 KB); + + memcpy(dctx->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize); + + dctx->dict = dctx->tmpOutBuffer; + dctx->dictSize = newDictSize; + dctx->tmpOut = dctx->tmpOutBuffer + newDictSize; + } + } + + *srcSizePtr = (size_t)(srcPtr - srcStart); + *dstSizePtr = (size_t)(dstPtr - dstStart); + return nextSrcSizeHint; +} + +/*! LZ4F_decompress_usingDict() : + * Same as LZ4F_decompress(), using a predefined dictionary. + * Dictionary is used "in place", without any preprocessing. + * It must remain accessible throughout the entire frame decoding. + */ +size_t LZ4F_decompress_usingDict(LZ4F_dctx* dctx, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const void* dict, size_t dictSize, + const LZ4F_decompressOptions_t* decompressOptionsPtr) +{ + if (dctx->dStage <= dstage_init) { + dctx->dict = (const BYTE*)dict; + dctx->dictSize = dictSize; + } + return LZ4F_decompress(dctx, dstBuffer, dstSizePtr, + srcBuffer, srcSizePtr, + decompressOptionsPtr); +} diff --git a/example/android/third_party/lz4/include/lz4frame.h b/example/android/third_party/lz4/include/lz4frame.h new file mode 100644 index 00000000..88d59df1 --- /dev/null +++ b/example/android/third_party/lz4/include/lz4frame.h @@ -0,0 +1,746 @@ +/* + LZ4F - LZ4-Frame library + Header File + Copyright (C) 2011-2020, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/lz4/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +/* LZ4F is a stand-alone API able to create and decode LZ4 frames + * conformant with specification v1.6.1 in doc/lz4_Frame_format.md . + * Generated frames are compatible with `lz4` CLI. + * + * LZ4F also offers streaming capabilities. + * + * lz4.h is not required when using lz4frame.h, + * except to extract common constants such as LZ4_VERSION_NUMBER. + * */ + +#ifndef LZ4F_H_09782039843 +#define LZ4F_H_09782039843 + +#if defined (__cplusplus) +extern "C" { +#endif + +/* --- Dependency --- */ +#include /* size_t */ + + +/** + * Introduction + * + * lz4frame.h implements LZ4 frame specification: see doc/lz4_Frame_format.md . + * LZ4 Frames are compatible with `lz4` CLI, + * and designed to be interoperable with any system. +**/ + +/*-*************************************************************** + * Compiler specifics + *****************************************************************/ +/* LZ4_DLL_EXPORT : + * Enable exporting of functions when building a Windows DLL + * LZ4FLIB_VISIBILITY : + * Control library symbols visibility. + */ +#ifndef LZ4FLIB_VISIBILITY +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4FLIB_VISIBILITY __attribute__ ((visibility ("default"))) +# else +# define LZ4FLIB_VISIBILITY +# endif +#endif +#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) +# define LZ4FLIB_API __declspec(dllexport) LZ4FLIB_VISIBILITY +#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) +# define LZ4FLIB_API __declspec(dllimport) LZ4FLIB_VISIBILITY +#else +# define LZ4FLIB_API LZ4FLIB_VISIBILITY +#endif + +#ifdef LZ4F_DISABLE_DEPRECATE_WARNINGS +# define LZ4F_DEPRECATE(x) x +#else +# if defined(_MSC_VER) +# define LZ4F_DEPRECATE(x) x /* __declspec(deprecated) x - only works with C++ */ +# elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 6)) +# define LZ4F_DEPRECATE(x) x __attribute__((deprecated)) +# else +# define LZ4F_DEPRECATE(x) x /* no deprecation warning for this compiler */ +# endif +#endif + + +/*-************************************ + * Error management + **************************************/ +typedef size_t LZ4F_errorCode_t; + +LZ4FLIB_API unsigned LZ4F_isError(LZ4F_errorCode_t code); /**< tells when a function result is an error code */ +LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return error code string; for debugging */ + + +/*-************************************ + * Frame compression types + ************************************* */ +/* #define LZ4F_ENABLE_OBSOLETE_ENUMS // uncomment to enable obsolete enums */ +#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS +# define LZ4F_OBSOLETE_ENUM(x) , LZ4F_DEPRECATE(x) = LZ4F_##x +#else +# define LZ4F_OBSOLETE_ENUM(x) +#endif + +/* The larger the block size, the (slightly) better the compression ratio, + * though there are diminishing returns. + * Larger blocks also increase memory usage on both compression and decompression sides. + */ +typedef enum { + LZ4F_default=0, + LZ4F_max64KB=4, + LZ4F_max256KB=5, + LZ4F_max1MB=6, + LZ4F_max4MB=7 + LZ4F_OBSOLETE_ENUM(max64KB) + LZ4F_OBSOLETE_ENUM(max256KB) + LZ4F_OBSOLETE_ENUM(max1MB) + LZ4F_OBSOLETE_ENUM(max4MB) +} LZ4F_blockSizeID_t; + +/* Linked blocks sharply reduce inefficiencies when using small blocks, + * they compress better. + * However, some LZ4 decoders are only compatible with independent blocks */ +typedef enum { + LZ4F_blockLinked=0, + LZ4F_blockIndependent + LZ4F_OBSOLETE_ENUM(blockLinked) + LZ4F_OBSOLETE_ENUM(blockIndependent) +} LZ4F_blockMode_t; + +typedef enum { + LZ4F_noContentChecksum=0, + LZ4F_contentChecksumEnabled + LZ4F_OBSOLETE_ENUM(noContentChecksum) + LZ4F_OBSOLETE_ENUM(contentChecksumEnabled) +} LZ4F_contentChecksum_t; + +typedef enum { + LZ4F_noBlockChecksum=0, + LZ4F_blockChecksumEnabled +} LZ4F_blockChecksum_t; + +typedef enum { + LZ4F_frame=0, + LZ4F_skippableFrame + LZ4F_OBSOLETE_ENUM(skippableFrame) +} LZ4F_frameType_t; + +#ifdef LZ4F_ENABLE_OBSOLETE_ENUMS +typedef LZ4F_blockSizeID_t blockSizeID_t; +typedef LZ4F_blockMode_t blockMode_t; +typedef LZ4F_frameType_t frameType_t; +typedef LZ4F_contentChecksum_t contentChecksum_t; +#endif + +/*! LZ4F_frameInfo_t : + * makes it possible to set or read frame parameters. + * Structure must be first init to 0, using memset() or LZ4F_INIT_FRAMEINFO, + * setting all parameters to default. + * It's then possible to update selectively some parameters */ +typedef struct { + LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB; 0 == default (LZ4F_max64KB) */ + LZ4F_blockMode_t blockMode; /* LZ4F_blockLinked, LZ4F_blockIndependent; 0 == default (LZ4F_blockLinked) */ + LZ4F_contentChecksum_t contentChecksumFlag; /* 1: add a 32-bit checksum of frame's decompressed data; 0 == default (disabled) */ + LZ4F_frameType_t frameType; /* read-only field : LZ4F_frame or LZ4F_skippableFrame */ + unsigned long long contentSize; /* Size of uncompressed content ; 0 == unknown */ + unsigned dictID; /* Dictionary ID, sent by compressor to help decoder select correct dictionary; 0 == no dictID provided */ + LZ4F_blockChecksum_t blockChecksumFlag; /* 1: each block followed by a checksum of block's compressed data; 0 == default (disabled) */ +} LZ4F_frameInfo_t; + +#define LZ4F_INIT_FRAMEINFO { LZ4F_max64KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame, 0ULL, 0U, LZ4F_noBlockChecksum } /* v1.8.3+ */ + +/*! LZ4F_preferences_t : + * makes it possible to supply advanced compression instructions to streaming interface. + * Structure must be first init to 0, using memset() or LZ4F_INIT_PREFERENCES, + * setting all parameters to default. + * All reserved fields must be set to zero. */ +typedef struct { + LZ4F_frameInfo_t frameInfo; + int compressionLevel; /* 0: default (fast mode); values > LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values < 0 trigger "fast acceleration" */ + unsigned autoFlush; /* 1: always flush; reduces usage of internal buffers */ + unsigned favorDecSpeed; /* 1: parser favors decompression speed vs compression ratio. Only works for high compression modes (>= LZ4HC_CLEVEL_OPT_MIN) */ /* v1.8.2+ */ + unsigned reserved[3]; /* must be zero for forward compatibility */ +} LZ4F_preferences_t; + +#define LZ4F_INIT_PREFERENCES { LZ4F_INIT_FRAMEINFO, 0, 0u, 0u, { 0u, 0u, 0u } } /* v1.8.3+ */ + + +/*-********************************* +* Simple compression function +***********************************/ + +/*! LZ4F_compressFrame() : + * Compress srcBuffer content into an LZ4-compressed frame. + * It's a one shot operation, all input content is consumed, and all output is generated. + * + * Note : it's a stateless operation (no LZ4F_cctx state needed). + * In order to reduce load on the allocator, LZ4F_compressFrame(), by default, + * uses the stack to allocate space for the compression state and some table. + * If this usage of the stack is too much for your application, + * consider compiling `lz4frame.c` with compile-time macro LZ4F_HEAPMODE set to 1 instead. + * All state allocations will use the Heap. + * It also means each invocation of LZ4F_compressFrame() will trigger several internal alloc/free invocations. + * + * @dstCapacity MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + * @preferencesPtr is optional : one can provide NULL, in which case all preferences are set to default. + * @return : number of bytes written into dstBuffer. + * or an error code if it fails (can be tested using LZ4F_isError()) + */ +LZ4FLIB_API size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_preferences_t* preferencesPtr); + +/*! LZ4F_compressFrameBound() : + * Returns the maximum possible compressed size with LZ4F_compressFrame() given srcSize and preferences. + * `preferencesPtr` is optional. It can be replaced by NULL, in which case, the function will assume default preferences. + * Note : this result is only usable with LZ4F_compressFrame(). + * It may also be relevant to LZ4F_compressUpdate() _only if_ no flush() operation is ever performed. + */ +LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); + + +/*! LZ4F_compressionLevel_max() : + * @return maximum allowed compression level (currently: 12) + */ +LZ4FLIB_API int LZ4F_compressionLevel_max(void); /* v1.8.0+ */ + + +/*-*********************************** +* Advanced compression functions +*************************************/ +typedef struct LZ4F_cctx_s LZ4F_cctx; /* incomplete type */ +typedef LZ4F_cctx* LZ4F_compressionContext_t; /* for compatibility with older APIs, prefer using LZ4F_cctx */ + +typedef struct { + unsigned stableSrc; /* 1 == src content will remain present on future calls to LZ4F_compress(); skip copying src content within tmp buffer */ + unsigned reserved[3]; +} LZ4F_compressOptions_t; + +/*--- Resource Management ---*/ + +#define LZ4F_VERSION 100 /* This number can be used to check for an incompatible API breaking change */ +LZ4FLIB_API unsigned LZ4F_getVersion(void); + +/*! LZ4F_createCompressionContext() : + * The first thing to do is to create a compressionContext object, + * which will keep track of operation state during streaming compression. + * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version, + * and a pointer to LZ4F_cctx*, to write the resulting pointer into. + * @version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL. + * The function provides a pointer to a fully allocated LZ4F_cctx object. + * @cctxPtr MUST be != NULL. + * If @return != zero, context creation failed. + * A created compression context can be employed multiple times for consecutive streaming operations. + * Once all streaming compression jobs are completed, + * the state object can be released using LZ4F_freeCompressionContext(). + * Note1 : LZ4F_freeCompressionContext() is always successful. Its return value can be ignored. + * Note2 : LZ4F_freeCompressionContext() works fine with NULL input pointers (do nothing). +**/ +LZ4FLIB_API LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version); +LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); + + +/*---- Compression ----*/ + +#define LZ4F_HEADER_SIZE_MIN 7 /* LZ4 Frame header size can vary, depending on selected parameters */ +#define LZ4F_HEADER_SIZE_MAX 19 + +/* Size in bytes of a block header in little-endian format. Highest bit indicates if block data is uncompressed */ +#define LZ4F_BLOCK_HEADER_SIZE 4 + +/* Size in bytes of a block checksum footer in little-endian format. */ +#define LZ4F_BLOCK_CHECKSUM_SIZE 4 + +/* Size in bytes of the content checksum. */ +#define LZ4F_CONTENT_CHECKSUM_SIZE 4 + +/*! LZ4F_compressBegin() : + * will write the frame header into dstBuffer. + * dstCapacity must be >= LZ4F_HEADER_SIZE_MAX bytes. + * `prefsPtr` is optional : NULL can be provided to set all preferences to default. + * @return : number of bytes written into dstBuffer for the header + * or an error code (which can be tested using LZ4F_isError()) + */ +LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_preferences_t* prefsPtr); + +/*! LZ4F_compressBound() : + * Provides minimum dstCapacity required to guarantee success of + * LZ4F_compressUpdate(), given a srcSize and preferences, for a worst case scenario. + * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() instead. + * Note that the result is only valid for a single invocation of LZ4F_compressUpdate(). + * When invoking LZ4F_compressUpdate() multiple times, + * if the output buffer is gradually filled up instead of emptied and re-used from its start, + * one must check if there is enough remaining capacity before each invocation, using LZ4F_compressBound(). + * @return is always the same for a srcSize and prefsPtr. + * prefsPtr is optional : when NULL is provided, preferences will be set to cover worst case scenario. + * tech details : + * @return if automatic flushing is not enabled, includes the possibility that internal buffer might already be filled by up to (blockSize-1) bytes. + * It also includes frame footer (ending + checksum), since it might be generated by LZ4F_compressEnd(). + * @return doesn't include frame header, as it was already generated by LZ4F_compressBegin(). + */ +LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr); + +/*! LZ4F_compressUpdate() : + * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. + * Important rule: dstCapacity MUST be large enough to ensure operation success even in worst case situations. + * This value is provided by LZ4F_compressBound(). + * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). + * After an error, the state is left in a UB state, and must be re-initialized or freed. + * If previously an uncompressed block was written, buffered data is flushed + * before appending compressed data is continued. + * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. + * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). + * or an error code if it fails (which can be tested using LZ4F_isError()) + */ +LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* cOptPtr); + +/*! LZ4F_flush() : + * When data must be generated and sent immediately, without waiting for a block to be completely filled, + * it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx. + * `dstCapacity` must be large enough to ensure the operation will be successful. + * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. + * @return : nb of bytes written into dstBuffer (can be zero, when there is no data stored within cctx) + * or an error code if it fails (which can be tested using LZ4F_isError()) + * Note : LZ4F_flush() is guaranteed to be successful when dstCapacity >= LZ4F_compressBound(0, prefsPtr). + */ +LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_compressOptions_t* cOptPtr); + +/*! LZ4F_compressEnd() : + * To properly finish an LZ4 frame, invoke LZ4F_compressEnd(). + * It will flush whatever data remained within `cctx` (like LZ4_flush()) + * and properly finalize the frame, with an endMark and a checksum. + * `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default. + * @return : nb of bytes written into dstBuffer, necessarily >= 4 (endMark), + * or an error code if it fails (which can be tested using LZ4F_isError()) + * Note : LZ4F_compressEnd() is guaranteed to be successful when dstCapacity >= LZ4F_compressBound(0, prefsPtr). + * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. + */ +LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_compressOptions_t* cOptPtr); + + +/*-********************************* +* Decompression functions +***********************************/ +typedef struct LZ4F_dctx_s LZ4F_dctx; /* incomplete type */ +typedef LZ4F_dctx* LZ4F_decompressionContext_t; /* compatibility with previous API versions */ + +typedef struct { + unsigned stableDst; /* pledges that last 64KB decompressed data is present right before @dstBuffer pointer. + * This optimization skips internal storage operations. + * Once set, this pledge must remain valid up to the end of current frame. */ + unsigned skipChecksums; /* disable checksum calculation and verification, even when one is present in frame, to save CPU time. + * Setting this option to 1 once disables all checksums for the rest of the frame. */ + unsigned reserved1; /* must be set to zero for forward compatibility */ + unsigned reserved0; /* idem */ +} LZ4F_decompressOptions_t; + + +/* Resource management */ + +/*! LZ4F_createDecompressionContext() : + * Create an LZ4F_dctx object, to track all decompression operations. + * @version provided MUST be LZ4F_VERSION. + * @dctxPtr MUST be valid. + * The function fills @dctxPtr with the value of a pointer to an allocated and initialized LZ4F_dctx object. + * The @return is an errorCode, which can be tested using LZ4F_isError(). + * dctx memory can be released using LZ4F_freeDecompressionContext(); + * Result of LZ4F_freeDecompressionContext() indicates current state of decompressionContext when being released. + * That is, it should be == 0 if decompression has been completed fully and correctly. + */ +LZ4FLIB_API LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version); +LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); + + +/*-*********************************** +* Streaming decompression functions +*************************************/ + +#define LZ4F_MAGICNUMBER 0x184D2204U +#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U +#define LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH 5 + +/*! LZ4F_headerSize() : v1.9.0+ + * Provide the header size of a frame starting at `src`. + * `srcSize` must be >= LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH, + * which is enough to decode the header length. + * @return : size of frame header + * or an error code, which can be tested using LZ4F_isError() + * note : Frame header size is variable, but is guaranteed to be + * >= LZ4F_HEADER_SIZE_MIN bytes, and <= LZ4F_HEADER_SIZE_MAX bytes. + */ +LZ4FLIB_API size_t LZ4F_headerSize(const void* src, size_t srcSize); + +/*! LZ4F_getFrameInfo() : + * This function extracts frame parameters (max blockSize, dictID, etc.). + * Its usage is optional: user can also invoke LZ4F_decompress() directly. + * + * Extracted information will fill an existing LZ4F_frameInfo_t structure. + * This can be useful for allocation and dictionary identification purposes. + * + * LZ4F_getFrameInfo() can work in the following situations : + * + * 1) At the beginning of a new frame, before any invocation of LZ4F_decompress(). + * It will decode header from `srcBuffer`, + * consuming the header and starting the decoding process. + * + * Input size must be large enough to contain the full frame header. + * Frame header size can be known beforehand by LZ4F_headerSize(). + * Frame header size is variable, but is guaranteed to be >= LZ4F_HEADER_SIZE_MIN bytes, + * and not more than <= LZ4F_HEADER_SIZE_MAX bytes. + * Hence, blindly providing LZ4F_HEADER_SIZE_MAX bytes or more will always work. + * It's allowed to provide more input data than the header size, + * LZ4F_getFrameInfo() will only consume the header. + * + * If input size is not large enough, + * aka if it's smaller than header size, + * function will fail and return an error code. + * + * 2) After decoding has been started, + * it's possible to invoke LZ4F_getFrameInfo() anytime + * to extract already decoded frame parameters stored within dctx. + * + * Note that, if decoding has barely started, + * and not yet read enough information to decode the header, + * LZ4F_getFrameInfo() will fail. + * + * The number of bytes consumed from srcBuffer will be updated in *srcSizePtr (necessarily <= original value). + * LZ4F_getFrameInfo() only consumes bytes when decoding has not yet started, + * and when decoding the header has been successful. + * Decompression must then resume from (srcBuffer + *srcSizePtr). + * + * @return : a hint about how many srcSize bytes LZ4F_decompress() expects for next call, + * or an error code which can be tested using LZ4F_isError(). + * note 1 : in case of error, dctx is not modified. Decoding operation can resume from beginning safely. + * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. + */ +LZ4FLIB_API size_t +LZ4F_getFrameInfo(LZ4F_dctx* dctx, + LZ4F_frameInfo_t* frameInfoPtr, + const void* srcBuffer, size_t* srcSizePtr); + +/*! LZ4F_decompress() : + * Call this function repetitively to regenerate data compressed in `srcBuffer`. + * + * The function requires a valid dctx state. + * It will read up to *srcSizePtr bytes from srcBuffer, + * and decompress data into dstBuffer, of capacity *dstSizePtr. + * + * The nb of bytes consumed from srcBuffer will be written into *srcSizePtr (necessarily <= original value). + * The nb of bytes decompressed into dstBuffer will be written into *dstSizePtr (necessarily <= original value). + * + * The function does not necessarily read all input bytes, so always check value in *srcSizePtr. + * Unconsumed source data must be presented again in subsequent invocations. + * + * `dstBuffer` can freely change between each consecutive function invocation. + * `dstBuffer` content will be overwritten. + * + * Note: if `LZ4F_getFrameInfo()` is called before `LZ4F_decompress()`, srcBuffer must be updated to reflect + * the number of bytes consumed after reading the frame header. Failure to update srcBuffer before calling + * `LZ4F_decompress()` will cause decompression failure or, even worse, successful but incorrect decompression. + * See the `LZ4F_getFrameInfo()` docs for details. + * + * @return : an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call. + * Schematically, it's the size of the current (or remaining) compressed block + header of next block. + * Respecting the hint provides some small speed benefit, because it skips intermediate buffers. + * This is just a hint though, it's always possible to provide any srcSize. + * + * When a frame is fully decoded, @return will be 0 (no more data expected). + * When provided with more bytes than necessary to decode a frame, + * LZ4F_decompress() will stop reading exactly at end of current frame, and @return 0. + * + * If decompression failed, @return is an error code, which can be tested using LZ4F_isError(). + * After a decompression error, the `dctx` context is not resumable. + * Use LZ4F_resetDecompressionContext() to return to clean state. + * + * After a frame is fully decoded, dctx can be used again to decompress another frame. + */ +LZ4FLIB_API size_t +LZ4F_decompress(LZ4F_dctx* dctx, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const LZ4F_decompressOptions_t* dOptPtr); + + +/*! LZ4F_resetDecompressionContext() : added in v1.8.0 + * In case of an error, the context is left in "undefined" state. + * In which case, it's necessary to reset it, before re-using it. + * This method can also be used to abruptly stop any unfinished decompression, + * and start a new one using same context resources. */ +LZ4FLIB_API void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); /* always successful */ + + + +#if defined (__cplusplus) +} +#endif + +#endif /* LZ4F_H_09782039843 */ + +#if defined(LZ4F_STATIC_LINKING_ONLY) && !defined(LZ4F_H_STATIC_09782039843) +#define LZ4F_H_STATIC_09782039843 + +/* Note : + * The below declarations are not stable and may change in the future. + * They are therefore only safe to depend on + * when the caller is statically linked against the library. + * To access their declarations, define LZ4F_STATIC_LINKING_ONLY. + * + * By default, these symbols aren't published into shared/dynamic libraries. + * You can override this behavior and force them to be published + * by defining LZ4F_PUBLISH_STATIC_FUNCTIONS. + * Use at your own risk. + */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifdef LZ4F_PUBLISH_STATIC_FUNCTIONS +# define LZ4FLIB_STATIC_API LZ4FLIB_API +#else +# define LZ4FLIB_STATIC_API +#endif + + +/* --- Error List --- */ +#define LZ4F_LIST_ERRORS(ITEM) \ + ITEM(OK_NoError) \ + ITEM(ERROR_GENERIC) \ + ITEM(ERROR_maxBlockSize_invalid) \ + ITEM(ERROR_blockMode_invalid) \ + ITEM(ERROR_parameter_invalid) \ + ITEM(ERROR_compressionLevel_invalid) \ + ITEM(ERROR_headerVersion_wrong) \ + ITEM(ERROR_blockChecksum_invalid) \ + ITEM(ERROR_reservedFlag_set) \ + ITEM(ERROR_allocation_failed) \ + ITEM(ERROR_srcSize_tooLarge) \ + ITEM(ERROR_dstMaxSize_tooSmall) \ + ITEM(ERROR_frameHeader_incomplete) \ + ITEM(ERROR_frameType_unknown) \ + ITEM(ERROR_frameSize_wrong) \ + ITEM(ERROR_srcPtr_wrong) \ + ITEM(ERROR_decompressionFailed) \ + ITEM(ERROR_headerChecksum_invalid) \ + ITEM(ERROR_contentChecksum_invalid) \ + ITEM(ERROR_frameDecoding_alreadyStarted) \ + ITEM(ERROR_compressionState_uninitialized) \ + ITEM(ERROR_parameter_null) \ + ITEM(ERROR_io_write) \ + ITEM(ERROR_io_read) \ + ITEM(ERROR_maxCode) + +#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, + +/* enum list is exposed, to handle specific errors */ +typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) + _LZ4F_dummy_error_enum_for_c89_never_used } LZ4F_errorCodes; + +LZ4FLIB_STATIC_API LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); + +/********************************** + * Advanced compression operations + *********************************/ + +/*! LZ4F_getBlockSize() : + * @return, in scalar format (size_t), + * the maximum block size associated with @blockSizeID, + * or an error code (can be tested using LZ4F_isError()) if @blockSizeID is invalid. +**/ +LZ4FLIB_STATIC_API size_t LZ4F_getBlockSize(LZ4F_blockSizeID_t blockSizeID); + +/*! LZ4F_uncompressedUpdate() : + * LZ4F_uncompressedUpdate() can be called repetitively to add data stored as uncompressed blocks. + * Important rule: dstCapacity MUST be large enough to store the entire source buffer as + * no compression is done for this operation + * If this condition is not respected, LZ4F_uncompressedUpdate() will fail (result is an errorCode). + * After an error, the state is left in a UB state, and must be re-initialized or freed. + * If previously a compressed block was written, buffered data is flushed first, + * before appending uncompressed data is continued. + * This operation is only supported when LZ4F_blockIndependent is used. + * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. + * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). + * or an error code if it fails (which can be tested using LZ4F_isError()) + */ +LZ4FLIB_STATIC_API size_t +LZ4F_uncompressedUpdate(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const void* srcBuffer, size_t srcSize, + const LZ4F_compressOptions_t* cOptPtr); + +/********************************** + * Dictionary compression API + *********************************/ + +/* A Dictionary is useful for the compression of small messages (KB range). + * It dramatically improves compression efficiency. + * + * LZ4 can ingest any input as dictionary, though only the last 64 KB are useful. + * Better results are generally achieved by using Zstandard's Dictionary Builder + * to generate a high-quality dictionary from a set of samples. + * + * The same dictionary will have to be used on the decompression side + * for decoding to be successful. + * To help identify the correct dictionary at decoding stage, + * the frame header allows optional embedding of a dictID field. + */ + +/*! LZ4F_compressBegin_usingDict() : + * Inits dictionary compression streaming, and writes the frame header into dstBuffer. + * `dstCapacity` must be >= LZ4F_HEADER_SIZE_MAX bytes. + * `prefsPtr` is optional : you may provide NULL as argument, + * however, it's the only way to provide dictID in the frame header. + * `dictBuffer` must outlive the compression session. + * @return : number of bytes written into dstBuffer for the header, + * or an error code (which can be tested using LZ4F_isError()) + * NOTE: this entry point doesn't fully exploit the spec, + * which allows each independent block to be compressed with the dictionary. + * Currently, only the first block uses the dictionary. + * This is still technically compliant, but less efficient for large inputs. + */ +LZ4FLIB_STATIC_API size_t +LZ4F_compressBegin_usingDict(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const void* dictBuffer, size_t dictSize, + const LZ4F_preferences_t* prefsPtr); + +/*! LZ4F_decompress_usingDict() : + * Same as LZ4F_decompress(), using a predefined dictionary. + * Dictionary is used "in place", without any preprocessing. +** It must remain accessible throughout the entire frame decoding. */ +LZ4FLIB_STATIC_API size_t +LZ4F_decompress_usingDict(LZ4F_dctx* dctxPtr, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const void* dict, size_t dictSize, + const LZ4F_decompressOptions_t* decompressOptionsPtr); + +/********************************** + * Bulk processing dictionary API + *********************************/ + +/* Loading a dictionary has a cost, since it involves construction of tables. + * The Bulk processing dictionary API makes it possible to share this cost + * over an arbitrary number of compression jobs, even concurrently, + * markedly improving compression latency for these cases. + */ +typedef struct LZ4F_CDict_s LZ4F_CDict; + +/*! LZ4_createCDict() : + * When compressing multiple messages / blocks using the same dictionary, it's recommended to load it just once. + * LZ4_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. + * LZ4_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * `dictBuffer` can be released after LZ4_CDict creation, since its content is copied within CDict */ +LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict(const void* dictBuffer, size_t dictSize); +LZ4FLIB_STATIC_API void LZ4F_freeCDict(LZ4F_CDict* CDict); + +/*! LZ4_compressFrame_usingCDict() : + * Compress an entire srcBuffer into a valid LZ4 frame using a digested Dictionary. + * cctx must point to a context created by LZ4F_createCompressionContext(). + * If cdict==NULL, compress without a dictionary. + * dstBuffer MUST be >= LZ4F_compressFrameBound(srcSize, preferencesPtr). + * If this condition is not respected, function will fail (@return an errorCode). + * The LZ4F_preferences_t structure is optional : you may provide NULL as argument, + * but it's not recommended, as it's the only way to provide dictID in the frame header. + * @return : number of bytes written into dstBuffer. + * or an error code if it fails (can be tested using LZ4F_isError()) */ +LZ4FLIB_STATIC_API size_t +LZ4F_compressFrame_usingCDict(LZ4F_cctx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* preferencesPtr); + +/*! LZ4F_compressBegin_usingCDict() : + * Inits streaming dictionary compression, and writes the frame header into dstBuffer. + * `dstCapacity` must be >= LZ4F_HEADER_SIZE_MAX bytes. + * `prefsPtr` is optional : you may provide NULL as argument, + * however, it's the only way to provide dictID in the frame header. + * `cdict` must outlive the compression session. + * @return : number of bytes written into dstBuffer for the header, + * or an error code (which can be tested using LZ4F_isError()) */ +LZ4FLIB_STATIC_API size_t +LZ4F_compressBegin_usingCDict(LZ4F_cctx* cctx, + void* dstBuffer, size_t dstCapacity, + const LZ4F_CDict* cdict, + const LZ4F_preferences_t* prefsPtr); + + +/********************************** + * Custom memory allocation + *********************************/ + +/*! Custom memory allocation : v1.9.4+ + * These prototypes make it possible to pass custom allocation/free functions. + * LZ4F_customMem is provided at state creation time, using LZ4F_create*_advanced() listed below. + * All allocation/free operations will be completed using these custom variants instead of regular ones. + */ +typedef void* (*LZ4F_AllocFunction) (void* opaqueState, size_t size); +typedef void* (*LZ4F_CallocFunction) (void* opaqueState, size_t size); +typedef void (*LZ4F_FreeFunction) (void* opaqueState, void* address); +typedef struct { + LZ4F_AllocFunction customAlloc; + LZ4F_CallocFunction customCalloc; /* optional; when not defined, uses customAlloc + memset */ + LZ4F_FreeFunction customFree; + void* opaqueState; +} LZ4F_CustomMem; +static +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif +LZ4F_CustomMem const LZ4F_defaultCMem = { NULL, NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ + +LZ4FLIB_STATIC_API LZ4F_cctx* LZ4F_createCompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version); +LZ4FLIB_STATIC_API LZ4F_dctx* LZ4F_createDecompressionContext_advanced(LZ4F_CustomMem customMem, unsigned version); +LZ4FLIB_STATIC_API LZ4F_CDict* LZ4F_createCDict_advanced(LZ4F_CustomMem customMem, const void* dictBuffer, size_t dictSize); + + +#if defined (__cplusplus) +} +#endif + +#endif /* defined(LZ4F_STATIC_LINKING_ONLY) && !defined(LZ4F_H_STATIC_09782039843) */ diff --git a/example/android/third_party/lz4/include/lz4frame_static.h b/example/android/third_party/lz4/include/lz4frame_static.h new file mode 100644 index 00000000..2b44a631 --- /dev/null +++ b/example/android/third_party/lz4/include/lz4frame_static.h @@ -0,0 +1,47 @@ +/* + LZ4 auto-framing library + Header File for static linking only + Copyright (C) 2011-2020, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/lz4/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +#ifndef LZ4FRAME_STATIC_H_0398209384 +#define LZ4FRAME_STATIC_H_0398209384 + +/* The declarations that formerly were made here have been merged into + * lz4frame.h, protected by the LZ4F_STATIC_LINKING_ONLY macro. Going forward, + * it is recommended to simply include that header directly. + */ + +#define LZ4F_STATIC_LINKING_ONLY +#include "lz4frame.h" + +#endif /* LZ4FRAME_STATIC_H_0398209384 */ diff --git a/example/android/third_party/lz4/include/lz4hc.c b/example/android/third_party/lz4/include/lz4hc.c new file mode 100644 index 00000000..a5bdbb5e --- /dev/null +++ b/example/android/third_party/lz4/include/lz4hc.c @@ -0,0 +1,2041 @@ +/* + LZ4 HC - High Compression Mode of LZ4 + Copyright (C) 2011-2020, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/lz4/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ +/* note : lz4hc is not an independent module, it requires lz4.h/lz4.c for proper compilation */ + + +/* ************************************* +* Tuning Parameter +***************************************/ + +/*! HEAPMODE : + * Select how stateless HC compression functions like `LZ4_compress_HC()` + * allocate memory for their workspace: + * in stack (0:fastest), or in heap (1:default, requires malloc()). + * Since workspace is rather large, heap mode is recommended. +**/ +#ifndef LZ4HC_HEAPMODE +# define LZ4HC_HEAPMODE 1 +#endif + + +/*=== Dependency ===*/ +#define LZ4_HC_STATIC_LINKING_ONLY +#include "lz4hc.h" +#include + + +/*=== Shared lz4.c code ===*/ +#ifndef LZ4_SRC_INCLUDED +# if defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wunused-function" +# endif +# if defined (__clang__) +# pragma clang diagnostic ignored "-Wunused-function" +# endif +# define LZ4_COMMONDEFS_ONLY +# include "lz4.c" /* LZ4_count, constants, mem */ +#endif + + +/*=== Enums ===*/ +typedef enum { noDictCtx, usingDictCtxHc } dictCtx_directive; + + +/*=== Constants ===*/ +#define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH) +#define LZ4_OPT_NUM (1<<12) + + +/*=== Macros ===*/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) +#define DELTANEXTU16(table, pos) table[(U16)(pos)] /* faster */ +/* Make fields passed to, and updated by LZ4HC_encodeSequence explicit */ +#define UPDATABLE(ip, op, anchor) &ip, &op, &anchor + + +/*=== Hashing ===*/ +#define LZ4HC_HASHSIZE 4 +#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) +static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } + +#if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2) +/* lie to the compiler about data alignment; use with caution */ +static U64 LZ4_read64(const void* memPtr) { return *(const U64*) memPtr; } + +#elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==1) +/* __pack instructions are safer, but compiler specific */ +LZ4_PACK(typedef struct { U64 u64; }) LZ4_unalign64; +static U64 LZ4_read64(const void* ptr) { return ((const LZ4_unalign64*)ptr)->u64; } + +#else /* safe and portable access using memcpy() */ +static U64 LZ4_read64(const void* memPtr) +{ + U64 val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; +} + +#endif /* LZ4_FORCE_MEMORY_ACCESS */ + +#define LZ4MID_HASHSIZE 8 +#define LZ4MID_HASHLOG (LZ4HC_HASH_LOG-1) +#define LZ4MID_HASHTABLESIZE (1 << LZ4MID_HASHLOG) + +static U32 LZ4MID_hash4(U32 v) { return (v * 2654435761U) >> (32-LZ4MID_HASHLOG); } +static U32 LZ4MID_hash4Ptr(const void* ptr) { return LZ4MID_hash4(LZ4_read32(ptr)); } +/* note: hash7 hashes the lower 56-bits. + * It presumes input was read using little endian.*/ +static U32 LZ4MID_hash7(U64 v) { return (U32)(((v << (64-56)) * 58295818150454627ULL) >> (64-LZ4MID_HASHLOG)) ; } +static U64 LZ4_readLE64(const void* memPtr); +static U32 LZ4MID_hash8Ptr(const void* ptr) { return LZ4MID_hash7(LZ4_readLE64(ptr)); } + +static U64 LZ4_readLE64(const void* memPtr) +{ + if (LZ4_isLittleEndian()) { + return LZ4_read64(memPtr); + } else { + const BYTE* p = (const BYTE*)memPtr; + /* note: relies on the compiler to simplify this expression */ + return (U64)p[0] + ((U64)p[1]<<8) + ((U64)p[2]<<16) + ((U64)p[3]<<24) + + ((U64)p[4]<<32) + ((U64)p[5]<<40) + ((U64)p[6]<<48) + ((U64)p[7]<<56); + } +} + + +/*=== Count match length ===*/ +LZ4_FORCE_INLINE +unsigned LZ4HC_NbCommonBytes32(U32 val) +{ + assert(val != 0); + if (LZ4_isLittleEndian()) { +# if defined(_MSC_VER) && (_MSC_VER >= 1400) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanReverse(&r, val); + return (unsigned)((31 - r) >> 3); +# elif (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_clz(val) >> 3; +# else + val >>= 8; + val = ((((val + 0x00FFFF00) | 0x00FFFFFF) + val) | + (val + 0x00FF0000)) >> 24; + return (unsigned)val ^ 3; +# endif + } else { +# if defined(_MSC_VER) && (_MSC_VER >= 1400) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward(&r, val); + return (unsigned)(r >> 3); +# elif (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_ctz(val) >> 3; +# else + const U32 m = 0x01010101; + return (unsigned)((((val - 1) ^ val) & (m - 1)) * m) >> 24; +# endif + } +} + +/** LZ4HC_countBack() : + * @return : negative value, nb of common bytes before ip/match */ +LZ4_FORCE_INLINE +int LZ4HC_countBack(const BYTE* const ip, const BYTE* const match, + const BYTE* const iMin, const BYTE* const mMin) +{ + int back = 0; + int const min = (int)MAX(iMin - ip, mMin - match); + assert(min <= 0); + assert(ip >= iMin); assert((size_t)(ip-iMin) < (1U<<31)); + assert(match >= mMin); assert((size_t)(match - mMin) < (1U<<31)); + + while ((back - min) > 3) { + U32 const v = LZ4_read32(ip + back - 4) ^ LZ4_read32(match + back - 4); + if (v) { + return (back - (int)LZ4HC_NbCommonBytes32(v)); + } else back -= 4; /* 4-byte step */ + } + /* check remainder if any */ + while ( (back > min) + && (ip[back-1] == match[back-1]) ) + back--; + return back; +} + + +/************************************** +* Init +**************************************/ +static void LZ4HC_clearTables (LZ4HC_CCtx_internal* hc4) +{ + MEM_INIT(hc4->hashTable, 0, sizeof(hc4->hashTable)); + MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); +} + +static void LZ4HC_init_internal (LZ4HC_CCtx_internal* hc4, const BYTE* start) +{ + size_t const bufferSize = (size_t)(hc4->end - hc4->prefixStart); + size_t newStartingOffset = bufferSize + hc4->dictLimit; + DEBUGLOG(5, "LZ4HC_init_internal"); + assert(newStartingOffset >= bufferSize); /* check overflow */ + if (newStartingOffset > 1 GB) { + LZ4HC_clearTables(hc4); + newStartingOffset = 0; + } + newStartingOffset += 64 KB; + hc4->nextToUpdate = (U32)newStartingOffset; + hc4->prefixStart = start; + hc4->end = start; + hc4->dictStart = start; + hc4->dictLimit = (U32)newStartingOffset; + hc4->lowLimit = (U32)newStartingOffset; +} + + +/************************************** +* Encode +**************************************/ +/* LZ4HC_encodeSequence() : + * @return : 0 if ok, + * 1 if buffer issue detected */ +LZ4_FORCE_INLINE int LZ4HC_encodeSequence ( + const BYTE** _ip, + BYTE** _op, + const BYTE** _anchor, + int matchLength, + int offset, + limitedOutput_directive limit, + BYTE* oend) +{ +#define ip (*_ip) +#define op (*_op) +#define anchor (*_anchor) + + size_t length; + BYTE* const token = op++; + +#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 6) + static const BYTE* start = NULL; + static U32 totalCost = 0; + U32 const pos = (start==NULL) ? 0 : (U32)(anchor - start); + U32 const ll = (U32)(ip - anchor); + U32 const llAdd = (ll>=15) ? ((ll-15) / 255) + 1 : 0; + U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0; + U32 const cost = 1 + llAdd + ll + 2 + mlAdd; + if (start==NULL) start = anchor; /* only works for single segment */ + /* g_debuglog_enable = (pos >= 2228) & (pos <= 2262); */ + DEBUGLOG(6, "pos:%7u -- literals:%4u, match:%4i, offset:%5i, cost:%4u + %5u", + pos, + (U32)(ip - anchor), matchLength, offset, + cost, totalCost); + totalCost += cost; +#endif + + /* Encode Literal length */ + length = (size_t)(ip - anchor); + LZ4_STATIC_ASSERT(notLimited == 0); + /* Check output limit */ + if (limit && ((op + (length / 255) + length + (2 + 1 + LASTLITERALS)) > oend)) { + DEBUGLOG(6, "Not enough room to write %i literals (%i bytes remaining)", + (int)length, (int)(oend - op)); + return 1; + } + if (length >= RUN_MASK) { + size_t len = length - RUN_MASK; + *token = (RUN_MASK << ML_BITS); + for(; len >= 255 ; len -= 255) *op++ = 255; + *op++ = (BYTE)len; + } else { + *token = (BYTE)(length << ML_BITS); + } + + /* Copy Literals */ + LZ4_wildCopy8(op, anchor, op + length); + op += length; + + /* Encode Offset */ + assert(offset <= LZ4_DISTANCE_MAX ); + assert(offset > 0); + LZ4_writeLE16(op, (U16)(offset)); op += 2; + + /* Encode MatchLength */ + assert(matchLength >= MINMATCH); + length = (size_t)matchLength - MINMATCH; + if (limit && (op + (length / 255) + (1 + LASTLITERALS) > oend)) { + DEBUGLOG(6, "Not enough room to write match length"); + return 1; /* Check output limit */ + } + if (length >= ML_MASK) { + *token += ML_MASK; + length -= ML_MASK; + for(; length >= 510 ; length -= 510) { *op++ = 255; *op++ = 255; } + if (length >= 255) { length -= 255; *op++ = 255; } + *op++ = (BYTE)length; + } else { + *token += (BYTE)(length); + } + + /* Prepare next loop */ + ip += matchLength; + anchor = ip; + + return 0; + +#undef ip +#undef op +#undef anchor +} + + +typedef struct { + int off; + int len; + int back; /* negative value */ +} LZ4HC_match_t; + +LZ4HC_match_t LZ4HC_searchExtDict(const BYTE* ip, U32 ipIndex, + const BYTE* const iLowLimit, const BYTE* const iHighLimit, + const LZ4HC_CCtx_internal* dictCtx, U32 gDictEndIndex, + int currentBestML, int nbAttempts) +{ + size_t const lDictEndIndex = (size_t)(dictCtx->end - dictCtx->prefixStart) + dictCtx->dictLimit; + U32 lDictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; + U32 matchIndex = lDictMatchIndex + gDictEndIndex - (U32)lDictEndIndex; + int offset = 0, sBack = 0; + assert(lDictEndIndex <= 1 GB); + if (lDictMatchIndex>0) + DEBUGLOG(7, "lDictEndIndex = %zu, lDictMatchIndex = %u", lDictEndIndex, lDictMatchIndex); + while (ipIndex - matchIndex <= LZ4_DISTANCE_MAX && nbAttempts--) { + const BYTE* const matchPtr = dictCtx->prefixStart - dictCtx->dictLimit + lDictMatchIndex; + + if (LZ4_read32(matchPtr) == LZ4_read32(ip)) { + int mlt; + int back = 0; + const BYTE* vLimit = ip + (lDictEndIndex - lDictMatchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + mlt = (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; + back = (ip > iLowLimit) ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->prefixStart) : 0; + mlt -= back; + if (mlt > currentBestML) { + currentBestML = mlt; + offset = (int)(ipIndex - matchIndex); + sBack = back; + DEBUGLOG(7, "found match of length %i within extDictCtx", currentBestML); + } } + + { U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, lDictMatchIndex); + lDictMatchIndex -= nextOffset; + matchIndex -= nextOffset; + } } + + { LZ4HC_match_t md; + md.len = currentBestML; + md.off = offset; + md.back = sBack; + return md; + } +} + +/************************************** +* Mid Compression (level 2) +**************************************/ + +LZ4_FORCE_INLINE void +LZ4MID_addPosition(U32* hTable, U32 hValue, U32 index) +{ + hTable[hValue] = index; +} + +#define ADDPOS8(_p, _idx) LZ4MID_addPosition(hash8Table, LZ4MID_hash8Ptr(_p), _idx) +#define ADDPOS4(_p, _idx) LZ4MID_addPosition(hash4Table, LZ4MID_hash4Ptr(_p), _idx) + +static int LZ4HC_compress_2hashes ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* srcSizePtr, + int const maxOutputSize, + const limitedOutput_directive limit, + const dictCtx_directive dict + ) +{ + U32* const hash4Table = ctx->hashTable; + U32* const hash8Table = hash4Table + LZ4MID_HASHTABLESIZE; + const BYTE* ip = (const BYTE*)src; + const BYTE* anchor = ip; + const BYTE* const iend = ip + *srcSizePtr; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = (iend - LASTLITERALS); + const BYTE* const ilimit = (iend - LZ4MID_HASHSIZE); + BYTE* op = (BYTE*)dst; + BYTE* oend = op + maxOutputSize; + + const BYTE* const prefixPtr = ctx->prefixStart; + const U32 prefixIdx = ctx->dictLimit; + const U32 ilimitIdx = (U32)(ilimit - prefixPtr) + prefixIdx; + const U32 gDictEndIndex = ctx->lowLimit; + unsigned matchLength; + unsigned matchDistance; + + /* input sanitization */ + DEBUGLOG(5, "LZ4HC_compress_2hashes (%i bytes)", *srcSizePtr); + assert(*srcSizePtr >= 0); + if (*srcSizePtr) assert(src != NULL); + if (maxOutputSize) assert(dst != NULL); + if (*srcSizePtr < 0) return 0; /* invalid */ + if (maxOutputSize < 0) return 0; /* invalid */ + if (*srcSizePtr > LZ4_MAX_INPUT_SIZE) { + /* forbidden: no input is allowed to be that large */ + return 0; + } + if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ + if (*srcSizePtr < LZ4_minLength) + goto _lz4mid_last_literals; /* Input too small, no compression (all literals) */ + + /* main loop */ + while (ip <= mflimit) { + const U32 ipIndex = (U32)(ip - prefixPtr) + prefixIdx; + /* search long match */ + { U32 h8 = LZ4MID_hash8Ptr(ip); + U32 pos8 = hash8Table[h8]; + assert(h8 < LZ4MID_HASHTABLESIZE); + assert(h8 < ipIndex); + LZ4MID_addPosition(hash8Table, h8, ipIndex); + if ( ipIndex - pos8 <= LZ4_DISTANCE_MAX + && pos8 >= prefixIdx /* note: currently only search within prefix */ + ) { + /* match candidate found */ + const BYTE* matchPtr = prefixPtr + pos8 - prefixIdx; + assert(matchPtr < ip); + matchLength = LZ4_count(ip, matchPtr, matchlimit); + if (matchLength >= MINMATCH) { + DEBUGLOG(7, "found candidate match at pos %u (len=%u)", pos8, matchLength); + matchDistance = ipIndex - pos8; + goto _lz4mid_encode_sequence; + } + } } + /* search short match */ + { U32 h4 = LZ4MID_hash4Ptr(ip); + U32 pos4 = hash4Table[h4]; + assert(h4 < LZ4MID_HASHTABLESIZE); + assert(pos4 < ipIndex); + LZ4MID_addPosition(hash4Table, h4, ipIndex); + if (ipIndex - pos4 <= LZ4_DISTANCE_MAX + && pos4 >= prefixIdx /* only search within prefix */ + ) { + /* match candidate found */ + const BYTE* const matchPtr = prefixPtr + (pos4 - prefixIdx); + assert(matchPtr < ip); + assert(matchPtr >= prefixPtr); + matchLength = LZ4_count(ip, matchPtr, matchlimit); + if (matchLength >= MINMATCH) { + /* short match found, let's just check ip+1 for longer */ + U32 const h8 = LZ4MID_hash8Ptr(ip+1); + U32 const pos8 = hash8Table[h8]; + U32 const m2Distance = ipIndex + 1 - pos8; + matchDistance = ipIndex - pos4; + if ( m2Distance <= LZ4_DISTANCE_MAX + && pos8 >= prefixIdx /* only search within prefix */ + && likely(ip < mflimit) + ) { + const BYTE* const m2Ptr = prefixPtr + (pos8 - prefixIdx); + unsigned ml2 = LZ4_count(ip+1, m2Ptr, matchlimit); + if (ml2 > matchLength) { + LZ4MID_addPosition(hash8Table, h8, ipIndex+1); + ip++; + matchLength = ml2; + matchDistance = m2Distance; + } } + goto _lz4mid_encode_sequence; + } + } } + /* no match found in prefix */ + if ( (dict == usingDictCtxHc) + && (ipIndex - gDictEndIndex < LZ4_DISTANCE_MAX - 8) ) { + /* search a match in dictionary */ + LZ4HC_match_t dMatch = LZ4HC_searchExtDict(ip, ipIndex, + anchor, matchlimit, + ctx->dictCtx, gDictEndIndex, + 0, 2); + if (dMatch.len >= MINMATCH) { + DEBUGLOG(7, "found Dictionary match (offset=%i)", dMatch.off); + ip += dMatch.back; + assert(ip >= anchor); + matchLength = (unsigned)dMatch.len; + matchDistance = (unsigned)dMatch.off; + goto _lz4mid_encode_sequence; + } + } + /* no match found */ + ip += 1 + ((ip-anchor) >> 9); /* skip faster over incompressible data */ + continue; + +_lz4mid_encode_sequence: + /* catch back */ + while (((ip > anchor) & ((U32)(ip-prefixPtr) > matchDistance)) && (unlikely(ip[-1] == ip[-(int)matchDistance-1]))) { + ip--; matchLength++; + }; + + /* fill table with beginning of match */ + ADDPOS8(ip+1, ipIndex+1); + ADDPOS8(ip+2, ipIndex+2); + ADDPOS4(ip+1, ipIndex+1); + + /* encode */ + { BYTE* const saved_op = op; + /* LZ4HC_encodeSequence always updates @op; on success, it updates @ip and @anchor */ + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), + (int)matchLength, (int)matchDistance, + limit, oend) ) { + op = saved_op; /* restore @op value before failed LZ4HC_encodeSequence */ + goto _lz4mid_dest_overflow; + } + } + + /* fill table with end of match */ + { U32 endMatchIdx = (U32)(ip-prefixPtr) + prefixIdx; + U32 pos_m2 = endMatchIdx - 2; + if (pos_m2 < ilimitIdx) { + if (likely(ip - prefixPtr > 5)) { + ADDPOS8(ip-5, endMatchIdx - 5); + } + ADDPOS8(ip-3, endMatchIdx - 3); + ADDPOS8(ip-2, endMatchIdx - 2); + ADDPOS4(ip-2, endMatchIdx - 2); + ADDPOS4(ip-1, endMatchIdx - 1); + } + } + } + +_lz4mid_last_literals: + /* Encode Last Literals */ + { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ + size_t llAdd = (lastRunSize + 255 - RUN_MASK) / 255; + size_t const totalSize = 1 + llAdd + lastRunSize; + if (limit == fillOutput) oend += LASTLITERALS; /* restore correct value */ + if (limit && (op + totalSize > oend)) { + if (limit == limitedOutput) return 0; /* not enough space in @dst */ + /* adapt lastRunSize to fill 'dest' */ + lastRunSize = (size_t)(oend - op) - 1 /*token*/; + llAdd = (lastRunSize + 256 - RUN_MASK) / 256; + lastRunSize -= llAdd; + } + DEBUGLOG(6, "Final literal run : %i literals", (int)lastRunSize); + ip = anchor + lastRunSize; /* can be != iend if limit==fillOutput */ + + if (lastRunSize >= RUN_MASK) { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = (RUN_MASK << ML_BITS); + for(; accumulator >= 255 ; accumulator -= 255) + *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRunSize << ML_BITS); + } + assert(lastRunSize <= (size_t)(oend - op)); + LZ4_memcpy(op, anchor, lastRunSize); + op += lastRunSize; + } + + /* End */ + DEBUGLOG(5, "compressed %i bytes into %i bytes", *srcSizePtr, (int)((char*)op - dst)); + assert(ip >= (const BYTE*)src); + assert(ip <= iend); + *srcSizePtr = (int)(ip - (const BYTE*)src); + assert((char*)op >= dst); + assert(op <= oend); + assert((char*)op - dst < INT_MAX); + return (int)((char*)op - dst); + +_lz4mid_dest_overflow: + if (limit == fillOutput) { + /* Assumption : @ip, @anchor, @optr and @matchLength must be set correctly */ + size_t const ll = (size_t)(ip - anchor); + size_t const ll_addbytes = (ll + 240) / 255; + size_t const ll_totalCost = 1 + ll_addbytes + ll; + BYTE* const maxLitPos = oend - 3; /* 2 for offset, 1 for token */ + DEBUGLOG(6, "Last sequence is overflowing : %u literals, %u remaining space", + (unsigned)ll, (unsigned)(oend-op)); + if (op + ll_totalCost <= maxLitPos) { + /* ll validated; now adjust match length */ + size_t const bytesLeftForMl = (size_t)(maxLitPos - (op+ll_totalCost)); + size_t const maxMlSize = MINMATCH + (ML_MASK-1) + (bytesLeftForMl * 255); + assert(maxMlSize < INT_MAX); + if ((size_t)matchLength > maxMlSize) matchLength= (unsigned)maxMlSize; + if ((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1 + matchLength >= MFLIMIT) { + DEBUGLOG(6, "Let's encode a last sequence (ll=%u, ml=%u)", (unsigned)ll, matchLength); + LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), + (int)matchLength, (int)matchDistance, + notLimited, oend); + } } + DEBUGLOG(6, "Let's finish with a run of literals (%u bytes left)", (unsigned)(oend-op)); + goto _lz4mid_last_literals; + } + /* compression failed */ + return 0; +} + + +/************************************** +* HC Compression - Search +**************************************/ + +/* Update chains up to ip (excluded) */ +LZ4_FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip) +{ + U16* const chainTable = hc4->chainTable; + U32* const hashTable = hc4->hashTable; + const BYTE* const prefixPtr = hc4->prefixStart; + U32 const prefixIdx = hc4->dictLimit; + U32 const target = (U32)(ip - prefixPtr) + prefixIdx; + U32 idx = hc4->nextToUpdate; + assert(ip >= prefixPtr); + assert(target >= prefixIdx); + + while (idx < target) { + U32 const h = LZ4HC_hashPtr(prefixPtr+idx-prefixIdx); + size_t delta = idx - hashTable[h]; + if (delta>LZ4_DISTANCE_MAX) delta = LZ4_DISTANCE_MAX; + DELTANEXTU16(chainTable, idx) = (U16)delta; + hashTable[h] = idx; + idx++; + } + + hc4->nextToUpdate = target; +} + +#if defined(_MSC_VER) +# define LZ4HC_rotl32(x,r) _rotl(x,r) +#else +# define LZ4HC_rotl32(x,r) ((x << r) | (x >> (32 - r))) +#endif + + +static U32 LZ4HC_rotatePattern(size_t const rotate, U32 const pattern) +{ + size_t const bitsToRotate = (rotate & (sizeof(pattern) - 1)) << 3; + if (bitsToRotate == 0) return pattern; + return LZ4HC_rotl32(pattern, (int)bitsToRotate); +} + +/* LZ4HC_countPattern() : + * pattern32 must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) */ +static unsigned +LZ4HC_countPattern(const BYTE* ip, const BYTE* const iEnd, U32 const pattern32) +{ + const BYTE* const iStart = ip; + reg_t const pattern = (sizeof(pattern)==8) ? + (reg_t)pattern32 + (((reg_t)pattern32) << (sizeof(pattern)*4)) : pattern32; + + while (likely(ip < iEnd-(sizeof(pattern)-1))) { + reg_t const diff = LZ4_read_ARCH(ip) ^ pattern; + if (!diff) { ip+=sizeof(pattern); continue; } + ip += LZ4_NbCommonBytes(diff); + return (unsigned)(ip - iStart); + } + + if (LZ4_isLittleEndian()) { + reg_t patternByte = pattern; + while ((ip>= 8; + } + } else { /* big endian */ + U32 bitOffset = (sizeof(pattern)*8) - 8; + while (ip < iEnd) { + BYTE const byte = (BYTE)(pattern >> bitOffset); + if (*ip != byte) break; + ip ++; bitOffset -= 8; + } } + + return (unsigned)(ip - iStart); +} + +/* LZ4HC_reverseCountPattern() : + * pattern must be a sample of repetitive pattern of length 1, 2 or 4 (but not 3!) + * read using natural platform endianness */ +static unsigned +LZ4HC_reverseCountPattern(const BYTE* ip, const BYTE* const iLow, U32 pattern) +{ + const BYTE* const iStart = ip; + + while (likely(ip >= iLow+4)) { + if (LZ4_read32(ip-4) != pattern) break; + ip -= 4; + } + { const BYTE* bytePtr = (const BYTE*)(&pattern) + 3; /* works for any endianness */ + while (likely(ip>iLow)) { + if (ip[-1] != *bytePtr) break; + ip--; bytePtr--; + } } + return (unsigned)(iStart - ip); +} + +/* LZ4HC_protectDictEnd() : + * Checks if the match is in the last 3 bytes of the dictionary, so reading the + * 4 byte MINMATCH would overflow. + * @returns true if the match index is okay. + */ +static int LZ4HC_protectDictEnd(U32 const dictLimit, U32 const matchIndex) +{ + return ((U32)((dictLimit - 1) - matchIndex) >= 3); +} + +typedef enum { rep_untested, rep_not, rep_confirmed } repeat_state_e; +typedef enum { favorCompressionRatio=0, favorDecompressionSpeed } HCfavor_e; + + +LZ4_FORCE_INLINE LZ4HC_match_t +LZ4HC_InsertAndGetWiderMatch ( + LZ4HC_CCtx_internal* const hc4, + const BYTE* const ip, + const BYTE* const iLowLimit, const BYTE* const iHighLimit, + int longest, + const int maxNbAttempts, + const int patternAnalysis, const int chainSwap, + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed) +{ + U16* const chainTable = hc4->chainTable; + U32* const hashTable = hc4->hashTable; + const LZ4HC_CCtx_internal* const dictCtx = hc4->dictCtx; + const BYTE* const prefixPtr = hc4->prefixStart; + const U32 prefixIdx = hc4->dictLimit; + const U32 ipIndex = (U32)(ip - prefixPtr) + prefixIdx; + const int withinStartDistance = (hc4->lowLimit + (LZ4_DISTANCE_MAX + 1) > ipIndex); + const U32 lowestMatchIndex = (withinStartDistance) ? hc4->lowLimit : ipIndex - LZ4_DISTANCE_MAX; + const BYTE* const dictStart = hc4->dictStart; + const U32 dictIdx = hc4->lowLimit; + const BYTE* const dictEnd = dictStart + prefixIdx - dictIdx; + int const lookBackLength = (int)(ip-iLowLimit); + int nbAttempts = maxNbAttempts; + U32 matchChainPos = 0; + U32 const pattern = LZ4_read32(ip); + U32 matchIndex; + repeat_state_e repeat = rep_untested; + size_t srcPatternLength = 0; + int offset = 0, sBack = 0; + + DEBUGLOG(7, "LZ4HC_InsertAndGetWiderMatch"); + /* First Match */ + LZ4HC_Insert(hc4, ip); /* insert all prior positions up to ip (excluded) */ + matchIndex = hashTable[LZ4HC_hashPtr(ip)]; + DEBUGLOG(7, "First candidate match for pos %u found at index %u / %u (lowestMatchIndex)", + ipIndex, matchIndex, lowestMatchIndex); + + while ((matchIndex>=lowestMatchIndex) && (nbAttempts>0)) { + int matchLength=0; + nbAttempts--; + assert(matchIndex < ipIndex); + if (favorDecSpeed && (ipIndex - matchIndex < 8)) { + /* do nothing: + * favorDecSpeed intentionally skips matches with offset < 8 */ + } else if (matchIndex >= prefixIdx) { /* within current Prefix */ + const BYTE* const matchPtr = prefixPtr + (matchIndex - prefixIdx); + assert(matchPtr < ip); + assert(longest >= 1); + if (LZ4_read16(iLowLimit + longest - 1) == LZ4_read16(matchPtr - lookBackLength + longest - 1)) { + if (LZ4_read32(matchPtr) == pattern) { + int const back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, prefixPtr) : 0; + matchLength = MINMATCH + (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit); + matchLength -= back; + if (matchLength > longest) { + longest = matchLength; + offset = (int)(ipIndex - matchIndex); + sBack = back; + DEBUGLOG(7, "Found match of len=%i within prefix, offset=%i, back=%i", longest, offset, -back); + } } } + } else { /* lowestMatchIndex <= matchIndex < dictLimit : within Ext Dict */ + const BYTE* const matchPtr = dictStart + (matchIndex - dictIdx); + assert(matchIndex >= dictIdx); + if ( likely(matchIndex <= prefixIdx - 4) + && (LZ4_read32(matchPtr) == pattern) ) { + int back = 0; + const BYTE* vLimit = ip + (prefixIdx - matchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + matchLength = (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; + if ((ip+matchLength == vLimit) && (vLimit < iHighLimit)) + matchLength += LZ4_count(ip+matchLength, prefixPtr, iHighLimit); + back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictStart) : 0; + matchLength -= back; + if (matchLength > longest) { + longest = matchLength; + offset = (int)(ipIndex - matchIndex); + sBack = back; + DEBUGLOG(7, "Found match of len=%i within dict, offset=%i, back=%i", longest, offset, -back); + } } } + + if (chainSwap && matchLength==longest) { /* better match => select a better chain */ + assert(lookBackLength==0); /* search forward only */ + if (matchIndex + (U32)longest <= ipIndex) { + int const kTrigger = 4; + U32 distanceToNextMatch = 1; + int const end = longest - MINMATCH + 1; + int step = 1; + int accel = 1 << kTrigger; + int pos; + for (pos = 0; pos < end; pos += step) { + U32 const candidateDist = DELTANEXTU16(chainTable, matchIndex + (U32)pos); + step = (accel++ >> kTrigger); + if (candidateDist > distanceToNextMatch) { + distanceToNextMatch = candidateDist; + matchChainPos = (U32)pos; + accel = 1 << kTrigger; + } } + if (distanceToNextMatch > 1) { + if (distanceToNextMatch > matchIndex) break; /* avoid overflow */ + matchIndex -= distanceToNextMatch; + continue; + } } } + + { U32 const distNextMatch = DELTANEXTU16(chainTable, matchIndex); + if (patternAnalysis && distNextMatch==1 && matchChainPos==0) { + U32 const matchCandidateIdx = matchIndex-1; + /* may be a repeated pattern */ + if (repeat == rep_untested) { + if ( ((pattern & 0xFFFF) == (pattern >> 16)) + & ((pattern & 0xFF) == (pattern >> 24)) ) { + DEBUGLOG(7, "Repeat pattern detected, char %02X", pattern >> 24); + repeat = rep_confirmed; + srcPatternLength = LZ4HC_countPattern(ip+sizeof(pattern), iHighLimit, pattern) + sizeof(pattern); + } else { + repeat = rep_not; + } } + if ( (repeat == rep_confirmed) && (matchCandidateIdx >= lowestMatchIndex) + && LZ4HC_protectDictEnd(prefixIdx, matchCandidateIdx) ) { + const int extDict = matchCandidateIdx < prefixIdx; + const BYTE* const matchPtr = extDict ? dictStart + (matchCandidateIdx - dictIdx) : prefixPtr + (matchCandidateIdx - prefixIdx); + if (LZ4_read32(matchPtr) == pattern) { /* good candidate */ + const BYTE* const iLimit = extDict ? dictEnd : iHighLimit; + size_t forwardPatternLength = LZ4HC_countPattern(matchPtr+sizeof(pattern), iLimit, pattern) + sizeof(pattern); + if (extDict && matchPtr + forwardPatternLength == iLimit) { + U32 const rotatedPattern = LZ4HC_rotatePattern(forwardPatternLength, pattern); + forwardPatternLength += LZ4HC_countPattern(prefixPtr, iHighLimit, rotatedPattern); + } + { const BYTE* const lowestMatchPtr = extDict ? dictStart : prefixPtr; + size_t backLength = LZ4HC_reverseCountPattern(matchPtr, lowestMatchPtr, pattern); + size_t currentSegmentLength; + if (!extDict + && matchPtr - backLength == prefixPtr + && dictIdx < prefixIdx) { + U32 const rotatedPattern = LZ4HC_rotatePattern((U32)(-(int)backLength), pattern); + backLength += LZ4HC_reverseCountPattern(dictEnd, dictStart, rotatedPattern); + } + /* Limit backLength not go further than lowestMatchIndex */ + backLength = matchCandidateIdx - MAX(matchCandidateIdx - (U32)backLength, lowestMatchIndex); + assert(matchCandidateIdx - backLength >= lowestMatchIndex); + currentSegmentLength = backLength + forwardPatternLength; + /* Adjust to end of pattern if the source pattern fits, otherwise the beginning of the pattern */ + if ( (currentSegmentLength >= srcPatternLength) /* current pattern segment large enough to contain full srcPatternLength */ + && (forwardPatternLength <= srcPatternLength) ) { /* haven't reached this position yet */ + U32 const newMatchIndex = matchCandidateIdx + (U32)forwardPatternLength - (U32)srcPatternLength; /* best position, full pattern, might be followed by more match */ + if (LZ4HC_protectDictEnd(prefixIdx, newMatchIndex)) + matchIndex = newMatchIndex; + else { + /* Can only happen if started in the prefix */ + assert(newMatchIndex >= prefixIdx - 3 && newMatchIndex < prefixIdx && !extDict); + matchIndex = prefixIdx; + } + } else { + U32 const newMatchIndex = matchCandidateIdx - (U32)backLength; /* farthest position in current segment, will find a match of length currentSegmentLength + maybe some back */ + if (!LZ4HC_protectDictEnd(prefixIdx, newMatchIndex)) { + assert(newMatchIndex >= prefixIdx - 3 && newMatchIndex < prefixIdx && !extDict); + matchIndex = prefixIdx; + } else { + matchIndex = newMatchIndex; + if (lookBackLength==0) { /* no back possible */ + size_t const maxML = MIN(currentSegmentLength, srcPatternLength); + if ((size_t)longest < maxML) { + assert(prefixPtr - prefixIdx + matchIndex != ip); + if ((size_t)(ip - prefixPtr) + prefixIdx - matchIndex > LZ4_DISTANCE_MAX) break; + assert(maxML < 2 GB); + longest = (int)maxML; + offset = (int)(ipIndex - matchIndex); + assert(sBack == 0); + DEBUGLOG(7, "Found repeat pattern match of len=%i, offset=%i", longest, offset); + } + { U32 const distToNextPattern = DELTANEXTU16(chainTable, matchIndex); + if (distToNextPattern > matchIndex) break; /* avoid overflow */ + matchIndex -= distToNextPattern; + } } } } } + continue; + } } + } } /* PA optimization */ + + /* follow current chain */ + matchIndex -= DELTANEXTU16(chainTable, matchIndex + matchChainPos); + + } /* while ((matchIndex>=lowestMatchIndex) && (nbAttempts)) */ + + if ( dict == usingDictCtxHc + && nbAttempts > 0 + && withinStartDistance) { + size_t const dictEndOffset = (size_t)(dictCtx->end - dictCtx->prefixStart) + dictCtx->dictLimit; + U32 dictMatchIndex = dictCtx->hashTable[LZ4HC_hashPtr(ip)]; + assert(dictEndOffset <= 1 GB); + matchIndex = dictMatchIndex + lowestMatchIndex - (U32)dictEndOffset; + if (dictMatchIndex>0) DEBUGLOG(7, "dictEndOffset = %zu, dictMatchIndex = %u => relative matchIndex = %i", dictEndOffset, dictMatchIndex, (int)dictMatchIndex - (int)dictEndOffset); + while (ipIndex - matchIndex <= LZ4_DISTANCE_MAX && nbAttempts--) { + const BYTE* const matchPtr = dictCtx->prefixStart - dictCtx->dictLimit + dictMatchIndex; + + if (LZ4_read32(matchPtr) == pattern) { + int mlt; + int back = 0; + const BYTE* vLimit = ip + (dictEndOffset - dictMatchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + mlt = (int)LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; + back = lookBackLength ? LZ4HC_countBack(ip, matchPtr, iLowLimit, dictCtx->prefixStart) : 0; + mlt -= back; + if (mlt > longest) { + longest = mlt; + offset = (int)(ipIndex - matchIndex); + sBack = back; + DEBUGLOG(7, "found match of length %i within extDictCtx", longest); + } } + + { U32 const nextOffset = DELTANEXTU16(dictCtx->chainTable, dictMatchIndex); + dictMatchIndex -= nextOffset; + matchIndex -= nextOffset; + } } } + + { LZ4HC_match_t md; + assert(longest >= 0); + md.len = longest; + md.off = offset; + md.back = sBack; + return md; + } +} + +LZ4_FORCE_INLINE LZ4HC_match_t +LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index table will be updated */ + const BYTE* const ip, const BYTE* const iLimit, + const int maxNbAttempts, + const int patternAnalysis, + const dictCtx_directive dict) +{ + DEBUGLOG(7, "LZ4HC_InsertAndFindBestMatch"); + /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), + * but this won't be the case here, as we define iLowLimit==ip, + * so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ + return LZ4HC_InsertAndGetWiderMatch(hc4, ip, ip, iLimit, MINMATCH-1, maxNbAttempts, patternAnalysis, 0 /*chainSwap*/, dict, favorCompressionRatio); +} + + +LZ4_FORCE_INLINE int LZ4HC_compress_hashChain ( + LZ4HC_CCtx_internal* const ctx, + const char* const source, + char* const dest, + int* srcSizePtr, + int const maxOutputSize, + int maxNbAttempts, + const limitedOutput_directive limit, + const dictCtx_directive dict + ) +{ + const int inputSize = *srcSizePtr; + const int patternAnalysis = (maxNbAttempts > 128); /* levels 9+ */ + + const BYTE* ip = (const BYTE*) source; + const BYTE* anchor = ip; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = (iend - LASTLITERALS); + + BYTE* optr = (BYTE*) dest; + BYTE* op = (BYTE*) dest; + BYTE* oend = op + maxOutputSize; + + const BYTE* start0; + const BYTE* start2 = NULL; + const BYTE* start3 = NULL; + LZ4HC_match_t m0, m1, m2, m3; + const LZ4HC_match_t nomatch = {0, 0, 0}; + + /* init */ + DEBUGLOG(5, "LZ4HC_compress_hashChain (dict?=>%i)", dict); + *srcSizePtr = 0; + if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ + if (inputSize < LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */ + + /* Main Loop */ + while (ip <= mflimit) { + m1 = LZ4HC_InsertAndFindBestMatch(ctx, ip, matchlimit, maxNbAttempts, patternAnalysis, dict); + if (m1.len encode ML1 immediately */ + optr = op; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), + m1.len, m1.off, + limit, oend) ) + goto _dest_overflow; + continue; + } + + if (start0 < ip) { /* first match was skipped at least once */ + if (start2 < ip + m0.len) { /* squeezing ML1 between ML0(original ML1) and ML2 */ + ip = start0; m1 = m0; /* restore initial Match1 */ + } } + + /* Here, start0==ip */ + if ((start2 - ip) < 3) { /* First Match too small : removed */ + ip = start2; + m1 = m2; + goto _Search2; + } + +_Search3: + if ((start2 - ip) < OPTIMAL_ML) { + int correction; + int new_ml = m1.len; + if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML; + if (ip+new_ml > start2 + m2.len - MINMATCH) + new_ml = (int)(start2 - ip) + m2.len - MINMATCH; + correction = new_ml - (int)(start2 - ip); + if (correction > 0) { + start2 += correction; + m2.len -= correction; + } + } + + if (start2 + m2.len <= mflimit) { + start3 = start2 + m2.len - 3; + m3 = LZ4HC_InsertAndGetWiderMatch(ctx, + start3, start2, matchlimit, m2.len, + maxNbAttempts, patternAnalysis, 0, dict, favorCompressionRatio); + start3 += m3.back; + } else { + m3 = nomatch; /* do not search further */ + } + + if (m3.len <= m2.len) { /* No better match => encode ML1 and ML2 */ + /* ip & ref are known; Now for ml */ + if (start2 < ip+m1.len) m1.len = (int)(start2 - ip); + /* Now, encode 2 sequences */ + optr = op; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), + m1.len, m1.off, + limit, oend) ) + goto _dest_overflow; + ip = start2; + optr = op; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), + m2.len, m2.off, + limit, oend) ) { + m1 = m2; + goto _dest_overflow; + } + continue; + } + + if (start3 < ip+m1.len+3) { /* Not enough space for match 2 : remove it */ + if (start3 >= (ip+m1.len)) { /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */ + if (start2 < ip+m1.len) { + int correction = (int)(ip+m1.len - start2); + start2 += correction; + m2.len -= correction; + if (m2.len < MINMATCH) { + start2 = start3; + m2 = m3; + } + } + + optr = op; + if (LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), + m1.len, m1.off, + limit, oend) ) + goto _dest_overflow; + ip = start3; + m1 = m3; + + start0 = start2; + m0 = m2; + goto _Search2; + } + + start2 = start3; + m2 = m3; + goto _Search3; + } + + /* + * OK, now we have 3 ascending matches; + * let's write the first one ML1. + * ip & ref are known; Now decide ml. + */ + if (start2 < ip+m1.len) { + if ((start2 - ip) < OPTIMAL_ML) { + int correction; + if (m1.len > OPTIMAL_ML) m1.len = OPTIMAL_ML; + if (ip + m1.len > start2 + m2.len - MINMATCH) + m1.len = (int)(start2 - ip) + m2.len - MINMATCH; + correction = m1.len - (int)(start2 - ip); + if (correction > 0) { + start2 += correction; + m2.len -= correction; + } + } else { + m1.len = (int)(start2 - ip); + } + } + optr = op; + if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), + m1.len, m1.off, + limit, oend) ) + goto _dest_overflow; + + /* ML2 becomes ML1 */ + ip = start2; m1 = m2; + + /* ML3 becomes ML2 */ + start2 = start3; m2 = m3; + + /* let's find a new ML3 */ + goto _Search3; + } + +_last_literals: + /* Encode Last Literals */ + { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ + size_t llAdd = (lastRunSize + 255 - RUN_MASK) / 255; + size_t const totalSize = 1 + llAdd + lastRunSize; + if (limit == fillOutput) oend += LASTLITERALS; /* restore correct value */ + if (limit && (op + totalSize > oend)) { + if (limit == limitedOutput) return 0; + /* adapt lastRunSize to fill 'dest' */ + lastRunSize = (size_t)(oend - op) - 1 /*token*/; + llAdd = (lastRunSize + 256 - RUN_MASK) / 256; + lastRunSize -= llAdd; + } + DEBUGLOG(6, "Final literal run : %i literals", (int)lastRunSize); + ip = anchor + lastRunSize; /* can be != iend if limit==fillOutput */ + + if (lastRunSize >= RUN_MASK) { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = (RUN_MASK << ML_BITS); + for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRunSize << ML_BITS); + } + LZ4_memcpy(op, anchor, lastRunSize); + op += lastRunSize; + } + + /* End */ + *srcSizePtr = (int) (((const char*)ip) - source); + return (int) (((char*)op)-dest); + +_dest_overflow: + if (limit == fillOutput) { + /* Assumption : @ip, @anchor, @optr and @m1 must be set correctly */ + size_t const ll = (size_t)(ip - anchor); + size_t const ll_addbytes = (ll + 240) / 255; + size_t const ll_totalCost = 1 + ll_addbytes + ll; + BYTE* const maxLitPos = oend - 3; /* 2 for offset, 1 for token */ + DEBUGLOG(6, "Last sequence overflowing"); + op = optr; /* restore correct out pointer */ + if (op + ll_totalCost <= maxLitPos) { + /* ll validated; now adjust match length */ + size_t const bytesLeftForMl = (size_t)(maxLitPos - (op+ll_totalCost)); + size_t const maxMlSize = MINMATCH + (ML_MASK-1) + (bytesLeftForMl * 255); + assert(maxMlSize < INT_MAX); assert(m1.len >= 0); + if ((size_t)m1.len > maxMlSize) m1.len = (int)maxMlSize; + if ((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1 + m1.len >= MFLIMIT) { + LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), m1.len, m1.off, notLimited, oend); + } } + goto _last_literals; + } + /* compression failed */ + return 0; +} + + +static int LZ4HC_compress_optimal( LZ4HC_CCtx_internal* ctx, + const char* const source, char* dst, + int* srcSizePtr, int dstCapacity, + int const nbSearches, size_t sufficient_len, + const limitedOutput_directive limit, int const fullUpdate, + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed); + + +LZ4_FORCE_INLINE int +LZ4HC_compress_generic_internal ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + const limitedOutput_directive limit, + const dictCtx_directive dict + ) +{ + typedef enum { lz4mid, lz4hc, lz4opt } lz4hc_strat_e; + typedef struct { + lz4hc_strat_e strat; + int nbSearches; + U32 targetLength; + } cParams_t; + static const cParams_t clTable[LZ4HC_CLEVEL_MAX+1] = { + { lz4mid, 2, 16 }, /* 0, unused */ + { lz4mid, 2, 16 }, /* 1, unused */ + { lz4mid, 2, 16 }, /* 2 */ + { lz4hc, 4, 16 }, /* 3 */ + { lz4hc, 8, 16 }, /* 4 */ + { lz4hc, 16, 16 }, /* 5 */ + { lz4hc, 32, 16 }, /* 6 */ + { lz4hc, 64, 16 }, /* 7 */ + { lz4hc, 128, 16 }, /* 8 */ + { lz4hc, 256, 16 }, /* 9 */ + { lz4opt, 96, 64 }, /*10==LZ4HC_CLEVEL_OPT_MIN*/ + { lz4opt, 512,128 }, /*11 */ + { lz4opt,16384,LZ4_OPT_NUM }, /* 12==LZ4HC_CLEVEL_MAX */ + }; + + DEBUGLOG(5, "LZ4HC_compress_generic_internal(src=%p, srcSize=%d)", + src, *srcSizePtr); + + if (limit == fillOutput && dstCapacity < 1) return 0; /* Impossible to store anything */ + if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size (too large or negative) */ + + ctx->end += *srcSizePtr; + /* note : clevel convention is a bit different from lz4frame, + * possibly something worth revisiting for consistency */ + if (cLevel < 1) + cLevel = LZ4HC_CLEVEL_DEFAULT; + cLevel = MIN(LZ4HC_CLEVEL_MAX, cLevel); + { cParams_t const cParam = clTable[cLevel]; + HCfavor_e const favor = ctx->favorDecSpeed ? favorDecompressionSpeed : favorCompressionRatio; + int result; + + if (cParam.strat == lz4mid) { + result = LZ4HC_compress_2hashes(ctx, + src, dst, srcSizePtr, dstCapacity, + limit, dict); + } else if (cParam.strat == lz4hc) { + result = LZ4HC_compress_hashChain(ctx, + src, dst, srcSizePtr, dstCapacity, + cParam.nbSearches, limit, dict); + } else { + assert(cParam.strat == lz4opt); + result = LZ4HC_compress_optimal(ctx, + src, dst, srcSizePtr, dstCapacity, + cParam.nbSearches, cParam.targetLength, limit, + cLevel == LZ4HC_CLEVEL_MAX, /* ultra mode */ + dict, favor); + } + if (result <= 0) ctx->dirty = 1; + return result; + } +} + +static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock); + +static int +LZ4HC_compress_generic_noDictCtx ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + limitedOutput_directive limit + ) +{ + assert(ctx->dictCtx == NULL); + return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, noDictCtx); +} + +static int +LZ4HC_compress_generic_dictCtx ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + limitedOutput_directive limit + ) +{ + const size_t position = (size_t)(ctx->end - ctx->prefixStart) + (ctx->dictLimit - ctx->lowLimit); + assert(ctx->dictCtx != NULL); + if (position >= 64 KB) { + ctx->dictCtx = NULL; + return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); + } else if (position == 0 && *srcSizePtr > 4 KB) { + LZ4_memcpy(ctx, ctx->dictCtx, sizeof(LZ4HC_CCtx_internal)); + LZ4HC_setExternalDict(ctx, (const BYTE *)src); + ctx->compressionLevel = (short)cLevel; + return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); + } else { + return LZ4HC_compress_generic_internal(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit, usingDictCtxHc); + } +} + +static int +LZ4HC_compress_generic ( + LZ4HC_CCtx_internal* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + int const dstCapacity, + int cLevel, + limitedOutput_directive limit + ) +{ + if (ctx->dictCtx == NULL) { + return LZ4HC_compress_generic_noDictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); + } else { + return LZ4HC_compress_generic_dictCtx(ctx, src, dst, srcSizePtr, dstCapacity, cLevel, limit); + } +} + + +int LZ4_sizeofStateHC(void) { return (int)sizeof(LZ4_streamHC_t); } + +static size_t LZ4_streamHC_t_alignment(void) +{ +#if LZ4_ALIGN_TEST + typedef struct { char c; LZ4_streamHC_t t; } t_a; + return sizeof(t_a) - sizeof(LZ4_streamHC_t); +#else + return 1; /* effectively disabled */ +#endif +} + +/* state is presumed correctly initialized, + * in which case its size and alignment have already been validate */ +int LZ4_compress_HC_extStateHC_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) +{ + LZ4HC_CCtx_internal* const ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; + if (!LZ4_isAligned(state, LZ4_streamHC_t_alignment())) return 0; + LZ4_resetStreamHC_fast((LZ4_streamHC_t*)state, compressionLevel); + LZ4HC_init_internal (ctx, (const BYTE*)src); + if (dstCapacity < LZ4_compressBound(srcSize)) + return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, limitedOutput); + else + return LZ4HC_compress_generic (ctx, src, dst, &srcSize, dstCapacity, compressionLevel, notLimited); +} + +int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) +{ + LZ4_streamHC_t* const ctx = LZ4_initStreamHC(state, sizeof(*ctx)); + if (ctx==NULL) return 0; /* init failure */ + return LZ4_compress_HC_extStateHC_fastReset(state, src, dst, srcSize, dstCapacity, compressionLevel); +} + +int LZ4_compress_HC(const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel) +{ + int cSize; +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 + LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)ALLOC(sizeof(LZ4_streamHC_t)); + if (statePtr==NULL) return 0; +#else + LZ4_streamHC_t state; + LZ4_streamHC_t* const statePtr = &state; +#endif + DEBUGLOG(5, "LZ4_compress_HC") + cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, dstCapacity, compressionLevel); +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 + FREEMEM(statePtr); +#endif + return cSize; +} + +/* state is presumed sized correctly (>= sizeof(LZ4_streamHC_t)) */ +int LZ4_compress_HC_destSize(void* state, const char* source, char* dest, int* sourceSizePtr, int targetDestSize, int cLevel) +{ + LZ4_streamHC_t* const ctx = LZ4_initStreamHC(state, sizeof(*ctx)); + if (ctx==NULL) return 0; /* init failure */ + LZ4HC_init_internal(&ctx->internal_donotuse, (const BYTE*) source); + LZ4_setCompressionLevel(ctx, cLevel); + return LZ4HC_compress_generic(&ctx->internal_donotuse, source, dest, sourceSizePtr, targetDestSize, cLevel, fillOutput); +} + + + +/************************************** +* Streaming Functions +**************************************/ +/* allocation */ +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +LZ4_streamHC_t* LZ4_createStreamHC(void) +{ + LZ4_streamHC_t* const state = + (LZ4_streamHC_t*)ALLOC_AND_ZERO(sizeof(LZ4_streamHC_t)); + if (state == NULL) return NULL; + LZ4_setCompressionLevel(state, LZ4HC_CLEVEL_DEFAULT); + return state; +} + +int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) +{ + DEBUGLOG(4, "LZ4_freeStreamHC(%p)", LZ4_streamHCPtr); + if (!LZ4_streamHCPtr) return 0; /* support free on NULL */ + FREEMEM(LZ4_streamHCPtr); + return 0; +} +#endif + + +LZ4_streamHC_t* LZ4_initStreamHC (void* buffer, size_t size) +{ + LZ4_streamHC_t* const LZ4_streamHCPtr = (LZ4_streamHC_t*)buffer; + DEBUGLOG(4, "LZ4_initStreamHC(%p, %u)", buffer, (unsigned)size); + /* check conditions */ + if (buffer == NULL) return NULL; + if (size < sizeof(LZ4_streamHC_t)) return NULL; + if (!LZ4_isAligned(buffer, LZ4_streamHC_t_alignment())) return NULL; + /* init */ + { LZ4HC_CCtx_internal* const hcstate = &(LZ4_streamHCPtr->internal_donotuse); + MEM_INIT(hcstate, 0, sizeof(*hcstate)); } + LZ4_setCompressionLevel(LZ4_streamHCPtr, LZ4HC_CLEVEL_DEFAULT); + return LZ4_streamHCPtr; +} + +/* just a stub */ +void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) +{ + LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); + LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); +} + +void LZ4_resetStreamHC_fast (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) +{ + LZ4HC_CCtx_internal* const s = &LZ4_streamHCPtr->internal_donotuse; + DEBUGLOG(5, "LZ4_resetStreamHC_fast(%p, %d)", LZ4_streamHCPtr, compressionLevel); + if (s->dirty) { + LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); + } else { + assert(s->end >= s->prefixStart); + s->dictLimit += (U32)(s->end - s->prefixStart); + s->prefixStart = NULL; + s->end = NULL; + s->dictCtx = NULL; + } + LZ4_setCompressionLevel(LZ4_streamHCPtr, compressionLevel); +} + +void LZ4_setCompressionLevel(LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) +{ + DEBUGLOG(5, "LZ4_setCompressionLevel(%p, %d)", LZ4_streamHCPtr, compressionLevel); + if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT; + if (compressionLevel > LZ4HC_CLEVEL_MAX) compressionLevel = LZ4HC_CLEVEL_MAX; + LZ4_streamHCPtr->internal_donotuse.compressionLevel = (short)compressionLevel; +} + +void LZ4_favorDecompressionSpeed(LZ4_streamHC_t* LZ4_streamHCPtr, int favor) +{ + LZ4_streamHCPtr->internal_donotuse.favorDecSpeed = (favor!=0); +} + +/* LZ4_loadDictHC() : + * LZ4_streamHCPtr is presumed properly initialized */ +int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, + const char* dictionary, int dictSize) +{ + LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; + DEBUGLOG(4, "LZ4_loadDictHC(ctx:%p, dict:%p, dictSize:%d)", LZ4_streamHCPtr, dictionary, dictSize); + assert(LZ4_streamHCPtr != NULL); + if (dictSize > 64 KB) { + dictionary += (size_t)dictSize - 64 KB; + dictSize = 64 KB; + } + /* need a full initialization, there are bad side-effects when using resetFast() */ + { int const cLevel = ctxPtr->compressionLevel; + LZ4_initStreamHC(LZ4_streamHCPtr, sizeof(*LZ4_streamHCPtr)); + LZ4_setCompressionLevel(LZ4_streamHCPtr, cLevel); + } + LZ4HC_init_internal (ctxPtr, (const BYTE*)dictionary); + ctxPtr->end = (const BYTE*)dictionary + dictSize; + if (dictSize >= LZ4HC_HASHSIZE) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); + return dictSize; +} + +void LZ4_attach_HC_dictionary(LZ4_streamHC_t *working_stream, const LZ4_streamHC_t *dictionary_stream) { + working_stream->internal_donotuse.dictCtx = dictionary_stream != NULL ? &(dictionary_stream->internal_donotuse) : NULL; +} + +/* compression */ + +static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) +{ + DEBUGLOG(4, "LZ4HC_setExternalDict(%p, %p)", ctxPtr, newBlock); + if (ctxPtr->end >= ctxPtr->prefixStart + 4) + LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ + + /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ + ctxPtr->lowLimit = ctxPtr->dictLimit; + ctxPtr->dictStart = ctxPtr->prefixStart; + ctxPtr->dictLimit += (U32)(ctxPtr->end - ctxPtr->prefixStart); + ctxPtr->prefixStart = newBlock; + ctxPtr->end = newBlock; + ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ + + /* cannot reference an extDict and a dictCtx at the same time */ + ctxPtr->dictCtx = NULL; +} + +static int +LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, + const char* src, char* dst, + int* srcSizePtr, int dstCapacity, + limitedOutput_directive limit) +{ + LZ4HC_CCtx_internal* const ctxPtr = &LZ4_streamHCPtr->internal_donotuse; + DEBUGLOG(5, "LZ4_compressHC_continue_generic(ctx=%p, src=%p, srcSize=%d, limit=%d)", + LZ4_streamHCPtr, src, *srcSizePtr, limit); + assert(ctxPtr != NULL); + /* auto-init if forgotten */ + if (ctxPtr->prefixStart == NULL) LZ4HC_init_internal (ctxPtr, (const BYTE*) src); + + /* Check overflow */ + if ((size_t)(ctxPtr->end - ctxPtr->prefixStart) + ctxPtr->dictLimit > 2 GB) { + size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->prefixStart); + if (dictSize > 64 KB) dictSize = 64 KB; + LZ4_loadDictHC(LZ4_streamHCPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); + } + + /* Check if blocks follow each other */ + if ((const BYTE*)src != ctxPtr->end) + LZ4HC_setExternalDict(ctxPtr, (const BYTE*)src); + + /* Check overlapping input/dictionary space */ + { const BYTE* sourceEnd = (const BYTE*) src + *srcSizePtr; + const BYTE* const dictBegin = ctxPtr->dictStart; + const BYTE* const dictEnd = ctxPtr->dictStart + (ctxPtr->dictLimit - ctxPtr->lowLimit); + if ((sourceEnd > dictBegin) && ((const BYTE*)src < dictEnd)) { + if (sourceEnd > dictEnd) sourceEnd = dictEnd; + ctxPtr->lowLimit += (U32)(sourceEnd - ctxPtr->dictStart); + ctxPtr->dictStart += (U32)(sourceEnd - ctxPtr->dictStart); + /* invalidate dictionary is it's too small */ + if (ctxPtr->dictLimit - ctxPtr->lowLimit < LZ4HC_HASHSIZE) { + ctxPtr->lowLimit = ctxPtr->dictLimit; + ctxPtr->dictStart = ctxPtr->prefixStart; + } } } + + return LZ4HC_compress_generic (ctxPtr, src, dst, srcSizePtr, dstCapacity, ctxPtr->compressionLevel, limit); +} + +int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int srcSize, int dstCapacity) +{ + DEBUGLOG(5, "LZ4_compress_HC_continue"); + if (dstCapacity < LZ4_compressBound(srcSize)) + return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, limitedOutput); + else + return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, src, dst, &srcSize, dstCapacity, notLimited); +} + +int LZ4_compress_HC_continue_destSize (LZ4_streamHC_t* LZ4_streamHCPtr, const char* src, char* dst, int* srcSizePtr, int targetDestSize) +{ + return LZ4_compressHC_continue_generic(LZ4_streamHCPtr, src, dst, srcSizePtr, targetDestSize, fillOutput); +} + + + +/* LZ4_saveDictHC : + * save history content + * into a user-provided buffer + * which is then used to continue compression + */ +int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize) +{ + LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse; + int const prefixSize = (int)(streamPtr->end - streamPtr->prefixStart); + DEBUGLOG(5, "LZ4_saveDictHC(%p, %p, %d)", LZ4_streamHCPtr, safeBuffer, dictSize); + assert(prefixSize >= 0); + if (dictSize > 64 KB) dictSize = 64 KB; + if (dictSize < 4) dictSize = 0; + if (dictSize > prefixSize) dictSize = prefixSize; + if (safeBuffer == NULL) assert(dictSize == 0); + if (dictSize > 0) + LZ4_memmove(safeBuffer, streamPtr->end - dictSize, (size_t)dictSize); + { U32 const endIndex = (U32)(streamPtr->end - streamPtr->prefixStart) + streamPtr->dictLimit; + streamPtr->end = (safeBuffer == NULL) ? NULL : (const BYTE*)safeBuffer + dictSize; + streamPtr->prefixStart = (const BYTE*)safeBuffer; + streamPtr->dictLimit = endIndex - (U32)dictSize; + streamPtr->lowLimit = endIndex - (U32)dictSize; + streamPtr->dictStart = streamPtr->prefixStart; + if (streamPtr->nextToUpdate < streamPtr->dictLimit) + streamPtr->nextToUpdate = streamPtr->dictLimit; + } + return dictSize; +} + + +/*************************************************** +* Deprecated Functions +***************************************************/ + +/* These functions currently generate deprecation warnings */ + +/* Wrappers for deprecated compression functions */ +int LZ4_compressHC(const char* src, char* dst, int srcSize) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), 0); } +int LZ4_compressHC_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, 0); } +int LZ4_compressHC2(const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); } +int LZ4_compressHC2_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, cLevel); } +int LZ4_compressHC_withStateHC (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, LZ4_compressBound(srcSize), 0); } +int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, maxDstSize, 0); } +int LZ4_compressHC2_withStateHC (void* state, const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); } +int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, maxDstSize, cLevel); } +int LZ4_compressHC_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, LZ4_compressBound(srcSize)); } +int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, maxDstSize); } + + +/* Deprecated streaming functions */ +int LZ4_sizeofStreamStateHC(void) { return sizeof(LZ4_streamHC_t); } + +/* state is presumed correctly sized, aka >= sizeof(LZ4_streamHC_t) + * @return : 0 on success, !=0 if error */ +int LZ4_resetStreamStateHC(void* state, char* inputBuffer) +{ + LZ4_streamHC_t* const hc4 = LZ4_initStreamHC(state, sizeof(*hc4)); + if (hc4 == NULL) return 1; /* init failed */ + LZ4HC_init_internal (&hc4->internal_donotuse, (const BYTE*)inputBuffer); + return 0; +} + +#if !defined(LZ4_STATIC_LINKING_ONLY_DISABLE_MEMORY_ALLOCATION) +void* LZ4_createHC (const char* inputBuffer) +{ + LZ4_streamHC_t* const hc4 = LZ4_createStreamHC(); + if (hc4 == NULL) return NULL; /* not enough memory */ + LZ4HC_init_internal (&hc4->internal_donotuse, (const BYTE*)inputBuffer); + return hc4; +} + +int LZ4_freeHC (void* LZ4HC_Data) +{ + if (!LZ4HC_Data) return 0; /* support free on NULL */ + FREEMEM(LZ4HC_Data); + return 0; +} +#endif + +int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int cLevel) +{ + return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, 0, cLevel, notLimited); +} + +int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* src, char* dst, int srcSize, int dstCapacity, int cLevel) +{ + return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, src, dst, &srcSize, dstCapacity, cLevel, limitedOutput); +} + +char* LZ4_slideInputBufferHC(void* LZ4HC_Data) +{ + LZ4HC_CCtx_internal* const s = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse; + const BYTE* const bufferStart = s->prefixStart - s->dictLimit + s->lowLimit; + LZ4_resetStreamHC_fast((LZ4_streamHC_t*)LZ4HC_Data, s->compressionLevel); + /* ugly conversion trick, required to evade (const char*) -> (char*) cast-qual warning :( */ + return (char*)(uptrval)bufferStart; +} + + +/* ================================================ + * LZ4 Optimal parser (levels [LZ4HC_CLEVEL_OPT_MIN - LZ4HC_CLEVEL_MAX]) + * ===============================================*/ +typedef struct { + int price; + int off; + int mlen; + int litlen; +} LZ4HC_optimal_t; + +/* price in bytes */ +LZ4_FORCE_INLINE int LZ4HC_literalsPrice(int const litlen) +{ + int price = litlen; + assert(litlen >= 0); + if (litlen >= (int)RUN_MASK) + price += 1 + ((litlen-(int)RUN_MASK) / 255); + return price; +} + + +/* requires mlen >= MINMATCH */ +LZ4_FORCE_INLINE int LZ4HC_sequencePrice(int litlen, int mlen) +{ + int price = 1 + 2 ; /* token + 16-bit offset */ + assert(litlen >= 0); + assert(mlen >= MINMATCH); + + price += LZ4HC_literalsPrice(litlen); + + if (mlen >= (int)(ML_MASK+MINMATCH)) + price += 1 + ((mlen-(int)(ML_MASK+MINMATCH)) / 255); + + return price; +} + + + +LZ4_FORCE_INLINE LZ4HC_match_t +LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx, + const BYTE* ip, const BYTE* const iHighLimit, + int minLen, int nbSearches, + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed) +{ + LZ4HC_match_t const match0 = { 0 , 0, 0 }; + /* note : LZ4HC_InsertAndGetWiderMatch() is able to modify the starting position of a match (*startpos), + * but this won't be the case here, as we define iLowLimit==ip, + ** so LZ4HC_InsertAndGetWiderMatch() won't be allowed to search past ip */ + LZ4HC_match_t md = LZ4HC_InsertAndGetWiderMatch(ctx, ip, ip, iHighLimit, minLen, nbSearches, 1 /*patternAnalysis*/, 1 /*chainSwap*/, dict, favorDecSpeed); + assert(md.back == 0); + if (md.len <= minLen) return match0; + if (favorDecSpeed) { + if ((md.len>18) & (md.len<=36)) md.len=18; /* favor dec.speed (shortcut) */ + } + return md; +} + + +static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx, + const char* const source, + char* dst, + int* srcSizePtr, + int dstCapacity, + int const nbSearches, + size_t sufficient_len, + const limitedOutput_directive limit, + int const fullUpdate, + const dictCtx_directive dict, + const HCfavor_e favorDecSpeed) +{ + int retval = 0; +#define TRAILING_LITERALS 3 +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 + LZ4HC_optimal_t* const opt = (LZ4HC_optimal_t*)ALLOC(sizeof(LZ4HC_optimal_t) * (LZ4_OPT_NUM + TRAILING_LITERALS)); +#else + LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS]; /* ~64 KB, which is a bit large for stack... */ +#endif + + const BYTE* ip = (const BYTE*) source; + const BYTE* anchor = ip; + const BYTE* const iend = ip + *srcSizePtr; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + BYTE* op = (BYTE*) dst; + BYTE* opSaved = (BYTE*) dst; + BYTE* oend = op + dstCapacity; + int ovml = MINMATCH; /* overflow - last sequence */ + int ovoff = 0; + + /* init */ +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 + if (opt == NULL) goto _return_label; +#endif + DEBUGLOG(5, "LZ4HC_compress_optimal(dst=%p, dstCapa=%u)", dst, (unsigned)dstCapacity); + *srcSizePtr = 0; + if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */ + if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1; + + /* Main Loop */ + while (ip <= mflimit) { + int const llen = (int)(ip - anchor); + int best_mlen, best_off; + int cur, last_match_pos = 0; + + LZ4HC_match_t const firstMatch = LZ4HC_FindLongerMatch(ctx, ip, matchlimit, MINMATCH-1, nbSearches, dict, favorDecSpeed); + if (firstMatch.len==0) { ip++; continue; } + + if ((size_t)firstMatch.len > sufficient_len) { + /* good enough solution : immediate encoding */ + int const firstML = firstMatch.len; + opSaved = op; + if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), firstML, firstMatch.off, limit, oend) ) { /* updates ip, op and anchor */ + ovml = firstML; + ovoff = firstMatch.off; + goto _dest_overflow; + } + continue; + } + + /* set prices for first positions (literals) */ + { int rPos; + for (rPos = 0 ; rPos < MINMATCH ; rPos++) { + int const cost = LZ4HC_literalsPrice(llen + rPos); + opt[rPos].mlen = 1; + opt[rPos].off = 0; + opt[rPos].litlen = llen + rPos; + opt[rPos].price = cost; + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup", + rPos, cost, opt[rPos].litlen); + } } + /* set prices using initial match */ + { int mlen = MINMATCH; + int const matchML = firstMatch.len; /* necessarily < sufficient_len < LZ4_OPT_NUM */ + int const offset = firstMatch.off; + assert(matchML < LZ4_OPT_NUM); + for ( ; mlen <= matchML ; mlen++) { + int const cost = LZ4HC_sequencePrice(llen, mlen); + opt[mlen].mlen = mlen; + opt[mlen].off = offset; + opt[mlen].litlen = llen; + opt[mlen].price = cost; + DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i) -- initial setup", + mlen, cost, mlen); + } } + last_match_pos = firstMatch.len; + { int addLit; + for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) { + opt[last_match_pos+addLit].mlen = 1; /* literal */ + opt[last_match_pos+addLit].off = 0; + opt[last_match_pos+addLit].litlen = addLit; + opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit); + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i) -- initial setup", + last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit); + } } + + /* check further positions */ + for (cur = 1; cur < last_match_pos; cur++) { + const BYTE* const curPtr = ip + cur; + LZ4HC_match_t newMatch; + + if (curPtr > mflimit) break; + DEBUGLOG(7, "rPos:%u[%u] vs [%u]%u", + cur, opt[cur].price, opt[cur+1].price, cur+1); + if (fullUpdate) { + /* not useful to search here if next position has same (or lower) cost */ + if ( (opt[cur+1].price <= opt[cur].price) + /* in some cases, next position has same cost, but cost rises sharply after, so a small match would still be beneficial */ + && (opt[cur+MINMATCH].price < opt[cur].price + 3/*min seq price*/) ) + continue; + } else { + /* not useful to search here if next position has same (or lower) cost */ + if (opt[cur+1].price <= opt[cur].price) continue; + } + + DEBUGLOG(7, "search at rPos:%u", cur); + if (fullUpdate) + newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, MINMATCH-1, nbSearches, dict, favorDecSpeed); + else + /* only test matches of minimum length; slightly faster, but misses a few bytes */ + newMatch = LZ4HC_FindLongerMatch(ctx, curPtr, matchlimit, last_match_pos - cur, nbSearches, dict, favorDecSpeed); + if (!newMatch.len) continue; + + if ( ((size_t)newMatch.len > sufficient_len) + || (newMatch.len + cur >= LZ4_OPT_NUM) ) { + /* immediate encoding */ + best_mlen = newMatch.len; + best_off = newMatch.off; + last_match_pos = cur + 1; + goto encode; + } + + /* before match : set price with literals at beginning */ + { int const baseLitlen = opt[cur].litlen; + int litlen; + for (litlen = 1; litlen < MINMATCH; litlen++) { + int const price = opt[cur].price - LZ4HC_literalsPrice(baseLitlen) + LZ4HC_literalsPrice(baseLitlen+litlen); + int const pos = cur + litlen; + if (price < opt[pos].price) { + opt[pos].mlen = 1; /* literal */ + opt[pos].off = 0; + opt[pos].litlen = baseLitlen+litlen; + opt[pos].price = price; + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", + pos, price, opt[pos].litlen); + } } } + + /* set prices using match at position = cur */ + { int const matchML = newMatch.len; + int ml = MINMATCH; + + assert(cur + newMatch.len < LZ4_OPT_NUM); + for ( ; ml <= matchML ; ml++) { + int const pos = cur + ml; + int const offset = newMatch.off; + int price; + int ll; + DEBUGLOG(7, "testing price rPos %i (last_match_pos=%i)", + pos, last_match_pos); + if (opt[cur].mlen == 1) { + ll = opt[cur].litlen; + price = ((cur > ll) ? opt[cur - ll].price : 0) + + LZ4HC_sequencePrice(ll, ml); + } else { + ll = 0; + price = opt[cur].price + LZ4HC_sequencePrice(0, ml); + } + + assert((U32)favorDecSpeed <= 1); + if (pos > last_match_pos+TRAILING_LITERALS + || price <= opt[pos].price - (int)favorDecSpeed) { + DEBUGLOG(7, "rPos:%3i => price:%3i (matchlen=%i)", + pos, price, ml); + assert(pos < LZ4_OPT_NUM); + if ( (ml == matchML) /* last pos of last match */ + && (last_match_pos < pos) ) + last_match_pos = pos; + opt[pos].mlen = ml; + opt[pos].off = offset; + opt[pos].litlen = ll; + opt[pos].price = price; + } } } + /* complete following positions with literals */ + { int addLit; + for (addLit = 1; addLit <= TRAILING_LITERALS; addLit ++) { + opt[last_match_pos+addLit].mlen = 1; /* literal */ + opt[last_match_pos+addLit].off = 0; + opt[last_match_pos+addLit].litlen = addLit; + opt[last_match_pos+addLit].price = opt[last_match_pos].price + LZ4HC_literalsPrice(addLit); + DEBUGLOG(7, "rPos:%3i => price:%3i (litlen=%i)", last_match_pos+addLit, opt[last_match_pos+addLit].price, addLit); + } } + } /* for (cur = 1; cur <= last_match_pos; cur++) */ + + assert(last_match_pos < LZ4_OPT_NUM + TRAILING_LITERALS); + best_mlen = opt[last_match_pos].mlen; + best_off = opt[last_match_pos].off; + cur = last_match_pos - best_mlen; + +encode: /* cur, last_match_pos, best_mlen, best_off must be set */ + assert(cur < LZ4_OPT_NUM); + assert(last_match_pos >= 1); /* == 1 when only one candidate */ + DEBUGLOG(6, "reverse traversal, looking for shortest path (last_match_pos=%i)", last_match_pos); + { int candidate_pos = cur; + int selected_matchLength = best_mlen; + int selected_offset = best_off; + while (1) { /* from end to beginning */ + int const next_matchLength = opt[candidate_pos].mlen; /* can be 1, means literal */ + int const next_offset = opt[candidate_pos].off; + DEBUGLOG(7, "pos %i: sequence length %i", candidate_pos, selected_matchLength); + opt[candidate_pos].mlen = selected_matchLength; + opt[candidate_pos].off = selected_offset; + selected_matchLength = next_matchLength; + selected_offset = next_offset; + if (next_matchLength > candidate_pos) break; /* last match elected, first match to encode */ + assert(next_matchLength > 0); /* can be 1, means literal */ + candidate_pos -= next_matchLength; + } } + + /* encode all recorded sequences in order */ + { int rPos = 0; /* relative position (to ip) */ + while (rPos < last_match_pos) { + int const ml = opt[rPos].mlen; + int const offset = opt[rPos].off; + if (ml == 1) { ip++; rPos++; continue; } /* literal; note: can end up with several literals, in which case, skip them */ + rPos += ml; + assert(ml >= MINMATCH); + assert((offset >= 1) && (offset <= LZ4_DISTANCE_MAX)); + opSaved = op; + if ( LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ml, offset, limit, oend) ) { /* updates ip, op and anchor */ + ovml = ml; + ovoff = offset; + goto _dest_overflow; + } } } + } /* while (ip <= mflimit) */ + +_last_literals: + /* Encode Last Literals */ + { size_t lastRunSize = (size_t)(iend - anchor); /* literals */ + size_t llAdd = (lastRunSize + 255 - RUN_MASK) / 255; + size_t const totalSize = 1 + llAdd + lastRunSize; + if (limit == fillOutput) oend += LASTLITERALS; /* restore correct value */ + if (limit && (op + totalSize > oend)) { + if (limit == limitedOutput) { /* Check output limit */ + retval = 0; + goto _return_label; + } + /* adapt lastRunSize to fill 'dst' */ + lastRunSize = (size_t)(oend - op) - 1 /*token*/; + llAdd = (lastRunSize + 256 - RUN_MASK) / 256; + lastRunSize -= llAdd; + } + DEBUGLOG(6, "Final literal run : %i literals", (int)lastRunSize); + ip = anchor + lastRunSize; /* can be != iend if limit==fillOutput */ + + if (lastRunSize >= RUN_MASK) { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = (RUN_MASK << ML_BITS); + for(; accumulator >= 255 ; accumulator -= 255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRunSize << ML_BITS); + } + LZ4_memcpy(op, anchor, lastRunSize); + op += lastRunSize; + } + + /* End */ + *srcSizePtr = (int) (((const char*)ip) - source); + retval = (int) ((char*)op-dst); + goto _return_label; + +_dest_overflow: +if (limit == fillOutput) { + /* Assumption : ip, anchor, ovml and ovref must be set correctly */ + size_t const ll = (size_t)(ip - anchor); + size_t const ll_addbytes = (ll + 240) / 255; + size_t const ll_totalCost = 1 + ll_addbytes + ll; + BYTE* const maxLitPos = oend - 3; /* 2 for offset, 1 for token */ + DEBUGLOG(6, "Last sequence overflowing (only %i bytes remaining)", (int)(oend-1-opSaved)); + op = opSaved; /* restore correct out pointer */ + if (op + ll_totalCost <= maxLitPos) { + /* ll validated; now adjust match length */ + size_t const bytesLeftForMl = (size_t)(maxLitPos - (op+ll_totalCost)); + size_t const maxMlSize = MINMATCH + (ML_MASK-1) + (bytesLeftForMl * 255); + assert(maxMlSize < INT_MAX); assert(ovml >= 0); + if ((size_t)ovml > maxMlSize) ovml = (int)maxMlSize; + if ((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1 + ovml >= MFLIMIT) { + DEBUGLOG(6, "Space to end : %i + ml (%i)", (int)((oend + LASTLITERALS) - (op + ll_totalCost + 2) - 1), ovml); + DEBUGLOG(6, "Before : ip = %p, anchor = %p", ip, anchor); + LZ4HC_encodeSequence(UPDATABLE(ip, op, anchor), ovml, ovoff, notLimited, oend); + DEBUGLOG(6, "After : ip = %p, anchor = %p", ip, anchor); + } } + goto _last_literals; +} +_return_label: +#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 + if (opt) FREEMEM(opt); +#endif + return retval; +} diff --git a/example/android/third_party/lz4/include/lz4hc.h b/example/android/third_party/lz4/include/lz4hc.h new file mode 100644 index 00000000..f23db5f6 --- /dev/null +++ b/example/android/third_party/lz4/include/lz4hc.h @@ -0,0 +1,413 @@ +/* + LZ4 HC - High Compression Mode of LZ4 + Header File + Copyright (C) 2011-2020, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/lz4/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ +#ifndef LZ4_HC_H_19834876238432 +#define LZ4_HC_H_19834876238432 + +#if defined (__cplusplus) +extern "C" { +#endif + +/* --- Dependency --- */ +/* note : lz4hc requires lz4.h/lz4.c for compilation */ +#include "lz4.h" /* stddef, LZ4LIB_API, LZ4_DEPRECATED */ + + +/* --- Useful constants --- */ +#define LZ4HC_CLEVEL_MIN 2 +#define LZ4HC_CLEVEL_DEFAULT 9 +#define LZ4HC_CLEVEL_OPT_MIN 10 +#define LZ4HC_CLEVEL_MAX 12 + + +/*-************************************ + * Block Compression + **************************************/ +/*! LZ4_compress_HC() : + * Compress data from `src` into `dst`, using the powerful but slower "HC" algorithm. + * `dst` must be already allocated. + * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h") + * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h") + * `compressionLevel` : any value between 1 and LZ4HC_CLEVEL_MAX will work. + * Values > LZ4HC_CLEVEL_MAX behave the same as LZ4HC_CLEVEL_MAX. + * @return : the number of bytes written into 'dst' + * or 0 if compression fails. + */ +LZ4LIB_API int LZ4_compress_HC (const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel); + + +/* Note : + * Decompression functions are provided within "lz4.h" (BSD license) + */ + + +/*! LZ4_compress_HC_extStateHC() : + * Same as LZ4_compress_HC(), but using an externally allocated memory segment for `state`. + * `state` size is provided by LZ4_sizeofStateHC(). + * Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() should do properly). + */ +LZ4LIB_API int LZ4_sizeofStateHC(void); +LZ4LIB_API int LZ4_compress_HC_extStateHC(void* stateHC, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); + + +/*! LZ4_compress_HC_destSize() : v1.9.0+ + * Will compress as much data as possible from `src` + * to fit into `targetDstSize` budget. + * Result is provided in 2 parts : + * @return : the number of bytes written into 'dst' (necessarily <= targetDstSize) + * or 0 if compression fails. + * `srcSizePtr` : on success, *srcSizePtr is updated to indicate how much bytes were read from `src` + */ +LZ4LIB_API int LZ4_compress_HC_destSize(void* stateHC, + const char* src, char* dst, + int* srcSizePtr, int targetDstSize, + int compressionLevel); + + +/*-************************************ + * Streaming Compression + * Bufferless synchronous API + **************************************/ + typedef union LZ4_streamHC_u LZ4_streamHC_t; /* incomplete type (defined later) */ + +/*! LZ4_createStreamHC() and LZ4_freeStreamHC() : + * These functions create and release memory for LZ4 HC streaming state. + * Newly created states are automatically initialized. + * A same state can be used multiple times consecutively, + * starting with LZ4_resetStreamHC_fast() to start a new stream of blocks. + */ +LZ4LIB_API LZ4_streamHC_t* LZ4_createStreamHC(void); +LZ4LIB_API int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr); + +/* + These functions compress data in successive blocks of any size, + using previous blocks as dictionary, to improve compression ratio. + One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks. + There is an exception for ring buffers, which can be smaller than 64 KB. + Ring-buffer scenario is automatically detected and handled within LZ4_compress_HC_continue(). + + Before starting compression, state must be allocated and properly initialized. + LZ4_createStreamHC() does both, though compression level is set to LZ4HC_CLEVEL_DEFAULT. + + Selecting the compression level can be done with LZ4_resetStreamHC_fast() (starts a new stream) + or LZ4_setCompressionLevel() (anytime, between blocks in the same stream) (experimental). + LZ4_resetStreamHC_fast() only works on states which have been properly initialized at least once, + which is automatically the case when state is created using LZ4_createStreamHC(). + + After reset, a first "fictional block" can be designated as initial dictionary, + using LZ4_loadDictHC() (Optional). + + Invoke LZ4_compress_HC_continue() to compress each successive block. + The number of blocks is unlimited. + Previous input blocks, including initial dictionary when present, + must remain accessible and unmodified during compression. + + It's allowed to update compression level anytime between blocks, + using LZ4_setCompressionLevel() (experimental). + + 'dst' buffer should be sized to handle worst case scenarios + (see LZ4_compressBound(), it ensures compression success). + In case of failure, the API does not guarantee recovery, + so the state _must_ be reset. + To ensure compression success + whenever `dst` buffer size cannot be made >= LZ4_compressBound(), + consider using LZ4_compress_HC_continue_destSize(). + + Whenever previous input blocks can't be preserved unmodified in-place during compression of next blocks, + it's possible to copy the last blocks into a more stable memory space, using LZ4_saveDictHC(). + Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer' (<= 64 KB) + + After completing a streaming compression, + it's possible to start a new stream of blocks, using the same LZ4_streamHC_t state, + just by resetting it, using LZ4_resetStreamHC_fast(). +*/ + +LZ4LIB_API void LZ4_resetStreamHC_fast(LZ4_streamHC_t* streamHCPtr, int compressionLevel); /* v1.9.0+ */ +LZ4LIB_API int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize); + +LZ4LIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, + const char* src, char* dst, + int srcSize, int maxDstSize); + +/*! LZ4_compress_HC_continue_destSize() : v1.9.0+ + * Similar to LZ4_compress_HC_continue(), + * but will read as much data as possible from `src` + * to fit into `targetDstSize` budget. + * Result is provided into 2 parts : + * @return : the number of bytes written into 'dst' (necessarily <= targetDstSize) + * or 0 if compression fails. + * `srcSizePtr` : on success, *srcSizePtr will be updated to indicate how much bytes were read from `src`. + * Note that this function may not consume the entire input. + */ +LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr, + const char* src, char* dst, + int* srcSizePtr, int targetDstSize); + +LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize); + + + +/*^********************************************** + * !!!!!! STATIC LINKING ONLY !!!!!! + ***********************************************/ + +/*-****************************************************************** + * PRIVATE DEFINITIONS : + * Do not use these definitions directly. + * They are merely exposed to allow static allocation of `LZ4_streamHC_t`. + * Declare an `LZ4_streamHC_t` directly, rather than any type below. + * Even then, only do so in the context of static linking, as definitions may change between versions. + ********************************************************************/ + +#define LZ4HC_DICTIONARY_LOGSIZE 16 +#define LZ4HC_MAXD (1<= LZ4HC_CLEVEL_OPT_MIN. + */ +LZ4LIB_STATIC_API void LZ4_favorDecompressionSpeed( + LZ4_streamHC_t* LZ4_streamHCPtr, int favor); + +/*! LZ4_resetStreamHC_fast() : v1.9.0+ + * When an LZ4_streamHC_t is known to be in a internally coherent state, + * it can often be prepared for a new compression with almost no work, only + * sometimes falling back to the full, expensive reset that is always required + * when the stream is in an indeterminate state (i.e., the reset performed by + * LZ4_resetStreamHC()). + * + * LZ4_streamHCs are guaranteed to be in a valid state when: + * - returned from LZ4_createStreamHC() + * - reset by LZ4_resetStreamHC() + * - memset(stream, 0, sizeof(LZ4_streamHC_t)) + * - the stream was in a valid state and was reset by LZ4_resetStreamHC_fast() + * - the stream was in a valid state and was then used in any compression call + * that returned success + * - the stream was in an indeterminate state and was used in a compression + * call that fully reset the state (LZ4_compress_HC_extStateHC()) and that + * returned success + * + * Note: + * A stream that was last used in a compression call that returned an error + * may be passed to this function. However, it will be fully reset, which will + * clear any existing history and settings from the context. + */ +LZ4LIB_STATIC_API void LZ4_resetStreamHC_fast( + LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel); + +/*! LZ4_compress_HC_extStateHC_fastReset() : + * A variant of LZ4_compress_HC_extStateHC(). + * + * Using this variant avoids an expensive initialization step. It is only safe + * to call if the state buffer is known to be correctly initialized already + * (see above comment on LZ4_resetStreamHC_fast() for a definition of + * "correctly initialized"). From a high level, the difference is that this + * function initializes the provided state with a call to + * LZ4_resetStreamHC_fast() while LZ4_compress_HC_extStateHC() starts with a + * call to LZ4_resetStreamHC(). + */ +LZ4LIB_STATIC_API int LZ4_compress_HC_extStateHC_fastReset ( + void* state, + const char* src, char* dst, + int srcSize, int dstCapacity, + int compressionLevel); + +/*! LZ4_attach_HC_dictionary() : + * This is an experimental API that allows for the efficient use of a + * static dictionary many times. + * + * Rather than re-loading the dictionary buffer into a working context before + * each compression, or copying a pre-loaded dictionary's LZ4_streamHC_t into a + * working LZ4_streamHC_t, this function introduces a no-copy setup mechanism, + * in which the working stream references the dictionary stream in-place. + * + * Several assumptions are made about the state of the dictionary stream. + * Currently, only streams which have been prepared by LZ4_loadDictHC() should + * be expected to work. + * + * Alternatively, the provided dictionary stream pointer may be NULL, in which + * case any existing dictionary stream is unset. + * + * A dictionary should only be attached to a stream without any history (i.e., + * a stream that has just been reset). + * + * The dictionary will remain attached to the working stream only for the + * current stream session. Calls to LZ4_resetStreamHC(_fast) will remove the + * dictionary context association from the working stream. The dictionary + * stream (and source buffer) must remain in-place / accessible / unchanged + * through the lifetime of the stream session. + */ +LZ4LIB_STATIC_API void LZ4_attach_HC_dictionary( + LZ4_streamHC_t *working_stream, + const LZ4_streamHC_t *dictionary_stream); + +#if defined (__cplusplus) +} +#endif + +#endif /* LZ4_HC_SLO_098092834 */ +#endif /* LZ4_HC_STATIC_LINKING_ONLY */ diff --git a/example/android/third_party/lz4/include/xxhash.c b/example/android/third_party/lz4/include/xxhash.c new file mode 100644 index 00000000..08d8b360 --- /dev/null +++ b/example/android/third_party/lz4/include/xxhash.c @@ -0,0 +1,1030 @@ +/* +* xxHash - Fast Hash algorithm +* Copyright (C) 2012-2016, Yann Collet +* +* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following disclaimer +* in the documentation and/or other materials provided with the +* distribution. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +* +* You can contact the author at : +* - xxHash homepage: http://www.xxhash.com +* - xxHash source repository : https://github.com/Cyan4973/xxHash +*/ + + +/* ************************************* +* Tuning parameters +***************************************/ +/*!XXH_FORCE_MEMORY_ACCESS : + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method doesn't depend on compiler but violate C standard. + * It can generate buggy code on targets which do not support unaligned memory accesses. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See http://stackoverflow.com/a/32095106/646947 for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) \ + || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) \ + || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define XXH_FORCE_MEMORY_ACCESS 2 +# elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) \ + || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) \ + || defined(__ARM_ARCH_7S__) )) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +/*!XXH_ACCEPT_NULL_INPUT_POINTER : + * If input pointer is NULL, xxHash default behavior is to dereference it, triggering a segfault. + * When this macro is enabled, xxHash actively checks input for null pointer. + * It it is, result for null input pointers is the same as a null-length input. + */ +#ifndef XXH_ACCEPT_NULL_INPUT_POINTER /* can be defined externally */ +# define XXH_ACCEPT_NULL_INPUT_POINTER 0 +#endif + +/*!XXH_FORCE_NATIVE_FORMAT : + * By default, xxHash library provides endian-independent Hash values, based on little-endian convention. + * Results are therefore identical for little-endian and big-endian CPU. + * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. + * Should endian-independence be of no importance for your application, you may set the #define below to 1, + * to improve speed for Big-endian CPU. + * This option has no impact on Little_Endian CPU. + */ +#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */ +# define XXH_FORCE_NATIVE_FORMAT 0 +#endif + +/*!XXH_FORCE_ALIGN_CHECK : + * This is a minor performance trick, only useful with lots of very small keys. + * It means : check for aligned/unaligned input. + * The check costs one initial branch per hash; + * set it to 0 when the input is guaranteed to be aligned, + * or when alignment doesn't matter for performance. + */ +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ +# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +/*! Modify the local functions below should you wish to use some other memory routines +* for malloc(), free() */ +#include +static void* XXH_malloc(size_t s) { return malloc(s); } +static void XXH_free (void* p) { free(p); } +/*! and for memcpy() */ +#include +static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } + +#include /* assert */ + +#define XXH_STATIC_LINKING_ONLY +#include "xxhash.h" + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#if defined (_MSC_VER) && !defined (__clang__) /* MSVC */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# define FORCE_INLINE static __forceinline +#else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# if defined (__GNUC__) || defined (__clang__) +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif + + +/* ************************************* +* Basic Types +***************************************/ +#ifndef MEM_MODULE +# if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; +# else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; +# endif +#endif + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +typedef union { U32 u32; } __attribute__((packed)) unalign; +static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } + +#else + +/* portable and safe solution. Generally efficient. + * see : http://stackoverflow.com/a/32095106/646947 + */ +static U32 XXH_read32(const void* memPtr) +{ + U32 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ +#if defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) +# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) +#endif + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static U32 XXH_swap32 (U32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +#endif + + +/* ************************************* +* Architecture Macros +***************************************/ +typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; + +/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ +#ifndef XXH_CPU_LITTLE_ENDIAN +static int XXH_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() +#endif + + +/* *************************** +* Memory reads +*****************************/ +typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; + +FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); + else + return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); +} + +FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE32_align(ptr, endian, XXH_unaligned); +} + +static U32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} + + +/* ************************************* +* Macros +***************************************/ +#define XXH_STATIC_ASSERT(c) { enum { XXH_sa = 1/(int)(!!(c)) }; } /* use after variable declarations */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ******************************************************************* +* 32-bit hash functions +*********************************************************************/ +static const U32 PRIME32_1 = 2654435761U; +static const U32 PRIME32_2 = 2246822519U; +static const U32 PRIME32_3 = 3266489917U; +static const U32 PRIME32_4 = 668265263U; +static const U32 PRIME32_5 = 374761393U; + +static U32 XXH32_round(U32 seed, U32 input) +{ + seed += input * PRIME32_2; + seed = XXH_rotl32(seed, 13); + seed *= PRIME32_1; + return seed; +} + +/* mix all bits */ +static U32 XXH32_avalanche(U32 h32) +{ + h32 ^= h32 >> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + return(h32); +} + +#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) + +static U32 +XXH32_finalize(U32 h32, const void* ptr, size_t len, + XXH_endianess endian, XXH_alignment align) + +{ + const BYTE* p = (const BYTE*)ptr; + +#define PROCESS1 \ + h32 += (*p++) * PRIME32_5; \ + h32 = XXH_rotl32(h32, 11) * PRIME32_1 ; + +#define PROCESS4 \ + h32 += XXH_get32bits(p) * PRIME32_3; \ + p+=4; \ + h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; + + switch(len&15) /* or switch(bEnd - p) */ + { + case 12: PROCESS4; + /* fallthrough */ + case 8: PROCESS4; + /* fallthrough */ + case 4: PROCESS4; + return XXH32_avalanche(h32); + + case 13: PROCESS4; + /* fallthrough */ + case 9: PROCESS4; + /* fallthrough */ + case 5: PROCESS4; + PROCESS1; + return XXH32_avalanche(h32); + + case 14: PROCESS4; + /* fallthrough */ + case 10: PROCESS4; + /* fallthrough */ + case 6: PROCESS4; + PROCESS1; + PROCESS1; + return XXH32_avalanche(h32); + + case 15: PROCESS4; + /* fallthrough */ + case 11: PROCESS4; + /* fallthrough */ + case 7: PROCESS4; + /* fallthrough */ + case 3: PROCESS1; + /* fallthrough */ + case 2: PROCESS1; + /* fallthrough */ + case 1: PROCESS1; + /* fallthrough */ + case 0: return XXH32_avalanche(h32); + } + assert(0); + return h32; /* reaching this point is deemed impossible */ +} + + +FORCE_INLINE U32 +XXH32_endian_align(const void* input, size_t len, U32 seed, + XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U32 h32; + +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + if (p==NULL) { + len=0; + bEnd=p=(const BYTE*)(size_t)16; + } +#endif + + if (len>=16) { + const BYTE* const limit = bEnd - 15; + U32 v1 = seed + PRIME32_1 + PRIME32_2; + U32 v2 = seed + PRIME32_2; + U32 v3 = seed + 0; + U32 v4 = seed - PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4; + v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; + v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; + v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; + } while (p < limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + PRIME32_5; + } + + h32 += (U32)len; + + return XXH32_finalize(h32, p, len&15, endian, align); +} + + +XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, input, len); + return XXH32_digest(&state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } } + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + + + +/*====== Hash streaming ======*/ + +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + memcpy(dstState, srcState, sizeof(*dstState)); +} + +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) +{ + XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)); + state.v1 = seed + PRIME32_1 + PRIME32_2; + state.v2 = seed + PRIME32_2; + state.v3 = seed + 0; + state.v4 = seed - PRIME32_1; + /* do not write into reserved, planned to be removed in a future version */ + memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved)); + return XXH_OK; +} + + +FORCE_INLINE XXH_errorcode +XXH32_update_endian(XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) +{ + if (input==NULL) +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + return XXH_OK; +#else + return XXH_ERROR; +#endif + + { const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + + state->total_len_32 += (unsigned)len; + state->large_len |= (len>=16) | (state->total_len_32>=16); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); + state->memsize += (unsigned)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const U32* p32 = state->mem32; + state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; + state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; + state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; + state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const BYTE* const limit = bEnd - 16; + U32 v1 = state->v1; + U32 v2 = state->v2; + U32 v3 = state->v3; + U32 v4 = state->v4; + + do { + v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; + v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; + v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; + v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH32_update_endian(state_in, input, len, XXH_bigEndian); +} + + +FORCE_INLINE U32 +XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) +{ + U32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v1, 1) + + XXH_rotl32(state->v2, 7) + + XXH_rotl32(state->v3, 12) + + XXH_rotl32(state->v4, 18); + } else { + h32 = state->v3 /* == seed */ + PRIME32_5; + } + + h32 += state->total_len_32; + + return XXH32_finalize(h32, state->mem32, state->memsize, endian, XXH_aligned); +} + + +XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_digest_endian(state_in, XXH_littleEndian); + else + return XXH32_digest_endian(state_in, XXH_bigEndian); +} + + +/*====== Canonical representation ======*/ + +/*! Default XXH result types are basic unsigned 32 and 64 bits. +* The canonical representation follows human-readable write convention, aka big-endian (large digits first). +* These functions allow transformation of hash result into and from its canonical format. +* This way, hash values can be written into a file or buffer, remaining comparable across different systems. +*/ + +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + memcpy(dst, &hash, sizeof(*dst)); +} + +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + + +#ifndef XXH_NO_LONG_LONG + +/* ******************************************************************* +* 64-bit hash functions +*********************************************************************/ + +/*====== Memory access ======*/ + +#ifndef MEM_MODULE +# define MEM_MODULE +# if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint64_t U64; +# else + /* if compiler doesn't support unsigned long long, replace by another 64-bit type */ + typedef unsigned long long U64; +# endif +#endif + + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign64; +static U64 XXH_read64(const void* ptr) { return ((const unalign64*)ptr)->u64; } + +#else + +/* portable and safe solution. Generally efficient. + * see : http://stackoverflow.com/a/32095106/646947 + */ + +static U64 XXH_read64(const void* memPtr) +{ + U64 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap64 _byteswap_uint64 +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap64 __builtin_bswap64 +#else +static U64 XXH_swap64 (U64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + +FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); + else + return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); +} + +FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE64_align(ptr, endian, XXH_unaligned); +} + +static U64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} + + +/*====== xxh64 ======*/ + +static const U64 PRIME64_1 = 11400714785074694791ULL; +static const U64 PRIME64_2 = 14029467366897019727ULL; +static const U64 PRIME64_3 = 1609587929392839161ULL; +static const U64 PRIME64_4 = 9650029242287828579ULL; +static const U64 PRIME64_5 = 2870177450012600261ULL; + +static U64 XXH64_round(U64 acc, U64 input) +{ + acc += input * PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= PRIME64_1; + return acc; +} + +static U64 XXH64_mergeRound(U64 acc, U64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * PRIME64_1 + PRIME64_4; + return acc; +} + +static U64 XXH64_avalanche(U64 h64) +{ + h64 ^= h64 >> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + return h64; +} + + +#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) + +static U64 +XXH64_finalize(U64 h64, const void* ptr, size_t len, + XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)ptr; + +#define PROCESS1_64 \ + h64 ^= (*p++) * PRIME64_5; \ + h64 = XXH_rotl64(h64, 11) * PRIME64_1; + +#define PROCESS4_64 \ + h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; \ + p+=4; \ + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + +#define PROCESS8_64 { \ + U64 const k1 = XXH64_round(0, XXH_get64bits(p)); \ + p+=8; \ + h64 ^= k1; \ + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; \ +} + + switch(len&31) { + case 24: PROCESS8_64; + /* fallthrough */ + case 16: PROCESS8_64; + /* fallthrough */ + case 8: PROCESS8_64; + return XXH64_avalanche(h64); + + case 28: PROCESS8_64; + /* fallthrough */ + case 20: PROCESS8_64; + /* fallthrough */ + case 12: PROCESS8_64; + /* fallthrough */ + case 4: PROCESS4_64; + return XXH64_avalanche(h64); + + case 25: PROCESS8_64; + /* fallthrough */ + case 17: PROCESS8_64; + /* fallthrough */ + case 9: PROCESS8_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 29: PROCESS8_64; + /* fallthrough */ + case 21: PROCESS8_64; + /* fallthrough */ + case 13: PROCESS8_64; + /* fallthrough */ + case 5: PROCESS4_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 26: PROCESS8_64; + /* fallthrough */ + case 18: PROCESS8_64; + /* fallthrough */ + case 10: PROCESS8_64; + PROCESS1_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 30: PROCESS8_64; + /* fallthrough */ + case 22: PROCESS8_64; + /* fallthrough */ + case 14: PROCESS8_64; + /* fallthrough */ + case 6: PROCESS4_64; + PROCESS1_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 27: PROCESS8_64; + /* fallthrough */ + case 19: PROCESS8_64; + /* fallthrough */ + case 11: PROCESS8_64; + PROCESS1_64; + PROCESS1_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 31: PROCESS8_64; + /* fallthrough */ + case 23: PROCESS8_64; + /* fallthrough */ + case 15: PROCESS8_64; + /* fallthrough */ + case 7: PROCESS4_64; + /* fallthrough */ + case 3: PROCESS1_64; + /* fallthrough */ + case 2: PROCESS1_64; + /* fallthrough */ + case 1: PROCESS1_64; + /* fallthrough */ + case 0: return XXH64_avalanche(h64); + } + + /* impossible to reach */ + assert(0); + return 0; /* unreachable, but some compilers complain without it */ +} + +FORCE_INLINE U64 +XXH64_endian_align(const void* input, size_t len, U64 seed, + XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U64 h64; + +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + if (p==NULL) { + len=0; + bEnd=p=(const BYTE*)(size_t)32; + } +#endif + + if (len>=32) { + const BYTE* const limit = bEnd - 32; + U64 v1 = seed + PRIME64_1 + PRIME64_2; + U64 v2 = seed + PRIME64_2; + U64 v3 = seed + 0; + U64 v4 = seed - PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8; + v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8; + v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8; + v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8; + } while (p<=limit); + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + + } else { + h64 = seed + PRIME64_5; + } + + h64 += (U64) len; + + return XXH64_finalize(h64, p, len, endian, align); +} + + +XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, input, len); + return XXH64_digest(&state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } } + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + +/*====== Hash Streaming ======*/ + +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +{ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState) +{ + memcpy(dstState, srcState, sizeof(*dstState)); +} + +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) +{ + XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)); + state.v1 = seed + PRIME64_1 + PRIME64_2; + state.v2 = seed + PRIME64_2; + state.v3 = seed + 0; + state.v4 = seed - PRIME64_1; + /* do not write into reserved, planned to be removed in a future version */ + memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved)); + return XXH_OK; +} + +FORCE_INLINE XXH_errorcode +XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) +{ + if (input==NULL) +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + return XXH_OK; +#else + return XXH_ERROR; +#endif + + { const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + + state->total_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); + state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); + state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); + state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); + p += 32-state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const BYTE* const limit = bEnd - 32; + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + do { + v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; + v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; + v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; + v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH64_update_endian(state_in, input, len, XXH_bigEndian); +} + +FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) +{ + U64 h64; + + if (state->total_len >= 32) { + U64 const v1 = state->v1; + U64 const v2 = state->v2; + U64 const v3 = state->v3; + U64 const v4 = state->v4; + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + } else { + h64 = state->v3 /*seed*/ + PRIME64_5; + } + + h64 += (U64) state->total_len; + + return XXH64_finalize(h64, state->mem64, (size_t)state->total_len, endian, XXH_aligned); +} + +XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_digest_endian(state_in, XXH_littleEndian); + else + return XXH64_digest_endian(state_in, XXH_bigEndian); +} + + +/*====== Canonical representation ======*/ + +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + memcpy(dst, &hash, sizeof(*dst)); +} + +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} + +#endif /* XXH_NO_LONG_LONG */ diff --git a/example/android/third_party/lz4/include/xxhash.h b/example/android/third_party/lz4/include/xxhash.h new file mode 100644 index 00000000..d6bad943 --- /dev/null +++ b/example/android/third_party/lz4/include/xxhash.h @@ -0,0 +1,328 @@ +/* + xxHash - Extremely Fast Hash algorithm + Header File + Copyright (C) 2012-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - xxHash source repository : https://github.com/Cyan4973/xxHash +*/ + +/* Notice extracted from xxHash homepage : + +xxHash is an extremely fast Hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MumurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. + +A 64-bit version, named XXH64, is available since r35. +It offers much better speed, but for 64-bit applications only. +Name Speed on 64 bits Speed on 32 bits +XXH64 13.8 GB/s 1.9 GB/s +XXH32 6.8 GB/s 6.0 GB/s +*/ + +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* **************************** +* Definitions +******************************/ +#include /* size_t */ +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + +/* **************************** + * API modifier + ******************************/ +/** XXH_INLINE_ALL (and XXH_PRIVATE_API) + * This is useful to include xxhash functions in `static` mode + * in order to inline them, and remove their symbol from the public list. + * Inlining can offer dramatic performance improvement on small keys. + * Methodology : + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * `xxhash.c` is automatically included. + * It's not useful to compile and link it as a separate module. + */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY +# endif +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else + /* this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static +# endif +#else +# define XXH_PUBLIC_API /* do nothing */ +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ + +/*! XXH_NAMESPACE, aka Namespace Emulation : + * + * If you want to include _and expose_ xxHash functions from within your own library, + * but also want to avoid symbol collisions with other libraries which may also include xxHash, + * + * you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library + * with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values). + * + * Note that no change is required within the calling program as long as it includes `xxhash.h` : + * regular symbol name will be automatically translated by this header. + */ +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +#endif + + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 6 +#define XXH_VERSION_RELEASE 5 +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) +XXH_PUBLIC_API unsigned XXH_versionNumber (void); + + +/*-********************************************************************** +* 32-bit hash +************************************************************************/ +typedef unsigned int XXH32_hash_t; + +/*! XXH32() : + Calculate the 32-bit hash of sequence "length" bytes stored at memory address "input". + The memory between input & input+length must be valid (allocated and read-accessible). + "seed" can be used to alter the result predictably. + Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); + +/*====== Streaming ======*/ +typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); + +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); + +/* + * Streaming functions generate the xxHash of an input provided in multiple segments. + * Note that, for small input, they are slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * XXH state must first be allocated, using XXH*_createState() . + * + * Start a new hash by initializing state with a seed, using XXH*_reset(). + * + * Then, feed the hash state by calling XXH*_update() as many times as necessary. + * The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using XXH*_digest(). + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a digest, + * and generate some new hashes later on, by calling again XXH*_digest(). + * + * When done, free XXH state space if it was allocated dynamically. + */ + +/*====== Canonical representation ======*/ + +typedef struct { unsigned char digest[4]; } XXH32_canonical_t; +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); + +/* Default result type for XXH functions are primitive unsigned 32 and 64 bits. + * The canonical representation uses human-readable write convention, aka big-endian (large digits first). + * These functions allow transformation of hash result into and from its canonical format. + * This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. + */ + + +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bit hash +************************************************************************/ +typedef unsigned long long XXH64_hash_t; + +/*! XXH64() : + Calculate the 64-bit hash of sequence of length "len" stored at memory address "input". + "seed" can be used to alter the result predictably. + This function runs faster on 64-bit systems, but slower on 32-bit systems (see benchmark). +*/ +XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); + +/*====== Streaming ======*/ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state); + +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); + +/*====== Canonical representation ======*/ +typedef struct { unsigned char digest[8]; } XXH64_canonical_t; +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); +#endif /* XXH_NO_LONG_LONG */ + + + +#ifdef XXH_STATIC_LINKING_ONLY + +/* ================================================================================================ + This section contains declarations which are not guaranteed to remain stable. + They may change in future versions, becoming incompatible with a different version of the library. + These declarations should only be used with static linking. + Never use them in association with dynamic linking ! +=================================================================================================== */ + +/* These definitions are only present to allow + * static allocation of XXH state, on stack or in a struct for example. + * Never **ever** use members directly. */ + +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + +struct XXH32_state_s { + uint32_t total_len_32; + uint32_t large_len; + uint32_t v1; + uint32_t v2; + uint32_t v3; + uint32_t v4; + uint32_t mem32[4]; + uint32_t memsize; + uint32_t reserved; /* never read nor write, might be removed in a future version */ +}; /* typedef'd to XXH32_state_t */ + +struct XXH64_state_s { + uint64_t total_len; + uint64_t v1; + uint64_t v2; + uint64_t v3; + uint64_t v4; + uint64_t mem64[4]; + uint32_t memsize; + uint32_t reserved[2]; /* never read nor write, might be removed in a future version */ +}; /* typedef'd to XXH64_state_t */ + +# else + +struct XXH32_state_s { + unsigned total_len_32; + unsigned large_len; + unsigned v1; + unsigned v2; + unsigned v3; + unsigned v4; + unsigned mem32[4]; + unsigned memsize; + unsigned reserved; /* never read nor write, might be removed in a future version */ +}; /* typedef'd to XXH32_state_t */ + +# ifndef XXH_NO_LONG_LONG /* remove 64-bit support */ +struct XXH64_state_s { + unsigned long long total_len; + unsigned long long v1; + unsigned long long v2; + unsigned long long v3; + unsigned long long v4; + unsigned long long mem64[4]; + unsigned memsize; + unsigned reserved[2]; /* never read nor write, might be removed in a future version */ +}; /* typedef'd to XXH64_state_t */ +# endif + +# endif + + +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# include "xxhash.c" /* include xxhash function bodies as `static`, for inlining */ +#endif + +#endif /* XXH_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* XXHASH_H_5627135585666179 */ diff --git a/example/android/third_party/lz4/x86-64/liblz4.a b/example/android/third_party/lz4/x86-64/liblz4.a new file mode 100644 index 00000000..d4764fe8 Binary files /dev/null and b/example/android/third_party/lz4/x86-64/liblz4.a differ diff --git a/example/android/third_party/lz4/x86-64/liblz4.so b/example/android/third_party/lz4/x86-64/liblz4.so new file mode 100644 index 00000000..91290e20 Binary files /dev/null and b/example/android/third_party/lz4/x86-64/liblz4.so differ diff --git a/example/android/third_party/secp256k1/armv7/libsecp256k1.a b/example/android/third_party/secp256k1/armv7/libsecp256k1.a new file mode 100644 index 00000000..f80ead61 Binary files /dev/null and b/example/android/third_party/secp256k1/armv7/libsecp256k1.a differ diff --git a/example/android/third_party/secp256k1/armv8/libsecp256k1.a b/example/android/third_party/secp256k1/armv8/libsecp256k1.a new file mode 100644 index 00000000..1b79e3af Binary files /dev/null and b/example/android/third_party/secp256k1/armv8/libsecp256k1.a differ diff --git a/example/android/third_party/secp256k1/build.sh b/example/android/third_party/secp256k1/build.sh new file mode 100755 index 00000000..68da6793 --- /dev/null +++ b/example/android/third_party/secp256k1/build.sh @@ -0,0 +1,36 @@ +#!/bin/sh +export PATH=$PATH:$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/linux-x86_64/bin +export NDK_PLATFORM="android-32" +export CC= +export CXX= + +rm -rf secp256k1 +git clone https://github.com/bitcoin-core/secp256k1 + +cd secp256k1 +git checkout v0.3.2 + +./autogen.sh +mkdir build +cd build + +cmake .. -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake" -DANDROID_ABI=armeabi-v7a -DANDROID_PLATFORM=21 -DBUILD_SHARED_LIBS=OFF -DANDROID_TOOLCHAIN_NAME=arm-linux-androideabi +make +cp lib/libsecp256k1.a ../../armv7/ +rm -rf * + +cmake .. -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake" -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=21 -DBUILD_SHARED_LIBS=OFF -DANDROID_TOOLCHAIN_NAME=aarch64-linux-android +make +cp lib/libsecp256k1.a ../../armv8/ +rm -rf * + +cmake .. -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake" -DANDROID_ABI=x86_64 -DANDROID_PLATFORM=21 -DBUILD_SHARED_LIBS=OFF -DANDROID_TOOLCHAIN_NAME=x86_64 +make +cp lib/libsecp256k1.a ../../x86-64/ +rm -rf * + +cmake .. -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake" -DANDROID_ABI= -DANDROID_PLATFORM=21 -DBUILD_SHARED_LIBS=OFF -DANDROID_TOOLCHAIN_NAME=x86- +make +cp lib/libsecp256k1.a ../../i686/ +rm -rf * +rm -rf ../secp256k1 diff --git a/example/android/third_party/secp256k1/i686/libsecp256k1.a b/example/android/third_party/secp256k1/i686/libsecp256k1.a new file mode 100644 index 00000000..014b1fc7 Binary files /dev/null and b/example/android/third_party/secp256k1/i686/libsecp256k1.a differ diff --git a/example/android/third_party/secp256k1/include/secp256k1.h b/example/android/third_party/secp256k1/include/secp256k1.h new file mode 100644 index 00000000..c6e9417f --- /dev/null +++ b/example/android/third_party/secp256k1/include/secp256k1.h @@ -0,0 +1,899 @@ +#ifndef SECP256K1_H +#define SECP256K1_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/** Unless explicitly stated all pointer arguments must not be NULL. + * + * The following rules specify the order of arguments in API calls: + * + * 1. Context pointers go first, followed by output arguments, combined + * output/input arguments, and finally input-only arguments. + * 2. Array lengths always immediately follow the argument whose length + * they describe, even if this violates rule 1. + * 3. Within the OUT/OUTIN/IN groups, pointers to data that is typically generated + * later go first. This means: signatures, public nonces, secret nonces, + * messages, public keys, secret keys, tweaks. + * 4. Arguments that are not data pointers go last, from more complex to less + * complex: function pointers, algorithm names, messages, void pointers, + * counts, flags, booleans. + * 5. Opaque data pointers follow the function pointer they are to be passed to. + */ + +/** Opaque data structure that holds context information + * + * The primary purpose of context objects is to store randomization data for + * enhanced protection against side-channel leakage. This protection is only + * effective if the context is randomized after its creation. See + * secp256k1_context_create for creation of contexts and + * secp256k1_context_randomize for randomization. + * + * A secondary purpose of context objects is to store pointers to callback + * functions that the library will call when certain error states arise. See + * secp256k1_context_set_error_callback as well as + * secp256k1_context_set_illegal_callback for details. Future library versions + * may use context objects for additional purposes. + * + * A constructed context can safely be used from multiple threads + * simultaneously, but API calls that take a non-const pointer to a context + * need exclusive access to it. In particular this is the case for + * secp256k1_context_destroy, secp256k1_context_preallocated_destroy, + * and secp256k1_context_randomize. + * + * Regarding randomization, either do it once at creation time (in which case + * you do not need any locking for the other calls), or use a read-write lock. + */ +typedef struct secp256k1_context_struct secp256k1_context; + +/** Opaque data structure that holds a parsed and valid public key. + * + * The exact representation of data inside is implementation defined and not + * guaranteed to be portable between different platforms or versions. It is + * however guaranteed to be 64 bytes in size, and can be safely copied/moved. + * If you need to convert to a format suitable for storage or transmission, + * use secp256k1_ec_pubkey_serialize and secp256k1_ec_pubkey_parse. To + * compare keys, use secp256k1_ec_pubkey_cmp. + */ +typedef struct secp256k1_pubkey { + unsigned char data[64]; +} secp256k1_pubkey; + +/** Opaque data structure that holds a parsed ECDSA signature. + * + * The exact representation of data inside is implementation defined and not + * guaranteed to be portable between different platforms or versions. It is + * however guaranteed to be 64 bytes in size, and can be safely copied/moved. + * If you need to convert to a format suitable for storage, transmission, or + * comparison, use the secp256k1_ecdsa_signature_serialize_* and + * secp256k1_ecdsa_signature_parse_* functions. + */ +typedef struct secp256k1_ecdsa_signature { + unsigned char data[64]; +} secp256k1_ecdsa_signature; + +/** A pointer to a function to deterministically generate a nonce. + * + * Returns: 1 if a nonce was successfully generated. 0 will cause signing to fail. + * Out: nonce32: pointer to a 32-byte array to be filled by the function. + * In: msg32: the 32-byte message hash being verified (will not be NULL) + * key32: pointer to a 32-byte secret key (will not be NULL) + * algo16: pointer to a 16-byte array describing the signature + * algorithm (will be NULL for ECDSA for compatibility). + * data: Arbitrary data pointer that is passed through. + * attempt: how many iterations we have tried to find a nonce. + * This will almost always be 0, but different attempt values + * are required to result in a different nonce. + * + * Except for test cases, this function should compute some cryptographic hash of + * the message, the algorithm, the key and the attempt. + */ +typedef int (*secp256k1_nonce_function)( + unsigned char *nonce32, + const unsigned char *msg32, + const unsigned char *key32, + const unsigned char *algo16, + void *data, + unsigned int attempt +); + +# if !defined(SECP256K1_GNUC_PREREQ) +# if defined(__GNUC__)&&defined(__GNUC_MINOR__) +# define SECP256K1_GNUC_PREREQ(_maj,_min) \ + ((__GNUC__<<16)+__GNUC_MINOR__>=((_maj)<<16)+(_min)) +# else +# define SECP256K1_GNUC_PREREQ(_maj,_min) 0 +# endif +# endif + +/* When this header is used at build-time the SECP256K1_BUILD define needs to be set + * to correctly setup export attributes and nullness checks. This is normally done + * by secp256k1.c but to guard against this header being included before secp256k1.c + * has had a chance to set the define (e.g. via test harnesses that just includes + * secp256k1.c) we set SECP256K1_NO_BUILD when this header is processed without the + * BUILD define so this condition can be caught. + */ +#ifndef SECP256K1_BUILD +# define SECP256K1_NO_BUILD +#endif + +/* Symbol visibility. */ +#if defined(_WIN32) + /* GCC for Windows (e.g., MinGW) accepts the __declspec syntax + * for MSVC compatibility. A __declspec declaration implies (but is not + * exactly equivalent to) __attribute__ ((visibility("default"))), and so we + * actually want __declspec even on GCC, see "Microsoft Windows Function + * Attributes" in the GCC manual and the recommendations in + * https://gcc.gnu.org/wiki/Visibility. */ +# if defined(SECP256K1_BUILD) +# if defined(DLL_EXPORT) || defined(SECP256K1_DLL_EXPORT) + /* Building libsecp256k1 as a DLL. + * 1. If using Libtool, it defines DLL_EXPORT automatically. + * 2. In other cases, SECP256K1_DLL_EXPORT must be defined. */ +# define SECP256K1_API extern __declspec (dllexport) +# else + /* Building libsecp256k1 as a static library on Windows. + * No declspec is needed, and so we would want the non-Windows-specific + * logic below take care of this case. However, this may result in setting + * __attribute__ ((visibility("default"))), which is supposed to be a noop + * on Windows but may trigger warnings when compiling with -flto due to a + * bug in GCC, see + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=116478 . */ +# define SECP256K1_API extern +# endif + /* The user must define SECP256K1_STATIC when consuming libsecp256k1 as a static + * library on Windows. */ +# elif !defined(SECP256K1_STATIC) + /* Consuming libsecp256k1 as a DLL. */ +# define SECP256K1_API extern __declspec (dllimport) +# endif +#endif +#ifndef SECP256K1_API +/* All cases not captured by the Windows-specific logic. */ +# if defined(__GNUC__) && (__GNUC__ >= 4) && defined(SECP256K1_BUILD) + /* Building libsecp256k1 using GCC or compatible. */ +# define SECP256K1_API extern __attribute__ ((visibility ("default"))) +# else + /* Fall back to standard C's extern. */ +# define SECP256K1_API extern +# endif +#endif + +/* Warning attributes + * NONNULL is not used if SECP256K1_BUILD is set to avoid the compiler optimizing out + * some paranoid null checks. */ +# if defined(__GNUC__) && SECP256K1_GNUC_PREREQ(3, 4) +# define SECP256K1_WARN_UNUSED_RESULT __attribute__ ((__warn_unused_result__)) +# else +# define SECP256K1_WARN_UNUSED_RESULT +# endif +# if !defined(SECP256K1_BUILD) && defined(__GNUC__) && SECP256K1_GNUC_PREREQ(3, 4) +# define SECP256K1_ARG_NONNULL(_x) __attribute__ ((__nonnull__(_x))) +# else +# define SECP256K1_ARG_NONNULL(_x) +# endif + +/* Attribute for marking functions, types, and variables as deprecated */ +#if !defined(SECP256K1_BUILD) && defined(__has_attribute) +# if __has_attribute(__deprecated__) +# define SECP256K1_DEPRECATED(_msg) __attribute__ ((__deprecated__(_msg))) +# else +# define SECP256K1_DEPRECATED(_msg) +# endif +#else +# define SECP256K1_DEPRECATED(_msg) +#endif + +/* All flags' lower 8 bits indicate what they're for. Do not use directly. */ +#define SECP256K1_FLAGS_TYPE_MASK ((1 << 8) - 1) +#define SECP256K1_FLAGS_TYPE_CONTEXT (1 << 0) +#define SECP256K1_FLAGS_TYPE_COMPRESSION (1 << 1) +/* The higher bits contain the actual data. Do not use directly. */ +#define SECP256K1_FLAGS_BIT_CONTEXT_VERIFY (1 << 8) +#define SECP256K1_FLAGS_BIT_CONTEXT_SIGN (1 << 9) +#define SECP256K1_FLAGS_BIT_CONTEXT_DECLASSIFY (1 << 10) +#define SECP256K1_FLAGS_BIT_COMPRESSION (1 << 8) + +/** Context flags to pass to secp256k1_context_create, secp256k1_context_preallocated_size, and + * secp256k1_context_preallocated_create. */ +#define SECP256K1_CONTEXT_NONE (SECP256K1_FLAGS_TYPE_CONTEXT) + +/** Deprecated context flags. These flags are treated equivalent to SECP256K1_CONTEXT_NONE. */ +#define SECP256K1_CONTEXT_VERIFY (SECP256K1_FLAGS_TYPE_CONTEXT | SECP256K1_FLAGS_BIT_CONTEXT_VERIFY) +#define SECP256K1_CONTEXT_SIGN (SECP256K1_FLAGS_TYPE_CONTEXT | SECP256K1_FLAGS_BIT_CONTEXT_SIGN) + +/* Testing flag. Do not use. */ +#define SECP256K1_CONTEXT_DECLASSIFY (SECP256K1_FLAGS_TYPE_CONTEXT | SECP256K1_FLAGS_BIT_CONTEXT_DECLASSIFY) + +/** Flag to pass to secp256k1_ec_pubkey_serialize. */ +#define SECP256K1_EC_COMPRESSED (SECP256K1_FLAGS_TYPE_COMPRESSION | SECP256K1_FLAGS_BIT_COMPRESSION) +#define SECP256K1_EC_UNCOMPRESSED (SECP256K1_FLAGS_TYPE_COMPRESSION) + +/** Prefix byte used to tag various encoded curvepoints for specific purposes */ +#define SECP256K1_TAG_PUBKEY_EVEN 0x02 +#define SECP256K1_TAG_PUBKEY_ODD 0x03 +#define SECP256K1_TAG_PUBKEY_UNCOMPRESSED 0x04 +#define SECP256K1_TAG_PUBKEY_HYBRID_EVEN 0x06 +#define SECP256K1_TAG_PUBKEY_HYBRID_ODD 0x07 + +/** A built-in constant secp256k1 context object with static storage duration, to be + * used in conjunction with secp256k1_selftest. + * + * This context object offers *only limited functionality* , i.e., it cannot be used + * for API functions that perform computations involving secret keys, e.g., signing + * and public key generation. If this restriction applies to a specific API function, + * it is mentioned in its documentation. See secp256k1_context_create if you need a + * full context object that supports all functionality offered by the library. + * + * It is highly recommended to call secp256k1_selftest before using this context. + */ +SECP256K1_API const secp256k1_context *secp256k1_context_static; + +/** Deprecated alias for secp256k1_context_static. */ +SECP256K1_API const secp256k1_context *secp256k1_context_no_precomp +SECP256K1_DEPRECATED("Use secp256k1_context_static instead"); + +/** Perform basic self tests (to be used in conjunction with secp256k1_context_static) + * + * This function performs self tests that detect some serious usage errors and + * similar conditions, e.g., when the library is compiled for the wrong endianness. + * This is a last resort measure to be used in production. The performed tests are + * very rudimentary and are not intended as a replacement for running the test + * binaries. + * + * It is highly recommended to call this before using secp256k1_context_static. + * It is not necessary to call this function before using a context created with + * secp256k1_context_create (or secp256k1_context_preallocated_create), which will + * take care of performing the self tests. + * + * If the tests fail, this function will call the default error handler to abort the + * program (see secp256k1_context_set_error_callback). + */ +SECP256K1_API void secp256k1_selftest(void); + + +/** Create a secp256k1 context object (in dynamically allocated memory). + * + * This function uses malloc to allocate memory. It is guaranteed that malloc is + * called at most once for every call of this function. If you need to avoid dynamic + * memory allocation entirely, see secp256k1_context_static and the functions in + * secp256k1_preallocated.h. + * + * Returns: pointer to a newly created context object. + * In: flags: Always set to SECP256K1_CONTEXT_NONE (see below). + * + * The only valid non-deprecated flag in recent library versions is + * SECP256K1_CONTEXT_NONE, which will create a context sufficient for all functionality + * offered by the library. All other (deprecated) flags will be treated as equivalent + * to the SECP256K1_CONTEXT_NONE flag. Though the flags parameter primarily exists for + * historical reasons, future versions of the library may introduce new flags. + * + * If the context is intended to be used for API functions that perform computations + * involving secret keys, e.g., signing and public key generation, then it is highly + * recommended to call secp256k1_context_randomize on the context before calling + * those API functions. This will provide enhanced protection against side-channel + * leakage, see secp256k1_context_randomize for details. + * + * Do not create a new context object for each operation, as construction and + * randomization can take non-negligible time. + */ +SECP256K1_API secp256k1_context *secp256k1_context_create( + unsigned int flags +) SECP256K1_WARN_UNUSED_RESULT; + +/** Copy a secp256k1 context object (into dynamically allocated memory). + * + * This function uses malloc to allocate memory. It is guaranteed that malloc is + * called at most once for every call of this function. If you need to avoid dynamic + * memory allocation entirely, see the functions in secp256k1_preallocated.h. + * + * Cloning secp256k1_context_static is not possible, and should not be emulated by + * the caller (e.g., using memcpy). Create a new context instead. + * + * Returns: pointer to a newly created context object. + * Args: ctx: pointer to a context to copy (not secp256k1_context_static). + */ +SECP256K1_API secp256k1_context *secp256k1_context_clone( + const secp256k1_context *ctx +) SECP256K1_ARG_NONNULL(1) SECP256K1_WARN_UNUSED_RESULT; + +/** Destroy a secp256k1 context object (created in dynamically allocated memory). + * + * The context pointer may not be used afterwards. + * + * The context to destroy must have been created using secp256k1_context_create + * or secp256k1_context_clone. If the context has instead been created using + * secp256k1_context_preallocated_create or secp256k1_context_preallocated_clone, the + * behaviour is undefined. In that case, secp256k1_context_preallocated_destroy must + * be used instead. + * + * Args: ctx: pointer to a context to destroy, constructed using + * secp256k1_context_create or secp256k1_context_clone + * (i.e., not secp256k1_context_static). + */ +SECP256K1_API void secp256k1_context_destroy( + secp256k1_context *ctx +) SECP256K1_ARG_NONNULL(1); + +/** Set a callback function to be called when an illegal argument is passed to + * an API call. It will only trigger for violations that are mentioned + * explicitly in the header. + * + * The philosophy is that these shouldn't be dealt with through a + * specific return value, as calling code should not have branches to deal with + * the case that this code itself is broken. + * + * On the other hand, during debug stage, one would want to be informed about + * such mistakes, and the default (crashing) may be inadvisable. + * When this callback is triggered, the API function called is guaranteed not + * to cause a crash, though its return value and output arguments are + * undefined. + * + * When this function has not been called (or called with fn==NULL), then the + * default handler will be used. The library provides a default handler which + * writes the message to stderr and calls abort. This default handler can be + * replaced at link time if the preprocessor macro + * USE_EXTERNAL_DEFAULT_CALLBACKS is defined, which is the case if the build + * has been configured with --enable-external-default-callbacks. Then the + * following two symbols must be provided to link against: + * - void secp256k1_default_illegal_callback_fn(const char *message, void *data); + * - void secp256k1_default_error_callback_fn(const char *message, void *data); + * The library can call these default handlers even before a proper callback data + * pointer could have been set using secp256k1_context_set_illegal_callback or + * secp256k1_context_set_error_callback, e.g., when the creation of a context + * fails. In this case, the corresponding default handler will be called with + * the data pointer argument set to NULL. + * + * Args: ctx: pointer to a context object. + * In: fun: pointer to a function to call when an illegal argument is + * passed to the API, taking a message and an opaque pointer. + * (NULL restores the default handler.) + * data: the opaque pointer to pass to fun above, must be NULL for the default handler. + * + * See also secp256k1_context_set_error_callback. + */ +SECP256K1_API void secp256k1_context_set_illegal_callback( + secp256k1_context *ctx, + void (*fun)(const char *message, void *data), + const void *data +) SECP256K1_ARG_NONNULL(1); + +/** Set a callback function to be called when an internal consistency check + * fails. + * + * The default callback writes an error message to stderr and calls abort + * to abort the program. + * + * This can only trigger in case of a hardware failure, miscompilation, + * memory corruption, serious bug in the library, or other error would can + * otherwise result in undefined behaviour. It will not trigger due to mere + * incorrect usage of the API (see secp256k1_context_set_illegal_callback + * for that). After this callback returns, anything may happen, including + * crashing. + * + * Args: ctx: pointer to a context object. + * In: fun: pointer to a function to call when an internal error occurs, + * taking a message and an opaque pointer (NULL restores the + * default handler, see secp256k1_context_set_illegal_callback + * for details). + * data: the opaque pointer to pass to fun above, must be NULL for the default handler. + * + * See also secp256k1_context_set_illegal_callback. + */ +SECP256K1_API void secp256k1_context_set_error_callback( + secp256k1_context *ctx, + void (*fun)(const char *message, void *data), + const void *data +) SECP256K1_ARG_NONNULL(1); + +/** Parse a variable-length public key into the pubkey object. + * + * Returns: 1 if the public key was fully valid. + * 0 if the public key could not be parsed or is invalid. + * Args: ctx: pointer to a context object. + * Out: pubkey: pointer to a pubkey object. If 1 is returned, it is set to a + * parsed version of input. If not, its value is undefined. + * In: input: pointer to a serialized public key + * inputlen: length of the array pointed to by input + * + * This function supports parsing compressed (33 bytes, header byte 0x02 or + * 0x03), uncompressed (65 bytes, header byte 0x04), or hybrid (65 bytes, header + * byte 0x06 or 0x07) format public keys. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_parse( + const secp256k1_context *ctx, + secp256k1_pubkey *pubkey, + const unsigned char *input, + size_t inputlen +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Serialize a pubkey object into a serialized byte sequence. + * + * Returns: 1 always. + * Args: ctx: pointer to a context object. + * Out: output: pointer to a 65-byte (if compressed==0) or 33-byte (if + * compressed==1) byte array to place the serialized key + * in. + * In/Out: outputlen: pointer to an integer which is initially set to the + * size of output, and is overwritten with the written + * size. + * In: pubkey: pointer to a secp256k1_pubkey containing an + * initialized public key. + * flags: SECP256K1_EC_COMPRESSED if serialization should be in + * compressed format, otherwise SECP256K1_EC_UNCOMPRESSED. + */ +SECP256K1_API int secp256k1_ec_pubkey_serialize( + const secp256k1_context *ctx, + unsigned char *output, + size_t *outputlen, + const secp256k1_pubkey *pubkey, + unsigned int flags +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +/** Compare two public keys using lexicographic (of compressed serialization) order + * + * Returns: <0 if the first public key is less than the second + * >0 if the first public key is greater than the second + * 0 if the two public keys are equal + * Args: ctx: pointer to a context object + * In: pubkey1: first public key to compare + * pubkey2: second public key to compare + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_cmp( + const secp256k1_context *ctx, + const secp256k1_pubkey *pubkey1, + const secp256k1_pubkey *pubkey2 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Sort public keys using lexicographic (of compressed serialization) order + * + * Returns: 0 if the arguments are invalid. 1 otherwise. + * + * Args: ctx: pointer to a context object + * In: pubkeys: array of pointers to pubkeys to sort + * n_pubkeys: number of elements in the pubkeys array + */ +SECP256K1_API int secp256k1_ec_pubkey_sort( + const secp256k1_context *ctx, + const secp256k1_pubkey **pubkeys, + size_t n_pubkeys +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); + +/** Parse an ECDSA signature in compact (64 bytes) format. + * + * Returns: 1 when the signature could be parsed, 0 otherwise. + * Args: ctx: pointer to a context object + * Out: sig: pointer to a signature object + * In: input64: pointer to the 64-byte array to parse + * + * The signature must consist of a 32-byte big endian R value, followed by a + * 32-byte big endian S value. If R or S fall outside of [0..order-1], the + * encoding is invalid. R and S with value 0 are allowed in the encoding. + * + * After the call, sig will always be initialized. If parsing failed or R or + * S are zero, the resulting sig value is guaranteed to fail verification for + * any message and public key. + */ +SECP256K1_API int secp256k1_ecdsa_signature_parse_compact( + const secp256k1_context *ctx, + secp256k1_ecdsa_signature *sig, + const unsigned char *input64 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Parse a DER ECDSA signature. + * + * Returns: 1 when the signature could be parsed, 0 otherwise. + * Args: ctx: pointer to a context object + * Out: sig: pointer to a signature object + * In: input: pointer to the signature to be parsed + * inputlen: the length of the array pointed to be input + * + * This function will accept any valid DER encoded signature, even if the + * encoded numbers are out of range. + * + * After the call, sig will always be initialized. If parsing failed or the + * encoded numbers are out of range, signature verification with it is + * guaranteed to fail for every message and public key. + */ +SECP256K1_API int secp256k1_ecdsa_signature_parse_der( + const secp256k1_context *ctx, + secp256k1_ecdsa_signature *sig, + const unsigned char *input, + size_t inputlen +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Serialize an ECDSA signature in DER format. + * + * Returns: 1 if enough space was available to serialize, 0 otherwise + * Args: ctx: pointer to a context object + * Out: output: pointer to an array to store the DER serialization + * In/Out: outputlen: pointer to a length integer. Initially, this integer + * should be set to the length of output. After the call + * it will be set to the length of the serialization (even + * if 0 was returned). + * In: sig: pointer to an initialized signature object + */ +SECP256K1_API int secp256k1_ecdsa_signature_serialize_der( + const secp256k1_context *ctx, + unsigned char *output, + size_t *outputlen, + const secp256k1_ecdsa_signature *sig +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +/** Serialize an ECDSA signature in compact (64 byte) format. + * + * Returns: 1 + * Args: ctx: pointer to a context object + * Out: output64: pointer to a 64-byte array to store the compact serialization + * In: sig: pointer to an initialized signature object + * + * See secp256k1_ecdsa_signature_parse_compact for details about the encoding. + */ +SECP256K1_API int secp256k1_ecdsa_signature_serialize_compact( + const secp256k1_context *ctx, + unsigned char *output64, + const secp256k1_ecdsa_signature *sig +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Verify an ECDSA signature. + * + * Returns: 1: correct signature + * 0: incorrect or unparseable signature + * Args: ctx: pointer to a context object + * In: sig: the signature being verified. + * msghash32: the 32-byte message hash being verified. + * The verifier must make sure to apply a cryptographic + * hash function to the message by itself and not accept an + * msghash32 value directly. Otherwise, it would be easy to + * create a "valid" signature without knowledge of the + * secret key. See also + * https://bitcoin.stackexchange.com/a/81116/35586 for more + * background on this topic. + * pubkey: pointer to an initialized public key to verify with. + * + * To avoid accepting malleable signatures, only ECDSA signatures in lower-S + * form are accepted. + * + * If you need to accept ECDSA signatures from sources that do not obey this + * rule, apply secp256k1_ecdsa_signature_normalize to the signature prior to + * verification, but be aware that doing so results in malleable signatures. + * + * For details, see the comments for that function. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ecdsa_verify( + const secp256k1_context *ctx, + const secp256k1_ecdsa_signature *sig, + const unsigned char *msghash32, + const secp256k1_pubkey *pubkey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +/** Convert a signature to a normalized lower-S form. + * + * Returns: 1 if sigin was not normalized, 0 if it already was. + * Args: ctx: pointer to a context object + * Out: sigout: pointer to a signature to fill with the normalized form, + * or copy if the input was already normalized. (can be NULL if + * you're only interested in whether the input was already + * normalized). + * In: sigin: pointer to a signature to check/normalize (can be identical to sigout) + * + * With ECDSA a third-party can forge a second distinct signature of the same + * message, given a single initial signature, but without knowing the key. This + * is done by negating the S value modulo the order of the curve, 'flipping' + * the sign of the random point R which is not included in the signature. + * + * Forgery of the same message isn't universally problematic, but in systems + * where message malleability or uniqueness of signatures is important this can + * cause issues. This forgery can be blocked by all verifiers forcing signers + * to use a normalized form. + * + * The lower-S form reduces the size of signatures slightly on average when + * variable length encodings (such as DER) are used and is cheap to verify, + * making it a good choice. Security of always using lower-S is assured because + * anyone can trivially modify a signature after the fact to enforce this + * property anyway. + * + * The lower S value is always between 0x1 and + * 0x7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF5D576E7357A4501DDFE92F46681B20A0, + * inclusive. + * + * No other forms of ECDSA malleability are known and none seem likely, but + * there is no formal proof that ECDSA, even with this additional restriction, + * is free of other malleability. Commonly used serialization schemes will also + * accept various non-unique encodings, so care should be taken when this + * property is required for an application. + * + * The secp256k1_ecdsa_sign function will by default create signatures in the + * lower-S form, and secp256k1_ecdsa_verify will not accept others. In case + * signatures come from a system that cannot enforce this property, + * secp256k1_ecdsa_signature_normalize must be called before verification. + */ +SECP256K1_API int secp256k1_ecdsa_signature_normalize( + const secp256k1_context *ctx, + secp256k1_ecdsa_signature *sigout, + const secp256k1_ecdsa_signature *sigin +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(3); + +/** An implementation of RFC6979 (using HMAC-SHA256) as nonce generation function. + * If a data pointer is passed, it is assumed to be a pointer to 32 bytes of + * extra entropy. + */ +SECP256K1_API const secp256k1_nonce_function secp256k1_nonce_function_rfc6979; + +/** A default safe nonce generation function (currently equal to secp256k1_nonce_function_rfc6979). */ +SECP256K1_API const secp256k1_nonce_function secp256k1_nonce_function_default; + +/** Create an ECDSA signature. + * + * Returns: 1: signature created + * 0: the nonce generation function failed, or the secret key was invalid. + * Args: ctx: pointer to a context object (not secp256k1_context_static). + * Out: sig: pointer to an array where the signature will be placed. + * In: msghash32: the 32-byte message hash being signed. + * seckey: pointer to a 32-byte secret key. + * noncefp: pointer to a nonce generation function. If NULL, + * secp256k1_nonce_function_default is used. + * ndata: pointer to arbitrary data used by the nonce generation function + * (can be NULL). If it is non-NULL and + * secp256k1_nonce_function_default is used, then ndata must be a + * pointer to 32-bytes of additional data. + * + * The created signature is always in lower-S form. See + * secp256k1_ecdsa_signature_normalize for more details. + */ +SECP256K1_API int secp256k1_ecdsa_sign( + const secp256k1_context *ctx, + secp256k1_ecdsa_signature *sig, + const unsigned char *msghash32, + const unsigned char *seckey, + secp256k1_nonce_function noncefp, + const void *ndata +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +/** Verify an elliptic curve secret key. + * + * A secret key is valid if it is not 0 and less than the secp256k1 curve order + * when interpreted as an integer (most significant byte first). The + * probability of choosing a 32-byte string uniformly at random which is an + * invalid secret key is negligible. However, if it does happen it should + * be assumed that the randomness source is severely broken and there should + * be no retry. + * + * Returns: 1: secret key is valid + * 0: secret key is invalid + * Args: ctx: pointer to a context object. + * In: seckey: pointer to a 32-byte secret key. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_seckey_verify( + const secp256k1_context *ctx, + const unsigned char *seckey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); + +/** Compute the public key for a secret key. + * + * Returns: 1: secret was valid, public key stores. + * 0: secret was invalid, try again. + * Args: ctx: pointer to a context object (not secp256k1_context_static). + * Out: pubkey: pointer to the created public key. + * In: seckey: pointer to a 32-byte secret key. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_create( + const secp256k1_context *ctx, + secp256k1_pubkey *pubkey, + const unsigned char *seckey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Negates a secret key in place. + * + * Returns: 0 if the given secret key is invalid according to + * secp256k1_ec_seckey_verify. 1 otherwise + * Args: ctx: pointer to a context object + * In/Out: seckey: pointer to the 32-byte secret key to be negated. If the + * secret key is invalid according to + * secp256k1_ec_seckey_verify, this function returns 0 and + * seckey will be set to some unspecified value. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_seckey_negate( + const secp256k1_context *ctx, + unsigned char *seckey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); + +/** Same as secp256k1_ec_seckey_negate, but DEPRECATED. Will be removed in + * future versions. */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_negate( + const secp256k1_context *ctx, + unsigned char *seckey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) + SECP256K1_DEPRECATED("Use secp256k1_ec_seckey_negate instead"); + +/** Negates a public key in place. + * + * Returns: 1 always + * Args: ctx: pointer to a context object + * In/Out: pubkey: pointer to the public key to be negated. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_negate( + const secp256k1_context *ctx, + secp256k1_pubkey *pubkey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); + +/** Tweak a secret key by adding tweak to it. + * + * Returns: 0 if the arguments are invalid or the resulting secret key would be + * invalid (only when the tweak is the negation of the secret key). 1 + * otherwise. + * Args: ctx: pointer to a context object. + * In/Out: seckey: pointer to a 32-byte secret key. If the secret key is + * invalid according to secp256k1_ec_seckey_verify, this + * function returns 0. seckey will be set to some unspecified + * value if this function returns 0. + * In: tweak32: pointer to a 32-byte tweak, which must be valid according to + * secp256k1_ec_seckey_verify or 32 zero bytes. For uniformly + * random 32-byte tweaks, the chance of being invalid is + * negligible (around 1 in 2^128). + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_seckey_tweak_add( + const secp256k1_context *ctx, + unsigned char *seckey, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Same as secp256k1_ec_seckey_tweak_add, but DEPRECATED. Will be removed in + * future versions. */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_tweak_add( + const secp256k1_context *ctx, + unsigned char *seckey, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) + SECP256K1_DEPRECATED("Use secp256k1_ec_seckey_tweak_add instead"); + +/** Tweak a public key by adding tweak times the generator to it. + * + * Returns: 0 if the arguments are invalid or the resulting public key would be + * invalid (only when the tweak is the negation of the corresponding + * secret key). 1 otherwise. + * Args: ctx: pointer to a context object. + * In/Out: pubkey: pointer to a public key object. pubkey will be set to an + * invalid value if this function returns 0. + * In: tweak32: pointer to a 32-byte tweak, which must be valid according to + * secp256k1_ec_seckey_verify or 32 zero bytes. For uniformly + * random 32-byte tweaks, the chance of being invalid is + * negligible (around 1 in 2^128). + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_tweak_add( + const secp256k1_context *ctx, + secp256k1_pubkey *pubkey, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Tweak a secret key by multiplying it by a tweak. + * + * Returns: 0 if the arguments are invalid. 1 otherwise. + * Args: ctx: pointer to a context object. + * In/Out: seckey: pointer to a 32-byte secret key. If the secret key is + * invalid according to secp256k1_ec_seckey_verify, this + * function returns 0. seckey will be set to some unspecified + * value if this function returns 0. + * In: tweak32: pointer to a 32-byte tweak. If the tweak is invalid according to + * secp256k1_ec_seckey_verify, this function returns 0. For + * uniformly random 32-byte arrays the chance of being invalid + * is negligible (around 1 in 2^128). + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_seckey_tweak_mul( + const secp256k1_context *ctx, + unsigned char *seckey, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Same as secp256k1_ec_seckey_tweak_mul, but DEPRECATED. Will be removed in + * future versions. */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_tweak_mul( + const secp256k1_context *ctx, + unsigned char *seckey, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) + SECP256K1_DEPRECATED("Use secp256k1_ec_seckey_tweak_mul instead"); + +/** Tweak a public key by multiplying it by a tweak value. + * + * Returns: 0 if the arguments are invalid. 1 otherwise. + * Args: ctx: pointer to a context object. + * In/Out: pubkey: pointer to a public key object. pubkey will be set to an + * invalid value if this function returns 0. + * In: tweak32: pointer to a 32-byte tweak. If the tweak is invalid according to + * secp256k1_ec_seckey_verify, this function returns 0. For + * uniformly random 32-byte arrays the chance of being invalid + * is negligible (around 1 in 2^128). + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_tweak_mul( + const secp256k1_context *ctx, + secp256k1_pubkey *pubkey, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Randomizes the context to provide enhanced protection against side-channel leakage. + * + * Returns: 1: randomization successful + * 0: error + * Args: ctx: pointer to a context object (not secp256k1_context_static). + * In: seed32: pointer to a 32-byte random seed (NULL resets to initial state). + * + * While secp256k1 code is written and tested to be constant-time no matter what + * secret values are, it is possible that a compiler may output code which is not, + * and also that the CPU may not emit the same radio frequencies or draw the same + * amount of power for all values. Randomization of the context shields against + * side-channel observations which aim to exploit secret-dependent behaviour in + * certain computations which involve secret keys. + * + * It is highly recommended to call this function on contexts returned from + * secp256k1_context_create or secp256k1_context_clone (or from the corresponding + * functions in secp256k1_preallocated.h) before using these contexts to call API + * functions that perform computations involving secret keys, e.g., signing and + * public key generation. It is possible to call this function more than once on + * the same context, and doing so before every few computations involving secret + * keys is recommended as a defense-in-depth measure. Randomization of the static + * context secp256k1_context_static is not supported. + * + * Currently, the random seed is mainly used for blinding multiplications of a + * secret scalar with the elliptic curve base point. Multiplications of this + * kind are performed by exactly those API functions which are documented to + * require a context that is not secp256k1_context_static. As a rule of thumb, + * these are all functions which take a secret key (or a keypair) as an input. + * A notable exception to that rule is the ECDH module, which relies on a different + * kind of elliptic curve point multiplication and thus does not benefit from + * enhanced protection against side-channel leakage currently. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_context_randomize( + secp256k1_context *ctx, + const unsigned char *seed32 +) SECP256K1_ARG_NONNULL(1); + +/** Add a number of public keys together. + * + * Returns: 1: the sum of the public keys is valid. + * 0: the sum of the public keys is not valid. + * Args: ctx: pointer to a context object. + * Out: out: pointer to a public key object for placing the resulting public key. + * In: ins: pointer to array of pointers to public keys. + * n: the number of public keys to add together (must be at least 1). + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_combine( + const secp256k1_context *ctx, + secp256k1_pubkey *out, + const secp256k1_pubkey * const *ins, + size_t n +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Compute a tagged hash as defined in BIP-340. + * + * This is useful for creating a message hash and achieving domain separation + * through an application-specific tag. This function returns + * SHA256(SHA256(tag)||SHA256(tag)||msg). Therefore, tagged hash + * implementations optimized for a specific tag can precompute the SHA256 state + * after hashing the tag hashes. + * + * Returns: 1 always. + * Args: ctx: pointer to a context object + * Out: hash32: pointer to a 32-byte array to store the resulting hash + * In: tag: pointer to an array containing the tag + * taglen: length of the tag array + * msg: pointer to an array containing the message + * msglen: length of the message array + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_tagged_sha256( + const secp256k1_context *ctx, + unsigned char *hash32, + const unsigned char *tag, + size_t taglen, + const unsigned char *msg, + size_t msglen +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(5); + +#ifdef __cplusplus +} +#endif + +#endif /* SECP256K1_H */ diff --git a/example/android/third_party/secp256k1/include/secp256k1_ecdh.h b/example/android/third_party/secp256k1/include/secp256k1_ecdh.h new file mode 100644 index 00000000..4d9da346 --- /dev/null +++ b/example/android/third_party/secp256k1/include/secp256k1_ecdh.h @@ -0,0 +1,63 @@ +#ifndef SECP256K1_ECDH_H +#define SECP256K1_ECDH_H + +#include "secp256k1.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** A pointer to a function that hashes an EC point to obtain an ECDH secret + * + * Returns: 1 if the point was successfully hashed. + * 0 will cause secp256k1_ecdh to fail and return 0. + * Other return values are not allowed, and the behaviour of + * secp256k1_ecdh is undefined for other return values. + * Out: output: pointer to an array to be filled by the function + * In: x32: pointer to a 32-byte x coordinate + * y32: pointer to a 32-byte y coordinate + * data: arbitrary data pointer that is passed through + */ +typedef int (*secp256k1_ecdh_hash_function)( + unsigned char *output, + const unsigned char *x32, + const unsigned char *y32, + void *data +); + +/** An implementation of SHA256 hash function that applies to compressed public key. + * Populates the output parameter with 32 bytes. */ +SECP256K1_API const secp256k1_ecdh_hash_function secp256k1_ecdh_hash_function_sha256; + +/** A default ECDH hash function (currently equal to secp256k1_ecdh_hash_function_sha256). + * Populates the output parameter with 32 bytes. */ +SECP256K1_API const secp256k1_ecdh_hash_function secp256k1_ecdh_hash_function_default; + +/** Compute an EC Diffie-Hellman secret in constant time + * + * Returns: 1: exponentiation was successful + * 0: scalar was invalid (zero or overflow) or hashfp returned 0 + * Args: ctx: pointer to a context object. + * Out: output: pointer to an array to be filled by hashfp. + * In: pubkey: pointer to a secp256k1_pubkey containing an initialized public key. + * seckey: a 32-byte scalar with which to multiply the point. + * hashfp: pointer to a hash function. If NULL, + * secp256k1_ecdh_hash_function_sha256 is used + * (in which case, 32 bytes will be written to output). + * data: arbitrary data pointer that is passed through to hashfp + * (can be NULL for secp256k1_ecdh_hash_function_sha256). + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ecdh( + const secp256k1_context *ctx, + unsigned char *output, + const secp256k1_pubkey *pubkey, + const unsigned char *seckey, + secp256k1_ecdh_hash_function hashfp, + void *data +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +#ifdef __cplusplus +} +#endif + +#endif /* SECP256K1_ECDH_H */ diff --git a/example/android/third_party/secp256k1/include/secp256k1_ellswift.h b/example/android/third_party/secp256k1/include/secp256k1_ellswift.h new file mode 100644 index 00000000..0d1293e9 --- /dev/null +++ b/example/android/third_party/secp256k1/include/secp256k1_ellswift.h @@ -0,0 +1,200 @@ +#ifndef SECP256K1_ELLSWIFT_H +#define SECP256K1_ELLSWIFT_H + +#include "secp256k1.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* This module provides an implementation of ElligatorSwift as well as a + * version of x-only ECDH using it (including compatibility with BIP324). + * + * ElligatorSwift is described in https://eprint.iacr.org/2022/759 by + * Chavez-Saab, Rodriguez-Henriquez, and Tibouchi. It permits encoding + * uniformly chosen public keys as 64-byte arrays which are indistinguishable + * from uniformly random arrays. + * + * Let f be the function from pairs of field elements to point X coordinates, + * defined as follows (all operations modulo p = 2^256 - 2^32 - 977) + * f(u,t): + * - Let C = 0xa2d2ba93507f1df233770c2a797962cc61f6d15da14ecd47d8d27ae1cd5f852, + * a square root of -3. + * - If u=0, set u=1 instead. + * - If t=0, set t=1 instead. + * - If u^3 + t^2 + 7 = 0, multiply t by 2. + * - Let X = (u^3 + 7 - t^2) / (2 * t) + * - Let Y = (X + t) / (C * u) + * - Return the first in [u + 4 * Y^2, (-X/Y - u) / 2, (X/Y - u) / 2] that is an + * X coordinate on the curve (at least one of them is, for any u and t). + * + * Then an ElligatorSwift encoding of x consists of the 32-byte big-endian + * encodings of field elements u and t concatenated, where f(u,t) = x. + * The encoding algorithm is described in the paper, and effectively picks a + * uniformly random pair (u,t) among those which encode x. + * + * If the Y coordinate is relevant, it is given the same parity as t. + * + * Changes w.r.t. the paper: + * - The u=0, t=0, and u^3+t^2+7=0 conditions result in decoding to the point + * at infinity in the paper. Here they are remapped to finite points. + * - The paper uses an additional encoding bit for the parity of y. Here the + * parity of t is used (negating t does not affect the decoded x coordinate, + * so this is possible). + * + * For mathematical background about the scheme, see the doc/ellswift.md file. + */ + +/** A pointer to a function used by secp256k1_ellswift_xdh to hash the shared X + * coordinate along with the encoded public keys to a uniform shared secret. + * + * Returns: 1 if a shared secret was successfully computed. + * 0 will cause secp256k1_ellswift_xdh to fail and return 0. + * Other return values are not allowed, and the behaviour of + * secp256k1_ellswift_xdh is undefined for other return values. + * Out: output: pointer to an array to be filled by the function + * In: x32: pointer to the 32-byte serialized X coordinate + * of the resulting shared point (will not be NULL) + * ell_a64: pointer to the 64-byte encoded public key of party A + * (will not be NULL) + * ell_b64: pointer to the 64-byte encoded public key of party B + * (will not be NULL) + * data: arbitrary data pointer that is passed through + */ +typedef int (*secp256k1_ellswift_xdh_hash_function)( + unsigned char *output, + const unsigned char *x32, + const unsigned char *ell_a64, + const unsigned char *ell_b64, + void *data +); + +/** An implementation of an secp256k1_ellswift_xdh_hash_function which uses + * SHA256(prefix64 || ell_a64 || ell_b64 || x32), where prefix64 is the 64-byte + * array pointed to by data. */ +SECP256K1_API const secp256k1_ellswift_xdh_hash_function secp256k1_ellswift_xdh_hash_function_prefix; + +/** An implementation of an secp256k1_ellswift_xdh_hash_function compatible with + * BIP324. It returns H_tag(ell_a64 || ell_b64 || x32), where H_tag is the + * BIP340 tagged hash function with tag "bip324_ellswift_xonly_ecdh". Equivalent + * to secp256k1_ellswift_xdh_hash_function_prefix with prefix64 set to + * SHA256("bip324_ellswift_xonly_ecdh")||SHA256("bip324_ellswift_xonly_ecdh"). + * The data argument is ignored. */ +SECP256K1_API const secp256k1_ellswift_xdh_hash_function secp256k1_ellswift_xdh_hash_function_bip324; + +/** Construct a 64-byte ElligatorSwift encoding of a given pubkey. + * + * Returns: 1 always. + * Args: ctx: pointer to a context object + * Out: ell64: pointer to a 64-byte array to be filled + * In: pubkey: pointer to a secp256k1_pubkey containing an + * initialized public key + * rnd32: pointer to 32 bytes of randomness + * + * It is recommended that rnd32 consists of 32 uniformly random bytes, not + * known to any adversary trying to detect whether public keys are being + * encoded, though 16 bytes of randomness (padded to an array of 32 bytes, + * e.g., with zeros) suffice to make the result indistinguishable from + * uniform. The randomness in rnd32 must not be a deterministic function of + * the pubkey (it can be derived from the private key, though). + * + * It is not guaranteed that the computed encoding is stable across versions + * of the library, even if all arguments to this function (including rnd32) + * are the same. + * + * This function runs in variable time. + */ +SECP256K1_API int secp256k1_ellswift_encode( + const secp256k1_context *ctx, + unsigned char *ell64, + const secp256k1_pubkey *pubkey, + const unsigned char *rnd32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +/** Decode a 64-bytes ElligatorSwift encoded public key. + * + * Returns: always 1 + * Args: ctx: pointer to a context object + * Out: pubkey: pointer to a secp256k1_pubkey that will be filled + * In: ell64: pointer to a 64-byte array to decode + * + * This function runs in variable time. + */ +SECP256K1_API int secp256k1_ellswift_decode( + const secp256k1_context *ctx, + secp256k1_pubkey *pubkey, + const unsigned char *ell64 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Compute an ElligatorSwift public key for a secret key. + * + * Returns: 1: secret was valid, public key was stored. + * 0: secret was invalid, try again. + * Args: ctx: pointer to a context object + * Out: ell64: pointer to a 64-byte array to receive the ElligatorSwift + * public key + * In: seckey32: pointer to a 32-byte secret key + * auxrnd32: (optional) pointer to 32 bytes of randomness + * + * Constant time in seckey and auxrnd32, but not in the resulting public key. + * + * It is recommended that auxrnd32 contains 32 uniformly random bytes, though + * it is optional (and does result in encodings that are indistinguishable from + * uniform even without any auxrnd32). It differs from the (mandatory) rnd32 + * argument to secp256k1_ellswift_encode in this regard. + * + * This function can be used instead of calling secp256k1_ec_pubkey_create + * followed by secp256k1_ellswift_encode. It is safer, as it uses the secret + * key as entropy for the encoding (supplemented with auxrnd32, if provided). + * + * Like secp256k1_ellswift_encode, this function does not guarantee that the + * computed encoding is stable across versions of the library, even if all + * arguments (including auxrnd32) are the same. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ellswift_create( + const secp256k1_context *ctx, + unsigned char *ell64, + const unsigned char *seckey32, + const unsigned char *auxrnd32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Given a private key, and ElligatorSwift public keys sent in both directions, + * compute a shared secret using x-only Elliptic Curve Diffie-Hellman (ECDH). + * + * Returns: 1: shared secret was successfully computed + * 0: secret was invalid or hashfp returned 0 + * Args: ctx: pointer to a context object. + * Out: output: pointer to an array to be filled by hashfp. + * In: ell_a64: pointer to the 64-byte encoded public key of party A + * (will not be NULL) + * ell_b64: pointer to the 64-byte encoded public key of party B + * (will not be NULL) + * seckey32: pointer to our 32-byte secret key + * party: boolean indicating which party we are: zero if we are + * party A, non-zero if we are party B. seckey32 must be + * the private key corresponding to that party's ell_?64. + * This correspondence is not checked. + * hashfp: pointer to a hash function. + * data: arbitrary data pointer passed through to hashfp. + * + * Constant time in seckey32. + * + * This function is more efficient than decoding the public keys, and performing + * ECDH on them. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ellswift_xdh( + const secp256k1_context *ctx, + unsigned char *output, + const unsigned char *ell_a64, + const unsigned char *ell_b64, + const unsigned char *seckey32, + int party, + secp256k1_ellswift_xdh_hash_function hashfp, + void *data +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(5) SECP256K1_ARG_NONNULL(7); + +#ifdef __cplusplus +} +#endif + +#endif /* SECP256K1_ELLSWIFT_H */ diff --git a/example/android/third_party/secp256k1/include/secp256k1_extrakeys.h b/example/android/third_party/secp256k1/include/secp256k1_extrakeys.h new file mode 100644 index 00000000..48c98693 --- /dev/null +++ b/example/android/third_party/secp256k1/include/secp256k1_extrakeys.h @@ -0,0 +1,250 @@ +#ifndef SECP256K1_EXTRAKEYS_H +#define SECP256K1_EXTRAKEYS_H + +#include "secp256k1.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** Opaque data structure that holds a parsed and valid "x-only" public key. + * An x-only pubkey encodes a point whose Y coordinate is even. It is + * serialized using only its X coordinate (32 bytes). See BIP-340 for more + * information about x-only pubkeys. + * + * The exact representation of data inside is implementation defined and not + * guaranteed to be portable between different platforms or versions. It is + * however guaranteed to be 64 bytes in size, and can be safely copied/moved. + * If you need to convert to a format suitable for storage, transmission, use + * use secp256k1_xonly_pubkey_serialize and secp256k1_xonly_pubkey_parse. To + * compare keys, use secp256k1_xonly_pubkey_cmp. + */ +typedef struct secp256k1_xonly_pubkey { + unsigned char data[64]; +} secp256k1_xonly_pubkey; + +/** Opaque data structure that holds a keypair consisting of a secret and a + * public key. + * + * The exact representation of data inside is implementation defined and not + * guaranteed to be portable between different platforms or versions. It is + * however guaranteed to be 96 bytes in size, and can be safely copied/moved. + */ +typedef struct secp256k1_keypair { + unsigned char data[96]; +} secp256k1_keypair; + +/** Parse a 32-byte sequence into a xonly_pubkey object. + * + * Returns: 1 if the public key was fully valid. + * 0 if the public key could not be parsed or is invalid. + * + * Args: ctx: pointer to a context object. + * Out: pubkey: pointer to a pubkey object. If 1 is returned, it is set to a + * parsed version of input. If not, it's set to an invalid value. + * In: input32: pointer to a serialized xonly_pubkey. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_xonly_pubkey_parse( + const secp256k1_context *ctx, + secp256k1_xonly_pubkey *pubkey, + const unsigned char *input32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Serialize an xonly_pubkey object into a 32-byte sequence. + * + * Returns: 1 always. + * + * Args: ctx: pointer to a context object. + * Out: output32: pointer to a 32-byte array to place the serialized key in. + * In: pubkey: pointer to a secp256k1_xonly_pubkey containing an initialized public key. + */ +SECP256K1_API int secp256k1_xonly_pubkey_serialize( + const secp256k1_context *ctx, + unsigned char *output32, + const secp256k1_xonly_pubkey *pubkey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Compare two x-only public keys using lexicographic order + * + * Returns: <0 if the first public key is less than the second + * >0 if the first public key is greater than the second + * 0 if the two public keys are equal + * Args: ctx: pointer to a context object. + * In: pubkey1: first public key to compare + * pubkey2: second public key to compare + */ +SECP256K1_API int secp256k1_xonly_pubkey_cmp( + const secp256k1_context *ctx, + const secp256k1_xonly_pubkey *pk1, + const secp256k1_xonly_pubkey *pk2 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Converts a secp256k1_pubkey into a secp256k1_xonly_pubkey. + * + * Returns: 1 always. + * + * Args: ctx: pointer to a context object. + * Out: xonly_pubkey: pointer to an x-only public key object for placing the converted public key. + * pk_parity: Ignored if NULL. Otherwise, pointer to an integer that + * will be set to 1 if the point encoded by xonly_pubkey is + * the negation of the pubkey and set to 0 otherwise. + * In: pubkey: pointer to a public key that is converted. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_xonly_pubkey_from_pubkey( + const secp256k1_context *ctx, + secp256k1_xonly_pubkey *xonly_pubkey, + int *pk_parity, + const secp256k1_pubkey *pubkey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(4); + +/** Tweak an x-only public key by adding the generator multiplied with tweak32 + * to it. + * + * Note that the resulting point can not in general be represented by an x-only + * pubkey because it may have an odd Y coordinate. Instead, the output_pubkey + * is a normal secp256k1_pubkey. + * + * Returns: 0 if the arguments are invalid or the resulting public key would be + * invalid (only when the tweak is the negation of the corresponding + * secret key). 1 otherwise. + * + * Args: ctx: pointer to a context object. + * Out: output_pubkey: pointer to a public key to store the result. Will be set + * to an invalid value if this function returns 0. + * In: internal_pubkey: pointer to an x-only pubkey to apply the tweak to. + * tweak32: pointer to a 32-byte tweak, which must be valid + * according to secp256k1_ec_seckey_verify or 32 zero + * bytes. For uniformly random 32-byte tweaks, the chance of + * being invalid is negligible (around 1 in 2^128). + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_xonly_pubkey_tweak_add( + const secp256k1_context *ctx, + secp256k1_pubkey *output_pubkey, + const secp256k1_xonly_pubkey *internal_pubkey, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +/** Checks that a tweaked pubkey is the result of calling + * secp256k1_xonly_pubkey_tweak_add with internal_pubkey and tweak32. + * + * The tweaked pubkey is represented by its 32-byte x-only serialization and + * its pk_parity, which can both be obtained by converting the result of + * tweak_add to a secp256k1_xonly_pubkey. + * + * Note that this alone does _not_ verify that the tweaked pubkey is a + * commitment. If the tweak is not chosen in a specific way, the tweaked pubkey + * can easily be the result of a different internal_pubkey and tweak. + * + * Returns: 0 if the arguments are invalid or the tweaked pubkey is not the + * result of tweaking the internal_pubkey with tweak32. 1 otherwise. + * Args: ctx: pointer to a context object. + * In: tweaked_pubkey32: pointer to a serialized xonly_pubkey. + * tweaked_pk_parity: the parity of the tweaked pubkey (whose serialization + * is passed in as tweaked_pubkey32). This must match the + * pk_parity value that is returned when calling + * secp256k1_xonly_pubkey with the tweaked pubkey, or + * this function will fail. + * internal_pubkey: pointer to an x-only public key object to apply the tweak to. + * tweak32: pointer to a 32-byte tweak. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_xonly_pubkey_tweak_add_check( + const secp256k1_context *ctx, + const unsigned char *tweaked_pubkey32, + int tweaked_pk_parity, + const secp256k1_xonly_pubkey *internal_pubkey, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(5); + +/** Compute the keypair for a valid secret key. + * + * See the documentation of `secp256k1_ec_seckey_verify` for more information + * about the validity of secret keys. + * + * Returns: 1: secret key is valid + * 0: secret key is invalid + * Args: ctx: pointer to a context object (not secp256k1_context_static). + * Out: keypair: pointer to the created keypair. + * In: seckey: pointer to a 32-byte secret key. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_keypair_create( + const secp256k1_context *ctx, + secp256k1_keypair *keypair, + const unsigned char *seckey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Get the secret key from a keypair. + * + * Returns: 1 always. + * Args: ctx: pointer to a context object. + * Out: seckey: pointer to a 32-byte buffer for the secret key. + * In: keypair: pointer to a keypair. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_keypair_sec( + const secp256k1_context *ctx, + unsigned char *seckey, + const secp256k1_keypair *keypair +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Get the public key from a keypair. + * + * Returns: 1 always. + * Args: ctx: pointer to a context object. + * Out: pubkey: pointer to a pubkey object, set to the keypair public key. + * In: keypair: pointer to a keypair. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_keypair_pub( + const secp256k1_context *ctx, + secp256k1_pubkey *pubkey, + const secp256k1_keypair *keypair +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Get the x-only public key from a keypair. + * + * This is the same as calling secp256k1_keypair_pub and then + * secp256k1_xonly_pubkey_from_pubkey. + * + * Returns: 1 always. + * Args: ctx: pointer to a context object. + * Out: pubkey: pointer to an xonly_pubkey object, set to the keypair + * public key after converting it to an xonly_pubkey. + * pk_parity: Ignored if NULL. Otherwise, pointer to an integer that will be set to the + * pk_parity argument of secp256k1_xonly_pubkey_from_pubkey. + * In: keypair: pointer to a keypair. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_keypair_xonly_pub( + const secp256k1_context *ctx, + secp256k1_xonly_pubkey *pubkey, + int *pk_parity, + const secp256k1_keypair *keypair +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(4); + +/** Tweak a keypair by adding tweak32 to the secret key and updating the public + * key accordingly. + * + * Calling this function and then secp256k1_keypair_pub results in the same + * public key as calling secp256k1_keypair_xonly_pub and then + * secp256k1_xonly_pubkey_tweak_add. + * + * Returns: 0 if the arguments are invalid or the resulting keypair would be + * invalid (only when the tweak is the negation of the keypair's + * secret key). 1 otherwise. + * + * Args: ctx: pointer to a context object. + * In/Out: keypair: pointer to a keypair to apply the tweak to. Will be set to + * an invalid value if this function returns 0. + * In: tweak32: pointer to a 32-byte tweak, which must be valid according to + * secp256k1_ec_seckey_verify or 32 zero bytes. For uniformly + * random 32-byte tweaks, the chance of being invalid is + * negligible (around 1 in 2^128). + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_keypair_xonly_tweak_add( + const secp256k1_context *ctx, + secp256k1_keypair *keypair, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +#ifdef __cplusplus +} +#endif + +#endif /* SECP256K1_EXTRAKEYS_H */ diff --git a/example/android/third_party/secp256k1/include/secp256k1_musig.h b/example/android/third_party/secp256k1/include/secp256k1_musig.h new file mode 100644 index 00000000..11b8f08c --- /dev/null +++ b/example/android/third_party/secp256k1/include/secp256k1_musig.h @@ -0,0 +1,588 @@ +#ifndef SECP256K1_MUSIG_H +#define SECP256K1_MUSIG_H + +#include "secp256k1_extrakeys.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +/** This module implements BIP 327 "MuSig2 for BIP340-compatible + * Multi-Signatures" + * (https://github.com/bitcoin/bips/blob/master/bip-0327.mediawiki) + * v1.0.0. You can find an example demonstrating the musig module in + * examples/musig.c. + * + * The module also supports BIP 341 ("Taproot") public key tweaking. + * + * It is recommended to read the documentation in this include file carefully. + * Further notes on API usage can be found in doc/musig.md + * + * Since the first version of MuSig is essentially replaced by MuSig2, we use + * MuSig, musig and MuSig2 synonymously unless noted otherwise. + */ + +/** Opaque data structures + * + * The exact representation of data inside the opaque data structures is + * implementation defined and not guaranteed to be portable between different + * platforms or versions. With the exception of `secp256k1_musig_secnonce`, the + * data structures can be safely copied/moved. If you need to convert to a + * format suitable for storage, transmission, or comparison, use the + * corresponding serialization and parsing functions. + */ + +/** Opaque data structure that caches information about public key aggregation. + * + * Guaranteed to be 197 bytes in size. No serialization and parsing functions + * (yet). + */ +typedef struct secp256k1_musig_keyagg_cache { + unsigned char data[197]; +} secp256k1_musig_keyagg_cache; + +/** Opaque data structure that holds a signer's _secret_ nonce. + * + * Guaranteed to be 132 bytes in size. + * + * WARNING: This structure MUST NOT be copied or read or written to directly. A + * signer who is online throughout the whole process and can keep this + * structure in memory can use the provided API functions for a safe standard + * workflow. + * + * Copying this data structure can result in nonce reuse which will leak the + * secret signing key. + */ +typedef struct secp256k1_musig_secnonce { + unsigned char data[132]; +} secp256k1_musig_secnonce; + +/** Opaque data structure that holds a signer's public nonce. + * + * Guaranteed to be 132 bytes in size. Serialized and parsed with + * `musig_pubnonce_serialize` and `musig_pubnonce_parse`. + */ +typedef struct secp256k1_musig_pubnonce { + unsigned char data[132]; +} secp256k1_musig_pubnonce; + +/** Opaque data structure that holds an aggregate public nonce. + * + * Guaranteed to be 132 bytes in size. Serialized and parsed with + * `musig_aggnonce_serialize` and `musig_aggnonce_parse`. + */ +typedef struct secp256k1_musig_aggnonce { + unsigned char data[132]; +} secp256k1_musig_aggnonce; + +/** Opaque data structure that holds a MuSig session. + * + * This structure is not required to be kept secret for the signing protocol to + * be secure. Guaranteed to be 133 bytes in size. No serialization and parsing + * functions (yet). + */ +typedef struct secp256k1_musig_session { + unsigned char data[133]; +} secp256k1_musig_session; + +/** Opaque data structure that holds a partial MuSig signature. + * + * Guaranteed to be 36 bytes in size. Serialized and parsed with + * `musig_partial_sig_serialize` and `musig_partial_sig_parse`. + */ +typedef struct secp256k1_musig_partial_sig { + unsigned char data[36]; +} secp256k1_musig_partial_sig; + +/** Parse a signer's public nonce. + * + * Returns: 1 when the nonce could be parsed, 0 otherwise. + * Args: ctx: pointer to a context object + * Out: nonce: pointer to a nonce object + * In: in66: pointer to the 66-byte nonce to be parsed + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_pubnonce_parse( + const secp256k1_context *ctx, + secp256k1_musig_pubnonce *nonce, + const unsigned char *in66 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Serialize a signer's public nonce + * + * Returns: 1 always + * Args: ctx: pointer to a context object + * Out: out66: pointer to a 66-byte array to store the serialized nonce + * In: nonce: pointer to the nonce + */ +SECP256K1_API int secp256k1_musig_pubnonce_serialize( + const secp256k1_context *ctx, + unsigned char *out66, + const secp256k1_musig_pubnonce *nonce +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Parse an aggregate public nonce. + * + * Returns: 1 when the nonce could be parsed, 0 otherwise. + * Args: ctx: pointer to a context object + * Out: nonce: pointer to a nonce object + * In: in66: pointer to the 66-byte nonce to be parsed + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_aggnonce_parse( + const secp256k1_context *ctx, + secp256k1_musig_aggnonce *nonce, + const unsigned char *in66 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Serialize an aggregate public nonce + * + * Returns: 1 always + * Args: ctx: pointer to a context object + * Out: out66: pointer to a 66-byte array to store the serialized nonce + * In: nonce: pointer to the nonce + */ +SECP256K1_API int secp256k1_musig_aggnonce_serialize( + const secp256k1_context *ctx, + unsigned char *out66, + const secp256k1_musig_aggnonce *nonce +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Parse a MuSig partial signature. + * + * Returns: 1 when the signature could be parsed, 0 otherwise. + * Args: ctx: pointer to a context object + * Out: sig: pointer to a signature object + * In: in32: pointer to the 32-byte signature to be parsed + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_partial_sig_parse( + const secp256k1_context *ctx, + secp256k1_musig_partial_sig *sig, + const unsigned char *in32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Serialize a MuSig partial signature + * + * Returns: 1 always + * Args: ctx: pointer to a context object + * Out: out32: pointer to a 32-byte array to store the serialized signature + * In: sig: pointer to the signature + */ +SECP256K1_API int secp256k1_musig_partial_sig_serialize( + const secp256k1_context *ctx, + unsigned char *out32, + const secp256k1_musig_partial_sig *sig +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Computes an aggregate public key and uses it to initialize a keyagg_cache + * + * Different orders of `pubkeys` result in different `agg_pk`s. + * + * Before aggregating, the pubkeys can be sorted with `secp256k1_ec_pubkey_sort` + * which ensures the same `agg_pk` result for the same multiset of pubkeys. + * This is useful to do before `pubkey_agg`, such that the order of pubkeys + * does not affect the aggregate public key. + * + * Returns: 0 if the arguments are invalid, 1 otherwise + * Args: ctx: pointer to a context object + * Out: agg_pk: the MuSig-aggregated x-only public key. If you do not need it, + * this arg can be NULL. + * keyagg_cache: if non-NULL, pointer to a musig_keyagg_cache struct that + * is required for signing (or observing the signing session + * and verifying partial signatures). + * In: pubkeys: input array of pointers to public keys to aggregate. The order + * is important; a different order will result in a different + * aggregate public key. + * n_pubkeys: length of pubkeys array. Must be greater than 0. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_pubkey_agg( + const secp256k1_context *ctx, + secp256k1_xonly_pubkey *agg_pk, + secp256k1_musig_keyagg_cache *keyagg_cache, + const secp256k1_pubkey * const *pubkeys, + size_t n_pubkeys +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(4); + +/** Obtain the aggregate public key from a keyagg_cache. + * + * This is only useful if you need the non-xonly public key, in particular for + * plain (non-xonly) tweaking or batch-verifying multiple key aggregations + * (not implemented). + * + * Returns: 0 if the arguments are invalid, 1 otherwise + * Args: ctx: pointer to a context object + * Out: agg_pk: the MuSig-aggregated public key. + * In: keyagg_cache: pointer to a `musig_keyagg_cache` struct initialized by + * `musig_pubkey_agg` + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_pubkey_get( + const secp256k1_context *ctx, + secp256k1_pubkey *agg_pk, + const secp256k1_musig_keyagg_cache *keyagg_cache +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Apply plain "EC" tweaking to a public key in a given keyagg_cache by adding + * the generator multiplied with `tweak32` to it. This is useful for deriving + * child keys from an aggregate public key via BIP 32 where `tweak32` is set to + * a hash as defined in BIP 32. + * + * Callers are responsible for deriving `tweak32` in a way that does not reduce + * the security of MuSig (for example, by following BIP 32). + * + * The tweaking method is the same as `secp256k1_ec_pubkey_tweak_add`. So after + * the following pseudocode buf and buf2 have identical contents (absent + * earlier failures). + * + * secp256k1_musig_pubkey_agg(..., keyagg_cache, pubkeys, ...) + * secp256k1_musig_pubkey_get(..., agg_pk, keyagg_cache) + * secp256k1_musig_pubkey_ec_tweak_add(..., output_pk, tweak32, keyagg_cache) + * secp256k1_ec_pubkey_serialize(..., buf, ..., output_pk, ...) + * secp256k1_ec_pubkey_tweak_add(..., agg_pk, tweak32) + * secp256k1_ec_pubkey_serialize(..., buf2, ..., agg_pk, ...) + * + * This function is required if you want to _sign_ for a tweaked aggregate key. + * If you are only computing a public key but not intending to create a + * signature for it, use `secp256k1_ec_pubkey_tweak_add` instead. + * + * Returns: 0 if the arguments are invalid, 1 otherwise + * Args: ctx: pointer to a context object + * Out: output_pubkey: pointer to a public key to store the result. Will be set + * to an invalid value if this function returns 0. If you + * do not need it, this arg can be NULL. + * In/Out: keyagg_cache: pointer to a `musig_keyagg_cache` struct initialized by + * `musig_pubkey_agg` + * In: tweak32: pointer to a 32-byte tweak. The tweak is valid if it passes + * `secp256k1_ec_seckey_verify` and is not equal to the + * secret key corresponding to the public key represented + * by keyagg_cache or its negation. For uniformly random + * 32-byte arrays the chance of being invalid is + * negligible (around 1 in 2^128). + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_pubkey_ec_tweak_add( + const secp256k1_context *ctx, + secp256k1_pubkey *output_pubkey, + secp256k1_musig_keyagg_cache *keyagg_cache, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +/** Apply x-only tweaking to a public key in a given keyagg_cache by adding the + * generator multiplied with `tweak32` to it. This is useful for creating + * Taproot outputs where `tweak32` is set to a TapTweak hash as defined in BIP + * 341. + * + * Callers are responsible for deriving `tweak32` in a way that does not reduce + * the security of MuSig (for example, by following Taproot BIP 341). + * + * The tweaking method is the same as `secp256k1_xonly_pubkey_tweak_add`. So in + * the following pseudocode xonly_pubkey_tweak_add_check (absent earlier + * failures) returns 1. + * + * secp256k1_musig_pubkey_agg(..., agg_pk, keyagg_cache, pubkeys, ...) + * secp256k1_musig_pubkey_xonly_tweak_add(..., output_pk, keyagg_cache, tweak32) + * secp256k1_xonly_pubkey_serialize(..., buf, output_pk) + * secp256k1_xonly_pubkey_tweak_add_check(..., buf, ..., agg_pk, tweak32) + * + * This function is required if you want to _sign_ for a tweaked aggregate key. + * If you are only computing a public key but not intending to create a + * signature for it, use `secp256k1_xonly_pubkey_tweak_add` instead. + * + * Returns: 0 if the arguments are invalid, 1 otherwise + * Args: ctx: pointer to a context object + * Out: output_pubkey: pointer to a public key to store the result. Will be set + * to an invalid value if this function returns 0. If you + * do not need it, this arg can be NULL. + * In/Out: keyagg_cache: pointer to a `musig_keyagg_cache` struct initialized by + * `musig_pubkey_agg` + * In: tweak32: pointer to a 32-byte tweak. The tweak is valid if it passes + * `secp256k1_ec_seckey_verify` and is not equal to the + * secret key corresponding to the public key represented + * by keyagg_cache or its negation. For uniformly random + * 32-byte arrays the chance of being invalid is + * negligible (around 1 in 2^128). + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_pubkey_xonly_tweak_add( + const secp256k1_context *ctx, + secp256k1_pubkey *output_pubkey, + secp256k1_musig_keyagg_cache *keyagg_cache, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +/** Starts a signing session by generating a nonce + * + * This function outputs a secret nonce that will be required for signing and a + * corresponding public nonce that is intended to be sent to other signers. + * + * MuSig differs from regular Schnorr signing in that implementers _must_ take + * special care to not reuse a nonce. This can be ensured by following these rules: + * + * 1. Each call to this function must have a UNIQUE session_secrand32 that must + * NOT BE REUSED in subsequent calls to this function and must be KEPT + * SECRET (even from other signers). + * 2. If you already know the seckey, message or aggregate public key + * cache, they can be optionally provided to derive the nonce and increase + * misuse-resistance. The extra_input32 argument can be used to provide + * additional data that does not repeat in normal scenarios, such as the + * current time. + * 3. Avoid copying (or serializing) the secnonce. This reduces the possibility + * that it is used more than once for signing. + * + * If you don't have access to good randomness for session_secrand32, but you + * have access to a non-repeating counter, then see + * secp256k1_musig_nonce_gen_counter. + * + * Remember that nonce reuse will leak the secret key! + * Note that using the same seckey for multiple MuSig sessions is fine. + * + * Returns: 0 if the arguments are invalid and 1 otherwise + * Args: ctx: pointer to a context object (not secp256k1_context_static) + * Out: secnonce: pointer to a structure to store the secret nonce + * pubnonce: pointer to a structure to store the public nonce + * In/Out: + * session_secrand32: a 32-byte session_secrand32 as explained above. Must be unique to this + * call to secp256k1_musig_nonce_gen and must be uniformly + * random. If the function call is successful, the + * session_secrand32 buffer is invalidated to prevent reuse. + * In: + * seckey: the 32-byte secret key that will later be used for signing, if + * already known (can be NULL) + * pubkey: public key of the signer creating the nonce. The secnonce + * output of this function cannot be used to sign for any + * other public key. While the public key should correspond + * to the provided seckey, a mismatch will not cause the + * function to return 0. + * msg32: the 32-byte message that will later be signed, if already known + * (can be NULL) + * keyagg_cache: pointer to the keyagg_cache that was used to create the aggregate + * (and potentially tweaked) public key if already known + * (can be NULL) + * extra_input32: an optional 32-byte array that is input to the nonce + * derivation function (can be NULL) + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_nonce_gen( + const secp256k1_context *ctx, + secp256k1_musig_secnonce *secnonce, + secp256k1_musig_pubnonce *pubnonce, + unsigned char *session_secrand32, + const unsigned char *seckey, + const secp256k1_pubkey *pubkey, + const unsigned char *msg32, + const secp256k1_musig_keyagg_cache *keyagg_cache, + const unsigned char *extra_input32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(6); + + +/** Alternative way to generate a nonce and start a signing session + * + * This function outputs a secret nonce that will be required for signing and a + * corresponding public nonce that is intended to be sent to other signers. + * + * This function differs from `secp256k1_musig_nonce_gen` by accepting a + * non-repeating counter value instead of a secret random value. This requires + * that a secret key is provided to `secp256k1_musig_nonce_gen_counter` + * (through the keypair argument), as opposed to `secp256k1_musig_nonce_gen` + * where the seckey argument is optional. + * + * MuSig differs from regular Schnorr signing in that implementers _must_ take + * special care to not reuse a nonce. This can be ensured by following these rules: + * + * 1. The nonrepeating_cnt argument must be a counter value that never repeats, + * i.e., you must never call `secp256k1_musig_nonce_gen_counter` twice with + * the same keypair and nonrepeating_cnt value. For example, this implies + * that if the same keypair is used with `secp256k1_musig_nonce_gen_counter` + * on multiple devices, none of the devices should have the same counter + * value as any other device. + * 2. If the seckey, message or aggregate public key cache is already available + * at this stage, any of these can be optionally provided, in which case + * they will be used in the derivation of the nonce and increase + * misuse-resistance. The extra_input32 argument can be used to provide + * additional data that does not repeat in normal scenarios, such as the + * current time. + * 3. Avoid copying (or serializing) the secnonce. This reduces the possibility + * that it is used more than once for signing. + * + * Remember that nonce reuse will leak the secret key! + * Note that using the same keypair for multiple MuSig sessions is fine. + * + * Returns: 0 if the arguments are invalid and 1 otherwise + * Args: ctx: pointer to a context object (not secp256k1_context_static) + * Out: secnonce: pointer to a structure to store the secret nonce + * pubnonce: pointer to a structure to store the public nonce + * In: + * nonrepeating_cnt: the value of a counter as explained above. Must be + * unique to this call to secp256k1_musig_nonce_gen. + * keypair: keypair of the signer creating the nonce. The secnonce + * output of this function cannot be used to sign for any + * other keypair. + * msg32: the 32-byte message that will later be signed, if already known + * (can be NULL) + * keyagg_cache: pointer to the keyagg_cache that was used to create the aggregate + * (and potentially tweaked) public key if already known + * (can be NULL) + * extra_input32: an optional 32-byte array that is input to the nonce + * derivation function (can be NULL) + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_nonce_gen_counter( + const secp256k1_context *ctx, + secp256k1_musig_secnonce *secnonce, + secp256k1_musig_pubnonce *pubnonce, + uint64_t nonrepeating_cnt, + const secp256k1_keypair *keypair, + const unsigned char *msg32, + const secp256k1_musig_keyagg_cache *keyagg_cache, + const unsigned char *extra_input32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(5); + +/** Aggregates the nonces of all signers into a single nonce + * + * This can be done by an untrusted party to reduce the communication + * between signers. Instead of everyone sending nonces to everyone else, there + * can be one party receiving all nonces, aggregating the nonces with this + * function and then sending only the aggregate nonce back to the signers. + * + * If the aggregator does not compute the aggregate nonce correctly, the final + * signature will be invalid. + * + * Returns: 0 if the arguments are invalid, 1 otherwise + * Args: ctx: pointer to a context object + * Out: aggnonce: pointer to an aggregate public nonce object for + * musig_nonce_process + * In: pubnonces: array of pointers to public nonces sent by the + * signers + * n_pubnonces: number of elements in the pubnonces array. Must be + * greater than 0. + */ +SECP256K1_API int secp256k1_musig_nonce_agg( + const secp256k1_context *ctx, + secp256k1_musig_aggnonce *aggnonce, + const secp256k1_musig_pubnonce * const *pubnonces, + size_t n_pubnonces +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Takes the aggregate nonce and creates a session that is required for signing + * and verification of partial signatures. + * + * Returns: 0 if the arguments are invalid, 1 otherwise + * Args: ctx: pointer to a context object + * Out: session: pointer to a struct to store the session + * In: aggnonce: pointer to an aggregate public nonce object that is the + * output of musig_nonce_agg + * msg32: the 32-byte message to sign + * keyagg_cache: pointer to the keyagg_cache that was used to create the + * aggregate (and potentially tweaked) pubkey + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_nonce_process( + const secp256k1_context *ctx, + secp256k1_musig_session *session, + const secp256k1_musig_aggnonce *aggnonce, + const unsigned char *msg32, + const secp256k1_musig_keyagg_cache *keyagg_cache +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(5); + +/** Produces a partial signature + * + * This function overwrites the given secnonce with zeros and will abort if given a + * secnonce that is all zeros. This is a best effort attempt to protect against nonce + * reuse. However, this is of course easily defeated if the secnonce has been + * copied (or serialized). Remember that nonce reuse will leak the secret key! + * + * For signing to succeed, the secnonce provided to this function must have + * been generated for the provided keypair. This means that when signing for a + * keypair consisting of a seckey and pubkey, the secnonce must have been + * created by calling musig_nonce_gen with that pubkey. Otherwise, the + * illegal_callback is called. + * + * This function does not verify the output partial signature, deviating from + * the BIP 327 specification. It is recommended to verify the output partial + * signature with `secp256k1_musig_partial_sig_verify` to prevent random or + * adversarially provoked computation errors. + * + * Returns: 0 if the arguments are invalid or the provided secnonce has already + * been used for signing, 1 otherwise + * Args: ctx: pointer to a context object + * Out: partial_sig: pointer to struct to store the partial signature + * In/Out: secnonce: pointer to the secnonce struct created in + * musig_nonce_gen that has been never used in a + * partial_sign call before and has been created for the + * keypair + * In: keypair: pointer to keypair to sign the message with + * keyagg_cache: pointer to the keyagg_cache that was output when the + * aggregate public key for this session + * session: pointer to the session that was created with + * musig_nonce_process + */ +SECP256K1_API int secp256k1_musig_partial_sign( + const secp256k1_context *ctx, + secp256k1_musig_partial_sig *partial_sig, + secp256k1_musig_secnonce *secnonce, + const secp256k1_keypair *keypair, + const secp256k1_musig_keyagg_cache *keyagg_cache, + const secp256k1_musig_session *session +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(5) SECP256K1_ARG_NONNULL(6); + +/** Verifies an individual signer's partial signature + * + * The signature is verified for a specific signing session. In order to avoid + * accidentally verifying a signature from a different or non-existing signing + * session, you must ensure the following: + * 1. The `keyagg_cache` argument is identical to the one used to create the + * `session` with `musig_nonce_process`. + * 2. The `pubkey` argument must be identical to the one sent by the signer + * before aggregating it with `musig_pubkey_agg` to create the + * `keyagg_cache`. + * 3. The `pubnonce` argument must be identical to the one sent by the signer + * before aggregating it with `musig_nonce_agg` and using the result to + * create the `session` with `musig_nonce_process`. + * + * It is not required to call this function in regular MuSig sessions, because + * if any partial signature does not verify, the final signature will not + * verify either, so the problem will be caught. However, this function + * provides the ability to identify which specific partial signature fails + * verification. + * + * Returns: 0 if the arguments are invalid or the partial signature does not + * verify, 1 otherwise + * Args ctx: pointer to a context object + * In: partial_sig: pointer to partial signature to verify, sent by + * the signer associated with `pubnonce` and `pubkey` + * pubnonce: public nonce of the signer in the signing session + * pubkey: public key of the signer in the signing session + * keyagg_cache: pointer to the keyagg_cache that was output when the + * aggregate public key for this signing session + * session: pointer to the session that was created with + * `musig_nonce_process` + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_partial_sig_verify( + const secp256k1_context *ctx, + const secp256k1_musig_partial_sig *partial_sig, + const secp256k1_musig_pubnonce *pubnonce, + const secp256k1_pubkey *pubkey, + const secp256k1_musig_keyagg_cache *keyagg_cache, + const secp256k1_musig_session *session +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(5) SECP256K1_ARG_NONNULL(6); + +/** Aggregates partial signatures + * + * Returns: 0 if the arguments are invalid, 1 otherwise (which does NOT mean + * the resulting signature verifies). + * Args: ctx: pointer to a context object + * Out: sig64: complete (but possibly invalid) Schnorr signature + * In: session: pointer to the session that was created with + * musig_nonce_process + * partial_sigs: array of pointers to partial signatures to aggregate + * n_sigs: number of elements in the partial_sigs array. Must be + * greater than 0. + */ +SECP256K1_API int secp256k1_musig_partial_sig_agg( + const secp256k1_context *ctx, + unsigned char *sig64, + const secp256k1_musig_session *session, + const secp256k1_musig_partial_sig * const *partial_sigs, + size_t n_sigs +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/secp256k1/include/secp256k1_preallocated.h b/example/android/third_party/secp256k1/include/secp256k1_preallocated.h new file mode 100644 index 00000000..f2d95c24 --- /dev/null +++ b/example/android/third_party/secp256k1/include/secp256k1_preallocated.h @@ -0,0 +1,134 @@ +#ifndef SECP256K1_PREALLOCATED_H +#define SECP256K1_PREALLOCATED_H + +#include "secp256k1.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* The module provided by this header file is intended for settings in which it + * is not possible or desirable to rely on dynamic memory allocation. It provides + * functions for creating, cloning, and destroying secp256k1 context objects in a + * contiguous fixed-size block of memory provided by the caller. + * + * Context objects created by functions in this module can be used like contexts + * objects created by functions in secp256k1.h, i.e., they can be passed to any + * API function that expects a context object (see secp256k1.h for details). The + * only exception is that context objects created by functions in this module + * must be destroyed using secp256k1_context_preallocated_destroy (in this + * module) instead of secp256k1_context_destroy (in secp256k1.h). + * + * It is guaranteed that functions in this module will not call malloc or its + * friends realloc, calloc, and free. + */ + +/** Determine the memory size of a secp256k1 context object to be created in + * caller-provided memory. + * + * The purpose of this function is to determine how much memory must be provided + * to secp256k1_context_preallocated_create. + * + * Returns: the required size of the caller-provided memory block + * In: flags: which parts of the context to initialize. + */ +SECP256K1_API size_t secp256k1_context_preallocated_size( + unsigned int flags +) SECP256K1_WARN_UNUSED_RESULT; + +/** Create a secp256k1 context object in caller-provided memory. + * + * The caller must provide a pointer to a rewritable contiguous block of memory + * of size at least secp256k1_context_preallocated_size(flags) bytes, suitably + * aligned to hold an object of any type. + * + * The block of memory is exclusively owned by the created context object during + * the lifetime of this context object, which begins with the call to this + * function and ends when a call to secp256k1_context_preallocated_destroy + * (which destroys the context object again) returns. During the lifetime of the + * context object, the caller is obligated not to access this block of memory, + * i.e., the caller may not read or write the memory, e.g., by copying the memory + * contents to a different location or trying to create a second context object + * in the memory. In simpler words, the prealloc pointer (or any pointer derived + * from it) should not be used during the lifetime of the context object. + * + * Returns: pointer to newly created context object. + * In: prealloc: pointer to a rewritable contiguous block of memory of + * size at least secp256k1_context_preallocated_size(flags) + * bytes, as detailed above. + * flags: which parts of the context to initialize. + * + * See secp256k1_context_create (in secp256k1.h) for further details. + * + * See also secp256k1_context_randomize (in secp256k1.h) + * and secp256k1_context_preallocated_destroy. + */ +SECP256K1_API secp256k1_context *secp256k1_context_preallocated_create( + void *prealloc, + unsigned int flags +) SECP256K1_ARG_NONNULL(1) SECP256K1_WARN_UNUSED_RESULT; + +/** Determine the memory size of a secp256k1 context object to be copied into + * caller-provided memory. + * + * Returns: the required size of the caller-provided memory block. + * In: ctx: pointer to a context to copy. + */ +SECP256K1_API size_t secp256k1_context_preallocated_clone_size( + const secp256k1_context *ctx +) SECP256K1_ARG_NONNULL(1) SECP256K1_WARN_UNUSED_RESULT; + +/** Copy a secp256k1 context object into caller-provided memory. + * + * The caller must provide a pointer to a rewritable contiguous block of memory + * of size at least secp256k1_context_preallocated_size(flags) bytes, suitably + * aligned to hold an object of any type. + * + * The block of memory is exclusively owned by the created context object during + * the lifetime of this context object, see the description of + * secp256k1_context_preallocated_create for details. + * + * Cloning secp256k1_context_static is not possible, and should not be emulated by + * the caller (e.g., using memcpy). Create a new context instead. + * + * Returns: pointer to a newly created context object. + * Args: ctx: pointer to a context to copy (not secp256k1_context_static). + * In: prealloc: pointer to a rewritable contiguous block of memory of + * size at least secp256k1_context_preallocated_size(flags) + * bytes, as detailed above. + */ +SECP256K1_API secp256k1_context *secp256k1_context_preallocated_clone( + const secp256k1_context *ctx, + void *prealloc +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_WARN_UNUSED_RESULT; + +/** Destroy a secp256k1 context object that has been created in + * caller-provided memory. + * + * The context pointer may not be used afterwards. + * + * The context to destroy must have been created using + * secp256k1_context_preallocated_create or secp256k1_context_preallocated_clone. + * If the context has instead been created using secp256k1_context_create or + * secp256k1_context_clone, the behaviour is undefined. In that case, + * secp256k1_context_destroy must be used instead. + * + * If required, it is the responsibility of the caller to deallocate the block + * of memory properly after this function returns, e.g., by calling free on the + * preallocated pointer given to secp256k1_context_preallocated_create or + * secp256k1_context_preallocated_clone. + * + * Args: ctx: pointer to a context to destroy, constructed using + * secp256k1_context_preallocated_create or + * secp256k1_context_preallocated_clone + * (i.e., not secp256k1_context_static). + */ +SECP256K1_API void secp256k1_context_preallocated_destroy( + secp256k1_context *ctx +) SECP256K1_ARG_NONNULL(1); + +#ifdef __cplusplus +} +#endif + +#endif /* SECP256K1_PREALLOCATED_H */ diff --git a/example/android/third_party/secp256k1/include/secp256k1_recovery.h b/example/android/third_party/secp256k1/include/secp256k1_recovery.h new file mode 100644 index 00000000..93a2e4cc --- /dev/null +++ b/example/android/third_party/secp256k1/include/secp256k1_recovery.h @@ -0,0 +1,113 @@ +#ifndef SECP256K1_RECOVERY_H +#define SECP256K1_RECOVERY_H + +#include "secp256k1.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** Opaque data structure that holds a parsed ECDSA signature, + * supporting pubkey recovery. + * + * The exact representation of data inside is implementation defined and not + * guaranteed to be portable between different platforms or versions. It is + * however guaranteed to be 65 bytes in size, and can be safely copied/moved. + * If you need to convert to a format suitable for storage or transmission, use + * the secp256k1_ecdsa_signature_serialize_* and + * secp256k1_ecdsa_signature_parse_* functions. + * + * Furthermore, it is guaranteed that identical signatures (including their + * recoverability) will have identical representation, so they can be + * memcmp'ed. + */ +typedef struct secp256k1_ecdsa_recoverable_signature { + unsigned char data[65]; +} secp256k1_ecdsa_recoverable_signature; + +/** Parse a compact ECDSA signature (64 bytes + recovery id). + * + * Returns: 1 when the signature could be parsed, 0 otherwise + * Args: ctx: pointer to a context object + * Out: sig: pointer to a signature object + * In: input64: pointer to a 64-byte compact signature + * recid: the recovery id (0, 1, 2 or 3) + */ +SECP256K1_API int secp256k1_ecdsa_recoverable_signature_parse_compact( + const secp256k1_context *ctx, + secp256k1_ecdsa_recoverable_signature *sig, + const unsigned char *input64, + int recid +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Convert a recoverable signature into a normal signature. + * + * Returns: 1 + * Args: ctx: pointer to a context object. + * Out: sig: pointer to a normal signature. + * In: sigin: pointer to a recoverable signature. + */ +SECP256K1_API int secp256k1_ecdsa_recoverable_signature_convert( + const secp256k1_context *ctx, + secp256k1_ecdsa_signature *sig, + const secp256k1_ecdsa_recoverable_signature *sigin +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Serialize an ECDSA signature in compact format (64 bytes + recovery id). + * + * Returns: 1 + * Args: ctx: pointer to a context object. + * Out: output64: pointer to a 64-byte array of the compact signature. + * recid: pointer to an integer to hold the recovery id. + * In: sig: pointer to an initialized signature object. + */ +SECP256K1_API int secp256k1_ecdsa_recoverable_signature_serialize_compact( + const secp256k1_context *ctx, + unsigned char *output64, + int *recid, + const secp256k1_ecdsa_recoverable_signature *sig +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +/** Create a recoverable ECDSA signature. + * + * Returns: 1: signature created + * 0: the nonce generation function failed, or the secret key was invalid. + * Args: ctx: pointer to a context object (not secp256k1_context_static). + * Out: sig: pointer to an array where the signature will be placed. + * In: msghash32: the 32-byte message hash being signed. + * seckey: pointer to a 32-byte secret key. + * noncefp: pointer to a nonce generation function. If NULL, + * secp256k1_nonce_function_default is used. + * ndata: pointer to arbitrary data used by the nonce generation function + * (can be NULL for secp256k1_nonce_function_default). + */ +SECP256K1_API int secp256k1_ecdsa_sign_recoverable( + const secp256k1_context *ctx, + secp256k1_ecdsa_recoverable_signature *sig, + const unsigned char *msghash32, + const unsigned char *seckey, + secp256k1_nonce_function noncefp, + const void *ndata +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +/** Recover an ECDSA public key from a signature. + * + * Returns: 1: public key successfully recovered (which guarantees a correct signature). + * 0: otherwise. + * Args: ctx: pointer to a context object. + * Out: pubkey: pointer to the recovered public key. + * In: sig: pointer to initialized signature that supports pubkey recovery. + * msghash32: the 32-byte message hash assumed to be signed. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ecdsa_recover( + const secp256k1_context *ctx, + secp256k1_pubkey *pubkey, + const secp256k1_ecdsa_recoverable_signature *sig, + const unsigned char *msghash32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +#ifdef __cplusplus +} +#endif + +#endif /* SECP256K1_RECOVERY_H */ diff --git a/example/android/third_party/secp256k1/include/secp256k1_schnorrsig.h b/example/android/third_party/secp256k1/include/secp256k1_schnorrsig.h new file mode 100644 index 00000000..013d4ee7 --- /dev/null +++ b/example/android/third_party/secp256k1/include/secp256k1_schnorrsig.h @@ -0,0 +1,190 @@ +#ifndef SECP256K1_SCHNORRSIG_H +#define SECP256K1_SCHNORRSIG_H + +#include "secp256k1.h" +#include "secp256k1_extrakeys.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** This module implements a variant of Schnorr signatures compliant with + * Bitcoin Improvement Proposal 340 "Schnorr Signatures for secp256k1" + * (https://github.com/bitcoin/bips/blob/master/bip-0340.mediawiki). + */ + +/** A pointer to a function to deterministically generate a nonce. + * + * Same as secp256k1_nonce function with the exception of accepting an + * additional pubkey argument and not requiring an attempt argument. The pubkey + * argument can protect signature schemes with key-prefixed challenge hash + * inputs against reusing the nonce when signing with the wrong precomputed + * pubkey. + * + * Returns: 1 if a nonce was successfully generated. 0 will cause signing to + * return an error. + * Out: nonce32: pointer to a 32-byte array to be filled by the function + * In: msg: the message being verified. Is NULL if and only if msglen + * is 0. + * msglen: the length of the message + * key32: pointer to a 32-byte secret key (will not be NULL) + * xonly_pk32: the 32-byte serialized xonly pubkey corresponding to key32 + * (will not be NULL) + * algo: pointer to an array describing the signature + * algorithm (will not be NULL) + * algolen: the length of the algo array + * data: arbitrary data pointer that is passed through + * + * Except for test cases, this function should compute some cryptographic hash of + * the message, the key, the pubkey, the algorithm description, and data. + */ +typedef int (*secp256k1_nonce_function_hardened)( + unsigned char *nonce32, + const unsigned char *msg, + size_t msglen, + const unsigned char *key32, + const unsigned char *xonly_pk32, + const unsigned char *algo, + size_t algolen, + void *data +); + +/** An implementation of the nonce generation function as defined in Bitcoin + * Improvement Proposal 340 "Schnorr Signatures for secp256k1" + * (https://github.com/bitcoin/bips/blob/master/bip-0340.mediawiki). + * + * If a data pointer is passed, it is assumed to be a pointer to 32 bytes of + * auxiliary random data as defined in BIP-340. If the data pointer is NULL, + * the nonce derivation procedure follows BIP-340 by setting the auxiliary + * random data to zero. The algo argument must be non-NULL, otherwise the + * function will fail and return 0. The hash will be tagged with algo. + * Therefore, to create BIP-340 compliant signatures, algo must be set to + * "BIP0340/nonce" and algolen to 13. + */ +SECP256K1_API const secp256k1_nonce_function_hardened secp256k1_nonce_function_bip340; + +/** Data structure that contains additional arguments for schnorrsig_sign_custom. + * + * A schnorrsig_extraparams structure object can be initialized correctly by + * setting it to SECP256K1_SCHNORRSIG_EXTRAPARAMS_INIT. + * + * Members: + * magic: set to SECP256K1_SCHNORRSIG_EXTRAPARAMS_MAGIC at initialization + * and has no other function than making sure the object is + * initialized. + * noncefp: pointer to a nonce generation function. If NULL, + * secp256k1_nonce_function_bip340 is used + * ndata: pointer to arbitrary data used by the nonce generation function + * (can be NULL). If it is non-NULL and + * secp256k1_nonce_function_bip340 is used, then ndata must be a + * pointer to 32-byte auxiliary randomness as per BIP-340. + */ +typedef struct secp256k1_schnorrsig_extraparams { + unsigned char magic[4]; + secp256k1_nonce_function_hardened noncefp; + void *ndata; +} secp256k1_schnorrsig_extraparams; + +#define SECP256K1_SCHNORRSIG_EXTRAPARAMS_MAGIC { 0xda, 0x6f, 0xb3, 0x8c } +#define SECP256K1_SCHNORRSIG_EXTRAPARAMS_INIT {\ + SECP256K1_SCHNORRSIG_EXTRAPARAMS_MAGIC,\ + NULL,\ + NULL\ +} + +/** Create a Schnorr signature. + * + * Does _not_ strictly follow BIP-340 because it does not verify the resulting + * signature. Instead, you can manually use secp256k1_schnorrsig_verify and + * abort if it fails. + * + * This function only signs 32-byte messages. If you have messages of a + * different size (or the same size but without a context-specific tag + * prefix), it is recommended to create a 32-byte message hash with + * secp256k1_tagged_sha256 and then sign the hash. Tagged hashing allows + * providing an context-specific tag for domain separation. This prevents + * signatures from being valid in multiple contexts by accident. + * + * Returns 1 on success, 0 on failure. + * Args: ctx: pointer to a context object (not secp256k1_context_static). + * Out: sig64: pointer to a 64-byte array to store the serialized signature. + * In: msg32: the 32-byte message being signed. + * keypair: pointer to an initialized keypair. + * aux_rand32: 32 bytes of fresh randomness. While recommended to provide + * this, it is only supplemental to security and can be NULL. A + * NULL argument is treated the same as an all-zero one. See + * BIP-340 "Default Signing" for a full explanation of this + * argument and for guidance if randomness is expensive. + */ +SECP256K1_API int secp256k1_schnorrsig_sign32( + const secp256k1_context *ctx, + unsigned char *sig64, + const unsigned char *msg32, + const secp256k1_keypair *keypair, + const unsigned char *aux_rand32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +/** Same as secp256k1_schnorrsig_sign32, but DEPRECATED. Will be removed in + * future versions. */ +SECP256K1_API int secp256k1_schnorrsig_sign( + const secp256k1_context *ctx, + unsigned char *sig64, + const unsigned char *msg32, + const secp256k1_keypair *keypair, + const unsigned char *aux_rand32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4) + SECP256K1_DEPRECATED("Use secp256k1_schnorrsig_sign32 instead"); + +/** Create a Schnorr signature with a more flexible API. + * + * Same arguments as secp256k1_schnorrsig_sign except that it allows signing + * variable length messages and accepts a pointer to an extraparams object that + * allows customizing signing by passing additional arguments. + * + * Equivalent to secp256k1_schnorrsig_sign32(..., aux_rand32) if msglen is 32 + * and extraparams is initialized as follows: + * ``` + * secp256k1_schnorrsig_extraparams extraparams = SECP256K1_SCHNORRSIG_EXTRAPARAMS_INIT; + * extraparams.ndata = (unsigned char*)aux_rand32; + * ``` + * + * Returns 1 on success, 0 on failure. + * Args: ctx: pointer to a context object (not secp256k1_context_static). + * Out: sig64: pointer to a 64-byte array to store the serialized signature. + * In: msg: the message being signed. Can only be NULL if msglen is 0. + * msglen: length of the message. + * keypair: pointer to an initialized keypair. + * extraparams: pointer to an extraparams object (can be NULL). + */ +SECP256K1_API int secp256k1_schnorrsig_sign_custom( + const secp256k1_context *ctx, + unsigned char *sig64, + const unsigned char *msg, + size_t msglen, + const secp256k1_keypair *keypair, + secp256k1_schnorrsig_extraparams *extraparams +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(5); + +/** Verify a Schnorr signature. + * + * Returns: 1: correct signature + * 0: incorrect signature + * Args: ctx: pointer to a context object. + * In: sig64: pointer to the 64-byte signature to verify. + * msg: the message being verified. Can only be NULL if msglen is 0. + * msglen: length of the message + * pubkey: pointer to an x-only public key to verify with + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_schnorrsig_verify( + const secp256k1_context *ctx, + const unsigned char *sig64, + const unsigned char *msg, + size_t msglen, + const secp256k1_xonly_pubkey *pubkey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(5); + +#ifdef __cplusplus +} +#endif + +#endif /* SECP256K1_SCHNORRSIG_H */ diff --git a/example/android/third_party/secp256k1/x86-64/libsecp256k1.a b/example/android/third_party/secp256k1/x86-64/libsecp256k1.a new file mode 100644 index 00000000..869dc5e6 Binary files /dev/null and b/example/android/third_party/secp256k1/x86-64/libsecp256k1.a differ diff --git a/fec/CMakeLists.txt b/fec/CMakeLists.txt index b1ac37b1..2a305607 100644 --- a/fec/CMakeLists.txt +++ b/fec/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) diff --git a/fec/fec.cpp b/fec/fec.cpp index 102df038..f6379fd7 100644 --- a/fec/fec.cpp +++ b/fec/fec.cpp @@ -99,7 +99,7 @@ td::uint32 FecType::symbol_size() const { } td::Result FecType::create(tl_object_ptr obj) { - td::int32 data_size_int, symbol_size_int, symbols_count_int; + td::int32 data_size_int = 0, symbol_size_int = 0, symbols_count_int = 0; ton_api::downcast_call(*obj, td::overloaded([&](const auto &obj) { data_size_int = obj.data_size_; symbol_size_int = obj.symbol_size_; diff --git a/git_watcher.cmake b/git_watcher.cmake index ab135e80..78e57ba1 100644 --- a/git_watcher.cmake +++ b/git_watcher.cmake @@ -139,6 +139,8 @@ function(GetGitState _working_dir) RunGitCommand(show -s "--format=%H" ${object}) if(exit_code EQUAL 0) set(ENV{GIT_HEAD_SHA1} ${output}) + else() + set(ENV{GIT_HEAD_SHA1} "$ENV{GIT_REVISION}") endif() RunGitCommand(show -s "--format=%an" ${object}) @@ -154,6 +156,8 @@ function(GetGitState _working_dir) RunGitCommand(show -s "--format=%ci" ${object}) if(exit_code EQUAL 0) set(ENV{GIT_COMMIT_DATE_ISO8601} "${output}") + else() + set(ENV{GIT_COMMIT_DATE_ISO8601} "$ENV{GIT_REVISION_DATE}") endif() RunGitCommand(show -s "--format=%s" ${object}) diff --git a/http/CMakeLists.txt b/http/CMakeLists.txt index 9437f244..4a3fccf8 100644 --- a/http/CMakeLists.txt +++ b/http/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) set(HTTP_SOURCE http.h @@ -23,3 +23,5 @@ target_link_libraries(tonhttp PUBLIC tdactor ton_crypto tl_api tdnet ) add_executable(http-proxy http-proxy.cpp) target_include_directories(http-proxy PUBLIC $) target_link_libraries(http-proxy PRIVATE tonhttp git) + +install(TARGETS http-proxy RUNTIME DESTINATION bin) diff --git a/keyring/CMakeLists.txt b/keyring/CMakeLists.txt index 29e48ee9..f8f610f2 100644 --- a/keyring/CMakeLists.txt +++ b/keyring/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) set(KEYRING_SOURCE keyring.h diff --git a/keyring/keyring.cpp b/keyring/keyring.cpp index 529a6b8b..a31d173e 100644 --- a/keyring/keyring.cpp +++ b/keyring/keyring.cpp @@ -27,6 +27,16 @@ namespace ton { namespace keyring { +KeyringImpl::PrivateKeyDescr::PrivateKeyDescr(PrivateKey private_key, bool is_temp) + : public_key(private_key.compute_public_key()), private_key(private_key), is_temp(is_temp) { + auto D = private_key.create_decryptor_async(); + D.ensure(); + decryptor_sign = D.move_as_ok(); + D = private_key.create_decryptor_async(); + D.ensure(); + decryptor_decrypt = D.move_as_ok(); +} + void KeyringImpl::start_up() { if (db_root_.size() > 0) { td::mkdir(db_root_).ensure(); @@ -45,23 +55,19 @@ td::Result KeyringImpl::load_key(PublicKeyHash k auto name = db_root_ + "/" + key_hash.bits256_value().to_hex(); - auto R = td::read_file(td::CSlice{name}); + auto R = td::read_file_secure(td::CSlice{name}); if (R.is_error()) { return R.move_as_error_prefix("key not in db: "); } auto data = R.move_as_ok(); - auto R2 = PrivateKey::import(td::SecureString(data)); + auto R2 = PrivateKey::import(data); R2.ensure(); auto key = R2.move_as_ok(); - auto pub = key.compute_public_key(); - auto short_id = pub.compute_short_id(); + auto desc = std::make_unique(key, false); + auto short_id = desc->public_key.compute_short_id(); CHECK(short_id == key_hash); - - auto D = key.create_decryptor_async(); - D.ensure(); - - return map_.emplace(short_id, std::make_unique(D.move_as_ok(), pub, false)).first->second.get(); + return map_.emplace(short_id, std::move(desc)).first->second.get(); } void KeyringImpl::add_key(PrivateKey key, bool is_temp, td::Promise promise) { @@ -76,10 +82,7 @@ void KeyringImpl::add_key(PrivateKey key, bool is_temp, td::Promise pr if (db_root_.size() == 0) { CHECK(is_temp); } - auto D = key.create_decryptor_async(); - D.ensure(); - - map_.emplace(short_id, std::make_unique(D.move_as_ok(), pub, is_temp)); + map_.emplace(short_id, std::make_unique(key, is_temp)); if (!is_temp && key.exportable()) { auto S = key.export_as_slice(); @@ -139,7 +142,7 @@ void KeyringImpl::sign_message(PublicKeyHash key_hash, td::BufferSlice data, td: if (S.is_error()) { promise.set_error(S.move_as_error()); } else { - td::actor::send_closure(S.move_as_ok()->decryptor, &DecryptorAsync::sign, std::move(data), std::move(promise)); + td::actor::send_closure(S.move_as_ok()->decryptor_sign, &DecryptorAsync::sign, std::move(data), std::move(promise)); } } @@ -161,7 +164,7 @@ void KeyringImpl::sign_add_get_public_key(PublicKeyHash key_hash, td::BufferSlic } promise.set_value(std::pair{R.move_as_ok(), id}); }); - td::actor::send_closure(D->decryptor, &DecryptorAsync::sign, std::move(data), std::move(P)); + td::actor::send_closure(D->decryptor_sign, &DecryptorAsync::sign, std::move(data), std::move(P)); } void KeyringImpl::sign_messages(PublicKeyHash key_hash, std::vector data, @@ -171,7 +174,7 @@ void KeyringImpl::sign_messages(PublicKeyHash key_hash, std::vectordecryptor, &DecryptorAsync::sign_batch, std::move(data), + td::actor::send_closure(S.move_as_ok()->decryptor_sign, &DecryptorAsync::sign_batch, std::move(data), std::move(promise)); } } @@ -182,10 +185,21 @@ void KeyringImpl::decrypt_message(PublicKeyHash key_hash, td::BufferSlice data, if (S.is_error()) { promise.set_error(S.move_as_error()); } else { - td::actor::send_closure(S.move_as_ok()->decryptor, &DecryptorAsync::decrypt, std::move(data), std::move(promise)); + td::actor::send_closure(S.move_as_ok()->decryptor_decrypt, &DecryptorAsync::decrypt, std::move(data), + std::move(promise)); } } +void KeyringImpl::export_all_private_keys(td::Promise> promise) { + std::vector keys; + for (auto& [_, descr] : map_) { + if (!descr->is_temp && descr->private_key.exportable()) { + keys.push_back(descr->private_key); + } + } + promise.set_value(std::move(keys)); +} + td::actor::ActorOwn Keyring::create(std::string db_root) { return td::actor::create_actor("keyring", db_root); } diff --git a/keyring/keyring.h b/keyring/keyring.h index 044d8d29..3b9064a7 100644 --- a/keyring/keyring.h +++ b/keyring/keyring.h @@ -44,6 +44,8 @@ class Keyring : public td::actor::Actor { virtual void decrypt_message(PublicKeyHash key_hash, td::BufferSlice data, td::Promise promise) = 0; + virtual void export_all_private_keys(td::Promise> promise) = 0; + static td::actor::ActorOwn create(std::string db_root); }; diff --git a/keyring/keyring.hpp b/keyring/keyring.hpp index fc67bd0f..eca9073a 100644 --- a/keyring/keyring.hpp +++ b/keyring/keyring.hpp @@ -30,12 +30,12 @@ namespace keyring { class KeyringImpl : public Keyring { private: struct PrivateKeyDescr { - td::actor::ActorOwn decryptor; + td::actor::ActorOwn decryptor_sign; + td::actor::ActorOwn decryptor_decrypt; PublicKey public_key; + PrivateKey private_key; bool is_temp; - PrivateKeyDescr(td::actor::ActorOwn decryptor, PublicKey public_key, bool is_temp) - : decryptor(std::move(decryptor)), public_key(public_key), is_temp(is_temp) { - } + PrivateKeyDescr(PrivateKey private_key, bool is_temp); }; public: @@ -57,6 +57,8 @@ class KeyringImpl : public Keyring { void decrypt_message(PublicKeyHash key_hash, td::BufferSlice data, td::Promise promise) override; + void export_all_private_keys(td::Promise> promise) override; + KeyringImpl(std::string db_root) : db_root_(db_root) { } diff --git a/keys/CMakeLists.txt b/keys/CMakeLists.txt index 486119de..e80436b7 100644 --- a/keys/CMakeLists.txt +++ b/keys/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) set(KEYS_SOURCE keys.cpp diff --git a/keys/encryptor.cpp b/keys/encryptor.cpp index 0b93d36f..8fef9a09 100644 --- a/keys/encryptor.cpp +++ b/keys/encryptor.cpp @@ -29,28 +29,6 @@ namespace ton { -td::Result> Encryptor::create(const ton_api::PublicKey *id) { - td::Result> res; - ton_api::downcast_call( - *const_cast(id), - td::overloaded([&](const ton_api::pub_unenc &obj) { res = std::make_unique(); }, - [&](const ton_api::pub_ed25519 &obj) { res = std::make_unique(obj.key_); }, - [&](const ton_api::pub_overlay &obj) { res = std::make_unique(); }, - [&](const ton_api::pub_aes &obj) { res = std::make_unique(obj.key_); })); - return res; -} - -td::Result> Decryptor::create(const ton_api::PrivateKey *id) { - td::Result> res; - ton_api::downcast_call( - *const_cast(id), - td::overloaded([&](const ton_api::pk_unenc &obj) { res = std::make_unique(); }, - [&](const ton_api::pk_ed25519 &obj) { res = std::make_unique(obj.key_); }, - [&](const ton_api::pk_overlay &obj) { res = std::make_unique(); }, - [&](const ton_api::pk_aes &obj) { res = std::make_unique(obj.key_); })); - return res; -} - td::Result EncryptorEd25519::encrypt(td::Slice data) { TRY_RESULT_PREFIX(pk, td::Ed25519::generate_private_key(), "failed to generate private key: "); TRY_RESULT_PREFIX(pubkey, pk.get_public_key(), "failed to get public key from private: "); diff --git a/keys/encryptor.h b/keys/encryptor.h index 3035a0ce..818c97d6 100644 --- a/keys/encryptor.h +++ b/keys/encryptor.h @@ -31,7 +31,6 @@ class Encryptor { virtual td::Result encrypt(td::Slice data) = 0; virtual td::Status check_signature(td::Slice message, td::Slice signature) = 0; virtual ~Encryptor() = default; - static td::Result> create(const ton_api::PublicKey *id); }; class Decryptor { @@ -40,7 +39,6 @@ class Decryptor { virtual td::Result sign(td::Slice data) = 0; virtual std::vector> sign_batch(std::vector data); virtual ~Decryptor() = default; - static td::Result> create(const ton_api::PrivateKey *id); }; class EncryptorAsync : public td::actor::Actor { @@ -61,16 +59,6 @@ class EncryptorAsync : public td::actor::Actor { void encrypt(td::BufferSlice data, td::Promise promise) { promise.set_result(encryptor_->encrypt(data.as_slice())); } - template - static td::Result> create(T &id) { - TRY_RESULT(d, Encryptor::create(id)); - return td::actor::create_actor("encryptor", std::move(d)); - } - template - static td::Result> create(T *id) { - TRY_RESULT(d, Encryptor::create(id)); - return td::actor::create_actor("encryptor", std::move(d)); - } }; class DecryptorAsync : public td::actor::Actor { @@ -94,16 +82,6 @@ class DecryptorAsync : public td::actor::Actor { } return decryptor_->sign_batch(v); } - template - static td::Result> create(T &id) { - TRY_RESULT(d, Decryptor::create(id)); - return td::actor::create_actor("decryptor", std::move(d)); - } - template - static td::Result> create(T *id) { - TRY_RESULT(d, Decryptor::create(id)); - return td::actor::create_actor("decryptor", std::move(d)); - } }; } // namespace ton diff --git a/keys/encryptor.hpp b/keys/encryptor.hpp index dbe88239..bcc841dc 100644 --- a/keys/encryptor.hpp +++ b/keys/encryptor.hpp @@ -83,7 +83,7 @@ class EncryptorEd25519 : public Encryptor { td::Result encrypt(td::Slice data) override; td::Status check_signature(td::Slice message, td::Slice signature) override; - EncryptorEd25519(td::Bits256 key) : pub_(td::SecureString(as_slice(key))) { + EncryptorEd25519(const td::Bits256& key) : pub_(td::SecureString(as_slice(key))) { } }; @@ -94,7 +94,7 @@ class DecryptorEd25519 : public Decryptor { public: td::Result decrypt(td::Slice data) override; td::Result sign(td::Slice data) override; - DecryptorEd25519(td::Bits256 key) : pk_(td::SecureString(as_slice(key))) { + DecryptorEd25519(const td::Bits256& key) : pk_(td::SecureString(as_slice(key))) { } }; @@ -129,12 +129,15 @@ class EncryptorAES : public Encryptor { td::Bits256 shared_secret_; public: + ~EncryptorAES() override { + shared_secret_.set_zero_s(); + } td::Result encrypt(td::Slice data) override; td::Status check_signature(td::Slice message, td::Slice signature) override { return td::Status::Error("can no sign channel messages"); } - EncryptorAES(td::Bits256 shared_secret) : shared_secret_(shared_secret) { + EncryptorAES(const td::Bits256& shared_secret) : shared_secret_(shared_secret) { } }; @@ -143,11 +146,14 @@ class DecryptorAES : public Decryptor { td::Bits256 shared_secret_; public: + ~DecryptorAES() override { + shared_secret_.set_zero_s(); + } td::Result decrypt(td::Slice data) override; td::Result sign(td::Slice data) override { return td::Status::Error("can no sign channel messages"); } - DecryptorAES(td::Bits256 shared_secret) : shared_secret_(shared_secret) { + DecryptorAES(const td::Bits256& shared_secret) : shared_secret_(shared_secret) { } }; diff --git a/keys/keys.cpp b/keys/keys.cpp index 7d6c0c2c..01afb26d 100644 --- a/keys/keys.cpp +++ b/keys/keys.cpp @@ -21,6 +21,7 @@ #include "td/utils/overloaded.h" #include "tl-utils/tl-utils.hpp" #include "encryptor.h" +#include "encryptor.hpp" #include "crypto/Ed25519.h" namespace ton { @@ -63,12 +64,31 @@ td::Result PublicKey::import(td::Slice s) { return PublicKey{x}; } +td::Result> pubkeys::Ed25519::create_encryptor() const { + return std::make_unique(data_); +} + +td::Result> pubkeys::AES::create_encryptor() const { + return std::make_unique(data_); +} + +td::Result> pubkeys::Unenc::create_encryptor() const { + return std::make_unique(); +} + +td::Result> pubkeys::Overlay::create_encryptor() const { + return std::make_unique(); +} + td::Result> PublicKey::create_encryptor() const { - return Encryptor::create(tl().get()); + td::Result> res; + pub_key_.visit([&](auto &obj) { res = obj.create_encryptor(); }); + return res; } td::Result> PublicKey::create_encryptor_async() const { - return EncryptorAsync::create(tl().get()); + TRY_RESULT(encryptor, create_encryptor()); + return td::actor::create_actor("encryptor", std::move(encryptor)); } bool PublicKey::empty() const { @@ -109,6 +129,22 @@ privkeys::Ed25519::Ed25519(td::Ed25519::PrivateKey key) { data_.as_slice().copy_from(td::Slice(s)); } +td::Result> privkeys::Ed25519::create_decryptor() const { + return std::make_unique(data_); +} + +td::Result> privkeys::AES::create_decryptor() const { + return std::make_unique(data_); +} + +td::Result> privkeys::Unenc::create_decryptor() const { + return std::make_unique(); +} + +td::Result> privkeys::Overlay::create_decryptor() const { + return std::make_unique(); +} + pubkeys::Ed25519::Ed25519(td::Ed25519::PublicKey key) { auto s = key.as_octet_string(); CHECK(s.length() == 32); @@ -188,11 +224,14 @@ tl_object_ptr PrivateKey::tl() const { } td::Result> PrivateKey::create_decryptor() const { - return Decryptor::create(tl().get()); + td::Result> res; + priv_key_.visit([&](auto &obj) { res = obj.create_decryptor(); }); + return res; } td::Result> PrivateKey::create_decryptor_async() const { - return DecryptorAsync::create(tl().get()); + TRY_RESULT(decryptor, create_decryptor()); + return td::actor::create_actor("decryptor", std::move(decryptor)); } } // namespace ton diff --git a/keys/keys.hpp b/keys/keys.hpp index cf883bbe..2e517b2e 100644 --- a/keys/keys.hpp +++ b/keys/keys.hpp @@ -110,6 +110,7 @@ class Ed25519 { tl_object_ptr tl() const { return create_tl_object(data_); } + td::Result> create_encryptor() const; bool operator==(const Ed25519 &with) const { return data_ == with.data_; } @@ -141,6 +142,7 @@ class AES { tl_object_ptr tl() const { return create_tl_object(data_); } + td::Result> create_encryptor() const; bool operator==(const AES &with) const { return data_ == with.data_; } @@ -172,6 +174,7 @@ class Unenc { tl_object_ptr tl() const { return create_tl_object(data_.clone_as_buffer_slice()); } + td::Result> create_encryptor() const; bool operator==(const Unenc &with) const { return data_.as_slice() == with.data_.as_slice(); } @@ -203,6 +206,7 @@ class Overlay { tl_object_ptr tl() const { return create_tl_object(data_.clone_as_buffer_slice()); } + td::Result> create_encryptor() const; bool operator==(const Overlay &with) const { return data_.as_slice() == with.data_.as_slice(); } @@ -223,6 +227,9 @@ class PublicKey { td::uint32 serialized_size() const { UNREACHABLE(); } + td::Result> create_encryptor() const { + UNREACHABLE(); + } bool operator==(const Empty &with) const { return false; } @@ -253,6 +260,10 @@ class PublicKey { td::BufferSlice export_as_slice() const; static td::Result import(td::Slice s); + bool is_ed25519() const { + return pub_key_.get_offset() == pub_key_.offset(); + } + pubkeys::Ed25519 ed25519_value() const { CHECK(pub_key_.get_offset() == pub_key_.offset()); return pub_key_.get(); @@ -316,6 +327,7 @@ class Ed25519 { } tl_object_ptr pub_tl() const; pubkeys::Ed25519 pub() const; + td::Result> create_decryptor() const; static Ed25519 random(); }; @@ -359,6 +371,7 @@ class AES { pubkeys::AES pub() const { return pubkeys::AES{data_}; } + td::Result> create_decryptor() const; }; class Unenc { @@ -393,6 +406,7 @@ class Unenc { pubkeys::Unenc pub() const { return pubkeys::Unenc{data_.clone()}; } + td::Result> create_decryptor() const; }; class Overlay { @@ -427,6 +441,7 @@ class Overlay { pubkeys::Overlay pub() const { return pubkeys::Overlay{data_.clone()}; } + td::Result> create_decryptor() const; }; } // namespace privkeys @@ -450,6 +465,9 @@ class PrivateKey { PublicKey pub() const { UNREACHABLE(); } + td::Result> create_decryptor() const { + UNREACHABLE(); + } }; td::Variant priv_key_{Empty{}}; diff --git a/lite-client/CMakeLists.txt b/lite-client/CMakeLists.txt index b8449516..b28a14e9 100644 --- a/lite-client/CMakeLists.txt +++ b/lite-client/CMakeLists.txt @@ -1,10 +1,10 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) -add_library(lite-client-common lite-client-common.cpp lite-client-common.h) -target_link_libraries(lite-client-common PUBLIC tdutils tdactor adnllite tl_api tl_lite_api tl-lite-utils ton_crypto ton_block) +add_library(lite-client-common STATIC lite-client-common.cpp lite-client-common.h ext-client.cpp ext-client.h + query-utils.hpp query-utils.cpp) +target_link_libraries(lite-client-common PUBLIC tdactor adnllite tl_api tl_lite_api tl-lite-utils ton_crypto) -add_executable(lite-client lite-client.cpp lite-client.h) -target_link_libraries(lite-client tdutils tdactor adnllite tl_api tl_lite_api tl-lite-utils ton_crypto ton_block - terminal lite-client-common git) +add_executable(lite-client lite-client.cpp lite-client.h ext-client.h ext-client.cpp) +target_link_libraries(lite-client tdutils tdactor adnllite tl_api tl_lite_api tl-lite-utils terminal lite-client-common git) install(TARGETS lite-client RUNTIME DESTINATION bin) diff --git a/lite-client/ext-client.cpp b/lite-client/ext-client.cpp new file mode 100644 index 00000000..a0e48e64 --- /dev/null +++ b/lite-client/ext-client.cpp @@ -0,0 +1,228 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "ext-client.h" +#include "td/utils/Random.h" +#include "ton/ton-shard.h" + +namespace liteclient { + +class ExtClientImpl : public ExtClient { + public: + ExtClientImpl(std::vector liteservers, td::unique_ptr callback, bool connect_to_all) + : callback_(std::move(callback)), connect_to_all_(connect_to_all) { + CHECK(!liteservers.empty()); + servers_.resize(liteservers.size()); + for (size_t i = 0; i < servers_.size(); ++i) { + servers_[i].config = std::move(liteservers[i]); + servers_[i].idx = i; + } + } + + void start_up() override { + LOG(INFO) << "Started ext client, " << servers_.size() << " liteservers"; + td::Random::Fast rnd; + td::random_shuffle(td::as_mutable_span(servers_), rnd); + server_indices_.resize(servers_.size()); + for (size_t i = 0; i < servers_.size(); ++i) { + server_indices_[servers_[i].idx] = i; + } + + if (connect_to_all_) { + for (size_t i = 0; i < servers_.size(); ++i) { + prepare_server(i, nullptr); + } + } + } + + void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, + td::Promise promise) override { + QueryInfo query_info = get_query_info(data); + TRY_RESULT_PROMISE(promise, server_idx, select_server(query_info)); + send_query_internal(std::move(name), std::move(data), std::move(query_info), server_idx, timeout, + std::move(promise)); + } + + void send_query_to_server(std::string name, td::BufferSlice data, size_t server_idx, td::Timestamp timeout, + td::Promise promise) override { + if (server_idx >= servers_.size()) { + promise.set_error(td::Status::Error(PSTRING() << "server idx " << server_idx << " is too big")); + return; + } + server_idx = server_indices_[server_idx]; + QueryInfo query_info = get_query_info(data); + prepare_server(server_idx, &query_info); + send_query_internal(std::move(name), std::move(data), std::move(query_info), server_idx, timeout, + std::move(promise)); + } + + void get_servers_status(td::Promise> promise) override { + std::vector status(servers_.size()); + for (const Server& s : servers_) { + status[s.idx] = s.alive; + } + promise.set_result(std::move(status)); + } + + void reset_servers() override { + LOG(INFO) << "Force resetting all liteservers"; + for (Server& server : servers_) { + server.alive = false; + server.timeout = {}; + server.ignore_until = {}; + server.client.reset(); + } + } + + private: + void send_query_internal(std::string name, td::BufferSlice data, QueryInfo query_info, size_t server_idx, + td::Timestamp timeout, td::Promise promise) { + auto& server = servers_[server_idx]; + CHECK(!server.client.empty()); + if (!connect_to_all_) { + alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); + } + td::Promise P = [SelfId = actor_id(this), server_idx, + promise = std::move(promise)](td::Result R) mutable { + if (R.is_error() && + (R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) { + td::actor::send_closure(SelfId, &ExtClientImpl::on_server_error, server_idx); + } + promise.set_result(std::move(R)); + }; + LOG(DEBUG) << "Sending query " << query_info.to_str() << " to server #" << server.idx << " (" + << server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")"; + send_closure(server.client, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout, + std::move(P)); + } + + td::Result select_server(const QueryInfo& query_info) { + for (size_t i = 0; i < servers_.size(); ++i) { + if (servers_[i].alive && servers_[i].config.accepts_query(query_info)) { + return i; + } + } + size_t server_idx = servers_.size(); + int cnt = 0; + int best_priority = -1; + for (size_t i = 0; i < servers_.size(); ++i) { + Server& server = servers_[i]; + if (!server.config.accepts_query(query_info)) { + continue; + } + int priority = 0; + priority += (server.ignore_until && !server.ignore_until.is_in_past() ? 0 : 10); + if (priority < best_priority) { + continue; + } + if (priority > best_priority) { + best_priority = priority; + cnt = 0; + } + if (td::Random::fast(0, cnt) == 0) { + server_idx = i; + } + ++cnt; + } + if (server_idx == servers_.size()) { + return td::Status::Error(PSTRING() << "no liteserver for query " << query_info.to_str()); + } + prepare_server(server_idx, &query_info); + return server_idx; + } + + void prepare_server(size_t server_idx, const QueryInfo* query_info) { + Server& server = servers_[server_idx]; + if (server.alive) { + return; + } + server.alive = true; + server.ignore_until = {}; + if (!connect_to_all_) { + alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); + } + if (!server.client.empty()) { + return; + } + + class Callback : public ton::adnl::AdnlExtClient::Callback { + public: + explicit Callback(td::actor::ActorId parent, size_t idx) : parent_(std::move(parent)), idx_(idx) { + } + void on_ready() override { + } + void on_stop_ready() override { + td::actor::send_closure(parent_, &ExtClientImpl::on_server_error, idx_); + } + + private: + td::actor::ActorId parent_; + size_t idx_; + }; + LOG(INFO) << "Connecting to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() << ":" + << server.config.addr.get_port() << ") for query " << (query_info ? query_info->to_str() : "[none]"); + server.client = ton::adnl::AdnlExtClient::create(server.config.adnl_id, server.config.addr, + std::make_unique(actor_id(this), server_idx)); + } + + struct Server { + LiteServerConfig config; + size_t idx = 0; + td::actor::ActorOwn client; + bool alive = false; + td::Timestamp timeout = td::Timestamp::never(); + td::Timestamp ignore_until = td::Timestamp::never(); + }; + std::vector servers_; + std::vector server_indices_; + + td::unique_ptr callback_; + bool connect_to_all_ = false; + static constexpr double MAX_NO_QUERIES_TIMEOUT = 100.0; + static constexpr double BAD_SERVER_TIMEOUT = 30.0; + + void alarm() override { + if (connect_to_all_) { + return; + } + for (Server& server : servers_) { + if (server.timeout && server.timeout.is_in_past()) { + LOG(INFO) << "Closing connection to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() + << ":" << server.config.addr.get_port() << ")"; + server.client.reset(); + server.alive = false; + server.ignore_until = {}; + } + } + } + + void on_server_error(size_t idx) { + servers_[idx].alive = false; + servers_[idx].ignore_until = td::Timestamp::in(BAD_SERVER_TIMEOUT); + } +}; + +td::actor::ActorOwn ExtClient::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, + td::unique_ptr callback) { + return create({LiteServerConfig{dst, dst_addr}}, std::move(callback)); +} + +td::actor::ActorOwn ExtClient::create(std::vector liteservers, + td::unique_ptr callback, bool connect_to_all) { + return td::actor::create_actor("ExtClient", std::move(liteservers), std::move(callback), + connect_to_all); +} +} // namespace liteclient diff --git a/lite-client/ext-client.h b/lite-client/ext-client.h new file mode 100644 index 00000000..ef4523fd --- /dev/null +++ b/lite-client/ext-client.h @@ -0,0 +1,48 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "td/actor/actor.h" +#include "ton/ton-types.h" +#include "adnl/adnl-ext-client.h" +#include "query-utils.hpp" + +namespace liteclient { +class ExtClient : public td::actor::Actor { + public: + class Callback { + public: + virtual ~Callback() = default; + }; + + virtual void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, + td::Promise promise) = 0; + virtual void send_query_to_server(std::string name, td::BufferSlice data, size_t server_idx, td::Timestamp timeout, + td::Promise promise) { + promise.set_error(td::Status::Error("not supported")); + } + virtual void get_servers_status(td::Promise> promise) { + promise.set_error(td::Status::Error("not supported")); + } + virtual void reset_servers() { + } + + static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, + td::unique_ptr callback); + static td::actor::ActorOwn create(std::vector liteservers, + td::unique_ptr callback, bool connect_to_all = false); +}; +} // namespace liteclient diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index feedbe40..1050e6d2 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -29,22 +29,16 @@ #include "lite-client-common.h" -#include "adnl/adnl-ext-client.h" #include "tl-utils/lite-utils.hpp" #include "auto/tl/ton_api_json.h" #include "auto/tl/lite_api.hpp" #include "td/utils/OptionParser.h" #include "td/utils/Time.h" #include "td/utils/filesystem.h" -#include "td/utils/format.h" #include "td/utils/Random.h" #include "td/utils/crypto.h" -#include "td/utils/overloaded.h" #include "td/utils/port/signals.h" -#include "td/utils/port/stacktrace.h" -#include "td/utils/port/StdStreams.h" #include "td/utils/port/FileFd.h" -#include "terminal/terminal.h" #include "ton/lite-tl.hpp" #include "block/block-db.h" #include "block/block.h" @@ -58,18 +52,14 @@ #include "vm/vm.h" #include "vm/cp0.h" #include "vm/memo.h" -#include "ton/ton-shard.h" -#include "openssl/rand.hpp" #include "crypto/vm/utils.h" #include "crypto/common/util.h" #include "common/checksum.h" #if TD_DARWIN || TD_LINUX #include -#include #endif #include -#include #include "git.h" using namespace std::literals::string_literals; @@ -77,24 +67,6 @@ using td::Ref; int verbosity; -std::unique_ptr TestNode::make_callback() { - class Callback : public ton::adnl::AdnlExtClient::Callback { - public: - void on_ready() override { - td::actor::send_closure(id_, &TestNode::conn_ready); - } - void on_stop_ready() override { - td::actor::send_closure(id_, &TestNode::conn_closed); - } - Callback(td::actor::ActorId id) : id_(std::move(id)) { - } - - private: - td::actor::ActorId id_; - }; - return std::make_unique(actor_id(this)); -} - void TestNode::run() { class Cb : public td::TerminalIO::Callback { public: @@ -110,19 +82,20 @@ void TestNode::run() { io_ = td::TerminalIO::create("> ", readline_enabled_, ex_mode_, std::make_unique(actor_id(this))); td::actor::send_closure(io_, &td::TerminalIO::set_log_interface); - if (remote_public_key_.empty()) { + std::vector servers; + if (!single_remote_public_key_.empty()) { // Use single provided liteserver + servers.push_back( + liteclient::LiteServerConfig{ton::adnl::AdnlNodeIdFull{single_remote_public_key_}, single_remote_addr_}); + td::TerminalIO::out() << "using liteserver " << single_remote_addr_ << "\n"; + } else { auto G = td::read_file(global_config_).move_as_ok(); auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); ton::ton_api::liteclient_config_global gc; ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); - CHECK(gc.liteservers_.size() > 0); - auto idx = liteserver_idx_ >= 0 ? liteserver_idx_ - : td::Random::fast(0, static_cast(gc.liteservers_.size() - 1)); - CHECK(idx >= 0 && static_cast(idx) <= gc.liteservers_.size()); - auto& cli = gc.liteservers_[idx]; - remote_addr_.init_host_port(td::IPAddress::ipv4_to_str(cli->ip_), cli->port_).ensure(); - remote_public_key_ = ton::PublicKey{cli->id_}; - td::TerminalIO::out() << "using liteserver " << idx << " with addr " << remote_addr_ << "\n"; + auto r_servers = liteclient::LiteServerConfig::parse_global_config(gc); + r_servers.ensure(); + servers = r_servers.move_as_ok(); + if (gc.validator_ && gc.validator_->zero_state_) { zstate_id_.workchain = gc.validator_->zero_state_->workchain_; if (zstate_id_.workchain != ton::workchainInvalid) { @@ -131,10 +104,19 @@ void TestNode::run() { td::TerminalIO::out() << "zerostate set to " << zstate_id_.to_str() << "\n"; } } - } - client_ = - ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull{remote_public_key_}, remote_addr_, make_callback()); + if (single_liteserver_idx_ != -1) { // Use single liteserver from config + CHECK(single_liteserver_idx_ >= 0 && (size_t)single_liteserver_idx_ < servers.size()); + td::TerminalIO::out() << "using liteserver #" << single_liteserver_idx_ << " with addr " + << servers[single_liteserver_idx_].addr << "\n"; + servers = {servers[single_liteserver_idx_]}; + } + } + CHECK(!servers.empty()); + client_ = liteclient::ExtClient::create(std::move(servers), nullptr); + ready_ = true; + + run_init_queries(); } void TestNode::got_result(td::Result R, td::Promise promise) { @@ -191,8 +173,8 @@ bool TestNode::envelope_send_query(td::BufferSlice query, td::Promise(std::move(query)), true); - td::actor::send_closure(client_, &ton::adnl::AdnlExtClient::send_query, "query", std::move(b), - td::Timestamp::in(10.0), std::move(P)); + td::actor::send_closure(client_, &liteclient::ExtClient::send_query, "query", std::move(b), td::Timestamp::in(10.0), + std::move(P)); return true; } @@ -319,9 +301,10 @@ bool TestNode::get_server_time() { if (F.is_error()) { LOG(ERROR) << "cannot parse answer to liteServer.getTime"; } else { - server_time_ = F.move_as_ok()->now_; - server_time_got_at_ = now(); - LOG(INFO) << "server time is " << server_time_ << " (delta " << server_time_ - server_time_got_at_ << ")"; + mc_server_time_ = F.move_as_ok()->now_; + mc_server_time_got_at_ = now(); + LOG(INFO) << "server time is " << mc_server_time_ << " (delta " << mc_server_time_ - mc_server_time_got_at_ + << ")"; } } }); @@ -335,7 +318,7 @@ bool TestNode::get_server_version(int mode) { }; void TestNode::got_server_version(td::Result res, int mode) { - server_ok_ = false; + mc_server_ok_ = false; if (res.is_error()) { LOG(ERROR) << "cannot get server version and time (server too old?)"; } else { @@ -344,11 +327,11 @@ void TestNode::got_server_version(td::Result res, int mode) { LOG(ERROR) << "cannot parse answer to liteServer.getVersion"; } else { auto a = F.move_as_ok(); - set_server_version(a->version_, a->capabilities_); - set_server_time(a->now_); + set_mc_server_version(a->version_, a->capabilities_); + set_mc_server_time(a->now_); } } - if (!server_ok_) { + if (!mc_server_ok_) { LOG(ERROR) << "server version is too old (at least " << (min_ls_version >> 8) << "." << (min_ls_version & 0xff) << " with capabilities " << min_ls_capabilities << " required), some queries are unavailable"; } @@ -357,24 +340,24 @@ void TestNode::got_server_version(td::Result res, int mode) { } } -void TestNode::set_server_version(td::int32 version, td::int64 capabilities) { - if (server_version_ != version || server_capabilities_ != capabilities) { - server_version_ = version; - server_capabilities_ = capabilities; - LOG(WARNING) << "server version is " << (server_version_ >> 8) << "." << (server_version_ & 0xff) - << ", capabilities " << server_capabilities_; +void TestNode::set_mc_server_version(td::int32 version, td::int64 capabilities) { + if (mc_server_version_ != version || mc_server_capabilities_ != capabilities) { + mc_server_version_ = version; + mc_server_capabilities_ = capabilities; + LOG(WARNING) << "server version is " << (mc_server_version_ >> 8) << "." << (mc_server_version_ & 0xff) + << ", capabilities " << mc_server_capabilities_; } - server_ok_ = (server_version_ >= min_ls_version) && !(~server_capabilities_ & min_ls_capabilities); + mc_server_ok_ = (mc_server_version_ >= min_ls_version) && !(~mc_server_capabilities_ & min_ls_capabilities); } -void TestNode::set_server_time(int server_utime) { - server_time_ = server_utime; - server_time_got_at_ = now(); - LOG(INFO) << "server time is " << server_time_ << " (delta " << server_time_ - server_time_got_at_ << ")"; +void TestNode::set_mc_server_time(int server_utime) { + mc_server_time_ = server_utime; + mc_server_time_got_at_ = now(); + LOG(INFO) << "server time is " << mc_server_time_ << " (delta " << mc_server_time_ - mc_server_time_got_at_ << ")"; } bool TestNode::get_server_mc_block_id() { - int mode = (server_capabilities_ & 2) ? 0 : -1; + int mode = (mc_server_capabilities_ & 2) ? 0 : -1; if (mode < 0) { auto b = ton::serialize_tl_object(ton::create_tl_object(), true); return envelope_send_query(std::move(b), [Self = actor_id(this)](td::Result res) -> void { @@ -448,8 +431,8 @@ void TestNode::got_server_mc_block_id(ton::BlockIdExt blkid, ton::ZeroStateIdExt void TestNode::got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateIdExt zstateid, int mode, int version, long long capabilities, int last_utime, int server_now) { - set_server_version(version, capabilities); - set_server_time(server_now); + set_mc_server_version(version, capabilities); + set_mc_server_time(server_now); if (last_utime > server_now) { LOG(WARNING) << "server claims to have a masterchain block " << blkid.to_str() << " created at " << last_utime << " (" << last_utime - server_now << " seconds in the future)"; @@ -457,10 +440,10 @@ void TestNode::got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateI LOG(WARNING) << "server appears to be out of sync: its newest masterchain block is " << blkid.to_str() << " created at " << last_utime << " (" << server_now - last_utime << " seconds ago according to the server's clock)"; - } else if (last_utime < server_time_got_at_ - 60) { + } else if (last_utime < mc_server_time_got_at_ - 60) { LOG(WARNING) << "either the server is out of sync, or the local clock is set incorrectly: the newest masterchain " "block known to server is " - << blkid.to_str() << " created at " << last_utime << " (" << server_now - server_time_got_at_ + << blkid.to_str() << " created at " << last_utime << " (" << server_now - mc_server_time_got_at_ << " seconds ago according to the local clock)"; } got_server_mc_block_id(blkid, zstateid, last_utime); @@ -926,7 +909,7 @@ bool TestNode::show_help(std::string command) { "saveaccount[code|data] []\tSaves into specified file the most recent state " "(StateInit) or just the code or data of specified account; is in " "[:] format\n" - "runmethod[full] [] ...\tRuns GET method of account " + "runmethod[full] [] ...\tRuns GET method of account " " " "with specified parameters\n" "dnsresolve [] []\tResolves a domain starting from root dns smart contract\n" @@ -949,8 +932,8 @@ bool TestNode::show_help(std::string command) { "lasttrans[dump] []\tShows or dumps specified transaction and " "several preceding " "ones\n" - "listblocktrans[rev] [ ]\tLists block transactions, " - "starting immediately after or before the specified one\n" + "listblocktrans[rev][meta] [ ]\tLists block " + "transactions, starting immediately after or before the specified one\n" "blkproofchain[step] []\tDownloads and checks proof of validity of the " "second " "indicated block (or the last known masterchain block) starting from given block\n" @@ -965,14 +948,20 @@ bool TestNode::show_help(std::string command) { "recentcreatorstats [ []]\tLists block creator statistics " "updated after by validator public " "key\n" - "checkload[all|severe] []\tChecks whether all validators worked " - "properly during specified time " + "checkload[all|severe][-v2] []\tChecks whether all validators " + "worked properly during specified time " "interval, and optionally saves proofs into -.boc\n" "loadproofcheck \tChecks a validator misbehavior proof previously created by checkload\n" "pastvalsets\tLists known past validator set ids and their hashes\n" "savecomplaints \tSaves all complaints registered for specified validator set id " "into files .boc\n" "complaintprice \tComputes the price (in nanograms) for creating a complaint\n" + "msgqueuesizes\tShows current sizes of outbound message queues in all shards\n" + "dispatchqueueinfo \tShows list of account dispatch queue of a block\n" + "dispatchqueuemessages []\tShows deferred messages from account , lt > " + "\n" + "dispatchqueuemessagesall [ []]\tShows messages from dispatch queue of a " + "block, starting after , \n" "known\tShows the list of all known block ids\n" "knowncells\tShows the list of hashes of all known (cached) cells\n" "dumpcell \nDumps a cached cell by a prefix of its hash\n" @@ -987,9 +976,9 @@ bool TestNode::show_help(std::string command) { bool TestNode::do_parse_line() { ton::WorkchainId workchain = ton::masterchainId; // change to basechain later int addr_ext = 0; - ton::StdSmcAddress addr{}; + ton::StdSmcAddress addr = ton::StdSmcAddress::zero(); ton::BlockIdExt blkid{}; - ton::LogicalTime lt{}; + ton::LogicalTime lt = 0; ton::Bits256 hash{}; ton::ShardIdFull shard{}; ton::BlockSeqno seqno{}; @@ -1073,6 +1062,13 @@ bool TestNode::do_parse_line() { return parse_block_id_ext(blkid) && parse_uint32(count) && (seekeoln() || (parse_hash(hash) && parse_lt(lt) && (mode |= 128) && seekeoln())) && get_block_transactions(blkid, mode, count, hash, lt); + } else if (word == "listblocktransmeta" || word == "listblocktransrevmeta") { + lt = 0; + int mode = (word == "listblocktransmeta" ? 7 : 0x47); + mode |= 256; + return parse_block_id_ext(blkid) && parse_uint32(count) && + (seekeoln() || (parse_hash(hash) && parse_lt(lt) && (mode |= 128) && seekeoln())) && + get_block_transactions(blkid, mode, count, hash, lt); } else if (word == "blkproofchain" || word == "blkproofchainstep") { ton::BlockIdExt blkid2{}; return parse_block_id_ext(blkid) && (seekeoln() || parse_block_id_ext(blkid2)) && seekeoln() && @@ -1089,8 +1085,15 @@ bool TestNode::do_parse_line() { return parse_block_id_ext(blkid) && (!mode || parse_uint32(utime)) && (seekeoln() ? (mode |= 0x100) : parse_uint32(count)) && (seekeoln() || (parse_hash(hash) && (mode |= 1))) && seekeoln() && get_creator_stats(blkid, mode, count, hash, utime); - } else if (word == "checkload" || word == "checkloadall" || word == "checkloadsevere") { - int time1, time2, mode = (word == "checkloadsevere"); + } else if (word == "checkload" || word == "checkloadall" || word == "checkloadsevere" || word == "checkload-v2" || + word == "checkloadall-v2" || word == "checkloadsevere-v2") { + int time1, time2, mode = 0; + if (word == "checkloadsevere" || word == "checkloadsevere-v2") { + mode |= 1; + } + if (td::ends_with(word, "-v2")) { + mode |= 4; + } std::string file_pfx; return parse_int32(time1) && parse_int32(time2) && (seekeoln() || ((mode |= 2) && get_word_to(file_pfx))) && seekeoln() && check_validator_load(time1, time2, mode, file_pfx); @@ -1108,6 +1111,18 @@ bool TestNode::do_parse_line() { std::string filename; return parse_uint32(expire_in) && get_word_to(filename) && seekeoln() && set_error(get_complaint_price(expire_in, filename)); + } else if (word == "msgqueuesizes") { + return get_msg_queue_sizes(); + } else if (word == "dispatchqueueinfo") { + return parse_block_id_ext(blkid) && seekeoln() && get_dispatch_queue_info(blkid); + } else if (word == "dispatchqueuemessages" || word == "dispatchqueuemessagesall") { + bool one_account = word == "dispatchqueuemessages"; + if (!parse_block_id_ext(blkid)) { + return false; + } + workchain = blkid.id.workchain; + return ((!one_account && seekeoln()) || parse_account_addr(workchain, addr)) && (seekeoln() || parse_lt(lt)) && + seekeoln() && get_dispatch_queue_messages(blkid, workchain, addr, lt, one_account); } else if (word == "known") { return eoln() && show_new_blkids(true); } else if (word == "knowncells") { @@ -1275,7 +1290,7 @@ bool TestNode::after_parse_run_method(ton::WorkchainId workchain, ton::StdSmcAdd } } }); - return start_run_method(workchain, addr, ref_blkid, method_name, std::move(params), ext_mode ? 0x1f : 0, + return start_run_method(workchain, addr, ref_blkid, method_name, std::move(params), ext_mode ? 0x17 : 0, std::move(P)); } @@ -1449,7 +1464,7 @@ bool TestNode::send_past_vset_query(ton::StdSmcAddress elector_addr) { } register_past_vset_info(std::move(S.back())); }); - return start_run_method(ton::masterchainId, elector_addr, mc_last_id_, "past_elections_list", std::move(params), 0x1f, + return start_run_method(ton::masterchainId, elector_addr, mc_last_id_, "past_elections_list", std::move(params), 0x17, std::move(P)); } @@ -1510,7 +1525,7 @@ void TestNode::send_get_complaints_query(unsigned elect_id, ton::StdSmcAddress e LOG(ERROR) << "vm virtualization error: " << err.get_msg(); } }); - start_run_method(ton::masterchainId, elector_addr, mc_last_id_, "get_past_complaints", std::move(params), 0x1f, + start_run_method(ton::masterchainId, elector_addr, mc_last_id_, "get_past_complaints", std::move(params), 0x17, std::move(P)); } @@ -1607,10 +1622,109 @@ void TestNode::send_compute_complaint_price_query(ton::StdSmcAddress elector_add LOG(ERROR) << "vm virtualization error: " << err.get_msg(); } }); - start_run_method(ton::masterchainId, elector_addr, mc_last_id_, "complaint_storage_price", std::move(params), 0x1f, + start_run_method(ton::masterchainId, elector_addr, mc_last_id_, "complaint_storage_price", std::move(params), 0x17, std::move(P)); } +bool TestNode::get_msg_queue_sizes() { + auto q = ton::serialize_tl_object(ton::create_tl_object(0, 0, 0), true); + return envelope_send_query(std::move(q), [Self = actor_id(this)](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "liteServer.getOutMsgQueueSizes error: " << res.move_as_error(); + return; + } + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getOutMsgQueueSizes"; + return; + } + td::actor::send_closure_later(Self, &TestNode::got_msg_queue_sizes, F.move_as_ok()); + }); +} + +void TestNode::got_msg_queue_sizes(ton::tl_object_ptr f) { + td::TerminalIO::out() << "Outbound message queue sizes:" << std::endl; + for (auto &x : f->shards_) { + td::TerminalIO::out() << ton::create_block_id(x->id_).id.to_str() << " " << x->size_ << std::endl; + } + td::TerminalIO::out() << "External message queue size limit: " << f->ext_msg_queue_size_limit_ << std::endl; +} + +bool TestNode::get_dispatch_queue_info(ton::BlockIdExt block_id) { + td::TerminalIO::out() << "Dispatch queue in block: " << block_id.id.to_str() << std::endl; + return get_dispatch_queue_info_cont(block_id, true, td::Bits256::zero()); +} + +bool TestNode::get_dispatch_queue_info_cont(ton::BlockIdExt block_id, bool first, td::Bits256 after_addr) { + auto q = ton::create_serialize_tl_object( + first ? 0 : 2, ton::create_tl_lite_block_id(block_id), after_addr, 32, false); + return envelope_send_query(std::move(q), [=, Self = actor_id(this)](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "liteServer.getDispatchQueueInfo error: " << res.move_as_error(); + return; + } + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getDispatchQueueInfo"; + return; + } + td::actor::send_closure_later(Self, &TestNode::got_dispatch_queue_info, block_id, F.move_as_ok()); + }); +} + +void TestNode::got_dispatch_queue_info(ton::BlockIdExt block_id, + ton::tl_object_ptr info) { + for (auto& acc : info->account_dispatch_queues_) { + td::TerminalIO::out() << block_id.id.workchain << ":" << acc->addr_.to_hex() << " : size=" << acc->size_ + << " lt=" << acc->min_lt_ << ".." << acc->max_lt_ << std::endl; + } + if (info->complete_) { + td::TerminalIO::out() << "Done" << std::endl; + return; + } + get_dispatch_queue_info_cont(block_id, false, info->account_dispatch_queues_.back()->addr_); +} + +bool TestNode::get_dispatch_queue_messages(ton::BlockIdExt block_id, ton::WorkchainId wc, ton::StdSmcAddress addr, + ton::LogicalTime lt, bool one_account) { + if (wc != block_id.id.workchain) { + return set_error("workchain mismatch"); + } + auto q = ton::create_serialize_tl_object( + one_account ? 2 : 0, ton::create_tl_lite_block_id(block_id), addr, lt, 64, false, one_account, false); + return envelope_send_query(std::move(q), [=, Self = actor_id(this)](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "liteServer.getDispatchQueueMessages error: " << res.move_as_error(); + return; + } + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getDispatchQueueMessages"; + return; + } + td::actor::send_closure_later(Self, &TestNode::got_dispatch_queue_messages, F.move_as_ok()); + }); +} + +void TestNode::got_dispatch_queue_messages(ton::tl_object_ptr msgs) { + td::TerminalIO::out() << "Dispatch queue messages (" << msgs->messages_.size() << "):\n"; + int count = 0; + for (auto& m : msgs->messages_) { + auto& meta = m->metadata_; + td::TerminalIO::out() << "Msg #" << ++count << ": " << msgs->id_->workchain_ << ":" << m->addr_.to_hex() << " " + << m->lt_ << " : " + << (meta->initiator_->workchain_ == ton::workchainInvalid + ? "[ no metadata ]" + : block::MsgMetadata{(td::uint32)meta->depth_, meta->initiator_->workchain_, + meta->initiator_->id_, (ton::LogicalTime)meta->initiator_lt_} + .to_str()) + << "\n"; + } + if (!msgs->complete_) { + td::TerminalIO::out() << "(incomplete list)\n"; + } +} + bool TestNode::dns_resolve_start(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::BlockIdExt blkid, std::string domain, td::Bits256 cat, int mode) { if (domain.size() >= 2 && domain[0] == '"' && domain.back() == '"') { @@ -1708,7 +1822,7 @@ bool TestNode::dns_resolve_send(ton::WorkchainId workchain, ton::StdSmcAddress a } return dns_resolve_finish(workchain, addr, blkid, domain, qdomain, cat, mode, (int)x->to_long(), std::move(cell)); }); - return start_run_method(workchain, addr, blkid, "dnsresolve", std::move(params), 0x1f, std::move(P)); + return start_run_method(workchain, addr, blkid, "dnsresolve", std::move(params), 0x17, std::move(P)); } bool TestNode::show_dns_record(std::ostream& os, td::Bits256 cat, Ref value, bool raw_dump) { @@ -2113,7 +2227,7 @@ void TestNode::run_smc_method(int mode, ton::BlockIdExt ref_blk, ton::BlockIdExt // auto log = create_vm_log(ctx.error_stream ? &ostream_logger : nullptr); vm::GasLimits gas{gas_limit}; LOG(DEBUG) << "creating VM"; - vm::VmState vm{code, std::move(stack), gas, 1, data, vm::VmLog()}; + vm::VmState vm{code, ton::SUPPORTED_VERSION, std::move(stack), gas, 1, data, vm::VmLog()}; vm.set_c7(liteclient::prepare_vm_c7(info.gen_utime, info.gen_lt, td::make_ref(acc.addr->clone()), balance)); // tuple with SmartContractInfo // vm.incr_stack_trace(1); // enable stack dump after each step @@ -2142,21 +2256,29 @@ void TestNode::run_smc_method(int mode, ton::BlockIdExt ref_blk, ton::BlockIdExt } } if (exit_code != 0) { - LOG(ERROR) << "VM terminated with error code " << exit_code; out << "result: error " << exit_code << std::endl; - promise.set_error(td::Status::Error(PSLICE() << "VM terminated with non-zero exit code " << exit_code)); - return; - } - stack = vm.get_stack_ref(); - { + } else { + stack = vm.get_stack_ref(); std::ostringstream os; os << "result: "; stack->dump(os, 3); out << os.str(); } - if (mode & 4) { - if (remote_result.empty()) { - out << "remote result: , exit code " << remote_exit_code; + if (!(mode & 4)) { + if (exit_code != 0) { + LOG(ERROR) << "VM terminated with error code " << exit_code; + promise.set_error(td::Status::Error(PSLICE() << "VM terminated with non-zero exit code " << exit_code)); + } else { + promise.set_result(stack->extract_contents()); + } + } else { + if (remote_exit_code != 0) { + out << "remote result: error " << remote_exit_code << std::endl; + LOG(ERROR) << "VM terminated with error code " << exit_code; + promise.set_error(td::Status::Error(PSLICE() << "VM terminated with non-zero exit code " << exit_code)); + } else if (remote_result.empty()) { + out << "remote result: " << std::endl; + promise.set_value({}); } else { auto res = vm::std_boc_deserialize(std::move(remote_result)); if (res.is_error()) { @@ -2177,10 +2299,10 @@ void TestNode::run_smc_method(int mode, ton::BlockIdExt ref_blk, ton::BlockIdExt os << "remote result (not to be trusted): "; remote_stack->dump(os, 3); out << os.str(); + promise.set_value(remote_stack->extract_contents()); } } out.flush(); - promise.set_result(stack->extract_contents()); } catch (vm::VmVirtError& err) { out << "virtualization error while parsing runSmcMethod result: " << err.get_msg(); promise.set_error( @@ -2458,23 +2580,40 @@ bool TestNode::get_block_transactions(ton::BlockIdExt blkid, int mode, unsigned } else { auto f = F.move_as_ok(); std::vector transactions; + std::vector> metadata; for (auto& id : f->ids_) { transactions.emplace_back(id->account_, id->lt_, id->hash_); + metadata.push_back(std::move(id->metadata_)); } td::actor::send_closure_later(Self, &TestNode::got_block_transactions, ton::create_block_id(f->id_), mode, - f->req_count_, f->incomplete_, std::move(transactions), std::move(f->proof_)); + f->req_count_, f->incomplete_, std::move(transactions), std::move(metadata), + std::move(f->proof_)); } }); } -void TestNode::got_block_transactions(ton::BlockIdExt blkid, int mode, unsigned req_count, bool incomplete, - std::vector trans, td::BufferSlice proof) { +void TestNode::got_block_transactions( + ton::BlockIdExt blkid, int mode, unsigned req_count, bool incomplete, std::vector trans, + std::vector> metadata, td::BufferSlice proof) { LOG(INFO) << "got up to " << req_count << " transactions from block " << blkid.to_str(); auto out = td::TerminalIO::out(); int count = 0; - for (auto& t : trans) { + for (size_t i = 0; i < trans.size(); ++i) { + auto& t = trans[i]; out << "transaction #" << ++count << ": account " << t.acc_addr.to_hex() << " lt " << t.trans_lt << " hash " << t.trans_hash.to_hex() << std::endl; + if (mode & 256) { + auto& meta = metadata.at(i); + if (meta == nullptr) { + out << " metadata: " << std::endl; + } else { + out << " metadata: " + << block::MsgMetadata{(td::uint32)meta->depth_, meta->initiator_->workchain_, meta->initiator_->id_, + (ton::LogicalTime)meta->initiator_lt_} + .to_str() + << std::endl; + } + } } out << (incomplete ? "(block transaction list incomplete)" : "(end of block transaction list)") << std::endl; } @@ -3370,9 +3509,7 @@ void TestNode::got_creator_stats(ton::BlockIdExt req_blkid, ton::BlockIdExt blki promise.set_error(td::Status::Error(PSLICE() << "invalid CreatorStats record with key " << key.to_hex())); return; } - if (mc_cnt.modified_since(min_utime) || shard_cnt.modified_since(min_utime)) { - func(key, mc_cnt, shard_cnt); - } + func(key, mc_cnt, shard_cnt); allow_eq = false; } if (complete) { @@ -3529,7 +3666,7 @@ void TestNode::continue_check_validator_load2(std::unique_ptr info2, int mode, std::string file_pfx) { LOG(INFO) << "continue_check_validator_load2 for blocks " << info1->blk_id.to_str() << " and " - << info1->blk_id.to_str() << " : requesting block creators data"; + << info2->blk_id.to_str() << " : requesting block creators data"; td::Status st = info1->unpack_vset(); if (st.is_error()) { LOG(ERROR) << "cannot unpack validator set from block " << info1->blk_id.to_str() << " :" << st.move_as_error(); @@ -3559,7 +3696,7 @@ void TestNode::continue_check_validator_load2(std::unique_ptr info2, int mode, std::string file_pfx) { LOG(INFO) << "continue_check_validator_load3 for blocks " << info1->blk_id.to_str() << " and " - << info1->blk_id.to_str() << " with mode=" << mode << " and file prefix `" << file_pfx - << "`: comparing block creators data"; + << info2->blk_id.to_str() << " with mode=" << mode << " and file prefix `" << file_pfx; + + if (mode & 4) { + ton::BlockSeqno start_seqno = info1->blk_id.seqno(); + ton::BlockSeqno end_seqno = info2->blk_id.seqno(); + block::ValidatorSet validator_set = *info1->vset; + if (info1->config->get_config_param(28)->get_hash() != info2->config->get_config_param(28)->get_hash()) { + LOG(ERROR) << "Catchain validator config (28) changed between the first and the last block"; + return; + } + auto catchain_config = std::make_unique( + block::Config::unpack_catchain_validators_config(info1->config->get_config_param(28))); + load_validator_shard_shares( + start_seqno, end_seqno, std::move(validator_set), std::move(catchain_config), + [=, this, info1 = std::move(info1), + info2 = std::move(info2)](td::Result> R) mutable { + if (R.is_error()) { + LOG(ERROR) << "failed to load validator shard shares: " << R.move_as_error(); + } else { + continue_check_validator_load4(std::move(info1), std::move(info2), mode, file_pfx, R.move_as_ok()); + } + }); + } else { + continue_check_validator_load4(std::move(info1), std::move(info2), mode, std::move(file_pfx), {}); + } +} + +void TestNode::continue_check_validator_load4(std::unique_ptr info1, + std::unique_ptr info2, int mode, + std::string file_pfx, + std::map exact_shard_shares) { + LOG(INFO) << "continue_check_validator_load4 for blocks " << info1->blk_id.to_str() << " and " + << info2->blk_id.to_str() << " with mode=" << mode << " and file prefix `" << file_pfx; if (info1->created_total.first <= 0 || info2->created_total.first <= 0) { LOG(ERROR) << "no total created blocks statistics"; return; } td::TerminalIO::out() << "total: (" << info1->created_total.first << "," << info1->created_total.second << ") -> (" << info2->created_total.first << "," << info2->created_total.second << ")\n"; - auto x = info2->created_total.first - info1->created_total.first; - auto y = info2->created_total.second - info1->created_total.second; - td::int64 xs = 0, ys = 0; - if (x <= 0 || y < 0 || (x | y) >= (1u << 31)) { - LOG(ERROR) << "impossible situation: zero or no blocks created: " << x << " masterchain blocks, " << y - << " shardchain blocks"; + auto created_total_mc = info2->created_total.first - info1->created_total.first; + auto created_total_bc = info2->created_total.second - info1->created_total.second; + td::int64 created_mc_sum = 0, created_bc_sum = 0; + if (created_total_mc <= 0 || created_total_bc < 0 || (created_total_mc | created_total_bc) >= (1U << 31)) { + LOG(ERROR) << "impossible situation: zero or no blocks created: " << created_total_mc << " masterchain blocks, " + << created_total_bc << " shardchain blocks"; return; } - std::pair created_total{(int)x, (int)y}; int count = info1->vset->total; CHECK(info2->vset->total == count); CHECK((int)info1->created.size() == count); CHECK((int)info2->created.size() == count); - std::vector> d; - d.reserve(count); + std::vector> vals_created; + vals_created.reserve(count); for (int i = 0; i < count; i++) { - auto x1 = info2->created[i].first - info1->created[i].first; - auto y1 = info2->created[i].second - info1->created[i].second; - if (x1 < 0 || y1 < 0 || (x1 | y1) >= (1u << 31)) { - LOG(ERROR) << "impossible situation: validator #i created a negative amount of blocks: " << x1 - << " masterchain blocks, " << y1 << " shardchain blocks"; + auto created_mc = info2->created[i].first - info1->created[i].first; + auto created_bc = info2->created[i].second - info1->created[i].second; + if (created_mc < 0 || created_bc < 0 || (created_mc | created_bc) >= (1u << 31)) { + LOG(ERROR) << "impossible situation: validator #" << i << " created a negative amount of blocks: " << created_mc + << " masterchain blocks, " << created_bc << " shardchain blocks"; return; } - xs += x1; - ys += y1; - d.emplace_back((int)x1, (int)y1); - td::TerminalIO::out() << "val #" << i << ": created (" << x1 << "," << y1 << ") ; was (" << info1->created[i].first - << "," << info1->created[i].second << ")\n"; + created_mc_sum += created_mc; + created_bc_sum += created_bc; + vals_created.emplace_back((int)created_mc, (int)created_bc); + td::TerminalIO::out() << "val #" << i << ": created (" << created_mc << "," << created_bc << ") ; was (" + << info1->created[i].first << "," << info1->created[i].second << ")\n"; } - if (xs != x || ys != y) { - LOG(ERROR) << "cannot account for all blocks created: total is (" << x << "," << y - << "), but the sum for all validators is (" << xs << "," << ys << ")"; + if (created_mc_sum != created_total_mc || created_bc_sum != created_total_bc) { + LOG(ERROR) << "cannot account for all blocks created: total is (" << created_total_mc << "," << created_total_bc + << "), but the sum for all validators is (" << created_mc_sum << "," << created_bc_sum << ")"; return; } - td::TerminalIO::out() << "total: (" << x << "," << y << ")\n"; + td::TerminalIO::out() << "total: (" << created_total_mc << "," << created_total_bc << ")\n"; auto ccfg = block::Config::unpack_catchain_validators_config(info2->config->get_config_param(28)); auto ccfg_old = block::Config::unpack_catchain_validators_config(info1->config->get_config_param(28)); if (ccfg.shard_val_num != ccfg_old.shard_val_num || ccfg.shard_val_num <= 0) { @@ -3650,56 +3817,216 @@ void TestNode::continue_check_validator_load3(std::unique_ptrvset->main; - if (info1->vset->main != main_count || main_count <= 0) { - LOG(ERROR) << "masterchain validator group size changed from " << info1->vset->main << " to " << main_count + int shard_vals = ccfg.shard_val_num, master_vals = info2->vset->main; + if (info1->vset->main != master_vals || master_vals <= 0) { + LOG(ERROR) << "masterchain validator group size changed from " << info1->vset->main << " to " << master_vals << ", or is not positive"; return; } - int cnt = 0, cnt_ok = 0; - double chunk_size = ccfg.shard_val_lifetime / 3. / shard_count; - block::MtCarloComputeShare shard_share(shard_count, info2->vset->export_scaled_validator_weights()); + + bool use_exact_shard_share = mode & 4; + int proofs_cnt = 0, proofs_cnt_ok = 0; + double chunk_size = ccfg.shard_val_lifetime / 3. / shard_vals; + + std::vector mtc_shard_share; + if (use_exact_shard_share) { + LOG(INFO) << "using exact shard shares"; + td::uint64 exact_shard_shares_sum = 0; + for (auto& [_, count] : exact_shard_shares) { + exact_shard_shares_sum += count; + } + if ((td::int64)exact_shard_shares_sum != shard_vals * created_bc_sum) { + LOG(ERROR) << "unexpected total shard shares: blocks=" << created_bc_sum << ", shard_vals=" << shard_vals + << ", expected_sum=" << shard_vals * created_bc_sum << ", found=" << exact_shard_shares_sum; + return; + } + } else { + LOG(INFO) << "using MtCarloComputeShare"; + block::MtCarloComputeShare mtc(shard_vals, info2->vset->export_scaled_validator_weights()); + if (!mtc.is_ok()) { + LOG(ERROR) << "failed to compute shard shares"; + return; + } + mtc_shard_share.resize(count); + for (size_t i = 0; i < count; ++i) { + mtc_shard_share[i] = mtc[i]; + } + } + + auto validators = info1->vset->export_validator_set(); for (int i = 0; i < count; i++) { - int x1 = d[i].first, y1 = d[i].second; - double xe = (i < main_count ? (double)xs / main_count : 0); - double ye = shard_share[i] * (double)ys / shard_count; + int created_mc = vals_created[i].first, created_bc = vals_created[i].second; + bool is_masterchain_validator = i < master_vals; + + double expected_created_mc = (is_masterchain_validator ? (double)created_mc_sum / master_vals : 0); + double prob_mc = create_prob(created_mc, .9 * expected_created_mc); + + double expected_created_bc, prob_bc; + if (use_exact_shard_share) { + expected_created_bc = (double)exact_shard_shares[validators[i].key.as_bits256()] / shard_vals; + prob_bc = create_prob(created_bc, .9 * expected_created_bc); + } else { + expected_created_bc = mtc_shard_share[i] * (double)created_bc_sum / shard_vals; + prob_bc = shard_create_prob(created_bc, .9 * expected_created_bc, chunk_size); + } + td::Bits256 pk = info2->vset->list[i].pubkey.as_bits256(); - double p1 = create_prob(x1, .9 * xe), p2 = shard_create_prob(y1, .9 * ye, chunk_size); - td::TerminalIO::out() << "val #" << i << ": pubkey " << pk.to_hex() << ", blocks created (" << x1 << "," << y1 - << "), expected (" << xe << "," << ye << "), probabilities " << p1 << " and " << p2 << "\n"; - if (std::min(p1, p2) < .00001) { + td::TerminalIO::out() << "val #" << i << ": pubkey " << pk.to_hex() << ", blocks created (" << created_mc << "," + << created_bc << "), expected (" << expected_created_mc << "," << expected_created_bc + << "), probabilities " << prob_mc << " and " << prob_bc << "\n"; + if ((is_masterchain_validator ? prob_mc : prob_bc) < .00001) { LOG(ERROR) << "validator #" << i << " with pubkey " << pk.to_hex() << " : serious misbehavior detected: created less than 90% of the expected amount of blocks with " "probability 99.999% : created (" - << x1 << "," << y1 << "), expected (" << xe << "," << ye << ") masterchain/shardchain blocks\n"; + << created_mc << "," << created_bc << "), expected (" << expected_created_mc << "," + << expected_created_bc << ") masterchain/shardchain blocks\n"; if (mode & 2) { - auto st = write_val_create_proof(*info1, *info2, i, true, file_pfx, ++cnt); + auto st = write_val_create_proof(*info1, *info2, i, true, file_pfx, ++proofs_cnt); if (st.is_error()) { LOG(ERROR) << "cannot create proof: " << st.move_as_error(); } else { - cnt_ok++; + proofs_cnt_ok++; } } - } else if (std::min(p1, p2) < .001) { + } else if ((is_masterchain_validator ? prob_mc : prob_bc) < .005) { LOG(ERROR) << "validator #" << i << " with pubkey " << pk.to_hex() << " : moderate misbehavior detected: created less than 90% of the expected amount of blocks with " - "probability 99.9% : created (" - << x1 << "," << y1 << "), expected (" << xe << "," << ye << ") masterchain/shardchain blocks\n"; + "probability 99.5% : created (" + << created_mc << "," << created_bc << "), expected (" << expected_created_mc << "," + << expected_created_bc << ") masterchain/shardchain blocks\n"; if ((mode & 3) == 2) { - auto st = write_val_create_proof(*info1, *info2, i, false, file_pfx, ++cnt); + auto st = write_val_create_proof(*info1, *info2, i, false, file_pfx, ++proofs_cnt); if (st.is_error()) { LOG(ERROR) << "cannot create proof: " << st.move_as_error(); } else { - cnt_ok++; + proofs_cnt_ok++; } } } } - if (cnt > 0) { - LOG(INFO) << cnt_ok << " out of " << cnt << " proofs written to " << file_pfx << "-*.boc"; + if (proofs_cnt > 0) { + LOG(INFO) << proofs_cnt_ok << " out of " << proofs_cnt << " proofs written to " << file_pfx << "-*.boc"; } } +void TestNode::load_validator_shard_shares(ton::BlockSeqno start_seqno, ton::BlockSeqno end_seqno, + block::ValidatorSet validator_set, + std::unique_ptr catchain_config, + td::Promise> promise) { + CHECK(start_seqno <= end_seqno); + LOG(INFO) << "loading shard shares from mc blocks " << start_seqno << ".." << end_seqno << " (" + << end_seqno - start_seqno + 1 << " blocks)"; + auto state = std::make_shared(); + state->start_seqno = start_seqno; + state->end_seqno = end_seqno; + state->validator_set = std::move(validator_set); + state->catchain_config = std::move(catchain_config); + state->shard_configs.resize(end_seqno - start_seqno + 1); + state->promise = std::move(promise); + load_validator_shard_shares_cont(std::move(state)); +} + +void TestNode::load_validator_shard_shares_cont(std::shared_ptr state) { + if (!state->promise) { + return; + } + if (state->loaded % 100 == 0) { + LOG(INFO) << "loaded " << state->loaded << "/" << state->shard_configs.size() << " mc blocks"; + } + while (state->cur_idx < state->shard_configs.size() && state->pending < 8) { + load_block_shard_configuration(state->start_seqno + state->cur_idx, + [this, state, idx = state->cur_idx](td::Result R) mutable { + if (R.is_error()) { + state->promise.set_error(R.move_as_error()); + state->promise = {}; + } else { + state->shard_configs[idx] = R.move_as_ok(); + --state->pending; + ++state->loaded; + load_validator_shard_shares_cont(std::move(state)); + } + }); + ++state->pending; + ++state->cur_idx; + } + + if (state->loaded != state->shard_configs.size()) { + return; + } + LOG(INFO) << "loaded all " << state->shard_configs.size() << " mc blocks, computing shard shares"; + std::map result; + try { + for (size_t idx = 0; idx + 1 < state->shard_configs.size(); ++idx) { + block::ShardConfig& shards1 = state->shard_configs[idx]; + block::ShardConfig& shards2 = state->shard_configs[idx + 1]; + + // Compute validator groups, see ValidatorManagerImpl::update_shards + auto process_shard = [&](ton::ShardIdFull shard, ton::BlockSeqno first_seqno) { + auto desc2 = shards2.get_shard_hash(shard); + if (desc2.is_null() || desc2->seqno() < first_seqno) { + return; + } + td::uint32 blocks_count = desc2->seqno() - first_seqno + 1; + ton::CatchainSeqno cc_seqno = shards1.get_shard_cc_seqno(shard); + auto val_set = + block::ConfigInfo::do_compute_validator_set(*state->catchain_config, shard, state->validator_set, cc_seqno); + for (const auto &val : val_set) { + result[val.key.as_bits256()] += blocks_count; + } + }; + + for (const ton::BlockId& id : shards1.get_shard_hash_ids()) { + ton::ShardIdFull shard = id.shard_full(); + auto desc = shards1.get_shard_hash(shard); + CHECK(desc.not_null()); + if (desc->before_split()) { + ton::ShardIdFull l_shard = shard_child(shard, true); + ton::ShardIdFull r_shard = shard_child(shard, false); + process_shard(l_shard, desc->seqno() + 1); + process_shard(r_shard, desc->seqno() + 1); + } else if (desc->before_merge()) { + if (is_right_child(shard)) { + continue; + } + ton::ShardIdFull sibling_shard = shard_sibling(shard); + auto sibling_desc = shards1.get_shard_hash(sibling_shard); + CHECK(sibling_desc.not_null()); + ton::ShardIdFull p_shard = shard_parent(shard); + process_shard(p_shard, std::max(desc->seqno(), sibling_desc->seqno()) + 1); + } else { + process_shard(shard, desc->seqno() + 1); + } + } + } + } catch (vm::VmError &e) { + state->promise.set_error(e.as_status("cannot parse shard hashes: ")); + return; + } + state->promise.set_value(std::move(result)); +} + +void TestNode::load_block_shard_configuration(ton::BlockSeqno seqno, td::Promise promise) { + lookup_block( + ton::ShardIdFull{ton::masterchainId}, 1, seqno, + [this, promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, res, std::move(R)); + auto b = ton::serialize_tl_object( + ton::create_tl_object(ton::create_tl_lite_block_id(res.blk_id)), + true); + envelope_send_query(std::move(b), [this, promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, data, std::move(R)); + TRY_RESULT_PROMISE(promise, f, ton::fetch_tl_object(data, true)); + TRY_RESULT_PROMISE(promise, root, vm::std_boc_deserialize(f->data_)); + block::ShardConfig sh_conf; + if (!sh_conf.unpack(load_cell_slice_ref(root))) { + promise.set_error(td::Status::Error("cannot extract shard block list from shard configuration")); + } else { + promise.set_value(std::move(sh_conf)); + } + }); + }); +} + bool compute_punishment_default(int interval, bool severe, td::RefInt256& fine, unsigned& fine_part) { if (interval <= 1000) { return false; // no punishments for less than 1000 seconds @@ -4265,7 +4592,7 @@ int main(int argc, char* argv[]) { }); p.add_option('V', "version", "shows lite-client build information", [&]() { std::cout << "lite-client build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; - + std::exit(0); }); p.add_option('i', "idx", "set liteserver idx", [&](td::Slice arg) { @@ -4307,7 +4634,7 @@ int main(int argc, char* argv[]) { }); #endif - vm::init_op_cp0(true); // enable vm debug + vm::init_vm(true).ensure(); // enable vm debug td::actor::Scheduler scheduler({2}); diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index 2602a0a7..721d2b20 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -26,6 +26,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "ext-client.h" #include "adnl/adnl-ext-client.h" #include "tl-utils/tl-utils.hpp" #include "ton/ton-types.h" @@ -35,6 +36,7 @@ #include "block/block.h" #include "block/mc-config.h" #include "td/utils/filesystem.h" +#include "auto/tl/lite_api.h" using td::Ref; @@ -45,22 +47,24 @@ class TestNode : public td::actor::Actor { min_ls_version = 0x101, min_ls_capabilities = 1 }; // server version >= 1.1, capabilities at least +1 = build proof chains - td::actor::ActorOwn client_; + td::actor::ActorOwn client_; td::actor::ActorOwn io_; + bool ready_ = false; + + td::int32 single_liteserver_idx_ = -1; + td::IPAddress single_remote_addr_; + ton::PublicKey single_remote_public_key_; bool readline_enabled_ = true; - bool server_ok_ = false; - td::int32 liteserver_idx_ = -1; int print_limit_ = 1024; - bool ready_ = false; - bool inited_ = false; std::string db_root_; - int server_time_ = 0; - int server_time_got_at_ = 0; - int server_version_ = 0; - long long server_capabilities_ = 0; + int mc_server_time_ = 0; + int mc_server_time_got_at_ = 0; + int mc_server_version_ = 0; + long long mc_server_capabilities_ = 0; + bool mc_server_ok_ = false; ton::ZeroStateIdExt zstate_id_; ton::BlockIdExt mc_last_id_; @@ -75,9 +79,6 @@ class TestNode : public td::actor::Actor { const char *parse_ptr_, *parse_end_; td::Status error_; - td::IPAddress remote_addr_; - ton::PublicKey remote_public_key_; - std::vector known_blk_ids_; std::size_t shown_blk_ids_ = 0; @@ -88,8 +89,6 @@ class TestNode : public td::actor::Actor { std::map> cell_cache_; - std::unique_ptr make_callback(); - using creator_stats_func_t = std::function; @@ -182,8 +181,8 @@ class TestNode : public td::actor::Actor { void got_server_mc_block_id(ton::BlockIdExt blkid, ton::ZeroStateIdExt zstateid, int created_at); void got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateIdExt zstateid, int mode, int version, long long capabilities, int last_utime, int server_now); - void set_server_version(td::int32 version, td::int64 capabilities); - void set_server_time(int server_utime); + void set_mc_server_version(td::int32 version, td::int64 capabilities); + void set_mc_server_time(int server_utime); bool request_block(ton::BlockIdExt blkid); bool request_state(ton::BlockIdExt blkid); void got_mc_block(ton::BlockIdExt blkid, td::BufferSlice data); @@ -257,7 +256,9 @@ class TestNode : public td::actor::Actor { bool get_block_transactions(ton::BlockIdExt blkid, int mode, unsigned count, ton::Bits256 acc_addr, ton::LogicalTime lt); void got_block_transactions(ton::BlockIdExt blkid, int mode, unsigned req_count, bool incomplete, - std::vector trans, td::BufferSlice proof); + std::vector trans, + std::vector> metadata, + td::BufferSlice proof); bool get_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mode); void got_block_proof(ton::BlockIdExt from, ton::BlockIdExt to, int mode, td::BufferSlice res); bool get_creator_stats(ton::BlockIdExt blkid, int mode, unsigned req_count, ton::Bits256 start_after, @@ -279,6 +280,26 @@ class TestNode : public td::actor::Actor { void continue_check_validator_load3(std::unique_ptr info1, std::unique_ptr info2, int mode = 0, std::string file_pfx = ""); + void continue_check_validator_load4(std::unique_ptr info1, + std::unique_ptr info2, int mode, std::string file_pfx, + std::map exact_shard_shares); + + struct LoadValidatorShardSharesState { + ton::BlockSeqno start_seqno; + ton::BlockSeqno end_seqno; + block::ValidatorSet validator_set; + std::unique_ptr catchain_config; + std::vector shard_configs; + td::uint32 cur_idx = 0, pending = 0, loaded = 0; + td::Promise> promise; + }; + void load_validator_shard_shares(ton::BlockSeqno start_seqno, ton::BlockSeqno end_seqno, + block::ValidatorSet validator_set, + std::unique_ptr catchain_config, + td::Promise> promise); + void load_validator_shard_shares_cont(std::shared_ptr state); + void load_block_shard_configuration(ton::BlockSeqno seqno, td::Promise promise); + td::Status write_val_create_proof(ValidatorLoadInfo& info1, ValidatorLoadInfo& info2, int idx, bool severe, std::string file_pfx, int cnt); bool load_creator_stats(std::unique_ptr load_to, @@ -302,6 +323,15 @@ class TestNode : public td::actor::Actor { td::Bits256 chash = td::Bits256::zero(), std::string filename = ""); void send_compute_complaint_price_query(ton::StdSmcAddress elector_addr, unsigned expires_in, unsigned bits, unsigned refs, td::Bits256 chash, std::string filename); + bool get_msg_queue_sizes(); + void got_msg_queue_sizes(ton::tl_object_ptr f); + bool get_dispatch_queue_info(ton::BlockIdExt block_id); + bool get_dispatch_queue_info_cont(ton::BlockIdExt block_id, bool first, td::Bits256 after_addr); + void got_dispatch_queue_info(ton::BlockIdExt block_id, + ton::tl_object_ptr info); + bool get_dispatch_queue_messages(ton::BlockIdExt block_id, ton::WorkchainId wc, ton::StdSmcAddress addr, + ton::LogicalTime lt, bool one_account); + void got_dispatch_queue_messages(ton::tl_object_ptr msgs); bool cache_cell(Ref cell); bool list_cached_cells() const; bool dump_cached_cell(td::Slice hash_pfx, td::Slice type_name = {}); @@ -338,9 +368,6 @@ class TestNode : public td::actor::Actor { bool parse_shard_id(ton::ShardIdFull& shard); bool parse_block_id_ext(ton::BlockIdExt& blkid, bool allow_incomplete = false); bool parse_block_id_ext(std::string blk_id_string, ton::BlockIdExt& blkid, bool allow_incomplete = false) const; - bool parse_stack_value(td::Slice str, vm::StackEntry& value); - bool parse_stack_value(vm::StackEntry& value); - bool parse_stack_values(std::vector& values); bool register_blkid(const ton::BlockIdExt& blkid); bool show_new_blkids(bool all = false); bool complete_blkid(ton::BlockId partial_blkid, ton::BlockIdExt& complete_blkid) const; @@ -359,16 +386,6 @@ class TestNode : public td::actor::Actor { static const tlb::TypenameLookup& get_tlb_dict(); public: - void conn_ready() { - LOG(ERROR) << "conn ready"; - ready_ = true; - if (!inited_) { - run_init_queries(); - } - } - void conn_closed() { - ready_ = false; - } void set_global_config(std::string str) { global_config_ = str; } @@ -379,10 +396,10 @@ class TestNode : public td::actor::Actor { readline_enabled_ = value; } void set_liteserver_idx(td::int32 idx) { - liteserver_idx_ = idx; + single_liteserver_idx_ = idx; } void set_remote_addr(td::IPAddress addr) { - remote_addr_ = addr; + single_remote_addr_ = addr; } void set_public_key(td::BufferSlice file_name) { auto R = [&]() -> td::Result { @@ -393,7 +410,7 @@ class TestNode : public td::actor::Actor { if (R.is_error()) { LOG(FATAL) << "bad server public key: " << R.move_as_error(); } - remote_public_key_ = R.move_as_ok(); + single_remote_public_key_ = R.move_as_ok(); } void decode_public_key(td::BufferSlice b64_key) { auto R = [&]() -> td::Result { @@ -405,7 +422,7 @@ class TestNode : public td::actor::Actor { if (R.is_error()) { LOG(FATAL) << "bad b64 server public key: " << R.move_as_error(); } - remote_public_key_ = R.move_as_ok(); + single_remote_public_key_ = R.move_as_ok(); } void set_fail_timeout(td::Timestamp ts) { fail_timeout_ = ts; @@ -443,8 +460,7 @@ class TestNode : public td::actor::Actor { bool envelope_send_query(td::BufferSlice query, td::Promise promise); void parse_line(td::BufferSlice data); - TestNode() { - } + TestNode() = default; void run(); }; diff --git a/lite-client/query-utils.cpp b/lite-client/query-utils.cpp new file mode 100644 index 00000000..b46d46a5 --- /dev/null +++ b/lite-client/query-utils.cpp @@ -0,0 +1,400 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "query-utils.hpp" + +#include "block-parse.h" +#include "td/utils/overloaded.h" +#include "tl-utils/common-utils.hpp" + +#include "block/block-auto.h" +#include "auto/tl/lite_api.hpp" +#include "overlay/overlay-broadcast.hpp" +#include "tl-utils/lite-utils.hpp" +#include "ton/lite-tl.hpp" +#include "ton/ton-shard.h" + +#include + +namespace liteclient { + +using namespace ton; + +std::string QueryInfo::to_str() const { + td::StringBuilder sb; + sb << "[ " << lite_query_name_by_id(query_id) << " " << shard_id.to_str(); + switch (type) { + case t_simple: + break; + case t_seqno: + sb << " seqno=" << value; + break; + case t_utime: + sb << " utime=" << value; + break; + case t_lt: + sb << " lt=" << value; + break; + case t_mc_seqno: + sb << " mc_seqno=" << value; + break; + } + sb << " ]"; + return sb.as_cslice().str(); +} + +QueryInfo get_query_info(td::Slice data) { + auto F = fetch_tl_object(data, true); + if (F.is_ok()) { + data = F.ok()->data_; + } else { + fetch_tl_prefix(data, true).ignore(); + } + fetch_tl_prefix(data, true).ignore(); + auto Q = fetch_tl_object(data, true); + if (Q.is_error()) { + return {}; + } + return get_query_info(*Q.ok()); +} + +QueryInfo get_query_info(const lite_api::Function& f) { + QueryInfo info; + info.query_id = f.get_id(); + auto from_block_id = [&](const tl_object_ptr& id) { + BlockIdExt block_id = create_block_id(id); + info.shard_id = block_id.shard_full(); + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno(); + }; + downcast_call( + const_cast(f), + td::overloaded([&](const lite_api::liteServer_getTime& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getVersion& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getMasterchainInfo& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getMasterchainInfoExt& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getBlock& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getBlockHeader& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getState& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getAccountState& q) { + BlockIdExt block_id = create_block_id(q.id_); + AccountIdPrefixFull acc_id_prefix = extract_addr_prefix(q.account_->workchain_, q.account_->id_); + info.shard_id = acc_id_prefix.as_leaf_shard(); + // See LiteQuery::perform_getAccountState + if (block_id.id.workchain != masterchainId) { + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno(); + } else if (block_id.id.seqno != ~0U) { + info.type = QueryInfo::t_mc_seqno; + info.value = block_id.seqno(); + } else { + info.type = QueryInfo::t_simple; + } + }, + [&](const lite_api::liteServer_getAccountStatePrunned& q) { + BlockIdExt block_id = create_block_id(q.id_); + AccountIdPrefixFull acc_id_prefix = extract_addr_prefix(q.account_->workchain_, q.account_->id_); + info.shard_id = acc_id_prefix.as_leaf_shard(); + // See LiteQuery::perform_getAccountState + if (block_id.id.workchain != masterchainId) { + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno(); + } else if (block_id.id.seqno != ~0U) { + info.type = QueryInfo::t_mc_seqno; + info.value = block_id.seqno(); + } else { + info.type = QueryInfo::t_simple; + } + }, + [&](const lite_api::liteServer_getOneTransaction& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getTransactions& q) { + AccountIdPrefixFull acc_id_prefix = extract_addr_prefix(q.account_->workchain_, q.account_->id_); + info.shard_id = acc_id_prefix.as_leaf_shard(); + info.type = QueryInfo::t_lt; + info.value = q.lt_; + }, + [&](const lite_api::liteServer_sendMessage& q) { + info.type = QueryInfo::t_simple; + auto r_root = vm::std_boc_deserialize(q.body_); + if (r_root.is_error()) { + return; + } + block::gen::CommonMsgInfo::Record_ext_in_msg_info msg_info; + if (!tlb::unpack_cell_inexact(r_root.ok(), msg_info)) { + return; + } + auto dest_prefix = block::tlb::MsgAddressInt::get_prefix(msg_info.dest); + if (!dest_prefix.is_valid()) { + return; + } + info.shard_id = dest_prefix.as_leaf_shard(); + }, + [&](const lite_api::liteServer_getShardInfo& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getAllShardsInfo& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_lookupBlock& q) { + BlockId block_id = create_block_id_simple(q.id_); + info.shard_id = block_id.shard_full(); + // See LiteQuery::perform_lookupBlock + if (q.mode_ & 1) { + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno; + } else if (q.mode_ == 2) { + info.type = QueryInfo::t_lt; + info.value = q.lt_; + } else if (q.mode_ == 4) { + info.type = QueryInfo::t_utime; + info.value = q.utime_; + } + }, + [&](const lite_api::liteServer_lookupBlockWithProof& q) { + BlockId block_id = create_block_id_simple(q.id_); + info.shard_id = block_id.shard_full(); + // See LiteQuery::perform_lookupBlockWithProof + if (q.mode_ & 1) { + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno; + } else if (q.mode_ == 2) { + info.type = QueryInfo::t_lt; + info.value = q.lt_; + } else if (q.mode_ == 4) { + info.type = QueryInfo::t_utime; + info.value = q.utime_; + } + }, + [&](const lite_api::liteServer_listBlockTransactions& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_listBlockTransactionsExt& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getConfigParams& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getConfigAll& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getBlockProof& q) { + info.shard_id = ShardIdFull{masterchainId}; + BlockIdExt from = create_block_id(q.known_block_); + // See LiteQuery::perform_getBlockProof + if ((q.mode_ & 1) && (q.mode_ & 0x1000)) { + BlockIdExt to = create_block_id(q.target_block_); // target_block is non-null if (mode & 1) + info.type = QueryInfo::t_seqno; + info.value = std::max(from.seqno(), to.seqno()); + } else { + info.type = QueryInfo::t_simple; + } + }, + [&](const lite_api::liteServer_getValidatorStats& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_runSmcMethod& q) { + BlockIdExt block_id = create_block_id(q.id_); + AccountIdPrefixFull acc_id_prefix = extract_addr_prefix(q.account_->workchain_, q.account_->id_); + info.shard_id = acc_id_prefix.as_leaf_shard(); + // See LiteQuery::perform_getAccountState + if (block_id.id.workchain != masterchainId) { + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno(); + } else if (block_id.id.seqno != ~0U) { + info.type = QueryInfo::t_mc_seqno; + info.value = block_id.seqno(); + } else { + info.type = QueryInfo::t_simple; + } + }, + [&](const lite_api::liteServer_getLibraries& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getLibrariesWithProof& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getShardBlockProof& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_nonfinal_getCandidate& q) { /* t_simple */ }, + [&](const lite_api::liteServer_nonfinal_getValidatorGroups& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getOutMsgQueueSizes& q) { + // This query is expected to be removed, as it is not fully compatible with separated liteservers + /* t_simple */ + }, + [&](const lite_api::liteServer_getBlockOutMsgQueueSize& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getDispatchQueueInfo& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getDispatchQueueMessages& q) { from_block_id(q.id_); }, + [&](const auto&) { /* t_simple */ })); + if (info.shard_id.workchain == masterchainId) { + info.shard_id.shard = shardIdAll; + } + if (!info.shard_id.is_valid_ext()) { + info.shard_id = ShardIdFull{masterchainId}; + info.type = QueryInfo::t_simple; + info.value = 0; + } + return info; +} + +bool LiteServerConfig::accepts_query(const QueryInfo& query_info) const { + if (is_full) { + return true; + } + for (const Slice& s : slices) { + if (s.accepts_query(query_info)) { + return true; + } + } + return false; +} + +bool LiteServerConfig::Slice::accepts_query(const QueryInfo& query_info) const { + if (unlimited) { + for (const ShardInfo& shard : shards_from) { + if (shard_intersects(shard.shard_id, query_info.shard_id)) { + return true; + } + } + return false; + } + if (!shards_from.empty()) { + bool from_ok = false; + DCHECK(shards_from[0].shard_id.is_masterchain()); + for (const ShardInfo& shard : shards_from) { + if (shard_intersects(shard.shard_id, query_info.shard_id)) { + switch (query_info.type) { + case QueryInfo::t_simple: + from_ok = true; + break; + case QueryInfo::t_seqno: + from_ok = shard.seqno <= query_info.value; + break; + case QueryInfo::t_utime: + from_ok = shard.utime <= query_info.value; + break; + case QueryInfo::t_lt: + from_ok = shard.lt <= query_info.value; + break; + case QueryInfo::t_mc_seqno: + from_ok = shards_from[0].seqno <= query_info.value; + break; + } + if (from_ok) { + break; + } + } + } + if (!from_ok) { + return false; + } + } + if (!shards_to.empty()) { + bool to_ok = false; + DCHECK(shards_to[0].shard_id.is_masterchain()); + for (const ShardInfo& shard : shards_to) { + if (shard_intersects(shard.shard_id, query_info.shard_id)) { + switch (query_info.type) { + case QueryInfo::t_simple: + break; + case QueryInfo::t_seqno: + to_ok = shard.seqno >= query_info.value; + break; + case QueryInfo::t_utime: + to_ok = shard.utime >= query_info.value; + break; + case QueryInfo::t_lt: + to_ok = shard.lt >= query_info.value; + break; + case QueryInfo::t_mc_seqno: + to_ok = shards_from[0].seqno >= query_info.value; + break; + } + if (to_ok) { + break; + } + } + } + if (!to_ok) { + return false; + } + } + return true; +} + +td::Result> LiteServerConfig::parse_global_config( + const ton_api::liteclient_config_global& config) { + std::vector servers; + for (const auto& f : config.liteservers_) { + LiteServerConfig server; + TRY_STATUS(server.addr.init_host_port(td::IPAddress::ipv4_to_str(f->ip_), f->port_)); + server.adnl_id = adnl::AdnlNodeIdFull{PublicKey{f->id_}}; + server.is_full = true; + servers.push_back(std::move(server)); + } + for (const auto& f : config.liteservers_v2_) { + LiteServerConfig server; + TRY_STATUS(server.addr.init_host_port(td::IPAddress::ipv4_to_str(f->ip_), f->port_)); + server.adnl_id = adnl::AdnlNodeIdFull{PublicKey{f->id_}}; + server.is_full = false; + for (const auto& slice_obj : f->slices_) { + Slice slice; + td::Status S = td::Status::OK(); + downcast_call(*slice_obj, + td::overloaded( + [&](const ton_api::liteserver_descV2_sliceSimple& s) { + slice.unlimited = true; + slice.shards_from.push_back({ShardIdFull{masterchainId}, 0, 0, 0}); + for (const auto& shard_obj : s.shards_) { + ShardIdFull shard_id = create_shard_id(shard_obj); + if (!shard_id.is_valid_ext()) { + S = td::Status::Error(PSTRING() << "invalid shard id " << shard_id.to_str()); + break; + } + if (!shard_id.is_masterchain()) { + slice.shards_from.push_back({shard_id, 0, 0, 0}); + } + } + }, + [&](const ton_api::liteserver_descV2_sliceTimed& s) { + auto parse_shards = + [](const std::vector>& shard_objs, + std::vector& shards) -> td::Status { + if (shard_objs.empty()) { + return td::Status::OK(); + } + size_t i = 0; + int mc_idx = -1; + for (const auto& shard_obj : shard_objs) { + ShardIdFull shard_id = create_shard_id(shard_obj->shard_id_); + if (!shard_id.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard id " << shard_id.to_str()); + } + if (shard_id.is_masterchain()) { + shard_id = ShardIdFull{masterchainId}; + if (mc_idx != -1) { + return td::Status::Error("duplicate masterchain shard in sliceTimed"); + } + mc_idx = (int)i; + } + shards.push_back({shard_id, (BlockSeqno)shard_obj->seqno_, (UnixTime)shard_obj->utime_, + (LogicalTime)shard_obj->lt_}); + ++i; + } + if (mc_idx == -1) { + return td::Status::Error("no masterchain shard in sliceTimed"); + } + std::swap(shards[0], shards[mc_idx]); + return td::Status::OK(); + }; + S = parse_shards(s.shards_from_, slice.shards_from); + if (S.is_ok()) { + S = parse_shards(s.shards_to_, slice.shards_to); + } + if (S.is_ok() && slice.shards_from.empty() && slice.shards_to.empty()) { + S = td::Status::Error("shards_from and shards_to are both empty"); + } + })); + TRY_STATUS(std::move(S)); + server.slices.push_back(slice); + } + + servers.push_back(std::move(server)); + } + return servers; +} + +} // namespace liteclient \ No newline at end of file diff --git a/lite-client/query-utils.hpp b/lite-client/query-utils.hpp new file mode 100644 index 00000000..28500e26 --- /dev/null +++ b/lite-client/query-utils.hpp @@ -0,0 +1,89 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "ton/ton-types.h" +#include "auto/tl/lite_api.h" +#include "td/utils/port/IPAddress.h" +#include "adnl/adnl-node-id.hpp" + +namespace liteclient { + +struct QueryInfo { + enum Type { t_simple, t_seqno, t_utime, t_lt, t_mc_seqno }; + int query_id = 0; + ton::ShardIdFull shard_id{ton::masterchainId}; + Type type = t_simple; + td::uint64 value = 0; + /* Query types and examples: + * t_simple - query to the recent blocks in a shard, or general info. value = 0. + * getTime, getMasterchainInfo (shard_id = masterchain) + * sendMessage + * getAccountState, runSmcMethod - when no block is given + * t_seqno - query to block with seqno in a shard. value = seqno. + * lookupBlock by seqno + * getBlock, getBlockHeader + * getAccountState, runSmcMethod - when shard block is given + * t_utime - query to a block with given unixtime in a shard. value = utime. + * lookupBlock by utime + * t_lt - query to a block with given lt in a shard. value = lt. + * lookupBlock by lt + * getTransactions + * t_mc_seqno - query to a block in a shard, masterchain seqno is given. value = mc_seqno. + * getAccountState, runSmcMethod - when mc block is given + */ + + std::string to_str() const; +}; + +QueryInfo get_query_info(td::Slice data); +QueryInfo get_query_info(const ton::lite_api::Function& f); + +struct LiteServerConfig { + private: + struct ShardInfo { + ton::ShardIdFull shard_id; + ton::BlockSeqno seqno; + ton::UnixTime utime; + ton::LogicalTime lt; + }; + + struct Slice { + std::vector shards_from, shards_to; + bool unlimited = false; + + bool accepts_query(const QueryInfo& query_info) const; + }; + + bool is_full = false; + std::vector slices; + + public: + ton::adnl::AdnlNodeIdFull adnl_id; + td::IPAddress addr; + + LiteServerConfig() = default; + LiteServerConfig(ton::adnl::AdnlNodeIdFull adnl_id, td::IPAddress addr) + : is_full(true), adnl_id(adnl_id), addr(addr) { + } + + bool accepts_query(const QueryInfo& query_info) const; + + static td::Result> parse_global_config( + const ton::ton_api::liteclient_config_global& config); +}; + +} // namespace liteclient diff --git a/memprof/CMakeLists.txt b/memprof/CMakeLists.txt index 8559c4d9..2ccf11df 100644 --- a/memprof/CMakeLists.txt +++ b/memprof/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) set(MEMPROF_SOURCE memprof/memprof.cpp diff --git a/overlay/CMakeLists.txt b/overlay/CMakeLists.txt index 7adc0584..ab9722a6 100644 --- a/overlay/CMakeLists.txt +++ b/overlay/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) diff --git a/overlay/overlay-broadcast.cpp b/overlay/overlay-broadcast.cpp index 03991b76..615b3e7c 100644 --- a/overlay/overlay-broadcast.cpp +++ b/overlay/overlay-broadcast.cpp @@ -68,7 +68,7 @@ td::Status BroadcastSimple::run_checks() { td::Status BroadcastSimple::distribute() { auto B = serialize(); - auto nodes = overlay_->get_neighbours(3); + auto nodes = overlay_->get_neighbours(overlay_->propagate_broadcast_to()); auto manager = overlay_->overlay_manager(); for (auto &n : nodes) { @@ -115,7 +115,8 @@ td::Status BroadcastSimple::run() { return run_continue(); } -td::Status BroadcastSimple::create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, tl_object_ptr broadcast) { +td::Status BroadcastSimple::create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, + tl_object_ptr broadcast) { auto src = PublicKey{broadcast->src_}; auto data_hash = sha256_bits256(broadcast->data_.as_slice()); auto broadcast_hash = compute_broadcast_id(src, data_hash, broadcast->flags_); diff --git a/overlay/overlay-fec-broadcast.cpp b/overlay/overlay-fec-broadcast.cpp index aed5248b..5a0ad10d 100644 --- a/overlay/overlay-fec-broadcast.cpp +++ b/overlay/overlay-fec-broadcast.cpp @@ -78,7 +78,6 @@ td::Status OverlayFecBroadcastPart::check_signature() { } td::Status OverlayFecBroadcastPart::run_checks() { - TRY_STATUS(check_time()); TRY_STATUS(check_duplicate()); TRY_STATUS(check_source()); @@ -94,14 +93,17 @@ void BroadcastFec::broadcast_checked(td::Result R) { overlay_->deliver_broadcast(get_source().compute_short_id(), data_.clone()); auto manager = overlay_->overlay_manager(); while (!parts_.empty()) { - distribute_part(parts_.begin()->first); + distribute_part(parts_.begin()->first); } + + is_checked_ = true; } // Do we need status here?? -td::Status BroadcastFec::distribute_part(td::uint32 seqno) { +td::Status BroadcastFec::distribute_part(td::uint32 seqno) { auto i = parts_.find(seqno); if (i == parts_.end()) { + VLOG(OVERLAY_WARNING) << "not distibuting empty part " << seqno; // should not get here return td::Status::OK(); } @@ -110,7 +112,7 @@ td::Status BroadcastFec::distribute_part(td::uint32 seqno) { td::BufferSlice data_short = std::move(tls.first); td::BufferSlice data = std::move(tls.second); - auto nodes = overlay_->get_neighbours(5); + auto nodes = overlay_->get_neighbours(overlay_->propagate_broadcast_to()); auto manager = overlay_->overlay_manager(); for (auto &n : nodes) { @@ -132,7 +134,6 @@ td::Status BroadcastFec::distribute_part(td::uint32 seqno) { } td::Status OverlayFecBroadcastPart::apply() { - if (!bcast_) { bcast_ = overlay_->get_fec_broadcast(broadcast_hash_); } @@ -165,16 +166,20 @@ td::Status OverlayFecBroadcastPart::apply() { return S; } } else { - if(untrusted_) { + if (untrusted_) { auto P = td::PromiseCreator::lambda( - [id = broadcast_hash_, overlay_id = actor_id(overlay_)](td::Result RR) mutable { - td::actor::send_closure(std::move(overlay_id), &OverlayImpl::broadcast_checked, id, std::move(RR)); - }); + [id = broadcast_hash_, overlay_id = actor_id(overlay_)](td::Result RR) mutable { + td::actor::send_closure(std::move(overlay_id), &OverlayImpl::broadcast_checked, id, std::move(RR)); + }); overlay_->check_broadcast(bcast_->get_source().compute_short_id(), R.move_as_ok(), std::move(P)); } else { overlay_->deliver_broadcast(bcast_->get_source().compute_short_id(), R.move_as_ok()); } } + } else { + bcast_->set_overlay(overlay_); + bcast_->set_src_peer_id(src_peer_id_); + TRY_STATUS(bcast_->add_part(seqno_, data_.clone(), export_serialized_short(), export_serialized())); } return td::Status::OK(); } @@ -304,7 +309,8 @@ td::Status OverlayFecBroadcastPart::create_new(OverlayImpl *overlay, td::actor:: auto B = std::make_unique( broadcast_hash, part_hash, PublicKey{}, overlay->get_certificate(local_id), data_hash, size, flags, - part_data_hash, std::move(part), seqno, std::move(fec_type), date, td::BufferSlice{}, false, nullptr, overlay, adnl::AdnlNodeIdShort::zero()); + part_data_hash, std::move(part), seqno, std::move(fec_type), date, td::BufferSlice{}, false, nullptr, overlay, + adnl::AdnlNodeIdShort::zero()); auto to_sign = B->to_sign(); auto P = td::PromiseCreator::lambda( diff --git a/overlay/overlay-fec-broadcast.hpp b/overlay/overlay-fec-broadcast.hpp index 612af22f..85de648e 100644 --- a/overlay/overlay-fec-broadcast.hpp +++ b/overlay/overlay-fec-broadcast.hpp @@ -82,15 +82,15 @@ class BroadcastFec : public td::ListNode { } } - td::Status add_part(td::uint32 seqno, td::BufferSlice data, - td::BufferSlice serialized_fec_part_short, + td::Status add_part(td::uint32 seqno, td::BufferSlice data, td::BufferSlice serialized_fec_part_short, td::BufferSlice serialized_fec_part) { - CHECK(decoder_); - td::fec::Symbol s; - s.id = seqno; - s.data = std::move(data); + if (decoder_) { + td::fec::Symbol s; + s.id = seqno; + s.data = std::move(data); - decoder_->add_symbol(std::move(s)); + decoder_->add_symbol(std::move(s)); + } parts_[seqno] = std::pair(std::move(serialized_fec_part_short), std::move(serialized_fec_part)); @@ -200,8 +200,13 @@ class BroadcastFec : public td::ListNode { td::Status distribute_part(td::uint32 seqno); + bool is_checked() const { + return is_checked_; + } + private: bool ready_ = false; + bool is_checked_ = false; Overlay::BroadcastHash hash_; Overlay::BroadcastDataHash data_hash_; @@ -281,7 +286,7 @@ class OverlayFecBroadcastPart : public td::ListNode { , signature_(std::move(signature)) , is_short_(is_short) , bcast_(bcast) - , overlay_(overlay) + , overlay_(overlay) , src_peer_id_(src_peer_id) { } @@ -300,7 +305,7 @@ class OverlayFecBroadcastPart : public td::ListNode { signature_ = std::move(signature); } void update_overlay(OverlayImpl *overlay); - + tl_object_ptr export_tl(); tl_object_ptr export_tl_short(); td::BufferSlice export_serialized(); @@ -310,14 +315,16 @@ class OverlayFecBroadcastPart : public td::ListNode { td::Status run() { TRY_STATUS(run_checks()); TRY_STATUS(apply()); - if(!untrusted_) { + if (!untrusted_ || bcast_->is_checked()) { TRY_STATUS(distribute()); } return td::Status::OK(); } - static td::Status create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, tl_object_ptr broadcast); - static td::Status create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, tl_object_ptr broadcast); + static td::Status create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, + tl_object_ptr broadcast); + static td::Status create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, + tl_object_ptr broadcast); static td::Status create_new(OverlayImpl *overlay, td::actor::ActorId overlay_actor_id, PublicKeyHash local_id, Overlay::BroadcastDataHash data_hash, td::uint32 size, td::uint32 flags, td::BufferSlice part, td::uint32 seqno, fec::FecType fec_type, diff --git a/overlay/overlay-fec.cpp b/overlay/overlay-fec.cpp index b29fce22..817d3b7c 100644 --- a/overlay/overlay-fec.cpp +++ b/overlay/overlay-fec.cpp @@ -32,7 +32,7 @@ void OverlayOutboundFecBroadcast::alarm() { fec_type_.size(), flags_, std::move(X.data), X.id, fec_type_, date_); } - alarm_timestamp() = td::Timestamp::in(0.010); + alarm_timestamp() = td::Timestamp::in(delay_); if (seqno_ >= to_send_) { stop(); @@ -46,8 +46,9 @@ void OverlayOutboundFecBroadcast::start_up() { OverlayOutboundFecBroadcast::OverlayOutboundFecBroadcast(td::BufferSlice data, td::uint32 flags, td::actor::ActorId overlay, - PublicKeyHash local_id) + PublicKeyHash local_id, double speed_multiplier) : flags_(flags) { + delay_ /= speed_multiplier; CHECK(data.size() <= (1 << 27)); local_id_ = local_id; overlay_ = std::move(overlay); @@ -63,9 +64,10 @@ OverlayOutboundFecBroadcast::OverlayOutboundFecBroadcast(td::BufferSlice data, t } td::actor::ActorId OverlayOutboundFecBroadcast::create( - td::BufferSlice data, td::uint32 flags, td::actor::ActorId overlay, PublicKeyHash local_id) { - return td::actor::create_actor(td::actor::ActorOptions().with_name("bcast"), - std::move(data), flags, overlay, local_id) + td::BufferSlice data, td::uint32 flags, td::actor::ActorId overlay, PublicKeyHash local_id, + double speed_multiplier) { + return td::actor::create_actor( + td::actor::ActorOptions().with_name("bcast"), std::move(data), flags, overlay, local_id, speed_multiplier) .release(); } diff --git a/overlay/overlay-fec.hpp b/overlay/overlay-fec.hpp index a9cc3634..b72e830e 100644 --- a/overlay/overlay-fec.hpp +++ b/overlay/overlay-fec.hpp @@ -37,6 +37,7 @@ class OverlayOutboundFecBroadcast : public td::actor::Actor { PublicKeyHash local_id_; Overlay::BroadcastDataHash data_hash_; td::uint32 flags_ = 0; + double delay_ = 0.010; td::int32 date_; std::unique_ptr encoder_; td::actor::ActorId overlay_; @@ -45,9 +46,9 @@ class OverlayOutboundFecBroadcast : public td::actor::Actor { public: static td::actor::ActorId create(td::BufferSlice data, td::uint32 flags, td::actor::ActorId overlay, - PublicKeyHash local_id); + PublicKeyHash local_id, double speed_multiplier = 1.0); OverlayOutboundFecBroadcast(td::BufferSlice data, td::uint32 flags, td::actor::ActorId overlay, - PublicKeyHash local_id); + PublicKeyHash local_id, double speed_multiplier = 1.0); void alarm() override; void start_up() override; diff --git a/overlay/overlay-id.hpp b/overlay/overlay-id.hpp index c22b0faa..2625773b 100644 --- a/overlay/overlay-id.hpp +++ b/overlay/overlay-id.hpp @@ -21,8 +21,14 @@ #include "auto/tl/ton_api.h" #include "adnl/adnl-node-id.hpp" #include "overlay/overlays.h" +#include "td/utils/SharedSlice.h" +#include "td/utils/buffer.h" #include "td/utils/overloaded.h" #include "keys/encryptor.h" +#include "td/utils/port/StdStreams.h" +#include "td/utils/unique_ptr.h" +#include +#include namespace ton { @@ -30,18 +36,30 @@ namespace overlay { class OverlayNode { public: - explicit OverlayNode(adnl::AdnlNodeIdShort self_id, OverlayIdShort overlay) { + explicit OverlayNode(adnl::AdnlNodeIdShort self_id, OverlayIdShort overlay, td::uint32 flags) { source_ = self_id; overlay_ = overlay; + flags_ = flags; version_ = static_cast(td::Clocks::system()); } static td::Result create(const tl_object_ptr &node) { TRY_RESULT(source, adnl::AdnlNodeIdFull::create(node->id_)); - return OverlayNode{source, OverlayIdShort{node->overlay_}, node->version_, node->signature_.as_slice()}; + return OverlayNode{source, OverlayIdShort{node->overlay_}, 0, node->version_, node->signature_.as_slice()}; } - OverlayNode(td::Variant source, OverlayIdShort overlay, + static td::Result create(const tl_object_ptr &node) { + TRY_RESULT(source, adnl::AdnlNodeIdFull::create(node->id_)); + auto res = OverlayNode{source, OverlayIdShort{node->overlay_}, (td::uint32)node->flags_, node->version_, + node->signature_.as_slice()}; + res.update_certificate(OverlayMemberCertificate(node->certificate_.get())); + return res; + } + OverlayNode(td::Variant source, OverlayIdShort overlay, td::uint32 flags, td::int32 version, td::Slice signature) - : source_(std::move(source)), overlay_(overlay), version_(version), signature_(td::SharedSlice(signature)) { + : source_(std::move(source)) + , overlay_(overlay) + , flags_(flags) + , version_(version) + , signature_(td::SharedSlice(signature)) { } OverlayNode(td::Variant source, OverlayIdShort overlay, td::int32 version, td::SharedSlice signature) @@ -64,10 +82,17 @@ class OverlayNode { } td::BufferSlice to_sign() const { - auto obj = create_tl_object(nullptr, overlay_.tl(), version_); - source_.visit(td::overloaded([&](const adnl::AdnlNodeIdShort &id) { obj->id_ = id.tl(); }, - [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.compute_short_id().tl(); })); - return serialize_tl_object(obj, true); + if (flags_ == 0) { + auto obj = create_tl_object(nullptr, overlay_.tl(), version_); + source_.visit(td::overloaded([&](const adnl::AdnlNodeIdShort &id) { obj->id_ = id.tl(); }, + [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.compute_short_id().tl(); })); + return serialize_tl_object(obj, true); + } else { + auto obj = create_tl_object(nullptr, overlay_.tl(), flags_, version_); + source_.visit(td::overloaded([&](const adnl::AdnlNodeIdShort &id) { obj->id_ = id.tl(); }, + [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.compute_short_id().tl(); })); + return serialize_tl_object(obj, true); + } } void update_adnl_id(adnl::AdnlNodeIdFull node_id) { source_ = node_id; @@ -81,6 +106,9 @@ class OverlayNode { td::int32 version() const { return version_; } + td::uint32 flags() const { + return flags_; + } td::BufferSlice signature() const { return signature_.clone_as_buffer_slice(); } @@ -103,15 +131,69 @@ class OverlayNode { [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.tl(); })); return obj; } + tl_object_ptr tl_v2() const { + tl_object_ptr cert; + if (cert_ && !cert_->empty()) { + cert = cert_->tl(); + } else { + cert = create_tl_object(); + } + auto obj = create_tl_object(nullptr, overlay_.tl(), flags_, version_, + signature_.clone_as_buffer_slice(), std::move(cert)); + source_.visit(td::overloaded([&](const adnl::AdnlNodeIdShort &id) { UNREACHABLE(); }, + [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.tl(); })); + return obj; + } OverlayNode clone() const { - return OverlayNode{source_, overlay_, version_, signature_.clone()}; + auto res = OverlayNode{source_, overlay_, version_, signature_.clone()}; + if (cert_) { + res.cert_ = td::make_unique(*cert_); + } + return res; + } + + const OverlayMemberCertificate *certificate() const { + if (cert_) { + return cert_.get(); + } + return &empty_certificate_; + } + + void update_certificate(OverlayMemberCertificate cert) { + if (!cert_ || cert_->empty() || cert_->is_expired() || cert.is_newer(*cert_)) { + cert_ = td::make_unique(std::move(cert)); + } + } + + void update(OverlayNode from) { + if (version_ < from.version_) { + source_ = from.source_; + overlay_ = from.overlay_; + flags_ = from.flags_; + version_ = from.version_; + signature_ = from.signature_.clone(); + } + if (from.cert_ && !from.cert_->empty()) { + update_certificate(std::move(*from.cert_)); + } + } + + void clear_certificate() { + cert_ = nullptr; + } + + bool has_full_id() const { + return source_.get_offset() == source_.offset(); } private: td::Variant source_; OverlayIdShort overlay_; + td::uint32 flags_; td::int32 version_; + td::unique_ptr cert_; td::SharedSlice signature_; + static const OverlayMemberCertificate empty_certificate_; }; } // namespace overlay diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index 43192190..f24c6cbc 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -18,6 +18,7 @@ */ #include "overlay-manager.h" #include "auto/tl/ton_api.h" +#include "auto/tl/ton_api.hpp" #include "overlay.h" #include "adnl/utils.hpp" @@ -28,9 +29,9 @@ #include "td/db/RocksDb.h" #include "td/utils/Status.h" +#include "td/utils/buffer.h" #include "td/utils/overloaded.h" -#include "keys/encryptor.h" #include "td/utils/port/Poll.h" #include @@ -42,13 +43,13 @@ void OverlayManager::update_dht_node(td::actor::ActorId dht) { dht_node_ = dht; for (auto &X : overlays_) { for (auto &Y : X.second) { - td::actor::send_closure(Y.second, &Overlay::update_dht_node, dht); + td::actor::send_closure(Y.second.overlay, &Overlay::update_dht_node, dht); } } } void OverlayManager::register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, - td::actor::ActorOwn overlay) { + OverlayMemberCertificate cert, td::actor::ActorOwn overlay) { auto it = overlays_.find(local_id); VLOG(OVERLAY_INFO) << this << ": registering overlay " << overlay_id << "@" << local_id; if (it == overlays_.end()) { @@ -58,19 +59,37 @@ void OverlayManager::register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdS td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id, adnl::Adnl::int_to_bytestring(ton_api::overlay_query::ID), std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id, + adnl::Adnl::int_to_bytestring(ton_api::overlay_messageWithExtra::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id, + adnl::Adnl::int_to_bytestring(ton_api::overlay_queryWithExtra::ID), + std::make_unique(actor_id(this))); } - overlays_[local_id][overlay_id] = std::move(overlay); + overlays_[local_id][overlay_id] = OverlayDescription{std::move(overlay), std::move(cert)}; - auto P = td::PromiseCreator::lambda([id = overlays_[local_id][overlay_id].get()](td::Result R) { - R.ensure(); - auto value = R.move_as_ok(); - if (value.status == td::KeyValue::GetStatus::Ok) { - auto F = fetch_tl_object(std::move(value.value), true); - F.ensure(); - auto nodes = std::move(F.move_as_ok()->nodes_); - td::actor::send_closure(id, &Overlay::receive_nodes_from_db, std::move(nodes)); - } - }); + if (!with_db_) { + return; + } + auto P = + td::PromiseCreator::lambda([id = overlays_[local_id][overlay_id].overlay.get()](td::Result R) { + R.ensure(); + auto value = R.move_as_ok(); + if (value.status == td::KeyValue::GetStatus::Ok) { + auto F = fetch_tl_object(std::move(value.value), true); + F.ensure(); + ton_api::downcast_call( + *F.move_as_ok(), td::overloaded( + [&](ton_api::overlay_db_nodes &V) { + auto nodes = std::move(V.nodes_); + td::actor::send_closure(id, &Overlay::receive_nodes_from_db, std::move(nodes)); + }, + [&](ton_api::overlay_db_nodesV2 &V) { + auto nodes = std::move(V.nodes_); + td::actor::send_closure(id, &Overlay::receive_nodes_from_db_v2, std::move(nodes)); + })); + } + }); auto key = create_hash_tl_object(local_id.bits256_value(), overlay_id.bits256_value()); db_.get(key, std::move(P)); } @@ -84,6 +103,10 @@ void OverlayManager::delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdSho adnl::Adnl::int_to_bytestring(ton_api::overlay_message::ID)); td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id, adnl::Adnl::int_to_bytestring(ton_api::overlay_query::ID)); + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id, + adnl::Adnl::int_to_bytestring(ton_api::overlay_messageWithExtra::ID)); + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id, + adnl::Adnl::int_to_bytestring(ton_api::overlay_queryWithExtra::ID)); overlays_.erase(it); } } @@ -93,81 +116,128 @@ void OverlayManager::create_public_overlay(adnl::AdnlNodeIdShort local_id, Overl std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) { create_public_overlay_ex(local_id, std::move(overlay_id), std::move(callback), std::move(rules), std::move(scope), - true); + {}); } void OverlayManager::create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, - td::string scope, bool announce_self) { + td::string scope, OverlayOptions opts) { CHECK(!dht_node_.empty()); auto id = overlay_id.compute_short_id(); - register_overlay(local_id, id, - Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), - std::move(callback), std::move(rules), scope, announce_self)); + register_overlay(local_id, id, OverlayMemberCertificate{}, + Overlay::create_public(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), + std::move(callback), std::move(rules), scope, std::move(opts))); } void OverlayManager::create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, - std::unique_ptr callback, OverlayPrivacyRules rules) { + std::unique_ptr callback, OverlayPrivacyRules rules, + std::string scope) { + create_private_overlay_ex(local_id, std::move(overlay_id), std::move(nodes), std::move(callback), std::move(rules), + std::move(scope), {}); +} + +void OverlayManager::create_private_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, + std::unique_ptr callback, OverlayPrivacyRules rules, + std::string scope, OverlayOptions opts) { auto id = overlay_id.compute_short_id(); - register_overlay(local_id, id, - Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), - std::move(nodes), std::move(callback), std::move(rules))); + register_overlay(local_id, id, OverlayMemberCertificate{}, + Overlay::create_private(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), + std::move(nodes), std::move(callback), std::move(rules), std::move(scope), + std::move(opts))); +} + +void OverlayManager::create_semiprivate_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, + std::vector root_public_keys, + OverlayMemberCertificate certificate, + std::unique_ptr callback, OverlayPrivacyRules rules, + td::string scope, OverlayOptions opts) { + auto id = overlay_id.compute_short_id(); + register_overlay( + local_id, id, certificate, + Overlay::create_semiprivate(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), + std::move(nodes), std::move(root_public_keys), certificate, std::move(callback), + std::move(rules), std::move(scope), std::move(opts))); } void OverlayManager::receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) { - auto R = fetch_tl_prefix(data, true); - - if (R.is_error()) { - VLOG(OVERLAY_WARNING) << this << ": can not parse overlay message: " << R.move_as_error(); - return; + OverlayIdShort overlay_id; + tl_object_ptr extra; + auto R = fetch_tl_prefix(data, true); + if (R.is_ok()) { + overlay_id = OverlayIdShort{R.ok()->overlay_}; + extra = std::move(R.ok()->extra_); + } else { + auto R2 = fetch_tl_prefix(data, true); + if (R2.is_ok()) { + overlay_id = OverlayIdShort{R2.ok()->overlay_}; + } else { + VLOG(OVERLAY_WARNING) << this << ": can not parse overlay message [" << src << "->" << dst + << "]: " << R2.move_as_error(); + return; + } } - auto M = R.move_as_ok(); - auto it = overlays_.find(dst); if (it == overlays_.end()) { - VLOG(OVERLAY_NOTICE) << this << ": message to unknown overlay " << M->overlay_ << "@" << dst; + VLOG(OVERLAY_NOTICE) << this << ": message to unknown overlay " << overlay_id << "@" << dst; return; } - auto it2 = it->second.find(OverlayIdShort{M->overlay_}); + auto it2 = it->second.find(overlay_id); if (it2 == it->second.end()) { - VLOG(OVERLAY_NOTICE) << this << ": message to localid is not in overlay " << M->overlay_ << "@" << dst; + VLOG(OVERLAY_NOTICE) << this << ": message to localid is not in overlay " << overlay_id << "@" << dst; return; } - td::actor::send_closure(it2->second, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), false); - td::actor::send_closure(it2->second, &Overlay::receive_message, src, std::move(data)); + td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_in_ctr, src, data.size(), false, false); + td::actor::send_closure(it2->second.overlay, &Overlay::receive_message, src, std::move(extra), std::move(data)); } void OverlayManager::receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { - auto R = fetch_tl_prefix(data, true); - - if (R.is_error()) { - VLOG(OVERLAY_WARNING) << this << ": can not parse overlay query [" << src << "->" << dst - << "]: " << R.move_as_error(); - promise.set_error(td::Status::Error(ErrorCode::protoviolation, "bad overlay query header")); - return; + OverlayIdShort overlay_id; + tl_object_ptr extra; + auto R = fetch_tl_prefix(data, true); + if (R.is_ok()) { + overlay_id = OverlayIdShort{R.ok()->overlay_}; + extra = std::move(R.ok()->extra_); + } else { + auto R2 = fetch_tl_prefix(data, true); + if (R2.is_ok()) { + overlay_id = OverlayIdShort{R2.ok()->overlay_}; + } else { + VLOG(OVERLAY_WARNING) << this << ": can not parse overlay query [" << src << "->" << dst + << "]: " << R2.move_as_error(); + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "bad overlay query header")); + return; + } } - auto M = R.move_as_ok(); - auto it = overlays_.find(dst); if (it == overlays_.end()) { - VLOG(OVERLAY_NOTICE) << this << ": query to unknown overlay " << M->overlay_ << "@" << dst << " from " << src; + VLOG(OVERLAY_NOTICE) << this << ": query to unknown overlay " << overlay_id << "@" << dst << " from " << src; promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad local_id " << dst)); return; } - auto it2 = it->second.find(OverlayIdShort{M->overlay_}); + auto it2 = it->second.find(overlay_id); if (it2 == it->second.end()) { - VLOG(OVERLAY_NOTICE) << this << ": query to localid not in overlay " << M->overlay_ << "@" << dst << " from " << src; - promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad overlay_id " << M->overlay_)); + VLOG(OVERLAY_NOTICE) << this << ": query to localid not in overlay " << overlay_id << "@" << dst << " from " << src; + promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad overlay_id " << overlay_id)); return; } - td::actor::send_closure(it2->second, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), true); - td::actor::send_closure(it2->second, &Overlay::receive_query, src, std::move(data), std::move(promise)); + td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_in_ctr, src, data.size(), true, false); + promise = [overlay = it2->second.overlay.get(), promise = std::move(promise), + src](td::Result R) mutable { + if (R.is_ok()) { + td::actor::send_closure(overlay, &Overlay::update_throughput_out_ctr, src, R.ok().size(), false, true); + } + promise.set_result(std::move(R)); + }; + td::actor::send_closure(it2->second.overlay, &Overlay::receive_query, src, std::move(extra), std::move(data), + std::move(promise)); } void OverlayManager::send_query_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, @@ -175,35 +245,70 @@ void OverlayManager::send_query_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdS td::BufferSlice query, td::uint64 max_answer_size, td::actor::ActorId via) { CHECK(query.size() <= adnl::Adnl::huge_packet_max_size()); - + + auto extra = create_tl_object(); + extra->flags_ = 0; + auto it = overlays_.find(src); if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::update_throughput_out_ctr, dst, (td::uint32)query.size(), true); + td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_out_ctr, dst, query.size(), true, false); + promise = [overlay = it2->second.overlay.get(), promise = std::move(promise), + dst](td::Result R) mutable { + if (R.is_ok()) { + td::actor::send_closure(overlay, &Overlay::update_throughput_in_ctr, dst, R.ok().size(), false, true); + } + promise.set_result(std::move(R)); + }; + if (!it2->second.member_certificate.empty()) { + extra->flags_ |= 1; + extra->certificate_ = it2->second.member_certificate.tl(); + } } } - - td::actor::send_closure( - via, &adnl::AdnlSenderInterface::send_query_ex, src, dst, std::move(name), std::move(promise), timeout, - create_serialize_tl_object_suffix(query.as_slice(), overlay_id.tl()), max_answer_size); + + auto extra_flags = extra->flags_; + td::BufferSlice serialized_query = + (extra_flags ? create_serialize_tl_object_suffix( + query.as_slice(), overlay_id.tl(), std::move(extra)) + : create_serialize_tl_object_suffix(query.as_slice(), overlay_id.tl())); + + td::actor::send_closure(via, &adnl::AdnlSenderInterface::send_query_ex, src, dst, std::move(name), std::move(promise), + timeout, std::move(serialized_query), max_answer_size); } void OverlayManager::send_message_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, td::BufferSlice object, td::actor::ActorId via) { CHECK(object.size() <= adnl::Adnl::huge_packet_max_size()); - + + auto extra = create_tl_object(); + extra->flags_ = 0; + auto it = overlays_.find(src); if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::update_throughput_out_ctr, dst, (td::uint32)object.size(), false); + td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_out_ctr, dst, object.size(), false, + false); + if (!it2->second.member_certificate.empty()) { + // do not send certificate here, we hope that all our neighbours already know of out certificate + // we send it every second to some random nodes. Here we don't want to increase the size of the message + if (false) { + extra->flags_ |= 1; + extra->certificate_ = it2->second.member_certificate.tl(); + } + } } } - - td::actor::send_closure( - via, &adnl::AdnlSenderInterface::send_message, src, dst, - create_serialize_tl_object_suffix(object.as_slice(), overlay_id.tl())); + + auto extra_flags = extra->flags_; + td::BufferSlice serialized_message = + (extra_flags ? create_serialize_tl_object_suffix( + object.as_slice(), overlay_id.tl(), std::move(extra)) + : create_serialize_tl_object_suffix(object.as_slice(), overlay_id.tl())); + + td::actor::send_closure(via, &adnl::AdnlSenderInterface::send_message, src, dst, std::move(serialized_message)); } void OverlayManager::send_broadcast(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, td::BufferSlice object) { @@ -217,7 +322,7 @@ void OverlayManager::send_broadcast_ex(adnl::AdnlNodeIdShort local_id, OverlayId if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::send_broadcast, send_as, flags, std::move(object)); + td::actor::send_closure(it2->second.overlay, &Overlay::send_broadcast, send_as, flags, std::move(object)); } } } @@ -234,7 +339,7 @@ void OverlayManager::send_broadcast_fec_ex(adnl::AdnlNodeIdShort local_id, Overl if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::send_broadcast_fec, send_as, flags, std::move(object)); + td::actor::send_closure(it2->second.overlay, &Overlay::send_broadcast_fec, send_as, flags, std::move(object)); } } } @@ -245,7 +350,7 @@ void OverlayManager::set_privacy_rules(adnl::AdnlNodeIdShort local_id, OverlayId if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::set_privacy_rules, std::move(rules)); + td::actor::send_closure(it2->second.overlay, &Overlay::set_privacy_rules, std::move(rules)); } } } @@ -256,7 +361,34 @@ void OverlayManager::update_certificate(adnl::AdnlNodeIdShort local_id, OverlayI if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::add_certificate, key, std::move(cert)); + td::actor::send_closure(it2->second.overlay, &Overlay::add_certificate, key, std::move(cert)); + } + } +} + +void OverlayManager::update_member_certificate(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + OverlayMemberCertificate certificate) { + auto it = overlays_.find(local_id); + if (it != overlays_.end()) { + auto it2 = it->second.find(overlay_id); + if (it2 != it->second.end()) { + it2->second.member_certificate = certificate; + td::actor::send_closure(it2->second.overlay, &Overlay::update_member_certificate, certificate); + } + } +} + +void OverlayManager::update_root_member_list(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + std::vector nodes, + std::vector root_public_keys, + OverlayMemberCertificate certificate) { + auto it = overlays_.find(local_id); + if (it != overlays_.end()) { + auto it2 = it->second.find(overlay_id); + if (it2 != it->second.end()) { + it2->second.member_certificate = certificate; + td::actor::send_closure(it2->second.overlay, &Overlay::update_root_member_list, std::move(nodes), + std::move(root_public_keys), std::move(certificate)); } } } @@ -265,12 +397,16 @@ void OverlayManager::get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, Ov td::uint32 max_peers, td::Promise> promise) { auto it = overlays_.find(local_id); - if (it != overlays_.end()) { - auto it2 = it->second.find(overlay_id); - if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::get_overlay_random_peers, max_peers, std::move(promise)); - } + if (it == overlays_.end()) { + promise.set_error(td::Status::Error(PSTRING() << "no such local id " << local_id)); + return; } + auto it2 = it->second.find(overlay_id); + if (it2 == it->second.end()) { + promise.set_error(td::Status::Error(PSTRING() << "no such overlay " << overlay_id)); + return; + } + td::actor::send_closure(it2->second.overlay, &Overlay::get_overlay_random_peers, max_peers, std::move(promise)); } td::actor::ActorOwn Overlays::create(std::string db_root, td::actor::ActorId keyring, @@ -284,13 +420,19 @@ OverlayManager::OverlayManager(std::string db_root, td::actor::ActorId kv = - std::make_shared(td::RocksDb::open(PSTRING() << db_root_ << "/overlays").move_as_ok()); - db_ = DbType{std::move(kv)}; + if (!db_root_.empty()) { + with_db_ = true; + std::shared_ptr kv = + std::make_shared(td::RocksDb::open(PSTRING() << db_root_ << "/overlays").move_as_ok()); + db_ = DbType{std::move(kv)}; + } } void OverlayManager::save_to_db(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, std::vector nodes) { + if (!with_db_) { + return; + } std::vector> nodes_vec; for (auto &n : nodes) { nodes_vec.push_back(n.tl()); @@ -333,7 +475,7 @@ void OverlayManager::get_stats(td::Promise> R) { if (R.is_ok()) { td::actor::send_closure(act, &Cb::receive_answer, R.move_as_ok()); @@ -347,6 +489,19 @@ void OverlayManager::get_stats(td::Promisesecond.find(overlay); + if (it2 == it->second.end()) { + return; + } + td::actor::send_closure(it2->second.overlay, &Overlay::forget_peer, peer_id); +} + Certificate::Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, td::BufferSlice signature) : issued_by_(issued_by) @@ -418,7 +573,7 @@ td::Result> Certificate::create(tl_object_ptr max_size_) { return BroadcastCheckResult::Forbidden; } @@ -429,16 +584,16 @@ BroadcastCheckResult Certificate::check(PublicKeyHash node, OverlayIdShort overl return BroadcastCheckResult::Forbidden; } - auto R1 = issued_by_.get().create_encryptor(); - if (R1.is_error()) { - return BroadcastCheckResult::Forbidden; - } - auto E = R1.move_as_ok(); - - auto B = to_sign(overlay_id, node); - - if (E->check_signature(B.as_slice(), signature_.as_slice()).is_error()) { - return BroadcastCheckResult::Forbidden; + if (!skip_check_signature) { + auto R1 = issued_by_.get().create_encryptor(); + if (R1.is_error()) { + return BroadcastCheckResult::Forbidden; + } + auto E = R1.move_as_ok(); + auto B = to_sign(overlay_id, node); + if (E->check_signature(B.as_slice(), signature_.as_slice()).is_error()) { + return BroadcastCheckResult::Forbidden; + } } return (flags_ & CertificateFlags::Trusted) ? BroadcastCheckResult::Allowed : BroadcastCheckResult::NeedCheck; @@ -453,6 +608,35 @@ tl_object_ptr Certificate::empty_tl() { return create_tl_object(); } +OverlayMemberCertificate::OverlayMemberCertificate(const ton_api::overlay_MemberCertificate *cert) { + if (!cert) { + expire_at_ = std::numeric_limits::max(); + return; + } + if (cert->get_id() == ton_api::overlay_emptyMemberCertificate::ID) { + expire_at_ = std::numeric_limits::max(); + return; + } + CHECK(cert->get_id() == ton_api::overlay_memberCertificate::ID); + const auto *real_cert = static_cast(cert); + signed_by_ = PublicKey(real_cert->issued_by_); + flags_ = real_cert->flags_; + slot_ = real_cert->slot_; + expire_at_ = real_cert->expire_at_; + signature_ = td::SharedSlice(real_cert->signature_.as_slice()); +} + +td::Status OverlayMemberCertificate::check_signature(const adnl::AdnlNodeIdShort &node) { + if (is_expired()) { + return td::Status::Error(ErrorCode::notready, "certificate is expired"); + } + td::BufferSlice data_to_sign = to_sign_data(node); + + TRY_RESULT(encryptor, signed_by_.create_encryptor()); + TRY_STATUS(encryptor->check_signature(data_to_sign.as_slice(), signature_.as_slice())); + return td::Status::OK(); +} + } // namespace overlay } // namespace ton diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index fe1166ac..68b033a3 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -53,11 +53,19 @@ class OverlayManager : public Overlays { void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) override; void create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope, - bool announce_self) override; + std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope, + OverlayOptions opts) override; + void create_semiprivate_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate certificate, + std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope, + OverlayOptions opts) override; void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, - OverlayPrivacyRules rules) override; + OverlayPrivacyRules rules, std::string scope) override; + void create_private_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, std::unique_ptr callback, + OverlayPrivacyRules rules, std::string scope, OverlayOptions opts) override; void delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id) override; void send_query(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice query) override { @@ -84,6 +92,11 @@ class OverlayManager : public Overlays { void set_privacy_rules(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, OverlayPrivacyRules rules) override; void update_certificate(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, PublicKeyHash key, std::shared_ptr cert) override; + void update_member_certificate(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + OverlayMemberCertificate certificate) override; + void update_root_member_list(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate certificate) override; void get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, td::uint32 max_peers, td::Promise> promise) override; @@ -92,10 +105,12 @@ class OverlayManager : public Overlays { td::Promise promise); void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data); - void register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + void register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, OverlayMemberCertificate cert, td::actor::ActorOwn overlay); void get_stats(td::Promise> promise) override; + void forget_peer(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, adnl::AdnlNodeIdShort peer_id) override; + struct PrintId {}; PrintId print_id() const { @@ -103,7 +118,11 @@ class OverlayManager : public Overlays { } private: - std::map>> overlays_; + struct OverlayDescription { + td::actor::ActorOwn overlay; + OverlayMemberCertificate member_certificate; + }; + std::map> overlays_; std::string db_root_; @@ -112,6 +131,7 @@ class OverlayManager : public Overlays { td::actor::ActorId dht_node_; using DbType = td::KeyValueAsync; + bool with_db_ = false; DbType db_; class AdnlCallback : public adnl::Adnl::Callback { diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index 409f0993..7def4a2d 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -16,89 +16,165 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "adnl/adnl-node-id.hpp" +#include "adnl/adnl-node.h" +#include "auto/tl/ton_api.h" #include "overlay.hpp" +#include "td/utils/Status.h" +#include "td/utils/Time.h" +#include "td/utils/port/signals.h" +#include +#include namespace ton { namespace overlay { -void OverlayImpl::del_peer(adnl::AdnlNodeIdShort id) { - auto P = peers_.get(id); - CHECK(P != nullptr); +void OverlayImpl::del_peer(const adnl::AdnlNodeIdShort &id) { + auto P = peer_list_.peers_.get(id); + if (P == nullptr) { + return; + } + if (P->is_permanent_member()) { + VLOG(OVERLAY_DEBUG) << this << ": not deleting peer " << id << ": a permanent member"; + return; + } VLOG(OVERLAY_DEBUG) << this << ": deleting peer " << id; if (P->is_neighbour()) { - VLOG(OVERLAY_INFO) << this << ": deleting neighbour " << id; - bool deleted = false; - for (auto &n : neighbours_) { - if (n == id) { - n = neighbours_[neighbours_.size() - 1]; - neighbours_.resize(neighbours_.size() - 1); - deleted = true; - break; - } - } - CHECK(deleted); - P->set_neighbour(false); + del_from_neighbour_list(P); } - peers_.remove(id); - bad_peers_.erase(id); - update_neighbours(0); + peer_list_.peers_.remove(id); + peer_list_.bad_peers_.erase(id); +} + +void OverlayImpl::del_from_neighbour_list(OverlayPeer *P) { + CHECK(P); + if (!P->is_neighbour()) { + return; + } + auto id = P->get_id(); + bool deleted = false; + auto &neighbours = peer_list_.neighbours_; + for (auto &n : neighbours) { + if (n == id) { + n = neighbours[neighbours.size() - 1]; + neighbours.resize(neighbours.size() - 1); + deleted = true; + break; + } + } + CHECK(deleted); + P->set_neighbour(false); +} + +void OverlayImpl::del_from_neighbour_list(const adnl::AdnlNodeIdShort &id) { + auto P = peer_list_.peers_.get(id); + CHECK(P != nullptr); + return del_from_neighbour_list(P); } void OverlayImpl::del_some_peers() { - if (!public_) { + if (overlay_type_ == OverlayType::FixedMemberList) { return; } - while (peers_.size() > max_peers()) { + const size_t max_iterations = 10; + size_t iteration_seqno = 0; + while (peer_list_.peers_.size() > max_peers() && iteration_seqno++ < max_iterations) { OverlayPeer *P; - if (bad_peers_.empty()) { + if (peer_list_.bad_peers_.empty()) { P = get_random_peer(); } else { - auto it = bad_peers_.upper_bound(next_bad_peer_); - if (it == bad_peers_.end()) { - it = bad_peers_.begin(); + auto it = peer_list_.bad_peers_.upper_bound(peer_list_.next_bad_peer_); + if (it == peer_list_.bad_peers_.end()) { + it = peer_list_.bad_peers_.begin(); } - P = peers_.get(next_bad_peer_ = *it); + P = peer_list_.peers_.get(peer_list_.next_bad_peer_ = *it); } - if (P) { + if (P && !P->is_permanent_member()) { auto id = P->get_id(); del_peer(id); } } + update_neighbours(0); } -void OverlayImpl::do_add_peer(OverlayNode node) { - auto id = node.adnl_id_short(); - - auto V = peers_.get(id); - if (V) { - VLOG(OVERLAY_DEBUG) << this << ": updating peer " << id << " up to version " << node.version(); - V->update(std::move(node)); - } else { - VLOG(OVERLAY_DEBUG) << this << ": adding peer " << id << " of version " << node.version(); - peers_.insert(id, OverlayPeer(std::move(node))); - - del_some_peers(); - update_neighbours(0); +td::Status OverlayImpl::validate_peer_certificate(const adnl::AdnlNodeIdShort &node, + const OverlayMemberCertificate &cert) { + if (cert.empty()) { + if (is_persistent_node(node) || overlay_type_ == OverlayType::Public) { + return td::Status::OK(); + } + return td::Status::Error(ErrorCode::protoviolation, "no member certificate found"); } + if (cert.is_expired()) { + return td::Status::Error(ErrorCode::timeout, "member certificate is expired"); + } + if (cert.slot() < 0 || cert.slot() >= opts_.max_slaves_in_semiprivate_overlay_) { + return td::Status::Error(ErrorCode::timeout, "member certificate has invalid slot"); + } + const auto &issued_by = cert.issued_by(); + auto it = peer_list_.root_public_keys_.find(issued_by.compute_short_id()); + if (it == peer_list_.root_public_keys_.end()) { + return td::Status::Error(ErrorCode::protoviolation, "member certificate is signed by unknown public key"); + } + if (it->second.size() > (size_t)cert.slot()) { + auto &el = it->second[cert.slot()]; + if (cert.expire_at() < el.expire_at) { + return td::Status::Error(ErrorCode::protoviolation, + "member certificate rejected, because we know of newer certificate at the same slot"); + } else if (cert.expire_at() == el.expire_at) { + if (node < el.node) { + return td::Status::Error(ErrorCode::protoviolation, + "member certificate rejected, because we know of newer certificate at the same slot"); + } else if (el.node == node) { + // we could return OK here, but we must make sure, that the unchecked signature will not be used for updating PeerNode. + } + } + } + auto R = get_encryptor(issued_by); + if (R.is_error()) { + return R.move_as_error_prefix("failed to check member certificate: failed to create encryptor: "); + } + auto enc = R.move_as_ok(); + auto S = enc->check_signature(cert.to_sign_data(node).as_slice(), cert.signature()); + if (S.is_error()) { + return S.move_as_error_prefix("failed to check member certificate: bad signature: "); + } + if (it->second.size() <= (size_t)cert.slot()) { + it->second.resize((size_t)cert.slot() + 1); + } + it->second[cert.slot()].expire_at = cert.expire_at(); + it->second[cert.slot()].node = node; + return td::Status::OK(); } -void OverlayImpl::add_peer_in_cont(OverlayNode node) { - CHECK(public_); - - do_add_peer(std::move(node)); +td::Status OverlayImpl::validate_peer_certificate(const adnl::AdnlNodeIdShort &node, + ton_api::overlay_MemberCertificate *cert) { + OverlayMemberCertificate ncert(cert); + return validate_peer_certificate(node, ncert); } -void OverlayImpl::add_peer_in(OverlayNode node) { - CHECK(public_); +td::Status OverlayImpl::validate_peer_certificate(const adnl::AdnlNodeIdShort &node, + const OverlayMemberCertificate *cert) { + if (!cert) { + if (is_persistent_node(node) || overlay_type_ == OverlayType::Public) { + return td::Status::OK(); + } + return td::Status::Error(ErrorCode::protoviolation, "no member certificate found"); + } + return validate_peer_certificate(node, *cert); +} + +void OverlayImpl::add_peer(OverlayNode node) { + CHECK(overlay_type_ != OverlayType::FixedMemberList); if (node.overlay_id() != overlay_id_) { VLOG(OVERLAY_WARNING) << this << ": received node with bad overlay"; return; } auto t = td::Clocks::system(); - if (node.version() + 600 < t || node.version() > t + 60) { + if (node.version() + Overlays::overlay_peer_ttl() < t || node.version() > t + 60) { VLOG(OVERLAY_INFO) << this << ": ignoring node of too old version " << node.version(); return; } @@ -115,35 +191,78 @@ void OverlayImpl::add_peer_in(OverlayNode node) { return; } - add_peer_in_cont(std::move(node)); + if (overlay_type_ == OverlayType::CertificatedMembers) { + auto R = validate_peer_certificate(node.adnl_id_short(), *node.certificate()); + if (R.is_error()) { + VLOG(OVERLAY_WARNING) << this << ": bad peer certificate node=" << node.adnl_id_short() << ": " + << R.move_as_error(); + UNREACHABLE(); + return; + } + } + + auto id = node.adnl_id_short(); + + auto V = peer_list_.peers_.get(id); + if (V) { + VLOG(OVERLAY_DEBUG) << this << ": updating peer " << id << " up to version " << node.version(); + V->update(std::move(node)); + } else { + VLOG(OVERLAY_DEBUG) << this << ": adding peer " << id << " of version " << node.version(); + CHECK(overlay_type_ != OverlayType::CertificatedMembers || (node.certificate() && !node.certificate()->empty())); + peer_list_.peers_.insert(id, OverlayPeer(std::move(node))); + del_some_peers(); + auto X = peer_list_.peers_.get(id); + if (X != nullptr && !X->is_neighbour() && peer_list_.neighbours_.size() < max_neighbours() && + !(X->get_node()->flags() & OverlayMemberFlags::DoNotReceiveBroadcasts) && X->get_id() != local_id_) { + peer_list_.neighbours_.push_back(X->get_id()); + X->set_neighbour(true); + } + + update_neighbours(0); + } } void OverlayImpl::add_peers(std::vector peers) { for (auto &node : peers) { - add_peer_in(std::move(node)); + add_peer(std::move(node)); } } -void OverlayImpl::add_peer(OverlayNode P) { - add_peer_in(std::move(P)); +void OverlayImpl::add_peers(const tl_object_ptr &nodes) { + for (auto &n : nodes->nodes_) { + auto N = OverlayNode::create(n); + if (N.is_ok()) { + add_peer(N.move_as_ok()); + } + } +} + +void OverlayImpl::add_peers(const tl_object_ptr &nodes) { + for (auto &n : nodes->nodes_) { + auto N = OverlayNode::create(n); + if (N.is_ok()) { + add_peer(N.move_as_ok()); + } + } } void OverlayImpl::on_ping_result(adnl::AdnlNodeIdShort peer, bool success) { - if (!public_) { + if (overlay_type_ == OverlayType::FixedMemberList) { return; } - if (OverlayPeer *p = peers_.get(peer)) { + if (OverlayPeer *p = peer_list_.peers_.get(peer)) { p->on_ping_result(success); if (p->is_alive()) { - bad_peers_.erase(peer); + peer_list_.bad_peers_.erase(peer); } else { - bad_peers_.insert(peer); + peer_list_.bad_peers_.insert(peer); } } } void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R) { - CHECK(public_); + CHECK(overlay_type_ != OverlayType::FixedMemberList); on_ping_result(src, R.is_ok()); if (R.is_error()) { VLOG(OVERLAY_NOTICE) << this << ": failed getRandomPeers query: " << R.move_as_error(); @@ -156,16 +275,24 @@ void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::Result nodes; - for (auto &n : res->nodes_) { - auto N = OverlayNode::create(n); - if (N.is_ok()) { - nodes.emplace_back(N.move_as_ok()); - } +void OverlayImpl::receive_random_peers_v2(adnl::AdnlNodeIdShort src, td::Result R) { + CHECK(overlay_type_ != OverlayType::FixedMemberList); + on_ping_result(src, R.is_ok()); + if (R.is_error()) { + VLOG(OVERLAY_NOTICE) << this << ": failed getRandomPeersV2 query: " << R.move_as_error(); + return; } - add_peers(std::move(nodes)); + auto R2 = fetch_tl_object(R.move_as_ok(), true); + if (R2.is_error()) { + VLOG(OVERLAY_WARNING) << this << ": dropping incorrect answer to overlay.getRandomPeers query from " << src << ": " + << R2.move_as_error(); + return; + } + + add_peers(R2.move_as_ok()); } void OverlayImpl::send_random_peers_cont(adnl::AdnlNodeIdShort src, OverlayNode node, @@ -175,10 +302,13 @@ void OverlayImpl::send_random_peers_cont(adnl::AdnlNodeIdShort src, OverlayNode vec.emplace_back(node.tl()); } - for (td::uint32 i = 0; i < nodes_to_send(); i++) { + td::uint32 max_iterations = nodes_to_send() + 16; + for (td::uint32 i = 0; i < max_iterations && vec.size() < nodes_to_send(); i++) { auto P = get_random_peer(true); if (P) { - vec.emplace_back(P->get().tl()); + if (P->has_full_id()) { + vec.emplace_back(P->get_node()->tl()); + } } else { break; } @@ -213,58 +343,111 @@ void OverlayImpl::send_random_peers(adnl::AdnlNodeIdShort src, td::Promise promise) { + std::vector> vec; + if (announce_self_) { + CHECK(is_persistent_node(node.adnl_id_short()) || !node.certificate()->empty()); + vec.emplace_back(node.tl_v2()); + } + + td::uint32 max_iterations = nodes_to_send() + 16; + for (td::uint32 i = 0; i < max_iterations && vec.size() < nodes_to_send(); i++) { + auto P = get_random_peer(true); + if (P) { + if (P->has_full_id() && !P->is_permanent_member()) { + vec.emplace_back(P->get_node()->tl_v2()); + } + } else { + break; + } + } + + if (promise) { + auto Q = create_tl_object(std::move(vec)); + promise.set_value(serialize_tl_object(Q, true)); + } else { + auto P = + td::PromiseCreator::lambda([SelfId = actor_id(this), src, oid = print_id()](td::Result res) { + td::actor::send_closure(SelfId, &OverlayImpl::receive_random_peers_v2, src, std::move(res)); + }); + auto Q = + create_tl_object(create_tl_object(std::move(vec))); + td::actor::send_closure(manager_, &OverlayManager::send_query, src, local_id_, overlay_id_, + "overlay getRandomPeers", std::move(P), + td::Timestamp::in(5.0 + td::Random::fast(0, 50) * 0.1), serialize_tl_object(Q, true)); + } +} + +void OverlayImpl::send_random_peers_v2(adnl::AdnlNodeIdShort src, td::Promise promise) { + auto P = td::PromiseCreator::lambda([src, promise = std::move(promise), + SelfId = actor_id(this)](td::Result res) mutable { + if (res.is_error()) { + promise.set_error(td::Status::Error(ErrorCode::error, "cannot get self node")); + return; + } + td::actor::send_closure(SelfId, &OverlayImpl::send_random_peers_v2_cont, src, res.move_as_ok(), std::move(promise)); + }); + + get_self_node(std::move(P)); +} + void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) { - if (peers_.size() == 0) { + if (peer_list_.peers_.size() == 0) { return; } td::uint32 iter = 0; - while (iter < 10 && (nodes_to_change > 0 || neighbours_.size() < max_neighbours())) { - auto X = peers_.get_random(); + while (iter++ < 10 && (nodes_to_change > 0 || peer_list_.neighbours_.size() < max_neighbours())) { + auto X = peer_list_.peers_.get_random(); if (!X) { break; } if (X->get_id() == local_id_) { - iter++; continue; } - if (X->get_version() <= td::Clocks::system() - 600) { - if (X->is_neighbour()) { - bool found = false; - for (auto &n : neighbours_) { - if (n == X->get_id()) { - n = *neighbours_.rbegin(); - found = true; - break; - } - } - CHECK(found); - neighbours_.pop_back(); - X->set_neighbour(false); + if (overlay_type_ != OverlayType::FixedMemberList && X->get_version() <= td::Clocks::system() - + Overlays::overlay_peer_ttl()) { + if (X->is_permanent_member()) { + del_from_neighbour_list(X); + } else { + auto id = X->get_id(); + del_peer(id); + } + continue; + } + + if (overlay_type_ == OverlayType::CertificatedMembers && !X->is_permanent_member() && + X->certificate()->is_expired()) { + auto id = X->get_id(); + del_peer(id); + continue; + } + + if (X->get_node()->flags() & OverlayMemberFlags::DoNotReceiveBroadcasts) { + if (X->is_neighbour()) { + del_from_neighbour_list(X); } - bad_peers_.erase(X->get_id()); - peers_.remove(X->get_id()); continue; } if (X->is_neighbour()) { - iter++; continue; } - if (neighbours_.size() < max_neighbours()) { + if (peer_list_.neighbours_.size() < max_neighbours()) { VLOG(OVERLAY_INFO) << this << ": adding new neighbour " << X->get_id(); - neighbours_.push_back(X->get_id()); + peer_list_.neighbours_.push_back(X->get_id()); X->set_neighbour(true); - } else { + } else if (X->is_alive()) { CHECK(nodes_to_change > 0); - auto i = td::Random::fast(0, static_cast(neighbours_.size()) - 1); - auto Y = peers_.get(neighbours_[i]); + auto i = td::Random::fast(0, static_cast(peer_list_.neighbours_.size()) - 1); + auto Y = peer_list_.peers_.get(peer_list_.neighbours_[i]); CHECK(Y != nullptr); CHECK(Y->is_neighbour()); Y->set_neighbour(false); - neighbours_[i] = X->get_id(); + peer_list_.neighbours_[i] = X->get_id(); X->set_neighbour(true); nodes_to_change--; VLOG(OVERLAY_INFO) << this << ": changing neighbour " << Y->get_id() << " -> " << X->get_id(); @@ -274,9 +457,11 @@ void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) { OverlayPeer *OverlayImpl::get_random_peer(bool only_alive) { size_t skip_bad = 3; - while (peers_.size() > (only_alive ? bad_peers_.size() : 0)) { - auto P = peers_.get_random(); - if (public_ && P->get_version() + 3600 < td::Clocks::system()) { + OverlayPeer *res = nullptr; + while (!res && peer_list_.peers_.size() > (only_alive ? peer_list_.bad_peers_.size() : 0)) { + auto P = peer_list_.peers_.get_random(); + if (!P->is_permanent_member() && + (P->get_version() + 3600 < td::Clocks::system() || P->certificate()->is_expired())) { VLOG(OVERLAY_INFO) << this << ": deleting outdated peer " << P->get_id(); del_peer(P->get_id()); continue; @@ -290,18 +475,19 @@ OverlayPeer *OverlayImpl::get_random_peer(bool only_alive) { continue; } } - return P; + res = P; } - return nullptr; + update_neighbours(0); + return res; } void OverlayImpl::get_overlay_random_peers(td::uint32 max_peers, td::Promise> promise) { std::vector v; auto t = td::Clocks::system(); - while (v.size() < max_peers && v.size() < peers_.size() - bad_peers_.size()) { - auto P = peers_.get_random(); - if (P->get_version() + 3600 < t) { + while (v.size() < max_peers && v.size() < peer_list_.peers_.size() - peer_list_.bad_peers_.size()) { + auto P = peer_list_.peers_.get_random(); + if (!P->is_permanent_member() && (P->get_version() + 3600 < t || P->certificate()->is_expired(t))) { VLOG(OVERLAY_INFO) << this << ": deleting outdated peer " << P->get_id(); del_peer(P->get_id()); } else if (P->is_alive()) { @@ -317,22 +503,229 @@ void OverlayImpl::get_overlay_random_peers(td::uint32 max_peers, } } } + update_neighbours(0); promise.set_result(std::move(v)); } void OverlayImpl::receive_nodes_from_db(tl_object_ptr tl_nodes) { - if (public_) { - std::vector nodes; - for (auto &n : tl_nodes->nodes_) { - auto N = OverlayNode::create(n); - if (N.is_ok()) { - nodes.emplace_back(N.move_as_ok()); + if (overlay_type_ != OverlayType::FixedMemberList) { + add_peers(tl_nodes); + } +} + +void OverlayImpl::receive_nodes_from_db_v2(tl_object_ptr tl_nodes) { + if (overlay_type_ != OverlayType::FixedMemberList) { + add_peers(tl_nodes); + } +} + +bool OverlayImpl::is_persistent_node(const adnl::AdnlNodeIdShort &id) { + auto P = peer_list_.peers_.get(id); + if (!P) { + return false; + } + return P->is_permanent_member(); +} + +bool OverlayImpl::is_valid_peer(const adnl::AdnlNodeIdShort &src, + const ton_api::overlay_MemberCertificate *certificate) { + if (overlay_type_ == OverlayType::Public) { + on_ping_result(src, true); + return true; + } else if (overlay_type_ == OverlayType::FixedMemberList) { + return peer_list_.peers_.get(src); + } else { + OverlayMemberCertificate cert(certificate); + if (cert.empty()) { + auto P = peer_list_.peers_.get(src); + if (P && !P->is_permanent_member()) { + auto C = P->certificate(); + if (C) { + cert = *C; + } } } - add_peers(std::move(nodes)); + + auto S = validate_peer_certificate(src, cert); + if (S.is_error()) { + VLOG(OVERLAY_WARNING) << "adnl=" << src << ": certificate is invalid: " << S; + return false; + } + auto P = peer_list_.peers_.get(src); + if (P) { + CHECK(P->is_permanent_member() || !cert.empty()); + P->update_certificate(std::move(cert)); + } + return true; } } +void OverlayImpl::iterate_all_peers(std::function cb) { + peer_list_.peers_.iterate([&](const adnl::AdnlNodeIdShort &key, OverlayPeer &peer) { cb(key, peer); }); +} + +void OverlayImpl::update_peer_err_ctr(adnl::AdnlNodeIdShort peer_id, bool is_fec) { + auto src_peer = peer_list_.peers_.get(peer_id); + if (src_peer) { + if (is_fec) { + src_peer->fec_broadcast_errors++; + } else { + src_peer->broadcast_errors++; + } + } +} + +void OverlayImpl::update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint64 msg_size, bool is_query, + bool is_response) { + auto out_peer = peer_list_.peers_.get(peer_id); + if (out_peer) { + out_peer->traffic_ctr.add_packet(msg_size, false); + if (is_response) { + out_peer->traffic_responses_ctr.add_packet(msg_size, false); + } + if (is_query) { + out_peer->last_out_query_at = td::Timestamp::now(); + } + } + total_traffic_ctr.add_packet(msg_size, false); + if (is_response) { + total_traffic_responses_ctr.add_packet(msg_size, false); + } +} + +void OverlayImpl::update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint64 msg_size, bool is_query, + bool is_response) { + auto in_peer = peer_list_.peers_.get(peer_id); + if (in_peer) { + in_peer->traffic_ctr.add_packet(msg_size, true); + if (is_response) { + in_peer->traffic_responses_ctr.add_packet(msg_size, true); + } + if (is_query) { + in_peer->last_in_query_at = td::Timestamp::now(); + } + } + total_traffic_ctr.add_packet(msg_size, true); + if (is_response) { + total_traffic_responses_ctr.add_packet(msg_size, true); + } +} + +void OverlayImpl::update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) { + auto fpeer = peer_list_.peers_.get(peer_id); + if (fpeer) { + fpeer->ip_addr_str = ip_str; + } +} + +bool OverlayImpl::has_good_peers() const { + return peer_list_.peers_.size() > peer_list_.bad_peers_.size(); +} + +bool OverlayImpl::is_root_public_key(const PublicKeyHash &key) const { + return peer_list_.root_public_keys_.count(key) > 0; +} + +std::vector OverlayImpl::get_neighbours(td::uint32 max_size) const { + if (max_size == 0 || max_size >= peer_list_.neighbours_.size()) { + return peer_list_.neighbours_; + } else { + std::vector vec; + std::vector ul; + for (td::uint32 i = 0; i < max_size; i++) { + td::uint32 t = td::Random::fast(0, static_cast(peer_list_.neighbours_.size()) - 1 - i); + td::uint32 j; + for (j = 0; j < i && ul[j] <= t; j++) { + t++; + } + ul.emplace(ul.begin() + j, t); + vec.push_back(peer_list_.neighbours_[t]); + } + return vec; + } +} + +void OverlayImpl::send_message_to_neighbours(td::BufferSlice data) { + for (auto &n : peer_list_.neighbours_) { + td::actor::send_closure(manager_, &OverlayManager::send_message, n, local_id_, overlay_id_, data.clone()); + } +} + +size_t OverlayImpl::neighbours_cnt() const { + return peer_list_.neighbours_.size(); +} + +void OverlayImpl::update_root_member_list(std::vector ids, + std::vector root_public_keys, OverlayMemberCertificate cert) { + auto expected_size = + (td::uint32)(ids.size() + root_public_keys.size() * opts_.max_slaves_in_semiprivate_overlay_); + opts_.max_peers_ = std::max(opts_.max_peers_, expected_size); + std::sort(ids.begin(), ids.end()); + auto old_root_public_keys = std::move(peer_list_.root_public_keys_); + for (const auto &pub_key : root_public_keys) { + auto it = old_root_public_keys.find(pub_key); + if (it != old_root_public_keys.end()) { + peer_list_.root_public_keys_.emplace(it->first, std::move(it->second)); + } else { + peer_list_.root_public_keys_.emplace(pub_key, PeerList::SlaveKeys{}); + } + } + std::vector to_del; + peer_list_.peers_.iterate([&](const adnl::AdnlNodeIdShort &key, OverlayPeer &peer) { + peer.set_permanent(std::binary_search(ids.begin(), ids.end(), key)); + if (peer.is_permanent_member()) { + peer.clear_certificate(); + } else { + auto S = validate_peer_certificate(peer.get_id(), peer.certificate()); + if (S.is_error()) { + to_del.push_back(peer.get_id()); + } + } + }); + for (const auto &id : to_del) { + del_peer(id); + } + for (const auto &id : ids) { + if (!peer_list_.peers_.exists(id)) { + OverlayNode node(id, overlay_id_, opts_.default_permanent_members_flags_); + OverlayPeer peer(std::move(node)); + peer.set_permanent(true); + CHECK(peer.is_permanent_member()); + peer_list_.peers_.insert(std::move(id), std::move(peer)); + } + } + + update_member_certificate(std::move(cert)); + update_neighbours(0); +} + +void OverlayImpl::update_member_certificate(OverlayMemberCertificate cert) { + peer_list_.cert_ = std::move(cert); + + if (is_persistent_node(local_id_)) { + peer_list_.local_cert_is_valid_until_ = td::Timestamp::in(86400.0 * 365 * 100); /* 100 years */ + } else { + auto R = validate_peer_certificate(local_id_, &peer_list_.cert_); + if (R.is_ok()) { + peer_list_.local_cert_is_valid_until_ = td::Timestamp::at_unix(cert.expire_at()); + } else { + peer_list_.local_cert_is_valid_until_ = td::Timestamp::never(); + } + } +} + +bool OverlayImpl::has_valid_membership_certificate() { + if (overlay_type_ != OverlayType::CertificatedMembers) { + return true; + } + + if (!peer_list_.local_cert_is_valid_until_) { + return false; + } + + return !peer_list_.local_cert_is_valid_until_.is_in_past(); +} + } // namespace overlay } // namespace ton diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index a562beb6..30a40b1c 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -18,6 +18,7 @@ */ #include "auto/tl/ton_api.h" #include "td/utils/Random.h" +#include "common/delay.h" #include "adnl/utils.hpp" #include "dht/dht.h" @@ -26,41 +27,63 @@ #include "auto/tl/ton_api.hpp" #include "keys/encryptor.h" +#include "td/utils/Status.h" #include "td/utils/StringBuilder.h" +#include "td/utils/port/signals.h" +#include namespace ton { namespace overlay { -td::actor::ActorOwn Overlay::create(td::actor::ActorId keyring, - td::actor::ActorId adnl, - td::actor::ActorId manager, - td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, - OverlayIdFull overlay_id, std::unique_ptr callback, - OverlayPrivacyRules rules, td::string scope, bool announce_self) { - auto R = td::actor::create_actor("overlay", keyring, adnl, manager, dht_node, local_id, - std::move(overlay_id), true, std::vector(), - std::move(callback), std::move(rules), scope, announce_self); - return td::actor::ActorOwn(std::move(R)); +const OverlayMemberCertificate OverlayNode::empty_certificate_{}; + +static std::string overlay_actor_name(const OverlayIdFull &overlay_id) { + return PSTRING() << "overlay." << overlay_id.compute_short_id().bits256_value().to_hex().substr(0, 4); } -td::actor::ActorOwn Overlay::create(td::actor::ActorId keyring, - td::actor::ActorId adnl, - td::actor::ActorId manager, - td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, - OverlayIdFull overlay_id, std::vector nodes, - std::unique_ptr callback, OverlayPrivacyRules rules) { - auto R = - td::actor::create_actor("overlay", keyring, adnl, manager, dht_node, local_id, std::move(overlay_id), - false, std::move(nodes), std::move(callback), std::move(rules)); - return td::actor::ActorOwn(std::move(R)); +td::actor::ActorOwn Overlay::create_public(td::actor::ActorId keyring, + td::actor::ActorId adnl, + td::actor::ActorId manager, + td::actor::ActorId dht_node, + adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::unique_ptr callback, + OverlayPrivacyRules rules, td::string scope, OverlayOptions opts) { + return td::actor::create_actor( + overlay_actor_name(overlay_id), keyring, adnl, manager, dht_node, local_id, std::move(overlay_id), + OverlayType::Public, std::vector(), std::vector(), + OverlayMemberCertificate{}, std::move(callback), std::move(rules), std::move(scope), std::move(opts)); +} + +td::actor::ActorOwn Overlay::create_private( + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, + OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, + OverlayPrivacyRules rules, std::string scope, OverlayOptions opts) { + return td::actor::create_actor( + overlay_actor_name(overlay_id), keyring, adnl, manager, dht_node, local_id, std::move(overlay_id), + OverlayType::FixedMemberList, std::move(nodes), std::vector(), OverlayMemberCertificate{}, + std::move(callback), std::move(rules), std::move(scope), std::move(opts)); +} + +td::actor::ActorOwn Overlay::create_semiprivate( + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, + OverlayIdFull overlay_id, std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate cert, std::unique_ptr callback, OverlayPrivacyRules rules, + std::string scope, OverlayOptions opts) { + return td::actor::create_actor(overlay_actor_name(overlay_id), keyring, adnl, manager, dht_node, + local_id, std::move(overlay_id), OverlayType::CertificatedMembers, + std::move(nodes), std::move(root_public_keys), std::move(cert), + std::move(callback), std::move(rules), std::move(scope), std::move(opts)); } OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId manager, td::actor::ActorId dht_node, - adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub, - std::vector nodes, std::unique_ptr callback, - OverlayPrivacyRules rules, td::string scope, bool announce_self) + adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, OverlayType overlay_type, + std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate cert, std::unique_ptr callback, + OverlayPrivacyRules rules, td::string scope, OverlayOptions opts) : keyring_(keyring) , adnl_(adnl) , manager_(manager) @@ -68,36 +91,29 @@ OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor , local_id_(local_id) , id_full_(std::move(overlay_id)) , callback_(std::move(callback)) - , public_(pub) + , overlay_type_(overlay_type) , rules_(std::move(rules)) , scope_(scope) - , announce_self_(announce_self) { + , announce_self_(opts.announce_self_) + , opts_(std::move(opts)) { overlay_id_ = id_full_.compute_short_id(); + frequent_dht_lookup_ = opts_.frequent_dht_lookup_; + peer_list_.local_member_flags_ = opts_.local_overlay_member_flags_; + opts_.broadcast_speed_multiplier_ = std::max(opts_.broadcast_speed_multiplier_, 1e-9); - VLOG(OVERLAY_INFO) << this << ": creating " << (public_ ? "public" : "private"); + VLOG(OVERLAY_INFO) << this << ": creating"; - for (auto &node : nodes) { - CHECK(!public_); - auto X = OverlayNode{node, overlay_id_}; - do_add_peer(std::move(X)); - } - - update_neighbours(static_cast(nodes.size())); + auto nodes_size = static_cast(nodes.size()); + OverlayImpl::update_root_member_list(std::move(nodes), std::move(root_public_keys), std::move(cert)); + update_neighbours(nodes_size); } void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeers &query, td::Promise promise) { - if (public_) { + if (overlay_type_ != OverlayType::FixedMemberList) { VLOG(OVERLAY_DEBUG) << this << ": received " << query.peers_->nodes_.size() << " nodes from " << src << " in getRandomPeers query"; - std::vector nodes; - for (auto &n : query.peers_->nodes_) { - auto N = OverlayNode::create(n); - if (N.is_ok()) { - nodes.emplace_back(N.move_as_ok()); - } - } - add_peers(std::move(nodes)); + add_peers(query.peers_); send_random_peers(src, std::move(promise)); } else { VLOG(OVERLAY_WARNING) << this << ": DROPPING getRandomPeers query from " << src << " in private overlay"; @@ -105,6 +121,19 @@ void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getR } } +void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeersV2 &query, + td::Promise promise) { + if (overlay_type_ != OverlayType::FixedMemberList) { + VLOG(OVERLAY_DEBUG) << this << ": received " << query.peers_->nodes_.size() << " nodes from " << src + << " in getRandomPeers query"; + add_peers(query.peers_); + send_random_peers_v2(src, std::move(promise)); + } else { + VLOG(OVERLAY_WARNING) << this << ": DROPPING getRandomPeers query from " << src << " in private overlay"; + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "overlay is private")); + } +} + void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcast &query, td::Promise promise) { auto it = broadcasts_.find(query.hash_); @@ -137,17 +166,14 @@ void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getB } */ -void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) { - if (!public_) { - auto P = peers_.get(src); - if (P == nullptr) { - VLOG(OVERLAY_WARNING) << this << ": received query in private overlay from unknown source " << src; - promise.set_error(td::Status::Error(ErrorCode::protoviolation, "overlay is private")); - return; - } - } else { - on_ping_result(src, true); +void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data, td::Promise promise) { + if (!is_valid_peer(src, extra ? extra->certificate_.get() : nullptr)) { + VLOG(OVERLAY_WARNING) << this << ": received query in private overlay from unknown source " << src; + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "overlay is not public")); + return; } + auto R = fetch_tl_object(data.clone(), true); if (R.is_error()) { @@ -165,16 +191,25 @@ void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr bcast) { + if (peer_list_.local_member_flags_ & OverlayMemberFlags::DoNotReceiveBroadcasts) { + return td::Status::OK(); + } return BroadcastSimple::create(this, message_from, std::move(bcast)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr b) { + if (peer_list_.local_member_flags_ & OverlayMemberFlags::DoNotReceiveBroadcasts) { + return td::Status::OK(); + } return OverlayFecBroadcastPart::create(this, message_from, std::move(b)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr b) { + if (peer_list_.local_member_flags_ & OverlayMemberFlags::DoNotReceiveBroadcasts) { + return td::Status::OK(); + } return OverlayFecBroadcastPart::create(this, message_from, std::move(b)); } @@ -186,6 +221,7 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg) { + return td::Status::OK(); // disable this logic for now auto it = fec_broadcasts_.find(msg->hash_); if (it != fec_broadcasts_.end()) { VLOG(OVERLAY_DEBUG) << this << ": received fec opt-out message from " << message_from << " for broadcast " @@ -200,6 +236,7 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg) { + return td::Status::OK(); // disable this logic for now auto it = fec_broadcasts_.find(msg->hash_); if (it != fec_broadcasts_.end()) { VLOG(OVERLAY_DEBUG) << this << ": received fec completed message from " << message_from << " for broadcast " @@ -219,15 +256,13 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, return td::Status::OK(); } -void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) { - if (!public_) { - if (peers_.get(src) == nullptr) { - VLOG(OVERLAY_WARNING) << this << ": received query in private overlay from unknown source " << src; - return; - } - } else { - on_ping_result(src, true); +void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data) { + if (!is_valid_peer(src, extra ? extra->certificate_.get() : nullptr)) { + VLOG(OVERLAY_WARNING) << this << ": received message in private overlay from unknown source " << src; + return; } + auto X = fetch_tl_object(data.clone(), true); if (X.is_error()) { VLOG(OVERLAY_DEBUG) << this << ": received custom message"; @@ -242,62 +277,83 @@ void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice dat void OverlayImpl::alarm() { bcast_gc(); - - if(update_throughput_at_.is_in_past()) { + + if (update_throughput_at_.is_in_past()) { double t_elapsed = td::Time::now() - last_throughput_update_.at(); auto SelfId = actor_id(this); - peers_.iterate([&](const adnl::AdnlNodeIdShort &key, OverlayPeer &peer) { - peer.throughput_out_bytes = static_cast(peer.throughput_out_bytes_ctr / t_elapsed); - peer.throughput_in_bytes = static_cast(peer.throughput_in_bytes_ctr / t_elapsed); - - peer.throughput_out_packets = static_cast(peer.throughput_out_packets_ctr / t_elapsed); - peer.throughput_in_packets = static_cast(peer.throughput_in_packets_ctr / t_elapsed); - - peer.throughput_out_bytes_ctr = 0; - peer.throughput_in_bytes_ctr = 0; - - peer.throughput_out_packets_ctr = 0; - peer.throughput_in_packets_ctr = 0; - + iterate_all_peers([&](const adnl::AdnlNodeIdShort &key, OverlayPeer &peer) { + peer.traffic = peer.traffic_ctr; + peer.traffic.normalize(t_elapsed); + peer.traffic_ctr = {}; + peer.traffic_responses = peer.traffic_responses_ctr; + peer.traffic_responses.normalize(t_elapsed); + peer.traffic_responses_ctr = {}; + auto P = td::PromiseCreator::lambda([SelfId, peer_id = key](td::Result result) { result.ensure(); td::actor::send_closure(SelfId, &Overlay::update_peer_ip_str, peer_id, result.move_as_ok()); }); - + td::actor::send_closure(adnl_, &adnl::AdnlSenderInterface::get_conn_ip_str, local_id_, key, std::move(P)); }); - + total_traffic = total_traffic_ctr; + total_traffic.normalize(t_elapsed); + total_traffic_ctr = {}; + total_traffic_responses = total_traffic_responses_ctr; + total_traffic_responses.normalize(t_elapsed); + total_traffic_responses_ctr = {}; + update_throughput_at_ = td::Timestamp::in(50.0); last_throughput_update_ = td::Timestamp::now(); } - - if (public_) { - if (peers_.size() > 0) { + + if (overlay_type_ != OverlayType::FixedMemberList) { + if (has_valid_membership_certificate()) { auto P = get_random_peer(); if (P) { - send_random_peers(P->get_id(), {}); - } - } - if (next_dht_query_.is_in_past()) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result res) { - td::actor::send_closure(SelfId, &OverlayImpl::receive_dht_nodes, std::move(res), true); - }); - td::actor::send_closure(dht_node_, &dht::Dht::get_value, dht::DhtKey{overlay_id_.pubkey_hash(), "nodes", 0}, - std::move(P)); - next_dht_query_ = td::Timestamp::in(td::Random::fast(60.0, 100.0)); - } - if (update_db_at_.is_in_past()) { - if (peers_.size() > 0) { - std::vector vec; - for (td::uint32 i = 0; i < 20; i++) { - vec.push_back(get_random_peer()->get()); + if (overlay_type_ == OverlayType::Public) { + send_random_peers(P->get_id(), {}); + } else { + send_random_peers_v2(P->get_id(), {}); } + } + } else { + VLOG(OVERLAY_WARNING) << "meber certificate ist invalid, valid_until=" + << peer_list_.local_cert_is_valid_until_.at_unix(); + } + if (next_dht_query_ && next_dht_query_.is_in_past() && overlay_type_ == OverlayType::Public) { + next_dht_query_ = td::Timestamp::never(); + std::function callback = [SelfId = actor_id(this)](dht::DhtValue value) { + td::actor::send_closure(SelfId, &OverlayImpl::receive_dht_nodes, std::move(value)); + }; + td::Promise on_finish = [SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &OverlayImpl::dht_lookup_finished, R.move_as_status()); + }; + td::actor::send_closure(dht_node_, &dht::Dht::get_value_many, dht::DhtKey{overlay_id_.pubkey_hash(), "nodes", 0}, + std::move(callback), std::move(on_finish)); + } + if (update_db_at_.is_in_past() && overlay_type_ == OverlayType::Public) { + std::vector vec; + for (td::uint32 i = 0; i < 20; i++) { + auto P = get_random_peer(); + if (!P) { + break; + } + vec.push_back(P->get_node()->clone()); + } + if (vec.size() > 0) { td::actor::send_closure(manager_, &OverlayManager::save_to_db, local_id_, overlay_id_, std::move(vec)); } update_db_at_ = td::Timestamp::in(60.0); } + if (update_neighbours_at_.is_in_past()) { + update_neighbours(2); + update_neighbours_at_ = td::Timestamp::in(td::Random::fast(30.0, 120.0)); + } else { + update_neighbours(0); + } alarm_timestamp() = td::Timestamp::in(1.0); } else { update_neighbours(0); @@ -305,31 +361,37 @@ void OverlayImpl::alarm() { } } -void OverlayImpl::receive_dht_nodes(td::Result res, bool dummy) { - CHECK(public_); - if (res.is_ok()) { - auto v = res.move_as_ok(); - auto R = fetch_tl_object(v.value().clone(), true); - if (R.is_ok()) { - auto r = R.move_as_ok(); - VLOG(OVERLAY_INFO) << this << ": received " << r->nodes_.size() << " nodes from overlay"; - VLOG(OVERLAY_EXTRA_DEBUG) << this << ": nodes: " << ton_api::to_string(r); - std::vector nodes; - for (auto &n : r->nodes_) { - auto N = OverlayNode::create(n); - if (N.is_ok()) { - nodes.emplace_back(N.move_as_ok()); - } +void OverlayImpl::receive_dht_nodes(dht::DhtValue v) { + CHECK(overlay_type_ == OverlayType::Public); + auto R = fetch_tl_object(v.value().clone(), true); + if (R.is_ok()) { + auto r = R.move_as_ok(); + VLOG(OVERLAY_INFO) << this << ": received " << r->nodes_.size() << " nodes from overlay"; + VLOG(OVERLAY_EXTRA_DEBUG) << this << ": nodes: " << ton_api::to_string(r); + std::vector nodes; + for (auto &n : r->nodes_) { + auto N = OverlayNode::create(n); + if (N.is_ok()) { + nodes.emplace_back(N.move_as_ok()); } - add_peers(std::move(nodes)); - } else { - VLOG(OVERLAY_WARNING) << this << ": incorrect value in DHT for overlay nodes: " << R.move_as_error(); } + add_peers(std::move(nodes)); } else { - VLOG(OVERLAY_NOTICE) << this << ": can not get value from DHT: " << res.move_as_error(); + VLOG(OVERLAY_WARNING) << this << ": incorrect value in DHT for overlay nodes: " << R.move_as_error(); } +} +void OverlayImpl::dht_lookup_finished(td::Status S) { + if (S.is_error()) { + VLOG(OVERLAY_NOTICE) << this << ": can not get value from DHT: " << S; + } + if (!(next_dht_store_query_ && next_dht_store_query_.is_in_past())) { + finish_dht_query(); + return; + } + next_dht_store_query_ = td::Timestamp::never(); if (!announce_self_) { + finish_dht_query(); return; } @@ -337,6 +399,7 @@ void OverlayImpl::receive_dht_nodes(td::Result res, bool dummy) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), oid = print_id()](td::Result R) { if (R.is_error()) { LOG(ERROR) << oid << "cannot get self node"; + td::actor::send_closure(SelfId, &OverlayImpl::finish_dht_query); return; } td::actor::send_closure(SelfId, &OverlayImpl::update_dht_nodes, R.move_as_ok()); @@ -345,7 +408,7 @@ void OverlayImpl::receive_dht_nodes(td::Result res, bool dummy) { } void OverlayImpl::update_dht_nodes(OverlayNode node) { - if (!public_) { + if (overlay_type_ != OverlayType::Public) { return; } @@ -361,10 +424,11 @@ void OverlayImpl::update_dht_nodes(OverlayNode node) { static_cast(td::Clocks::system() + 3600), td::BufferSlice()}; value.check().ensure(); - auto P = td::PromiseCreator::lambda([oid = print_id()](td::Result res) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), oid = print_id()](td::Result res) { if (res.is_error()) { VLOG(OVERLAY_NOTICE) << oid << ": error storing to DHT: " << res.move_as_error(); } + td::actor::send_closure(SelfId, &OverlayImpl::finish_dht_query); }); td::actor::send_closure(dht_node_, &dht::Dht::set_value, std::move(value), std::move(P)); @@ -401,13 +465,16 @@ void OverlayImpl::bcast_gc() { CHECK(delivered_broadcasts_.size() == bcast_lru_.size()); } -void OverlayImpl::send_message_to_neighbours(td::BufferSlice data) { - for (auto &n : neighbours_) { - td::actor::send_closure(manager_, &OverlayManager::send_message, n, local_id_, overlay_id_, data.clone()); - } -} - void OverlayImpl::send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) { + if (!has_valid_membership_certificate()) { + VLOG(OVERLAY_WARNING) << "member certificate is invalid, valid_until=" + << peer_list_.local_cert_is_valid_until_.at_unix(); + return; + } + if (!has_valid_broadcast_certificate(send_as, data.size(), false)) { + VLOG(OVERLAY_WARNING) << "broadcast source certificate is invalid"; + return; + } auto S = BroadcastSimple::create_new(actor_id(this), keyring_, send_as, std::move(data), flags); if (S.is_error()) { LOG(WARNING) << "failed to send broadcast: " << S; @@ -415,7 +482,17 @@ void OverlayImpl::send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::Bu } void OverlayImpl::send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) { - OverlayOutboundFecBroadcast::create(std::move(data), flags, actor_id(this), send_as); + if (!has_valid_membership_certificate()) { + VLOG(OVERLAY_WARNING) << "meber certificate is invalid, valid_until=" + << peer_list_.local_cert_is_valid_until_.at_unix(); + return; + } + if (!has_valid_broadcast_certificate(send_as, data.size(), true)) { + VLOG(OVERLAY_WARNING) << "broadcast source certificate is invalid"; + return; + } + OverlayOutboundFecBroadcast::create(std::move(data), flags, actor_id(this), send_as, + opts_.broadcast_speed_multiplier_); } void OverlayImpl::print(td::StringBuilder &sb) { @@ -433,23 +510,46 @@ td::Status OverlayImpl::check_date(td::uint32 date) { return td::Status::OK(); } -BroadcastCheckResult OverlayImpl::check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size, - bool is_fec) { +BroadcastCheckResult OverlayImpl::check_source_eligible(const PublicKeyHash& source, const Certificate* cert, + td::uint32 size, bool is_fec) { if (size == 0) { return BroadcastCheckResult::Forbidden; } - auto short_id = source.compute_short_id(); - - auto r = rules_.check_rules(source.compute_short_id(), size, is_fec); + auto r = rules_.check_rules(source, size, is_fec); if (!cert || r == BroadcastCheckResult::Allowed) { return r; } + td::Bits256 cert_hash = get_tl_object_sha_bits256(cert->tl()); + auto cached_cert = checked_certificates_cache_.find(source); + bool cached = cached_cert != checked_certificates_cache_.end() && cached_cert->second->cert_hash == cert_hash; - auto r2 = cert->check(short_id, overlay_id_, static_cast(td::Clocks::system()), size, is_fec); + auto r2 = cert->check(source, overlay_id_, static_cast(td::Clocks::system()), size, is_fec, + /* skip_check_signature = */ cached); + if (r2 != BroadcastCheckResult::Forbidden) { + if (cached_cert == checked_certificates_cache_.end()) { + cached_cert = checked_certificates_cache_.emplace( + source, std::make_unique(source, cert_hash)).first; + } else { + cached_cert->second->cert_hash = cert_hash; + cached_cert->second->remove(); + } + checked_certificates_cache_lru_.put(cached_cert->second.get()); + while (checked_certificates_cache_.size() > max_checked_certificates_cache_size_) { + auto to_remove = (CachedCertificate*)checked_certificates_cache_lru_.get(); + CHECK(to_remove); + to_remove->remove(); + checked_certificates_cache_.erase(to_remove->source); + } + } r2 = broadcast_check_result_min(r2, rules_.check_rules(cert->issuer_hash(), size, is_fec)); return broadcast_check_result_max(r, r2); } +BroadcastCheckResult OverlayImpl::check_source_eligible(PublicKey source, const Certificate* cert, td::uint32 size, + bool is_fec) { + return check_source_eligible(source.compute_short_id(), cert, size, is_fec); +} + td::Status OverlayImpl::check_delivered(BroadcastHash hash) { if (delivered_broadcasts_.count(hash) == 1 || broadcasts_.count(hash) == 1) { return td::Status::Error(ErrorCode::notready, "duplicate broadcast"); @@ -475,21 +575,23 @@ void OverlayImpl::register_fec_broadcast(std::unique_ptr bcast) { } void OverlayImpl::get_self_node(td::Promise promise) { - OverlayNode s{local_id_, overlay_id_}; + OverlayNode s{local_id_, overlay_id_, peer_list_.local_member_flags_}; auto to_sign = s.to_sign(); - auto P = td::PromiseCreator::lambda([oid = print_id(), s = std::move(s), promise = std::move(promise)]( - td::Result> R) mutable { - if (R.is_error()) { - auto S = R.move_as_error(); - LOG(ERROR) << oid << ": failed to get self node: " << S; - promise.set_error(std::move(S)); - return; - } - auto V = R.move_as_ok(); - s.update_signature(std::move(V.first)); - s.update_adnl_id(adnl::AdnlNodeIdFull{V.second}); - promise.set_value(std::move(s)); - }); + auto P = td::PromiseCreator::lambda( + [oid = print_id(), s = std::move(s), cert = peer_list_.cert_, + promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + auto S = R.move_as_error(); + LOG(ERROR) << oid << ": failed to get self node: " << S; + promise.set_error(std::move(S)); + return; + } + auto V = R.move_as_ok(); + s.update_signature(std::move(V.first)); + s.update_adnl_id(adnl::AdnlNodeIdFull{V.second}); + s.update_certificate(std::move(cert)); + promise.set_value(std::move(s)); + }); td::actor::send_closure(keyring_, &keyring::Keyring::sign_add_get_public_key, local_id_.pubkey_hash(), std::move(to_sign), std::move(P)); @@ -581,17 +683,6 @@ void OverlayImpl::check_broadcast(PublicKeyHash src, td::BufferSlice data, td::P callback_->check_broadcast(src, overlay_id_, std::move(data), std::move(promise)); } -void OverlayImpl::update_peer_err_ctr(adnl::AdnlNodeIdShort peer_id, bool is_fec) { - auto src_peer = peers_.get(peer_id); - if(src_peer) { - if(is_fec) { - src_peer->fec_broadcast_errors++; - } else { - src_peer->broadcast_errors++; - } - } -} - void OverlayImpl::broadcast_checked(Overlay::BroadcastHash hash, td::Result R) { { auto it = broadcasts_.find(hash); @@ -613,30 +704,67 @@ void OverlayImpl::get_stats(td::Promiseoverlay_id_ = overlay_id_.bits256_value(); res->overlay_id_full_ = id_full_.pubkey().tl(); res->scope_ = scope_; - peers_.iterate([&](const adnl::AdnlNodeIdShort &key, const OverlayPeer &peer) { + iterate_all_peers([&](const adnl::AdnlNodeIdShort &key, const OverlayPeer &peer) { auto node_obj = create_tl_object(); node_obj->adnl_id_ = key.bits256_value(); - node_obj->t_out_bytes_ = peer.throughput_out_bytes; - node_obj->t_in_bytes_ = peer.throughput_in_bytes; - - node_obj->t_out_pckts_ = peer.throughput_out_packets; - node_obj->t_in_pckts_ = peer.throughput_in_packets; - + node_obj->traffic_ = peer.traffic.tl(); + node_obj->traffic_responses_ = peer.traffic_responses.tl(); node_obj->ip_addr_ = peer.ip_addr_str; - + node_obj->last_in_query_ = static_cast(peer.last_in_query_at.at_unix()); node_obj->last_out_query_ = static_cast(peer.last_out_query_at.at_unix()); - + node_obj->bdcst_errors_ = peer.broadcast_errors; node_obj->fec_bdcst_errors_ = peer.fec_broadcast_errors; - + + node_obj->is_neighbour_ = peer.is_neighbour(); + node_obj->is_alive_ = peer.is_alive(); + node_obj->node_flags_ = peer.get_node()->flags(); + res->nodes_.push_back(std::move(node_obj)); }); + res->total_traffic_ = total_traffic.tl(); + res->total_traffic_responses_ = total_traffic_responses.tl(); res->stats_.push_back( - create_tl_object("neighbours_cnt", PSTRING() << neighbours_.size())); + create_tl_object("neighbours_cnt", PSTRING() << neighbours_cnt())); - promise.set_value(std::move(res)); + callback_->get_stats_extra([promise = std::move(promise), res = std::move(res)](td::Result R) mutable { + if (R.is_ok()) { + res->extra_ = R.move_as_ok(); + } + promise.set_value(std::move(res)); + }); +} + +bool OverlayImpl::has_valid_broadcast_certificate(const PublicKeyHash &source, size_t size, bool is_fec) { + if (size > std::numeric_limits::max()) { + return false; + } + auto it = certs_.find(source); + return check_source_eligible(source, it == certs_.end() ? nullptr : it->second.get(), (td::uint32)size, is_fec) != + BroadcastCheckResult::Forbidden; +} + +void TrafficStats::add_packet(td::uint64 size, bool in) { + if (in) { + ++in_packets; + in_bytes += size; + } else { + ++out_packets; + out_bytes += size; + } +} + +void TrafficStats::normalize(double elapsed) { + out_bytes = static_cast(out_bytes / elapsed); + in_bytes = static_cast(in_bytes / elapsed); + out_packets = static_cast(out_packets / elapsed); + in_packets = static_cast(in_packets / elapsed); +} + +tl_object_ptr TrafficStats::tl() const { + return create_tl_object(out_bytes, in_bytes, out_packets, in_packets); } } // namespace overlay diff --git a/overlay/overlay.h b/overlay/overlay.h index a5f7b3a4..bfab4987 100644 --- a/overlay/overlay.h +++ b/overlay/overlay.h @@ -18,6 +18,7 @@ */ #pragma once +#include "auto/tl/ton_api.h" #include "td/utils/buffer.h" #include "td/utils/int_types.h" @@ -37,23 +38,29 @@ class Overlay : public td::actor::Actor { using BroadcastDataHash = td::Bits256; using BroadcastPartHash = td::Bits256; - static td::actor::ActorOwn create(td::actor::ActorId keyring, - td::actor::ActorId adnl, - td::actor::ActorId manager, - td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, - OverlayIdFull overlay_id, std::unique_ptr callback, - OverlayPrivacyRules rules, td::string scope, bool announce_self = true); - static td::actor::ActorOwn create(td::actor::ActorId keyring, - td::actor::ActorId adnl, - td::actor::ActorId manager, - td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, - OverlayIdFull overlay_id, std::vector nodes, - std::unique_ptr callback, OverlayPrivacyRules rules); + static td::actor::ActorOwn create_public( + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, + OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, + td::string scope, OverlayOptions opts = {}); + static td::actor::ActorOwn create_private( + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, + OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, + OverlayPrivacyRules rules, std::string scope, OverlayOptions opts = {}); + static td::actor::ActorOwn create_semiprivate( + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, + OverlayIdFull overlay_id, std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate cert, std::unique_ptr callback, OverlayPrivacyRules rules, + std::string scope, OverlayOptions opts = {}); virtual void update_dht_node(td::actor::ActorId dht) = 0; - virtual void receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) = 0; - virtual void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) = 0; + virtual void receive_message(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data) = 0; + virtual void receive_query(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data, td::Promise promise) = 0; virtual void send_message_to_neighbours(td::BufferSlice data) = 0; virtual void send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) = 0; virtual void send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) = 0; @@ -63,12 +70,19 @@ class Overlay : public td::actor::Actor { virtual void add_certificate(PublicKeyHash key, std::shared_ptr) = 0; virtual void set_privacy_rules(OverlayPrivacyRules rules) = 0; virtual void receive_nodes_from_db(tl_object_ptr nodes) = 0; + virtual void receive_nodes_from_db_v2(tl_object_ptr nodes) = 0; virtual void get_stats(td::Promise> promise) = 0; - virtual void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) = 0; - virtual void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) = 0; + virtual void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint64 msg_size, bool is_query, + bool is_response) = 0; + virtual void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint64 msg_size, bool is_query, + bool is_response) = 0; virtual void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) = 0; + virtual void update_member_certificate(OverlayMemberCertificate cert) = 0; + virtual void update_root_member_list(std::vector ids, + std::vector root_public_keys, OverlayMemberCertificate cert) = 0; //virtual void receive_broadcast(td::BufferSlice data) = 0; //virtual void subscribe(std::unique_ptr callback) = 0; + virtual void forget_peer(adnl::AdnlNodeIdShort peer_id) = 0; }; } // namespace overlay diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 86d37d5b..41a04dec 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -18,11 +18,15 @@ */ #pragma once +#include +#include #include #include #include +#include #include +#include "adnl/adnl-node-id.hpp" #include "overlay.h" #include "overlay-manager.h" #include "overlay-fec.hpp" @@ -32,6 +36,9 @@ #include "td/utils/DecTree.h" #include "td/utils/List.h" +#include "td/utils/Status.h" +#include "td/utils/Time.h" +#include "td/utils/buffer.h" #include "td/utils/overloaded.h" #include "fec/fec.h" @@ -40,6 +47,8 @@ #include "auto/tl/ton_api.h" #include "auto/tl/ton_api.hpp" +#include "td/utils/port/signals.h" +#include "tl-utils/common-utils.hpp" namespace ton { @@ -50,6 +59,18 @@ namespace overlay { class OverlayImpl; +struct TrafficStats { + td::uint64 out_bytes = 0; + td::uint64 in_bytes = 0; + + td::uint32 out_packets = 0; + td::uint32 in_packets = 0; + + void add_packet(td::uint64 size, bool in); + void normalize(double elapsed); + tl_object_ptr tl() const; +}; + class OverlayPeer { public: adnl::AdnlNodeIdShort get_id() const { @@ -58,15 +79,17 @@ class OverlayPeer { adnl::AdnlNodeIdFull get_full_id() const { return node_.adnl_id_full(); } - OverlayNode get() const { - return node_.clone(); + const OverlayNode *get_node() const { + return &node_; } void update(OverlayNode node) { CHECK(get_id() == node.adnl_id_short()); - if (node.version() > node_.version()) { - node_ = std::move(node); - } + node_.update(std::move(node)); } + void update_certificate(OverlayMemberCertificate cert) { + node_.update_certificate(std::move(cert)); + } + OverlayPeer(OverlayNode node) : node_(std::move(node)) { id_ = node_.adnl_id_short(); } @@ -82,32 +105,48 @@ class OverlayPeer { void on_ping_result(bool success) { if (success) { missed_pings_ = 0; + last_ping_at_ = td::Timestamp::now(); + is_alive_ = true; } else { ++missed_pings_; + if (missed_pings_ >= 3 && last_ping_at_.is_in_past(td::Timestamp::in(-15.0))) { + is_alive_ = false; + } } } bool is_alive() const { - return missed_pings_ < 3; + return is_alive_; } - td::uint32 throughput_out_bytes = 0; - td::uint32 throughput_in_bytes = 0; - - td::uint32 throughput_out_packets = 0; - td::uint32 throughput_in_packets = 0; - - td::uint32 throughput_out_bytes_ctr = 0; - td::uint32 throughput_in_bytes_ctr = 0; - - td::uint32 throughput_out_packets_ctr = 0; - td::uint32 throughput_in_packets_ctr = 0; - + bool is_permanent_member() const { + return is_permanent_member_; + } + + void set_permanent(bool value) { + is_permanent_member_ = value; + } + + void clear_certificate() { + node_.clear_certificate(); + } + + auto certificate() const { + return node_.certificate(); + } + + bool has_full_id() const { + return node_.has_full_id(); + } + + TrafficStats traffic, traffic_ctr; + TrafficStats traffic_responses, traffic_responses_ctr; + td::uint32 broadcast_errors = 0; td::uint32 fec_broadcast_errors = 0; - + td::Timestamp last_in_query_at = td::Timestamp::now(); td::Timestamp last_out_query_at = td::Timestamp::now(); - + td::string ip_addr_str = "undefined"; private: @@ -116,25 +155,32 @@ class OverlayPeer { bool is_neighbour_ = false; size_t missed_pings_ = 0; + bool is_alive_ = true; + bool is_permanent_member_ = false; + td::Timestamp last_ping_at_ = td::Timestamp::now(); }; class OverlayImpl : public Overlay { public: OverlayImpl(td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId manager, td::actor::ActorId dht_node, - adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub, - std::vector nodes, std::unique_ptr callback, - OverlayPrivacyRules rules, td::string scope = "{ \"type\": \"undefined\" }", bool announce_self = true); + adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, OverlayType overlay_type, + std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate cert, std::unique_ptr callback, OverlayPrivacyRules rules, + td::string scope = "{ \"type\": \"undefined\" }", OverlayOptions opts = {}); void update_dht_node(td::actor::ActorId dht) override { dht_node_ = dht; } - void receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) override; - void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) override; + void receive_message(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data) override; + void receive_query(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data, td::Promise promise) override; void send_message_to_neighbours(td::BufferSlice data) override; void send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) override; void send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) override; void receive_nodes_from_db(tl_object_ptr nodes) override; + void receive_nodes_from_db_v2(tl_object_ptr nodes) override; void get_self_node(td::Promise promise); @@ -142,8 +188,8 @@ class OverlayImpl : public Overlay { void start_up() override { update_throughput_at_ = td::Timestamp::in(50.0); last_throughput_update_ = td::Timestamp::now(); - - if (public_) { + + if (overlay_type_ == OverlayType::Public) { update_db_at_ = td::Timestamp::in(60.0); } alarm_timestamp() = td::Timestamp::in(1); @@ -151,15 +197,20 @@ class OverlayImpl : public Overlay { void on_ping_result(adnl::AdnlNodeIdShort peer, bool success); void receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R); + void receive_random_peers_v2(adnl::AdnlNodeIdShort src, td::Result R); void send_random_peers(adnl::AdnlNodeIdShort dst, td::Promise promise); + void send_random_peers_v2(adnl::AdnlNodeIdShort dst, td::Promise promise); void send_random_peers_cont(adnl::AdnlNodeIdShort dst, OverlayNode node, td::Promise promise); + void send_random_peers_v2_cont(adnl::AdnlNodeIdShort dst, OverlayNode node, td::Promise promise); void get_overlay_random_peers(td::uint32 max_peers, td::Promise> promise) override; void set_privacy_rules(OverlayPrivacyRules rules) override; void add_certificate(PublicKeyHash key, std::shared_ptr cert) override { certs_[key] = std::move(cert); } + void update_member_certificate(OverlayMemberCertificate cert) override; - void receive_dht_nodes(td::Result res, bool dummy); + void receive_dht_nodes(dht::DhtValue v); + void dht_lookup_finished(td::Status S); void update_dht_nodes(OverlayNode node); void update_neighbours(td::uint32 nodes_to_change); @@ -180,6 +231,8 @@ class OverlayImpl : public Overlay { td::Status check_date(td::uint32 date); BroadcastCheckResult check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size, bool is_fec); + BroadcastCheckResult check_source_eligible(const PublicKeyHash &source, const Certificate *cert, td::uint32 size, + bool is_fec); td::Status check_delivered(BroadcastHash hash); void broadcast_checked(Overlay::BroadcastHash hash, td::Result R); @@ -198,17 +251,7 @@ class OverlayImpl : public Overlay { void send_new_fec_broadcast_part(PublicKeyHash local_id, Overlay::BroadcastDataHash data_hash, td::uint32 size, td::uint32 flags, td::BufferSlice part, td::uint32 seqno, fec::FecType fec_type, td::uint32 date); - std::vector get_neighbours(td::uint32 max_size = 0) const { - if (max_size == 0 || max_size >= neighbours_.size()) { - return neighbours_; - } else { - std::vector vec; - for (td::uint32 i = 0; i < max_size; i++) { - vec.push_back(neighbours_[td::Random::fast(0, static_cast(neighbours_.size()) - 1)]); - } - return vec; - } - } + std::vector get_neighbours(td::uint32 max_size = 0) const; td::actor::ActorId overlay_manager() const { return manager_; } @@ -228,38 +271,58 @@ class OverlayImpl : public Overlay { td::Result get_encryptor(PublicKey source); void get_stats(td::Promise> promise) override; - - void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override { - auto out_peer = peers_.get(peer_id); - if(out_peer) { - out_peer->throughput_out_bytes_ctr += msg_size; - out_peer->throughput_out_packets_ctr++; - - if(is_query) - { - out_peer->last_out_query_at = td::Timestamp::now(); - } - } + + void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint64 msg_size, bool is_query, + bool is_response) override; + + void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint64 msg_size, bool is_query, + bool is_response) override; + + void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) override; + + void update_root_member_list(std::vector ids, std::vector root_public_keys, + OverlayMemberCertificate cert) override; + + bool is_valid_peer(const adnl::AdnlNodeIdShort &id, const ton_api::overlay_MemberCertificate *certificate); + bool is_persistent_node(const adnl::AdnlNodeIdShort &id); + + td::uint32 max_data_bcasts() const { + return 100; } - - void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override { - auto in_peer = peers_.get(peer_id); - if(in_peer) { - in_peer->throughput_in_bytes_ctr += msg_size; - in_peer->throughput_in_packets_ctr++; - - if(is_query) - { - in_peer->last_in_query_at = td::Timestamp::now(); - } - } + td::uint32 max_bcasts() const { + return 1000; } - - void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) override { - auto fpeer = peers_.get(peer_id); - if(fpeer) { - fpeer->ip_addr_str = ip_str; - } + td::uint32 max_fec_bcasts() const { + return 20; + } + td::uint32 max_sources() const { + return 10; + } + td::uint32 max_encryptors() const { + return 16; + } + + td::uint32 max_neighbours() const { + return opts_.max_neighbours_; + } + + td::uint32 max_peers() const { + return opts_.max_peers_; + } + + td::uint32 nodes_to_send() const { + return opts_.nodes_to_send_; + } + + td::uint32 propagate_broadcast_to() const { + return opts_.propagate_broadcast_to_; + } + + bool has_valid_membership_certificate(); + bool has_valid_broadcast_certificate(const PublicKeyHash &source, size_t size, bool is_fec); + + void forget_peer(adnl::AdnlNodeIdShort peer_id) override { + del_peer(peer_id); } private: @@ -270,6 +333,8 @@ class OverlayImpl : public Overlay { void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeers &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeersV2 &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcast &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcastList &query, @@ -286,14 +351,33 @@ class OverlayImpl : public Overlay { td::Status process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg); td::Status process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg); - void do_add_peer(OverlayNode node); - void add_peer_in_cont(OverlayNode node); - void add_peer_in(OverlayNode node); + td::Status validate_peer_certificate(const adnl::AdnlNodeIdShort &node, const OverlayMemberCertificate &cert); + td::Status validate_peer_certificate(const adnl::AdnlNodeIdShort &node, const OverlayMemberCertificate *cert); + td::Status validate_peer_certificate(const adnl::AdnlNodeIdShort &node, ton_api::overlay_MemberCertificate *cert); void add_peer(OverlayNode node); void add_peers(std::vector nodes); + void add_peers(const tl_object_ptr &nodes); + void add_peers(const tl_object_ptr &nodes); void del_some_peers(); - void del_peer(adnl::AdnlNodeIdShort id); + void del_peer(const adnl::AdnlNodeIdShort &id); + void del_from_neighbour_list(OverlayPeer *P); + void del_from_neighbour_list(const adnl::AdnlNodeIdShort &id); + void iterate_all_peers(std::function cb); OverlayPeer *get_random_peer(bool only_alive = false); + bool is_root_public_key(const PublicKeyHash &key) const; + bool has_good_peers() const; + size_t neighbours_cnt() const; + + void finish_dht_query() { + if (!next_dht_store_query_) { + next_dht_store_query_ = td::Timestamp::in(td::Random::fast(60.0, 100.0)); + } + if (frequent_dht_lookup_ && !has_good_peers()) { + next_dht_query_ = td::Timestamp::in(td::Random::fast(6.0, 10.0)); + } else { + next_dht_query_ = next_dht_store_query_; + } + } td::actor::ActorId keyring_; td::actor::ActorId adnl_; @@ -303,13 +387,12 @@ class OverlayImpl : public Overlay { OverlayIdFull id_full_; OverlayIdShort overlay_id_; - td::DecTree peers_; td::Timestamp next_dht_query_ = td::Timestamp::in(1.0); + td::Timestamp next_dht_store_query_ = td::Timestamp::in(1.0); td::Timestamp update_db_at_; td::Timestamp update_throughput_at_; + td::Timestamp update_neighbours_at_; td::Timestamp last_throughput_update_; - std::set bad_peers_; - adnl::AdnlNodeIdShort next_bad_peer_ = adnl::AdnlNodeIdShort::zero(); std::unique_ptr callback_; @@ -317,7 +400,6 @@ class OverlayImpl : public Overlay { std::map> fec_broadcasts_; std::set delivered_broadcasts_; - std::vector neighbours_; td::ListNode bcast_data_lru_; td::ListNode bcast_fec_lru_; std::queue bcast_lru_; @@ -326,33 +408,6 @@ class OverlayImpl : public Overlay { void bcast_gc(); - static td::uint32 max_data_bcasts() { - return 100; - } - static td::uint32 max_bcasts() { - return 1000; - } - static td::uint32 max_fec_bcasts() { - return 20; - } - static td::uint32 max_sources() { - return 10; - } - static td::uint32 max_neighbours() { - return 5; - } - static td::uint32 max_encryptors() { - return 16; - } - - static td::uint32 max_peers() { - return 20; - } - - static td::uint32 nodes_to_send() { - return 4; - } - static BroadcastHash get_broadcast_hash(adnl::AdnlNodeIdShort &src, td::Bits256 &data_hash) { td::uint8 buf[64]; td::MutableSlice m{buf, 64}; @@ -362,11 +417,11 @@ class OverlayImpl : public Overlay { return td::sha256_bits256(td::Slice(buf, 64)); } - bool public_; - bool semi_public_ = false; + OverlayType overlay_type_; OverlayPrivacyRules rules_; td::string scope_; bool announce_self_ = true; + bool frequent_dht_lookup_ = false; std::map> certs_; class CachedEncryptor : public td::ListNode { @@ -391,6 +446,40 @@ class OverlayImpl : public Overlay { td::ListNode encryptor_lru_; std::map> encryptor_map_; + + struct PeerList { + struct SlaveKey { + td::int32 expire_at{0}; + adnl::AdnlNodeIdShort node{}; + }; + using SlaveKeys = std::vector; + std::map root_public_keys_; + OverlayMemberCertificate cert_; + std::set bad_peers_; + adnl::AdnlNodeIdShort next_bad_peer_ = adnl::AdnlNodeIdShort::zero(); + td::DecTree peers_; + std::vector neighbours_; + + td::Timestamp local_cert_is_valid_until_; + td::uint32 local_member_flags_{0}; + } peer_list_; + TrafficStats total_traffic, total_traffic_ctr; + TrafficStats total_traffic_responses, total_traffic_responses_ctr; + + OverlayOptions opts_; + + struct CachedCertificate : td::ListNode { + CachedCertificate(PublicKeyHash source, td::Bits256 cert_hash) + : source(source) + , cert_hash(cert_hash) { + } + + PublicKeyHash source; + td::Bits256 cert_hash; + }; + std::map> checked_certificates_cache_; + td::ListNode checked_certificates_cache_lru_; + size_t max_checked_certificates_cache_size_ = 1000; }; } // namespace overlay diff --git a/overlay/overlays.h b/overlay/overlays.h index e12bbbdb..5eb63b13 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -18,7 +18,9 @@ */ #pragma once +#include "adnl/adnl-node-id.hpp" #include "adnl/adnl.h" +#include "auto/tl/ton_api.h" #include "dht/dht.h" #include "td/actor/PromiseFuture.h" @@ -33,6 +35,8 @@ namespace ton { namespace overlay { +enum class OverlayType { Public, FixedMemberList, CertificatedMembers }; + class OverlayIdShort { public: OverlayIdShort() { @@ -88,6 +92,10 @@ struct CertificateFlags { enum Values : td::uint32 { AllowFec = 1, Trusted = 2 }; }; +struct OverlayMemberFlags { + enum Values : td::uint32 { DoNotReceiveBroadcasts = 1 }; +}; + enum BroadcastCheckResult { Forbidden = 1, NeedCheck = 2, Allowed = 3 }; inline BroadcastCheckResult broadcast_check_result_max(BroadcastCheckResult l, BroadcastCheckResult r) { @@ -108,7 +116,6 @@ class OverlayPrivacyRules { } BroadcastCheckResult check_rules(PublicKeyHash hash, td::uint32 size, bool is_fec) { - auto it = authorized_keys_.find(hash); if (it == authorized_keys_.end()) { if (size > max_unath_size_) { @@ -142,7 +149,7 @@ class Certificate { td::BufferSlice to_sign(OverlayIdShort overlay_id, PublicKeyHash issued_to) const; BroadcastCheckResult check(PublicKeyHash node, OverlayIdShort overlay_id, td::int32 unix_time, td::uint32 size, - bool is_fec) const; + bool is_fec, bool skip_check_signature = false) const; tl_object_ptr tl() const; const PublicKey &issuer() const; const PublicKeyHash issuer_hash() const; @@ -158,6 +165,113 @@ class Certificate { td::SharedSlice signature_; }; +class OverlayMemberCertificate { + public: + OverlayMemberCertificate() { + expire_at_ = std::numeric_limits::max(); + } + OverlayMemberCertificate(PublicKey signed_by, td::uint32 flags, td::int32 slot, td::int32 expire_at, + td::BufferSlice signature) + : signed_by_(std::move(signed_by)) + , flags_(flags) + , slot_(slot) + , expire_at_(expire_at) + , signature_(std::move(signature)) { + } + OverlayMemberCertificate(const OverlayMemberCertificate &other) + : signed_by_(other.signed_by_) + , flags_(other.flags_) + , slot_(other.slot_) + , expire_at_(other.expire_at_) + , signature_(other.signature_.clone()) { + } + OverlayMemberCertificate(OverlayMemberCertificate &&) = default; + OverlayMemberCertificate &operator=(OverlayMemberCertificate &&) = default; + OverlayMemberCertificate &operator=(const OverlayMemberCertificate &other) { + signed_by_ = other.signed_by_; + flags_ = other.flags_; + slot_ = other.slot_; + expire_at_ = other.expire_at_; + signature_ = other.signature_.clone(); + return *this; + } + explicit OverlayMemberCertificate(const ton_api::overlay_MemberCertificate *cert); + td::Status check_signature(const adnl::AdnlNodeIdShort &node); + + bool is_expired() const { + return expire_at_ < td::Clocks::system() - 3; + } + + bool is_expired(double cur_time) const { + return expire_at_ < cur_time - 3; + } + + tl_object_ptr tl() const { + if (empty()) { + return create_tl_object(); + } + return create_tl_object(signed_by_.tl(), flags_, slot_, expire_at_, + signature_.clone_as_buffer_slice()); + } + + const auto &issued_by() const { + return signed_by_; + } + + td::Slice signature() const { + return signature_.as_slice(); + } + + td::BufferSlice to_sign_data(const adnl::AdnlNodeIdShort &node) const { + return ton::create_serialize_tl_object(node.tl(), flags_, slot_, + expire_at_); + } + + bool empty() const { + return signed_by_.empty(); + } + + bool is_newer(const OverlayMemberCertificate &other) const { + return !empty() && expire_at_ > other.expire_at_; + } + + auto slot() const { + return slot_; + } + + auto expire_at() const { + return expire_at_; + } + + void set_signature(td::Slice signature) { + signature_ = td::SharedSlice(signature); + } + void set_signature(td::SharedSlice signature) { + signature_ = std::move(signature); + } + + private: + PublicKey signed_by_; + td::uint32 flags_; + td::int32 slot_; + td::int32 expire_at_ = std::numeric_limits::max(); + td::SharedSlice signature_; +}; + + +struct OverlayOptions { + bool announce_self_ = true; + bool frequent_dht_lookup_ = false; + td::uint32 local_overlay_member_flags_ = 0; + td::int32 max_slaves_in_semiprivate_overlay_ = 5; + td::uint32 max_peers_ = 20; + td::uint32 max_neighbours_ = 5; + td::uint32 nodes_to_send_ = 4; + td::uint32 propagate_broadcast_to_ = 5; + td::uint32 default_permanent_members_flags_ = 0; + double broadcast_speed_multiplier_ = 1.0; +}; + class Overlays : public td::actor::Actor { public: class Callback { @@ -170,6 +284,9 @@ class Overlays : public td::actor::Actor { td::Promise promise) { promise.set_value(td::Unit()); } + virtual void get_stats_extra(td::Promise promise) { + promise.set_result(""); + } virtual ~Callback() = default; }; @@ -187,6 +304,10 @@ class Overlays : public td::actor::Actor { return 1; } + static constexpr td::uint32 overlay_peer_ttl() { + return 600; + } + static td::actor::ActorOwn create(std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId dht); @@ -196,11 +317,20 @@ class Overlays : public td::actor::Actor { std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) = 0; virtual void create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - std::unique_ptr callback, OverlayPrivacyRules rules, - td::string scope, bool announce_self) = 0; + std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope, + OverlayOptions opts) = 0; + virtual void create_semiprivate_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, + std::vector root_public_keys, + OverlayMemberCertificate certificate, + std::unique_ptr callback, OverlayPrivacyRules rules, + td::string scope, OverlayOptions opts) = 0; virtual void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, - OverlayPrivacyRules rules) = 0; + OverlayPrivacyRules rules, std::string scope) = 0; + virtual void create_private_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, std::unique_ptr callback, + OverlayPrivacyRules rules, std::string scope, OverlayOptions opts) = 0; virtual void delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id) = 0; virtual void send_query(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, @@ -234,9 +364,18 @@ class Overlays : public td::actor::Actor { virtual void update_certificate(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, PublicKeyHash key, std::shared_ptr cert) = 0; + virtual void update_member_certificate(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + OverlayMemberCertificate certificate) = 0; + virtual void update_root_member_list(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + std::vector nodes, + std::vector root_public_keys, + OverlayMemberCertificate certificate) = 0; + virtual void get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, td::uint32 max_peers, td::Promise> promise) = 0; virtual void get_stats(td::Promise> promise) = 0; + + virtual void forget_peer(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, adnl::AdnlNodeIdShort peer_id) = 0; }; } // namespace overlay diff --git a/recent_changelog.md b/recent_changelog.md new file mode 100644 index 00000000..820d2aa4 --- /dev/null +++ b/recent_changelog.md @@ -0,0 +1,13 @@ +## 2025.03 Update +1. New extracurrency behavior introduced, check [GlobalVersions.md](./doc/GlobalVersions.md#version-10) +2. Optmization of validation process, in particular CellStorageStat. +3. Flag for speeding up broadcasts in various overlays. +4. Fixes for static builds for emulator and tonlibjson +5. Improving getstats output: add + * Liteserver queries count + * Collated/validated blocks count, number of active sessions + * Persistent state sizes + * Initial sync progress +6. Fixes in logging, TON Storage, external message checking, persistent state downloading, UB in tonlib + +Besides the work of the core team, this update is based on the efforts of @Sild from StonFi(UB in tonlib). diff --git a/rldp-http-proxy/CMakeLists.txt b/rldp-http-proxy/CMakeLists.txt index eefd1dd8..f7e30c80 100644 --- a/rldp-http-proxy/CMakeLists.txt +++ b/rldp-http-proxy/CMakeLists.txt @@ -1,5 +1,7 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) add_executable(rldp-http-proxy rldp-http-proxy.cpp DNSResolver.h DNSResolver.cpp) target_include_directories(rldp-http-proxy PUBLIC $) -target_link_libraries(rldp-http-proxy PRIVATE tonhttp rldp dht tonlib git) +target_link_libraries(rldp-http-proxy PRIVATE tonhttp rldp rldp2 dht tonlib git) + +install(TARGETS rldp-http-proxy RUNTIME DESTINATION bin) diff --git a/rldp-http-proxy/rldp-http-proxy.cpp b/rldp-http-proxy/rldp-http-proxy.cpp index eb04ef9e..0d518d6d 100644 --- a/rldp-http-proxy/rldp-http-proxy.cpp +++ b/rldp-http-proxy/rldp-http-proxy.cpp @@ -45,6 +45,7 @@ #include "adnl/adnl.h" #include "rldp/rldp.h" +#include "rldp2/rldp.h" #include "dht/dht.h" #include @@ -53,6 +54,7 @@ #include "git.h" #include "td/utils/BufferedFd.h" #include "common/delay.h" +#include "td/utils/port/path.h" #include "tonlib/tonlib/TonlibClientWrapper.h" #include "DNSResolver.h" @@ -63,6 +65,53 @@ class RldpHttpProxy; +class RldpDispatcher : public ton::adnl::AdnlSenderInterface { + public: + RldpDispatcher(td::actor::ActorId rldp, td::actor::ActorId rldp2) + : rldp_(std::move(rldp)), rldp2_(std::move(rldp2)) { + } + + void send_message(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { + td::actor::send_closure(dispatch(dst), &ton::adnl::AdnlSenderInterface::send_message, src, dst, std::move(data)); + } + + void send_query(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, std::string name, + td::Promise promise, td::Timestamp timeout, td::BufferSlice data) override { + td::actor::send_closure(dispatch(dst), &ton::adnl::AdnlSenderInterface::send_query, src, dst, std::move(name), + std::move(promise), timeout, std::move(data)); + } + void send_query_ex(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, std::string name, + td::Promise promise, td::Timestamp timeout, td::BufferSlice data, + td::uint64 max_answer_size) override { + td::actor::send_closure(dispatch(dst), &ton::adnl::AdnlSenderInterface::send_query_ex, src, dst, std::move(name), + std::move(promise), timeout, std::move(data), max_answer_size); + } + void get_conn_ip_str(ton::adnl::AdnlNodeIdShort l_id, ton::adnl::AdnlNodeIdShort p_id, + td::Promise promise) override { + td::actor::send_closure(rldp_, &ton::adnl::AdnlSenderInterface::get_conn_ip_str, l_id, p_id, std::move(promise)); + } + + void set_supports_rldp2(ton::adnl::AdnlNodeIdShort dst, bool supports) { + if (supports) { + supports_rldp2_.insert(dst); + } else { + supports_rldp2_.erase(dst); + } + } + + private: + td::actor::ActorId rldp_; + td::actor::ActorId rldp2_; + std::set supports_rldp2_; + + td::actor::ActorId dispatch(ton::adnl::AdnlNodeIdShort dst) const { + if (supports_rldp2_.count(dst)) { + return rldp2_; + } + return rldp_; + } +}; + class HttpRemote : public td::actor::Actor { public: struct Query { @@ -137,6 +186,8 @@ const std::string PROXY_SITE_VERISON_HEADER_NAME = "Ton-Proxy-Site-Version"; const std::string PROXY_ENTRY_VERISON_HEADER_NAME = "Ton-Proxy-Entry-Version"; const std::string PROXY_VERSION_HEADER = PSTRING() << "Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate(); +const td::uint64 CAPABILITY_RLDP2 = 1; +const td::uint64 CAPABILITIES = 1; using RegisteredPayloadSenderGuard = std::unique_ptr, td::Bits256>, @@ -146,8 +197,8 @@ class HttpRldpPayloadReceiver : public td::actor::Actor { public: HttpRldpPayloadReceiver(std::shared_ptr payload, td::Bits256 transfer_id, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort local_id, - td::actor::ActorId adnl, td::actor::ActorId rldp, - bool is_tunnel = false) + td::actor::ActorId adnl, + td::actor::ActorId rldp, bool is_tunnel = false) : payload_(std::move(payload)) , id_(transfer_id) , src_(src) @@ -204,8 +255,8 @@ class HttpRldpPayloadReceiver : public td::actor::Actor { auto f = ton::create_serialize_tl_object( id_, seqno_++, static_cast(chunk_size())); auto timeout = td::Timestamp::in(is_tunnel_ ? 60.0 : 15.0); - td::actor::send_closure(rldp_, &ton::rldp::Rldp::send_query_ex, local_id_, src_, "payload part", std::move(P), - timeout, std::move(f), 2 * chunk_size() + 1024); + td::actor::send_closure(rldp_, &ton::adnl::AdnlSenderInterface::send_query_ex, local_id_, src_, "payload part", + std::move(P), timeout, std::move(f), 2 * chunk_size() + 1024); } void add_data(td::BufferSlice data) { @@ -265,7 +316,7 @@ class HttpRldpPayloadReceiver : public td::actor::Actor { ton::adnl::AdnlNodeIdShort src_; ton::adnl::AdnlNodeIdShort local_id_; td::actor::ActorId adnl_; - td::actor::ActorId rldp_; + td::actor::ActorId rldp_; bool sent_ = false; td::int32 seqno_ = 0; @@ -276,8 +327,8 @@ class HttpRldpPayloadSender : public td::actor::Actor { public: HttpRldpPayloadSender(std::shared_ptr payload, td::Bits256 transfer_id, ton::adnl::AdnlNodeIdShort local_id, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId proxy, - bool is_tunnel = false) + td::actor::ActorId rldp, + td::actor::ActorId proxy, bool is_tunnel = false) : payload_(std::move(payload)) , id_(transfer_id) , local_id_(local_id) @@ -407,7 +458,7 @@ class HttpRldpPayloadSender : public td::actor::Actor { ton::adnl::AdnlNodeIdShort local_id_; td::actor::ActorId adnl_; - td::actor::ActorId rldp_; + td::actor::ActorId rldp_; td::actor::ActorId proxy_; size_t cur_query_size_; @@ -424,9 +475,8 @@ class TcpToRldpRequestSender : public td::actor::Actor { std::shared_ptr request_payload, td::Promise, std::shared_ptr>> promise, td::actor::ActorId adnl, td::actor::ActorId dht, - td::actor::ActorId rldp, td::actor::ActorId proxy, - td::actor::ActorId dns_resolver, - ton::adnl::AdnlNodeIdShort storage_gateway) + td::actor::ActorId rldp, td::actor::ActorId proxy, + td::actor::ActorId dns_resolver, ton::adnl::AdnlNodeIdShort storage_gateway) : local_id_(local_id) , host_(std::move(host)) , request_(std::move(request)) @@ -447,26 +497,7 @@ class TcpToRldpRequestSender : public td::actor::Actor { } void resolve(std::string host); - - void resolved(ton::adnl::AdnlNodeIdShort id) { - dst_ = id; - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &TcpToRldpRequestSender::abort_query, R.move_as_error()); - } else { - td::actor::send_closure(SelfId, &TcpToRldpRequestSender::got_result, R.move_as_ok()); - } - }); - - td::actor::create_actor("HttpPayloadSender", request_payload_, id_, local_id_, adnl_, rldp_, - proxy_, is_tunnel()) - .release(); - - auto f = ton::serialize_tl_object(request_tl_, true); - td::actor::send_closure(rldp_, &ton::rldp::Rldp::send_query_ex, local_id_, dst_, "http request over rldp", - std::move(P), td::Timestamp::in(30.0), std::move(f), 16 << 10); - } + void resolved(ton::adnl::AdnlNodeIdShort id); void got_result(td::BufferSlice data) { auto F = ton::fetch_tl_object(data, true); @@ -548,7 +579,7 @@ class TcpToRldpRequestSender : public td::actor::Actor { td::actor::ActorId adnl_; td::actor::ActorId dht_; - td::actor::ActorId rldp_; + td::actor::ActorId rldp_; td::actor::ActorId proxy_; td::actor::ActorId dns_resolver_; ton::adnl::AdnlNodeIdShort storage_gateway_ = ton::adnl::AdnlNodeIdShort::zero(); @@ -562,7 +593,7 @@ class TcpToRldpRequestSender : public td::actor::Actor { class RldpTcpTunnel : public td::actor::Actor, private td::ObserverBase { public: RldpTcpTunnel(td::Bits256 transfer_id, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort local_id, - td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId proxy, td::SocketFd fd) : id_(transfer_id) , src_(src) @@ -599,8 +630,8 @@ class RldpTcpTunnel : public td::actor::Actor, private td::ObserverBase { auto f = ton::create_serialize_tl_object(id_, out_seqno_++, (1 << 21) - (1 << 11)); - td::actor::send_closure(rldp_, &ton::rldp::Rldp::send_query_ex, local_id_, src_, "payload part", std::move(P), - td::Timestamp::in(60.0), std::move(f), (1 << 21) + 1024); + td::actor::send_closure(rldp_, &ton::adnl::AdnlSenderInterface::send_query_ex, local_id_, src_, "payload part", + std::move(P), td::Timestamp::in(60.0), std::move(f), (1 << 21) + 1024); } void receive_query(ton::tl_object_ptr f, @@ -727,7 +758,7 @@ class RldpTcpTunnel : public td::actor::Actor, private td::ObserverBase { ton::adnl::AdnlNodeIdShort src_; ton::adnl::AdnlNodeIdShort local_id_; td::actor::ActorId adnl_; - td::actor::ActorId rldp_; + td::actor::ActorId rldp_; td::actor::ActorId proxy_; td::BufferedFd fd_; @@ -746,7 +777,8 @@ class RldpToTcpRequestSender : public td::actor::Actor { RldpToTcpRequestSender(td::Bits256 id, ton::adnl::AdnlNodeIdShort local_id, ton::adnl::AdnlNodeIdShort dst, std::unique_ptr request, std::shared_ptr request_payload, td::Promise promise, - td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId proxy, td::actor::ActorId remote) : id_(id) , local_id_(local_id) @@ -806,7 +838,7 @@ class RldpToTcpRequestSender : public td::actor::Actor { td::Promise promise_; td::actor::ActorId adnl_; - td::actor::ActorId rldp_; + td::actor::ActorId rldp_; td::actor::ActorId proxy_; td::actor::ActorId remote_; @@ -889,6 +921,12 @@ class RldpHttpProxy : public td::actor::Actor { } void run() { + if (!db_root_.empty()) { + td::mkpath(db_root_ + "/").ensure(); + } else if (!is_client_) { + LOG(ERROR) << "DB root is required for server proxy"; + std::_Exit(2); + } keyring_ = ton::keyring::Keyring::create(is_client_ ? std::string("") : (db_root_ + "/keyring")); { auto S = load_global_config(); @@ -924,9 +962,16 @@ class RldpHttpProxy : public td::actor::Actor { auto conf_dataR = td::read_file(global_config_); conf_dataR.ensure(); + ton::tl_object_ptr key_store; + if (db_root_.empty()) { + key_store = tonlib_api::make_object(); + } else { + td::mkpath(db_root_ + "/tonlib-cache/").ensure(); + key_store = tonlib_api::make_object(db_root_ + "/tonlib-cache/"); + } auto tonlib_options = tonlib_api::make_object( tonlib_api::make_object(conf_dataR.move_as_ok().as_slice().str(), "", false, false), - tonlib_api::make_object()); + std::move(key_store)); tonlib_client_ = td::actor::create_actor("tonlibclient", std::move(tonlib_options)); dns_resolver_ = td::actor::create_actor("dnsresolver", tonlib_client_.get()); } @@ -1032,35 +1077,58 @@ class RldpHttpProxy : public td::actor::Actor { private: td::actor::ActorId self_id_; }; - for (auto &serv_id : server_ids_) { - class AdnlCb : public ton::adnl::Adnl::Callback { - public: - AdnlCb(td::actor::ActorId id) : self_id_(id) { - } - void receive_message(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, - td::BufferSlice data) override { - } - void receive_query(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, - td::Promise promise) override { - td::actor::send_closure(self_id_, &RldpHttpProxy::receive_rldp_request, src, dst, std::move(data), - std::move(promise)); - } + class AdnlCapabilitiesCb : public ton::adnl::Adnl::Callback { + public: + AdnlCapabilitiesCb(td::actor::ActorId id) : self_id_(id) { + } + void receive_message(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, + td::BufferSlice data) override { + } + void receive_query(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + TRY_RESULT_PROMISE(promise, query, ton::fetch_tl_object(data, true)); + promise.set_result(ton::create_serialize_tl_object(CAPABILITIES)); + td::actor::send_closure(self_id_, &RldpHttpProxy::update_peer_capabilities, src, query->capabilities_); + } - private: - td::actor::ActorId self_id_; - }; + private: + td::actor::ActorId self_id_; + }; + class AdnlServerCb : public ton::adnl::Adnl::Callback { + public: + AdnlServerCb(td::actor::ActorId id) : self_id_(id) { + } + void receive_message(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, + td::BufferSlice data) override { + } + void receive_query(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(self_id_, &RldpHttpProxy::receive_rldp_request, src, dst, std::move(data), + std::move(promise)); + } + + private: + td::actor::ActorId self_id_; + }; + for (auto &serv_id : server_ids_) { td::actor::send_closure(adnl_, &ton::adnl::Adnl::subscribe, serv_id, ton::adnl::Adnl::int_to_bytestring(ton::ton_api::http_request::ID), - std::make_unique(actor_id(this))); + std::make_unique(actor_id(this))); if (local_id_ != serv_id) { td::actor::send_closure(adnl_, &ton::adnl::Adnl::subscribe, serv_id, ton::adnl::Adnl::int_to_bytestring(ton::ton_api::http_getNextPayloadPart::ID), std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_, &ton::adnl::Adnl::subscribe, serv_id, + ton::adnl::Adnl::int_to_bytestring(ton::ton_api::http_proxy_getCapabilities::ID), + std::make_unique(actor_id(this))); } } td::actor::send_closure(adnl_, &ton::adnl::Adnl::subscribe, local_id_, ton::adnl::Adnl::int_to_bytestring(ton::ton_api::http_getNextPayloadPart::ID), std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_, &ton::adnl::Adnl::subscribe, local_id_, + ton::adnl::Adnl::int_to_bytestring(ton::ton_api::http_proxy_getCapabilities::ID), + std::make_unique(actor_id(this))); rldp_ = ton::rldp::Rldp::create(adnl_.get()); td::actor::send_closure(rldp_, &ton::rldp::Rldp::set_default_mtu, 16 << 10); @@ -1069,6 +1137,15 @@ class RldpHttpProxy : public td::actor::Actor { td::actor::send_closure(rldp_, &ton::rldp::Rldp::add_id, serv_id); } + rldp2_ = ton::rldp2::Rldp::create(adnl_.get()); + td::actor::send_closure(rldp2_, &ton::rldp2::Rldp::set_default_mtu, 16 << 10); + td::actor::send_closure(rldp2_, &ton::rldp2::Rldp::add_id, local_id_); + for (auto &serv_id : server_ids_) { + td::actor::send_closure(rldp2_, &ton::rldp2::Rldp::add_id, serv_id); + } + + rldp_dispatcher_ = td::actor::create_actor("RldpDispatcher", rldp_.get(), rldp2_.get()); + store_dht(); } @@ -1107,7 +1184,7 @@ class RldpHttpProxy : public td::actor::Actor { } std::transform(host.begin(), host.end(), host.begin(), [](unsigned char c) { return std::tolower(c); }); bool allow = proxy_all_; - for (const char* suffix : {".adnl", ".ton", ".bag"}) { + for (const char *suffix : {".adnl", ".ton", ".bag"}) { if (td::ends_with(host, td::Slice(suffix))) { allow = true; } @@ -1117,9 +1194,9 @@ class RldpHttpProxy : public td::actor::Actor { return; } - td::actor::create_actor("outboundreq", local_id_, host, std::move(request), - std::move(payload), std::move(promise), adnl_.get(), dht_.get(), - rldp_.get(), actor_id(this), dns_resolver_.get(), storage_gateway_) + td::actor::create_actor( + "outboundreq", local_id_, host, std::move(request), std::move(payload), std::move(promise), adnl_.get(), + dht_.get(), rldp_dispatcher_.get(), actor_id(this), dns_resolver_.get(), storage_gateway_) .release(); } @@ -1127,6 +1204,7 @@ class RldpHttpProxy : public td::actor::Actor { td::Promise promise) { LOG(INFO) << "got HTTP request over rldp from " << src; TRY_RESULT_PROMISE(promise, f, ton::fetch_tl_object(data, true)); + ask_peer_capabilities(src); std::unique_ptr request; auto S = [&]() { TRY_RESULT_ASSIGN(request, ton::http::HttpRequest::create(f->method_, f->url_, f->http_version_)); @@ -1214,8 +1292,8 @@ class RldpHttpProxy : public td::actor::Actor { LOG(INFO) << "starting HTTP over RLDP request"; td::actor::create_actor("inboundreq", f->id_, dst, src, std::move(request), - payload.move_as_ok(), std::move(promise), adnl_.get(), rldp_.get(), - actor_id(this), server.http_remote_.get()) + payload.move_as_ok(), std::move(promise), adnl_.get(), + rldp_dispatcher_.get(), actor_id(this), server.http_remote_.get()) .release(); } @@ -1227,7 +1305,7 @@ class RldpHttpProxy : public td::actor::Actor { return; } td::actor::create_actor(td::actor::ActorOptions().with_name("tunnel").with_poll(), id, src, local_id, - adnl_.get(), rldp_.get(), actor_id(this), fd.move_as_ok()) + adnl_.get(), rldp_dispatcher_.get(), actor_id(this), fd.move_as_ok()) .release(); std::vector> headers; headers.push_back( @@ -1291,6 +1369,47 @@ class RldpHttpProxy : public td::actor::Actor { storage_gateway_ = id; } + void update_peer_capabilities(ton::adnl::AdnlNodeIdShort peer, td::uint64 capabilities) { + auto &c = peer_capabilities_[peer]; + if (c.capabilities != capabilities) { + LOG(DEBUG) << "Update capabilities of peer " << peer << " : " << capabilities; + } + c.capabilities = capabilities; + c.received = true; + td::actor::send_closure(rldp_dispatcher_, &RldpDispatcher::set_supports_rldp2, peer, + capabilities & CAPABILITY_RLDP2); + } + + void ask_peer_capabilities(ton::adnl::AdnlNodeIdShort peer) { + auto &c = peer_capabilities_[peer]; + if (!c.received && c.retry_at.is_in_past()) { + c.retry_at = td::Timestamp::in(30.0); + auto send_query = [&, this, SelfId = actor_id(this)](const ton::adnl::AdnlNodeIdShort &local_id) { + td::actor::send_closure( + adnl_, &ton::adnl::Adnl::send_query, local_id, peer, "q", + [SelfId, peer](td::Result R) { + if (R.is_error()) { + return; + } + auto r_obj = ton::fetch_tl_object(R.move_as_ok(), true); + if (r_obj.is_error()) { + return; + } + td::actor::send_closure(SelfId, &RldpHttpProxy::update_peer_capabilities, peer, + r_obj.ok()->capabilities_); + }, + td::Timestamp::in(3.0), + ton::create_serialize_tl_object(CAPABILITIES)); + }; + for (const ton::adnl::AdnlNodeIdShort &local_id : server_ids_) { + if (local_id != local_id_) { + send_query(local_id); + } + } + send_query(local_id_); + } + } + private: struct Host { struct Server { @@ -1320,6 +1439,8 @@ class RldpHttpProxy : public td::actor::Actor { td::actor::ActorOwn adnl_; td::actor::ActorOwn dht_; td::actor::ActorOwn rldp_; + td::actor::ActorOwn rldp2_; + td::actor::ActorOwn rldp_dispatcher_; std::shared_ptr dht_config_; @@ -1333,6 +1454,13 @@ class RldpHttpProxy : public td::actor::Actor { std::map, td::Promise)>> payload_senders_; + + struct PeerCapabilities { + td::uint64 capabilities = 0; + bool received = false; + td::Timestamp retry_at = td::Timestamp::now(); + }; + std::map peer_capabilities_; }; void TcpToRldpRequestSender::resolve(std::string host) { @@ -1355,7 +1483,7 @@ void TcpToRldpRequestSender::resolve(std::string host) { } request_tl_->url_ = (PSTRING() << "/gateway/" << bag_id << url); host = storage_gateway_.serialize() + ".adnl"; - for (auto& header : request_tl_->headers_) { + for (auto &header : request_tl_->headers_) { if (td::to_lower(header->name_) == "host") { header->value_ = host; break; @@ -1390,6 +1518,27 @@ void TcpToRldpRequestSender::resolve(std::string host) { }); } +void TcpToRldpRequestSender::resolved(ton::adnl::AdnlNodeIdShort id) { + dst_ = id; + td::actor::send_closure(proxy_, &RldpHttpProxy::ask_peer_capabilities, id); + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &TcpToRldpRequestSender::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &TcpToRldpRequestSender::got_result, R.move_as_ok()); + } + }); + + td::actor::create_actor("HttpPayloadSender", request_payload_, id_, local_id_, adnl_, rldp_, + proxy_, is_tunnel()) + .release(); + + auto f = ton::serialize_tl_object(request_tl_, true); + td::actor::send_closure(rldp_, &ton::adnl::AdnlSenderInterface::send_query_ex, local_id_, dst_, + "http request over rldp", std::move(P), td::Timestamp::in(30.0), std::move(f), 16 << 10); +} + void HttpRldpPayloadSender::start_up() { td::actor::send_closure( proxy_, &RldpHttpProxy::register_payload_sender, id_, diff --git a/rldp/CMakeLists.txt b/rldp/CMakeLists.txt index 813d346d..39e0d3ca 100644 --- a/rldp/CMakeLists.txt +++ b/rldp/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) diff --git a/rldp2/CMakeLists.txt b/rldp2/CMakeLists.txt index 1bfeb0bb..c144ec01 100644 --- a/rldp2/CMakeLists.txt +++ b/rldp2/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) diff --git a/rldp2/RldpConnection.cpp b/rldp2/RldpConnection.cpp index c6f96728..47506178 100644 --- a/rldp2/RldpConnection.cpp +++ b/rldp2/RldpConnection.cpp @@ -170,6 +170,9 @@ td::Timestamp RldpConnection::run(ConnectionCallback &callback) { if (in_flight_count_ > congestion_window_) { bdw_stats_.on_pause(now); } + if (in_flight_count_ == 0) { + bdw_stats_.on_pause(now); + } for (auto &inbound : inbound_transfers_) { alarm_timestamp.relax(run(inbound.first, inbound.second)); diff --git a/rldp2/rldp-in.hpp b/rldp2/rldp-in.hpp index c2e46d2a..095c1479 100644 --- a/rldp2/rldp-in.hpp +++ b/rldp2/rldp-in.hpp @@ -92,6 +92,8 @@ class RldpIn : public RldpImpl { void get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p_id, td::Promise promise) override; + void set_default_mtu(td::uint64 mtu) override; + RldpIn(td::actor::ActorId adnl) : adnl_(adnl) { } @@ -107,6 +109,8 @@ class RldpIn : public RldpImpl { std::set local_ids_; + td::optional custom_default_mtu_; + td::actor::ActorId create_connection(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst); }; diff --git a/rldp2/rldp.cpp b/rldp2/rldp.cpp index 765e38a5..95780b23 100644 --- a/rldp2/rldp.cpp +++ b/rldp2/rldp.cpp @@ -44,6 +44,9 @@ class RldpConnectionActor : public td::actor::Actor, private ConnectionCallback connection_.receive_raw(std::move(data)); yield(); } + void set_default_mtu(td::uint64 mtu) { + connection_.set_default_mtu(mtu); + } private: td::actor::ActorId rldp_; @@ -129,6 +132,9 @@ td::actor::ActorId RldpIn::create_connection(adnl::AdnlNode return it->second.get(); } auto connection = td::actor::create_actor("RldpConnection", actor_id(this), src, dst, adnl_); + if (custom_default_mtu_) { + td::actor::send_closure(connection, &RldpConnectionActor::set_default_mtu, custom_default_mtu_.value()); + } auto res = connection.get(); connections_[std::make_pair(src, dst)] = std::move(connection); return res; @@ -221,6 +227,13 @@ void RldpIn::get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p td::actor::send_closure(adnl_, &adnl::AdnlPeerTable::get_conn_ip_str, l_id, p_id, std::move(promise)); } +void RldpIn::set_default_mtu(td::uint64 mtu) { + custom_default_mtu_ = mtu; + for (auto &connection : connections_) { + td::actor::send_closure(connection.second, &RldpConnectionActor::set_default_mtu, mtu); + } +} + std::unique_ptr RldpIn::make_adnl_callback() { class Callback : public adnl::Adnl::Callback { private: diff --git a/rldp2/rldp.h b/rldp2/rldp.h index f4b26985..97fc126a 100644 --- a/rldp2/rldp.h +++ b/rldp2/rldp.h @@ -37,6 +37,8 @@ class Rldp : public adnl::AdnlSenderInterface { virtual void send_message_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout, td::BufferSlice data) = 0; + virtual void set_default_mtu(td::uint64 mtu) = 0; + static td::actor::ActorOwn create(td::actor::ActorId adnl); }; diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index a53a42b8..30403f5e 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) @@ -8,9 +8,11 @@ endif() set(STORAGE_SOURCE LoadSpeed.cpp MerkleTree.cpp + MicrochunkTree.cpp NodeActor.cpp PeerActor.cpp PeerState.cpp + SpeedLimiter.cpp Torrent.cpp TorrentCreator.cpp TorrentHeader.cpp @@ -25,26 +27,25 @@ set(STORAGE_SOURCE PartsHelper.h PeerActor.h PeerState.h + SpeedLimiter.h Torrent.h TorrentCreator.h TorrentHeader.h TorrentInfo.h TorrentMeta.h PeerManager.h - MicrochunkTree.h MicrochunkTree.cpp) + MicrochunkTree.h) set(STORAGE_CLI_SOURCE storage-cli.cpp ) -add_library(storage ${STORAGE_SOURCE}) +add_library(storage STATIC ${STORAGE_SOURCE}) target_link_libraries(storage tdutils tdactor tddb ton_crypto tl_api ${JEMALLOC_LIBRARIES}) -target_include_directories(storage PUBLIC - $ +target_include_directories(storage PUBLIC $ ) add_executable(storage-cli ${STORAGE_CLI_SOURCE}) -target_link_libraries(storage-cli storage overlay tdutils tdactor adnl tl_api dht - rldp rldp2 fift-lib memprof terminal git ${JEMALLOC_LIBRARIES}) +target_link_libraries(storage-cli storage overlay tdutils tdactor adnl tl_api dht rldp rldp2 fift-lib memprof terminal git ${JEMALLOC_LIBRARIES}) set(STORAGE_TEST_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/test/storage.cpp @@ -54,4 +55,4 @@ set(STORAGE_TEST_SOURCE add_subdirectory(storage-daemon) # Do not install it yet -#install(TARGETS storage-cli RUNTIME DESTINATION bin) +install(TARGETS storage-cli storage-daemon storage-daemon-cli RUNTIME DESTINATION bin) diff --git a/storage/LoadSpeed.cpp b/storage/LoadSpeed.cpp index b47f4120..46aac2ff 100644 --- a/storage/LoadSpeed.cpp +++ b/storage/LoadSpeed.cpp @@ -31,13 +31,17 @@ double LoadSpeed::speed(td::Timestamp now) const { update(now); return (double)total_size_ / duration(now); } +void LoadSpeed::reset() { + events_ = {}; + total_size_ = 0; +} td::StringBuilder &operator<<(td::StringBuilder &sb, const LoadSpeed &speed) { return sb << td::format::as_size(static_cast(speed.speed())) << "/s"; } void LoadSpeed::update(td::Timestamp now) const { - while (duration(now) > 30) { + while (duration(now) > 10) { total_size_ -= events_.front().size; events_.pop(); } diff --git a/storage/LoadSpeed.h b/storage/LoadSpeed.h index 92d947f7..37115b88 100644 --- a/storage/LoadSpeed.h +++ b/storage/LoadSpeed.h @@ -28,6 +28,7 @@ class LoadSpeed { public: void add(td::uint64 size, td::Timestamp now = td::Timestamp::now()); double speed(td::Timestamp now = td::Timestamp::now()) const; + void reset(); friend td::StringBuilder &operator<<(td::StringBuilder &sb, const LoadSpeed &speed); private: diff --git a/storage/NodeActor.cpp b/storage/NodeActor.cpp index 041d74cd..e24fffff 100644 --- a/storage/NodeActor.cpp +++ b/storage/NodeActor.cpp @@ -25,27 +25,14 @@ #include "td/utils/Enumerator.h" #include "td/utils/tests.h" #include "td/utils/overloaded.h" -#include "tl-utils/common-utils.hpp" #include "tl-utils/tl-utils.hpp" #include "auto/tl/ton_api.hpp" #include "td/actor/MultiPromise.h" namespace ton { NodeActor::NodeActor(PeerId self_id, Torrent torrent, td::unique_ptr callback, - td::unique_ptr node_callback, std::shared_ptr db, bool should_download, - bool should_upload) - : self_id_(self_id) - , torrent_(std::move(torrent)) - , callback_(std::move(callback)) - , node_callback_(std::move(node_callback)) - , db_(std::move(db)) - , should_download_(should_download) - , should_upload_(should_upload) { -} - -NodeActor::NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr callback, - td::unique_ptr node_callback, std::shared_ptr db, bool should_download, - bool should_upload, DbInitialData db_initial_data) + td::unique_ptr node_callback, std::shared_ptr db, + SpeedLimiters speed_limiters, bool should_download, bool should_upload) : self_id_(self_id) , torrent_(std::move(torrent)) , callback_(std::move(callback)) @@ -53,6 +40,23 @@ NodeActor::NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr callback, + td::unique_ptr node_callback, std::shared_ptr db, + SpeedLimiters speed_limiters, bool should_download, bool should_upload, + DbInitialData db_initial_data) + : self_id_(self_id) + , torrent_(std::move(torrent)) + , callback_(std::move(callback)) + , node_callback_(std::move(node_callback)) + , db_(std::move(db)) + , should_download_(should_download) + , should_upload_(should_upload) + , added_at_(db_initial_data.added_at) + , speed_limiters_(std::move(speed_limiters)) , pending_set_file_priority_(std::move(db_initial_data.priorities)) , pieces_in_db_(std::move(db_initial_data.pieces_in_db)) { } @@ -100,12 +104,12 @@ void NodeActor::init_torrent() { } } - torrent_info_str_ = - std::make_shared(vm::std_boc_serialize(torrent_.get_info().as_cell()).move_as_ok()); + torrent_info_shared_ = std::make_shared(torrent_.get_info()); for (auto &p : peers_) { auto &state = p.second.state; - state->torrent_info_str_ = torrent_info_str_; + state->torrent_info_ = torrent_info_shared_; CHECK(!state->torrent_info_ready_.exchange(true)); + state->notify_peer(); } LOG(INFO) << "Inited torrent info for " << torrent_.get_hash().to_hex() << ": size=" << torrent_.get_info().file_size << ", pieces=" << torrent_.get_info().pieces_count(); @@ -185,7 +189,7 @@ void NodeActor::loop_will_upload() { auto &state = it.second.state; bool needed = false; if (state->peer_state_ready_) { - needed = state->peer_state_.load().want_download; + needed = state->peer_state_.load().want_download && state->peer_online_; } peers.emplace_back(!needed, !state->node_state_.load().want_download, -it.second.download_speed.speed(), it.first); } @@ -247,6 +251,10 @@ void NodeActor::loop() { } wait_for_completion_.clear(); is_completed_ = true; + download_speed_.reset(); + for (auto &peer : peers_) { + peer.second.download_speed.reset(); + } callback_->on_completed(); } } @@ -398,6 +406,12 @@ void NodeActor::set_should_download(bool should_download) { return; } should_download_ = should_download; + if (!should_download_) { + download_speed_.reset(); + for (auto &peer : peers_) { + peer.second.download_speed.reset(); + } + } db_store_torrent(); yield(); } @@ -407,7 +421,14 @@ void NodeActor::set_should_upload(bool should_upload) { return; } should_upload_ = should_upload; + if (!should_upload_) { + upload_speed_.reset(); + for (auto &peer : peers_) { + peer.second.upload_speed.reset(); + } + } db_store_torrent(); + will_upload_at_ = td::Timestamp::now(); yield(); } @@ -478,6 +499,7 @@ void NodeActor::loop_start_stop_peers() { if (peer.actor.empty()) { auto &state = peer.state = std::make_shared(peer.notifier.get()); + state->speed_limiters_ = speed_limiters_; if (torrent_.inited_info()) { std::vector node_ready_parts; for (td::uint32 i = 0; i < parts_.parts.size(); i++) { @@ -486,7 +508,7 @@ void NodeActor::loop_start_stop_peers() { } } state->node_ready_parts_.add_elements(std::move(node_ready_parts)); - state->torrent_info_str_ = torrent_info_str_; + state->torrent_info_ = torrent_info_shared_; state->torrent_info_ready_ = true; } else { state->torrent_info_response_callback_ = [SelfId = actor_id(this)](td::BufferSlice data) { @@ -575,6 +597,11 @@ void NodeActor::loop_peer(const PeerId &peer_id, Peer &peer) { for (auto part_id : state->peer_ready_parts_.read()) { parts_helper_.on_peer_part_ready(peer.peer_token, part_id); } + if (state->peer_online_set_.load()) { + state->peer_online_set_ = false; + will_upload_at_ = td::Timestamp::now(); + loop_will_upload(); + } // Answer queries from peer bool should_notify_peer = false; @@ -600,7 +627,7 @@ void NodeActor::loop_peer(const PeerId &peer_id, Peer &peer) { TRY_RESULT(proof_serialized, vm::std_boc_serialize(std::move(proof))); res.proof = std::move(proof_serialized); res.data = td::BufferSlice(std::move(data)); - td::uint64 size = res.data.size() + res.proof.size(); + td::uint64 size = res.data.size(); upload_speed_.add(size); peer.upload_speed.add(size); return std::move(res); @@ -701,10 +728,11 @@ void NodeActor::db_store_torrent() { if (!db_) { return; } - auto obj = create_tl_object(); + auto obj = create_tl_object(); obj->active_download_ = should_download_; obj->active_upload_ = should_upload_; obj->root_dir_ = torrent_.get_root_dir(); + obj->added_at_ = added_at_; db_->set(create_hash_tl_object(torrent_.get_hash()), serialize_tl_object(obj, true), [](td::Result R) { if (R.is_error()) { @@ -823,16 +851,18 @@ void NodeActor::db_update_pieces_list() { } void NodeActor::load_from_db(std::shared_ptr db, td::Bits256 hash, td::unique_ptr callback, - td::unique_ptr node_callback, + td::unique_ptr node_callback, SpeedLimiters speed_limiters, td::Promise> promise) { class Loader : public td::actor::Actor { public: Loader(std::shared_ptr db, td::Bits256 hash, td::unique_ptr callback, - td::unique_ptr node_callback, td::Promise> promise) + td::unique_ptr node_callback, SpeedLimiters speed_limiters, + td::Promise> promise) : db_(std::move(db)) , hash_(hash) , callback_(std::move(callback)) , node_callback_(std::move(node_callback)) + , speed_limiters_(std::move(speed_limiters)) , promise_(std::move(promise)) { } @@ -842,9 +872,9 @@ void NodeActor::load_from_db(std::shared_ptr db, td::Bits256 hash, t } void start_up() override { - db::db_get( + db::db_get( *db_, create_hash_tl_object(hash_), false, - [SelfId = actor_id(this)](td::Result> R) { + [SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &Loader::finish, R.move_as_error_prefix("Torrent: ")); } else { @@ -853,10 +883,20 @@ void NodeActor::load_from_db(std::shared_ptr db, td::Bits256 hash, t }); } - void got_torrent(tl_object_ptr obj) { - root_dir_ = std::move(obj->root_dir_); - active_download_ = obj->active_download_; - active_upload_ = obj->active_upload_; + void got_torrent(tl_object_ptr obj) { + ton_api::downcast_call(*obj, td::overloaded( + [&](ton_api::storage_db_torrent &t) { + root_dir_ = std::move(t.root_dir_); + active_download_ = t.active_download_; + active_upload_ = t.active_upload_; + added_at_ = (td::uint32)td::Clocks::system(); + }, + [&](ton_api::storage_db_torrentV2 &t) { + root_dir_ = std::move(t.root_dir_); + active_download_ = t.active_download_; + active_upload_ = t.active_upload_; + added_at_ = t.added_at_; + })); db_->get(create_hash_tl_object(hash_), [SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { @@ -980,9 +1020,10 @@ void NodeActor::load_from_db(std::shared_ptr db, td::Bits256 hash, t DbInitialData data; data.priorities = std::move(priorities_); data.pieces_in_db = std::move(pieces_in_db_); + data.added_at = added_at_; finish(td::actor::create_actor("Node", 1, torrent_.unwrap(), std::move(callback_), - std::move(node_callback_), std::move(db_), active_download_, - active_upload_, std::move(data))); + std::move(node_callback_), std::move(db_), std::move(speed_limiters_), + active_download_, active_upload_, std::move(data))); } private: @@ -990,18 +1031,20 @@ void NodeActor::load_from_db(std::shared_ptr db, td::Bits256 hash, t td::Bits256 hash_; td::unique_ptr callback_; td::unique_ptr node_callback_; + SpeedLimiters speed_limiters_; td::Promise> promise_; std::string root_dir_; bool active_download_{false}; bool active_upload_{false}; + td::uint32 added_at_; td::optional torrent_; std::vector priorities_; std::set pieces_in_db_; size_t remaining_pieces_in_db_ = 0; }; td::actor::create_actor("loader", std::move(db), hash, std::move(callback), std::move(node_callback), - std::move(promise)) + std::move(speed_limiters), std::move(promise)) .release(); } diff --git a/storage/NodeActor.h b/storage/NodeActor.h index f8cadce2..2f0a3232 100644 --- a/storage/NodeActor.h +++ b/storage/NodeActor.h @@ -23,6 +23,7 @@ #include "PartsHelper.h" #include "PeerActor.h" #include "Torrent.h" +#include "SpeedLimiter.h" #include "td/utils/Random.h" #include "td/utils/Variant.h" @@ -31,6 +32,7 @@ #include "db.h" namespace ton { + class NodeActor : public td::actor::Actor { public: class NodeCallback { @@ -60,14 +62,15 @@ class NodeActor : public td::actor::Actor { struct DbInitialData { std::vector priorities; std::set pieces_in_db; + td::uint32 added_at; }; NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr callback, - td::unique_ptr node_callback, std::shared_ptr db, bool should_download = true, - bool should_upload = true); + td::unique_ptr node_callback, std::shared_ptr db, SpeedLimiters speed_limiters, + bool should_download = true, bool should_upload = true); NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr callback, - td::unique_ptr node_callback, std::shared_ptr db, bool should_download, - bool should_upload, DbInitialData db_initial_data); + td::unique_ptr node_callback, std::shared_ptr db, SpeedLimiters speed_limiters, + bool should_download, bool should_upload, DbInitialData db_initial_data); void start_peer(PeerId peer_id, td::Promise> promise); struct NodeState { @@ -76,11 +79,12 @@ class NodeActor : public td::actor::Actor { bool active_upload; double download_speed; double upload_speed; + td::uint32 added_at; const std::vector &file_priority; }; void with_torrent(td::Promise promise) { promise.set_value(NodeState{torrent_, should_download_, should_upload_, download_speed_.speed(), - upload_speed_.speed(), file_priority_}); + upload_speed_.speed(), added_at_, file_priority_}); } std::string get_stats_str(); @@ -98,20 +102,22 @@ class NodeActor : public td::actor::Actor { void get_peers_info(td::Promise> promise); static void load_from_db(std::shared_ptr db, td::Bits256 hash, td::unique_ptr callback, - td::unique_ptr node_callback, + td::unique_ptr node_callback, SpeedLimiters speed_limiters, td::Promise> promise); static void cleanup_db(std::shared_ptr db, td::Bits256 hash, td::Promise promise); private: PeerId self_id_; ton::Torrent torrent_; - std::shared_ptr torrent_info_str_; + std::shared_ptr torrent_info_shared_; std::vector file_priority_; td::unique_ptr callback_; td::unique_ptr node_callback_; std::shared_ptr db_; bool should_download_{false}; bool should_upload_{false}; + td::uint32 added_at_{0}; + SpeedLimiters speed_limiters_; class Notifier : public td::actor::Actor { public: diff --git a/storage/PeerActor.cpp b/storage/PeerActor.cpp index b2bf5a89..e140b4ce 100644 --- a/storage/PeerActor.cpp +++ b/storage/PeerActor.cpp @@ -24,6 +24,8 @@ #include "td/utils/overloaded.h" #include "td/utils/Random.h" +#include "vm/boc.h" +#include "common/delay.h" namespace ton { @@ -44,7 +46,7 @@ PeerActor::PeerActor(td::unique_ptr callback, std::shared_ptr -td::uint64 PeerActor::create_and_send_query(ArgsT &&... args) { +td::uint64 PeerActor::create_and_send_query(ArgsT &&...args) { return send_query(ton::create_serialize_tl_object(std::forward(args)...)); } @@ -86,7 +88,9 @@ void PeerActor::on_ping_result(td::Result r_answer) { void PeerActor::on_pong() { wait_pong_till_ = td::Timestamp::in(10); - state_->peer_online_ = true; + if (!state_->peer_online_.exchange(true)) { + state_->peer_online_set_ = true; + } notify_node(); } @@ -115,6 +119,11 @@ void PeerActor::on_get_piece_result(PartId piece_id, td::Result res.proof = std::move(piece->proof_); return std::move(res); }(); + if (res.is_error()) { + LOG(DEBUG) << "getPiece " << piece_id << " query: " << res.error(); + } else { + LOG(DEBUG) << "getPiece " << piece_id << " query: OK"; + } state_->node_queries_results_.add_element(std::make_pair(piece_id, std::move(res))); notify_node(); } @@ -130,13 +139,16 @@ void PeerActor::on_get_info_result(td::Result r_answer) { next_get_info_at_ = td::Timestamp::in(5.0); alarm_timestamp().relax(next_get_info_at_); if (r_answer.is_error()) { + LOG(DEBUG) << "getTorrentInfo query: " << r_answer.move_as_error(); return; } auto R = fetch_tl_object(r_answer.move_as_ok(), true); if (R.is_error()) { + LOG(DEBUG) << "getTorrentInfo query: " << R.move_as_error(); return; } td::BufferSlice data = std::move(R.ok_ref()->data_); + LOG(DEBUG) << "getTorrentInfo query: got result (" << data.size() << " bytes)"; if (!data.empty() && !state_->torrent_info_ready_) { state_->torrent_info_response_callback_(std::move(data)); } @@ -239,12 +251,14 @@ void PeerActor::loop_update_init() { } s = s.substr(peer_init_offset_, UPDATE_INIT_BLOCK_SIZE); auto query = create_update_query(ton::create_tl_object( - td::BufferSlice(s), (int)peer_init_offset_, to_ton_api(node_state))); + td::BufferSlice(s), (int)peer_init_offset_ * 8, to_ton_api(node_state))); // take care about update_state_query initial state update_state_query_.state = node_state; update_state_query_.query_id = 0; + LOG(DEBUG) << "Sending updateInit query (" << update_state_query_.state.want_download << ", " + << update_state_query_.state.will_upload << ", offset=" << peer_init_offset_ * 8 << ")"; update_query_id_ = send_query(std::move(query)); } @@ -265,13 +279,15 @@ void PeerActor::loop_update_state() { auto query = create_update_query( ton::create_tl_object(to_ton_api(update_state_query_.state))); + LOG(DEBUG) << "Sending updateState query (" << update_state_query_.state.want_download << ", " + << update_state_query_.state.will_upload << ")"; update_state_query_.query_id = send_query(std::move(query)); } void PeerActor::update_have_pieces() { auto node_ready_parts = state_->node_ready_parts_.read(); for (auto piece_id : node_ready_parts) { - if (piece_id < peer_init_offset_ + UPDATE_INIT_BLOCK_SIZE) { + if (piece_id < (peer_init_offset_ + UPDATE_INIT_BLOCK_SIZE) * 8 && !have_pieces_.get(piece_id)) { have_pieces_list_.push_back(piece_id); } have_pieces_.set_one(piece_id); @@ -291,23 +307,58 @@ void PeerActor::loop_update_pieces() { have_pieces_list_.erase(have_pieces_list_.end() - count, have_pieces_list_.end()); auto query = create_update_query(ton::create_tl_object( td::transform(sent_have_pieces_list_, [](auto x) { return static_cast(x); }))); + LOG(DEBUG) << "Sending updateHavePieces query (" << sent_have_pieces_list_.size() << " pieces)"; update_query_id_ = send_query(std::move(query)); } } void PeerActor::loop_get_torrent_info() { - if (get_info_query_id_ || state_->torrent_info_ready_) { + if (state_->torrent_info_ready_) { + if (!torrent_info_) { + torrent_info_ = state_->torrent_info_; + for (const auto &u : pending_update_peer_parts_) { + process_update_peer_parts(u); + } + pending_update_peer_parts_.clear(); + } + return; + } + if (get_info_query_id_) { return; } if (next_get_info_at_ && !next_get_info_at_.is_in_past()) { return; } + LOG(DEBUG) << "Sending getTorrentInfo query"; get_info_query_id_ = create_and_send_query(); } void PeerActor::loop_node_get_piece() { for (auto part : state_->node_queries_.read()) { - node_get_piece_.emplace(part, NodePieceQuery{}); + if (state_->speed_limiters_.download.empty()) { + node_get_piece_.emplace(part, NodePieceQuery{}); + } else { + if (!torrent_info_) { + CHECK(state_->torrent_info_ready_); + loop_get_torrent_info(); + } + auto piece_size = + std::min(torrent_info_->piece_size, torrent_info_->file_size - part * torrent_info_->piece_size); + td::Timestamp timeout = td::Timestamp::in(3.0); + td::actor::send_closure( + state_->speed_limiters_.download, &SpeedLimiter::enqueue, (double)piece_size, timeout, + [=, SelfId = actor_id(this)](td::Result R) { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &PeerActor::node_get_piece_query_ready, part, std::move(R)); + } else { + delay_action( + [=, R = std::move(R)]() mutable { + td::actor::send_closure(SelfId, &PeerActor::node_get_piece_query_ready, part, std::move(R)); + }, + timeout); + } + }); + } } for (auto &query_it : node_get_piece_) { @@ -315,11 +366,21 @@ void PeerActor::loop_node_get_piece() { continue; } + LOG(DEBUG) << "Sending getPiece " << query_it.first << " query"; query_it.second.query_id = create_and_send_query(static_cast(query_it.first)); } } +void PeerActor::node_get_piece_query_ready(PartId part, td::Result R) { + if (R.is_error()) { + on_get_piece_result(part, R.move_as_error()); + } else { + node_get_piece_.emplace(part, NodePieceQuery{}); + } + schedule_loop(); +} + void PeerActor::loop_peer_get_piece() { // process answers for (auto &p : state_->peer_queries_results_.read()) { @@ -328,9 +389,26 @@ void PeerActor::loop_peer_get_piece() { if (promise_it == peer_get_piece_.end()) { continue; } - promise_it->second.promise.set_result(p.second.move_map([](PeerState::Part part) { - return ton::create_serialize_tl_object(std::move(part.proof), std::move(part.data)); - })); + td::Promise promise = + [i = p.first, promise = std::move(promise_it->second.promise)](td::Result R) mutable { + LOG(DEBUG) << "Responding to getPiece " << i << ": " << (R.is_ok() ? "OK" : R.error().to_string()); + promise.set_result(R.move_map([](PeerState::Part part) { + return create_serialize_tl_object(std::move(part.proof), std::move(part.data)); + })); + }; + if (p.second.is_error()) { + promise.set_error(p.second.move_as_error()); + } else { + auto part = p.second.move_as_ok(); + auto size = (double)part.data.size(); + td::Promise P = promise.wrap([part = std::move(part)](td::Unit) mutable { return std::move(part); }); + if (state_->speed_limiters_.upload.empty()) { + P.set_result(td::Unit()); + } else { + td::actor::send_closure(state_->speed_limiters_.upload, &SpeedLimiter::enqueue, size, td::Timestamp::in(3.0), + std::move(P)); + } + } peer_get_piece_.erase(promise_it); notify_node(); } @@ -373,18 +451,8 @@ void PeerActor::execute_add_update(ton::ton_api::storage_addUpdate &add_update, promise.set_error(td::Status::Error(404, "INVALID_SESSION")); return; } - promise.set_value(ton::create_serialize_tl_object()); - std::vector new_peer_ready_parts; - auto add_piece = [&](PartId id) { - if (!peer_have_pieces_.get(id)) { - peer_have_pieces_.set_one(id); - new_peer_ready_parts.push_back(id); - notify_node(); - } - }; - auto seqno = static_cast(add_update.seqno_); auto update_peer_state = [&](PeerState::State peer_state) { if (peer_seqno_ >= seqno) { @@ -393,26 +461,52 @@ void PeerActor::execute_add_update(ton::ton_api::storage_addUpdate &add_update, if (state_->peer_state_ready_ && state_->peer_state_.load() == peer_state) { return; } + LOG(DEBUG) << "Processing update peer state query (" << peer_state.want_download << ", " << peer_state.will_upload + << ")"; peer_seqno_ = seqno; state_->peer_state_.exchange(peer_state); state_->peer_state_ready_ = true; notify_node(); }; + downcast_call( + *add_update.update_, + td::overloaded( + [&](const ton::ton_api::storage_updateHavePieces &have_pieces) {}, + [&](const ton::ton_api::storage_updateState &state) { update_peer_state(from_ton_api(*state.state_)); }, + [&](const ton::ton_api::storage_updateInit &init) { update_peer_state(from_ton_api(*init.state_)); })); + if (torrent_info_) { + process_update_peer_parts(add_update.update_); + } else { + pending_update_peer_parts_.push_back(std::move(add_update.update_)); + } +} - downcast_call(*add_update.update_, +void PeerActor::process_update_peer_parts(const tl_object_ptr &update) { + CHECK(torrent_info_); + td::uint64 pieces_count = torrent_info_->pieces_count(); + std::vector new_peer_ready_parts; + auto add_piece = [&](PartId id) { + if (id < pieces_count && !peer_have_pieces_.get(id)) { + peer_have_pieces_.set_one(id); + new_peer_ready_parts.push_back(id); + notify_node(); + } + }; + downcast_call(*update, td::overloaded( - [&](ton::ton_api::storage_updateHavePieces &have_pieces) { + [&](const ton::ton_api::storage_updateHavePieces &have_pieces) { + LOG(DEBUG) << "Processing updateHavePieces query (" << have_pieces.piece_id_ << " pieces)"; for (auto id : have_pieces.piece_id_) { add_piece(id); } }, - [&](ton::ton_api::storage_updateState &state) { update_peer_state(from_ton_api(*state.state_)); }, - [&](ton::ton_api::storage_updateInit &init) { - update_peer_state(from_ton_api(*init.state_)); + [&](const ton::ton_api::storage_updateState &state) {}, + [&](const ton::ton_api::storage_updateInit &init) { + LOG(DEBUG) << "Processing updateInit query (offset=" << init.have_pieces_offset_ << ")"; td::Bitset new_bitset; new_bitset.set_raw(init.have_pieces_.as_slice().str()); - size_t offset = init.have_pieces_offset_ * 8; - for (auto size = new_bitset.size(), i = size_t(0); i < size; i++) { + size_t offset = init.have_pieces_offset_; + for (auto size = new_bitset.size(), i = (size_t)0; i < size; i++) { if (new_bitset.get(i)) { add_piece(static_cast(offset + i)); } @@ -428,7 +522,13 @@ void PeerActor::execute_get_piece(ton::ton_api::storage_getPiece &get_piece, td: void PeerActor::execute_get_torrent_info(td::Promise promise) { td::BufferSlice result = create_serialize_tl_object( - state_->torrent_info_ready_ ? state_->torrent_info_str_->clone() : td::BufferSlice()); + state_->torrent_info_ready_ ? vm::std_boc_serialize(state_->torrent_info_->as_cell()).move_as_ok() + : td::BufferSlice()); + if (state_->torrent_info_ready_) { + LOG(DEBUG) << "Responding to getTorrentInfo: " << result.size() << " bytes"; + } else { + LOG(DEBUG) << "Responding to getTorrentInfo: no info"; + } promise.set_result(std::move(result)); } } // namespace ton diff --git a/storage/PeerActor.h b/storage/PeerActor.h index fd34bc88..a4229e16 100644 --- a/storage/PeerActor.h +++ b/storage/PeerActor.h @@ -59,6 +59,10 @@ class PeerActor : public td::actor::Actor { // startSession td::uint64 node_session_id_; td::Bitset peer_have_pieces_; + std::shared_ptr torrent_info_; + std::vector> pending_update_peer_parts_; + + void process_update_peer_parts(const tl_object_ptr &update); // update td::optional peer_session_id_; @@ -112,6 +116,7 @@ class PeerActor : public td::actor::Actor { td::BufferSlice create_update_query(ton::tl_object_ptr update); void loop_node_get_piece(); + void node_get_piece_query_ready(PartId part, td::Result R); void loop_peer_get_piece(); diff --git a/storage/PeerManager.h b/storage/PeerManager.h index 52297ac5..38d1494b 100644 --- a/storage/PeerManager.h +++ b/storage/PeerManager.h @@ -143,9 +143,11 @@ class PeerManager : public td::actor::Actor { td::actor::ActorId peer_manager_; ton::adnl::AdnlNodeIdShort dst_; }; + ton::overlay::OverlayOptions opts; + opts.announce_self_ = !client_mode_; + opts.frequent_dht_lookup_ = true; send_closure(overlays_, &ton::overlay::Overlays::create_public_overlay_ex, src_id, overlay_id_.clone(), - std::make_unique(actor_id(this), src_id), rules, R"({ "type": "storage" })", - !client_mode_); + std::make_unique(actor_id(this), src_id), rules, R"({ "type": "storage" })", opts); } promise.set_value({}); } diff --git a/storage/PeerState.h b/storage/PeerState.h index c578b544..29c080a2 100644 --- a/storage/PeerState.h +++ b/storage/PeerState.h @@ -23,6 +23,8 @@ #include "td/utils/optional.h" #include "td/actor/actor.h" +#include "TorrentInfo.h" +#include "SpeedLimiter.h" #include #include @@ -100,6 +102,8 @@ struct PeerState { std::atomic_bool peer_state_ready_{false}; std::atomic peer_state_{State{false, false}}; std::atomic_bool peer_online_{false}; + std::atomic_bool peer_online_set_{false}; + SpeedLimiters speed_limiters_; struct Part { td::BufferSlice proof; @@ -121,7 +125,7 @@ struct PeerState { // Node -> Peer std::atomic_bool torrent_info_ready_{false}; - std::shared_ptr torrent_info_str_; + std::shared_ptr torrent_info_; std::function torrent_info_response_callback_; const td::actor::ActorId<> node; diff --git a/storage/SpeedLimiter.cpp b/storage/SpeedLimiter.cpp new file mode 100644 index 00000000..704c7402 --- /dev/null +++ b/storage/SpeedLimiter.cpp @@ -0,0 +1,83 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ + +#include "SpeedLimiter.h" +#include "common/errorcode.h" + +namespace ton { + +SpeedLimiter::SpeedLimiter(double max_speed) : max_speed_(max_speed) { +} + +void SpeedLimiter::set_max_speed(double max_speed) { + max_speed_ = max_speed; + auto old_queue = std::move(queue_); + unlock_at_ = (queue_.empty() ? td::Timestamp::now() : queue_.front().execute_at_); + queue_ = {}; + while (!old_queue.empty()) { + auto &e = old_queue.front(); + enqueue(e.size_, e.timeout_, std::move(e.promise_)); + old_queue.pop(); + } + process_queue(); +} + +void SpeedLimiter::enqueue(double size, td::Timestamp timeout, td::Promise promise) { + if (max_speed_ < 0.0) { + promise.set_result(td::Unit()); + return; + } + if (max_speed_ == 0.0) { + promise.set_error(td::Status::Error(ErrorCode::timeout, "Speed limit is 0")); + return; + } + if (timeout < unlock_at_) { + promise.set_error(td::Status::Error(ErrorCode::timeout, "Timeout caused by speed limit")); + return; + } + if (queue_.empty() && unlock_at_.is_in_past()) { + unlock_at_ = td::Timestamp::now(); + promise.set_result(td::Unit()); + } else { + queue_.push({unlock_at_, size, timeout, std::move(promise)}); + } + unlock_at_ = td::Timestamp::in(size / max_speed_, unlock_at_); + if (!queue_.empty()) { + alarm_timestamp() = queue_.front().execute_at_; + } +} + +void SpeedLimiter::alarm() { + process_queue(); +} + +void SpeedLimiter::process_queue() { + while (!queue_.empty()) { + auto &e = queue_.front(); + if (e.execute_at_.is_in_past()) { + e.promise_.set_result(td::Unit()); + queue_.pop(); + } else { + break; + } + } + if (!queue_.empty()) { + alarm_timestamp() = queue_.front().execute_at_; + } +} + +} // namespace ton diff --git a/tonlib/tonlib/ExtClientLazy.h b/storage/SpeedLimiter.h similarity index 57% rename from tonlib/tonlib/ExtClientLazy.h rename to storage/SpeedLimiter.h index dc4490b3..b6230732 100644 --- a/tonlib/tonlib/ExtClientLazy.h +++ b/storage/SpeedLimiter.h @@ -13,29 +13,40 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP */ + #pragma once #include "td/actor/actor.h" +#include -#include "adnl/adnl-ext-client.h" +namespace ton { -namespace tonlib { -class ExtClientLazy : public ton::adnl::AdnlExtClient { +class SpeedLimiter : public td::actor::Actor { public: - class Callback { - public: - virtual ~Callback() { - } + explicit SpeedLimiter(double max_speed); + + void set_max_speed(double max_speed); // Negative = unlimited + void enqueue(double size, td::Timestamp timeout, td::Promise promise); + + void alarm() override; + + private: + double max_speed_ = -1.0; + td::Timestamp unlock_at_ = td::Timestamp::never(); + + struct Event { + td::Timestamp execute_at_; + double size_; + td::Timestamp timeout_; + td::Promise promise_; }; + std::queue queue_; - virtual void force_change_liteserver() = 0; - - static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, - td::unique_ptr callback); - static td::actor::ActorOwn create( - std::vector> servers, td::unique_ptr callback); + void process_queue(); }; -} // namespace tonlib +struct SpeedLimiters { + td::actor::ActorId download, upload; +}; + +} // namespace ton diff --git a/storage/Torrent.cpp b/storage/Torrent.cpp index 1dafb999..545b6d10 100644 --- a/storage/Torrent.cpp +++ b/storage/Torrent.cpp @@ -266,7 +266,9 @@ td::Result Torrent::get_piece_data(td::uint64 piece_i) { if (!inited_info_) { return td::Status::Error("Torrent info not inited"); } - CHECK(piece_i < info_.pieces_count()); + if (piece_i >= info_.pieces_count()) { + return td::Status::Error("Piece idx is too big"); + } if (!piece_is_ready_[piece_i]) { return td::Status::Error("Piece is not ready"); } @@ -291,7 +293,9 @@ td::Result> Torrent::get_piece_proof(td::uint64 piece_i) { if (!inited_info_) { return td::Status::Error("Torrent info not inited"); } - CHECK(piece_i < info_.pieces_count()); + if (piece_i >= info_.pieces_count()) { + return td::Status::Error("Piece idx is too big"); + } return merkle_tree_.gen_proof(piece_i, piece_i); } @@ -305,7 +309,9 @@ td::Status Torrent::add_piece(td::uint64 piece_i, td::Slice data, td::Ref= info_.pieces_count()) { + return td::Status::Error("Piece idx is too big"); + } if (piece_is_ready_[piece_i]) { return td::Status::OK(); } @@ -318,7 +324,7 @@ td::Status Torrent::add_piece(td::uint64 piece_i, td::Slice data, td::Ref Torrent::read_file(td::Slice name) { @@ -482,7 +488,7 @@ Torrent::Torrent(Info info, td::optional header, ton::MerkleTree , info_(info) , root_dir_(std::move(root_dir)) , header_(std::move(header)) - , enabled_wirte_to_files_(true) + , enabled_write_to_files_(true) , merkle_tree_(std::move(tree)) , piece_is_ready_(info_.pieces_count(), true) , ready_parts_count_{info_.pieces_count()} @@ -580,7 +586,7 @@ void Torrent::set_file_excluded(size_t i, bool excluded) { included_ready_size_ += chunk.ready_size; } chunk.excluded = excluded; - if (!enabled_wirte_to_files_ || excluded) { + if (!enabled_write_to_files_ || excluded) { return; } auto range = get_file_parts_range(i); diff --git a/storage/Torrent.h b/storage/Torrent.h index 6401e405..1ae30fc2 100644 --- a/storage/Torrent.h +++ b/storage/Torrent.h @@ -173,7 +173,7 @@ class Torrent { size_t not_ready_pending_piece_count_{0}; size_t header_pieces_count_{0}; std::map pending_pieces_; - bool enabled_wirte_to_files_ = false; + bool enabled_write_to_files_ = false; struct InMemoryPiece { std::string data; std::set pending_chunks; diff --git a/storage/TorrentCreator.cpp b/storage/TorrentCreator.cpp index 35fb8212..ab060023 100644 --- a/storage/TorrentCreator.cpp +++ b/storage/TorrentCreator.cpp @@ -29,18 +29,22 @@ #include "TorrentHeader.hpp" namespace ton { +static bool is_dir_slash(char c) { + return (c == TD_DIR_SLASH) | (c == '/'); +} + td::Result Torrent::Creator::create_from_path(Options options, td::CSlice raw_path) { TRY_RESULT(path, td::realpath(raw_path)); TRY_RESULT(stat, td::stat(path)); std::string root_dir = path; - while (!root_dir.empty() && root_dir.back() == TD_DIR_SLASH) { + while (!root_dir.empty() && is_dir_slash(root_dir.back())) { root_dir.pop_back(); } - while (!root_dir.empty() && root_dir.back() != TD_DIR_SLASH) { + while (!root_dir.empty() && !is_dir_slash(root_dir.back())) { root_dir.pop_back(); } if (stat.is_dir_) { - if (!path.empty() && path.back() != TD_DIR_SLASH) { + if (!path.empty() && !is_dir_slash(path.back())) { path += TD_DIR_SLASH; } if (!options.dir_name) { @@ -50,7 +54,20 @@ td::Result Torrent::Creator::create_from_path(Options options, td::CSli td::Status status; auto walk_status = td::WalkPath::run(path, [&](td::CSlice name, td::WalkPath::Type type) { if (type == td::WalkPath::Type::NotDir) { - status = creator.add_file(td::PathView::relative(name, path), name); + std::string rel_name = td::PathView::relative(name, path).str(); + td::Slice file_name = rel_name; + for (size_t i = 0; i < rel_name.size(); ++i) { + if (is_dir_slash(rel_name[i])) { + rel_name[i] = '/'; + file_name = td::Slice(rel_name.c_str() + i + 1, rel_name.c_str() + rel_name.size()); + } + } + // Exclude OS-created files that can be modified automatically and thus break some torrent pieces + if (file_name == ".DS_Store" || td::to_lower(file_name) == "desktop.ini" || + td::to_lower(file_name) == "thumbs.db") { + return td::WalkPath::Action::Continue; + } + status = creator.add_file(rel_name, name); if (status.is_error()) { return td::WalkPath::Action::Abort; } diff --git a/storage/TorrentHeader.hpp b/storage/TorrentHeader.hpp index d8f8f719..15b2d8b9 100644 --- a/storage/TorrentHeader.hpp +++ b/storage/TorrentHeader.hpp @@ -68,13 +68,23 @@ void TorrentHeader::parse(ParserT &parser) { parser.set_error("Unknown fec type"); return; } - name_index.resize(files_count); - for (auto &x : name_index) { + name_index.clear(); + for (size_t i = 0; i < files_count; ++i) { + td::uint64 x; parse(x, parser); + if (parser.get_error()) { + return; + } + name_index.push_back(x); } - data_index.resize(files_count); - for (auto &x : data_index) { + data_index.clear(); + for (size_t i = 0; i < files_count; ++i) { + td::uint64 x; parse(x, parser); + if (parser.get_error()) { + return; + } + data_index.push_back(x); } names = parser.template fetch_string_raw(tot_names_size); } diff --git a/storage/TorrentInfo.cpp b/storage/TorrentInfo.cpp index 48b0d592..26aa092e 100644 --- a/storage/TorrentInfo.cpp +++ b/storage/TorrentInfo.cpp @@ -73,6 +73,12 @@ td::Status TorrentInfo::validate() const { if (description.size() > 1024) { return td::Status::Error("Description is too long"); } + if (piece_size > (1 << 23)) { + return td::Status::Error("Piece size is too big"); + } + if (pieces_count() >= (1ULL << 31)) { + return td::Status::Error("Too many pieces"); + } return td::Status::OK(); } } // namespace ton diff --git a/storage/storage-cli.cpp b/storage/storage-cli.cpp index 43ddbeaf..858433d1 100644 --- a/storage/storage-cli.cpp +++ b/storage/storage-cli.cpp @@ -458,9 +458,9 @@ class StorageCli : public td::actor::Actor { } auto callback = td::make_unique(actor_id(this), ptr->id, std::move(on_completed)); auto context = PeerManager::create_callback(ptr->peer_manager.get()); - ptr->node = - td::actor::create_actor(PSLICE() << "Node#" << self_id, self_id, ptr->torrent.unwrap(), - std::move(callback), std::move(context), nullptr, should_download); + ptr->node = td::actor::create_actor(PSLICE() << "Node#" << self_id, self_id, ptr->torrent.unwrap(), + std::move(callback), std::move(context), nullptr, + ton::SpeedLimiters{}, should_download); td::TerminalIO::out() << "Torrent #" << ptr->id << " started\n"; promise.release().release(); if (promise) { diff --git a/storage/storage-daemon/CMakeLists.txt b/storage/storage-daemon/CMakeLists.txt index 4880eece..cabc6143 100644 --- a/storage/storage-daemon/CMakeLists.txt +++ b/storage/storage-daemon/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) add_executable(embed-provider-code smartcont/embed-provider-code.cpp) @@ -25,8 +25,7 @@ set(STORAGE_DAEMON_CLI_SOURCE ) add_executable(storage-daemon ${STORAGE_DAEMON_SOURCE}) -target_link_libraries(storage-daemon storage overlay tdutils tdactor adnl tl_api dht - rldp rldp2 fift-lib memprof git tonlib) +target_link_libraries(storage-daemon storage overlay tdutils tdactor adnl tl_api dht rldp rldp2 fift-lib memprof git tonlib) add_executable(storage-daemon-cli ${STORAGE_DAEMON_CLI_SOURCE}) -target_link_libraries(storage-daemon-cli tdutils tdactor adnllite tl_api tl_lite_api tl-lite-utils ton_block terminal git) \ No newline at end of file +target_link_libraries(storage-daemon-cli tdutils tdactor adnllite tl_api tl_lite_api tl-lite-utils ton_block terminal git) diff --git a/storage/storage-daemon/StorageManager.cpp b/storage/storage-daemon/StorageManager.cpp index 8edd5224..cf93f735 100644 --- a/storage/storage-daemon/StorageManager.cpp +++ b/storage/storage-daemon/StorageManager.cpp @@ -50,6 +50,29 @@ void StorageManager::start_up() { db_ = std::make_shared( std::make_shared(td::RocksDb::open(db_root_ + "/torrent-db").move_as_ok())); + + db::db_get( + *db_, create_hash_tl_object(), true, + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + LOG(ERROR) << "Failed to load config from db: " << R.move_as_error(); + td::actor::send_closure(SelfId, &StorageManager::loaded_config_from_db, nullptr); + } else { + td::actor::send_closure(SelfId, &StorageManager::loaded_config_from_db, R.move_as_ok()); + } + }); +} + +void StorageManager::loaded_config_from_db(tl_object_ptr config) { + if (config) { + LOG(INFO) << "Loaded config from DB. Speed limits: download=" << config->download_speed_limit_ + << ", upload=" << config->upload_speed_limit_; + download_speed_limit_ = config->download_speed_limit_; + upload_speed_limit_ = config->upload_speed_limit_; + td::actor::send_closure(download_speed_limiter_, &SpeedLimiter::set_max_speed, download_speed_limit_); + td::actor::send_closure(upload_speed_limiter_, &SpeedLimiter::set_max_speed, upload_speed_limit_); + } + db::db_get( *db_, create_hash_tl_object(), true, [SelfId = actor_id(this)](td::Result> R) { @@ -79,6 +102,7 @@ void StorageManager::load_torrents_from_db(std::vector torrents) { client_mode_, overlays_, adnl_, rldp_); NodeActor::load_from_db( db_, hash, create_callback(hash, entry.closing_state), PeerManager::create_callback(entry.peer_manager.get()), + SpeedLimiters{download_speed_limiter_.get(), upload_speed_limiter_.get()}, [SelfId = actor_id(this), hash, promise = ig.get_promise()](td::Result> R) mutable { td::actor::send_closure(SelfId, &StorageManager::loaded_torrent_from_db, hash, std::move(R)); @@ -162,9 +186,9 @@ td::Status StorageManager::add_torrent_impl(Torrent torrent, bool start_download client_mode_, overlays_, adnl_, rldp_); auto context = PeerManager::create_callback(entry.peer_manager.get()); LOG(INFO) << "Added torrent " << hash.to_hex() << " , root_dir = " << torrent.get_root_dir(); - entry.actor = - td::actor::create_actor("Node", 1, std::move(torrent), create_callback(hash, entry.closing_state), - std::move(context), db_, start_download, allow_upload); + entry.actor = td::actor::create_actor( + "Node", 1, std::move(torrent), create_callback(hash, entry.closing_state), std::move(context), db_, + SpeedLimiters{download_speed_limiter_.get(), upload_speed_limiter_.get()}, start_download, allow_upload); return td::Status::OK(); } @@ -210,6 +234,18 @@ void StorageManager::get_all_torrents(td::Promise> prom promise.set_result(std::move(result)); } +void StorageManager::db_store_config() { + auto config = create_tl_object(); + config->download_speed_limit_ = download_speed_limit_; + config->upload_speed_limit_ = upload_speed_limit_; + db_->set(create_hash_tl_object(), serialize_tl_object(config, true), + [](td::Result R) { + if (R.is_error()) { + LOG(ERROR) << "Failed to save config to db: " << R.move_as_error(); + } + }); +} + void StorageManager::db_store_torrent_list() { std::vector torrents; for (const auto& p : torrents_) { @@ -259,19 +295,55 @@ void StorageManager::load_from(td::Bits256 hash, td::optional meta, std::move(promise)); } +static bool try_rm_empty_dir(const std::string& path) { + auto stat = td::stat(path); + if (stat.is_error() || !stat.ok().is_dir_) { + return true; + } + size_t cnt = 0; + td::WalkPath::run(path, [&](td::CSlice name, td::WalkPath::Type type) { + if (type != td::WalkPath::Type::ExitDir) { + ++cnt; + } + if (cnt < 2) { + return td::WalkPath::Action::Continue; + } else { + return td::WalkPath::Action::Abort; + } + }).ignore(); + if (cnt == 1) { + td::rmdir(path).ignore(); + return true; + } + return false; +} + void StorageManager::on_torrent_closed(Torrent torrent, std::shared_ptr closing_state) { if (!closing_state->removing) { return; } if (closing_state->remove_files && torrent.inited_header()) { + // Ignore all errors: files may just not exist size_t files_count = torrent.get_files_count().unwrap(); for (size_t i = 0; i < files_count; ++i) { std::string path = torrent.get_file_path(i); td::unlink(path).ignore(); - // TODO: Check errors, remove empty directories + std::string name = torrent.get_file_name(i).str(); + for (int j = (int)name.size() - 1; j >= 0; --j) { + if (name[j] == '/') { + name.resize(j + 1); + if (!try_rm_empty_dir(torrent.get_root_dir() + '/' + torrent.get_header().dir_name + '/' + name)) { + break; + } + } + } + if (!torrent.get_header().dir_name.empty()) { + try_rm_empty_dir(torrent.get_root_dir() + '/' + torrent.get_header().dir_name); + } } } - td::rmrf(db_root_ + "/torrent-files/" + torrent.get_hash().to_hex()).ignore(); + std::string path = db_root_ + "/torrent-files/" + torrent.get_hash().to_hex(); + td::rmrf(path).ignore(); NodeActor::cleanup_db(db_, torrent.get_hash(), [promise = std::move(closing_state->promise)](td::Result R) mutable { if (R.is_error()) { @@ -292,4 +364,28 @@ void StorageManager::get_peers_info(td::Bits256 hash, td::actor::send_closure(entry->actor, &NodeActor::get_peers_info, std::move(promise)); } +void StorageManager::get_speed_limits(td::Promise> promise) { + promise.set_result(std::make_pair(download_speed_limit_, upload_speed_limit_)); +} + +void StorageManager::set_download_speed_limit(double max_speed) { + if (max_speed < 0.0) { + max_speed = -1.0; + } + LOG(INFO) << "Set download speed limit to " << max_speed; + download_speed_limit_ = max_speed; + td::actor::send_closure(download_speed_limiter_, &SpeedLimiter::set_max_speed, max_speed); + db_store_config(); +} + +void StorageManager::set_upload_speed_limit(double max_speed) { + if (max_speed < 0.0) { + max_speed = -1.0; + } + LOG(INFO) << "Set upload speed limit to " << max_speed; + upload_speed_limit_ = max_speed; + td::actor::send_closure(upload_speed_limiter_, &SpeedLimiter::set_max_speed, max_speed); + db_store_config(); +} + } // namespace ton \ No newline at end of file diff --git a/storage/storage-daemon/StorageManager.h b/storage/storage-daemon/StorageManager.h index 102e1b9a..fbfd68eb 100644 --- a/storage/storage-daemon/StorageManager.h +++ b/storage/storage-daemon/StorageManager.h @@ -22,6 +22,7 @@ #include "overlay/overlays.h" #include "storage/PeerManager.h" #include "storage/db.h" +#include "SpeedLimiter.h" namespace ton { @@ -63,6 +64,10 @@ class StorageManager : public td::actor::Actor { void wait_for_completion(td::Bits256 hash, td::Promise promise); void get_peers_info(td::Bits256 hash, td::Promise> promise); + void get_speed_limits(td::Promise> promise); // Download, upload + void set_download_speed_limit(double max_speed); + void set_upload_speed_limit(double max_speed); + private: adnl::AdnlNodeIdShort local_id_; std::string db_root_; @@ -89,6 +94,13 @@ class StorageManager : public td::actor::Actor { std::map torrents_; + double download_speed_limit_ = -1.0; + double upload_speed_limit_ = -1.0; + td::actor::ActorOwn download_speed_limiter_ = + td::actor::create_actor("DownloadRateLimitrer", -1.0); + td::actor::ActorOwn upload_speed_limiter_ = + td::actor::create_actor("DownloadRateLimitrer", -1.0); + td::Status add_torrent_impl(Torrent torrent, bool start_download, bool allow_upload); td::Result get_torrent(td::Bits256 hash) { @@ -102,9 +114,11 @@ class StorageManager : public td::actor::Actor { td::unique_ptr create_callback(td::Bits256 hash, std::shared_ptr closing_state); + void loaded_config_from_db(tl_object_ptr config); void load_torrents_from_db(std::vector torrents); void loaded_torrent_from_db(td::Bits256 hash, td::Result> R); void after_load_torrents_from_db(); + void db_store_config(); void db_store_torrent_list(); void on_torrent_closed(Torrent torrent, std::shared_ptr closing_state); diff --git a/storage/storage-daemon/StorageProvider.cpp b/storage/storage-daemon/StorageProvider.cpp index d09b66cd..e1eb55d6 100644 --- a/storage/storage-daemon/StorageProvider.cpp +++ b/storage/storage-daemon/StorageProvider.cpp @@ -258,9 +258,13 @@ void StorageProvider::process_transaction(tl_object_ptr body = r_body.move_as_ok(); vm::CellSlice cs = vm::load_cell_slice(body); - // const op::offer_storage_contract = 0x107c49ef; - if (cs.size() >= 32 && cs.prefetch_long(32) == 0x107c49ef) { - new_contract_address = message->destination_->account_address_; + if (cs.size() >= 32) { + long long op_code = cs.prefetch_ulong(32); + // const op::offer_storage_contract = 0x107c49ef; -- old versions + // const op::deploy_storage_contract = 0xe4748df1; -- new versions + if((op_code == 0x107c49ef) || (op_code == 0xe4748df1)) { + new_contract_address = message->destination_->account_address_; + } } } if (!new_contract_address.empty()) { diff --git a/storage/storage-daemon/smartcont/constants.fc b/storage/storage-daemon/smartcont/constants.fc index 479cdfa3..19cc28b1 100644 --- a/storage/storage-daemon/smartcont/constants.fc +++ b/storage/storage-daemon/smartcont/constants.fc @@ -1,4 +1,5 @@ const op::offer_storage_contract = 0x107c49ef; +const op::deploy_storage_contract = 0xe4748df1; const op::close_contract = 0x79f937ea; const op::contract_deployed = 0xbf7bd0c1; const op::storage_contract_confirmed = 0xd4caedcd; diff --git a/storage/storage-daemon/smartcont/storage-contract-code.boc b/storage/storage-daemon/smartcont/storage-contract-code.boc index 03dd18ea..647b3a79 100644 Binary files a/storage/storage-daemon/smartcont/storage-contract-code.boc and b/storage/storage-daemon/smartcont/storage-contract-code.boc differ diff --git a/storage/storage-daemon/smartcont/storage-contract.fc b/storage/storage-daemon/smartcont/storage-contract.fc index 41e3ee5b..b9087f41 100644 --- a/storage/storage-daemon/smartcont/storage-contract.fc +++ b/storage/storage-daemon/smartcont/storage-contract.fc @@ -76,7 +76,7 @@ int check_proof(int merkle_hash, int byte_to_proof, int file_size, cell file_dic int query_id = in_msg_body~load_uint(64); - if(op == op::offer_storage_contract) { + if(op == op::deploy_storage_contract) { add_to_balance(msg_value - 2 * fee::receipt_value); var (client, torrent_hash) = get_client_data(get_data().begin_parse()); var msg = begin_cell() @@ -263,4 +263,4 @@ _ get_next_proof_info() method_id { next_proof, rate_per_mb_day, max_span, last_proof_time, client, torrent_hash) = get_storage_contract_data(); return (next_proof, last_proof_time, max_span); -} \ No newline at end of file +} diff --git a/storage/storage-daemon/smartcont/storage-contract.fif b/storage/storage-daemon/smartcont/storage-contract.fif index 5dd86983..05641521 100644 --- a/storage/storage-daemon/smartcont/storage-contract.fif +++ b/storage/storage-daemon/smartcont/storage-contract.fif @@ -111,7 +111,7 @@ PROGRAM{ }> // msg_value sender_address op in_msg_body 64 LDU // msg_value sender_address op query_id in_msg_body s2 PUSH - 276580847 PUSHINT // msg_value sender_address op query_id in_msg_body op _27=276580847 + 3832843761 PUSHINT // msg_value sender_address op query_id in_msg_body op _27=3832843761 EQUAL // msg_value sender_address op query_id in_msg_body _28 IF:<{ // msg_value sender_address op query_id in_msg_body s0 s4 XCHG diff --git a/storage/storage-daemon/smartcont/storage-provider-code.boc b/storage/storage-daemon/smartcont/storage-provider-code.boc index e7cee4ff..e331154f 100644 Binary files a/storage/storage-daemon/smartcont/storage-provider-code.boc and b/storage/storage-daemon/smartcont/storage-provider-code.boc differ diff --git a/storage/storage-daemon/smartcont/storage-provider.fc b/storage/storage-daemon/smartcont/storage-provider.fc index 17223833..07d6b563 100644 --- a/storage/storage-daemon/smartcont/storage-provider.fc +++ b/storage/storage-daemon/smartcont/storage-provider.fc @@ -2,9 +2,12 @@ #include "constants.fc"; -const min_deploy_amount = 50000000; +const min_deploy_amount = 60000000; + +;; cell storage_contract_code() asm """ "storage-contract-code.boc" file>B B>boc PUSHREF """; +;; the same constant but more "compiler" friendly +cell storage_contract_code() asm " B{B5EE9C72010213010002FD000114FF00F4A413F4BCF2C80B0102016202030202CE04050201200D0E020120060700115D74CD0FA40D3FF30804E1007434C0C05C6C2497C0F83E900C0871C023A0D6F6CF380074C7C8700023A117C0F6CF3834CFC8A084391D237C6EA3AD4120829896802876CF3B51343C00E0842FDEF4305C20063232C1540133C5A0824C4B403E8084F2DA84B2C7D48832CFF2FFF25C3EC0244D388860841E8D85A22EA008080809006F35CE6CE4D7C11C3834C1C070C0E4D7C11C3834FFC12F64D7C0DC3800B50C1C25A010086B092E64693A0CC06AC140BD039BE84C645FF81C20002CED44D0D200FA0003A001C8CA0001FA0201CF16C9ED5403E68E4FED44D0D200FA00FA4003B3F2E3EF5350C705F2E1917FC8CA0058FA0201CF1621CF16C9ED54F003F8258210D4CAEDCD708018C8CB055005CF168209312D00FA0214CB6A13CB1F12CB3FCBFFC970FB00DE21821079F937EABAE30221821046ED2E94BA9130E30D8210419D5D4DBA915BE30D0A0B0C00EA10235F03ED44D0D200FA00FA40F0035352C7055162C70516B1F2E191F8258210B6236D63708018C8CB055004CF165005FA0212CB6A13CB1F12CB3F5230CBFFC902B39730318100A0FB00E0018040FB00F8258210B6236D63708018C8CB055004CF1623FA0213CB6A12CB1FCB3FCBFFC98100A0FB000092ED44D0D200FA00FA403002F2E3EB5341C705F2E19120C200998208989680A072FB029130E28210A91BAF56708018C8CB055003CF168209312D00FA0212CB6ACB1FCB3FC98100A0FB0000FA01D430ED44D0D200FA00FA40D3FFD33FD33FFA00D31FD31FD43009F2E3EB53A6C705F2E191544540525BF001F2E3EA22F811F8235003A128B6085331A8018102A3AA1AA984067007A116B609F8237FC8CA0058FA025005CF1613CBFFCB3FCB3F58FA0213CB1F12CB1FCCC9ED54708018C8CB0558CF16CB6EC98042FB000201200F100011BEE6576A2686B8500402012011120033B9241ED44D0D200FA00FA40D3FFD33FD33FFA00D31FD31FF00380017B6E4F0402A483DA87B0D9430001BB7CA50402A483DA87B0B664D8A70} B>boc PUSHREF "; -cell storage_contract_code() asm """ "storage-contract-code.boc" file>B B>boc PUSHREF """; slice calculate_address_by_stateinit(cell state_init) { return begin_cell().store_uint(4, 3) @@ -67,7 +70,7 @@ cell build_storage_contract_stateinit(int merkle_hash, int file_size, int rate_p .store_coins(0) .store_uint(4 + 2, 1 + 4 + 4 + 64 + 32 + 1 + 1 + 1) .store_ref(state_init) - .store_uint(op::offer_storage_contract, 32) + .store_uint(op::deploy_storage_contract, 32) .store_uint(query_id, 64) .end_cell(); send_raw_message(msg, 64); diff --git a/storage/storage-daemon/smartcont/storage-provider.fif b/storage/storage-daemon/smartcont/storage-provider.fif index 28dddc7c..d926c1c6 100644 --- a/storage/storage-daemon/smartcont/storage-provider.fif +++ b/storage/storage-daemon/smartcont/storage-provider.fif @@ -52,7 +52,7 @@ PROGRAM{ STREF // _33 ENDC // data 0 PUSHINT // data _36=0 - "storage-contract-code.boc" file>B B>boc PUSHREF // data _36=0 _37 + B{B5EE9C72010213010002FD000114FF00F4A413F4BCF2C80B0102016202030202CE04050201200D0E020120060700115D74CD0FA40D3FF30804E1007434C0C05C6C2497C0F83E900C0871C023A0D6F6CF380074C7C8700023A117C0F6CF3834CFC8A084391D237C6EA3AD4120829896802876CF3B51343C00E0842FDEF4305C20063232C1540133C5A0824C4B403E8084F2DA84B2C7D48832CFF2FFF25C3EC0244D388860841E8D85A22EA008080809006F35CE6CE4D7C11C3834C1C070C0E4D7C11C3834FFC12F64D7C0DC3800B50C1C25A010086B092E64693A0CC06AC140BD039BE84C645FF81C20002CED44D0D200FA0003A001C8CA0001FA0201CF16C9ED5403E68E4FED44D0D200FA00FA4003B3F2E3EF5350C705F2E1917FC8CA0058FA0201CF1621CF16C9ED54F003F8258210D4CAEDCD708018C8CB055005CF168209312D00FA0214CB6A13CB1F12CB3FCBFFC970FB00DE21821079F937EABAE30221821046ED2E94BA9130E30D8210419D5D4DBA915BE30D0A0B0C00EA10235F03ED44D0D200FA00FA40F0035352C7055162C70516B1F2E191F8258210B6236D63708018C8CB055004CF165005FA0212CB6A13CB1F12CB3F5230CBFFC902B39730318100A0FB00E0018040FB00F8258210B6236D63708018C8CB055004CF1623FA0213CB6A12CB1FCB3FCBFFC98100A0FB000092ED44D0D200FA00FA403002F2E3EB5341C705F2E19120C200998208989680A072FB029130E28210A91BAF56708018C8CB055003CF168209312D00FA0212CB6ACB1FCB3FC98100A0FB0000FA01D430ED44D0D200FA00FA40D3FFD33FD33FFA00D31FD31FD43009F2E3EB53A6C705F2E191544540525BF001F2E3EA22F811F8235003A128B6085331A8018102A3AA1AA984067007A116B609F8237FC8CA0058FA025005CF1613CBFFCB3FCB3F58FA0213CB1F12CB1FCCC9ED54708018C8CB0558CF16CB6EC98042FB000201200F100011BEE6576A2686B8500402012011120033B9241ED44D0D200FA00FA40D3FFD33FD33FFA00D31FD31FF00380017B6E4F0402A483DA87B0D9430001BB7CA50402A483DA87B0B664D8A70} B>boc PUSHREF // data _36=0 _37 OVER // data _36=0 _37 _38=0 NEWC // data _36=0 _37 _38=0 _39 2 STU // data _36=0 _37 _41 @@ -93,19 +93,19 @@ PROGRAM{ s1 s5 XCHG s1 s6 XCHG // query_id merkle_hash file_size rate_per_mb_day max_span client torrent_hash build_storage_contract_stateinit CALLDICT // query_id state_init - 276580847 PUSHINT // query_id state_init _54=276580847 - 6 PUSHINT // query_id state_init _54=276580847 _57 - 24 PUSHINT // query_id state_init _54=276580847 _57 _58=24 - NEWC // query_id state_init _54=276580847 _57 _58=24 _59 - 6 STU // query_id state_init _54=276580847 _57 _61 - s3 PUSH // query_id state_init _54=276580847 _57 _61 state_init - calculate_address_by_stateinit CALLDICT // query_id state_init _54=276580847 _57 _61 _62 - STSLICER // query_id state_init _54=276580847 _57 _63 - 0 PUSHINT // query_id state_init _54=276580847 _57 _63 _64=0 - STGRAMS // query_id state_init _54=276580847 _57 _65 - 108 STU // query_id state_init _54=276580847 _81 - s1 s2 XCHG // query_id _54=276580847 state_init _81 - STREF // query_id _54=276580847 _82 + 3832843761 PUSHINT // query_id state_init _54=3832843761 + 6 PUSHINT // query_id state_init _54=3832843761 _57 + 24 PUSHINT // query_id state_init _54=3832843761 _57 _58=24 + NEWC // query_id state_init _54=3832843761 _57 _58=24 _59 + 6 STU // query_id state_init _54=3832843761 _57 _61 + s3 PUSH // query_id state_init _54=3832843761 _57 _61 state_init + calculate_address_by_stateinit CALLDICT // query_id state_init _54=3832843761 _57 _61 _62 + STSLICER // query_id state_init _54=3832843761 _57 _63 + 0 PUSHINT // query_id state_init _54=3832843761 _57 _63 _64=0 + STGRAMS // query_id state_init _54=3832843761 _57 _65 + 108 STU // query_id state_init _54=3832843761 _81 + s1 s2 XCHG // query_id _54=3832843761 state_init _81 + STREF // query_id _54=3832843761 _82 32 STU // query_id _84 64 STU // _86 ENDC // msg @@ -113,6 +113,7 @@ PROGRAM{ SENDRAWMSG }> recv_internal PROC:<{ + c2 SAVE SAMEALTSAVE // msg_value in_msg_full in_msg_body SWAP // msg_value in_msg_body in_msg_full CTOS // msg_value in_msg_body cs @@ -142,7 +143,7 @@ PROGRAM{ IFJMP:<{ // msg_value sender_address op query_id in_msg_body s2 POP // msg_value sender_address in_msg_body query_id s0 s3 XCHG - 50000000 PUSHINT // query_id sender_address in_msg_body msg_value _29=50000000 + 60000000 PUSHINT // query_id sender_address in_msg_body msg_value _29=60000000 GEQ // query_id sender_address in_msg_body _30 1001 THROWIFNOT LDREF // query_id sender_address torrent_info in_msg_body diff --git a/storage/storage-daemon/storage-daemon-cli.cpp b/storage/storage-daemon/storage-daemon-cli.cpp index 4d9345a8..9cb1d417 100644 --- a/storage/storage-daemon/storage-daemon-cli.cpp +++ b/storage/storage-daemon/storage-daemon-cli.cpp @@ -107,6 +107,29 @@ std::string size_to_str(td::uint64 size) { return s.as_cslice().str(); } +td::Result str_to_size(std::string s) { + if (!s.empty() && std::tolower(s.back()) == 'b') { + s.pop_back(); + } + int shift = 0; + if (!s.empty()) { + auto c = std::tolower(s.back()); + static std::string suffixes = "kmgtpe"; + for (size_t i = 0; i < suffixes.size(); ++i) { + if (c == suffixes[i]) { + shift = 10 * (int)(i + 1); + s.pop_back(); + break; + } + } + } + TRY_RESULT(x, td::to_integer_safe(s)); + if (td::count_leading_zeroes64(x) < shift) { + return td::Status::Error("Number is too big"); + } + return x << shift; +} + std::string time_to_str(td::uint32 time) { char time_buffer[80]; time_t rawtime = time; @@ -285,10 +308,13 @@ class StorageDaemonCli : public td::actor::Actor { } std::_Exit(0); } else if (tokens[0] == "help") { - if (tokens.size() != 1) { + std::string category; + if (tokens.size() == 2) { + category = tokens[1]; + } else if (tokens.size() != 1) { return td::Status::Error("Unexpected tokens"); } - return execute_help(); + return execute_help(category); } else if (tokens[0] == "setverbosity") { if (tokens.size() != 2) { return td::Status::Error("Expected level"); @@ -460,6 +486,46 @@ class StorageDaemonCli : public td::actor::Actor { return td::Status::Error("Unexpected EOLN"); } return execute_get_peers(hash, json); + } else if (tokens[0] == "get-pieces-info") { + td::Bits256 hash; + bool found_hash = false; + bool json = false; + bool files = false; + td::uint64 offset = 0; + td::optional max_pieces; + for (size_t i = 1; i < tokens.size(); ++i) { + if (!tokens[i].empty() && tokens[i][0] == '-') { + if (tokens[i] == "--json") { + json = true; + continue; + } + if (tokens[i] == "--files") { + files = true; + continue; + } + if (tokens[i] == "--offset") { + TRY_RESULT_PREFIX_ASSIGN(offset, td::to_integer_safe(tokens[i + 1]), "Invalid offset: "); + ++i; + continue; + } + if (tokens[i] == "--max-pieces") { + TRY_RESULT_PREFIX_ASSIGN(max_pieces, td::to_integer_safe(tokens[i + 1]), "Invalid offset: "); + max_pieces.value() = std::min(max_pieces.value(), ((td::uint64)1 << 62)); + ++i; + continue; + } + return td::Status::Error(PSTRING() << "Unknown flag " << tokens[i]); + } + if (found_hash) { + return td::Status::Error("Unexpected token"); + } + TRY_RESULT_ASSIGN(hash, parse_torrent(tokens[i])); + found_hash = true; + } + if (!found_hash) { + return td::Status::Error("Unexpected EOLN"); + } + return execute_get_pieces_info(hash, files, offset, max_pieces, json); } else if (tokens[0] == "download-pause" || tokens[0] == "download-resume") { if (tokens.size() != 2) { return td::Status::Error("Expected bag"); @@ -544,6 +610,48 @@ class StorageDaemonCli : public td::actor::Actor { return td::Status::Error("Unexpected EOLN"); } return execute_load_from(hash, std::move(meta), std::move(path)); + } else if (tokens[0] == "get-speed-limits") { + bool json = false; + for (size_t i = 1; i < tokens.size(); ++i) { + if (!tokens[i].empty() && tokens[i][0] == '-') { + if (tokens[i] == "--json") { + json = true; + continue; + } + return td::Status::Error(PSTRING() << "Unknown flag " << tokens[i]); + } + return td::Status::Error("Unexpected token"); + } + return execute_get_speed_limits(json); + } else if (tokens[0] == "set-speed-limits") { + td::optional download, upload; + for (size_t i = 1; i < tokens.size(); ++i) { + if (!tokens[i].empty() && tokens[i][0] == '-') { + if (tokens[i] == "--download") { + ++i; + if (tokens[i] == "unlimited") { + download = -1.0; + } else { + TRY_RESULT_PREFIX(x, str_to_size(tokens[i]), "Invalid download speed: "); + download = (double)x; + } + continue; + } + if (tokens[i] == "--upload") { + ++i; + if (tokens[i] == "unlimited") { + upload = -1.0; + } else { + TRY_RESULT_PREFIX(x, str_to_size(tokens[i]), "Invalid upload speed: "); + upload = (double)x; + } + continue; + } + return td::Status::Error(PSTRING() << "Unknown flag " << tokens[i]); + } + return td::Status::Error("Unexpected token"); + } + return execute_set_speed_limits(download, upload); } else if (tokens[0] == "new-contract-message") { td::Bits256 hash; std::string file; @@ -659,12 +767,12 @@ class StorageDaemonCli : public td::actor::Actor { continue; } if (tokens[i] == "--min-file-size") { - TRY_RESULT_PREFIX(x, td::to_integer_safe(tokens[i + 1]), "Invalid value for --min-file-size: "); + TRY_RESULT_PREFIX(x, str_to_size(tokens[i + 1]), "Invalid value for --min-file-size: "); new_params.minimal_file_size = x; continue; } if (tokens[i] == "--max-file-size") { - TRY_RESULT_PREFIX(x, td::to_integer_safe(tokens[i + 1]), "Invalid value for --max-file-size: "); + TRY_RESULT_PREFIX(x, str_to_size(tokens[i + 1]), "Invalid value for --max-file-size: "); new_params.maximal_file_size = x; continue; } @@ -708,7 +816,7 @@ class StorageDaemonCli : public td::actor::Actor { continue; } if (tokens[i] == "--max-total-size") { - TRY_RESULT_PREFIX(x, td::to_integer_safe(tokens[i + 1]), "Invalid value for --max-total-size: "); + TRY_RESULT_PREFIX(x, str_to_size(tokens[i + 1]), "Invalid value for --max-total-size: "); new_config.max_total_size = x; continue; } @@ -765,93 +873,111 @@ class StorageDaemonCli : public td::actor::Actor { } } - td::Status execute_help() { - td::TerminalIO::out() << "help\tPrint this help\n"; - td::TerminalIO::out() - << "create [-d description] [--no-upload] [--copy] [--json] \tCreate bag of files from \n"; - td::TerminalIO::out() << "\t-d\tDescription will be stored in torrent info\n"; - td::TerminalIO::out() << "\t--no-upload\tDon't share bag with peers\n"; - td::TerminalIO::out() << "\t--copy\tFiles will be copied to an internal directory of storage-daemon\n"; - td::TerminalIO::out() << "\t--json\tOutput in json\n"; - td::TerminalIO::out() << "add-by-hash [-d root_dir] [--paused] [--no-upload] [--json] [--partial file1 " - "file2 ...]\tAdd bag with given BagID (in hex)\n"; - td::TerminalIO::out() << "\t-d\tTarget directory, default is an internal directory of storage-daemon\n"; - td::TerminalIO::out() << "\t--paused\tDon't start download immediately\n"; - td::TerminalIO::out() << "\t--no-upload\tDon't share bag with peers\n"; - td::TerminalIO::out() - << "\t--partial\tEverything after this flag is a list of filenames. Only these files will be downloaded.\n"; - td::TerminalIO::out() << "\t--json\tOutput in json\n"; - td::TerminalIO::out() << "add-by-meta [-d root_dir] [--paused] [--no-upload] [--json] [--partial file1 " - "file2 ...]\tLoad meta from file and add bag\n"; - td::TerminalIO::out() << "\tFlags are the same as in add-by-hash\n"; - td::TerminalIO::out() << "list [--hashes] [--json]\tPrint list of bags\n"; - td::TerminalIO::out() << "\t--hashes\tPrint full BagID\n"; - td::TerminalIO::out() << "\t--json\tOutput in json\n"; - td::TerminalIO::out() << "get [--json]\tPrint information about \n"; - td::TerminalIO::out() << "\t--json\tOutput in json\n"; - td::TerminalIO::out() << "\tHere and below bags are identified by BagID (in hex) or index (see bag list)\n"; - td::TerminalIO::out() << "get-meta \tSave bag meta of to \n"; - td::TerminalIO::out() << "get-peers [--json]\tPrint a list of peers\n"; - td::TerminalIO::out() << "\t--json\tOutput in json\n"; - td::TerminalIO::out() << "download-pause \tPause download of \n"; - td::TerminalIO::out() << "download-resume \tResume download of \n"; - td::TerminalIO::out() << "upload-pause \tPause upload of \n"; - td::TerminalIO::out() << "upload-resume \tResume upload of \n"; - td::TerminalIO::out() << "priority-all

\tSet priority of all files in to

\n"; - td::TerminalIO::out() << "\tPriority is in [0..255], 0 - don't download\n"; - td::TerminalIO::out() << "priority-idx

\tSet priority of file # in to

\n"; - td::TerminalIO::out() << "\tPriority is in [0..255], 0 - don't download\n"; - td::TerminalIO::out() << "priority-name

\tSet priority of file in to

\n"; - td::TerminalIO::out() << "\tPriority is in [0..255], 0 - don't download\n"; - td::TerminalIO::out() << "remove [--remove-files]\tRemove \n"; - td::TerminalIO::out() << "\t--remove-files - also remove all files\n"; - td::TerminalIO::out() << "load-from [--meta meta] [--files path]\tProvide meta and data for an existing " - "incomplete bag.\n"; - td::TerminalIO::out() << "\t--meta meta\ttorrent info and header will be inited (if not ready) from meta file\n"; - td::TerminalIO::out() << "\t--files path\tdata for files will be taken from here\n"; - td::TerminalIO::out() << "new-contract-message [--query-id id] --provider \tCreate " - "\"new contract message\" for storage provider. Saves message body to .\n"; - td::TerminalIO::out() << "\t\tAddress of storage provider account to take parameters from.\n"; - td::TerminalIO::out() << "new-contract-message [--query-id id] --rate --max-span " - "\tSame thing, but parameters are not fetched automatically.\n"; - td::TerminalIO::out() << "exit\tExit\n"; - td::TerminalIO::out() << "quit\tExit\n"; - td::TerminalIO::out() << "setverbosity \tSet vetbosity to in [0..10]\n"; - td::TerminalIO::out() << "\nStorage provider control:\n"; - td::TerminalIO::out() << "import-pk \tImport private key from \n"; - td::TerminalIO::out() << "deploy-provider\tInit storage provider by deploying a new provider smart contract\n"; - td::TerminalIO::out() - << "init-provider \tInit storage provider using the existing provider smart contract\n"; - td::TerminalIO::out() << "remove-storage-provider\tRemove storage provider\n"; - td::TerminalIO::out() - << "\tSmart contracts in blockchain and bags will remain intact, but they will not be managed anymore\n"; - td::TerminalIO::out() << "get-provider-params [address] [--json]\tPrint parameters of the smart contract\n"; - td::TerminalIO::out() - << "\taddress\tAddress of a smart contract. Default is the provider managed by this daemon.\n"; - td::TerminalIO::out() << "\t--json\tOutput in json\n"; - td::TerminalIO::out() << "set-provider-params [--accept x] [--rate x] [--max-span x] [--min-file-size x] " - "[--max-file-size x]\tSet parameters of the smart contract\n"; - td::TerminalIO::out() << "\t--accept\tAccept new contracts: 0 (no) or 1 (yes)\n"; - td::TerminalIO::out() << "\t--rate\tPrice of storage, nanoTON per MB*day\n"; - td::TerminalIO::out() << "\t--max-span\n"; - td::TerminalIO::out() << "\t--min-file-size\tMinimal total size of a bag of files (bytes)\n"; - td::TerminalIO::out() << "\t--max-file-size\tMaximal total size of a bag of files (bytes)\n"; - td::TerminalIO::out() - << "get-provider-info [--balances] [--contracts] [--json]\tPrint information about storage provider\n"; - td::TerminalIO::out() << "\t--contracts\tPrint list of storage contracts\n"; - td::TerminalIO::out() << "\t--balances\tPrint balances of the main contract and storage contracts\n"; - td::TerminalIO::out() << "\t--json\tOutput in json\n"; - td::TerminalIO::out() - << "set-provider-config [--max-contracts x] [--max-total-size x]\tSet configuration parameters\n"; - td::TerminalIO::out() << "\t--max-contracts\tMaximal number of storage contracts\n"; - td::TerminalIO::out() << "\t--max-total-size\tMaximal total size storage contracts (in bytes)\n"; - td::TerminalIO::out() << "withdraw

\tSend bounty from storage contract
to the main contract\n"; - td::TerminalIO::out() << "withdraw-all\tSend bounty from all storage contracts (where at least 1 TON is available) " - "to the main contract\n"; - td::TerminalIO::out() - << "send-coins
[--message msg]\tSend nanoTON to
from the main contract\n"; - td::TerminalIO::out() - << "close-contract
\tClose storage contract
and delete bag (if possible)\n"; + td::Status execute_help(std::string category) { + if (category == "") { + td::TerminalIO::out() << "create [-d description] [--no-upload] [--copy] [--json] \tCreate bag of " + "files from \n"; + td::TerminalIO::out() << "\t-d\tDescription will be stored in torrent info\n"; + td::TerminalIO::out() << "\t--no-upload\tDon't share bag with peers\n"; + td::TerminalIO::out() << "\t--copy\tFiles will be copied to an internal directory of storage-daemon\n"; + td::TerminalIO::out() << "\t--json\tOutput in json\n"; + td::TerminalIO::out() << "add-by-hash [-d root_dir] [--paused] [--no-upload] [--json] [--partial file1 " + "file2 ...]\tAdd bag with given BagID (in hex)\n"; + td::TerminalIO::out() << "\t-d\tTarget directory, default is an internal directory of storage-daemon\n"; + td::TerminalIO::out() << "\t--paused\tDon't start download immediately\n"; + td::TerminalIO::out() << "\t--no-upload\tDon't share bag with peers\n"; + td::TerminalIO::out() + << "\t--partial\tEverything after this flag is a list of filenames. Only these files will be downloaded.\n"; + td::TerminalIO::out() << "\t--json\tOutput in json\n"; + td::TerminalIO::out() << "add-by-meta [-d root_dir] [--paused] [--no-upload] [--json] [--partial file1 " + "file2 ...]\tLoad meta from file and add bag\n"; + td::TerminalIO::out() << "\tFlags are the same as in add-by-hash\n"; + td::TerminalIO::out() << "list [--hashes] [--json]\tPrint list of bags\n"; + td::TerminalIO::out() << "\t--hashes\tPrint full BagID\n"; + td::TerminalIO::out() << "\t--json\tOutput in json\n"; + td::TerminalIO::out() << "get [--json]\tPrint information about \n"; + td::TerminalIO::out() << "\t--json\tOutput in json\n"; + td::TerminalIO::out() << "\tHere and below bags are identified by BagID (in hex) or index (see bag list)\n"; + td::TerminalIO::out() << "get-meta \tSave bag meta of to \n"; + td::TerminalIO::out() << "get-peers [--json]\tPrint a list of peers\n"; + td::TerminalIO::out() << "get-pieces-info [--files] [--offset l] [--max-pieces m] [--json]\tPrint " + "information about ready pieces\n"; + td::TerminalIO::out() << "\t--files\tShow piece ranges for each file\n"; + td::TerminalIO::out() << "\t--offset l\tShow pieces starting from l (deafault: 0)\n"; + td::TerminalIO::out() << "\t--max-pieces m\tShow no more than m pieces (deafault: unlimited)\n"; + td::TerminalIO::out() << "\t--json\tOutput in json\n"; + td::TerminalIO::out() << "download-pause \tPause download of \n"; + td::TerminalIO::out() << "download-resume \tResume download of \n"; + td::TerminalIO::out() << "upload-pause \tPause upload of \n"; + td::TerminalIO::out() << "upload-resume \tResume upload of \n"; + td::TerminalIO::out() << "priority-all

\tSet priority of all files in to

\n"; + td::TerminalIO::out() << "\tPriority is in [0..255], 0 - don't download\n"; + td::TerminalIO::out() << "priority-idx

\tSet priority of file # in to

\n"; + td::TerminalIO::out() << "\tPriority is in [0..255], 0 - don't download\n"; + td::TerminalIO::out() << "priority-name

\tSet priority of file in to

\n"; + td::TerminalIO::out() << "\tPriority is in [0..255], 0 - don't download\n"; + td::TerminalIO::out() << "remove [--remove-files]\tRemove \n"; + td::TerminalIO::out() << "\t--remove-files - also remove all files\n"; + td::TerminalIO::out() << "load-from [--meta meta] [--files path]\tProvide meta and data for an existing " + "incomplete bag.\n"; + td::TerminalIO::out() << "\t--meta meta\ttorrent info and header will be inited (if not ready) from meta file\n"; + td::TerminalIO::out() << "\t--files path\tdata for files will be taken from here\n"; + td::TerminalIO::out() << "get-speed-limits [--json]\tShow global limits for download and upload speed\n"; + td::TerminalIO::out() << "\t--json\tOutput in json\n"; + td::TerminalIO::out() + << "set-speed-limits [--download x] [--upload x]\tSet global limits for download and upload speed\n"; + td::TerminalIO::out() << "\t--download x\tDownload speed limit in bytes/s, or \"unlimited\"\n"; + td::TerminalIO::out() << "\t--upload x\tUpload speed limit in bytes/s, or \"unlimited\"\n"; + td::TerminalIO::out() << "new-contract-message [--query-id id] --provider \tCreate " + "\"new contract message\" for storage provider. Saves message body to .\n"; + td::TerminalIO::out() << "\t\tAddress of storage provider account to take parameters from.\n"; + td::TerminalIO::out() << "new-contract-message [--query-id id] --rate --max-span " + "\tSame thing, but parameters are not fetched automatically.\n"; + td::TerminalIO::out() << "exit\tExit\n"; + td::TerminalIO::out() << "quit\tExit\n"; + td::TerminalIO::out() << "setverbosity \tSet vetbosity to in [0..10]\n"; + td::TerminalIO::out() << "help\tPrint this help\n"; + td::TerminalIO::out() << "help provider\tcommands for deploying and controling storage provider\n"; + } else if (category == "provider") { + td::TerminalIO::out() << "\nStorage provider control:\n"; + td::TerminalIO::out() << "import-pk \tImport private key from \n"; + td::TerminalIO::out() << "deploy-provider\tInit storage provider by deploying a new provider smart contract\n"; + td::TerminalIO::out() + << "init-provider \tInit storage provider using the existing provider smart contract\n"; + td::TerminalIO::out() << "remove-storage-provider\tRemove storage provider\n"; + td::TerminalIO::out() + << "\tSmart contracts in blockchain and bags will remain intact, but they will not be managed anymore\n"; + td::TerminalIO::out() << "get-provider-params [address] [--json]\tPrint parameters of the smart contract\n"; + td::TerminalIO::out() + << "\taddress\tAddress of a smart contract. Default is the provider managed by this daemon.\n"; + td::TerminalIO::out() << "\t--json\tOutput in json\n"; + td::TerminalIO::out() << "set-provider-params [--accept x] [--rate x] [--max-span x] [--min-file-size x] " + "[--max-file-size x]\tSet parameters of the smart contract\n"; + td::TerminalIO::out() << "\t--accept\tAccept new contracts: 0 (no) or 1 (yes)\n"; + td::TerminalIO::out() << "\t--rate\tPrice of storage, nanoTON per MB*day\n"; + td::TerminalIO::out() << "\t--max-span\n"; + td::TerminalIO::out() << "\t--min-file-size\tMinimal total size of a bag of files (bytes)\n"; + td::TerminalIO::out() << "\t--max-file-size\tMaximal total size of a bag of files (bytes)\n"; + td::TerminalIO::out() + << "get-provider-info [--balances] [--contracts] [--json]\tPrint information about storage provider\n"; + td::TerminalIO::out() << "\t--contracts\tPrint list of storage contracts\n"; + td::TerminalIO::out() << "\t--balances\tPrint balances of the main contract and storage contracts\n"; + td::TerminalIO::out() << "\t--json\tOutput in json\n"; + td::TerminalIO::out() + << "set-provider-config [--max-contracts x] [--max-total-size x]\tSet configuration parameters\n"; + td::TerminalIO::out() << "\t--max-contracts\tMaximal number of storage contracts\n"; + td::TerminalIO::out() << "\t--max-total-size\tMaximal total size storage contracts (in bytes)\n"; + td::TerminalIO::out() << "withdraw

\tSend bounty from storage contract
to the main contract\n"; + td::TerminalIO::out() + << "withdraw-all\tSend bounty from all storage contracts (where at least 1 TON is available) " + "to the main contract\n"; + td::TerminalIO::out() << "send-coins
[--message msg]\tSend nanoTON to
from " + "the main contract\n"; + td::TerminalIO::out() + << "close-contract
\tClose storage contract
and delete bag (if possible)\n"; + } else { + td::TerminalIO::out() << "Unknown command 'help " << category << "'\n"; + } command_finished(td::Status::OK()); return td::Status::OK(); } @@ -871,7 +997,7 @@ class StorageDaemonCli : public td::actor::Actor { td::Status execute_create(std::string path, std::string description, bool upload, bool copy, bool json) { TRY_RESULT_PREFIX_ASSIGN(path, td::realpath(path), "Invalid path: "); - auto query = create_tl_object(path, description, upload, copy); + auto query = create_tl_object(path, description, upload, copy, 0); send_query(std::move(query), [=, SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { @@ -904,7 +1030,7 @@ class StorageDaemonCli : public td::actor::Actor { } } auto query = create_tl_object(hash, std::move(root_dir), !paused, upload, - std::move(priorities)); + std::move(priorities), 0); send_query(std::move(query), [=, SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { @@ -938,7 +1064,7 @@ class StorageDaemonCli : public td::actor::Actor { } } auto query = create_tl_object(std::move(meta), std::move(root_dir), !paused, - upload, std::move(priorities)); + upload, std::move(priorities), 0); send_query(std::move(query), [=, SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { @@ -957,7 +1083,7 @@ class StorageDaemonCli : public td::actor::Actor { } td::Status execute_list(bool with_hashes, bool json) { - auto query = create_tl_object(); + auto query = create_tl_object(0); send_query(std::move(query), [=, SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { @@ -975,7 +1101,7 @@ class StorageDaemonCli : public td::actor::Actor { } td::Status execute_get(td::Bits256 hash, bool json) { - auto query = create_tl_object(hash); + auto query = create_tl_object(hash, 0); send_query(std::move(query), [=, SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { @@ -993,7 +1119,7 @@ class StorageDaemonCli : public td::actor::Actor { } td::Status execute_get_meta(td::Bits256 hash, std::string meta_file) { - auto query = create_tl_object(hash); + auto query = create_tl_object(hash, 0); send_query(std::move(query), [SelfId = actor_id(this), meta_file](td::Result> R) { if (R.is_error()) { @@ -1014,7 +1140,7 @@ class StorageDaemonCli : public td::actor::Actor { } td::Status execute_get_peers(td::Bits256 hash, bool json) { - auto query = create_tl_object(hash); + auto query = create_tl_object(hash, 0); send_query( std::move(query), [=, SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { @@ -1054,6 +1180,63 @@ class StorageDaemonCli : public td::actor::Actor { return td::Status::OK(); } + td::Status execute_get_pieces_info(td::Bits256 hash, bool files, td::uint64 offset, + td::optional max_pieces, bool json) { + auto query = create_tl_object(hash, files ? 1 : 0, offset, + max_pieces ? max_pieces.value() : -1); + send_query(std::move(query), + [=, SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + return; + } + if (json) { + print_json(R.ok()); + td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK()); + return; + } + auto obj = R.move_as_ok(); + td::TerminalIO::out() << "BagID " << hash.to_hex() << "\n"; + td::TerminalIO::out() << "Total pieces: " << obj->total_pieces_ << ", piece size: " << obj->piece_size_ + << "\n"; + if (files) { + if (obj->flags_ & 1) { + td::TerminalIO::out() << "Files:\n"; + std::vector> table; + table.push_back({"#####", "Piece range", "Name"}); + size_t i = 0; + for (const auto& f : obj->files_) { + table.push_back({i == 0 ? "" : td::to_string(i - 1), + PSTRING() << "[" << f->range_l_ << ".." << f->range_r_ << ")", + f->name_.empty() ? "[HEADER]" : f->name_}); + ++i; + } + print_table(table, {1, 2}); + } else { + td::TerminalIO::out() << "Cannot show files: torrent header is not available\n"; + } + } + td::uint64 l = obj->range_l_, r = obj->range_r_; + td::TerminalIO::out() << "Pieces [" << l << ".." << r << ")\n"; + if (obj->range_l_ != obj->range_r_) { + std::vector> table; + td::uint64 i = l; + while (i < r) { + td::uint64 ir = std::min(i + 100, r); + std::string s = "["; + for (td::uint64 j = i; j < ir; ++j) { + s += (obj->piece_ready_bitset_[(j - l) / 8] & (1 << ((j - l) % 8)) ? '#' : '-'); + } + s += ']'; + table.push_back({std::to_string(i), s}); + i = ir; + } + print_table(table, {1}); + } + td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK()); + }); + return td::Status::OK(); + } + td::Status execute_set_active_download(td::Bits256 hash, bool active) { auto query = create_tl_object(hash, active); send_query(std::move(query), @@ -1156,7 +1339,7 @@ class StorageDaemonCli : public td::actor::Actor { if (!path.empty()) { TRY_RESULT_PREFIX_ASSIGN(path, td::realpath(path), "Invalid path: "); } - auto query = create_tl_object(hash, std::move(meta_data), std::move(path)); + auto query = create_tl_object(hash, std::move(meta_data), std::move(path), 0); send_query(std::move(query), [SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { @@ -1183,6 +1366,59 @@ class StorageDaemonCli : public td::actor::Actor { return td::Status::OK(); } + td::Status execute_get_speed_limits(bool json) { + auto query = create_tl_object(0); + send_query(std::move(query), [=, SelfId = actor_id(this)]( + td::Result> R) { + if (R.is_error()) { + return; + } + if (json) { + print_json(R.ok()); + td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK()); + return; + } + auto obj = R.move_as_ok(); + if (obj->download_ < 0.0) { + td::TerminalIO::out() << "Download speed limit: unlimited\n"; + } else { + td::TerminalIO::out() << "Download speed limit: " << td::format::as_size((td::uint64)obj->download_) << "/s\n"; + } + if (obj->upload_ < 0.0) { + td::TerminalIO::out() << "Upload speed limit: unlimited\n"; + } else { + td::TerminalIO::out() << "Upload speed limit: " << td::format::as_size((td::uint64)obj->upload_) << "/s\n"; + } + td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK()); + }); + return td::Status::OK(); + } + + td::Status execute_set_speed_limits(td::optional download, td::optional upload) { + if (!download && !upload) { + return td::Status::Error("No parameters are set"); + } + auto query = create_tl_object(); + query->flags_ = 0; + if (download) { + query->flags_ |= 1; + query->download_ = download.value(); + } + if (upload) { + query->flags_ |= 2; + query->upload_ = upload.value(); + } + send_query(std::move(query), + [SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + return; + } + td::TerminalIO::out() << "Speed limits were set\n"; + td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK()); + }); + return td::Status::OK(); + } + td::Status execute_new_contract_message(td::Bits256 hash, std::string file, td::uint64 query_id, td::optional provider_address, td::optional rate, td::optional max_span) { @@ -1282,25 +1518,26 @@ class StorageDaemonCli : public td::actor::Actor { td::Status execute_get_provider_params(std::string address, bool json) { auto query = create_tl_object(address); - send_query(std::move(query), - [=, SelfId = actor_id(this)](td::Result> R) { - if (R.is_error()) { - return; - } - if (json) { - print_json(R.ok()); - td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK()); - return; - } - auto params = R.move_as_ok(); - td::TerminalIO::out() << "Storage provider parameters:\n"; - td::TerminalIO::out() << "Accept new contracts: " << params->accept_new_contracts_ << "\n"; - td::TerminalIO::out() << "Rate (nanoTON per day*MB): " << params->rate_per_mb_day_ << "\n"; - td::TerminalIO::out() << "Max span: " << (td::uint32)params->max_span_ << "\n"; - td::TerminalIO::out() << "Min file size: " << (td::uint64)params->minimal_file_size_ << "\n"; - td::TerminalIO::out() << "Max file size: " << (td::uint64)params->maximal_file_size_ << "\n"; - td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK()); - }); + send_query(std::move(query), [=, SelfId = actor_id(this)]( + td::Result> R) { + if (R.is_error()) { + return; + } + if (json) { + print_json(R.ok()); + td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK()); + return; + } + auto params = R.move_as_ok(); + td::TerminalIO::out() << "Storage provider parameters:\n"; + td::TerminalIO::out() << "Accept new contracts: " << params->accept_new_contracts_ << "\n"; + td::TerminalIO::out() << "Rate (nanoTON per day*MB): " << params->rate_per_mb_day_ << "\n"; + td::TerminalIO::out() << "Max span: " << (td::uint32)params->max_span_ << "\n"; + auto min_size = (td::uint64)params->minimal_file_size_, max_size = (td::uint64)params->maximal_file_size_; + td::TerminalIO::out() << "Min file size: " << td::format::as_size(min_size) << " (" << min_size << ")\n"; + td::TerminalIO::out() << "Max file size: " << td::format::as_size(max_size) << " (" << max_size << ")\n"; + td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK()); + }); return td::Status::OK(); } @@ -1632,6 +1869,7 @@ class StorageDaemonCli : public td::actor::Actor { add_id(obj.torrent_->hash_); td::TerminalIO::out() << "BagID = " << obj.torrent_->hash_.to_hex() << "\n"; td::TerminalIO::out() << "Index = " << hash_to_id_[obj.torrent_->hash_] << "\n"; + td::TerminalIO::out() << "Added: " << time_to_str(obj.torrent_->added_at_) << "\n"; if (obj.torrent_->flags_ & 4) { // fatal error td::TerminalIO::out() << "FATAL ERROR: " << obj.torrent_->fatal_error_ << "\n"; } diff --git a/storage/storage-daemon/storage-daemon.cpp b/storage/storage-daemon/storage-daemon.cpp index 7563332a..98388428 100644 --- a/storage/storage-daemon/storage-daemon.cpp +++ b/storage/storage-daemon/storage-daemon.cpp @@ -180,6 +180,9 @@ class StorageDaemon : public td::actor::Actor { dht_id_ = dht_id_full.compute_short_id(); td::actor::send_closure(adnl_, &adnl::Adnl::add_id, dht_id_full, addr_list, static_cast(0)); + LOG(INFO) << "Storage daemon ADNL id is " << local_id_; + LOG(INFO) << "DHT id is " << dht_id_; + if (client_mode_) { auto D = dht::Dht::create_client(dht_id_, db_root_, dht_config_, keyring_.get(), adnl_.get()); D.ensure(); @@ -431,6 +434,52 @@ class StorageDaemon : public td::actor::Actor { })); } + void run_control_query(ton_api::storage_daemon_getTorrentPiecesInfo &query, td::Promise promise) { + td::Bits256 hash = query.hash_; + td::actor::send_closure( + manager_, &StorageManager::with_torrent, hash, + promise.wrap([query = std::move(query)](NodeActor::NodeState state) -> td::Result { + Torrent &torrent = state.torrent; + if (!torrent.inited_info()) { + return td::Status::Error("Torrent info is not available"); + } + td::uint64 total_pieces = torrent.get_info().pieces_count(); + td::uint64 range_l = std::min(total_pieces, query.offset_); + td::uint64 size = query.max_pieces_ != -1 ? std::min(total_pieces - range_l, query.max_pieces_) + : total_pieces - range_l; + td::BufferSlice piece_ready((size + 7) / 8); + std::fill(piece_ready.data(), piece_ready.data() + piece_ready.size(), 0); + for (td::uint64 i = range_l; i < range_l + size; ++i) { + if (torrent.is_piece_ready(i)) { + piece_ready.data()[(i - range_l) / 8] |= (1 << ((i - range_l) % 8)); + } + } + + auto result = create_tl_object(); + result->total_pieces_ = total_pieces; + result->piece_size_ = torrent.get_info().piece_size; + result->range_l_ = range_l; + result->range_r_ = range_l + size; + result->piece_ready_bitset_ = std::move(piece_ready); + + if ((query.flags_ & 1) && torrent.inited_header()) { + result->flags_ = 1; + auto range = torrent.get_header_parts_range(); + result->files_.push_back( + create_tl_object("", range.begin, range.end)); + for (size_t i = 0; i < torrent.get_files_count().value(); ++i) { + auto range = torrent.get_file_parts_range(i); + result->files_.push_back(create_tl_object( + torrent.get_file_name(i).str(), range.begin, range.end)); + } + } else { + result->flags_ = 0; + } + + return serialize_tl_object(result, true); + })); + } + void run_control_query(ton_api::storage_daemon_setFilePriorityAll &query, td::Promise promise) { TRY_RESULT_PROMISE(promise, priority, td::narrow_cast_safe(query.priority_)); td::actor::send_closure(manager_, &StorageManager::set_all_files_priority, query.hash_, priority, @@ -491,6 +540,24 @@ class StorageDaemon : public td::actor::Actor { }); } + void run_control_query(ton_api::storage_daemon_getSpeedLimits &query, td::Promise promise) { + td::actor::send_closure(manager_, &StorageManager::get_speed_limits, + promise.wrap([](std::pair limits) -> td::BufferSlice { + return create_serialize_tl_object(limits.first, + limits.second); + })); + } + + void run_control_query(ton_api::storage_daemon_setSpeedLimits &query, td::Promise promise) { + if (query.flags_ & 1) { + td::actor::send_closure(manager_, &StorageManager::set_download_speed_limit, query.download_); + } + if (query.flags_ & 2) { + td::actor::send_closure(manager_, &StorageManager::set_upload_speed_limit, query.upload_); + } + promise.set_result(create_serialize_tl_object()); + } + void run_control_query(ton_api::storage_daemon_getNewContractMessage &query, td::Promise promise) { td::Promise> P = [promise = std::move(promise), hash = query.hash_, query_id = query.query_id_, @@ -779,6 +846,7 @@ class StorageDaemon : public td::actor::Actor { file->name_ = torrent.get_file_name(i).str(); file->size_ = torrent.get_file_size(i); file->downloaded_size_ = torrent.get_file_ready_size(i); + file->flags_ = 0; obj.files_.push_back(std::move(file)); } } @@ -798,6 +866,7 @@ class StorageDaemon : public td::actor::Actor { obj->active_upload_ = state.active_upload; obj->download_speed_ = state.download_speed; obj->upload_speed_ = state.upload_speed; + obj->added_at_ = state.added_at; promise.set_result(std::move(obj)); }); } @@ -816,6 +885,7 @@ class StorageDaemon : public td::actor::Actor { obj->torrent_->active_upload_ = state.active_upload; obj->torrent_->download_speed_ = state.download_speed; obj->torrent_->upload_speed_ = state.upload_speed; + obj->torrent_->added_at_ = state.added_at; for (size_t i = 0; i < obj->files_.size(); ++i) { obj->files_[i]->priority_ = (i < state.file_priority.size() ? state.file_priority[i] : 1); @@ -857,9 +927,11 @@ class StorageDaemon : public td::actor::Actor { } auto r_conf_data = td::read_file(global_config_); r_conf_data.ensure(); + std::string key_store = db_root_ + "/tonlib/"; + td::mkpath(key_store).ensure(); auto tonlib_options = tonlib_api::make_object( tonlib_api::make_object(r_conf_data.move_as_ok().as_slice().str(), "", false, false), - tonlib_api::make_object()); + tonlib_api::make_object(key_store)); tonlib_client_ = td::actor::create_actor("tonlibclient", std::move(tonlib_options)); } diff --git a/storage/test/storage.cpp b/storage/test/storage.cpp index b4f67b9b..ff5a4831 100644 --- a/storage/test/storage.cpp +++ b/storage/test/storage.cpp @@ -400,7 +400,6 @@ class NetChannel : public td::actor::Actor { break; } else if (l > alive_end - eps) { alive_begin += alive_step + sleep_step; - alive_end = alive_begin + alive_step; } else { double new_l = td::min(alive_end, r); res += (new_l - l) * speed; @@ -516,9 +515,7 @@ class NetChannel : public td::actor::Actor { queue_ = {}; } - bool ok = false; while (!queue_.empty() && (double)queue_.front().size < got_) { - ok = true; auto query = queue_.pop(); got_ -= (double)query.size; total_size_ -= (double)query.size; @@ -1342,7 +1339,7 @@ TEST(Torrent, Peer) { guard->push_back(td::actor::create_actor( "Node#1", 1, std::move(torrent), td::make_unique(stop_watcher, complete_watcher), - td::make_unique(peer_manager.get(), 1, gen_peers(1, 2)), nullptr)); + td::make_unique(peer_manager.get(), 1, gen_peers(1, 2)), nullptr, ton::SpeedLimiters{})); for (size_t i = 2; i <= peers_n; i++) { ton::Torrent::Options options; options.in_memory = true; @@ -1351,7 +1348,7 @@ TEST(Torrent, Peer) { PSLICE() << "Node#" << i, i, std::move(other_torrent), td::make_unique(stop_watcher, complete_watcher), td::make_unique(peer_manager.get(), i, gen_peers(i, 2)), - nullptr); + nullptr, ton::SpeedLimiters{}); if (i == 3) { td::actor::create_actor("StatsActor", node_actor.get()).release(); diff --git a/tdactor/CMakeLists.txt b/tdactor/CMakeLists.txt index 3490eb17..98b900a1 100644 --- a/tdactor/CMakeLists.txt +++ b/tdactor/CMakeLists.txt @@ -1,18 +1,21 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) #SOURCE SETS set(TDACTOR_SOURCE td/actor/core/ActorExecutor.cpp + td/actor/core/ActorTypeStat.cpp td/actor/core/CpuWorker.cpp td/actor/core/IoWorker.cpp td/actor/core/Scheduler.cpp + td/actor/ActorStats.cpp td/actor/MultiPromise.cpp td/actor/actor.h td/actor/ActorId.h td/actor/ActorOwn.h td/actor/ActorShared.h + td/actor/ActorStats.h td/actor/common.h td/actor/PromiseFuture.h td/actor/MultiPromise.h @@ -27,6 +30,7 @@ set(TDACTOR_SOURCE td/actor/core/ActorMessage.h td/actor/core/ActorSignals.h td/actor/core/ActorState.h + td/actor/core/ActorTypeStat.h td/actor/core/CpuWorker.h td/actor/core/Context.h td/actor/core/IoWorker.h diff --git a/tdactor/benchmark/CMakeLists.txt b/tdactor/benchmark/CMakeLists.txt index e01d33dc..c4ff79a1 100644 --- a/tdactor/benchmark/CMakeLists.txt +++ b/tdactor/benchmark/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) set(BENCHMARK_SOURCE benchmark.cpp diff --git a/tdactor/td/actor/ActorStats.cpp b/tdactor/td/actor/ActorStats.cpp new file mode 100644 index 00000000..ad037204 --- /dev/null +++ b/tdactor/td/actor/ActorStats.cpp @@ -0,0 +1,245 @@ +#include "ActorStats.h" + +#include "td/utils/ThreadSafeCounter.h" +namespace td { +namespace actor { +void td::actor::ActorStats::start_up() { + auto now = td::Time::now(); + for (std::size_t i = 0; i < SIZE; i++) { + stat_[i] = td::TimedStat>(DURATIONS[i], now); + stat_[i].add_event(ActorTypeStats(), now); + } + begin_ts_ = td::Timestamp::now(); + begin_ticks_ = Clocks::rdtsc(); + loop(); +} +double ActorStats::estimate_inv_ticks_per_second() { + auto now = td::Timestamp::now(); + auto elapsed_seconds = now.at() - begin_ts_.at(); + auto now_ticks = td::Clocks::rdtsc(); + auto elapsed_ticks = now_ticks - begin_ticks_; + auto estimated_inv_ticks_per_second = + elapsed_seconds > 0.1 ? elapsed_seconds / double(elapsed_ticks) : Clocks::inv_ticks_per_second(); + return estimated_inv_ticks_per_second; +} + +std::string ActorStats::prepare_stats() { + auto estimated_inv_ticks_per_second = estimate_inv_ticks_per_second(); + + auto current_stats = td::actor::ActorTypeStatManager::get_stats(estimated_inv_ticks_per_second); + auto now = td::Timestamp::now(); + auto now_ticks = Clocks::rdtsc(); + + update(now); + + // Lets look at recent stats first + auto load_stats = [&](auto &timed_stat) { + auto res = current_stats; + auto &since = timed_stat.get_stat(now.at()); + auto duration = since.get_duration(estimated_inv_ticks_per_second); + if (since.first_) { + res -= since.first_.value(); + } + res /= duration; + return res.stats; + }; + auto stats_10s = load_stats(stat_[0]); + auto stats_10m = load_stats(stat_[1]); + current_stats /= double(now_ticks - begin_ticks_) * estimated_inv_ticks_per_second; + auto stats_forever = current_stats.stats; + + std::map current_perf_map; + std::map perf_map_10s; + std::map perf_map_10m; + std::map perf_values; + td::NamedPerfCounter::get_default().for_each( + [&](td::Slice name, td::int64 value_int64) { perf_values[name.str()] = double(value_int64); }); + for (auto &value_it : perf_values) { + const auto &name = value_it.first; + auto value = value_it.second; + + auto &perf_stat = pef_stats_[name]; + auto load_perf_stats = [&](auto &timed_stat, auto &m) { + double res = double(value); + auto &since = timed_stat.get_stat(now.at()); + auto duration = since.get_duration(estimated_inv_ticks_per_second); + if (since.first_) { + res -= since.first_.value(); + } + if (td::ends_with(name, ".duration")) { + res *= estimated_inv_ticks_per_second; + } + // m[name + ".raw"] = res; + // m[name + ".range"] = duration; + res /= duration; + return res; + }; + perf_map_10s[name] = load_perf_stats(perf_stat.perf_stat_[0], perf_map_10s); + perf_map_10m[name] = load_perf_stats(perf_stat.perf_stat_[1], perf_map_10m); + + auto current_duration = (double(now_ticks - begin_ticks_) * estimated_inv_ticks_per_second); + if (td::ends_with(name, ".duration")) { + value *= estimated_inv_ticks_per_second; + } + current_perf_map[name] = double(value) / current_duration; + // current_perf_map[name + ".raw"] = double(value); + // current_perf_map[name + ".range"] = double(now_ticks - begin_ticks_) * estimated_inv_ticks_per_second; + }; + + td::StringBuilder sb; + sb << "================================= PERF COUNTERS ================================\n"; + sb << "ticks_per_second_estimate\t" << 1.0 / estimated_inv_ticks_per_second << "\n"; + for (auto &it : perf_map_10s) { + const std::string &name = it.first; + auto dot_at = name.rfind('.'); + CHECK(dot_at != std::string::npos); + auto base_name = name.substr(0, dot_at); + auto rest_name = name.substr(dot_at + 1); + td::Slice new_rest_name = rest_name; + if (rest_name == "count") { + new_rest_name = "qps"; + } + if (rest_name == "duration") { + new_rest_name = "load"; + } + auto rewrite_name = PSTRING() << base_name << "." << new_rest_name; + sb << rewrite_name << "\t" << perf_map_10s[name] << " " << perf_map_10m[name] << " " << current_perf_map[name] + << "\n"; + } + sb << "\n"; + sb << "================================= ACTORS STATS =================================\n"; + double max_delay = 0; + ActorTypeStat sum_stat_forever; + ActorTypeStat sum_stat_10m; + ActorTypeStat sum_stat_10s; + for (auto &it : stats_forever) { + sum_stat_forever += it.second; + } + for (auto &it : stats_10m) { + sum_stat_10m += it.second; + } + for (auto &it : stats_10s) { + sum_stat_10s += it.second; + } + sb << "\n"; + + auto do_describe = [&](auto &&sb, const ActorTypeStat &stat_10s, const ActorTypeStat &stat_10m, + const ActorTypeStat &stat_forever) { + sb() << "load_per_second:\t" << stat_10s.seconds << " " << stat_10m.seconds << " " << stat_forever.seconds << "\n"; + sb() << "messages_per_second:\t" << stat_10s.messages << " " << stat_10m.messages << " " << stat_forever.messages + << "\n"; + + sb() << "max_execute_messages:\t" << stat_forever.max_execute_messages.value_10s << " " + << stat_forever.max_execute_messages.value_10m << " " << stat_forever.max_execute_messages.value_forever + << "\n"; + + sb() << "max_execute_seconds:\t" << stat_forever.max_execute_seconds.value_10s << "s" + << " " << stat_forever.max_execute_seconds.value_10m << "s" + << " " << stat_forever.max_execute_seconds.value_forever << "s\n"; + sb() << "max_message_seconds:\t" << stat_forever.max_message_seconds.value_10s << " " + << stat_forever.max_message_seconds.value_10m << " " << stat_forever.max_message_seconds.value_forever << "\n"; + sb() << "created_per_second:\t" << stat_10s.created << " " << stat_10m.created << " " << stat_forever.created + << "\n"; + + auto executing_for = + stat_forever.executing_start > 1e15 + ? 0 + : double(td::Clocks::rdtsc()) * estimated_inv_ticks_per_second - stat_forever.executing_start; + sb() << "max_delay:\t" << stat_forever.max_delay_seconds.value_10s << "s " + << stat_forever.max_delay_seconds.value_10m << "s " << stat_forever.max_delay_seconds.value_forever << "s\n"; + sb() << "" + << "alive: " << stat_forever.alive << " executing: " << stat_forever.executing + << " max_executing_for: " << executing_for << "s\n"; + }; + + auto describe = [&](td::StringBuilder &sb, std::type_index actor_type_index) { + auto stat_10s = stats_10s[actor_type_index]; + auto stat_10m = stats_10m[actor_type_index]; + auto stat_forever = stats_forever[actor_type_index]; + do_describe([&sb]() -> td::StringBuilder & { return sb << "\t\t"; }, stat_10s, stat_10m, stat_forever); + }; + + sb << "Cummulative stats:\n"; + do_describe([&sb]() -> td::StringBuilder & { return sb << "\t"; }, sum_stat_10s, sum_stat_10m, sum_stat_forever); + sb << "\n"; + + auto top_k_by = [&](auto &stats_map, size_t k, std::string description, auto by) { + auto stats = td::transform(stats_map, [](auto &it) { return std::make_pair(it.first, it.second); }); + k = std::min(k, stats.size()); + std::partial_sort(stats.begin(), stats.begin() + k, stats.end(), [&](auto &a, auto &b) { return by(a) > by(b); }); + bool is_first = true; + for (size_t i = 0; i < k; i++) { + auto value = by(stats[i]); + if (value < 1e-9) { + break; + } + if (is_first) { + sb << "top actors by " << description << "\n"; + is_first = false; + } + sb << "\t#" << i << ": " << ActorTypeStatManager::get_class_name(stats[i].first.name()) << "\t" << value << "\n"; + } + sb << "\n"; + }; + using Entry = std::pair; + static auto cutoff = [](auto x, auto min_value) { return x < min_value ? decltype(x){} : x; }; + top_k_by(stats_10s, 10, "load_10s", [](auto &x) { return cutoff(x.second.seconds, 0.005); }); + + top_k_by(stats_10m, 10, "load_10m", [](auto &x) { return cutoff(x.second.seconds, 0.005); }); + top_k_by(stats_forever, 10, "max_execute_seconds_10m", + [](Entry &x) { return cutoff(x.second.max_execute_seconds.value_10m, 0.5); }); + auto rdtsc_seconds = double(now_ticks) * estimated_inv_ticks_per_second; + top_k_by(stats_forever, 10, "executing_for", [&](Entry &x) { + if (x.second.executing_start > 1e15) { + return 0.0; + } + return rdtsc_seconds - x.second.executing_start; + }); + top_k_by(stats_forever, 10, "max_execute_messages_10m", + [](Entry &x) { return cutoff(x.second.max_execute_messages.value_10m, 10u); }); + + auto stats = td::transform(stats_forever, [](auto &it) { return std::make_pair(it.first, it.second); }); + + auto main_key = [&](std::type_index actor_type_index) { + auto stat_10s = stats_10s[actor_type_index]; + auto stat_10m = stats_10m[actor_type_index]; + auto stat_forever = stats_forever[actor_type_index]; + return std::make_tuple(cutoff(std::max(stat_10s.seconds, stat_10m.seconds), 0.1), + cutoff(stat_forever.max_execute_seconds.value_10m, 0.5), stat_forever.seconds); + }; + std::sort(stats.begin(), stats.end(), + [&](auto &left, auto &right) { return main_key(left.first) > main_key(right.first); }); + auto debug = Debug(SchedulerContext::get()->scheduler_group()); + debug.dump(sb); + sb << "All actors:\n"; + for (auto &it : stats) { + sb << "\t" << ActorTypeStatManager::get_class_name(it.first.name()) << "\n"; + auto key = main_key(it.first); + describe(sb, it.first); + } + sb << "\n"; + return sb.as_cslice().str(); +} +ActorStats::PefStat::PefStat() { + for (std::size_t i = 0; i < SIZE; i++) { + perf_stat_[i] = td::TimedStat>(DURATIONS[i], td::Time::now()); + perf_stat_[i].add_event(0, td::Time::now()); + } +} + +void ActorStats::update(td::Timestamp now) { + auto stat = td::actor::ActorTypeStatManager::get_stats(estimate_inv_ticks_per_second()); + for (auto &timed_stat : stat_) { + timed_stat.add_event(stat, now.at()); + } + NamedPerfCounter::get_default().for_each([&](td::Slice name, td::int64 value) { + auto &stat = pef_stats_[name.str()].perf_stat_; + for (auto &timed_stat : stat) { + timed_stat.add_event(value, now.at()); + } + }); +} +constexpr int ActorStats::DURATIONS[SIZE]; +constexpr const char *ActorStats::DESCR[SIZE]; +} // namespace actor +} // namespace td diff --git a/tdactor/td/actor/ActorStats.h b/tdactor/td/actor/ActorStats.h new file mode 100644 index 00000000..00f7ebeb --- /dev/null +++ b/tdactor/td/actor/ActorStats.h @@ -0,0 +1,52 @@ +#pragma once +#include "td/actor/actor.h" +#include "td/utils/TimedStat.h" +namespace td { +namespace actor { + +class ActorStats : public td::actor::Actor { + public: + ActorStats() { + } + void start_up() override; + double estimate_inv_ticks_per_second(); + std::string prepare_stats(); + + private: + template + struct StatStorer { + void on_event(const T &event) { + if (!first_) { + first_ = event; + first_ts_ = Clocks::rdtsc(); + } + } + double get_duration(double inv_ticks_per_second) const { + if (first_) { + return std::max(1.0, (Clocks::rdtsc() - first_ts_) * inv_ticks_per_second); + } + return 1.0; + } + td::optional first_; + td::uint64 first_ts_; + }; + static constexpr std::size_t SIZE = 2; + static constexpr const char *DESCR[SIZE] = {"10sec", "10min"}; + static constexpr int DURATIONS[SIZE] = {10, 10 * 60}; + td::TimedStat> stat_[SIZE]; + struct PefStat { + PefStat(); + td::TimedStat> perf_stat_[SIZE]; + }; + std::map pef_stats_; + td::Timestamp begin_ts_; + td::uint64 begin_ticks_{}; + void loop() override { + alarm_timestamp() = td::Timestamp::in(5.0); + update(td::Timestamp::now()); + } + void update(td::Timestamp now); +}; + +} // namespace actor +} // namespace td diff --git a/tdactor/td/actor/common.h b/tdactor/td/actor/common.h index d3858974..222db743 100644 --- a/tdactor/td/actor/common.h +++ b/tdactor/td/actor/common.h @@ -19,6 +19,7 @@ #pragma once #include "td/actor/core/Actor.h" #include "td/actor/core/ActorSignals.h" +#include "td/actor/core/ActorTypeStat.h" #include "td/actor/core/SchedulerId.h" #include "td/actor/core/SchedulerContext.h" #include "td/actor/core/Scheduler.h" @@ -68,7 +69,7 @@ using core::set_debug; struct Debug { public: Debug() = default; - Debug(std::shared_ptr group_info) : group_info_(std::move(group_info)) { + Debug(core::SchedulerGroupInfo *group_info) : group_info_(group_info) { } template void for_each(F &&f) { @@ -80,18 +81,29 @@ struct Debug { } } - void dump() { - for_each([](core::Debug &debug) { + void dump(td::StringBuilder &sb) { + sb << "list of active actors with names:\n"; + for_each([&](core::Debug &debug) { core::DebugInfo info; debug.read(info); if (info.is_active) { - LOG(ERROR) << info.name << " " << td::format::as_time(Time::now() - info.start_at); + sb << "\t\"" << info.name << "\" is active for " << Time::now() - info.start_at << "s\n"; } }); + sb << "\nsizes of cpu local queues:\n"; + for (auto &scheduler : group_info_->schedulers) { + for (size_t i = 0; i < scheduler.cpu_threads_count; i++) { + auto size = scheduler.cpu_local_queue[i].size(); + if (size != 0) { + sb << "\tcpu#" << i << " queue.size() = " << size << "\n"; + } + } + } + sb << "\n"; } private: - std::shared_ptr group_info_; + core::SchedulerGroupInfo *group_info_; }; class Scheduler { @@ -142,7 +154,7 @@ class Scheduler { } Debug get_debug() { - return Debug{group_info_}; + return Debug{group_info_.get()}; } bool run() { @@ -200,6 +212,10 @@ class Scheduler { } }; +using core::ActorTypeStat; +using core::ActorTypeStatManager; +using core::ActorTypeStats; + // Some helper functions. Not part of public interface and not part // of namespace core namespace detail { @@ -324,7 +340,7 @@ void send_closure_impl(ActorRef actor_ref, ClosureT &&closure) { } template -void send_closure(ActorRef actor_ref, ArgsT &&... args) { +void send_closure(ActorRef actor_ref, ArgsT &&...args) { send_closure_impl(actor_ref, create_immediate_closure(std::forward(args)...)); } @@ -365,7 +381,7 @@ void send_closure_with_promise_later(ActorRef actor_ref, ClosureT &&closure, Pro } template -void send_closure_later(ActorRef actor_ref, ArgsT &&... args) { +void send_closure_later(ActorRef actor_ref, ArgsT &&...args) { send_closure_later_impl(actor_ref, create_delayed_closure(std::forward(args)...)); } @@ -396,15 +412,17 @@ inline void send_signals_later(ActorRef actor_ref, ActorSignals signals) { inline void register_actor_info_ptr(core::ActorInfoPtr actor_info_ptr) { auto state = actor_info_ptr->state().get_flags_unsafe(); + actor_info_ptr->on_add_to_queue(); core::SchedulerContext::get()->add_to_queue(std::move(actor_info_ptr), state.get_scheduler_id(), !state.is_shared()); } template -core::ActorInfoPtr create_actor(core::ActorOptions &options, ArgsT &&... args) noexcept { +core::ActorInfoPtr create_actor(core::ActorOptions &options, ArgsT &&...args) noexcept { auto *scheduler_context = core::SchedulerContext::get(); if (!options.has_scheduler()) { options.on_scheduler(scheduler_context->get_scheduler_id()); } + options.with_actor_stat_id(core::ActorTypeStatImpl::get_unique_id()); auto res = scheduler_context->get_actor_info_creator().create(std::make_unique(std::forward(args)...), options); register_actor_info_ptr(res); diff --git a/tdactor/td/actor/core/ActorExecutor.cpp b/tdactor/td/actor/core/ActorExecutor.cpp index 267758d5..d1542242 100644 --- a/tdactor/td/actor/core/ActorExecutor.cpp +++ b/tdactor/td/actor/core/ActorExecutor.cpp @@ -114,6 +114,8 @@ void ActorExecutor::start() noexcept { actor_execute_context_.set_actor(&actor_info_.actor()); + actor_stats_ = actor_info_.actor_type_stat(); + auto execute_timer = actor_stats_.create_execute_timer(); while (flush_one_signal(signals)) { if (actor_execute_context_.has_immediate_flags()) { return; @@ -175,6 +177,11 @@ void ActorExecutor::finish() noexcept { if (add_to_queue) { actor_info_ptr = actor_info_.actor().get_actor_info_ptr(); } + if (!flags().is_closed() && flags().is_in_queue()) { + // Must do it while we are locked, so to it optimistically + // we will add actor to queue after unlock OR we are already in a queue OR we will be closed + actor_info_.on_add_to_queue(); + } if (actor_locker_.try_unlock(flags())) { if (add_to_queue) { dispatcher_.add_to_queue(std::move(actor_info_ptr), flags().get_scheduler_id(), !flags().is_shared()); @@ -193,23 +200,31 @@ bool ActorExecutor::flush_one_signal(ActorSignals &signals) { } switch (signal) { //NB: Signals will be handled in order of their value. - // For clarity it conincides with order in this switch + // For clarity, it coincides with order in this switch case ActorSignals::Pause: actor_execute_context_.set_pause(); break; - case ActorSignals::Kill: + case ActorSignals::Kill: { + auto message_timer = actor_stats_.create_message_timer(); actor_execute_context_.set_stop(); break; - case ActorSignals::StartUp: + } + case ActorSignals::StartUp: { + auto message_timer = actor_stats_.create_message_timer(); + actor_stats_.created(); actor_info_.actor().start_up(); break; - case ActorSignals::Wakeup: + } + case ActorSignals::Wakeup: { + auto message_timer = actor_stats_.create_message_timer(); actor_info_.actor().wake_up(); break; + } case ActorSignals::Alarm: if (actor_execute_context_.get_alarm_timestamp() && actor_execute_context_.get_alarm_timestamp().is_in_past()) { actor_execute_context_.alarm_timestamp() = Timestamp::never(); actor_info_.set_alarm_timestamp(Timestamp::never()); + auto message_timer = actor_stats_.create_message_timer(); actor_info_.actor().alarm(); } break; @@ -245,6 +260,7 @@ bool ActorExecutor::flush_one_message() { } actor_execute_context_.set_link_token(message.get_link_token()); + auto message_timer = actor_stats_.create_message_timer(); message.run(); return true; } @@ -257,7 +273,9 @@ void ActorExecutor::flush_context_flags() { } flags_.set_closed(true); if (!flags_.get_signals().has_signal(ActorSignals::Signal::StartUp)) { + auto message_timer = actor_stats_.create_message_timer(); actor_info_.actor().tear_down(); + actor_stats_.destroyed(); } actor_info_.destroy_actor(); } else { diff --git a/tdactor/td/actor/core/ActorExecutor.h b/tdactor/td/actor/core/ActorExecutor.h index 366cb754..dd86b5ae 100644 --- a/tdactor/td/actor/core/ActorExecutor.h +++ b/tdactor/td/actor/core/ActorExecutor.h @@ -24,6 +24,7 @@ #include "td/actor/core/ActorMessage.h" #include "td/actor/core/ActorSignals.h" #include "td/actor/core/ActorState.h" +#include "td/actor/core/ActorTypeStat.h" #include "td/actor/core/SchedulerContext.h" #include "td/utils/format.h" @@ -95,6 +96,7 @@ class ActorExecutor { ActorInfo &actor_info_; SchedulerDispatcher &dispatcher_; Options options_; + ActorTypeStatRef actor_stats_; ActorLocker actor_locker_{&actor_info_.state(), ActorLocker::Options() .with_can_execute_paused(options_.from_queue) .with_is_shared(!options_.has_poll) diff --git a/tdactor/td/actor/core/ActorInfo.h b/tdactor/td/actor/core/ActorInfo.h index 97419293..690b06d9 100644 --- a/tdactor/td/actor/core/ActorInfo.h +++ b/tdactor/td/actor/core/ActorInfo.h @@ -19,6 +19,7 @@ #pragma once #include "td/actor/core/ActorState.h" +#include "td/actor/core/ActorTypeStat.h" #include "td/actor/core/ActorMailbox.h" #include "td/utils/Heap.h" @@ -34,8 +35,8 @@ class ActorInfo; using ActorInfoPtr = SharedObjectPool::Ptr; class ActorInfo : private HeapNode, private ListNode { public: - ActorInfo(std::unique_ptr actor, ActorState::Flags state_flags, Slice name) - : actor_(std::move(actor)), name_(name.begin(), name.size()) { + ActorInfo(std::unique_ptr actor, ActorState::Flags state_flags, Slice name, td::uint32 actor_stat_id) + : actor_(std::move(actor)), name_(name.begin(), name.size()), actor_stat_id_(actor_stat_id) { state_.set_flags_unsafe(state_flags); VLOG(actor) << "Create actor [" << name_ << "]"; } @@ -58,6 +59,18 @@ class ActorInfo : private HeapNode, private ListNode { Actor *actor_ptr() const { return actor_.get(); } + // NB: must be called only when actor is locked + ActorTypeStatRef actor_type_stat() { + auto res = ActorTypeStatManager::get_actor_type_stat(actor_stat_id_, actor_.get()); + if (in_queue_since_) { + res.pop_from_queue(in_queue_since_); + in_queue_since_ = 0; + } + return res; + } + void on_add_to_queue() { + in_queue_since_ = td::Clocks::rdtsc(); + } void destroy_actor() { actor_.reset(); } @@ -103,6 +116,8 @@ class ActorInfo : private HeapNode, private ListNode { std::atomic alarm_timestamp_at_{0}; ActorInfoPtr pin_; + td::uint64 in_queue_since_{0}; + td::uint32 actor_stat_id_{0}; }; } // namespace core diff --git a/tdactor/td/actor/core/ActorInfoCreator.h b/tdactor/td/actor/core/ActorInfoCreator.h index 49c7611a..d818dc63 100644 --- a/tdactor/td/actor/core/ActorInfoCreator.h +++ b/tdactor/td/actor/core/ActorInfoCreator.h @@ -46,10 +46,16 @@ class ActorInfoCreator { return *this; } + Options& with_actor_stat_id(td::uint32 new_id) { + actor_stat_id = new_id; + return *this; + } + private: friend class ActorInfoCreator; Slice name; SchedulerId scheduler_id; + td::uint32 actor_stat_id{0}; bool is_shared{true}; bool in_queue{true}; //TODO: rename @@ -65,7 +71,7 @@ class ActorInfoCreator { flags.set_in_queue(args.in_queue); flags.set_signals(ActorSignals::one(ActorSignals::StartUp)); - auto actor_info_ptr = pool_.alloc(std::move(actor), flags, args.name); + auto actor_info_ptr = pool_.alloc(std::move(actor), flags, args.name, args.actor_stat_id); actor_info_ptr->actor().set_actor_info_ptr(actor_info_ptr); return actor_info_ptr; } diff --git a/tdactor/td/actor/core/ActorTypeStat.cpp b/tdactor/td/actor/core/ActorTypeStat.cpp new file mode 100644 index 00000000..4de6f105 --- /dev/null +++ b/tdactor/td/actor/core/ActorTypeStat.cpp @@ -0,0 +1,124 @@ +#include "td/actor/core/Actor.h" +#include "td/actor/core/ActorTypeStat.h" +#include "td/actor/core/Scheduler.h" +#include "td/utils/port/thread_local.h" +#include +#include +#include +#include +#include +#include + +#ifdef __has_include +#if __has_include() +#include +#define CXXABI_AVAILABLE 1 +#else +#define CXXABI_AVAILABLE 0 +#endif +#else +#define CXXABI_AVAILABLE 0 +#endif + +namespace td { +namespace actor { +namespace core { + +class ActorTypeStatRef; +struct ActorTypeStatsTlsEntry { + struct Entry { + std::unique_ptr stat; + std::optional o_type_index; + }; + std::vector by_id; + std::mutex mutex; + + template + void foreach_entry(F &&f) { + std::lock_guard guard(mutex); + for (auto &entry : by_id) { + f(entry); + } + } + ActorTypeStatRef get_actor_type_stat(td::uint32 id, Actor &actor) { + if (id >= by_id.size()) { + std::lock_guard guard(mutex); + by_id.resize(id + 1); + } + auto &entry = by_id.at(id); + if (!entry.o_type_index) { + std::lock_guard guard(mutex); + entry.o_type_index = std::type_index(typeid(actor)); + entry.stat = std::make_unique(); + } + return ActorTypeStatRef{entry.stat.get()}; + } +}; + +struct ActorTypeStatsRegistry { + std::mutex mutex; + std::vector> entries; + void registry_entry(std::shared_ptr entry) { + std::lock_guard guard(mutex); + entries.push_back(std::move(entry)); + } + template + void foreach_entry(F &&f) { + std::lock_guard guard(mutex); + for (auto &entry : entries) { + f(*entry); + } + } +}; + +ActorTypeStatsRegistry registry; + +struct ActorTypeStatsTlsEntryRef { + ActorTypeStatsTlsEntryRef() { + entry_ = std::make_shared(); + registry.registry_entry(entry_); + } + std::shared_ptr entry_; +}; + +static TD_THREAD_LOCAL ActorTypeStatsTlsEntryRef *actor_type_stats_tls_entry = nullptr; + +ActorTypeStatRef ActorTypeStatManager::get_actor_type_stat(td::uint32 id, Actor *actor) { + if (!actor || !need_debug()) { + return ActorTypeStatRef{nullptr}; + } + td::init_thread_local(actor_type_stats_tls_entry); + ActorTypeStatsTlsEntry &tls_entry = *actor_type_stats_tls_entry->entry_; + return tls_entry.get_actor_type_stat(id, *actor); +} + +std::string ActorTypeStatManager::get_class_name(const char *name) { +#if CXXABI_AVAILABLE + int status; + char *real_name = abi::__cxa_demangle(name, nullptr, nullptr, &status); + if (status < 0) { + return name; + } + + std::string result = real_name; + std::free(real_name); + return result; +#else + return name; +#endif +} + +ActorTypeStats ActorTypeStatManager::get_stats(double inv_ticks_per_second) { + std::map stats; + registry.foreach_entry([&](ActorTypeStatsTlsEntry &tls_entry) { + tls_entry.foreach_entry([&](ActorTypeStatsTlsEntry::Entry &entry) { + if (entry.o_type_index) { + stats[entry.o_type_index.value()] += entry.stat->to_stat(inv_ticks_per_second); + } + }); + }); + return ActorTypeStats{.stats = std::move(stats)}; +} +} // namespace core +} // namespace actor +} // namespace td diff --git a/tdactor/td/actor/core/ActorTypeStat.h b/tdactor/td/actor/core/ActorTypeStat.h new file mode 100644 index 00000000..5b0bd18d --- /dev/null +++ b/tdactor/td/actor/core/ActorTypeStat.h @@ -0,0 +1,395 @@ +#pragma once +#include "td/utils/int_types.h" +#include "td/utils/port/Clocks.h" +#include +#include +#include + +namespace td { +namespace actor { +namespace core { +class Actor; + +struct ActorTypeStat { + // diff (speed) + double created{0}; + double executions{0}; + double messages{0}; + double seconds{0}; + + // current statistics + td::int64 alive{0}; + td::int32 executing{0}; + double executing_start{1e20}; + + // max statistics (TODO: recent_max) + template + struct MaxStatGroup { + T value_forever{}; + T value_10s{}; + T value_10m{}; + MaxStatGroup &operator+=(const MaxStatGroup &other) { + value_forever = std::max(value_forever, other.value_forever); + value_10s = std::max(value_10s, other.value_10s); + value_10m = std::max(value_10m, other.value_10m); + return *this; + } + }; + MaxStatGroup max_execute_messages; + MaxStatGroup max_message_seconds; + MaxStatGroup max_execute_seconds; + MaxStatGroup max_delay_seconds; + + ActorTypeStat &operator+=(const ActorTypeStat &other) { + created += other.created; + executions += other.executions; + messages += other.messages; + seconds += other.seconds; + + alive += other.alive; + executing += other.executing; + executing_start = std::min(other.executing_start, executing_start); + + max_execute_messages += other.max_execute_messages; + max_message_seconds += other.max_message_seconds; + max_execute_seconds += other.max_execute_seconds; + max_delay_seconds += other.max_delay_seconds; + return *this; + } + + ActorTypeStat &operator-=(const ActorTypeStat &other) { + created -= other.created; + executions -= other.executions; + messages -= other.messages; + seconds -= other.seconds; + return *this; + } + ActorTypeStat &operator/=(double t) { + if (t > 1e-2) { + created /= t; + executions /= t; + messages /= t; + seconds /= t; + } else { + created = 0; + executions = 0; + messages = 0; + seconds = 0; + } + return *this; + } +}; + +struct ActorTypeStatImpl { + public: + ActorTypeStatImpl() { + } + + class MessageTimer { + public: + MessageTimer(ActorTypeStatImpl *stat, td::uint64 started_at = Clocks::rdtsc()) + : stat_(stat), started_at_(started_at) { + } + MessageTimer(const MessageTimer &) = delete; + MessageTimer(MessageTimer &&) = delete; + MessageTimer &operator=(const MessageTimer &) = delete; + MessageTimer &operator=(MessageTimer &&) = delete; + ~MessageTimer() { + if (stat_) { + auto ts = td::Clocks::rdtsc(); + stat_->message_finish(ts, ts - started_at_); + } + } + + private: + ActorTypeStatImpl *stat_; + td::uint64 started_at_; + }; + void created() { + inc(total_created_); + inc(alive_); + } + void destroyed() { + dec(alive_); + } + MessageTimer create_run_timer() { + return MessageTimer{this}; + } + + void message_finish(td::uint64 ts, td::uint64 ticks) { + inc(total_messages_); + inc(execute_messages_); + add(total_ticks_, ticks); + max_message_ticks_.update(ts, ticks); + } + void on_delay(td::uint64 ts, td::uint64 ticks) { + max_delay_ticks_.update(ts, ticks); + } + + void execute_start(td::uint64 ts) { + // TODO: this is mostly protection for recursive actor calls, which curretly should be almost impossible + // But too full handle it, one would use one executing_cnt per thread, so only upper level execution is counted + if (inc(executing_) == 1) { + store(execute_start_, ts); + store(execute_messages_, 0); + } + } + void execute_finish(td::uint64 ts) { + CHECK(executing_ > 0); + if (dec(executing_) == 0) { + max_execute_messages_.update(ts, load(execute_messages_)); + max_execute_ticks_.update(ts, ts - load(execute_start_)); + + inc(total_executions_); + store(execute_start_, 0); + store(execute_messages_, 0); + } + } + + template + static td::uint32 get_unique_id() { + static td::uint32 value = get_next_unique_id(); + return value; + } + + static td::uint32 get_next_unique_id() { + static std::atomic next_id_{}; + return ++next_id_; + } + ActorTypeStat to_stat(double inv_ticks_per_second) const { + auto execute_start_copy = load(execute_start_); + auto actual_total_ticks = load(total_ticks_); + auto ts = Clocks::rdtsc(); + if (execute_start_copy != 0) { + actual_total_ticks += ts - execute_start_copy; + } + auto execute_start = ticks_to_seconds(load(execute_start_), inv_ticks_per_second); + return ActorTypeStat{.created = double(load(total_created_)), + .executions = double(load(total_executions_)), + .messages = double(load(total_messages_)), + .seconds = ticks_to_seconds(actual_total_ticks, inv_ticks_per_second), + + .alive = load(alive_), + .executing = load(executing_), + .executing_start = execute_start < 1e-9 ? 1e20 : execute_start, + .max_execute_messages = load(max_execute_messages_), + .max_message_seconds = load_seconds(max_message_ticks_, inv_ticks_per_second), + .max_execute_seconds = load_seconds(max_execute_ticks_, inv_ticks_per_second), + .max_delay_seconds = load_seconds(max_delay_ticks_, inv_ticks_per_second)}; + } + + private: + static double ticks_to_seconds(td::uint64 ticks, double inv_tick_per_second) { + return double(ticks) * inv_tick_per_second; + } + + template + static T load(const std::atomic &a) { + return a.load(std::memory_order_relaxed); + } + template + static void store(std::atomic &a, S value) { + a.store(value, std::memory_order_relaxed); + } + template + static T add(std::atomic &a, S value) { + T new_value = load(a) + value; + store(a, new_value); + return new_value; + } + template + static T inc(std::atomic &a) { + return add(a, 1); + } + + template + static T dec(std::atomic &a) { + return add(a, -1); + } + template + static void relax_max(std::atomic &a, T value) { + auto old_value = load(a); + if (value > old_value) { + store(a, value); + } + } + + template + class MaxCounter { + alignas(64) std::atomic max_values[2] = {0}; + std::atomic last_update_segment_time = 0; + + void update_current_segment(uint64 current_segment_time, uint64 segment_difference) { + if (segment_difference >= 2) { + store(max_values[0], 0); + store(max_values[1], 0); + } else if (segment_difference == 1) { + store(max_values[1 - (current_segment_time & 1)], 0); + } + store(last_update_segment_time, current_segment_time); + } + + public: + inline void update(td::uint64 rdtsc, ValueT value) { + auto current_segment_time = rdtsc / (Clocks::rdtsc_frequency() * Interval); + + auto segment_difference = current_segment_time - last_update_segment_time; + + if (unlikely(segment_difference != 0)) { + update_current_segment(current_segment_time, segment_difference); + } + + relax_max(max_values[current_segment_time & 1], value); + } + + inline ValueT get_max(uint64_t rdtsc) const { + uint64_t current_segment_time = rdtsc / (Clocks::rdtsc_frequency() * Interval); + uint64_t segment_difference = current_segment_time - load(last_update_segment_time); + + if (segment_difference >= 2) { + return 0; + } else if (segment_difference == 1) { + return load(max_values[current_segment_time & 1]); + } else { + return std::max(load(max_values[0]), load(max_values[1])); + } + } + }; + + template + struct MaxCounterGroup { + std::atomic max_forever{}; + MaxCounter max_10m; + MaxCounter max_10s; + + inline void update(td::uint64 rdtsc, T value) { + relax_max(max_forever, value); + max_10m.update(rdtsc, value); + max_10s.update(rdtsc, value); + } + }; + template + static ActorTypeStat::MaxStatGroup load(const MaxCounterGroup &src) { + auto ts = Clocks::rdtsc(); + return {.value_forever = load(src.max_forever), + .value_10s = src.max_10s.get_max(ts), + .value_10m = src.max_10m.get_max(ts)}; + } + template + static ActorTypeStat::MaxStatGroup load_seconds(const MaxCounterGroup &src, double inv_ticks_per_second) { + auto ts = Clocks::rdtsc(); + return {.value_forever = ticks_to_seconds(load(src.max_forever), inv_ticks_per_second), + .value_10s = ticks_to_seconds(src.max_10s.get_max(ts), inv_ticks_per_second), + .value_10m = ticks_to_seconds(src.max_10m.get_max(ts), inv_ticks_per_second)}; + } + + // total (increment only statistics) + std::atomic total_created_{0}; + std::atomic total_executions_{0}; + std::atomic total_messages_{0}; + std::atomic total_ticks_{0}; + + // current statistics + std::atomic alive_{0}; + std::atomic executing_{0}; + + // max statistics (TODO: recent_max) + MaxCounterGroup max_execute_messages_; + MaxCounterGroup max_message_ticks_; + MaxCounterGroup max_execute_ticks_; + MaxCounterGroup max_delay_ticks_; + + // execute state + std::atomic execute_start_{0}; + std::atomic execute_messages_{0}; +}; + +class ActorTypeStatRef { + public: + ActorTypeStatImpl *ref_{nullptr}; + + void created() { + if (!ref_) { + return; + } + ref_->created(); + } + void destroyed() { + if (!ref_) { + return; + } + ref_->destroyed(); + } + void pop_from_queue(td::uint64 in_queue_since) { + if (!ref_) { + return; + } + CHECK(in_queue_since); + auto ts = td::Clocks::rdtsc(); + ref_->on_delay(ts, ts - in_queue_since); + } + void start_execute() { + if (!ref_) { + return; + } + ref_->execute_start(td::Clocks::rdtsc()); + } + void finish_execute() { + if (!ref_) { + return; + } + ref_->execute_finish(td::Clocks::rdtsc()); + } + ActorTypeStatImpl::MessageTimer create_message_timer() { + if (!ref_) { + return ActorTypeStatImpl::MessageTimer{nullptr, 0}; + } + return ActorTypeStatImpl::MessageTimer{ref_}; + } + + struct ExecuteTimer { + ExecuteTimer() = delete; + ExecuteTimer(const ExecuteTimer &) = delete; + ExecuteTimer(ExecuteTimer &&) = delete; + ExecuteTimer &operator=(const ExecuteTimer &) = delete; + ExecuteTimer &operator=(ExecuteTimer &&) = delete; + + ExecuteTimer(ActorTypeStatRef *stat) : stat(stat) { + stat->start_execute(); + } + ActorTypeStatRef *stat{}; + ~ExecuteTimer() { + stat->finish_execute(); + } + }; + ExecuteTimer create_execute_timer() { + return ExecuteTimer(this); + } +}; + +// TODO: currently it is implemented via TD_THREAD_LOCAL, so the statistics is global across different schedulers +struct ActorTypeStats { + std::map stats; + ActorTypeStats &operator-=(const ActorTypeStats &other) { + for (auto &it : other.stats) { + stats.at(it.first) -= it.second; + } + return *this; + } + ActorTypeStats &operator/=(double x) { + for (auto &it : stats) { + it.second /= x; + } + return *this; + } +}; +class ActorTypeStatManager { + public: + static ActorTypeStatRef get_actor_type_stat(td::uint32 id, Actor *actor); + static ActorTypeStats get_stats(double inv_ticks_per_second); + static std::string get_class_name(const char *name); +}; + +} // namespace core +} // namespace actor +} // namespace td \ No newline at end of file diff --git a/tdactor/td/actor/core/Scheduler.h b/tdactor/td/actor/core/Scheduler.h index 3de519e0..e76b919e 100644 --- a/tdactor/td/actor/core/Scheduler.h +++ b/tdactor/td/actor/core/Scheduler.h @@ -130,27 +130,20 @@ struct LocalQueue { public: template bool push(T value, F &&overflow_f) { - auto res = std::move(next_); - next_ = std::move(value); - if (res) { - queue_.local_push(res.unwrap(), overflow_f); - return true; - } - return false; - } - bool try_pop(T &message) { - if (!next_) { - return queue_.local_pop(message); - } - message = next_.unwrap(); + queue_.local_push(std::move(value), overflow_f); return true; } - bool steal(T &message, LocalQueue &other) { + bool try_pop(T &message) { + return queue_.local_pop(message); + } + bool steal(T &message, LocalQueue &other) { return queue_.steal(message, other.queue_); } + size_t size() const { + return queue_.size(); + } private: - td::optional next_; StealingQueue queue_; char pad[TD_CONCURRENCY_PAD - sizeof(optional)]; }; @@ -267,11 +260,12 @@ class Scheduler { bool is_stop_requested() override; void stop() override; - private: - SchedulerGroupInfo *scheduler_group() const { + SchedulerGroupInfo *scheduler_group() const override { return scheduler_group_; } + private: + ActorInfoCreator *creator_; SchedulerId scheduler_id_; CpuWorkerId cpu_worker_id_; diff --git a/tdactor/td/actor/core/SchedulerContext.h b/tdactor/td/actor/core/SchedulerContext.h index e46aef7d..99b1922f 100644 --- a/tdactor/td/actor/core/SchedulerContext.h +++ b/tdactor/td/actor/core/SchedulerContext.h @@ -38,6 +38,7 @@ class SchedulerDispatcher { }; struct Debug; +struct SchedulerGroupInfo; class SchedulerContext : public Context, public SchedulerDispatcher { public: virtual ~SchedulerContext() = default; @@ -59,6 +60,7 @@ class SchedulerContext : public Context, public SchedulerDispa // Debug virtual Debug &get_debug() = 0; + virtual SchedulerGroupInfo *scheduler_group() const = 0; }; } // namespace core } // namespace actor diff --git a/tdactor/test/actors_core.cpp b/tdactor/test/actors_core.cpp index ae10eb9b..1c56d1e5 100644 --- a/tdactor/test/actors_core.cpp +++ b/tdactor/test/actors_core.cpp @@ -19,15 +19,20 @@ #include "td/actor/core/ActorLocker.h" #include "td/actor/actor.h" #include "td/actor/PromiseFuture.h" +#include "td/actor/ActorStats.h" #include "td/utils/format.h" #include "td/utils/logging.h" +#include "td/utils/misc.h" +#include "td/utils/port/thread.h" #include "td/utils/port/thread.h" #include "td/utils/Random.h" #include "td/utils/Slice.h" #include "td/utils/StringBuilder.h" #include "td/utils/tests.h" #include "td/utils/Time.h" +#include "td/utils/TimedStat.h" +#include "td/utils/port/sleep.h" #include #include @@ -720,7 +725,7 @@ TEST(Actor2, actor_function_result) { } TEST(Actor2, actor_ping_pong) { - Scheduler scheduler{{3}, Scheduler::Paused}; + Scheduler scheduler{{3}, false, Scheduler::Paused}; sb.clear(); scheduler.start(); @@ -799,7 +804,7 @@ TEST(Actor2, Schedulers) { for (auto run_count : {0, 1, 2}) { for (auto stop_count : {0, 1, 2}) { for (size_t threads : {0, 1}) { - Scheduler scheduler({threads}, mode); + Scheduler scheduler({threads}, false, mode); for (int i = 0; i < start_count; i++) { scheduler.start(); } @@ -1102,4 +1107,59 @@ TEST(Actor2, send_vs_close2) { scheduler.run(); } } + +TEST(Actor2, test_stats) { + Scheduler scheduler({8}); + td::actor::set_debug(true); + + auto watcher = td::create_shared_destructor([] { SchedulerContext::get()->stop(); }); + scheduler.run_in_context([watcher = std::move(watcher)] { + class SleepWorker : public Actor { + void loop() override { + // 0.8 load + td::usleep_for(800000); + alarm_timestamp() = td::Timestamp::in(0.2); + } + }; + class QueueWorker : public Actor { + void loop() override { + for (int i = 0; i < 20; i++) { + send_closure(actor_id(this), &QueueWorker::ping); + } + alarm_timestamp() = td::Timestamp::in(1.0); + } + void ping() { + } + }; + class Master : public Actor { + public: + Master(std::shared_ptr watcher) : watcher_(std::move(watcher)) { + } + void start_up() override { + alarm_timestamp() = td::Timestamp::in(1); + stats_ = td::actor::create_actor("actor_stats"); + td::actor::create_actor("sleep_worker").release(); + td::actor::create_actor("queue_worker").release(); + } + void alarm() override { + td::actor::send_closure(stats_, &ActorStats::prepare_stats, td::promise_send_closure(actor_id(this), &Master::on_stats)); + alarm_timestamp() = td::Timestamp::in(5); + } + void on_stats(td::Result r_stats) { + LOG(ERROR) << "\n" << r_stats.ok(); + if (--cnt_ == 0) { + stop(); + } + } + + private: + std::shared_ptr watcher_; + td::actor::ActorOwn stats_; + int cnt_={2}; + }; + td::actor::create_actor("Master", watcher).release(); + }); + + scheduler.run(); +} #endif //!TD_THREAD_UNSUPPORTED diff --git a/tdactor/test/actors_promise.cpp b/tdactor/test/actors_promise.cpp index f1d57069..5717b394 100644 --- a/tdactor/test/actors_promise.cpp +++ b/tdactor/test/actors_promise.cpp @@ -210,7 +210,7 @@ TEST(Actor, promise_future) { TEST(Actor2, actor_lost_promise) { using namespace td::actor; using namespace td; - Scheduler scheduler({1}, Scheduler::Paused); + Scheduler scheduler({1}, false, Scheduler::Paused); auto watcher = td::create_shared_destructor([] { LOG(ERROR) << "STOP"; diff --git a/tddb/CMakeLists.txt b/tddb/CMakeLists.txt index 1acd5420..55476e64 100644 --- a/tddb/CMakeLists.txt +++ b/tddb/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) #SOURCE SETS set(TDDB_UTILS_SOURCE @@ -59,7 +59,7 @@ target_link_libraries(tddb PUBLIC tdutils tdactor tddb_utils) if (TDDB_USE_ROCKSDB) target_sources(tddb PRIVATE ${TDDB_ROCKSDB_SOURCE}) target_compile_definitions(tddb PUBLIC -DTDDB_USE_ROCKSDB) - target_link_libraries(tddb PRIVATE rocksdb) + target_link_libraries(tddb PUBLIC rocksdb) target_include_directories(tddb PRIVATE $) endif() diff --git a/tddb/td/db/KeyValue.h b/tddb/td/db/KeyValue.h index 4e0d8538..12c3a4f8 100644 --- a/tddb/td/db/KeyValue.h +++ b/tddb/td/db/KeyValue.h @@ -18,7 +18,9 @@ */ #pragma once #include "td/utils/Status.h" +#include "td/utils/Time.h" #include "td/utils/logging.h" +#include namespace td { class KeyValueReader { public: @@ -27,6 +29,12 @@ class KeyValueReader { virtual Result get(Slice key, std::string &value) = 0; virtual Result count(Slice prefix) = 0; + virtual Status for_each(std::function f) { + return Status::Error("for_each is not supported"); + } + virtual Status for_each_in_range (Slice begin, Slice end, std::function f) { + return td::Status::Error("foreach_range is not supported"); + } }; class PrefixedKeyValueReader : public KeyValueReader { diff --git a/tddb/td/db/MemoryKeyValue.cpp b/tddb/td/db/MemoryKeyValue.cpp index aaf5472d..08013360 100644 --- a/tddb/td/db/MemoryKeyValue.cpp +++ b/tddb/td/db/MemoryKeyValue.cpp @@ -29,6 +29,24 @@ Result MemoryKeyValue::get(Slice key, std::string &va value = it->second; return GetStatus::Ok; } + +Status MemoryKeyValue::for_each(std::function f) { + for (auto &it : map_) { + TRY_STATUS(f(it.first, it.second)); + } + return Status::OK(); +} + +Status MemoryKeyValue::for_each_in_range(Slice begin, Slice end, std::function f) { + for (auto it = map_.lower_bound(begin); it != map_.end(); it++) { + if (it->first < end) { + TRY_STATUS(f(it->first, it->second)); + } else { + break; + } + } + return Status::OK(); +} Status MemoryKeyValue::set(Slice key, Slice value) { map_[key.str()] = value.str(); return Status::OK(); diff --git a/tddb/td/db/MemoryKeyValue.h b/tddb/td/db/MemoryKeyValue.h index c9d584bc..f0b5faa0 100644 --- a/tddb/td/db/MemoryKeyValue.h +++ b/tddb/td/db/MemoryKeyValue.h @@ -25,6 +25,8 @@ namespace td { class MemoryKeyValue : public KeyValue { public: Result get(Slice key, std::string &value) override; + Status for_each(std::function f) override; + Status for_each_in_range(Slice begin, Slice end, std::function f) override; Status set(Slice key, Slice value) override; Status erase(Slice key) override; Result count(Slice prefix) override; diff --git a/tddb/td/db/RocksDb.cpp b/tddb/td/db/RocksDb.cpp index 500985e2..f1aa64a5 100644 --- a/tddb/td/db/RocksDb.cpp +++ b/tddb/td/db/RocksDb.cpp @@ -56,42 +56,67 @@ RocksDb::~RocksDb() { } RocksDb RocksDb::clone() const { - return RocksDb{db_, statistics_}; + return RocksDb{db_, options_}; } -Result RocksDb::open(std::string path) { +Result RocksDb::open(std::string path, RocksDbOptions options) { rocksdb::OptimisticTransactionDB *db; - auto statistics = rocksdb::CreateDBStatistics(); { - rocksdb::Options options; + rocksdb::Options db_options; - static auto cache = rocksdb::NewLRUCache(1 << 30); + static auto default_cache = rocksdb::NewLRUCache(1 << 30); + if (!options.no_block_cache && options.block_cache == nullptr) { + options.block_cache = default_cache; + } rocksdb::BlockBasedTableOptions table_options; - table_options.block_cache = cache; - options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); + if (options.no_block_cache) { + table_options.no_block_cache = true; + } else { + table_options.block_cache = options.block_cache; + } + db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); - options.manual_wal_flush = true; - options.create_if_missing = true; - options.max_background_compactions = 4; - options.max_background_flushes = 2; - options.bytes_per_sync = 1 << 20; - options.writable_file_max_buffer_size = 2 << 14; - options.statistics = statistics; + db_options.use_direct_reads = options.use_direct_reads; + db_options.manual_wal_flush = true; + db_options.create_if_missing = true; + db_options.max_background_compactions = 4; + db_options.max_background_flushes = 2; + db_options.bytes_per_sync = 1 << 20; + db_options.writable_file_max_buffer_size = 2 << 14; + db_options.statistics = options.statistics; + db_options.max_log_file_size = 100 << 20; + db_options.keep_log_file_num = 1; rocksdb::OptimisticTransactionDBOptions occ_options; occ_options.validate_policy = rocksdb::OccValidationPolicy::kValidateSerial; - rocksdb::ColumnFamilyOptions cf_options(options); + rocksdb::ColumnFamilyOptions cf_options(db_options); std::vector column_families; column_families.push_back(rocksdb::ColumnFamilyDescriptor(rocksdb::kDefaultColumnFamilyName, cf_options)); std::vector handles; - TRY_STATUS(from_rocksdb( - rocksdb::OptimisticTransactionDB::Open(options, occ_options, std::move(path), column_families, &handles, &db))); + TRY_STATUS(from_rocksdb(rocksdb::OptimisticTransactionDB::Open(db_options, occ_options, std::move(path), + column_families, &handles, &db))); CHECK(handles.size() == 1); // i can delete the handle since DBImpl is always holding a reference to // default column family delete handles[0]; } - return RocksDb(std::shared_ptr(db), std::move(statistics)); + return RocksDb(std::shared_ptr(db), std::move(options)); +} + +std::shared_ptr RocksDb::create_statistics() { + return rocksdb::CreateDBStatistics(); +} + +std::string RocksDb::statistics_to_string(const std::shared_ptr statistics) { + return statistics->ToString(); +} + +void RocksDb::reset_statistics(const std::shared_ptr statistics) { + statistics->Reset(); +} + +std::shared_ptr RocksDb::create_cache(size_t capacity) { + return rocksdb::NewLRUCache(capacity); } std::unique_ptr RocksDb::snapshot() { @@ -105,7 +130,6 @@ std::string RocksDb::stats() const { db_->GetProperty("rocksdb.stats", &out); //db_->GetProperty("rocksdb.cur-size-all-mem-tables", &out); return out; - return statistics_->ToString(); } Result RocksDb::get(Slice key, std::string &value) { @@ -172,6 +196,54 @@ Result RocksDb::count(Slice prefix) { return res; } +Status RocksDb::for_each(std::function f) { + rocksdb::ReadOptions options; + options.snapshot = snapshot_.get(); + std::unique_ptr iterator; + if (snapshot_ || !transaction_) { + iterator.reset(db_->NewIterator(options)); + } else { + iterator.reset(transaction_->GetIterator(options)); + } + + iterator->SeekToFirst(); + for (; iterator->Valid(); iterator->Next()) { + auto key = from_rocksdb(iterator->key()); + auto value = from_rocksdb(iterator->value()); + TRY_STATUS(f(key, value)); + } + if (!iterator->status().ok()) { + return from_rocksdb(iterator->status()); + } + return Status::OK(); +} + +Status RocksDb::for_each_in_range(Slice begin, Slice end, std::function f) { + rocksdb::ReadOptions options; + options.snapshot = snapshot_.get(); + std::unique_ptr iterator; + if (snapshot_ || !transaction_) { + iterator.reset(db_->NewIterator(options)); + } else { + iterator.reset(transaction_->GetIterator(options)); + } + + auto comparator = rocksdb::BytewiseComparator(); + iterator->Seek(to_rocksdb(begin)); + for (; iterator->Valid(); iterator->Next()) { + auto key = from_rocksdb(iterator->key()); + if (comparator->Compare(to_rocksdb(key), to_rocksdb(end)) >= 0) { + break; + } + auto value = from_rocksdb(iterator->value()); + TRY_STATUS(f(key, value)); + } + if (!iterator->status().ok()) { + return from_rocksdb(iterator->status()); + } + return td::Status::OK(); +} + Status RocksDb::begin_write_batch() { CHECK(!transaction_); write_batch_ = std::make_unique(); @@ -218,17 +290,61 @@ Status RocksDb::flush() { Status RocksDb::begin_snapshot() { snapshot_.reset(db_->GetSnapshot()); + if (options_.snapshot_statistics) { + options_.snapshot_statistics->begin_snapshot(snapshot_.get()); + } return td::Status::OK(); } Status RocksDb::end_snapshot() { if (snapshot_) { + if (options_.snapshot_statistics) { + options_.snapshot_statistics->end_snapshot(snapshot_.get()); + } db_->ReleaseSnapshot(snapshot_.release()); } return td::Status::OK(); } -RocksDb::RocksDb(std::shared_ptr db, std::shared_ptr statistics) - : db_(std::move(db)), statistics_(std::move(statistics)) { +RocksDb::RocksDb(std::shared_ptr db, RocksDbOptions options) + : db_(std::move(db)), options_(options) { } + +void RocksDbSnapshotStatistics::begin_snapshot(const rocksdb::Snapshot *snapshot) { + auto lock = std::unique_lock(mutex_); + auto id = reinterpret_cast(snapshot); + auto ts = td::Timestamp::now().at(); + CHECK(id_to_ts_.emplace(id, ts).second); + CHECK(by_ts_.emplace(ts, id).second); +} + +void RocksDbSnapshotStatistics::end_snapshot(const rocksdb::Snapshot *snapshot) { + auto lock = std::unique_lock(mutex_); + auto id = reinterpret_cast(snapshot); + auto it = id_to_ts_.find(id); + CHECK(it != id_to_ts_.end()); + auto ts = it->second; + CHECK(by_ts_.erase(std::make_pair(ts, id)) == 1u); + CHECK(id_to_ts_.erase(id) == 1u); +} + +td::Timestamp RocksDbSnapshotStatistics::oldest_snapshot_timestamp() const { + auto lock = std::unique_lock(mutex_); + if (by_ts_.empty()) { + return {}; + } + return td::Timestamp::at(by_ts_.begin()->first); +} + +std::string RocksDbSnapshotStatistics::to_string() const { + td::Timestamp oldest_snapshot = oldest_snapshot_timestamp(); + double value; + if (oldest_snapshot) { + value = td::Timestamp::now().at() - oldest_snapshot.at(); + } else { + value = -1; + } + return PSTRING() << "td.rocksdb.snapshot.oldest_snapshot_ago.seconds : " << value << "\n"; +} + } // namespace td diff --git a/tddb/td/db/RocksDb.h b/tddb/td/db/RocksDb.h index b8bfaf9d..499a3328 100644 --- a/tddb/td/db/RocksDb.h +++ b/tddb/td/db/RocksDb.h @@ -23,9 +23,20 @@ #endif #include "td/db/KeyValue.h" +#include "td/utils/Span.h" #include "td/utils/Status.h" +#include "td/utils/optional.h" + +#include "td/utils/Time.h" + +#include +#include +#include + +#include namespace rocksdb { +class Cache; class OptimisticTransactionDB; class Transaction; class WriteBatch; @@ -34,16 +45,38 @@ class Statistics; } // namespace rocksdb namespace td { +struct RocksDbSnapshotStatistics { + void begin_snapshot(const rocksdb::Snapshot *snapshot); + void end_snapshot(const rocksdb::Snapshot *snapshot); + td::Timestamp oldest_snapshot_timestamp() const; + std::string to_string() const; + + private: + mutable std::mutex mutex_; + std::map id_to_ts_; + std::set> by_ts_; +}; + +struct RocksDbOptions { + std::shared_ptr statistics = nullptr; + std::shared_ptr block_cache; // Default - one 1G cache for all RocksDb + std::shared_ptr snapshot_statistics = nullptr; + bool use_direct_reads = false; + bool no_block_cache = false; +}; + class RocksDb : public KeyValue { public: static Status destroy(Slice path); RocksDb clone() const; - static Result open(std::string path); + static Result open(std::string path, RocksDbOptions options = {}); Result get(Slice key, std::string &value) override; Status set(Slice key, Slice value) override; Status erase(Slice key) override; Result count(Slice prefix) override; + Status for_each(std::function f) override; + Status for_each_in_range (Slice begin, Slice end, std::function f) override; Status begin_write_batch() override; Status commit_write_batch() override; @@ -60,6 +93,12 @@ class RocksDb : public KeyValue { std::unique_ptr snapshot() override; std::string stats() const override; + static std::shared_ptr create_statistics(); + static std::string statistics_to_string(const std::shared_ptr statistics); + static void reset_statistics(const std::shared_ptr statistics); + + static std::shared_ptr create_cache(size_t capacity); + RocksDb(RocksDb &&); RocksDb &operator=(RocksDb &&); ~RocksDb(); @@ -70,7 +109,7 @@ class RocksDb : public KeyValue { private: std::shared_ptr db_; - std::shared_ptr statistics_; + RocksDbOptions options_; std::unique_ptr transaction_; std::unique_ptr write_batch_; @@ -83,7 +122,6 @@ class RocksDb : public KeyValue { }; std::unique_ptr snapshot_; - explicit RocksDb(std::shared_ptr db, - std::shared_ptr statistics); + explicit RocksDb(std::shared_ptr db, RocksDbOptions options); }; } // namespace td diff --git a/tddb/td/db/utils/BlobView.cpp b/tddb/td/db/utils/BlobView.cpp index 7011a00e..ebfbc6d5 100644 --- a/tddb/td/db/utils/BlobView.cpp +++ b/tddb/td/db/utils/BlobView.cpp @@ -311,6 +311,7 @@ td::Result FileMemoryMappingBlobView::create(td::CSlice file_path, td: class CyclicBlobViewImpl : public BlobViewImpl { public: CyclicBlobViewImpl(td::BufferSlice data, td::uint64 total_size) : data_(std::move(data)), total_size_(total_size) { + CHECK(!data_.empty()); } td::Result view_impl(td::MutableSlice slice, td::uint64 offset) override { auto res = slice; diff --git a/tddb/td/db/utils/StreamInterface.h b/tddb/td/db/utils/StreamInterface.h index 5262f3df..cf8f1d51 100644 --- a/tddb/td/db/utils/StreamInterface.h +++ b/tddb/td/db/utils/StreamInterface.h @@ -27,7 +27,7 @@ namespace td { // Generic stream interface // Will to hide implementations details. // CyclicBuffer, ChainBuffer, Bounded ChainBuffer, some clever writers. They all should be interchangable -// Most implementaions will assume that reading and writing may happen concurrently +// Most implementations will assume that reading and writing may happen concurrently class StreamReaderInterface { public: diff --git a/tddb/td/db/utils/StreamToFileActor.cpp b/tddb/td/db/utils/StreamToFileActor.cpp index 24202da4..fb733d02 100644 --- a/tddb/td/db/utils/StreamToFileActor.cpp +++ b/tddb/td/db/utils/StreamToFileActor.cpp @@ -73,7 +73,7 @@ Result StreamToFileActor::do_loop() { // Also it could be useful to check error and stop immediately. TRY_RESULT(is_closed, is_closed()); - // Flush all data that is awailable on the at the beginning of loop + // Flush all data that is available on the at the beginning of loop TRY_STATUS(do_flush_once()); if ((sync_at_ && sync_at_.is_in_past()) || is_closed) { @@ -81,7 +81,7 @@ Result StreamToFileActor::do_loop() { sync_at_ = {}; } - bool need_update = sync_state_.set_synced_size(synced_size_) | sync_state_.set_flushed_size(flushed_size_); + bool need_update = int(sync_state_.set_synced_size(synced_size_)) | int(sync_state_.set_flushed_size(flushed_size_)); if (need_update && callback_) { callback_->on_sync_state_changed(); } diff --git a/tddb/test/key_value.cpp b/tddb/test/key_value.cpp index e04e7ee9..921ad059 100644 --- a/tddb/test/key_value.cpp +++ b/tddb/test/key_value.cpp @@ -60,9 +60,20 @@ TEST(KeyValue, simple) { ensure_value(as_slice(x), as_slice(x)); kv.reset(); - kv = std::make_unique(td::RocksDb::open(db_name.str()).move_as_ok()); + td::RocksDbOptions options; + options.snapshot_statistics = std::make_shared(); + kv = std::make_unique(td::RocksDb::open(db_name.str(), options).move_as_ok()); ensure_value("A", "HELLO"); ensure_value(as_slice(x), as_slice(x)); + + CHECK(!options.snapshot_statistics->oldest_snapshot_timestamp()); + auto snapshot = kv->snapshot(); + CHECK(options.snapshot_statistics->oldest_snapshot_timestamp()); + auto snapshot2 = kv->snapshot(); + snapshot.reset(); + CHECK(options.snapshot_statistics->oldest_snapshot_timestamp()); + snapshot2.reset(); + CHECK(!options.snapshot_statistics->oldest_snapshot_timestamp()); }; TEST(KeyValue, async_simple) { diff --git a/tdfec/CMakeLists.txt b/tdfec/CMakeLists.txt index adfe2fdb..828ff90d 100644 --- a/tdfec/CMakeLists.txt +++ b/tdfec/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) set(TDFEC_SOURCE td/fec/raptorq/Rfc.cpp diff --git a/tdfec/benchmark/CMakeLists.txt b/tdfec/benchmark/CMakeLists.txt index 93ec575d..ee8f72cb 100644 --- a/tdfec/benchmark/CMakeLists.txt +++ b/tdfec/benchmark/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) add_executable(benchmark-fec benchmark.cpp ) target_include_directories(benchmark-fec PUBLIC $) diff --git a/tdfec/td/fec/raptorq/Rfc.h b/tdfec/td/fec/raptorq/Rfc.h index 1f5c27f0..e3a33131 100644 --- a/tdfec/td/fec/raptorq/Rfc.h +++ b/tdfec/td/fec/raptorq/Rfc.h @@ -61,7 +61,7 @@ class Rfc { template void encoding_row_for_each(EncodingRow t, F &&f) const { f(t.b); - for (uint16 j = 1; j < t.d; ++j) { + for (uint32 j = 1; j < t.d; ++j) { t.b = (t.b + t.a) % W; f(t.b); } diff --git a/tdfec/td/fec/raptorq/Solver.cpp b/tdfec/td/fec/raptorq/Solver.cpp index 02d8102f..772271fa 100644 --- a/tdfec/td/fec/raptorq/Solver.cpp +++ b/tdfec/td/fec/raptorq/Solver.cpp @@ -19,6 +19,7 @@ #include "td/fec/raptorq/Solver.h" #include "td/fec/algebra/GaussianElimination.h" #include "td/fec/algebra/InactivationDecoding.h" +#include "td/utils/ThreadSafeCounter.h" #include "td/utils/Timer.h" #include @@ -70,6 +71,7 @@ Result Solver::run(const Rfc::Parameters &p, Span symbol auto C = GaussianElimination::run(std::move(A), std::move(D)); return C; } + TD_PERF_COUNTER(raptor_solve); PerfWarningTimer x("solve"); Timer timer; auto perf_log = [&](Slice message) { diff --git a/tdnet/CMakeLists.txt b/tdnet/CMakeLists.txt index d5ae7086..bc00a676 100644 --- a/tdnet/CMakeLists.txt +++ b/tdnet/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) set(TDNET_SOURCE td/net/FdListener.cpp diff --git a/tdnet/td/net/TcpListener.cpp b/tdnet/td/net/TcpListener.cpp index 7b7364ba..e711cbbd 100644 --- a/tdnet/td/net/TcpListener.cpp +++ b/tdnet/td/net/TcpListener.cpp @@ -46,9 +46,11 @@ void TcpListener::start_up() { } void TcpListener::tear_down() { - // unsubscribe from socket updates - // nb: interface will be changed - td::actor::SchedulerContext::get()->get_poll().unsubscribe(server_socket_fd_.get_poll_info().get_pollable_fd_ref()); + if (!server_socket_fd_.empty()) { + // unsubscribe from socket updates + // nb: interface will be changed + td::actor::SchedulerContext::get()->get_poll().unsubscribe(server_socket_fd_.get_poll_info().get_pollable_fd_ref()); + } } void TcpListener::loop() { diff --git a/tdnet/test/net-test.cpp b/tdnet/test/net-test.cpp index bb084a67..d20be504 100644 --- a/tdnet/test/net-test.cpp +++ b/tdnet/test/net-test.cpp @@ -158,9 +158,11 @@ void run_server(int from_port, int to_port, bool is_first, bool use_tcp) { TEST(Net, PingPong) { SET_VERBOSITY_LEVEL(VERBOSITY_NAME(ERROR)); + int port1 = td::Random::fast(10000, 10999); + int port2 = td::Random::fast(11000, 11999); for (auto use_tcp : {false, true}) { - auto a = td::thread([use_tcp] { run_server(8091, 8092, true, use_tcp); }); - auto b = td::thread([use_tcp] { run_server(8092, 8091, false, use_tcp); }); + auto a = td::thread([=] { run_server(port1, port2, true, use_tcp); }); + auto b = td::thread([=] { run_server(port2, port1, false, use_tcp); }); a.join(); b.join(); } diff --git a/tdtl/CMakeLists.txt b/tdtl/CMakeLists.txt index b0f83cd9..482bd0f7 100644 --- a/tdtl/CMakeLists.txt +++ b/tdtl/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) #SOURCE SETS set(TDTL_SOURCE diff --git a/tdtl/td/tl/tl_writer.cpp b/tdtl/td/tl/tl_writer.cpp index 5c672f26..f4f5f27c 100644 --- a/tdtl/td/tl/tl_writer.cpp +++ b/tdtl/td/tl/tl_writer.cpp @@ -28,7 +28,7 @@ namespace tl { std::string TL_writer::int_to_string(int x) { char buf[15]; - std::sprintf(buf, "%d", x); + std::snprintf(buf, sizeof(buf), "%d", x); return buf; } diff --git a/tdutils/CMakeLists.txt b/tdutils/CMakeLists.txt index fbcc74fd..f8c191a1 100644 --- a/tdutils/CMakeLists.txt +++ b/tdutils/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) option(TDUTILS_MIME_TYPE "Generate mime types conversion (gperf is required)" ON) @@ -14,8 +14,9 @@ if (NOT DEFINED CMAKE_INSTALL_LIBDIR) set(CMAKE_INSTALL_LIBDIR "lib") endif() +find_package(PkgConfig REQUIRED) if (NOT ZLIB_FOUND) - find_package(ZLIB) + pkg_check_modules(ZLIB zlib) endif() if (ZLIB_FOUND) set(TD_HAVE_ZLIB 1) @@ -279,6 +280,19 @@ if (TDUTILS_MIME_TYPE) ) endif() +if (NOT LZ4_FOUND) + pkg_check_modules(LZ4 REQUIRED liblz4) +endif() + +if (LZ4_FOUND) + set(TD_HAVE_LZ4 1) + set(TDUTILS_SOURCE + ${TDUTILS_SOURCE} + td/utils/lz4.cpp + td/utils/lz4.h + ) +endif() + set(TDUTILS_TEST_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/test/buffer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test/ConcurrentHashMap.cpp @@ -313,7 +327,7 @@ if (WIN32) # find_library(WS2_32_LIBRARY ws2_32) # find_library(MSWSOCK_LIBRARY Mswsock) # target_link_libraries(tdutils PRIVATE ${WS2_32_LIBRARY} ${MSWSOCK_LIBRARY}) - target_link_libraries(tdutils PRIVATE ws2_32 Mswsock Normaliz psapi) + target_link_libraries(tdutils PRIVATE ws2_32 Mswsock Normaliz psapi DbgHelp) endif() if (NOT CMAKE_CROSSCOMPILING AND TDUTILS_MIME_TYPE) add_dependencies(tdutils tdmime_auto) @@ -337,6 +351,14 @@ endif() if (CRC32C_FOUND) target_link_libraries(tdutils PRIVATE crc32c) endif() + +if (LZ4_FOUND) + message(STATUS "Found LZ4 ${LZ4_LIBRARIES} ${LZ4_INCLUDE_DIRS}") + target_link_libraries(tdutils PRIVATE ${LZ4_LIBRARIES}) + target_include_directories(tdutils SYSTEM PRIVATE ${LZ4_INCLUDE_DIRS}) + target_link_directories(tdutils PUBLIC ${LZ4_LIBRARY_DIRS}) +endif() + if (ABSL_FOUND) target_link_libraries_system(tdutils absl::flat_hash_map absl::flat_hash_set absl::hash) endif() diff --git a/tdutils/generate/CMakeLists.txt b/tdutils/generate/CMakeLists.txt index 07353e51..194fda39 100644 --- a/tdutils/generate/CMakeLists.txt +++ b/tdutils/generate/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) # Generates files for MIME type <-> extension conversions # DEPENDS ON: gperf grep bash/powershell diff --git a/tdutils/td/utils/BigNum.cpp b/tdutils/td/utils/BigNum.cpp index 36dde064..9de11fca 100644 --- a/tdutils/td/utils/BigNum.cpp +++ b/tdutils/td/utils/BigNum.cpp @@ -159,7 +159,11 @@ bool BigNum::is_bit_set(int num) const { } bool BigNum::is_prime(BigNumContext &context) const { +#if OPENSSL_VERSION_MAJOR >= 3 + int result = BN_check_prime(impl_->big_num, context.impl_->big_num_context, nullptr); +#else int result = BN_is_prime_ex(impl_->big_num, BN_prime_checks, context.impl_->big_num_context, nullptr); +#endif LOG_IF(FATAL, result == -1); return result == 1; } diff --git a/tdutils/td/utils/BufferedUdp.h b/tdutils/td/utils/BufferedUdp.h index bf4aa1b8..3fa93e9d 100644 --- a/tdutils/td/utils/BufferedUdp.h +++ b/tdutils/td/utils/BufferedUdp.h @@ -106,6 +106,7 @@ class UdpReader { } if (status.is_error() && !UdpSocketFd::is_critical_read_error(status)) { queue.push(UdpMessage{{}, {}, std::move(status)}); + return td::Status::OK(); } return status; } diff --git a/tdutils/td/utils/CancellationToken.h b/tdutils/td/utils/CancellationToken.h index 9f30d204..7ef30497 100644 --- a/tdutils/td/utils/CancellationToken.h +++ b/tdutils/td/utils/CancellationToken.h @@ -20,6 +20,7 @@ #include #include +#include "Status.h" namespace td { @@ -38,6 +39,12 @@ class CancellationToken { } return token_->is_cancelled_.load(std::memory_order_acquire); } + Status check() const { + if (*this) { + return Status::Error(653, "cancelled"); // cancelled = 653 + } + return Status::OK(); + } CancellationToken() = default; explicit CancellationToken(std::shared_ptr token) : token_(std::move(token)) { } diff --git a/tdutils/td/utils/HashMap.h b/tdutils/td/utils/HashMap.h index 9646bef9..1380f029 100644 --- a/tdutils/td/utils/HashMap.h +++ b/tdutils/td/utils/HashMap.h @@ -22,6 +22,7 @@ #if TD_HAVE_ABSL #include +#include #else #include #endif @@ -31,9 +32,13 @@ namespace td { #if TD_HAVE_ABSL template > using HashMap = absl::flat_hash_map; +template , class E = std::equal_to<>> +using NodeHashMap = absl::node_hash_map; #else template > using HashMap = std::unordered_map; +template > +using NodeHashMap = std::unordered_map; #endif } // namespace td diff --git a/tdutils/td/utils/HashSet.h b/tdutils/td/utils/HashSet.h index b00d5a92..85254b32 100644 --- a/tdutils/td/utils/HashSet.h +++ b/tdutils/td/utils/HashSet.h @@ -22,6 +22,7 @@ #if TD_HAVE_ABSL #include +#include #else #include #endif @@ -29,11 +30,15 @@ namespace td { #if TD_HAVE_ABSL -template > -using HashSet = absl::flat_hash_set; +template , class E = std::equal_to<>> +using HashSet = absl::flat_hash_set; +template , class E = std::equal_to<>> +using NodeHashSet = absl::node_hash_set; #else -template > -using HashSet = std::unordered_set; +template , class E = std::equal_to<>> +using HashSet = std::unordered_set; +template , class E = std::equal_to<>> +using NodeHashSet = HashSet; #endif } // namespace td diff --git a/tdutils/td/utils/HazardPointers.h b/tdutils/td/utils/HazardPointers.h index 954d6efd..7465140a 100644 --- a/tdutils/td/utils/HazardPointers.h +++ b/tdutils/td/utils/HazardPointers.h @@ -32,12 +32,7 @@ class HazardPointers { explicit HazardPointers(size_t threads_n) : threads_(threads_n) { for (auto &data : threads_) { for (auto &ptr : data.hazard_) { -// workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=64658 -#if TD_GCC && GCC_VERSION <= 40902 ptr = nullptr; -#else - std::atomic_init(&ptr, static_cast(nullptr)); -#endif } } } diff --git a/tdutils/td/utils/OptionParser.cpp b/tdutils/td/utils/OptionParser.cpp index b9584856..634570e1 100644 --- a/tdutils/td/utils/OptionParser.cpp +++ b/tdutils/td/utils/OptionParser.cpp @@ -33,7 +33,7 @@ void OptionParser::set_description(string description) { void OptionParser::add_option(Option::Type type, char short_key, Slice long_key, Slice description, std::function callback) { for (auto &option : options_) { - if (option.short_key == short_key || (!long_key.empty() && long_key == option.long_key)) { + if ((short_key != '\0' && option.short_key == short_key) || (!long_key.empty() && long_key == option.long_key)) { LOG(ERROR) << "Ignore duplicated option '" << short_key << "' '" << long_key << "'"; } } diff --git a/tdutils/td/utils/SpinLock.h b/tdutils/td/utils/SpinLock.h index b5bb62db..f0856f0c 100644 --- a/tdutils/td/utils/SpinLock.h +++ b/tdutils/td/utils/SpinLock.h @@ -63,7 +63,10 @@ class SpinLock { } private: +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-pragma" std::atomic_flag flag_ = ATOMIC_FLAG_INIT; +#pragma clang diagnostic pop void unlock() { flag_.clear(std::memory_order_release); } diff --git a/tdutils/td/utils/Status.h b/tdutils/td/utils/Status.h index 9c4f1b7d..cff80814 100644 --- a/tdutils/td/utils/Status.h +++ b/tdutils/td/utils/Status.h @@ -80,7 +80,7 @@ TRY_RESULT_PREFIX_IMPL(TD_CONCAT(TD_CONCAT(r_, name), __LINE__), auto name, result, prefix) #define TRY_RESULT_PREFIX_ASSIGN(name, result, prefix) \ - TRY_RESULT_PREFIX_IMPL(TD_CONCAT(TD_CONCAT(r_, name), __LINE__), name, result, prefix) + TRY_RESULT_PREFIX_IMPL(TD_CONCAT(r_response, __LINE__), name, result, prefix) #define TRY_RESULT_PROMISE_PREFIX(promise_name, name, result, prefix) \ TRY_RESULT_PROMISE_PREFIX_IMPL(promise_name, TD_CONCAT(TD_CONCAT(r_, name), __LINE__), auto name, result, prefix) @@ -555,6 +555,12 @@ class Result { }; return status_.move_as_error_suffix(suffix); } + Status move_as_status() TD_WARN_UNUSED_RESULT { + if (status_.is_error()) { + return move_as_error(); + } + return Status::OK(); + } const T &ok() const { LOG_CHECK(status_.is_ok()) << status_; return value_; diff --git a/tdutils/td/utils/StealingQueue.h b/tdutils/td/utils/StealingQueue.h index 20a39dc0..86c2f0ef 100644 --- a/tdutils/td/utils/StealingQueue.h +++ b/tdutils/td/utils/StealingQueue.h @@ -115,6 +115,22 @@ class StealingQueue { std::atomic_thread_fence(std::memory_order_seq_cst); } + size_t size() const { + while (true) { + auto head = head_.load(); + auto tail = tail_.load(std::memory_order_acquire); + + if (tail < head) { + continue; + } + size_t n = tail - head; + if (n > N) { + continue; + } + return n; + } + } + private: std::atomic head_{0}; std::atomic tail_{0}; diff --git a/tdutils/td/utils/StringBuilder.h b/tdutils/td/utils/StringBuilder.h index 99e9d517..685416fe 100644 --- a/tdutils/td/utils/StringBuilder.h +++ b/tdutils/td/utils/StringBuilder.h @@ -149,4 +149,19 @@ std::enable_if_t::value, string> to_string(const T &x) { return sb.as_cslice().str(); } +template +struct LambdaPrintHelper { + SB& sb; +}; +template +SB& operator<<(const LambdaPrintHelper& helper, F&& f) { + f(helper.sb); + return helper.sb; +} +struct LambdaPrint {}; + +inline LambdaPrintHelper operator<<(td::StringBuilder& sb, const LambdaPrint&) { + return LambdaPrintHelper{sb}; +} + } // namespace td diff --git a/tdutils/td/utils/ThreadSafeCounter.h b/tdutils/td/utils/ThreadSafeCounter.h index 55bf94b5..aa976b2f 100644 --- a/tdutils/td/utils/ThreadSafeCounter.h +++ b/tdutils/td/utils/ThreadSafeCounter.h @@ -137,4 +137,55 @@ class NamedThreadSafeCounter { Counter counter_; }; +// another class for simplicity, it +struct NamedPerfCounter { + public: + static NamedPerfCounter &get_default() { + static NamedPerfCounter res; + return res; + } + struct PerfCounterRef { + NamedThreadSafeCounter::CounterRef count; + NamedThreadSafeCounter::CounterRef duration; + }; + PerfCounterRef get_counter(Slice name) { + return {.count = counter_.get_counter(PSLICE() << name << ".count"), + .duration = counter_.get_counter(PSLICE() << name << ".duration")}; + } + + struct ScopedPerfCounterRef : public NoCopyOrMove { + PerfCounterRef perf_counter; + uint64 started_at_ticks{td::Clocks::rdtsc()}; + + ~ScopedPerfCounterRef() { + perf_counter.count.add(1); + perf_counter.duration.add(td::Clocks::rdtsc() - started_at_ticks); + } + }; + + template + void for_each(F &&f) const { + counter_.for_each(f); + } + + void clear() { + counter_.clear(); + } + + friend StringBuilder &operator<<(StringBuilder &sb, const NamedPerfCounter &counter) { + return sb << counter.counter_; + } + private: + NamedThreadSafeCounter counter_; +}; + } // namespace td + +#define TD_PERF_COUNTER(name) \ + static auto perf_##name = td::NamedPerfCounter::get_default().get_counter(td::Slice(#name)); \ + auto scoped_perf_##name = td::NamedPerfCounter::ScopedPerfCounterRef{.perf_counter = perf_##name}; + +#define TD_PERF_COUNTER_SINCE(name, since) \ + static auto perf_##name = td::NamedPerfCounter::get_default().get_counter(td::Slice(#name)); \ + auto scoped_perf_##name = \ + td::NamedPerfCounter::ScopedPerfCounterRef{.perf_counter = perf_##name, .started_at_ticks = since}; diff --git a/tdutils/td/utils/Time.h b/tdutils/td/utils/Time.h index 5151b818..c7795ae4 100644 --- a/tdutils/td/utils/Time.h +++ b/tdutils/td/utils/Time.h @@ -110,6 +110,7 @@ class Timestamp { } friend bool operator==(Timestamp a, Timestamp b); + friend Timestamp &operator+=(Timestamp &a, double b); private: double at_{0}; @@ -122,6 +123,15 @@ inline bool operator<(const Timestamp &a, const Timestamp &b) { return a.at() < b.at(); } +inline Timestamp &operator+=(Timestamp &a, double b) { + a.at_ += b; + return a; +} + +inline double operator-(const Timestamp &a, const Timestamp &b) { + return a.at() - b.at(); +} + template void store(const Timestamp ×tamp, StorerT &storer) { storer.store_binary(timestamp.at() - Time::now() + Clocks::system()); diff --git a/tdutils/td/utils/Timer.cpp b/tdutils/td/utils/Timer.cpp index f33a30dc..24de099a 100644 --- a/tdutils/td/utils/Timer.cpp +++ b/tdutils/td/utils/Timer.cpp @@ -78,10 +78,60 @@ void PerfWarningTimer::reset() { return; } double duration = Time::now() - start_at_; - //LOG_IF(WARNING, duration > max_duration_) - //<< "SLOW: " << tag("name", name_) << tag("duration", format::as_time(duration)); - callback_(duration); + if (callback_) { + callback_(duration); + } else { + LOG_IF(WARNING, duration > max_duration_) + << "SLOW: " << tag("name", name_) << tag("duration", format::as_time(duration)); + } start_at_ = 0; } +double PerfWarningTimer::elapsed() const { + return Time::now() - start_at_; +} + +static double thread_cpu_clock() { +#if defined(CLOCK_THREAD_CPUTIME_ID) + timespec ts; + int result = clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts); + CHECK(result == 0); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1e-9; +#else + return 0.0; // TODO: MacOS and Windows support (currently cpu timer is used only in validators) +#endif +} + +ThreadCpuTimer::ThreadCpuTimer(bool is_paused) : is_paused_(is_paused) { + if (is_paused_) { + start_time_ = 0; + } else { + start_time_ = thread_cpu_clock(); + } +} + +void ThreadCpuTimer::pause() { + if (is_paused_) { + return; + } + elapsed_ += thread_cpu_clock() - start_time_; + is_paused_ = true; +} + +void ThreadCpuTimer::resume() { + if (!is_paused_) { + return; + } + start_time_ = thread_cpu_clock(); + is_paused_ = false; +} + +double ThreadCpuTimer::elapsed() const { + double res = elapsed_; + if (!is_paused_) { + res += thread_cpu_clock() - start_time_; + } + return res; +} + } // namespace td diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index 64f3e934..a27cac8a 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -46,13 +46,14 @@ class Timer { class PerfWarningTimer { public: - explicit PerfWarningTimer(string name, double max_duration = 0.1, std::function&& callback = [] (double) {}); + explicit PerfWarningTimer(string name, double max_duration = 0.1, std::function&& callback = {}); PerfWarningTimer(const PerfWarningTimer &) = delete; PerfWarningTimer &operator=(const PerfWarningTimer &) = delete; PerfWarningTimer(PerfWarningTimer &&other); PerfWarningTimer &operator=(PerfWarningTimer &&) = delete; ~PerfWarningTimer(); void reset(); + double elapsed() const; private: string name_; @@ -61,4 +62,22 @@ class PerfWarningTimer { std::function callback_; }; +class ThreadCpuTimer { + public: + ThreadCpuTimer() : ThreadCpuTimer(false) { + } + explicit ThreadCpuTimer(bool is_paused); + ThreadCpuTimer(const ThreadCpuTimer &other) = default; + ThreadCpuTimer &operator=(const ThreadCpuTimer &other) = default; + + double elapsed() const; + void pause(); + void resume(); + + private: + double elapsed_{0}; + double start_time_; + bool is_paused_{false}; +}; + } // namespace td diff --git a/tdutils/td/utils/as.h b/tdutils/td/utils/as.h index c60c74e2..6015af29 100644 --- a/tdutils/td/utils/as.h +++ b/tdutils/td/utils/as.h @@ -76,12 +76,7 @@ class ConstAs { } // namespace detail -// no std::is_trivially_copyable in libstdc++ before 5.0 -#if __GLIBCXX__ -#define TD_IS_TRIVIALLY_COPYABLE(T) __has_trivial_copy(T) -#else #define TD_IS_TRIVIALLY_COPYABLE(T) std::is_trivially_copyable::value -#endif template = 0> diff --git a/tdutils/td/utils/common.h b/tdutils/td/utils/common.h index 79d3dc52..f9f86c5c 100644 --- a/tdutils/td/utils/common.h +++ b/tdutils/td/utils/common.h @@ -127,4 +127,12 @@ struct Auto { } }; +struct NoCopyOrMove { + NoCopyOrMove() = default; + NoCopyOrMove(NoCopyOrMove &&) = delete; + NoCopyOrMove(const NoCopyOrMove &) = delete; + NoCopyOrMove &operator=(NoCopyOrMove &&) = delete; + NoCopyOrMove &operator=(const NoCopyOrMove &) = delete; +}; + } // namespace td diff --git a/tdutils/td/utils/config.h.in b/tdutils/td/utils/config.h.in index f8b89aeb..3f4e1bf2 100644 --- a/tdutils/td/utils/config.h.in +++ b/tdutils/td/utils/config.h.in @@ -3,6 +3,7 @@ #cmakedefine01 TD_HAVE_OPENSSL #cmakedefine01 TD_HAVE_ZLIB #cmakedefine01 TD_HAVE_CRC32C +#cmakedefine01 TD_HAVE_LZ4 #cmakedefine01 TD_HAVE_COROUTINES #cmakedefine01 TD_HAVE_ABSL #cmakedefine01 TD_FD_DEBUG diff --git a/tdutils/td/utils/crypto.cpp b/tdutils/td/utils/crypto.cpp index 27313cf3..ea1efbe7 100644 --- a/tdutils/td/utils/crypto.cpp +++ b/tdutils/td/utils/crypto.cpp @@ -25,7 +25,6 @@ #include "td/utils/logging.h" #include "td/utils/misc.h" #include "td/utils/port/RwMutex.h" -#include "td/utils/port/thread_local.h" #include "td/utils/Random.h" #include "td/utils/ScopeGuard.h" #include "td/utils/SharedSlice.h" @@ -598,16 +597,23 @@ void aes_ige_decrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlic static void aes_cbc_xcrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to, bool encrypt_flag) { CHECK(aes_key.size() == 32); CHECK(aes_iv.size() == 16); - AES_KEY key; - int err; - if (encrypt_flag) { - err = AES_set_encrypt_key(aes_key.ubegin(), 256, &key); - } else { - err = AES_set_decrypt_key(aes_key.ubegin(), 256, &key); - } - LOG_IF(FATAL, err != 0); CHECK(from.size() <= to.size()); - AES_cbc_encrypt(from.ubegin(), to.ubegin(), from.size(), &key, aes_iv.ubegin(), encrypt_flag); + CHECK(from.size() % 16 == 0); + int out_len = 0; + EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); + CHECK(ctx); + if (encrypt_flag) { + CHECK(EVP_EncryptInit_ex(ctx, EVP_aes_256_cbc(), nullptr, aes_key.ubegin(), aes_iv.ubegin()) == 1); + CHECK(EVP_CIPHER_CTX_set_padding(ctx, 0) == 1); + CHECK(EVP_EncryptUpdate(ctx, to.ubegin(), &out_len, from.ubegin(), td::narrow_cast(from.size())) == 1); + CHECK(EVP_EncryptFinal_ex(ctx, to.ubegin() + out_len, &out_len) == 1); + } else { + CHECK(EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), nullptr, aes_key.ubegin(), aes_iv.ubegin()) == 1); + CHECK(EVP_CIPHER_CTX_set_padding(ctx, 0) == 1); + CHECK(EVP_DecryptUpdate(ctx, to.ubegin(), &out_len, from.ubegin(), td::narrow_cast(from.size())) == 1); + CHECK(EVP_DecryptFinal_ex(ctx, to.ubegin() + out_len, &out_len) == 1); + } + EVP_CIPHER_CTX_free(ctx); } void aes_cbc_encrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to) { @@ -723,7 +729,18 @@ string sha512(Slice data) { class Sha256State::Impl { public: - SHA256_CTX ctx_; + EVP_MD_CTX *ctx_ = nullptr; + + Impl() { + ctx_ = EVP_MD_CTX_new(); + CHECK(ctx_); + } + + ~Impl() { + if (ctx_) { + EVP_MD_CTX_free(ctx_); + } + } }; Sha256State::Sha256State() = default; @@ -755,24 +772,23 @@ void Sha256State::init() { impl_ = make_unique(); } CHECK(!is_inited_); - int err = SHA256_Init(&impl_->ctx_); - LOG_IF(FATAL, err != 1); + CHECK(EVP_DigestInit_ex(impl_->ctx_, EVP_sha256(), nullptr) == 1); is_inited_ = true; } void Sha256State::feed(Slice data) { CHECK(impl_); CHECK(is_inited_); - int err = SHA256_Update(&impl_->ctx_, data.ubegin(), data.size()); - LOG_IF(FATAL, err != 1); + CHECK(EVP_DigestUpdate(impl_->ctx_, data.ubegin(), data.size()) == 1); } void Sha256State::extract(MutableSlice output, bool destroy) { CHECK(output.size() >= 32); CHECK(impl_); CHECK(is_inited_); - int err = SHA256_Final(output.ubegin(), &impl_->ctx_); - LOG_IF(FATAL, err != 1); + unsigned size; + CHECK(EVP_DigestFinal_ex(impl_->ctx_, output.ubegin(), &size) == 1); + CHECK(size == 32); is_inited_ = false; if (destroy) { impl_.reset(); diff --git a/tdutils/td/utils/crypto.h b/tdutils/td/utils/crypto.h index 592a8a00..4494ef48 100644 --- a/tdutils/td/utils/crypto.h +++ b/tdutils/td/utils/crypto.h @@ -151,7 +151,7 @@ class Sha256State { bool is_inited_ = false; }; -void md5(Slice input, MutableSlice output); +[[deprecated("MD5 is not cryptographically secure")]] void md5(Slice input, MutableSlice output); void pbkdf2_sha256(Slice password, Slice salt, int iteration_count, MutableSlice dest); void pbkdf2_sha512(Slice password, Slice salt, int iteration_count, MutableSlice dest); diff --git a/tdutils/td/utils/filesystem.cpp b/tdutils/td/utils/filesystem.cpp index 18fb5717..b84b6b3f 100644 --- a/tdutils/td/utils/filesystem.cpp +++ b/tdutils/td/utils/filesystem.cpp @@ -60,7 +60,7 @@ Result read_file_impl(CSlice path, int64 size, int64 offset) { if (size == -1) { size = file_size - offset; } else if (size >= 0) { - if (size + offset > file_size) { + if (size > file_size - offset) { size = file_size - offset; } } @@ -68,9 +68,14 @@ Result read_file_impl(CSlice path, int64 size, int64 offset) { return Status::Error("Failed to read file: invalid size"); } auto content = create_empty(narrow_cast(size)); - TRY_RESULT(got_size, from_file.pread(as_mutable_slice(content), offset)); - if (got_size != static_cast(size)) { - return Status::Error("Failed to read file"); + MutableSlice slice = as_mutable_slice(content); + while (!slice.empty()) { + TRY_RESULT(got_size, from_file.pread(slice, offset)); + if (got_size == 0) { + return Status::Error("Failed to read file"); + } + offset += got_size; + slice.remove_prefix(got_size); } from_file.close(); return std::move(content); @@ -103,9 +108,15 @@ Status write_file(CSlice to, Slice data, WriteFileOptions options) { TRY_STATUS(to_file.lock(FileFd::LockFlags::Write, to.str(), 10)); TRY_STATUS(to_file.truncate_to_current_position(0)); } - TRY_RESULT(written, to_file.write(data)); - if (written != size) { - return Status::Error(PSLICE() << "Failed to write file: written " << written << " bytes instead of " << size); + size_t total_written = 0; + while (!data.empty()) { + TRY_RESULT(written, to_file.write(data)); + if (written == 0) { + return Status::Error(PSLICE() << "Failed to write file: written " << total_written << " bytes instead of " + << size); + } + total_written += written; + data.remove_prefix(written); } if (options.need_sync) { TRY_STATUS(to_file.sync()); diff --git a/tdutils/td/utils/logging.cpp b/tdutils/td/utils/logging.cpp index 477823b8..03d32ee2 100644 --- a/tdutils/td/utils/logging.cpp +++ b/tdutils/td/utils/logging.cpp @@ -18,6 +18,7 @@ */ #include "td/utils/logging.h" +#include "ThreadSafeCounter.h" #include "td/utils/port/Clocks.h" #include "td/utils/port/StdStreams.h" #include "td/utils/port/thread_local.h" @@ -127,6 +128,9 @@ Logger::~Logger() { slice = MutableCSlice(slice.begin(), slice.begin() + slice.size() - 1); } log_.append(slice, log_level_); + + // put stats here to avoid conflict with PSTRING and PSLICE + TD_PERF_COUNTER_SINCE(logger, start_at_); } else { log_.append(as_cslice(), log_level_); } @@ -172,7 +176,10 @@ void TsCerr::enterCritical() { void TsCerr::exitCritical() { lock_.clear(std::memory_order_release); } +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-pragma" TsCerr::Lock TsCerr::lock_ = ATOMIC_FLAG_INIT; +#pragma clang diagnostic pop class DefaultLog : public LogInterface { public: @@ -301,5 +308,4 @@ ScopedDisableLog::~ScopedDisableLog() { set_verbosity_level(sdl_verbosity); } } - } // namespace td diff --git a/tdutils/td/utils/logging.h b/tdutils/td/utils/logging.h index e5278b4b..bb28f6df 100644 --- a/tdutils/td/utils/logging.h +++ b/tdutils/td/utils/logging.h @@ -40,6 +40,7 @@ #include "td/utils/Slice.h" #include "td/utils/StackAllocator.h" #include "td/utils/StringBuilder.h" +#include "td/utils/port/Clocks.h" #include #include @@ -73,6 +74,7 @@ #define LOG(level) LOG_IMPL(level, level, true, ::td::Slice()) #define LOG_IF(level, condition) LOG_IMPL(level, level, condition, #condition) +#define FLOG(level) LOG_IMPL(level, level, true, ::td::Slice()) << td::LambdaPrint{} << [&](auto &sb) #define VLOG(level) LOG_IMPL(DEBUG, level, true, TD_DEFINE_STR(level)) #define VLOG_IF(level, condition) LOG_IMPL(DEBUG, level, condition, TD_DEFINE_STR(level) " " #condition) @@ -94,13 +96,13 @@ inline bool no_return_func() { #define DUMMY_LOG_CHECK(condition) LOG_IF(NEVER, !(condition)) #ifdef TD_DEBUG - #if TD_MSVC +#if TD_MSVC #define LOG_CHECK(condition) \ __analysis_assume(!!(condition)); \ LOG_IMPL(FATAL, FATAL, !(condition), #condition) - #else +#else #define LOG_CHECK(condition) LOG_IMPL(FATAL, FATAL, !(condition) && no_return_func(), #condition) - #endif +#endif #else #define LOG_CHECK DUMMY_LOG_CHECK #endif @@ -251,7 +253,8 @@ class Logger { , log_(log) , sb_(buffer_.as_slice()) , options_(options) - , log_level_(log_level) { + , log_level_(log_level) + , start_at_(Clocks::rdtsc()) { } Logger(LogInterface &log, const LogOptions &options, int log_level, Slice file_name, int line_num, Slice comment); @@ -261,6 +264,9 @@ class Logger { sb_ << other; return *this; } + LambdaPrintHelper operator<<(const LambdaPrint &) { + return LambdaPrintHelper{sb_}; + } MutableCSlice as_cslice() { return sb_.as_cslice(); @@ -283,6 +289,7 @@ class Logger { StringBuilder sb_; const LogOptions &options_; int log_level_; + td::uint64 start_at_; }; namespace detail { @@ -336,7 +343,10 @@ class TsLog : public LogInterface { private: LogInterface *log_ = nullptr; +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-pragma" std::atomic_flag lock_ = ATOMIC_FLAG_INIT; +#pragma clang diagnostic pop void enter_critical() { while (lock_.test_and_set(std::memory_order_acquire)) { // spin @@ -346,5 +356,4 @@ class TsLog : public LogInterface { lock_.clear(std::memory_order_release); } }; - } // namespace td diff --git a/tdutils/td/utils/lz4.cpp b/tdutils/td/utils/lz4.cpp new file mode 100644 index 00000000..ebf456aa --- /dev/null +++ b/tdutils/td/utils/lz4.cpp @@ -0,0 +1,48 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "td/utils/buffer.h" +#include "td/utils/misc.h" +#include + +namespace td { + +td::BufferSlice lz4_compress(td::Slice data) { + int size = narrow_cast(data.size()); + int buf_size = LZ4_compressBound(size); + td::BufferSlice compressed(buf_size); + int compressed_size = LZ4_compress_default(data.data(), compressed.data(), size, buf_size); + CHECK(compressed_size > 0); + return td::BufferSlice{compressed.as_slice().substr(0, compressed_size)}; +} + +td::Result lz4_decompress(td::Slice data, int max_decompressed_size) { + TRY_RESULT(size, narrow_cast_safe(data.size())); + if (max_decompressed_size < 0) { + return td::Status::Error("invalid max_decompressed_size"); + } + td::BufferSlice decompressed(max_decompressed_size); + int result = LZ4_decompress_safe(data.data(), decompressed.data(), size, max_decompressed_size); + if (result < 0) { + return td::Status::Error(PSTRING() << "lz4 decompression failed, error code: " << result); + } + if (result == max_decompressed_size) { + return decompressed; + } + return td::BufferSlice{decompressed.as_slice().substr(0, result)}; +} + +} // namespace td diff --git a/tdutils/td/utils/lz4.h b/tdutils/td/utils/lz4.h new file mode 100644 index 00000000..fbbc470f --- /dev/null +++ b/tdutils/td/utils/lz4.h @@ -0,0 +1,27 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "td/utils/buffer.h" +#include "td/utils/Status.h" + +namespace td { + +td::BufferSlice lz4_compress(td::Slice data); +td::Result lz4_decompress(td::Slice data, int max_decompressed_size); + +} // namespace td diff --git a/tdutils/td/utils/misc.cpp b/tdutils/td/utils/misc.cpp index d592593a..caff44e3 100644 --- a/tdutils/td/utils/misc.cpp +++ b/tdutils/td/utils/misc.cpp @@ -153,9 +153,9 @@ string buffer_to_hex(Slice buffer) { const char *hex = "0123456789ABCDEF"; string res(2 * buffer.size(), '\0'); for (std::size_t i = 0; i < buffer.size(); i++) { - auto c = buffer.ubegin()[i]; - res[2 * i] = hex[c & 15]; - res[2 * i + 1] = hex[c >> 4]; + unsigned char c = buffer[i]; + res[2 * i] = hex[c >> 4]; + res[2 * i + 1] = hex[c & 15]; } return res; } diff --git a/tdutils/td/utils/optional.h b/tdutils/td/utils/optional.h index 44575948..7723d2c3 100644 --- a/tdutils/td/utils/optional.h +++ b/tdutils/td/utils/optional.h @@ -66,6 +66,12 @@ class optional { DCHECK(*this); return impl_.ok_ref(); } + T &value_force() { + if (!*this) { + *this = T(); + } + return value(); + } T &operator*() { return value(); } @@ -88,6 +94,14 @@ class optional { impl_.emplace(std::forward(args)...); } + bool operator==(const optional& other) const { + return (bool)*this == (bool)other && (!(bool)*this || value() == other.value()); + } + + bool operator!=(const optional& other) const { + return !(*this == other); + } + private: Result impl_; }; diff --git a/tdutils/td/utils/port/Clocks.cpp b/tdutils/td/utils/port/Clocks.cpp index 59e80f61..7e27d51a 100644 --- a/tdutils/td/utils/port/Clocks.cpp +++ b/tdutils/td/utils/port/Clocks.cpp @@ -22,6 +22,10 @@ #include namespace td { +int64 Clocks::monotonic_nano() { + auto duration = std::chrono::steady_clock::now().time_since_epoch(); + return std::chrono::duration_cast(duration).count(); +} double Clocks::monotonic() { // TODO write system specific functions, because std::chrono::steady_clock is steady only under Windows diff --git a/tdutils/td/utils/port/Clocks.h b/tdutils/td/utils/port/Clocks.h index 2d7d9e0f..b2054365 100644 --- a/tdutils/td/utils/port/Clocks.h +++ b/tdutils/td/utils/port/Clocks.h @@ -17,15 +17,89 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "td/utils/int_types.h" namespace td { struct Clocks { + static int64 monotonic_nano(); + static double monotonic(); static double system(); static int tz_offset(); + +#if defined(__i386__) or defined(__M_IX86) + static uint64 rdtsc() { + unsigned long long int x; + __asm__ volatile("rdtsc" : "=A"(x)); + return x; + } + + static constexpr uint64 rdtsc_frequency() { + return 2000'000'000; + } + + static constexpr double ticks_per_second() { + return 2e9; + } + + static constexpr double inv_ticks_per_second() { + return 0.5e-9; + } +#elif defined(__x86_64__) or defined(__M_X64) + static uint64 rdtsc() { + unsigned hi, lo; + __asm__ __volatile__("rdtsc" : "=a"(lo), "=d"(hi)); + return ((unsigned long long)lo) | (((unsigned long long)hi) << 32); + } + static constexpr uint64 rdtsc_frequency() { + return 2000'000'000; + } + + static constexpr double ticks_per_second() { + return 2e9; + } + + static constexpr double inv_ticks_per_second() { + return 0.5e-9; + } +#elif defined(__aarch64__) or defined(_M_ARM64) + static uint64 rdtsc() { + unsigned long long val; + asm volatile("mrs %0, cntvct_el0" : "=r"(val)); + return val; + } + static uint64 rdtsc_frequency() { + unsigned long long val; + asm volatile("mrs %0, cntfrq_el0" : "=r"(val)); + return val; + } + + static double ticks_per_second() { + return static_cast(rdtsc_frequency()); + } + + static double inv_ticks_per_second() { + return 1.0 / static_cast(rdtsc_frequency()); + } +#else + static uint64 rdtsc() { + return monotonic_nano(); + } + static uint64 rdtsc_frequency() { + return 1000'000'000; + } + + static double ticks_per_second() { + return static_cast(rdtsc_frequency()); + } + + static double inv_ticks_per_second() { + return 1.0 / static_cast(rdtsc_frequency()); + } +#endif }; } // namespace td diff --git a/tdutils/td/utils/port/Stat.cpp b/tdutils/td/utils/port/Stat.cpp index 00e63438..73b00608 100644 --- a/tdutils/td/utils/port/Stat.cpp +++ b/tdutils/td/utils/port/Stat.cpp @@ -58,8 +58,7 @@ #define PSAPI_VERSION 1 #endif #include -#pragma comment( lib, "psapi.lib" ) - +#pragma comment(lib, "psapi.lib") #endif @@ -413,4 +412,105 @@ Result cpu_stat() { #endif } +Result get_total_mem_stat() { +#if TD_LINUX + TRY_RESULT(fd, FileFd::open("/proc/meminfo", FileFd::Read)); + SCOPE_EXIT { + fd.close(); + }; + constexpr int TMEM_SIZE = 10000; + char mem[TMEM_SIZE]; + TRY_RESULT(size, fd.read(MutableSlice(mem, TMEM_SIZE - 1))); + if (size >= TMEM_SIZE - 1) { + return Status::Error("Failed for read /proc/meminfo"); + } + TotalMemStat stat; + mem[size] = 0; + const char *s = mem; + size_t got = 0; + while (*s) { + const char *name_begin = s; + while (*s != 0 && *s != '\n') { + s++; + } + auto name_end = name_begin; + while (is_alpha(*name_end)) { + name_end++; + } + Slice name(name_begin, name_end); + td::uint64 *dest = nullptr; + if (name == "MemTotal") { + dest = &stat.total_ram; + } else if (name == "MemAvailable") { + dest = &stat.available_ram; + } + if (dest != nullptr) { + Slice value(name_end, s); + if (!value.empty() && value[0] == ':') { + value.remove_prefix(1); + } + value = trim(value); + value = split(value).first; + TRY_RESULT_PREFIX(mem, to_integer_safe(value), PSLICE() << "Invalid value of " << name); + if (mem >= 1ULL << (64 - 10)) { + return Status::Error("Invalid value of MemTotal"); + } + *dest = mem * 1024; + got++; + if (got == 2) { + return stat; + } + } + if (*s == 0) { + break; + } + s++; + } + return Status::Error("No MemTotal in /proc/meminfo"); +#else + return Status::Error("Not supported"); +#endif +} + +Result get_cpu_cores() { +#if TD_LINUX + uint32 result = 0; + TRY_RESULT(fd, FileFd::open("/proc/cpuinfo", FileFd::Read)); + SCOPE_EXIT { + fd.close(); + }; + std::string data; + char buf[10000]; + while (true) { + TRY_RESULT(size, fd.read(MutableSlice{buf, sizeof(buf) - 1})); + if (size == 0) { + break; + } + buf[size] = '\0'; + data += buf; + } + size_t i = 0; + while (i < data.size()) { + const char *line_begin = data.data() + i; + while (i < data.size() && data[i] != '\n') { + ++i; + } + auto line_end = data.data() + i; + ++i; + Slice line{line_begin, line_end}; + size_t j = 0; + while (j < line.size() && line[j] != ' ' && line[j] != '\t' && line[j] != ':') { + ++j; + } + Slice name = line.substr(0, j); + if (name == "processor") { + ++result; + } + } + return result; +#else + return Status::Error("Not supported"); +#endif +} + } // namespace td diff --git a/tdutils/td/utils/port/Stat.h b/tdutils/td/utils/port/Stat.h index f446c39d..82e1832a 100644 --- a/tdutils/td/utils/port/Stat.h +++ b/tdutils/td/utils/port/Stat.h @@ -64,4 +64,12 @@ Status update_atime(CSlice path) TD_WARN_UNUSED_RESULT; #endif +struct TotalMemStat { + uint64 total_ram; + uint64 available_ram; +}; +Result get_total_mem_stat() TD_WARN_UNUSED_RESULT; + +Result get_cpu_cores() TD_WARN_UNUSED_RESULT; + } // namespace td diff --git a/tdutils/td/utils/port/config.h b/tdutils/td/utils/port/config.h index 77143668..f89082f3 100644 --- a/tdutils/td/utils/port/config.h +++ b/tdutils/td/utils/port/config.h @@ -28,35 +28,33 @@ #define TD_PORT_POSIX 1 #endif -#if TD_LINUX || TD_ANDROID || TD_TIZEN +#if TD_EMSCRIPTEN + #define TD_POLL_POLL 1 +#elif TD_LINUX || TD_ANDROID || TD_TIZEN #define TD_POLL_EPOLL 1 - #define TD_EVENTFD_LINUX 1 #elif TD_FREEBSD || TD_OPENBSD || TD_NETBSD #define TD_POLL_KQUEUE 1 - #define TD_EVENTFD_BSD 1 #elif TD_CYGWIN #define TD_POLL_SELECT 1 - #define TD_EVENTFD_BSD 1 -#elif TD_EMSCRIPTEN - #define TD_POLL_POLL 1 - // #define TD_EVENTFD_UNSUPPORTED 1 #elif TD_DARWIN #define TD_POLL_KQUEUE 1 - #define TD_EVENTFD_BSD 1 #elif TD_WINDOWS #define TD_POLL_WINEVENT 1 - #define TD_EVENTFD_WINDOWS 1 #else #error "Poll's implementation is not defined" #endif -#if TD_EMSCRIPTEN - // #define TD_THREAD_UNSUPPORTED 1 - #define TD_POLL_EPOLL 1 - #define TD_EVENTFD_UNSUPPORTED 0 - #define TD_THREAD_PTHREAD 1 +#if TD_LINUX || TD_ANDROID || TD_TIZEN #define TD_EVENTFD_LINUX 1 -#elif TD_TIZEN || TD_LINUX || TD_DARWIN +#elif TD_FREEBSD || TD_OPENBSD || TD_NETBSD || TD_CYGWIN || TD_DARWIN + #define TD_EVENTFD_BSD 1 +#elif TD_WINDOWS + #define TD_EVENTFD_WINDOWS 1 +#else + #error "eventfd's implementation is not defined" +#endif + +#if TD_TIZEN || TD_LINUX || TD_DARWIN || TD_EMSCRIPTEN #define TD_THREAD_PTHREAD 1 #else #define TD_THREAD_STL 1 diff --git a/tdutils/td/utils/port/detail/PollableFd.h b/tdutils/td/utils/port/detail/PollableFd.h index dceea4f3..6cb621e8 100644 --- a/tdutils/td/utils/port/detail/PollableFd.h +++ b/tdutils/td/utils/port/detail/PollableFd.h @@ -149,7 +149,10 @@ class PollableFdInfo : private ListNode { private: NativeFd fd_{}; +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-pragma" std::atomic_flag lock_ = ATOMIC_FLAG_INIT; +#pragma clang diagnostic pop PollFlagsSet flags_; #if TD_PORT_WINDOWS SpinLock observer_lock_; diff --git a/tdutils/td/utils/port/platform.h b/tdutils/td/utils/port/platform.h index 783dd399..a7b19220 100644 --- a/tdutils/td/utils/port/platform.h +++ b/tdutils/td/utils/port/platform.h @@ -20,6 +20,11 @@ // clang-format off +/*** Determine emscripten ***/ +#if defined(__EMSCRIPTEN__) + #define TD_EMSCRIPTEN 1 +#endif + /*** Platform macros ***/ #if defined(_WIN32) || defined(_WINDOWS) // _WINDOWS is defined by CMake #if defined(__cplusplus_winrt) @@ -63,10 +68,11 @@ #define TD_NETBSD 1 #elif defined(__CYGWIN__) #define TD_CYGWIN 1 -#elif defined(__EMSCRIPTEN__) - #define TD_EMSCRIPTEN 1 -#elif defined(__unix__) // all unices not caught above - #warning "Probably unsupported Unix platform. Feel free to try to compile" +#elif defined(__unix__) // all unices not caught above + // supress if emscripten + #if !TD_EMSCRIPTEN + #warning "Probably unsupported Unix platform. Feel free to try to compile" + #endif #define TD_CYGWIN 1 #else #error "Probably unsupported platform. Feel free to remove the error and try to recompile" diff --git a/tdutils/td/utils/port/stacktrace.cpp b/tdutils/td/utils/port/stacktrace.cpp index 2c025d2e..e89daec6 100644 --- a/tdutils/td/utils/port/stacktrace.cpp +++ b/tdutils/td/utils/port/stacktrace.cpp @@ -20,9 +20,13 @@ #include "td/utils/port/signals.h" -#if __GLIBC__ +#if TD_WINDOWS +#include +#else +#if TD_DARWIN || __GLIBC__ #include #endif +#endif #if TD_LINUX || TD_FREEBSD #include @@ -39,13 +43,48 @@ namespace td { namespace { void print_backtrace(void) { -#if __GLIBC__ +#if TD_WINDOWS + void *stack[100]; + HANDLE process = GetCurrentProcess(); + SymInitialize(process, nullptr, 1); + unsigned frames = CaptureStackBackTrace(0, 100, stack, nullptr); + signal_safe_write("------- Stack Backtrace -------\n", false); + for (unsigned i = 0; i < frames; i++) { + td::uint8 symbol_buf[sizeof(SYMBOL_INFO) + 256]; + auto symbol = (SYMBOL_INFO *)symbol_buf; + memset(symbol_buf, 0, sizeof(symbol_buf)); + symbol->MaxNameLen = 255; + symbol->SizeOfStruct = sizeof(SYMBOL_INFO); + SymFromAddr(process, (DWORD64)(stack[i]), nullptr, symbol); + // Don't use sprintf here because it is not signal-safe + char buf[256 + 32]; + char* buf_ptr = buf; + if (frames - i - 1 < 10) { + strcpy(buf_ptr, " "); + buf_ptr += strlen(buf_ptr); + } + _itoa(frames - i - 1, buf_ptr, 10); + buf_ptr += strlen(buf_ptr); + strcpy(buf_ptr, ": ["); + buf_ptr += strlen(buf_ptr); + _ui64toa(td::uint64(symbol->Address), buf_ptr, 16); + buf_ptr += strlen(buf_ptr); + strcpy(buf_ptr, "] "); + buf_ptr += strlen(buf_ptr); + strcpy(buf_ptr, symbol->Name); + buf_ptr += strlen(buf_ptr); + strcpy(buf_ptr, "\n"); + signal_safe_write(td::Slice{buf, strlen(buf)}, false); + } +#else +#if TD_DARWIN || __GLIBC__ void *buffer[128]; int nptrs = backtrace(buffer, 128); signal_safe_write("------- Stack Backtrace -------\n", false); backtrace_symbols_fd(buffer, nptrs, 2); signal_safe_write("-------------------------------\n", false); #endif +#endif } void print_backtrace_gdb(void) { @@ -129,7 +168,7 @@ void Stacktrace::print_to_stderr(const PrintOptions &options) { } void Stacktrace::init() { -#if __GLIBC__ +#if TD_DARWIN || __GLIBC__ // backtrace needs to be called once to ensure that next calls are async-signal-safe void *buffer[1]; backtrace(buffer, 1); diff --git a/tdutils/test/List.cpp b/tdutils/test/List.cpp index ae00b499..74fe9c21 100644 --- a/tdutils/test/List.cpp +++ b/tdutils/test/List.cpp @@ -170,8 +170,8 @@ TEST(Misc, TsList) { TEST(Misc, TsListConcurrent) { td::TsList root; - td::vector threads; std::atomic id{0}; + td::vector threads; for (std::size_t i = 0; i < 4; i++) { threads.emplace_back( [&] { do_run_list_test, td::TsList, td::TsListNode>(root, id); }); diff --git a/tdutils/test/MpmcWaiter.cpp b/tdutils/test/MpmcWaiter.cpp index 9cb5b363..d0a9fc84 100644 --- a/tdutils/test/MpmcWaiter.cpp +++ b/tdutils/test/MpmcWaiter.cpp @@ -75,9 +75,9 @@ void test_waiter_stress_one_one() { TEST(MpmcEagerWaiter, stress_one_one) { test_waiter_stress_one_one(); } -TEST(MpmcSleepyWaiter, stress_one_one) { - test_waiter_stress_one_one(); -} +// TEST(MpmcSleepyWaiter, stress_one_one) { +// test_waiter_stress_one_one(); +// } template void test_waiter_stress() { diff --git a/terminal/CMakeLists.txt b/terminal/CMakeLists.txt index ae8c70bd..af51153f 100644 --- a/terminal/CMakeLists.txt +++ b/terminal/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) diff --git a/test/ed25519_crypto.cpp b/test/ed25519_crypto.cpp deleted file mode 100644 index 371b7247..00000000 --- a/test/ed25519_crypto.cpp +++ /dev/null @@ -1,2053 +0,0 @@ -/* - This file is part of TON Blockchain source code. - - TON Blockchain is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - TON Blockchain is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with TON Blockchain. If not, see . - - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement - from all source files in the program, then also delete it here. - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include -#include -#include -#include -#include - -// ****************************************************** - -namespace openssl { -#include -} - -namespace arith { -struct dec_string { - std::string str; - explicit dec_string(const std::string& s) : str(s) { - } -}; - -struct hex_string { - std::string str; - explicit hex_string(const std::string& s) : str(s) { - } -}; -} // namespace arith - -namespace arith { - -using namespace openssl; - -inline void bn_assert(int cond); -BN_CTX* get_ctx(); - -class BignumBitref { - BIGNUM* ptr; - int n; - - public: - BignumBitref(BIGNUM& x, int _n) : ptr(&x), n(_n){}; - operator bool() const { - return BN_is_bit_set(ptr, n); - } - BignumBitref& operator=(bool val); -}; - -class Bignum { - BIGNUM val; - - public: - class bignum_error {}; - Bignum() { - BN_init(&val); - } - Bignum(long x) { - BN_init(&val); - set_long(x); - } - ~Bignum() { - BN_free(&val); - } - Bignum(const dec_string& ds) { - BN_init(&val); - set_dec_str(ds.str); - } - Bignum(const hex_string& hs) { - BN_init(&val); - set_hex_str(hs.str); - } - Bignum(const Bignum& x) { - BN_init(&val); - BN_copy(&val, &x.val); - } - //Bignum (Bignum&& x) { val = x.val; } - void clear() { - BN_clear(&val); - } // use this for sensitive data - Bignum& operator=(const Bignum& x) { - BN_copy(&val, &x.val); - return *this; - } - Bignum& operator=(Bignum&& x) { - swap(x); - return *this; - } - Bignum& operator=(long x) { - return set_long(x); - } - Bignum& operator=(const dec_string& ds) { - return set_dec_str(ds.str); - } - Bignum& operator=(const hex_string& hs) { - return set_hex_str(hs.str); - } - Bignum& swap(Bignum& x) { - BN_swap(&val, &x.val); - return *this; - } - BIGNUM* bn_ptr() { - return &val; - } - const BIGNUM* bn_ptr() const { - return &val; - } - bool is_zero() const { - return BN_is_zero(&val); - } - int sign() const { - return BN_is_zero(&val) ? 0 : (BN_is_negative(&val) ? -1 : 1); - } - bool odd() const { - return BN_is_odd(&val); - } - int num_bits() const { - return BN_num_bits(&val); - } - int num_bytes() const { - return BN_num_bytes(&val); - } - bool operator[](int n) const { - return BN_is_bit_set(&val, n); - } - BignumBitref operator[](int n) { - return BignumBitref(val, n); - } - void export_msb(unsigned char* buffer, std::size_t size) const; - Bignum& import_msb(const unsigned char* buffer, std::size_t size); - Bignum& import_msb(const std::string& s) { - return import_msb((const unsigned char*)s.c_str(), s.size()); - } - void export_lsb(unsigned char* buffer, std::size_t size) const; - Bignum& import_lsb(const unsigned char* buffer, std::size_t size); - Bignum& import_lsb(const std::string& s) { - return import_lsb((const unsigned char*)s.c_str(), s.size()); - } - - Bignum& set_dec_str(std::string s) { - BIGNUM* tmp = &val; - bn_assert(BN_dec2bn(&tmp, s.c_str())); - return *this; - } - - Bignum& set_hex_str(std::string s) { - BIGNUM* tmp = &val; - bn_assert(BN_hex2bn(&tmp, s.c_str())); - return *this; - } - - Bignum& set_ulong(unsigned long x) { - bn_assert(BN_set_word(&val, x)); - return *this; - } - - Bignum& set_long(long x) { - set_ulong(std::abs(x)); - return x < 0 ? negate() : *this; - } - - Bignum& negate() { - BN_set_negative(&val, !BN_is_negative(&val)); - return *this; - } - - Bignum& operator+=(const Bignum& y) { - bn_assert(BN_add(&val, &val, &y.val)); - return *this; - } - - Bignum& operator+=(long y) { - bn_assert((y >= 0 ? BN_add_word : BN_sub_word)(&val, std::abs(y))); - return *this; - } - - Bignum& operator-=(long y) { - bn_assert((y >= 0 ? BN_sub_word : BN_add_word)(&val, std::abs(y))); - return *this; - } - - Bignum& operator*=(const Bignum& y) { - bn_assert(BN_mul(&val, &val, &y.val, get_ctx())); - return *this; - } - - Bignum& operator*=(long y) { - if (y < 0) { - negate(); - } - bn_assert(BN_mul_word(&val, std::abs(y))); - return *this; - } - - Bignum& operator<<=(int r) { - bn_assert(BN_lshift(&val, &val, r)); - return *this; - } - - Bignum& operator>>=(int r) { - bn_assert(BN_rshift(&val, &val, r)); - return *this; - } - - Bignum& operator/=(const Bignum& y) { - Bignum w; - bn_assert(BN_div(&val, &w.val, &val, &y.val, get_ctx())); - return *this; - } - - Bignum& operator/=(long y) { - bn_assert(BN_div_word(&val, std::abs(y)) != (BN_ULONG)(-1)); - return y < 0 ? negate() : *this; - } - - Bignum& operator%=(const Bignum& y) { - bn_assert(BN_mod(&val, &val, &y.val, get_ctx())); - return *this; - } - - Bignum& operator%=(long y) { - BN_ULONG rem = BN_mod_word(&val, std::abs(y)); - bn_assert(rem != (BN_ULONG)(-1)); - return set_long(y < 0 ? -rem : rem); - } - - unsigned long divmod(unsigned long y) { - BN_ULONG rem = BN_div_word(&val, y); - bn_assert(rem != (BN_ULONG)(-1)); - return rem; - } - - const Bignum divmod(const Bignum& y); - - std::string to_str() const; - std::string to_hex() const; -}; - -inline void bn_assert(int cond) { - if (!cond) { - throw Bignum::bignum_error(); - } -} - -BN_CTX* get_ctx(void) { - static BN_CTX* ctx = BN_CTX_new(); - return ctx; -} - -BignumBitref& BignumBitref::operator=(bool val) { - if (val) { - BN_set_bit(ptr, n); - } else { - BN_clear_bit(ptr, n); - } - return *this; -} - -const Bignum operator+(const Bignum& x, const Bignum& y) { - Bignum z; - bn_assert(BN_add(z.bn_ptr(), x.bn_ptr(), y.bn_ptr())); - return z; -} - -const Bignum operator+(const Bignum& x, long y) { - if (y > 0) { - Bignum z(x); - bn_assert(BN_add_word(z.bn_ptr(), y)); - return z; - } else if (y < 0) { - Bignum z(x); - bn_assert(BN_sub_word(z.bn_ptr(), -y)); - return z; - } else { - return x; - } -} - -/* - const Bignum operator+ (Bignum&& x, long y) { - if (y > 0) { - bn_assert (BN_add_word (x.bn_ptr(), y)); - } else if (y < 0) { - bn_assert (BN_sub_word (x.bn_ptr(), -y)); - } - return std::move (x); - } - */ - -const Bignum operator+(long y, const Bignum& x) { - return x + y; -} - -/* - const Bignum operator+ (long y, Bignum&& x) { - return x + y; - } - */ - -const Bignum operator-(const Bignum& x, const Bignum& y) { - Bignum z; - bn_assert(BN_sub(z.bn_ptr(), x.bn_ptr(), y.bn_ptr())); - return z; -} - -const Bignum operator-(const Bignum& x, long y) { - return x + (-y); -} - -/* - const Bignum operator- (Bignum&& x, long y) { - return x + (-y); - } - */ - -const Bignum operator*(const Bignum& x, const Bignum& y) { - Bignum z; - bn_assert(BN_mul(z.bn_ptr(), x.bn_ptr(), y.bn_ptr(), get_ctx())); - return z; -} - -const Bignum operator*(const Bignum& x, long y) { - if (y > 0) { - Bignum z(x); - bn_assert(BN_mul_word(z.bn_ptr(), y)); - return z; - } else if (y < 0) { - Bignum z(x); - z.negate(); - bn_assert(BN_mul_word(z.bn_ptr(), -y)); - return z; - } else { - Bignum z(0); - return z; - } -} - -/* - const Bignum operator* (Bignum&& x, long y) { - if (y > 0) { - bn_assert (BN_mul_word (x.bn_ptr(), y)); - } else if (y < 0) { - x.negate(); - bn_assert (BN_mul_word (x.bn_ptr(), -y)); - } else { - x = 0; - } - return std::move (x); - } - */ - -const Bignum operator*(long y, const Bignum& x) { - return x * y; -} - -const Bignum operator/(const Bignum& x, const Bignum& y) { - Bignum z, w; - bn_assert(BN_div(z.bn_ptr(), w.bn_ptr(), x.bn_ptr(), y.bn_ptr(), get_ctx())); - return z; -} - -const Bignum Bignum::divmod(const Bignum& y) { - Bignum w; - bn_assert(BN_div(&val, w.bn_ptr(), &val, y.bn_ptr(), get_ctx())); - return w; -} - -const Bignum operator%(const Bignum& x, const Bignum& y) { - Bignum z; - bn_assert(BN_mod(z.bn_ptr(), x.bn_ptr(), y.bn_ptr(), get_ctx())); - return z; -} - -unsigned long operator%(const Bignum& x, unsigned long y) { - BN_ULONG rem = BN_mod_word(x.bn_ptr(), y); - bn_assert(rem != (BN_ULONG)(-1)); - return rem; -} - -const Bignum operator<<(const Bignum& x, int r) { - Bignum z; - bn_assert(BN_lshift(z.bn_ptr(), x.bn_ptr(), r)); - return z; -} - -const Bignum operator>>(const Bignum& x, int r) { - Bignum z; - bn_assert(BN_rshift(z.bn_ptr(), x.bn_ptr(), r)); - return z; -} - -const Bignum abs(const Bignum& x) { - Bignum T(x); - if (T.sign() < 0) { - T.negate(); - } - return T; -} - -const Bignum sqr(const Bignum& x) { - Bignum z; - bn_assert(BN_sqr(z.bn_ptr(), x.bn_ptr(), get_ctx())); - return z; -} - -void Bignum::export_msb(unsigned char* buffer, std::size_t size) const { - bn_assert(size >= 0 && size <= (1 << 20)); - bn_assert(sign() >= 0); - int n = BN_num_bytes(&val); - bn_assert(n >= 0 && (unsigned)n <= size); - bn_assert(BN_bn2bin(&val, buffer + size - n) == n); - std::memset(buffer, 0, size - n); -} - -Bignum& Bignum::import_msb(const unsigned char* buffer, std::size_t size) { - bn_assert(size >= 0 && size <= (1 << 20)); - std::size_t i = 0; - while (i < size && !buffer[i]) { - i++; - } - bn_assert(BN_bin2bn(buffer + i, size - i, &val) == &val); - return *this; -} - -void Bignum::export_lsb(unsigned char* buffer, std::size_t size) const { - bn_assert(size >= 0 && size <= (1 << 20)); - bn_assert(sign() >= 0); - std::size_t n = BN_num_bytes(&val); - bn_assert(n >= 0 && (unsigned)n <= size); - bn_assert(BN_bn2bin(&val, buffer) == (int)n); - std::memset(buffer + n, 0, size - n); - for (std::size_t i = 0; 2 * i + 1 < n; i++) { - std::swap(buffer[i], buffer[n - 1 - i]); - } -} - -Bignum& Bignum::import_lsb(const unsigned char* buffer, std::size_t size) { - bn_assert(size >= 0 && size <= (1 << 20)); - while (size > 0 && !buffer[size - 1]) { - size--; - } - if (!size) { - bn_assert(BN_zero(&val)); - return *this; - } - unsigned char tmp[size], *ptr = tmp + size; - for (std::size_t i = 0; i < size; i++) { - *--ptr = buffer[i]; - } - bn_assert(BN_bin2bn(tmp, size, &val) == &val); - return *this; -} - -int cmp(const Bignum& x, const Bignum& y) { - return BN_cmp(x.bn_ptr(), y.bn_ptr()); -} - -bool operator==(const Bignum& x, const Bignum& y) { - return cmp(x, y) == 0; -} - -bool operator!=(const Bignum& x, const Bignum& y) { - return cmp(x, y) != 0; -} - -bool operator<(const Bignum& x, const Bignum& y) { - return cmp(x, y) < 0; -} - -bool operator<=(const Bignum& x, const Bignum& y) { - return cmp(x, y) <= 0; -} - -bool operator>(const Bignum& x, const Bignum& y) { - return cmp(x, y) > 0; -} - -bool operator>=(const Bignum& x, const Bignum& y) { - return cmp(x, y) >= 0; -} - -bool operator==(const Bignum& x, long y) { - if (y >= 0) { - return BN_is_word(x.bn_ptr(), y); - } else { - return x == Bignum(y); - } -} - -bool operator!=(const Bignum& x, long y) { - if (y >= 0) { - return !BN_is_word(x.bn_ptr(), y); - } else { - return x != Bignum(y); - } -} - -std::string Bignum::to_str() const { - char* ptr = BN_bn2dec(&val); - std::string z(ptr); - OPENSSL_free(ptr); - return z; -} - -std::string Bignum::to_hex() const { - char* ptr = BN_bn2hex(&val); - std::string z(ptr); - OPENSSL_free(ptr); - return z; -} - -std::ostream& operator<<(std::ostream& os, const Bignum& x) { - return os << x.to_str(); -} - -std::istream& operator>>(std::istream& is, Bignum& x) { - std::string word; - is >> word; - x = dec_string(word); - return is; -} - -bool is_prime(const Bignum& p, int nchecks = 64, bool trial_div = true) { - return BN_is_prime_fasttest_ex(p.bn_ptr(), BN_prime_checks, get_ctx(), trial_div, 0); -} -} // namespace arith - -namespace arith { -using namespace openssl; - -class Residue; -class ResidueRing; - -class ResidueRing { - public: - struct bad_modulus {}; - struct elem_cnt_mismatch { - int cnt; - elem_cnt_mismatch(int x) : cnt(x) { - } - }; - - private: - const Bignum modulus; - mutable int cnt; - bool prime; - void cnt_assert(bool b) { - if (!b) { - throw elem_cnt_mismatch(cnt); - } - } - Residue* Zero; - Residue* One; - Residue* Img_i; - void init(); - - public: - typedef Residue element; - explicit ResidueRing(Bignum mod) : modulus(mod), cnt(0), prime(arith::is_prime(mod)), Zero(0), One(0) { - init(); - } - ~ResidueRing(); - int incr_count() { - return ++cnt; - } - int decr_count() { - --cnt; - cnt_assert(cnt >= 0); - return cnt; - } - const Bignum& get_modulus() const { - return modulus; - } - bool is_prime() const { - return prime; - } - const Residue& zero() const { - return *Zero; - } - const Residue& one() const { - return *One; - } - const Residue& img_i(); - Residue frac(long num, long denom = 1); - Residue convert(long num); - Residue convert(const Bignum& x); - - Bignum reduce(const Bignum& x) { - Bignum r = x % modulus; - if (r.sign() < 0) { - r += modulus; - } - return r; - } - - Bignum& do_reduce(Bignum& x) { - x %= modulus; - if (x.sign() < 0) { - x += modulus; - } - return x; - } -}; - -class Residue { - public: - struct not_same_ring {}; - - private: - ResidueRing* ring; - mutable Bignum val; - Residue& reduce() { - ring->do_reduce(val); - return *this; - } - - public: - explicit Residue(ResidueRing& R) : ring(&R) { - R.incr_count(); - } - Residue(const Bignum& x, ResidueRing& R) : ring(&R), val(R.reduce(x)) { - R.incr_count(); - } - ~Residue() { - ring->decr_count(); - ring = 0; - } - Residue(const Residue& x) : ring(x.ring), val(x.val) { - ring->incr_count(); - } - Bignum extract() const { - return val; - } - const Bignum& extract_raw() const { - return val; - } - const Bignum& modulus() const { - return ring->get_modulus(); - } - void same_ring(const Residue& y) const { - if (ring != y.ring) { - throw not_same_ring(); - } - } - ResidueRing& ring_of() const { - return *ring; - } - bool is_zero() const { - return (val == 0); - } - Residue& operator=(const Residue& x) { - same_ring(x); - val = x.val; - return *this; - } - Residue& operator=(const Bignum& x) { - val = ring->reduce(x); - return *this; - } - Residue& operator+=(const Residue& y); - Residue& operator-=(const Residue& y); - Residue& operator*=(const Residue& y); - Residue& operator+=(long y) { - val += y; - return reduce(); - } - Residue& operator-=(long y) { - val -= y; - return reduce(); - } - Residue& operator*=(long y) { - val *= y; - return reduce(); - } - Residue& negate() { - val.negate(); - return reduce(); - } - friend const Residue operator+(const Residue& x, const Residue& y); - friend const Residue operator-(const Residue& x, const Residue& y); - friend const Residue operator*(const Residue& x, const Residue& y); - friend const Residue operator-(const Residue& x); - friend Residue sqr(const Residue& x); - friend Residue power(const Residue& x, const Bignum& y); - friend Residue inverse(const Residue& x); - std::string to_str() const; -}; - -void ResidueRing::init() { - Zero = new Residue(0, *this); - One = new Residue(1, *this); -} - -ResidueRing::~ResidueRing() { - delete Zero; - delete One; - Zero = One = 0; - cnt_assert(!cnt); -} - -const Residue operator+(const Residue& x, const Residue& y) { - x.same_ring(y); - Residue z(x.ring_of()); - bn_assert(BN_mod_add(z.val.bn_ptr(), x.val.bn_ptr(), y.val.bn_ptr(), x.modulus().bn_ptr(), get_ctx())); - return z; -} - -const Residue operator-(const Residue& x, const Residue& y) { - x.same_ring(y); - Residue z(x.ring_of()); - bn_assert(BN_mod_sub(z.val.bn_ptr(), x.val.bn_ptr(), y.val.bn_ptr(), x.modulus().bn_ptr(), get_ctx())); - return z; -} - -const Residue operator*(const Residue& x, const Residue& y) { - x.same_ring(y); - Residue z(x.ring_of()); - bn_assert(BN_mod_mul(z.val.bn_ptr(), x.val.bn_ptr(), y.val.bn_ptr(), x.modulus().bn_ptr(), get_ctx())); - return z; -} - -const Residue operator-(const Residue& x) { - Residue z(x); - z.val.negate(); - return z.reduce(); -} - -Residue& Residue::operator+=(const Residue& y) { - same_ring(y); - bn_assert(BN_mod_add(val.bn_ptr(), val.bn_ptr(), y.val.bn_ptr(), modulus().bn_ptr(), get_ctx())); - return *this; -} - -Residue& Residue::operator-=(const Residue& y) { - same_ring(y); - bn_assert(BN_mod_sub(val.bn_ptr(), val.bn_ptr(), y.val.bn_ptr(), modulus().bn_ptr(), get_ctx())); - return *this; -} - -Residue& Residue::operator*=(const Residue& y) { - same_ring(y); - bn_assert(BN_mod_mul(val.bn_ptr(), val.bn_ptr(), y.val.bn_ptr(), modulus().bn_ptr(), get_ctx())); - return *this; -} - -bool operator==(const Residue& x, const Residue& y) { - x.same_ring(y); - return x.extract() == y.extract(); -} - -bool operator!=(const Residue& x, const Residue& y) { - x.same_ring(y); - return x.extract() != y.extract(); -} - -Residue sqr(const Residue& x) { - Residue z(x.ring_of()); - bn_assert(BN_mod_sqr(z.val.bn_ptr(), x.val.bn_ptr(), x.modulus().bn_ptr(), get_ctx())); - return z; -} - -Residue power(const Residue& x, const Bignum& y) { - Residue z(x.ring_of()); - bn_assert(BN_mod_exp(z.val.bn_ptr(), x.val.bn_ptr(), y.bn_ptr(), x.modulus().bn_ptr(), get_ctx())); - return z; -} - -Residue inverse(const Residue& x) { - assert(x.ring_of().is_prime()); - return power(x, x.ring_of().get_modulus() - 2); -} - -const Residue& ResidueRing::img_i() { - if (!Img_i) { - assert(is_prime()); - assert(modulus % 4 == 1); - int g = 2; - Bignum n = (modulus - 1) / 4; - while (true) { - Residue t = power(frac(g), n); - if (t != one() && t != frac(-1)) { - Img_i = new Residue(t); - break; - } - } - } - return *Img_i; -} - -Residue sqrt(const Residue& x) { - assert(x.ring_of().is_prime()); - ResidueRing& R = x.ring_of(); - const Bignum& p = R.get_modulus(); - if (x.is_zero() || !p.odd()) { - return x; - } - if (p[1]) { // p=3 (mod 4) - return power(x, (p + 1) >> 2); - } else if (p[2]) { - // p=5 (mod 8) - Residue t = power(x, (p + 3) >> 3); - return (sqr(t) == x) ? t : R.img_i() * t; - } else { - assert(p[2]); - return R.zero(); - } -} - -Residue ResidueRing::frac(long num, long denom) { - assert(denom); - if (denom < 0) { - num = -num; - denom = -denom; - } - if (!(num % denom)) { - return Residue(num / denom, *this); - } else { - return Residue(num, *this) * inverse(Residue(denom, *this)); - } -} - -inline Residue ResidueRing::convert(long x) { - return Residue(x, *this); -} - -inline Residue ResidueRing::convert(const Bignum& x) { - return Residue(x, *this); -} - -std::string Residue::to_str() const { - return "Mod(" + val.to_str() + "," + modulus().to_str() + ")"; -} - -std::ostream& operator<<(std::ostream& os, const Residue& x) { - return os << x.to_str(); -} - -std::istream& operator>>(std::istream& is, Residue& x) { - std::string word; - is >> word; - x = dec_string(word); - return is; -} -} // namespace arith - -// ****************************************************** - -namespace ellcurve { -using namespace arith; - -const Bignum& P25519() { - static Bignum P25519 = (Bignum(1) << 255) - 19; - return P25519; -} - -ResidueRing& Fp25519() { - static ResidueRing Fp25519(P25519()); - return Fp25519; -} -} // namespace ellcurve - -// ****************************************************** - -namespace ellcurve { -using namespace arith; - -class MontgomeryCurve { - ResidueRing& ring; - int A_short; // v^2 = u^2 + Au + 1 - int Gu_short; // u(G) - int a_short; // (A+2)/4 - Residue A; - Residue Gu; - Bignum P; - Bignum L; - Bignum Order; - Bignum cofactor; - int cofactor_short; - - void init(); - - public: - MontgomeryCurve(int _A, int _Gu, ResidueRing& _R) - : ring(_R) - , A_short(_A) - , Gu_short(_Gu) - , a_short((_A + 2) / 4) - , A(_A, _R) - , Gu(_Gu, _R) - , P(_R.get_modulus()) - , cofactor_short(0) { - init(); - } - - const Residue& get_gen_u() const { - return Gu; - } - const Bignum& get_ell() const { - return L; - } - const Bignum& get_order() const { - return Order; - } - ResidueRing& get_base_ring() const { - return ring; - } - const Bignum& get_p() const { - return P; - } - - void set_order_cofactor(const Bignum& order, int cof); - - struct PointXZ { - Residue X, Z; - PointXZ(Residue x, Residue z) : X(x), Z(z) { - x.same_ring(z); - } - PointXZ(ResidueRing& r) : X(r.one()), Z(r.zero()) { - } - explicit PointXZ(Residue u) : X(u), Z(u.ring_of().one()) { - } - explicit PointXZ(Residue y, bool) : X(y.ring_of().one() - y), Z(y + y.ring_of().one()) { - } - PointXZ(const PointXZ& P) : X(P.X), Z(P.Z) { - } - PointXZ& operator=(const PointXZ& P) { - X = P.X; - Z = P.Z; - return *this; - } - Residue get_u() const { - return X * inverse(Z); - } - Residue get_v(bool sign_v = false) const; - bool is_infty() const { - return Z.is_zero(); - } - Residue get_y() const { - return (X - Z) * inverse(X + Z); - } - bool export_point_y(unsigned char buffer[32]) const; - bool export_point_u(unsigned char buffer[32]) const; - void zeroize() { - X = Z = Z.ring_of().zero(); - } - }; - - PointXZ power_gen_xz(const Bignum& n) const; - PointXZ power_xz(const Residue& u, const Bignum& n) const; - PointXZ power_xz(const PointXZ& P, const Bignum& n) const; - PointXZ add_xz(const PointXZ& P, const PointXZ& Q) const; - PointXZ double_xz(const PointXZ& P) const; - - PointXZ import_point_u(const unsigned char point[32]) const; - PointXZ import_point_y(const unsigned char point[32]) const; -}; - -void MontgomeryCurve::init() { - assert(!((a_short + 2) & 3) && a_short >= 0); -} - -void MontgomeryCurve::set_order_cofactor(const Bignum& order, int cof) { - assert(order > 0); - assert(cof >= 0); - assert(cof == 0 || (order % cof) == 0); - Order = order; - cofactor = cofactor_short = cof; - if (cof > 0) { - L = order / cof; - assert(is_prime(L)); - } - assert(!power_gen_xz(1).is_infty()); - assert(power_gen_xz(Order).is_infty()); -} - -// computes u(P+Q)*u(P-Q) as X/Z -MontgomeryCurve::PointXZ MontgomeryCurve::add_xz(const MontgomeryCurve::PointXZ& P, - const MontgomeryCurve::PointXZ& Q) const { - Residue u = (P.X + P.Z) * (Q.X - Q.Z); - Residue v = (P.X - P.Z) * (Q.X + Q.Z); - return MontgomeryCurve::PointXZ(sqr(u + v), sqr(u - v)); -} - -// computes u(2P) as X/Z -MontgomeryCurve::PointXZ MontgomeryCurve::double_xz(const MontgomeryCurve::PointXZ& P) const { - Residue u = sqr(P.X + P.Z); - Residue v = sqr(P.X - P.Z); - Residue w = u - v; - return PointXZ(u * v, w * (v + Residue(a_short, ring) * w)); -} - -MontgomeryCurve::PointXZ MontgomeryCurve::power_gen_xz(const Bignum& n) const { - return power_xz(Gu, n); -} - -MontgomeryCurve::PointXZ MontgomeryCurve::power_xz(const Residue& u, const Bignum& n) const { - return power_xz(PointXZ(u), n); -} - -// computes u([n]P) in form X/Z -MontgomeryCurve::PointXZ MontgomeryCurve::power_xz(const PointXZ& A, const Bignum& n) const { - assert(n >= 0); - if (n == 0) { - return PointXZ(ring); - } - - int k = n.num_bits(); - PointXZ P(A); - PointXZ Q(double_xz(P)); - for (int i = k - 2; i >= 0; --i) { - PointXZ PQ(add_xz(P, Q)); - PQ.X *= A.Z; - PQ.Z *= A.X; - if (n[i]) { - P = PQ; - Q = double_xz(Q); - } else { - Q = PQ; - P = double_xz(P); - } - } - return P; -} - -bool MontgomeryCurve::PointXZ::export_point_y(unsigned char buffer[32]) const { - if ((X + Z).is_zero()) { - std::memset(buffer, 0xff, 32); - return false; - } else { - get_y().extract().export_lsb(buffer, 32); - return true; - } -} - -bool MontgomeryCurve::PointXZ::export_point_u(unsigned char buffer[32]) const { - if (Z.is_zero()) { - std::memset(buffer, 0xff, 32); - return false; - } else { - get_u().extract().export_lsb(buffer, 32); - return true; - } -} - -MontgomeryCurve::PointXZ MontgomeryCurve::import_point_u(const unsigned char point[32]) const { - Bignum u; - u.import_lsb(point, 32); - u[255] = 0; - return PointXZ(Residue(u, ring)); -} - -MontgomeryCurve::PointXZ MontgomeryCurve::import_point_y(const unsigned char point[32]) const { - Bignum y; - y.import_lsb(point, 32); - y[255] = 0; - return PointXZ(Residue(y, ring), true); -} - -MontgomeryCurve& Curve25519() { - static MontgomeryCurve Curve25519(486662, 9, Fp25519()); - static bool init = false; - if (!init) { - Curve25519.set_order_cofactor(hex_string{"80000000000000000000000000000000a6f7cef517bce6b2c09318d2e7ae9f68"}, 8); - init = true; - } - return Curve25519; -} -} // namespace ellcurve - -// ****************************************************** - -namespace ellcurve { -using namespace arith; - -class TwEdwardsCurve; - -class TwEdwardsCurve { - public: - struct SegrePoint { - Residue XY, X, Y, Z; // if x=X/Z and y=Y/T, stores (xy,x,y,1)*Z*T - SegrePoint(ResidueRing& R) : XY(R), X(R), Y(R), Z(R) { - } - SegrePoint(const Residue& x, const Residue& y) : XY(x * y), X(x), Y(y), Z(y.ring_of().one()) { - } - SegrePoint(const TwEdwardsCurve& E, const Residue& y, bool x_sign); - SegrePoint(const SegrePoint& P) : XY(P.XY), X(P.X), Y(P.Y), Z(P.Z) { - } - SegrePoint& operator=(const SegrePoint& P) { - XY = P.XY; - X = P.X; - Y = P.Y; - Z = P.Z; - return *this; - } - bool is_zero() const { - return X.is_zero() && (Y == Z); - } - bool is_valid() const { - return (XY * Z == X * Y) && !(XY.is_zero() && X.is_zero() && Y.is_zero() && Z.is_zero()); - } - bool is_finite() const { - return !Z.is_zero(); - } - bool is_normalized() const { - return Z == Z.ring_of().one(); - } - SegrePoint& normalize() { - auto f = inverse(Z); - XY *= f; - X *= f; - Y *= f; - Z = Z.ring_of().one(); - return *this; - } - SegrePoint& zeroize() { - XY = X = Y = Z = Z.ring_of().zero(); - return *this; - } - bool export_point(unsigned char buffer[32], bool need_x = true) const; - bool export_point_y(unsigned char buffer[32]) const { - return export_point(buffer, false); - } - bool export_point_u(unsigned char buffer[32]) const; - Residue get_y() const { - return Y * inverse(Z); - } - Residue get_x() const { - return X * inverse(Z); - } - Residue get_u() const { - return (Z + Y) * inverse(Z - Y); - } - void negate() { - XY.negate(); - X.negate(); - } - }; - - private: - ResidueRing& ring; - Residue D; - Residue D2; - Residue Gy; - Bignum P; - Bignum L; - Bignum Order; - Bignum cofactor; - int cofactor_short; - SegrePoint G; - SegrePoint O; - void init(); - - public: - TwEdwardsCurve(const Residue& _D, const Residue& _Gy, ResidueRing& _R) - : ring(_R), D(_D), D2(_D + _D), Gy(_Gy), P(_R.get_modulus()), cofactor_short(0), G(_R), O(_R) { - init(); - } - - const Residue& get_gen_y() const { - return Gy; - } - const Bignum& get_ell() const { - return L; - } - const Bignum& get_order() const { - return Order; - } - ResidueRing& get_base_ring() const { - return ring; - } - const Bignum& get_p() const { - return P; - } - const SegrePoint& get_base_point() const { - return G; - } - - void set_order_cofactor(const Bignum& order, int cof); - bool recover_x(Residue& x, const Residue& y, bool x_sign) const; - - void add_points(SegrePoint& R, const SegrePoint& P, const SegrePoint& Q) const; - SegrePoint add_points(const SegrePoint& P, const SegrePoint& Q) const; - void double_point(SegrePoint& R, const SegrePoint& P) const; - SegrePoint double_point(const SegrePoint& P) const; - SegrePoint power_point(const SegrePoint& A, const Bignum& n) const; - SegrePoint power_gen(const Bignum& n) const; - - SegrePoint import_point(const unsigned char point[32], bool& ok) const; -}; - -std::ostream& operator<<(std::ostream& os, const TwEdwardsCurve::SegrePoint& P) { - return os << "[" << P.XY << ":" << P.X << ":" << P.Y << ":" << P.Z << "]"; -} - -void TwEdwardsCurve::init() { - assert(D != ring.zero() && D != ring.convert(-1)); - O.X = O.Z = ring.one(); - G = SegrePoint(*this, Gy, 0); - assert(!G.XY.is_zero()); -} - -void TwEdwardsCurve::set_order_cofactor(const Bignum& order, int cof) { - assert(order > 0); - assert(cof >= 0); - assert(cof == 0 || (order % cof) == 0); - Order = order; - cofactor = cofactor_short = cof; - if (cof > 0) { - L = order / cof; - assert(is_prime(L)); - assert(!power_gen(1).is_zero()); - assert(power_gen(L).is_zero()); - } -} - -TwEdwardsCurve::SegrePoint::SegrePoint(const TwEdwardsCurve& E, const Residue& y, bool x_sign) - : XY(y), X(E.get_base_ring()), Y(y), Z(E.get_base_ring().one()) { - Residue x(y.ring_of()); - if (E.recover_x(x, y, x_sign)) { - XY *= x; - X = x; - } else { - XY = Y = Z = E.get_base_ring().zero(); - } -} - -bool TwEdwardsCurve::recover_x(Residue& x, const Residue& y, bool x_sign) const { - // recovers x from equation -x^2+y^2 = 1+d*x^2*y^2 - Residue z = inverse(ring.one() + D * sqr(y)); - if (z.is_zero()) { - return false; - } - z *= sqr(y) - ring.one(); - Residue t = sqrt(z); - if (sqr(t) == z) { - x = (t.extract().odd() == x_sign) ? t : -t; - //std::cout << "x=" << x << ", y=" << y << std::endl; - return true; - } else { - return false; - } -} - -void TwEdwardsCurve::add_points(SegrePoint& Res, const SegrePoint& P, const SegrePoint& Q) const { - Residue a((P.X + P.Y) * (Q.X + Q.Y)); - Residue b((P.X - P.Y) * (Q.X - Q.Y)); - Residue c(P.Z * Q.Z * ring.convert(2)); - Residue d(P.XY * Q.XY * D2); - Residue x_num(a - b); // 2(x1y2+x2y1) - Residue y_num(a + b); // 2(x1x2+y1y2) - Residue x_den(c + d); // 2(1+dx1x2y1y2) - Residue y_den(c - d); // 2(1-dx1x2y1y2) - Res.X = x_num * y_den; // x = x_num/x_den, y = y_num/y_den - Res.Y = y_num * x_den; - Res.XY = x_num * y_num; - Res.Z = x_den * y_den; -} - -TwEdwardsCurve::SegrePoint TwEdwardsCurve::add_points(const SegrePoint& P, const SegrePoint& Q) const { - SegrePoint Res(ring); - add_points(Res, P, Q); - return Res; -} - -void TwEdwardsCurve::double_point(SegrePoint& Res, const SegrePoint& P) const { - add_points(Res, P, P); -} - -TwEdwardsCurve::SegrePoint TwEdwardsCurve::double_point(const SegrePoint& P) const { - SegrePoint Res(ring); - double_point(Res, P); - return Res; -} - -// computes u([n]P) in form (xy,x,y,1)*Z -TwEdwardsCurve::SegrePoint TwEdwardsCurve::power_point(const SegrePoint& A, const Bignum& n) const { - assert(n >= 0); - if (n == 0) { - return O; - } - - int k = n.num_bits(); - SegrePoint P(A); - SegrePoint Q(double_point(A)); - for (int i = k - 2; i >= 0; --i) { - if (n[i]) { - add_points(P, P, Q); - double_point(Q, Q); - } else { - // we do more operations than necessary for uniformicity - add_points(Q, P, Q); - double_point(P, P); - } - } - return P; -} - -TwEdwardsCurve::SegrePoint TwEdwardsCurve::power_gen(const Bignum& n) const { - return power_point(G, n); -} - -bool TwEdwardsCurve::SegrePoint::export_point(unsigned char buffer[32], bool need_x) const { - if (!is_normalized()) { - if (Z.is_zero()) { - std::memset(buffer, 0xff, 32); - return false; - } - Residue f(inverse(Z)); - Bignum y((Y * f).extract()); - assert(!y[255]); - if (need_x) { - y[255] = (X * f).extract().odd(); - } - y.export_lsb(buffer, 32); - } else { - Bignum y(Y.extract()); - assert(!y[255]); - if (need_x) { - y[255] = X.extract().odd(); - } - y.export_lsb(buffer, 32); - } - return true; -} - -bool TwEdwardsCurve::SegrePoint::export_point_u(unsigned char buffer[32]) const { - if (Z == Y) { - std::memset(buffer, 0xff, 32); - return false; - } - Residue f(inverse(Z - Y)); - ((Z + Y) * f).extract().export_lsb(buffer, 32); - assert(!(buffer[31] & 0x80)); - return true; -} - -TwEdwardsCurve::SegrePoint TwEdwardsCurve::import_point(const unsigned char point[32], bool& ok) const { - Bignum y; - y.import_lsb(point, 32); - bool x_sign = y[255]; - y[255] = 0; - Residue yr(y, ring); - Residue xr(ring); - ok = recover_x(xr, yr, x_sign); - return ok ? SegrePoint(xr, yr) : SegrePoint(ring); -} - -TwEdwardsCurve& Ed25519() { - static TwEdwardsCurve Ed25519(Fp25519().frac(-121665, 121666), Fp25519().frac(4, 5), Fp25519()); - static bool init = false; - if (!init) { - Ed25519.set_order_cofactor(hex_string{"80000000000000000000000000000000a6f7cef517bce6b2c09318d2e7ae9f68"}, 8); - init = true; - } - return Ed25519; -} -} // namespace ellcurve - -// ****************************************************** - -namespace openssl { -#include -} - -namespace digest { -using namespace openssl; - -struct OpensslEVP_SHA1 { - enum { digest_bytes = 20 }; - static const EVP_MD* get_evp() { - return EVP_sha1(); - } -}; - -struct OpensslEVP_SHA256 { - enum { digest_bytes = 32 }; - static const EVP_MD* get_evp() { - return EVP_sha256(); - } -}; - -struct OpensslEVP_SHA512 { - enum { digest_bytes = 64 }; - static const EVP_MD* get_evp() { - return EVP_sha512(); - } -}; - -template -class HashCtx { - EVP_MD_CTX ctx; - void init(); - void clear(); - - public: - enum { digest_bytes = H::digest_bytes }; - HashCtx() { - init(); - } - HashCtx(const void* data, std::size_t len) { - init(); - feed(data, len); - } - ~HashCtx() { - clear(); - } - void feed(const void* data, std::size_t len); - std::size_t extract(unsigned char buffer[digest_bytes]); - std::string extract(); -}; - -template -void HashCtx::init() { - EVP_MD_CTX_init(&ctx); - EVP_DigestInit_ex(&ctx, H::get_evp(), 0); -} - -template -void HashCtx::clear() { - EVP_MD_CTX_cleanup(&ctx); -} - -template -void HashCtx::feed(const void* data, std::size_t len) { - EVP_DigestUpdate(&ctx, data, len); -} - -template -std::size_t HashCtx::extract(unsigned char buffer[digest_bytes]) { - unsigned olen = 0; - EVP_DigestFinal_ex(&ctx, buffer, &olen); - assert(olen == digest_bytes); - return olen; -} - -template -std::string HashCtx::extract() { - unsigned char buffer[digest_bytes]; - unsigned olen = 0; - EVP_DigestFinal_ex(&ctx, buffer, &olen); - assert(olen == digest_bytes); - return std::string((char*)buffer, olen); -} - -typedef HashCtx SHA1; -typedef HashCtx SHA256; -typedef HashCtx SHA512; - -template -std::size_t hash_str(unsigned char buffer[T::digest_bytes], const void* data, std::size_t size) { - T hasher(data, size); - return hasher.extract(buffer); -} - -template -std::size_t hash_two_str(unsigned char buffer[T::digest_bytes], const void* data1, std::size_t size1, const void* data2, - std::size_t size2) { - T hasher(data1, size1); - hasher.feed(data2, size2); - return hasher.extract(buffer); -} - -template -std::string hash_str(const void* data, std::size_t size) { - T hasher(data, size); - return hasher.extract(); -} - -template -std::string hash_two_str(const void* data1, std::size_t size1, const void* data2, std::size_t size2) { - T hasher(data1, size1); - hasher.feed(data2, size2); - return hasher.extract(); -} -} // namespace digest - -// ****************************************************** - -namespace openssl { -#include -} - -#include -#include - -namespace prng { - -int os_get_random_bytes(void* buf, int n) { - using namespace std; - int r = 0, h = open("/dev/random", O_RDONLY | O_NONBLOCK); - if (h >= 0) { - r = read(h, buf, n); - if (r > 0) { - //std::cerr << "added " << r << " bytes of real entropy to secure random numbers seed" << std::endl; - } else { - r = 0; - } - close(h); - } - - if (r < n) { - h = open("/dev/urandom", O_RDONLY); - if (h < 0) { - return r; - } - int s = read(h, (char*)buf + r, n - r); - close(h); - if (s < 0) { - return r; - } - r += s; - } - - if (r >= 8) { - *(long*)buf ^= lrand48(); - srand48(*(long*)buf); - } - - return r; -} -} // namespace prng - -namespace prng { -using namespace openssl; - -class RandomGen { - public: - struct rand_error {}; - void randomize(bool force = true); - void seed_add(const void* data, std::size_t size, double entropy = 0); - bool ok() const { - return RAND_status(); - } - RandomGen() { - randomize(false); - } - RandomGen(const void* seed, std::size_t size) { - seed_add(seed, size); - randomize(false); - } - bool rand_bytes(void* data, std::size_t size, bool strong = false); - bool strong_rand_bytes(void* data, std::size_t size) { - return rand_bytes(data, size, true); - } - template - bool rand_obj(T& obj) { - return rand_bytes(&obj, sizeof(T)); - } - template - bool rand_objs(T* ptr, std::size_t count) { - return rand_bytes(ptr, sizeof(T) * count); - } - std::string rand_string(std::size_t size, bool strong = false); -}; - -void RandomGen::seed_add(const void* data, std::size_t size, double entropy) { - RAND_add(data, size, entropy > 0 ? entropy : size); -} - -void RandomGen::randomize(bool force) { - if (!force && ok()) { - return; - } - unsigned char buffer[128]; - int n = os_get_random_bytes(buffer, 128); - seed_add(buffer, n); - assert(ok()); -} - -bool RandomGen::rand_bytes(void* data, std::size_t size, bool strong) { - int res = (strong ? RAND_bytes : RAND_pseudo_bytes)((unsigned char*)data, size); - if (res != 0 && res != 1) { - throw rand_error(); - } - return res; -} - -std::string RandomGen::rand_string(std::size_t size, bool strong) { - char buffer[size]; - if (!rand_bytes(buffer, size, strong)) { - throw rand_error(); - } - return std::string(buffer, size); -} - -RandomGen& rand_gen() { - static RandomGen MainPRNG; - return MainPRNG; -} - -} // namespace prng - -// ****************************************************** - -namespace crypto { -namespace Ed25519 { - -const int privkey_bytes = 32; -const int pubkey_bytes = 32; -const int sign_bytes = 64; -const int shared_secret_bytes = 32; - -bool all_bytes_same(const unsigned char* str, std::size_t size) { - unsigned char c = str[0]; - for (std::size_t i = 0; i < size; i++) { - if (str[i] != c) { - return false; - } - } - return true; -} - -class PublicKey { - enum { pk_empty, pk_xz, pk_init } inited; - unsigned char pubkey[pubkey_bytes]; - ellcurve::TwEdwardsCurve::SegrePoint PubKey; - ellcurve::MontgomeryCurve::PointXZ PubKey_xz; - - public: - PublicKey() : inited(pk_empty), PubKey(ellcurve::Fp25519()), PubKey_xz(ellcurve::Fp25519()) { - } - PublicKey(const unsigned char pub_key[pubkey_bytes]); - PublicKey(const ellcurve::TwEdwardsCurve::SegrePoint& Pub_Key); - - bool import_public_key(const unsigned char pub_key[pubkey_bytes]); - bool import_public_key(const ellcurve::TwEdwardsCurve::SegrePoint& Pub_Key); - bool export_public_key(unsigned char pubkey_buffer[pubkey_bytes]) const; - bool check_message_signature(unsigned char signature[sign_bytes], const unsigned char* message, std::size_t msg_size); - - void clear(); - bool ok() const { - return inited == pk_init; - } - - const unsigned char* get_pubkey_ptr() const { - return inited == pk_init ? pubkey : 0; - } - const ellcurve::TwEdwardsCurve::SegrePoint& get_point() const { - return PubKey; - } - const ellcurve::MontgomeryCurve::PointXZ& get_point_xz() const { - return PubKey_xz; - } -}; - -void PublicKey::clear(void) { - if (inited != pk_empty) { - std::memset(pubkey, 0, pubkey_bytes); - PubKey.zeroize(); - PubKey_xz.zeroize(); - } - inited = pk_empty; -} - -PublicKey::PublicKey(const unsigned char pub_key[pubkey_bytes]) - : inited(pk_empty), PubKey(ellcurve::Fp25519()), PubKey_xz(ellcurve::Fp25519()) { - import_public_key(pub_key); -} - -PublicKey::PublicKey(const ellcurve::TwEdwardsCurve::SegrePoint& Pub_Key) - : inited(pk_empty), PubKey(ellcurve::Fp25519()), PubKey_xz(ellcurve::Fp25519()) { - import_public_key(Pub_Key); -} - -bool PublicKey::import_public_key(const unsigned char pub_key[pubkey_bytes]) { - clear(); - if (all_bytes_same(pub_key, pubkey_bytes)) { - return false; - } - bool ok = false; - PubKey = ellcurve::Ed25519().import_point(pub_key, ok); - if (!ok) { - clear(); - return false; - } - std::memcpy(pubkey, pub_key, pubkey_bytes); - PubKey_xz.X = PubKey.Z + PubKey.Y; - PubKey_xz.Z = PubKey.Z - PubKey.Y; - inited = pk_init; - return true; -} - -bool PublicKey::import_public_key(const ellcurve::TwEdwardsCurve::SegrePoint& Pub_Key) { - clear(); - if (!Pub_Key.is_valid()) { - return false; - } - PubKey = Pub_Key; - PubKey_xz.X = PubKey.Z + PubKey.Y; - PubKey_xz.Z = PubKey.Z - PubKey.Y; - inited = pk_init; - - if (!PubKey.export_point(pubkey)) { - clear(); - return false; - } - return true; -} - -bool PublicKey::export_public_key(unsigned char pubkey_buffer[pubkey_bytes]) const { - if (inited != pk_init) { - std::memset(pubkey_buffer, 0, pubkey_bytes); - return false; - } else { - std::memcpy(pubkey_buffer, pubkey, pubkey_bytes); - return true; - } -} - -bool PublicKey::check_message_signature(unsigned char signature[sign_bytes], const unsigned char* message, - std::size_t msg_size) { - if (inited != pk_init) { - return false; - } - unsigned char hash[64]; - { - digest::SHA512 hasher(signature, 32); - hasher.feed(pubkey, 32); - hasher.feed(message, msg_size); - hasher.extract(hash); - } - auto& E = ellcurve::Ed25519(); - const arith::Bignum& L = E.get_ell(); - arith::Bignum H, S; - S.import_lsb(signature + 32, 32); - H.import_lsb(hash, 64); - H %= L; - H = L - H; - auto sG = E.power_gen(S); - auto hA = E.power_point(PubKey, H); - auto pR1 = E.add_points(sG, hA); - unsigned char pR1_bytes[32]; - if (!pR1.export_point(pR1_bytes)) { - return false; - } - return !std::memcmp(pR1_bytes, signature, 32); -} - -class PrivateKey { - public: - struct priv_key_no_copy {}; - PrivateKey() : inited(false) { - std::memset(privkey, 0, privkey_bytes); - } - PrivateKey(const unsigned char pk[privkey_bytes]) : inited(false) { - std::memset(privkey, 0, privkey_bytes); - import_private_key(pk); - } - ~PrivateKey() { - clear(); - } - bool random_private_key(bool strong = false); - bool import_private_key(const unsigned char pk[privkey_bytes]); - bool export_private_key(unsigned char pk[privkey_bytes]) const; // careful! - bool export_public_key(unsigned char pubk[pubkey_bytes]) const { - return PubKey.export_public_key(pubk); - } - void clear(); - bool ok() const { - return inited; - } - - // used for EdDSA (sign) - bool sign_message(unsigned char signature[sign_bytes], const unsigned char* message, std::size_t msg_size); - // used for ECDH (encrypt / decrypt) - bool compute_shared_secret(unsigned char secret[shared_secret_bytes], const PublicKey& Pub); - // used for EC asymmetric decryption - bool compute_temp_shared_secret(unsigned char secret[shared_secret_bytes], - const unsigned char temp_pub_key[pubkey_bytes]); - - const PublicKey& get_public_key() const { - return PubKey; - } - - private: - bool inited; - unsigned char privkey[privkey_bytes]; - unsigned char priv_salt[32]; - arith::Bignum priv_exp; - PublicKey PubKey; - - bool process_private_key(); - PrivateKey(const PrivateKey&) { - throw priv_key_no_copy(); - } - PrivateKey& operator=(const PrivateKey&) { - throw priv_key_no_copy(); - } -}; - -bool PrivateKey::random_private_key(bool strong) { - inited = false; - if (!prng::rand_gen().rand_bytes(privkey, privkey_bytes, strong)) { - clear(); - return false; - } - return process_private_key(); -} - -void PrivateKey::clear(void) { - std::memset(privkey, 0, privkey_bytes); - std::memset(priv_salt, 0, sizeof(priv_salt)); - priv_exp.clear(); - PubKey.clear(); - inited = false; -} - -bool PrivateKey::import_private_key(const unsigned char pk[privkey_bytes]) { - clear(); - if (all_bytes_same(pk, privkey_bytes)) { - return false; - } - std::memcpy(privkey, pk, privkey_bytes); - return process_private_key(); -} - -bool PrivateKey::export_private_key(unsigned char pk[privkey_bytes]) const { // careful! - if (!inited) { - std::memset(pk, 0, privkey_bytes); - return false; - } else { - std::memcpy(pk, privkey, privkey_bytes); - return true; - } -} - -bool PrivateKey::process_private_key() { - unsigned char buff[64]; - digest::hash_str(buff, privkey, privkey_bytes); - std::memcpy(priv_salt, buff + 32, 32); - buff[0] &= -8; - buff[31] = ((buff[31] | 0x40) & ~0x80); - priv_exp.import_lsb(buff, 32); - PubKey = ellcurve::Ed25519().power_gen(priv_exp); - inited = PubKey.ok(); - if (!inited) { - clear(); - } - return inited; -} - -bool PrivateKey::compute_shared_secret(unsigned char secret[shared_secret_bytes], const PublicKey& Pub) { - if (!inited || !Pub.ok()) { - std::memset(secret, 0, shared_secret_bytes); - *(long*)secret = lrand48(); - return false; - } - auto P = ellcurve::Curve25519().power_xz(Pub.get_point_xz(), priv_exp); - if (P.is_infty()) { - std::memset(secret, 0, shared_secret_bytes); - *(long*)secret = lrand48(); - return false; - } - P.export_point_y(secret); - return true; -} - -bool PrivateKey::compute_temp_shared_secret(unsigned char secret[shared_secret_bytes], - const unsigned char temp_pub_key[pubkey_bytes]) { - PublicKey tempPubkey(temp_pub_key); - if (!tempPubkey.ok()) { - return false; - } - return compute_shared_secret(secret, tempPubkey); -} - -bool PrivateKey::sign_message(unsigned char signature[sign_bytes], const unsigned char* message, std::size_t msg_size) { - if (!inited) { - std::memset(signature, 0, sign_bytes); - return false; - } - unsigned char r_bytes[64]; - digest::hash_two_str(r_bytes, priv_salt, 32, message, msg_size); - const arith::Bignum& L = ellcurve::Ed25519().get_ell(); - arith::Bignum eR; - eR.import_lsb(r_bytes, 64); - eR %= L; - - auto pR = ellcurve::Ed25519().power_gen(eR); - - assert(pR.export_point(signature, 32)); - { - digest::SHA512 hasher(signature, 32); - hasher.feed(PubKey.get_pubkey_ptr(), 32); - hasher.feed(message, msg_size); - hasher.extract(r_bytes); - } - arith::Bignum S; - S.import_lsb(r_bytes, 64); - S %= L; - S *= priv_exp; - S += eR; - S %= L; - S.export_lsb(signature + 32, 32); - return true; -} - -// use one TempKeyGenerator object a lot of times -class TempKeyGenerator { - enum { salt_size = 64 }; - unsigned char random_salt[salt_size]; - unsigned char buffer[privkey_bytes]; - - public: - TempKeyGenerator() { - prng::rand_gen().strong_rand_bytes(random_salt, salt_size); - } - ~TempKeyGenerator() { - std::memset(random_salt, 0, salt_size); - std::memset(buffer, 0, privkey_bytes); - } - - unsigned char* get_temp_private_key(unsigned char* to, const unsigned char* message, std::size_t size, - const unsigned char* rand = 0, std::size_t rand_size = 0); // rand may be 0 - void create_temp_private_key(PrivateKey& pk, const unsigned char* message, std::size_t size, - const unsigned char* rand = 0, std::size_t rand_size = 0); - - // sets temp_pub_key and shared_secret for one-time asymmetric encryption of message - bool create_temp_shared_secret(unsigned char temp_pub_key[pubkey_bytes], unsigned char secret[shared_secret_bytes], - const PublicKey& recipientPubKey, const unsigned char* message, std::size_t size, - const unsigned char* rand = 0, std::size_t rand_size = 0); -}; - -unsigned char* TempKeyGenerator::get_temp_private_key(unsigned char* to, const unsigned char* message, std::size_t size, - const unsigned char* rand, - std::size_t rand_size) { // rand may be 0 - digest::SHA256 hasher(message, size); - hasher.feed(random_salt, salt_size); - if (rand && rand_size) { - hasher.feed(rand, rand_size); - } - if (!to) { - to = buffer; - } - hasher.extract(to); - //++ *((long *)random_salt); - return to; -} - -void TempKeyGenerator::create_temp_private_key(PrivateKey& pk, const unsigned char* message, std::size_t size, - const unsigned char* rand, std::size_t rand_size) { - pk.import_private_key(get_temp_private_key(buffer, message, size, rand, rand_size)); - std::memset(buffer, 0, privkey_bytes); -} - -bool TempKeyGenerator::create_temp_shared_secret(unsigned char temp_pub_key[pubkey_bytes], - unsigned char shared_secret[shared_secret_bytes], - const PublicKey& recipientPubKey, const unsigned char* message, - std::size_t size, const unsigned char* rand, std::size_t rand_size) { - PrivateKey tmpPk; - create_temp_private_key(tmpPk, message, size, rand, rand_size); - return tmpPk.export_public_key(temp_pub_key) && tmpPk.compute_shared_secret(shared_secret, recipientPubKey); -} - -} // namespace Ed25519 -} // namespace crypto - -// ****************************************************** - -void print_buffer(const unsigned char buffer[32]) { - for (int i = 0; i < 32; i++) { - char buff[4]; - sprintf(buff, "%02x", buffer[i]); - std::cout << buff; - } -} - -std::string buffer_to_hex(const unsigned char* buffer, std::size_t size = 32) { - char str[2 * size + 1]; - for (std::size_t i = 0; i < size; i++) { - sprintf(str + 2 * i, "%02x", buffer[i]); - } - return str; -} - -int main(void) { - arith::Bignum x = (3506824292LL << 31); - x = (2948877059LL << 31); - arith::Bignum L = (((36 * x + 36) * x + 18) * x + 6) * x + 1; - arith::Bignum P = L + 6 * sqr(x); - std::cout << "x= " << x << "; l= " << L << "; p= " << P << std::endl; - std::cout << "x= " << x.to_hex() << "; l= " << L.to_hex() << "; p= " << P.to_hex() << std::endl; - std::cout << "x mod 3=" << x % 3 << "; p mod 9=" << P % 9 << "; x/2^31=" << (x >> 31).to_hex() << "=" << (x >> 31) - << std::endl; - - crypto::Ed25519::PrivateKey PK1, PK2, PK3; - PK1.random_private_key(); - PK2.random_private_key(); - unsigned char priv2_export[32]; - bool ok = PK2.export_private_key(priv2_export); - std::cout << "PK2 = " << ok << " " << buffer_to_hex(priv2_export) << std::endl; - PK3.import_private_key(priv2_export); - std::cout << "PK3 = " << PK3.ok() << std::endl; - - unsigned char pub_export[32]; - ok = PK1.export_public_key(pub_export); - std::cout << "PubK1 = " << ok << " " << buffer_to_hex(pub_export) << std::endl; - crypto::Ed25519::PublicKey PubK1(pub_export); - ok = PK2.export_public_key(pub_export); - std::cout << "PubK2 = " << ok << " " << buffer_to_hex(pub_export) << std::endl; - crypto::Ed25519::PublicKey PubK2(pub_export); - ok = PK3.export_public_key(pub_export); - std::cout << "PubK3 = " << ok << " " << buffer_to_hex(pub_export) << std::endl; - crypto::Ed25519::PublicKey PubK3(pub_export); - ok = PubK1.export_public_key(pub_export); - std::cout << "PubK1 = " << ok << " " << buffer_to_hex(pub_export) << std::endl; - - unsigned char secret12[32], secret21[32]; - ok = PK1.compute_shared_secret(secret12, PK3.get_public_key()); - std::cout << "secret(PK1,PubK2)=" << ok << " " << buffer_to_hex(secret12) << std::endl; - ok = PK2.compute_shared_secret(secret21, PubK1); - std::cout << "secret(PK2,PubK1)=" << ok << " " << buffer_to_hex(secret21) << std::endl; - - unsigned char signature[64]; - ok = PK1.sign_message(signature, (const unsigned char*)"abc", 3); - std::cout << "PK1.signature=" << ok << " " << buffer_to_hex(signature) << std::endl; - - // signature[63] ^= 1; - ok = PubK1.check_message_signature(signature, (const unsigned char*)"abc", 3); - std::cout << "PubK1.check_signature=" << ok << std::endl; - - unsigned char temp_pubkey[32]; - crypto::Ed25519::TempKeyGenerator TKG; // use one generator a lot of times - - TKG.create_temp_shared_secret(temp_pubkey, secret12, PubK1, (const unsigned char*)"abc", 3); - std::cout << "secret12=" << buffer_to_hex(secret12) << "; temp_pubkey=" << buffer_to_hex(temp_pubkey) << std::endl; - - PK1.compute_temp_shared_secret(secret21, temp_pubkey); - std::cout << "secret21=" << buffer_to_hex(secret21) << std::endl; -} diff --git a/test/regression-tests.ans b/test/regression-tests.ans index 93fa16ae..14d5958b 100644 --- a/test/regression-tests.ans +++ b/test/regression-tests.ans @@ -1,16 +1,30 @@ abce -Test_Bigint_main_default 0327a04f1252c37f77b6706b902ab2c3235c47738bca3f183c837a2c5d22bb6f +Test_Bigint_main_default 76f38492ec19464a1d0eac51d389023a31ce10396b3894061361d159567ce8cd Test_Bitstrings_main_default a8b08af3116923c4c2a14e138d168375abd0c059f2f780d3267b294929a1110e -Test_Cells_simple_default 832502642fe4fe5db70de82681aedb7d54d7f3530e0069861fff405fe6f6cf23 +Test_Cells_simple_default 414f68da0a2f6fa09b2bdb99c453cdf919db48b5b4ca1c6ac1dfcd837e2d8170 Test_Fift_bug_div_default 1ac42861ce96b2896001c587f65e9afe1617db48859f19c2f4e3063a20ea60b0 Test_Fift_bug_newlize_default e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 Test_Fift_bug_ufits_default 51bf5a9f1ed7633a193f6fdd17a7a3af8e032dfe72a9669c85e8639aa8a7c195 Test_Fift_contfrac_default 09ebce5c91bcb70696c6fb6981d82dc3b9e3444dab608a7a1b044c0ddd778a96 +Test_Fift_test_adddiv_default 8516934b6fe143062864a90c76271c1d6b9b83bcb07cd09c43ac5d3e41997e6b +Test_Fift_test_asm_nested_program_default 2a19decac67adb719c444ab42879a5d894447d450d1998848c469605531076ad +Test_Fift_test_bls_default 362b67d27e3081f75a59d3d2ca5891fd000cbc061d741764924362aae4235737 +Test_Fift_test_bls_ops_default fb0a81d4f247ab806318b051d12158f7f1aacc5513db5f8cb0fdca21dbb4f9f4 +Test_Fift_test_deep_stack_ops_default df812efbadfffa8a3f553416f68c8c4435bac07266f84562cf98fe5f0dd62a52 Test_Fift_test_default 4e44b3382963ec89f7b5c8f2ebd85da3bc8aebad5b49f5b11b14075061477b4d Test_Fift_test_dict_default a9c8cbcfdece5573185022cea07f59f1bc404e5d879e5157a5745757f8ee0525 +Test_Fift_test_disasm_default 412cf37d37c5d9d81f44dbf4e3d3e7cda173c23b890614eb8a3bc5f2b92f13e6 +Test_Fift_test_fiftext_default 2b0db5d4d4bfbc705b959cc787540d7b3a21a71469eac54756e76953f0d9afca Test_Fift_test_fixed_default 278a19d56b773102caf5c1fe2997ea6c8d0d9e720eff8503feede6398a197eec +Test_Fift_test_hash_ext_default 686fc5680feca5b3bb207768215b27f6872a95128762dee0d7f2c88bc492d62d +Test_Fift_test_hmap_default c269246882039824bb5822e896c3e6e82ef8e1251b6b251f5af8ea9fb8d05067 +Test_Fift_test_levels_default 9fba4a7c98aec9000f42846d6e5fd820343ba61d68f9139dd16c88ccda757cf3 +Test_Fift_test_namespaces_default e6419619c51332fb5e8bf22043ef415db686c47fe24f03061e5ad831014e7c6c +Test_Fift_test_rist255_default f4d7558f200a656934f986145c19b1dedbe2ad029292a5a975576d6891e25fc4 +Test_Fift_test_secp256k1_default 3118450dace6af05fcdbd54a87d9446162ce11ac6ef6dfc57998cf113587d602 Test_Fift_test_sort2_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a Test_Fift_test_sort_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a +Test_Fift_test_tvm_runvm_default ff3d2a4031b543c18d6b555f0a1f1a891c7825e6d1e2e9beb4bf13b37441450b Test_Fift_testvm2_default 8a6e35fc0224398be9d2de39d31c86ea96965ef1eca2aa9e0af2303150ed4a7b Test_Fift_testvm3_default 3c1b77471c5fd914ed8b5f528b9faed618e278693f5030b953ff150e543864ae Test_Fift_testvm4_default 8a6e35fc0224398be9d2de39d31c86ea96965ef1eca2aa9e0af2303150ed4a7b @@ -25,40 +39,40 @@ Test_Fift_testvm8_default 17c9e2205ccecfd8549328b4a501d07dde0336899a7a496e747e10 Test_Fift_testvm_default ee4cbfec76c050b6de7877cfc39817d594cd1e175b6265b76fb642e30b940437 Test_Fift_testvmprog_default e5d0b2c68ee568280877c8495be558bfd0054ca5d99a99eebb525bbeca8a65af Test_RefInt_main_default 768493e0aef8e09a401a6d369edd1ef503a9215fb09dc460f52b27a8bde767cb -Test_VM_assert_code_not_null_default 05bc07e129181c972b976442f200de9487dee8bfb5ac53dd36ff61c5d4d4291d -Test_VM_assert_extract_minmax_key_default c352309c61bdf62ba7a0ba7280d303c88b0696fe7efa550c05feb2c662275297 -Test_VM_assert_lookup_prefix_default c5b45999b46d324e4008c07e5ce671bbcd833f4e15fb21a4a5136f7b980ca6fc -Test_VM_assert_pfx_dict_lookup_default fa6e3f96b31cf2ed9a9dac6b279ec05acfedf13b8ed7b815789f167d1ed7352f +Test_VM_assert_code_not_null_default 09f75cb845e0df27f3ec92405ccb4018484711a79813fd47fe8e158762c1cb93 +Test_VM_assert_extract_minmax_key_default 756337c2b2ce489243956a6608d6934ba9f76124a9435f045fc3a3b65c113d41 +Test_VM_assert_lookup_prefix_default f7683f9d2010bca53b1ef20c0e82427fb04ed62fa5fea1ee986f005ecfc9a27a +Test_VM_assert_pfx_dict_lookup_default 6d7c80d94dbc6d3ae4bafa216667b95ede4f2cbd44a59384abace84270417ef8 Test_VM_bigint_default feeb473a4ac51133989e1c145d0f49defa77117d2ae8b66bd7d12e3579e91b9f -Test_VM_bug_div_short_any_default f69aca6873f75d53dd37b6952151a2d858407a04589330762827dbc96d8b7c04 -Test_VM_bug_exec_dict_getnear_default db314c2e25b49c1f7f044d271e225f36da546c66242a8ab12f6afae37628a81e -Test_VM_bug_stack_overflow_default 7e0e3e96ca438ac96648d569c55213aa82154cf004e80265b1c481b1c4219719 -Test_VM_infinity_loop_1_default 670beda76229922806805e558d50d8f320017c642c3e7e34a7e1f2b7edb83cee -Test_VM_infinity_loop_2_default 22d9bd8cb41ff7b6cced5825e4ab73275b2fc07b1e3cd4588de815e2e6df2963 -Test_VM_memory_leak_default e10dc118f3538720a16bcbd39be9a68c3ea07f76b3d2ed5719a5e866d91f0ab3 -Test_VM_memory_leak_new_default fd2eec0a1d5ae49fb5ff8ba4b938fd9d0fe330be4a07b2b8be12bab249b00d90 -Test_VM_memory_leak_old_default f3076ae38d14000c021597b824d2f0e51de4f00601429ec3e23cca1b32dba844 -Test_VM_oom_1_default 90862ddf3270840fbc9263c003c628ddd4a8bf6548b9bd3d53eb35a5c34bc325 -Test_VM_report3_1_default 7bc6a8e66f9a0e40cd131e9829ff36fed16b464170d27c0b365a3f549df57282 -Test_VM_report3_2_default 2231bc352cf431e72a84abad2261969bd5b0ee3d9051bb7a53b69fd8ea05f951 -Test_VM_report3_3_default 9416187eb0600ed247795837ca820bccaffb39841bd9d2ff625816bfbba35d6d -Test_VM_report3_4_default 11661eb00ea37c68e3483a8e048f922f73070c6da8219247663e3d6471c5c0cc -Test_VM_report3_6_default 1d7be98a8b07f803e80168247459e620ce4b91df634ad896e878d21a3ed757c0 -Test_VM_report3_int_overflow_1_default a0c2414ca2c9672d54409ee375a6aad6e2233306eaa3dfd33a82de3c90bba3ba -Test_VM_report3_int_overflow_2_default 01cd461802e532a6830705ad50eaa1760278737ff7beeb654e3c59ceb4aa8e2e -Test_VM_report3_loop_1_default b28b35d057a1b4fa2282d6f422ecd822b18cc4344733d923ef7b002f64bc4d72 -Test_VM_report3_loop_2_default 9f8236535902b04e403d412fcf1f79e64d0f2eb25b3cc014b7d61b2d7a34b9ef -Test_VM_report3_loop_3_default 7ee05ea553c48a2476035817b9d860f614a355927c9e011b2f824dc6e5f7b0cf -Test_VM_report3_loop_4_default 4b6c2f65fda3c9e9c6660b6cbbcb1b2103c5b52870cb5daa8876bbed0ca9bbc9 -Test_VM_report3_loop_5_default 0d5d504884172ef8513757d7f6b2a3870dbd28efd5960857441c032e1c67d836 -Test_VM_report3_loop_6_default 5c35b92144debdb61b2020d690669bffbdd96f75ecde827fd0c75c05da22b5a0 -Test_VM_report3_qnot_default dc280444c7e3886cc3412f96f44c803c45287a07fcb9c638643e21bcdfe3905d -Test_VM_simple_default f6733549069427c2beb1a85ee25635540e27aa68cb8ad101d8435e19afeae862 -Test_VM_unhandled_exception_1_default 0abe2740dd3b6a6b91eb67fee573f638086fecc72653d2d81c956782186b5d78 -Test_VM_unhandled_exception_2_default 5ca67db5a0e957cc106bb47b744177ca959632a352f3629df376c34cbf03d51b -Test_VM_unhandled_exception_3_default b354e897e35a1177fd66d2c6ad7d77ae33a4e18f8678a518d79fea1388853307 -Test_VM_unhandled_exception_4_default 412cbfe13745fde55cdcc5e41d7b15ba4d09f0e723f8e4421ae0b7066ca07b8f -Test_VM_unhandled_exception_5_default d760e540cd9c200c207f71c540cdaf06d11c96e32ec19860b9c1046cb1e38855 +Test_VM_bug_div_short_any_default 49c9588b2b25b08979016f8b7ca42ae9fa4904a1dc6a2093a7dae6dce0cdf42f +Test_VM_bug_exec_dict_getnear_default 0b0cb6c1fef773f8b5a4aab8d575ba941f3b85dd449f85051296954028e59781 +Test_VM_bug_stack_overflow_default 31950eb2ed62bd1ce1c18e0031a81390ff3a3feee61ff23a09181995917137d0 +Test_VM_infinity_loop_1_default 6b8cc0ff85efa6882ffdf6e9e4333967976a29c4ce32a25b42c4c53370ad3024 +Test_VM_infinity_loop_2_default 4be08957dc86dfde3dfadd8c2f961ef2f1fa839788bbf7affea754115cee9e18 +Test_VM_memory_leak_default da588f89f3bc3ef7496bbab61e2d993f0c84bba80bb28d9c20c6eac0f7f57dd3 +Test_VM_memory_leak_new_default d25e8602c88c454ded6271d0f7afbc556820cc9942c56de9e0bd95b329f8783e +Test_VM_memory_leak_old_default 563f5a02130f231823099985c77d09913db07d2d8782edf431822f6afe4911b8 +Test_VM_oom_1_default 354934173c82e4f7bca5063846abd35cb47c4fcf1c3ba8c2fd04a4b186fcbf18 +Test_VM_report3_1_default 26bb43b5100e94791911a66226ec6545422749e0ea9e6279983b00ef4b506601 +Test_VM_report3_2_default 07a84726217f362fd71b3ceab96112ffc7aa40ed44636cf0128205d85798c66c +Test_VM_report3_3_default 0d9a92491c88ada92283691debd123724db2b7c1bd345200a53c057032e9cc07 +Test_VM_report3_4_default eb23c8e1219aed91b1b4f519efcac87018a5cf8e0ce473cfa641f8221f3c5d20 +Test_VM_report3_6_default 769ce8f9bb6fb9b8619afdb8e9d621b6199466f07c37eeea8edf3c21bf05a101 +Test_VM_report3_int_overflow_1_default 7aaf32ec7ace54b93d6b55f3ac9642572f348ebd64412afdda24849f8e4eb1dc +Test_VM_report3_int_overflow_2_default 572d197681654c94951280448ea3cf448613391633383c2424d719b03b6ec939 +Test_VM_report3_loop_1_default c9b00b32a024c65e0a8019c86e94ee365a823ff26e2420e1797902841abab57c +Test_VM_report3_loop_2_default 3654949987ddb44d8e11e84fe907d43707eaed910b9d0ad15dd68b531df1444a +Test_VM_report3_loop_3_default c1fc7e0d160b334fe8a4735a2a9d36c3b10530edaad5c1859df88382ff82a2d9 +Test_VM_report3_loop_4_default 5ad7cc51a6e553ee3d4a427229908a51692e117624838190311c7023df7a5e5b +Test_VM_report3_loop_5_default 068f81caecc344132a601259d9f73eea7089b1399793661ba1954483e0d5682c +Test_VM_report3_loop_6_default bf2e45709fceeed0192ec34af618cba3b85b90f71071e018afba686167618a90 +Test_VM_report3_qnot_default 7fcbda7e3fc4853a36e6b02e9d346f039690b1879d40850f561ea4123452d3ec +Test_VM_simple_default c96d70ea853828c89cd38fcf22543289335f3086a53301a1d0f186753ba9975b +Test_VM_unhandled_exception_1_default 80fe0e4c2ae19ae73e67e4355548d0afa59ea286be2d75a91db4529618dba008 +Test_VM_unhandled_exception_2_default 1362ba3a6ddbf5a30aba07ad58e8c24b0f85bdc53525e3eaa27af7248c62525a +Test_VM_unhandled_exception_3_default e381ce751cbd0e2994d7f60df7746b9ed7783198cfbcb31dccf02fafe68b6733 +Test_VM_unhandled_exception_4_default 51dd501ec0514f3b388145761b252f09d6ef3e831ea450605ae30511688dd708 +Test_VM_unhandled_exception_5_default 8231cfe1fb6ce6107b592f2c8f6a4eae0d123fc399163c81e8e0d5228b68bc91 Test_base64_main_default e90d541bd810871c4a81e162f1fffb555024b72807cb895414d16bc11494b789 Test_bigexp_main_default 45a1f51fb2abcc1ebf8569e1a57bebee04c334a15e03535ff5869bc9a9db8956 Test_bits256_scan_main_default 3ec7434e1cabc8e08eb2e79064e67747ffbfed177473c7873b88c144a7ed6f42 diff --git a/test/test-adnl.cpp b/test/test-adnl.cpp index d9ae4abe..45011fbc 100644 --- a/test/test-adnl.cpp +++ b/test/test-adnl.cpp @@ -52,7 +52,7 @@ int main() { td::to_integer_safe("0").ensure(); - std::string db_root_ = "tmp-ee"; + std::string db_root_ = "tmp-dir-test-adnl"; td::rmrf(db_root_).ignore(); td::mkdir(db_root_).ensure(); @@ -225,13 +225,19 @@ int main() { auto f = td::Clocks::system(); scheduler.run_in_context([&] { - for (td::uint32 i = 1; i <= ton::adnl::Adnl::huge_packet_max_size(); i++) { + // Don't send too many packets + // Channels are disabled, so packet rate is limited + for (td::uint32 i : {1, 2, 3, 4, 100, 500, 900}) { + remaining++; + td::actor::send_closure(adnl, &ton::adnl::Adnl::send_message, src, dst, send_packet(i)); + } + for (td::uint32 i = 1024; i <= ton::adnl::Adnl::huge_packet_max_size() /* 1024 * 8 */; i += 1024) { remaining++; td::actor::send_closure(adnl, &ton::adnl::Adnl::send_message, src, dst, send_packet(i)); } }); - auto t = td::Timestamp::in(320.0); + auto t = td::Timestamp::in(60.0); while (scheduler.run(1)) { if (!remaining) { break; @@ -241,7 +247,7 @@ int main() { } } - LOG(ERROR) << "successfully tested delivering of packets of all sizes. Time=" << (td::Clocks::system() - f); + LOG(ERROR) << "successfully tested delivering of packets of various sizes. Time=" << (td::Clocks::system() - f); scheduler.run_in_context([&] { td::actor::send_closure(network_manager, &ton::adnl::TestLoopbackNetworkManager::add_node_id, src, true, true); diff --git a/test/test-catchain.cpp b/test/test-catchain.cpp index 53b32924..3131c2b9 100644 --- a/test/test-catchain.cpp +++ b/test/test-catchain.cpp @@ -186,6 +186,7 @@ class CatChainInst : public td::actor::Actor { void create_fork() { auto height = height_ - 1; //td::Random::fast(0, height_ - 1); + LOG(WARNING) << "Creating fork, source_id=" << idx_ << ", height=" << height; auto sum = prev_values_[height] + 1; td::uint64 x[2]; @@ -219,7 +220,7 @@ int main(int argc, char *argv[]) { SET_VERBOSITY_LEVEL(verbosity_INFO); td::set_default_failure_signal_handler().ensure(); - std::string db_root_ = "tmp-ee"; + std::string db_root_ = "tmp-dir-test-catchain"; td::rmrf(db_root_).ignore(); td::mkdir(db_root_).ensure(); @@ -241,7 +242,8 @@ int main(int argc, char *argv[]) { td::actor::send_closure(adnl, &ton::adnl::Adnl::register_network_manager, network_manager.get()); }); - for (td::uint32 att = 0; att < 10; att++) { + for (td::uint32 att = 0; att < 20; att++) { + LOG(WARNING) << "Test #" << att; nodes.resize(total_nodes); scheduler.run_in_context([&] { @@ -274,8 +276,6 @@ int main(int argc, char *argv[]) { } }); - auto t = td::Timestamp::in(1.0); - ton::catchain::CatChainSessionId unique_id; td::Random::secure_bytes(unique_id.as_slice()); @@ -287,7 +287,7 @@ int main(int argc, char *argv[]) { } }); - t = td::Timestamp::in(10.0); + auto t = td::Timestamp::in(10.0); while (scheduler.run(1)) { if (t.is_in_past()) { break; @@ -298,9 +298,12 @@ int main(int argc, char *argv[]) { std::cout << "value=" << n.get_actor_unsafe().value() << std::endl; } - scheduler.run_in_context([&] { td::actor::send_closure(inst[0], &CatChainInst::create_fork); }); + td::uint32 fork_cnt = att < 10 ? 1 : (att - 10) / 5 + 2; + for (td::uint32 idx = 0; idx < fork_cnt; ++idx) { + scheduler.run_in_context([&] { td::actor::send_closure(inst[idx], &CatChainInst::create_fork); }); + } - t = td::Timestamp::in(10.0); + t = td::Timestamp::in(1.0); while (scheduler.run(1)) { if (t.is_in_past()) { break; diff --git a/test/test-dht.cpp b/test/test-dht.cpp index 2391fd9c..8d814f6e 100644 --- a/test/test-dht.cpp +++ b/test/test-dht.cpp @@ -41,7 +41,7 @@ int main() { SET_VERBOSITY_LEVEL(verbosity_INFO); - std::string db_root_ = "tmp-ee"; + std::string db_root_ = "tmp-dir-test-dht"; td::rmrf(db_root_).ignore(); td::mkdir(db_root_).ensure(); diff --git a/test/test-ext-client.cpp b/test/test-ext-client.cpp deleted file mode 100644 index a1187d78..00000000 --- a/test/test-ext-client.cpp +++ /dev/null @@ -1,220 +0,0 @@ -/* - This file is part of TON Blockchain source code. - - TON Blockchain is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - TON Blockchain is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with TON Blockchain. If not, see . - - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement - from all source files in the program, then also delete it here. - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "adnl/adnl.h" -#include "adnl/utils.hpp" -#include "auto/tl/ton_api_json.h" -#include "dht/dht.h" -#include "overlay/overlays.h" -#include "td/utils/OptionParser.h" -#include "td/utils/Time.h" -#include "td/utils/filesystem.h" -#include "td/utils/format.h" -#include "td/utils/Random.h" -#include "td/utils/port/signals.h" -#include "td/utils/port/FileFd.h" -#include "adnl/adnl-ext-client.h" - -#if TD_DARWIN || TD_LINUX -#include -#endif -#include -#include - -template -std::ostream &operator<<(std::ostream &stream, const td::UInt &x) { - for (size_t i = 0; i < size / 8; i++) { - stream << td::format::hex_digit((x.raw[i] >> 4) & 15) << td::format::hex_digit(x.raw[i] & 15); - } - - return stream; -} - -class TestNode : public td::actor::Actor { - private: - std::string local_config_ = "ton-local.config"; - std::string global_config_ = "ton-global.config"; - - td::actor::ActorOwn client_; - - std::unique_ptr make_callback() { - class Callback : public ton::adnl::AdnlExtClient::Callback { - public: - void on_ready() override { - td::actor::send_closure(id_, &TestNode::conn_ready); - } - void on_stop_ready() override { - td::actor::send_closure(id_, &TestNode::conn_closed); - } - Callback(td::actor::ActorId id) : id_(std::move(id)) { - } - - private: - td::actor::ActorId id_; - }; - - return std::make_unique(actor_id(this)); - } - - bool ready_ = false; - std::string db_root_; - - public: - void conn_ready() { - LOG(ERROR) << "conn ready"; - ready_ = true; - } - void conn_closed() { - ready_ = false; - } - void set_local_config(std::string str) { - local_config_ = str; - } - void set_global_config(std::string str) { - global_config_ = str; - } - void set_db_root(std::string db_root) { - db_root_ = db_root; - } - void start_up() override { - } - void alarm() override { - if (ready_ && !client_.empty()) { - LOG(ERROR) << "sending query"; - auto P = td::PromiseCreator::lambda([](td::Result R) { - if (R.is_error()) { - LOG(ERROR) << "failed query: " << R.move_as_error(); - return; - } - auto F = ton::fetch_tl_object(R.move_as_ok(), true); - if (F.is_error()) { - LOG(ERROR) << "failed to pasrse answer: " << F.move_as_error(); - return; - } - auto obj = F.move_as_ok(); - LOG(ERROR) << "got answer: " << ton::ton_api::to_string(obj); - }); - td::BufferSlice b = ton::serialize_tl_object(ton::create_tl_object(), true); - td::actor::send_closure(client_, &ton::adnl::AdnlExtClient::send_query, "query", std::move(b), - td::Timestamp::in(10.0), std::move(P)); - } - alarm_timestamp() = td::Timestamp::in(2.0); - } - TestNode() { - } - void run() { - auto L = td::read_file(local_config_).move_as_ok(); - auto lc_j = td::json_decode(L.as_slice()).move_as_ok(); - ton::ton_api::config_local lc; - ton::ton_api::from_json(lc, lc_j.get_object()).ensure(); - - auto G = td::read_file(global_config_).move_as_ok(); - auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); - ton::ton_api::config_global gc; - ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); - - CHECK(gc.liteclients_.size() > 0); - auto &cli = gc.liteclients_[0]; - td::IPAddress addr; - addr.init_host_port(td::IPAddress::ipv4_to_str(cli->ip_), cli->port_).ensure(); - - client_ = ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull::create(cli->id_).move_as_ok(), addr, - make_callback()); - alarm_timestamp() = td::Timestamp::in(2.0); - } -}; - -td::Result get_uint256(std::string str) { - if (str.size() != 64) { - return td::Status::Error("uint256 must have 64 bytes"); - } - td::UInt256 res; - for (size_t i = 0; i < 32; i++) { - res.raw[i] = static_cast(td::hex_to_int(str[2 * i]) * 16 + td::hex_to_int(str[2 * i + 1])); - } - return res; -} - -int main(int argc, char *argv[]) { - SET_VERBOSITY_LEVEL(verbosity_DEBUG); - td::set_default_failure_signal_handler().ensure(); - - td::actor::ActorOwn x; - - td::OptionParser p; - p.set_description("test basic adnl functionality"); - p.add_option('h', "help", "prints_help", [&]() { - char b[10240]; - td::StringBuilder sb(td::MutableSlice{b, 10000}); - sb << p; - std::cout << sb.as_cslice().c_str(); - std::exit(2); - return td::Status::OK(); - }); - p.add_option('C', "global-config", "file to read global config", [&](td::Slice fname) { - td::actor::send_closure(x, &TestNode::set_global_config, fname.str()); - return td::Status::OK(); - }); - p.add_option('c', "local-config", "file to read local config", [&](td::Slice fname) { - td::actor::send_closure(x, &TestNode::set_local_config, fname.str()); - return td::Status::OK(); - }); - p.add_option('D', "db", "root for dbs", [&](td::Slice fname) { - td::actor::send_closure(x, &TestNode::set_db_root, fname.str()); - return td::Status::OK(); - }); - p.add_option('d', "daemonize", "set SIGHUP", [&]() { - td::set_signal_handler(td::SignalType::HangUp, [](int sig) { -#if TD_DARWIN || TD_LINUX - close(0); - setsid(); -#endif - }).ensure(); - return td::Status::OK(); - }); -#if TD_DARWIN || TD_LINUX - p.add_option('l', "logname", "log to file", [&](td::Slice fname) { - auto FileLog = td::FileFd::open(td::CSlice(fname.str().c_str()), - td::FileFd::Flags::Create | td::FileFd::Flags::Append | td::FileFd::Flags::Write) - .move_as_ok(); - - dup2(FileLog.get_native_fd().fd(), 1); - dup2(FileLog.get_native_fd().fd(), 2); - return td::Status::OK(); - }); -#endif - - td::actor::Scheduler scheduler({2}); - - scheduler.run_in_context([&] { x = td::actor::create_actor("testnode"); }); - - scheduler.run_in_context([&] { p.run(argc, argv).ensure(); }); - scheduler.run_in_context([&] { td::actor::send_closure(x, &TestNode::run); }); - scheduler.run(); - - return 0; -} diff --git a/test/test-ext-server.cpp b/test/test-ext-server.cpp deleted file mode 100644 index b4b78728..00000000 --- a/test/test-ext-server.cpp +++ /dev/null @@ -1,221 +0,0 @@ -/* - This file is part of TON Blockchain source code. - - TON Blockchain is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - TON Blockchain is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with TON Blockchain. If not, see . - - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement - from all source files in the program, then also delete it here. - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "adnl/adnl.h" -#include "adnl/utils.hpp" -#include "auto/tl/ton_api_json.h" -#include "dht/dht.h" -#include "overlay/overlays.h" -#include "td/utils/OptionParser.h" -#include "td/utils/Time.h" -#include "td/utils/filesystem.h" -#include "td/utils/format.h" -#include "td/utils/Random.h" -#include "td/utils/port/signals.h" -#include "td/utils/port/FileFd.h" - -#if TD_DARWIN || TD_LINUX -#include -#endif -#include -#include - -template -std::ostream &operator<<(std::ostream &stream, const td::UInt &x) { - for (size_t i = 0; i < size / 8; i++) { - stream << td::format::hex_digit((x.raw[i] >> 4) & 15) << td::format::hex_digit(x.raw[i] & 15); - } - - return stream; -} - -class TestNode : public td::actor::Actor { - private: - td::actor::ActorOwn keyring_; - td::actor::ActorOwn adnl_; - - std::string local_config_ = "ton-local.config"; - std::string global_config_ = "ton-global.config"; - - std::unique_ptr make_callback() { - class Callback : public ton::adnl::Adnl::Callback { - public: - void receive_message(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, - td::BufferSlice data) override { - td::actor::send_closure(id_, &TestNode::adnl_receive_message, src, dst, std::move(data)); - } - void receive_query(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, - td::Promise promise) override { - td::actor::send_closure(id_, &TestNode::adnl_receive_query, src, dst, std::move(data), std::move(promise)); - } - Callback(td::actor::ActorId id) : id_(std::move(id)) { - } - - private: - td::actor::ActorId id_; - }; - - return std::make_unique(actor_id(this)); - } - - std::string db_root_; - - public: - void adnl_receive_message(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data) { - LOG(ERROR) << "ADNL MESSAGE FROM " << src << ": size=" << data.size() << "\n"; - } - - void adnl_receive_query(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, - td::Promise promise) { - LOG(ERROR) << "ADNL QUERY FROM " << src << ": size=" << data.size() << "\n"; - promise.set_value(ton::serialize_tl_object(ton::create_tl_object("xxx"), true)); - } - void set_local_config(std::string str) { - local_config_ = str; - } - void set_global_config(std::string str) { - global_config_ = str; - } - void set_db_root(std::string db_root) { - db_root_ = db_root; - } - void start_up() override { - } - void alarm() override { - } - TestNode() { - } - void run() { - keyring_ = ton::keyring::Keyring::create(db_root_ + "/keyring/"); - adnl_ = ton::adnl::Adnl::create(db_root_, keyring_.get()); - - auto L = td::read_file(local_config_).move_as_ok(); - auto lc_j = td::json_decode(L.as_slice()).move_as_ok(); - ton::ton_api::config_local lc; - ton::ton_api::from_json(lc, lc_j.get_object()).ensure(); - - auto G = td::read_file(global_config_).move_as_ok(); - auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); - ton::ton_api::config_global gc; - ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); - - for (auto &port : lc.udp_ports_) { - td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_listening_udp_port, "0.0.0.0", - static_cast(port)); - } - - //td::actor::send_closure(network_manager_, &ton::adnl::AdnlNetworkManager::load_local_config, std::move(lc.net_)); - //td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_ids_from_config, std::move(lc.local_ids_)); - if (gc.adnl_) { - td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_static_nodes_from_config, - std::move(gc.adnl_->static_nodes_)); - } - for (auto &x : lc.liteservers_) { - auto pk = ton::PrivateKey{x->id_}; - auto pub_k = ton::adnl::AdnlNodeIdFull{pk.compute_public_key()}; - auto id = pub_k.compute_short_id(); - - td::actor::send_closure(keyring_, &ton::keyring::Keyring::add_key, std::move(pk), false); - td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_id, pub_k, ton::adnl::AdnlAddressList{}); - td::actor::send_closure(adnl_, &ton::adnl::Adnl::subscribe, id, - ton::adnl::Adnl::int_to_bytestring(ton::ton_api::getTestObject::ID), make_callback()); - td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_ext_local_id, id); - td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_ext_tcp_port, static_cast(x->port_)); - } - } -}; - -td::Result get_uint256(std::string str) { - if (str.size() != 64) { - return td::Status::Error("uint256 must have 64 bytes"); - } - td::UInt256 res; - for (size_t i = 0; i < 32; i++) { - res.raw[i] = static_cast(td::hex_to_int(str[2 * i]) * 16 + td::hex_to_int(str[2 * i + 1])); - } - return res; -} - -int main(int argc, char *argv[]) { - SET_VERBOSITY_LEVEL(verbosity_DEBUG); - td::set_default_failure_signal_handler().ensure(); - - td::actor::ActorOwn x; - - td::OptionParser p; - p.set_description("test basic adnl functionality"); - p.add_option('h', "help", "prints_help", [&]() { - char b[10240]; - td::StringBuilder sb(td::MutableSlice{b, 10000}); - sb << p; - std::cout << sb.as_cslice().c_str(); - std::exit(2); - return td::Status::OK(); - }); - p.add_option('C', "global-config", "file to read global config", [&](td::Slice fname) { - td::actor::send_closure(x, &TestNode::set_global_config, fname.str()); - return td::Status::OK(); - }); - p.add_option('c', "local-config", "file to read local config", [&](td::Slice fname) { - td::actor::send_closure(x, &TestNode::set_local_config, fname.str()); - return td::Status::OK(); - }); - p.add_option('D', "db", "root for dbs", [&](td::Slice fname) { - td::actor::send_closure(x, &TestNode::set_db_root, fname.str()); - return td::Status::OK(); - }); - p.add_option('d', "daemonize", "set SIGHUP", [&]() { - td::set_signal_handler(td::SignalType::HangUp, [](int sig) { -#if TD_DARWIN || TD_LINUX - close(0); - setsid(); -#endif - }).ensure(); - return td::Status::OK(); - }); -#if TD_DARWIN || TD_LINUX - p.add_option('l', "logname", "log to file", [&](td::Slice fname) { - auto FileLog = td::FileFd::open(td::CSlice(fname.str().c_str()), - td::FileFd::Flags::Create | td::FileFd::Flags::Append | td::FileFd::Flags::Write) - .move_as_ok(); - - dup2(FileLog.get_native_fd().fd(), 1); - dup2(FileLog.get_native_fd().fd(), 2); - return td::Status::OK(); - }); -#endif - - td::actor::Scheduler scheduler({2}); - - scheduler.run_in_context([&] { x = td::actor::create_actor("testnode"); }); - - scheduler.run_in_context([&] { p.run(argc, argv).ensure(); }); - scheduler.run_in_context([&] { td::actor::send_closure(x, &TestNode::run); }); - scheduler.run(); - - return 0; -} diff --git a/test/test-node.cpp b/test/test-node.cpp deleted file mode 100644 index d771d3ae..00000000 --- a/test/test-node.cpp +++ /dev/null @@ -1,376 +0,0 @@ -/* - This file is part of TON Blockchain source code. - - TON Blockchain is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - TON Blockchain is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with TON Blockchain. If not, see . - - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement - from all source files in the program, then also delete it here. - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "adnl/adnl.h" -#include "adnl/utils.hpp" -#include "auto/tl/ton_api_json.h" -#include "dht/dht.h" -#include "overlay/overlays.h" -#include "td/utils/OptionParser.h" -#include "td/utils/Time.h" -#include "td/utils/filesystem.h" -#include "td/utils/format.h" -#include "td/utils/Random.h" -#include "td/utils/port/signals.h" -#include "td/utils/port/FileFd.h" -#include "catchain/catchain.h" - -#include "crypto/common/refvector.hpp" - -#if TD_DARWIN || TD_LINUX -#include -#endif -#include -#include - -template -std::ostream &operator<<(std::ostream &stream, const td::UInt &x) { - for (size_t i = 0; i < size / 8; i++) { - stream << td::format::hex_digit((x.raw[i] >> 4) & 15) << td::format::hex_digit(x.raw[i] & 15); - } - - return stream; -} - -class TestNode : public td::actor::Actor { - private: - std::vector ping_ids_; - td::Timestamp next_dht_dump_; - - td::actor::ActorOwn adnl_; - std::vector> dht_nodes_; - td::actor::ActorOwn overlay_manager_; - std::vector> overlays_; - std::vector> catchains_; - - std::string local_config_ = "ton-local.config"; - std::string global_config_ = "ton-global.config"; - - td::int32 broadcast_size_ = 100; - - void receive_message(td::UInt256 src, td::UInt256 dst, td::BufferSlice data) { - LOG(ERROR) << "MESSAGE FROM " << src << " to " << dst << " of size " << std::to_string(data.size()) << "\n"; - } - - void receive_broadcast(td::UInt256 overlay_id, td::BufferSlice data) { - LOG(ERROR) << "BROADCAST IN " << overlay_id << " hash=" << td::sha256(data.as_slice()) << "\n"; - } - - void receive_query(td::UInt256 src, td::UInt256 dst, td::BufferSlice data, td::Promise promise) { - auto Q = ton::fetch_tl_object(std::move(data), true); - CHECK(Q.is_ok()); - auto R = Q.move_as_ok(); - LOG(ERROR) << "QUERY " - << " FROM " << src << " to " << dst << ": " << ton::ton_api::to_string(R) << "\n"; - promise.set_value(serialize_tl_object(ton::create_tl_object(), true)); - } - - void catchain_new_block(td::UInt256 src, td::uint64 height, td::BufferSlice data) { - LOG(ERROR) << "CATCHAIN BLOCK: " << src << "@" << height << ": " << td::sha256_uint256(data.as_slice()) << "\n"; - } - void catchain_bad_block(td::UInt256 src) { - LOG(ERROR) << "CATCHAIN BAD BLOCK\n"; - } - void catchain_broadcast(td::BufferSlice data) { - LOG(ERROR) << "CATCHAIN BROADCAST " << td::sha256_uint256(data.as_slice()) << "\n"; - } - - std::unique_ptr make_callback() { - class Callback : public ton::adnl::Adnl::Callback { - public: - void receive_message(td::UInt256 src, td::UInt256 dst, td::BufferSlice data) override { - td::actor::send_closure(id_, &TestNode::receive_message, src, dst, std::move(data)); - } - void receive_query(td::UInt256 src, td::UInt256 dst, td::BufferSlice data, - td::Promise promise) override { - td::actor::send_closure(id_, &TestNode::receive_query, src, dst, std::move(data), std::move(promise)); - } - Callback(td::actor::ActorId id) : id_(std::move(id)) { - } - - private: - td::actor::ActorId id_; - }; - - return std::make_unique(actor_id(this)); - } - - std::unique_ptr make_catchain_callback() { - class Callback : public ton::CatChainActor::Callback { - public: - void new_block(td::UInt256 src, td::uint64 height, td::BufferSlice data) override { - td::actor::send_closure(id_, &TestNode::catchain_new_block, src, height, std::move(data)); - } - void bad_block(td::UInt256 src) override { - td::actor::send_closure(id_, &TestNode::catchain_bad_block, src); - } - void broadcast(td::BufferSlice data) override { - td::actor::send_closure(id_, &TestNode::catchain_broadcast, std::move(data)); - } - Callback(td::actor::ActorId id) : id_(std::move(id)) { - } - - private: - td::actor::ActorId id_; - }; - - return std::make_unique(actor_id(this)); - } - - std::unique_ptr make_overlay_callback() { - class Callback : public ton::overlay::Overlays::Callback { - public: - void receive_message(td::UInt256 src, td::UInt256 overlay_id, td::BufferSlice data) override { - } - void receive_query(td::UInt256 src, td::uint64 query_id, td::UInt256 overlay_id, td::BufferSlice data) override { - } - - void receive_broadcast(td::UInt256 overlay_id, td::BufferSlice data) override { - td::actor::send_closure(id_, &TestNode::receive_broadcast, overlay_id, std::move(data)); - } - Callback(td::actor::ActorId id) : id_(std::move(id)) { - } - - private: - td::actor::ActorId id_; - }; - - return std::make_unique(actor_id(this)); - } - - public: - void set_broadcast_size(td::int32 size) { - broadcast_size_ = size; - } - void set_local_config(std::string str) { - local_config_ = str; - } - void set_global_config(std::string str) { - global_config_ = str; - } - void start_up() override { - alarm_timestamp() = td::Timestamp::in(1); - } - void alarm() override { - /*if (overlays_.size() > 0 && broadcast_size_ > 0) { - td::BufferSlice s(broadcast_size_); - td::Random::secure_bytes(s.as_slice()); - - td::actor::send_closure(overlay_manager_, &ton::overlay::OverlayManager::send_broadcast_fer, overlays_[0].first, - overlays_[0].second, ton::create_tl_object(s.as_slice().str())); - }*/ - for (auto &chain : catchains_) { - td::BufferSlice s(broadcast_size_); - td::Random::secure_bytes(s.as_slice()); - - td::actor::send_closure(chain, &ton::CatChainActor::add_event, std::move(s)); - } - alarm_timestamp() = td::Timestamp::in(1.0); - if (next_dht_dump_.is_in_past()) { - /*for (auto &node : dht_nodes_) { - char b[10240]; - td::StringBuilder sb({b, 10000}); - node->get_actor_unsafe().dump(sb); - LOG(DEBUG) << sb.as_cslice().c_str(); - }*/ - next_dht_dump_ = td::Timestamp::in(60.0); - } - } - TestNode() { - adnl_ = ton::adnl::Adnl::create("/var/ton-work/db.adnl"); - } - void run() { - auto L = td::read_file(local_config_).move_as_ok(); - auto lc_j = td::json_decode(L.as_slice()).move_as_ok(); - ton::ton_api::config_local lc; - ton::ton_api::from_json(lc, lc_j.get_object()).ensure(); - - auto G = td::read_file(global_config_).move_as_ok(); - auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); - ton::ton_api::config_global gc; - ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); - - for (auto &port : lc.udp_ports_) { - td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_listening_udp_port, "0.0.0.0", - static_cast(port)); - } - /*if (!lc.net_) { - LOG(FATAL) << "local config does not contain NET section"; - }*/ - - //td::actor::send_closure(network_manager_, &ton::adnl::AdnlNetworkManager::load_local_config, std::move(lc.net_)); - td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_ids_from_config, std::move(lc.local_ids_)); - if (gc.adnl_) { - td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_static_nodes_from_config, - std::move(gc.adnl_->static_nodes_)); - } - if (!gc.dht_) { - LOG(FATAL) << "global config does not contain dht section"; - } - - for (auto &it : lc.dht_) { - if (it->get_id() == ton::ton_api::dht_config_local::ID) { - auto R = ton::dht::Dht::create_from_json( - ton::clone_tl_object(gc.dht_), ton::move_tl_object_as(it), adnl_.get()); - if (R.is_error()) { - LOG(FATAL) << "fail creating dht node: " << R.move_as_error(); - } - dht_nodes_.push_back(R.move_as_ok()); - } else { - auto I = ton::move_tl_object_as(it); - for (int i = 0; i < I->cnt_; i++) { - auto R = ton::dht::Dht::create_random(ton::clone_tl_object(gc.dht_), ton::clone_tl_object(I->addr_list_), - adnl_.get()); - if (R.is_error()) { - LOG(FATAL) << "fail creating dht node: " << R.move_as_error(); - } - dht_nodes_.push_back(R.move_as_ok()); - } - } - } - - CHECK(dht_nodes_.size() > 0); - - td::actor::send_closure(adnl_, &ton::adnl::Adnl::register_dht_node, dht_nodes_[0].get()); - //td::actor::send_closure(overlay_manager_, &ton::overlay::Overlays::register_dht_node, dht_nodes_[0].get()); - - overlay_manager_ = ton::overlay::Overlays::create(adnl_.get(), dht_nodes_[0].get()); - - for (auto &it : lc.public_overlays_) { - if (it->get_id() == ton::ton_api::overlay_config_local::ID) { - auto X = ton::move_tl_object_as(it); - auto id = ton::create_tl_object(X->name_.clone()); - auto Id = ton::move_tl_object_as(id); - auto sid = ton::adnl_short_id(Id); - overlays_.emplace_back(X->id_->id_, sid); - td::actor::send_closure(overlay_manager_, &ton::overlay::Overlays::create_public_overlay, X->id_->id_, - std::move(Id), make_overlay_callback()); - } else { - auto X = ton::move_tl_object_as(it); - for (int i = 0; i < X->cnt_; i++) { - auto pk = ton::adnl_generate_random_pk(); - auto local_id = ton::adnl_short_id(ton::get_public_key(pk)); - - td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_id, std::move(pk), ton::clone_tl_object(X->addr_list_)); - - auto id = ton::create_tl_object(X->name_.clone()); - auto Id = ton::move_tl_object_as(id); - auto sid = ton::adnl_short_id(Id); - overlays_.emplace_back(local_id, sid); - td::actor::send_closure(overlay_manager_, &ton::overlay::Overlays::create_public_overlay, local_id, - std::move(Id), make_overlay_callback()); - } - } - } - - //auto C = ton::CatChainActor::create(nullptr, adnl_.get(), overlay_manager_.get(), - // std::vector>()); - - for (auto &it : lc.catchains_) { - auto tag = it->tag_; - for (auto &V : gc.catchains_) { - if (V->tag_ == tag) { - auto v = std::move(clone_tl_object(V)->nodes_); - auto C = ton::CatChainActor::create(make_catchain_callback(), adnl_.get(), overlay_manager_.get(), - std::move(v), it->id_->id_, tag); - catchains_.push_back(std::move(C)); - } - } - } - } -}; - -td::Result get_uint256(std::string str) { - if (str.size() != 64) { - return td::Status::Error("uint256 must have 64 bytes"); - } - td::UInt256 res; - for (size_t i = 0; i < 32; i++) { - res.raw[i] = static_cast(td::hex_to_int(str[2 * i]) * 16 + td::hex_to_int(str[2 * i + 1])); - } - return res; -} - -int main(int argc, char *argv[]) { - SET_VERBOSITY_LEVEL(verbosity_DEBUG); - td::set_default_failure_signal_handler().ensure(); - - td::actor::ActorOwn x; - - td::OptionParser p; - p.set_description("test basic adnl functionality"); - p.add_option('h', "help", "prints_help", [&]() { - char b[10240]; - td::StringBuilder sb({b, 10000}); - sb << p; - std::cout << sb.as_cslice().c_str(); - std::exit(2); - return td::Status::OK(); - }); - p.add_option('C', "global-config", "file to read global config", [&](td::Slice fname) { - td::actor::send_closure(x, &TestNode::set_global_config, fname.str()); - return td::Status::OK(); - }); - p.add_option('c', "local-config", "file to read local config", [&](td::Slice fname) { - td::actor::send_closure(x, &TestNode::set_local_config, fname.str()); - return td::Status::OK(); - }); - p.add_option('s', "broadcast-size", "size of broadcast", [&](td::Slice fname) { - td::actor::send_closure(x, &TestNode::set_broadcast_size, std::atoi(fname.str().c_str())); - return td::Status::OK(); - }); - p.add_option('d', "daemonize", "set SIGHUP", [&]() { - td::set_signal_handler(td::SignalType::HangUp, [](int sig) { -#if TD_DARWIN || TD_LINUX - close(0); - setsid(); -#endif - }).ensure(); - return td::Status::OK(); - }); -#if TD_DARWIN || TD_LINUX - p.add_option('l', "logname", "log to file", [&](td::Slice fname) { - auto FileLog = td::FileFd::open(td::CSlice(fname.str().c_str()), - td::FileFd::Flags::Create | td::FileFd::Flags::Append | td::FileFd::Flags::Write) - .move_as_ok(); - - dup2(FileLog.get_native_fd().fd(), 1); - dup2(FileLog.get_native_fd().fd(), 2); - return td::Status::OK(); - }); -#endif - - td::actor::Scheduler scheduler({2}); - - scheduler.run_in_context([&] { x = td::actor::create_actor("testnode"); }); - - scheduler.run_in_context([&] { p.run(argc, argv).ensure(); }); - scheduler.run_in_context([&] { td::actor::send_closure(x, &TestNode::run); }); - scheduler.run(); - - return 0; -} diff --git a/test/test-overlay.cpp b/test/test-overlay.cpp new file mode 100644 index 00000000..31db3e78 --- /dev/null +++ b/test/test-overlay.cpp @@ -0,0 +1,450 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. + + Copyright 2017-2020 Telegram Systems LLP +*/ +#include "adnl/adnl-node-id.hpp" +#include "adnl/adnl.h" +#include "adnl/utils.hpp" +#include "adnl/adnl-test-loopback-implementation.h" +#include "auto/tl/ton_api.h" +#include "checksum.h" +#include "common/bitstring.h" +#include "dht/dht.h" +#include "keys/keys.hpp" +#include "overlay-manager.h" +#include "overlay.h" +#include "overlay-id.hpp" +#include "overlay/overlays.h" +#include "td/actor/actor.h" +#include "td/utils/OptionParser.h" +#include "td/utils/Status.h" +#include "td/utils/Time.h" +#include "td/utils/UInt.h" +#include "td/utils/buffer.h" +#include "td/utils/crypto.h" +#include "td/utils/filesystem.h" +#include "td/utils/format.h" +#include "td/utils/port/path.h" +#include "td/utils/Random.h" +#include "td/utils/port/signals.h" +#include "td/utils/port/FileFd.h" +#include "td/utils/overloaded.h" +#include "common/errorlog.h" +#include "tl-utils/common-utils.hpp" +#include "tl/TlObject.h" +#include +#include + +#if TD_DARWIN || TD_LINUX +#include +#endif +#include +#include + +#include + +struct Node { + ton::PrivateKey pk; + ton::PublicKeyHash id; + ton::PublicKey id_full; + ton::adnl::AdnlNodeIdShort adnl_id; + ton::adnl::AdnlNodeIdFull adnl_id_full; + bool can_receive; +}; + +static std::vector root_nodes; +static std::vector slave_nodes; +static std::vector all_nodes; +static td::uint32 total_nodes = 4; +static td::int32 node_slaves_cnt = 3; +static size_t remaining = 0; +static td::Bits256 bcast_hash; + +class Callback : public ton::overlay::Overlays::Callback { + public: + Callback(bool can_receive) : can_receive_(can_receive) { + } + void receive_message(ton::adnl::AdnlNodeIdShort src, ton::overlay::OverlayIdShort overlay_id, + td::BufferSlice data) override { + UNREACHABLE(); + } + void receive_query(ton::adnl::AdnlNodeIdShort src, ton::overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + UNREACHABLE(); + } + void receive_broadcast(ton::PublicKeyHash src, ton::overlay::OverlayIdShort overlay_id, + td::BufferSlice data) override { + CHECK(can_receive_); + CHECK(td::sha256_bits256(data.as_slice()) == bcast_hash); + CHECK(remaining > 0); + remaining--; + } + + private: + bool can_receive_; +}; + +int main(int argc, char *argv[]) { + SET_VERBOSITY_LEVEL(verbosity_INFO); + td::set_default_failure_signal_handler().ensure(); + + std::string db_root_ = "tmp-dir-test-catchain"; + td::rmrf(db_root_).ignore(); + td::mkdir(db_root_).ensure(); + + td::set_default_failure_signal_handler().ensure(); + + td::actor::ActorOwn keyring; + td::actor::ActorOwn network_manager; + td::actor::ActorOwn adnl; + td::actor::ActorOwn overlay_manager; + + td::actor::Scheduler scheduler({7}); + scheduler.run_in_context([&] { + ton::errorlog::ErrorLog::create(db_root_); + keyring = ton::keyring::Keyring::create(db_root_); + network_manager = td::actor::create_actor("test net"); + adnl = ton::adnl::Adnl::create(db_root_, keyring.get()); + overlay_manager = + ton::overlay::Overlays::create(db_root_, keyring.get(), adnl.get(), td::actor::ActorId{}); + td::actor::send_closure(adnl, &ton::adnl::Adnl::register_network_manager, network_manager.get()); + }); + + td::uint32 att = 0; + for (td::uint32 start = att; att < start + 5; att++) { + LOG(WARNING) << "Test #" << att; + root_nodes.resize(total_nodes); + slave_nodes.resize(total_nodes * node_slaves_cnt); + + auto overlay_id_full = + ton::create_serialize_tl_object(td::BufferSlice(PSTRING() << "TEST" << att)); + ton::overlay::OverlayIdFull overlay_id(overlay_id_full.clone()); + auto overlay_id_short = overlay_id.compute_short_id(); + + ton::overlay::OverlayOptions opts; + opts.max_slaves_in_semiprivate_overlay_ = node_slaves_cnt; + opts.default_permanent_members_flags_ = ton::overlay::OverlayMemberFlags::DoNotReceiveBroadcasts; + + ton::overlay::OverlayPrivacyRules rules( + 20 << 20, ton::overlay::CertificateFlags::AllowFec | ton::overlay::CertificateFlags::Trusted, {}); + + std::vector root_keys; + std::vector root_adnl; + + size_t real_members = 0; + + scheduler.run_in_context([&] { + auto addr = ton::adnl::TestLoopbackNetworkManager::generate_dummy_addr_list(); + + for (auto &n : root_nodes) { + bool receive_bcasts = (real_members == 0) ? true : (td::Random::fast_uint32() & 1); + if (receive_bcasts) { + real_members++; + } + n.can_receive = receive_bcasts; + + auto pk1 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub1 = pk1.compute_public_key(); + n.adnl_id_full = ton::adnl::AdnlNodeIdFull{pub1}; + n.adnl_id = ton::adnl::AdnlNodeIdShort{pub1.compute_short_id()}; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk1), true, [](td::Unit) {}); + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_id, ton::adnl::AdnlNodeIdFull{pub1}, addr, + static_cast(0)); + td::actor::send_closure(network_manager, &ton::adnl::TestLoopbackNetworkManager::add_node_id, n.adnl_id, true, + true); + + auto pk2 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub2 = pk2.compute_public_key(); + n.id_full = pub2; + n.id = pub2.compute_short_id(); + n.pk = pk2; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk2), true, [](td::Unit) {}); + + LOG(DEBUG) << "created node " << n.adnl_id << " " << n.id; + + all_nodes.push_back(&n); + root_keys.push_back(n.id); + root_adnl.push_back(n.adnl_id); + } + + for (auto &n : slave_nodes) { + bool receive_bcasts = (real_members == 0) ? true : (td::Random::fast_uint32() & 1); + if (receive_bcasts) { + real_members++; + } + n.can_receive = receive_bcasts; + + auto pk1 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub1 = pk1.compute_public_key(); + n.adnl_id_full = ton::adnl::AdnlNodeIdFull{pub1}; + n.adnl_id = ton::adnl::AdnlNodeIdShort{pub1.compute_short_id()}; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk1), true, [](td::Unit) {}); + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_id, ton::adnl::AdnlNodeIdFull{pub1}, addr, + static_cast(0)); + td::actor::send_closure(network_manager, &ton::adnl::TestLoopbackNetworkManager::add_node_id, n.adnl_id, true, + true); + + auto pk2 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub2 = pk2.compute_public_key(); + n.id_full = pub2; + n.id = pub2.compute_short_id(); + n.pk = pk2; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk2), true, [](td::Unit) {}); + + LOG(DEBUG) << "created node " << n.adnl_id << " " << n.id; + all_nodes.push_back(&n); + } + + for (auto &n1 : all_nodes) { + for (auto &n2 : all_nodes) { + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_peer, n1->adnl_id, n2->adnl_id_full, addr); + } + } + + for (auto &n1 : root_nodes) { + opts.local_overlay_member_flags_ = + (n1.can_receive ? 0 : ton::overlay::OverlayMemberFlags::DoNotReceiveBroadcasts); + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::create_semiprivate_overlay, n1.adnl_id, + ton::overlay::OverlayIdFull(overlay_id_full.clone()), root_adnl, root_keys, + ton::overlay::OverlayMemberCertificate{}, std::make_unique(n1.can_receive), + rules, "", opts); + } + for (size_t i = 0; i < slave_nodes.size(); i++) { + auto &n1 = slave_nodes[i]; + opts.local_overlay_member_flags_ = + (n1.can_receive ? 0 : ton::overlay::OverlayMemberFlags::DoNotReceiveBroadcasts); + + ton::overlay::OverlayMemberCertificate cert(root_nodes[i / node_slaves_cnt].id_full, 0, i % node_slaves_cnt, + 2000000000, td::BufferSlice()); + auto buf = cert.to_sign_data(n1.adnl_id); + auto dec = root_nodes[i / node_slaves_cnt].pk.create_decryptor().move_as_ok(); + auto signature = dec->sign(buf.as_slice()).move_as_ok(); + cert.set_signature(signature.as_slice()); + auto enc = root_nodes[i / node_slaves_cnt].id_full.create_encryptor().move_as_ok(); + enc->check_signature(cert.to_sign_data(n1.adnl_id), cert.signature()).ensure(); + + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::create_semiprivate_overlay, n1.adnl_id, + ton::overlay::OverlayIdFull(overlay_id_full.clone()), root_adnl, root_keys, cert, + std::make_unique(n1.can_receive), rules, "", opts); + } + }); + + td::BufferSlice broadcast(1 << 20); + td::Random::secure_bytes(broadcast.as_slice()); + remaining = real_members; + bcast_hash = td::sha256_bits256(broadcast.as_slice()); + + auto t = td::Timestamp::in(20.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + } + + scheduler.run_in_context([&] { + /*td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::get_stats, + [&](td::Result> R) { + if (R.is_ok()) { + auto res = R.move_as_ok(); + for (auto &o : res->overlays_) { + if (o->overlay_id_ == overlay_id_short.bits256_value()) { + LOG(ERROR) << "NODE " << o->adnl_id_ << " nodes=" << o->nodes_.size(); + for (auto &x : o->stats_) { + LOG(ERROR) << "\t" << x->key_ << " " << x->value_; + } + for (auto &x : o->nodes_) { + LOG(ERROR) << "\t\t" << x->adnl_id_; + } + } + } + } + });*/ + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::send_broadcast_fec_ex, root_nodes[0].adnl_id, + overlay_id_short, root_nodes[0].id, 0, std::move(broadcast)); + }); + + t = td::Timestamp::in(10.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + if (!remaining) { + break; + } + } + + LOG_CHECK(!remaining) << "remaining=" << remaining << " all=" << real_members; + + broadcast = td::BufferSlice(700); + td::Random::secure_bytes(broadcast.as_slice()); + remaining = real_members; + bcast_hash = td::sha256_bits256(broadcast.as_slice()); + scheduler.run_in_context([&] { + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::send_broadcast_ex, root_nodes[0].adnl_id, + overlay_id_short, root_nodes[0].id, 0, std::move(broadcast)); + }); + + t = td::Timestamp::in(10.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + if (!remaining) { + break; + } + } + + LOG_CHECK(!remaining) << "remaining=" << remaining; + + scheduler.run_in_context([&] { + root_nodes.clear(); + slave_nodes.clear(); + all_nodes.clear(); + }); + } + + for (td::uint32 start = att; att < start + 5; att++) { + LOG(WARNING) << "Test #" << att; + root_nodes.resize(total_nodes); + + auto overlay_id_full = + ton::create_serialize_tl_object(td::BufferSlice(PSTRING() << "TEST" << att)); + ton::overlay::OverlayIdFull overlay_id(overlay_id_full.clone()); + auto overlay_id_short = overlay_id.compute_short_id(); + + ton::overlay::OverlayOptions opts; + + ton::overlay::OverlayPrivacyRules rules( + 20 << 20, ton::overlay::CertificateFlags::AllowFec | ton::overlay::CertificateFlags::Trusted, {}); + + std::vector root_keys; + std::vector root_adnl; + + size_t real_members = 0; + + scheduler.run_in_context([&] { + auto addr = ton::adnl::TestLoopbackNetworkManager::generate_dummy_addr_list(); + + for (auto &n : root_nodes) { + real_members++; + auto pk1 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub1 = pk1.compute_public_key(); + n.adnl_id_full = ton::adnl::AdnlNodeIdFull{pub1}; + n.adnl_id = ton::adnl::AdnlNodeIdShort{pub1.compute_short_id()}; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk1), true, [](td::Unit) {}); + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_id, ton::adnl::AdnlNodeIdFull{pub1}, addr, + static_cast(0)); + td::actor::send_closure(network_manager, &ton::adnl::TestLoopbackNetworkManager::add_node_id, n.adnl_id, true, + true); + + auto pk2 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub2 = pk2.compute_public_key(); + n.id_full = pub2; + n.id = pub2.compute_short_id(); + n.pk = pk2; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk2), true, [](td::Unit) {}); + + LOG(DEBUG) << "created node " << n.adnl_id << " " << n.id; + + all_nodes.push_back(&n); + root_keys.push_back(n.id); + root_adnl.push_back(n.adnl_id); + } + + for (auto &n1 : all_nodes) { + for (auto &n2 : all_nodes) { + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_peer, n1->adnl_id, n2->adnl_id_full, addr); + } + } + + for (auto &n1 : root_nodes) { + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::create_private_overlay_ex, n1.adnl_id, + ton::overlay::OverlayIdFull(overlay_id_full.clone()), root_adnl, + std::make_unique(true), rules, "", opts); + } + }); + + auto t = td::Timestamp::in(10.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + } + + td::BufferSlice broadcast(1 << 20); + td::Random::secure_bytes(broadcast.as_slice()); + remaining = real_members; + bcast_hash = td::sha256_bits256(broadcast.as_slice()); + + scheduler.run_in_context([&] { + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::send_broadcast_fec_ex, root_nodes[0].adnl_id, + overlay_id_short, root_nodes[0].id, 0, std::move(broadcast)); + }); + + t = td::Timestamp::in(10.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + if (!remaining) { + break; + } + } + + LOG_CHECK(!remaining) << "remaining=" << remaining; + + broadcast = td::BufferSlice(700); + td::Random::secure_bytes(broadcast.as_slice()); + remaining = real_members; + bcast_hash = td::sha256_bits256(broadcast.as_slice()); + scheduler.run_in_context([&] { + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::send_broadcast_ex, root_nodes[0].adnl_id, + overlay_id_short, root_nodes[0].id, 0, std::move(broadcast)); + }); + + t = td::Timestamp::in(10.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + if (!remaining) { + break; + } + } + + LOG_CHECK(!remaining) << "remaining=" << remaining; + + scheduler.run_in_context([&] { + root_nodes.clear(); + slave_nodes.clear(); + all_nodes.clear(); + }); + } + + td::rmrf(db_root_).ensure(); + std::_Exit(0); + return 0; +} diff --git a/test/test-rldp.cpp b/test/test-rldp.cpp index 11344c55..b07f5f7d 100644 --- a/test/test-rldp.cpp +++ b/test/test-rldp.cpp @@ -40,7 +40,7 @@ int main() { SET_VERBOSITY_LEVEL(verbosity_INFO); - std::string db_root_ = "tmp-ee"; + std::string db_root_ = "tmp-dir-test-rldp"; td::rmrf(db_root_).ignore(); td::mkdir(db_root_).ensure(); diff --git a/test/test-rldp2.cpp b/test/test-rldp2.cpp index 5367ffd3..646b27d5 100644 --- a/test/test-rldp2.cpp +++ b/test/test-rldp2.cpp @@ -40,7 +40,7 @@ int main() { SET_VERBOSITY_LEVEL(verbosity_INFO); - std::string db_root_ = "tmp-ee"; + std::string db_root_ = "tmp-dir-test-rldp2"; td::rmrf(db_root_).ignore(); td::mkdir(db_root_).ensure(); diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 9ed5c781..0fde1e68 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -50,7 +50,7 @@ #include "validator/fabric.h" #include "validator/impl/collator.h" -#include "crypto/vm/cp0.h" +#include "crypto/vm/vm.h" #include "crypto/block/block-db.h" #include "common/errorlog.h" @@ -300,7 +300,7 @@ class TestNode : public td::actor::Actor { shard_top_block_id_, db_root_); for (auto &msg : ext_msgs_) { td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManager::new_external_message, - std::move(msg)); + std::move(msg), 0); } for (auto &topmsg : top_shard_descrs_) { td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManager::new_shard_block, ton::BlockIdExt{}, @@ -323,9 +323,8 @@ class TestNode : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } - void add_shard(ton::ShardIdFull) override { - } - void del_shard(ton::ShardIdFull) override { + void on_new_masterchain_block(td::Ref state, + std::set shards_to_monitor) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } @@ -347,7 +346,10 @@ class TestNode : public td::actor::Actor { } } } - void send_broadcast(ton::BlockBroadcast broadcast) override { + void send_block_candidate(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override { + } + void send_broadcast(ton::BlockBroadcast broadcast, int mode) override { } void download_block(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override { @@ -368,13 +370,19 @@ class TestNode : public td::actor::Actor { void get_next_key_blocks(ton::BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) override { } - void download_archive(ton::BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - - td::Promise promise) override { + void download_archive(ton::BlockSeqno masterchain_seqno, ton::ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override { + } + void download_out_msg_queue_proof( + ton::ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, + td::Timestamp timeout, td::Promise>> promise) override { } void new_key_block(ton::validator::BlockHandle handle) override { } + void send_validator_telemetry(ton::PublicKeyHash key, + ton::tl_object_ptr telemetry) override { + } }; td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::install_callback, @@ -408,7 +416,7 @@ int main(int argc, char *argv[]) { SET_VERBOSITY_LEVEL(verbosity_INFO); td::set_default_failure_signal_handler().ensure(); - CHECK(vm::init_op_cp0()); + vm::init_vm().ensure(); td::actor::ActorOwn x; diff --git a/test/test-validator-session-state.cpp b/test/test-validator-session-state.cpp index 819c1cc2..a675070c 100644 --- a/test/test-validator-session-state.cpp +++ b/test/test-validator-session-state.cpp @@ -34,6 +34,7 @@ #include "validator-session/validator-session-description.h" #include "validator-session/validator-session-state.h" +#include "validator-session/validator-session-description.hpp" #include #include @@ -48,16 +49,13 @@ class Description : public ton::validatorsession::ValidatorSessionDescription { return 0; } void *alloc(size_t size, size_t align, bool temp) override { - td::uint32 idx = temp ? 1 : 0; - auto s = pdata_cur_[idx].fetch_add(size); - CHECK(s + size <= pdata_size_[idx]); - return static_cast(pdata_[idx] + s); + return (temp ? mem_temp_ : mem_perm_).alloc(size, align); } bool is_persistent(const void *ptr) const override { - return ptr == nullptr || (ptr >= pdata_[0] && ptr < pdata_[0] + pdata_size_[0]); + return mem_perm_.contains(ptr); } void clear_temp_memory() override { - pdata_cur_[1] = 0; + mem_temp_.clear(); } ton::PublicKeyHash get_source_id(td::uint32 idx) const override { @@ -103,6 +101,10 @@ class Description : public ton::validatorsession::ValidatorSessionDescription { td::uint32 get_max_priority() const override { return opts_.round_candidates - 1; } + td::uint32 get_node_by_priority(td::uint32 round, td::uint32 priority) const override { + CHECK(priority <= get_max_priority()); + return (round + priority) % get_total_nodes(); + } td::uint32 get_unixtime(td::uint64 ts) const override { return static_cast(ts >> 32); } @@ -188,21 +190,8 @@ class Description : public ton::validatorsession::ValidatorSessionDescription { return opts_; } - ~Description() { - delete[] pdata_[0]; - delete[] pdata_[1]; - } - Description(ton::validatorsession::ValidatorSessionOptions opts, td::uint32 total_nodes) - : opts_(opts), total_nodes_(total_nodes) { - pdata_size_[0] = - static_cast(std::numeric_limits::max() < (1ull << 32) ? 1ull << 30 : 1ull << 33); - pdata_size_[1] = 1 << 22; - pdata_[0] = new td::uint8[pdata_size_[0]]; - pdata_[1] = new td::uint8[pdata_size_[1]]; - pdata_cur_[0] = 0; - pdata_cur_[1] = 0; - + : opts_(opts), total_nodes_(total_nodes), mem_perm_(1 << 30), mem_temp_(1 << 22) { for (auto &el : cache_) { Cached v{nullptr}; el.store(v, std::memory_order_relaxed); @@ -223,9 +212,7 @@ class Description : public ton::validatorsession::ValidatorSessionDescription { }; std::array, cache_size> cache_; - td::uint8 *pdata_[2]; - std::atomic pdata_cur_[2]; - size_t pdata_size_[2]; + ton::validatorsession::ValidatorSessionDescriptionImpl::MemPool mem_perm_, mem_temp_; }; double myrand() { diff --git a/test/test-validator-session.cpp b/test/test-validator-session.cpp deleted file mode 100644 index e986795a..00000000 --- a/test/test-validator-session.cpp +++ /dev/null @@ -1,356 +0,0 @@ -/* - This file is part of TON Blockchain source code. - - TON Blockchain is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - TON Blockchain is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with TON Blockchain. If not, see . - - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement - from all source files in the program, then also delete it here. - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "adnl/adnl.h" -#include "rldp/rldp.h" -#include "adnl/utils.hpp" -#include "auto/tl/ton_api_json.h" -#include "dht/dht.h" -#include "overlay/overlays.h" -#include "td/utils/OptionParser.h" -#include "td/utils/Time.h" -#include "td/utils/filesystem.h" -#include "td/utils/format.h" -#include "td/utils/Random.h" -#include "td/utils/port/signals.h" -#include "td/utils/port/FileFd.h" -#include "td/utils/overloaded.h" -#include "catchain/catchain.h" -#include "validator-session/validator-session.h" - -#if TD_DARWIN || TD_LINUX -#include -#endif -#include -#include - -class TestNode : public td::actor::Actor { - private: - td::actor::ActorOwn keyring_; - td::actor::ActorOwn adnl_; - td::actor::ActorOwn rldp_; - std::vector> dht_nodes_; - td::actor::ActorOwn overlay_manager_; - std::vector> validator_sessions_; - - std::string local_config_ = "ton-local.config"; - std::string global_config_ = "ton-global.config"; - - std::unique_ptr make_vs_callback() { - class Callback : public ton::validatorsession::ValidatorSession::Callback { - public: - void on_candidate(td::uint32 round, ton::PublicKeyHash source, - ton::validatorsession::ValidatorSessionRootHash root_hash, td::BufferSlice data, - td::BufferSlice extra, - td::Promise promise) override { - td::actor::send_closure(id_, &TestNode::on_candidate, round, source, root_hash, std::move(data), - std::move(extra), std::move(promise)); - } - void on_generate_slot(td::uint32 round, td::Promise promise) override { - td::actor::send_closure(id_, &TestNode::on_generate_slot, round, std::move(promise)); - } - void on_block_committed(td::uint32 round, ton::PublicKeyHash src, - ton::validatorsession::ValidatorSessionRootHash root_hash, - ton::validatorsession::ValidatorSessionFileHash file_hash, td::BufferSlice data, - std::vector> signatures, - ton::validatorsession::ValidatorSessionStats stats) override { - td::actor::send_closure(id_, &TestNode::on_block_committed, round, root_hash, std::move(data), - std::move(signatures)); - } - /*void on_missing_block_committed( - td::uint32 round, ton::validatorsession::ValidatorSessionRootHash root_hash, ton::validatorsession::ValidatorSessionFileHash file_hash, - td::BufferSlice data, std::vector> signatures) override { - td::actor::send_closure(id_, &TestNode::on_block_committed_abscent, round, root_hash, file_hash, - std::move(data), std::move(signatures)); - }*/ - void on_block_skipped(td::uint32 round) override { - td::actor::send_closure(id_, &TestNode::on_block_skipped, round); - } - void get_approved_candidate(ton::validatorsession::ValidatorSessionRootHash root_hash, - ton::validatorsession::ValidatorSessionFileHash file_hash, - ton::validatorsession::ValidatorSessionFileHash collated_data_file_hash, - td::Promise promise) override { - UNREACHABLE(); - } - - Callback(td::actor::ActorId id) : id_(std::move(id)) { - } - - private: - td::actor::ActorId id_; - }; - - return std::make_unique(actor_id(this)); - } - - td::uint64 height_ = 0; - - public: - void on_candidate(td::uint32 round, ton::PublicKeyHash source, - ton::validatorsession::ValidatorSessionRootHash root_hash, td::BufferSlice data, - td::BufferSlice collated, - td::Promise promise) { - auto sh = sha256_bits256(data.as_slice()); - auto B = ton::fetch_tl_object(std::move(data), true); - if (B.is_error()) { - promise.set_result( - ton::validatorsession::ValidatorSession::CandidateDecision{B.move_as_error().to_string(), td::BufferSlice()}); - return; - } - if (collated.size() != 32) { - promise.set_result( - ton::validatorsession::ValidatorSession::CandidateDecision{"bad collated data length", td::BufferSlice()}); - return; - } - td::Bits256 x; - x.as_slice().copy_from(collated.as_slice().truncate(32)); - if (x != sh) { - promise.set_result( - ton::validatorsession::ValidatorSession::CandidateDecision{"bad block hash", td::BufferSlice()}); - return; - } - auto block = B.move_as_ok(); - if (block->root_hash_ != root_hash) { - promise.set_result( - ton::validatorsession::ValidatorSession::CandidateDecision{"bad root hash", td::BufferSlice()}); - return; - } - if (block->root_hash_ != sha256_bits256(block->data_.as_slice())) { - promise.set_result( - ton::validatorsession::ValidatorSession::CandidateDecision{"bad root hash (2)", td::BufferSlice()}); - return; - } - if (block->height_ != static_cast(height_) + 1) { - promise.set_result( - ton::validatorsession::ValidatorSession::CandidateDecision{"bad root height", td::BufferSlice()}); - return; - } - promise.set_result(ton::validatorsession::ValidatorSession::CandidateDecision{0}); - } - void on_generate_slot(td::uint32 round, td::Promise promise) { - auto data = td::BufferSlice{10000}; - td::Random::secure_bytes(data.as_slice()); - auto root_hash = sha256_bits256(data.as_slice()); - auto block = - ton::create_tl_object(root_hash, height_ + 1, std::move(data)); - - auto B = ton::serialize_tl_object(block, true); - auto hash = sha256_bits256(B.as_slice()); - auto collated = td::BufferSlice{32}; - collated.as_slice().copy_from(as_slice(hash)); - - /*BlockId id; - BlockStatus status; - RootHash root_hash; - FileHash file_hash; - FileHash collated_file_hash; - td::BufferSlice data; - td::BufferSlice collated_data;*/ - auto collated_file_hash = td::sha256_bits256(collated.as_slice()); - ton::BlockCandidate candidate{ton::BlockIdExt{ton::BlockId{0, 0, 0}, root_hash, td::sha256_bits256(B.as_slice())}, - collated_file_hash, std::move(B), std::move(collated)}; - promise.set_result(std::move(candidate)); - } - void on_block_committed(td::uint32 round, ton::validatorsession::ValidatorSessionRootHash root_hash, - td::BufferSlice data, - std::vector> signatures) { - LOG(ERROR) << "COMITTED BLOCK: ROUND=" << round << " ROOT_HASH=" << root_hash - << " DATA_HASH=" << sha256_bits256(data.as_slice()) << " SIGNED BY " << signatures.size(); - } - void on_block_skipped(td::uint32 round) { - LOG(ERROR) << "SKIPPED ROUND=" << round; - } - - void set_local_config(std::string str) { - local_config_ = str; - } - void set_global_config(std::string str) { - global_config_ = str; - } - void start_up() override { - } - void alarm() override { - } - TestNode() { - } - void run() { - keyring_ = ton::keyring::Keyring::create("/var/ton-work/db.keyring"); - adnl_ = ton::adnl::Adnl::create("/var/ton-work/db.adnl", keyring_.get()); - rldp_ = ton::rldp::Rldp::create(adnl_.get()); - - auto L = td::read_file(local_config_).move_as_ok(); - auto lc_j = td::json_decode(L.as_slice()).move_as_ok(); - ton::ton_api::config_local lc; - ton::ton_api::from_json(lc, lc_j.get_object()).ensure(); - - auto G = td::read_file(global_config_).move_as_ok(); - auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); - ton::ton_api::config_global gc; - ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); - - for (auto &port : lc.udp_ports_) { - td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_listening_udp_port, "0.0.0.0", - static_cast(port)); - } - /*if (!lc.net_) { - LOG(FATAL) << "local config does not contain NET section"; - }*/ - - //td::actor::send_closure(network_manager_, &ton::adnl::AdnlNetworkManager::load_local_config, std::move(lc.net_)); - //td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_ids_from_config, std::move(lc.local_ids_)); - if (gc.adnl_) { - td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_static_nodes_from_config, - std::move(gc.adnl_->static_nodes_)); - } - if (!gc.dht_) { - LOG(FATAL) << "global config does not contain dht section"; - } - auto dhtR = ton::dht::Dht::create_global_config(std::move(gc.dht_)); - if (dhtR.is_error()) { - LOG(FATAL) << "bad dht config: " << dhtR.move_as_error(); - } - auto dht = dhtR.move_as_ok(); - - for (auto &it : lc.dht_) { - std::vector adnl_ids; - ton::ton_api::downcast_call( - *it.get(), td::overloaded( - [&](ton::ton_api::dht_config_local &obj) { - adnl_ids.push_back(ton::adnl::AdnlNodeIdShort{obj.id_->id_}); - }, - [&](ton::ton_api::dht_config_random_local &obj) { - auto addrR = ton::adnl::AdnlAddressList::create(std::move(obj.addr_list_)); - addrR.ensure(); - auto addr = addrR.move_as_ok(); - for (td::int32 i = 0; i < obj.cnt_; i++) { - auto pk = ton::PrivateKey{ton::privkeys::Ed25519::random()}; - auto pub = pk.compute_public_key(); - td::actor::send_closure(keyring_, &ton::keyring::Keyring::add_key, std::move(pk), false); - td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_id, ton::adnl::AdnlNodeIdFull{pub}, - addr); - auto adnl_id = ton::adnl::AdnlNodeIdShort{pub.compute_short_id()}; - adnl_ids.push_back(adnl_id); - } - })); - for (auto &id : adnl_ids) { - auto R = ton::dht::Dht::create(id, "/var/ton-work/db/", dht, keyring_.get(), adnl_.get()); - R.ensure(); - dht_nodes_.push_back(R.move_as_ok()); - } - } - - CHECK(dht_nodes_.size() > 0); - - td::actor::send_closure(adnl_, &ton::adnl::Adnl::register_dht_node, dht_nodes_[0].get()); - //td::actor::send_closure(overlay_manager_, &ton::overlay::Overlays::register_dht_node, dht_nodes_[0].get()); - - overlay_manager_ = - ton::overlay::Overlays::create("/var/ton-work/db.overlays", keyring_.get(), adnl_.get(), dht_nodes_[0].get()); - - //auto C = ton::CatChainActor::create(nullptr, adnl_.get(), overlay_manager_.get(), - // std::vector>()); - - for (auto &it : lc.catchains_) { - auto tag = it->tag_; - for (auto &V : gc.catchains_) { - if (V->tag_ == tag) { - auto v = std::move(clone_tl_object(V)->nodes_); - - std::vector w; - w.resize(v.size()); - for (size_t i = 0; i < w.size(); i++) { - w[i].pub_key = ton::PublicKey{v[i]}; - w[i].adnl_id = ton::adnl::AdnlNodeIdShort{w[i].pub_key.compute_short_id()}; - w[i].weight = 1; - } - - auto C = ton::validatorsession::ValidatorSession::create( - tag, ton::PublicKeyHash{it->id_->id_}, std::move(w), make_vs_callback(), keyring_.get(), adnl_.get(), - rldp_.get(), overlay_manager_.get(), "/var/ton-work/db/"); - td::actor::send_closure(C, &ton::validatorsession::ValidatorSession::start); - validator_sessions_.emplace_back(std::move(C)); - } - } - } - } -}; - -int main(int argc, char *argv[]) { - SET_VERBOSITY_LEVEL(verbosity_INFO); - td::set_default_failure_signal_handler().ensure(); - - td::actor::ActorOwn x; - - td::OptionParser p; - p.set_description("test basic adnl functionality"); - p.add_option('h', "help", "prints_help", [&]() { - char b[10240]; - td::StringBuilder sb(td::MutableSlice{b, 10000}); - sb << p; - std::cout << sb.as_cslice().c_str(); - std::exit(2); - return td::Status::OK(); - }); - p.add_option('C', "global-config", "file to read global config", [&](td::Slice fname) { - td::actor::send_closure(x, &TestNode::set_global_config, fname.str()); - return td::Status::OK(); - }); - p.add_option('c', "local-config", "file to read local config", [&](td::Slice fname) { - td::actor::send_closure(x, &TestNode::set_local_config, fname.str()); - return td::Status::OK(); - }); - p.add_option('d', "daemonize", "set SIGHUP", [&]() { - td::set_signal_handler(td::SignalType::HangUp, [](int sig) { -#if TD_DARWIN || TD_LINUX - close(0); - setsid(); -#endif - }).ensure(); - return td::Status::OK(); - }); -#if TD_DARWIN || TD_LINUX - p.add_option('l', "logname", "log to file", [&](td::Slice fname) { - auto FileLog = td::FileFd::open(td::CSlice(fname.str().c_str()), - td::FileFd::Flags::Create | td::FileFd::Flags::Append | td::FileFd::Flags::Write) - .move_as_ok(); - - dup2(FileLog.get_native_fd().fd(), 1); - dup2(FileLog.get_native_fd().fd(), 2); - return td::Status::OK(); - }); -#endif - - td::actor::Scheduler scheduler({7}); - - scheduler.run_in_context([&] { x = td::actor::create_actor("testnode"); }); - - scheduler.run_in_context([&] { p.run(argc, argv).ensure(); }); - scheduler.run_in_context([&] { td::actor::send_closure(x, &TestNode::run); }); - scheduler.run(); - - return 0; -} diff --git a/test/test-validator.cpp b/test/test-validator.cpp deleted file mode 100644 index 7bc018f7..00000000 --- a/test/test-validator.cpp +++ /dev/null @@ -1,356 +0,0 @@ -/* - This file is part of TON Blockchain source code. - - TON Blockchain is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - TON Blockchain is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with TON Blockchain. If not, see . - - In addition, as a special exception, the copyright holders give permission - to link the code of portions of this program with the OpenSSL library. - You must obey the GNU General Public License in all respects for all - of the code used other than OpenSSL. If you modify file(s) with this - exception, you may extend this exception to your version of the file(s), - but you are not obligated to do so. If you do not wish to do so, delete this - exception statement from your version. If you delete this exception statement - from all source files in the program, then also delete it here. - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "adnl/adnl.h" -#include "rldp/rldp.h" -#include "adnl/utils.hpp" -#include "auto/tl/ton_api_json.h" -#include "auto/tl/ton_api.hpp" -#include "dht/dht.h" -#include "overlay/overlays.h" -#include "td/utils/OptionParser.h" -#include "td/utils/Time.h" -#include "td/utils/TsFileLog.h" -#include "td/utils/filesystem.h" -#include "td/utils/format.h" -#include "td/utils/Random.h" -#include "td/utils/port/signals.h" -#include "td/utils/port/FileFd.h" -#include "catchain/catchain.h" -#include "validator-session/validator-session.h" -#include "ton-node/ton-node.h" -#include "validator/manager.h" -#include "td/utils/filesystem.h" -#include "td/utils/ThreadSafeCounter.h" -#include "td/utils/port/path.h" -#include "crypto/vm/cp0.h" -#include "td/utils/overloaded.h" - -#include "memprof/memprof.h" - -#if TD_DARWIN || TD_LINUX -#include -#endif -#include -#include -#include - -class TestNode : public td::actor::Actor { - private: - td::actor::ActorOwn keyring_; - td::actor::ActorOwn adnl_; - td::actor::ActorOwn rldp_; - std::vector> dht_nodes_; - td::actor::ActorOwn overlay_manager_; - td::actor::ActorOwn validator_manager_; - td::actor::ActorOwn ton_node_; - - std::string local_config_ = "ton-local.config"; - std::string global_config_ = "ton-global.config"; - - std::string db_root_ = "/var/ton-work/db/"; - std::string zero_state_ = ""; - - public: - void set_local_config(std::string str) { - local_config_ = str; - } - void set_global_config(std::string str) { - global_config_ = str; - } - void set_db_root(std::string db_root) { - db_root_ = db_root; - } - void set_zero_state(std::string zero_state) { - zero_state_ = zero_state; - } - void start_up() override { - } - void alarm() override { - } - TestNode() { - } - void run() { - td::mkdir(db_root_).ensure(); - - keyring_ = ton::keyring::Keyring::create(db_root_ + "/keyring"); - adnl_ = ton::adnl::Adnl::create(db_root_, keyring_.get()); - rldp_ = ton::rldp::Rldp::create(adnl_.get()); - - auto L = td::read_file(local_config_).move_as_ok(); - auto lc_j = td::json_decode(L.as_slice()).move_as_ok(); - ton::ton_api::config_local lc; - ton::ton_api::from_json(lc, lc_j.get_object()).ensure(); - - auto G = td::read_file(global_config_).move_as_ok(); - auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); - ton::ton_api::config_global gc; - ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); - - for (auto &port : lc.udp_ports_) { - td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_listening_udp_port, "0.0.0.0", - static_cast(port)); - } - /*if (!lc.net_) { - LOG(FATAL) << "local config does not contain NET section"; - }*/ - - //td::actor::send_closure(network_manager_, &ton::adnl::AdnlNetworkManager::load_local_config, std::move(lc.net_)); - //td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_ids_from_config, std::move(lc.local_ids_)); - for (auto &local_id : lc.local_ids_) { - auto pk = ton::PrivateKey{local_id->id_}; - auto pub = pk.compute_public_key(); - auto addr_list = ton::adnl::AdnlAddressList::create(local_id->addr_list_); - addr_list.ensure(); - td::actor::send_closure(keyring_, &ton::keyring::Keyring::add_key, std::move(pk), false); - td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_id, ton::adnl::AdnlNodeIdFull{pub}, addr_list.move_as_ok()); - } - if (gc.adnl_) { - td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_static_nodes_from_config, - std::move(gc.adnl_->static_nodes_)); - } - if (!gc.dht_) { - LOG(FATAL) << "global config does not contain dht section"; - } - - auto dhtR = ton::dht::Dht::create_global_config(std::move(gc.dht_)); - if (dhtR.is_error()) { - LOG(FATAL) << "bad dht config: " << dhtR.move_as_error(); - } - auto dht = dhtR.move_as_ok(); - - for (auto &it : lc.dht_) { - std::vector adnl_ids; - ton::ton_api::downcast_call( - *it.get(), td::overloaded( - [&](ton::ton_api::dht_config_local &obj) { - adnl_ids.push_back(ton::adnl::AdnlNodeIdShort{obj.id_->id_}); - }, - [&](ton::ton_api::dht_config_random_local &obj) { - auto addrR = ton::adnl::AdnlAddressList::create(std::move(obj.addr_list_)); - addrR.ensure(); - auto addr = addrR.move_as_ok(); - for (td::int32 i = 0; i < obj.cnt_; i++) { - auto pk = ton::PrivateKey{ton::privkeys::Ed25519::random()}; - auto pub = pk.compute_public_key(); - td::actor::send_closure(keyring_, &ton::keyring::Keyring::add_key, std::move(pk), false); - td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_id, ton::adnl::AdnlNodeIdFull{pub}, - addr); - auto adnl_id = ton::adnl::AdnlNodeIdShort{pub.compute_short_id()}; - adnl_ids.push_back(adnl_id); - } - })); - for (auto &id : adnl_ids) { - auto R = ton::dht::Dht::create(id, db_root_, dht, keyring_.get(), adnl_.get()); - R.ensure(); - dht_nodes_.push_back(R.move_as_ok()); - } - } - - CHECK(dht_nodes_.size() > 0); - - td::actor::send_closure(adnl_, &ton::adnl::Adnl::register_dht_node, dht_nodes_[0].get()); - overlay_manager_ = ton::overlay::Overlays::create(db_root_, keyring_.get(), adnl_.get(), dht_nodes_[0].get()); - - CHECK(lc.validators_.size() <= 1); - CHECK(gc.validators_.size() <= 1); - - bool is_validator = false; - if (lc.validators_.size() == 1) { - CHECK(gc.validators_.size() == 1); - auto zero_state_id = - ton::BlockIdExt{ton::masterchainId, ton::shardIdAll, 0, gc.validators_[0]->zero_state_root_hash_, - gc.validators_[0]->zero_state_file_hash_}; - ton::PublicKeyHash id; - ton::adnl::AdnlNodeIdShort adnl_id; - ton::ton_api::downcast_call(*lc.validators_[0].get(), - td::overloaded( - [&](ton::ton_api::validator_config_local &cfg) { - id = ton::PublicKeyHash{cfg.id_->id_}; - adnl_id = ton::adnl::AdnlNodeIdShort{id}; - is_validator = true; - }, - [&](ton::ton_api::validator_config_random_local &cfg) { - auto privkey = ton::PrivateKey{ton::privkeys::Ed25519::random()}; - auto pubkey = ton::adnl::AdnlNodeIdFull{privkey.compute_public_key()}; - auto addrR = ton::adnl::AdnlAddressList::create(std::move(cfg.addr_list_)); - addrR.ensure(); - auto addr = addrR.move_as_ok(); - id = privkey.compute_short_id(); - td::actor::send_closure(keyring_, &ton::keyring::Keyring::add_key, - std::move(privkey), false); - td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_id, pubkey, addr); - adnl_id = ton::adnl::AdnlNodeIdShort{id}; - })); - - auto opts = ton::ValidatorManagerOptions::create( - zero_state_id, std::vector{ton::ShardIdFull{ton::basechainId, ton::shardIdAll}}); - CHECK(!opts.is_null()); - opts.write().set_allow_blockchain_init(is_validator); - validator_manager_ = - ton::ValidatorManagerFactory::create(is_validator ? id : ton::PublicKeyHash::zero(), opts, db_root_, - keyring_.get(), adnl_.get(), rldp_.get(), overlay_manager_.get()); - ton_node_ = - ton::TonNodeManager::create(adnl_id, gc.validators_[0]->zero_state_file_hash_, adnl_.get(), rldp_.get(), - dht_nodes_[0].get(), overlay_manager_.get(), validator_manager_.get(), db_root_); - - for (auto &x : lc.liteservers_) { - auto pk = ton::PrivateKey{x->id_}; - auto pub_k = ton::adnl::AdnlNodeIdFull{pk.compute_public_key()}; - auto id = pub_k.compute_short_id(); - - td::actor::send_closure(keyring_, &ton::keyring::Keyring::add_key, std::move(pk), false); - td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_id, pub_k, ton::adnl::AdnlAddressList{}); - td::actor::send_closure(validator_manager_, &ton::ValidatorManager::add_ext_server_id, id); - td::actor::send_closure(validator_manager_, &ton::ValidatorManager::add_ext_server_port, - static_cast(x->port_)); - } - } - } -}; - -td::Result get_uint256(std::string str) { - if (str.size() != 64) { - return td::Status::Error("uint256 must have 64 bytes"); - } - td::UInt256 res; - for (size_t i = 0; i < 32; i++) { - res.raw[i] = static_cast(td::hex_to_int(str[2 * i]) * 16 + td::hex_to_int(str[2 * i + 1])); - } - return res; -} - -std::atomic need_stats_flag{false}; -void need_stats(int sig) { - need_stats_flag.store(true); -} -void dump_memory_stats() { - if (!is_memprof_on()) { - return; - } - LOG(WARNING) << "memory_dump"; - std::vector v; - dump_alloc([&](const AllocInfo &info) { v.push_back(info); }); - std::sort(v.begin(), v.end(), [](const AllocInfo &a, const AllocInfo &b) { return a.size > b.size; }); - size_t total_size = 0; - size_t other_size = 0; - int cnt = 0; - for (auto &info : v) { - if (cnt++ < 50) { - LOG(WARNING) << td::format::as_size(info.size) << td::format::as_array(info.backtrace); - } else { - other_size += info.size; - } - total_size += info.size; - } - LOG(WARNING) << td::tag("other", td::format::as_size(other_size)); - LOG(WARNING) << td::tag("total", td::format::as_size(total_size)); - LOG(WARNING) << td::tag("total traces", get_ht_size()); - LOG(WARNING) << td::tag("fast_backtrace_success_rate", get_fast_backtrace_success_rate()); -} -void dump_stats() { - dump_memory_stats(); - LOG(WARNING) << td::NamedThreadSafeCounter::get_default(); -} - -int main(int argc, char *argv[]) { - SET_VERBOSITY_LEVEL(verbosity_INFO); - - td::set_default_failure_signal_handler().ensure(); - - CHECK(vm::init_op_cp0()); - - td::actor::ActorOwn x; - td::unique_ptr logger_; - SCOPE_EXIT { - td::log_interface = td::default_log_interface; - }; - - td::OptionParser p; - p.set_description("test basic adnl functionality"); - p.add_option('v', "verbosity", "set verbosity level", [&](td::Slice arg) { - int v = VERBOSITY_NAME(FATAL) + (td::to_integer(arg)); - SET_VERBOSITY_LEVEL(v); - return td::Status::OK(); - }); - p.add_option('h', "help", "prints_help", [&]() { - char b[10240]; - td::StringBuilder sb(td::MutableSlice{b, 10000}); - sb << p; - std::cout << sb.as_cslice().c_str(); - std::exit(2); - return td::Status::OK(); - }); - p.add_option('C', "global-config", "file to read global config", [&](td::Slice fname) { - td::actor::send_closure(x, &TestNode::set_global_config, fname.str()); - return td::Status::OK(); - }); - p.add_option('c', "local-config", "file to read local config", [&](td::Slice fname) { - td::actor::send_closure(x, &TestNode::set_local_config, fname.str()); - return td::Status::OK(); - }); - p.add_option('i', "id", "id of instance", [&](td::Slice fname) { return td::Status::OK(); }); - p.add_option('D', "db", "root for dbs", [&](td::Slice fname) { - td::actor::send_closure(x, &TestNode::set_db_root, fname.str()); - return td::Status::OK(); - }); - p.add_option('z', "zero-state", "file with serialized zero state", [&](td::Slice fname) { - td::actor::send_closure(x, &TestNode::set_zero_state, fname.str()); - return td::Status::OK(); - }); - p.add_option('d', "daemonize", "set SIGHUP", [&]() { - td::set_signal_handler(td::SignalType::HangUp, [](int sig) { -#if TD_DARWIN || TD_LINUX - close(0); - setsid(); -#endif - }).ensure(); - return td::Status::OK(); - }); -#if TD_DARWIN || TD_LINUX - p.add_option('l', "logname", "log to file", [&](td::Slice fname) { - logger_ = td::TsFileLog::create(fname.str()).move_as_ok(); - td::log_interface = logger_.get(); - return td::Status::OK(); - }); -#endif - td::set_runtime_signal_handler(1, need_stats).ensure(); - - td::actor::Scheduler scheduler({7}); - - scheduler.run_in_context([&] { x = td::actor::create_actor("testnode"); }); - - scheduler.run_in_context([&] { p.run(argc, argv).ensure(); }); - scheduler.run_in_context([&] { td::actor::send_closure(x, &TestNode::run); }); - while (scheduler.run(1)) { - if (need_stats_flag.exchange(false)) { - dump_stats(); - } - } - - return 0; -} diff --git a/third-party/blst b/third-party/blst new file mode 160000 index 00000000..3dd0f804 --- /dev/null +++ b/third-party/blst @@ -0,0 +1 @@ +Subproject commit 3dd0f804b1819e5d03fb22ca2e6fac105932043a diff --git a/third-party/rocksdb b/third-party/rocksdb index fcf3d75f..cb7a5e02 160000 --- a/third-party/rocksdb +++ b/third-party/rocksdb @@ -1 +1 @@ -Subproject commit fcf3d75f3f022a6a55ff1222d6b06f8518d38c7c +Subproject commit cb7a5e02edeb883193eb5b4901d5943f58e9add9 diff --git a/third-party/secp256k1 b/third-party/secp256k1 new file mode 160000 index 00000000..acf5c55a --- /dev/null +++ b/third-party/secp256k1 @@ -0,0 +1 @@ +Subproject commit acf5c55ae6a94e5ca847e07def40427547876101 diff --git a/tl-utils/CMakeLists.txt b/tl-utils/CMakeLists.txt index b17b7dc9..d5c52d48 100644 --- a/tl-utils/CMakeLists.txt +++ b/tl-utils/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) set(TL_UTILS_SOURCE common-utils.hpp diff --git a/tl-utils/common-utils.hpp b/tl-utils/common-utils.hpp index d61bd79c..05f8a825 100644 --- a/tl-utils/common-utils.hpp +++ b/tl-utils/common-utils.hpp @@ -148,6 +148,39 @@ td::Result::value, T>>> } } +template +td::Result::value, T>>> fetch_tl_prefix(td::Slice &data, + bool boxed) { + td::TlParser p(data); + tl_object_ptr R; + if (boxed) { + R = TlFetchBoxed, T::ID>::parse(p); + } else { + R = move_tl_object_as(T::fetch(p)); + } + if (p.get_status().is_ok()) { + data.remove_prefix(data.size() - p.get_left_len()); + return std::move(R); + } else { + return p.get_status(); + } +} + +template +td::Result::value, T>>> fetch_tl_prefix(td::Slice &data, + bool boxed) { + CHECK(boxed); + td::TlParser p(data); + tl_object_ptr R; + R = move_tl_object_as(T::fetch(p)); + if (p.get_status().is_ok()) { + data.remove_prefix(data.size() - p.get_left_len()); + return std::move(R); + } else { + return p.get_status(); + } +} + template [[deprecated]] tl_object_ptr clone_tl_object(const tl_object_ptr &obj) { auto B = serialize_tl_object(obj, true); diff --git a/tl-utils/lite-utils.cpp b/tl-utils/lite-utils.cpp index cd2ace0a..387474cf 100644 --- a/tl-utils/lite-utils.cpp +++ b/tl-utils/lite-utils.cpp @@ -22,6 +22,7 @@ #include "td/utils/tl_storers.h" #include "td/utils/crypto.h" #include "crypto/common/bitstring.h" +#include namespace ton { @@ -129,4 +130,45 @@ td::Bits256 get_tl_object_sha_bits256(const lite_api::Object *T) { return id256; } +std::string lite_query_name_by_id(int id) { + static std::map names = { + {lite_api::liteServer_getMasterchainInfo::ID, "getMasterchainInfo"}, + {lite_api::liteServer_getMasterchainInfoExt::ID, "getMasterchainInfoExt"}, + {lite_api::liteServer_getTime::ID, "getTime"}, + {lite_api::liteServer_getVersion::ID, "getVersion"}, + {lite_api::liteServer_getBlock::ID, "getBlock"}, + {lite_api::liteServer_getState::ID, "getState"}, + {lite_api::liteServer_getBlockHeader::ID, "getBlockHeader"}, + {lite_api::liteServer_sendMessage::ID, "sendMessage"}, + {lite_api::liteServer_getAccountState::ID, "getAccountState"}, + {lite_api::liteServer_getAccountStatePrunned::ID, "getAccountStatePrunned"}, + {lite_api::liteServer_runSmcMethod::ID, "runSmcMethod"}, + {lite_api::liteServer_getShardInfo::ID, "getShardInfo"}, + {lite_api::liteServer_getAllShardsInfo::ID, "getAllShardsInfo"}, + {lite_api::liteServer_getOneTransaction::ID, "getOneTransaction"}, + {lite_api::liteServer_getTransactions::ID, "getTransactions"}, + {lite_api::liteServer_lookupBlock::ID, "lookupBlock"}, + {lite_api::liteServer_lookupBlockWithProof::ID, "lookupBlockWithProof"}, + {lite_api::liteServer_listBlockTransactions::ID, "listBlockTransactions"}, + {lite_api::liteServer_listBlockTransactionsExt::ID, "listBlockTransactionsExt"}, + {lite_api::liteServer_getBlockProof::ID, "getBlockProof"}, + {lite_api::liteServer_getConfigAll::ID, "getConfigAll"}, + {lite_api::liteServer_getConfigParams::ID, "getConfigParams"}, + {lite_api::liteServer_getValidatorStats::ID, "getValidatorStats"}, + {lite_api::liteServer_getLibraries::ID, "getLibraries"}, + {lite_api::liteServer_getLibrariesWithProof::ID, "getLibrariesWithProof"}, + {lite_api::liteServer_getShardBlockProof::ID, "getShardBlockProof"}, + {lite_api::liteServer_getOutMsgQueueSizes::ID, "getOutMsgQueueSizes"}, + {lite_api::liteServer_getBlockOutMsgQueueSize::ID, "getBlockOutMsgQueueSize"}, + {lite_api::liteServer_getDispatchQueueInfo::ID, "getDispatchQueueInfo"}, + {lite_api::liteServer_getDispatchQueueMessages::ID, "getDispatchQueueMessages"}, + {lite_api::liteServer_nonfinal_getCandidate::ID, "nonfinal.getCandidate"}, + {lite_api::liteServer_nonfinal_getValidatorGroups::ID, "nonfinal.getValidatorGroups"}}; + auto it = names.find(id); + if (it == names.end()) { + return "unknown"; + } + return it->second; +} + } // namespace ton diff --git a/tl-utils/lite-utils.hpp b/tl-utils/lite-utils.hpp index 520ff710..fb3c1dc8 100644 --- a/tl-utils/lite-utils.hpp +++ b/tl-utils/lite-utils.hpp @@ -46,4 +46,6 @@ template ::valu td::Bits256 get_tl_object_sha_bits256(const Tp &T) { return get_tl_object_sha_bits256(static_cast(&T)); } + +std::string lite_query_name_by_id(int id); } // namespace ton diff --git a/tl/CMakeLists.txt b/tl/CMakeLists.txt index 8adabeda..d0760a34 100644 --- a/tl/CMakeLists.txt +++ b/tl/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) add_subdirectory(generate) set_source_files_properties(${TL_TON_API} PROPERTIES GENERATED TRUE) diff --git a/tl/generate/CMakeLists.txt b/tl/generate/CMakeLists.txt index 61d66c93..083d3973 100644 --- a/tl/generate/CMakeLists.txt +++ b/tl/generate/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) file(MAKE_DIRECTORY auto/tl) diff --git a/tl/generate/scheme/lite_api.tl b/tl/generate/scheme/lite_api.tl index a01da11a..b00951da 100644 --- a/tl/generate/scheme/lite_api.tl +++ b/tl/generate/scheme/lite_api.tl @@ -41,9 +41,11 @@ liteServer.shardInfo id:tonNode.blockIdExt shardblk:tonNode.blockIdExt shard_pro liteServer.allShardsInfo id:tonNode.blockIdExt proof:bytes data:bytes = liteServer.AllShardsInfo; liteServer.transactionInfo id:tonNode.blockIdExt proof:bytes transaction:bytes = liteServer.TransactionInfo; liteServer.transactionList ids:(vector tonNode.blockIdExt) transactions:bytes = liteServer.TransactionList; -liteServer.transactionId mode:# account:mode.0?int256 lt:mode.1?long hash:mode.2?int256 = liteServer.TransactionId; +liteServer.transactionMetadata mode:# depth:int initiator:liteServer.accountId initiator_lt:long = liteServer.TransactionMetadata; +liteServer.transactionId#b12f65af mode:# account:mode.0?int256 lt:mode.1?long hash:mode.2?int256 metadata:mode.8?liteServer.transactionMetadata = liteServer.TransactionId; liteServer.transactionId3 account:int256 lt:long = liteServer.TransactionId3; liteServer.blockTransactions id:tonNode.blockIdExt req_count:# incomplete:Bool ids:(vector liteServer.transactionId) proof:bytes = liteServer.BlockTransactions; +liteServer.blockTransactionsExt id:tonNode.blockIdExt req_count:# incomplete:Bool transactions:bytes proof:bytes = liteServer.BlockTransactionsExt; liteServer.signature node_id_short:int256 signature:bytes = liteServer.Signature; liteServer.signatureSet validator_set_hash:int catchain_seqno:int signatures:(vector liteServer.signature) = liteServer.SignatureSet; liteServer.blockLinkBack to_key_block:Bool from:tonNode.blockIdExt to:tonNode.blockIdExt dest_proof:bytes proof:bytes state_proof:bytes = liteServer.BlockLink; @@ -52,11 +54,27 @@ liteServer.partialBlockProof complete:Bool from:tonNode.blockIdExt to:tonNode.bl liteServer.configInfo mode:# id:tonNode.blockIdExt state_proof:bytes config_proof:bytes = liteServer.ConfigInfo; liteServer.validatorStats mode:# id:tonNode.blockIdExt count:int complete:Bool state_proof:bytes data_proof:bytes = liteServer.ValidatorStats; liteServer.libraryResult result:(vector liteServer.libraryEntry) = liteServer.LibraryResult; +liteServer.libraryResultWithProof id:tonNode.blockIdExt mode:# result:(vector liteServer.libraryEntry) state_proof:bytes data_proof:bytes = liteServer.LibraryResultWithProof; liteServer.shardBlockLink id:tonNode.blockIdExt proof:bytes = liteServer.ShardBlockLink; liteServer.shardBlockProof masterchain_id:tonNode.blockIdExt links:(vector liteServer.shardBlockLink) = liteServer.ShardBlockProof; +liteServer.lookupBlockResult id:tonNode.blockIdExt mode:# mc_block_id:tonNode.blockIdExt client_mc_state_proof:bytes mc_block_proof:bytes shard_links:(vector liteServer.shardBlockLink) header:bytes prev_header:bytes = liteServer.LookupBlockResult; +liteServer.outMsgQueueSize id:tonNode.blockIdExt size:int = liteServer.OutMsgQueueSize; +liteServer.outMsgQueueSizes shards:(vector liteServer.outMsgQueueSize) ext_msg_queue_size_limit:int = liteServer.OutMsgQueueSizes; +liteServer.blockOutMsgQueueSize mode:# id:tonNode.blockIdExt size:long proof:mode.0?bytes = liteServer.BlockOutMsgQueueSize; +liteServer.accountDispatchQueueInfo addr:int256 size:long min_lt:long max_lt:long = liteServer.AccountDispatchQueueInfo; +liteServer.dispatchQueueInfo mode:# id:tonNode.blockIdExt account_dispatch_queues:(vector liteServer.accountDispatchQueueInfo) complete:Bool proof:mode.0?bytes = liteServer.DispatchQueueInfo; +liteServer.dispatchQueueMessage addr:int256 lt:long hash:int256 metadata:liteServer.transactionMetadata = liteServer.DispatchQueueMessage; +liteServer.dispatchQueueMessages mode:# id:tonNode.blockIdExt messages:(vector liteServer.dispatchQueueMessage) complete:Bool + proof:mode.0?bytes messages_boc:mode.2?bytes = liteServer.DispatchQueueMessages; liteServer.debug.verbosity value:int = liteServer.debug.Verbosity; +liteServer.nonfinal.candidateId block_id:tonNode.blockIdExt creator:int256 collated_data_hash:int256 = liteServer.nonfinal.CandidateId; +liteServer.nonfinal.candidate id:liteServer.nonfinal.candidateId data:bytes collated_data:bytes = liteServer.nonfinal.Candidate; +liteServer.nonfinal.candidateInfo id:liteServer.nonfinal.candidateId available:Bool approved_weight:long signed_weight:long total_weight:long = liteServer.nonfinal.CandidateInfo; +liteServer.nonfinal.validatorGroupInfo next_block_id:tonNode.blockId cc_seqno:int prev:(vector tonNode.blockIdExt) candidates:(vector liteServer.nonfinal.candidateInfo) = liteServer.nonfinal.ValidatorGroupInfo; +liteServer.nonfinal.validatorGroups groups:(vector liteServer.nonfinal.validatorGroupInfo) = liteServer.nonfinal.ValidatorGroups; + ---functions--- liteServer.getMasterchainInfo = liteServer.MasterchainInfo; @@ -75,13 +93,24 @@ liteServer.getAllShardsInfo id:tonNode.blockIdExt = liteServer.AllShardsInfo; liteServer.getOneTransaction id:tonNode.blockIdExt account:liteServer.accountId lt:long = liteServer.TransactionInfo; liteServer.getTransactions count:# account:liteServer.accountId lt:long hash:int256 = liteServer.TransactionList; liteServer.lookupBlock mode:# id:tonNode.blockId lt:mode.1?long utime:mode.2?int = liteServer.BlockHeader; +liteServer.lookupBlockWithProof mode:# id:tonNode.blockId mc_block_id:tonNode.blockIdExt lt:mode.1?long utime:mode.2?int = liteServer.LookupBlockResult; liteServer.listBlockTransactions id:tonNode.blockIdExt mode:# count:# after:mode.7?liteServer.transactionId3 reverse_order:mode.6?true want_proof:mode.5?true = liteServer.BlockTransactions; +liteServer.listBlockTransactionsExt id:tonNode.blockIdExt mode:# count:# after:mode.7?liteServer.transactionId3 reverse_order:mode.6?true want_proof:mode.5?true = liteServer.BlockTransactionsExt; liteServer.getBlockProof mode:# known_block:tonNode.blockIdExt target_block:mode.0?tonNode.blockIdExt = liteServer.PartialBlockProof; liteServer.getConfigAll mode:# id:tonNode.blockIdExt = liteServer.ConfigInfo; liteServer.getConfigParams mode:# id:tonNode.blockIdExt param_list:(vector int) = liteServer.ConfigInfo; liteServer.getValidatorStats#091a58bc mode:# id:tonNode.blockIdExt limit:int start_after:mode.0?int256 modified_after:mode.2?int = liteServer.ValidatorStats; liteServer.getLibraries library_list:(vector int256) = liteServer.LibraryResult; +liteServer.getLibrariesWithProof id:tonNode.blockIdExt mode:# library_list:(vector int256) = liteServer.LibraryResultWithProof; liteServer.getShardBlockProof id:tonNode.blockIdExt = liteServer.ShardBlockProof; +liteServer.getOutMsgQueueSizes mode:# wc:mode.0?int shard:mode.0?long = liteServer.OutMsgQueueSizes; +liteServer.getBlockOutMsgQueueSize mode:# id:tonNode.blockIdExt want_proof:mode.0?true = liteServer.BlockOutMsgQueueSize; +liteServer.getDispatchQueueInfo mode:# id:tonNode.blockIdExt after_addr:mode.1?int256 max_accounts:int want_proof:mode.0?true = liteServer.DispatchQueueInfo; +liteServer.getDispatchQueueMessages mode:# id:tonNode.blockIdExt addr:int256 after_lt:long max_messages:int + want_proof:mode.0?true one_account:mode.1?true messages_boc:mode.2?true = liteServer.DispatchQueueMessages; + +liteServer.nonfinal.getValidatorGroups mode:# wc:mode.0?int shard:mode.0?long = liteServer.nonfinal.ValidatorGroups; +liteServer.nonfinal.getCandidate id:liteServer.nonfinal.candidateId = liteServer.nonfinal.Candidate; liteServer.queryPrefix = Object; liteServer.query data:bytes = Object; diff --git a/tl/generate/scheme/lite_api.tlo b/tl/generate/scheme/lite_api.tlo index 0572fd46..c576a7f4 100644 Binary files a/tl/generate/scheme/lite_api.tlo and b/tl/generate/scheme/lite_api.tlo differ diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 07aab3da..cfc9f3a1 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -144,7 +144,26 @@ adnl.message.part hash:int256 total_size:int offset:int data:bytes = adnl.Messag ---types--- adnl.db.node.key local_id:int256 peer_id:int256 = adnl.db.Key; -adnl.db.node.value date:int id:PublicKey addr_list:adnl.addressList priority_addr_list:adnl.addressList = adnl.db.node.Value; +adnl.db.node.value date:int id:PublicKey addr_list:adnl.addressList priority_addr_list:adnl.addressList = adnl.db.node.Value; + +adnl.stats.packets ts_start:double ts_end:double + in_packets:long in_bytes:long in_packets_channel:long in_bytes_channel:long + out_packets:long out_bytes:long out_packets_channel:long out_bytes_channel:long + out_expired_messages:long out_expired_bytes:long = adnl.stats.Packets; +adnl.stats.peerPair local_id:int256 peer_id:int256 ip_str:string + packets_recent:adnl.stats.packets packets_total:adnl.stats.packets + last_out_packet_ts:double last_in_packet_ts:double + connection_ready:Bool channel_status:int try_reinit_at:double + out_queue_messages:long out_queue_bytes:long + = adnl.stats.PeerPair; +adnl.stats.ipPackets ip_str:string packets:long = adnl.stats.IpPackets; +adnl.stats.localIdPackets ts_start:double ts_end:double + decrypted_packets:(vector adnl.stats.ipPackets) dropped_packets:(vector adnl.stats.ipPackets) = adnl.stats.LocalIdPackets; +adnl.stats.localId short_id:int256 + current_decrypt:(vector adnl.stats.ipPackets) + packets_recent:adnl.stats.localIdPackets packets_total:adnl.stats.localIdPackets + peers:(vector adnl.stats.peerPair) = adnl.stats.LocalId; +adnl.stats timestamp:double local_ids:(vector adnl.stats.localId) = adnl.Stats; ---functions--- @@ -209,10 +228,15 @@ dht.query node:dht.node = True; ---types--- overlay.node.toSign id:adnl.id.short overlay:int256 version:int = overlay.node.ToSign; +overlay.node.toSignEx id:adnl.id.short overlay:int256 flags:int version:int = overlay.node.ToSign; overlay.node id:PublicKey overlay:int256 version:int signature:bytes = overlay.Node; +overlay.nodeV2 id:PublicKey overlay:int256 flags:int version:int signature:bytes certificate:overlay.MemberCertificate = overlay.NodeV2; overlay.nodes nodes:(vector overlay.node) = overlay.Nodes; +overlay.nodesV2 nodes:(vector overlay.NodeV2) = overlay.NodesV2; +overlay.messageExtra flags:# certificate:flags.0?overlay.MemberCertificate = overlay.MessageExtra; overlay.message overlay:int256 = overlay.Message; +overlay.messageWithExtra overlay:int256 extra:overlay.messageExtra = overlay.Message; //overlay.randomPeers peers:(vector adnl.node) = overlay.RandomPeers; overlay.broadcastList hashes:(vector int256) = overlay.BroadcastList; @@ -225,6 +249,9 @@ overlay.broadcastFec.partId broadcast_hash:int256 data_hash:int256 seqno:int = o overlay.broadcast.toSign hash:int256 date:int = overlay.broadcast.ToSign; +overlay.memberCertificateId node:adnl.id.short flags:int slot:int expire_at:int = overlay.MemberCertificateId; +overlay.memberCertificate issued_by:PublicKey flags:int slot:int expire_at:int signature:bytes = overlay.MemberCertificate; +overlay.emptyMemberCertificate = overlay.MemberCertificate; overlay.certificate issued_by:PublicKey expire_at:int max_size:int signature:bytes = overlay.Certificate; overlay.certificateV2 issued_by:PublicKey expire_at:int max_size:int flags:int signature:bytes = overlay.Certificate; overlay.emptyCertificate = overlay.Certificate; @@ -242,13 +269,16 @@ overlay.broadcastNotFound = overlay.Broadcast; ---functions--- overlay.getRandomPeers peers:overlay.nodes = overlay.Nodes; +overlay.getRandomPeersV2 peers:overlay.NodesV2 = overlay.NodesV2; overlay.query overlay:int256 = True; +overlay.queryWithExtra overlay:int256 extra:overlay.messageExtra = True; overlay.getBroadcast hash:int256 = overlay.Broadcast; overlay.getBroadcastList list:overlay.broadcastList = overlay.BroadcastList; ---types--- +overlay.db.nodesV2 nodes:overlay.nodesV2 = overlay.db.Nodes; overlay.db.nodes nodes:overlay.nodes = overlay.db.Nodes; overlay.db.key.nodes local_id:int256 overlay:int256 = overlay.db.Key; @@ -277,14 +307,10 @@ catchain.differenceFork left:catchain.block.dep right:catchain.block.dep = catch catchain.blockNotFound = catchain.BlockResult; catchain.blockResult block:catchain.block = catchain.BlockResult; -catchain.sent cnt:int = catchain.Sent; - ---functions--- catchain.getBlock block:int256 = catchain.BlockResult; -catchain.getBlocks blocks:(vector int256) = catchain.Sent; catchain.getDifference rt:(vector int) = catchain.Difference; -catchain.getBlockHistory block:int256 height:long stop_if:(vector int256) = catchain.Sent; //catchain.getForkDifference src:int fork:catchain.fork = catchain.ForkDifference; ---types--- @@ -313,6 +339,7 @@ validatorSession.candidateId src:int256 root_hash:int256 file_hash:int256 collat validatorSession.blockUpdate ts:long actions:(vector validatorSession.round.Message) state:int = validatorSession.BlockUpdate; validatorSession.candidate src:int256 round:int root_hash:int256 data:bytes collated_data:bytes = validatorSession.Candidate; +validatorSession.compressedCandidate flags:# src:int256 round:int root_hash:int256 decompressed_size:int data:bytes = validatorSession.Candidate; validatorSession.config catchain_idle_timeout:double catchain_max_deps:int round_candidates:int next_candidate_delay:double round_attempt_duration:int max_round_attempts:int max_block_size:int max_collated_data_size:int = validatorSession.Config; @@ -369,6 +396,7 @@ tonNode.blockSignature who:int256 signature:bytes = tonNode.BlockSignature; tonNode.blockId workchain:int shard:long seqno:int = tonNode.BlockId; tonNode.blockIdExt workchain:int shard:long seqno:int root_hash:int256 file_hash:int256 = tonNode.BlockIdExt; tonNode.zeroStateIdExt workchain:int root_hash:int256 file_hash:int256 = tonNode.ZeroStateIdExt; +tonNode.shardId workchain:int shard:long = tonNode.ShardId; tonNode.blockDescriptionEmpty = tonNode.BlockDescription; tonNode.blockDescription id:tonNode.blockIdExt = tonNode.BlockDescription; @@ -389,32 +417,49 @@ tonNode.externalMessage data:bytes = tonNode.ExternalMessage; tonNode.newShardBlock block:tonNode.blockIdExt cc_seqno:int data:bytes = tonNode.NewShardBlock; -tonNode.blockBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int - signatures:(vector tonNode.blockSignature) +tonNode.blockBroadcastCompressed.data signatures:(vector tonNode.blockSignature) proof_data:bytes = tonNode.blockBroadcaseCompressed.Data; + +tonNode.blockBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + signatures:(vector tonNode.blockSignature) proof:bytes data:bytes = tonNode.Broadcast; +tonNode.blockBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + flags:# compressed:bytes = tonNode.Broadcast; tonNode.ihrMessageBroadcast message:tonNode.ihrMessage = tonNode.Broadcast; tonNode.externalMessageBroadcast message:tonNode.externalMessage = tonNode.Broadcast; tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast; +// signature may be empty, at least for now +tonNode.newBlockCandidateBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + collator_signature:tonNode.blockSignature data:bytes = tonNode.Broadcast; +tonNode.newBlockCandidateBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + collator_signature:tonNode.blockSignature flags:# compressed:bytes = tonNode.Broadcast; tonNode.shardPublicOverlayId workchain:int shard:long zero_state_file_hash:int256 = tonNode.ShardPublicOverlayId; +tonNode.privateBlockOverlayId zero_state_file_hash:int256 nodes:(vector int256) = tonNode.PrivateBlockOverlayId; +tonNode.customOverlayId zero_state_file_hash:int256 name:string nodes:(vector int256) = tonNode.CustomOverlayId; + tonNode.keyBlocks blocks:(vector tonNode.blockIdExt) incomplete:Bool error:Bool = tonNode.KeyBlocks; ton.blockId root_cell_hash:int256 file_hash:int256 = ton.BlockId; ton.blockIdApprove root_cell_hash:int256 file_hash:int256 = ton.BlockId; -tonNode.dataList data:(vector bytes) = tonNode.DataList; - tonNode.dataFull id:tonNode.blockIdExt proof:bytes block:bytes is_link:Bool = tonNode.DataFull; +tonNode.dataFullCompressed id:tonNode.blockIdExt flags:# compressed:bytes is_link:Bool = tonNode.DataFull; tonNode.dataFullEmpty = tonNode.DataFull; -tonNode.capabilities version:int capabilities:long = tonNode.Capabilities; +tonNode.capabilities#f5bf60c0 version_major:int version_minor:int flags:# = tonNode.Capabilities; tonNode.success = tonNode.Success; tonNode.archiveNotFound = tonNode.ArchiveInfo; tonNode.archiveInfo id:long = tonNode.ArchiveInfo; +tonNode.importedMsgQueueLimits max_bytes:int max_msgs:int = ImportedMsgQueueLimits; +tonNode.outMsgQueueProof queue_proofs:bytes block_state_proofs:bytes msg_counts:(vector int) = tonNode.OutMsgQueueProof; +tonNode.outMsgQueueProofEmpty = tonNode.OutMsgQueueProof; + +tonNode.forgetPeer = tonNode.ForgetPeer; + ---functions--- tonNode.getNextBlockDescription prev_block:tonNode.blockIdExt = tonNode.BlockDescription; @@ -432,20 +477,18 @@ tonNode.getNextKeyBlockIds block:tonNode.blockIdExt max_size:int = tonNode.KeyBl tonNode.downloadNextBlockFull prev_block:tonNode.blockIdExt = tonNode.DataFull; tonNode.downloadBlockFull block:tonNode.blockIdExt = tonNode.DataFull; tonNode.downloadBlock block:tonNode.blockIdExt = tonNode.Data; -tonNode.downloadBlocks blocks:(vector tonNode.blockIdExt) = tonNode.DataList; tonNode.downloadPersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadPersistentStateSlice block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt offset:long max_size:long = tonNode.Data; tonNode.downloadZeroState block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadBlockProof block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadKeyBlockProof block:tonNode.blockIdExt = tonNode.Data; -tonNode.downloadBlockProofs blocks:(vector tonNode.blockIdExt) = tonNode.DataList; -tonNode.downloadKeyBlockProofs blocks:(vector tonNode.blockIdExt) = tonNode.DataList; tonNode.downloadBlockProofLink block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadKeyBlockProofLink block:tonNode.blockIdExt = tonNode.Data; -tonNode.downloadBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.DataList; -tonNode.downloadKeyBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.DataList; tonNode.getArchiveInfo masterchain_seqno:int = tonNode.ArchiveInfo; +tonNode.getShardArchiveInfo masterchain_seqno:int shard_prefix:tonNode.shardId = tonNode.ArchiveInfo; tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data; +tonNode.getOutMsgQueueProof dst_shard:tonNode.shardId blocks:(vector tonNode.blockIdExt) + limits:tonNode.importedMsgQueueLimits = tonNode.OutMsgQueueProof; tonNode.getCapabilities = tonNode.Capabilities; @@ -499,6 +542,7 @@ db.filedb.key.proof block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.proofLink block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.signatures block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.candidate id:db.candidate.id = db.filedb.Key; +db.filedb.key.candidateRef id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.blockInfo block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.value key:db.filedb.Key prev:int256 next:int256 file_hash:int256 = db.filedb.Value; @@ -510,6 +554,9 @@ db.state.shardClient block:tonNode.blockIdExt = db.state.ShardClient; db.state.asyncSerializer block:tonNode.blockIdExt last:tonNode.blockIdExt last_ts:int = db.state.AsyncSerializer; db.state.hardforks blocks:(vector tonNode.blockIdExt) = db.state.Hardforks; db.state.dbVersion version:int = db.state.DbVersion; +db.state.persistentStateDescriptionShards shard_blocks:(vector tonNode.blockIdExt) = db.state.PersistentStateDescriptionShards; +db.state.persistentStateDescriptionHeader masterchain_id:tonNode.blockIdExt start_time:int end_time:int = db.state.PersistentStateDescriptionHeader; +db.state.persistentStateDescriptionsList list:(vector db.state.persistentStateDescriptionHeader) = db.state.PersistentStateDescriptionsList; db.state.key.destroyedSessions = db.state.Key; db.state.key.initBlockId = db.state.Key; @@ -518,6 +565,8 @@ db.state.key.shardClient = db.state.Key; db.state.key.asyncSerializer = db.state.Key; db.state.key.hardforks = db.state.Key; db.state.key.dbVersion = db.state.Key; +db.state.key.persistentStateDescriptionShards masterchain_seqno:int = db.state.Key; +db.state.key.persistentStateDescriptionsList = db.state.Key; db.lt.el.key workchain:int shard:long idx:int = db.lt.Key; db.lt.desc.key workchain:int shard:long = db.lt.Key; @@ -545,6 +594,10 @@ validator.group workchain:int shard:long catchain_seqno:int config_hash:int256 m validator.groupEx workchain:int shard:long vertical_seqno:int catchain_seqno:int config_hash:int256 members:(vector validator.groupMember) = validator.Group; validator.groupNew workchain:int shard:long vertical_seqno:int last_key_block_seqno:int catchain_seqno:int config_hash:int256 members:(vector validator.groupMember) = validator.Group; +validator.telemetry flags:# timestamp:double adnl_id:int256 + node_version:string os_version:string node_started_at:int + ram_size:long cpu_cores:int node_threads:int = validator.Telemetry; + ---functions--- @@ -569,8 +622,13 @@ dummyworkchain0.config.global zero_state_hash:int256 = dummyworkchain0.config.Gl validator.config.global zero_state:tonNode.blockIdExt init_block:tonNode.blockIdExt hardforks:(vector tonNode.blockIdExt) = validator.config.Global; config.global adnl:adnl.config.global dht:dht.config.Global validator:validator.config.global = config.Global; +liteserver.descV2.sliceSimple shards:(vector tonNode.shardId) = liteserver.descV2.Slice; +liteserver.descV2.shardInfo shard_id:tonNode.shardId seqno:int utime:int lt:long = liteserver.descV2.ShardInfo; +liteserver.descV2.sliceTimed shards_from:(vector liteserver.descV2.shardInfo) shards_to:(vector liteserver.descV2.shardInfo) = liteserver.descV2.Slice; + liteserver.desc id:PublicKey ip:int port:int = liteserver.Desc; -liteclient.config.global liteservers:(vector liteserver.desc) validator:validator.config.global = liteclient.config.Global; +liteserver.descV2 id:PublicKey ip:int port:int slices:(vector liteserver.descV2.Slice) = liteserver.DescV2; +liteclient.config.global liteservers:(vector liteserver.desc) liteservers_v2:(vector liteserver.descV2) validator:validator.config.global = liteclient.config.Global; engine.adnl id:int256 category:int = engine.Adnl; engine.addr ip:int port:int categories:(vector int) priority_categories:(vector int) = engine.Addr; @@ -588,12 +646,28 @@ engine.gc ids:(vector int256) = engine.Gc; engine.dht.config dht:(vector engine.dht) gc:engine.gc = engine.dht.Config; engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNodeMaster; engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave; -engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) +engine.validator.fullNodeConfig ext_messages_broadcast_disabled:Bool = engine.validator.FullNodeConfig; +engine.validator.extraConfig state_serializer_enabled:Bool = engine.validator.ExtraConfig; +engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) dht:(vector engine.dht) validators:(vector engine.validator) fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) fullnodemasters:(vector engine.validator.fullNodeMaster) + fullnodeconfig:engine.validator.fullNodeConfig + extraconfig:engine.validator.extraConfig liteservers:(vector engine.liteServer) control:(vector engine.controlInterface) - gc:engine.gc = engine.validator.Config; + shards_to_monitor:(vector tonNode.shardId) + gc:engine.gc = engine.validator.Config; + +engine.validator.customOverlayNode adnl_id:int256 msg_sender:Bool msg_sender_priority:int block_sender:Bool = engine.validator.CustomOverlayNode; +engine.validator.customOverlay name:string nodes:(vector engine.validator.customOverlayNode) sender_shards:(vector tonNode.shardId) + = engine.validator.CustomOverlay; +engine.validator.customOverlaysConfig overlays:(vector engine.validator.customOverlay) = engine.validator.CustomOverlaysConfig; + +engine.validator.collatorOptions + deferring_enabled:Bool defer_messages_after:int defer_out_queue_size_limit:long + dispatch_phase_2_max_total:int dispatch_phase_3_max_total:int + dispatch_phase_2_max_per_initiator:int dispatch_phase_3_max_per_initiator:int + whitelist:(vector string) prioritylist:(vector string) = engine.validator.CollatorOptions; ---functions--- ---types--- @@ -619,6 +693,7 @@ engine.validator.signature signature:bytes = engine.validator.Signature; engine.validator.oneStat key:string value:string = engine.validator.OneStat; engine.validator.stats stats:(vector engine.validator.oneStat) = engine.validator.Stats; +engine.validator.textStats data:string = engine.validator.TextStats; engine.validator.controlQueryError code:int message:string = engine.validator.ControlQueryError; @@ -633,15 +708,31 @@ engine.validator.proposalVote perm_key:int256 to_send:bytes = engine.validator.P engine.validator.dhtServerStatus id:int256 status:int = engine.validator.DhtServerStatus; engine.validator.dhtServersStatus servers:(vector engine.validator.dhtServerStatus) = engine.validator.DhtServersStatus; -engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string bdcst_errors:int fec_bdcst_errors:int last_in_query:int last_out_query:int t_out_bytes:int t_in_bytes:int t_out_pckts:int t_in_pckts:int = engine.validator.OverlayStatsNode; +engine.validator.overlayStatsTraffic t_out_bytes:long t_in_bytes:long t_out_pckts:int t_in_pckts:int = engine.validator.OverlayStatsTraffic; -engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) = engine.validator.OverlayStats; +engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string is_neighbour:Bool is_alive:Bool node_flags:int + bdcst_errors:int fec_bdcst_errors:int last_in_query:int last_out_query:int + traffic:engine.validator.overlayStatsTraffic traffic_responses:engine.validator.overlayStatsTraffic = engine.validator.OverlayStatsNode; + +engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string + nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) + total_traffic:engine.validator.overlayStatsTraffic total_traffic_responses:engine.validator.overlayStatsTraffic + extra:string = engine.validator.OverlayStats; engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats; +engine.validator.shardOverlayStats.neighbour id:string verison_major:int version_minor:int flags:# + roundtrip:double unreliability:double = engine.validator.shardOverlayStats.Neighbour; +engine.validator.shardOverlayStats shard:string active:Bool + neighbours:(vector engine.validator.shardOverlayStats.neighbour) = engine.validator.ShardOverlayStats; + engine.validator.onePerfTimerStat time:int min:double avg:double max:double = engine.validator.OnePerfTimerStat; engine.validator.perfTimerStatsByName name:string stats:(vector engine.validator.OnePerfTimerStat) = engine.validator.PerfTimerStatsByName; engine.validator.perfTimerStats stats:(vector engine.validator.PerfTimerStatsByName) = engine.validator.PerfTimerStats; +engine.validator.shardOutQueueSize size:long = engine.validator.ShardOutQueueSize; + +engine.validator.exportedPrivateKeys encrypted_data:bytes = engine.validator.ExportedPrivateKeys; + ---functions--- @@ -672,6 +763,7 @@ engine.validator.delListeningPort ip:int port:int categories:(vector int) priori engine.validator.delProxy out_ip:int out_port:int categories:(vector int) priority_categories:(vector int) = engine.validator.Success; engine.validator.sign key_hash:int256 data:bytes = engine.validator.Signature; +engine.validator.exportAllPrivateKeys encryption_key:PublicKey = engine.validator.ExportedPrivateKeys; engine.validator.getStats = engine.validator.Stats; engine.validator.getConfig = engine.validator.JsonConfig; @@ -693,6 +785,23 @@ engine.validator.signShardOverlayCertificate workchain:int shard:long signed_key engine.validator.importShardOverlayCertificate workchain:int shard:long signed_key:engine.validator.KeyHash cert:overlay.Certificate = engine.validator.Success; engine.validator.getPerfTimerStats name:string = engine.validator.PerfTimerStats; +engine.validator.getShardOutQueueSize flags:# block_id:tonNode.blockId dest_wc:flags.0?int dest_shard:flags.0?long = engine.validator.ShardOutQueueSize; +engine.validator.setExtMessagesBroadcastDisabled disabled:Bool = engine.validator.Success; + +engine.validator.addCustomOverlay overlay:engine.validator.customOverlay = engine.validator.Success; +engine.validator.delCustomOverlay name:string = engine.validator.Success; +engine.validator.showCustomOverlays = engine.validator.CustomOverlaysConfig; + +engine.validator.setStateSerializerEnabled enabled:Bool = engine.validator.Success; + +engine.validator.setCollatorOptionsJson json:string = engine.validator.Success; +engine.validator.getCollatorOptionsJson = engine.validator.JsonConfig; + +engine.validator.getAdnlStats all:Bool = adnl.Stats; +engine.validator.getActorTextStats = engine.validator.TextStats; + +engine.validator.addShard shard:tonNode.shardId = engine.validator.Success; +engine.validator.delShard shard:tonNode.shardId = engine.validator.Success; ---types--- @@ -720,11 +829,13 @@ storage.getPiece piece_id:int = storage.Piece; http.header name:string value:string = http.Header; http.payloadPart data:bytes trailer:(vector http.header) last:Bool = http.PayloadPart; http.response http_version:string status_code:int reason:string headers:(vector http.header) no_payload:Bool = http.Response; +http.proxy.capabilities capabilities:long = http.proxy.Capabilities; ---functions--- http.request id:int256 method:string url:string http_version:string headers:(vector http.header) = http.Response; http.getNextPayloadPart id:int256 seqno:int max_chunk_size:int = http.PayloadPart; +http.proxy.getCapabilities capabilities:long = http.proxy.Capabilities; ---types--- @@ -738,14 +849,37 @@ http.server.config dhs:(vector http.server.dnsEntry) local_hosts:(vector http.se ---types--- -validatorSession.statsProducer id:int256 block_status:int block_timestamp:long = validatorSession.StatsProducer; +validatorSession.collationStats bytes:int gas:int lt_delta:int cat_bytes:int cat_gas:int cat_lt_delta:int + limits_log:string ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int = validadorSession.CollationStats; -validatorSession.statsRound timestamp:long producers:(vector validatorSession.statsProducer) = validatorSession.StatsRound; +validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int root_hash:int256 file_hash:int256 + comment:string block_timestamp:double is_accepted:Bool is_ours:Bool got_submit_at:double + collation_time:double collated_at:double collation_cached:Bool + collation_work_time:double collation_cpu_work_time:double + collation_stats:validatorSession.collationStats + validation_time:double validated_at:double validation_cached:Bool + validation_work_time:double validation_cpu_work_time:double + gen_utime:double + approved_weight:long approved_33pct_at:double approved_66pct_at:double approvers:string + signed_weight:long signed_33pct_at:double signed_66pct_at:double signers:string + serialize_time:double deserialize_time:double serialized_size:int = validatorSession.StatsProducer; -validatorSession.stats id:tonNode.blockId timestamp:long self:int256 creator:int256 total_validators:int total_weight:long +validatorSession.statsRound timestamp:double producers:(vector validatorSession.statsProducer) = validatorSession.StatsRound; + +validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:double self:int256 session_id:int256 cc_seqno:int + creator:int256 total_validators:int total_weight:long signatures:int signatures_weight:long approve_signatures:int approve_signatures_weight:long first_round:int rounds:(vector validatorSession.statsRound) = validatorSession.Stats; +validatorSession.newValidatorGroupStats.node id:int256 weight:long = validatorSession.newValidatorGroupStats.Node; +validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:long cc_seqno:int + last_key_block_seqno:int timestamp:double + self_idx:int nodes:(vector validatorSession.newValidatorGroupStats.node) = validatorSession.NewValidatorGroupStats; + +validatorSession.endValidatorGroupStats.node id:int256 catchain_blocks:int = validatorSession.endValidatorGroupStats.Node; +validatorSession.endValidatorGroupStats session_id:int256 timestamp:double + nodes:(vector validatorSession.endValidatorGroupStats.node) = validatorSession.EndValidatorGroupStats; + ---functions--- ---types--- @@ -756,9 +890,12 @@ storage.db.key.torrentMeta hash:int256 = storage.db.key.TorrentMeta; storage.db.key.priorities hash:int256 = storage.db.key.Priorities; storage.db.key.piecesInDb hash:int256 = storage.db.key.PiecesInDb; storage.db.key.pieceInDb hash:int256 idx:long = storage.db.key.PieceInDb; +storage.db.key.config = storage.db.key.Config; +storage.db.config flags:# download_speed_limit:double upload_speed_limit:double = storage.db.Config; storage.db.torrentList torrents:(vector int256) = storage.db.TorrentList; storage.db.torrent root_dir:string active_download:Bool active_upload:Bool = storage.db.TorrentShort; +storage.db.torrentV2 flags:# root_dir:string added_at:int active_download:Bool active_upload:Bool = storage.db.TorrentShort; storage.db.priorities actions:(vector storage.PriorityAction) = storage.db.Priorities; storage.db.piecesInDb pieces:(vector long) = storage.db.PiecesInDb; @@ -786,6 +923,7 @@ storage.provider.db.microchunkTree data:bytes = storage.provider.db.MicrochunkTr storage.daemon.queryError message:string = storage.daemon.QueryError; storage.daemon.success = storage.daemon.Success; + storage.daemon.torrent hash:int256 flags:# // 0 - info ready @@ -794,19 +932,29 @@ storage.daemon.torrent total_size:flags.0?long description:flags.0?string files_count:flags.1?long included_size:flags.1?long dir_name:flags.1?string downloaded_size:long - root_dir:string active_download:Bool active_upload:Bool completed:Bool + added_at:int root_dir:string active_download:Bool active_upload:Bool completed:Bool download_speed:double upload_speed:double fatal_error:flags.2?string = storage.daemon.Torrent; + storage.daemon.fileInfo - name:string size:long + name:string size:long flags:# priority:int downloaded_size:long = storage.daemon.FileInfo; + storage.daemon.torrentFull torrent:storage.daemon.torrent files:(vector storage.daemon.fileInfo) = storage.daemon.TorrentFull; storage.daemon.torrentList torrents:(vector storage.daemon.torrent) = storage.daemon.TorrentList; storage.daemon.torrentMeta meta:bytes = storage.daemon.TorrentMeta; +storage.daemon.filePiecesInfo name:string range_l:long range_r:long = storage.daemon.FilePiecesInfo; +storage.daemon.torrentPiecesInfo + flags:# // 0 - with file ranges + total_pieces:long piece_size:int + range_l:long range_r:long piece_ready_bitset:bytes + files:flags.0?(vector storage.daemon.filePiecesInfo) // files[0] is header + = storage.daemon.TorrentPiecesInfo; + storage.daemon.newContractParams rate:string max_span:int = storage.daemon.NewContractParams; storage.daemon.newContractParamsAuto provider_address:string = storage.daemon.NewContractParams; storage.daemon.newContractMessage body:bytes rate:string max_span:int = storage.daemon.NewContractMessage; @@ -819,6 +967,8 @@ storage.daemon.priorityPending = storage.daemon.SetPriorityStatus; storage.daemon.keyHash key_hash:int256 = storage.daemon.KeyHash; +storage.daemon.speedLimits download:double upload:double = storage.daemon.SpeedLimits; + storage.daemon.providerConfig max_contracts:int max_total_size:long = storage.daemon.ProviderConfig; storage.daemon.contractInfo address:string state:int torrent:int256 created_time:int file_size:long downloaded_size:long rate:string max_span:int client_balance:string contract_balance:string = storage.daemon.ContractInfo; @@ -829,24 +979,32 @@ storage.daemon.providerAddress address:string = storage.daemon.ProviderAddress; ---functions--- storage.daemon.setVerbosity verbosity:int = storage.daemon.Success; -storage.daemon.createTorrent path:string description:string allow_upload:Bool copy_inside:Bool = storage.daemon.TorrentFull; -storage.daemon.addByHash hash:int256 root_dir:string start_download:Bool allow_upload:Bool priorities:(vector storage.PriorityAction) = storage.daemon.TorrentFull; -storage.daemon.addByMeta meta:bytes root_dir:string start_download:Bool allow_upload:Bool priorities:(vector storage.PriorityAction) = storage.daemon.TorrentFull; +storage.daemon.createTorrent path:string description:string allow_upload:Bool copy_inside:Bool flags:# = storage.daemon.TorrentFull; +storage.daemon.addByHash hash:int256 root_dir:string start_download:Bool allow_upload:Bool priorities:(vector storage.PriorityAction) flags:# = storage.daemon.TorrentFull; +storage.daemon.addByMeta meta:bytes root_dir:string start_download:Bool allow_upload:Bool priorities:(vector storage.PriorityAction) flags:# = storage.daemon.TorrentFull; storage.daemon.setActiveDownload hash:int256 active:Bool = storage.daemon.Success; storage.daemon.setActiveUpload hash:int256 active:Bool = storage.daemon.Success; -storage.daemon.getTorrents = storage.daemon.TorrentList; -storage.daemon.getTorrentFull hash:int256 = storage.daemon.TorrentFull; -storage.daemon.getTorrentMeta hash:int256 = storage.daemon.TorrentMeta; +storage.daemon.getTorrents flags:# = storage.daemon.TorrentList; +storage.daemon.getTorrentFull hash:int256 flags:# = storage.daemon.TorrentFull; +storage.daemon.getTorrentMeta hash:int256 flags:# = storage.daemon.TorrentMeta; storage.daemon.getNewContractMessage hash:int256 query_id:long params:storage.daemon.NewContractParams = storage.daemon.NewContractMessage; -storage.daemon.getTorrentPeers hash:int256 = storage.daemon.PeerList; +storage.daemon.getTorrentPeers hash:int256 flags:# = storage.daemon.PeerList; +storage.daemon.getTorrentPiecesInfo hash:int256 + flags:# // 0 - with file ranges + offset:long max_pieces:long + = storage.daemon.TorrentPiecesInfo; storage.daemon.setFilePriorityAll hash:int256 priority:int = storage.daemon.SetPriorityStatus; storage.daemon.setFilePriorityByIdx hash:int256 idx:long priority:int = storage.daemon.SetPriorityStatus; storage.daemon.setFilePriorityByName hash:int256 name:string priority:int = storage.daemon.SetPriorityStatus; storage.daemon.removeTorrent hash:int256 remove_files:Bool = storage.daemon.Success; -storage.daemon.loadFrom hash:int256 meta:bytes path:string = storage.daemon.Torrent; +storage.daemon.loadFrom hash:int256 meta:bytes path:string flags:# = storage.daemon.Torrent; + +storage.daemon.getSpeedLimits flags:# = storage.daemon.SpeedLimits; +storage.daemon.setSpeedLimits flags:# download:flags.0?double upload:flags.1?double = storage.daemon.Success; + storage.daemon.importPrivateKey key:PrivateKey = storage.daemon.KeyHash; storage.daemon.initProvider account_address:string = storage.daemon.Success; @@ -859,3 +1017,6 @@ storage.daemon.withdraw contract:string = storage.daemon.Success; storage.daemon.sendCoins address:string amount:string message:string = storage.daemon.Success; storage.daemon.closeStorageContract address:string = storage.daemon.Success; storage.daemon.removeStorageProvider = storage.daemon.Success; + +---types--- +proxyLiteserver.config port:int id:PublicKey = proxyLiteserver.Config; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 6a2e18f1..96ecb775 100644 Binary files a/tl/generate/scheme/ton_api.tlo and b/tl/generate/scheme/ton_api.tlo differ diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index 10a5896c..31ca6fd4 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -50,9 +50,11 @@ internal.transactionId lt:int64 hash:bytes = internal.TransactionId; ton.blockId workchain:int32 shard:int64 seqno:int32 = internal.BlockId; ton.blockIdExt workchain:int32 shard:int64 seqno:int32 root_hash:bytes file_hash:bytes = ton.BlockIdExt; -raw.fullAccountState balance:int64 code:bytes data:bytes last_transaction_id:internal.transactionId block_id:ton.blockIdExt frozen_hash:bytes sync_utime:int53 = raw.FullAccountState; -raw.message source:accountAddress destination:accountAddress value:int64 fwd_fee:int64 ihr_fee:int64 created_lt:int64 body_hash:bytes msg_data:msg.Data = raw.Message; -raw.transaction utime:int53 data:bytes transaction_id:internal.transactionId fee:int64 storage_fee:int64 other_fee:int64 in_msg:raw.message out_msgs:vector = raw.Transaction; +extraCurrency id:int32 amount:int64 = ExtraCurrency; + +raw.fullAccountState balance:int64 extra_currencies:vector code:bytes data:bytes last_transaction_id:internal.transactionId block_id:ton.blockIdExt frozen_hash:bytes sync_utime:int53 = raw.FullAccountState; +raw.message hash:bytes source:accountAddress destination:accountAddress value:int64 extra_currencies:vector fwd_fee:int64 ihr_fee:int64 created_lt:int64 body_hash:bytes msg_data:msg.Data = raw.Message; +raw.transaction address:accountAddress utime:int53 data:bytes transaction_id:internal.transactionId fee:int64 storage_fee:int64 other_fee:int64 in_msg:raw.message out_msgs:vector = raw.Transaction; raw.transactions transactions:vector previous_transaction_id:internal.transactionId = raw.Transactions; raw.extMessageInfo hash:bytes = raw.ExtMessageInfo; @@ -61,6 +63,7 @@ pchan.config alice_public_key:string alice_address:accountAddress bob_public_key raw.initialAccountState code:bytes data:bytes = InitialAccountState; wallet.v3.initialAccountState public_key:string wallet_id:int64 = InitialAccountState; +wallet.v4.initialAccountState public_key:string wallet_id:int64 = InitialAccountState; wallet.highload.v1.initialAccountState public_key:string wallet_id:int64 = InitialAccountState; wallet.highload.v2.initialAccountState public_key:string wallet_id:int64 = InitialAccountState; @@ -73,6 +76,7 @@ pchan.initialAccountState config:pchan.config = InitialAccountState; raw.accountState code:bytes data:bytes frozen_hash:bytes = AccountState; wallet.v3.accountState wallet_id:int64 seqno:int32 = AccountState; +wallet.v4.accountState wallet_id:int64 seqno:int32 = AccountState; wallet.highload.v1.accountState wallet_id:int64 seqno:int32 = AccountState; wallet.highload.v2.accountState wallet_id:int64 = AccountState; dns.accountState wallet_id:int64 = AccountState; @@ -85,7 +89,7 @@ pchan.statePayout A:int64 B:int64 = pchan.State; pchan.accountState config:pchan.config state:pchan.State description:string = AccountState; uninited.accountState frozen_hash:bytes = AccountState; -fullAccountState address:accountAddress balance:int64 last_transaction_id:internal.transactionId block_id:ton.blockIdExt sync_utime:int53 account_state:AccountState revision:int32 = FullAccountState; +fullAccountState address:accountAddress balance:int64 extra_currencies:vector last_transaction_id:internal.transactionId block_id:ton.blockIdExt sync_utime:int53 account_state:AccountState revision:int32 = FullAccountState; accountRevisionList revisions:vector = AccountRevisionList; accountList accounts:vector = AccountList; @@ -108,7 +112,7 @@ msg.dataDecrypted proof:bytes data:msg.Data = msg.DataDecrypted; msg.dataEncryptedArray elements:vector = msg.DataEncryptedArray; msg.dataDecryptedArray elements:vector = msg.DataDecryptedArray; -msg.message destination:accountAddress public_key:string amount:int64 data:msg.Data send_mode:int32 = msg.Message; +msg.message destination:accountAddress public_key:string amount:int64 extra_currencies:vector data:msg.Data send_mode:int32 = msg.Message; // // DNS @@ -184,6 +188,10 @@ smc.runResult gas_used:int53 stack:vector exit_code:int32 = smc. smc.libraryEntry hash:int256 data:bytes = smc.LibraryEntry; smc.libraryResult result:(vector smc.libraryEntry) = smc.LibraryResult; +smc.libraryQueryExt.one hash:int256 = smc.LibraryQueryExt; +smc.libraryQueryExt.scanBoc boc:bytes max_libs:int32 = smc.LibraryQueryExt; +smc.libraryResultExt dict_boc:bytes libs_ok:(vector int256) libs_not_found:(vector int256) = smc.LibraryResultExt; + updateSendLiteServerQuery id:int64 data:bytes = Update; updateSyncState sync_state:SyncState = Update; @@ -215,6 +223,7 @@ blocks.shards shards:vector = blocks.Shards; blocks.accountTransactionId account:bytes lt:int64 = blocks.AccountTransactionId; blocks.shortTxId mode:# account:mode.0?bytes lt:mode.1?int64 hash:mode.2?bytes = liteServer.TransactionId; blocks.transactions id:ton.blockIdExt req_count:int32 incomplete:Bool transactions:vector = blocks.Transactions; +blocks.transactionsExt id:ton.blockIdExt req_count:int32 incomplete:Bool transactions:vector = blocks.TransactionsExt; blocks.header id:ton.blockIdExt global_id:int32 version:int32 flags:# after_merge:Bool after_split:Bool before_split:Bool want_merge:Bool want_split:Bool validator_list_hash_short:int32 catchain_seqno:int32 min_ref_mc_seqno:int32 is_key_block:Bool prev_key_block_seqno:int32 start_lt:int64 end_lt:int64 gen_utime:int53 vert_seqno:# prev_blocks:vector = blocks.Header; //blocks.shortData header:blocks.Header transactions:blocks.Header = blocks.BlockData; @@ -223,6 +232,8 @@ blocks.blockSignatures id:ton.blockIdExt signatures:(vector blocks.signature) = blocks.shardBlockLink id:ton.blockIdExt proof:bytes = blocks.ShardBlockLink; blocks.blockLinkBack to_key_block:Bool from:ton.blockIdExt to:ton.blockIdExt dest_proof:bytes proof:bytes state_proof:bytes = blocks.BlockLinkBack; blocks.shardBlockProof from:ton.blockIdExt mc_id:ton.blockIdExt links:(vector blocks.shardBlockLink) mc_proof:(vector blocks.blockLinkBack) = blocks.ShardBlockProof; +blocks.outMsgQueueSize id:ton.blockIdExt size:int32 = blocks.OutMsgQueueSize; +blocks.outMsgQueueSizes shards:(vector blocks.outMsgQueueSize) ext_msg_queue_size_limit:int32 = blocks.OutMsgQueueSizes; configInfo config:tvm.cell = ConfigInfo; @@ -257,6 +268,7 @@ getBip39Hints prefix:string = Bip39Hints; //raw.init initial_account_state:raw.initialAccountState = Ok; raw.getAccountState account_address:accountAddress = raw.FullAccountState; +raw.getAccountStateByTransaction account_address:accountAddress transaction_id:internal.transactionId = raw.FullAccountState; raw.getTransactions private_key:InputKey account_address:accountAddress from_transaction_id:internal.transactionId = raw.Transactions; raw.getTransactionsV2 private_key:InputKey account_address:accountAddress from_transaction_id:internal.transactionId count:# try_decode_messages:Bool = raw.Transactions; raw.sendMessage body:bytes = Ok; @@ -278,9 +290,13 @@ guessAccountRevision initial_account_state:InitialAccountState workchain_id:int3 guessAccount public_key:string rwallet_init_public_key:string = AccountRevisionList; getAccountState account_address:accountAddress = FullAccountState; +getAccountStateByTransaction account_address:accountAddress transaction_id:internal.transactionId = FullAccountState; +getShardAccountCell account_address:accountAddress = tvm.Cell; +getShardAccountCellByTransaction account_address:accountAddress transaction_id:internal.transactionId = tvm.Cell; createQuery private_key:InputKey address:accountAddress timeout:int32 action:Action initial_account_state:InitialAccountState = query.Info; -getConfigParam mode:# id:ton.blockIdExt param:# = ConfigInfo; +getConfigParam mode:# param:# = ConfigInfo; +getConfigAll mode:# = ConfigInfo; msg.decrypt input_key:InputKey data:msg.dataEncryptedArray = msg.DataDecryptedArray; msg.decryptWithProof proof:bytes data:msg.dataEncrypted = msg.Data; @@ -292,13 +308,16 @@ query.estimateFees id:int53 ignore_chksig:Bool = query.Fees; query.getInfo id:int53 = query.Info; smc.load account_address:accountAddress = smc.Info; +smc.loadByTransaction account_address:accountAddress transaction_id:internal.transactionId = smc.Info; smc.forget id:int53 = Ok; smc.getCode id:int53 = tvm.Cell; smc.getData id:int53 = tvm.Cell; smc.getState id:int53 = tvm.Cell; +smc.getRawFullAccountState id:int53 = raw.FullAccountState; smc.runGetMethod id:int53 method:smc.MethodId stack:vector = smc.RunResult; smc.getLibraries library_list:(vector int256) = smc.LibraryResult; +smc.getLibrariesExt list:(vector smc.LibraryQueryExt) = smc.LibraryResultExt; dns.resolve account_address:accountAddress name:string category:int256 ttl:int32 = dns.Resolved; @@ -313,9 +332,11 @@ blocks.getMasterchainInfo = blocks.MasterchainInfo; blocks.getShards id:ton.blockIdExt = blocks.Shards; blocks.lookupBlock mode:int32 id:ton.blockId lt:int64 utime:int32 = ton.BlockIdExt; blocks.getTransactions id:ton.blockIdExt mode:# count:# after:blocks.accountTransactionId = blocks.Transactions; +blocks.getTransactionsExt id:ton.blockIdExt mode:# count:# after:blocks.accountTransactionId = blocks.TransactionsExt; blocks.getBlockHeader id:ton.blockIdExt = blocks.Header; blocks.getMasterchainBlockSignatures seqno:int32 = blocks.BlockSignatures; blocks.getShardBlockProof id:ton.blockIdExt mode:# from:mode.0?ton.blockIdExt = blocks.ShardBlockProof; +blocks.getOutMsgQueueSizes mode:# wc:mode.0?int32 shard:mode.0?int64 = blocks.OutMsgQueueSizes; onLiteServerQueryResult id:int64 bytes:bytes = Ok; onLiteServerQueryError id:int64 error:error = Ok; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index 075c5a3a..10b9ed8d 100644 Binary files a/tl/generate/scheme/tonlib_api.tlo and b/tl/generate/scheme/tonlib_api.tlo differ diff --git a/tl/generate/tl_writer_hpp.cpp b/tl/generate/tl_writer_hpp.cpp index 028aa927..cdcf26c4 100644 --- a/tl/generate/tl_writer_hpp.cpp +++ b/tl/generate/tl_writer_hpp.cpp @@ -27,13 +27,14 @@ bool TD_TL_writer_hpp::is_documentation_generated() const { } int TD_TL_writer_hpp::get_additional_function_type(const std::string &additional_function_name) const { - assert(additional_function_name == "downcast_call"); + assert(additional_function_name == "downcast_call" || additional_function_name == "downcast_construct"); return 2; } std::vector TD_TL_writer_hpp::get_additional_functions() const { std::vector additional_functions; additional_functions.push_back("downcast_call"); + additional_functions.push_back("downcast_construct"); return additional_functions; } @@ -194,7 +195,7 @@ std::string TD_TL_writer_hpp::gen_fetch_switch_end() const { std::string TD_TL_writer_hpp::gen_additional_function(const std::string &function_name, const tl::tl_combinator *t, bool is_function) const { - assert(function_name == "downcast_call"); + assert(function_name == "downcast_call" || function_name == "downcast_construct"); return ""; } @@ -202,24 +203,40 @@ std::string TD_TL_writer_hpp::gen_additional_proxy_function_begin(const std::str const tl::tl_type *type, const std::string &class_name, int arity, bool is_function) const { - assert(function_name == "downcast_call"); - return "/**\n" - " * Calls specified function object with the specified object downcasted to the most-derived type.\n" - " * \\param[in] obj Object to pass as an argument to the function object.\n" - " * \\param[in] func Function object to which the object will be passed.\n" - " * \\returns whether function object call has happened. Should always return true for correct parameters.\n" - " */\n" - "template \n" - "bool downcast_call(" + - class_name + - " &obj, const T &func) {\n" - " switch (obj.get_id()) {\n"; + if (function_name == "downcast_call") { + return "/**\n" + " * Calls specified function object with the specified object downcasted to the most-derived type.\n" + " * \\param[in] obj Object to pass as an argument to the function object.\n" + " * \\param[in] func Function object to which the object will be passed.\n" + " * \\returns whether function object call has happened. Should always return true for correct parameters.\n" + " */\n" + "template \n" + "bool downcast_call(" + + class_name + + " &obj, const T &func) {\n" + " switch (obj.get_id()) {\n"; + } + if (function_name == "downcast_construct") { + return "/**\n" + "* Constructs tl_object_ptr with the object of the same type as the specified object, calls the specified " + "function.\n" + " * \\param[in] obj Object to get the type from.\n" + " * \\param[in] func Function object to which the new object will be passed.\n" + " * \\returns whether function object call has happened. Should always return true for correct parameters.\n" + "*/" + "template \n" + "bool downcast_construct(" + + class_name + + " &obj, const T &func) {\n" + "switch (obj.get_id()) {"; + } + assert(false); } std::string TD_TL_writer_hpp::gen_additional_proxy_function_case(const std::string &function_name, const tl::tl_type *type, const std::string &class_name, int arity) const { - assert(function_name == "downcast_call"); + //assert(function_name == "downcast_call"); assert(false); return ""; } @@ -227,18 +244,28 @@ std::string TD_TL_writer_hpp::gen_additional_proxy_function_case(const std::stri std::string TD_TL_writer_hpp::gen_additional_proxy_function_case(const std::string &function_name, const tl::tl_type *type, const tl::tl_combinator *t, int arity, bool is_function) const { - assert(function_name == "downcast_call"); - return " case " + gen_class_name(t->name) + - "::ID:\n" - " func(static_cast<" + - gen_class_name(t->name) + - " &>(obj));\n" - " return true;\n"; + if (function_name == "downcast_call") { + return " case " + gen_class_name(t->name) + + "::ID:\n" + " func(static_cast<" + + gen_class_name(t->name) + + " &>(obj));\n" + " return true;\n"; + } + if (function_name == "downcast_construct") { + return " case " + gen_class_name(t->name) + + "::ID:\n" + " func(create_tl_object<" + + gen_class_name(t->name) + + ">());\n" + " return true;\n"; + } + assert(false); } std::string TD_TL_writer_hpp::gen_additional_proxy_function_end(const std::string &function_name, const tl::tl_type *type, bool is_function) const { - assert(function_name == "downcast_call"); + assert(function_name == "downcast_call" || function_name == "downcast_construct"); return " default:\n" " return false;\n" " }\n" diff --git a/tl/tl/tl_jni_object.cpp b/tl/tl/tl_jni_object.cpp index e7e69789..26a94d62 100644 --- a/tl/tl/tl_jni_object.cpp +++ b/tl/tl/tl_jni_object.cpp @@ -115,8 +115,9 @@ static size_t get_utf8_from_utf16_length(const jchar *p, jsize len) { for (jsize i = 0; i < len; i++) { unsigned int cur = p[i]; if ((cur & 0xF800) == 0xD800) { + ++i; if (i < len) { - unsigned int next = p[++i]; + unsigned int next = p[i]; if ((next & 0xFC00) == 0xDC00 && (cur & 0x400) == 0) { result += 4; continue; diff --git a/tl/tl/tl_json.h b/tl/tl/tl_json.h index 66c639b5..8eee3aad 100644 --- a/tl/tl/tl_json.h +++ b/tl/tl/tl_json.h @@ -108,12 +108,13 @@ inline Status from_json(std::int32_t &to, JsonValue from) { inline Status from_json(bool &to, JsonValue from) { if (from.type() != JsonValue::Type::Boolean) { int32 x; + auto type = from.type(); auto status = from_json(x, std::move(from)); if (status.is_ok()) { to = x != 0; return Status::OK(); } - return Status::Error(PSLICE() << "Expected bool, got " << from.type()); + return Status::Error(PSLICE() << "Expected bool, got " << type); } to = from.get_boolean(); return Status::OK(); @@ -277,8 +278,7 @@ std::enable_if_t::value, Status> from_json(ton::tl_obj DowncastHelper helper(constructor); Status status; - bool ok = downcast_call(static_cast(helper), [&](auto &dummy) { - auto result = ton::create_tl_object>(); + bool ok = downcast_construct(static_cast(helper), [&](auto result) { status = from_json(*result, object); to = std::move(result); }); diff --git a/tolk-tester/tests/a10.tolk b/tolk-tester/tests/a10.tolk new file mode 100644 index 00000000..9d24f38d --- /dev/null +++ b/tolk-tester/tests/a10.tolk @@ -0,0 +1,159 @@ +import "@stdlib/tvm-lowlevel" + +fun pair_first(p: [X, Y]): X asm "FIRST"; + +fun one(dummy: tuple?) { + return 1; +} + +fun main(a: int, x: int) { + var y: int = 0; + var z: int = 0; + while ((y = x * x) > a) { + x -= 1; + z = one(null); + } + return (y, z); +} + +fun throwIfLt10(x: int): void { + if (x > 10) { + return; + } + throw 234; + return; +} + +@method_id(88) +fun test88(x: int) { + try { + var x: void = throwIfLt10(x); + return 0; + } catch(code) { + return code; + } +} + +@method_id(89) +fun test89(last: int): (int, int, int, int) { + var t: tuple = createEmptyTuple(); + t.tuplePush(1); + t.tuplePush(2); + t.tuplePush(3); + t.tuplePush(last); + return (t.tupleAt(0), t.tupleAt(t.tupleSize() - 1), t.tupleFirst(), t.tupleLast()); +} + +@pure fun get10() { return 10; } + +@method_id(91) +fun touchCodegen2() { + var f = get10(); + f.stackMoveToTop(); + return f; +} + +@method_id(92) +fun testDumpDontPolluteStack() { + var f = get10(); + f.debugPrint(); + debugPrint(10); + var s = "asdf"; + s.debugPrintString(); + debugDumpStack(); + debugPrintString("my"); + return (f, getRemainingBitsCount(s)); +} + +@method_id(93) +fun testStartBalanceCodegen1() { + var t = getMyOriginalBalanceWithExtraCurrencies(); + var first = t.pair_first(); + return first; +} + +@method_id(94) +fun testStartBalanceCodegen2() { + var first = getMyOriginalBalance(); + return first; +} + +global cur: [int, int, int]; +global next: [int, int, int]; + +@method_id(95) +fun test95() { + cur = [1, 2, 3]; + next = [2, 3, 4]; + (cur, next) = (next, [3, 4, 5]); + return (cur, next); +} + +/** + method_id | in | out +@testcase | 0 | 101 15 | 100 1 +@testcase | 0 | 101 14 | 100 1 +@testcase | 0 | 101 10 | 100 0 +@testcase | 0 | 100 10 | 100 0 +@testcase | 0 | 100 10 | 100 0 +@testcase | 88 | 5 | 234 +@testcase | 88 | 50 | 0 +@testcase | 89 | 4 | 1 4 1 4 +@testcase | 91 | | 10 +@testcase | 92 | | 10 32 +@testcase | 95 | | [ 2 3 4 ] [ 3 4 5 ] + +@fif_codegen +""" + touchCodegen2 PROC:<{ + // + get10 CALLDICT // f + }> +""" + +@fif_codegen +""" + testDumpDontPolluteStack PROC:<{ + ... + DUMPSTK + x{6d79} PUSHSLICE // f s '5 + STRDUMP DROP + SBITS // f '6 + }> +""" + +@fif_codegen +""" + testStartBalanceCodegen1 PROC:<{ + // + BALANCE // t + FIRST // first + }> +""" + +@fif_codegen +""" + testStartBalanceCodegen2 PROC:<{ + // + BALANCE + FIRST // first + }> +""" + +@fif_codegen +""" + test95 PROC:<{ + ... + next GETGLOB // g_next + 3 PUSHINT // g_next '14=3 + 4 PUSHINT // g_next '14=3 '15=4 + 5 PUSHINT // g_next '14=3 '15=4 '16=5 + TRIPLE // '10 '11 + SWAP + cur SETGLOB + next SETGLOB + cur GETGLOB // g_cur + next GETGLOB // g_cur g_next + }> +""" +*/ diff --git a/tolk-tester/tests/a6.tolk b/tolk-tester/tests/a6.tolk new file mode 100644 index 00000000..94494546 --- /dev/null +++ b/tolk-tester/tests/a6.tolk @@ -0,0 +1,83 @@ +fun f(a: int, b: int, c: int, d: int, e: int, f: int): (int, int) { + // solve a 2x2 linear equation + var D: int = a*d - b*c;;;; var Dx: int = e*d-b*f ;;;; var Dy: int = a * f - e * c; + __expect_type(D, "int"); + __expect_type(D*D, "int"); + __expect_type(calc_phi, "() -> int"); + return (Dx/D,Dy/D); +};;;; + +fun calc_phi(): int { + var n = 1; + repeat (70) { n*=10; }; + var p= 1; + var `q`=1; + _=`q`; + do { + (p,q)=(q,p+q); + } while (q <= n); //;; + return mulDivRound(p, n, q); +} + +fun calc_sqrt2(): int { + var n = 1; + repeat (70) { n *= 10; } + var p = 1; + var q = 1; + do { + var t = p + q; + (p, q) = (q, t + q); + } while (q <= n); + return mulDivRound(p, n, q); +} + +fun calc_root(m: int) { + var base: int=1; + repeat(70) { base *= 10; } + var (a, b, c) = (1,0,-m); + var (p1, q1, p2, q2) = (1, 0, 0, 1); + do { + var k: int=-1; + var (a1, b1, c1) = (0, 0, 0); + do { + k+=1; + (a1, b1, c1) = (a, b, c); + c+=b; + c += b += a; + } while (c <= 0); + (a, b, c) = (-c1, -b1, -a1); + (p1, q1) = (k * p1+q1, p1); + (p2, q2) = (k * p2+q2, p2); + } while (p1 <= base); + return (p1, q1, p2, q2); +} + +fun ataninv(base: int, q: int): int { // computes base*atan(1/q) + base=base~/q; + q*=-q; + var sum: int = 0; + var n: int = 1; + do { + sum += base~/n; + base = base~/q; + n += 2; + } while (base != 0); + return sum; +} + +fun calc_pi(): int { + var base: int = 64; + repeat (70) { base *= 10; } + return (ataninv(base << 2, 5) - ataninv(base, 239))~>>4; +} + +fun main(): int { + return calc_pi(); +} + +/** + method_id | in | out +@testcase | 0 | | 31415926535897932384626433832795028841971693993751058209749445923078164 + +@code_hash 84337043972311674339187056298873613816389434478842780265748859098303774481976 +*/ diff --git a/tolk-tester/tests/a6_1.tolk b/tolk-tester/tests/a6_1.tolk new file mode 100644 index 00000000..8079972b --- /dev/null +++ b/tolk-tester/tests/a6_1.tolk @@ -0,0 +1,22 @@ +fun main(a: int, b: int, c: int, d: int, e: int, f: int): (int, int) { + var D: int = a * d - b * c; + var Dx: int = e * d - b * f; + var Dy: int = a * f - e * c; + return (Dx / D, Dy / D); +} + +@method_id(101) +fun testDivMod(x: int, y: int) { + return (divMod(x, y), modDiv(x, y), mulDivMod(x, y, 10)); +} + +/** + method_id | in | out +@testcase | 0 | 1 1 1 -1 10 6 | 8 2 +@testcase | 0 | 817 -31 624 -241 132272 272276 | 132 -788 +@testcase | 0 | -886 562 498 -212 -36452 -68958 | -505 -861 +@testcase | 0 | 448 -433 -444 792 150012 -356232 | -218 -572 +@testcase | 0 | -40 -821 433 -734 -721629 -741724 | -206 889 +@testcase | 0 | -261 -98 -494 868 -166153 733738 | 263 995 +@testcase | 101 | 112 3 | 37 1 1 37 33 6 +*/ diff --git a/tolk-tester/tests/a6_5.tolk b/tolk-tester/tests/a6_5.tolk new file mode 100644 index 00000000..43fd59c5 --- /dev/null +++ b/tolk-tester/tests/a6_5.tolk @@ -0,0 +1,26 @@ +@deprecated +fun twice(f: int -> int, x: int) { + return f (f (x)); +} + +fun sqr(x: int) { + return x * x; +} + +fun main(x: int): int { + var f = sqr; + return twice(f, x) * f(x); +} + +@method_id(4) +fun pow6(x: int): int { + return twice(sqr, x) * sqr(x); +} + +/** + method_id | in | out +@testcase | 0 | 3 | 729 +@testcase | 0 | 10 | 1000000 +@testcase | 4 | 3 | 729 +@testcase | 4 | 10 | 1000000 +*/ diff --git a/tolk-tester/tests/a7.tolk b/tolk-tester/tests/a7.tolk new file mode 100644 index 00000000..1c0ae2eb --- /dev/null +++ b/tolk-tester/tests/a7.tolk @@ -0,0 +1,24 @@ +fun main() { } +@method_id(1) +fun steps(x: int): int { + var n = 0; + while (x > 1) { + n += 1; + if (x & 1) { + x = 3 * x + 1; + } else { + x >>= 1; + } + } + return n; +} + +/** + method_id | in | out +@testcase | 1 | 1 | 0 +@testcase | 1 | 2 | 1 +@testcase | 1 | 5 | 5 +@testcase | 1 | 19 | 20 +@testcase | 1 | 27 | 111 +@testcase | 1 | 100 | 25 +*/ diff --git a/tolk-tester/tests/allow_post_modification.tolk b/tolk-tester/tests/allow_post_modification.tolk new file mode 100644 index 00000000..e20e8218 --- /dev/null +++ b/tolk-tester/tests/allow_post_modification.tolk @@ -0,0 +1,151 @@ +fun unsafe_tuple(x: X): tuple + asm "NOP"; + +fun inc(x: int, y: int): (int, int) { + return (x + y, y * 10); +} +fun `~inc`(mutate self: int, y: int): int { + val (newX, newY) = inc(self, y); + self = newX; + return newY; +} + +fun eq(v: X): X { return v; } +fun eq2(v: (int, int)) { return v; } +fun mul2(mutate dest: int, v: int): int { dest = v*2; return dest; } +fun multens(mutate self: (int, int), v: (int, int)): (int, int) { var (f, s) = self; var (m1, m2) = v; self = (f*m1, s*m2); return self; } + +@method_id(11) +fun test_return(x: int): (int, int, int, int, int, int, int) { + return (x, x.`~inc`(x / 20), x, x = x * 2, x, x += 1, x); +} + +@method_id(12) +fun test_assign(x: int): (int, int, int, int, int, int, int) { + var (x1: int, x2: int, x3: int, x4: int, x5: int, x6: int, x7: int) = (x, x.`~inc`(x / 20), x, x=x*2, x, x+=1, x); + return (x1, x2, x3, x4, x5, x6, x7); +} + +@method_id(13) +fun test_tuple(x: int): tuple { + var t: tuple = unsafe_tuple([x, x.`~inc`(x / 20), x, x = x * 2, x, x += 1, x]); + return t; +} + +@method_id(14) +fun test_tuple_assign(x: int): (int, int, int, int, int, int, int) { + var [x1: int, x2: int, x3: int, x4: int, x5: int, x6: int, x7: int] = [x, x.`~inc`(x / 20), x, x = x * 2, x, x += 1, x]; + return (x1, x2, x3, x4, x5, x6, x7); +} + +fun foo1(x1: int, x2: int, x3: int, x4: int, x5: int, x6: int, x7: int): (int, int, int, int, int, int, int) { + return (x1, x2, x3, x4, x5, x6, x7); +} + +@method_id(15) +fun test_call_1(x: int): (int, int, int, int, int, int, int) { + return foo1(x, x.`~inc`(x / 20), x, x = x * 2, x, x += 1, x); +} + +fun foo2(x1: int, x2: int, x3456: (int, int, int, int), x7: int): (int, int, int, int, int, int, int) { + var (x3: int, x4: int, x5: int, x6: int) = x3456; + return (x1, x2, x3, x4, x5, x6, x7); +} + +@method_id(16) +fun test_call_2(x: int): (int, int, int, int, int, int, int) { + return foo2(x, x.`~inc`(x / 20), (x, x = x * 2, x, x += 1), x); +} + +fun asm_func(x1: int, x2: int, x3: int, x4: int, x5: int, x6: int, x7: int): (int, int, int, int, int, int, int) + asm (x4 x5 x6 x7 x1 x2 x3->0 1 2 3 4 5 6) "NOP"; + +@method_id(17) +fun test_call_asm_old(x: int): (int, int, int, int, int, int, int) { + return asm_func(x, x += 1, x, x, x.`~inc`(x / 20), x, x = x * 2); +} + +@method_id(18) +fun test_call_asm_new(x: int): (int, int, int, int, int, int, int) { + return asm_func(x, x.`~inc`(x / 20), x, x = x * 2, x, x += 1, x); +} + +global xx: int; +@method_id(19) +fun test_global(x: int) { + xx = x; + return (x, xx, xx.`~inc`(xx / 20), eq(xx += (x *= 0)), xx = xx * 2, xx, xx += 1, xx, x); +} + +@method_id(20) +fun test_if_else(x: int): (int, int, int, int, int) { + if (x > 10) { + return (x.`~inc`(8), x + 1, x = 1, x <<= 3, x); + } else { + xx = 9; + return (x, x.`~inc`(-4), x.`~inc`(-1), (x >= 1) as int, x = x + xx); + } +} + +@method_id(21) +fun test_assign_with_inner(x: int) { + var result = (x, x += 10, [x, x += 20, eq(x -= 50), x], eq2((x, x *= eq(x /= 2)))); + return result; +} + +@method_id(22) +fun test_assign_with_mutate(x: int) { + var (result, _) = ((x, mul2(mutate x, x += 5), x.`~inc`(mul2(mutate x, x)), x), 0); + return result; +} + +@method_id(23) +fun test_assign_tensor(x: (int, int)) { + var fs = (0, 0); + return (x, x = (20, 30), fs = x.multens((1, 2)), fs.multens(multens(mutate x, (-1, -1))), x, fs); +} + +global fs: (int, int); +@method_id(24) +fun test_assign_tensor_global(x: (int, int)) { + fs = (0, 0); + return (x, x = (20, 30), fs = x.multens((1, 2)), fs.multens(multens(mutate x, (-1, -1))), x, fs); +} + +fun main() { +} + +/** + method_id | in | out +@testcase | 11 | 100 | 100 50 105 210 210 211 211 +@testcase | 12 | 100 | 100 50 105 210 210 211 211 +@testcase | 13 | 100 | [ 100 50 105 210 210 211 211 ] +@testcase | 14 | 100 | 100 50 105 210 210 211 211 +@testcase | 15 | 100 | 100 50 105 210 210 211 211 +@testcase | 16 | 100 | 100 50 105 210 210 211 211 +@testcase | 17 | 100 | 101 50 106 212 100 101 101 +@testcase | 18 | 100 | 210 210 211 211 100 50 105 +@testcase | 19 | 100 | 100 100 50 105 210 210 211 211 0 +@testcase | 20 | 80 | 80 89 1 8 8 +@testcase | 20 | 9 | 9 -40 -10 -1 13 +@testcase | 21 | 100 | 100 110 [ 110 130 80 80 ] 80 3200 +@testcase | 22 | 100 | 100 210 4200 630 +@testcase | 23 | 1 1 | 1 1 20 30 20 60 -400 -3600 -20 -60 -400 -3600 +@testcase | 24 | 1 1 | 1 1 20 30 20 60 -400 -3600 -20 -60 -400 -3600 + +@fif_codegen +""" + ~inc PROC:<{ + // self y + inc CALLDICT // self newY + }> +""" + +@fif_codegen +""" + test_assign_tensor_global PROC:<{ + // x.0 x.1 +""" + +@code_hash 61280273714870328160131559159866470128402169974050439159015534193532598351244 +*/ diff --git a/tolk-tester/tests/asm_arg_order.tolk b/tolk-tester/tests/asm_arg_order.tolk new file mode 100644 index 00000000..b96e09ec --- /dev/null +++ b/tolk-tester/tests/asm_arg_order.tolk @@ -0,0 +1,139 @@ +@pure +fun empty_tuple2(): tuple +asm "NIL"; +@pure +fun tpush2(mutate self: tuple, x: X): void +asm "TPUSH"; + +@pure +fun asm_func_1(x: int, y: int, z: int): tuple +asm "3 TUPLE"; +@pure +fun asm_func_2(x: int, y: int, z: int): tuple +asm (z y x -> 0) "3 TUPLE"; +@pure +fun asm_func_3(x: int, y: int, z: int): tuple +asm (y z x -> 0) "3 TUPLE"; +@pure +fun asm_func_4(a: int, b: (int, (int, int)), c: int): tuple +asm (b a c -> 0) "5 TUPLE"; + +@pure +fun asm_func_modify(mutate self: tuple, b: int, c: int): void +asm (c b self) "SWAP TPUSH SWAP TPUSH"; + +global t: tuple; + +fun foo(x: int): int { + t.tpush2(x); + return x * 10; +} + +@method_id(11) +fun test_old_1(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = asm_func_1(foo(11), foo(22), foo(33)); + return (t, t2); +} + +@method_id(12) +fun test_old_2(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = asm_func_2(foo(11), foo(22), foo(33)); + return (t, t2); +} + +@method_id(13) +fun test_old_3(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = asm_func_3(foo(11), foo(22), foo(33)); + return (t, t2); +} + +@method_id(14) +fun test_old_4(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = empty_tuple2(); + // This actually computes left-to-right even without compute-asm-ltr + t2 = asm_func_4(foo(11), (foo(22), (foo(33), foo(44))), foo(55)); + return (t, t2); +} + +@method_id(15) +fun test_old_modify(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = empty_tuple2(); + t2.asm_func_modify(foo(22), foo(33)); + return (t, t2); +} + +@method_id(16) +fun test_old_dot(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = foo(11).asm_func_3(foo(22), foo(33)); + return (t, t2); +} + +@method_id(21) +fun test_new_1(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = asm_func_1(foo(11), foo(22), foo(33)); + return (t, t2); +} + +@method_id(22) +fun test_new_2(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = asm_func_2(foo(11), foo(22), foo(33)); + return (t, t2); +} + +@method_id(23) +fun test_new_3(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = asm_func_3(foo(11), foo(22), foo(33)); + return (t, t2); +} + +@method_id(24) +fun test_new_4(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = asm_func_4(foo(11), (foo(22), (foo(33), foo(44))), foo(55)); + return (t, t2); +} + +@method_id(25) +fun test_new_modify(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = empty_tuple2(); + t2.asm_func_modify(foo(22), foo(33)); + return (t, t2); +} + +@method_id(26) +fun test_new_dot(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = foo(11).asm_func_3(foo(22), foo(33)); + return (t, t2); +} + +fun main() { +} + +/** + method_id | in | out +@testcase | 11 | | [ 11 22 33 ] [ 110 220 330 ] +@testcase | 12 | | [ 11 22 33 ] [ 330 220 110 ] +@testcase | 13 | | [ 11 22 33 ] [ 220 330 110 ] +@testcase | 14 | | [ 11 22 33 44 55 ] [ 220 330 440 110 550 ] +@testcase | 15 | | [ 22 33 ] [ 220 330 ] +@testcase | 16 | | [ 11 22 33 ] [ 220 330 110 ] +@testcase | 21 | | [ 11 22 33 ] [ 110 220 330 ] +@testcase | 22 | | [ 11 22 33 ] [ 330 220 110 ] +@testcase | 23 | | [ 11 22 33 ] [ 220 330 110 ] +@testcase | 24 | | [ 11 22 33 44 55 ] [ 220 330 440 110 550 ] +@testcase | 25 | | [ 22 33 ] [ 220 330 ] +@testcase | 26 | | [ 11 22 33 ] [ 220 330 110 ] + +@code_hash 93068291567112337250118419287631047120002003622184251973082208096953112184588 +*/ diff --git a/tolk-tester/tests/assignment-tests.tolk b/tolk-tester/tests/assignment-tests.tolk new file mode 100644 index 00000000..bb647652 --- /dev/null +++ b/tolk-tester/tests/assignment-tests.tolk @@ -0,0 +1,250 @@ +fun extractFromTypedTuple(params: [int]) { + var [payload: int] = params; + return payload + 10; +} + +@method_id(101) +fun test101(x: int) { + var params = [x]; + return extractFromTypedTuple(params); +} + +fun autoInferIntNull(x: int) { + if (x > 10) { return null; } + return x; +} + +fun typesAsIdentifiers(builder: builder) { + var int = 1; + var cell = builder.endCell(); + var slice = cell.beginParse(); + { + var cell: cell = cell; + var tuple: tuple = createEmptyTuple(); + var bool: bool = tuple.tupleAt(0) > 0; + } + return int; +} + +global callOrder: tuple; + +fun getTensor_12() { + callOrder.tuplePush(100); + return (1, 2); +} +fun getTensor_1X(x: int) { + callOrder.tuplePush(101); + return (1, x); +} +fun getTuple_12() { + callOrder.tuplePush(110); + return [1, 2]; +} +fun getTuple_1X(x: int) { + callOrder.tuplePush(111); + return [1, x]; +} +fun getUntypedTuple_12() { + callOrder.tuplePush(120); + var t = createEmptyTuple(); t.tuplePush(1); t.tuplePush(2); + return t; +} +fun getUntypedTuple_1X(x: int) { + callOrder.tuplePush(121); + var t = createEmptyTuple(); t.tuplePush(1); t.tuplePush(x); + return t; +} +fun getIntValue5() { + callOrder.tuplePush(10); + return 5; +} +fun getIntValueX(x: int) { + callOrder.tuplePush(11); + return x; +} + +@method_id(102) +fun test102() { + callOrder = createEmptyTuple(); + var x = 0; + getTensor_12().0 = getIntValue5(); + getTensor_1X(5).1 = getIntValue5(); + getTensor_1X(x = 10).0 = getIntValueX(x); + return (callOrder, x); +} + +@method_id(103) +fun test103() { + callOrder = createEmptyTuple(); + var x = 0; + getTuple_12().0 = getIntValue5(); + getTuple_1X(5).1 = getIntValue5(); + getTuple_1X(x = 10).0 = getIntValueX(x); + return (callOrder, x); +} + +@method_id(104) +fun test104() { + callOrder = createEmptyTuple(); + var x = 0; + getUntypedTuple_12().0 = getIntValue5(); + getUntypedTuple_1X(5).1 = getIntValue5(); + getUntypedTuple_1X(x = 10).0 = getIntValueX(x); + return (callOrder, x); +} + +@method_id(105) +fun test105() { + callOrder = createEmptyTuple(); + getTensor_12().0 = getTensor_1X(getIntValue5()).1 = getIntValueX(getTensor_12().1); + return callOrder; +} + +@method_id(106) +fun test106() { + callOrder = createEmptyTuple(); + getTuple_12().0 = getTuple_1X(getIntValue5()).1 = getIntValueX(getTuple_12().1); + return callOrder; +} + +global t107: (int, int); + +@method_id(107) +fun test107() { + ((t107 = (1, 2)).0, (t107 = (3, 4)).1) = (5, 6); + return t107; +} + +global g108: int; +fun assertEq(a: int, b: int) { + assert(a == b, 10); + return b; +} + +@method_id(108) +fun test108() { + callOrder = createEmptyTuple(); + g108 = 0; + getTensor_1X(g108 = 8).1 = assertEq(g108, 8); + return (callOrder, g108); +} + +@method_id(109) +fun test109() { + callOrder = createEmptyTuple(); + var x = 0; + [getTuple_12().0, getTuple_1X(x = getIntValue5()).1, getTuple_1X(x += 10).0] = [getIntValue5(), getIntValue5(), getIntValueX(x)]; + return (callOrder, x); +} + +global g110: int; +global t110: (int, int); + +@method_id(110) +fun test110() { + callOrder = createEmptyTuple(); + var xy = [0, 0]; + [xy.0, getTuple_1X(g110 = 8).0] = [g110 += 5, getIntValueX(g110 += 10)]; + [xy.1, getTuple_1X((t110 = (8, 9)).0).1] = [t110.0 += 5, getIntValueX(t110.1 += 10)]; + return (xy, callOrder, g110, t110); +} + +@method_id(111) +fun test111() { + callOrder = createEmptyTuple(); + var z = -1; + var xy = [0, z = 0]; + var rhs = [getIntValueX(xy.1 += 10), xy.1, xy.0, z += 50]; + [xy.0, getTuple_1X(g110 = 8 + getIntValueX(xy.1)).0, xy.1, z] = rhs; + return (xy, g110, callOrder, z); +} + +@method_id(112) +fun test112() { + var xy = [1, 2]; + ((((xy))).0, ((xy.1))) = ((xy).1, ((xy.0))); + return xy; +} + +@method_id(113) +fun test113() { + var (a, t, z) = (1, [2,3], (-1,-1)); + (a, t, a, z, t.1, z.1) = (10, [a,12], 13, (a, t.1), 14, t.1); + return (a, t, z); +} + +global g114: int; +global t114: [int, int]; +global z114: (int, int); + +@method_id(114) +fun test114() { + g114 = 1; + t114 = [2, 3]; + (g114, t114, g114, z114, t114.1, z114.1) = (10, [g114,12], 13, (g114, t114.1), 14, t114.1); + return (g114, t114, z114); +} + +@method_id(115) +fun test115() { + callOrder = createEmptyTuple(); + var x = 0; + var y = 0; + [getTensor_1X(x = 5).0, y] = getTuple_1X(x = 9); + return (callOrder, x, y); +} + +@method_id(116) +fun test116() { + var (a,b,c,d) = (0,0,0,0); + var rhs = [1, 2, 3, 4]; + var rhs2 = ([a,b,c,d] = rhs); + __expect_type(rhs2, "[int, int, int, int]"); + return (a, b, c, d, rhs2); +} + + + +fun main(value: int) { + var (x: int?, y) = (autoInferIntNull(value), autoInferIntNull(value * 2)); + if (x == null && y == null) { return null; } + return x == null || y == null ? -1 : x! + y!; +} + +/** +@testcase | 0 | 3 | 9 +@testcase | 0 | 6 | -1 +@testcase | 0 | 11 | (null) +@testcase | 101 | 78 | 88 +@testcase | 102 | | [ 100 10 101 10 101 11 ] 10 +@testcase | 103 | | [ 110 10 111 10 111 11 ] 10 +@testcase | 104 | | [ 120 10 121 10 121 11 ] 10 +@testcase | 105 | | [ 100 10 101 100 11 ] +@testcase | 106 | | [ 110 10 111 110 11 ] +@testcase | 107 | | 3 4 +@testcase | 108 | | [ 101 ] 8 +@testcase | 109 | | [ 110 10 111 111 10 10 11 ] 15 +@testcase | 110 | | [ 13 13 ] [ 111 11 111 11 ] 23 13 19 +@testcase | 111 | | [ 10 0 ] 18 [ 11 11 111 ] 50 +@testcase | 112 | | [ 2 1 ] +@testcase | 113 | | 13 [ 1 14 ] 1 3 +@testcase | 114 | | 13 [ 1 14 ] 1 3 +@testcase | 115 | | [ 101 111 ] 9 9 +@testcase | 116 | | 1 2 3 4 [ 1 2 3 4 ] + + +@fif_codegen +""" + test116 PROC:<{ + // + 1 PUSHINT // '10=1 + 2 PUSHINT // '10=1 '11=2 + 3 PUSHINT // '10=1 '11=2 '12=3 + 4 PUSHINT // '10=1 '11=2 '12=3 '13=4 + 4 TUPLE // rhs + DUP // rhs rhs + 4 UNTUPLE // rhs2 a b c d + 4 ROLL // a b c d rhs2 + }> +""" +*/ diff --git a/tolk-tester/tests/bit-operators.tolk b/tolk-tester/tests/bit-operators.tolk new file mode 100644 index 00000000..b0106883 --- /dev/null +++ b/tolk-tester/tests/bit-operators.tolk @@ -0,0 +1,204 @@ +fun lshift(): bool { + return (1 << 0) == 1; +} + +fun rshift(): bool { + return (1 >> 0) == 1; +} + +fun lshift_var(i: int): bool { + return (1 << i) == 1; +} + +fun rshift_var(i: int): bool { + return (1 >> i) == 1; +} + +fun main(x: int): bool { + if (x == 0) { + return lshift(); + } else if (x == 1) { + return rshift(); + } else if (x == 2) { + return lshift_var(0); + } else if (x == 3) { + return rshift_var(0); + } else if (x == 4) { + return lshift_var(1); + } else { + return rshift_var(1); + } +} + +@method_id(11) +fun is_claimed(index: int): bool { + var claim_bit_index: int = index % 256; + var mask: int = 1 << claim_bit_index; + return (255 & mask) == mask; +} + +@method_id(12) +fun bit_not(i: int, b: bool): (int, bool, bool, bool, int, bool) { + var i2 = ~i; + var b2 = !b; + var (i3: int, b3: bool) = (i2, b2); + return (i3, b3, !i, !b, ~~~i, !!!b); +} + +@method_id(13) +fun boolWithBitwiseConst() { + var found = true; + return (found & false, found | true, found ^ true, found & found); +} + +global g14: int; +fun getBool() { return (g14 += 1) > 2; } + +@method_id(14) +fun boolWithBitwise(b: bool) { + g14 = 0; + return (b & getBool(), !b & getBool(), b | getBool(), !b | getBool(), b ^ getBool(), !b & getBool(), g14); +} + +@method_id(15) +fun boolWithBitwiseSet(b1: bool, b2: bool) { + b1 &= b2; + b2 |= true; + b1 |= b1 == false; + b2 ^= (b1 ^= b2); + return (b1, b2); +} + +@method_id(16) +fun testDoUntilCodegen(i: bool, n: int) { + var cnt = 0; + do { cnt += 1; } while (i); + do { cnt += 1; } while (!!i); + do { cnt += 1; } while (n); + return (cnt, !i, !n); +} + +@method_id(17) +fun testConstNegateCodegen() { + return (!0, !1, !true, !false, !!true, !!false); +} + +@method_id(18) +fun testBoolNegateOptimized(x: bool) { + return (x, !x, !!x, !!!x, !!!!true); +} + +fun eqX(x: bool) { return x; } + +@method_id(19) +fun testBoolCompareOptimized(x: bool) { + return (x == true, x != true, eqX(x) == false, eqX(x) != false, !!(x == !false)); +} + + + +/** + method_id | in | out +@testcase | 0 | 0 | -1 +@testcase | 0 | 1 | -1 +@testcase | 0 | 2 | -1 +@testcase | 0 | 3 | -1 +@testcase | 0 | 4 | 0 +@testcase | 0 | 5 | 0 +@testcase | 11 | 0 | -1 +@testcase | 11 | 1 | -1 +@testcase | 11 | 256 | -1 +@testcase | 11 | 8 | 0 +@testcase | 12 | 0 0 | -1 -1 -1 -1 -1 -1 +@testcase | 12 | -1 -1 | 0 0 0 0 0 0 +@testcase | 12 | 7 0 | -8 -1 0 -1 -8 -1 +@testcase | 14 | -1 | 0 0 -1 -1 0 0 6 +@testcase | 14 | 0 | 0 0 -1 -1 -1 -1 6 +@testcase | 15 | -1 -1 | 0 -1 +@testcase | 15 | -1 0 | 0 -1 +@testcase | 16 | 0 0 | 3 -1 -1 +@testcase | 17 | | -1 0 0 -1 -1 0 +@testcase | 18 | 0 | 0 -1 0 -1 -1 +@testcase | 18 | -1 | -1 0 -1 0 -1 +@testcase | 19 | 0 | 0 -1 -1 0 0 +@testcase | 19 | -1 | -1 0 0 -1 -1 + +@fif_codegen +""" + boolWithBitwiseConst PROC:<{ + // + 0 PUSHINT // '3 + -1 PUSHINT // '3 '5 + 0 PUSHINT // '3 '5 '7 + -1 PUSHINT // '3 '5 '7 '8 + }> +""" + +@fif_codegen +""" + testDoUntilCodegen PROC:<{ + // i n + 0 PUSHINT // i n cnt=0 + UNTIL:<{ + INC // i n cnt + s2 PUSH // i n cnt i + NOT // i n cnt '6 + }> // i n cnt + UNTIL:<{ + INC // i n cnt + s2 PUSH // i n cnt i + NOT // i n cnt '9 + }> // i n cnt + UNTIL:<{ + INC // i n cnt + OVER // i n cnt n + 0 EQINT // i n cnt '12 + }> // i n cnt + s0 s2 XCHG // cnt n i + NOT // cnt n '13 + SWAP // cnt '13 n + 0 EQINT // cnt '13 '14 + }> +""" + +@fif_codegen +""" + testConstNegateCodegen PROC:<{ + // + TRUE // '0 + FALSE // '0 '1 + FALSE // '0 '1 '2 + TRUE // '0 '1 '2 '3 + TRUE // '0 '1 '2 '3 '4 + FALSE // '0 '1 '2 '3 '4 '5 + }> +""" + +@fif_codegen +""" + testBoolNegateOptimized PROC:<{ + // x + DUP // x x + NOT // x '1 + OVER // x '1 x + NOT // x '1 '2 + s2 s(-1) PUXC + TRUE // x '1 x '2 '3 + }> +""" + +@fif_codegen +""" + testBoolCompareOptimized PROC:<{ + // x + DUP // x x + NOT // x '1 + OVER // x '1 x + eqX CALLDICT // x '1 '2 + NOT // x '1 '3 + s2 PUSH // x '1 '3 x + eqX CALLDICT // x '1 '3 '4 + s3 PUSH // x '1 '3 '4 x + }> +""" +*/ diff --git a/tolk-tester/tests/c2.tolk b/tolk-tester/tests/c2.tolk new file mode 100644 index 00000000..bcbc6c93 --- /dev/null +++ b/tolk-tester/tests/c2.tolk @@ -0,0 +1,28 @@ +global op: (int, int) -> int; + +fun check_assoc(a: int, b: int, c: int): bool { + return op(op(a, b), c) == op(a, op(b, c)); +} + +fun unnamed_args(_: int, _: slice, _: int) { + return true; +} + +fun main(x: int, y: int, z: int): bool? { + op = `_+_`; + if (0) { return null; } + return check_assoc(x, y, z); +} + +@method_id(101) +fun test101(x: int, z: int) { + return unnamed_args(x, "asdf", z); +} + +/** + method_id | in | out +@testcase | 0 | 2 3 9 | -1 +@testcase | 0 | 11 22 44 | -1 +@testcase | 0 | -1 -10 -20 | -1 +@testcase | 101 | 1 10 | -1 +*/ diff --git a/tolk-tester/tests/c2_1.tolk b/tolk-tester/tests/c2_1.tolk new file mode 100644 index 00000000..ef1e589a --- /dev/null +++ b/tolk-tester/tests/c2_1.tolk @@ -0,0 +1,14 @@ +fun check_assoc(op: (int, int) -> int, a: int, b: int, c: int) { + return op(op(a, b), c) == op(a, op(b, c)); +} + +fun main(x: int, y: int, z: int): bool { + return check_assoc(`_+_`, x, y, z); +} + +/** + method_id | in | out +@testcase | 0 | 2 3 9 | -1 +@testcase | 0 | 11 22 44 | -1 +@testcase | 0 | -1 -10 -20 | -1 +*/ diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk new file mode 100644 index 00000000..772812eb --- /dev/null +++ b/tolk-tester/tests/cells-slices.tolk @@ -0,0 +1,232 @@ +fun store_u32(mutate self: builder, value: int): self { + return self.storeUint(value, 32); +} + +fun load_u32(mutate self: slice): int { + return self.loadUint(32); +} + +fun myLoadInt(mutate self: slice, len: int): int + asm(-> 1 0) "LDIX"; +fun myStoreInt(mutate self: builder, x: int, len: int): self + asm(x self len) "STIX"; + +@method_id(101) +fun test1(): [int,int,int,int,int] { + var b: builder = beginCell().storeUint(1, 32); + b = b.storeUint(2, 32); + b.storeUint(3, 32); + b = b.store_u32(4); + b.store_u32(5); + + var cs: slice = b.endCell().beginParse(); + var one: int = cs.loadUint(32); + var (two: int, three: int) = (cs.loadUint(32), cs.load_u32()); + var four: int = cs.load_u32(); + var five: int = cs.load_u32(); + + return [one,two,three,four,five]; +} + +@method_id(102) +fun test2(): [int,int,int] { + var b: builder = beginCell().myStoreInt(1, 32); + b = b.myStoreInt(2, 32); + // operator ! here and below is used just for testing purposes, it doesn't affect the result + b!.myStoreInt(3, 32); + + var cs: slice = b.endCell().beginParse(); + var one: int = cs.myLoadInt(32); + var (two: int, three: int) = (cs.myLoadInt(32), cs.myLoadInt(32)); + + return [one,two,three]; +} + +@method_id(103) +fun test3(ret: int): int { + val same: int = beginCell()!.storeUint(ret,32).endCell().beginParse().loadUint(32); + return same; +} + +@method_id(104) +fun test4(): [int,int] { + var b: builder = (beginCell() as builder).myStoreInt(1, 32); + b = b!.storeInt(2, 32)!.storeInt(3, 32); + + var cs: slice = b.endCell().beginParse(); + var (one, _, three) = (cs.getFirstBits(32).loadUint(32), cs.skipBits(64), cs.load_u32()); + + return [one,three]; +} + +@method_id(105) +fun test5(): [int,int] { + var cref: cell = endCell(beginCell().store_u32(105)); + var c: cell = beginCell().storeRef(cref).storeRef(cref).store_u32(1).endCell(); + + var cs: slice = beginParse(c); + var sto5x2: int = cs.loadRef().beginParse().load_u32() + cs.loadRef().beginParse().loadUint(32); + return [sto5x2, cs.load_u32()]; +} + +@method_id(106) +fun test6() { + return beginCell().storeUint(1, 32).storeUint(2, 32).storeUint(3, 32); +} + +@method_id(107) +fun test7() { + // since .store() methods now mutate, this piece of code works not as earlier (mutates uri_builder) + var uri_builder = beginCell(); + var uri_slice = uri_builder.storeSlice(".json").endCell().beginParse(); + var image_slice = uri_builder.storeSlice(".png").endCell().beginParse(); + return (uri_builder.getBuilderBitsCount(), uri_slice.getRemainingBitsCount(), image_slice.getRemainingBitsCount()); +} + +@method_id(108) +fun test8() { + var uri_builder = beginCell(); + var fresh = uri_builder; + var uri_slice = fresh.storeSlice(".json").endCell().beginParse(); + var fresh redef = uri_builder; + var image_slice = fresh.storeSlice(".png").endCell().beginParse(); + return (uri_builder.getBuilderBitsCount(), uri_slice.getRemainingBitsCount(), image_slice.getRemainingBitsCount()); +} + + +fun sumNumbersInSlice(mutate self: slice): int { + var result = 0; + while (!self.isEndOfSliceBits()) { + result += self.loadUint(32); + } + return result; +} + +@method_id(110) +fun test10() { + var ref = beginCell().storeInt(100, 32).endCell(); + var s: slice = beginCell().storeInt(1, 32).storeInt(2, 32).storeRef(ref).endCell().beginParse(); + var result = (getRemainingBitsCount(s), s.sumNumbersInSlice(), getRemainingBitsCount(s), isEndOfSlice(s), isEndOfSliceBits(s), isEndOfSliceRefs(s)); + var ref2: cell = s.loadRef(); + var s2: slice = ref2.beginParse(); + s.assertEndOfSlice(); + return (result, s2.loadInt(32), s2.isEndOfSlice()); +} + +@method_id(111) +fun test11() { + var s: slice = beginCell().storeInt(1, 32).storeInt(2, 32).storeInt(3, 32).storeInt(4, 32).storeInt(5, 32).storeInt(6, 32).storeInt(7, 32).endCell().beginParse(); + var size1 = getRemainingBitsCount(s); + s!.skipBits(32); + var s1: slice = s.getFirstBits(64); + var n1 = s1.loadInt(32); + var size2 = getRemainingBitsCount(s); + s.loadInt(32); + var size3 = getRemainingBitsCount(s); + s.removeLastBits(32); + var size4 = getRemainingBitsCount(s); + var n2 = s.loadInt(32); + var size5 = getRemainingBitsCount(s); + return (n1, n2, size1, size2, size3, size4, size5); +} + +@method_id(112) +fun test12() { + var (result1, result2) = (0, 0); + try { + beginCell().storeRef(beginCell().endCell()).endCell().beginParse().assertEndOfSlice(); + result1 = 100; + } catch (code) { + result1 = code; + } + try { + beginCell().endCell().beginParse().assertEndOfSlice(); + result2 = 100; + } catch (code) { + result2 = code; + } + return (result1, result2); +} + +@method_id(113) +fun test13() { + var ref2 = beginCell().storeInt(1, 32).endCell(); + var ref1 = beginCell().storeInt(1, 32).storeRef(ref2).endCell(); + var c = beginCell().storeInt(444, 32).storeRef(ref1).storeRef(ref1).storeRef(ref1).storeRef(ref2).storeInt(4, 32).endCell(); + var (n_cells1, n_bits1, n_refs1) = c.calculateCellSizeStrict(10); + var s = c.beginParse(); + s.loadRef(); + s.loadRef(); + var n = s.loadInt(32); + var (n_cells2, n_bits2, n_refs2) = s.calculateSliceSizeStrict(10); + return ([n_cells1, n_bits1, n_refs1], [n_cells2, n_bits2, n_refs2], n); +} + +@method_id(114) +fun test110(x: bool) { + var s = beginCell().storeBool(x == true).storeBool(false).storeBool(x).endCell().beginParse(); + return (s.loadBool(), s.loadBool(), s.loadBool()); +} + +@method_id(115) +fun test111() { + var s = beginCell().storeMessageOp(123).storeMessageQueryId(456) + .storeAddressNone().storeAddressNone() + .storeUint(0, 32) + .storeUint(123, 32).storeUint(456, 64).storeUint(789, 64) + .endCell().beginParse(); + var op1 = s.loadUint(32); + var q1 = s.loadUint(64); + if (s.addressIsNone()) { + s.skipBits(2); + } + if (s.loadBool() == false) { + assert(!s.loadBool()) throw 444; + s.skipBouncedPrefix(); + } + var op2 = s.loadMessageOp(); + var q2 = s.loadMessageQueryId(); + s.skipBits(64); + s.assertEndOfSlice(); + assert(isMessageBounced(0x001) && !isMessageBounced(0x002)) throw 444; + return (op1, q1, op2, q2); +} + +fun main(): int { + return 0; +} + +/** +@testcase | 101 | | [ 1 2 3 4 5 ] +@testcase | 102 | | [ 1 2 3 ] +@testcase | 103 | 103 | 103 +@testcase | 104 | | [ 1 3 ] +@testcase | 105 | | [ 210 1 ] +@testcase | 107 | | 72 40 72 +@testcase | 108 | | 0 40 32 +@testcase | 110 | | 64 3 0 0 -1 0 100 -1 +@testcase | 111 | | 2 3 224 192 160 128 96 +@testcase | 112 | | 9 100 +@testcase | 113 | | [ 3 128 5 ] [ 2 96 3 ] 444 +@testcase | 114 | -1 | -1 0 -1 +@testcase | 114 | 0 | 0 0 0 +@testcase | 115 | | 123 456 123 456 + +Note, that since 'compute-asm-ltr' became on be default, chaining methods codegen is not quite optimal. +@fif_codegen +""" + test6 PROC:<{ + // + NEWC // '0 + 1 PUSHINT // '0 '1=1 + SWAP // '1=1 '0 + 32 STU // '0 + 2 PUSHINT // '0 '4=2 + SWAP // '4=2 '0 + 32 STU // '0 + 3 PUSHINT // '0 '7=3 + SWAP // '7=3 '0 + 32 STU // '0 + }> +""" + */ diff --git a/tolk-tester/tests/co1.tolk b/tolk-tester/tests/co1.tolk new file mode 100644 index 00000000..f124e1de --- /dev/null +++ b/tolk-tester/tests/co1.tolk @@ -0,0 +1,72 @@ +const int1 = 1; +const int2 = 2; + +const int101: int = 101; +const int111: int = 111; + +const int1r = int1; + +const str1 = "const1"; +const str2 = "aabbcc"s; + +const str2r: slice = str2; + +const str1int = 0x636f6e737431; +const str2int = 0xAABBCC; + +const nibbles: int = 4; + +fun iget1(): int { return int1; } +fun iget2(): int { return int2; } +fun iget3(): int { return int1+int2; } + +fun iget1r(): int { return int1r; } + +fun sget1(): slice { return str1; } +fun sget2(): slice { return str2; } +fun sget2r(): slice { return str2r; } + +const int240: int = ((int1+int2)*10)<<3; + +fun iget240(): int { return int240; } + +@pure +fun newc(): builder +asm "NEWC"; +@pure +fun endcs(b: builder): slice +asm "ENDC" "CTOS"; +@pure +fun sdeq(s1: slice, s2: slice): int +asm "SDEQ"; +@pure +fun stslicer(b: builder, s: slice): builder +asm "STSLICER"; + +fun main() { + var i1: int = iget1(); + var i2: int = iget2(); + var i3: int = iget3(); + + assert(i1 == 1) throw int101; + assert(i2 == 2) throw 102; + assert(i3 == 3) throw 103; + + var s1: slice = sget1(); + var s2: slice = sget2(); + var s3: slice = newc().stslicer(str1).stslicer(str2r).endcs(); + + assert(sdeq(s1, newc().storeUint(str1int, 12 * nibbles).endcs())) throw int111; + assert(sdeq(s2, newc().storeUint(str2int, 6 * nibbles).endcs())) throw 112; + assert(sdeq(s3, newc().storeUint(0x636f6e737431AABBCC, 18 * nibbles).endcs())) throw 113; + + var i4: int = iget240(); + assert(i4 == 240) throw ((104)); + return 0; +} + +/** +@testcase | 0 | | 0 + +@code_hash 61273295789179921867241079778489100375537711211918844448475493726205774530743 +*/ diff --git a/tolk-tester/tests/code_after_ifelse.tolk b/tolk-tester/tests/code_after_ifelse.tolk new file mode 100644 index 00000000..6a16262f --- /dev/null +++ b/tolk-tester/tests/code_after_ifelse.tolk @@ -0,0 +1,41 @@ +fun elseif(cond: int) { + if (cond > 0) { + throw(cond); + } +} + +@inline +@method_id(101) +fun foo(x: int): int { + if (x==1) { + return 111; + } else { + x *= 2; + } + return x + 1; +} + +fun main(x: int): (int, int) { + return (foo(x), 222); +} + +@method_id(102) +fun test2(x: int) { + try { + if (x < 0) { return -1; } + elseif (x); + } catch(excNo) { + return excNo * 1000; + } + return 0; +} + +/** + method_id | in | out +@testcase | 0 | 1 | 111 222 +@testcase | 0 | 3 | 7 222 +@testcase | 101 | 1 | 111 +@testcase | 101 | 3 | 7 +@testcase | 102 | -5 | -1 +@testcase | 102 | 5 | 5000 +*/ diff --git a/tolk-tester/tests/codegen_check_demo.tolk b/tolk-tester/tests/codegen_check_demo.tolk new file mode 100644 index 00000000..5b46c093 --- /dev/null +++ b/tolk-tester/tests/codegen_check_demo.tolk @@ -0,0 +1,96 @@ +@method_id(101) +fun test1(): int { + var x: int = false as int; + if (x == true as int) { + x= 100500; + } + return x; +} + +fun main(s: int) { + var (z, t) = (17, s); + while (z > 0) { + t = s; + z -= 1; + } + return ~ t; +} + +/** + method_id | in | out +@testcase | 0 | 1 | -2 +@testcase | 0 | 5 | -6 +@testcase | 101 | | 0 + +Below, I just give examples of @fif_codegen tag: +* a pattern can be single-line (after the tag), or multi-line, surrounded with """ +* there may be multiple @fif_codegen, they all will be checked +* identation (spaces) is not checked intentionally +* "..." means any number of any lines +* lines not divided with "..." are expected to be consecutive in fif output +* //comments can be omitted, but if present, they are also expected to be equal +* there is also a tag @fif_codegen_avoid to check a pattern does not occur + +@fif_codegen +""" +main PROC:<{ + // s + 17 PUSHINT // s '3=17 + OVER // s z=17 t + WHILE:<{ + ... + }>DO<{ // s z t + ... + s1 s(-1) PUXC // s t z + ... + 2 1 BLKDROP2 + ... +}> +""" + +@fif_codegen +""" +main PROC:<{ + ... + WHILE:<{ + ... + }>DO<{ + ... + }> + }END>c +""" + +@fif_codegen +""" + OVER + 0 GTINT // s z t '5 +""" + +@fif_codegen +""" + "Asm.fif" include + ... + PROGRAM{ + ... + }END>c +""" + +@fif_codegen +""" +test1 PROC:<{ +// +FALSE +}> +""" + +@fif_codegen NOT // '8 +@fif_codegen main PROC:<{ + +@fif_codegen_avoid PROCINLINE +@fif_codegen_avoid END c +@fif_codegen_avoid +""" +multiline +can also be +""" +*/ diff --git a/tolk-tester/tests/comments.tolk b/tolk-tester/tests/comments.tolk new file mode 100644 index 00000000..cd287747 --- /dev/null +++ b/tolk-tester/tests/comments.tolk @@ -0,0 +1,31 @@ + +fun main(): int + +// inside a comment, /* doesn't start a new one +/* but if // is inside, a comment may end at this line*/ { + var cc = "a string may contain /* or // or /*, not parsed"; + // return 1; + return get10() + /* + traditional comment /* may not be nested + // line comment + // ends */1 + + 1; + /* moreover, different comment styles + may be used for opening and closing + */ +} + +/*** + first line + //two-lined*/ + +@method_id(10) +fun get10(): int { + return 10; +} + + +/** +@testcase | 0 | | 12 +@testcase | 10 | | 10 +*/ diff --git a/tolk-tester/tests/dicts-demo.tolk b/tolk-tester/tests/dicts-demo.tolk new file mode 100644 index 00000000..606318cb --- /dev/null +++ b/tolk-tester/tests/dicts-demo.tolk @@ -0,0 +1,105 @@ +import "@stdlib/tvm-dicts" + +fun addIntToIDict(mutate self: cell?, key: int, number: int): void { + return self.iDictSetBuilder(32, key, beginCell().storeInt(number, 32)); +} + +fun calculateDictLen(d: cell?) { + var len = 0; + var (k, v, f) = d.uDictGetFirst(32); + while (f) { + len += 1; + (k, v, f) = d.uDictGetNext(32, k!); + } + return len; +} + +fun loadTwoDigitNumberFromSlice(mutate self: slice): int { + var n1 = self.loadInt(8); + var n2 = self.loadInt(8); + return (n1 - 48) * 10 + (n2 - 48); +} + + +@method_id(101) +fun test101(getK1: int, getK2: int, getK3: int) { + var dict = createEmptyDict(); + dict.uDictSetBuilder(32, 1, beginCell().storeUint(1, 32)); + var (old1: slice?, found1) = dict.uDictSetAndGet(32, getK1, beginCell().storeUint(2, 32).endCell().beginParse()); + var (old2: slice?, found2) = dict.uDictSetAndGet(32, getK2, beginCell().storeUint(3, 32).endCell().beginParse()); + var (cur3: slice?, found3) = dict.uDictGet(32, getK3); + return ( + found1 ? old1!.loadUint(32) : -1, + found2 ? old2!.loadUint(32) : -1, + found3 ? cur3!.loadUint(32) : -1 + ); +} + +@method_id(102) +fun test102() { + var dict = createEmptyDict(); + dict.addIntToIDict(2, 102); + dict.addIntToIDict(1, 101); + dict.addIntToIDict(4, 104); + dict.addIntToIDict(3, 103); + var deleted = createEmptyTuple(); + var shouldBreak = false; + while (!shouldBreak) { + var (kDel, kVal, wasDel) = dict.iDictDeleteLastAndGet(32); + if (wasDel) { + deleted.tuplePush([kDel, kVal!.loadInt(32)]); + } else { + shouldBreak = true; + } + } + return deleted; +} + +@method_id(103) +fun test103() { + var dict = createEmptyDict(); + dict.uDictSetBuilderIfNotExists(32, 1,beginCell().storeInt(1, 32)); + dict.uDictSetBuilderIfNotExists(32, 1,beginCell().storeInt(1, 32)); + var len1 = calculateDictLen(dict); + dict.uDictSetBuilderIfExists(32, 2,beginCell().storeInt(1, 32)); + dict.uDictSetBuilderIfExists(32, 2,beginCell().storeInt(1, 32)); + var len2 = calculateDictLen(dict); + dict.uDictSetBuilder(32, 3,beginCell().storeInt(1, 32)); + dict.uDictSetBuilderIfExists(32, 3,beginCell().storeInt(1, 32)); + var len3 = calculateDictLen(dict); + var (delK1, _, _) = dict.uDictDeleteFirstAndGet(32); + var (delK2, _, _) = dict.uDictDeleteFirstAndGet(32); + var (delK3, _, _) = dict.uDictDeleteFirstAndGet(32); + return (len1, len2, len3, delK1, delK2, delK3); +} + +@method_id(104) +fun test104() { + var dict = createEmptyDict(); + dict.sDictSetBuilder(32, "7800", beginCell().storeUint(5 + 48, 8).storeUint(6 + 48, 8)); + dict.sDictSet(32, "key1", "12"); + var (old1, _) = dict.sDictSetAndGet(32, "key1", "34"); + var (old2, _) = dict.sDictDeleteAndGet(32, "key1"); + var (restK, restV, _) = dict.sDictGetFirst(32); + var (restK1, restV1, _) = dict.sDictDeleteLastAndGet(32); + assert (restK!.isSliceBitsEqual(restK1!)) throw 123; + assert (restV!.isSliceBitsEqual(restV1!)) throw 123; + return ( + old1!.loadTwoDigitNumberFromSlice(), + old2!.loadTwoDigitNumberFromSlice(), + restV!.loadTwoDigitNumberFromSlice(), + restK!.loadTwoDigitNumberFromSlice(), + restK!.loadTwoDigitNumberFromSlice() + ); +} + +fun main() {} + +/** +@testcase | 101 | 1 1 1 | 1 2 3 +@testcase | 101 | 1 2 1 | 1 -1 2 +@testcase | 101 | 1 2 3 | 1 -1 -1 +@testcase | 102 | | [ [ 4 104 ] [ 3 103 ] [ 2 102 ] [ 1 101 ] ] +@testcase | 103 | | 1 1 2 1 3 (null) +@testcase | 104 | | 12 34 56 78 0 + */ diff --git a/tolk-tester/tests/generics-1.tolk b/tolk-tester/tests/generics-1.tolk new file mode 100644 index 00000000..ca310927 --- /dev/null +++ b/tolk-tester/tests/generics-1.tolk @@ -0,0 +1,160 @@ +fun eq1(value: X): X { return value; } +fun eq2(value: X) { return value; } +fun eq3(value: X): X { var cp: [X] = [eq1(value)]; var ((([v: X]))) = cp; return v; } +fun eq4(value: X) { return eq1(value); } + +@method_id(101) +fun test101(x: int) { + var (a, b, c) = (x, (x,x), [x,x]); + return (eq1(a), eq1(b), eq1(c), eq2(a), eq2(b), eq2(c), eq3(a), eq4(b), eq3(createEmptyTuple())); +} + +fun getTwo(): X { return 2 as X; } + +fun takeInt(a: int) { return a; } + +@method_id(102) +fun test102(): (int, int, int, [int, int]) { + var a: int = getTwo(); + var _: int = getTwo(); + var b = getTwo() as int; + var c: int = 1 ? getTwo() : getTwo(); + var c redef = getTwo(); + var ab_tens = (0, (1, 2)); + ab_tens.0 = getTwo(); + ab_tens.1.1 = getTwo(); + var ab_tup = [0, [1, 2]]; + ab_tup.0 = getTwo(); + ab_tup.1.1 = getTwo(); + return (eq1(a), eq2(b), takeInt(getTwo()), [getTwo(), ab_tens.1.1]); +} + +@method_id(103) +fun test103(first: int): (int, int, int) { + var t = createEmptyTuple(); + var cs = beginCell().storeInt(100, 32).endCell().beginParse(); + t.tuplePush(first); + t.tuplePush(2); + t.tuplePush(cs); + cs = t.tupleAt(2); + cs = t.tupleAt(2) as slice; + return (t.tupleAt(0), cs.loadInt(32), t.tupleAt(2).loadInt(32)); +} + +fun manyEq(a: T1, b: T2, c: T3): [T1, T2, T3] { + return [a, b, c]; +} + +@method_id(104) +fun test104(f: int) { + var result = ( + manyEq(1 ? 1 : 1, f ? 0 : null, !f ? getTwo() as int : null), + manyEq(f ? null as int? : eq2(2), beginCell().storeBool(true).endCell().beginParse().loadBool(), eq4(f)) + ); + __expect_type(result, "([int, int?, int?], [int?, bool, int])"); + return result; +} + +fun calcSum(x: X, y: X) { return x! + y!; } + +@method_id(105) +fun test105() { + if (0) { calcSum(((0 as int?)), null); } + return (calcSum(1, 2)); +} + +fun calcYPlus1(value: Y) { return value + 1; } +fun calcLoad32(cs: slice) { return cs.loadInt(32); } +fun calcTensorPlus1(tens: (int, int)) { var (f, s) = tens; return (f + 1, s + 1); } +fun calcTensorMul2(tens: (int, int)) { var (f, s) = tens; return (f * 2, s * 2); } +fun cellToSlice(c: cell) { return c.beginParse(); } +fun abstractTransform(xToY: (X) -> Y, yToR: (((Y))) -> R, initialX: X): R { + var y = xToY(initialX); + return yToR(y); +} + +@method_id(106) +fun test106() { + var c = beginCell().storeInt(106, 32).endCell(); + __expect_type(calcYPlus1, "(int) -> int"); + return [ + abstractTransform(cellToSlice, calcLoad32, c), + abstractTransform(calcYPlus1, calcYPlus1, 0), + abstractTransform(calcTensorPlus1, calcTensorMul2, (2, 2)).0, + abstractTransform(calcTensorPlus1, calcTensorMul2, (2, 2)).1 + ]; +} + +fun callTupleFirst(t: X): Y { return t.tupleFirst(); } +fun callTuplePush(mutate self: T, v1: V, v2: V): self { self.tuplePush(v1); tuplePush(mutate self, v2); return self; } +fun getTupleLastInt(t: tuple) { return t.tupleLast(); } +fun getTupleSize(t: tuple) { return t.tupleSize(); } +fun callAnyFn(f: (TObj) -> TResult, arg: TObj) { return f(arg); } +fun callAnyFn2(f: TCallback, arg: tuple) { return f(arg); } + +global t107: tuple; + +@method_id(107) +fun test107() { + t107 = createEmptyTuple(); + callTuplePush(mutate t107, 1, 2); + t107.callTuplePush(3, 4).callTuplePush(5, 6); + var first: int = t107.callTupleFirst(); + return ( + callAnyFn(getTupleSize, t107), + callAnyFn2(getTupleSize, t107), + first, + callTupleFirst(t107) as int, + callAnyFn(getTupleLastInt, t107), + callAnyFn2(getTupleLastInt, t107) + ); +} + +global g108: int; + +fun inc108(by: int) { g108 += by; } +fun getInc108() { return inc108; } +fun returnResult(f: () -> RetT): RetT { return f(); } +fun applyAndReturn(f: () -> (ArgT) -> RetT, arg: ArgT): () -> ArgT -> RetT { + f()(arg); + return f; +} + +@method_id(108) +fun test108() { + g108 = 0; + getInc108()(1); + returnResult<(int) -> void>(getInc108)(2); + applyAndReturn(getInc108, 10)()(10); + returnResult(getInc108)(2); + applyAndReturn(getInc108, 10)()(10); + return g108; +} + +fun main(x: int): (int, [[int, int]]) { + try { if(x) { throw (1, x); } } + catch (excNo, arg) { return (arg as int, [[eq2(arg as int), getTwo()]]); } + return (0, [[x, 1]]); +} + +/** +@testcase | 0 | 1 | 1 [ [ 1 2 ] ] +@testcase | 101 | 0 | 0 0 0 [ 0 0 ] 0 0 0 [ 0 0 ] 0 0 0 [] +@testcase | 102 | | 2 2 2 [ 2 2 ] +@testcase | 103 | 0 | 0 100 100 +@testcase | 104 | 0 | [ 1 (null) 2 ] [ 2 -1 0 ] +@testcase | 105 | | 3 +@testcase | 106 | | [ 106 2 6 6 ] +@testcase | 107 | | 6 6 1 1 6 6 +@testcase | 108 | | 45 + +@fif_codegen DECLPROC eq1 +@fif_codegen DECLPROC eq1 +@fif_codegen DECLPROC eq1<(int,int)> +@fif_codegen DECLPROC eq1<[int,int]> +@fif_codegen DECLPROC getTwo + +@fif_codegen_avoid DECLPROC eq1 +@fif_codegen_avoid DECLPROC eq2 +@fif_codegen_avoid DECLPROC eq3 + */ diff --git a/tolk-tester/tests/if_stmt.tolk b/tolk-tester/tests/if_stmt.tolk new file mode 100644 index 00000000..0f78a516 --- /dev/null +++ b/tolk-tester/tests/if_stmt.tolk @@ -0,0 +1,66 @@ +@method_id(101) +fun test1(x: int): int { + if (x > 200) { + return 200; + } else if (x > 100) { + return 100; + } else if (!(x <= 50)) { + if (!(x > 90)) { + return x; + } else { + return 90; + } + } else { + return 0; + } +} + +@method_id(102) +fun test2(x: int) { + if (x == 20) { return 20; } + if (x != 50) { return 50; } + if (x == 0) { return 0; } + return -1; +} + +@method_id(103) +fun test3(x: int) { + if (!(x != 20)) { return 20; } + if (!(x == 50)) { return 50; } + if (!x) { return 0; } + return -1; +} + +fun main() { + +} + +/** +@testcase | 101 | 0 | 0 +@testcase | 101 | 1000 | 200 +@testcase | 101 | 150 | 100 +@testcase | 101 | -1 | 0 +@testcase | 101 | 87 | 87 +@testcase | 101 | 94 | 90 +@testcase | 102 | 20 | 20 +@testcase | 102 | 40 | 50 +@testcase | 102 | 50 | -1 +@testcase | 103 | 20 | 20 +@testcase | 103 | 40 | 50 +@testcase | 103 | 50 | -1 + +@fif_codegen +""" + test3 PROC:<{ + // x + DUP // x x + 20 NEQINT // x '2 + IFNOTJMP:<{ // x + DROP // + 20 PUSHINT // '3=20 + }> // x + DUP // x x + 50 EQINT // x '5 + IFNOTJMP:<{ // x +""" +*/ diff --git a/tolk-tester/tests/imports/invalid-no-import.tolk b/tolk-tester/tests/imports/invalid-no-import.tolk new file mode 100644 index 00000000..6c4ab6ce --- /dev/null +++ b/tolk-tester/tests/imports/invalid-no-import.tolk @@ -0,0 +1,4 @@ +fun demoOfInvalid(): (int) { + var f = someAdd; + return f(1, 2); +} diff --git a/tolk-tester/tests/imports/some-math.tolk b/tolk-tester/tests/imports/some-math.tolk new file mode 100644 index 00000000..dc0c9c9b --- /dev/null +++ b/tolk-tester/tests/imports/some-math.tolk @@ -0,0 +1,3 @@ +fun someAdd(a: int, b: int): int { + return a + b + 0; +} diff --git a/tolk-tester/tests/imports/use-dicts-err.tolk b/tolk-tester/tests/imports/use-dicts-err.tolk new file mode 100644 index 00000000..c5ba89d2 --- /dev/null +++ b/tolk-tester/tests/imports/use-dicts-err.tolk @@ -0,0 +1,21 @@ +fun prepareDict_3_30_4_40_5_x(valueAt5: int): cell { + var dict: cell = createEmptyDict(); + dict.idict_set_builder(32, 3, begin_cell().store_int(30, 32)); + dict.idict_set_builder(32, 4, begin_cell().store_int(40, 32)); + dict.idict_set_builder(32, 5, begin_cell().store_int(valueAt5, 32)); + return dict; +} + +fun lookupIdxByValue(idict32: cell, value: int): int { + var cur_key = -1; + do { + var (cur_key redef, cs: slice, found: int) = idict32.idictGetNext(32, cur_key); + // one-line condition (via &) doesn't work, since right side is calculated immediately + if (found) { + if (cs.loadInt(32) == value) { + return cur_key; + } + } + } while (found); + return -1; +} diff --git a/tolk-tester/tests/imports/use-dicts.tolk b/tolk-tester/tests/imports/use-dicts.tolk new file mode 100644 index 00000000..2daaf2b1 --- /dev/null +++ b/tolk-tester/tests/imports/use-dicts.tolk @@ -0,0 +1,23 @@ +import "@stdlib/tvm-dicts" + +fun prepareDict_3_30_4_40_5_x(valueAt5: int): cell? { + var dict: cell? = createEmptyDict(); + dict.iDictSetBuilder(32, 3, beginCell().storeInt(30, 32)); + dict.iDictSetBuilder(32, 4, beginCell().storeInt(40, 32)); + dict.iDictSetBuilder(32, 5, beginCell().storeInt(valueAt5, 32)); + return dict; +} + +fun lookupIdxByValue(idict32: cell?, value: int): int { + var cur_key: int? = -1; + do { + var (cur_key redef, cs: slice?, found: bool) = idict32.iDictGetNext(32, cur_key!); + // one-line condition (via &) doesn't work, since right side is calculated immediately + if (found) { + if (cs!.loadInt(32) == value) { + return cur_key!; + } + } + } while (found); + return -1; +} diff --git a/tolk-tester/tests/indexed-access.tolk b/tolk-tester/tests/indexed-access.tolk new file mode 100644 index 00000000..e2bd3dd9 --- /dev/null +++ b/tolk-tester/tests/indexed-access.tolk @@ -0,0 +1,358 @@ + +fun increment(mutate self: int) { + self += 1; +} + +fun increment2(mutate a: int, mutate b: int) { + a += 1; + b += 1; +} + +fun assign1020(mutate a: int, mutate b: int) { + a = 10; + b = 20; +} + +fun plus(mutate self: int, y: int): int { + val newVals = (self + y, y * 10); + self = newVals.0; + return newVals.1; +} + +fun eq(v: X): X { return v; } + +global gTup: [int]; +global gTens: (int, int); + +@method_id(100) +fun testCodegenSimple() { + var t1 = [1]; + t1.0 = 2; + debugPrintString(""); + var t2 = [[1]]; + t2.0.0 = 2; + debugPrintString(""); + gTup = [1]; + gTup.0 = 2; + debugPrintString(""); + gTens = (1,2); + gTens.1 = 4; + debugPrintString(""); + return (t1, t2, gTup, gTens); +} + +@method_id(101) +fun test101() { + var t = (1, (2, 3), [4, 5, [6, 7]], 8); + (t.0, t.1.0, t.2.0) = (2, 3, 5); + t.3.increment(); + t.2.1 += (t.1.1 += 1) - t.1.1 + 1; + increment2(mutate t.2.2.0, mutate t.2.2.1); + return t; +} + +global t102: (int, (int, int), [int, int, [int, int]], int); + +@method_id(102) +fun test102() { + t102 = (1, (2, 3), [4, 5, [6, 7]], 8); + (t102.0, t102.1.0, t102.2.0) = (2, 3, 5); + t102.3.increment(); + t102.2.1 += (t102.1.1 += 1) - t102.1.1 + 1; + increment2(mutate t102.2.2.0, mutate t102.2.2.1); + return t102; +} + +global t103: (int, int); + +@method_id(103) +fun test103() { + t103 = (5, 5); + assign1020(mutate t103.0, mutate t103.1); + var t = (5, 5); + assign1020(mutate t.0, mutate t.1); + return (t103, t); +} + +global t104: [[int, int]]; + +@method_id(104) +fun test104() { + var m = [[5, 5]]; + (m.0.0, m.0.1) = (10, 20); + t104 = [[5, 5]]; + (t104.0.0, t104.0.1) = (10, 20); + return (t104, m); +} + +@method_id(105) +fun test105(x: int, y: int): (tuple, int, (int?, int), int, int) { + var ab = (createEmptyTuple(), (x as int?, y), tupleSize); + ab.0.tuplePush(1); + tuplePush(mutate ab.0, 2); + ab.1.0 = null; + ab.1.1 += 10; + var cb = ab.2; + return (ab.0, ab.0.1, ab.1, cb(ab.0), ab.2(ab.0)); +} + +@method_id(106) +fun test106(x: int, y: int) { + var ab = [createEmptyTuple(), [x as int?, y], tupleSize]; + ab.0.tuplePush(1); + tuplePush(mutate ab.0, 2); + ab.1.0 = null; + ab.1.1 += 10; + var cb = ab.2; + return (ab.0, ab.1, cb(ab.0), ab.2(ab.0)); +} + +@method_id(107) +fun test107() { + var ab = createEmptyTuple(); + ab.tuplePush(1); + ab.tuplePush(beginCell().storeInt(1, 32)); + return (ab.0 as int, getBuilderBitsCount(ab.1)); +} + +global t108: [int, [int, [int]]]; + +@method_id(108) +fun test108(last: int) { + t108 = [1, [2, [last]]]; + t108.1.1.0.increment(); + var t = [1, [2, [last]]]; + t.1.1.0.increment(); + return (t108, t); +} + +@method_id(109) +fun test109(x: (int, int)): (int, int, int, int, int, int, int) { + return (x.1, x.1.plus(x.1 / 20), x.1, x.1 = x.1 * 2, x.1, x.1 += 1, x.1); +} + +@method_id(110) +fun test110(f: int, s: int) { + var x = [f, s]; + return (x, x.1, x.1.plus(x.1 / 20), x.1, x.1 = x.1 * 2, x.1, x.1 += 1, x.1, x); +} + +global xx: (int, int); + +@method_id(111) +fun test111(x: (int, int)) { + xx = x; + return (x, xx.1, xx.1.plus(xx.1 / 20), eq(xx.1 += (x.1 *= 0)), xx.1 = xx.1 * 2, xx.1, xx.1 += 1, xx.1, x); +} + +global yy: [int, int]; + +@method_id(112) +fun test112(f: int, s: int) { + yy = [f, s]; + return (yy, yy.1, yy.1.plus(yy.1 / 20), eq(yy.1 += (yy.1 *= 0)), yy.1 = yy.1 * 2, yy.1, yy.1 += 1, yy.1, yy); +} + +@pure +fun getConstTuple() { + return [1,2]; +} + +fun testCodegenNoPureIndexedAccess() { + (getConstTuple().1, getConstTuple().0) = (3, 4); + return 0; +} + +@method_id(113) +fun test113() { + var x = [[1, 2]]; + return (x, x.0, plus(mutate x.0.0, 10), x.0, x, x.0 = [10, 20], x); +} + +@method_id(114) +fun test114(f: int, s: int) { + var x = ((), (f, s), ()); + return (x, x.1, plus(mutate x.1.0, 10), x.1, x, x.1 = (10, 20), x); +} + +@method_id(115) +fun test115() { + var y = [[[[true]]]]; + return (y, ((((y).0).0).0).0 = !y.0.0.0.0, y.0); +} + +@method_id(116) +fun test116() { + var t = createEmptyTuple(); + t.tuplePush(1); + try { + return t.100500 as int; + } catch(excNo) { + return excNo; + } +} + +@method_id(117) +fun test117() { + var t = createEmptyTuple(); + t.tuplePush(1); + try { + return (t.0 as tuple).0 as int; + } catch(excNo) { + return excNo; + } +} + +@method_id(118) +fun testCodegenIndexPostfix1(x: (int, int)) { + var ab = (x.1, x.0); + return ab; +} + +@method_id(119) +fun testCodegenIndexPostfix2(x: (int, (int, int), int)) { + var y = x; + return (y.2, y.0, y.1.1); +} + +fun getT() { return (1, 2); } + +@method_id(120) +fun test120() { + return (getT().0 = 3, getT().0 = 4, [getT().0 = 5, getT().0 = 6]); +} + +@method_id(121) +fun test121(zero: int) { + var t = createEmptyTuple(); + t.tuplePush(-100); + t.tupleSetAt(0, zero); + (t.0 as int).increment(); + (((t.0) as int) as int).increment(); + increment(mutate t.0 as int); + return t; +} + +fun isFirstComponentGt0(t: (T1, T2)): bool { + return t.0 > 0; +} + +@method_id(122) +fun test122(x: (int, int)) { + return ( + isFirstComponentGt0(x), isFirstComponentGt0((2, beginCell())), isFirstComponentGt0((0, null)), + x.isFirstComponentGt0(), (2, beginCell()).isFirstComponentGt0(), (0, null).isFirstComponentGt0() + ); +} + +@method_id(123) +fun test123() { + var t = [[10, 20]] as [[int,int]]?; + ((t!).0).0 = ((t!).0).1 = 100; + return t; +} + +fun main(){} + + +/** +@testcase | 101 | | 2 3 4 [ 5 6 [ 7 8 ] ] 9 +@testcase | 102 | | 2 3 4 [ 5 6 [ 7 8 ] ] 9 +@testcase | 103 | | 10 20 10 20 +@testcase | 104 | | [ [ 10 20 ] ] [ [ 10 20 ] ] +@testcase | 105 | 5 6 | [ 1 2 ] 2 (null) 16 2 2 +@testcase | 106 | 5 6 | [ 1 2 ] [ (null) 16 ] 2 2 +@testcase | 107 | | 1 32 +@testcase | 108 | 3 | [ 1 [ 2 [ 4 ] ] ] [ 1 [ 2 [ 4 ] ] ] +@testcase | 109 | 0 100 | 100 50 105 210 210 211 211 +@testcase | 110 | 0 100 | [ 0 100 ] 100 50 105 210 210 211 211 [ 0 211 ] +@testcase | 111 | 0 100 | 0 100 100 50 105 210 210 211 211 0 0 +@testcase | 112 | 0 100 | [ 0 100 ] 100 50 105 210 210 211 211 [ 0 211 ] +@testcase | 113 | | [ [ 1 2 ] ] [ 1 2 ] 100 [ 11 2 ] [ [ 11 2 ] ] [ 10 20 ] [ [ 10 20 ] ] +@testcase | 114 | 1 2 | 1 2 1 2 100 11 2 11 2 10 20 10 20 +@testcase | 115 | | [ [ [ [ -1 ] ] ] ] 0 [ [ [ 0 ] ] ] +@testcase | 116 | | 5 +@testcase | 117 | | 7 +@testcase | 118 | 1 2 | 2 1 +@testcase | 119 | 1 2 3 4 | 4 1 3 +@testcase | 120 | | 3 4 [ 5 6 ] +@testcase | 121 | 0 | [ 3 ] +@testcase | 122 | 1 2 | -1 -1 0 -1 -1 0 +@testcase | 123 | | [ [ 100 100 ] ] + +@fif_codegen +""" + testCodegenSimple PROC:<{ + // + 1 PUSHINT // '2=1 + SINGLE // t1 + 2 PUSHINT // t1 '3=2 + 0 SETINDEX // t1 + x{} PUSHSLICE // t1 '6 + STRDUMP DROP + 1 PUSHINT // t1 '10=1 + SINGLE // t1 '9 + SINGLE // t1 t2 + 2 PUSHINT // t1 t2 '11=2 + OVER // t1 t2 '11=2 t2 + 0 INDEX // t1 t2 '11=2 '14 + SWAP // t1 t2 '14 '11=2 + 0 SETINDEX // t1 t2 '14 + 0 SETINDEX // t1 t2 + x{} PUSHSLICE // t1 t2 '17 + STRDUMP DROP + 1 PUSHINT // t1 t2 '20=1 + SINGLE // t1 t2 '18 + gTup SETGLOB + 2 PUSHINT // t1 t2 '21=2 + gTup GETGLOB // t1 t2 '21=2 g_gTup + SWAP // t1 t2 g_gTup '21=2 + 0 SETINDEX // t1 t2 g_gTup + gTup SETGLOB + x{} PUSHSLICE // t1 t2 '25 + STRDUMP DROP + 1 PUSHINT // t1 t2 '28=1 + 2 PUSHINT // t1 t2 '26=1 '27=2 + PAIR + gTens SETGLOB + 4 PUSHINT // t1 t2 g_gTens.1=4 + gTens GETGLOB + UNPAIR // t1 t2 g_gTens.1=4 g_gTens.0 g_gTens.1 + DROP // t1 t2 g_gTens.1=4 g_gTens.0 + SWAP // t1 t2 g_gTens.0 g_gTens.1=4 + PAIR + gTens SETGLOB + x{} PUSHSLICE // t1 t2 '36 + STRDUMP DROP + gTup GETGLOB // t1 t2 g_gTup + gTens GETGLOB + UNPAIR // t1 t2 g_gTup g_gTens.0 g_gTens.1 + }> +""" + +@fif_codegen +""" + testCodegenNoPureIndexedAccess PROC:<{ + // + 0 PUSHINT // '8=0 + }> +""" + +@fif_codegen +""" + testCodegenIndexPostfix1 PROC:<{ + // x.0 x.1 + // ab.1 ab.0 + SWAP // ab.0 ab.1 + }> +""" + +@fif_codegen +""" + testCodegenIndexPostfix2 PROC:<{ + // x.0 x.1.0 x.1.1 x.2 + s2 POP // y.0 y.2 y.1.1 + s1 s2 XCHG // y.2 y.0 y.1.1 + }> +""" + */ diff --git a/tolk-tester/tests/inference-tests.tolk b/tolk-tester/tests/inference-tests.tolk new file mode 100644 index 00000000..5020d0dd --- /dev/null +++ b/tolk-tester/tests/inference-tests.tolk @@ -0,0 +1,107 @@ +// the goal of this file is not only to @testcase results — +// but to check that this file compiles + +fun eq(value: X): X { return value; } + +fun test1(x: int, y: int) { + __expect_type(0, "int"); + __expect_type("0"c, "int"); + __expect_type(x, "int"); + __expect_type(x + y, "int"); + __expect_type(x * y, "int"); + __expect_type(x & y, "int"); + __expect_type(x << y, "int"); + __expect_type((((x))), "int"); + __expect_type(x = x, "int"); + __expect_type(x += x, "int"); + __expect_type(x &= x, "int"); + __expect_type(random() ? x : y, "int"); + __expect_type(eq(x), "int"); + __expect_type(eq(x), "int"); + __expect_type(eq(null), "int?"); + __expect_type(x as int, "int"); + __expect_type(+x, "int"); + __expect_type(~x, "int"); + __expect_type(x!, "int"); + __expect_type(x!!!, "int"); + { + var x: slice = beginCell().endCell().beginParse(); + __expect_type(x, "slice"); + __expect_type(beginCell(), "builder"); + __expect_type(beginCell().endCell(), "cell"); + } +} + +fun test2(x: int, y: bool) { + __expect_type(!x, "bool"); + __expect_type(x != x, "bool"); + __expect_type(x <= x, "bool"); + __expect_type(x <=> x, "bool"); + __expect_type(x <=> x, "bool"); + __expect_type(!random(), "bool"); + __expect_type(!!(x != null), "bool"); + __expect_type(x ? x != null : null == x, "bool"); + __expect_type(y & true, "bool"); + __expect_type(y ^= false, "bool"); + __expect_type(x && y, "bool"); + __expect_type(true && false && true, "bool"); + __expect_type(x || x, "bool"); + __expect_type(x || !x || (true & false), "bool"); +} + +fun test3() { + __expect_type(true as int, "int"); + __expect_type(!random() as int, "int"); +} + +fun test4(x: int) { + __expect_type((), "()"); + __expect_type((x, x), "(int, int)"); + __expect_type((x, (x, x), x), "(int, (int, int), int)"); +} + +fun test5(x: int) { + __expect_type([], "[]"); + __expect_type([x], "[int]"); + __expect_type([x, x >= 1], "[int, bool]"); + __expect_type([x, x >= 1, null as slice?], "[int, bool, slice?]"); + __expect_type((x, [x], [[x], x]), "(int, [int], [[int], int])"); + __expect_type(getMyOriginalBalanceWithExtraCurrencies(), "[int, cell?]"); +} + +fun test6() { + var t = createEmptyTuple(); + __expect_type(t, "tuple"); + t.tuplePush(1); + __expect_type(t, "tuple"); +} + +fun test7() { + __expect_type(test3(), "void"); + __expect_type(test3, "() -> void"); + var cb = test1; + __expect_type(cb, "(int, int) -> void"); + var t = createEmptyTuple(); + __expect_type(beginCell().endCell, "(builder) -> cell"); + // __expect_type(eq<(int, slice)>, "(int, slice) -> (int, slice)"); +} + +fun alwaysThrows(): never { throw 123; } +fun alwaysThrowsNotAnnotated() { throw 123; } +fun alwaysThrowsNotAnnotated2() { alwaysThrows(); } + +fun test9() { + __expect_type(alwaysThrows(), "never"); + __expect_type(alwaysThrows, "() -> never"); + __expect_type(alwaysThrowsNotAnnotated(), "void"); + __expect_type(alwaysThrowsNotAnnotated2(), "void"); +} + + +fun main() { + return 0; +} + +/** +@testcase | 0 | | 0 +*/ diff --git a/tolk-tester/tests/inline_big.tolk b/tolk-tester/tests/inline_big.tolk new file mode 100644 index 00000000..be014eb5 --- /dev/null +++ b/tolk-tester/tests/inline_big.tolk @@ -0,0 +1,62 @@ +@inline +fun foo(x: int): int { + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + return x; +} + +fun main(x: int): int { + return foo(x) * 10 + 5; +} +/** + method_id | in | out +@testcase | 0 | 9 | 9111111111111111111111111111111111111111111111111115 +*/ diff --git a/tolk-tester/tests/inline_if.tolk b/tolk-tester/tests/inline_if.tolk new file mode 100644 index 00000000..9f1fa8c1 --- /dev/null +++ b/tolk-tester/tests/inline_if.tolk @@ -0,0 +1,28 @@ +fun foo1(x: int): int { + if (x == 1) { + return 1; + } + return 2; +} +@inline +fun foo2(x: int): int { + if (x == 1) { + return 11; + } + return 22; +} +@inline_ref +fun foo3(x: int): int { + if (x == 1) { + return 111; + } + return 222; +} +fun main(x: int): (int, int, int) { + return (foo1(x)+1, foo2(x)+1, foo3(x)+1); +} +/** + method_id | in | out +@testcase | 0 | 1 | 2 12 112 +@testcase | 0 | 2 | 3 23 223 +*/ diff --git a/tolk-tester/tests/inline_loops.tolk b/tolk-tester/tests/inline_loops.tolk new file mode 100644 index 00000000..eba595a5 --- /dev/null +++ b/tolk-tester/tests/inline_loops.tolk @@ -0,0 +1,48 @@ +global g: int; + +@inline +fun foo_repeat() { + g = 1; + repeat(5) { + g *= 2; + } +} + +@inline +fun foo_until(): int { + g = 1; + var i: int = 0; + do { + g *= 2; + i += 1; + } while (i < 8); + return i; +} + +@inline +fun foo_while(): int { + g = 1; + var i: int = 0; + while (i < 10) { + g *= 2; + i += 1; + } + return i; +} + +fun main() { + foo_repeat(); + var x: int = g; + foo_until(); + var y: int = g; + foo_while(); + var z: int = g; + return (x, y, z); +} + +/** + method_id | in | out +@testcase | 0 | | 32 256 1024 + +@code_hash 102749806552989901976653997041637095139193406161777448419603700344770997608788 +*/ diff --git a/tolk-tester/tests/invalid-assign-1.tolk b/tolk-tester/tests/invalid-assign-1.tolk new file mode 100644 index 00000000..799176df --- /dev/null +++ b/tolk-tester/tests/invalid-assign-1.tolk @@ -0,0 +1,9 @@ +fun main() { + var t = createEmptyTuple(); + t.0 = (1, 2); +} + +/** +@compilation_should_fail +@stderr a tuple can not have `(int, int)` inside, because it occupies 2 stack slots in TVM, not 1 +*/ diff --git a/tolk-tester/tests/invalid-assign-2.tolk b/tolk-tester/tests/invalid-assign-2.tolk new file mode 100644 index 00000000..6a33e696 --- /dev/null +++ b/tolk-tester/tests/invalid-assign-2.tolk @@ -0,0 +1,8 @@ +fun main(cs: slice) { + var cb = cs.tupleSize; +} + +/** +@compilation_should_fail +@stderr referencing a method for `tuple` with object of type `slice` +*/ diff --git a/tolk-tester/tests/invalid-assign-3.tolk b/tolk-tester/tests/invalid-assign-3.tolk new file mode 100644 index 00000000..567ace33 --- /dev/null +++ b/tolk-tester/tests/invalid-assign-3.tolk @@ -0,0 +1,9 @@ +fun main() { + var t = createEmptyTuple(); + var xy = t.0 as (int, int); +} + +/** +@compilation_should_fail +@stderr a tuple can not have `(int, int)` inside, because it occupies 2 stack slots in TVM, not 1 +*/ diff --git a/tolk-tester/tests/invalid-bitwise-1.tolk b/tolk-tester/tests/invalid-bitwise-1.tolk new file mode 100644 index 00000000..f939d60d --- /dev/null +++ b/tolk-tester/tests/invalid-bitwise-1.tolk @@ -0,0 +1,9 @@ +fun main(flags: int): int { + return flags&0xFF!=0; +} + +/** +@compilation_should_fail +@stderr & has lower precedence than != +@stderr Use parenthesis +*/ diff --git a/tolk-tester/tests/invalid-bitwise-2.tolk b/tolk-tester/tests/invalid-bitwise-2.tolk new file mode 100644 index 00000000..e6fcd1e5 --- /dev/null +++ b/tolk-tester/tests/invalid-bitwise-2.tolk @@ -0,0 +1,8 @@ +fun justTrue(): int { return true; } + +const a = justTrue() | 1 < 9; + +/** +@compilation_should_fail +@stderr | has lower precedence than < +*/ diff --git a/tolk-tester/tests/invalid-bitwise-3.tolk b/tolk-tester/tests/invalid-bitwise-3.tolk new file mode 100644 index 00000000..ee43860b --- /dev/null +++ b/tolk-tester/tests/invalid-bitwise-3.tolk @@ -0,0 +1,8 @@ +fun justTrue(): int { return true; } + +const a = justTrue() | (1 < 9) | justTrue() != true; + +/** +@compilation_should_fail +@stderr | has lower precedence than != +*/ diff --git a/tolk-tester/tests/invalid-bitwise-4.tolk b/tolk-tester/tests/invalid-bitwise-4.tolk new file mode 100644 index 00000000..563ed535 --- /dev/null +++ b/tolk-tester/tests/invalid-bitwise-4.tolk @@ -0,0 +1,6 @@ +const a = (1) <=> (0) ^ 8; + +/** +@compilation_should_fail +@stderr ^ has lower precedence than <=> +*/ diff --git a/tolk-tester/tests/invalid-bitwise-5.tolk b/tolk-tester/tests/invalid-bitwise-5.tolk new file mode 100644 index 00000000..1030ed8d --- /dev/null +++ b/tolk-tester/tests/invalid-bitwise-5.tolk @@ -0,0 +1,11 @@ +const MAX_SLIPAGE = 100; + +fun main(jetton_amount: int, msg_value: int, slippage: int) { + if ((0 == jetton_amount) | (msg_value == 0) | true | false | slippage > MAX_SLIPAGE) { + } +} + +/** +@compilation_should_fail +@stderr | has lower precedence than > +*/ diff --git a/tolk-tester/tests/invalid-bitwise-6.tolk b/tolk-tester/tests/invalid-bitwise-6.tolk new file mode 100644 index 00000000..9c4dc67e --- /dev/null +++ b/tolk-tester/tests/invalid-bitwise-6.tolk @@ -0,0 +1,9 @@ +fun main() { + if ((1==1)|(2==2)&(3==3)) { + } +} + +/** +@compilation_should_fail +@stderr mixing | with & without parenthesis +*/ diff --git a/tolk-tester/tests/invalid-bitwise-7.tolk b/tolk-tester/tests/invalid-bitwise-7.tolk new file mode 100644 index 00000000..39fba401 --- /dev/null +++ b/tolk-tester/tests/invalid-bitwise-7.tolk @@ -0,0 +1,8 @@ +fun main() { + var c = x && y || x && y; +} + +/** +@compilation_should_fail +@stderr mixing && with || without parenthesis +*/ diff --git a/tolk-tester/tests/invalid-builtin-1.tolk b/tolk-tester/tests/invalid-builtin-1.tolk new file mode 100644 index 00000000..6a7f1ca7 --- /dev/null +++ b/tolk-tester/tests/invalid-builtin-1.tolk @@ -0,0 +1,10 @@ +fun moddiv2(x: int, y: int): (int, int) builtin; + +/** +@compilation_should_fail +@stderr +""" +`builtin` used for non-builtin function +fun moddiv2 +""" +*/ diff --git a/tolk-tester/tests/invalid-call-1.tolk b/tolk-tester/tests/invalid-call-1.tolk new file mode 100644 index 00000000..7435bb3c --- /dev/null +++ b/tolk-tester/tests/invalid-call-1.tolk @@ -0,0 +1,10 @@ +const asdf = 1; + +fun main(x: int) { + return x.asdf(); +} + +/** +@compilation_should_fail +@stderr non-existing method `asdf` of type `int` + */ diff --git a/tolk-tester/tests/invalid-call-10.tolk b/tolk-tester/tests/invalid-call-10.tolk new file mode 100644 index 00000000..4da85f4f --- /dev/null +++ b/tolk-tester/tests/invalid-call-10.tolk @@ -0,0 +1,11 @@ +fun takeInvalidTuple(t: [int, (int, builder), int]) { +} + +fun main() { + takeInvalidTuple([1, (2, beginCell()), 0]); +} + +/** +@compilation_should_fail +@stderr a tuple can not have `(int, builder)` inside, because it occupies 2 stack slots in TVM, not 1 + */ diff --git a/tolk-tester/tests/invalid-call-11.tolk b/tolk-tester/tests/invalid-call-11.tolk new file mode 100644 index 00000000..f631c546 --- /dev/null +++ b/tolk-tester/tests/invalid-call-11.tolk @@ -0,0 +1,11 @@ +fun main() { + var functions = (beginCell, beginCell); + var b = functions.1(); // ok + var c = functions.2(); // error +} + +/** +@compilation_should_fail +@stderr invalid tensor index, expected 0..1 +@stderr functions.2() + */ diff --git a/tolk-tester/tests/invalid-call-2.tolk b/tolk-tester/tests/invalid-call-2.tolk new file mode 100644 index 00000000..5a8c9fa5 --- /dev/null +++ b/tolk-tester/tests/invalid-call-2.tolk @@ -0,0 +1,14 @@ +fun add1(x: int) { + return x + 1; +} + +fun main() { + val adder_fn = add1; + var x = 10; + return adder_fn(mutate x); +} + +/** +@compilation_should_fail +@stderr `mutate` used for non-mutate argument + */ diff --git a/tolk-tester/tests/invalid-call-3.tolk b/tolk-tester/tests/invalid-call-3.tolk new file mode 100644 index 00000000..ac98df70 --- /dev/null +++ b/tolk-tester/tests/invalid-call-3.tolk @@ -0,0 +1,12 @@ +fun with2Params(x: int, y: int) { + +} + +fun main() { + return with2Params(1); +} + +/** +@compilation_should_fail +@stderr too few arguments in call to `with2Params`, expected 2, have 1 + */ diff --git a/tolk-tester/tests/invalid-call-4.tolk b/tolk-tester/tests/invalid-call-4.tolk new file mode 100644 index 00000000..c8f7dceb --- /dev/null +++ b/tolk-tester/tests/invalid-call-4.tolk @@ -0,0 +1,13 @@ +fun methodWith1Param(self: int, param: int) { + +} + +fun main() { + val x = 10; + x.methodWith1Param(2, "asdf"); +} + +/** +@compilation_should_fail +@stderr too many arguments in call to `methodWith1Param`, expected 1, have 2 + */ diff --git a/tolk-tester/tests/invalid-call-5.tolk b/tolk-tester/tests/invalid-call-5.tolk new file mode 100644 index 00000000..32905cd7 --- /dev/null +++ b/tolk-tester/tests/invalid-call-5.tolk @@ -0,0 +1,13 @@ +fun inc(x: int) { + return x + 1; +} + +fun main() { + return inc(_); +} + +/** +@compilation_should_fail +@stderr `_` can't be used as a value; it's a placeholder for a left side of assignment +@stderr inc(_) + */ diff --git a/tolk-tester/tests/invalid-call-6.tolk b/tolk-tester/tests/invalid-call-6.tolk new file mode 100644 index 00000000..cbf59806 --- /dev/null +++ b/tolk-tester/tests/invalid-call-6.tolk @@ -0,0 +1,12 @@ +fun nothing() { +} + +fun main() { + val x = 0; + return x.nothing(); +} + +/** +@compilation_should_fail +@stderr `nothing` has no parameters and can not be called as method + */ diff --git a/tolk-tester/tests/invalid-call-7.tolk b/tolk-tester/tests/invalid-call-7.tolk new file mode 100644 index 00000000..cf8c788c --- /dev/null +++ b/tolk-tester/tests/invalid-call-7.tolk @@ -0,0 +1,14 @@ +fun main() { + beginCell() + .storeAddressNone() + .storeUint(3, 32) + .storeUnexisting() + .storeInt(1, 32) + .endCell(); +} + +/** +@compilation_should_fail +@stderr non-existing method `storeUnexisting` of type `builder` +@stderr .storeUnexisting() + */ diff --git a/tolk-tester/tests/invalid-call-8.tolk b/tolk-tester/tests/invalid-call-8.tolk new file mode 100644 index 00000000..199aa681 --- /dev/null +++ b/tolk-tester/tests/invalid-call-8.tolk @@ -0,0 +1,10 @@ +fun get_incoming_value() { return 3; } + +fun main() { + var incoming_ton: int = get_incoming_value().3(); +} + +/** +@compilation_should_fail +@stderr type `int` is not indexable + */ diff --git a/tolk-tester/tests/invalid-call-9.tolk b/tolk-tester/tests/invalid-call-9.tolk new file mode 100644 index 00000000..87eb61e8 --- /dev/null +++ b/tolk-tester/tests/invalid-call-9.tolk @@ -0,0 +1,10 @@ +fun getOne() { return 1; } + +fun main() { + return getOne(); +} + +/** +@compilation_should_fail +@stderr calling a not generic function with generic T + */ diff --git a/tolk-tester/tests/invalid-catch-1.tolk b/tolk-tester/tests/invalid-catch-1.tolk new file mode 100644 index 00000000..54f6e182 --- /dev/null +++ b/tolk-tester/tests/invalid-catch-1.tolk @@ -0,0 +1,12 @@ +fun main() { + try { + + } catch(if, arg) {} + return 0; +} + +/** +@compilation_should_fail +@stderr expected identifier, got `if` +@stderr catch(if + */ diff --git a/tolk-tester/tests/invalid-catch-2.tolk b/tolk-tester/tests/invalid-catch-2.tolk new file mode 100644 index 00000000..a0276146 --- /dev/null +++ b/tolk-tester/tests/invalid-catch-2.tolk @@ -0,0 +1,9 @@ +fun main() { + try {} + catch(err, arg, more) {} +} + +/** +@compilation_should_fail +@stderr expected `)`, got `,` + */ diff --git a/tolk-tester/tests/invalid-cmt-nested.tolk b/tolk-tester/tests/invalid-cmt-nested.tolk new file mode 100644 index 00000000..807e7be8 --- /dev/null +++ b/tolk-tester/tests/invalid-cmt-nested.tolk @@ -0,0 +1,11 @@ +/* +in tolk we decided to drop nested comments support +/* +not nested + */ +*/ + +/** +@compilation_should_fail +@stderr error: expected fun or get, got `*` +*/ diff --git a/tolk-tester/tests/invalid-cmt-old.tolk b/tolk-tester/tests/invalid-cmt-old.tolk new file mode 100644 index 00000000..58927d3a --- /dev/null +++ b/tolk-tester/tests/invalid-cmt-old.tolk @@ -0,0 +1,8 @@ +fun main(): int { + ;; here is not a comment +} + +/** +@compilation_should_fail +@stderr error: expected `;`, got `is` + */ diff --git a/tolk-tester/tests/invalid-const-1.tolk b/tolk-tester/tests/invalid-const-1.tolk new file mode 100644 index 00000000..10e8303a --- /dev/null +++ b/tolk-tester/tests/invalid-const-1.tolk @@ -0,0 +1,8 @@ +fun main() { + return 9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999; +} + +/** +@compilation_should_fail +@stderr invalid integer constant + */ diff --git a/tolk-tester/tests/invalid-cyclic-1.tolk b/tolk-tester/tests/invalid-cyclic-1.tolk new file mode 100644 index 00000000..c46b1640 --- /dev/null +++ b/tolk-tester/tests/invalid-cyclic-1.tolk @@ -0,0 +1,8 @@ +const ONE = TWO - 1; +const TWO = ONE + 1; + +/** +@compilation_should_fail +@stderr const ONE +@stderr undefined symbol `TWO` + */ diff --git a/tolk-tester/tests/invalid-declaration-1.tolk b/tolk-tester/tests/invalid-declaration-1.tolk new file mode 100644 index 00000000..ea27e723 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-1.tolk @@ -0,0 +1,6 @@ +const a = 10, b = 20; + +/** +@compilation_should_fail +@stderr multiple declarations are not allowed + */ diff --git a/tolk-tester/tests/invalid-declaration-10.tolk b/tolk-tester/tests/invalid-declaration-10.tolk new file mode 100644 index 00000000..7ccb182d --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-10.tolk @@ -0,0 +1,8 @@ +get fun onInternalMessage() { + return 0; +} + +/** +@compilation_should_fail +@stderr invalid declaration of a reserved function + */ diff --git a/tolk-tester/tests/invalid-declaration-11.tolk b/tolk-tester/tests/invalid-declaration-11.tolk new file mode 100644 index 00000000..75ebb450 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-11.tolk @@ -0,0 +1,13 @@ +// this function is declared incorrectly, +// since it should return 2 values onto a stack (1 for returned slice, 1 for mutated int) +// but contains not 2 numbers in asm ret_order +fun loadAddress2(mutate self: int): slice + asm( -> 1 0 2) "LDMSGADDR"; + +fun main(){} + +/** +@compilation_should_fail +@stderr ret_order (after ->) expected to contain 2 numbers +@stderr asm( -> 1 0 2) + */ diff --git a/tolk-tester/tests/invalid-declaration-12.tolk b/tolk-tester/tests/invalid-declaration-12.tolk new file mode 100644 index 00000000..25ae9de6 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-12.tolk @@ -0,0 +1,16 @@ +fun proxy(x: int) { + return factorial(x); +} + +fun factorial(x: int) { + if (x <= 0) { + return 1; + } + return x * proxy(x-1); +} + +/** +@compilation_should_fail +@stderr could not infer return type of `factorial`, because it appears in a recursive call chain +@stderr fun factorial + */ diff --git a/tolk-tester/tests/invalid-declaration-13.tolk b/tolk-tester/tests/invalid-declaration-13.tolk new file mode 100644 index 00000000..758a4f21 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-13.tolk @@ -0,0 +1,7 @@ +const c: slice = 123 + 456; + +/** +@compilation_should_fail +@stderr expression type does not match declared type +@stderr const c + */ diff --git a/tolk-tester/tests/invalid-declaration-2.tolk b/tolk-tester/tests/invalid-declaration-2.tolk new file mode 100644 index 00000000..07ffb683 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-2.tolk @@ -0,0 +1,8 @@ +fun main(int): int { + +} + +/** +@compilation_should_fail +@stderr expected `: `, got `)` +*/ diff --git a/tolk-tester/tests/invalid-declaration-3.tolk b/tolk-tester/tests/invalid-declaration-3.tolk new file mode 100644 index 00000000..3edc09fd --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-3.tolk @@ -0,0 +1,8 @@ +int main() { + +} + +/** +@compilation_should_fail +@stderr expected fun or get, got `int` +*/ diff --git a/tolk-tester/tests/invalid-declaration-4.tolk b/tolk-tester/tests/invalid-declaration-4.tolk new file mode 100644 index 00000000..62fd6c56 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-4.tolk @@ -0,0 +1,8 @@ +fun main() { + int x = 0; +} + +/** +@compilation_should_fail +@stderr expected `;`, got `x` +*/ diff --git a/tolk-tester/tests/invalid-declaration-5.tolk b/tolk-tester/tests/invalid-declaration-5.tolk new file mode 100644 index 00000000..bf23d857 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-5.tolk @@ -0,0 +1,6 @@ +enum MyKind { } + +/** +@compilation_should_fail +@stderr `enum` is not supported yet +*/ diff --git a/tolk-tester/tests/invalid-declaration-6.tolk b/tolk-tester/tests/invalid-declaration-6.tolk new file mode 100644 index 00000000..42cb7b95 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-6.tolk @@ -0,0 +1,8 @@ +get seqno(self: int) { + return 0; +} + +/** +@compilation_should_fail +@stderr get methods can't have `mutate` and `self` params + */ diff --git a/tolk-tester/tests/invalid-declaration-7.tolk b/tolk-tester/tests/invalid-declaration-7.tolk new file mode 100644 index 00000000..8d188ea0 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-7.tolk @@ -0,0 +1,8 @@ +fun main() { + var a = 10, b = 20; +} + +/** +@compilation_should_fail +@stderr multiple declarations are not allowed + */ diff --git a/tolk-tester/tests/invalid-declaration-8.tolk b/tolk-tester/tests/invalid-declaration-8.tolk new file mode 100644 index 00000000..06cb9a98 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-8.tolk @@ -0,0 +1,8 @@ +fun someDemo() { + return 0; +} + +/** +@compilation_should_fail +@stderr the contract has no entrypoint + */ diff --git a/tolk-tester/tests/invalid-declaration-9.tolk b/tolk-tester/tests/invalid-declaration-9.tolk new file mode 100644 index 00000000..8cb71c73 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-9.tolk @@ -0,0 +1,9 @@ +fun recv_internal() { + return 0; +} + +/** +@compilation_should_fail +@stderr this is a reserved FunC/Fift identifier +@stderr you need `onInternalMessage` + */ diff --git a/tolk-tester/tests/invalid-generics-1.tolk b/tolk-tester/tests/invalid-generics-1.tolk new file mode 100644 index 00000000..0bbdeee6 --- /dev/null +++ b/tolk-tester/tests/invalid-generics-1.tolk @@ -0,0 +1,10 @@ +fun f(v: int, x: X) {} + +fun failCantDeduceWithoutArgument() { + return f(1); +} + +/** +@compilation_should_fail +@stderr too few arguments in call to `f`, expected 2, have 1 + */ diff --git a/tolk-tester/tests/invalid-generics-10.tolk b/tolk-tester/tests/invalid-generics-10.tolk new file mode 100644 index 00000000..c7f72bf4 --- /dev/null +++ b/tolk-tester/tests/invalid-generics-10.tolk @@ -0,0 +1,9 @@ +fun invalidReferencingGenericMethodWithoutGeneric() { + var t = createEmptyTuple(); + var cb = t.tupleLast; +} + +/** +@compilation_should_fail +@stderr can not use a generic function `tupleLast` as non-call + */ diff --git a/tolk-tester/tests/invalid-generics-11.tolk b/tolk-tester/tests/invalid-generics-11.tolk new file mode 100644 index 00000000..a399bc91 --- /dev/null +++ b/tolk-tester/tests/invalid-generics-11.tolk @@ -0,0 +1,11 @@ +global gVar: int; + +fun main() { + var x = gVar; + return x; +} + +/** +@compilation_should_fail +@stderr generic T not expected here + */ diff --git a/tolk-tester/tests/invalid-generics-12.tolk b/tolk-tester/tests/invalid-generics-12.tolk new file mode 100644 index 00000000..62a6f5da --- /dev/null +++ b/tolk-tester/tests/invalid-generics-12.tolk @@ -0,0 +1,15 @@ +fun getTwo(): X { return 2; } + +fun cantDeduceNonArgumentGeneric() { + var t1: [int] = [0]; + t1.0 = getTwo(); // ok + var t2 = createEmptyTuple(); + t2.tuplePush(0); + t2.0 = getTwo(); // error, can't decude X +} + +/** +@compilation_should_fail +@stderr can not deduce X for generic function `getTwo` +@stderr t2.0 = getTwo(); + */ diff --git a/tolk-tester/tests/invalid-generics-13.tolk b/tolk-tester/tests/invalid-generics-13.tolk new file mode 100644 index 00000000..d10e2174 --- /dev/null +++ b/tolk-tester/tests/invalid-generics-13.tolk @@ -0,0 +1,11 @@ +fun calcSum(x: X, y: X) { return x + y; } + +fun cantApplyPlusOnNullable() { + return calcSum(((0 as int?)), null); +} + +/** +@compilation_should_fail +@stderr in function `calcSum` +@stderr can not apply operator `+` to `int?` and `int?` + */ diff --git a/tolk-tester/tests/invalid-generics-14.tolk b/tolk-tester/tests/invalid-generics-14.tolk new file mode 100644 index 00000000..eb3adc92 --- /dev/null +++ b/tolk-tester/tests/invalid-generics-14.tolk @@ -0,0 +1,17 @@ +fun eq(v: X) {} + +fun cantDeduceWhenNotInferred() { + // at type inferring (before type checking) they are unknown + var (x, y) = 2; + + eq(x as int); // ok (since execution doesn't reach type checking) + eq(x); // ok (since execution doesn't reach type checking) + eq(x); +} + +/** +@compilation_should_fail +@stderr in function `cantDeduceWhenNotInferred` +@stderr can not deduce X for generic function `eq` +@stderr eq(x); + */ diff --git a/tolk-tester/tests/invalid-generics-2.tolk b/tolk-tester/tests/invalid-generics-2.tolk new file mode 100644 index 00000000..15594433 --- /dev/null +++ b/tolk-tester/tests/invalid-generics-2.tolk @@ -0,0 +1,10 @@ +fun f(v: int, x: T) {} + +fun failCantDeduceWithPlainNull() { + return f(0, null); +} + +/** +@compilation_should_fail +@stderr can not deduce T for generic function `f` + */ diff --git a/tolk-tester/tests/invalid-generics-3.tolk b/tolk-tester/tests/invalid-generics-3.tolk new file mode 100644 index 00000000..72b7df0e --- /dev/null +++ b/tolk-tester/tests/invalid-generics-3.tolk @@ -0,0 +1,11 @@ +fun f(x: T, y: T) {} + +fun failIncompatibleTypesForT() { + return f(32, ""); +} + +/** +@compilation_should_fail +@stderr T is both int and slice for generic function `f` +@stderr f(32 + */ diff --git a/tolk-tester/tests/invalid-generics-4.tolk b/tolk-tester/tests/invalid-generics-4.tolk new file mode 100644 index 00000000..07472ba3 --- /dev/null +++ b/tolk-tester/tests/invalid-generics-4.tolk @@ -0,0 +1,10 @@ +fun f(x: T): void asm "NOP"; + +fun failInstantiatingAsmFunctionWithNon1Slot() { + f((1, 2)); +} + +/** +@compilation_should_fail +@stderr can not call `f` with T=(int, int), because it occupies 2 stack slots in TVM, not 1 + */ diff --git a/tolk-tester/tests/invalid-generics-5.tolk b/tolk-tester/tests/invalid-generics-5.tolk new file mode 100644 index 00000000..4d4f2967 --- /dev/null +++ b/tolk-tester/tests/invalid-generics-5.tolk @@ -0,0 +1,10 @@ +fun f(x: T): void asm "NOP"; + +fun failUsingGenericFunctionPartially() { + var cb = f; +} + +/** +@compilation_should_fail +@stderr can not use a generic function `f` as non-call + */ diff --git a/tolk-tester/tests/invalid-generics-6.tolk b/tolk-tester/tests/invalid-generics-6.tolk new file mode 100644 index 00000000..73e6403f --- /dev/null +++ b/tolk-tester/tests/invalid-generics-6.tolk @@ -0,0 +1,10 @@ +fun eq(t: X) { return t; } + +fun failUsingGenericFunctionPartially() { + var cb = createEmptyTuple().eq().eq().tuplePush; +} + +/** +@compilation_should_fail +@stderr can not use a generic function `tuplePush` as non-call + */ diff --git a/tolk-tester/tests/invalid-generics-7.tolk b/tolk-tester/tests/invalid-generics-7.tolk new file mode 100644 index 00000000..076b7804 --- /dev/null +++ b/tolk-tester/tests/invalid-generics-7.tolk @@ -0,0 +1,17 @@ +fun failOnInstantiation(a: slice) { + var b: slice = foo(a); +} + +fun bar(value: X) : X { + return 1; +} +fun foo(value: X) : X { + return bar(value); +} + +/** +@compilation_should_fail +@stderr in function `bar` +@stderr can not convert type `int` to return type `slice` +@stderr return 1 + */ diff --git a/tolk-tester/tests/invalid-generics-8.tolk b/tolk-tester/tests/invalid-generics-8.tolk new file mode 100644 index 00000000..d2c24e53 --- /dev/null +++ b/tolk-tester/tests/invalid-generics-8.tolk @@ -0,0 +1,11 @@ +fun withT1T2(a: (T1, T2)) {} + +fun wrongTCountPassed() { + withT1T2((5, "")); +} + +/** +@compilation_should_fail +@stderr wrong count of generic T: expected 2, got 1 +@stderr + */ diff --git a/tolk-tester/tests/invalid-generics-9.tolk b/tolk-tester/tests/invalid-generics-9.tolk new file mode 100644 index 00000000..73fd6f87 --- /dev/null +++ b/tolk-tester/tests/invalid-generics-9.tolk @@ -0,0 +1,8 @@ +fun invalidProvidingGenericTsToNotGeneric() { + beginCell(); +} + +/** +@compilation_should_fail +@stderr calling a not generic function with generic T + */ diff --git a/tolk-tester/tests/invalid-get-method-1.tolk b/tolk-tester/tests/invalid-get-method-1.tolk new file mode 100644 index 00000000..263370d4 --- /dev/null +++ b/tolk-tester/tests/invalid-get-method-1.tolk @@ -0,0 +1,9 @@ +@method_id(123) +get fun hello(x: int, y: int): (int, int) { + return (x, y); +} + +/** +@compilation_should_fail +@stderr @method_id can be specified only for regular functions +*/ diff --git a/tolk-tester/tests/invalid-get-method-2.tolk b/tolk-tester/tests/invalid-get-method-2.tolk new file mode 100644 index 00000000..7c7a1413 --- /dev/null +++ b/tolk-tester/tests/invalid-get-method-2.tolk @@ -0,0 +1,17 @@ +@pure +get fun secret(): int { + return 0; +} +@pure +get fun balanced(): int { + return 1; +} + +fun main(): int { + return secret() + balanced(); +} + +/** +@compilation_should_fail +@stderr GET methods hash collision: `secret` and `balanced` produce the same hash +*/ diff --git a/tolk-tester/tests/invalid-import.tolk b/tolk-tester/tests/invalid-import.tolk new file mode 100644 index 00000000..416764b6 --- /dev/null +++ b/tolk-tester/tests/invalid-import.tolk @@ -0,0 +1,11 @@ +// line1 +/* */ import "unexisting.tolk"; +// line3 + +/** +@compilation_should_fail +On Linux/Mac, `realpath()` returns an error, and the error message is `cannot find file` +On Windows, it fails after, on reading, with a message "cannot open file" +@stderr invalid-import.tolk:2:7: error: Failed to import: cannot +@stderr import "unexisting.tolk"; + */ diff --git a/tolk-tester/tests/invalid-mutate-1.tolk b/tolk-tester/tests/invalid-mutate-1.tolk new file mode 100644 index 00000000..280d1e99 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-1.tolk @@ -0,0 +1,11 @@ +fun f(x: int) {} + +fun cantAssignToVal() { + val x = 10; + f(x += 1); +} + +/** +@compilation_should_fail +@stderr modifying immutable variable `x` + */ diff --git a/tolk-tester/tests/invalid-mutate-10.tolk b/tolk-tester/tests/invalid-mutate-10.tolk new file mode 100644 index 00000000..8cd37c51 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-10.tolk @@ -0,0 +1,16 @@ +fun increment(mutate x: int) { + x = x + 1; +} + +fun cantCallMutatingAsAMember() { + var x = 0; + x.increment(); + return x; +} + +/** +@compilation_should_fail +@stderr function `increment` mutates parameter `x` +@stderr consider calling `increment(mutate x)`, not `x.increment`() +@stderr alternatively, rename parameter to `self` to make it a method + */ diff --git a/tolk-tester/tests/invalid-mutate-11.tolk b/tolk-tester/tests/invalid-mutate-11.tolk new file mode 100644 index 00000000..dfc69851 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-11.tolk @@ -0,0 +1,8 @@ +fun load32(self: slice): int { + return self.loadUint(32); +} + +/** +@compilation_should_fail +@stderr modifying `self`, which is immutable by default + */ diff --git a/tolk-tester/tests/invalid-mutate-12.tolk b/tolk-tester/tests/invalid-mutate-12.tolk new file mode 100644 index 00000000..c8c8c68e --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-12.tolk @@ -0,0 +1,14 @@ +fun increment(mutate x: int) { + +} + +fun main() { + var x = 0; + var inc = increment; + inc(x); +} + +/** +@compilation_should_fail +@stderr saving `increment` into a variable is impossible, since it has `mutate` parameters and thus can only be called directly + */ diff --git a/tolk-tester/tests/invalid-mutate-13.tolk b/tolk-tester/tests/invalid-mutate-13.tolk new file mode 100644 index 00000000..ad861fd8 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-13.tolk @@ -0,0 +1,8 @@ +fun onInternalMessage(mutate in_msg_body: slice) { + +} + +/** +@compilation_should_fail +@stderr invalid declaration of a reserved function + */ diff --git a/tolk-tester/tests/invalid-mutate-14.tolk b/tolk-tester/tests/invalid-mutate-14.tolk new file mode 100644 index 00000000..2ba645d1 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-14.tolk @@ -0,0 +1,8 @@ +fun main(cs: slice) { + return loadInt(cs, 32); +} + +/** +@compilation_should_fail +@stderr `loadInt` is a mutating method; consider calling `cs.loadInt()`, not `loadInt(cs)` + */ diff --git a/tolk-tester/tests/invalid-mutate-15.tolk b/tolk-tester/tests/invalid-mutate-15.tolk new file mode 100644 index 00000000..f6874fb8 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-15.tolk @@ -0,0 +1,12 @@ +fun asdf(mutate cs: slice) {} + +fun main(cs: slice) { + cs.asdf(); +} + +/** +@compilation_should_fail +@stderr function `asdf` mutates parameter `cs` +@stderr consider calling `asdf(mutate cs)`, not `cs.asdf`() +@stderr alternatively, rename parameter to `self` to make it a method + */ diff --git a/tolk-tester/tests/invalid-mutate-16.tolk b/tolk-tester/tests/invalid-mutate-16.tolk new file mode 100644 index 00000000..9da6e253 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-16.tolk @@ -0,0 +1,9 @@ +fun cantCallMutatingFunctionWithAssignmentLValue() { + var t: tuple = createEmptyTuple(); + (t = createEmptyTuple()).tuplePush(1); +} + +/** +@compilation_should_fail +@stderr assignment can not be used as lvalue + */ diff --git a/tolk-tester/tests/invalid-mutate-17.tolk b/tolk-tester/tests/invalid-mutate-17.tolk new file mode 100644 index 00000000..9327f07d --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-17.tolk @@ -0,0 +1,13 @@ +@pure +fun tupleMut(mutate self: tuple): int + asm "TLEN"; + +fun main() { + var t = createEmptyTuple(); + return [[t.tupleMut]]; +} + +/** +@compilation_should_fail +@stderr saving `tupleMut` into a variable is impossible, since it has `mutate` parameters + */ diff --git a/tolk-tester/tests/invalid-mutate-18.tolk b/tolk-tester/tests/invalid-mutate-18.tolk new file mode 100644 index 00000000..bb8cde05 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-18.tolk @@ -0,0 +1,10 @@ +fun getNullableTuple(): tuple? { return createEmptyTuple(); } + +fun cantUseLValueUnwrappedNotNull() { + tuplePush(mutate getNullableTuple()!, 1); +} + +/** +@compilation_should_fail +@stderr function call can not be used as lvalue + */ diff --git a/tolk-tester/tests/invalid-mutate-19.tolk b/tolk-tester/tests/invalid-mutate-19.tolk new file mode 100644 index 00000000..bb8cde05 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-19.tolk @@ -0,0 +1,10 @@ +fun getNullableTuple(): tuple? { return createEmptyTuple(); } + +fun cantUseLValueUnwrappedNotNull() { + tuplePush(mutate getNullableTuple()!, 1); +} + +/** +@compilation_should_fail +@stderr function call can not be used as lvalue + */ diff --git a/tolk-tester/tests/invalid-mutate-2.tolk b/tolk-tester/tests/invalid-mutate-2.tolk new file mode 100644 index 00000000..71afe730 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-2.tolk @@ -0,0 +1,10 @@ +fun cantAssignToVal() { + val x = 10; + var y = 20; + [y, x] = [30, 40]; +} + +/** +@compilation_should_fail +@stderr modifying immutable variable `x` + */ diff --git a/tolk-tester/tests/invalid-mutate-20.tolk b/tolk-tester/tests/invalid-mutate-20.tolk new file mode 100644 index 00000000..f6eb2f9f --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-20.tolk @@ -0,0 +1,13 @@ +fun acceptMutateNullableTensor(mutate self: (int, int)?) { +} + +fun cantModifyTupleIndexWithTypeTransition() { + var t = [1, null]; + t.1.acceptMutateNullableTensor(); +} + +/** +@compilation_should_fail +@stderr can not call method for mutate `(int, int)?` with object of type `null` +@stderr because mutation is not type compatible + */ diff --git a/tolk-tester/tests/invalid-mutate-3.tolk b/tolk-tester/tests/invalid-mutate-3.tolk new file mode 100644 index 00000000..d556c9ed --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-3.tolk @@ -0,0 +1,11 @@ +const op_increase = 0x123; + +fun cantAssignToConst() { + var x = 10; + (x, op_increase) = (20, 30); +} + +/** +@compilation_should_fail +@stderr modifying immutable constant + */ diff --git a/tolk-tester/tests/invalid-mutate-4.tolk b/tolk-tester/tests/invalid-mutate-4.tolk new file mode 100644 index 00000000..5f2c111d --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-4.tolk @@ -0,0 +1,14 @@ + +fun inc(mutate x: int) { + x += 1; +} + +fun cantPassToMutatingFunction() { + val myVal = 10; + inc(mutate myVal); +} + +/** +@compilation_should_fail +@stderr modifying immutable variable `myVal` + */ diff --git a/tolk-tester/tests/invalid-mutate-5.tolk b/tolk-tester/tests/invalid-mutate-5.tolk new file mode 100644 index 00000000..2b282cf0 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-5.tolk @@ -0,0 +1,14 @@ +fun cantCallMutatingMethod(c: cell) { + val s: slice = c.beginParse(); + if (1) { + var s: slice = c.beginParse(); + s.loadRef(); // this is ok, 's' is another variable + } + val i = s.loadUint(32); +} + +/** +@compilation_should_fail +@stderr modifying immutable variable `s` +@stderr s.loadUint + */ diff --git a/tolk-tester/tests/invalid-mutate-6.tolk b/tolk-tester/tests/invalid-mutate-6.tolk new file mode 100644 index 00000000..749d9cab --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-6.tolk @@ -0,0 +1,16 @@ +const op_increase = 0x123; + +fun inc(mutate x: int): int { + x += 10; + return x + 1; +} + +fun cantCallMutatingFunctionWithImmutable() { + return inc(mutate op_increase); +} + +/** +@compilation_should_fail +@stderr modifying immutable constant +@stderr inc(mutate op_increase) + */ diff --git a/tolk-tester/tests/invalid-mutate-7.tolk b/tolk-tester/tests/invalid-mutate-7.tolk new file mode 100644 index 00000000..de3bce45 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-7.tolk @@ -0,0 +1,15 @@ +fun incBoth(mutate x: int, mutate y: int) { + x += 10; + y += 10; +} + +fun cantCallMutatingFunctionWithRvalue() { + var x = 10; + incBoth(mutate x, mutate 30); +} + +/** +@compilation_should_fail +@stderr literal can not be used as lvalue +@stderr incBoth(mutate x, mutate 30) + */ diff --git a/tolk-tester/tests/invalid-mutate-8.tolk b/tolk-tester/tests/invalid-mutate-8.tolk new file mode 100644 index 00000000..9b14e28f --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-8.tolk @@ -0,0 +1,10 @@ +fun cantRedefImmutable() { + val x = 10; + var (y: int, x redef) = (20, 30); + return (y, x); +} + +/** +@compilation_should_fail +@stderr `redef` for immutable variable + */ diff --git a/tolk-tester/tests/invalid-mutate-9.tolk b/tolk-tester/tests/invalid-mutate-9.tolk new file mode 100644 index 00000000..3489a288 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-9.tolk @@ -0,0 +1,9 @@ +fun increment(self: int) { + self = self + 1; +} + +/** +@compilation_should_fail +@stderr modifying `self`, which is immutable by default +@stderr probably, you want to declare `mutate self` + */ diff --git a/tolk-tester/tests/invalid-never-1.tolk b/tolk-tester/tests/invalid-never-1.tolk new file mode 100644 index 00000000..68c6c804 --- /dev/null +++ b/tolk-tester/tests/invalid-never-1.tolk @@ -0,0 +1,8 @@ +fun invalidNever(): never { + if (random()) { throw 123; } +} + +/** +@compilation_should_fail +@stderr a function returning `never` can not have a reachable endpoint + */ diff --git a/tolk-tester/tests/invalid-no-import-1.tolk b/tolk-tester/tests/invalid-no-import-1.tolk new file mode 100644 index 00000000..89f879a3 --- /dev/null +++ b/tolk-tester/tests/invalid-no-import-1.tolk @@ -0,0 +1,8 @@ +import "imports/some-math.tolk"; +import "imports/invalid-no-import.tolk"; + +/** +@compilation_should_fail +@stderr imports/invalid-no-import.tolk:2:13 +@stderr Using a non-imported symbol `someAdd` + */ diff --git a/tolk-tester/tests/invalid-no-import-2.tolk b/tolk-tester/tests/invalid-no-import-2.tolk new file mode 100644 index 00000000..d78346b9 --- /dev/null +++ b/tolk-tester/tests/invalid-no-import-2.tolk @@ -0,0 +1,9 @@ +import "@stdlib/tvm-dicts" +import "imports/use-dicts-err.tolk" + +/** +@compilation_should_fail +@stderr imports/use-dicts-err.tolk:2:22 +@stderr Using a non-imported symbol `createEmptyDict` +@stderr Forgot to import "@stdlib/tvm-dicts"? + */ diff --git a/tolk-tester/tests/invalid-nopar-1.tolk b/tolk-tester/tests/invalid-nopar-1.tolk new file mode 100644 index 00000000..a9a84865 --- /dev/null +++ b/tolk-tester/tests/invalid-nopar-1.tolk @@ -0,0 +1,12 @@ +fun eq(x: int): int { + return x; +} + +fun main(x: int): int { + return eq x; +} + +/** +@compilation_should_fail +@stderr expected `;`, got `x` + */ diff --git a/tolk-tester/tests/invalid-nopar-2.tolk b/tolk-tester/tests/invalid-nopar-2.tolk new file mode 100644 index 00000000..c7c13650 --- /dev/null +++ b/tolk-tester/tests/invalid-nopar-2.tolk @@ -0,0 +1,12 @@ + +fun main(x: int): int { + if x { + return 10; + } + return 0; +} + +/** +@compilation_should_fail +@stderr expected `(`, got `x` + */ diff --git a/tolk-tester/tests/invalid-nopar-3.tolk b/tolk-tester/tests/invalid-nopar-3.tolk new file mode 100644 index 00000000..8249ca28 --- /dev/null +++ b/tolk-tester/tests/invalid-nopar-3.tolk @@ -0,0 +1,12 @@ + +fun main(x: int): int { + if (x, 1) { + return 10; + } + return 0; +} + +/** +@compilation_should_fail +@stderr expected `)`, got `,` + */ diff --git a/tolk-tester/tests/invalid-nopar-4.tolk b/tolk-tester/tests/invalid-nopar-4.tolk new file mode 100644 index 00000000..033c483e --- /dev/null +++ b/tolk-tester/tests/invalid-nopar-4.tolk @@ -0,0 +1,8 @@ +fun load_u32(cs: slice): (slice, int) { + return cs.load_uint 32; +} + +/** +@compilation_should_fail +@stderr expected `;`, got `32` + */ diff --git a/tolk-tester/tests/invalid-pure-1.tolk b/tolk-tester/tests/invalid-pure-1.tolk new file mode 100644 index 00000000..4f0e9142 --- /dev/null +++ b/tolk-tester/tests/invalid-pure-1.tolk @@ -0,0 +1,20 @@ + +@pure +fun f_pure(): int { + return f_impure(); +} + +fun f_impure(): int { return 0; } + +fun main(): int { + return f_pure(); +} + +/** +@compilation_should_fail +@stderr +""" +an impure operation in a pure function +return f_impure(); +""" +*/ diff --git a/tolk-tester/tests/invalid-pure-2.tolk b/tolk-tester/tests/invalid-pure-2.tolk new file mode 100644 index 00000000..21320683 --- /dev/null +++ b/tolk-tester/tests/invalid-pure-2.tolk @@ -0,0 +1,23 @@ +global g: int; + +@pure +fun f_pure(): builder { + var b: builder = beginCell(); + g = g + 1; + return b; +} + +fun main(): int { + g = 0; + f_pure(); + return g; +} + +/** +@compilation_should_fail +@stderr +""" +an impure operation in a pure function +g = g + 1; +""" +*/ diff --git a/tolk-tester/tests/invalid-pure-3.tolk b/tolk-tester/tests/invalid-pure-3.tolk new file mode 100644 index 00000000..31d4f021 --- /dev/null +++ b/tolk-tester/tests/invalid-pure-3.tolk @@ -0,0 +1,24 @@ +@pure +fun validate_input(input: cell): (int, int) { + var (x, y, z, correct) = calculateCellSize(input, 10); + assert(correct) throw 102; + return (x, y); +} + +@pure +fun someF(): int { + var c: cell = beginCell().endCell(); + validate_input(c); + return 0; +} + +fun main() {} + +/** +@compilation_should_fail +@stderr +""" +an impure operation in a pure function +assert(correct) +""" +*/ diff --git a/tolk-tester/tests/invalid-redefinition-1.tolk b/tolk-tester/tests/invalid-redefinition-1.tolk new file mode 100644 index 00000000..5238a680 --- /dev/null +++ b/tolk-tester/tests/invalid-redefinition-1.tolk @@ -0,0 +1,7 @@ +global mulDivMod: int; + +/** +@compilation_should_fail +@stderr global mulDivMod: int; +@stderr redefinition of built-in symbol + */ diff --git a/tolk-tester/tests/invalid-redefinition-2.tolk b/tolk-tester/tests/invalid-redefinition-2.tolk new file mode 100644 index 00000000..3a300dc2 --- /dev/null +++ b/tolk-tester/tests/invalid-redefinition-2.tolk @@ -0,0 +1,12 @@ +global hello: int; + +fun hello(): int { + +} + +/** +@compilation_should_fail +@stderr fun hello() +@stderr redefinition of symbol, previous was at +@stderr invalid-redefinition-2.tolk:1:1 + */ diff --git a/tolk-tester/tests/invalid-redefinition-3.tolk b/tolk-tester/tests/invalid-redefinition-3.tolk new file mode 100644 index 00000000..04ed9383 --- /dev/null +++ b/tolk-tester/tests/invalid-redefinition-3.tolk @@ -0,0 +1,8 @@ +fun main(): int { + var demo_10: int = demo_10; +} + +/** +@compilation_should_fail +@stderr undefined symbol `demo_10` + */ diff --git a/tolk-tester/tests/invalid-redefinition-4.tolk b/tolk-tester/tests/invalid-redefinition-4.tolk new file mode 100644 index 00000000..993a869b --- /dev/null +++ b/tolk-tester/tests/invalid-redefinition-4.tolk @@ -0,0 +1,9 @@ +fun main(): int { + var (a: int, b: int) = (10, 20); + var (a, b: int) = (10, 20); +} + +/** +@compilation_should_fail +@stderr redeclaration of local variable `a` + */ diff --git a/tolk-tester/tests/invalid-redefinition-5.tolk b/tolk-tester/tests/invalid-redefinition-5.tolk new file mode 100644 index 00000000..4a8f5ea1 --- /dev/null +++ b/tolk-tester/tests/invalid-redefinition-5.tolk @@ -0,0 +1,9 @@ +fun main(x: int): int { + var (a: int, b: int) = (10, 20); + var (a redef, x: int) = (10, 20); +} + +/** +@compilation_should_fail +@stderr redeclaration of local variable `x` + */ diff --git a/tolk-tester/tests/invalid-redefinition-6.tolk b/tolk-tester/tests/invalid-redefinition-6.tolk new file mode 100644 index 00000000..e6b087c6 --- /dev/null +++ b/tolk-tester/tests/invalid-redefinition-6.tolk @@ -0,0 +1,10 @@ +const s1 = "asdf"; + +fun main() { + var s1 redef = "d"; +} + +/** +@compilation_should_fail +@stderr `redef` for unknown variable + */ diff --git a/tolk-tester/tests/invalid-self-1.tolk b/tolk-tester/tests/invalid-self-1.tolk new file mode 100644 index 00000000..40b54f16 --- /dev/null +++ b/tolk-tester/tests/invalid-self-1.tolk @@ -0,0 +1,8 @@ +fun cantReturnFromSelf(mutate self: int): self { + return 2; +} + +/** +@compilation_should_fail +@stderr invalid return from `self` function + */ diff --git a/tolk-tester/tests/invalid-self-2.tolk b/tolk-tester/tests/invalid-self-2.tolk new file mode 100644 index 00000000..c4aa758b --- /dev/null +++ b/tolk-tester/tests/invalid-self-2.tolk @@ -0,0 +1,8 @@ +fun cantUseSelfAsType(mutate x: int) { + var y: self = x; +} + +/** +@compilation_should_fail +@stderr `self` type can be used only as a return type of a function (enforcing it to be chainable) + */ diff --git a/tolk-tester/tests/invalid-self-3.tolk b/tolk-tester/tests/invalid-self-3.tolk new file mode 100644 index 00000000..330ac249 --- /dev/null +++ b/tolk-tester/tests/invalid-self-3.tolk @@ -0,0 +1,10 @@ +fun cantReturnSelf(mutate x: int): int { + x += 1; + return self; +} + +/** +@compilation_should_fail +@stderr using `self` in a non-member function (it does not accept the first `self` parameter) +@stderr return self + */ diff --git a/tolk-tester/tests/invalid-self-4.tolk b/tolk-tester/tests/invalid-self-4.tolk new file mode 100644 index 00000000..0be6b9e4 --- /dev/null +++ b/tolk-tester/tests/invalid-self-4.tolk @@ -0,0 +1,9 @@ +fun cantReturnNothingFromSelf(mutate self: int): self { + self = self + 1; +} + +/** +@compilation_should_fail +@stderr missing return +@stderr } + */ diff --git a/tolk-tester/tests/invalid-self-5.tolk b/tolk-tester/tests/invalid-self-5.tolk new file mode 100644 index 00000000..a007a93c --- /dev/null +++ b/tolk-tester/tests/invalid-self-5.tolk @@ -0,0 +1,15 @@ +fun increment(mutate self: int): self { + self = self + 1; + return self; +} + +fun cantReturnAnotherSelf(mutate self: int): self { + self = self + 1; + var x = 0; + return x.increment(); +} + +/** +@compilation_should_fail +@stderr invalid return from `self` function + */ diff --git a/tolk-tester/tests/invalid-self-6.tolk b/tolk-tester/tests/invalid-self-6.tolk new file mode 100644 index 00000000..588c70ab --- /dev/null +++ b/tolk-tester/tests/invalid-self-6.tolk @@ -0,0 +1,8 @@ +fun increment(x: int, self: int): int { + return x + self; +} + +/** +@compilation_should_fail +@stderr `self` can only be the first parameter + */ diff --git a/tolk-tester/tests/invalid-self-7.tolk b/tolk-tester/tests/invalid-self-7.tolk new file mode 100644 index 00000000..2fa2da49 --- /dev/null +++ b/tolk-tester/tests/invalid-self-7.tolk @@ -0,0 +1,8 @@ +fun increment(x: int): int { + return self + 1; +} + +/** +@compilation_should_fail +@stderr using `self` in a non-member function + */ diff --git a/tolk-tester/tests/invalid-shift-1.tolk b/tolk-tester/tests/invalid-shift-1.tolk new file mode 100644 index 00000000..5127ce05 --- /dev/null +++ b/tolk-tester/tests/invalid-shift-1.tolk @@ -0,0 +1,8 @@ +fun main(flags: int) { + return flags << 1 + 32; +} + +/** +@compilation_should_fail +@stderr << has lower precedence than + +*/ diff --git a/tolk-tester/tests/invalid-symbol-1.tolk b/tolk-tester/tests/invalid-symbol-1.tolk new file mode 100644 index 00000000..08a86f17 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol-1.tolk @@ -0,0 +1,14 @@ +fun main(x: int): int { + if (x > 0) { + var y: int = 10; + } else { + var y: slice = "20"; + } + debugPrint(y); +} + +/** +@compilation_should_fail +@stderr debugPrint(y); +@stderr undefined symbol `y` + */ diff --git a/tolk-tester/tests/invalid-symbol-2.tolk b/tolk-tester/tests/invalid-symbol-2.tolk new file mode 100644 index 00000000..f55e15ce --- /dev/null +++ b/tolk-tester/tests/invalid-symbol-2.tolk @@ -0,0 +1,12 @@ +fun main(x: int): int { + try { + if (x > 10) { throw(44); } + } catch(code) {} + return code; +} + +/** +@compilation_should_fail +@stderr return code; +@stderr undefined symbol `code` + */ diff --git a/tolk-tester/tests/invalid-syntax-1.tolk b/tolk-tester/tests/invalid-syntax-1.tolk new file mode 100644 index 00000000..4ccc8f22 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax-1.tolk @@ -0,0 +1,15 @@ +fun main(x: int): int { + if (x > 0) { + return 1; + } + // 'elseif' doesn't exist anymore, it's treated as 'someFunction(arg)' + elseif(x < 0) { + return -1; + } + return x; +} + +/** +@compilation_should_fail +@stderr expected `;`, got `{` + */ diff --git a/tolk-tester/tests/invalid-syntax-2.tolk b/tolk-tester/tests/invalid-syntax-2.tolk new file mode 100644 index 00000000..1180dbbf --- /dev/null +++ b/tolk-tester/tests/invalid-syntax-2.tolk @@ -0,0 +1,13 @@ +fun main(x: int) { + while (x > 0) { + if (x == 10) { + break; + } + x = x -1; + } +} + +/** +@compilation_should_fail +@stderr break/continue from loops are not supported yet + */ diff --git a/tolk-tester/tests/invalid-syntax-3.tolk b/tolk-tester/tests/invalid-syntax-3.tolk new file mode 100644 index 00000000..259ea795 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax-3.tolk @@ -0,0 +1,8 @@ +fun main(x: int) { + return null(); +} + +/** +@compilation_should_fail +@stderr calling a non-function + */ diff --git a/tolk-tester/tests/invalid-syntax-4.tolk b/tolk-tester/tests/invalid-syntax-4.tolk new file mode 100644 index 00000000..044dd329 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax-4.tolk @@ -0,0 +1,8 @@ +fun main(x: int) { + assert(x > 0); +} + +/** +@compilation_should_fail +@stderr expected `throw excNo` after assert, got `;` + */ diff --git a/tolk-tester/tests/invalid-syntax-5.tolk b/tolk-tester/tests/invalid-syntax-5.tolk new file mode 100644 index 00000000..21774774 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax-5.tolk @@ -0,0 +1,8 @@ +fun main(s: auto) { + var (z, t) = ; + +/** +@compilation_should_fail +@stderr expected , got `;` +@stderr var (z, t) = ; +*/ diff --git a/tolk-tester/tests/invalid-syntax-6.tolk b/tolk-tester/tests/invalid-syntax-6.tolk new file mode 100644 index 00000000..12e02645 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax-6.tolk @@ -0,0 +1,9 @@ +fun main() { + var a = 1; + (a += 1) += 2; +} + +/** +@compilation_should_fail +@stderr assignment can not be used as lvalue +*/ diff --git a/tolk-tester/tests/invalid-syntax-7.tolk b/tolk-tester/tests/invalid-syntax-7.tolk new file mode 100644 index 00000000..9f63ac10 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax-7.tolk @@ -0,0 +1,9 @@ +fun main() { + var x = 1; + x += (var y = 2); +} + +/** +@compilation_should_fail +@stderr expected , got `var` +*/ diff --git a/tolk-tester/tests/invalid-tolk-version.tolk b/tolk-tester/tests/invalid-tolk-version.tolk new file mode 100644 index 00000000..d66de9ff --- /dev/null +++ b/tolk-tester/tests/invalid-tolk-version.tolk @@ -0,0 +1,7 @@ +tolk asdf; + +/** +@compilation_should_fail +@stderr semver expected +@stderr tolk asdf; + */ diff --git a/tolk-tester/tests/invalid-typing-1.tolk b/tolk-tester/tests/invalid-typing-1.tolk new file mode 100644 index 00000000..0089bd62 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-1.tolk @@ -0,0 +1,10 @@ +fun main() { + var tri: [int, scli] = [10, null()]; + return; +} + +/** +@compilation_should_fail +@stderr .tolk:2 +@stderr unknown type name `scli` + */ diff --git a/tolk-tester/tests/invalid-typing-10.tolk b/tolk-tester/tests/invalid-typing-10.tolk new file mode 100644 index 00000000..8c1df4a2 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-10.tolk @@ -0,0 +1,8 @@ +fun failMathOnBoolean(c: cell) { + return (null == c) * 10; +} + +/** +@compilation_should_fail +@stderr can not apply operator `*` to `bool` and `int` + */ diff --git a/tolk-tester/tests/invalid-typing-11.tolk b/tolk-tester/tests/invalid-typing-11.tolk new file mode 100644 index 00000000..f6e89d08 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-11.tolk @@ -0,0 +1,12 @@ +fun failBitwiseNotOnBool() { + var eq = 1 == 0; + if (~eq) { + return 0; + } + return -1; +} + +/** +@compilation_should_fail +@stderr can not apply operator `~` to `bool` + */ diff --git a/tolk-tester/tests/invalid-typing-12.tolk b/tolk-tester/tests/invalid-typing-12.tolk new file mode 100644 index 00000000..3a5b1fe2 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-12.tolk @@ -0,0 +1,10 @@ +fun failAssignNullToTensor() { + var ab = (1, 2); + ab = null; + return ab; +} + +/** +@compilation_should_fail +@stderr can not assign `null` to variable of type `(int, int)` + */ diff --git a/tolk-tester/tests/invalid-typing-13.tolk b/tolk-tester/tests/invalid-typing-13.tolk new file mode 100644 index 00000000..e356d0f3 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-13.tolk @@ -0,0 +1,9 @@ +fun failAssignToInvalidTupleIndex() { + var ab = [1, 2]; + ab.100500 = 5; +} + +/** +@compilation_should_fail +@stderr invalid tuple index, expected 0..1 + */ diff --git a/tolk-tester/tests/invalid-typing-14.tolk b/tolk-tester/tests/invalid-typing-14.tolk new file mode 100644 index 00000000..657ab5f4 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-14.tolk @@ -0,0 +1,14 @@ + +fun autoGetIntOrNull() { + if (random()) { return 1; } + return null; +} + +fun testAutoInferredIntOrNull() { + var b: builder = autoGetIntOrNull() as builder; +} + +/** +@compilation_should_fail +@stderr type `int?` can not be cast to `builder` + */ diff --git a/tolk-tester/tests/invalid-typing-15.tolk b/tolk-tester/tests/invalid-typing-15.tolk new file mode 100644 index 00000000..fbcff8a2 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-15.tolk @@ -0,0 +1,13 @@ + +fun getNullable4(): int? { + return 4; +} + +fun testCantSumNullable() { + return 1 + getNullable4(); +} + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `int` and `int?` + */ diff --git a/tolk-tester/tests/invalid-typing-16.tolk b/tolk-tester/tests/invalid-typing-16.tolk new file mode 100644 index 00000000..1dca7822 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-16.tolk @@ -0,0 +1,13 @@ +@pure +fun myDictDeleteStrict(mutate self: cell, keyLen: int, key: int): bool + asm(key self keyLen) "DICTIDEL"; + + +fun testCantCallDictMethodsOnNullable(c: cell) { + c.beginParse().loadDict().myDictDeleteStrict(16, 1); +} + +/** +@compilation_should_fail +@stderr can not call method for `cell` with object of type `cell?` + */ diff --git a/tolk-tester/tests/invalid-typing-17.tolk b/tolk-tester/tests/invalid-typing-17.tolk new file mode 100644 index 00000000..b7302684 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-17.tolk @@ -0,0 +1,10 @@ + +fun testCantUseNullableAsCondition(x: int?) { + if (x) { return 1; } + return 0; +} + +/** +@compilation_should_fail +@stderr can not use `int?` as a boolean condition + */ diff --git a/tolk-tester/tests/invalid-typing-18.tolk b/tolk-tester/tests/invalid-typing-18.tolk new file mode 100644 index 00000000..cf985add --- /dev/null +++ b/tolk-tester/tests/invalid-typing-18.tolk @@ -0,0 +1,16 @@ +fun incrementOrSetNull(mutate x: int?) { + if (random()) { x! += 1; } + else { x = null; } +} + +fun cantCallMutateMethodNotNullable() { + var x = 1; + incrementOrSetNull(mutate x); + return x; +} + +/** +@compilation_should_fail +@stderr can not pass `int` to mutate `int?` +@stderr because mutation is not type compatible + */ diff --git a/tolk-tester/tests/invalid-typing-19.tolk b/tolk-tester/tests/invalid-typing-19.tolk new file mode 100644 index 00000000..58b6c1fc --- /dev/null +++ b/tolk-tester/tests/invalid-typing-19.tolk @@ -0,0 +1,12 @@ +fun getNullableInt(): int? { return 5; } + +fun testCantApplyNotNullForAlwaysNull() { + var x: int? = getNullableInt(); + if (x != null) { return 0; } + return x! + 1; +} + +/** +@compilation_should_fail +@stderr operator `!` used for always null expression + */ diff --git a/tolk-tester/tests/invalid-typing-2.tolk b/tolk-tester/tests/invalid-typing-2.tolk new file mode 100644 index 00000000..052596e4 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-2.tolk @@ -0,0 +1,9 @@ +fun main() { + var tri: (int, int) = (10, false); + return; +} + +/** +@compilation_should_fail +@stderr can not assign `(int, bool)` to variable of type `(int, int)` + */ diff --git a/tolk-tester/tests/invalid-typing-20.tolk b/tolk-tester/tests/invalid-typing-20.tolk new file mode 100644 index 00000000..457bc97a --- /dev/null +++ b/tolk-tester/tests/invalid-typing-20.tolk @@ -0,0 +1,15 @@ +fun getNullableInt(): int? { return 5; } + +fun testFlowContextAppliedInBinaryOperator() { + var x: int? = getNullableInt(); + var y: int? = getNullableInt(); + if ((y = null) < y) { + return -100; + } + return 0; +} + +/** +@compilation_should_fail +@stderr can not apply operator `<` to `null` and `null` + */ diff --git a/tolk-tester/tests/invalid-typing-21.tolk b/tolk-tester/tests/invalid-typing-21.tolk new file mode 100644 index 00000000..d2a815ee --- /dev/null +++ b/tolk-tester/tests/invalid-typing-21.tolk @@ -0,0 +1,14 @@ +fun getNullableInt(): int? { return 5; } + +fun testNeverTypeOccurs() { + var x: int? = getNullableInt(); + if (x == null && x != null) { + return x + 0; + } + return 0; +} + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `never` and `int` + */ diff --git a/tolk-tester/tests/invalid-typing-22.tolk b/tolk-tester/tests/invalid-typing-22.tolk new file mode 100644 index 00000000..f962f364 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-22.tolk @@ -0,0 +1,9 @@ +fun testLogicalAndNotConditionDoesntAffect(x: int?) { + var gt1 = x != null && x > 1; + return x + 0; +} + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `int?` and `int` + */ diff --git a/tolk-tester/tests/invalid-typing-23.tolk b/tolk-tester/tests/invalid-typing-23.tolk new file mode 100644 index 00000000..74feed52 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-23.tolk @@ -0,0 +1,15 @@ +fun getTensor(): (int?, int?) { return (5, null); } + +fun testSmartCastsForFieldsDropAfterAssign() { + var t = getTensor(); + if (t.0 != null && t.1 != null) { + t = getTensor(); + return t.0 + t.1; + } + return -1; +} + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `int?` and `int?` + */ diff --git a/tolk-tester/tests/invalid-typing-24.tolk b/tolk-tester/tests/invalid-typing-24.tolk new file mode 100644 index 00000000..75f61be9 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-24.tolk @@ -0,0 +1,16 @@ +fun getNullableInt(): int? { return 5; } + +fun getTensor(x: int?): (int?, int) { return (x, 0); } + +fun testSmartCastsDropAfterAssign() { + var x: int? = 0; + var y: int? = 0; + (getTensor(x = getNullableInt()).0, getTensor(y = getNullableInt()).0) = (x + y, x - y); + return x+y; +} + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `int?` and `int?` +@stderr x + y, x - y + */ diff --git a/tolk-tester/tests/invalid-typing-25.tolk b/tolk-tester/tests/invalid-typing-25.tolk new file mode 100644 index 00000000..1621bab1 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-25.tolk @@ -0,0 +1,14 @@ +fun takeNullableTensor(mutate ij: (int, int)?) { } + +fun testSmartCastsDropAfterMutate() { + var x: (int, int)? = (1, 2); + return x.0; // ok + takeNullableTensor(mutate x); + return x.1; // error +} + +/** +@compilation_should_fail +@stderr type `(int, int)?` is not indexable +@stderr return x.1 + */ diff --git a/tolk-tester/tests/invalid-typing-26.tolk b/tolk-tester/tests/invalid-typing-26.tolk new file mode 100644 index 00000000..bf5a1165 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-26.tolk @@ -0,0 +1,12 @@ +fun getNullableInt(): int? { return 5; } + +fun testAssertThrowIsConditional() { + var (x, y) = (getNullableInt(), getNullableInt()); + assert(x != null) throw(y = 10); + return x + y; +} + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `int` and `int?` + */ diff --git a/tolk-tester/tests/invalid-typing-27.tolk b/tolk-tester/tests/invalid-typing-27.tolk new file mode 100644 index 00000000..3861403b --- /dev/null +++ b/tolk-tester/tests/invalid-typing-27.tolk @@ -0,0 +1,18 @@ +fun assignNull2(mutate x: T1?, mutate y: T2?) { + if (false) { + x = null; + y = null; + } +} + +fun testSmartCastsDropAfterNullableGeneric() { + var (x: int?, y: int?) = (1, 2); + x * y; // ok + assignNull2(x, y); // treated like assignments to nullable + x << y; // error +} + +/** +@compilation_should_fail +@stderr can not apply operator `<<` to `int?` and `int?` + */ diff --git a/tolk-tester/tests/invalid-typing-28.tolk b/tolk-tester/tests/invalid-typing-28.tolk new file mode 100644 index 00000000..5d60ff22 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-28.tolk @@ -0,0 +1,15 @@ +fun getNullableInt(): int? { return 5; } + +fun testReassignInRedef() { + var t1: int? = getNullableInt(); + if (t1 != null) { + var (t1 redef, t2) = (getNullableInt(), 5); + return t1 + t2; + } + return -1; +} + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `int?` and `int` + */ diff --git a/tolk-tester/tests/invalid-typing-29.tolk b/tolk-tester/tests/invalid-typing-29.tolk new file mode 100644 index 00000000..e8a4e5e2 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-29.tolk @@ -0,0 +1,14 @@ +fun getNullableInt(): int? { return 5; } + +fun testTryBodyDontSmartCast() { + var x = getNullableInt(); + try { + x = 5; + } catch {} + return x * 10; // x is not int here; for now, we have no exception edges, assuming it can be anywhere inside try +} + +/** +@compilation_should_fail +@stderr can not apply operator `*` to `int?` and `int` + */ diff --git a/tolk-tester/tests/invalid-typing-3.tolk b/tolk-tester/tests/invalid-typing-3.tolk new file mode 100644 index 00000000..ac019a42 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-3.tolk @@ -0,0 +1,19 @@ +fun incInt(mutate self: int): self { + self += 1; + return self; +} + +fun appendBuilder(mutate self: builder): self { + self.storeUint(1, 32); + return self; +} + +fun cantMixDifferentThis() { + var x = 0; + return x.incInt().appendBuilder().incInt(); +} + +/** +@compilation_should_fail +@stderr can not call method for `builder` with object of type `int` + */ diff --git a/tolk-tester/tests/invalid-typing-30.tolk b/tolk-tester/tests/invalid-typing-30.tolk new file mode 100644 index 00000000..53dfc5ca --- /dev/null +++ b/tolk-tester/tests/invalid-typing-30.tolk @@ -0,0 +1,15 @@ +fun getNullableInt(): int? { return 5; } + +fun testDoWhileCondition() { + var (x: int?, y: int?) = (10, 20); + do { + x = getNullableInt(); + y = getNullableInt(); + } while(x == null); + return x * y; // x is 100% int, but y is not +} + +/** +@compilation_should_fail +@stderr can not apply operator `*` to `int` and `int?` + */ diff --git a/tolk-tester/tests/invalid-typing-4.tolk b/tolk-tester/tests/invalid-typing-4.tolk new file mode 100644 index 00000000..1ee71290 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-4.tolk @@ -0,0 +1,12 @@ +fun incNotChained(mutate self: int) { + self = self + 1; +} + +fun cantCallNotChainedMethodsInAChain(x: int) { + return x.incNotChained().incNotChained(); +} + +/** +@compilation_should_fail +@stderr can not call method for `int` with object of type `void` + */ diff --git a/tolk-tester/tests/invalid-typing-44.tolk b/tolk-tester/tests/invalid-typing-44.tolk new file mode 100644 index 00000000..2ec5d0e8 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-44.tolk @@ -0,0 +1,9 @@ +fun cantAssignIntToTensor() { + var (x, y) = 2; + x + y; +} + +/** +@compilation_should_fail +@stderr can not assign `int` to a tensor + */ diff --git a/tolk-tester/tests/invalid-typing-45.tolk b/tolk-tester/tests/invalid-typing-45.tolk new file mode 100644 index 00000000..b357b637 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-45.tolk @@ -0,0 +1,9 @@ +fun cantAssignSizesMismatch() { + var [x, y] = [2, 3, 4]; + x + y; +} + +/** +@compilation_should_fail +@stderr can not assign `[int, int, int]`, sizes mismatch + */ diff --git a/tolk-tester/tests/invalid-typing-5.tolk b/tolk-tester/tests/invalid-typing-5.tolk new file mode 100644 index 00000000..9d8cd480 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-5.tolk @@ -0,0 +1,13 @@ +fun incNotChained(mutate self: int) { + self = self + 1; +} + +fun failWhenReturnANotChainedValue(x: int): int { + return x.incNotChained(); +} + +/** +@compilation_should_fail +@stderr x.incNotChained() +@stderr can not convert type `void` to return type `int` + */ diff --git a/tolk-tester/tests/invalid-typing-6.tolk b/tolk-tester/tests/invalid-typing-6.tolk new file mode 100644 index 00000000..f2e99c7d --- /dev/null +++ b/tolk-tester/tests/invalid-typing-6.tolk @@ -0,0 +1,8 @@ +fun failWhenTernaryConditionNotInt(cs: slice) { + return cs ? 1 : 0; +} + +/** +@compilation_should_fail +@stderr can not use `slice` as a boolean condition + */ diff --git a/tolk-tester/tests/invalid-typing-7.tolk b/tolk-tester/tests/invalid-typing-7.tolk new file mode 100644 index 00000000..c192a05b --- /dev/null +++ b/tolk-tester/tests/invalid-typing-7.tolk @@ -0,0 +1,9 @@ +fun failAssignPlainNullToVariable() { + var x = null; +} + +/** +@compilation_should_fail +@stderr can not infer type of `x`, it's always null +@stderr specify its type with `x: ` or use `null as ` + */ diff --git a/tolk-tester/tests/invalid-typing-8.tolk b/tolk-tester/tests/invalid-typing-8.tolk new file mode 100644 index 00000000..d696e132 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-8.tolk @@ -0,0 +1,8 @@ +fun failExplicitCastIncompatible(c: cell) { + return c as slice; +} + +/** +@compilation_should_fail +@stderr type `cell` can not be cast to `slice` + */ diff --git a/tolk-tester/tests/invalid-typing-9.tolk b/tolk-tester/tests/invalid-typing-9.tolk new file mode 100644 index 00000000..a0d5ee04 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-9.tolk @@ -0,0 +1,13 @@ +fun getTupleLastGetter(): tuple -> X { + return tupleLast; +} + +fun failTypeMismatch() { + var t = createEmptyTuple(); + var c: cell = getTupleLastGetter()(t); +} + +/** +@compilation_should_fail +@stderr can not assign `int` to variable of type `cell` + */ diff --git a/tolk-tester/tests/logical-operators.tolk b/tolk-tester/tests/logical-operators.tolk new file mode 100644 index 00000000..700f2a3c --- /dev/null +++ b/tolk-tester/tests/logical-operators.tolk @@ -0,0 +1,346 @@ +import "imports/use-dicts.tolk" + +fun simpleAllConst() { + return (!0, !!0 & !false, !!!0, !1, !!1, !-1, !!-1, (!5 as int == 0) == !0, !0 == true); +} + +fun compileTimeEval1(x: int) { + // todo now compiler doesn't understand that bool can't be equal to number other than 0/-1 + // (but understands that it can't be positive) + // that's why for now, the last condition is evaluated at runtime + return (!x, !x as int > 10, (!x as int) < 10, !!x as int == 5, !x as int == -10); +} + +@method_id(101) +fun withIfNot(x: int, y: int) { + if (!x) { return 10; } + else if (!y) { return 20; } + return x+y; +} + +@method_id(102) +fun withAndOr(x: int, y: int, z: int) { + var return_at_end = -1; + if (!x & !y) { + if (!z & !y) { return 10; } + else if ((z != 0) | !!y) { return_at_end = 20; } + } else if (!!x & !!y & !z) { + if (!z & (x > 10)) { return_at_end = 30; } + if ((x != 11) & !z) { return 40; } + return_at_end = 50; + } else { + return_at_end = !x ? !y as int : (!z as int) | 1; + } + return return_at_end; +} + +@method_id(103) +fun someSum(upto: int) { + var x = 0; + var should_break = false; + while (!x & !should_break) { + if (upto < 10) { x = upto; should_break = true; } + else { upto = upto - 1; } + } + return x; +} + +@method_id(104) +fun testDict(last: int) { + // prepare dict: [3 => 30, 4 => 40, 5 => x] + var dict = prepareDict_3_30_4_40_5_x(!last ? 100 : last); + return (lookupIdxByValue(dict, 30), lookupIdxByValue(dict, last), lookupIdxByValue(dict, 100)); +} + +@method_id(105) +fun testNotNull(x: int?) { + return [x == null, null == x, !(x == null), null == null, (null != null) as int]; +} + +@method_id(106) +fun testAndConstCodegen() { + return ( + [1 && 0, 0 && 1, 0 && 0, 1 && 1], + [4 && 3 && 0, 5 && 0 && 7 && 8, (7 && 0) && -19], + [4 && 3 && -1, 5 && -100 && 7 && 8, (7 && (1 + 2)) && -19], + [true && false, true && true] + ); +} + +@method_id(107) +fun testOrConstCodegen() { + return ( + [1 || 0, 0 || 1, 0 || 0, 1 || 1], + [0 || 0 || 0, 0 || (0 || 0), ((0 || 0) || 0) || 0], + [4 || 3 || -1, 0 || -100 || 0 || 0, (0 || (1 + -1)) || -19], + [true || false, false || false] + ); +} + +global eqCallsCnt: int; + +fun eq(x: int) { return x; } +fun eqCnt(x: int) { eqCallsCnt += 1; return x; } +fun isGt0(x: int) { return x > 0; } + +fun alwaysThrows(): int { throw 444 ; return 444; } + +@method_id(108) +fun testAndSimpleCodegen(a: int, b: int) { + return a && b; +} + +@method_id(109) +fun testOrSimpleCodegen(a: int, b: int) { + return a > 0 || b > 0; +} + +@method_id(110) +fun testLogicalOps1(x: int) { + eqCallsCnt = 0; + return ( + isGt0(x) || !isGt0(x) || alwaysThrows(), + x && eqCnt(x) && eqCnt(x - 1) && eqCnt(x - 2), + (400 == eq(x)) && alwaysThrows(), + (500 == eq(x)) || eqCnt(x) || false, + (500 == eq(x)) || eqCnt(x) || true, + eqCallsCnt + ); +} + +@method_id(111) +fun testLogicalOps2(first: int) { + var s = beginCell().storeInt(1, 32).storeInt(2, 32).storeInt(3, 32).storeInt(4, 32).storeInt(5, 32).endCell().beginParse(); + var sum = 0; + if (first && s.loadUint(32)) { + (2 == s.loadUint(32)) && (sum += s.loadUint(32)); + (3 == s.loadUint(32)) && (sum += s.loadUint(32)); + (5 == s.preloadUint(32)) && (sum += s.loadUint(32)); + } else { + (10 == s.loadUint(32)) || (20 == s.loadUint(32)) || (3 == s.loadUint(32)) || (4 == s.loadUint(32)); + sum += s.loadUint(32); + } + return (s.getRemainingBitsCount(), sum); +} + +@method_id(112) +fun mixLogicalIntsAndBools(first: int, cond: bool) { + return ( + (first && cond) || (!first && cond), + ((first & -1) & cond as int) == ((first && true) && cond) as int, + 7 && cond, + first || cond || !cond || alwaysThrows(), + cond || first || !first || alwaysThrows() + ); +} + +@method_id(113) +fun testConvertIfToIfnot(x: bool) { + assert(!!(x == false), 100); + assert(!x, 100); + if (x == !!false) { + return 1; + } + if (!!(x != !false)) { + return 1; + } + assert(!!x, 100); + return -4; +} + +fun main() { + +} + +/** +@testcase | 101 | 0 0 | 10 +@testcase | 101 | 5 0 | 20 +@testcase | 101 | 5 8 | 13 +@testcase | 102 | 0 0 0 | 10 +@testcase | 102 | 0 0 5 | 20 +@testcase | 102 | 1 2 0 | 40 +@testcase | 102 | 11 2 0 | 50 +@testcase | 102 | 1 0 0 | -1 +@testcase | 102 | 0 1 0 | 0 +@testcase | 102 | 1 0 1 | 1 +@testcase | 103 | 15 | 9 +@testcase | 103 | 6 | 6 +@testcase | 103 | -1 | -1 +@testcase | 104 | 50 | 3 5 -1 +@testcase | 104 | 100 | 3 5 5 +@testcase | 104 | 0 | 3 -1 5 +@testcase | 105 | 0 | [ 0 0 -1 -1 0 ] +@testcase | 105 | null | [ -1 -1 0 -1 0 ] +@testcase | 106 | | [ 0 0 0 -1 ] [ 0 0 0 ] [ -1 -1 -1 ] [ 0 -1 ] +@testcase | 107 | | [ -1 -1 0 -1 ] [ 0 0 0 ] [ -1 -1 -1 ] [ -1 0 ] +@testcase | 108 | 1 2 | -1 +@testcase | 108 | 1 0 | 0 +@testcase | 109 | -5 -4 | 0 +@testcase | 109 | -5 4 | -1 +@testcase | 109 | 1 99 | -1 +@testcase | 110 | 0 | -1 0 0 0 -1 2 +@testcase | 110 | 1 | -1 0 0 -1 -1 4 +@testcase | 110 | 2 | -1 0 0 -1 -1 5 +@testcase | 110 | 500 | -1 -1 0 -1 -1 3 +@testcase | 111 | 0 | 32 4 +@testcase | 111 | -1 | 0 8 +@testcase | 112 | 5 0 | 0 -1 0 -1 -1 +@testcase | 112 | 0 -1 | -1 -1 -1 -1 -1 +@testcase | 113 | 0 | 1 + +@fif_codegen +""" + simpleAllConst PROC:<{ + // + TRUE + 0 PUSHINT + TRUE + FALSE + TRUE + FALSE + TRUE + TRUE + TRUE + }> +""" + +@fif_codegen +""" + compileTimeEval1 PROC:<{ + // x + DUP // x x + 0 EQINT // x '1 + FALSE // x '1 '4 + TRUE // x '1 '4 '7 + FALSE // x '1 '4 '7 '11 + s0 s4 XCHG // '11 '1 '4 '7 x + 0 EQINT // '11 '1 '4 '7 '12 + -10 EQINT // '11 '1 '4 '7 '14 + s3 s4 XCHG + s1 s3 s0 XCHG3 // '1 '4 '7 '11 '14 + }> +""" + +@fif_codegen +""" + withIfNot PROC:<{ + c2 SAVE + SAMEALTSAVE // x y + OVER // x y x + IFNOTJMP:<{ // x y + 2DROP // + 10 PUSHINT // '2=10 + }> // x y + DUP // x y y + IFNOTJMP:<{ // x y + 2DROP // + 20 PUSHINT // '3=20 + RETALT + }> // x y + ADD // '4 + }> +""" + +@fif_codegen +""" + testAndConstCodegen PROC:<{ + // + FALSE + 0 PUSHINT + DUP + TRUE + 4 TUPLE + FALSE + 0 PUSHINT + DUP + TRIPLE + TRUE + TRUE + TRUE + TRIPLE + FALSE + TRUE + PAIR + }> +""" + +@fif_codegen +""" + testOrConstCodegen PROC:<{ + // + -1 PUSHINT + TRUE + FALSE + s2 PUSH + 4 TUPLE + FALSE + FALSE + FALSE + TRIPLE + -1 PUSHINT + DUP + TRUE + TRIPLE + -1 PUSHINT + FALSE + PAIR + }> +""" + +Currently, && operator is implemented via ?: and is not optimal in primitive cases. +For example, `a && b` can be expressed without IFs. +These are moments of future optimizations. For now, it's more than enough. +@fif_codegen +""" + testAndSimpleCodegen PROC:<{ + // a b + SWAP // b a + IF:<{ // b + 0 NEQINT // '2 + }>ELSE<{ // b + DROP // + 0 PUSHINT // '2=0 + }> + }> +""" + +@fif_codegen +""" + testOrSimpleCodegen PROC:<{ + // a b + SWAP // b a + 0 GTINT // b '3 + IF:<{ // b + DROP // + -1 PUSHINT // '4=-1 + }>ELSE<{ // b + 0 GTINT // '7 + 0 NEQINT // '4 + }> + }> +""" + +@fif_codegen +""" + testConvertIfToIfnot PROC:<{ + // x + DUP // x x + 100 THROWIF + DUP // x x + 100 THROWIF + DUP // x x + IFNOTJMP:<{ // x + DROP // + 1 PUSHINT // '5=1 + }> // x + DUP // x x + IFNOTJMP:<{ // x + DROP // + 1 PUSHINT // '6=1 + }> // x + 100 THROWIFNOT + -4 PUSHINT // '9=-4 + }> +""" + + */ diff --git a/tolk-tester/tests/method_id.tolk b/tolk-tester/tests/method_id.tolk new file mode 100644 index 00000000..e7e70d24 --- /dev/null +++ b/tolk-tester/tests/method_id.tolk @@ -0,0 +1,17 @@ +@method_id(1) +fun foo1(): int { return 111; } +@method_id(3) +fun foo2(): int { return 222; } +@method_id(10) +fun foo3(): int { return 333; } +@method_id(11) +fun slice(slice: slice): slice { return slice; } +fun main(): int { return 999; } + +/** + method_id | in | out +@testcase | 1 | | 111 +@testcase | 3 | | 222 +@testcase | 10 | | 333 +@testcase | 0 | | 999 +*/ diff --git a/tolk-tester/tests/mutate-methods.tolk b/tolk-tester/tests/mutate-methods.tolk new file mode 100644 index 00000000..9ebf8b1d --- /dev/null +++ b/tolk-tester/tests/mutate-methods.tolk @@ -0,0 +1,344 @@ +fun incrementInPlace(mutate self: int, byValue: int): void { + self = self + byValue; +} + +fun incrementTwoInPlace(mutate self: int, mutate y: int, byValue: int): int { + self.incrementInPlace(byValue); + y += byValue; + return self + y; +} + +@method_id(101) +fun testIncrement1() { + var x = 50; + var y = 30; + incrementInPlace(mutate x, 10); + incrementInPlace(mutate x, 10); + incrementInPlace(mutate y, 10); + y.incrementInPlace(10); + incrementInPlace(mutate y, 10); + return (x, y); +} + +@method_id(102) +fun testIncrement2() { + var x = 50; + var y = 30; + val sum1 = incrementTwoInPlace(mutate x, mutate y, 10); + val sum2 = x.incrementTwoInPlace(mutate y, 10); + return (x, y, sum1, sum2); +} + + +fun load_next(mutate cs: slice): int { + return loadInt(mutate cs, 32); +} + +fun myLoadInt(mutate self: slice, len: int): int + asm(-> 1 0) "LDIX"; +fun myStoreInt(mutate self: builder, x: int, len: int): self + asm(x self len) "STIX"; + +@inline_ref +fun unpack_utils_info(mutate utils_info_sl: slice): (int, int) { + return ( + utils_info_sl.myLoadInt(32), + utils_info_sl.myLoadInt(32) + ); +} + +@method_id(103) +fun testSlices1() { + var b: builder = beginCell().storeInt(1, 32).myStoreInt(2, 32); + b.myStoreInt(3, 32); + var c: cell = b.myStoreInt(4, 32).storeInt(5, 32).endCell(); + var cs = c.beginParse(); + var first = cs.preloadInt(32); + unpack_utils_info(mutate cs); + return (first, cs.myLoadInt(32), cs.loadInt(32)); +} + +fun load_decimal_symbol(mutate self: slice): int { + // load decimal from bits using utf-8 table + var n: int = self.loadUint(8); + n = n - 48; + assert(n >= 0) throw 400; + assert(n <= 9) throw 400; + return n; +} + +@method_id(104) +fun testSlices2() { + var cs = "123"; + return (cs.load_decimal_symbol(), cs.load_decimal_symbol(), cs.load_decimal_symbol()); +} + +global v1: int; +global v2: int; +global v3: int; + +@method_id(105) +fun testGlobals() { + v1 = 0; + v2 = 0; + v3 = 100; + v3 += incrementTwoInPlace(mutate v1, mutate v2, 5); + return (v1, v2, v3); +} + +fun withNameShadowing(mutate x: int, pivot: int, extra: int) { + x += pivot; + if (pivot < 100) { + var x = 100 + extra; + if (pivot < 50) { + var x = 50 + extra; + return x + extra; + } else { + x += extra; + return x + extra; + } + } else { + x += extra; + return -100 + extra; + } +} + +@method_id(106) +fun testNameShadowing() { + var x = 0; + var sum = 0; + sum += withNameShadowing(mutate x, 100, 10); + sum += withNameShadowing(mutate x, 50, 10); + sum += withNameShadowing(mutate x, 0, 10); + return (x, sum); +} + +fun updateTwoItems(mutate self: (int, int), byValue: int) { + val (first, second) = self; + self = (first + byValue, second + byValue); +} + +global t107_1: int; +global t107_2: int; + +@method_id(107) +fun testMutableTensor() { + var t = (40, 50); + t.updateTwoItems(10); + updateTwoItems(mutate t, 10); + t107_1 = 1; + t107_2 = 2; + (t107_1, t107_2).updateTwoItems(10); + updateTwoItems(mutate (t107_1, t107_2), 10); + return (t, t107_1, t107_2); +} + +@pure +fun myStoreUint(mutate self: builder, x: int, len: int): self + asm(x self len) "STIX"; + +@pure +fun myStoreU32(mutate self: builder, x: int): self { + return self.storeUint(x, 32); +} + +fun getSumOfNumbersInCell(c: cell): int { + var sum = 0; + var s = c.beginParse(); + var n_numbers = s.getRemainingBitsCount() / 32; + repeat (n_numbers) { + sum += s.loadUint(32); + } + return sum; +} + +@method_id(110) +fun testStoreChaining() { + var b = ((beginCell()).storeUint(1, 32)).storeUint(2, 32).storeUint(3, 32); + b.storeUint(4, 32); + b.myStoreUint(5, 32).storeUint(6, 32); + storeUint(mutate b, 7, 32); + b = b.storeUint(8, 32); + b = b.storeUint(9, 32).storeUint(10, 32); + + return getBuilderBitsCount(b); +} + +@method_id(111) +fun testStoreChainingCustom() { + var b = beginCell().myStoreUint(1, 32).myStoreUint(2, 32).myStoreUint(3, 32); + b.myStoreUint(4, 32); + b.myStoreUint(5, 32).myStoreUint(6, 32); + myStoreUint(mutate b, 7, 32); + b = b.myStoreUint(8, 32); + b = b.myStoreUint(9, 32).myStoreUint(10, 32); + val sum1 = getSumOfNumbersInCell(b.endCell()); + + b = beginCell().myStoreU32(1).storeUint(2, 32).myStoreU32(3); + b.myStoreU32(4); + b.myStoreU32(5).myStoreU32(6); + myStoreU32(mutate b, 7); + b = b.myStoreU32(8); + b = b.storeUint(9, 32).myStoreU32(10); + val sum2 = getSumOfNumbersInCell(b.endCell()); + + return (sum1, sum2); +} + +fun myStoreU32_and_mutate_x(mutate self: builder, mutate x: int): void { + return myStoreUint(mutate self, x += 10, 32); +} + +@method_id(112) +fun testStoreAndMutateBoth() { + var x = 3; + var b: builder = beginCell().myStoreUint(1, 32); + b.myStoreU32_and_mutate_x(mutate x); + b.myStoreU32(3).myStoreU32_and_mutate_x(mutate x); + b.myStoreU32_and_mutate_x(mutate x); + + var cs: slice = b.endCell().beginParse(); + var (n1,n2,n3,n4,n5) = (cs.loadUint(32),((cs)).loadUint(32),cs.loadUint(32),cs.loadUint(32),cs.loadUint(32)); + assert(n5 == x) throw 100; + + return [n1,n2,n3,n4,n5]; +} + +global ccc: builder; + +@method_id(113) +fun testStoreChainingForGlobal() { + ccc = beginCell().storeUint(1, 32).myStoreUint(2, 32).myStoreU32(3); + ccc.storeUint(4, 32); + ccc.storeUint(5, 32).myStoreU32(6); + storeUint(mutate ccc, 7, 32); + ccc = ccc.myStoreU32(8); + ccc = ccc.storeUint(9, 32).myStoreUint(10, 32); + + return getBuilderBitsCount(ccc); +} + +fun alwaysThrows(): int { throw 123; return 123; } +fun loadIntFromCell(c: cell, len: int) { return c.beginParse().loadUint(len); } + +@method_id(114) +fun testLoadIntForTemporaryObject() { + val c0 = beginCell().storeUint(0, 32).endCell(); + val c4 = beginCell().storeUint(4, 32).endCell(); + return ( + beginCell().storeUint(1, 32).endCell().beginParse().loadUint(32), + beginCell().storeUint(2, 32).endCell().beginParse().loadUint(32), + c0.beginParse().loadUint(32) ? alwaysThrows() : loadIntFromCell(c4, 32) + ); +} + +@pure +fun myStoreUint_pure(mutate self: builder): void + asm "STIX"; + +fun myStoreUint_impure(mutate self: builder): void + asm "STIX"; + +fun testStoreUintPureUnusedResult() { + var b = beginCell(); + b.myStoreUint_pure(); + var s = b.endCell().beginParse(); + val ii = s.loadUint(32); + return 0; +} + +fun testStoreUintImpureUnusedResult() { + var b = beginCell(); + b.myStoreUint_impure(); + var s = b.endCell().beginParse(); + val ii = s.loadUint(32); + return 0; +} + +global counter: int; + +fun writeNext2(mutate self: builder): self { + return self.storeUint(counter += 1, 32).storeUint(counter += 1, 32); +} + +fun resetCounter(mutate self: builder): self { + counter = 0; + return self; +} + +@method_id(115) +fun testExplicitReturn() { + counter = 0; + return ( + beginCell().writeNext2().writeNext2().resetCounter().writeNext2().endCell().getSumOfNumbersInCell(), + counter + ); +} + + +fun main(){} + +/** +@testcase | 101 | | 70 60 +@testcase | 102 | | 70 50 100 120 +@testcase | 103 | | 1 3 4 +@testcase | 104 | | 1 2 3 +@testcase | 105 | | 5 5 110 +@testcase | 106 | | 160 110 +@testcase | 107 | | 60 70 21 22 +@testcase | 110 | | 320 +@testcase | 111 | | 55 55 +@testcase | 112 | | [ 1 13 3 23 33 ] +@testcase | 113 | | 320 +@testcase | 114 | | 1 2 4 +@testcase | 115 | | 13 2 + +@fif_codegen +""" + incrementInPlace PROC:<{ + // self byValue + ADD // self + }> +""" + +@fif_codegen +""" + testIncrement2 PROC:<{ + ... + incrementTwoInPlace CALLDICT // x y sum1 + -ROT + 10 PUSHINT // sum1 x y '11=10 + incrementTwoInPlace CALLDICT // sum1 x y sum2 + s1 s3 s0 XCHG3 // x y sum1 sum2 + }> +""" + +@fif_codegen +""" + load_next PROC:<{ + // cs + 32 LDI // '4 cs + SWAP // cs '4 + }> +""" + +@fif_codegen +""" + testStoreUintPureUnusedResult PROC:<{ + // + 0 PUSHINT // '11=0 + }> +""" + +@fif_codegen +""" + testStoreUintImpureUnusedResult PROC:<{ + // + NEWC // b + STIX // '2 + DROP // + 0 PUSHINT // '11=0 + }> +""" + + */ diff --git a/tolk-tester/tests/never-type-tests.tolk b/tolk-tester/tests/never-type-tests.tolk new file mode 100644 index 00000000..89447389 --- /dev/null +++ b/tolk-tester/tests/never-type-tests.tolk @@ -0,0 +1,28 @@ +fun takeInt(a: int) {} + +@method_id(101) +fun test1(x: int?) { + if (x == null && x != null) { + var y = x; + __expect_type(y, "never"); + __expect_type(y!, "never"); + // `never` type is assignable to anything, flow won't reach this point + var t: (int, int) = x; + t = y; + takeInt(x); + var cb: (int) -> int = x; + x as int?; + x as (int, int)?; + x as never; + return x; + } + return 123; +} + +fun main() { + __expect_type(test1, "(int?) -> int"); +} + +/** +@testcase | 101 | null | 123 + */ diff --git a/tolk-tester/tests/no-spaces.tolk b/tolk-tester/tests/no-spaces.tolk new file mode 100644 index 00000000..409bb342 --- /dev/null +++ b/tolk-tester/tests/no-spaces.tolk @@ -0,0 +1,115 @@ +const int10:int=10; + +fun just10(): int { return int10; } +fun eq(v: int): int { return`v`; } + +@method_id(101) fun `get_-1` (): int {return-1;} +@method_id(102) fun `get_--1` (): int {return--1;} +@method_id(103) fun `get_---1`(): int {return---1;} +@method_id(104) fun `get_+++1`(): int {return+++1;} +@method_id(105) fun `get_+-+1`(): int {return+-+1;} + +global `some()var`:int; + +@method_id(110) fun `some_math`(): int { + `some()var`=--6; + return 1*-2*-3*-4*just10()*-5+-`some()var`+--`some()var`---`some()var`; +} + +@method_id(111) fun `negative_nums`(a:int):int { + var m$0:int=1; + var m1:int=-(+0x1)*m$0; + return `a`*-1*-(1)*---(1)*+just10()+-`just10`()*m1*-m1+-eq(m1)----0x1; +} + +@method_id(112) fun `bitwise~ops`(flags:int):[bool,bool] { + return[ + (just10()-3==just10()-(4)--1)|((2==2)&(eq(eq(10)) -3==just10()--13)), + ((flags&0xFF)!=0) + ]; +} + +@method_id(113)fun`unary+bitwise-constant`():[int,int,int]{ + return [~-~~+-3, ~+3-~9, -(-~+-20-~ 10+3+~38&39)]; +} + +@method_id(114)fun`unary+bitwize-parametrized`(c3:int, c9:int, c20:int, c10:int, c38:int):[int,int,int]{ + return [~-~~+-c3, ~+c3-~`c9`, -(-~+-c20-~c10+c3+~c38&39)]; +} + +fun add3(a: int, b: int, c: int) { return a+b+c; } + +@method_id(115) fun unary_const_check(): [int,int] { + var fst1: int=-1; + var snd1: int=-1; + var trd1: int=+2; + var (fst2,snd2,trd2)=(-1,-1,+2); + return [add3(fst2,snd2,trd2),add3(fst1,snd1,trd1)]; +} + +fun `load:u32`(mutate self: slice): int { + return self.loadUint(32); +} + +@method_id(116) fun `call_~_via_backticks`():[int,int,int,int] { + var b:builder = beginCell().storeUint(1, 32).storeUint(2, 32).storeUint(3, 32).storeUint(4, 32); + var `cs`:slice = b.endCell().beginParse(); + val one:int=`cs`.`loadUint`(32); + val (two:int,three:int) = (`cs`.`loadUint`(32), cs.`load:u32`()); + val four:int = cs.`load:u32`(); + return [one,two,three,four]; +} + +fun`main`(){} + +/** + method_id | in | out +@testcase | 101 | | -1 +@testcase | 102 | | 1 +@testcase | 103 | | -1 +@testcase | 104 | | 1 +@testcase | 105 | | -1 +@testcase | 110 | | 1194 +@testcase | 111 | -1 | 22 +@testcase | 112 | 0 | [ -1 0 ] +@testcase | 113 | | [ -4 6 -4 ] +@testcase | 114 | 3 9 20 10 38 | [ -4 6 -4 ] +@testcase | 115 | | [ 0 0 ] +@testcase | 116 | | [ 1 2 3 4 ] + +@fif_codegen +""" + get_+-+1 PROC:<{ + // + -1 PUSHINT + }> +""" + +@fif_codegen +""" + unary+bitwise-constant PROC:<{ + // + -4 PUSHINT + 6 PUSHINT + -4 PUSHINT + TRIPLE + }> +""" + +@fif_codegen +""" + unary_const_check PROC:<{ + // + -1 PUSHINT // fst1=-1 + DUP // fst1=-1 snd1=-1 + 2 PUSHINT // fst1=-1 snd1=-1 trd1=2 + s1 s1 s0 PUSH3 // fst1=-1 snd1=-1 trd1=2 fst2=-1 snd2=-1 trd2=2 + add3 CALLDICT // fst1=-1 snd1=-1 trd1=2 '13 + 3 -ROLL // '13 fst1=-1 snd1=-1 trd1=2 + add3 CALLDICT // '13 '14 + PAIR // '12 + }> +""" + + */ + diff --git a/tolk-tester/tests/null-keyword.tolk b/tolk-tester/tests/null-keyword.tolk new file mode 100644 index 00000000..65890a92 --- /dev/null +++ b/tolk-tester/tests/null-keyword.tolk @@ -0,0 +1,136 @@ +import "@stdlib/lisp-lists" + +@method_id(101) +fun test1() { + var numbers: tuple? = createEmptyList(); + numbers = listPrepend(1, numbers); + numbers = listPrepend(2, numbers); + numbers = listPrepend(3, numbers); + numbers = listPrepend(4, numbers); + var (h: int, numbers redef) = listSplit(numbers!); + h += listGetHead(numbers!); + + _ = null; + (_, _) = (null, null); + var t = createEmptyTuple(); + do { + var num: int = numbers.listNext(); + t.tuplePush(num); + } while (numbers != null); + + return (h, numbers == null, t); +} + +@method_id(102) +fun test2(x: int?) { + if (null != x) { + var y: int? = null; + if (y != null) { return 10; } + if (10 < 20) { // always true at runtime (not at compile-time) + return y; + } + } + try { + return x! + 10; // will throw, since not a number + } catch { + return -1; + } + return 100; +} + +fun myIsNull(x: int?): int { + return x == null ? -1 : x!; +} + +@method_id(103) +fun test3(x: int) { + return myIsNull(x > 10 ? null : x); +} + +@method_id(104) +fun test4(): null { + var (_, (_, untyped: null)) = (3, (createEmptyTuple, null)); + if (true) { + return untyped; + } + return untyped; +} + +@method_id(107) +fun test7() { + var b = beginCell().storeMaybeRef(null) as builder?; + var s = b!.endCell().beginParse(); + var c = s.loadMaybeRef(); + return (null == c) as int * 10 + (b != null) as int; +} + +fun test8() { + __expect_type(null, "null"); + __expect_type([[null]], "[[null]]"); + __expect_type(null as tuple?, "tuple?"); + __expect_type(null as [int]?, "[int]?"); + __expect_type(((null)) as (int, int)?, "(int, int)?"); +} + +fun main() { + // the compiler optimizes this at compile-time + var i: int? = null; + if (i == null) { + return 1; + } + return 10; +} + +/** +@testcase | 101 | | 7 -1 [ 3 2 1 ] +@testcase | 102 | 5 | (null) +@testcase | 102 | null | -1 +@testcase | 103 | 5 | 5 +@testcase | 103 | 15 | -1 +@testcase | 104 | | (null) +@testcase | 107 | | -11 +@fif_codegen +""" + test1 PROC:<{ + // + PUSHNULL // numbers + 1 PUSHINT // numbers '2=1 + SWAP // '2=1 numbers + CONS // numbers + 2 PUSHINT // numbers '4=2 + SWAP // '4=2 numbers + CONS // numbers + 3 PUSHINT // numbers '6=3 + SWAP // '6=3 numbers + CONS // numbers + 4 PUSHINT // numbers '8=4 + SWAP // '8=4 numbers + CONS // numbers + UNCONS // h numbers + DUP // h numbers numbers + CAR // h numbers '13 +""" + +@fif_codegen +""" + main PROC:<{ + // + 1 PUSHINT // '3=1 + }> +""" + +@fif_codegen +""" + test7 PROC:<{ + ... + LDOPTREF // b '9 '8 + DROP // b c + ISNULL // b '11 + 10 MULCONST // b '13 + SWAP // '13 b + ISNULL // '13 '14 + NOT // '13 '14 + ADD // '15 + }> +""" +*/ diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk new file mode 100644 index 00000000..d0720273 --- /dev/null +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -0,0 +1,492 @@ +fun getNullableInt(): int? { return 5; } + +fun sumOfNullableTensorComponents(t: (int, int)?): int { + if (t == null) { return 0; } + return t!.0 + t!.1; +} + +fun isTensorNull(t: (int, int)?) { + return t == null; +} + +fun incrementNullableTensorComponents(mutate self: (int, int)?): self { + if (self != null) { + self!.0 += 1; + self!.1 += 1; + } + return self; +} + +fun incrementTensorComponents(mutate self: (int, int)): self { + self.0 += 1; + self.1 += 1; + return self; +} + +fun assignFirstComponent(mutate t: (int, int), first: int) { + t!.0 = first; +} + +fun assignFirstComponentNullable(mutate t: (int, int)?, first: int) { + if (t == null) { + t = (first, 0); + } else { + t!.0 = first; + } +} + +fun getNullableTensor(firstComponent: int?): (int, int)? { + return firstComponent == null ? null : (firstComponent!, 2); +} + +fun sumOfTensor(x: (int, int)) { + return x.0 + x.1; +} + +fun assignNullTo(mutate x: T?) { + x = null; +} + +fun getTensor12() { + return (1,2); +} + +@method_id(101) +fun test101(): (int, int)? { + return (1, 2); +} + +@method_id(102) +fun test102(): ((int, int)?, (int, int)?) { + var t = (1, 2); + return (t, null); +} + +@method_id(103) +fun test103(t: (int, int)) { + var t2: (int, int)? = t; + return (sumOfNullableTensorComponents(t), sumOfNullableTensorComponents(t2), sumOfNullableTensorComponents(null), t2); +} + +@method_id(104) +fun test104() { + var t1_1: (int, int)? = (1, 2); + var t1_2: (int, int)? = t1_1; + var t1_3: (int, int)? = t1_1!; + var t2_1: (int, int)? = getNullableTensor(null); + var t2_2 = t2_1; + return (t1_3, t2_2); +} + +@method_id(105) +fun test105() { + return (null as (int, slice, cell)?, (1, 2, 3) as (int, int, int)?); +} + +@method_id(106) +fun test106() { + var t: (int?, int?)? = (((((1, 2))) as (int, int))); + return t; +} + +@method_id(107) +fun test107() { + var ab = (1, 2); + var ab2: (int, int)? = ab; + return (isTensorNull(ab), isTensorNull(ab2), isTensorNull(null), ab.isTensorNull(), ab2.isTensorNull(), null.isTensorNull()); +} + +@method_id(108) +fun test108(x1: (int, int)) { + incrementTensorComponents(mutate x1); + x1.incrementTensorComponents(); + var x2: (int, int)? = x1; + __expect_type(x2, "(int, int)"); + x2.incrementNullableTensorComponents().incrementNullableTensorComponents(); + incrementNullableTensorComponents(mutate x2); + __expect_type(x2, "(int, int)?"); + var x3: (int, int)? = null; + __expect_type(x3, "null"); + x3.incrementNullableTensorComponents().incrementNullableTensorComponents(); + incrementNullableTensorComponents(mutate x3); + return (x1, x2, x3); +} + +fun isTensorNullGen(t: (T1, T2)?) { + return t == null; +} + +@method_id(109) +fun test109() { + var x1 = (1, 2); + var x2: (int, int)? = x1; + var x3: (int, int)? = x1.1 > 10 ? (1, 2) : null; + return ( + isTensorNullGen(x1), isTensorNullGen(x2), isTensorNullGen(null), + isTensorNullGen(x1), isTensorNullGen(x3), + x1.isTensorNullGen(), x2.isTensorNullGen(), x3.isTensorNullGen(), null.isTensorNullGen() + ); +} + +global g110_1: (int, int); +global g110_2: (int, int)?; + +@method_id(110) +fun test110() { + g110_1 = getNullableTensor(1)!; + incrementTensorComponents(mutate g110_1); + g110_1.incrementTensorComponents(); + g110_2 = g110_1; + g110_2.incrementNullableTensorComponents().incrementNullableTensorComponents(); + incrementNullableTensorComponents(mutate g110_2); + var tmp = g110_2; + g110_2 = null; + g110_2.incrementNullableTensorComponents(); + incrementNullableTensorComponents(mutate g110_2); + return (g110_1, g110_2, tmp); +} + +@method_id(111) +fun test111() { + var x = (1, 2); + assignFirstComponent(mutate x, 50); + var x2: (int, int)? = null; + var x3 = x2 as (int, int)?; + assignFirstComponentNullable(mutate x2, 30); + assignFirstComponentNullable(mutate x3, 70); + g110_1 = (1, 2); + g110_2 = null; + assignFirstComponent(mutate g110_1, 90); + assignFirstComponentNullable(mutate g110_2, 100); + return (x.0, x2!.0, x3!.0, g110_1.0, g110_2!.0); +} + +@method_id(112) +fun test112() { + var x: (int, int)? = (10, 20); + incrementTensorComponents(mutate x!); + x!.incrementTensorComponents(); + return x; +} + +@method_id(113) +fun test113() { + var t = [1, null]; // t.1 is always null + return isTensorNull(t.1); +} + +@method_id(114) +fun test114(): ((slice, (cell, [int, slice, tuple]))?, slice?, (int?, bool?)?) { + var t = [[null]]; + return (t.0.0, t.0.0, t.0.0); +} + +@method_id(115) +fun test115() { + var tt = getNullableTensor(null); + assignFirstComponentNullable(mutate tt, 5); + return ( + getNullableTensor(1)!.incrementTensorComponents(), + sumOfNullableTensorComponents(getNullableTensor(1).incrementNullableTensorComponents().incrementNullableTensorComponents()), + getNullableTensor(null).incrementNullableTensorComponents(), + tt, + sumOfNullableTensorComponents(getNullableTensor(null)) + ); +} + +@method_id(116) +fun test116(returnNull: bool) { + var t1: (int, int)? = returnNull ? null : getTensor12(); + var t2 = returnNull ? null as (int, int)? : getTensor12() as (int, int)?; + returnNull ? null : (1, 2); + return (t1, t2); +} + +@method_id(117) +fun test117() { + var (a, b: (int, int)?, c) = (1, null, 3); + return (b, a, c); +} + +fun autoInferNullableTensor(a: int?, b: int) { + if (a != null) { + return (a!, b); + } + return null; +} + +@method_id(118) +fun test118(a: int?) { + return autoInferNullableTensor(a, 10); +} + +@method_id(119) +fun test119() { + var x: (int, int)? = (1, 2); + x = null; + var tt: (int, (int, int)?) = (0, (1, 2)); + tt.1 = null; + var third: (int, (int, int)?, int) = (0, (1, 2), 3); + third.2 = 100; + return (x, tt.1, third.1, third.2); +} + +@method_id(120) +fun test120(setNull: bool) { + var x: (int, int)? = (1, 2); + if (setNull) { + assignNullTo(mutate x); + } + return x; +} + +@method_id(121) +fun test121() { + var t: [int?, [int?, int?]?] = [1, [2, 3]]; + t.1 = [3, 4]; + return t; +} + +@method_id(122) +fun test122(setNull: bool) { + var t: [int?, [int?, int?]?, int?, [int?, int?]?]? = [1, [2, 3], 4, null]; + if (setNull) { + assignNullTo(mutate t!.1); + } else { + var rhs = [3, 4]; + t!!.1 = rhs; + } + return t; +} + +@method_id(123) +fun test123() { + var t: (int?, (int?, int?)?) = (1, (2, 3)); + t.1 = (3, 4); + return t; +} + +@method_id(124) +fun test124(setNull: bool) { + var t: (int?, (int?, int?)?, int?, (int?, int?)?)? = (1, (2, 3), 4, null); + if (setNull) { + assignNullTo(mutate t!.1); + } else { + var rhs = (3, 4); + t!!.1 = rhs; + } + return t; +} + +global g125: int; +fun getT125(): (int, (int, int)?, (int?, int)?) { return (g125 += 1, null, null); } + +@method_id(125) +fun test125() { + g125 = 0; + getT125().1 = null; + getT125().2 = (1, 2); + (getT125()!! as (int, (int, int)?, (int?, int)?)).2 = null; + // test that nothing left on a stack + return g125; +} + +@method_id(126) +fun test126() { + var tt1: (int, null, int) = (1, null, 2); + var (a: int, b: (int, int)?, c: int) = tt1; + return (a, b, c); +} + +@method_id(127) +fun test127(choice: int) { + var tt1: (int, null, int) = (1, null, 2); + var tt2: (int, (int, int), int) = (1, (2, 3), 4); + var tt3: (int, (int, int)?, int) = (1, null, 5); + var abc: (int, (int, int)?, int) = choice == 1 ? tt1 : choice == 2 ? tt2 : tt3; + return abc; +} + +fun get128_1() { return (1, null, 2); } +fun get128_2() { return null; } +fun get128_3() { return (1, (2, 3), 4); } +fun takeT128(abc: (int, (int, int)?, int)?) { return abc; } + +@method_id(128) +fun test128(choice: int) { + if (choice == 1) { + return takeT128(get128_1())!; + } + if (choice == 2) { + return takeT128(get128_2()); + } + return takeT128(get128_3()); +} + +@method_id(129) +fun test129(setNull: bool) { + var t: (int?, int?) = (getNullableInt(), getNullableInt()); + var r1 = (t, t == null, t != null); + t = (setNull ? null : 1, setNull ? null : 2); + var r2 = (t, t == null, t != null); + return (r1, r2); +} + +@method_id(130) +fun test130(setNull: bool) { + var os: (int, (int, int)?) = (1, setNull ? null : (2, 3)); + return os; +} + +fun getEmptyNullableTensor(getNull: bool): ()? { + return getNull ? null : (); +} + +@method_id(131) +fun test131() { + var nonNullEmptyT = getEmptyNullableTensor(false); + var nullEmptyT = getEmptyNullableTensor(true); + var emptyT = nonNullEmptyT!; + __expect_type(emptyT, "()"); + var doubleNulls1 = (null, null) as (()?, ()?); + var doubleNulls2 = ((), ()) as (()?, ()?); + var doubleNulls3 = ((), ()) as (()?, ()?)?; + var stillEmpty = ((), ()); + return (nonNullEmptyT, 777, nullEmptyT, 777, emptyT, 777, nullEmptyT!, 777, doubleNulls1, doubleNulls2, 777, doubleNulls3, 777, stillEmpty); +} + +@method_id(132) +fun test132() { + var doubleNulls: (()?, ()?) = (getEmptyNullableTensor(true), getEmptyNullableTensor(false)); + var result = ((null as ()?) == null, (() as ()?) == null, doubleNulls.0 == null, doubleNulls.1 == null); + var aln1: int? = (doubleNulls.1 = null); + var aln2: null = (doubleNulls.1 = null); + return (result, 777, aln1, aln2, doubleNulls.1 == null, doubleNulls); +} + +@method_id(133) +fun test133() { + var x: (int, int)? = (10, 20); + return sumOfTensor(x) + x.0 + x.1; // smart casted +} + +@method_id(134) +fun test134(): (int, int)? { + var x: (int, int)? = (10, 20); + incrementTensorComponents(mutate x); // smart casted + return x; +} + + +fun getNormalNullableTensorWidth1(vLess100: int?): ([int?], ())? { + if (vLess100 != null && vLess100 >= 100) { + return null; + } + return ([vLess100], ()); // such a nullable tensor can store NULL in the same slot +} + +fun getTrickyNullableTensorWidth1(vLess100: int?): (int?, ())? { + if (vLess100 != null && vLess100 >= 100) { + return null; + } + return (vLess100, ()); // such a nullable tensor requires an extra stack slot for null presence +} + +fun getEvenTrickierNullableWidth1(vLess100: int?): ((), (int?, ()), ())? { + if (vLess100 != null && vLess100 >= 100) { + return null; + } + return ((), (vLess100, ()), ()); +} + +@method_id(135) +fun test135() { + var n1 = getNormalNullableTensorWidth1(10); // ([10], ()) + var n2 = getNormalNullableTensorWidth1(null); // ([null], ()) + var n3 = getNormalNullableTensorWidth1(100); // null + var t1 = getTrickyNullableTensorWidth1(10); // (10, ()) + var t2 = getTrickyNullableTensorWidth1(null); // (null, ()) + var t3 = getTrickyNullableTensorWidth1(100); // null + var e1 = getEvenTrickierNullableWidth1(10); // ((), (10, ()), ()) + var e2 = getEvenTrickierNullableWidth1(null); // ((), (null, (), ()) + var e3 = getEvenTrickierNullableWidth1(100); // null + return (n1, n2, n3, 777, t1, t2, t3, 777, e1, e2, e3, 777, + n1 == null, n2 == null, n3 == null, t1 == null, t2 == null, t3 == null, e1 == null, e2 == null, e3 == null, 777, + t1!.0 == null, t2!.0 == null, e1!.1.0 == null, e1!.1.1 == null, e2!.1.0 == null, e2!.1.1 == null); +} + + + +fun main(){} + +/** +@testcase | 101 | | 1 2 -1 +@testcase | 102 | | 1 2 -1 (null) (null) 0 +@testcase | 103 | 1 2 | 3 3 0 1 2 +@testcase | 104 | | 1 2 (null) (null) 0 +@testcase | 105 | | (null) (null) (null) 0 1 2 3 -1 +@testcase | 106 | | 1 2 +@testcase | 107 | | 0 0 -1 0 0 -1 +@testcase | 108 | 5 6 | 7 8 10 11 -1 (null) (null) 0 +@testcase | 109 | | 0 0 -1 0 -1 0 0 -1 -1 +@testcase | 110 | | 3 4 (null) (null) 0 6 7 -1 +@testcase | 111 | | 50 30 70 90 100 +@testcase | 112 | | 12 22 +@testcase | 113 | | -1 +@testcase | 114 | | (null) (null) (null) 0 (null) (null) (null) 0 +@testcase | 115 | | 2 3 7 (null) (null) 0 5 0 -1 0 +@testcase | 116 | -1 | (null) (null) 0 (null) (null) 0 +@testcase | 116 | 0 | 1 2 -1 1 2 -1 +@testcase | 117 | | (null) 1 3 +@testcase | 118 | 5 | 5 10 -1 +@testcase | 118 | null | (null) (null) 0 +@testcase | 119 | | (null) (null) 1 2 -1 100 +@testcase | 120 | -1 | (null) (null) 0 +@testcase | 120 | 0 | 1 2 -1 +@testcase | 121 | | [ 1 [ 3 4 ] ] +@testcase | 122 | 0 | [ 1 [ 3 4 ] 4 (null) ] +@testcase | 122 | -1 | [ 1 (null) 4 (null) ] +@testcase | 123 | | 1 3 4 -1 +@testcase | 124 | 0 | 1 3 4 -1 4 (null) (null) 0 +@testcase | 124 | -1 | 1 (null) (null) 0 4 (null) (null) 0 +@testcase | 125 | | 3 +@testcase | 126 | | 1 (null) 2 +@testcase | 127 | 1 | 1 (null) (null) 0 2 +@testcase | 127 | 2 | 1 2 3 -1 4 +@testcase | 127 | 3 | 1 (null) (null) 0 5 +@testcase | 128 | 1 | 1 (null) (null) 0 2 -1 +@testcase | 128 | 2 | (null) (null) (null) (null) (null) 0 +@testcase | 128 | 3 | 1 2 3 -1 4 -1 +@testcase | 129 | 0 | 5 5 0 -1 1 2 0 -1 +@testcase | 129 | -1 | 5 5 0 -1 (null) (null) 0 -1 +@testcase | 130 | 0 | 1 2 3 -1 +@testcase | 130 | -1 | 1 (null) (null) 0 +@testcase | 131 | | -1 777 0 777 777 777 0 0 -1 -1 777 -1 -1 -1 777 +@testcase | 132 | | -1 0 -1 0 777 (null) (null) -1 0 0 +@testcase | 133 | | 60 +@testcase | 134 | | 11 21 -1 +@testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 -1 (null) -1 (null) 0 777 10 -1 (null) -1 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 + +@fif_codegen +""" + isTensorNull PROC:<{ + // t.0 t.1 t.NNFlag + 2 1 BLKDROP2 // t.NNFlag + 0 EQINT // '3 + }> +""" + +@fif_codegen +""" + test113 PROC:<{ + // + 1 PUSHINT // '2=1 + PUSHNULL // '2=1 '3 + PAIR // t + 1 INDEX // '5 + PUSHNULL // '5 '6 + 0 PUSHINT // '5 '6 '7=0 + isTensorNull CALLDICT // '8 + }> +""" +*/ diff --git a/tolk-tester/tests/nullable-types.tolk b/tolk-tester/tests/nullable-types.tolk new file mode 100644 index 00000000..24aa7f8a --- /dev/null +++ b/tolk-tester/tests/nullable-types.tolk @@ -0,0 +1,109 @@ + +fun getNullable4(): int? { return 4; } +fun getNullableIntNull(): int? asm "PUSHNULL"; + +fun eqInt(x: int) { return x; } +fun eq(x: T) { return x; } + +fun unwrap(x: T?): T { return x!; } +fun intOr0(x: int?): int { return null == x ? 0 : x!; } + +@method_id(101) +fun test101(x: int) { + var re = x == 0 ? null : 100; + return re == null ? re : 200 + getNullable4()!; +} + +@method_id(102) +fun test102(a: int) { + try { + throw (123, a > 10 ? null : a); + return 0; + } catch (excno, arg) { + var i = arg as int?; + return excno + (i != null ? i!!!!! : -100); + } +} + +@method_id(103) +fun test103(x: int?): (bool, bool, int) { + var x_gt_0 = x != null && eqInt(x!) > 0; + var x_lt_0 = x != null && eq(x)! < 0; + if (x == null) { + return (x_gt_0, x_lt_0, 0); + } + return (x_gt_0, x_lt_0, x!); +} + +@method_id(104) +fun test104(x: int?) { + var x2 = eq(x = 10); + var ab = (x2, getNullableIntNull()); + return (unwrap(ab.0) + (ab.1 == null ? -100 : ab.1!), ab.1); +} + +@method_id(105) +fun test105() { + var xy: (int?, int?) = (5, null); + var ab = [1 ? [xy.0, xy.1] : null]; + ab.0!.0 = intOr0(ab.0!.0); + ab.0!.1 = intOr0(ab.0!.1); + return ab.0!.0! + ab.0!.1!; +} + +global gTup106: tuple?; +global gInt106: int?; + +@method_id(106) +fun test106() { + gInt106 = 0; + gInt106! += 5; + var int106: int? = 0; + var gTup106 = createEmptyTuple(); + gTup106!.tuplePush(createEmptyTuple()); + (gTup106!.0 as tuple?)!.tuplePush(0 as int?); + tuplePush(mutate gTup106!, gInt106); + tuplePush(mutate gTup106!.0, int106! += 1); + return (gTup106 == null, null != gTup106, gTup106, gTup106!.0 as tuple?); +} + +@method_id(107) +fun test107() { + var b: builder? = beginCell(); + b!.storeInt(1, 32).storeInt(2, 32); + b = b!.storeInt(3, 32); + storeInt(mutate b!, 4, 32); + (b! as builder).storeInt(5, 32); + return b!.getBuilderBitsCount(); +} + +@method_id(108) +fun test108() { + var (a, b: cell?, c) = (1, beginCell().endCell(), 3); + if (10>3) { b = null; } + return a + (b == null ? 0 : b!.beginParse().loadInt(32)) + c; +} + +@method_id(109) +fun test109() { + var a = getNullable4(); + var b = getNullable4(); + return ([a, b] = [3, 4], a, b); +} + +fun main(x: int?, y: int?) { +} + +/** +@testcase | 101 | 0 | (null) +@testcase | 101 | -1 | 204 +@testcase | 102 | 5 | 128 +@testcase | 102 | 15 | 23 +@testcase | 103 | 10 | -1 0 10 +@testcase | 104 | 8 | -90 (null) +@testcase | 105 | | 5 +@testcase | 106 | | 0 -1 [ [ 0 1 ] 5 ] [ 0 1 ] +@testcase | 107 | | 160 +@testcase | 108 | | 4 +@testcase | 109 | | [ 3 4 ] 3 4 + */ diff --git a/tolk-tester/tests/op-priority.tolk b/tolk-tester/tests/op-priority.tolk new file mode 100644 index 00000000..e3c6bb6e --- /dev/null +++ b/tolk-tester/tests/op-priority.tolk @@ -0,0 +1,121 @@ +fun justTrue(): bool { return true; } + +fun unary_minus_1(a: int, b: int, c: int): int{return -(a+b) *c;} +fun unary_minus_2(a: int, b: int, c: int): int{return(-(a+b))*c;} +fun unary_minus_3(a: int, b: int, c: int): int{return-((a+b) *c);} + + +@method_id(101) +fun test1(x: int, y: int, z: int): bool { + return (x > 0) & (y > 0) & (z > 0); +} + +@method_id(102) +fun test2(x: int, y: int, z: int): bool { + return x > (0 & (y > 0) as int & (z > 0) as int); +} + +@method_id(103) +fun test3(x: int, y: int, z: int): bool { + if ((x < 0) | (y < 0)) { + return z < 0; + } + return (x > 0) & (y > 0); +} + +@method_id(104) +fun test4(x: int, y: int, mode: int): bool { + if (mode == 1) { + return (x == 10) | (y == 20); + } if (mode == 2) { + return (x == 10) | (y == 20); + } else { + return x == (10 | (y == 20) as int); + } +} + +@method_id(105) +fun test5(status: int): bool { + return justTrue() & (status == 1) & ((justTrue() as int & status) == 1); +} + +@method_id(106) +fun test6(a: int, b: int, c: int): bool { + return (unary_minus_1(a,b,c) == unary_minus_2(a,b,c)) & (unary_minus_1(a,b,c) == unary_minus_3(a,b,c)); +} + +@method_id(107) +fun test7(b: int): int { + var a = b == 3 ? 3 : b == 4 ? 4 : (b == 5) & true ? 5 : 100; + return a; +} + +@method_id(108) +fun test8(b: int): int { + var a = b == 3 ? 3 : b == 4 ? 4 : b = 5 ? 5 : 100; + return a; +} + +fun `_ 0, 3 & (3 > 0) as int, 3 & (`_<_`(3, 0)), + 3 & `_ + unary_minus_2 PROC:<{ + // a b c + -ROT // c a b + ADD // c '3 + NEGATE // c '4 + SWAP // '4 c + MUL // '5 + }> + unary_minus_3 PROC:<{ + // a b c + -ROT // c a b + ADD // c '3 + SWAP // '3 c + MUL // '4 + NEGATE // '5 + }> +""" + + */ diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk new file mode 100644 index 00000000..385aa3b5 --- /dev/null +++ b/tolk-tester/tests/parse-address.tolk @@ -0,0 +1,113 @@ +const cc1 = "0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e"a; +const cc2 = "EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"a; + +fun verifyAddr(addr: slice, workchain: int, number: int) { + assert (addr.getRemainingBitsCount() == 3 + 8 + 256) throw 112; + addr.skipBits(3); + assert (addr.loadUint(8) == workchain) throw 111; + assert (addr.loadUint(256) == number) throw 111; +} + +fun main() { + verifyAddr("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"a, 255, 23158417847463239084714197001737581570653996933128112807891516801582625927987); + verifyAddr("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"a, 0, 0); + verifyAddr("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"a, 0, 65607996509792174074532427555986248720836864382484024657400295821210434460432); + verifyAddr("UQCOgxbCOjOLH_cEuQdGgS23zBM5SrQQepMFedjK-oixYbis"a, 0, 64460038539088394980732229180523693489682583665805557562964506609821558550881); + verifyAddr("EQDa4VOnTYlLvDJ0gZjNYm5PXfSmmtL6Vs6A_CZEtXCNICq_"a, 0, 99002318936150612861744867526221033858534811876886359650897405270877291973920); + verifyAddr("Ef8BtXO9bcTMXjg9bgivKh4lhJmZWQPP6_rb9vfjlTP5FJtM"a, 255, 772910975127952880303441415761050161913031788763061162001556772893733681428); + verifyAddr("Ef89xh-uy860-mCcvS8zcAUs8bApmxLGygDLEKjUk5RL-311"a, 255, 27941138149036269893630478666581900122707382189183906805784676408403709676539); + verifyAddr("Ef_vA6yRfmt2P4UHnxlrQUZFcBnKux8mL2eMqBgpeMFPorr4"a, 255, 108109262375472472702582493362335418330829651067377177643099076957184687427490); + verifyAddr("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"a, 255, 18502444830824300068094395885436326119386947594392869497312068745716154912158); + verifyAddr("Ef_fvrd0hBoVJUxoi7wH173Zk8NPiyVvxh5IoYSjEYZbOhsu"a, 255, 101202732337223525952216789200341489000836292542250083765062769181728788863802); + verifyAddr("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf"a, 255, 46952625717497919357580310066854892621799390294920450816077086267929711460872); + verifyAddr("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA"a, 255, 48545777798729612074233611768739897492467685225150339217043102685589809464695); + verifyAddr("Ef9LynHHKgBxY6-l-W_dWN-CtGT2_ji5rN3EzOI-p9zWEfq6"a, 255, 34281152017620085319078796986198022632548048219136747083019177301186013091345); + verifyAddr("Ef9hMd78gzSiVsK0zz0AHtEja8x1UoB_NDZMjn-l86NQK_2Y"a, 255, 43962460814164090767878334494257755557842170134382045184921495822637115592747); + verifyAddr("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6"a, 255, 23557057702048801338698514499604413540742716310574705490458593067566768087319); + verifyAddr("Ef_fdIbThooPs4_r2DE_Z6ZsWycJdHLnsuKAJHTcbaZaipez"a, 255, 101071650030310556115830521522496708686577365303530257137459798093298869361290); + verifyAddr("Ef_lva0qEiZhWrrZJl-IJxyCcTQmmTo71fIWyQ31HxJ8NurV"a, 255, 103914771557158282349484109182290824591675204108148026180964788916630125182006); + verifyAddr("Ef8sMGKypw006AeRYqimLjmY2Ufp-SHk8C0ZJBNgVBlzw_Nr"a, 255, 19987255184378161380023126214650814972824352533523055905552702178965809886147); + verifyAddr("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff+W72r5gqPrHF"a, 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr("EQCaSCHVak-jIc9ANutTAfHpZNM3YdGky7yaDzsTrg0WhFlm"a, 0, 69783625181781015447914682554083924798054947959007050695795761257887453484676); + verifyAddr("EQBS9U3AfD15fGmOtRMXQAxcPVBwNuItfLcDni9fkbTyyNX0"a, 0, 37523067738561024305547433298623118197038688994386001017161816416175242146504); + verifyAddr("EQBiMNL9qNWMAkJHuM0BFneYcuHL17kzS4pswpaEO-NGWrFG"a, 0, 44412924025649114419413541526870954696667907029239618728289150652715284776538); + verifyAddr("EQAUzE-Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU-KeaYLNUi"a, 0, 9407242825041766837311851458322335726136775042891143504070507665010681354284); + verifyAddr("EQD-nhrinjv0B4LTgr0dRHTHwH1MOsgGhKBXJZd7vESMZUf1"a, 0, 115166810931401616117484448645661180241548402534908005320733783571353775148133); + verifyAddr("EQAVD3Fni9I6j8XeSIl-wAGBEhqhame6OtAY0GScKT0D9X6f"a, 0, 9525855215156855607080079714361451576383963668563135377495902959388099150837); + verifyAddr("EQC6ACq3VANZjqfRBy7JMHkpLwqQ9qyYJsCIGx1mYbQgxaKw"a, 0, 84130484652351964071210477536969520113177637645401392541565606610268614566085); + verifyAddr("EQCIJLNFIko5CvpKn9oAkrDgLocDOoD4vwmHxNx_fsG_LkwW"a, 0, 61579391178099797614367237687950512448308156724136883899001108680249616482094); + verifyAddr("EQCe4AYIBce1pAk2qJJPSs1OzyZRlKjkfq8zuC8D7erv6DUP"a, 0, 71861245445432818728925844931259040612664802586395398157190478191760507596776); + verifyAddr("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"a, 0, 78559023162479717496981724991265882229440558807791659796411897368395464230649); + verifyAddr("EQBBlraAps0OZaB9Q8ePQn2wVAaL1G411A-dNppyWe3X3GIT"a, 0, 29666621803903557832193058147214384979915773445007872807927344851911086823388); + verifyAddr("EQBiASqUqaVizrozLRbszkWC2kETbkhpO2qniDVDPPg2_0W8"a, 0, 44328719889509369519441680467651025944540360433148852643949783408843779749631); + verifyAddr("EQBu2Q1EO8gIoNA1qoGWnHUudKfmqlKEDTQE-DxN-_4sdg14"a, 0, 50137910719490808065414827264266674858051167131188257457782826342827836714102); + verifyAddr("EQA5bvxWd5-q2vJUVqR9AlbEIfdFysLR0PXGgVlBf8x5hWuF"a, 0, 25977927117604457079092522008276392864656238504700352770597256138254994667909); + verifyAddr("EQBguMSHjFv5bfoOdshr3ruS9ymSZzhRKMovoNrxGxZXvmee"a, 0, 43748489720571123896506696370504498290006245978262404519821633796370658121662); + verifyAddr("EQAxL0oF1-zNgimPKthbDnYS4xj94rHtfNRN7_Pd1r2LNNv3"a, 0, 22246882279393590648219842750911786376759362211171398419754426796438233910068); + verifyAddr("EQANX1uRKGZfyPIwEaIXrR0ZOqadct5q10dvKxWIxx7SQqzW"a, 0, 6048549475100840191738010856156544571222758030966479209409932714701987172930); + verifyAddr("EQBitdFDoU5DWSjfKq7AsO29RIwAnBzzvcVVSn5ekQoB9Liv"a, 0, 44647902768175374073183447303109856895983123510911038181495138821771906122228); + verifyAddr("EQBgbux7VSjqJHP7ByRK1q4QuVZbpSCesNgvz5qad3lfXX_B"a, 0, 43618018778298854282398238948198420936771670943015013768514626213000552996701); + verifyAddr("EQDisBd8U7M3CEOZ8gcWCdetdmJi3AI31zIT5qBwOdmUbsxY"a, 0, 102533830955233207294921564956803510155400341370448300698800842506363763004526); + verifyAddr("EQAZpn_eynVlf7Ii2d6jP_p1URPrdF9F3S7DiudQyelkjzwE"a, 0, 11602000355550451044739442929923326898313570892134000961608306166632391730319); + verifyAddr("EQDE0HBgfkOiqHezLtExBGTvOs8eitthHQosBjW3BmDy1y2K"a, 0, 89021598108837008984355105304701054698583123510131754065320641619941010764503); + verifyAddr("EQDyT36zktBN9PVWvZ1joRxhIfEUgCPt4F2isa-enUA_d6CP"a, 0, 109600164736599393471831241268953938618560132398692391517933933264745646800759); + verifyAddr("EQDSMUGwt25IQd3_yHjI03F71G8Kp2GMaMEv2TiWoTKbsyRH"a, 0, 95072727086440754059372943502908629555499501854161516009430039520728770059187); + verifyAddr("EQAgK1EcrvEuL9sCtoj3cNhVNOuf3lo5GIPE2gn1fwZZYB3j"a, 0, 14550545393206146289454646242321274637527057595221202748348667645886114191712); + verifyAddr("EQCDKqL5w_6MD-Z7AOButu-uR-ZJTsgNU1fu464hn9grY81U"a, 0, 59328315557704100696483472039557119625141880163887490602190749720459366378339); + verifyAddr("EQB1aVMyFBhnlYXmQjsma0S63kvxKU7ccZKFNCFTwX7ASPv4"a, 0, 53106696421104300082516512931084483581353095629408473618166869610568148238408); + verifyAddr("EQBbjrXHoxDyh1ZYGBdBoQgLaScxW6pZR1hEhJC8BqF-5Kgq"a, 0, 41412616102566803060532874463898939692666425753852274254609049615175463829220); + verifyAddr("EQC-QeZ13QP0lszxNKt380fCWuaV94vwC/bfuqmrlg1/fJPA"a, 0, 86055876869280374285292827775555707420719385459150221433115419095878595346300); + verifyAddr("EQAiUwpF27vXCngqNhf_TQ5E_06ah0G4zuSrnfU7CLLaht5H"a, 0, 15525356059048115813946213102829493539706126913595626308144289257869196581510); + verifyAddr("EQBqiVjmhe2iVGmgOSDO1FGjSiz_AMtb1w7lLEiP4XIF_SFy"a, 0, 48187833566271418625754761625661652107159264793429628379411792200127405491709); + verifyAddr("EQDmwvaK2d_SbaPdpOM60effPWeKsksgDVwFPEyxuftM396K"a, 0, 104376425077737068747642645125299653296942252727305637929378053253273342397663); + verifyAddr("EQDWtPZZgF7wvIMUHZQojuD3utiuivsW7WslRJ33dgv-5yc8"a, 0, 97114682311034709685427168495629428400170984047839002197324103884924936519399); + verifyAddr("EQAA7z0JI0JKqbN-1uENKz9JrxIO5ZRY-ehMeg9fPncx50Ck"a, 0, 422697701361909095759185681783393186844038628935759044330165207027374567911); + verifyAddr("EQBVUHRoCq6coQYUwOAhGSoAmQ6Mpm7dFlDYon6HMgWV8Ftr"a, 0, 38588743302295548905191533977469452945717219128199196974980570837505276220912); + verifyAddr("EQCTdvDCf0bA5dOPI1-44tB2ZfNcMGiklzvg27TovgDEqM6E"a, 0, 66700138358140658950710678965721715920748906761125730971082529064117803730088); + verifyAddr("EQBDBKE5WGKIlnoi3OOzw7vkKKIX55eWjPvgxJWwek8AyL2J"a, 0, 30313140970524770883308749215942283658935592719811899513010665548955593408712); + verifyAddr("EQAvCSyLCo21GrqLAifdov4WkOxuGQCjMRxgF1cXSaNzLHZe"a, 0, 21274912932379789207153885262858116665851037273450532982121514600400844714796); + verifyAddr("EQCsLpDeHB2qpRbmsCb_0xmsYVNx1NgeYrvHGT1TDrHkDgL4"a, 0, 77880084760844670718511036557364450189323549135231036747480176919181282894862); + verifyAddr("EQCTQ8kPwyX92r48gCIL_pLN_RcQT9ghZygnmDTYkOkuW_j5"a, 0, 66609755171046741472463433430016629628609840960137226492652665879987546041947); + verifyAddr("EQCTrFRSHt-tfk7WxK9ZHQmqLcgxXxTK7wGfCEbqgY2W9Mcx"a, 0, 66794468397542182731534559853537484892417154018190804733043974345563210356468); + verifyAddr("EQCv28y49GdaLncoclv0ISdDlMUY_cxDPGNWFCPT8t4GuqUJ"a, 0, 79543100951881731989812212377176715376834409392116644269458867858071577560762); + verifyAddr("EQCVL-k6deDR56Z8pcb0Btg0lGfaivOGfdDCD1vvyRsyL9vS"a, 0, 67479265933941008511790471646564661743401752930295407567346938670637286896175); + verifyAddr("EQD6t2dXDjZxF1DqunKF-8dEWivJdliY_0FYiCXnthuqnDCa"a, 0, 113402258385556889021060606279033166272577193563727959698596277924908309916316); + verifyAddr("EQDE98XNzXiPq7VnbJJ2M4-Ht3tX_OWR0xUTTnDC8NObLmyU"a, 0, 89091094739778473356272490822716056624384395256388481953562551087642791090990); + verifyAddr("EQDfeRDE1TDhwt478CDR0Q7MDwqcTUhfjqyTT59mgoAaF6f7"a, 0, 101079669463449311486034260688909914923832300293253430457119371423825321269783); + verifyAddr("EQDijcEyUKa-QgCbeGlggQk1uBtt2ZRHyW4Y4gB4R6MN6RLW"a, 0, 102473162609487797404330889623966425536887610061087715571345738626121871855081); + verifyAddr("EQDOtFOt41skbjBkZF89oYXpoDECjlxIzD-ShWAOYyzuxqLA"a, 0, 93495056812773926196963707371665512785268729004579280701087533371037976424134); + verifyAddr("EQDuJKSFWU7AYqH6KLFfAbYvMuz346eWmJvG6_2NYE42_B4T"a, 0, 107715199938628393100813870735031557263256555616273999363057194279168054802172); + verifyAddr("EQDwGu4vFv1e3wn8min_iy7OPJXegOYTFQ5bZFZ5a5ZPiBpX"a, 0, 108602665568837301744601989570019709742180613578164394799026726718721456754568); + verifyAddr("EQC4G2ph6AS_mD_-cIv4aIYm1z5jAgCW_TTDEr72ygXOP2X-"a, 0, 83274003234732023403481554420859495155084746906198543572711543697320249249343); + verifyAddr("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"a, 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); + verifyAddr("EQDoIA20MF1qEcSPtROdCu5ukGx9dVjgMeJh1oQ4A4cf_Jif"a, 0, 104993214557977037193613824776415934089204193426692473563548548423424814817276); + verifyAddr("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"a, 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); + verifyAddr("EQClLO4EnZ_rTyV1GVpWy53pLgWJRki5c4ZzuM_1O_ClBkO9"a, 0, 74711004027159342540251007601464500186374346239921204216319145006974068892934); + verifyAddr("EQDmkj65Ab_m0aZaW8IpKw4kYqIgITw_HRstYEkVQ6NIYCyW"a, 0, 104290347741656803921830951060768893809692975574470790497562993373950614128736); + verifyAddr("EQCqNTwAYUNhPFS0RgqZoTLGJcQQxbAJ7csUo4YO3_TONLab"a, 0, 76987241268612358571638783428744566580605181728938801022059780105627411729972); + verifyAddr("EQCL3DmCynaRK7-vsfeNmd4Jj-UxAIHPvA4qS2xwaL6UpLbF"a, 0, 63260589232981964910240894899061676480139492286430589202252472895352724165796); + verifyAddr("EQDbU1SVEjBE73oUqgAoM9gDcShUkM5EC2PgoCjuwVUKo-Ee"a, 0, 99203745911752606845646497420891218522647962685916739950275357890977532807843); + verifyAddr("EQD02VdcF4TDbCKLLhZJ39NQTu6aWq2LnLjp0oXqbNu_BANK"a, 0, 110748343802097970709980079967961144373090790244250392237586606542170934198020); + verifyAddr("EQBynBO23ywHy_CgarY9NK9FTz0yDsG82PtcbSTQgGoXwiuA"a, 0, 51839428943991432793039248316067731096592274748149794482308513726460953499586); + verifyAddr("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"a, 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr("EQAUTbQiM522Y_XJ_T98QPhPhTmb4nV--VSPiha8kC6kRfPO"a, 0, 9183547432069678364603018431103042146626948674383548774683927217595824907333); + verifyAddr("EQBlqsm144Dq6SjbPI4jjZvA1hqTIP3CvHovbIfW_t-SCALE"a, 0, 45985353862647206060987594732861817093328871106941773337270673759241903247880); + verifyAddr("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"a, 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr("kQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPgpP"a, 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr("kf-Dfdg-YQXaR2Q97gZJ4fGBtmV1DHOU1y1RPyyZZtRy_Ikh"a, 255, 59475331506450494976393625198911249698879029820580340449086829444312920781564); + verifyAddr("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 0, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr("0:0000000000000000000000000000000000000000000000000000000000000000"a, 0, 0); + verifyAddr("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff"a, 0, 115792089237316195423570985008687907853269984665640564039457584007913129639935); + verifyAddr("0:zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"a, 0, 23158417847463239084714197001737581570653996933128112807891516801582625927987); + verifyAddr("0:0000000000000000000000000000000000000000000000000000000000000000"a, 0, 0); + verifyAddr("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 1, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 9, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr("99:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 99, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr("-1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 255, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + + return cc1.isSliceBitsEqual(cc2); +} + +/** +@testcase | 0 | | -1 + */ diff --git a/tolk-tester/tests/pure-functions.tolk b/tolk-tester/tests/pure-functions.tolk new file mode 100644 index 00000000..bb1a7d59 --- /dev/null +++ b/tolk-tester/tests/pure-functions.tolk @@ -0,0 +1,46 @@ + +@pure +fun f_pure1(): int { + return f_pure2(); +} + +@pure +fun f_pure2(): int { + return 2; +} + +@pure +fun get_contract_data(): (int, int) { + var c: cell = getContractData(); + var cs: slice = c.beginParse(); + cs.loadBits(32); + var value: int = cs.loadUint(16); + return (1, value); +} + +fun save_contract_data(value: int) { + var b: builder = beginCell().storeInt(1, 32).storeUint(value, 16); + setContractData(b.endCell()); +} + +@pure +@method_id(101) +fun test1(): int { + return f_pure1(); +} + +@method_id(102) +fun test2(value: int): int { + save_contract_data(value); + var (_, restored) = get_contract_data(); + return restored; +} + +fun main() { return; } + +/** + +@testcase | 101 | | 2 +@testcase | 102 | 44 | 44 + +*/ diff --git a/tolk-tester/tests/remove-unused-functions.tolk b/tolk-tester/tests/remove-unused-functions.tolk new file mode 100644 index 00000000..8e748ecf --- /dev/null +++ b/tolk-tester/tests/remove-unused-functions.tolk @@ -0,0 +1,48 @@ +fun unused1(): int { return 2; } +fun unused2(): int { return unused1(); } +fun unused3(x: int): int { return x * 2+unused2(); } + +fun used_from_noncall1(): int { return 10; } +fun used_as_noncall1(): int { return used_from_noncall1(); } + +const int20: int = 20; +fun used_from_noncall2(): int { return int20; } +fun used_as_noncall2(): int { return 0 * 0 + used_from_noncall2() + (0 << 0); } + +global unused_gv: int; +global used_gv: int; + +fun receiveGetter(): () -> int { return used_as_noncall2; } + +@pure +fun usedButOptimizedOut(x: int): int { return x + 2; } + +fun main(): (int, int, int) { + used_gv = 1; + used_gv = used_gv + 2; + var getter1 = used_as_noncall1; + var getter2 = receiveGetter(); + usedButOptimizedOut(used_gv); + return (used_gv, getter1(), getter2()); +} + +/** +@experimental_options remove-unused-functions + +@testcase | 0 | | 3 10 20 + +@fif_codegen DECLPROC used_as_noncall1 +@fif_codegen DECLGLOBVAR used_gv + +@fif_codegen_avoid DECLPROC unused1 +@fif_codegen_avoid DECLPROC unused2 +@fif_codegen_avoid DECLPROC unused3 +@fif_codegen_avoid DECLGLOBVAR unused_gv + +Note, that `usedButOptimizedOut()` (a pure function which result is unused) +is currently codegenerated, since it's formally reachable. +This is because optimizing code is a moment of codegen for now (later than marking unused symbols). + +@fif_codegen DECLPROC usedButOptimizedOut +@fif_codegen_avoid usedButOptimizedOut CALLDICT +*/ diff --git a/tolk-tester/tests/s1.tolk b/tolk-tester/tests/s1.tolk new file mode 100644 index 00000000..c7c4f694 --- /dev/null +++ b/tolk-tester/tests/s1.tolk @@ -0,0 +1,61 @@ +get ascii_slice(): slice { + return"string"; +} + +get raw_slice(): slice { + return "abcdef"s; +} + +get addr_slice(): slice { + return "Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"a; +} + +get string_hex(): int { + return "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"u; +} + +get fun string_minihash(): int { // 'get' and 'get fun' both possible + return "transfer(slice, int)"h; +} + +get fun string_maxihash(): int { + return "transfer(slice, int)"H; +} + +get fun string_crc32(): int { + return "transfer(slice, int)"c; +} + +@pure +fun newc(): builder +asm "NEWC"; +fun endcs(b: builder): slice +asm "ENDC" "CTOS"; +@pure +fun sdeq(s1: slice, s2: slice): int +asm "SDEQ"; + +fun main() { + var s_ascii: slice = ascii_slice(); + var s_raw: slice = raw_slice(); + var s_addr: slice = addr_slice(); + var i_hex: int = string_hex(); + var i_mini: int = string_minihash(); + var i_maxi: int = string_maxihash(); + var i_crc: int = string_crc32(); + assert(sdeq(s_ascii, newc().storeUint(0x737472696E67, 12 * 4).endcs())) throw 101; + assert(sdeq(s_raw, newc().storeUint(0xABCDEF, 6 * 4).endcs())) throw 102; + assert(sdeq(s_addr, newc().storeUint(4, 3).storeInt(-1, 8) + .storeUint(0x3333333333333333333333333333333333333333333333333333333333333333, 256).endcs()), 103); + assert(i_hex == 0x4142434445464748494A4B4C4D4E4F505152535455565758595A303132333435) throw 104; + assert(i_mini == 0x7a62e8a8) throw 105; + assert(i_maxi == 0x7a62e8a8ebac41bd6de16c65e7be363bc2d2cbc6a0873778dead4795c13db979) throw 106; + assert(i_crc == 2235694568) throw 107; + return 0; +} + +/** +@testcase | 0 | | 0 + +@code_hash 13830542019509784148027107880226447201604257839069192762244575629978154217223 +*/ diff --git a/tolk-tester/tests/self-keyword.tolk b/tolk-tester/tests/self-keyword.tolk new file mode 100644 index 00000000..b0567696 --- /dev/null +++ b/tolk-tester/tests/self-keyword.tolk @@ -0,0 +1,253 @@ +fun incChained(mutate self: int): self { + self = self + 1; + return self; +} + +fun incChained2(mutate self: int): self { + return self.incChained(); +} + +fun incChained3(mutate self: int): self { + incChained(mutate self); + return self; +} + +fun incChained4(mutate self: int): self { + self.incChained(); + return self; +} + +@method_id(101) +fun testIncChainedCodegen(x: int) { + return x.incChained().incChained2().incChained3().incChained4(); +} + +@method_id(102) +fun testIncChained() { + var x: int = 10; + incChained(mutate x); + x.incChained(); + x.incChained2(); + x.incChained2().incChained(); + x = x.incChained(); + x = x.incChained2().incChained().incChained2(); + return x.incChained(); +} + +fun incChainedWithMiddleReturn(mutate self: int, maxValue: int): self { + if (self >= maxValue) { + return self; + } + self += 1; + return self; +} + +@method_id(103) +fun testIncChainedWithMiddleReturn(x: int) { + x.incChainedWithMiddleReturn(10).incChainedWithMiddleReturn(10); + x = x.incChainedWithMiddleReturn(10).incChainedWithMiddleReturn(10); + return x.incChainedWithMiddleReturn(10).incChainedWithMiddleReturn(999); +} + +fun incChainedMutatingBoth(mutate self: int, mutate y: int): self { + self += 1; + y += 1; + return self; +} + +global c104: int; + +@method_id(104) +fun testIncChainedMutatingBoth() { + var (x, y) = (0, 0); + c104 = 0; + x.incChainedMutatingBoth(mutate y).incChainedMutatingBoth(mutate y); + incChainedMutatingBoth(mutate x, mutate y); + x = x.incChainedMutatingBoth(mutate c104).incChainedMutatingBoth(mutate c104).incChainedMutatingBoth(mutate y); + return (x, y, c104); +} + +fun incTensorChained(mutate self: (int, int)): self { + val (f, s) = self; + self = (f + 1, s + 1); + return self; +} + +@method_id(105) +fun testIncTensorChained(f: int, s: int) { + var tens = (f, s); + tens.incTensorChained().incTensorChained(); + return tens.incTensorChained().incTensorChained(); +} + +fun incConditionalChainable(mutate self: int, mutate another: int, ifLessThan: int): self { + another += 1; + return self.incChained() < ifLessThan ? self.incChained().incChained() : self; +} + +@method_id(106) +fun testIncConditionalChainable(x: int) { + var y = 0; + x.incConditionalChainable(mutate y, 5).incConditionalChainable(mutate y, 5); + x = x.incConditionalChainable(mutate y, 5).incConditionalChainable(mutate y, 5); + return (x.incConditionalChainable(mutate y, 5), y); +} + +fun checkNotEq(self: int, throwIfEq: int): void { + if (self == throwIfEq) { + throw 100 + throwIfEq; + } +} + +@method_id(107) +fun testNotMutatingSelf(arg: int) { + try { + arg.checkNotEq(100); + arg.checkNotEq(101); + arg.checkNotEq(102); + return 0; + } catch (code) { + return code; + } +} + +global c108: int; + +fun checkNotEqChainable(self: int, throwIfEq: int): self { + c108 += 1; + if (self != throwIfEq) { + return self; + } + throw 100 + throwIfEq; + return self; +} + +@method_id(108) +fun testNotMutatingChainableSelf(arg: int) { + c108 = 0; + try { + arg.checkNotEqChainable(100).checkNotEqChainable(101).checkNotEqChainable(102); + arg = arg.checkNotEqChainable(100).checkNotEqChainable(101).checkNotEqChainable(102); + return (arg, c108); + } catch (code) { + return (code, c108); + } +} + +global onceFailed109: int; + +fun checkNotEqChainableMutateAnother(self: int, throwIfEq: int, mutate toInc: int): self { + if (onceFailed109) { return self; } + toInc += 1; + try { return self.checkNotEqChainable(throwIfEq); } + catch { onceFailed109 = 1; return self; } +} + +global c109: int; + +@method_id(109) +fun testNotMutatingChainableSelfMutateAnother(initial: int) { + val arg = initial; + var x = 0; + c108 = 0; + c109 = 0; + onceFailed109 = 0; + arg.checkNotEqChainableMutateAnother(100, mutate x) + .checkNotEqChainableMutateAnother(101, mutate c109) + .checkNotEqChainableMutateAnother(102, mutate x); + return (arg, c108, c109, x); +} + +fun pickG110(mutate self: int, mutate pushTo: tuple): self { + self += 10; + pushTo.tuplePush(c110); + return self; +} + +global tup110: tuple; +global c110: int; + +@method_id(110) +fun testMutateGlobalsLValue(init: int) { + c110 = init; + tup110 = createEmptyTuple(); + c110.incChained().incChained().pickG110(mutate tup110).incChained().pickG110(mutate tup110).incChained(); + return (c110, tup110); +} + +fun myTuplePush(mutate self: tuple, value: T): self { + self.tuplePush(value); + return self; +} + +fun myTupleAt(self: tuple, idx: int): T { + return self.tupleAt(idx); +} + +global tup111: tuple; + +@method_id(111) +fun testForallFunctionsWithSelf(): (int, int, tuple) { + var t = createEmptyTuple(); + tup111 = createEmptyTuple(); + t.myTuplePush(10); + tup111.myTuplePush(1).myTuplePush(2).myTuplePush(3); + return (t.myTupleAt(0), tup111.myTupleAt(tup111.tupleSize() - 1), tup111); +} + + + +fun main() { } + +/** +@testcase | 101 | 5 | 9 +@testcase | 102 | | 20 +@testcase | 103 | 1 | 7 +@testcase | 103 | 100 | 101 +@testcase | 103 | 8 | 11 +@testcase | 104 | | 6 4 2 +@testcase | 105 | 1 2 | 5 6 +@testcase | 106 | -20 | -5 5 +@testcase | 106 | -1 | 8 5 +@testcase | 106 | 7 | 12 5 +@testcase | 107 | 200 | 0 +@testcase | 107 | 102 | 202 +@testcase | 108 | 200 | 200 6 +@testcase | 108 | 101 | 201 0 +@testcase | 109 | 200 | 200 3 1 2 +@testcase | 109 | 100 | 100 0 0 1 +@testcase | 109 | 102 | 102 2 1 2 +@testcase | 110 | 0 | 24 [ 2 13 ] +@testcase | 111 | | 10 3 [ 1 2 3 ] + +@fif_codegen +""" + incChained PROC:<{ + // self + INC // self + }> + incChained2 PROC:<{ + // self + incChained CALLDICT // self + }> + incChained3 PROC:<{ + // self + incChained CALLDICT // self + }> + incChained4 PROC:<{ + // self + incChained CALLDICT // self + }> +""" + +@fif_codegen +""" + testIncChainedCodegen PROC:<{ + // x + incChained CALLDICT // x + incChained2 CALLDICT // x + incChained3 CALLDICT // x + incChained4 CALLDICT // x + }> +""" + */ diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk new file mode 100644 index 00000000..4d71bb63 --- /dev/null +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -0,0 +1,678 @@ +// the goal of this file is not only to @testcase results — +// but to check that this file compiles + +fun getNullableInt(): int? { return 5; } +fun getNullableSlice(): slice? { return null; } +fun takeNullableInt(a: int?) {} +fun takeNullableSlice(a: slice?) {} +fun increment(mutate self: int) { self += 1; } +fun assignToInt(mutate self: int, value: int) { self = value; } +fun assignToNullableInt(mutate self: int?, value: int) { self = value; } +fun sameTensor(t: (int, int)) { return t; } +fun sameTensor2(t: (int?, (slice, slice, slice, builder)?)) { return t; } +fun eq(v: T) { return v; } +fun getTwo(): X { return 2 as X; } + +fun test1(): int { + var x = getNullableInt(); + var y = getNullableInt(); + if (x != null && y != null) { + __expect_type(x, "int"); + __expect_type(y, "int"); + return x + y; + } + return -1; +} + +fun test2() { + var (x, y) = (getNullableInt(), getNullableInt()); + if (x == null || y == null) { + return null; + } + __expect_type(x, "int"); + __expect_type(y, "int"); + return x + y; +} + +fun test3(): int { + var ([x, y]) = [getNullableInt(), getNullableInt()]; + if (x != null) { + if (((y)) != null) { + __expect_type(x, "int"); + __expect_type(y, "int"); + return x + y; + } + return x; + } + if (random() > -1) { + if (y == null) { return -1; } + else { return y; } + } + return 0; +} + +fun test4() { + var x = getNullableInt(); + if (x != null && x > 0) { + var x = getNullableInt(); + if ((x) != null && x + 10 < 0) { + var x = getNullableInt(); + return 10 > 3 && 10 < 10 && x != null && x + 8 > 10; + } + } + if (x != null && x < 1) { + return false; + } + if (x == null && x == null) { + __expect_type(x, "null"); + return true; + } + return x < x + 3; +} + +fun test5() { + var (a, (b, c)) = (getNullableInt(), (getNullableInt(), getNullableInt())); + if (a == null) { return -1; } + if (!(b != null)) { return -2; } + if (random() ? c == null && c == null : c == null) { return -3; } + return a + b + c; +} + +fun test6() { + var a: int? = 5; + __expect_type(a, "int"); + __expect_type(a != null ? a : null, "int"); + __expect_type(a == null ? "" : a, "int"); + takeNullableInt(a); + __expect_type(a, "int"); + if (random()) { + a = null; + } else { + if (random()) { a = null; } + else { a = null; } + } + __expect_type(a, "null"); + takeNullableSlice(a); // ok, `slice?` is `slice | null`, here a definitely null + var b: int? = true ? null : "sl"; + __expect_type(b, "null"); + takeNullableInt(b); + takeNullableSlice(b); // same reason + var c: int? = 10; + __expect_type(c, "int"); + takeNullableSlice(c = null); +} + +fun test7() { + var (a, b, c, d) = (getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt()); + if (a == null && true) { return -1; } + if (true && true && 1 && !0 && b == null) { return -2; } + if (true ? c == null && (((c))) == null && true : false) { return -3; } + if (!true ? random() > 0 : a != null && (d == null && b != null)) { return -4; } + return a + b + c + d; +} + +fun test8(x: int?, y: int?) { + var allGt1 = x != null && x > 1 && y != null && y > 1; + var xGtY = x != null && y != null && x > y; + var xLtEq0 = x == null || x < 0; + (x = 0) < random() || x > 10; + return x + 0; +} + +fun test9() { + var x = getNullableInt(); + var y = getNullableInt(); + if (x == null || y == null) { + return -1; + } + __expect_type(x, "int"); + __expect_type(y, "int"); + return x + y; +} + +fun test10(): int { + var (x, y) = (getNullableInt(), getNullableInt()); + if (x == null) { + if (y == null) { return -1; } + __expect_type(x, "null"); + __expect_type(y, "int"); + return y; + } + if (y == null) { + return x; + } + __expect_type(x, "int"); + __expect_type(y, "int"); + return x + y; +} + +fun test11() { + var [x, y] = [getNullableInt(), getNullableInt()]; + if (random()) { return x == null || y == null ? -1 : x + y; } + if (true && (x == null || y == null) && !!true) { return 0; } + return x + y; +} + +fun test12() { + var (x, y) = (getNullableInt(), getNullableInt()); + if (random() ? x == null || y == null : x == null || y == null) { return -1; } + __expect_type(x, "int"); + __expect_type(y, "int"); + return x + y; +} + +fun test13() { + var x: int? = getNullableInt(); + var y: int? = 10; + var z = getNullableInt(); + var w = getNullableInt(); + beginCell().storeInt(x!, 32).storeInt(x = getNullableInt()!, 32).storeInt(x, 32) + .storeInt(y, 32).storeInt(z = 10, 32).storeInt(x + y + z, 32) + .storeInt(w == null ? -1 : w, 32).storeInt(!(null == w) ? w : -1, 32); +} + +fun test14() { + var (x, y) = (getNullableInt(), getNullableInt()); + if (x == null) { + x = 0; + } + if (y == null) { + if (random()) { return 0; } + else { y = 0; } + } + return x + y; +} + +fun test20() { + var t = (getNullableInt(), getNullableInt()); + if (t.0 != null && t.1 != null) { + __expect_type(t.0, "int"); + __expect_type(t.1, "int"); + return t.0 + t.1; + } + t.0 = 10; + if (t.1 == null) { + t.1 = 20; + } + __expect_type(t.0, "int"); + __expect_type(t.1, "int"); + return t.0 + t.1; +} + +fun test21() { + var t = (getNullableInt(), (getNullableInt(), getNullableInt())); + if (t.0 != null && t.1.0 != null) { + if (t.1.1 != null) { return t.0 + t.1.0 + t.1.1; } + return t.0 + t.1.0; + } + if (t.0 != null) { + return t.0 + 0; + } + __expect_type(t.0, "null"); + __expect_type(t.1.0, "int?"); + return t.1.0 == null ? -1 : t.1.0 + 0; +} + +fun test22() { + var t = (getNullableInt(), (getNullableInt(), getNullableInt())); + if (t.0 == null || t.1.0 == null || t.1.1 == null) { + return -1; + } + return t.0 + t.1.0 + t.1.1; +} + +@method_id(123) +fun test23() { + var (x: int?, y: int?, z: int?) = (getNullableInt(), getNullableInt(), getNullableInt()); + ((x = 1, 0).0, (y = 2, 1).0) = (3, z = 4); + return x + y + z; +} + +@method_id(124) +fun test24(x: int?) { + if (x == null) { + __expect_type(x, "null"); + assignToNullableInt(mutate x, 10); + __expect_type(x, "int?"); + x.assignToNullableInt(x! + 5); + } else { + __expect_type(x, "int"); + increment(mutate x); + x.increment(); + __expect_type(x, "int"); + } + __expect_type(x, "int?"); + return x; +} + +fun test25() { + var x = (getNullableInt(), getNullableInt(), getNullableInt()); + x.0 = x.2 = random(); + return (x.0) + ((x.2)); +} + +fun test26() { + var x = [getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), + getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt()]; + if (~(x.0 = random())) { return; } + if ((x.1 = random()) < (x.2 = random())) { return; } + else if (!(x.2 <=> (x.3 = random()))) { return; } + x.5 = (x.4 = random()) ? (x.6 = random()) : (x.6 = random()); + if ((x.7 = random()) as int) { return; } + if (((((x.8 = random()) != null)))) { return; } + if ([x.1, (x.9 = random())!].1) { return; } + val result = x.0+x.1+x.2+x.3+x.4+x.5+x.6+x.7+x.8+x.9; +} + +fun test27() { + var (x, _) = ([getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), + getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt()], []); + +(x.0 = random()); + x.0 += [((x.1 = random()) < (x.2 = random() + x.1)) as int].0; + !(x.2 <=> (x.3 = random() + x.2)); + x.5 = (x.4 = random()) ? (x.6 = random()) : (x.6 = random()); + (x.7 = random()) as int; + (((((x.8 = random()) != null)))); + [x.1, (x.9 = random())!].1; + return x.0+x.1+x.2+x.3+x.4+x.5+x.6+x.7+x.8+x.9; +} + +fun test28() { + var x = (getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt()); + __expect_type((x.0 = random(), x.0 += (x.1 = random()) as int, !(x.1 <=> (x.2 = random() + x.0)) == null, (x.3 = random()) ? x.3 : (!x.3) as int), + "(int, int, bool, int)"); +} + +fun test29() { + var x = (getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt()); + __expect_type([x.0 = random(), ((x.0 += (x.1 = random()) as int)), !(x.1 <=> (x.2 = random() + x.0)) == null, (x.3 = random()) ? x.3 : (!x.3) as int], + "[int, int, bool, int]"); +} + +@method_id(130) +fun test30(initial5: bool) { + var t: (int?, (int?, (int?, int?))) = initial5 + ? (getNullableInt(), (getNullableInt(), (getNullableInt(), getNullableInt()))) + : (null, (null, (null, null))); + if (t.0 == null || t.1.0 == null || t.1.1.0 == null || t.1.1.1 == null) { + if (t.1.0 == null || t.1.1.0 == null) { + if (t.1.1.0 == null) { + t.1.1.0 = 4; + } + __expect_type(t.1.1.0, "int"); + __expect_type(t.1.1.1, "int?"); + __expect_type(t.1.0, "int?"); + t.1.1.1 = 3; + t.1.0 = 2; + __expect_type(t.1.1.1, "int"); + __expect_type(t.1.0, "int"); + } + if (((((t.1.1.1)))) != null) {} + else { t.1.1.1 = 3; } + t.0 = 1; + } + return t.0 + t.1.0 + t.1.1.0 + t.1.1.1; +} + +fun test31() { + var t = (getNullableInt(), getNullableInt()); + t.0 == null ? (t.0, t.1) = (1, 2) : (t.1, t.0) = (4, 3); + return t.0 + t.1; +} + +@method_id(132) +fun test32() { + var t: (int?, (int?, int?)?, (int?, int?)) = (getNullableInt(), (getNullableInt(), getNullableInt()), (getNullableInt(), getNullableInt())); + if (t.0 == null) { return -1; } + t.1 != null && t.1.0 == null ? t.1 = (1, 2) : t.1 = (3, 4); + if (t.2.1 != null) { t.2.0 = 1; t.2.1 = 2; } + else { [t.2.0, t.2.1] = [3, 4]; } + return t.0 + t.1.0! + t.1.1! + t.2.0 + t.2.1; +} + +@method_id(133) +fun test33(): int { + var x = getNullableInt(); + repeat (eq(x = 5)) { + __expect_type(x, "int"); + increment(mutate x); + } + return x; +} + +fun test34() { + var (x, y) = (getNullableInt(), getNullableInt()); + if (random()) { throw (x = 1, y = 2); } + else { throw (x = 3, y = (1, getNullableInt()!).1); } + return x + y; +} + +fun test35() { + var (x, y, z, t) = (getNullableInt(), getNullableInt(), getNullableInt(), (getNullableInt(), getNullableInt())); + assert (x != null, 404); + assert (t.0 != null && true && !(t.1 == null) && !(z = 4)) throw (y = 404); + __expect_type(y, "int?"); + return x + t.0 + t.1 + z; +} + +fun test36() { + var x = getNullableInt(); + assert (x == null, x + 0); // check that x is int there + __expect_type(x, "null"); +} + +fun test37() { + var (x, code) = (getNullableInt()!, getNullableInt()); + try { + } catch(code) { + x = 20; + return x + code; // code is scoped + } + return code == null ? x : x + code; +} + +fun assignNull2(mutate x: T1?, mutate y: T2?) { + x = null; + y = null; +} + +fun test38() { + var (x: int?, y: int?) = (1, 2); + __expect_type(x, "int"); + __expect_type(y, "int"); + assignNull2(mutate x, mutate y); + __expect_type(x, "int?"); + __expect_type(y, "int?"); + if (x != null) { + if (y == null) { return -1; } + return x + y; + } + var t: (int?, slice?) = (null, null); + if (!false) { t.0 = 1; } + if (true) { t.1 = beginCell().endCell().beginParse(); } + __expect_type(t.0, "int"); + __expect_type(t.1, "slice"); + t.0 + t.1.loadInt(32); + assignNull2(mutate t.0, mutate t.1); + __expect_type(t.0, "int?"); + __expect_type(t.1, "slice?"); + t.0 != null && t.1 != null ? t.0 + loadInt(mutate t.1, 32) : -1; + return t.0 != null && t.1 != null ? t.0 + loadInt(mutate t.1, 32) : -1; +} + +@method_id(139) +fun test39() { + var x: (int?, int?)? = (4, null); + x.1 = 10; + x.1 += 1; + x!.1 += 1; + return (x!.0! + x.1); +} + +@method_id(140) +fun test40(second: int?) { + var x: (int?, int?)? = (4, second); + if (x.1 != null) { + val result = x.1 + x!.1 + x!!.1 + x.1! + x!!.1!!; + } + if (x!.1 != null) { + val result = x.1 + x!.1 + x!!.1 + x.1! + x!!.1!!; + } + if (!(x!!.1 != null)) { + return -1; + } + return x.1 + x!.1 + x!!.1 + x.1! + x!!.1!!; +} + +@method_id(141) +fun test41() { + var t: (int, int)? = null; + return sameTensor(t = (1, 2)); +} + +@method_id(142) +fun test42() { + var t: (int?, (int?, (int, int)?)?) = (getNullableInt(), (1, (2, 3))); + t.1 = (3,null); + __expect_type(t.1, "(int?, (int, int)?)"); + __expect_type(t, "(int?, (int?, (int, int)?)?)"); + return (t, t.1); +} + +@method_id(143) +fun test43() { + var t1: ((int, int), int?) = ((1, 2), 3); + var t2: ((int?, int?), (int?,int?)?) = ((null, null), (null, 5)); + t2.0 = t1.0 = (10, 11); + t2.1 = t1.1 = null; + return (t1, t2); +} + +@method_id(144) +fun test44() { + var t1: ((int, int), int?) = ((1, 2), 3); + var t2: ((int?, int?), (int?,int?)?) = ((null, null), (null, 5)); + t1.0 = t2.0 = (10, 11); + t1.1 = t2.1 = null; + __expect_type(t1, "((int, int), int?)"); + __expect_type(t2, "((int?, int?), (int?, int?)?)"); + return (t1, t2); +} + +@method_id(145) +fun test45() { + var t: (int?, (int?, (int, int)?)?) = (getNullableInt(), (1, (2, 3))); + var t2 = sameTensor2(t.1 = (3,null)); + return (t, t2, t.1); +} + +fun autoInfer46() { + var t1: int? = 3; + var t2: (int, int)? = (4, 5); + __expect_type(t1, "int"); + __expect_type(t2, "(int, int)"); + return (t1, t2); // proven to be not null, inferred (int, (int,int)) +} + +@method_id(146) +fun test46() { + var r46_1: (int, (int,int)) = autoInfer46(); + var r46_2: (int, (int,int)?) = autoInfer46(); + return (r46_1, r46_2); +} + +@method_id(147) +fun test47() { + var t1: int? = 3; + var t2: (int, int)? = (4, 5); + t1 = t2 = null; + __expect_type(t1, "null"); + __expect_type(t2, "null"); + var result = (t1, t2); // proven to be always null, inferred (null, null), 2 slots on a stack + return (result, 100, result.1, 100, t2 as (int, int)?); +} + +fun test48() { + var t1: int? = getNullableInt(); + if (t1 != null) { + var (t1 redef, t2) = (10, 5); + return t1 + t2; + var t2 redef = getNullableInt()!; + return t1 + t2; + } + return -1; +} + +fun test49(x: int?) { + while (x == null) { + x = getNullableInt(); + } + __expect_type(x, "int"); + return x + 1; +} + +fun test50() { + var (x: int?, y: int?) = (1, 2); + do { + x = getNullableInt(); + y = getNullableInt(); + } while (x == null || y == null); + return x + y; +} + +fun test51() { + while (true) { return; } + // test that no error "control reaches end of function" +} + +fun test52() { + do { } while (true); +} + +fun test53() { + var x1: int? = getNullableInt(); + var x2: int? = 5; + var x3: int? = 5; + var x10: int? = null; + var x11: int? = 5; + var x12: int? = 5; + while (x1 != null) { + __expect_type(x1, "int"); // because condition + __expect_type(x2, "int?"); // because re-assigned + __expect_type(x3, "int?"); // because re-assigned + __expect_type(x10, "null"); + __expect_type(x11, "int"); + x1 = getNullableInt(); + __expect_type(x1, "int?"); + assignToNullableInt(mutate x2, 5); + x3.assignToNullableInt(5); + x11 = 10; + assignToInt(mutate x12, 5); + } + __expect_type(x1, "null"); + __expect_type(x2, "int?"); + __expect_type(x3, "int?"); +} + +fun test54() { + var x1: int? = null; + var x2: int? = 5; + var x3: int? = 5; + var x10: int? = null; + var x11: int? = 5; + var x12: int? = 5; + do { + __expect_type(x1, "int?"); // because re-assigned + __expect_type(x2, "int?"); // because re-assigned + __expect_type(x3, "int?"); // because re-assigned + __expect_type(x10, "null"); + __expect_type(x11, "int"); + x1 = getNullableInt(); + __expect_type(x1, "int?"); + assignToNullableInt(mutate x2, 5); + if (random()) { x3.assignToNullableInt(5); } + x11 = 10; + assignToInt(mutate x12, 5); + } while (x1 != null); + __expect_type(x1, "null"); + __expect_type(x2, "int?"); + __expect_type(x3, "int?"); +} + +fun eq55(v: T) { return v; } + +fun test55() { + var x: int? = 4; + while (true) { + // currently, generic functions are instantiated at the type inferring step + // in case of loops, type inferring is re-enterable + // first iteration: x is int, eq instantiated + // second (final) iteration: x is int?, eq instantiated + // (checked via codegen) + eq55(x); + __expect_type(x, "int?"); // types are checked (unlike generics instantiated) after inferring + x = random() ? 1 : null; + } + __expect_type(x, "int?"); +} + +fun test56() { + var i: int? = null; + var (j: int?, k: int?) = (null, null); + __expect_type(i, "null"); + __expect_type(k, "null"); + i = getTwo(); + [j, ((k))] = [getTwo(), ((getTwo()))]; + __expect_type(i, "int?"); + __expect_type(j, "int?"); + __expect_type(k, "int?"); +} + +fun test57(mutate x: int?): int { + if (x == null) { x = 5; } + else { + if (x < 10) { x = 10; } + else { x = 20; } + } + if (x != null) { + return 123; + } + __expect_type(x, "int"); + // no "return" needed, because end of function is unreachable +} + +@method_id(158) +fun test58() { + var (x1, x2: int?) = (getNullableInt(), null); + return (test57(mutate x1), x1, test57(mutate x2), x2); +} + +fun test59() { + var (x1: int?, x2, x3) = (getNullableInt()!, getNullableInt(), 5); + if ((x2 = x3) != null) { + __expect_type(x2, "int"); + } + __expect_type(x2, "int"); + if ((x2 = getNullableInt()) != null) { + __expect_type(x2, "int"); + } + __expect_type(x2, "int?"); + if (((x1) = x2) == null) { + return; + } + __expect_type(x1, "int"); +} + + + +fun main(x: int?): int { + return x == null ? -1 : x; +} + +/** +@testcase | 0 | 1 | 1 +@testcase | 123 | | 7 +@testcase | 124 | 4 | 6 +@testcase | 124 | null | 15 +@testcase | 130 | -1 | 20 +@testcase | 130 | 0 | 10 +@testcase | 132 | | 15 +@testcase | 133 | | 10 +@testcase | 139 | | 16 +@testcase | 140 | 5 | 25 +@testcase | 141 | | 1 2 +@testcase | 142 | | 5 3 (null) (null) 0 -1 3 (null) (null) 0 +@testcase | 143 | | 10 11 (null) 10 11 (null) (null) 0 +@testcase | 144 | | 10 11 (null) 10 11 (null) (null) 0 +@testcase | 145 | | 5 3 (null) (null) 0 -1 3 (null) (null) (null) (null) 0 3 (null) (null) 0 +@testcase | 146 | | 3 4 5 3 4 5 -1 +@testcase | 147 | | (null) (null) 100 (null) 100 (null) (null) 0 +@testcase | 158 | | 123 10 123 5 + +@stderr warning: expression of type `int` is always not null, this condition is always true +@stderr warning: unreachable code +@stderr var t2 redef = getNullableInt()!; + +@fif_codegen eq55 PROC:<{ +@fif_codegen eq55 PROC:<{ +*/ diff --git a/tolk-tester/tests/special-fun-names.tolk b/tolk-tester/tests/special-fun-names.tolk new file mode 100644 index 00000000..8fae6d5d --- /dev/null +++ b/tolk-tester/tests/special-fun-names.tolk @@ -0,0 +1,24 @@ +fun onInternalMessage() { return 0; } +fun onExternalMessage() { return -1; } +fun onRunTickTock() { return -2; } +fun onSplitPrepare() { return -3; } +fun onSplitInstall() { return -4; } + +/** +@experimental_options remove-unused-functions + +@testcase | 0 | | 0 +@testcase | -1 | | -1 +@testcase | -2 | | -2 +@testcase | -3 | | -3 +@testcase | -4 | | -4 + +@fif_codegen +""" + 0 DECLMETHOD onInternalMessage + -1 DECLMETHOD onExternalMessage + -2 DECLMETHOD onRunTickTock + -3 DECLMETHOD onSplitPrepare + -4 DECLMETHOD onSplitInstall +""" + */ diff --git a/tolk-tester/tests/test-math.tolk b/tolk-tester/tests/test-math.tolk new file mode 100644 index 00000000..6b147c77 --- /dev/null +++ b/tolk-tester/tests/test-math.tolk @@ -0,0 +1,1288 @@ +// this is actually `mathlib.fc` transformed to Tolk + +import "@stdlib/tvm-lowlevel" + +/*--------------- MISSING OPERATIONS AND BUILT-INS ----------------*/ + +/// compute floor(log2(x))+1 +@pure +fun log2_floor_p1(x: int): int + asm "UBITSIZE"; + +@pure +fun mulrshiftr(x: int, y: int, s: int): int + asm "MULRSHIFTR"; + +@pure +fun mulrshiftr256(x: int, y: int): int + asm "256 MULRSHIFTR#"; + +@pure +fun mulrshift256mod(x: int, y: int): (int, int) + asm "256 MULRSHIFT#MOD"; + +@pure +fun mulrshiftr256mod(x: int, y: int): (int, int) + asm "256 MULRSHIFTR#MOD"; + +@pure +fun mulrshiftr255mod(x: int, y: int): (int, int) + asm "255 MULRSHIFTR#MOD"; + +@pure +fun mulrshiftr248mod(x: int, y: int): (int, int) + asm "248 MULRSHIFTR#MOD"; + +@pure +fun mulrshiftr5mod(x: int, y: int): (int, int) + asm "5 MULRSHIFTR#MOD"; + +@pure +fun mulrshiftr6mod(x: int, y: int): (int, int) + asm "6 MULRSHIFTR#MOD"; + +@pure +fun mulrshiftr7mod(x: int, y: int): (int, int) + asm "7 MULRSHIFTR#MOD"; + +@pure +fun lshift256divr(x: int, y: int): int + asm "256 LSHIFT#DIVR"; + +@pure +fun lshift256divmodr(x: int, y: int): (int, int) + asm "256 LSHIFT#DIVMODR"; + +@pure +fun lshift255divmodr(x: int, y: int): (int, int) + asm "255 LSHIFT#DIVMODR"; + +@pure +fun lshift2divmodr(x: int, y: int): (int, int) + asm "2 LSHIFT#DIVMODR"; + +@pure +fun lshift7divmodr(x: int, y: int): (int, int) + asm "7 LSHIFT#DIVMODR"; + +@pure +fun lshiftdivmodr(x: int, y: int, s: int): (int, int) + asm "LSHIFTDIVMODR"; + +@pure +fun rshiftr256mod(x: int): (int, int) + asm "256 RSHIFTR#MOD"; + +@pure +fun rshiftr248mod(x: int): (int, int) + asm "248 RSHIFTR#MOD"; + +@pure +fun rshiftr4mod(x: int): (int, int) + asm "4 RSHIFTR#MOD"; + +@pure +fun rshift3mod(x: int): (int, int) + asm "3 RSHIFT#MOD"; + +/// computes y - x (Tolk compiler does not try to use this by itself) +@pure +fun sub_rev(x: int, y: int): int + asm "SUBR"; + +@pure +fun nan(): int + asm "PUSHNAN"; + +@pure +fun is_nan(x: int): int + asm "ISNAN"; + +/*----------------------- SQUARE ROOTS ---------------------------*/ + +/// computes sqrt(a*b) exactly rounded to the nearest integer +/// for all 0 <= a, b <= 2^256-1 +/// may be used with b=1 or b=scale of fixed-point numbers +@pure +@inline_ref +fun geom_mean(a: int, b: int): int { + if (!min(a, b)) { + return 0; + } + var s: int = log2_floor_p1(a); // throws out of range error if a < 0 or b < 0 + var t: int = log2_floor_p1(b); + // NB: (a-b)/2+b == (a+b)/2, but without overflow for large a and b + var x: int = (s == t ? (a - b) / 2 + b : 1 << ((s + t) / 2)); + do { + // if always used with b=2^const, may be optimized to "const LSHIFTDIVC#" + // it is important to use `mulDivCeil` here, not `mulDivFloor` or `mulDivRound` + var q: int = (mulDivCeil(a, b, x) - x) / 2; + x += q; + } while (q); + return x; +} + +/// integer square root, computes round(sqrt(a)) for all a>=0. +/// note: `inline` is better than `inline_ref` for such simple functions +@pure +@inline +fun sqrt(a: int): int { + return geom_mean(a, 1); +} + +/// version for fixed248 = fixed-point numbers with scale 2^248 +/// fixed248 sqrt(fixed248 x) +@pure +@inline +fun fixed248_sqrt(x: int): int { + return geom_mean(x, 1 << 248); +} + +/// fixed255 sqrt(fixed255 x) +@pure +@inline +fun fixed255_sqrt(x: int): int { + return geom_mean(x, 1 << 255); +} + +/// fixed248 sqr(fixed248 x); +@pure +@inline +fun fixed248_sqr(x: int): int { + return mulDivRound(x, x, 1 << 248); +} + +/// fixed255 sqr(fixed255 x); +@pure +@inline +fun fixed255_sqr(x: int): int { + return mulDivRound(x, x, 1 << 255); +} + +const fixed248_One: int = (1 << 248); +const fixed255_One: int = (1 << 255); + +/*------------------- USEFUL CONSTANTS -------------------*/ + +/// store huge constants in inline_ref functions for reuse +/// (y,z) where y=round(log(2)*2^256), z=round((log(2)*2^256-y)*2^128) +/// then log(2) = y/2^256 + z/2^384 +@pure +@inline_ref +fun log2_xconst_f256(): (int, int) { + return (80260960185991308862233904206310070533990667611589946606122867505419956976172, -32272921378999278490133606779486332143); +} + +/// (y,z) where Pi = y/2^254 + z/2^382 +@pure +@inline_ref +fun Pi_xconst_f254(): (int, int) { + return (90942894222941581070058735694432465663348344332098107489693037779484723616546, 108051869516004014909778934258921521947); +} + +/// atan(1/16) as fixed260 +@pure +@inline_ref +fun Atan1_16_f260(): int { + return 115641670674223639132965820642403718536242645001775371762318060545014644837101; // true value is ...101.0089... +} + +/// atan(1/8) as fixed259 +@pure +@inline_ref +fun Atan1_8_f259(): int { + return 115194597005316551477397594802136977648153890007566736408151129975021336532841; // correction -0.1687... +} + +/// atan(1/32) as fixed261 +@pure +@inline_ref +fun Atan1_32_f261(): int { + return 115754418570128574501879331591757054405465733718902755858991306434399246026247; // correction 0.395... +} + +/// inline is better than inline_ref for such very small functions +@pure +@inline +fun log2_const_f256(): int { + var (c: int, _) = log2_xconst_f256(); + return c; +} + +@pure +@inline +fun fixed248_log2_const(): int { + return log2_const_f256() ~>> 8; +} + +@pure +@inline +fun Pi_const_f254(): int { + var (c, _) = Pi_xconst_f254(); + return c; +} + +@pure +@inline +fun fixed248_Pi_const(): int { + return Pi_const_f254() ~>> 6; +} + +/*-------------- HYPERBOLIC TANGENT AND EXPONENT ------------------*/ + +/// hyperbolic tangent of small x via n+2 terms of Lambert's continued fraction +/// n=17: good for |x| < log(2)/4 = 0.173 +/// fixed258 tanh_f258(fixed258 x, int n) +@pure +@inline_ref +fun tanh_f258(x: int, n: int): int { + var x2: int = mulDivRound(x, x, 1 << 255); // x^2 as fixed261 + var a: int = (2 * n + 5) << 250; // a=2n+5 as fixed250 + var c = a; + var Two: int = (1 << 251); // 2. as fixed250 + repeat (n) { + a = (c -= Two) + mulDivRound(x2, 1 << 239, a); // a := 2k+1+x^2/a as fixed250, k=n+1,n,...,2 + } + a = (3 << 254) + mulDivRound(x2, 1 << 243, a); // a := 3+x^2/a as fixed254 + // y = x/(1+a') = x - x*a'/(1+a') = x - x*x^2/(a+x^2) where a' = x^2/a + return x - (mulDivRound(x, x2, a + (x2 ~>> 7)) ~>> 7); +} + +/// fixed257 expm1_f257(fixed257 x) +/// computes exp(x)-1 for small x via 19 terms of Lambert's continued fraction for tanh(x/2) +/// good for |x| < log(2)/2 = 0.347 (n=17); consumes ~3500 gas +@pure +@inline_ref +fun expm1_f257(x: int): int { + // (almost) compute tanh(x/2) first; x/2 as fixed258 = x as fixed257 + var x2: int = mulDivRound(x, x, 1 << 255); // x^2 as fixed261 + var Two: int = (1 << 251); // 2. as fixed250 + var a: int = 39 << 250; // a=2n+5 as fixed250 + var c = a; + repeat (17) { + a = (c -= Two) + mulDivRound(x2, 1 << 239, a); // a := 2k+1+x^2/a as fixed250, k=n+1,n,...,2 + } + a = (3 << 254) + mulDivRound(x2, 1 << 243, a); // a := 3+x^2/a as fixed254 + // now tanh(x/2) = x/(1+a') where a'=x^2/a ; apply exp(x)-1=2*tanh(x/2)/(1-tanh(x/2)) + var t: int = (x ~>> 4) - a; // t:=x-a as fixed254 + return x - mulDivRound(x2, t / 2, a + mulrshiftr256(x, t) ~/ 4) ~/ 4; // x - x^2 * (x-a) / (a + x*(x-a)) +} + +/// expm1_f257() may be used to implement specific fixed-point exponentials +/// example: +/// fixed248 exp(fixed248 x) +@pure +@inline_ref +fun fixed248_exp(x: int): int { + var (l2c, l2d) = log2_xconst_f256(); + // divide x by log(2) and convert to fixed257 + // (int q, x) = muldivmodr(x, 256, l2c); // unfortunately, no such built-in + var (q: int, x redef) = lshiftdivmodr(x, l2c, 8); + x = 2 * x - mulDivRound(q, l2d, 1 << 127); + var y: int = expm1_f257(x); + // result is (1 + y) * (2^q) --> ((1 << 257) + y) >> (9 - q) + return (y ~>> (9 - q)) - (-1 << (248 + q)); + // note that (y ~>> (9 - q)) + (1 << (248 + q)) leads to overflow when q=8 +} + +/// compute 2^x in fixed248 +/// fixed248 exp2(fixed248 x) +@pure +@inline_ref +fun fixed248_exp2(x: int): int { + // (int q, x) = divmodr(x, 1 << 248); // no such built-in + var (q: int, x redef) = rshiftr248mod(x); + x = mulDivRound(x, log2_const_f256(), 1 << 247); + var y: int = expm1_f257(x); + return (y ~>> (9 - q)) - (-1 << (248 + q)); +} + +/*-------------------- TRIGONOMETRIC FUNCTIONS ----------------------*/ + +/// fixed260 tan(fixed260 x); +/// computes tan(x) for small |x|> 10)) ~>> 9); +} + +/// fixed260 tan(fixed260 x); +@pure +@inline_ref +fun tan_f260(x: int): int { + return tan_f260_inlined(x); +} + +/// fixed258 tan(fixed258 x); +/// computes tan(x) for small |x|> 6)) ~>> 5); +} + +/// fixed258 tan(fixed258 x); +@pure +@inline_ref +fun tan_f258(x: int): int { + return tan_f258_inlined(x); +} + +/// (fixed259, fixed263) sincosm1(fixed259 x) +/// computes (sin(x), 1-cos(x)) for small |x|<2*atan(1/16) +@pure +@inline +fun sincosm1_f259_inlined(x: int): (int, int) { + var t: int = tan_f260_inlined(x); // t=tan(x/2) as fixed260 + var tt: int = mulrshiftr256(t, t); // t^2 as fixed264 + var y: int = tt ~/ 512 + (1 << 255); // 1+t^2 as fixed255 + // 2*t/(1+t^2) as fixed259 and 2*t^2/(1+t^2) as fixed263 + // return (mulDivRound(t, 1 << 255, y), mulDivRound(tt, 1 << 255, y)); + return (t - mulDivRound(t / 2, tt, y) ~/ 256, tt - mulDivRound(tt / 2, tt, y) ~/ 256); +} + +@pure +@inline_ref +fun sincosm1_f259(x: int): (int, int) { + return sincosm1_f259_inlined(x); +} + +/// computes (sin(x+xe),-cos(x+xe)) for |x| <= Pi/4, xe very small +/// this function is very accurate, error less than 0.7 ulp (consumes ~ 5500 gas) +/// (fixed256, fixed256) sincosn(fixed256 x, fixed259 xe) +@pure +@inline_ref +fun sincosn_f256(x: int, xe: int): (int, int) { + // var (q, x1) = muldivmodr(x, 8, Atan1_8_f259()); // no muldivmodr() builtin + var (q, x1) = lshift2divmodr(abs(x), Atan1_8_f259()); // reduce mod theta where theta=2*atan(1/8) + var (si, co) = sincosm1_f259(x1 * 2 + xe); + var (a, b, c) = (-1, 0, 1); + repeat (q) { + // (a+b*I) *= (8+I)^2 = 63+16*I + (a, b, c) = (63 * a - 16 * b, 16 * a + 63 * b, 65 * c); + } + // now a/c = cos(q*theta), b/c = sin(q*theta) exactly(!) + // compute (a+b*I)*(1-co+si*I)/c + // (b, a) = (lshift256divr(b, c), lshift256divr(a, c)); + var (b redef, br: int) = lshift256divmodr(b, c); br = mulDivRound(br, 128, c); + var (a redef, ar: int) = lshift256divmodr(a, c); ar = mulDivRound(ar, 128, c); + return (sign(x) * (((mulrshiftr256(b, co) - br) ~/ 16 - mulrshiftr256(a, si)) ~/ 8 - b), + a - ((mulrshiftr256(a, co) - ar) ~/ 16 + mulrshiftr256(b, si)) ~/ 8); +} + +/// compute (sin(x),1-cos(x)) in fixed256 for |x| < 16*atan(1/16) = 0.9987 +/// (fixed256, fixed257) sincosm1_f256(fixed256 x); +/// slightly less accurate than sincosn_f256() (error up to 3/2^256), but faster (~ 4k gas) and shorter +@pure +@inline_ref +fun sincosm1_f256(x: int): (int, int) { + var (si, co) = sincosm1_f259_inlined(x); // compute (sin,1-cos)(x/8) in (fixed259,fixed263) + var r: int = 7; + repeat (r / 2) { + // 1-cos(2*x) = 2*sin(x)^2, sin(2*x) = 2*sin(x)*cos(x) + (co, si) = (mulrshiftr256(si, si), si - (mulrshiftr256(si, co) ~>> r)); + r -= 2; + } + return (si, co); +} + +/// compute (p, q) such that p/q = tan(x) for |x|<2*atan(1/2)=1899/2048=0.927 +/// (int, int) tan_aux(fixed256 x); +@pure +@inline_ref +fun tan_aux_f256(x: int): (int, int) { + var t: int = tan_f258_inlined(x); // t=tan(x/4) as fixed258 + // t:=2*t/(1-t^2)=2*(t-t^3/(t^2-1)) + var tt: int = mulrshiftr256(t, t); // t^2 as fixed260 + t = mulDivRound(t, tt, tt ~/ 16 + (-1 << 256)) ~/ 16 - t; // now t=-tan(x/2) as fixed259 + return (t, mulrshiftr256(t, t) ~/ 4 + (-1 << 256)); // return (2*t, t^2-1) as fixed256 +} + +/// sincosm1_f256() and sincosn_f256() may be used to implement trigonometric functions for different fixed-point types +/// example: +/// (fixed248, fixed248) sincos(fixed248 x); +@pure +@inline_ref +fun fixed248_sincos(x: int): (int, int) { + var (Pic, Pid) = Pi_xconst_f254(); + // (int q, x) = muldivmodr(x, 128, Pic); // no muldivmodr() builtin + var (q: int, x redef) = lshift7divmodr(x, Pic); // reduce mod Pi/2 + x = 2 * x - mulDivRound(q, Pid, 1 << 127); + var (si: int, co: int) = sincosm1_f256(x); // doesn't make sense to use more accurate sincosn_f256() + co = (1 << 248) - (co ~>> 9); + si = si ~>> 8; + repeat (q & 3) { + (si, co) = (co, -si); + } + return (si, co); +} + +/// fixed248 sin(fixed248 x); +/// inline is better than inline_ref for such simple functions +@pure +@inline +fun fixed248_sin(x: int): int { + var (si: int, _) = fixed248_sincos(x); + return si; +} + +/// fixed248 cos(fixed248 x); +@pure +@inline +fun fixed248_cos(x: int): int { + var (_, co: int) = fixed248_sincos(x); + return co; +} + +/// similarly, tan_aux_f256() may be used to implement tan() and cot() for specific fixed-point formats +/// fixed248 tan(fixed248 x); +/// not very accurate when |tan(x)| is very large (difficult to do better without floating-point numbers) +/// however, the relative accuracy is approximately 2^-247 in all cases, which is good enough for arguments given up to 2^-249 +@pure +@inline_ref +fun fixed248_tan(x: int): int { + var (Pic, Pid) = Pi_xconst_f254(); + // (int q, x) = muldivmodr(x, 128, Pic); // no muldivmodr() builtin + var (q: int, x redef) = lshift7divmodr(x, Pic); // reduce mod Pi/2 + x = 2 * x - mulDivRound(q, Pid, 1 << 127); + var (a, b) = tan_aux_f256(x); // now a/b = tan(x') + if (q & 1) { + (a, b) = (b, -a); + } + return mulDivRound(a, 1 << 248, b); // either -b/a or a/b as fixed248 +} + +/// fixed248 cot(fixed248 x); +@pure +@inline_ref +fun fixed248_cot(x: int): int { + var (Pic, Pid) = Pi_xconst_f254(); + var (q: int, x redef) = lshift7divmodr(x, Pic); // reduce mod Pi/2 + x = 2 * x - mulDivRound(q, Pid, 1 << 127); + var (b, a) = tan_aux_f256(x); // now b/a = tan(x') + if (q & 1) { + (a, b) = (b, -a); + } + return mulDivRound(a, 1 << 248, b); // either -b/a or a/b as fixed248 +} + +/*---------------- INVERSE HYPERBOLIC TANGENT AND LOGARITHMS ----------------*/ + +/// inverse hyperbolic tangent of small x, evaluated by means of n terms of the continued fraction +/// valid for |x| < 2^-2.5 ~ 0.18 if n=37 (slightly less accurate with n=36) +/// |x| < 1/8 if n=32; |x| < 2^-3.5 if n=28; |x| < 1/16 if n=25 +/// |x| < 2^-4.5 if n=23; |x| < 1/32 if n=21; |x| < 1/64 if n=18 +/// fixed258 atanh(fixed258 x); +@pure +@inline_ref +fun atanh_f258(x: int, n: int): int { + var x2: int = mulrshiftr256(x, x); // x^2 as fixed260 + var One: int = (1 << 254); + var a: int = One ~/ n + (1 << 255); // a := 2 + 1/n as fixed254 + repeat (n - 1) { + // a := 1 + (1 - x^2 / a)(1 + 1/n) as fixed254 + var t: int = One - mulDivRound(x2, 1 << 248, a); // t := 1 - x^2 / a + var n1: int = n - 1; + a = mulDivRound(t, n, n1) + One; + n = n1; + } + // x / (1 - x^2 / a) = x / (1 - d) = x + x * d / (1 - d) for d = x^2 / a + // int d = mulDivRound(x2, 1 << 255, a - (x2 ~>> 6)); // d/(1-d) = x^2/(a-x^2) as fixed261 + // return x + (mulrshiftr256(x, d) ~>> 5); + return x + mulDivRound(x, x2 / 2, a - x2 ~/ 64) ~/ 32; +} + +/// number of terms n should be chosen as for atanh_f258() +/// fixed261 atanh(fixed261 x); +@pure +@inline +fun atanh_f261_inlined(x: int, n: int): int { + var x2: int = mulrshiftr256(x, x); // x^2 as fixed266 + var One: int = (1 << 254); + var a: int = One ~/ n + (1 << 255); // a := 2 + 1/n as fixed254 + repeat (n - 1) { + // a := 1 + (1 - x^2 / a)(1 + 1/n) as fixed254 + var t: int = One - mulDivRound(x2, 1 << 242, a); // t := 1 - x^2 / a + var n1: int = n - 1; + a = mulDivRound(t, n, n1) + One; + n = n1; + } + // x / (1 - x^2 / a) = x / (1 - d) = x + x * d / (1 - d) for d = x^2 / a + // int d = mulDivRound(x2, 1 << 255, a - (x2 ~>> 12)); // d/(1-d) = x^2/(a-x^2) as fixed267 + // return x + (mulrshiftr256(x, d) ~>> 11); + return x + mulDivRound(x, x2, a - x2 ~/ 4096) ~/ 4096; +} + +/// fixed261 atanh(fixed261 x); +@pure +@inline_ref +fun atanh_f261(x: int, n: int): int { + return atanh_f261_inlined(x, n); +} + +/// returns (y, s) such that log(x) = y/2^257 + s*log(2) for positive integer x +/// (fixed257, int) log_aux(int x) +@pure +@inline_ref +fun log_aux_f257(x: int): (int, int) { + var s: int = log2_floor_p1(x); + x <<= 256 - s; + var t: int = -1 << 256; + if ((x >> 249) <= 90) { + t >>= 1; + s -= 1; + } + x += t; + var `2x`: int = 2 * x; + var y: int = lshift256divr(`2x`, (x >> 1) - t); + // y = `2x` - (mulrshiftr256(2x, y) ~>> 2); // this line could improve precision on very rare occasions + return (atanh_f258(y, 36), s); +} + +/// computes 33^m for small m +@pure +@inline +fun pow33(m: int): int { + var t: int = 1; + repeat (m) { + t *= 33; + } + return t; +} + +/// computes 33^m for small 0<=m<=22 +/// slightly faster than pow33() +@pure +@inline +fun pow33b(m: int): int { + var (mh: int, ml: int) = divMod(m, 5); + var t: int = 1; + repeat (ml) { + t *= 33; + } + repeat (mh) { + t *= 33 * 33 * 33 * 33 * 33; + } + return t; +} + +/// returns (s, q, y) such that log(x) = s*log(2) + q*log(33/32) + y/2^260 for positive integer x +/// (int, int, fixed260) log_auxx_f260(int x); +@pure +@inline_ref +fun log_auxx_f260(x: int): (int, int, int) { + var s: int = log2_floor_p1(x) - 1; + x <<= 255 - s; // rescale to 1 <= x < 2 as fixed255 + var t: int = 2873 << 244; // ~ (33/32)^11 ~ sqrt(2) as fixed255 + var x1: int = (x - t) >> 1; + var q: int = mulDivRound(x1, 65, x1 + t) + 11; // crude approximation to round(log(x)/log(33/32)) + // t = 1; repeat (q) { t *= 33; } // t:=33^q, 0<=q<=22 + t = pow33b(q); + t <<= (51 - q) * 5; // t:=(33/32)^q as fixed255, nearest power of 33/32 to x + x -= t; + var y: int = lshift256divr(x << 4, (x >> 1) + t); // y = (x-t)/(x+t) as fixed261 + y = atanh_f261(y, 18); // atanh((x-t)/(x+t)) as fixed261, or log(x/t) as fixed260 + return (s, q, y); +} + +/// returns (y, s) such that log(x) = y/2^256 + s*log(2) for positive integer x +/// this function is very precise (error less than 0.6 ulp) and consumes < 7k gas +/// may be used to implement specific fixed-point instances of log() and log2() +/// (fixed256, int) log_aux_f256(int x); +@pure +@inline_ref +fun log_aux_f256(x: int): (int, int) { + var (s, q, y) = log_auxx_f260(x); + var (yh, yl) = rshiftr4mod(y); // y ~/% 16 , but Tolk does not optimize this to RSHIFTR#MOD + // int Log33_32 = 3563114646320977386603103333812068872452913448227778071188132859183498739150; // log(33/32) as fixed256 + // int Log33_32_l = -3769; // log(33/32) = Log33_32 / 2^256 + Log33_32_l / 2^269 + yh += (yl * 512 + q * -3769) ~>> 13; // compensation, may be removed if slightly worse accuracy is acceptable + var Log33_32: int = 3563114646320977386603103333812068872452913448227778071188132859183498739150; // log(33/32) as fixed256 + return (yh + q * Log33_32, s); +} + +/// returns (y, s) such that log2(x) = y/2^256 + s for positive integer x +/// this function is very precise (error less than 0.6 ulp) and consumes < 7k gas +/// may be used to implement specific fixed-point instances of log() and log2() +/// (fixed256, int) log2_aux_f256(int x); +@pure +@inline_ref +fun log2_aux_f256(x: int): (int, int) { + var (s, q, y) = log_auxx_f260(x); + y = lshift256divr(y, log2_const_f256()) ~>> 4; // y/log(2) as fixed256 + var Log33_32: int = 5140487830366106860412008603913034462883915832139695448455767612111363481357; // log_2(33/32) as fixed256 + // Log33_32/2^256 happens to be a very precise approximation to log_2(33/32), no compensation required + return (y + q * Log33_32, s); +} + + +/// fixed248 log(fixed248 x) +@pure +@inline_ref +fun fixed248_log(x: int): int { + var (y, s) = log_aux_f256(x); + return mulDivRound(s - 248, log2_const_f256(), 1 << 8) + (y ~>> 8); + // return mulDivRound(s - 248, 80260960185991308862233904206310070533990667611589946606122867505419956976172, 1 << 8) + (y ~>> 8); +} + +/// fixed248 log2(fixed248 x) +@pure +@inline +fun fixed248_log2(x: int): int { + var (y, s) = log2_aux_f256(x); + return ((s - 248) << 248) + (y ~>> 8); +} + +/// computes x^y as exp(y*log(x)), x >= 0 +/// fixed248 pow(fixed248 x, fixed248 y); +@pure +@inline_ref +fun fixed248_pow(x: int, y: int): int { + if (!y) { + return 1 << 248; // x^0 = 1 + } + if (x <= 0) { + var bad: int = ((x | y) < 0) as int; + return 0 >> bad; // 0^y = 0 if x=0 and y>=0; "out of range" exception otherwise + } + var (l, s) = log2_aux_f256(x); + s -= 248; // log_2(x) = s+l, l is fixed256, 0<=l<1 + // compute (s+l)*y = q+ll + var (q1, r1) = mulrshiftr248mod(s, y); // muldivmodr(s, y, 1 << 248) + var (q2, r2) = mulrshift256mod(l, y); + r2 >>= 247; + var (q3, r3) = rshiftr248mod(q2); // divmodr(q2, 1 << 248); + var (q, ll) = rshiftr248mod(r1 + r3); + ll = 512 * ll + r2; + q += q1 + q3; + // now log_2(x^y) = y*log_2(x) = q + ll, ss integer, ll fixed257, -1/2<=ll<1/2 + var sq: int = q + 248; + if (sq <= 0) { + return -((sq == 0) as int); // underflow + } + y = expm1_f257(mulrshiftr256(ll, log2_const_f256())); + return (y ~>> (9 - q)) - (-1 << sq); +} + +/*-------------------- INVERSE TRIGONOMETRIC FUNCTIONS ------------------*/ + +/// number of terms n should be chosen as for atanh_f258() +/// fixed259 atan(fixed259 x); +@pure +@inline_ref +fun atan_f259(x: int, n: int): int { + var x2: int = mulrshiftr256(x, x); // x^2 as fixed262 + var One: int = (1 << 254); + var a: int = One ~/ n + (1 << 255); // a := 2 + 1/n as fixed254 + repeat (n - 1) { + // a := 1 + (1 + x^2 / a)(1 + 1/n) as fixed254 + var t: int = One + mulDivRound(x2, 1 << 246, a); // t := 1 + x^2 / a + var n1: int = n - 1; + a = mulDivRound(t, n, n1) + One; + n = n1; + } + // x / (1 + x^2 / a) = x / (1 + d) = x - x * d / (1 + d) = x - x * x^2/(a+x^2) for d = x^2 / a + return x - mulDivRound(x, x2, a + x2 ~/ 256) ~/ 256; +} + +/// number of terms n should be chosen as for atanh_f261() +/// fixed261 atan(fixed261 x); +@pure +@inline +fun atan_f261_inlined(x: int, n: int): int { + var x2: int = mulrshiftr256(x, x); // x^2 as fixed266 + var One: int = (1 << 254); + var a: int = One ~/ n + (1 << 255); // a := 2 + 1/n as fixed254 + repeat (n - 1) { + // a := 1 + (1 + x^2 / a)(1 + 1/n) as fixed254 + var t: int = One + mulDivRound(x2, 1 << 242, a); // t := 1 + x^2 / a + var n1: int = n - 1; + a = mulDivRound(t, n, n1) + One; + n = n1; + } + // x / (1 + x^2 / a) = x / (1 + d) = x - x * d / (1 + d) = x - x * x^2/(a+x^2) for d = x^2 / a + return x - mulDivRound(x, x2, a + x2 ~/ 4096) ~/ 4096; +} + +/// fixed261 atan(fixed261 x); +@pure +@inline_ref +fun atan_f261(x: int, n: int): int { + return atan_f261_inlined(x, n); +} + +/// computes (q,a,b) such that q is approximately atan(x)/atan(1/32) and a+b*I=(1+I/32)^q as fixed255 +/// then b/a=atan(q*atan(1/32)) exactly, and (a,b) is almost a unit vector pointing in the direction of (1,x) +/// must have |x|<1.1, x is fixed24 +/// (int, fixed255, fixed255) atan_aux_prereduce(fixed24 x); +@pure +@inline_ref +fun atan_aux_prereduce(x: int): (int, int, int) { + var xu: int = abs(x); + var tc: int = 7214596; // tan(13*theta) as fixed24 where theta=atan(1/32) + var t1: int = mulDivRound(xu - tc, 1 << 88, xu * tc + (1 << 48)); // tan(x') as fixed64 where x'=atan(x)-13*theta + // t1/(3+t1^2) * 3073/32 = x'/3 * 3072/32 = x' / (96/3072) = x' / theta + var q: int = mulDivRound(t1 * 3073, 1 << 59, t1 * t1 + (3 << 128)) + 13; // approximately round(atan(x)/theta), 0<=q<=25 + var (pa, pb) = (33226912, 5232641); // (32+I)^5 + var (qh, ql) = divMod(q, 5); + var (a, b) = (1 << (5 * (51 - q)), 0); // (1/32^q, 0) as fixed255 + repeat (ql) { + // a+b*I *= 32+I + b.stackMoveToTop(); + (a, b) = (sub_rev(b, 32 * a), a + 32 * b); // same as (32 * a - b, 32 * b + a), but more efficient + } + repeat (qh) { + // a+b*I *= (32+I)^5 = pa + pb*I + (a, b) = (a * pa - b * pb, a * pb + b * pa); + } + var xs: int = sign(x); + return (xs * q, a, xs * b); +} + +/// compute (q, z) such that atan(x)=q*atan(1/32)+z for -1 <= x < 1 +/// this function is reasonably accurate (error < 7 ulp with ulp = 2^-261), but it consumes >7k gas +/// this is sufficient for most purposes +/// (int, fixed261) atan_aux(fixed256 x) +@pure +@inline_ref +fun atan_aux_f256(x: int): (int, int) { + var (q, a, b) = atan_aux_prereduce(x ~>> 232); // convert x to fixed24 + // now b/a = tan(q*atan(1/32)) exactly, where q is near atan(x)/atan(1/32); so b/a is near x + // compute y = u/v = (a*x-b)/(a+b*x) as fixed261 ; then |y|<0.0167 = 1.07/64 and atan(x)=atan(y)+q*atan(1/32) + var (u, ul) = mulrshiftr256mod(a, x); + u = (ul ~>> 250) + ((u - b) << 6); // |u| < 1/32, convert fixed255 -> fixed261 + var v: int = a + mulrshiftr256(b, x); // v is scalar product of (a,b) and (1,x), it is approximately in [1..sqrt(2)] as fixed255 + var y: int = mulDivRound(u, 1 << 255, v); // y = u/v as fixed261 + var z: int = atan_f261_inlined(y, 18); // z = atan(x)-q*atan(1/32) + return (q, z); +} + +/// compute (q, z) such that atan(x)=q*atan(1/32)+z for -1 <= x < 1 +/// this function is very accurate (error < 2 ulp), but it consumes >7k gas +/// in most cases, faster function atan_aux_f256() should be used +/// (int, fixed261) atan_auxx(fixed256 x) +@pure +@inline_ref +fun atan_auxx_f256(x: int): (int, int) { + var (q, a, b) = atan_aux_prereduce(x ~>> 232); // convert x to fixed24 + // now b/a = tan(q*atan(1/32)) exactly, where q is near atan(x)/atan(1/32); so b/a is near x + // compute y = (a*x-b)/(a+b*x) as fixed261 ; then |y|<0.0167 = 1.07/64 and atan(x)=atan(y)+q*atan(1/32) + // use sort of double precision arithmetic for this + var (u, ul) = mulrshiftr256mod(a, x); + ul /= 2; + u -= b; // |u| < 1/32 as fixed255 + var (v, vl) = mulrshiftr256mod(b, x); + vl /= 2; + v += a; // v is scalar product of (a,b) and (1,x), it is approximately in [1..sqrt(2)] as fixed255 + // y = (u + ul*eps) / (v + vl*eps) = u/v + (ul - vl * u/v)/v * eps where eps=1/2^255 + var (y, r) = lshift255divmodr(u, v); // y = u/v as fixed255 + var yl: int = mulDivRound(ul + r, 1 << 255, v) - mulDivRound(vl, y, v); // y/2^255 + yl/2^510 represent u/v + y = (yl ~>> 249) + (y << 6); // convert y to fixed261 + var z: int = atan_f261_inlined(y, 18); // z = atan(x)-q*atan(1/32) + return (q, z); +} + +/// consumes ~ 8k gas +/// fixed255 atan(fixed255 x); +@pure +@inline_ref +fun atan_f255(x: int): int { + var s: int = (x ~>> 256); + x.stackMoveToTop(); + if (s) { + x = lshift256divr(-1 << 255, x); // x:=-1/x as fixed256 + } else { + x *= 2; // convert to fixed256 + } + var (q, z) = atan_aux_f256(x); + // now atan(x) = z + q*atan(1/32) + s*(Pi/2), z is fixed261 + var (Pi_h, Pi_l) = Pi_xconst_f254(); // Pi/2 as fixed255 + fixed383 + var (qh, ql) = mulrshiftr6mod(q, Atan1_32_f261()); + return qh + s * Pi_h + (z + ql + mulDivRound(s, Pi_l, 1 << 122)) ~/ 64; +} + +/// computes atan(x) for -1 <= x < 1 only +/// fixed256 atan_small(fixed256 x); +@pure +@inline_ref +fun atan_f256_small(x: int): int { + var (q, z) = atan_aux_f256(x); + // now atan(x) = z + q*atan(1/32), z is fixed261 + var (qh, ql) = mulrshiftr5mod(q, Atan1_32_f261()); + return qh + (z + ql) ~/ 32; +} + +/// fixed255 asin(fixed255 x); +@pure +@inline_ref +fun asin_f255(x: int): int { + var a: int = fixed255_One - fixed255_sqr(x); // a:=1-x^2 + if (!a) { + return sign(x) * Pi_const_f254(); // Pi/2 or -Pi/2 + } + var y: int = fixed255_sqrt(a); // sqrt(1-x^2) + var t: int = -lshift256divr(x, (-1 << 255) - y); // t = x/(1+sqrt(1-x^2)) avoiding overflow + return atan_f256_small(t); // asin(x)=2*atan(t) +} + +/// fixed254 acos(fixed255 x); +@pure +@inline_ref +fun acos_f255(x: int): int { + var Pi: int = Pi_const_f254(); + if (x == (-1 << 255)) { + return Pi; // acos(-1) = Pi + } + Pi /= 2; + var y: int = fixed255_sqrt(fixed255_One - fixed255_sqr(x)); // sqrt(1-x^2) + var t: int = lshift256divr(x, (-1 << 255) - y); // t = -x/(1+sqrt(1-x^2)) avoiding overflow + return Pi + atan_f256_small(t) ~/ 2; // acos(x)=Pi/2 + 2*atan(t) +} + +/// consumes ~ 10k gas +/// fixed248 asin(fixed248 x) +@pure +@inline +fun fixed248_asin(x: int): int { + return asin_f255(x << 7) ~>> 7; +} + +/// consumes ~ 10k gas +/// fixed248 acos(fixed248 x) +@pure +@inline +fun fixed248_acos(x: int): int { + return acos_f255(x << 7) ~>> 6; +} + +/// consumes ~ 7500 gas +/// fixed248 atan(fixed248 x); +@pure +@inline_ref +fun fixed248_atan(x: int): int { + var s: int = (x ~>> 249); + x.stackMoveToTop(); + if (s) { + s = sign(s); + x = lshift256divr(-1 << 248, x); // x:=-1/x as fixed256 + } else { + x <<= 8; // convert to fixed256 + } + var (q, z) = atan_aux_f256(x); + // now atan(x) = z + q*atan(1/32) + s*(Pi/2), z is fixed261 + return (z ~/ 64 + s * Pi_const_f254() + mulDivRound(q, Atan1_32_f261(), 64)) ~/ 128; // compute in fixed255, then convert +} + +/// fixed248 acot(fixed248 x); +@pure +@inline_ref +fun fixed248_acot(x: int): int { + var s: int = (x ~>> 249); + x.stackMoveToTop(); + if (s) { + x = lshift256divr(-1 << 248, x); // x:=-1/x as fixed256 + s = 0; + } else { + x <<= 8; // convert to fixed256 + s = sign(x); + } + var (q, z) = atan_aux_f256(x); + // now acot(x) = - z - q*atan(1/32) + s*(Pi/2), z is fixed261 + return (s * Pi_const_f254() - z ~/ 64 - mulDivRound(q, Atan1_32_f261(), 64)) ~/ 128; // compute in fixed255, then convert +} + +/*-------------------- PSEUDO-RANDOM NUMBERS ------------------*/ + +/// random number with standard normal distribution N(0,1) +/// generated by Kinderman--Monahan ratio method modified by J.Leva +/// spends ~ 2k..3k gas on average +/// fixed252 nrand(); +@inline_ref +fun nrand_f252(): int { + var (x, s, t, A, B, r0) = (nan(), 29483 << 236, -3167 << 239, 12845, 16693, 9043); + // 4/sqrt(e*Pi) = 1.369 loop iterations on average + do { + var (u, v) = (random() / 16 + 1, mulDivRound(random() - (1 << 255), 7027, 1 << 16)); // fixed252; 7027=ceil(sqrt(8/e)*2^12) + var va: int = abs(v); + var (u1, v1) = (u - s, va - t); // (u - 29483/2^16, abs(v) + 3167/2^13) as fixed252 + // Q := u1^2 + v1 * (A*v1 - B*u1) as fixed252 where A=12845/2^16, B=16693/2^16 + var Q: int = mulDivRound(u1, u1, 1 << 252) + mulDivRound(v1, mulDivRound(v1, A, 1 << 16) - mulDivRound(u1, B, 1 << 16), 1 << 252); + // must have 9043 / 2^15 < Q < 9125 / 2^15, otherwise accept if smaller, reject if larger + var Qd: int = (Q >> 237) - r0; + if ((Qd < 9125 - 9043) & (va / u < 16)) { + x = mulDivRound(v, 1 << 252, u); // x:=v/u as fixed252; reject immediately if |v/u| >= 16 + if (Qd >= 0) { + // immediately accept if Qd < 0 + // rarely taken branch - 0.012 times per call on average + // check condition v^2 < -4*u^2*log(u), or equivalent condition u < exp(-x^2/4) for x=v/u + var xx: int = mulrshiftr256(x, x) ~/ 4; // x^2/4 as fixed248 + var ex: int = fixed248_exp(-xx) * 16; // exp(-x^2/4) as fixed252 + if (u > ex) { + x = nan(); // condition false, reject + } + } + } + } while (!(~ is_nan(x))); + return x; +} + +/// generates a random number approximately distributed according to the standard normal distribution +/// much faster than nrand_f252(), should be suitable for most purposes when only several random numbers are needed +/// fixed252 nrand_fast(); +@inline_ref +fun nrand_fast_f252(): int { + var t: int = -3 << 253; // -6. as fixed252 + repeat (12) { + t += random() / 16; // add together 12 uniformly random numbers + } + return t; +} + +/// random number uniformly distributed in [0..1) +/// fixed248 random(); +@inline +fun fixed248_random(): int { + return random() >> 8; +} + +/// random number with standard normal distribution +/// fixed248 nrand(); +@inline +fun fixed248_nrand(): int { + return nrand_f252() ~>> 4; +} + +/// generates a random number approximately distributed according to the standard normal distribution +/// fixed248 nrand_fast(); +@inline +fun fixed248_nrand_fast(): int { + return nrand_fast_f252() ~>> 4; +} + +@pure +fun tset(mutate self: tuple, idx: int, value: X): void + asm(self value idx) "SETINDEXVAR"; + +// computes 1-acos(x)/Pi by a very simple, extremely slow (~70k gas) and imprecise method +// fixed256 acos_prepare_slow(fixed255 x); +@inline +fun acos_prepare_slow_f255(x: int): int { + x -= (x == 0) as int; + var t: int = 1; + repeat (255) { + t = t * sign(x) * 2 + 1; // decode Gray code (sign(x_0), sign(x_1), ...) + x = (-1 << 255) - mulDivRound(x, - x, 1 << 254); // iterate x := 2*x^2 - 1 = cos(2*acos(x)) + } + return abs(t); +} + +// extremely slow (~70k gas) and somewhat imprecise (very imprecise when x is small), for testing only +// fixed254 acos_slow(fixed255 x); +@inline_ref +fun acos_slow_f255(x: int): int { + var t: int = acos_prepare_slow_f255(x); + return - mulrshiftr256(t + (-1<<256), Pi_const_f254()); +} + +// fixed255 asin_slow(fixed255 x); +@inline_ref +fun asin_slow_f255(x: int): int { + var t: int = acos_prepare_slow_f255(abs(x)) % (1 << 255); + return mulDivRound(t, Pi_const_f254(), 1 << 255) * sign(x); +} + +@inline_ref +fun test_nrand(n: int): tuple { + var t: tuple = createEmptyTuple(); + repeat (255) { + t.tuplePush(0); + } + repeat (n) { + var x: int = fixed248_nrand(); + var bucket: int = (abs(x) >> 243); // 255 buckets starting from x=0, each 1/32 wide + var at_bucket: int = t.tupleAt(bucket); + t.tset(bucket, at_bucket + 1); + } + return t; +} + +@method_id(10000) +fun geom_mean_test(x: int, y: int): int { + return geom_mean(x, y); +} +@method_id(10001) +fun tan_f260_test(x: int): int { + return tan_f260(x); +} +@method_id(10002) +fun sincosm1_f259_test(x: int): (int, int) { + return sincosm1_f259(x); +} +@method_id(10003) +fun sincosn_f256_test(x: int, y: int): (int, int) { + return sincosn_f256(x, y); +} +@method_id(10004) +fun sincosm1_f256_test(x: int): (int, int) { + return sincosm1_f256(x); +} +@method_id(10005) +fun tan_aux_f256_test(x: int): (int, int) { + return tan_aux_f256(x); +} +@method_id(10006) +fun fixed248_tan_test(x: int): int { + return fixed248_tan(x); +} +/* + (int) atanh_alt_f258_test(x) method_id(10007) { + return atanh_alt_f258(x); + } +*/ +@method_id(10008) +fun atanh_f258_test(x:int, y:int): int { + return atanh_f258(x, y); +} +@method_id(10009) +fun atanh_f261_test(x:int, y:int): int { + return atanh_f261(x, y); +} + +@method_id(10010) +fun log2_aux_f256_test(x:int): (int, int) { + return log2_aux_f256(x); +} +@method_id(10011) +fun log_aux_f256_test(x:int): (int, int) { + return log_aux_f256(x); +} +@method_id(10012) +fun fixed248_pow_test(x:int, y:int): int { + return fixed248_pow(x, y); +} +@method_id(10013) +fun exp_log_div(x:int, y:int): int { + return fixed248_exp(fixed248_log(x << 248) ~/ y); +} +@method_id(10014) +fun fixed248_log_test(x:int): int { + return fixed248_log(x); +} +@method_id(10015) +fun log_aux_f257_test(x:int): (int,int) { + return log_aux_f257(x); +} +@method_id(10016) +fun fixed248_sincos_test(x:int): (int,int) { + return fixed248_sincos(x); +} +@method_id(10017) +fun fixed248_exp_test(x:int): int { + return fixed248_exp(x); +} +@method_id(10018) +fun fixed248_exp2_test(x:int): int { + return fixed248_exp2(x); +} +@method_id(10019) +fun expm1_f257_test(x:int): int { + return expm1_f257(x); +} +@method_id(10020) +fun atan_f255_test(x:int): int { + return atan_f255(x); +} +@method_id(10021) +fun atan_f259_test(x:int, n:int): int { + return atan_f259(x, n); +} +@method_id(10022) +fun atan_aux_f256_test(x:int): (int, int) { + return atan_aux_f256(x); +} +@method_id(10023) +fun asin_f255_test(x:int): int { + return asin_f255(x); +} +@method_id(10024) +fun asin_slow_f255_test(x:int): int { + return asin_slow_f255(x); +} +@method_id(10025) +fun acos_f255_test(x:int): int { + return acos_f255(x); +} +@method_id(10026) +fun acos_slow_f255_test(x:int): int { + return acos_slow_f255(x); +} +@method_id(10027) +fun fixed248_atan_test(x:int): int { + return fixed248_atan(x); +} +@method_id(10028) +fun fixed248_acot_test(x:int): int { + return fixed248_acot(x); +} + +fun main() { + var One: int = 1; + // repeat(76 / 4) { One *= 10000; } + var sqrt2: int = geom_mean(One, 2 * One); + var sqrt3: int = geom_mean(One, 3 * One); + // return geom_mean(-1 - (-1 << 256), -1 - (-1 << 256)); + // return geom_mean(-1 - (-1 << 256), -2 - (-1 << 256)); + // return geom_mean(-1 - (-1 << 256), 1 << 255); + // return (sqrt2, geom_mean(sqrt2, One)); // (sqrt(2), 2^(1/4)) + // return (sqrt3, geom_mean(sqrt3, One)); // (sqrt(3), 3^(1/4)) + // return geom_mean(3 << 254, 1 << 254); + // return geom_mean(3, 5); + // return tan_f260(115641670674223639132965820642403718536242645001775371762318060545014644837101 - 1); + // return tan_f260(15 << 252); // tan(15/256) * 2^260 + // return sincosm1_f259(1 << 255); // (sin,1-cos)(1/16) * 2^259 + // return sincosm1_f259(115641670674223639132965820642403718536242645001775371762318060545014644837101 - 1); + // return sincosm1_f256((1 << 255) - 1 + (1 << 255)); // (sin,1-cos)(1-2^(-256)) + // return sincosm1_f256(Pi_const_f254()); // (sin,1-cos)(Pi/4) + // return sincosn_f256(Pi_const_f254(), 0); // (sin,-cos)(Pi/4) + // return sincosn_f256((1 << 255) + 1, 0); // (sin,-cos)(1/2+1/2^256) + // return sincosn_f256(1 << 254, 0); + // return sincosn_f256(stackMoveToTop(15) << 252, 0); // (sin,-cos)(15/16) + // return sincosm1_f256(stackMoveToTop(15) << 252); // (sin,1-cos)(15/16) + // return sincosn_f256(60628596148627720713372490462954977108898896221398738326462025186323149077698, 0); // (sin,-cos)(Pi/6) + // return sincosm1_f256(60628596148627720713372490462954977108898896221398738326462025186323149077698); // (sin,1-cos)(Pi/6) + // return tan_aux_f256(1899 << 245); // (p,q) such that p/q=tan(1899/2048) + // return fixed248_tan(11 << 248); // tan(11) + // return atanh_alt_f258(1 << 252); // atanh(1/64) * 2^258 + // return atanh_f258(1 << 252, 18); // atanh(1/64) * 2^258 + // return atanh_f261(mulDivRound(64, 1 << 255, 55), 18); // atanh(1/55) * 2^261 + // return log2_aux_f256(1 << 255); + // return log2_aux_f256(-1 - (-1 << 256)); // log2(2-1/2^255))*2^256 ~ 2^256 - 1.43 + // return log_aux_f256(-1 - (-1 << 256)); + // return log_aux_f256(3); // log(3/2)*2^256 + // return fixed248_pow(3 << 248, 3 << 248); // 3^3 + // return fixed248_exp(fixed248_log(5 << 248) ~/ 7); // exp(log(5)/7) = 5^(1/7) + // return fixed248_log(Pi_const_f254() ~>> 6); // log(Pi) + // return atanh_alt_f258(1 << 255); // atanh(1/8) * 2^258 + // return atanh_f258(1 << 255, 37); // atanh(1/8) * 2^258 + // return atanh_f258(81877371507464127617551201542979628307507432471243237061821853600756754782485, 36); // atanh(sqrt(2)/8) * 2^258 + // return log_aux_f257(Pi_const_f254()); // log(Pi/4) + // return log_aux_f257(3 << 254); // log(3) + // return atanh_alt_f258(81877371507464127617551201542979628307507432471243237061821853600756754782485); // atanh(sqrt(2)/8) * 2^258 + // return fixed248_sincos(Pi_const_f254() ~/ (64 * 3)); // (sin,cos)(Pi/3) + // return fixed248_exp(3 << 248); // exp(3)*2^248 + // return fixed248_exp2((1 << 248) ~/ 5); // 2^(1/5)*2^248 + // return fixed248_pow(3 << 248, -3 << 247); // 3^(-1.5) + // return fixed248_pow(10 << 248, -70 << 248); // 10^(-70) + // return fixed248_pow(fixed248_Pi_const(), stackMoveToTop(3) << 248); // Pi^3 ~ 31.006, computed more precisely + // return fixed248_pow(fixed248_Pi_const(), fixed248_Pi_const()); // Pi^Pi, more precisely + // return fixed248_exp(fixed248_log(fixed248_Pi_const()) * 3); // Pi^3 ~ 31.006 + // return fixed248_exp(mulDivRound(fixed248_log(fixed248_Pi_const()), fixed248_Pi_const(), 1 << 248)); // Pi^Pi + // return fixed248_sin(fixed248_log(fixed248_exp(fixed248_Pi_const()))); // sin(log(e^Pi)) + // return expm1_f257(1 << 255); // (exp(1/4)-1)*2^256 + // return expm1_f257(-1 << 256); // (exp(-1/2)-1)*2^256 (argument out of range, will overflow) + // return expm1_f257(log2_const_f256()); // (exp(log(2)/2)-1)*2^256 + // return expm1_f257(- log2_const_f256()); // (exp(-log(2)/2)-1)*2^256 + // return tanh_f258(log2_const_f256(), 17); // tanh(log(2)/4)*2^258 + // return atan_f255(0xa0 << 247); + // return atan_f259(1 << 255, 26); // atan(1/16) + // return atan_f259(stackMoveToTop(2273) << 244, 26); // atan(2273/2^15) + // return atan_aux_f256(0xa0 << 248); + // return atan_aux_f256(-1 - (-1 << 256)); + // return atan_aux_f256(-1 << 256); + // return atan_aux_f256(1); // atan(1/2^256)*2^261 = 32 + //return fixed248_nrand(); + // return test_nrand(100000); + var One2: int = 1 << 255; + // return asin_f255(One); + // return asin_f255(-2 * One ~/ -3); + var arg: int = mulDivRound(12, One2, 17); // 12/17 + // return [ asin_slow_f255(arg), asin_f255(arg) ]; + // return [ acos_slow_f255(arg), acos_f255(arg) ]; + // return 4 * atan_f255(One ~/ 5) - atan_f255(One ~/ 239); // 4 * atan(1/5) - atan(1/239) = Pi/4 as fixed255 + var One3: int = 1 << 248; + // return fixed248_atan(One) ~/ 5); // atan(1/5) + // return fixed248_acot(One ~/ 239); // atan(1/5) +} + +/** + method_id | in | out +@testcase | 10000 | -1-(-1<<256) -1-(-1<<256) | 115792089237316195423570985008687907853269984665640564039457584007913129639935 +@testcase | 10000 | -1-(-1<<256) -2-(-1<<256) | 115792089237316195423570985008687907853269984665640564039457584007913129639934 +@testcase | 10000 | -1-(-1<<256) 1<<255 | 81877371507464127617551201542979628307507432471243237061821853600756754782485 +@testcase | 10000 | 1 2 | 1 +@testcase | 10000 | 1 3 | 2 +@testcase | 10000 | 3<<254 1<<254 | 50139445418395255283694704271811692336355250894665672355503583528635147053497 +@testcase | 10000 | 3 5 | 4 +@testcase | 10001 | 115641670674223639132965820642403718536242645001775371762318060545014644837101-1 | 115792089237316195423570985008687907853269984665640564039457584007913129639935 +@testcase | 10001 | 15<<252 | 108679485937549714997960660780289583146059954551846264494610741505469565211201 + +@testcase | 10002 | 1<<255 | 57858359242454268843682786479537198006144860419130642837770554273561536355094 28938600351875109040123440645416448095273333920390487381363947585666516031269 +@testcase | 10002 | 90942894222941581070058735694432465663348344332098107489693037779484723616546 | 90796875678616203090520439851979829600860326752181983760731669850687818036503 71369031536005973567205947792557760023823761636922618688720973932041901854510 +@testcase | 10002 | 115641670674223639132965820642403718536242645001775371762318060545014644837100 | 115341536360906404779899502576747487978354537254490211650198994186870666100480 115341536360906404779899502576747487978354537254490211650198994186870666100479 +@testcase | 10003 | 90942894222941581070058735694432465663348344332098107489693037779484723616546 0 | 81877371507464127617551201542979628307507432471243237061821853600756754782485 -81877371507464127617551201542979628307507432471243237061821853600756754782486 +@testcase | 10003 | (1<<255)+1 0 | 55513684748706254392157395574451324146997108788015526773113170656738693667657 -101617118319522600545601981648807607350213579319835970884288805016705398675944 +@testcase | 10003 | 1<<254 0 | 28647421327665059059430596260119789787021370826354543144805343654507971817712 -112192393597863122712065585177748900737784171216163716639418346853706594800924 +@testcase | 10003 | 15<<252 0 | 93337815620236900315136494926097782162348358704087992554326802765553037216157 -68526346066204767396483080633934170508153877799043171682610011603005473885083 +@testcase | 10004 | 15<<252 | 93337815620236900315136494926097782162348358704087992554326802765553037216158 94531486342222856054175808749507474690232213733194784713695144809815311509707 +@testcase | 10003 | 60628596148627720713372490462954977108898896221398738326462025186323149077698 0 | 57896044618658097711785492504343953926634992332820282019728792003956564819968 -100278890836790510567389408543623384672710501789331344711007167057270294106993 +@testcase | 10004 | 60628596148627720713372490462954977108898896221398738326462025186323149077698 | 57896044618658097711785492504343953926634992332820282019728792003956564819968 31026396801051369712363152930129046361118965752618438656900833901285671065886 +@testcase | 10005 | 1899<<245 | -115784979074977116522606932816046735344768048129666123117516779696532375620701 -86847621900007587791673148476644866514014227467564880140262768165345715058771 +@testcase | 10006 | 11<<248 | -102200470999497240398685962406597118965525125432278008915850368651878945159221 +@testcase | 10008 | 1<<252 18 | 7237594612640731814076778712183932891481921212865048737772958953246047977071 +@testcase! | 10009 | 64*(1<<255)//55 18 | 67377367986958444187782963285047188951340314639925508148698906136973510008513 +@testcase | 10010 | 1<<255 | 0 255 +@testcase | 10011 | -1-(-1<<256) | 80260960185991308862233904206310070533990667611589946606122867505419956976171 255 +@testcase | 10012 | 3<<248 3<<248 | 12212446911748192486079752325135052781399568695204278238536542063334587891712 +@testcase | 10013 | 5 7 | 569235245303856216139605450142923208167703167128528666640203654338408315932 +@testcase | 10014 | 1420982722233462204219667745225507275989817880189032929526453715304448806508 | 517776035526939558040896860590142614178014859368681705591403663865964112176 +@testcase | 10008 | 1<<255 37 | 58200445412255555045265806996802932280233368707362818578692888102488340124094 +@testcase | 10008 | 81877371507464127617551201542979628307507432471243237061821853600756754782485 36 | 82746618329032515754939514227666784789465120373484337368014239356561508382845 +@testcase | 10015 | 90942894222941581070058735694432465663348344332098107489693037779484723616546 | -55942510554172181731996424203087263676819062449594753161692794122306202470292 256 +@testcase | 10015 | 3<<254 | -66622616410625360568738677407433830899150908037353507097280251369610028875158 256 +@testcase | 10016 | 90942894222941581070058735694432465663348344332098107489693037779484723616546//(64*3) | 391714417331212931903864877123528846377775397614575565277371746317462086355 226156424291633194186662080095093570025917938800079226639565593765455331328 +@testcase | 10017 | 3<<248 | 9084946421051389814103830025729847734065792062362132089390904679466687950835 +@testcase | 10018 | (1<<248)//5 | 519571025111621076330285524602776985448579272766894385941850747946908706857 +@testcase | 10012 | 3<<248 -3<<247 | 87047648295825095978636639360784188083950088358794570061638165848324908079 +@testcase | 10012 | 10<<248 -70<<248 | 45231 +@testcase | 10012 | 1420982722233462204219667745225507275989817880189032929526453715304448806508 3<<248 | 14024537329227316173680050897643053638073167245065581681188087336877135047241 +@testcase | 10012 | 1420982722233462204219667745225507275989817880189032929526453715304448806508 1420982722233462204219667745225507275989817880189032929526453715304448806508 | 16492303277433924047657446877966346821161732581471802839855102123372676002295 +@testcase | 10019 | 1<<255 | 65775792789545756849501669218806308540691279864498696756901136302101823231959 +@testcase | 10019 | -1<<255 | -51226238931640701466578648374135745377468902266335737558089915608594425303282 + +@testcase | 10020 | 160<<247 | 32340690885082755723307749066376646841771751777398167772823878380310576779097 +@testcase | 10021 | 1<<255 26 | 57820835337111819566482910321201859268121322500887685881159030272507322418551 +@testcase | 10021 | 2273<<244 26 | 64153929153128256059565403901040178355488584937372975321150754259394300105908 +@testcase | 10022 | 160<<248 | 18 -13775317617017974742132028403521581424991093186766868001115299479309514610238 +@testcase | 10022 | -1-(-1<<256) | 25 16312150880916231694896252427912541090503675654570543195394548083530005073282 +@testcase | 10022 | -1<<256 | -25 -16312150880916231694896252427912541090503675654570543195394548083530005073298 +@testcase | 10022 | 1 | 0 32 + +@testcase | 10023 | 1<<255 | 90942894222941581070058735694432465663348344332098107489693037779484723616546 +@testcase | 10023 | (1-(1<<255))//-3 | 19675212872822715586637341573564384553677006914302429002469838095945333339604 +@testcase | 10023 | 12*(1<<255)//17 | 45371280744427205854111943101074857545572584208710061167826656461897302968384 +@testcase | 10024 | 12*(1<<255)//17 | 45371280744427205854111943101074857545572584208710061167826656461897302968387 +@testcase | 10025 | 12*(1<<255)//17 | 22785806739257187607973396296678804058887880061694023160933190658793710324081 +@testcase | 10026 | 12*(1<<255)//17 | 22785806739257187607973396296678804058887880061694023160933190658793710324080 + +@testcase | 10027 | (1<<248)//5 | 89284547973388213553327350968415123522888028497458323165947767504203347189 +@testcase | 10028 | (1<<248)//239 | 708598849781543798951441405045469962900811296151941404481049216461523216127 +*/ diff --git a/tolk-tester/tests/try-func.tolk b/tolk-tester/tests/try-func.tolk new file mode 100644 index 00000000..4ac86d96 --- /dev/null +++ b/tolk-tester/tests/try-func.tolk @@ -0,0 +1,323 @@ +fun foo(x: int): int { + try { + if (x == 7) { + throw 44; + } + return x; + } catch { + return 2; + } +} + +@inline +fun foo_inline(x: int): int { + try { + assert(!(x == 7)) throw 44; + return x; + } catch { + return 2; + } +} + +@inline_ref +fun foo_inlineref(x: int): int { + try { + if (x == 7) { throw (44, 2); } + return x; + } catch (_, arg) { + return arg as int; + } +} + +@method_id(101) +fun test(x: int, y: int, z: int): int { + y = foo(y); + return x * 100 + y * 10 + z; +} + +@method_id(102) +fun test_inline(x: int, y: int, z: int): int { + y = foo_inline(y); + return x * 100 + y * 10 + z; +} + +@method_id(103) +fun test_inlineref(x: int, y: int, z: int): int { + y = foo_inlineref(y); + return x * 100 + y * 10 + z; +} + +@inline +fun foo_inline_big( + x1: int, x2: int, x3: int, x4: int, x5: int, x6: int, x7: int, x8: int, x9: int, x10: int, + x11: int, x12: int, x13: int, x14: int, x15: int, x16: int, x17: int, x18: int, x19: int, x20: int +): int { + try { + if (x1 == 7) { + throw 44; + } + return x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + x10 + x11 + x12 + x13 + x14 + x15 + x16 + x17 + x18 + x19 + x20; + } catch { + return 1; + } +} + +@method_id(104) +fun test_inline_big(x: int, y: int, z: int): int { + y = foo_inline_big( + y, y + 1, y + 2, y + 3, y + 4, y + 5, y + 6, y + 7, y + 8, y + 9, + y + 10, y + 11, y + 12, y + 13, y + 14, y + 15, y + 16, y + 17, y + 18, y + 19); + return x * 1000000 + y * 1000 + z; +} + +fun foo_big( + x1: int, x2: int, x3: int, x4: int, x5: int, x6: int, x7: int, x8: int, x9: int, x10: int, + x11: int, x12: int, x13: int, x14: int, x15: int, x16: int, x17: int, x18: int, x19: int, x20: int +): int { + try { + if (x1 == 7) { + throw (44, 1); + } + return x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + x10 + x11 + x12 + x13 + x14 + x15 + x16 + x17 + x18 + x19 + x20; + } catch (code, arg) { + return arg as int; + } +} + +@method_id(105) +fun test_big(x: int, y: int, z: int): int { + y = foo_big( + y, y + 1, y + 2, y + 3, y + 4, y + 5, y + 6, y + 7, y + 8, y + 9, + y + 10, y + 11, y + 12, y + 13, y + 14, y + 15, y + 16, y + 17, y + 18, y + 19); + return x * 1000000 + y * 1000 + z; +} + +@method_id(106) +fun test_catch_into_same(x: int): int { + var code = x; + try { + assert(x <= 10, 44); + } catch(code) { + return code; + } + return code; +} + + +@method_id(107) +fun test_catch_into_same_2(x: int): int { + var code = x; + try { + if (x > 10) { + throw 44; + } + } catch(code) { + } + return code; +} + +global after046: int; + +// this bug existed in FunC and is fixed in v0.4.6 +fun bug_046_internal(op: int) { + if (op == 1) { + return; + } else if (op == 2) { + return; + } else { + throw 1; + } +} + +fun bug_046_called() { + after046 = 0; + try { + bug_046_internal(1337); + after046 = 1; // shouldn't be called + } catch(n) { + return; + } + return; +} + +@method_id(108) +fun bug_046_entrypoint() { + bug_046_called(); + return after046; +} + +global g_reg: int; + +@method_id(109) +fun test109(): (int, int) { + var l_reg = 10; + g_reg = 10; + try { + // note, that regardless of assignment, an exception RESTORES them to previous (to 10) + // it's very unexpected, but is considered to be a TVM feature, not a bug + g_reg = 999; + l_reg = 999; + bug_046_internal(999); // throws + } catch { + } + // returns (10,10) because of an exception, see a comment above + return (g_reg, l_reg); +} + +fun alwaysThrow123(): never { + throw 123; +} + +fun alwaysThrowX(x: int): never { + if (x > 10) { throw (x, beginCell()); } + else { throw (x, null); } +} + +fun anotherNever(throw123: bool): never { + if (throw123) { alwaysThrow123(); } + alwaysThrowX(456); +} + +fun testCodegen1(x: int) { + if (x > 10) { + throw 123; + anotherNever(true); // unreachable, will be dropped + } + else if (x < 10) { + throw x; + return -123; // unreachable, will be dropped + } + return 0; +} + +fun testCodegen2(x: int) { + if (x > 10) { + alwaysThrow123(); + anotherNever(true); // unreachable, will be dropped + } + else if (x < 10) { + anotherNever(false); + return -123; // unreachable, will be dropped + } + return 0; +} + +@method_id(110) +fun test110(b: bool) { + try { + if (b == true) { testCodegen1(100); } + testCodegen1(5); + return -1; + } catch (ex) { + return ex; + } +} + +@method_id(111) +fun test111(b: bool) { + try { + if (b == true) { testCodegen2(100); } + testCodegen2(5); + return -1; + } catch (ex) { + return ex; + } +} + +fun mySetCode(newCode: slice): void + asm "SETCODE"; + +fun testCodegen3(numberId: int, paramVal: cell) { + if (numberId == -1000) { + var cs = paramVal.beginParse(); + mySetCode(cs); + throw 0; + } + paramVal.beginParse(); +} + +fun main() { +} + +/** + method_id | in | out +@testcase | 101 | 1 2 3 | 123 +@testcase | 101 | 3 8 9 | 389 +@testcase | 101 | 3 7 9 | 329 +@testcase | 102 | 1 2 3 | 123 +@testcase | 102 | 3 8 9 | 389 +@testcase | 102 | 3 7 9 | 329 +@testcase | 103 | 1 2 3 | 123 +@testcase | 103 | 3 8 9 | 389 +@testcase | 103 | 3 7 9 | 329 +@testcase | 104 | 4 8 9 | 4350009 +@testcase | 104 | 4 7 9 | 4001009 +@testcase | 105 | 4 8 9 | 4350009 +@testcase | 105 | 4 7 9 | 4001009 +@testcase | 106 | 5 | 5 +@testcase | 106 | 20 | 44 +@testcase | 107 | 5 | 5 +@testcase | 107 | 20 | 20 +@testcase | 108 | | 0 +@testcase | 109 | | 10 10 +@testcase | 110 | -1 | 123 +@testcase | 110 | 0 | 5 +@testcase | 111 | -1 | 123 +@testcase | 111 | 0 | 456 + +@code_hash 57361460846265694653029920796509802052573595128418810728101968091567195330515 + +@fif_codegen +""" + testCodegen1 PROC:<{ + // x + DUP // x x + 10 GTINT // x '2 + IFJMP:<{ // x + 123 THROW + }> // x + DUP // x x + 10 LESSINT // x '6 + IFJMP:<{ // x + THROWANY + }> // x + DROP // + 0 PUSHINT // '8=0 + }> +""" + +@fif_codegen +""" + testCodegen2 PROC:<{ + // x + DUP // x x + 10 GTINT // x '2 + IFJMP:<{ // x + DROP // + alwaysThrow123 CALLDICT + }> // x + 10 LESSINT // '5 + IFJMP:<{ // + FALSE // '6 + anotherNever CALLDICT + }> // + 0 PUSHINT // '8=0 + }> +""" + +@fif_codegen +""" + testCodegen3 PROC:<{ + // numberId paramVal + SWAP + -1000 PUSHINT // paramVal numberId '2=-1000 + EQUAL // paramVal '3 + IFJMP:<{ // paramVal + CTOS // cs + SETCODE + 0 THROW + }> // paramVal + DROP // + }> +""" +*/ diff --git a/tolk-tester/tests/unbalanced_ret.tolk b/tolk-tester/tests/unbalanced_ret.tolk new file mode 100644 index 00000000..6cf42643 --- /dev/null +++ b/tolk-tester/tests/unbalanced_ret.tolk @@ -0,0 +1,17 @@ +fun main(x: int): (int, int) { + var y: int = 5; + if (x < 0) { + x *= 2; + y += 1; + if (x == -10) { + return (111, 0); + } + } + return (x + 1, y); +} +/** + method_id | in | out +@testcase | 0 | 10 | 11 5 +@testcase | 0 | -5 | 111 0 +@testcase | 0 | -4 | -7 6 +*/ diff --git a/tolk-tester/tests/unbalanced_ret_inline.tolk b/tolk-tester/tests/unbalanced_ret_inline.tolk new file mode 100644 index 00000000..4e24fbd8 --- /dev/null +++ b/tolk-tester/tests/unbalanced_ret_inline.tolk @@ -0,0 +1,19 @@ +@inline +fun foo(x: int): int { + if (x < 0) { + x *= 2; + if (x == -10) { + return 111; + } + } + return x + 1; +} +fun main(x: int): int { + return foo(x) * 10; +} +/** + method_id | in | out +@testcase | 0 | 10 | 110 +@testcase | 0 | -5 | 1110 +@testcase | 0 | -4 | -70 +*/ diff --git a/tolk-tester/tests/unbalanced_ret_loops.tolk b/tolk-tester/tests/unbalanced_ret_loops.tolk new file mode 100644 index 00000000..292b48da --- /dev/null +++ b/tolk-tester/tests/unbalanced_ret_loops.tolk @@ -0,0 +1,68 @@ +fun main() { } + +@method_id(1) +fun foo_repeat(x: int): int { + repeat(10) { + x += 10; + if (x >= 100) { + return x; + } + } + return -1; +} + +@method_id(2) +fun foo_while(x: int): int { + var i: int = 0; + while (i < 10) { + x += 10; + if (x >= 100) { + return x; + } + i += 1; + } + return -1; +} + +@method_id(3) +fun foo_until(x: int): int { + var i: int = 0; + do { + x += 10; + if (x >= 100) { + return x; + } + i += 1; + } while (i < 10); + return -1; +} + +@method_id(4) +fun test4(x: int): (int, bool) { + var s = 0; + var reached = false; + do { + x = x - 1; + s = s + 1; + if (x < 10) { + reached = true; + } + } while (!reached); + return (s, reached); +} + +/** + method_id | in | out +@testcase | 1 | 40 | 100 +@testcase | 1 | 33 | 103 +@testcase | 1 | -5 | -1 +@testcase | 2 | 40 | 100 +@testcase | 2 | 33 | 103 +@testcase | 2 | -5 | -1 +@testcase | 3 | 40 | 100 +@testcase | 3 | 33 | 103 +@testcase | 3 | -5 | -1 +@testcase | 4 | 18 | 9 -1 + +@code_hash 12359153928622198176298534554187062238616102949658930329300859312625793323482 +*/ diff --git a/tolk-tester/tests/unbalanced_ret_nested.tolk b/tolk-tester/tests/unbalanced_ret_nested.tolk new file mode 100644 index 00000000..05e60924 --- /dev/null +++ b/tolk-tester/tests/unbalanced_ret_nested.tolk @@ -0,0 +1,37 @@ +fun foo(y: int): int { + if (y < 0) { + y *= 2; + if (y == -10) { + return 111; + } + } + return y + 1; +} +fun bar(x: int, y: int): (int, int) { + if (x < 0) { + y = foo(y); + x *= 2; + if (x == -10) { + return (111, y); + } + } + return (x + 1, y); +} +fun main(x: int, y: int): (int, int) { + (x, y) = bar(x, y); + return (x, y * 10); +} +/** + method_id | in | out +@testcase | 0 | 3 3 | 4 30 +@testcase | 0 | 3 -5 | 4 -50 +@testcase | 0 | 3 -4 | 4 -40 +@testcase | 0 | -5 3 | 111 40 +@testcase | 0 | -5 -5 | 111 1110 +@testcase | 0 | -5 -4 | 111 -70 +@testcase | 0 | -4 3 | -7 40 +@testcase | 0 | -4 -5 | -7 1110 +@testcase | 0 | -4 -4 | -7 -70 + +@code_hash 68625253347714662162648433047986779710161195298061582217368558479961252943991 +*/ diff --git a/tolk-tester/tests/unreachable-1.tolk b/tolk-tester/tests/unreachable-1.tolk new file mode 100644 index 00000000..5b3cb1b0 --- /dev/null +++ b/tolk-tester/tests/unreachable-1.tolk @@ -0,0 +1,14 @@ +fun main(x: int) { + if (x) { + x = 10;;;;; + return x;;; + x = 20; + } + return -1; +} + +/** +@testcase | 0 | 1 | 10 +@stderr warning: unreachable code +@stderr x = 20; + */ diff --git a/tolk-tester/tests/unreachable-2.tolk b/tolk-tester/tests/unreachable-2.tolk new file mode 100644 index 00000000..aeadd8c6 --- /dev/null +++ b/tolk-tester/tests/unreachable-2.tolk @@ -0,0 +1,22 @@ +fun main(x: int) { + if (x) { + if (x > 10) { + return 1; // throw 1; + } else if (true) { + return -1; + } else { + return 2; // throw 2; + } + } else { + {{return 1;} + x = 30;} + } + assert(false, 10); +} + +/** +@testcase | 0 | 1 | -1 +@stderr warning: unreachable code +@stderr assert(false, 10) +@stderr x = 30 + */ diff --git a/tolk-tester/tests/unreachable-3.tolk b/tolk-tester/tests/unreachable-3.tolk new file mode 100644 index 00000000..fab21fd2 --- /dev/null +++ b/tolk-tester/tests/unreachable-3.tolk @@ -0,0 +1,22 @@ +fun main(x: int?) { + if (x != null && x == null) { + return 1 + 2; + } + if (x == null) { + return -1; + } + if (x != null) { + return -2; + } + return 3 + 4; +} + +/** +@testcase | 0 | 5 | -2 +@testcase | 0 | null | -1 + +@stderr warning: variable `x` of type `int` is always not null +@stderr if (x != null) +@stderr warning: unreachable code +@stderr return 3 + 4 + */ diff --git a/tolk-tester/tests/unreachable-4.tolk b/tolk-tester/tests/unreachable-4.tolk new file mode 100644 index 00000000..6b25b3d9 --- /dev/null +++ b/tolk-tester/tests/unreachable-4.tolk @@ -0,0 +1,24 @@ +fun alwaysThrows(): never { + throw 456; +} + +fun testUnreachable(x: int) { + if (x) { throw 123; } + else { alwaysThrows(); } + return 1; +} + +fun main() { + try { + testUnreachable(100); + throw 80; + } catch (excNo) { + return excNo; + } +} + +/** +@testcase | 0 | | 123 +@stderr warning: unreachable code +@stderr return 1; + */ diff --git a/tolk-tester/tests/use-before-declare.tolk b/tolk-tester/tests/use-before-declare.tolk new file mode 100644 index 00000000..2a0e0e7f --- /dev/null +++ b/tolk-tester/tests/use-before-declare.tolk @@ -0,0 +1,49 @@ +fun main(): int { + var c: cell = my_begin_cell().storeInt(demo_10, 32).my_end_cell(); + var cs: slice = my_begin_parse(c); + var ten: int = cs.loadInt(32); + return 1 + demo1(ten) + demo_var; +} + +@pure +fun my_begin_cell(): builder +asm "NEWC"; +@pure +fun my_end_cell(b: builder): cell +asm "ENDC"; +@pure +fun my_begin_parse(c: cell): slice +asm "CTOS"; + +fun demo1(v: int): int { + demo_var = 23; + return v; +} + +global demo_var: int; +const demo_10: int = 10; + +fun test1(): int { + var demo_var: int = demo_10; + var demo_slice: int = demo_20; + if (demo_var > 0) { + var demo_var: tuple? = null; + var demo_slice: tuple? = null; + } + return demo_var + demo_slice; +} + +global demo_slice: slice; +const demo_20: int = 20; + +/** +@testcase | 0 | | 34 + +@fif_codegen +""" + test1 PROC:<{ + // + 30 PUSHINT // '10 + }> +""" + */ diff --git a/tolk-tester/tests/var-apply.tolk b/tolk-tester/tests/var-apply.tolk new file mode 100644 index 00000000..d189430f --- /dev/null +++ b/tolk-tester/tests/var-apply.tolk @@ -0,0 +1,192 @@ +fun getBeginCell() { + return beginCell; +} + +fun getBeginParse() { + return beginParse; +} + +@method_id(101) +fun testVarApply1() { + var (_, f_end_cell) = (0, endCell); + var b: builder = (getBeginCell())().storeInt(1, 32); + b.storeInt(2, 32); + var s = (getBeginParse())(f_end_cell(b)); + return (s.loadInt(32), s.loadInt(32)); +} + +@inline +fun my_throw_always() { + throw 1000; +} + +@inline +fun get_raiser() { + return my_throw_always; +} + +@method_id(102) +fun testVarApplyWithoutSavingResult() { + try { + var raiser = get_raiser(); + raiser(); // `some_var()` is always impure, the compiler has no considerations about its runtime value + return 0; + } catch (code) { + return code; + } +} + +@inline +fun sum(a: int, b: int) { + assert(a + b < 24, 1000); + return a + b; +} + +@inline +fun mul(a: int, b: int) { + assert(a * b < 24, 1001); + return a * b; +} + +fun demo_handler(op: int, query_id: int, a: int, b: int): int { + if (op == 0xF2) { + val func = query_id % 2 == 0 ? sum : mul; + val result = func(a, b); + return 0; // result not used, we test that func is nevertheless called + } + if (op == 0xF4) { + val func = query_id % 2 == 0 ? sum : mul; + val result = func(a, b); + return result; + } + return -1; +} + +@method_id(103) +fun testVarApplyInTernary() { + var t: tuple = createEmptyTuple(); + try { + t.tuplePush(demo_handler(0xF2, 122, 100, 200)); + } catch(code) { + t.tuplePush(code); + } + try { + t.tuplePush(demo_handler(0xF4, 122, 100, 200)); + } catch(code) { + t.tuplePush(code); + } + try { + t.tuplePush(demo_handler(0xF2, 122, 10, 10)); + } catch(code) { + t.tuplePush(code); + } + try { + t.tuplePush(demo_handler(0xF2, 123, 10, 10)); + } catch(code) { + t.tuplePush(code); + } + return t; +} + +fun always_throw2(x: int) { + throw 239 + x; +} + +global global_f: int -> void; + +@method_id(104) +fun testGlobalVarApply() { + try { + global_f = always_throw2; + global_f(1); + return 0; + } catch (code) { + return code; + } +} + +@method_id(105) +fun testVarApply2() { + var creator = createEmptyTuple; + var t = creator(); + t.tuplePush(1); + var sizer = t.tupleSize; + return sizer(t); +} + +fun getTupleLastGetter(): (tuple) -> X { + return tupleLast; +} + +@method_id(106) +fun testVarApply3() { + var t = createEmptyTuple(); + t.tuplePush(1); + t.tuplePush([2]); + var getIntAt = t.tupleAt; + var getTupleFirstInt = createEmptyTuple().tupleFirst; + var getTupleLastTuple = getTupleLastGetter(); + return (getIntAt(t, 0), getTupleFirstInt(t), getTupleLastTuple(t), getTupleLastGetter()(t)); +} + +@method_id(107) +fun testIndexedAccessApply() { + var functions1 = (beginCell, endCell); + var functions2 = [beginParse]; + var b = functions1.0().storeInt(1, 16); + b.storeInt(1, 16); + return functions2.0(functions1.1(b)).loadInt(32); +} + +fun getNullable4(): int? { return 4; } +fun myBeginCell(): builder? asm "NEWC"; + +@method_id(108) +fun testCallingNotNull() { + var n4: () -> int? = getNullable4; + var creator: (() -> builder?)? = myBeginCell; + var end2: [int, (builder -> cell)?] = [0, endCell]; + var c: cell = end2.1!((creator!()!)!.storeInt(getNullable4()!, 32)); + return c.beginParse().loadInt(32); +} + +fun sumOfTensorIfNotNull(t: (int, int)?) { + if (t == null) { return 0; } + return t!.0 + t!.1; +} + +@method_id(109) +fun testTypeTransitionOfVarCall() { + var summer = sumOfTensorIfNotNull; + var hh1 = [1, null]; + var tt1 = (3, 4); + return (summer(null), summer((1,2)), summer(hh1.1), summer(tt1)); +} + +fun makeTensor(x1: int, x2: int, x3: int, x4: int, x5: int) { + return (x1, x2, x3, x4, x5); +} + +fun eq(x: T): T { return x; } + +@method_id(110) +fun testVarsModificationInsideVarCall(x: int) { + var cb = makeTensor; + return x > 3 ? cb(x, x += 5, eq(x *= x), x, eq(x)) : null; +} + +fun main() {} + +/** +@testcase | 101 | | 1 2 +@testcase | 102 | | 1000 +@testcase | 103 | | [ 1000 1000 0 1001 ] +@testcase | 104 | | 240 +@testcase | 105 | | 1 +@testcase | 106 | | 1 1 [ 2 ] [ 2 ] +@testcase | 107 | | 65537 +@testcase | 108 | | 4 +@testcase | 109 | | 0 3 0 7 +@testcase | 110 | 5 | 5 10 100 100 100 -1 +@testcase | 110 | 0 | (null) (null) (null) (null) (null) 0 + */ diff --git a/tolk-tester/tests/w1.tolk b/tolk-tester/tests/w1.tolk new file mode 100644 index 00000000..eb06bec6 --- /dev/null +++ b/tolk-tester/tests/w1.tolk @@ -0,0 +1,14 @@ +fun main(id: int): (int, int) { + if (id > 0) { + if (id > 10) { + return (2 * id, 3 * id); + } + } + return (5, 6); +} +/** + method_id | in | out +@testcase | 0 | 0 | 5 6 +@testcase | 0 | 4 | 5 6 +@testcase | 0 | 11 | 22 33 +*/ diff --git a/tolk-tester/tests/w2.tolk b/tolk-tester/tests/w2.tolk new file mode 100644 index 00000000..728b18d3 --- /dev/null +++ b/tolk-tester/tests/w2.tolk @@ -0,0 +1,34 @@ +@method_id(101) +fun test1(cs: slice) { + return cs.loadUint(8)+cs.loadUint(8)+cs.loadUint(8)+cs.loadUint(8); +} + +@method_id(102) +fun test2(cs: slice) { + var (x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, + x11, x12, x13, x14, x15, x16, x17, x18, x19) = f(cs); + return x0 + x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + + x10+ x11+ x12+ x13+ x14+ x15+ x16+ x17+ x18+ x19; +} + +fun main(cs: slice) { + return (cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8)); +} + +fun f(cs: slice) { + return (cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), + cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), + cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), + cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), + cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8)); +} + + +/** + method_id | in | out +@testcase | 102 | x{000102030405060708090a0b0c0d0e0f10111213} | 190 +@testcase | 101 | x{000102030405060708090a0b0c0d0e0f10111213} | 6 +@testcase | 0 | x{000102030405060708090a0b0c0d0e0f10111213} | 0 1 2 3 + +@code_hash 58474889199998908444151060994149070836199913191952040273624197630531731101157 +*/ diff --git a/tolk-tester/tests/w6.tolk b/tolk-tester/tests/w6.tolk new file mode 100644 index 00000000..489ffa8c --- /dev/null +++ b/tolk-tester/tests/w6.tolk @@ -0,0 +1,19 @@ +fun main(x: int): int { + var i: int = 0; + // int f = false; + do { + i = i + 1; + if (i > 5) { + return 1; + } + var f: bool = (i * i == 64); + } while (!f); + return -1; +} + +/** + method_id | in | out +@testcase | 0 | 0 | 1 + +@code_hash 36599880583276393028571473830850694081778552118303309411432666239740650614479 +*/ diff --git a/tolk-tester/tests/w7.tolk b/tolk-tester/tests/w7.tolk new file mode 100644 index 00000000..3d68c775 --- /dev/null +++ b/tolk-tester/tests/w7.tolk @@ -0,0 +1,26 @@ +@method_id(1) +fun test(y: int): int { + var x: int = 1; + if (y > 0) { + return 1; + } + return x > 0 ? -1 : 0; +} + +@method_id(2) +fun f(y: int): int { + if (y > 0) { + return 1; + } + return 2; +} + +fun main() { } + +/** + method_id | in | out +@testcase | 1 | 10 | 1 +@testcase | 1 | -5 | -1 +@testcase | 2 | 10 | 1 +@testcase | 2 | -5 | 2 +*/ diff --git a/tolk-tester/tests/w9.tolk b/tolk-tester/tests/w9.tolk new file mode 100644 index 00000000..b88dc736 --- /dev/null +++ b/tolk-tester/tests/w9.tolk @@ -0,0 +1,14 @@ +fun main(s: int) { + var (z, t) = (17, s); + while (z > 0) { + t = s; + z -= 1; + } + return ~ t; +} + +/** + method_id | in | out +@testcase | 0 | 1 | -2 +@testcase | 0 | 5 | -6 +*/ diff --git a/tolk-tester/tests/warnings-1.tolk b/tolk-tester/tests/warnings-1.tolk new file mode 100644 index 00000000..040057d1 --- /dev/null +++ b/tolk-tester/tests/warnings-1.tolk @@ -0,0 +1,28 @@ +fun getNullableInt(): int? { return null; } + +fun main() { + var c: int? = 6; + __expect_type(c, "int"); + if (c == null) {} + + var d: int? = c; + if (((d)) != null && tupleSize(createEmptyTuple())) {} + + var e: int? = getNullableInt(); + if (e != null) { + return true; + } + __expect_type(e, "null"); + null == e; + + return null != null; +} + +/** +@testcase | 0 | | 0 + +@stderr warning: variable `c` of type `int` is always not null, this condition is always false +@stderr warning: variable `d` of type `int` is always not null, this condition is always true +@stderr warning: variable `e` is always null, this condition is always true +@stderr warning: expression is always null, this condition is always false + */ diff --git a/tolk-tester/tests/warnings-2.tolk b/tolk-tester/tests/warnings-2.tolk new file mode 100644 index 00000000..57ecb21a --- /dev/null +++ b/tolk-tester/tests/warnings-2.tolk @@ -0,0 +1,26 @@ +fun main() { + var (a, b, c, d, e) = (1, beginCell(), beginCell().endCell().beginParse(), [1], true as bool?); + + var alwaysInt = a != null ? 1 : null; + __expect_type(alwaysInt, "int"); + + if (!(c == null)) { + if (10 < 3) { assert(b == null, 100); } + } + while (d == null || false) {} + + return e! != null; +} + +/** +@testcase | 0 | | -1 + +@stderr warning: variable `a` of type `int` is always not null, this condition is always true +@stderr warning: condition of ternary operator is always true +@stderr warning: variable `c` of type `slice` is always not null, this condition is always false +@stderr warning: condition of `if` is always true +@stderr warning: variable `b` of type `builder` is always not null, this condition is always false +@stderr warning: condition of `assert` is always false +@stderr warning: condition of `while` is always false +@stderr warning: expression of type `bool` is always not null, this condition is always true + */ diff --git a/tolk-tester/tolk-tester.js b/tolk-tester/tolk-tester.js new file mode 100644 index 00000000..c7e71021 --- /dev/null +++ b/tolk-tester/tolk-tester.js @@ -0,0 +1,533 @@ +// Usage: `node tolk-tester.js tests_dir` OR `node tolk-tester.js test_file.tolk` +// from current dir, providing some env (see getenv() calls). +// This is a JS version of tolk-tester.py to test Tolk compiled to WASM. +// Don't forget to keep it identical to Python version! + +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const child_process = require('child_process'); + +function print(...args) { + console.log(...args) +} + +/** @return {string} */ +function getenv(name, def = null) { + if (name in process.env) + return process.env[name] + if (def === null) { + print(`Environment variable ${name} is not set`) + process.exit(1) + } + return def +} + +const TOLKFIFTLIB_MODULE = getenv('TOLKFIFTLIB_MODULE') +const TOLKFIFTLIB_WASM = getenv('TOLKFIFTLIB_WASM') +const FIFT_EXECUTABLE = getenv('FIFT_EXECUTABLE') +const FIFT_LIBS_FOLDER = getenv('FIFTPATH') // this env is needed for fift to work properly +const STDLIB_FOLDER = __dirname + '/../crypto/smartcont/tolk-stdlib' +const TMP_DIR = os.tmpdir() + +class CmdLineOptions { + constructor(/**string[]*/ argv) { + if (argv.length !== 3) { + print("Usage: node tolk-tester.js tests_dir OR node tolk-tester.js test_file.tolk") + process.exit(1) + } + if (!fs.existsSync(argv[2])) { + print(`Input '${argv[2]}' doesn't exist`) + process.exit(1) + } + + if (fs.lstatSync(argv[2]).isDirectory()) { + this.tests_dir = argv[2] + this.test_file = null + } else { + this.tests_dir = path.dirname(argv[2]) + this.test_file = argv[2] + } + } + + /** @return {string[]} */ + find_tests() { + if (this.test_file) // an option to run (debug) a single test + return [this.test_file] + + let tests = fs.readdirSync(this.tests_dir).filter(f => f.endsWith('.tolk') || f.endsWith('.ton')) + tests.sort() + return tests.map(f => path.join(this.tests_dir, f)) + } +} + + +class ParseInputError extends Error { +} + +class TolkCompilationFailedError extends Error { + constructor(/**string*/ message, /**string*/ stderr) { + super(message); + this.stderr = stderr + } +} + +class TolkCompilationSucceededError extends Error { +} + +class FiftExecutionFailedError extends Error { + constructor(/**string*/ message, /**string*/ stderr) { + super(message); + this.stderr = stderr + } +} + +class CompareOutputError extends Error { + constructor(/**string*/ message, /**string*/ output) { + super(message); + this.output = output + } +} + +class CompareFifCodegenError extends Error { +} + +class CompareCodeHashError extends Error { +} + + +/* + * In positive tests, there are several testcases "input X should produce output Y". + */ +class TolkTestCaseInputOutput { + static reJustNumber = /^[-+]?\d+$/ + static reMathExpr = /^[0x123456789()+\-*/<>]*$/ + + constructor(/**string*/ method_id_str, /**string*/ input_str, /**string*/ output_str) { + let processed_inputs = [] + for (let in_arg of input_str.split(' ')) { + if (in_arg.length === 0) + continue + else if (in_arg.startsWith("x{") || TolkTestCaseInputOutput.reJustNumber.test(in_arg)) + processed_inputs.push(in_arg) + else if (TolkTestCaseInputOutput.reMathExpr.test(in_arg)) + // replace "3<<254" with "3n<<254n" (big number) before eval (in Python we don't need this) + processed_inputs.push(eval(in_arg.replace('//', '/').replace(/(\d)($|\D)/gmi, '$1n$2')).toString()) + else if (in_arg === "null") + processed_inputs.push("null") + else + throw new ParseInputError(`'${in_arg}' can't be evaluated`) + } + + this.method_id = +method_id_str + this.input = processed_inputs.join(' ') + this.expected_output = output_str + } + + check(/**string[]*/ stdout_lines, /**number*/ line_idx) { + if (stdout_lines[line_idx] !== this.expected_output) + throw new CompareOutputError(`error on case #${line_idx + 1} (${this.method_id} | ${this.input}): expected '${this.expected_output}', found '${stdout_lines[line_idx]}'`, stdout_lines.join("\n")) + } +} + +/* + * @stderr checks, when compilation fails, that stderr (compilation error) is expected. + * If it's multiline, all lines must be present in specified order. + */ +class TolkTestCaseStderr { + constructor(/**string[]*/ stderr_pattern, /**boolean*/ avoid) { + this.stderr_pattern = stderr_pattern + this.avoid = avoid + } + + check(/**string*/ stderr) { + const line_match = this.find_pattern_in_stderr(stderr.split(/\n/)) + if (line_match === -1 && !this.avoid) + throw new CompareOutputError("pattern not found in stderr:\n" + + this.stderr_pattern.map(x => " " + x).join("\n"), stderr) + else if (line_match !== -1 && this.avoid) + throw new CompareOutputError(`pattern found (line ${line_match + 1}), but not expected to be:\n` + + this.stderr_pattern.map(x => " " + x).join("\n"), stderr) + } + + find_pattern_in_stderr(/**string[]*/ stderr) { + for (let line_start = 0; line_start < stderr.length; ++line_start) + if (this.try_match_pattern(0, stderr, line_start)) + return line_start + return -1 + } + + try_match_pattern(/**number*/ pattern_offset, /**string[]*/ stderr, /**number*/ offset) { + if (pattern_offset >= this.stderr_pattern.length) + return true + if (offset >= stderr.length) + return false + + const line_pattern = this.stderr_pattern[pattern_offset] + const line_output = stderr[offset] + return line_output.includes(line_pattern) && this.try_match_pattern(pattern_offset + 1, stderr, offset + 1) + } +} + +/* + * @fif_codegen checks that contents of compiled.fif matches the expected pattern. + * @fif_codegen_avoid checks that is does not match the pattern. + * See comments in run_tests.py. + */ +class TolkTestCaseFifCodegen { + constructor(/**string[]*/ fif_pattern, /**boolean*/ avoid) { + /** @type {string[]} */ + this.fif_pattern = fif_pattern.map(s => s.trim()) + this.avoid = avoid + } + + check(/**string[]*/ fif_output) { + const line_match = this.find_pattern_in_fif_output(fif_output) + if (line_match === -1 && !this.avoid) + throw new CompareFifCodegenError("pattern not found:\n" + + this.fif_pattern.map(x => " " + x).join("\n")) + else if (line_match !== -1 && this.avoid) + throw new CompareFifCodegenError(`pattern found (line ${line_match + 1}), but not expected to be:\n` + + this.fif_pattern.map(x => " " + x).join("\n")) + } + + find_pattern_in_fif_output(/**string[]*/ fif_output) { + for (let line_start = 0; line_start < fif_output.length; ++line_start) + if (this.try_match_pattern(0, fif_output, line_start)) + return line_start + return -1 + } + + try_match_pattern(/**number*/ pattern_offset, /**string[]*/ fif_output, /**number*/ offset) { + if (pattern_offset >= this.fif_pattern.length) + return true + if (offset >= fif_output.length) + return false + const line_pattern = this.fif_pattern[pattern_offset] + const line_output = fif_output[offset] + + if (line_pattern !== "...") { + if (!TolkTestCaseFifCodegen.does_line_match(line_pattern, line_output)) + return false + return this.try_match_pattern(pattern_offset + 1, fif_output, offset + 1) + } + while (offset < fif_output.length) { + if (this.try_match_pattern(pattern_offset + 1, fif_output, offset)) + return true + offset = offset + 1 + } + return false + } + + static split_line_to_cmd_and_comment(/**string*/ trimmed_line) { + const pos = trimmed_line.indexOf("//") + if (pos === -1) + return [trimmed_line, null] + else + return [trimmed_line.substring(0, pos).trimEnd(), trimmed_line.substring(pos + 2).trimStart()] + } + + static does_line_match(/**string*/ line_pattern, /**string*/ line_output) { + const [cmd_pattern, comment_pattern] = TolkTestCaseFifCodegen.split_line_to_cmd_and_comment(line_pattern) + const [cmd_output, comment_output] = TolkTestCaseFifCodegen.split_line_to_cmd_and_comment(line_output.trim()) + return cmd_pattern === cmd_output && (comment_pattern === null || comment_pattern === comment_output) + } +} + +/* + * @code_hash checks that hash of compiled output.fif matches the provided value. + * It's used to "record" code boc hash and to check that it remains the same on compiler modifications. + * Being much less flexible than @fif_codegen, it nevertheless gives a guarantee of bytecode stability. + */ +class TolkTestCaseExpectedHash { + constructor(/**string*/ expected_hash) { + this.code_hash = expected_hash + } + + check(/**string*/ fif_code_hash) { + if (this.code_hash !== fif_code_hash) + throw new CompareCodeHashError(`expected ${this.code_hash}, actual ${fif_code_hash}`) + } +} + + +class TolkTestFile { + constructor(/**string*/ tolk_filename, /**string*/ artifacts_folder) { + this.line_idx = 0 + this.tolk_filename = tolk_filename + this.artifacts_folder = artifacts_folder + this.compilation_should_fail = false + /** @type {TolkTestCaseStderr[]} */ + this.stderr_includes = [] + /** @type {TolkTestCaseInputOutput[]} */ + this.input_output = [] + /** @type {TolkTestCaseFifCodegen[]} */ + this.fif_codegen = [] + /** @type {TolkTestCaseExpectedHash | null} */ + this.expected_hash = null + /** @type {string | null} */ + this.experimental_options = null + } + + parse_input_from_tolk_file() { + const lines = fs.readFileSync(this.tolk_filename, 'utf-8').split(/\r?\n/) + this.line_idx = 0 + + while (this.line_idx < lines.length) { + const line = lines[this.line_idx] + if (line.startsWith('@testcase')) { + let s = line.split("|").map(p => p.trim()) + if (s.length !== 4) + throw new ParseInputError(`incorrect format of @testcase: ${line}`) + this.input_output.push(new TolkTestCaseInputOutput(s[1], s[2], s[3])) + } else if (line.startsWith('@compilation_should_fail')) { + this.compilation_should_fail = true + } else if (line.startsWith('@stderr')) { + this.stderr_includes.push(new TolkTestCaseStderr(this.parse_string_value(lines), false)) + } else if (line.startsWith("@fif_codegen_avoid")) { + this.fif_codegen.push(new TolkTestCaseFifCodegen(this.parse_string_value(lines), true)) + } else if (line.startsWith("@fif_codegen")) { + this.fif_codegen.push(new TolkTestCaseFifCodegen(this.parse_string_value(lines), false)) + } else if (line.startsWith("@code_hash")) { + this.expected_hash = new TolkTestCaseExpectedHash(this.parse_string_value(lines, false)[0]) + } else if (line.startsWith("@experimental_options")) { + this.experimental_options = line.substring(22) + } + this.line_idx++ + } + + if (this.input_output.length === 0 && !this.compilation_should_fail) + throw new ParseInputError("no @testcase present") + if (this.input_output.length !== 0 && this.compilation_should_fail) + throw new ParseInputError("@testcase present, but compilation_should_fail") + } + + /** @return {string[]} */ + parse_string_value(/**string[]*/ lines, allow_multiline = true) { + // a tag must be followed by a space (single-line), e.g. '@stderr some text' + // or be a multi-line value, surrounded by """ + const line = lines[this.line_idx] + const pos_sp = line.indexOf(' ') + const is_multi_line = lines[this.line_idx + 1] === '"""' + const is_single_line = pos_sp !== -1 + if (!is_single_line && !is_multi_line) + throw new ParseInputError(`${line} value is empty (not followed by a string or a multiline """)`) + if (is_single_line && is_multi_line) + throw new ParseInputError(`${line.substring(0, pos_sp)} value is both single-line and followed by """`) + if (is_multi_line && !allow_multiline) + throw new ParseInputError(`${line} value should be single-line`); + + if (is_single_line) + return [line.substring(pos_sp + 1).trim()] + + this.line_idx += 2 + let s_multiline = [] + while (this.line_idx < lines.length && lines[this.line_idx] !== '"""') { + s_multiline.push(lines[this.line_idx]) + this.line_idx = this.line_idx + 1 + } + return s_multiline + } + + get_compiled_fif_filename() { + return this.artifacts_folder + "/compiled.fif" + } + + get_runner_fif_filename() { + return this.artifacts_folder + "/runner.fif" + } + + async run_and_check() { + const wasmModule = await compileWasm(TOLKFIFTLIB_MODULE, TOLKFIFTLIB_WASM) + let res = compileFile(wasmModule, this.tolk_filename, this.experimental_options) + let exit_code = res.status === 'ok' ? 0 : 1 + let stderr = res.message + let stdout = '' + + if (exit_code === 0 && this.compilation_should_fail) + throw new TolkCompilationSucceededError("compilation succeeded, but it should have failed") + + for (let should_include of this.stderr_includes) // @stderr is used to check errors and warnings + should_include.check(stderr) + + if (exit_code !== 0 && this.compilation_should_fail) + return + + if (exit_code !== 0 && !this.compilation_should_fail) + throw new TolkCompilationFailedError(`tolk exit_code = ${exit_code}`, stderr) + + fs.writeFileSync(this.get_compiled_fif_filename(), `"Asm.fif" include\n${res.fiftCode}`) + { + let runner = `"${this.get_compiled_fif_filename()}" include x.trim()).filter(s => s.length > 0) + let fif_code_hash = null + if (this.expected_hash !== null) { // then the last stdout line is a hash + fif_code_hash = stdout_lines[stdout_lines.length - 1] + stdout_lines = stdout_lines.slice(0, stdout_lines.length - 1) + } + + if (stdout_lines.length !== this.input_output.length) + throw new CompareOutputError(`unexpected number of fift output: ${stdout_lines.length} lines, but ${this.input_output.length} testcases`, stdout) + + for (let i = 0; i < stdout_lines.length; ++i) + this.input_output[i].check(stdout_lines, i) + + if (this.fif_codegen.length) { + const fif_output = fs.readFileSync(this.get_compiled_fif_filename(), 'utf-8').split(/\r?\n/) + for (let fif_codegen of this.fif_codegen) + fif_codegen.check(fif_output) + } + + if (this.expected_hash !== null) + this.expected_hash.check(fif_code_hash) + } +} + +async function run_all_tests(/**string[]*/ tests) { + for (let ti = 0; ti < tests.length; ++ti) { + let tolk_filename = tests[ti] + print(`Running test ${ti + 1}/${tests.length}: ${tolk_filename}`) + + let artifacts_folder = path.join(TMP_DIR, tolk_filename) + let testcase = new TolkTestFile(tolk_filename, artifacts_folder) + + try { + if (!fs.existsSync(artifacts_folder)) + fs.mkdirSync(artifacts_folder, {recursive: true}) + testcase.parse_input_from_tolk_file() + await testcase.run_and_check() + fs.rmSync(artifacts_folder, {recursive: true}) + + if (testcase.compilation_should_fail) + print(" OK, compilation failed as it should") + else + print(` OK, ${testcase.input_output.length} cases`) + } catch (e) { + if (e instanceof ParseInputError) { + print(` Error parsing input (cur line #${testcase.line_idx + 1}):`, e.message) + process.exit(2) + } else if (e instanceof TolkCompilationFailedError) { + print(" Error compiling tolk:", e.message) + print(" stderr:") + print(e.stderr.trimEnd()) + process.exit(2) + } else if (e instanceof FiftExecutionFailedError) { + print(" Error executing fift:", e.message) + print(" stderr:") + print(e.stderr.trimEnd()) + print(" compiled.fif at:", testcase.get_compiled_fif_filename()) + process.exit(2) + } else if (e instanceof CompareOutputError) { + print(" Mismatch in output:", e.message) + print(" Full output:") + print(e.output.trimEnd()) + print(" Was compiled to:", testcase.get_compiled_fif_filename()) + process.exit(2) + } else if (e instanceof CompareFifCodegenError) { + print(" Mismatch in fif codegen:", e.message) + print(" Was compiled to:", testcase.get_compiled_fif_filename()) + print(fs.readFileSync(testcase.get_compiled_fif_filename(), 'utf-8')) + process.exit(2) + } else if (e instanceof CompareCodeHashError) { + print(" Mismatch in code hash:", e.message) + print(" Was compiled to:", testcase.get_compiled_fif_filename()) + process.exit(2) + } + throw e + } + } +} + +const tests = new CmdLineOptions(process.argv).find_tests() +print(`Found ${tests.length} tests`) +run_all_tests(tests).then( + () => print(`Done, ${tests.length} tests`), + console.error +) + +// below are WASM helpers, which don't exist in Python version + +process.setMaxListeners(0); + +function copyToCString(mod, str) { + const len = mod.lengthBytesUTF8(str) + 1; + const ptr = mod._malloc(len); + mod.stringToUTF8(str, ptr, len); + return ptr; +} + +function copyToCStringPtr(mod, str, ptr) { + const allocated = copyToCString(mod, str); + mod.setValue(ptr, allocated, '*'); + return allocated; +} + +/** @return {string} */ +function copyFromCString(mod, ptr) { + return mod.UTF8ToString(ptr); +} + +/** @return {{status: string, message: string, fiftCode: string, codeBoc: string, codeHashHex: string}} */ +function compileFile(mod, filename, experimentalOptions) { + // see tolk-wasm.cpp: typedef void (*WasmFsReadCallback)(int, char const*, char**, char**) + const callbackPtr = mod.addFunction((kind, dataPtr, destContents, destError) => { + if (kind === 0) { // realpath + try { + let relative = copyFromCString(mod, dataPtr) + if (relative.startsWith('@stdlib/')) { + // import "@stdlib/filename" or import "@stdlib/filename.tolk" + relative = STDLIB_FOLDER + '/' + relative.substring(7) + if (!relative.endsWith('.tolk')) { + relative += '.tolk' + } + } + copyToCStringPtr(mod, fs.realpathSync(relative), destContents); + } catch (err) { + copyToCStringPtr(mod, 'cannot find file', destError); + } + } else if (kind === 1) { // read file + try { + const absolute = copyFromCString(mod, dataPtr) // already normalized (as returned above) + copyToCStringPtr(mod, fs.readFileSync(absolute).toString('utf-8'), destContents); + } catch (err) { + copyToCStringPtr(mod, err.message || err.toString(), destError); + } + } else { + copyToCStringPtr(mod, 'Unknown callback kind=' + kind, destError); + } + }, 'viiii'); + + const config = { + optimizationLevel: 2, + withStackComments: true, + experimentalOptions: experimentalOptions || undefined, + entrypointFileName: filename + }; + + const configPtr = copyToCString(mod, JSON.stringify(config)); + + const responsePtr = mod._tolk_compile(configPtr, callbackPtr); + + return JSON.parse(copyFromCString(mod, responsePtr)); +} + +async function compileWasm(tolkFiftLibJsFileName, tolkFiftLibWasmFileName) { + const wasmModule = require(tolkFiftLibJsFileName) + const wasmBinary = new Uint8Array(fs.readFileSync(tolkFiftLibWasmFileName)) + + return await wasmModule({ wasmBinary }) +} diff --git a/tolk-tester/tolk-tester.py b/tolk-tester/tolk-tester.py new file mode 100644 index 00000000..0b3c774c --- /dev/null +++ b/tolk-tester/tolk-tester.py @@ -0,0 +1,431 @@ +# Usage: `tolk-tester.py tests_dir` OR `tolk-tester.py test_file.tolk` +# from current dir, providing some env (see getenv() calls). +# Every .tolk file should provide /* testcase description in a comment */, consider tests/ folder. +# +# Tests for Tolk can be +# * positive (compiled to .fif, run with fift, compared output with the one expected) +# * negative (compilation fails, and it's expected; patterns in stderr can be specified) +# +# Note, that there is also tolk-tester.js to test Tolk compiled to WASM. +# Don't forget to keep it identical to Python version! + +import os +import os.path +import re +import shutil +import subprocess +import sys +import tempfile +from typing import List + + +def getenv(name, default=None): + if name in os.environ: + return os.environ[name] + if default is None: + print("Environment variable", name, "is not set", file=sys.stderr) + exit(1) + return default + + +TOLK_EXECUTABLE = getenv("TOLK_EXECUTABLE", "tolk") +FIFT_EXECUTABLE = getenv("FIFT_EXECUTABLE", "fift") +FIFT_LIBS_FOLDER = getenv("FIFTPATH") # this env is needed for fift to work properly +TMP_DIR = tempfile.mkdtemp() + + +class CmdLineOptions: + def __init__(self, argv: List[str]): + if len(argv) != 2: + print("Usage: tolk-tester.py tests_dir OR tolk-tester.py test_file.tolk", file=sys.stderr) + exit(1) + if not os.path.exists(argv[1]): + print("Input '%s' doesn't exist" % argv[1], file=sys.stderr) + exit(1) + + if os.path.isdir(argv[1]): + self.tests_dir = argv[1] + self.test_file = None + else: + self.tests_dir = os.path.dirname(argv[1]) + self.test_file = argv[1] + + def find_tests(self) -> List[str]: + if self.test_file is not None: # an option to run (debug) a single test + return [self.test_file] + + tests = [f for f in os.listdir(self.tests_dir) if f.endswith(".tolk") or f.endswith(".ton")] + tests.sort() + return [os.path.join(self.tests_dir, f) for f in tests] + + +class ParseInputError(Exception): + pass + + +class TolkCompilationFailedError(Exception): + def __init__(self, message: str, stderr: str): + super().__init__(message) + self.stderr = stderr + + +class TolkCompilationSucceededError(Exception): + pass + + +class FiftExecutionFailedError(Exception): + def __init__(self, message: str, stderr: str): + super().__init__(message) + self.stderr = stderr + + +class CompareOutputError(Exception): + def __init__(self, message: str, output: str): + super().__init__(message) + self.output = output + + +class CompareFifCodegenError(Exception): + pass + + +class CompareCodeHashError(Exception): + pass + + +class TolkTestCaseInputOutput: + """ + In positive tests, there are several testcases "input X should produce output Y". + They are written as a table: + @testcase | method_id | input (one or several) | output + """ + reJustNumber = re.compile(r"[-+]?\d+") + reMathExpr = re.compile(r"[0x123456789()+\-*/<>]+") + + def __init__(self, method_id_str: str, input_str: str, output_str: str): + processed_inputs = [] + for in_arg in input_str.split(" "): + if len(in_arg) == 0: + continue + elif in_arg.startswith("x{") or TolkTestCaseInputOutput.reJustNumber.fullmatch(in_arg): + processed_inputs.append(in_arg) + elif TolkTestCaseInputOutput.reMathExpr.fullmatch(in_arg): + processed_inputs.append(str(eval(in_arg))) + elif in_arg == "null": + processed_inputs.append("null") + else: + raise ParseInputError("'%s' can't be evaluated" % in_arg) + + self.method_id = int(method_id_str) + self.input = " ".join(processed_inputs) + self.expected_output = output_str + + def check(self, stdout_lines: List[str], line_idx: int): + if stdout_lines[line_idx] != self.expected_output: + raise CompareOutputError("error on case #%d (%d | %s): expected '%s', found '%s'" % (line_idx + 1, self.method_id, self.input, self.expected_output, stdout_lines[line_idx]), "\n".join(stdout_lines)) + + +class TolkTestCaseStderr: + """ + @stderr checks, when compilation fails, that stderr (compilation error) is expected. + If it's multiline, all lines must be present in specified order. + """ + + def __init__(self, stderr_pattern: List[str], avoid: bool): + self.stderr_pattern = stderr_pattern + self.avoid = avoid + + def check(self, stderr: str): + line_match = self.find_pattern_in_stderr(stderr.splitlines()) + if line_match == -1 and not self.avoid: + raise CompareOutputError("pattern not found in stderr:\n%s" % + "\n".join(map(lambda x: " " + x, self.stderr_pattern)), stderr) + elif line_match != -1 and self.avoid: + raise CompareOutputError("pattern found (line %d), but not expected to be:\n%s" % + (line_match + 1, "\n".join(map(lambda x: " " + x, self.stderr_pattern))), stderr) + + def find_pattern_in_stderr(self, stderr: List[str]) -> int: + for line_start in range(len(stderr)): + if self.try_match_pattern(0, stderr, line_start): + return line_start + return -1 + + def try_match_pattern(self, pattern_offset: int, stderr: List[str], offset: int) -> bool: + if pattern_offset >= len(self.stderr_pattern): + return True + if offset >= len(stderr): + return False + + line_pattern = self.stderr_pattern[pattern_offset] + line_output = stderr[offset] + return line_output.find(line_pattern) != -1 and self.try_match_pattern(pattern_offset + 1, stderr, offset + 1) + + +class TolkTestCaseFifCodegen: + """ + @fif_codegen checks that contents of compiled.fif matches the expected pattern. + @fif_codegen_avoid checks that is does not match the pattern. + The pattern is a multiline piece of fift code, optionally with "..." meaning "any lines here". + See tests/codegen_check_demo.tolk of how it looks. + A notable thing about indentations (spaces at line starts): + Taking them into account will complicate the code without reasonable profit, + that's why we just trim every string. + And one more word about //comments. Tolk inserts them into fift output. + If a line in the pattern contains a //comment, it's expected to be equal. + If a line does not, we just compare a command. + """ + + def __init__(self, fif_pattern: List[str], avoid: bool): + self.fif_pattern = [s.strip() for s in fif_pattern] + self.avoid = avoid + + def check(self, fif_output: List[str]): + line_match = self.find_pattern_in_fif_output(fif_output) + if line_match == -1 and not self.avoid: + raise CompareFifCodegenError("pattern not found:\n%s" % + "\n".join(map(lambda x: " " + x, self.fif_pattern))) + elif line_match != -1 and self.avoid: + raise CompareFifCodegenError("pattern found (line %d), but not expected to be:\n%s" % + (line_match + 1, "\n".join(map(lambda x: " " + x, self.fif_pattern)))) + + def find_pattern_in_fif_output(self, fif_output: List[str]) -> int: + for line_start in range(len(fif_output)): + if self.try_match_pattern(0, fif_output, line_start): + return line_start + return -1 + + def try_match_pattern(self, pattern_offset: int, fif_output: List[str], offset: int) -> bool: + if pattern_offset >= len(self.fif_pattern): + return True + if offset >= len(fif_output): + return False + line_pattern = self.fif_pattern[pattern_offset] + line_output = fif_output[offset] + + if line_pattern != "...": + if not TolkTestCaseFifCodegen.does_line_match(line_pattern, line_output): + return False + return self.try_match_pattern(pattern_offset + 1, fif_output, offset + 1) + while offset < len(fif_output): + if self.try_match_pattern(pattern_offset + 1, fif_output, offset): + return True + offset = offset + 1 + return False + + @staticmethod + def split_line_to_cmd_and_comment(trimmed_line: str) -> tuple: + pos = trimmed_line.find("//") + if pos == -1: + return trimmed_line, None + else: + return trimmed_line[:pos].rstrip(), trimmed_line[pos + 2:].lstrip() + + @staticmethod + def does_line_match(line_pattern: str, line_output: str) -> bool: + cmd_pattern, comment_pattern = TolkTestCaseFifCodegen.split_line_to_cmd_and_comment(line_pattern) + cmd_output, comment_output = TolkTestCaseFifCodegen.split_line_to_cmd_and_comment(line_output.strip()) + return cmd_pattern == cmd_output and (comment_pattern is None or comment_pattern == comment_output) + + +class TolkTestCaseExpectedHash: + """ + @code_hash checks that hash of compiled output.fif matches the provided value. + It's used to "record" code boc hash and to check that it remains the same on compiler modifications. + Being much less flexible than @fif_codegen, it nevertheless gives a guarantee of bytecode stability. + """ + + def __init__(self, expected_hash: str): + self.code_hash = expected_hash + + def check(self, fif_code_hash: str): + if self.code_hash != fif_code_hash: + raise CompareCodeHashError("expected %s, actual %s" % (self.code_hash, fif_code_hash)) + + +class TolkTestFile: + def __init__(self, tolk_filename: str, artifacts_folder: str): + self.line_idx = 0 + self.tolk_filename = tolk_filename + self.artifacts_folder = artifacts_folder + self.compilation_should_fail = False + self.stderr_includes: List[TolkTestCaseStderr] = [] + self.input_output: List[TolkTestCaseInputOutput] = [] + self.fif_codegen: List[TolkTestCaseFifCodegen] = [] + self.expected_hash: TolkTestCaseExpectedHash | None = None + self.experimental_options: str | None = None + + def parse_input_from_tolk_file(self): + with open(self.tolk_filename, "r") as fd: + lines = fd.read().splitlines() + self.line_idx = 0 + + while self.line_idx < len(lines): + line = lines[self.line_idx] + if line.startswith("@testcase"): + s = [x.strip() for x in line.split("|")] + if len(s) != 4: + raise ParseInputError("incorrect format of @testcase: %s" % line) + self.input_output.append(TolkTestCaseInputOutput(s[1], s[2], s[3])) + elif line.startswith("@compilation_should_fail"): + self.compilation_should_fail = True + elif line.startswith("@stderr"): + self.stderr_includes.append(TolkTestCaseStderr(self.parse_string_value(lines), False)) + elif line.startswith("@fif_codegen_avoid"): + self.fif_codegen.append(TolkTestCaseFifCodegen(self.parse_string_value(lines), True)) + elif line.startswith("@fif_codegen"): + self.fif_codegen.append(TolkTestCaseFifCodegen(self.parse_string_value(lines), False)) + elif line.startswith("@code_hash"): + self.expected_hash = TolkTestCaseExpectedHash(self.parse_string_value(lines, False)[0]) + elif line.startswith("@experimental_options"): + self.experimental_options = line[22:] + self.line_idx = self.line_idx + 1 + + if len(self.input_output) == 0 and not self.compilation_should_fail: + raise ParseInputError("no @testcase present") + if len(self.input_output) != 0 and self.compilation_should_fail: + raise ParseInputError("@testcase present, but compilation_should_fail") + + def parse_string_value(self, lines: List[str], allow_multiline = True) -> List[str]: + # a tag must be followed by a space (single-line), e.g. '@stderr some text' + # or be a multi-line value, surrounded by """ + line = lines[self.line_idx] + pos_sp = line.find(' ') + is_multi_line = lines[self.line_idx + 1] == '"""' + is_single_line = pos_sp != -1 + if not is_single_line and not is_multi_line: + raise ParseInputError('%s value is empty (not followed by a string or a multiline """)' % line) + if is_single_line and is_multi_line: + raise ParseInputError('%s value is both single-line and followed by """' % line[:pos_sp]) + if is_multi_line and not allow_multiline: + raise ParseInputError("%s value should be single-line" % line) + + if is_single_line: + return [line[pos_sp + 1:].strip()] + + self.line_idx += 2 + s_multiline = [] + while self.line_idx < len(lines) and lines[self.line_idx] != '"""': + s_multiline.append(lines[self.line_idx]) + self.line_idx = self.line_idx + 1 + return s_multiline + + def get_compiled_fif_filename(self): + return self.artifacts_folder + "/compiled.fif" + + def get_runner_fif_filename(self): + return self.artifacts_folder + "/runner.fif" + + def run_and_check(self): + cmd_args = [TOLK_EXECUTABLE, "-o", self.get_compiled_fif_filename()] + if self.experimental_options: + cmd_args = cmd_args + ["-x", self.experimental_options] + res = subprocess.run(cmd_args + [self.tolk_filename], capture_output=True, timeout=10) + exit_code = res.returncode + stderr = str(res.stderr, "utf-8") + stdout = str(res.stdout, "utf-8") + + if exit_code == 0 and self.compilation_should_fail: + raise TolkCompilationSucceededError("compilation succeeded, but it should have failed") + + for should_include in self.stderr_includes: # @stderr is used to check errors and warnings + should_include.check(stderr) + + if exit_code != 0 and self.compilation_should_fail: + return + + if exit_code != 0 and not self.compilation_should_fail: + raise TolkCompilationFailedError("tolk exit_code = %d" % exit_code, stderr) + + with open(self.get_runner_fif_filename(), "w") as fd: + fd.write("\"%s\" include ) +target_link_libraries(tolk PUBLIC git ton_crypto_core) +if (WINGETOPT_FOUND) + target_link_libraries_system(tolk wingetopt) +endif () +if (${TOLK_DEBUG}) # -DTOLK_DEBUG=1 in CMake options => #define TOLK_DEBUG (for development purposes) + message(STATUS "TOLK_DEBUG is ON") + target_compile_definitions(tolk PRIVATE TOLK_DEBUG=1) +endif() + +if (USE_EMSCRIPTEN) + add_executable(tolkfiftlib tolk-wasm.cpp ${TOLK_SOURCE}) + target_include_directories(tolkfiftlib PUBLIC $) + target_link_libraries(tolkfiftlib PUBLIC fift-lib git) + target_link_options(tolkfiftlib PRIVATE + -sEXPORTED_RUNTIME_METHODS=FS,ccall,cwrap,UTF8ToString,stringToUTF8,lengthBytesUTF8,addFunction,removeFunction,setValue + -sEXPORTED_FUNCTIONS=_tolk_compile,_version,_malloc,_free,_setThrew + -sEXPORT_NAME=CompilerModule + -sERROR_ON_UNDEFINED_SYMBOLS=0 + -sFILESYSTEM=1 -lnodefs.js + -Oz + -sIGNORE_MISSING_MAIN=1 + -sAUTO_NATIVE_LIBRARIES=0 + -sMODULARIZE=1 + -sTOTAL_MEMORY=33554432 + -sALLOW_MEMORY_GROWTH=1 + -sALLOW_TABLE_GROWTH=1 + --embed-file ${CMAKE_CURRENT_SOURCE_DIR}/../crypto/fift/lib@/fiftlib + -fexceptions + ) + target_compile_options(tolkfiftlib PRIVATE -fexceptions -fno-stack-protector) +endif () + +install(TARGETS tolk RUNTIME DESTINATION bin) diff --git a/tolk/abscode.cpp b/tolk/abscode.cpp new file mode 100644 index 00000000..fc160984 --- /dev/null +++ b/tolk/abscode.cpp @@ -0,0 +1,429 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" +#include "compiler-state.h" +#include "type-system.h" + +namespace tolk { + +/* + * + * ABSTRACT CODE + * + */ + +void TmpVar::show_as_stack_comment(std::ostream& os) const { + if (!name.empty()) { + os << name; + } else { + os << '\'' << ir_idx; + } +#ifdef TOLK_DEBUG + // uncomment for detailed stack output, like `'15(binary-op) '16(glob-var)` + // if (desc) os << desc; +#endif +} + +void TmpVar::show(std::ostream& os) const { + os << '\'' << ir_idx; // vars are printed out as `'1 '2` (in stack comments, debug info, etc.) + if (!name.empty()) { + os << '_' << name; + } +#ifdef TOLK_DEBUG + if (desc) { + os << ' ' << desc; // "origin" of implicitly created tmp var, like `'15 (binary-op) '16 (glob-var)` + } +#endif +} + +std::ostream& operator<<(std::ostream& os, const TmpVar& var) { + var.show(os); + return os; +} + +void VarDescr::show_value(std::ostream& os) const { + if (val & _Int) { + os << 'i'; + } + if (val & _Const) { + os << 'c'; + } + if (val & _Zero) { + os << '0'; + } + if (val & _NonZero) { + os << '!'; + } + if (val & _Pos) { + os << '>'; + } + if (val & _Neg) { + os << '<'; + } + if (val & _Even) { + os << 'E'; + } + if (val & _Odd) { + os << 'O'; + } + if (val & _Finite) { + os << 'f'; + } + if (val & _Nan) { + os << 'N'; + } + if (int_const.not_null()) { + os << '=' << int_const; + } +} + +void VarDescr::show(std::ostream& os, const char* name) const { + if (flags & _Last) { + os << '*'; + } + if (flags & _Unused) { + os << '?'; + } + if (name) { + os << name; + } + os << '\'' << idx; + show_value(os); +} + +void VarDescr::set_const(long long value) { + return set_const(td::make_refint(value)); +} + +void VarDescr::set_const(td::RefInt256 value) { + int_const = std::move(value); + if (!int_const->signed_fits_bits(257)) { + int_const.write().invalidate(); + } + val = _Const | _Int; + int s = sgn(int_const); + if (s < -1) { + val |= _Nan | _NonZero; + } else if (s < 0) { + val |= _NonZero | _Neg | _Finite; + } else if (s > 0) { + val |= _NonZero | _Pos | _Finite; + } else { + val |= _Zero | _Neg | _Pos | _Finite; + } + if (val & _Finite) { + val |= int_const->get_bit(0) ? _Odd : _Even; + } +} + +void VarDescr::set_const(std::string value) { + str_const = value; + val = _Const; +} + +void VarDescr::operator|=(const VarDescr& y) { + val &= y.val; + if (is_int_const() && y.is_int_const() && cmp(int_const, y.int_const) != 0) { + val &= ~_Const; + } + if (!(val & _Const)) { + int_const.clear(); + } +} + +void VarDescr::operator&=(const VarDescr& y) { + val |= y.val; + if (y.int_const.not_null() && int_const.is_null()) { + int_const = y.int_const; + } +} + +void VarDescr::set_value(const VarDescr& y) { + val = y.val; + int_const = y.int_const; +} + +void VarDescr::set_value(VarDescr&& y) { + val = y.val; + int_const = std::move(y.int_const); +} + +void VarDescr::clear_value() { + val = 0; + int_const.clear(); +} + +void VarDescrList::show(std::ostream& os) const { + if (unreachable) { + os << " "; + } + os << "["; + for (const auto& v : list) { + os << ' ' << v; + } + os << " ]\n"; +} + +void Op::show(std::ostream& os, const std::vector& vars, std::string pfx, int mode) const { + if (mode & 2) { + os << pfx << " ["; + for (const auto& v : var_info.list) { + os << ' '; + if (v.flags & VarDescr::_Last) { + os << '*'; + } + if (v.flags & VarDescr::_Unused) { + os << '?'; + } + os << vars[v.idx]; + if (mode & 4) { + os << ':'; + v.show_value(os); + } + } + os << " ]\n"; + } + std::string dis = disabled() ? " " : ""; + if (noreturn()) { + dis += " "; + } + if (impure()) { + dis += " "; + } + switch (cl) { + case _Undef: + os << pfx << dis << "???\n"; + break; + case _Nop: + os << pfx << dis << "NOP\n"; + break; + case _Call: + os << pfx << dis << "CALL: "; + show_var_list(os, left, vars); + os << " := " << (f_sym ? f_sym->name : "(null)") << " "; + if ((mode & 4) && args.size() == right.size()) { + show_var_list(os, args, vars); + } else { + show_var_list(os, right, vars); + } + os << std::endl; + break; + case _CallInd: + os << pfx << dis << "CALLIND: "; + show_var_list(os, left, vars); + os << " := EXEC "; + show_var_list(os, right, vars); + os << std::endl; + break; + case _Let: + os << pfx << dis << "LET "; + show_var_list(os, left, vars); + os << " := "; + show_var_list(os, right, vars); + os << std::endl; + break; + case _Tuple: + os << pfx << dis << "MKTUPLE "; + show_var_list(os, left, vars); + os << " := "; + show_var_list(os, right, vars); + os << std::endl; + break; + case _UnTuple: + os << pfx << dis << "UNTUPLE "; + show_var_list(os, left, vars); + os << " := "; + show_var_list(os, right, vars); + os << std::endl; + break; + case _IntConst: + os << pfx << dis << "CONST "; + show_var_list(os, left, vars); + os << " := " << int_const << std::endl; + break; + case _SliceConst: + os << pfx << dis << "SCONST "; + show_var_list(os, left, vars); + os << " := " << str_const << std::endl; + break; + case _Import: + os << pfx << dis << "IMPORT "; + show_var_list(os, left, vars); + os << std::endl; + break; + case _Return: + os << pfx << dis << "RETURN "; + show_var_list(os, left, vars); + os << std::endl; + break; + case _GlobVar: + os << pfx << dis << "GLOBVAR "; + show_var_list(os, left, vars); + os << " := " << (g_sym ? g_sym->name : "(null)") << std::endl; + break; + case _SetGlob: + os << pfx << dis << "SETGLOB "; + os << (g_sym ? g_sym->name : "(null)") << " := "; + show_var_list(os, right, vars); + os << std::endl; + break; + case _Repeat: + os << pfx << dis << "REPEAT "; + show_var_list(os, left, vars); + os << ' '; + show_block(os, block0.get(), vars, pfx, mode); + os << std::endl; + break; + case _If: + os << pfx << dis << "IF "; + show_var_list(os, left, vars); + os << ' '; + show_block(os, block0.get(), vars, pfx, mode); + os << " ELSE "; + show_block(os, block1.get(), vars, pfx, mode); + os << std::endl; + break; + case _While: + os << pfx << dis << "WHILE "; + show_var_list(os, left, vars); + os << ' '; + show_block(os, block0.get(), vars, pfx, mode); + os << " DO "; + show_block(os, block1.get(), vars, pfx, mode); + os << std::endl; + break; + case _Until: + os << pfx << dis << "UNTIL "; + show_var_list(os, left, vars); + os << ' '; + show_block(os, block0.get(), vars, pfx, mode); + os << std::endl; + break; + case _Again: + os << pfx << dis << "AGAIN "; + show_var_list(os, left, vars); + os << ' '; + show_block(os, block0.get(), vars, pfx, mode); + os << std::endl; + break; + default: + os << pfx << dis << " "; + show_var_list(os, left, vars); + os << " -- "; + show_var_list(os, right, vars); + os << std::endl; + break; + } +} + +void Op::show_var_list(std::ostream& os, const std::vector& idx_list, + const std::vector& vars) const { + if (!idx_list.size()) { + os << "()"; + } else if (idx_list.size() == 1) { + os << vars.at(idx_list[0]); + } else { + os << "(" << vars.at(idx_list[0]); + for (std::size_t i = 1; i < idx_list.size(); i++) { + os << ", " << vars.at(idx_list[i]); + } + os << ")"; + } +} + +void Op::show_var_list(std::ostream& os, const std::vector& list, const std::vector& vars) const { + auto n = list.size(); + if (!n) { + os << "()"; + } else { + os << "( "; + for (std::size_t i = 0; i < list.size(); i++) { + if (i) { + os << ", "; + } + if (list[i].is_unused()) { + os << '?'; + } + os << vars.at(list[i].idx) << ':'; + list[i].show_value(os); + } + os << " )"; + } +} + +void Op::show_block(std::ostream& os, const Op* block, const std::vector& vars, std::string pfx, int mode) { + os << "{" << std::endl; + std::string pfx2 = pfx + " "; + for (const Op& op : block) { + op.show(os, vars, pfx2, mode); + } + os << pfx << "}"; +} + +std::ostream& operator<<(std::ostream& os, const CodeBlob& code) { + code.print(os); + return os; +} + +// flags: +1 = show variable definition locations; +2 = show vars after each op; +4 = show var abstract value info after each op; +8 = show all variables at start +void CodeBlob::print(std::ostream& os, int flags) const { + os << "CODE BLOB: " << var_cnt << " variables, " << in_var_cnt << " input\n"; + if ((flags & 8) != 0) { + for (const auto& var : vars) { + var.show(os); + os << " : " << var.v_type << std::endl; + if (var.loc.is_defined() && (flags & 1) != 0) { + var.loc.show(os); + os << " defined here:\n"; + var.loc.show_context(os); + } + } + } + os << "------- BEGIN --------\n"; + for (const auto& op : ops) { + op.show(os, vars, "", flags); + } + os << "-------- END ---------\n\n"; +} + +std::vector CodeBlob::create_var(TypePtr var_type, SrcLocation loc, std::string name) { + std::vector ir_idx; + int stack_w = var_type->get_width_on_stack(); + ir_idx.reserve(stack_w); + if (const TypeDataTensor* t_tensor = var_type->try_as()) { + for (int i = 0; i < t_tensor->size(); ++i) { + std::string sub_name = name.empty() ? name : name + "." + std::to_string(i); + std::vector nested = create_var(t_tensor->items[i], loc, std::move(sub_name)); + ir_idx.insert(ir_idx.end(), nested.begin(), nested.end()); + } + } else if (const TypeDataNullable* t_nullable = var_type->try_as(); t_nullable && stack_w != 1) { + std::string null_flag_name = name.empty() ? name : name + ".NNFlag"; + ir_idx = create_var(t_nullable->inner, loc, std::move(name)); + ir_idx.emplace_back(create_var(TypeDataBool::create(), loc, std::move(null_flag_name))[0]); + } else if (var_type != TypeDataVoid::create() && var_type != TypeDataNever::create()) { +#ifdef TOLK_DEBUG + tolk_assert(stack_w == 1); +#endif + vars.emplace_back(var_cnt, var_type, std::move(name), loc); + ir_idx.emplace_back(var_cnt); + var_cnt++; + } + tolk_assert(static_cast(ir_idx.size()) == stack_w); + return ir_idx; +} + +} // namespace tolk diff --git a/tolk/analyzer.cpp b/tolk/analyzer.cpp new file mode 100644 index 00000000..c38b0bfa --- /dev/null +++ b/tolk/analyzer.cpp @@ -0,0 +1,902 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" +#include "compiler-state.h" +#include "type-system.h" + +namespace tolk { + +// functions returning "never" are assumed to interrupt flow +// for instance, variables after their call aren't considered used +// its main purpose is `throw` statement, it's a call to a built-in `__throw` function +static bool does_function_always_throw(FunctionPtr fun_ref) { + return fun_ref->declared_return_type == TypeDataNever::create(); +} + +/* + * + * ANALYZE AND PREPROCESS ABSTRACT CODE + * + */ + +bool CodeBlob::compute_used_code_vars() { + VarDescrList empty_var_info; + return compute_used_code_vars(ops, empty_var_info, true); +} + +bool CodeBlob::compute_used_code_vars(std::unique_ptr& ops_ptr, const VarDescrList& var_info, bool edit) const { + tolk_assert(ops_ptr); + if (!ops_ptr->next) { + tolk_assert(ops_ptr->cl == Op::_Nop); + return ops_ptr->set_var_info(var_info); + } + // here and below, bitwise | (not logical ||) are used to execute both left and right parts + return static_cast(compute_used_code_vars(ops_ptr->next, var_info, edit)) | + static_cast(ops_ptr->compute_used_vars(*this, edit)); +} + +bool operator==(const VarDescrList& x, const VarDescrList& y) { + if (x.size() != y.size()) { + return false; + } + for (std::size_t i = 0; i < x.size(); i++) { + if (x.list[i].idx != y.list[i].idx || x.list[i].flags != y.list[i].flags) { + return false; + } + } + return true; +} + +bool same_values(const VarDescr& x, const VarDescr& y) { + if (x.val != y.val || x.int_const.is_null() != y.int_const.is_null()) { + return false; + } + if (x.int_const.not_null() && cmp(x.int_const, y.int_const) != 0) { + return false; + } + return true; +} + +bool same_values(const VarDescrList& x, const VarDescrList& y) { + if (x.size() != y.size()) { + return false; + } + for (std::size_t i = 0; i < x.size(); i++) { + if (x.list[i].idx != y.list[i].idx || !same_values(x.list[i], y.list[i])) { + return false; + } + } + return true; +} + +bool Op::set_var_info(const VarDescrList& new_var_info) { + if (var_info == new_var_info) { + return false; + } + var_info = new_var_info; + return true; +} + +bool Op::set_var_info(VarDescrList&& new_var_info) { + if (var_info == new_var_info) { + return false; + } + var_info = std::move(new_var_info); + return true; +} + +bool Op::set_var_info_except(const VarDescrList& new_var_info, const std::vector& var_list) { + if (!var_list.size()) { + return set_var_info(new_var_info); + } + VarDescrList tmp_info{new_var_info}; + tmp_info -= var_list; + return set_var_info(tmp_info); +} + +bool Op::set_var_info_except(VarDescrList&& new_var_info, const std::vector& var_list) { + if (var_list.size()) { + new_var_info -= var_list; + } + return set_var_info(std::move(new_var_info)); +} +std::vector sort_unique_vars(const std::vector& var_list) { + std::vector vars{var_list}, unique_vars; + std::sort(vars.begin(), vars.end()); + vars.erase(std::unique(vars.begin(), vars.end()), vars.end()); + return vars; +} + +VarDescr* VarDescrList::operator[](var_idx_t idx) { + auto it = std::lower_bound(list.begin(), list.end(), idx); + return it != list.end() && it->idx == idx ? &*it : nullptr; +} + +const VarDescr* VarDescrList::operator[](var_idx_t idx) const { + auto it = std::lower_bound(list.begin(), list.end(), idx); + return it != list.end() && it->idx == idx ? &*it : nullptr; +} + +std::size_t VarDescrList::count(const std::vector idx_list) const { + std::size_t res = 0; + for (var_idx_t idx : idx_list) { + if (operator[](idx)) { + ++res; + } + } + return res; +} + +std::size_t VarDescrList::count_used(const std::vector idx_list) const { + std::size_t res = 0; + for (var_idx_t idx : idx_list) { + auto v = operator[](idx); + if (v && !v->is_unused()) { + ++res; + } + } + return res; +} + +VarDescrList& VarDescrList::operator-=(var_idx_t idx) { + auto it = std::lower_bound(list.begin(), list.end(), idx); + if (it != list.end() && it->idx == idx) { + list.erase(it); + } + return *this; +} + +VarDescrList& VarDescrList::operator-=(const std::vector& idx_list) { + for (var_idx_t idx : idx_list) { + *this -= idx; + } + return *this; +} + +VarDescrList& VarDescrList::add_var(var_idx_t idx, bool unused) { + auto it = std::lower_bound(list.begin(), list.end(), idx); + if (it == list.end() || it->idx != idx) { + list.emplace(it, idx, VarDescr::_Last | (unused ? VarDescr::_Unused : 0)); + } else if (it->is_unused() && !unused) { + it->clear_unused(); + } + return *this; +} + +VarDescrList& VarDescrList::add_vars(const std::vector& idx_list, bool unused) { + for (var_idx_t idx : idx_list) { + add_var(idx, unused); + } + return *this; +} + +VarDescr& VarDescrList::add(var_idx_t idx) { + auto it = std::lower_bound(list.begin(), list.end(), idx); + if (it == list.end() || it->idx != idx) { + it = list.emplace(it, idx); + } + return *it; +} + +VarDescr& VarDescrList::add_newval(var_idx_t idx) { + auto it = std::lower_bound(list.begin(), list.end(), idx); + if (it == list.end() || it->idx != idx) { + return *list.emplace(it, idx); + } else { + it->clear_value(); + return *it; + } +} + +VarDescrList& VarDescrList::clear_last() { + for (auto& var : list) { + if (var.flags & VarDescr::_Last) { + var.flags &= ~VarDescr::_Last; + } + } + return *this; +} + +VarDescrList VarDescrList::operator+(const VarDescrList& y) const { + VarDescrList res; + auto it1 = list.cbegin(); + auto it2 = y.list.cbegin(); + while (it1 != list.cend() && it2 != y.list.cend()) { + if (it1->idx < it2->idx) { + res.list.push_back(*it1++); + } else if (it1->idx > it2->idx) { + res.list.push_back(*it2++); + } else { + res.list.push_back(*it1++); + res.list.back() += *it2++; + } + } + while (it1 != list.cend()) { + res.list.push_back(*it1++); + } + while (it2 != y.list.cend()) { + res.list.push_back(*it2++); + } + return res; +} + +VarDescrList& VarDescrList::operator+=(const VarDescrList& y) { + return *this = *this + y; +} + +VarDescrList VarDescrList::operator|(const VarDescrList& y) const { + if (y.unreachable) { + return *this; + } + if (unreachable) { + return y; + } + VarDescrList res; + auto it1 = list.cbegin(); + auto it2 = y.list.cbegin(); + while (it1 != list.cend() && it2 != y.list.cend()) { + if (it1->idx < it2->idx) { + it1++; + } else if (it1->idx > it2->idx) { + it2++; + } else { + res.list.push_back(*it1++); + res.list.back() |= *it2++; + } + } + return res; +} + +VarDescrList& VarDescrList::operator|=(const VarDescrList& y) { + if (y.unreachable) { + return *this; + } else { + return *this = *this | y; + } +} + +VarDescrList& VarDescrList::import_values(const VarDescrList& values) { + if (values.unreachable) { + set_unreachable(); + } else + for (auto& vd : list) { + auto new_vd = values[vd.idx]; + if (new_vd) { + vd.set_value(*new_vd); + } else { + vd.clear_value(); + } + } + return *this; +} + +bool Op::std_compute_used_vars(bool disabled) { + // left = OP right + // var_info := (var_info - left) + right + VarDescrList new_var_info{next->var_info}; + new_var_info -= left; + new_var_info.clear_last(); + if (args.size() == right.size() && !disabled) { + for (const VarDescr& arg : args) { + new_var_info.add_var(arg.idx, arg.is_unused()); + } + } else { + new_var_info.add_vars(right, disabled); + } + return set_var_info(std::move(new_var_info)); +} + +bool Op::compute_used_vars(const CodeBlob& code, bool edit) { + tolk_assert(next); + const VarDescrList& next_var_info = next->var_info; + if (cl == _Nop) { + return set_var_info_except(next_var_info, left); + } + switch (cl) { + case _IntConst: + case _SliceConst: + case _GlobVar: + case _Call: + case _CallInd: + case _Tuple: + case _UnTuple: { + // left = EXEC right; + if (!next_var_info.count_used(left) && !impure()) { + // all variables in `left` are not needed + if (edit) { + set_disabled(); + } + return std_compute_used_vars(true); + } + if (cl == _Call && does_function_always_throw(f_sym)) { + VarDescrList new_var_info; // empty, not next->var_info + if (args.size() == right.size()) { + for (const VarDescr& arg : args) { + new_var_info.add_var(arg.idx, arg.is_unused()); + } + } else { + new_var_info.add_vars(right, false); + } + return set_var_info(std::move(new_var_info)); + } + return std_compute_used_vars(); + } + case _SetGlob: { + // GLOB = right + if (right.empty() && edit) { + set_disabled(); + } + return std_compute_used_vars(right.empty()); + } + case _Let: { + // left = right + std::size_t cnt = next_var_info.count_used(left); + tolk_assert(left.size() == right.size()); + auto l_it = left.cbegin(), r_it = right.cbegin(); + VarDescrList new_var_info{next_var_info}; + new_var_info -= left; + new_var_info.clear_last(); + std::vector new_left, new_right; + for (; l_it < left.cend(); ++l_it, ++r_it) { + if (std::find(l_it + 1, left.cend(), *l_it) == left.cend()) { + auto p = next_var_info[*l_it]; + new_var_info.add_var(*r_it, edit && (!p || p->is_unused())); + new_left.push_back(*l_it); + new_right.push_back(*r_it); + } + } + if (new_left.size() < left.size()) { + left = std::move(new_left); + right = std::move(new_right); + } + if (!cnt && edit) { + // all variables in `left` are not needed + set_disabled(); + } + return set_var_info(std::move(new_var_info)); + } + case _Return: { + // return left + if (var_info.count(left) == left.size()) { + return false; + } + std::vector unique_vars = sort_unique_vars(left); + var_info.list.clear(); + for (var_idx_t i : unique_vars) { + var_info.list.emplace_back(i, VarDescr::_Last); + } + return true; + } + case _Import: { + // import left + std::vector unique_vars = sort_unique_vars(left); + var_info.list.clear(); + for (var_idx_t i : unique_vars) { + var_info.list.emplace_back(i, next_var_info[i] ? 0 : VarDescr::_Last); + } + return true; + } + case _If: { + // if (left) then block0 else block1 + // VarDescrList nx_var_info = next_var_info; + // nx_var_info.clear_last(); + code.compute_used_code_vars(block0, next_var_info, edit); + VarDescrList merge_info; + if (block1) { + code.compute_used_code_vars(block1, next_var_info, edit); + merge_info = block0->var_info + block1->var_info; + } else { + merge_info = block0->var_info + next_var_info; + } + merge_info.clear_last(); + merge_info += left; + return set_var_info(std::move(merge_info)); + } + case _While: { + // while (block0 || left) block1; + // ... block0 left { block1 block0 left } next + VarDescrList new_var_info{next_var_info}; + bool changes = false; + do { + VarDescrList after_cond{new_var_info}; + after_cond += left; + code.compute_used_code_vars(block0, after_cond, changes); + code.compute_used_code_vars(block1, block0->var_info, changes); + std::size_t n = new_var_info.size(); + new_var_info += block1->var_info; + new_var_info.clear_last(); + if (changes) { + break; + } + changes = (new_var_info.size() == n); + } while (changes <= edit); + new_var_info += left; + code.compute_used_code_vars(block0, new_var_info, edit); + return set_var_info(block0->var_info); + } + case _Until: { + // until (block0 || left); + // .. { block0 left } block0 left next + VarDescrList after_cond_first{next_var_info}; + after_cond_first += left; + code.compute_used_code_vars(block0, after_cond_first, false); + VarDescrList new_var_info{block0->var_info}; + bool changes = false; + do { + VarDescrList after_cond{new_var_info}; + after_cond += next_var_info; + after_cond += left; + code.compute_used_code_vars(block0, after_cond, changes); + std::size_t n = new_var_info.size(); + new_var_info += block0->var_info; + new_var_info.clear_last(); + if (changes) { + break; + } + changes = (new_var_info.size() == n); + } while (changes <= edit); + return set_var_info(std::move(new_var_info) + next_var_info); + } + case _Repeat: { + // repeat (left) block0 + // left { block0 } next + VarDescrList new_var_info{next_var_info}; + bool changes = false; + do { + code.compute_used_code_vars(block0, new_var_info, changes); + std::size_t n = new_var_info.size(); + new_var_info += block0->var_info; + new_var_info.clear_last(); + if (changes) { + break; + } + changes = (new_var_info.size() == n); + } while (changes <= edit); + tolk_assert(left.size() == 1); + bool last = new_var_info.count_used(left) == 0; + new_var_info += left; + if (last) { + new_var_info[left[0]]->flags |= VarDescr::_Last; + } + return set_var_info(std::move(new_var_info)); + } + case _Again: { + // for(;;) block0 + // { block0 } + VarDescrList new_var_info; + bool changes = false; + do { + code.compute_used_code_vars(block0, new_var_info, changes); + std::size_t n = new_var_info.size(); + new_var_info += block0->var_info; + new_var_info.clear_last(); + if (changes) { + break; + } + changes = (new_var_info.size() == n); + } while (changes <= edit); + return set_var_info(std::move(new_var_info)); + } + case _TryCatch: { + code.compute_used_code_vars(block0, next_var_info, edit); + code.compute_used_code_vars(block1, next_var_info, edit); + VarDescrList merge_info = block0->var_info + block1->var_info + next_var_info; + merge_info -= left; + merge_info.clear_last(); + return set_var_info(std::move(merge_info)); + } + default: + std::cerr << "fatal: unknown operation in compute_used_vars()\n"; + throw ParseError{where, "unknown operation"}; + } +} + +bool prune_unreachable(std::unique_ptr& ops) { + if (!ops) { + return true; + } + Op& op = *ops; + if (op.cl == Op::_Nop) { + if (op.next) { + ops = std::move(op.next); + return prune_unreachable(ops); + } + return true; + } + bool reach; + switch (op.cl) { + case Op::_IntConst: + case Op::_SliceConst: + case Op::_GlobVar: + case Op::_SetGlob: + case Op::_CallInd: + case Op::_Tuple: + case Op::_UnTuple: + case Op::_Import: + case Op::_Let: + reach = true; + break; + case Op::_Return: + reach = false; + break; + case Op::_Call: + reach = !does_function_always_throw(op.f_sym); + break; + case Op::_If: { + // if left then block0 else block1; ... + VarDescr* c_var = op.var_info[op.left[0]]; + if (c_var && c_var->always_true()) { + op.block0->last().next = std::move(op.next); + ops = std::move(op.block0); + return prune_unreachable(ops); + } else if (c_var && c_var->always_false()) { + op.block1->last().next = std::move(op.next); + ops = std::move(op.block1); + return prune_unreachable(ops); + } else { + reach = static_cast(prune_unreachable(op.block0)) | static_cast(prune_unreachable(op.block1)); + } + break; + } + case Op::_While: { + // while (block0 || left) block1; + if (!prune_unreachable(op.block0)) { + // computation of block0 never returns + ops = std::move(op.block0); + return prune_unreachable(ops); + } + VarDescr* c_var = op.block0->last().var_info[op.left[0]]; + if (c_var && c_var->always_false()) { + // block1 never executed + op.block0->last().next = std::move(op.next); + ops = std::move(op.block0); + return prune_unreachable(ops); + } else if (c_var && c_var->always_true()) { + if (!prune_unreachable(op.block1)) { + // block1 never returns + op.block0->last().next = std::move(op.block1); + ops = std::move(op.block0); + return false; + } + // infinite loop + op.cl = Op::_Again; + op.block0->last().next = std::move(op.block1); + op.left.clear(); + reach = false; + } else { + if (!prune_unreachable(op.block1)) { + // block1 never returns, while equivalent to block0 ; if left then block1 else next + op.cl = Op::_If; + std::unique_ptr new_op = std::move(op.block0); + op.block0 = std::move(op.block1); + op.block1 = std::make_unique(op.next->where, Op::_Nop); + new_op->last().next = std::move(ops); + ops = std::move(new_op); + } + reach = true; // block1 may be never executed + } + break; + } + case Op::_Repeat: { + // repeat (left) block0 + VarDescr* c_var = op.var_info[op.left[0]]; + if (c_var && c_var->always_nonpos()) { + // loop never executed + ops = std::move(op.next); + return prune_unreachable(ops); + } + if (c_var && c_var->always_pos()) { + if (!prune_unreachable(op.block0)) { + // block0 executed at least once, and it never returns + // replace code with block0 + ops = std::move(op.block0); + return false; + } + } else { + prune_unreachable(op.block0); + } + reach = true; + break; + } + case Op::_Until: + case Op::_Again: { + // do block0 until left; ... + if (!prune_unreachable(op.block0)) { + // block0 never returns, replace loop by block0 + ops = std::move(op.block0); + return false; + } + reach = (op.cl != Op::_Again); + break; + } + case Op::_TryCatch: { + reach = static_cast(prune_unreachable(op.block0)) | static_cast(prune_unreachable(op.block1)); + break; + } + default: + std::cerr << "fatal: unknown operation \n"; + throw ParseError{op.where, "unknown operation in prune_unreachable()"}; + } + if (reach) { + return prune_unreachable(op.next); + } else { + while (op.next->next) { + op.next = std::move(op.next->next); + } + return false; + } +} + +void CodeBlob::prune_unreachable_code() { + if (prune_unreachable(ops)) { + throw ParseError{loc, "control reaches end of function"}; + } +} + +void CodeBlob::fwd_analyze() { + VarDescrList values; + tolk_assert(ops && ops->cl == Op::_Import); + for (var_idx_t i : ops->left) { + values += i; + if (vars[i].v_type == TypeDataInt::create()) { + values[i]->val |= VarDescr::_Int; + } + } + ops->fwd_analyze(values); +} + +void Op::prepare_args(VarDescrList values) { + if (args.size() != right.size()) { + args.clear(); + for (var_idx_t i : right) { + args.emplace_back(i); + } + } + for (std::size_t i = 0; i < right.size(); i++) { + const VarDescr* val = values[right[i]]; + if (val) { + args[i].set_value(*val); + // args[i].clear_unused(); + } else { + args[i].clear_value(); + } + args[i].clear_unused(); + } +} + +VarDescrList Op::fwd_analyze(VarDescrList values) { + var_info.import_values(values); + switch (cl) { + case _Nop: + case _Import: + break; + case _Return: + values.set_unreachable(); + break; + case _IntConst: { + values.add_newval(left[0]).set_const(int_const); + break; + } + case _SliceConst: { + values.add_newval(left[0]).set_const(str_const); + break; + } + case _Call: { + prepare_args(values); + if (!f_sym->is_code_function()) { + std::vector res; + res.reserve(left.size()); + for (var_idx_t i : left) { + res.emplace_back(i); + } + AsmOpList tmp; + if (f_sym->is_asm_function()) { + std::get(f_sym->body)->compile(tmp); // abstract interpretation of res := f (args) + } else { + std::get(f_sym->body)->compile(tmp, res, args, where); + } + int j = 0; + for (var_idx_t i : left) { + values.add_newval(i).set_value(res[j++]); + } + } else { + for (var_idx_t i : left) { + values.add_newval(i); + } + } + if (does_function_always_throw(f_sym)) { + values.set_unreachable(); + } + break; + } + case _Tuple: + case _UnTuple: + case _GlobVar: + case _CallInd: { + for (var_idx_t i : left) { + values.add_newval(i); + } + break; + } + case _SetGlob: + break; + case _Let: { + std::vector old_val; + tolk_assert(left.size() == right.size()); + for (std::size_t i = 0; i < right.size(); i++) { + const VarDescr* ov = values[right[i]]; + if (!ov && G.is_verbosity(5)) { + std::cerr << "FATAL: error in assignment at right component #" << i << " (no value for _" << right[i] << ")" + << std::endl; + for (auto x : left) { + std::cerr << '_' << x << " "; + } + std::cerr << "= "; + for (auto x : right) { + std::cerr << '_' << x << " "; + } + std::cerr << std::endl; + } + // tolk_assert(ov); + if (ov) { + old_val.push_back(*ov); + } else { + old_val.emplace_back(); + } + } + for (std::size_t i = 0; i < left.size(); i++) { + values.add_newval(left[i]).set_value(std::move(old_val[i])); + } + break; + } + case _If: { + VarDescrList val1 = block0->fwd_analyze(values); + VarDescrList val2 = block1 ? block1->fwd_analyze(std::move(values)) : std::move(values); + values = val1 | val2; + break; + } + case _Repeat: { + bool atl1 = (values[left[0]] && values[left[0]]->always_pos()); + VarDescrList next_values = block0->fwd_analyze(values); + while (true) { + VarDescrList new_values = values | next_values; + if (same_values(new_values, values)) { + break; + } + values = std::move(new_values); + next_values = block0->fwd_analyze(values); + } + if (atl1) { + values = std::move(next_values); + } + break; + } + case _While: { + auto values0 = values; + values = block0->fwd_analyze(values); + if (values[left[0]] && values[left[0]]->always_false()) { + // block1 never executed + block1->fwd_analyze(values); + break; + } + while (true) { + VarDescrList next_values = values | block0->fwd_analyze(values0 | block1->fwd_analyze(values)); + if (same_values(next_values, values)) { + break; + } + values = std::move(next_values); + } + break; + } + case _Until: + case _Again: { + while (true) { + VarDescrList next_values = values | block0->fwd_analyze(values); + if (same_values(next_values, values)) { + break; + } + values = std::move(next_values); + } + values = block0->fwd_analyze(values); + break; + } + case _TryCatch: { + VarDescrList val1 = block0->fwd_analyze(values); + VarDescrList val2 = block1->fwd_analyze(std::move(values)); + values = val1 | val2; + break; + } + default: + std::cerr << "fatal: unknown operation \n"; + throw ParseError{where, "unknown operation in fwd_analyze()"}; + } + if (next) { + return next->fwd_analyze(std::move(values)); + } else { + return values; + } +} + +void Op::set_disabled(bool flag) { + if (flag) { + flags |= _Disabled; + } else { + flags &= ~_Disabled; + } +} + + +bool Op::set_noreturn(bool flag) { + if (flag) { + flags |= _NoReturn; + } else { + flags &= ~_NoReturn; + } + return flag; +} + +void Op::set_impure_flag() { + flags |= _Impure; +} + +bool Op::mark_noreturn() { + switch (cl) { + case _Nop: + if (!next) { + return set_noreturn(false); + } + // fallthrough + case _Import: + case _IntConst: + case _SliceConst: + case _Let: + case _Tuple: + case _UnTuple: + case _SetGlob: + case _GlobVar: + case _CallInd: + return set_noreturn(next->mark_noreturn()); + case _Return: + return set_noreturn(); + case _Call: + return set_noreturn(next->mark_noreturn() || does_function_always_throw(f_sym)); + case _If: + case _TryCatch: + // note, that & | (not && ||) here and below is mandatory to invoke both left and right calls + return set_noreturn((static_cast(block0->mark_noreturn()) & static_cast(block1 && block1->mark_noreturn())) | static_cast(next->mark_noreturn())); + case _Again: + block0->mark_noreturn(); + return set_noreturn(); + case _Until: + return set_noreturn(static_cast(block0->mark_noreturn()) | static_cast(next->mark_noreturn())); + case _While: + block1->mark_noreturn(); + return set_noreturn(static_cast(block0->mark_noreturn()) | static_cast(next->mark_noreturn())); + case _Repeat: + block0->mark_noreturn(); + return set_noreturn(next->mark_noreturn()); + default: + std::cerr << "fatal: unknown operation \n"; + throw ParseError{where, "unknown operation in mark_noreturn()"}; + } +} + +void CodeBlob::mark_noreturn() { + ops->mark_noreturn(); +} + +} // namespace tolk diff --git a/tolk/asmops.cpp b/tolk/asmops.cpp new file mode 100644 index 00000000..2618ed26 --- /dev/null +++ b/tolk/asmops.cpp @@ -0,0 +1,364 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" +#include + +namespace tolk { + +/* + * + * ASM-OP LIST FUNCTIONS + * + */ + +int is_pos_pow2(td::RefInt256 x) { + if (sgn(x) > 0 && !sgn(x & (x - 1))) { + return x->bit_size(false) - 1; + } else { + return -1; + } +} + +int is_neg_pow2(td::RefInt256 x) { + return sgn(x) < 0 ? is_pos_pow2(-x) : 0; +} + +std::ostream& operator<<(std::ostream& os, AsmOp::SReg stack_reg) { + int i = stack_reg.idx; + if (i >= 0) { + if (i < 16) { + return os << 's' << i; + } else { + return os << i << " s()"; + } + } else if (i >= -2) { + return os << "s(" << i << ')'; + } else { + return os << i << " s()"; + } +} + +AsmOp AsmOp::Const(int arg, const std::string& push_op) { + std::ostringstream os; + os << arg << ' ' << push_op; + return AsmOp::Const(os.str()); +} + +AsmOp AsmOp::make_stk2(int a, int b, const char* str, int delta) { + std::ostringstream os; + os << SReg(a) << ' ' << SReg(b) << ' ' << str; + int c = std::max(a, b) + 1; + return AsmOp::Custom(os.str(), c, c + delta); +} + +AsmOp AsmOp::make_stk3(int a, int b, int c, const char* str, int delta) { + std::ostringstream os; + os << SReg(a) << ' ' << SReg(b) << ' ' << SReg(c) << ' ' << str; + int m = std::max(a, std::max(b, c)) + 1; + return AsmOp::Custom(os.str(), m, m + delta); +} + +AsmOp AsmOp::BlkSwap(int a, int b) { + std::ostringstream os; + if (a == 1 && b == 1) { + return AsmOp::Xchg(0, 1); + } else if (a == 1) { + if (b == 2) { + os << "ROT"; + } else { + os << b << " ROLL"; + } + } else if (b == 1) { + if (a == 2) { + os << "-ROT"; + } else { + os << a << " -ROLL"; + } + } else { + os << a << " " << b << " BLKSWAP"; + } + return AsmOp::Custom(os.str(), a + b, a + b); +} + +AsmOp AsmOp::BlkPush(int a, int b) { + std::ostringstream os; + if (a == 1) { + return AsmOp::Push(b); + } else if (a == 2 && b == 1) { + os << "2DUP"; + } else { + os << a << " " << b << " BLKPUSH"; + } + return AsmOp::Custom(os.str(), b + 1, a + b + 1); +} + +AsmOp AsmOp::BlkDrop(int a) { + std::ostringstream os; + if (a == 1) { + return AsmOp::Pop(); + } else if (a == 2) { + os << "2DROP"; + } else { + os << a << " BLKDROP"; + } + return AsmOp::Custom(os.str(), a, 0); +} + +AsmOp AsmOp::BlkDrop2(int a, int b) { + if (!b) { + return BlkDrop(a); + } + std::ostringstream os; + os << a << " " << b << " BLKDROP2"; + return AsmOp::Custom(os.str(), a + b, b); +} + +AsmOp AsmOp::BlkReverse(int a, int b) { + std::ostringstream os; + os << a << " " << b << " REVERSE"; + return AsmOp::Custom(os.str(), a + b, a + b); +} + +AsmOp AsmOp::Tuple(int a) { + switch (a) { + case 1: + return AsmOp::Custom("SINGLE", 1, 1); + case 2: + return AsmOp::Custom("PAIR", 2, 1); + case 3: + return AsmOp::Custom("TRIPLE", 3, 1); + } + std::ostringstream os; + os << a << " TUPLE"; + return AsmOp::Custom(os.str(), a, 1); +} + +AsmOp AsmOp::UnTuple(int a) { + switch (a) { + case 1: + return AsmOp::Custom("UNSINGLE", 1, 1); + case 2: + return AsmOp::Custom("UNPAIR", 1, 2); + case 3: + return AsmOp::Custom("UNTRIPLE", 1, 3); + } + std::ostringstream os; + os << a << " UNTUPLE"; + return AsmOp::Custom(os.str(), 1, a); +} + +AsmOp AsmOp::IntConst(const td::RefInt256& x) { + if (x->signed_fits_bits(8)) { + return AsmOp::Const(dec_string(x) + " PUSHINT"); + } + if (!x->is_valid()) { + return AsmOp::Const("PUSHNAN"); + } + int k = is_pos_pow2(x); + if (k >= 0) { + return AsmOp::Const(k, "PUSHPOW2"); + } + k = is_pos_pow2(x + 1); + if (k >= 0) { + return AsmOp::Const(k, "PUSHPOW2DEC"); + } + k = is_pos_pow2(-x); + if (k >= 0) { + return AsmOp::Const(k, "PUSHNEGPOW2"); + } + if (!x->mod_pow2_short(23)) { + return AsmOp::Const(dec_string(x) + " PUSHINTX"); + } + return AsmOp::Const(dec_string(x) + " PUSHINT"); +} + +AsmOp AsmOp::BoolConst(bool f) { + return AsmOp::Const(f ? "TRUE" : "FALSE"); +} + +AsmOp AsmOp::Parse(const std::string& custom_op) { + if (custom_op == "NOP") { + return AsmOp::Nop(); + } else if (custom_op == "SWAP") { + return AsmOp::Xchg(1); + } else if (custom_op == "DROP") { + return AsmOp::Pop(0); + } else if (custom_op == "NIP") { + return AsmOp::Pop(1); + } else if (custom_op == "DUP") { + return AsmOp::Push(0); + } else if (custom_op == "OVER") { + return AsmOp::Push(1); + } else { + return AsmOp::Custom(custom_op); + } +} + +AsmOp AsmOp::Parse(std::string custom_op, int args, int retv) { + auto res = Parse(custom_op); + if (res.is_custom()) { + res.a = args; + res.b = retv; + } + return res; +} + +void AsmOp::out(std::ostream& os) const { + if (!op.empty()) { + os << op; + return; + } + switch (t) { + case a_none: + break; + case a_xchg: + if (!a && !(b & -2)) { + os << (b ? "SWAP" : "NOP"); + break; + } + os << SReg(a) << ' ' << SReg(b) << " XCHG"; + break; + case a_push: + if (!(a & -2)) { + os << (a ? "OVER" : "DUP"); + break; + } + os << SReg(a) << " PUSH"; + break; + case a_pop: + if (!(a & -2)) { + os << (a ? "NIP" : "DROP"); + break; + } + os << SReg(a) << " POP"; + break; + default: + throw Fatal{"unknown assembler operation"}; + } +} + +void AsmOp::out_indent_nl(std::ostream& os, bool no_eol) const { + for (int i = 0; i < indent; i++) { + os << " "; + } + out(os); + if (!no_eol) { + os << std::endl; + } +} + +std::string AsmOp::to_string() const { + if (!op.empty()) { + return op; + } else { + std::ostringstream os; + out(os); + return os.str(); + } +} + +bool AsmOpList::append(const std::vector& ops) { + for (const auto& op : ops) { + if (!append(op)) { + return false; + } + } + return true; +} + +const_idx_t AsmOpList::register_const(Const new_const) { + if (new_const.is_null()) { + return not_const; + } + unsigned idx; + for (idx = 0; idx < constants_.size(); idx++) { + if (!td::cmp(new_const, constants_[idx])) { + return idx; + } + } + constants_.push_back(std::move(new_const)); + return (const_idx_t)idx; +} + +Const AsmOpList::get_const(const_idx_t idx) { + if ((unsigned)idx < constants_.size()) { + return constants_[idx]; + } else { + return {}; + } +} + +void AsmOpList::show_var_ext(std::ostream& os, std::pair idx_pair) const { + var_idx_t i = idx_pair.first; + const_idx_t j = idx_pair.second; + if (!var_names_ || (unsigned)i >= var_names_->size()) { + os << '\'' << i; + } else { + var_names_->at(i).show_as_stack_comment(os); + } + if ((unsigned)j < constants_.size() && constants_[j].not_null()) { + os << '=' << constants_[j]; + } +} + +void AsmOpList::out(std::ostream& os, int mode) const { + if (!(mode & 2)) { + for (const auto& op : list_) { + op.out_indent_nl(os); + } + } else { + std::size_t n = list_.size(); + for (std::size_t i = 0; i < n; i++) { + const auto& op = list_[i]; + if (!op.is_comment() && i + 1 < n && list_[i + 1].is_comment()) { + op.out_indent_nl(os, true); + os << '\t'; + do { + i++; + } while (i + 1 < n && list_[i + 1].is_comment()); + list_[i].out(os); + os << std::endl; + } else { + op.out_indent_nl(os, false); + } + } + } +} + +bool apply_op(StackTransform& trans, const AsmOp& op) { + if (!trans.is_valid()) { + return false; + } + switch (op.t) { + case AsmOp::a_none: + return true; + case AsmOp::a_xchg: + return trans.apply_xchg(op.a, op.b, true); + case AsmOp::a_push: + return trans.apply_push(op.a); + case AsmOp::a_pop: + return trans.apply_pop(op.a); + case AsmOp::a_const: + return !op.a && op.b == 1 && trans.apply_push_newconst(); + case AsmOp::a_custom: + return op.is_gconst() && trans.apply_push_newconst(); + default: + return false; + } +} + +} // namespace tolk diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp new file mode 100644 index 00000000..fcaa1157 --- /dev/null +++ b/tolk/ast-from-tokens.cpp @@ -0,0 +1,1172 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "ast-from-tokens.h" +#include "ast.h" +#include "type-system.h" +#include "platform-utils.h" +#include "tolk-version.h" + +/* + * Here we construct AST for a tolk file. + * While constructing, no global state is modified. + * Historically, in FunC, there was no AST: while lexing, symbols were registered, types were inferred, and so on. + * There was no way to perform any more or less semantic analysis. + * Implementing AST gives a giant advance for future modifications and stability. + */ + +namespace tolk { + +// given a token, determine whether it's <, or >, or similar +static bool is_comparison_binary_op(TokenType tok) { + return tok == tok_lt || tok == tok_gt || tok == tok_leq || tok == tok_geq || tok == tok_eq || tok == tok_neq || tok == tok_spaceship; +} + +// same as above, but to detect bitwise operators: & | ^ +static bool is_bitwise_binary_op(TokenType tok) { + return tok == tok_bitwise_and || tok == tok_bitwise_or || tok == tok_bitwise_xor; +} + +// same as above, but to detect logical operators: && || +static bool is_logical_binary_op(TokenType tok) { + return tok == tok_logical_and || tok == tok_logical_or; +} + +// same as above, but to detect addition/subtraction +static bool is_add_or_sub_binary_op(TokenType tok) { + return tok == tok_plus || tok == tok_minus; +} + +// fire an error for a case "flags & 0xFF != 0" (equivalent to "flags & 1", probably unexpected) +// it would better be a warning, but we decided to make it a strict error +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_lower_precedence(SrcLocation loc, std::string_view op_lower, std::string_view op_higher) { + std::string name_lower = static_cast(op_lower); + std::string name_higher = static_cast(op_higher); + throw ParseError(loc, name_lower + " has lower precedence than " + name_higher + + ", probably this code won't work as you expected. " + "Use parenthesis: either (... " + name_lower + " ...) to evaluate it first, or (... " + name_higher + " ...) to suppress this error."); +} + +// fire an error for a case "arg1 & arg2 | arg3" +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_mix_and_or_no_parenthesis(SrcLocation loc, std::string_view op1, std::string_view op2) { + std::string name1 = static_cast(op1); + std::string name2 = static_cast(op2); + throw ParseError(loc, "mixing " + name1 + " with " + name2 + " without parenthesis may lead to accidental errors. " + "Use parenthesis to emphasize operator precedence."); +} + +// diagnose when bitwise operators are used in a probably wrong way due to tricky precedence +// example: "flags & 0xFF != 0" is equivalent to "flags & 1", most likely it's unexpected +// the only way to suppress this error for the programmer is to use parenthesis +// (how do we detect presence of parenthesis? simple: (0!=1) is ast_parenthesized_expr{ast_binary_operator}, +// that's why if rhs->type == ast_binary_operator, it's not surrounded by parenthesis) +static void diagnose_bitwise_precedence(SrcLocation loc, std::string_view operator_name, AnyExprV lhs, AnyExprV rhs) { + // handle "flags & 0xFF != 0" (rhs = "0xFF != 0") + if (rhs->type == ast_binary_operator && is_comparison_binary_op(rhs->as()->tok)) { + fire_error_lower_precedence(loc, operator_name, rhs->as()->operator_name); + } + + // handle "0 != flags & 0xFF" (lhs = "0 != flags") + if (lhs->type == ast_binary_operator && is_comparison_binary_op(lhs->as()->tok)) { + fire_error_lower_precedence(loc, operator_name, lhs->as()->operator_name); + } +} + +// similar to above, but detect potentially invalid usage of && and || +// since anyway, using parenthesis when both && and || occur in the same expression, +// && and || have equal operator precedence in Tolk +static void diagnose_and_or_precedence(SrcLocation loc, AnyExprV lhs, TokenType rhs_tok, std::string_view rhs_operator_name) { + if (auto lhs_op = lhs->try_as()) { + // handle "arg1 & arg2 | arg3" (lhs = "arg1 & arg2") + if (is_bitwise_binary_op(lhs_op->tok) && is_bitwise_binary_op(rhs_tok) && lhs_op->tok != rhs_tok) { + fire_error_mix_and_or_no_parenthesis(loc, lhs_op->operator_name, rhs_operator_name); + } + + // handle "arg1 && arg2 || arg3" (lhs = "arg1 && arg2") + if (is_logical_binary_op(lhs_op->tok) && is_logical_binary_op(rhs_tok) && lhs_op->tok != rhs_tok) { + fire_error_mix_and_or_no_parenthesis(loc, lhs_op->operator_name, rhs_operator_name); + } + } +} + +// diagnose "a << 8 + 1" (equivalent to "a << 9", probably unexpected) +static void diagnose_addition_in_bitshift(SrcLocation loc, std::string_view bitshift_operator_name, AnyExprV rhs) { + if (rhs->type == ast_binary_operator && is_add_or_sub_binary_op(rhs->as()->tok)) { + fire_error_lower_precedence(loc, bitshift_operator_name, rhs->as()->operator_name); + } +} + +// replace (a == null) and similar to ast_is_null_check(a) (special AST vertex) +static AnyExprV maybe_replace_eq_null_with_isNull_check(V v) { + bool has_null = v->get_lhs()->type == ast_null_keyword || v->get_rhs()->type == ast_null_keyword; + bool replace = has_null && (v->tok == tok_eq || v->tok == tok_neq); + if (!replace) { + return v; + } + + AnyExprV v_nullable = v->get_lhs()->type == ast_null_keyword ? v->get_rhs() : v->get_lhs(); + return createV(v->loc, v_nullable, v->tok == tok_neq); +} + + +/* + * + * PARSE SOURCE + * + */ + + +AnyExprV parse_expr(Lexer& lex); + +static AnyV parse_parameter(Lexer& lex, bool is_first) { + SrcLocation loc = lex.cur_location(); + + // optional keyword `mutate` meaning that a function will mutate a passed argument (like passed by reference) + bool declared_as_mutate = false; + if (lex.tok() == tok_mutate) { + lex.next(); + declared_as_mutate = true; + } + + // parameter name (or underscore for an unnamed parameter) + std::string_view param_name; + if (lex.tok() == tok_identifier) { + param_name = lex.cur_str(); + } else if (lex.tok() == tok_self) { + if (!is_first) { + lex.error("`self` can only be the first parameter"); + } + param_name = "self"; + } else if (lex.tok() != tok_underscore) { + lex.unexpected("parameter name"); + } + lex.next(); + + // parameter type after colon are mandatory + lex.expect(tok_colon, "`: `"); + TypePtr param_type = parse_type_from_tokens(lex); + + return createV(loc, param_name, param_type, declared_as_mutate); +} + +static AnyV parse_global_var_declaration(Lexer& lex, const std::vector>& annotations) { + if (!annotations.empty()) { + lex.error("@annotations are not applicable to global var declaration"); + } + SrcLocation loc = lex.cur_location(); + lex.expect(tok_global, "`global`"); + lex.check(tok_identifier, "global variable name"); + auto v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + lex.expect(tok_colon, "`:`"); + TypePtr declared_type = parse_type_from_tokens(lex); + if (lex.tok() == tok_comma) { + lex.error("multiple declarations are not allowed, split globals on separate lines"); + } + if (lex.tok() == tok_assign) { + lex.error("assigning to a global is not allowed at declaration"); + } + lex.expect(tok_semicolon, "`;`"); + return createV(loc, v_ident, declared_type); +} + +static AnyV parse_constant_declaration(Lexer& lex, const std::vector>& annotations) { + if (!annotations.empty()) { + lex.error("@annotations are not applicable to global var declaration"); + } + SrcLocation loc = lex.cur_location(); + lex.expect(tok_const, "`const`"); + lex.check(tok_identifier, "constant name"); + auto v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + TypePtr declared_type = nullptr; + if (lex.tok() == tok_colon) { + lex.next(); + declared_type = parse_type_from_tokens(lex); + } + lex.expect(tok_assign, "`=`"); + AnyExprV init_value = parse_expr(lex); + if (lex.tok() == tok_comma) { + lex.error("multiple declarations are not allowed, split constants on separate lines"); + } + lex.expect(tok_semicolon, "`;`"); + return createV(loc, v_ident, declared_type, init_value); +} + +// "parameters" are at function declaration: `fun f(param1: int, mutate param2: slice)` +static V parse_parameter_list(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + std::vector params; + lex.expect(tok_oppar, "parameter list"); + if (lex.tok() != tok_clpar) { + params.push_back(parse_parameter(lex, true)); + while (lex.tok() == tok_comma) { + lex.next(); + params.push_back(parse_parameter(lex, false)); + } + } + lex.expect(tok_clpar, "`)`"); + return createV(loc, std::move(params)); +} + +// "arguments" are at function call: `f(arg1, mutate arg2)` +static AnyExprV parse_argument(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + + // keyword `mutate` is necessary when a parameter is declared `mutate` (to make mutation obvious for the reader) + bool passed_as_mutate = false; + if (lex.tok() == tok_mutate) { + lex.next(); + passed_as_mutate = true; + } + + AnyExprV expr = parse_expr(lex); + return createV(loc, expr, passed_as_mutate); +} + +static V parse_argument_list(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + std::vector args; + lex.expect(tok_oppar, "`(`"); + if (lex.tok() != tok_clpar) { + args.push_back(parse_argument(lex)); + while (lex.tok() == tok_comma) { + lex.next(); + args.push_back(parse_argument(lex)); + } + } + lex.expect(tok_clpar, "`)`"); + return createV(loc, std::move(args)); +} + +static V parse_maybe_instantiationTs_after_identifier(Lexer& lex) { + lex.check(tok_lt, "`<`"); + Lexer::SavedPositionForLookahead backup = lex.save_parsing_position(); + try { + SrcLocation loc = lex.cur_location(); + lex.next(); + std::vector instantiationTs; + instantiationTs.push_back(createV(lex.cur_location(), parse_type_from_tokens(lex))); + while (lex.tok() == tok_comma) { + lex.next(); + instantiationTs.push_back(createV(lex.cur_location(), parse_type_from_tokens(lex))); + } + lex.expect(tok_gt, "`>`"); + return createV(loc, std::move(instantiationTs)); + } catch (const ParseError&) { + lex.restore_position(backup); + return nullptr; + } +} + +// parse (expr) / [expr] / identifier / number +static AnyExprV parse_expr100(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + switch (lex.tok()) { + case tok_oppar: { + lex.next(); + if (lex.tok() == tok_clpar) { + lex.next(); + return createV(loc, {}); + } + AnyExprV first = parse_expr(lex); + if (lex.tok() == tok_clpar) { + lex.next(); + return createV(loc, first); + } + std::vector items(1, first); + while (lex.tok() == tok_comma) { + lex.next(); + items.emplace_back(parse_expr(lex)); + } + lex.expect(tok_clpar, "`)`"); + return createV(loc, std::move(items)); + } + case tok_opbracket: { + lex.next(); + if (lex.tok() == tok_clbracket) { + lex.next(); + return createV(loc, {}); + } + std::vector items(1, parse_expr(lex)); + while (lex.tok() == tok_comma) { + lex.next(); + items.emplace_back(parse_expr(lex)); + } + lex.expect(tok_clbracket, "`]`"); + return createV(loc, std::move(items)); + } + case tok_int_const: { + std::string_view orig_str = lex.cur_str(); + td::RefInt256 intval = td::string_to_int256(static_cast(orig_str)); + if (intval.is_null() || !intval->signed_fits_bits(257)) { + lex.error("invalid integer constant"); + } + lex.next(); + return createV(loc, std::move(intval), orig_str); + } + case tok_string_const: { + std::string_view str_val = lex.cur_str(); + lex.next(); + char modifier = 0; + if (lex.tok() == tok_string_modifier) { + modifier = lex.cur_str()[0]; + lex.next(); + } + return createV(loc, str_val, modifier); + } + case tok_underscore: { + lex.next(); + return createV(loc); + } + case tok_true: { + lex.next(); + return createV(loc, true); + } + case tok_false: { + lex.next(); + return createV(loc, false); + } + case tok_null: { + lex.next(); + return createV(loc); + } + case tok_self: { + lex.next(); + auto v_ident = createV(loc, "self"); + return createV(loc, v_ident, nullptr); + } + case tok_identifier: { + auto v_ident = createV(loc, lex.cur_str()); + V v_instantiationTs = nullptr; + lex.next(); + if (lex.tok() == tok_lt) { + v_instantiationTs = parse_maybe_instantiationTs_after_identifier(lex); + } + return createV(loc, v_ident, v_instantiationTs); + } + default: + lex.unexpected(""); + } +} + +// parse E(...) and E! having parsed E already (left-to-right) +static AnyExprV parse_fun_call_postfix(Lexer& lex, AnyExprV lhs) { + while (true) { + if (lex.tok() == tok_oppar) { + lhs = createV(lhs->loc, lhs, parse_argument_list(lex)); + } else if (lex.tok() == tok_logical_not) { + lex.next(); + lhs = createV(lhs->loc, lhs); + } else { + break; + } + } + return lhs; +} + +// parse E(...) and E! (left-to-right) +static AnyExprV parse_expr90(Lexer& lex) { + AnyExprV res = parse_expr100(lex); + if (lex.tok() == tok_oppar || lex.tok() == tok_logical_not) { + res = parse_fun_call_postfix(lex, res); + } + return res; +} + +// parse E.field and E.method(...) and E.field! (left-to-right) +static AnyExprV parse_expr80(Lexer& lex) { + AnyExprV lhs = parse_expr90(lex); + while (lex.tok() == tok_dot) { + SrcLocation loc = lex.cur_location(); + lex.next(); + V v_ident = nullptr; + V v_instantiationTs = nullptr; + if (lex.tok() == tok_identifier) { // obj.field / obj.method + v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + if (lex.tok() == tok_lt) { + v_instantiationTs = parse_maybe_instantiationTs_after_identifier(lex); + } + } else if (lex.tok() == tok_int_const) { // obj.0 (indexed access) + v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + } else { + lex.unexpected("method name"); + } + lhs = createV(loc, lhs, v_ident, v_instantiationTs); + if (lex.tok() == tok_oppar || lex.tok() == tok_logical_not) { + lhs = parse_fun_call_postfix(lex, lhs); + } + } + return lhs; +} + +// parse ! ~ - + E (unary) +static AnyExprV parse_expr75(Lexer& lex) { + TokenType t = lex.tok(); + if (t == tok_logical_not || t == tok_bitwise_not || t == tok_minus || t == tok_plus) { + SrcLocation loc = lex.cur_location(); + std::string_view operator_name = lex.cur_str(); + lex.next(); + AnyExprV rhs = parse_expr75(lex); + return createV(loc, operator_name, t, rhs); + } + return parse_expr80(lex); +} + +// parse E as +static AnyExprV parse_expr40(Lexer& lex) { + AnyExprV lhs = parse_expr75(lex); + if (lex.tok() == tok_as) { + SrcLocation loc = lex.cur_location(); + lex.next(); + TypePtr cast_to_type = parse_type_from_tokens(lex); + lhs = createV(loc, lhs, cast_to_type); + } + return lhs; +} + +// parse E * / % ^/ ~/ E (left-to-right) +static AnyExprV parse_expr30(Lexer& lex) { + AnyExprV lhs = parse_expr40(lex); + TokenType t = lex.tok(); + while (t == tok_mul || t == tok_div || t == tok_mod || t == tok_divC || t == tok_divR) { + SrcLocation loc = lex.cur_location(); + std::string_view operator_name = lex.cur_str(); + lex.next(); + AnyExprV rhs = parse_expr40(lex); + lhs = createV(loc, operator_name, t, lhs, rhs); + t = lex.tok(); + } + return lhs; +} + +// parse E + - E (left-to-right) +static AnyExprV parse_expr20(Lexer& lex) { + AnyExprV lhs = parse_expr30(lex); + TokenType t = lex.tok(); + while (t == tok_minus || t == tok_plus) { + SrcLocation loc = lex.cur_location(); + std::string_view operator_name = lex.cur_str(); + lex.next(); + AnyExprV rhs = parse_expr30(lex); + lhs = createV(loc, operator_name, t, lhs, rhs); + t = lex.tok(); + } + return lhs; +} + +// parse E << >> ~>> ^>> E (left-to-right) +static AnyExprV parse_expr17(Lexer& lex) { + AnyExprV lhs = parse_expr20(lex); + TokenType t = lex.tok(); + while (t == tok_lshift || t == tok_rshift || t == tok_rshiftC || t == tok_rshiftR) { + SrcLocation loc = lex.cur_location(); + std::string_view operator_name = lex.cur_str(); + lex.next(); + AnyExprV rhs = parse_expr20(lex); + diagnose_addition_in_bitshift(loc, operator_name, rhs); + lhs = createV(loc, operator_name, t, lhs, rhs); + t = lex.tok(); + } + return lhs; +} + +// parse E == < > <= >= != <=> E (left-to-right) +static AnyExprV parse_expr15(Lexer& lex) { + AnyExprV lhs = parse_expr17(lex); + TokenType t = lex.tok(); + if (t == tok_eq || t == tok_lt || t == tok_gt || t == tok_leq || t == tok_geq || t == tok_neq || t == tok_spaceship) { + SrcLocation loc = lex.cur_location(); + std::string_view operator_name = lex.cur_str(); + lex.next(); + AnyExprV rhs = parse_expr17(lex); + lhs = createV(loc, operator_name, t, lhs, rhs); + if (t == tok_eq || t == tok_neq) { + lhs = maybe_replace_eq_null_with_isNull_check(lhs->as()); + } + } + return lhs; +} + +// parse E & | ^ E (left-to-right) +static AnyExprV parse_expr14(Lexer& lex) { + AnyExprV lhs = parse_expr15(lex); + TokenType t = lex.tok(); + while (t == tok_bitwise_and || t == tok_bitwise_or || t == tok_bitwise_xor) { + SrcLocation loc = lex.cur_location(); + std::string_view operator_name = lex.cur_str(); + lex.next(); + AnyExprV rhs = parse_expr15(lex); + diagnose_bitwise_precedence(loc, operator_name, lhs, rhs); + diagnose_and_or_precedence(loc, lhs, t, operator_name); + lhs = createV(loc, operator_name, t, lhs, rhs); + t = lex.tok(); + } + return lhs; +} + +// parse E && || E (left-to-right) +static AnyExprV parse_expr13(Lexer& lex) { + AnyExprV lhs = parse_expr14(lex); + TokenType t = lex.tok(); + while (t == tok_logical_and || t == tok_logical_or) { + SrcLocation loc = lex.cur_location(); + std::string_view operator_name = lex.cur_str(); + lex.next(); + AnyExprV rhs = parse_expr14(lex); + diagnose_and_or_precedence(loc, lhs, t, operator_name); + lhs = createV(loc, operator_name, t, lhs, rhs); + t = lex.tok(); + } + return lhs; +} + +// parse E = += -= E and E ? E : E (right-to-left) +static AnyExprV parse_expr10(Lexer& lex) { + AnyExprV lhs = parse_expr13(lex); + TokenType t = lex.tok(); + if (t == tok_assign) { + SrcLocation loc = lex.cur_location(); + lex.next(); + AnyExprV rhs = parse_expr10(lex); + return createV(loc, lhs, rhs); + } + if (t == tok_set_plus || t == tok_set_minus || t == tok_set_mul || t == tok_set_div || + t == tok_set_mod || t == tok_set_lshift || t == tok_set_rshift || + t == tok_set_bitwise_and || t == tok_set_bitwise_or || t == tok_set_bitwise_xor) { + SrcLocation loc = lex.cur_location(); + std::string_view operator_name = lex.cur_str().substr(0, lex.cur_str().size() - 1); // "+" for += + lex.next(); + AnyExprV rhs = parse_expr10(lex); + return createV(loc, operator_name, t, lhs, rhs); + } + if (t == tok_question) { + SrcLocation loc = lex.cur_location(); + lex.next(); + AnyExprV when_true = parse_expr10(lex); + lex.expect(tok_colon, "`:`"); + AnyExprV when_false = parse_expr10(lex); + return createV(loc, lhs, when_true, when_false); + } + return lhs; +} + +AnyExprV parse_expr(Lexer& lex) { + return parse_expr10(lex); +} + +AnyV parse_statement(Lexer& lex); + +static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { + SrcLocation loc = lex.cur_location(); + if (lex.tok() == tok_oppar) { + lex.next(); + AnyExprV first = parse_var_declaration_lhs(lex, is_immutable); + if (lex.tok() == tok_clpar) { + lex.next(); + return first; + } + std::vector args(1, first); + while (lex.tok() == tok_comma) { + lex.next(); + args.push_back(parse_var_declaration_lhs(lex, is_immutable)); + } + lex.expect(tok_clpar, "`)`"); + return createV(loc, std::move(args)); + } + if (lex.tok() == tok_opbracket) { + lex.next(); + std::vector args(1, parse_var_declaration_lhs(lex, is_immutable)); + while (lex.tok() == tok_comma) { + lex.next(); + args.push_back(parse_var_declaration_lhs(lex, is_immutable)); + } + lex.expect(tok_clbracket, "`]`"); + return createV(loc, std::move(args)); + } + if (lex.tok() == tok_identifier) { + auto v_ident = createV(loc, lex.cur_str()); + TypePtr declared_type = nullptr; + bool marked_as_redef = false; + lex.next(); + if (lex.tok() == tok_colon) { + lex.next(); + declared_type = parse_type_from_tokens(lex); + } else if (lex.tok() == tok_redef) { + lex.next(); + marked_as_redef = true; + } + return createV(loc, v_ident, declared_type, is_immutable, marked_as_redef); + } + if (lex.tok() == tok_underscore) { + TypePtr declared_type = nullptr; + lex.next(); + if (lex.tok() == tok_colon) { + lex.next(); + declared_type = parse_type_from_tokens(lex); + } + return createV(loc, createV(loc, ""), declared_type, true, false); + } + lex.unexpected("variable name"); +} + +static AnyV parse_local_vars_declaration_assignment(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + bool is_immutable = lex.tok() == tok_val; + lex.next(); + + AnyExprV lhs = createV(loc, parse_var_declaration_lhs(lex, is_immutable)); + if (lex.tok() != tok_assign) { + lex.error("variables declaration must be followed by assignment: `var xxx = ...`"); + } + lex.next(); + AnyExprV rhs = parse_expr(lex); + + if (lex.tok() == tok_comma) { + lex.error("multiple declarations are not allowed, split variables on separate lines"); + } + lex.expect(tok_semicolon, "`;`"); + return createV(loc, lhs, rhs); +} + +static V parse_sequence(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_opbrace, "`{`"); + std::vector items; + while (lex.tok() != tok_clbrace) { + items.push_back(parse_statement(lex)); + } + SrcLocation loc_end = lex.cur_location(); + lex.expect(tok_clbrace, "`}`"); + return createV(loc, loc_end, items); +} + +static AnyV parse_return_statement(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_return, "`return`"); + AnyExprV child = lex.tok() == tok_semicolon // `return;` actually means "nothing" (inferred as void) + ? createV(lex.cur_location()) + : parse_expr(lex); + lex.expect(tok_semicolon, "`;`"); + return createV(loc, child); +} + +static AnyV parse_if_statement(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_if, "`if`"); + + lex.expect(tok_oppar, "`(`"); + AnyExprV cond = parse_expr(lex); + lex.expect(tok_clpar, "`)`"); + + V if_body = parse_sequence(lex); + V else_body = nullptr; + if (lex.tok() == tok_else) { // else if(e) { } or else { } + lex.next(); + if (lex.tok() == tok_if) { + AnyV v_inner_if = parse_if_statement(lex); + else_body = createV(v_inner_if->loc, lex.cur_location(), {v_inner_if}); + } else { + else_body = parse_sequence(lex); + } + } else { // no 'else', create empty block + else_body = createV(lex.cur_location(), lex.cur_location(), {}); + } + return createV(loc, false, cond, if_body, else_body); +} + +static AnyV parse_repeat_statement(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_repeat, "`repeat`"); + lex.expect(tok_oppar, "`(`"); + AnyExprV cond = parse_expr(lex); + lex.expect(tok_clpar, "`)`"); + V body = parse_sequence(lex); + return createV(loc, cond, body); +} + +static AnyV parse_while_statement(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_while, "`while`"); + lex.expect(tok_oppar, "`(`"); + AnyExprV cond = parse_expr(lex); + lex.expect(tok_clpar, "`)`"); + V body = parse_sequence(lex); + return createV(loc, cond, body); +} + +static AnyV parse_do_while_statement(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_do, "`do`"); + V body = parse_sequence(lex); + lex.expect(tok_while, "`while`"); + lex.expect(tok_oppar, "`(`"); + AnyExprV cond = parse_expr(lex); + lex.expect(tok_clpar, "`)`"); + lex.expect(tok_semicolon, "`;`"); + return createV(loc, body, cond); +} + +static AnyExprV parse_catch_variable(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + if (lex.tok() == tok_identifier) { + std::string_view var_name = lex.cur_str(); + lex.next(); + auto v_ident = createV(loc, var_name); + return createV(loc, v_ident, nullptr); + } + if (lex.tok() == tok_underscore) { + lex.next(); + auto v_ident = createV(loc, ""); + return createV(loc, v_ident, nullptr); + } + lex.unexpected("identifier"); +} + +static AnyExprV create_catch_underscore_variable(const Lexer& lex) { + auto v_ident = createV(lex.cur_location(), ""); + return createV(lex.cur_location(), v_ident, nullptr); +} + +static AnyV parse_throw_statement(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_throw, "`throw`"); + + AnyExprV thrown_code, thrown_arg; + if (lex.tok() == tok_oppar) { // throw (code) or throw (code, arg) + lex.next(); + thrown_code = parse_expr(lex); + if (lex.tok() == tok_comma) { + lex.next(); + thrown_arg = parse_expr(lex); + } else { + thrown_arg = createV(loc); + } + lex.expect(tok_clpar, "`)`"); + } else { // throw code + thrown_code = parse_expr(lex); + thrown_arg = createV(loc); + } + + lex.expect(tok_semicolon, "`;`"); + return createV(loc, thrown_code, thrown_arg); +} + +static AnyV parse_assert_statement(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_assert, "`assert`"); + + lex.expect(tok_oppar, "`(`"); + AnyExprV cond = parse_expr(lex); + AnyExprV thrown_code; + if (lex.tok() == tok_comma) { // assert(cond, code) + lex.next(); + thrown_code = parse_expr(lex); + lex.expect(tok_clpar, "`)`"); + } else { // assert(cond) throw code + lex.expect(tok_clpar, "`)`"); + lex.expect(tok_throw, "`throw excNo` after assert"); + thrown_code = parse_expr(lex); + } + + lex.expect(tok_semicolon, "`;`"); + return createV(loc, cond, thrown_code); +} + +static AnyV parse_try_catch_statement(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_try, "`try`"); + V try_body = parse_sequence(lex); + + std::vector catch_args; + lex.expect(tok_catch, "`catch`"); + SrcLocation catch_loc = lex.cur_location(); + if (lex.tok() == tok_oppar) { + lex.next(); + catch_args.push_back(parse_catch_variable(lex)); + if (lex.tok() == tok_comma) { // catch (excNo, arg) + lex.next(); + catch_args.push_back(parse_catch_variable(lex)); + } else { // catch (excNo) -> catch (excNo, _) + catch_args.push_back(create_catch_underscore_variable(lex)); + } + lex.expect(tok_clpar, "`)`"); + } else { // catch -> catch (_, _) + catch_args.push_back(create_catch_underscore_variable(lex)); + catch_args.push_back(create_catch_underscore_variable(lex)); + } + V catch_expr = createV(catch_loc, std::move(catch_args)); + + V catch_body = parse_sequence(lex); + return createV(loc, try_body, catch_expr, catch_body); +} + +AnyV parse_statement(Lexer& lex) { + switch (lex.tok()) { + case tok_var: // `var x = 0` is technically an expression, but can not appear in "any place", + case tok_val: // only as a separate declaration + return parse_local_vars_declaration_assignment(lex); + case tok_opbrace: + return parse_sequence(lex); + case tok_return: + return parse_return_statement(lex); + case tok_if: + return parse_if_statement(lex); + case tok_repeat: + return parse_repeat_statement(lex); + case tok_do: + return parse_do_while_statement(lex); + case tok_while: + return parse_while_statement(lex); + case tok_throw: + return parse_throw_statement(lex); + case tok_assert: + return parse_assert_statement(lex); + case tok_try: + return parse_try_catch_statement(lex); + case tok_semicolon: { + SrcLocation loc = lex.cur_location(); + lex.next(); + return createV(loc); + } + case tok_break: + case tok_continue: + lex.error("break/continue from loops are not supported yet"); + default: { + AnyExprV expr = parse_expr(lex); + lex.expect(tok_semicolon, "`;`"); + return expr; + } + } +} + +static AnyV parse_func_body(Lexer& lex) { + return parse_sequence(lex); +} + +static AnyV parse_asm_func_body(Lexer& lex, V param_list) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_asm, "`asm`"); + size_t n_params = param_list->size(); + if (n_params > 16) { + throw ParseError{loc, "assembler built-in function can have at most 16 arguments"}; + } + std::vector arg_order, ret_order; + if (lex.tok() == tok_oppar) { + lex.next(); + while (lex.tok() == tok_identifier || lex.tok() == tok_self) { + int arg_idx = param_list->lookup_idx(lex.cur_str()); + if (arg_idx == -1) { + lex.unexpected("parameter name"); + } + arg_order.push_back(arg_idx); + lex.next(); + } + if (lex.tok() == tok_arrow) { + lex.next(); + while (lex.tok() == tok_int_const) { + int ret_idx = std::atoi(static_cast(lex.cur_str()).c_str()); + ret_order.push_back(ret_idx); + lex.next(); + } + } + lex.expect(tok_clpar, "`)`"); + } + std::vector asm_commands; + lex.check(tok_string_const, "\"ASM COMMAND\""); + while (lex.tok() == tok_string_const) { + std::string_view asm_command = lex.cur_str(); + asm_commands.push_back(createV(lex.cur_location(), asm_command, 0)); + lex.next(); + } + lex.expect(tok_semicolon, "`;`"); + return createV(loc, std::move(arg_order), std::move(ret_order), std::move(asm_commands)); +} + +static AnyV parse_genericsT_list(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + std::vector genericsT_items; + lex.expect(tok_lt, "`<`"); + while (true) { + lex.check(tok_identifier, "T"); + std::string_view nameT = lex.cur_str(); + genericsT_items.emplace_back(createV(lex.cur_location(), nameT)); + lex.next(); + if (lex.tok() != tok_comma) { + break; + } + lex.next(); + } + lex.expect(tok_gt, "`>`"); + return createV{loc, std::move(genericsT_items)}; +} + +static V parse_annotation(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.check(tok_annotation_at, "`@`"); + std::string_view name = lex.cur_str(); + AnnotationKind kind = Vertex::parse_kind(name); + lex.next(); + + V v_arg = nullptr; + if (lex.tok() == tok_oppar) { + SrcLocation loc_args = lex.cur_location(); + lex.next(); + std::vector args; + args.push_back(parse_expr(lex)); + while (lex.tok() == tok_comma) { + lex.next(); + args.push_back(parse_expr(lex)); + } + lex.expect(tok_clpar, "`)`"); + v_arg = createV(loc_args, std::move(args)); + } + + switch (kind) { + case AnnotationKind::unknown: + throw ParseError(loc, "unknown annotation " + static_cast(name)); + case AnnotationKind::inline_simple: + case AnnotationKind::inline_ref: + case AnnotationKind::pure: + case AnnotationKind::deprecated: + if (v_arg) { + throw ParseError(v_arg->loc, "arguments aren't allowed for " + static_cast(name)); + } + v_arg = createV(loc, {}); + break; + case AnnotationKind::method_id: + if (!v_arg || v_arg->size() != 1 || v_arg->get_item(0)->type != ast_int_const) { + throw ParseError(loc, "expecting `(number)` after " + static_cast(name)); + } + break; + } + + return createV(loc, kind, v_arg); +} + +static AnyV parse_function_declaration(Lexer& lex, const std::vector>& annotations) { + SrcLocation loc = lex.cur_location(); + bool is_get_method = lex.tok() == tok_get; + lex.next(); + if (is_get_method && lex.tok() == tok_fun) { + lex.next(); // 'get f()' and 'get fun f()' both correct + } + + lex.check(tok_identifier, "function name identifier"); + + std::string_view f_name = lex.cur_str(); + bool is_entrypoint = + f_name == "main" || f_name == "onInternalMessage" || f_name == "onExternalMessage" || + f_name == "onRunTickTock" || f_name == "onSplitPrepare" || f_name == "onSplitInstall"; + bool is_FunC_entrypoint = + f_name == "recv_internal" || f_name == "recv_external" || + f_name == "run_ticktock" || f_name == "split_prepare" || f_name == "split_install"; + if (is_FunC_entrypoint) { + lex.error("this is a reserved FunC/Fift identifier; you need `onInternalMessage`"); + } + + auto v_ident = createV(lex.cur_location(), f_name); + lex.next(); + + V genericsT_list = nullptr; + if (lex.tok() == tok_lt) { // 'fun f' + genericsT_list = parse_genericsT_list(lex)->as(); + } + + V v_param_list = parse_parameter_list(lex)->as(); + bool accepts_self = !v_param_list->empty() && v_param_list->get_param(0)->param_name == "self"; + int n_mutate_params = v_param_list->get_mutate_params_count(); + + TypePtr ret_type = nullptr; + bool returns_self = false; + if (lex.tok() == tok_colon) { // : (if absent, it means "auto infer", not void) + lex.next(); + if (lex.tok() == tok_self) { + if (!accepts_self) { + lex.error("only a member function can return `self` (which accepts `self` first parameter)"); + } + lex.next(); + returns_self = true; + ret_type = TypeDataVoid::create(); + } else { + ret_type = parse_type_from_tokens(lex); + } + } + + if (is_entrypoint && (is_get_method || genericsT_list || n_mutate_params || accepts_self)) { + throw ParseError(loc, "invalid declaration of a reserved function"); + } + if (is_get_method && (genericsT_list || n_mutate_params || accepts_self)) { + throw ParseError(loc, "get methods can't have `mutate` and `self` params"); + } + + AnyV v_body = nullptr; + + if (lex.tok() == tok_builtin) { + v_body = createV(lex.cur_location()); + lex.next(); + lex.expect(tok_semicolon, "`;`"); + } else if (lex.tok() == tok_opbrace) { + v_body = parse_func_body(lex); + } else if (lex.tok() == tok_asm) { + if (!ret_type) { + lex.error("asm function must specify return type"); + } + v_body = parse_asm_func_body(lex, v_param_list); + } else { + lex.unexpected("{ function body }"); + } + + int flags = 0; + if (is_entrypoint) { + flags |= FunctionData::flagIsEntrypoint; + } + if (is_get_method) { + flags |= FunctionData::flagGetMethod; + } + if (accepts_self) { + flags |= FunctionData::flagAcceptsSelf; + } + if (returns_self) { + flags |= FunctionData::flagReturnsSelf; + } + + td::RefInt256 method_id; + for (auto v_annotation : annotations) { + switch (v_annotation->kind) { + case AnnotationKind::inline_simple: + flags |= FunctionData::flagInline; + break; + case AnnotationKind::inline_ref: + flags |= FunctionData::flagInlineRef; + break; + case AnnotationKind::pure: + flags |= FunctionData::flagMarkedAsPure; + break; + case AnnotationKind::method_id: { + if (is_get_method || genericsT_list || is_entrypoint || n_mutate_params || accepts_self) { + v_annotation->error("@method_id can be specified only for regular functions"); + } + auto v_int = v_annotation->get_arg()->get_item(0)->as(); + if (v_int->intval.is_null() || !v_int->intval->signed_fits_bits(32)) { + v_int->error("invalid integer constant"); + } + method_id = v_int->intval; + break; + } + case AnnotationKind::deprecated: + // no special handling + break; + + default: + v_annotation->error("this annotation is not applicable to functions"); + } + } + + return createV(loc, v_ident, v_param_list, v_body, ret_type, genericsT_list, std::move(method_id), flags); +} + +static AnyV parse_tolk_required_version(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.next_special(tok_semver, "semver"); // syntax: "tolk 0.6" + std::string semver = static_cast(lex.cur_str()); + lex.next(); + + // for simplicity, there is no syntax ">= version" and so on, just strict compare + if (TOLK_VERSION != semver && TOLK_VERSION != semver + ".0") { // 0.6 = 0.6.0 + loc.show_warning("the contract is written in Tolk v" + semver + ", but you use Tolk compiler v" + TOLK_VERSION + "; probably, it will lead to compilation errors or hash changes"); + } + + return createV(loc, semver); // semicolon is not necessary +} + +static AnyV parse_import_directive(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_import, "`import`"); + lex.check(tok_string_const, "source file name"); + std::string_view rel_filename = lex.cur_str(); + if (rel_filename.empty()) { + lex.error("imported file name is an empty string"); + } + auto v_str = createV(lex.cur_location(), rel_filename, 0); + lex.next(); + return createV(loc, v_str); // semicolon is not necessary +} + +// the main (exported) function +AnyV parse_src_file_to_ast(const SrcFile* file) { + std::vector toplevel_declarations; + std::vector> annotations; + Lexer lex(file); + + while (!lex.is_eof()) { + switch (lex.tok()) { + case tok_tolk: + if (!annotations.empty()) { + lex.unexpected("declaration after @annotations"); + } + toplevel_declarations.push_back(parse_tolk_required_version(lex)); + break; + case tok_import: + if (!annotations.empty()) { + lex.unexpected("declaration after @annotations"); + } + toplevel_declarations.push_back(parse_import_directive(lex)); + break; + case tok_semicolon: + if (!annotations.empty()) { + lex.unexpected("declaration after @annotations"); + } + lex.next(); // don't add ast_empty, no need + break; + + case tok_annotation_at: + annotations.push_back(parse_annotation(lex)); + break; + case tok_global: + toplevel_declarations.push_back(parse_global_var_declaration(lex, annotations)); + annotations.clear(); + break; + case tok_const: + toplevel_declarations.push_back(parse_constant_declaration(lex, annotations)); + annotations.clear(); + break; + case tok_fun: + case tok_get: + toplevel_declarations.push_back(parse_function_declaration(lex, annotations)); + annotations.clear(); + break; + + case tok_export: + case tok_struct: + case tok_enum: + case tok_operator: + case tok_infix: + lex.error("`" + static_cast(lex.cur_str()) +"` is not supported yet"); + + default: + lex.unexpected("fun or get"); + } + } + + return createV(file, std::move(toplevel_declarations)); +} + +} // namespace tolk diff --git a/tolk/ast-from-tokens.h b/tolk/ast-from-tokens.h new file mode 100644 index 00000000..39574f9c --- /dev/null +++ b/tolk/ast-from-tokens.h @@ -0,0 +1,25 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "fwd-declarations.h" + +namespace tolk { + +AnyV parse_src_file_to_ast(const SrcFile* file); + +} // namespace tolk diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h new file mode 100644 index 00000000..5103cc92 --- /dev/null +++ b/tolk/ast-replacer.h @@ -0,0 +1,201 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "ast.h" +#include "platform-utils.h" + +/* + * A module of implementing traversing a vertex tree and replacing any vertex to another. + * For example, to replace "beginCell()" call to "begin_cell()" in a function body (in V) + * regardless of the place this call is performed, you need to iterate over all the function AST, + * to find ast_function_call(beginCell), create ast_function_call(begin_cell) instead and to replace + * a pointer inside its parent. + * Inheriting from ASTVisitor makes this task quite simple, without any boilerplate. + * + * If you need just to traverse a vertex tree without replacing vertices, + * consider another api: ast-visitor.h. + */ + +namespace tolk { + +class ASTReplacer { +protected: + GNU_ATTRIBUTE_ALWAYS_INLINE static AnyExprV replace_children(const ASTExprLeaf* v) { + return v; + } + + GNU_ATTRIBUTE_ALWAYS_INLINE AnyExprV replace_children(const ASTExprUnary* v) { + auto* v_mutable = const_cast(v); + v_mutable->child = replace(v_mutable->child); + return v_mutable; + } + + GNU_ATTRIBUTE_ALWAYS_INLINE AnyExprV replace_children(const ASTExprBinary* v) { + auto* v_mutable = const_cast(v); + v_mutable->lhs = replace(v->lhs); + v_mutable->rhs = replace(v->rhs); + return v_mutable; + } + + GNU_ATTRIBUTE_ALWAYS_INLINE AnyExprV replace_children(const ASTExprVararg* v) { + auto* v_mutable = const_cast(v); + for (AnyExprV& child : v_mutable->children) { + child = replace(child); + } + return v_mutable; + } + + GNU_ATTRIBUTE_ALWAYS_INLINE AnyV replace_children(const ASTStatementUnary* v) { + auto* v_mutable = const_cast(v); + v_mutable->child = replace(v_mutable->child); + return v_mutable; + } + + GNU_ATTRIBUTE_ALWAYS_INLINE AnyV replace_children(const ASTStatementVararg* v) { + auto* v_mutable = const_cast(v); + for (AnyV& child : v_mutable->children) { + child = replace(child); + } + return v_mutable; + } + +public: + virtual ~ASTReplacer() = default; + + virtual AnyV replace(AnyV v) = 0; + virtual AnyExprV replace(AnyExprV v) = 0; +}; + +class ASTReplacerInFunctionBody : public ASTReplacer { +protected: + using parent = ASTReplacerInFunctionBody; + + // expressions + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + // statements + virtual AnyV replace(V v) { return replace_children(v); } + virtual AnyV replace(V v) { return replace_children(v); } + virtual AnyV replace(V v) { return replace_children(v); } + virtual AnyV replace(V v) { return replace_children(v); } + virtual AnyV replace(V v) { return replace_children(v); } + virtual AnyV replace(V v) { return replace_children(v); } + virtual AnyV replace(V v) { return replace_children(v); } + virtual AnyV replace(V v) { return replace_children(v); } + virtual AnyV replace(V v) { return replace_children(v); } + virtual AnyV replace(V v) { return replace_children(v); } + + AnyExprV replace(AnyExprV v) final { + switch (v->type) { + case ast_empty_expression: return replace(v->as()); + case ast_parenthesized_expression: return replace(v->as()); + case ast_tensor: return replace(v->as()); + case ast_typed_tuple: return replace(v->as()); + case ast_reference: return replace(v->as()); + case ast_local_var_lhs: return replace(v->as()); + case ast_local_vars_declaration: return replace(v->as()); + case ast_int_const: return replace(v->as()); + case ast_string_const: return replace(v->as()); + case ast_bool_const: return replace(v->as()); + case ast_null_keyword: return replace(v->as()); + case ast_argument: return replace(v->as()); + case ast_argument_list: return replace(v->as()); + case ast_dot_access: return replace(v->as()); + case ast_function_call: return replace(v->as()); + case ast_underscore: return replace(v->as()); + case ast_assign: return replace(v->as()); + case ast_set_assign: return replace(v->as()); + case ast_unary_operator: return replace(v->as()); + case ast_binary_operator: return replace(v->as()); + case ast_ternary_operator: return replace(v->as()); + case ast_cast_as_operator: return replace(v->as()); + case ast_not_null_operator: return replace(v->as()); + case ast_is_null_check: return replace(v->as()); + default: + throw UnexpectedASTNodeType(v, "ASTReplacerInFunctionBody::replace"); + } + } + + AnyV replace(AnyV v) final { + switch (v->type) { + case ast_empty_statement: return replace(v->as()); + case ast_sequence: return replace(v->as()); + case ast_return_statement: return replace(v->as()); + case ast_if_statement: return replace(v->as()); + case ast_repeat_statement: return replace(v->as()); + case ast_while_statement: return replace(v->as()); + case ast_do_while_statement: return replace(v->as()); + case ast_throw_statement: return replace(v->as()); + case ast_assert_statement: return replace(v->as()); + case ast_try_catch_statement: return replace(v->as()); +#ifdef TOLK_DEBUG + case ast_asm_body: + throw UnexpectedASTNodeType(v, "ASTReplacer::replace"); +#endif + default: { + // be very careful, don't forget to handle all statements (not expressions) above! + AnyExprV as_expr = reinterpret_cast(v); + return replace(as_expr); + } + } + } + +public: + virtual bool should_visit_function(FunctionPtr fun_ref) = 0; + + void start_replacing_in_function(FunctionPtr fun_ref, V v_function) { + replace(v_function->get_body()); + } +}; + + +const std::vector& get_all_not_builtin_functions(); + +template +void replace_ast_of_all_functions() { + BodyReplacerT visitor; + for (FunctionPtr fun_ref : get_all_not_builtin_functions()) { + if (visitor.should_visit_function(fun_ref)) { + visitor.start_replacing_in_function(fun_ref, fun_ref->ast_root->as()); + } + } +} + +} // namespace tolk diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h new file mode 100644 index 00000000..16bbbeb8 --- /dev/null +++ b/tolk/ast-replicator.h @@ -0,0 +1,263 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "ast.h" +#include "platform-utils.h" + +namespace tolk { + +class ASTReplicator { +protected: + virtual AnyV clone(AnyV v) = 0; + virtual AnyExprV clone(AnyExprV v) = 0; + virtual TypePtr clone(TypePtr) = 0; + +public: + virtual ~ASTReplicator() = default; +}; + +class ASTReplicatorFunction : public ASTReplicator { +protected: + using parent = ASTReplicatorFunction; + + std::vector clone(const std::vector& items) { + std::vector result; + result.reserve(items.size()); + for (AnyV item : items) { + result.push_back(clone(item)); + } + return result; + } + + std::vector clone(const std::vector& items) { + std::vector result; + result.reserve(items.size()); + for (AnyExprV item : items) { + result.push_back(clone(item)); + } + return result; + } + + // expressions + + virtual V clone(V v) { + return createV(v->loc); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_expr())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_items())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_items())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_identifier()), v->has_instantiationTs() ? clone(v->get_instantiationTs()) : nullptr); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_identifier()), clone(v->declared_type), v->is_immutable, v->marked_as_redef); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_expr())); + } + virtual V clone(V v) { + return createV(v->loc, v->intval, v->orig_str); + } + virtual V clone(V v) { + return createV(v->loc, v->str_val, v->modifier); + } + virtual V clone(V v) { + return createV(v->loc, v->bool_val); + } + virtual V clone(V v) { + return createV(v->loc); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_expr()), v->passed_as_mutate); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_arguments())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_obj()), clone(v->get_identifier()), v->has_instantiationTs() ? clone(v->get_instantiationTs()) : nullptr); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_callee()), clone(v->get_arg_list())); + } + virtual V clone(V v) { + return createV(v->loc); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_lhs()), clone(v->get_rhs())); + } + virtual V clone(V v) { + return createV(v->loc, v->operator_name, v->tok, clone(v->get_lhs()), clone(v->get_rhs())); + } + virtual V clone(V v) { + return createV(v->loc, v->operator_name, v->tok, clone(v->get_rhs())); + } + virtual V clone(V v) { + return createV(v->loc, v->operator_name, v->tok, clone(v->get_lhs()), clone(v->get_rhs())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_cond()), clone(v->get_when_true()), clone(v->get_when_false())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_expr()), clone(v->cast_to_type)); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_expr())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_expr()), v->is_negated); + } + + // statements + + virtual V clone(V v) { + return createV(v->loc); + } + virtual V clone(V v) { + return createV(v->loc, v->loc_end, clone(v->get_items())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_return_value())); + } + virtual V clone(V v) { + return createV(v->loc, v->is_ifnot, clone(v->get_cond()), clone(v->get_if_body()), clone(v->get_else_body())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_cond()), clone(v->get_body())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_cond()), clone(v->get_body())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_body()), clone(v->get_cond())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_thrown_code()), clone(v->get_thrown_arg())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_cond()), clone(v->get_thrown_code())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_try_body()), clone(v->get_catch_expr()), clone(v->get_catch_body())); + } + virtual V clone(V v) { + return createV(v->loc, v->arg_order, v->ret_order, clone(v->get_asm_commands())); + } + + // other + + virtual V clone(V v) { + return createV(v->loc, v->name); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->substituted_type)); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_items())); + } + virtual V clone(V v) { + return createV(v->loc, v->param_name, clone(v->declared_type), v->declared_as_mutate); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_params())); + } + + AnyExprV clone(AnyExprV v) final { + switch (v->type) { + case ast_empty_expression: return clone(v->as()); + case ast_parenthesized_expression: return clone(v->as()); + case ast_tensor: return clone(v->as()); + case ast_typed_tuple: return clone(v->as()); + case ast_reference: return clone(v->as()); + case ast_local_var_lhs: return clone(v->as()); + case ast_local_vars_declaration: return clone(v->as()); + case ast_int_const: return clone(v->as()); + case ast_string_const: return clone(v->as()); + case ast_bool_const: return clone(v->as()); + case ast_null_keyword: return clone(v->as()); + case ast_argument: return clone(v->as()); + case ast_argument_list: return clone(v->as()); + case ast_dot_access: return clone(v->as()); + case ast_function_call: return clone(v->as()); + case ast_underscore: return clone(v->as()); + case ast_assign: return clone(v->as()); + case ast_set_assign: return clone(v->as()); + case ast_unary_operator: return clone(v->as()); + case ast_binary_operator: return clone(v->as()); + case ast_ternary_operator: return clone(v->as()); + case ast_cast_as_operator: return clone(v->as()); + case ast_not_null_operator: return clone(v->as()); + case ast_is_null_check: return clone(v->as()); + default: + throw UnexpectedASTNodeType(v, "ASTReplicatorFunction::clone"); + } + } + + AnyV clone(AnyV v) final { + switch (v->type) { + case ast_empty_statement: return clone(v->as()); + case ast_sequence: return clone(v->as()); + case ast_return_statement: return clone(v->as()); + case ast_if_statement: return clone(v->as()); + case ast_repeat_statement: return clone(v->as()); + case ast_while_statement: return clone(v->as()); + case ast_do_while_statement: return clone(v->as()); + case ast_throw_statement: return clone(v->as()); + case ast_assert_statement: return clone(v->as()); + case ast_try_catch_statement: return clone(v->as()); + case ast_asm_body: return clone(v->as()); + // other AST nodes that can be children of ast nodes of function body + case ast_identifier: return clone(v->as()); + case ast_instantiationT_item: return clone(v->as()); + case ast_instantiationT_list: return clone(v->as()); + case ast_parameter: return clone(v->as()); + case ast_parameter_list: return clone(v->as()); + + default: { + // be very careful, don't forget to handle all statements/other (not expressions) above! + AnyExprV as_expr = reinterpret_cast(v); + return clone(as_expr); + } + } + } + + TypePtr clone(TypePtr t) override { + return t; + } + + public: + virtual V clone_function_body(V v_function) { + return createV( + v_function->loc, + clone(v_function->get_identifier()), + clone(v_function->get_param_list()), + clone(v_function->get_body()->as()), + clone(v_function->declared_return_type), + v_function->genericsT_list, + v_function->method_id, + v_function->flags + ); + } +}; + +} // namespace tolk diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h new file mode 100644 index 00000000..a7f260de --- /dev/null +++ b/tolk/ast-stringifier.h @@ -0,0 +1,309 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#ifdef TOLK_DEBUG + +#include "ast.h" +#include "ast-visitor.h" +#include "type-system.h" +#include + +/* + * ASTStringifier is used to print out the whole vertex tree in a human-readable format. + * To stringify any vertex, call v->debug_print(), which uses this class. + */ + +namespace tolk { + +class ASTStringifier final : public ASTVisitor { + constexpr static std::pair name_pairs[] = { + {ast_identifier, "ast_identifier"}, + // expressions + {ast_empty_expression, "ast_empty_expression"}, + {ast_parenthesized_expression, "ast_parenthesized_expression"}, + {ast_tensor, "ast_tensor"}, + {ast_typed_tuple, "ast_typed_tuple"}, + {ast_reference, "ast_reference"}, + {ast_local_var_lhs, "ast_local_var_lhs"}, + {ast_local_vars_declaration, "ast_local_vars_declaration"}, + {ast_int_const, "ast_int_const"}, + {ast_string_const, "ast_string_const"}, + {ast_bool_const, "ast_bool_const"}, + {ast_null_keyword, "ast_null_keyword"}, + {ast_argument, "ast_argument"}, + {ast_argument_list, "ast_argument_list"}, + {ast_dot_access, "ast_dot_access"}, + {ast_function_call, "ast_function_call"}, + {ast_underscore, "ast_underscore"}, + {ast_assign, "ast_assign"}, + {ast_set_assign, "ast_set_assign"}, + {ast_unary_operator, "ast_unary_operator"}, + {ast_binary_operator, "ast_binary_operator"}, + {ast_ternary_operator, "ast_ternary_operator"}, + {ast_cast_as_operator, "ast_cast_as_operator"}, + {ast_not_null_operator, "ast_not_null_operator"}, + {ast_is_null_check, "ast_is_null_check"}, + // statements + {ast_empty_statement, "ast_empty_statement"}, + {ast_sequence, "ast_sequence"}, + {ast_return_statement, "ast_return_statement"}, + {ast_if_statement, "ast_if_statement"}, + {ast_repeat_statement, "ast_repeat_statement"}, + {ast_while_statement, "ast_while_statement"}, + {ast_do_while_statement, "ast_do_while_statement"}, + {ast_throw_statement, "ast_throw_statement"}, + {ast_assert_statement, "ast_assert_statement"}, + {ast_try_catch_statement, "ast_try_catch_statement"}, + {ast_asm_body, "ast_asm_body"}, + // other + {ast_genericsT_item, "ast_genericsT_item"}, + {ast_genericsT_list, "ast_genericsT_list"}, + {ast_instantiationT_item, "ast_instantiationT_item"}, + {ast_instantiationT_list, "ast_instantiationT_list"}, + {ast_parameter, "ast_parameter"}, + {ast_parameter_list, "ast_parameter_list"}, + {ast_annotation, "ast_annotation"}, + {ast_function_declaration, "ast_function_declaration"}, + {ast_global_var_declaration, "ast_global_var_declaration"}, + {ast_constant_declaration, "ast_constant_declaration"}, + {ast_tolk_required_version, "ast_tolk_required_version"}, + {ast_import_directive, "ast_import_directive"}, + {ast_tolk_file, "ast_tolk_file"}, + }; + + static_assert(std::size(name_pairs) == ast_tolk_file + 1, "name_pairs needs to be updated"); + + constexpr static std::pair annotation_kinds[] = { + {AnnotationKind::inline_simple, "@inline"}, + {AnnotationKind::inline_ref, "@inline_ref"}, + {AnnotationKind::method_id, "@method_id"}, + {AnnotationKind::pure, "@pure"}, + {AnnotationKind::deprecated, "@deprecated"}, + }; + + static_assert(std::size(annotation_kinds) == static_cast(AnnotationKind::unknown), "annotation_kinds needs to be updated"); + + template + constexpr static const char* ast_node_type_to_string() { + return name_pairs[node_type].second; + } + + int depth = 0; + std::string out; + bool colored = false; + + template + void handle_vertex(V v) { + out += std::string(depth * 2, ' '); + out += ast_node_type_to_string(); + if (std::string postfix = specific_str(v); !postfix.empty()) { + out += colored ? " \x1b[34m" : " // "; + out += postfix; + out += colored ? "\x1b[0m" : ""; + } + out += '\n'; + depth++; + visit_children(v); + depth--; + } + + static std::string specific_str(AnyV v) { + switch (v->type) { + case ast_identifier: + return static_cast(v->as()->name); + case ast_reference: { + std::string result(v->as()->get_name()); + if (v->as()->has_instantiationTs()) { + result += specific_str(v->as()->get_instantiationTs()); + } + return result; + } + case ast_int_const: + return static_cast(v->as()->orig_str); + case ast_string_const: + if (char modifier = v->as()->modifier) { + return "\"" + static_cast(v->as()->str_val) + "\"" + std::string(1, modifier); + } else { + return "\"" + static_cast(v->as()->str_val) + "\""; + } + case ast_bool_const: + return v->as()->bool_val ? "true" : "false"; + case ast_dot_access: { + std::string result = "." + static_cast(v->as()->get_field_name()); + if (v->as()->has_instantiationTs()) { + result += specific_str(v->as()->get_instantiationTs()); + } + return result; + } + case ast_function_call: { + std::string inner = specific_str(v->as()->get_callee()); + if (int n_args = v->as()->get_num_args()) { + return inner + "(..." + std::to_string(n_args) + ")"; + } + return inner + "()"; + } + case ast_global_var_declaration: + return static_cast(v->as()->get_identifier()->name); + case ast_constant_declaration: + return static_cast(v->as()->get_identifier()->name); + case ast_assign: + return "="; + case ast_set_assign: + return static_cast(v->as()->operator_name) + "="; + case ast_unary_operator: + return static_cast(v->as()->operator_name); + case ast_binary_operator: + return static_cast(v->as()->operator_name); + case ast_cast_as_operator: + return v->as()->cast_to_type->as_human_readable(); + case ast_sequence: + return "↓" + std::to_string(v->as()->get_items().size()); + case ast_instantiationT_item: + return v->as()->substituted_type->as_human_readable(); + case ast_if_statement: + return v->as()->is_ifnot ? "ifnot" : ""; + case ast_annotation: + return annotation_kinds[static_cast(v->as()->kind)].second; + case ast_parameter: { + std::ostringstream os; + os << v->as()->declared_type; + return static_cast(v->as()->param_name) + ": " + os.str(); + } + case ast_function_declaration: { + std::string param_names; + for (int i = 0; i < v->as()->get_num_params(); i++) { + if (!param_names.empty()) + param_names += ","; + param_names += v->as()->get_param(i)->param_name; + } + return "fun " + static_cast(v->as()->get_identifier()->name) + "(" + param_names + ")"; + } + case ast_local_var_lhs: { + std::ostringstream os; + os << (v->as()->inferred_type ? v->as()->inferred_type->as_human_readable() : v->as()->declared_type->as_human_readable()); + if (v->as()->get_name().empty()) { + return "_: " + os.str(); + } + return static_cast(v->as()->get_name()) + ":" + os.str(); + } + case ast_instantiationT_list: { + std::string result = "<"; + for (AnyV item : v->as()->get_items()) { + if (result.size() > 1) + result += ","; + result += item->as()->substituted_type->as_human_readable(); + } + return result + ">"; + } + case ast_tolk_required_version: + return static_cast(v->as()->semver); + case ast_import_directive: + return static_cast(v->as()->get_file_leaf()->str_val); + case ast_tolk_file: + return v->as()->file->rel_filename; + default: + return {}; + } + } + +public: + explicit ASTStringifier(bool colored) : colored(colored) { + } + + std::string to_string_with_children(AnyV v) { + out.clear(); + visit(v); + return std::move(out); + } + + static std::string to_string_without_children(AnyV v) { + std::string result = ast_node_type_to_string(v->type); + if (std::string postfix = specific_str(v); !postfix.empty()) { + result += ' '; + result += specific_str(v); + } + return result; + } + + static const char* ast_node_type_to_string(ASTNodeType node_type) { + return name_pairs[node_type].second; + } + + void visit(AnyV v) override { + switch (v->type) { + case ast_identifier: return handle_vertex(v->as()); + // expressions + case ast_empty_expression: return handle_vertex(v->as()); + case ast_parenthesized_expression: return handle_vertex(v->as()); + case ast_tensor: return handle_vertex(v->as()); + case ast_typed_tuple: return handle_vertex(v->as()); + case ast_reference: return handle_vertex(v->as()); + case ast_local_var_lhs: return handle_vertex(v->as()); + case ast_local_vars_declaration: return handle_vertex(v->as()); + case ast_int_const: return handle_vertex(v->as()); + case ast_string_const: return handle_vertex(v->as()); + case ast_bool_const: return handle_vertex(v->as()); + case ast_null_keyword: return handle_vertex(v->as()); + case ast_argument: return handle_vertex(v->as()); + case ast_argument_list: return handle_vertex(v->as()); + case ast_dot_access: return handle_vertex(v->as()); + case ast_function_call: return handle_vertex(v->as()); + case ast_underscore: return handle_vertex(v->as()); + case ast_assign: return handle_vertex(v->as()); + case ast_set_assign: return handle_vertex(v->as()); + case ast_unary_operator: return handle_vertex(v->as()); + case ast_binary_operator: return handle_vertex(v->as()); + case ast_ternary_operator: return handle_vertex(v->as()); + case ast_cast_as_operator: return handle_vertex(v->as()); + case ast_not_null_operator: return handle_vertex(v->as()); + case ast_is_null_check: return handle_vertex(v->as()); + // statements + case ast_empty_statement: return handle_vertex(v->as()); + case ast_sequence: return handle_vertex(v->as()); + case ast_return_statement: return handle_vertex(v->as()); + case ast_if_statement: return handle_vertex(v->as()); + case ast_repeat_statement: return handle_vertex(v->as()); + case ast_while_statement: return handle_vertex(v->as()); + case ast_do_while_statement: return handle_vertex(v->as()); + case ast_throw_statement: return handle_vertex(v->as()); + case ast_assert_statement: return handle_vertex(v->as()); + case ast_try_catch_statement: return handle_vertex(v->as()); + case ast_asm_body: return handle_vertex(v->as()); + // other + case ast_genericsT_item: return handle_vertex(v->as()); + case ast_genericsT_list: return handle_vertex(v->as()); + case ast_instantiationT_item: return handle_vertex(v->as()); + case ast_instantiationT_list: return handle_vertex(v->as()); + case ast_parameter: return handle_vertex(v->as()); + case ast_parameter_list: return handle_vertex(v->as()); + case ast_annotation: return handle_vertex(v->as()); + case ast_function_declaration: return handle_vertex(v->as()); + case ast_global_var_declaration: return handle_vertex(v->as()); + case ast_constant_declaration: return handle_vertex(v->as()); + case ast_tolk_required_version: return handle_vertex(v->as()); + case ast_import_directive: return handle_vertex(v->as()); + case ast_tolk_file: return handle_vertex(v->as()); + default: + throw UnexpectedASTNodeType(v, "ASTStringifier::visit"); + } + } +}; + +} // namespace tolk + +#endif // TOLK_DEBUG diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h new file mode 100644 index 00000000..d697aa82 --- /dev/null +++ b/tolk/ast-visitor.h @@ -0,0 +1,194 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "ast.h" +#include "platform-utils.h" + +/* + * A module implementing base functionality of read-only traversing a vertex tree. + * Since a vertex in general doesn't store a vector of children, iterating is possible only for concrete node_type. + * E.g., for ast_if_statement, visit nodes cond, if-body and else-body. For ast_string_const, nothing. And so on. + * Visitors below are helpers to inherit from and handle specific vertex types. + * + * Note, that absence of "children" in ASTNodeBase is not a drawback. Instead, it encourages you to think + * about types and match the type system. + * + * The visitor is read-only, it does not modify visited nodes (except if you purposely call mutating methods). + * For example, if you want to replace "beginCell()" call with "begin_cell", a visitor isn't enough for you. + * To replace vertices, consider another API: ast-replacer.h. + */ + +namespace tolk { + +class ASTVisitor { +protected: + GNU_ATTRIBUTE_ALWAYS_INLINE static void visit_children(const ASTExprLeaf* v) { + static_cast(v); + } + + GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTExprUnary* v) { + visit(v->child); + } + + GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTExprBinary* v) { + visit(v->lhs); + visit(v->rhs); + } + + GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTExprVararg* v) { + for (AnyExprV child : v->children) { + visit(child); + } + } + + GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTStatementUnary* v) { + visit(v->child); + } + + GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTStatementVararg* v) { + for (AnyV child : v->children) { + visit(child); + } + } + + GNU_ATTRIBUTE_ALWAYS_INLINE static void visit_children(const ASTOtherLeaf* v) { + static_cast(v); + } + + GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTOtherVararg* v) { + for (AnyV child : v->children) { + visit(child); + } + } + + virtual void visit(AnyV v) = 0; + +public: + virtual ~ASTVisitor() = default; +}; + +class ASTVisitorFunctionBody : public ASTVisitor { +protected: + using parent = ASTVisitorFunctionBody; + + // expressions + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + // statements + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + + void visit(AnyV v) final { + switch (v->type) { + // expressions + case ast_empty_expression: return visit(v->as()); + case ast_parenthesized_expression: return visit(v->as()); + case ast_tensor: return visit(v->as()); + case ast_typed_tuple: return visit(v->as()); + case ast_reference: return visit(v->as()); + case ast_local_var_lhs: return visit(v->as()); + case ast_local_vars_declaration: return visit(v->as()); + case ast_int_const: return visit(v->as()); + case ast_string_const: return visit(v->as()); + case ast_bool_const: return visit(v->as()); + case ast_null_keyword: return visit(v->as()); + case ast_argument: return visit(v->as()); + case ast_argument_list: return visit(v->as()); + case ast_dot_access: return visit(v->as()); + case ast_function_call: return visit(v->as()); + case ast_underscore: return visit(v->as()); + case ast_assign: return visit(v->as()); + case ast_set_assign: return visit(v->as()); + case ast_unary_operator: return visit(v->as()); + case ast_binary_operator: return visit(v->as()); + case ast_ternary_operator: return visit(v->as()); + case ast_cast_as_operator: return visit(v->as()); + case ast_not_null_operator: return visit(v->as()); + case ast_is_null_check: return visit(v->as()); + // statements + case ast_empty_statement: return visit(v->as()); + case ast_sequence: return visit(v->as()); + case ast_return_statement: return visit(v->as()); + case ast_if_statement: return visit(v->as()); + case ast_repeat_statement: return visit(v->as()); + case ast_while_statement: return visit(v->as()); + case ast_do_while_statement: return visit(v->as()); + case ast_throw_statement: return visit(v->as()); + case ast_assert_statement: return visit(v->as()); + case ast_try_catch_statement: return visit(v->as()); +#ifdef TOLK_DEBUG + case ast_asm_body: + throw UnexpectedASTNodeType(v, "ASTVisitor; forgot to filter out asm functions in should_visit_function()?"); +#endif + default: + throw UnexpectedASTNodeType(v, "ASTVisitorFunctionBody::visit"); + } + } + +public: + virtual bool should_visit_function(FunctionPtr fun_ref) = 0; + + virtual void start_visiting_function(FunctionPtr fun_ref, V v_function) { + visit(v_function->get_body()); + } +}; + + +const std::vector& get_all_not_builtin_functions(); + +template +void visit_ast_of_all_functions() { + BodyVisitorT visitor; + for (FunctionPtr fun_ref : get_all_not_builtin_functions()) { + if (visitor.should_visit_function(fun_ref)) { + visitor.start_visiting_function(fun_ref, fun_ref->ast_root->as()); + } + } +} + +} // namespace tolk diff --git a/tolk/ast.cpp b/tolk/ast.cpp new file mode 100644 index 00000000..26eaacd5 --- /dev/null +++ b/tolk/ast.cpp @@ -0,0 +1,209 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "ast.h" +#ifdef TOLK_DEBUG +#include "ast-stringifier.h" +#endif + +namespace tolk { + +static_assert(sizeof(ASTNodeBase) == 12); + +#ifdef TOLK_DEBUG + +std::string ASTNodeBase::to_debug_string(bool colored) const { + ASTStringifier s(colored); + return s.to_string_with_children(this); +} + +void ASTNodeBase::debug_print() const { + std::cerr << to_debug_string(true) << std::endl; +} + +#endif // TOLK_DEBUG + +UnexpectedASTNodeType::UnexpectedASTNodeType(AnyV v_unexpected, const char* place_where): v_unexpected(v_unexpected) { + message = "Unexpected ASTNodeType "; +#ifdef TOLK_DEBUG + message += ASTStringifier::ast_node_type_to_string(v_unexpected->type); + message += " "; +#endif + message += "in "; + message += place_where; +} + +void ASTNodeBase::error(const std::string& err_msg) const { + throw ParseError(loc, err_msg); +} + +AnnotationKind Vertex::parse_kind(std::string_view name) { + if (name == "@pure") { + return AnnotationKind::pure; + } + if (name == "@inline") { + return AnnotationKind::inline_simple; + } + if (name == "@inline_ref") { + return AnnotationKind::inline_ref; + } + if (name == "@method_id") { + return AnnotationKind::method_id; + } + if (name == "@deprecated") { + return AnnotationKind::deprecated; + } + return AnnotationKind::unknown; +} + +int Vertex::lookup_idx(std::string_view nameT) const { + for (size_t idx = 0; idx < children.size(); ++idx) { + if (children[idx] && children[idx]->as()->nameT == nameT) { + return static_cast(idx); + } + } + return -1; +} + +int Vertex::lookup_idx(std::string_view param_name) const { + for (size_t idx = 0; idx < children.size(); ++idx) { + if (children[idx] && children[idx]->as()->param_name == param_name) { + return static_cast(idx); + } + } + return -1; +} + +int Vertex::get_mutate_params_count() const { + int n = 0; + for (AnyV param : children) { + if (param->as()->declared_as_mutate) { + n++; + } + } + return n; +} + +// --------------------------------------------------------- +// "assign" methods +// +// From the user's point of view, all AST vertices are constant, fields are public, but can't be modified. +// The only way to modify a field is to call "mutate()" and then use these "assign_*" methods. +// Therefore, there is a guarantee, that all AST mutations are done via these methods, +// easily searched by usages, and there is no another way to modify any other field. + +void ASTNodeExpressionBase::assign_inferred_type(TypePtr type) { + this->inferred_type = type; +} + +void ASTNodeExpressionBase::assign_rvalue_true() { + this->is_rvalue = true; +} + +void ASTNodeExpressionBase::assign_lvalue_true() { + this->is_lvalue = true; +} + +void ASTNodeExpressionBase::assign_always_true_or_false(int flow_true_false_state) { + this->is_always_true = flow_true_false_state == 1; // see smart-casts-cfg.h + this->is_always_false = flow_true_false_state == 2; +} + +void Vertex::assign_sym(const Symbol* sym) { + this->sym = sym; +} + +void Vertex::assign_fun_ref(FunctionPtr fun_ref) { + this->fun_maybe = fun_ref; +} + +void Vertex::assign_resolved_type(TypePtr cast_to_type) { + this->cast_to_type = cast_to_type; +} + +void Vertex::assign_var_ref(GlobalVarPtr var_ref) { + this->var_ref = var_ref; +} + +void Vertex::assign_resolved_type(TypePtr declared_type) { + this->declared_type = declared_type; +} + +void Vertex::assign_const_ref(GlobalConstPtr const_ref) { + this->const_ref = const_ref; +} + +void Vertex::assign_resolved_type(TypePtr declared_type) { + this->declared_type = declared_type; +} + +void Vertex::assign_resolved_type(TypePtr substituted_type) { + this->substituted_type = substituted_type; +} + +void Vertex::assign_param_ref(LocalVarPtr param_ref) { + this->param_ref = param_ref; +} + +void Vertex::assign_resolved_type(TypePtr declared_type) { + this->declared_type = declared_type; +} + +void Vertex::assign_fun_ref(FunctionPtr fun_ref) { + this->fun_ref = fun_ref; +} + +void Vertex::assign_fun_ref(FunctionPtr fun_ref) { + this->fun_ref = fun_ref; +} + +void Vertex::assign_fun_ref(FunctionPtr fun_ref) { + this->fun_ref = fun_ref; +} + +void Vertex::assign_is_negated(bool is_negated) { + this->is_negated = is_negated; +} + +void Vertex::assign_first_unreachable(AnyV first_unreachable) { + this->first_unreachable = first_unreachable; +} + +void Vertex::assign_target(const DotTarget& target) { + this->target = target; +} + +void Vertex::assign_fun_ref(FunctionPtr fun_ref) { + this->fun_ref = fun_ref; +} + +void Vertex::assign_resolved_type(TypePtr declared_return_type) { + this->declared_return_type = declared_return_type; +} + +void Vertex::assign_var_ref(LocalVarPtr var_ref) { + this->var_ref = var_ref; +} + +void Vertex::assign_resolved_type(TypePtr declared_type) { + this->declared_type = declared_type; +} + +void Vertex::assign_src_file(const SrcFile* file) { + this->file = file; +} + +} // namespace tolk diff --git a/tolk/ast.h b/tolk/ast.h new file mode 100644 index 00000000..9b7c5d1a --- /dev/null +++ b/tolk/ast.h @@ -0,0 +1,1090 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include +#include "fwd-declarations.h" +#include "platform-utils.h" +#include "src-file.h" +#include "lexer.h" +#include "symtable.h" + +/* + * Here we introduce AST representation of Tolk source code. + * Historically, in FunC, there was no AST: while lexing, symbols were registered, types were inferred, and so on. + * There was no way to perform any more or less semantic analysis. + * In Tolk, I've implemented parsing .tolk files into AST at first, and then converting this AST + * into legacy representation (see pipe-ast-to-legacy.cpp). + * In the future, more and more code analysis will be moved out of legacy to AST-level. + * + * From the user's point of view, all AST vertices are constant. All API is based on constancy. + * Even though fields of vertex structs are public, they can't be modified, since vertices are accepted by const ref. + * Generally, there are three ways of accepting a vertex: + * * AnyV (= const ASTNodeBase*) + * the only you can do with this vertex is to see v->type (ASTNodeType) and to cast via v->as() + * * AnyExprV (= const ASTNodeExpressionBase*) + * in contains expression-specific properties (lvalue/rvalue, inferred type) + * * V (= const Vertex*) + * a specific type of vertex, you can use its fields and methods + * There is one way of creating a vertex: + * * createV(...constructor_args) (= new Vertex(...)) + * vertices are currently created on a heap, without any custom memory arena, just allocated and never deleted + * The only way to modify a field is to use "mutate()" method (drops constancy, the only point of mutation) + * and then to call "assign_*" method, like "assign_sym", "assign_src_file", etc. + * + * Having AnyV and knowing its node_type, a call + * v->as() + * will return a typed vertex. + * There is also a shorthand v->try_as() which returns V or nullptr if types don't match: + * if (auto v_int = v->try_as()) + * Note, that there casts are NOT DYNAMIC. ASTNode is not a virtual base, it has no vtable. + * So, as<...>() is just a compile-time casting, without any runtime overhead. + * + * Note, that ASTNodeBase doesn't store any vector of children. That's why there is no way to loop over + * a random (unknown) vertex. Only a concrete Vertex stores its children (if any). + * Hence, to iterate over a custom vertex (e.g., a function body), one should inherit some kind of ASTVisitor. + * Besides read-only visiting, there is a "visit and replace" pattern. + * See ast-visitor.h and ast-replacer.h. + */ + +namespace tolk { + +enum ASTNodeType { + ast_identifier, + // expressions + ast_empty_expression, + ast_parenthesized_expression, + ast_tensor, + ast_typed_tuple, + ast_reference, + ast_local_var_lhs, + ast_local_vars_declaration, + ast_int_const, + ast_string_const, + ast_bool_const, + ast_null_keyword, + ast_argument, + ast_argument_list, + ast_dot_access, + ast_function_call, + ast_underscore, + ast_assign, + ast_set_assign, + ast_unary_operator, + ast_binary_operator, + ast_ternary_operator, + ast_cast_as_operator, + ast_not_null_operator, + ast_is_null_check, + // statements + ast_empty_statement, + ast_sequence, + ast_return_statement, + ast_if_statement, + ast_repeat_statement, + ast_while_statement, + ast_do_while_statement, + ast_throw_statement, + ast_assert_statement, + ast_try_catch_statement, + ast_asm_body, + // other + ast_genericsT_item, + ast_genericsT_list, + ast_instantiationT_item, + ast_instantiationT_list, + ast_parameter, + ast_parameter_list, + ast_annotation, + ast_function_declaration, + ast_global_var_declaration, + ast_constant_declaration, + ast_tolk_required_version, + ast_import_directive, + ast_tolk_file, +}; + +enum class AnnotationKind { + inline_simple, + inline_ref, + method_id, + pure, + deprecated, + unknown, +}; + +template +struct Vertex; + +template +using V = const Vertex*; + +#define createV new Vertex + +struct UnexpectedASTNodeType final : std::exception { + AnyV v_unexpected; + std::string message; + + explicit UnexpectedASTNodeType(AnyV v_unexpected, const char* place_where); + + const char* what() const noexcept override { + return message.c_str(); + } +}; + +// --------------------------------------------------------- + +struct ASTNodeBase { + const ASTNodeType type; + const SrcLocation loc; + + ASTNodeBase(ASTNodeType type, SrcLocation loc) : type(type), loc(loc) {} + ASTNodeBase(const ASTNodeBase&) = delete; + + template + V as() const { +#ifdef TOLK_DEBUG + if (type != node_type) { + throw Fatal("v->as<...> to wrong node_type"); + } +#endif + return static_cast>(this); + } + + template + V try_as() const { + return type == node_type ? static_cast>(this) : nullptr; + } + +#ifdef TOLK_DEBUG + std::string to_debug_string() const { return to_debug_string(false); } + std::string to_debug_string(bool colored) const; + void debug_print() const; +#endif + + GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD + void error(const std::string& err_msg) const; +}; + +struct ASTNodeExpressionBase : ASTNodeBase { + friend class ASTDuplicatorFunction; + + TypePtr inferred_type = nullptr; + bool is_rvalue: 1 = false; + bool is_lvalue: 1 = false; + bool is_always_true: 1 = false; // inside `if`, `while`, ternary condition, `== null`, etc. + bool is_always_false: 1 = false; // (when expression is guaranteed to be always true or always false) + + ASTNodeExpressionBase* mutate() const { return const_cast(this); } + void assign_inferred_type(TypePtr type); + void assign_rvalue_true(); + void assign_lvalue_true(); + void assign_always_true_or_false(int flow_true_false_state); + + ASTNodeExpressionBase(ASTNodeType type, SrcLocation loc) : ASTNodeBase(type, loc) {} +}; + +struct ASTNodeStatementBase : ASTNodeBase { + ASTNodeStatementBase(ASTNodeType type, SrcLocation loc) : ASTNodeBase(type, loc) {} +}; + +struct ASTExprLeaf : ASTNodeExpressionBase { + friend class ASTVisitor; + friend class ASTReplacer; + +protected: + ASTExprLeaf(ASTNodeType type, SrcLocation loc) + : ASTNodeExpressionBase(type, loc) {} +}; + +struct ASTExprUnary : ASTNodeExpressionBase { + friend class ASTVisitor; + friend class ASTReplacer; + +protected: + AnyExprV child; + + ASTExprUnary(ASTNodeType type, SrcLocation loc, AnyExprV child) + : ASTNodeExpressionBase(type, loc), child(child) {} +}; + +struct ASTExprBinary : ASTNodeExpressionBase { + friend class ASTVisitor; + friend class ASTReplacer; + +protected: + AnyExprV lhs; + AnyExprV rhs; + + ASTExprBinary(ASTNodeType type, SrcLocation loc, AnyExprV lhs, AnyExprV rhs) + : ASTNodeExpressionBase(type, loc), lhs(lhs), rhs(rhs) {} +}; + +struct ASTExprVararg : ASTNodeExpressionBase { + friend class ASTVisitor; + friend class ASTReplacer; + +protected: + std::vector children; + + AnyExprV child(int i) const { return children.at(i); } + + ASTExprVararg(ASTNodeType type, SrcLocation loc, std::vector children) + : ASTNodeExpressionBase(type, loc), children(std::move(children)) {} + +public: + int size() const { return static_cast(children.size()); } + bool empty() const { return children.empty(); } +}; + +struct ASTStatementUnary : ASTNodeStatementBase { + friend class ASTVisitor; + friend class ASTReplacer; + +protected: + AnyV child; + + AnyExprV child_as_expr() const { return reinterpret_cast(child); } + + ASTStatementUnary(ASTNodeType type, SrcLocation loc, AnyV child) + : ASTNodeStatementBase(type, loc), child(child) {} +}; + +struct ASTStatementVararg : ASTNodeStatementBase { + friend class ASTVisitor; + friend class ASTReplacer; + +protected: + std::vector children; + + AnyExprV child_as_expr(int i) const { return reinterpret_cast(children.at(i)); } + + ASTStatementVararg(ASTNodeType type, SrcLocation loc, std::vector children) + : ASTNodeStatementBase(type, loc), children(std::move(children)) {} + +public: + int size() const { return static_cast(children.size()); } + bool empty() const { return children.empty(); } +}; + +struct ASTOtherLeaf : ASTNodeBase { + friend class ASTVisitor; + friend class ASTReplacer; + +protected: + ASTOtherLeaf(ASTNodeType type, SrcLocation loc) + : ASTNodeBase(type, loc) {} +}; + +struct ASTOtherVararg : ASTNodeBase { + friend class ASTVisitor; + friend class ASTReplacer; + +protected: + std::vector children; + + AnyExprV child_as_expr(int i) const { return reinterpret_cast(children.at(i)); } + + ASTOtherVararg(ASTNodeType type, SrcLocation loc, std::vector children) + : ASTNodeBase(type, loc), children(std::move(children)) {} + +public: + int size() const { return static_cast(children.size()); } + bool empty() const { return children.empty(); } +}; + + +template<> +// ast_identifier is "a name" in AST structure +// it's NOT a standalone expression, it's "implementation details" of other AST vertices +// example: `var x = 5` then "x" is identifier (inside local var declaration) +// example: `global g: int` then "g" is identifier +// example: `someF` is a reference, which contains identifier +// example: `someF` is a reference which contains identifier and generics instantiation +// example: `fun f()` then "f" is identifier, "" is a generics declaration +struct Vertex final : ASTOtherLeaf { + std::string_view name; // empty for underscore + + Vertex(SrcLocation loc, std::string_view name) + : ASTOtherLeaf(ast_identifier, loc) + , name(name) {} +}; + + +// +// --------------------------------------------------------- +// expressions +// + + +template<> +// ast_empty_expression is "nothing" in context of expression, it has "unknown" type +// example: `throw 123;` then "throw arg" is empty expression (opposed to `throw (123, arg)`) +struct Vertex final : ASTExprLeaf { + explicit Vertex(SrcLocation loc) + : ASTExprLeaf(ast_empty_expression, loc) {} +}; + + +template<> +// ast_parenthesized_expression is something surrounded embraced by (parenthesis) +// example: `(1)`, `((f()))` (two nested) +struct Vertex final : ASTExprUnary { + AnyExprV get_expr() const { return child; } + + Vertex(SrcLocation loc, AnyExprV expr) + : ASTExprUnary(ast_parenthesized_expression, loc, expr) {} +}; + +template<> +// ast_tensor is a set of expressions embraced by (parenthesis) +// in most languages, it's called "tuple", but in TVM, "tuple" is a TVM primitive, that's why "tensor" +// example: `(1, 2)`, `(1, (2, 3))` (nested), `()` (empty tensor) +// note, that `(1)` is not a tensor, it's a parenthesized expression +// a tensor of N elements occupies N slots on a stack (opposed to TVM tuple primitive, 1 slot) +struct Vertex final : ASTExprVararg { + const std::vector& get_items() const { return children; } + AnyExprV get_item(int i) const { return child(i); } + + Vertex(SrcLocation loc, std::vector items) + : ASTExprVararg(ast_tensor, loc, std::move(items)) {} +}; + +template<> +// ast_typed_tuple is a set of expressions in [square brackets] +// in TVM, it's a TVM tuple, that occupies 1 slot, but the compiler knows its "typed structure" +// example: `[1, x]`, `[[0]]` (nested) +// typed tuples can be assigned to N variables, like `[one, _, three] = [1,2,3]` +struct Vertex final : ASTExprVararg { + const std::vector& get_items() const { return children; } + AnyExprV get_item(int i) const { return child(i); } + + Vertex(SrcLocation loc, std::vector items) + : ASTExprVararg(ast_typed_tuple, loc, std::move(items)) {} +}; + +template<> +// ast_reference is "something that references a symbol" +// examples: `x` / `someF` / `someF` +// it's a leaf expression from traversing point of view, but actually, has children (not expressions) +// note, that both `someF()` and `someF()` are function calls, where a callee is just a reference +struct Vertex final : ASTExprLeaf { +private: + V identifier; // its name, `x` / `someF` + V instantiationTs; // not null if ``, otherwise nullptr + +public: + const Symbol* sym = nullptr; // filled on resolve or type inferring; points to local / global / function / constant + + auto get_identifier() const { return identifier; } + bool has_instantiationTs() const { return instantiationTs != nullptr; } + auto get_instantiationTs() const { return instantiationTs; } + std::string_view get_name() const { return identifier->name; } + + Vertex* mutate() const { return const_cast(this); } + void assign_sym(const Symbol* sym); + + Vertex(SrcLocation loc, V name_identifier, V instantiationTs) + : ASTExprLeaf(ast_reference, loc) + , identifier(name_identifier), instantiationTs(instantiationTs) {} +}; + +template<> +// ast_local_var_lhs is one variable inside `var` declaration +// example: `var x = 0;` then "x" is local var lhs +// example: `val (x: int, [y redef], _) = rhs` then "x" and "y" and "_" are +// it's a leaf from expression's point of view, though technically has an "identifier" child +struct Vertex final : ASTExprLeaf { +private: + V identifier; + +public: + LocalVarPtr var_ref = nullptr; // filled on resolve identifiers; for `redef` points to declared above; for underscore, name is empty + TypePtr declared_type; // not null for `var x: int = rhs`, otherwise nullptr + bool is_immutable; // declared via 'val', not 'var' + bool marked_as_redef; // var (existing_var redef, new_var: int) = ... + + V get_identifier() const { return identifier; } + std::string_view get_name() const { return identifier->name; } // empty for underscore + + Vertex* mutate() const { return const_cast(this); } + void assign_var_ref(LocalVarPtr var_ref); + void assign_resolved_type(TypePtr declared_type); + + Vertex(SrcLocation loc, V identifier, TypePtr declared_type, bool is_immutable, bool marked_as_redef) + : ASTExprLeaf(ast_local_var_lhs, loc) + , identifier(identifier), declared_type(declared_type), is_immutable(is_immutable), marked_as_redef(marked_as_redef) {} +}; + +template<> +// ast_local_vars_declaration is an expression declaring local variables on the left side of assignment +// examples: see above +// for `var (x, [y])` its expr is "tensor (local var, typed tuple (local var))" +// for assignment `var x = 5`, this node is `var x`, lhs of assignment +struct Vertex final : ASTExprUnary { + AnyExprV get_expr() const { return child; } // ast_local_var_lhs / ast_tensor / ast_typed_tuple + + Vertex(SrcLocation loc, AnyExprV expr) + : ASTExprUnary(ast_local_vars_declaration, loc, expr) {} +}; + +template<> +// ast_int_const is an integer literal +// examples: `0` / `0xFF` +// note, that `-1` is unary minus of `1` int const +struct Vertex final : ASTExprLeaf { + td::RefInt256 intval; // parsed value, 255 for "0xFF" + std::string_view orig_str; // original "0xFF"; empty for nodes generated by compiler (e.g. in constant folding) + + Vertex(SrcLocation loc, td::RefInt256 intval, std::string_view orig_str) + : ASTExprLeaf(ast_int_const, loc) + , intval(std::move(intval)) + , orig_str(orig_str) {} +}; + +template<> +// ast_string_const is a string literal in double quotes or """ when multiline +// examples: "asdf" / "Ef8zMz..."a / "to_calc_crc32_from"c +// an optional modifier specifies how a string is parsed (probably, like an integer) +// note, that TVM doesn't have strings, it has only slices, so "hello" has type slice +struct Vertex final : ASTExprLeaf { + std::string_view str_val; + char modifier; + + bool is_bitslice() const { + char m = modifier; + return m == 0 || m == 's' || m == 'a'; + } + bool is_intval() const { + char m = modifier; + return m == 'u' || m == 'h' || m == 'H' || m == 'c'; + } + + Vertex(SrcLocation loc, std::string_view str_val, char modifier) + : ASTExprLeaf(ast_string_const, loc) + , str_val(str_val), modifier(modifier) {} +}; + +template<> +// ast_bool_const is either `true` or `false` +struct Vertex final : ASTExprLeaf { + bool bool_val; + + Vertex(SrcLocation loc, bool bool_val) + : ASTExprLeaf(ast_bool_const, loc) + , bool_val(bool_val) {} +}; + +template<> +// ast_null_keyword is the `null` literal +// it should be handled with care; for instance, `null` takes special place in the type system +struct Vertex final : ASTExprLeaf { + explicit Vertex(SrcLocation loc) + : ASTExprLeaf(ast_null_keyword, loc) {} +}; + +template<> +// ast_argument is an element of an argument list of a function/method call +// example: `f(1, x)` has 2 arguments, `t.tupleFirst()` has no arguments (though `t` is passed as `self`) +// example: `f(mutate arg)` has 1 argument with `passed_as_mutate` flag +// (without `mutate` keyword, the entity "argument" could be replaced just by "any expression") +struct Vertex final : ASTExprUnary { + bool passed_as_mutate; + + AnyExprV get_expr() const { return child; } + + Vertex(SrcLocation loc, AnyExprV expr, bool passed_as_mutate) + : ASTExprUnary(ast_argument, loc, expr) + , passed_as_mutate(passed_as_mutate) {} +}; + +template<> +// ast_argument_list contains N arguments of a function/method call +struct Vertex final : ASTExprVararg { + const std::vector& get_arguments() const { return children; } + auto get_arg(int i) const { return child(i)->as(); } + + Vertex(SrcLocation loc, std::vector arguments) + : ASTExprVararg(ast_argument_list, loc, std::move(arguments)) {} +}; + +template<> +// ast_dot_access is "object before dot, identifier + optional after dot" +// examples: `tensorVar.0` / `obj.field` / `getObj().method` / `t.tupleFirst` +// from traversing point of view, it's an unary expression: only obj is expression, field name is not +// note, that `obj.method()` is a function call with "dot access `obj.method`" callee +struct Vertex final : ASTExprUnary { +private: + V identifier; // `0` / `field` / `method` + V instantiationTs; // not null if ``, otherwise nullptr + +public: + + typedef std::variant< + FunctionPtr, // for `t.tupleAt` target is `tupleAt` global function + int // for `t.0` target is "indexed access" 0 + > DotTarget; + DotTarget target = static_cast(nullptr); // filled at type inferring + + bool is_target_fun_ref() const { return std::holds_alternative(target); } + bool is_target_indexed_access() const { return std::holds_alternative(target); } + + AnyExprV get_obj() const { return child; } + auto get_identifier() const { return identifier; } + bool has_instantiationTs() const { return instantiationTs != nullptr; } + auto get_instantiationTs() const { return instantiationTs; } + std::string_view get_field_name() const { return identifier->name; } + + Vertex* mutate() const { return const_cast(this); } + void assign_target(const DotTarget& target); + + Vertex(SrcLocation loc, AnyExprV obj, V identifier, V instantiationTs) + : ASTExprUnary(ast_dot_access, loc, obj) + , identifier(identifier), instantiationTs(instantiationTs) {} +}; + +template<> +// ast_function_call is "calling some lhs with parenthesis", lhs is arbitrary expression (callee) +// example: `globalF()` then callee is reference +// example: `globalF()` then callee is reference (with instantiation Ts filled) +// example: `local_var()` then callee is reference (points to local var, filled at resolve identifiers) +// example: `getF()()` then callee is another func call (which type is TypeDataFunCallable) +// example: `obj.method()` then callee is dot access (resolved while type inferring) +struct Vertex final : ASTExprBinary { + FunctionPtr fun_maybe = nullptr; // filled while type inferring for `globalF()` / `obj.f()`; remains nullptr for `local_var()` / `getF()()` + + AnyExprV get_callee() const { return lhs; } + bool is_dot_call() const { return lhs->type == ast_dot_access; } + AnyExprV get_dot_obj() const { return lhs->as()->get_obj(); } + auto get_arg_list() const { return rhs->as(); } + int get_num_args() const { return rhs->as()->size(); } + auto get_arg(int i) const { return rhs->as()->get_arg(i); } + + Vertex* mutate() const { return const_cast(this); } + void assign_fun_ref(FunctionPtr fun_ref); + + Vertex(SrcLocation loc, AnyExprV lhs_f, V arguments) + : ASTExprBinary(ast_function_call, loc, lhs_f, arguments) {} +}; + +template<> +// ast_underscore represents `_` symbol used for left side of assignment +// example: `(cs, _) = cs.loadAndReturn()` +// though it's the only correct usage, using _ as rvalue like `var x = _;` is correct from AST point of view +// note, that for declaration `var _ = 1` underscore is a regular local var declared (with empty name) +// but for `_ = 1` (not declaration) it's underscore; it's because `var _:int` is also correct +struct Vertex final : ASTExprLeaf { + explicit Vertex(SrcLocation loc) + : ASTExprLeaf(ast_underscore, loc) {} +}; + +template<> +// ast_assign represents assignment "lhs = rhs" +// examples: `a = 4` / `var a = 4` / `(cs, b, mode) = rhs` / `f() = g()` +// note, that `a = 4` lhs is ast_reference, `var a = 4` lhs is ast_local_vars_declaration +struct Vertex final : ASTExprBinary { + AnyExprV get_lhs() const { return lhs; } + AnyExprV get_rhs() const { return rhs; } + + explicit Vertex(SrcLocation loc, AnyExprV lhs, AnyExprV rhs) + : ASTExprBinary(ast_assign, loc, lhs, rhs) {} +}; + +template<> +// ast_set_assign represents assignment-and-set operation "lhs = rhs" +// examples: `a += 4` / `b <<= c` +struct Vertex final : ASTExprBinary { + FunctionPtr fun_ref = nullptr; // filled at type inferring, points to `_+_` built-in for += + std::string_view operator_name; // without equal sign, "+" for operator += + TokenType tok; // tok_set_* + + AnyExprV get_lhs() const { return lhs; } + AnyExprV get_rhs() const { return rhs; } + + Vertex* mutate() const { return const_cast(this); } + void assign_fun_ref(FunctionPtr fun_ref); + + Vertex(SrcLocation loc, std::string_view operator_name, TokenType tok, AnyExprV lhs, AnyExprV rhs) + : ASTExprBinary(ast_set_assign, loc, lhs, rhs) + , operator_name(operator_name), tok(tok) {} +}; + +template<> +// ast_unary_operator is "some operator over one expression" +// examples: `-1` / `~found` +struct Vertex final : ASTExprUnary { + FunctionPtr fun_ref = nullptr; // filled at type inferring, points to some built-in function + std::string_view operator_name; + TokenType tok; + + AnyExprV get_rhs() const { return child; } + + Vertex* mutate() const { return const_cast(this); } + void assign_fun_ref(FunctionPtr fun_ref); + + Vertex(SrcLocation loc, std::string_view operator_name, TokenType tok, AnyExprV rhs) + : ASTExprUnary(ast_unary_operator, loc, rhs) + , operator_name(operator_name), tok(tok) {} +}; + +template<> +// ast_binary_operator is "some operator over two expressions" +// examples: `a + b` / `x & true` / `(a, b) << g()` +// note, that `a = b` is NOT a binary operator, it's ast_assign, also `a += b`, it's ast_set_assign +struct Vertex final : ASTExprBinary { + FunctionPtr fun_ref = nullptr; // filled at type inferring, points to some built-in function + std::string_view operator_name; + TokenType tok; + + AnyExprV get_lhs() const { return lhs; } + AnyExprV get_rhs() const { return rhs; } + + Vertex* mutate() const { return const_cast(this); } + void assign_fun_ref(FunctionPtr fun_ref); + + Vertex(SrcLocation loc, std::string_view operator_name, TokenType tok, AnyExprV lhs, AnyExprV rhs) + : ASTExprBinary(ast_binary_operator, loc, lhs, rhs) + , operator_name(operator_name), tok(tok) {} +}; + +template<> +// ast_ternary_operator is a traditional ternary construction +// example: `cond ? a : b` +struct Vertex final : ASTExprVararg { + AnyExprV get_cond() const { return child(0); } + AnyExprV get_when_true() const { return child(1); } + AnyExprV get_when_false() const { return child(2); } + + Vertex(SrcLocation loc, AnyExprV cond, AnyExprV when_true, AnyExprV when_false) + : ASTExprVararg(ast_ternary_operator, loc, {cond, when_true, when_false}) {} +}; + +template<> +// ast_cast_as_operator is explicit casting with "as" keyword +// examples: `arg as int` / `null as cell` / `t.tupleAt(2) as slice` +struct Vertex final : ASTExprUnary { + AnyExprV get_expr() const { return child; } + + TypePtr cast_to_type; + + Vertex* mutate() const { return const_cast(this); } + void assign_resolved_type(TypePtr cast_to_type); + + Vertex(SrcLocation loc, AnyExprV expr, TypePtr cast_to_type) + : ASTExprUnary(ast_cast_as_operator, loc, expr) + , cast_to_type(cast_to_type) {} +}; + +template<> +// ast_not_null_operator is non-null assertion: like TypeScript ! or Kotlin !! +// examples: `nullableInt!` / `getNullableBuilder()!` +struct Vertex final : ASTExprUnary { + AnyExprV get_expr() const { return child; } + + Vertex(SrcLocation loc, AnyExprV expr) + : ASTExprUnary(ast_not_null_operator, loc, expr) {} +}; + +template<> +// ast_is_null_check is an artificial vertex for "expr == null" / "expr != null" / same but null on the left +// it's created instead of a general binary expression to emphasize its purpose +struct Vertex final : ASTExprUnary { + bool is_negated; + + AnyExprV get_expr() const { return child; } + + Vertex* mutate() const { return const_cast(this); } + void assign_is_negated(bool is_negated); + + Vertex(SrcLocation loc, AnyExprV expr, bool is_negated) + : ASTExprUnary(ast_is_null_check, loc, expr) + , is_negated(is_negated) {} +}; + + +// +// --------------------------------------------------------- +// statements +// + + +template<> +// ast_empty_statement is very similar to "empty sequence" but has a special treatment +// example: `;` (just semicolon) +// example: body of `builtin` function is empty statement (not a zero sequence) +struct Vertex final : ASTStatementVararg { + explicit Vertex(SrcLocation loc) + : ASTStatementVararg(ast_empty_statement, loc, {}) {} +}; + +template<> +// ast_sequence is "some sequence of statements" +// example: function body is a sequence +// example: do while body is a sequence +struct Vertex final : ASTStatementVararg { + SrcLocation loc_end; + AnyV first_unreachable = nullptr; + + const std::vector& get_items() const { return children; } + AnyV get_item(int i) const { return children.at(i); } + + Vertex* mutate() const { return const_cast(this); } + void assign_first_unreachable(AnyV first_unreachable); + + Vertex(SrcLocation loc, SrcLocation loc_end, std::vector items) + : ASTStatementVararg(ast_sequence, loc, std::move(items)) + , loc_end(loc_end) {} +}; + +template<> +// ast_return_statement is "return something from a function" +// examples: `return a` / `return any_expr()()` / `return;` +// note, that for `return;` (without a value, meaning "void"), in AST, it's stored as empty expression +struct Vertex : ASTStatementUnary { + AnyExprV get_return_value() const { return child_as_expr(); } + bool has_return_value() const { return child->type != ast_empty_expression; } + + Vertex(SrcLocation loc, AnyExprV child) + : ASTStatementUnary(ast_return_statement, loc, child) {} +}; + +template<> +// ast_if_statement is a traditional if statement, probably followed by an else branch +// examples: `if (cond) { ... } else { ... }` / `if (cond) { ... }` +// when else branch is missing, it's stored as empty statement +// for "else if", it's just "if statement" inside a sequence of else branch +struct Vertex final : ASTStatementVararg { + bool is_ifnot; // if(!cond), to generate more optimal fift code + + AnyExprV get_cond() const { return child_as_expr(0); } + auto get_if_body() const { return children.at(1)->as(); } + auto get_else_body() const { return children.at(2)->as(); } // always exists (when else omitted, it's empty) + + Vertex(SrcLocation loc, bool is_ifnot, AnyExprV cond, V if_body, V else_body) + : ASTStatementVararg(ast_if_statement, loc, {cond, if_body, else_body}) + , is_ifnot(is_ifnot) {} +}; + +template<> +// ast_repeat_statement is "repeat something N times" +// example: `repeat (10) { ... }` +struct Vertex final : ASTStatementVararg { + AnyExprV get_cond() const { return child_as_expr(0); } + auto get_body() const { return children.at(1)->as(); } + + Vertex(SrcLocation loc, AnyExprV cond, V body) + : ASTStatementVararg(ast_repeat_statement, loc, {cond, body}) {} +}; + +template<> +// ast_while_statement is a standard "while" loop +// example: `while (x > 0) { ... }` +struct Vertex final : ASTStatementVararg { + AnyExprV get_cond() const { return child_as_expr(0); } + auto get_body() const { return children.at(1)->as(); } + + Vertex(SrcLocation loc, AnyExprV cond, V body) + : ASTStatementVararg(ast_while_statement, loc, {cond, body}) {} +}; + +template<> +// ast_do_while_statement is a standard "do while" loop +// example: `do { ... } while (x > 0);` +struct Vertex final : ASTStatementVararg { + auto get_body() const { return children.at(0)->as(); } + AnyExprV get_cond() const { return child_as_expr(1); } + + Vertex(SrcLocation loc, V body, AnyExprV cond) + : ASTStatementVararg(ast_do_while_statement, loc, {body, cond}) {} +}; + +template<> +// ast_throw_statement is throwing an exception, it accepts excNo and optional arg +// examples: `throw 10` / `throw (ERR_LOW_BALANCE)` / `throw (1001, incomingAddr)` +// when thrown arg is missing, it's stored as empty expression +struct Vertex final : ASTStatementVararg { + AnyExprV get_thrown_code() const { return child_as_expr(0); } + bool has_thrown_arg() const { return child_as_expr(1)->type != ast_empty_expression; } + AnyExprV get_thrown_arg() const { return child_as_expr(1); } + + Vertex(SrcLocation loc, AnyExprV thrown_code, AnyExprV thrown_arg) + : ASTStatementVararg(ast_throw_statement, loc, {thrown_code, thrown_arg}) {} +}; + +template<> +// ast_assert_statement is "assert that cond is true, otherwise throw an exception" +// examples: `assert (balance > 0, ERR_ZERO_BALANCE)` / `assert (balance > 0) throw (ERR_ZERO_BALANCE)` +struct Vertex final : ASTStatementVararg { + AnyExprV get_cond() const { return child_as_expr(0); } + AnyExprV get_thrown_code() const { return child_as_expr(1); } + + Vertex(SrcLocation loc, AnyExprV cond, AnyExprV thrown_code) + : ASTStatementVararg(ast_assert_statement, loc, {cond, thrown_code}) {} +}; + +template<> +// ast_try_catch_statement is a standard try catch (finally block doesn't exist) +// example: `try { ... } catch (excNo) { ... }` +// there are two formal "arguments" of catch: excNo and arg, but both can be omitted +// when omitted, they are stored as underscores, so len of a catch tensor is always 2 +struct Vertex final : ASTStatementVararg { + auto get_try_body() const { return children.at(0)->as(); } + auto get_catch_expr() const { return children.at(1)->as(); } // (excNo, arg), always len 2 + auto get_catch_body() const { return children.at(2)->as(); } + + Vertex(SrcLocation loc, V try_body, V catch_expr, V catch_body) + : ASTStatementVararg(ast_try_catch_statement, loc, {try_body, catch_expr, catch_body}) {} +}; + +template<> +// ast_asm_body is a body of `asm` function — a set of strings, and optionally stack order manipulations +// example: `fun skipMessageOp... asm "32 PUSHINT" "SDSKIPFIRST";` +// user can specify "arg order"; example: `fun store(self: builder, op: int) asm (op self)` then [1, 0] +// user can specify "ret order"; example: `fun modDiv... asm(-> 1 0) "DIVMOD";` then [1, 0] +struct Vertex final : ASTStatementVararg { + std::vector arg_order; + std::vector ret_order; + + const std::vector& get_asm_commands() const { return children; } // ast_string_const[] + + Vertex(SrcLocation loc, std::vector arg_order, std::vector ret_order, std::vector asm_commands) + : ASTStatementVararg(ast_asm_body, loc, std::move(asm_commands)) + , arg_order(std::move(arg_order)), ret_order(std::move(ret_order)) {} +}; + + +// +// --------------------------------------------------------- +// other +// + + +template<> +// ast_genericsT_item is generics T at declaration +// example: `fun f` has a list of 2 generic Ts +struct Vertex final : ASTOtherLeaf { + std::string_view nameT; + + Vertex(SrcLocation loc, std::string_view nameT) + : ASTOtherLeaf(ast_genericsT_item, loc) + , nameT(nameT) {} +}; + +template<> +// ast_genericsT_list is a container for generics T at declaration +// example: see above +struct Vertex final : ASTOtherVararg { + std::vector get_items() const { return children; } + auto get_item(int i) const { return children.at(i)->as(); } + + Vertex(SrcLocation loc, std::vector genericsT_items) + : ASTOtherVararg(ast_genericsT_list, loc, std::move(genericsT_items)) {} + + int lookup_idx(std::string_view nameT) const; +}; + + +template<> +// ast_instantiationT_item is manual substitution of generic T used in code, mostly for func calls +// examples: `g()` / `t.tupleFirst()` / `f<(int, slice), builder>()` +struct Vertex final : ASTOtherLeaf { + TypePtr substituted_type; + + Vertex* mutate() const { return const_cast(this); } + void assign_resolved_type(TypePtr substituted_type); + + Vertex(SrcLocation loc, TypePtr substituted_type) + : ASTOtherLeaf(ast_instantiationT_item, loc) + , substituted_type(substituted_type) {} +}; + +template<> +// ast_instantiationT_list is a container for generic T substitutions used in code +// examples: see above +struct Vertex final : ASTOtherVararg { + std::vector get_items() const { return children; } + auto get_item(int i) const { return children.at(i)->as(); } + + Vertex(SrcLocation loc, std::vector instantiationTs) + : ASTOtherVararg(ast_instantiationT_list, loc, std::move(instantiationTs)) {} +}; + +template<> +// ast_parameter is a parameter of a function in its declaration +// example: `fun f(a: int, mutate b: slice)` has 2 parameters +struct Vertex final : ASTOtherLeaf { + LocalVarPtr param_ref = nullptr; // filled on resolve identifiers + std::string_view param_name; + TypePtr declared_type; + bool declared_as_mutate; // declared as `mutate param_name` + + bool is_underscore() const { return param_name.empty(); } + + Vertex* mutate() const { return const_cast(this); } + void assign_param_ref(LocalVarPtr param_ref); + void assign_resolved_type(TypePtr declared_type); + + Vertex(SrcLocation loc, std::string_view param_name, TypePtr declared_type, bool declared_as_mutate) + : ASTOtherLeaf(ast_parameter, loc) + , param_name(param_name), declared_type(declared_type), declared_as_mutate(declared_as_mutate) {} +}; + +template<> +// ast_parameter_list is a container of parameters +// example: see above +struct Vertex final : ASTOtherVararg { + const std::vector& get_params() const { return children; } + auto get_param(int i) const { return children.at(i)->as(); } + + Vertex(SrcLocation loc, std::vector params) + : ASTOtherVararg(ast_parameter_list, loc, std::move(params)) {} + + int lookup_idx(std::string_view param_name) const; + int get_mutate_params_count() const; + bool has_mutate_params() const { return get_mutate_params_count() > 0; } +}; + +template<> +// ast_annotation is @annotation above a declaration +// example: `@pure fun ...` +struct Vertex final : ASTOtherVararg { + AnnotationKind kind; + + auto get_arg() const { return children.at(0)->as(); } + + static AnnotationKind parse_kind(std::string_view name); + + Vertex(SrcLocation loc, AnnotationKind kind, V arg_probably_empty) + : ASTOtherVararg(ast_annotation, loc, {arg_probably_empty}) + , kind(kind) {} +}; + +template<> +// ast_function_declaration is declaring a function/method +// methods are still global functions, just accepting "self" first parameter +// example: `fun f() { ... }` +// functions can be generic, `fun f(params) { ... }` +// their body is either sequence (regular code function), or `asm`, or `builtin` +struct Vertex final : ASTOtherVararg { + auto get_identifier() const { return children.at(0)->as(); } + int get_num_params() const { return children.at(1)->as()->size(); } + auto get_param_list() const { return children.at(1)->as(); } + auto get_param(int i) const { return children.at(1)->as()->get_param(i); } + AnyV get_body() const { return children.at(2); } // ast_sequence / ast_asm_body + + FunctionPtr fun_ref = nullptr; // filled after register + TypePtr declared_return_type; // filled at ast parsing; if unspecified (nullptr), means "auto infer" + V genericsT_list; // for non-generics it's nullptr + td::RefInt256 method_id; // specified via @method_id annotation + int flags; // from enum in FunctionData + + bool is_asm_function() const { return children.at(2)->type == ast_asm_body; } + bool is_code_function() const { return children.at(2)->type == ast_sequence; } + bool is_builtin_function() const { return children.at(2)->type == ast_empty_statement; } + + Vertex* mutate() const { return const_cast(this); } + void assign_fun_ref(FunctionPtr fun_ref); + void assign_resolved_type(TypePtr declared_return_type); + + Vertex(SrcLocation loc, V name_identifier, V parameters, AnyV body, TypePtr declared_return_type, V genericsT_list, td::RefInt256 method_id, int flags) + : ASTOtherVararg(ast_function_declaration, loc, {name_identifier, parameters, body}) + , declared_return_type(declared_return_type), genericsT_list(genericsT_list), method_id(std::move(method_id)), flags(flags) {} +}; + +template<> +// ast_global_var_declaration is declaring a global var, outside a function +// example: `global g: int;` +// note, that globals don't have default values, since there is no single "entrypoint" for a contract +struct Vertex final : ASTOtherVararg { + GlobalVarPtr var_ref = nullptr; // filled after register + TypePtr declared_type; // filled always, typing globals is mandatory + + auto get_identifier() const { return children.at(0)->as(); } + + Vertex* mutate() const { return const_cast(this); } + void assign_var_ref(GlobalVarPtr var_ref); + void assign_resolved_type(TypePtr declared_type); + + Vertex(SrcLocation loc, V name_identifier, TypePtr declared_type) + : ASTOtherVararg(ast_global_var_declaration, loc, {name_identifier}) + , declared_type(declared_type) {} +}; + +template<> +// ast_constant_declaration is declaring a global constant, outside a function +// example: `const op = 0x123;` +struct Vertex final : ASTOtherVararg { + GlobalConstPtr const_ref = nullptr; // filled after register + TypePtr declared_type; // not null for `const op: int = ...` + + auto get_identifier() const { return children.at(0)->as(); } + AnyExprV get_init_value() const { return child_as_expr(1); } + + Vertex* mutate() const { return const_cast(this); } + void assign_const_ref(GlobalConstPtr const_ref); + void assign_resolved_type(TypePtr declared_type); + + Vertex(SrcLocation loc, V name_identifier, TypePtr declared_type, AnyExprV init_value) + : ASTOtherVararg(ast_constant_declaration, loc, {name_identifier, init_value}) + , declared_type(declared_type) {} +}; + +template<> +// ast_tolk_required_version is a preamble fixating compiler's version at the top of the file +// example: `tolk 0.6` +// when compiler version mismatches, it means, that another compiler was earlier for that sources, a warning is emitted +struct Vertex final : ASTOtherLeaf { + std::string_view semver; + + Vertex(SrcLocation loc, std::string_view semver) + : ASTOtherLeaf(ast_tolk_required_version, loc) + , semver(semver) {} +}; + +template<> +// ast_import_directive is an import at the top of the file +// examples: `import "another.tolk"` / `import "@stdlib/tvm-dicts"` +struct Vertex final : ASTOtherVararg { + const SrcFile* file = nullptr; // assigned after imports have been resolved, just after parsing a file to ast + + auto get_file_leaf() const { return children.at(0)->as(); } + + std::string get_file_name() const { return static_cast(children.at(0)->as()->str_val); } + + Vertex* mutate() const { return const_cast(this); } + void assign_src_file(const SrcFile* file); + + Vertex(SrcLocation loc, V file_name) + : ASTOtherVararg(ast_import_directive, loc, {file_name}) {} +}; + +template<> +// ast_tolk_file represents a whole parsed input .tolk file +// with functions, constants, etc. +// particularly, it contains imports that lead to loading other files +// a whole program consists of multiple parsed files, each of them has a parsed ast tree (stdlib is also parsed) +struct Vertex final : ASTOtherVararg { + const SrcFile* const file; + + const std::vector& get_toplevel_declarations() const { return children; } + + Vertex(const SrcFile* file, std::vector toplevel_declarations) + : ASTOtherVararg(ast_tolk_file, SrcLocation(file), std::move(toplevel_declarations)) + , file(file) {} +}; + +} // namespace tolk diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp new file mode 100644 index 00000000..cb89c984 --- /dev/null +++ b/tolk/builtins.cpp @@ -0,0 +1,1282 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" +#include "compiler-state.h" +#include "type-system.h" +#include "generics-helpers.h" + +namespace tolk { +using namespace std::literals::string_literals; + +// given func_type = `(slice, int) -> slice` and func flags, create SymLocalVarOrParameter +// currently (see at the bottom) parameters of built-in functions are unnamed: +// built-in functions are created using a resulting type +static std::vector define_builtin_parameters(const std::vector& params_types, int func_flags) { + // `loadInt()`, `storeInt()`: they accept `self` and mutate it; no other options available in built-ins for now + bool is_mutate_self = func_flags & FunctionData::flagHasMutateParams; + std::vector parameters; + parameters.reserve(params_types.size()); + + for (int i = 0; i < static_cast(params_types.size()); ++i) { + LocalVarData p_sym("", {}, params_types[i], (i == 0 && is_mutate_self) * LocalVarData::flagMutateParameter, i); + parameters.push_back(std::move(p_sym)); + } + + return parameters; +} + +static void define_builtin_func(const std::string& name, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags) { + auto* f_sym = new FunctionData(name, {}, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); + G.symtable.add_function(f_sym); +} + +static void define_builtin_func(const std::string& name, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const AsmOp& macro, int flags) { + auto* f_sym = new FunctionData(name, {}, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(make_simple_compile(macro)), nullptr); + G.symtable.add_function(f_sym); +} + +static void define_builtin_func(const std::string& name, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags, + std::initializer_list arg_order, std::initializer_list ret_order) { + auto* f_sym = new FunctionData(name, {}, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); + f_sym->arg_order = arg_order; + f_sym->ret_order = ret_order; + G.symtable.add_function(f_sym); +} + +void FunctionBodyBuiltin::compile(AsmOpList& dest, std::vector& out, std::vector& in, + SrcLocation where) const { + dest.append(simple_compile(out, in, where)); +} + +void FunctionBodyAsm::compile(AsmOpList& dest) const { + dest.append(ops); +} + + +/* + * + * DEFINE BUILT-IN FUNCTIONS + * + */ + +int emulate_negate(int a) { + int f = VarDescr::_Pos | VarDescr::_Neg; + if ((a & f) && (~a & f)) { + a ^= f; + } + return a; +} + +int emulate_add(int a, int b) { + if (b & VarDescr::_Zero) { + return a; + } else if (a & VarDescr::_Zero) { + return b; + } + int u = a & b, v = a | b; + int r = VarDescr::_Int; + int t = u & (VarDescr::_Pos | VarDescr::_Neg); + if (v & VarDescr::_Nan) { + return r | VarDescr::_Nan; + } + // non-quiet addition always returns finite results! + r |= t | VarDescr::_Finite; + if (t) { + r |= v & VarDescr::_NonZero; + } + r |= v & VarDescr::_Nan; + if (u & (VarDescr::_Odd | VarDescr::_Even)) { + r |= VarDescr::_Even; + } else if (!(~v & (VarDescr::_Odd | VarDescr::_Even))) { + r |= VarDescr::_Odd | VarDescr::_NonZero; + } + return r; +} + +int emulate_sub(int a, int b) { + return emulate_add(a, emulate_negate(b)); +} + +int emulate_mul(int a, int b) { + if ((b & VarDescr::ConstOne) == VarDescr::ConstOne) { + return a; + } else if ((a & VarDescr::ConstOne) == VarDescr::ConstOne) { + return b; + } + int u = a & b, v = a | b; + int r = VarDescr::_Int; + if (v & VarDescr::_Nan) { + return r | VarDescr::_Nan; + } + // non-quiet multiplication always yields finite results, if any + r |= VarDescr::_Finite; + if (v & VarDescr::_Zero) { + // non-quiet multiplication + // the result is zero, if any result at all + return VarDescr::ConstZero; + } + if (u & (VarDescr::_Pos | VarDescr::_Neg)) { + r |= VarDescr::_Pos; + } else if (!(~v & (VarDescr::_Pos | VarDescr::_Neg))) { + r |= VarDescr::_Neg; + } + r |= v & VarDescr::_Even; + r |= u & (VarDescr::_Odd | VarDescr::_NonZero); + return r; +} + +int emulate_bitwise_and(int a, int b) { + int both = a & b, any = a | b; + int r = VarDescr::_Int; + if (any & VarDescr::_Nan) { + return r | VarDescr::_Nan; + } + r |= VarDescr::_Finite; + if (any & VarDescr::_Zero) { + return VarDescr::ConstZero; + } + r |= both & (VarDescr::_Even | VarDescr::_Odd); + if (both & VarDescr::_Odd) { + r |= VarDescr::_NonZero; + } + return r; +} + +int emulate_bitwise_or(int a, int b) { + if (b & VarDescr::_Zero) { + return a; + } else if (a & VarDescr::_Zero) { + return b; + } + int both = a & b, any = a | b; + int r = VarDescr::_Int; + if (any & VarDescr::_Nan) { + return r | VarDescr::_Nan; + } + r |= VarDescr::_Finite; + r |= any & VarDescr::_NonZero; + r |= any & VarDescr::_Odd; + r |= both & VarDescr::_Even; + return r; +} + +int emulate_bitwise_xor(int a, int b) { + if (b & VarDescr::_Zero) { + return a; + } else if (a & VarDescr::_Zero) { + return b; + } + int both = a & b, any = a | b; + int r = VarDescr::_Int; + if (any & VarDescr::_Nan) { + return r | VarDescr::_Nan; + } + r |= VarDescr::_Finite; + r |= both & VarDescr::_Even; + if (both & VarDescr::_Odd) { + r |= VarDescr::_Even; + } + return r; +} + +int emulate_bitwise_not(int a) { + if ((a & VarDescr::ConstZero) == VarDescr::ConstZero) { + return VarDescr::ConstTrue; + } + if ((a & VarDescr::ConstTrue) == VarDescr::ConstTrue) { + return VarDescr::ConstZero; + } + int a2 = a; + int f = VarDescr::_Even | VarDescr::_Odd; + if ((a2 & f) && (~a2 & f)) { + a2 ^= f; + } + a2 &= ~(VarDescr::_Zero | VarDescr::_NonZero | VarDescr::_Pos | VarDescr::_Neg); + if ((a & VarDescr::_Neg) && (a & VarDescr::_NonZero)) { + a2 |= VarDescr::_Pos; + } + if (a & VarDescr::_Pos) { + a2 |= VarDescr::_Neg; + } + return a2; +} + +int emulate_lshift(int a, int b) { + if (((a | b) & VarDescr::_Nan) || !(~b & (VarDescr::_Neg | VarDescr::_NonZero))) { + return VarDescr::_Int | VarDescr::_Nan; + } + if (b & VarDescr::_Zero) { + return a; + } + int t = ((b & VarDescr::_NonZero) ? VarDescr::_Even : 0); + t |= b & VarDescr::_Finite; + return emulate_mul(a, VarDescr::_Int | VarDescr::_Pos | VarDescr::_NonZero | t); +} + +int emulate_div(int a, int b) { + if ((b & VarDescr::ConstOne) == VarDescr::ConstOne) { + return a; + } else if ((b & VarDescr::ConstOne) == VarDescr::ConstOne) { + return emulate_negate(a); + } + if (b & VarDescr::_Zero) { + return VarDescr::_Int | VarDescr::_Nan; + } + int u = a & b, v = a | b; + int r = VarDescr::_Int; + if (v & VarDescr::_Nan) { + return r | VarDescr::_Nan; + } + // non-quiet division always yields finite results, if any + r |= VarDescr::_Finite; + if (a & VarDescr::_Zero) { + // non-quiet division + // the result is zero, if any result at all + return VarDescr::ConstZero; + } + if (u & (VarDescr::_Pos | VarDescr::_Neg)) { + r |= VarDescr::_Pos; + } else if (!(~v & (VarDescr::_Pos | VarDescr::_Neg))) { + r |= VarDescr::_Neg; + } + return r; +} + +int emulate_rshift(int a, int b) { + if (((a | b) & VarDescr::_Nan) || !(~b & (VarDescr::_Neg | VarDescr::_NonZero))) { + return VarDescr::_Int | VarDescr::_Nan; + } + if (b & VarDescr::_Zero) { + return a; + } + int t = ((b & VarDescr::_NonZero) ? VarDescr::_Even : 0); + t |= b & VarDescr::_Finite; + return emulate_div(a, VarDescr::_Int | VarDescr::_Pos | VarDescr::_NonZero | t); +} + +int emulate_mod(int a, int b, int round_mode = -1) { + if ((b & VarDescr::ConstOne) == VarDescr::ConstOne) { + return VarDescr::ConstZero; + } + if (b & VarDescr::_Zero) { + return VarDescr::_Int | VarDescr::_Nan; + } + int r = VarDescr::_Int; + if ((a | b) & VarDescr::_Nan) { + return r | VarDescr::_Nan; + } + // non-quiet division always yields finite results, if any + r |= VarDescr::_Finite; + if (a & VarDescr::_Zero) { + // non-quiet division + // the result is zero, if any result at all + return VarDescr::ConstZero; + } + if (round_mode < 0) { + r |= b & (VarDescr::_Pos | VarDescr::_Neg); + } else if (round_mode > 0) { + r |= emulate_negate(b) & (VarDescr::_Pos | VarDescr::_Neg); + } + if (b & VarDescr::_Even) { + r |= a & (VarDescr::_Even | VarDescr::_Odd); + } + return r; +} + +bool VarDescr::always_less(const VarDescr& other) const { + if (is_int_const() && other.is_int_const()) { + return int_const < other.int_const; + } + return (always_nonpos() && other.always_pos()) || (always_neg() && other.always_nonneg()); +} + +bool VarDescr::always_leq(const VarDescr& other) const { + if (is_int_const() && other.is_int_const()) { + return int_const <= other.int_const; + } + return always_nonpos() && other.always_nonneg(); +} + +bool VarDescr::always_greater(const VarDescr& other) const { + return other.always_less(*this); +} + +bool VarDescr::always_geq(const VarDescr& other) const { + return other.always_leq(*this); +} + +bool VarDescr::always_equal(const VarDescr& other) const { + return is_int_const() && other.is_int_const() && *int_const == *other.int_const; +} + +bool VarDescr::always_neq(const VarDescr& other) const { + if (is_int_const() && other.is_int_const()) { + return *int_const != *other.int_const; + } + return always_greater(other) || always_less(other) || (always_even() && other.always_odd()) || + (always_odd() && other.always_even()); +} + +AsmOp exec_op(std::string op) { + return AsmOp::Custom(op); +} + +AsmOp exec_op(std::string op, int args, int retv = 1) { + return AsmOp::Custom(op, args, retv); +} + +AsmOp exec_arg_op(std::string op, long long arg) { + std::ostringstream os; + os << arg << ' ' << op; + return AsmOp::Custom(os.str()); +} + +AsmOp exec_arg_op(std::string op, long long arg, int args, int retv) { + std::ostringstream os; + os << arg << ' ' << op; + return AsmOp::Custom(os.str(), args, retv); +} + +AsmOp exec_arg_op(std::string op, td::RefInt256 arg) { + std::ostringstream os; + os << arg << ' ' << op; + return AsmOp::Custom(os.str()); +} + +AsmOp exec_arg_op(std::string op, td::RefInt256 arg, int args, int retv) { + std::ostringstream os; + os << arg << ' ' << op; + return AsmOp::Custom(os.str(), args, retv); +} + +AsmOp exec_arg2_op(std::string op, long long imm1, long long imm2, int args, int retv) { + std::ostringstream os; + os << imm1 << ' ' << imm2 << ' ' << op; + return AsmOp::Custom(os.str(), args, retv); +} + +AsmOp push_const(td::RefInt256 x) { + return AsmOp::IntConst(std::move(x)); +} + +AsmOp compile_add(std::vector& res, std::vector& args, SrcLocation where) { + tolk_assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (x.is_int_const() && y.is_int_const()) { + r.set_const(x.int_const + y.int_const); + if (!r.int_const->is_valid()) { + throw ParseError(where, "integer overflow"); + } + x.unused(); + y.unused(); + return push_const(r.int_const); + } + r.val = emulate_add(x.val, y.val); + if (y.is_int_const() && y.int_const->signed_fits_bits(8)) { + y.unused(); + if (y.always_zero()) { + return AsmOp::Nop(); + } + if (*y.int_const == 1) { + return exec_op("INC", 1); + } + if (*y.int_const == -1) { + return exec_op("DEC", 1); + } + return exec_arg_op("ADDCONST", y.int_const, 1); + } + if (x.is_int_const() && x.int_const->signed_fits_bits(8)) { + x.unused(); + if (x.always_zero()) { + return AsmOp::Nop(); + } + if (*x.int_const == 1) { + return exec_op("INC", 1); + } + if (*x.int_const == -1) { + return exec_op("DEC", 1); + } + return exec_arg_op("ADDCONST", x.int_const, 1); + } + return exec_op("ADD", 2); +} + +AsmOp compile_sub(std::vector& res, std::vector& args, SrcLocation where) { + tolk_assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (x.is_int_const() && y.is_int_const()) { + r.set_const(x.int_const - y.int_const); + if (!r.int_const->is_valid()) { + throw ParseError(where, "integer overflow"); + } + x.unused(); + y.unused(); + return push_const(r.int_const); + } + r.val = emulate_sub(x.val, y.val); + if (y.is_int_const() && (-y.int_const)->signed_fits_bits(8)) { + y.unused(); + if (y.always_zero()) { + return {}; + } + if (*y.int_const == 1) { + return exec_op("DEC", 1); + } + if (*y.int_const == -1) { + return exec_op("INC", 1); + } + return exec_arg_op("ADDCONST", -y.int_const, 1); + } + if (x.always_zero()) { + x.unused(); + return exec_op("NEGATE", 1); + } + return exec_op("SUB", 2); +} + +AsmOp compile_unary_minus(std::vector& res, std::vector& args, SrcLocation where) { + tolk_assert(res.size() == 1 && args.size() == 1); + VarDescr &r = res[0], &x = args[0]; + if (x.is_int_const()) { + r.set_const(-x.int_const); + if (!r.int_const->is_valid()) { + throw ParseError(where, "integer overflow"); + } + x.unused(); + return push_const(r.int_const); + } + r.val = emulate_negate(x.val); + return exec_op("NEGATE", 1); +} + +AsmOp compile_unary_plus(std::vector& res, std::vector& args, SrcLocation where) { + tolk_assert(res.size() == 1 && args.size() == 1); + VarDescr &r = res[0], &x = args[0]; + if (x.is_int_const()) { + r.set_const(x.int_const); + x.unused(); + return push_const(r.int_const); + } + r.val = x.val; + return AsmOp::Nop(); +} + +AsmOp compile_logical_not(std::vector& res, std::vector& args, SrcLocation where, bool for_int_arg) { + tolk_assert(res.size() == 1 && args.size() == 1); + VarDescr &r = res[0], &x = args[0]; + if (x.is_int_const()) { + r.set_const(x.int_const == 0 ? -1 : 0); + x.unused(); + return push_const(r.int_const); + } + r.val = VarDescr::ValBool; + // for integers, `!var` is `var != 0` + // for booleans, `!var` can be shortened to `~var` (works the same for 0/-1 but consumes less) + return for_int_arg ? exec_op("0 EQINT", 1) : exec_op("NOT", 1); +} + +AsmOp compile_bitwise_and(std::vector& res, std::vector& args, SrcLocation where) { + tolk_assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (x.is_int_const() && y.is_int_const()) { + r.set_const(x.int_const & y.int_const); + x.unused(); + y.unused(); + return push_const(r.int_const); + } + r.val = emulate_bitwise_and(x.val, y.val); + return exec_op("AND", 2); +} + +AsmOp compile_bitwise_or(std::vector& res, std::vector& args, SrcLocation where) { + tolk_assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (x.is_int_const() && y.is_int_const()) { + r.set_const(x.int_const | y.int_const); + x.unused(); + y.unused(); + return push_const(r.int_const); + } + r.val = emulate_bitwise_or(x.val, y.val); + return exec_op("OR", 2); +} + +AsmOp compile_bitwise_xor(std::vector& res, std::vector& args, SrcLocation where) { + tolk_assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (x.is_int_const() && y.is_int_const()) { + r.set_const(x.int_const ^ y.int_const); + x.unused(); + y.unused(); + return push_const(r.int_const); + } + r.val = emulate_bitwise_xor(x.val, y.val); + return exec_op("XOR", 2); +} + +AsmOp compile_bitwise_not(std::vector& res, std::vector& args, SrcLocation where) { + tolk_assert(res.size() == 1 && args.size() == 1); + VarDescr &r = res[0], &x = args[0]; + if (x.is_int_const()) { + r.set_const(~x.int_const); + x.unused(); + return push_const(r.int_const); + } + r.val = emulate_bitwise_not(x.val); + return exec_op("NOT", 1); +} + +AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation where) { + if (x.is_int_const() && y.is_int_const()) { + r.set_const(x.int_const * y.int_const); + if (!r.int_const->is_valid()) { + throw ParseError(where, "integer overflow"); + } + x.unused(); + y.unused(); + return push_const(r.int_const); + } + r.val = emulate_mul(x.val, y.val); + if (y.is_int_const()) { + int k = is_pos_pow2(y.int_const); + if (y.int_const->signed_fits_bits(8) && k < 0) { + y.unused(); + if (y.always_zero() && x.always_finite()) { + // dubious optimization: NaN * 0 = ? + r.set_const(y.int_const); + x.unused(); + return push_const(r.int_const); + } + if (*y.int_const == 1 && x.always_finite()) { + return AsmOp::Nop(); + } + if (*y.int_const == -1) { + return exec_op("NEGATE", 1); + } + return exec_arg_op("MULCONST", y.int_const, 1); + } + if (k > 0) { + y.unused(); + return exec_arg_op("LSHIFT#", k, 1); + } + if (k == 0) { + y.unused(); + return AsmOp::Nop(); + } + } + if (x.is_int_const()) { + int k = is_pos_pow2(x.int_const); + if (x.int_const->signed_fits_bits(8) && k < 0) { + x.unused(); + if (x.always_zero() && y.always_finite()) { + // dubious optimization: NaN * 0 = ? + r.set_const(x.int_const); + y.unused(); + return push_const(r.int_const); + } + if (*x.int_const == 1 && y.always_finite()) { + return AsmOp::Nop(); + } + if (*x.int_const == -1) { + return exec_op("NEGATE", 1); + } + return exec_arg_op("MULCONST", x.int_const, 1); + } + if (k > 0) { + x.unused(); + return exec_arg_op("LSHIFT#", k, 1); + } + if (k == 0) { + x.unused(); + return AsmOp::Nop(); + } + } + return exec_op("MUL", 2); +} + +AsmOp compile_mul(std::vector& res, std::vector& args, SrcLocation where) { + tolk_assert(res.size() == 1 && args.size() == 2); + return compile_mul_internal(res[0], args[0], args[1], where); +} + +AsmOp compile_lshift(std::vector& res, std::vector& args, SrcLocation where) { + tolk_assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (y.is_int_const()) { + auto yv = y.int_const->to_long(); + if (yv < 0 || yv > 256) { + throw ParseError(where, "lshift argument is out of range"); + } else if (x.is_int_const()) { + r.set_const(x.int_const << (int)yv); + if (!r.int_const->is_valid()) { + throw ParseError(where, "integer overflow"); + } + x.unused(); + y.unused(); + return push_const(r.int_const); + } + } + r.val = emulate_lshift(x.val, y.val); + if (y.is_int_const()) { + int k = (int)(y.int_const->to_long()); + if (!k /* && x.always_finite() */) { + // dubious optimization: what if x=NaN ? + y.unused(); + return AsmOp::Nop(); + } + y.unused(); + return exec_arg_op("LSHIFT#", k, 1); + } + if (x.is_int_const()) { + auto xv = x.int_const->to_long(); + if (xv == 1) { + x.unused(); + return exec_op("POW2", 1); + } + if (xv == -1) { + x.unused(); + return exec_op("-1 PUSHINT SWAP LSHIFT", 1); + } + } + return exec_op("LSHIFT", 2); +} + +AsmOp compile_rshift(std::vector& res, std::vector& args, SrcLocation where, + int round_mode) { + tolk_assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (y.is_int_const()) { + auto yv = y.int_const->to_long(); + if (yv < 0 || yv > 256) { + throw ParseError(where, "rshift argument is out of range"); + } else if (x.is_int_const()) { + r.set_const(td::rshift(x.int_const, (int)yv, round_mode)); + x.unused(); + y.unused(); + return push_const(r.int_const); + } + } + r.val = emulate_rshift(x.val, y.val); + std::string rshift = (round_mode < 0 ? "RSHIFT" : (round_mode ? "RSHIFTC" : "RSHIFTR")); + if (y.is_int_const()) { + int k = (int)(y.int_const->to_long()); + if (!k /* && x.always_finite() */) { + // dubious optimization: what if x=NaN ? + y.unused(); + return AsmOp::Nop(); + } + y.unused(); + return exec_arg_op(rshift + "#", k, 1); + } + return exec_op(rshift, 2); +} + +AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation where, int round_mode) { + if (x.is_int_const() && y.is_int_const()) { + r.set_const(div(x.int_const, y.int_const, round_mode)); + if (!r.int_const->is_valid()) { + throw ParseError(where, *y.int_const == 0 ? "division by zero" : "integer overflow"); + } + x.unused(); + y.unused(); + return push_const(r.int_const); + } + r.val = emulate_div(x.val, y.val); + if (y.is_int_const()) { + if (*y.int_const == 0) { + throw ParseError(where, "division by zero"); + } + if (*y.int_const == 1 && x.always_finite()) { + y.unused(); + return AsmOp::Nop(); + } + if (*y.int_const == -1) { + y.unused(); + return exec_op("NEGATE", 1); + } + int k = is_pos_pow2(y.int_const); + if (k > 0) { + y.unused(); + std::string op = "RSHIFT"; + if (round_mode >= 0) { + op += (round_mode > 0 ? 'C' : 'R'); + } + return exec_arg_op(op + '#', k, 1); + } + } + std::string op = "DIV"; + if (round_mode >= 0) { + op += (round_mode > 0 ? 'C' : 'R'); + } + return exec_op(op, 2); +} + +AsmOp compile_div(std::vector& res, std::vector& args, SrcLocation where, int round_mode) { + tolk_assert(res.size() == 1 && args.size() == 2); + return compile_div_internal(res[0], args[0], args[1], where, round_mode); +} + +AsmOp compile_mod(std::vector& res, std::vector& args, SrcLocation where, + int round_mode) { + tolk_assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (x.is_int_const() && y.is_int_const()) { + r.set_const(mod(x.int_const, y.int_const, round_mode)); + if (!r.int_const->is_valid()) { + throw ParseError(where, *y.int_const == 0 ? "division by zero" : "integer overflow"); + } + x.unused(); + y.unused(); + return push_const(r.int_const); + } + r.val = emulate_mod(x.val, y.val); + if (y.is_int_const()) { + if (*y.int_const == 0) { + throw ParseError(where, "division by zero"); + } + if ((*y.int_const == 1 || *y.int_const == -1) && x.always_finite()) { + x.unused(); + y.unused(); + r.set_const(td::zero_refint()); + return push_const(r.int_const); + } + int k = is_pos_pow2(y.int_const); + if (k > 0) { + y.unused(); + std::string op = "MODPOW2"; + if (round_mode >= 0) { + op += (round_mode > 0 ? 'C' : 'R'); + } + return exec_arg_op(op + '#', k, 1); + } + } + std::string op = "MOD"; + if (round_mode >= 0) { + op += (round_mode > 0 ? 'C' : 'R'); + } + return exec_op(op, 2); +} + +AsmOp compile_muldiv(std::vector& res, std::vector& args, SrcLocation where, + int round_mode) { + tolk_assert(res.size() == 1 && args.size() == 3); + VarDescr &r = res[0], &x = args[0], &y = args[1], &z = args[2]; + if (x.is_int_const() && y.is_int_const() && z.is_int_const()) { + r.set_const(muldiv(x.int_const, y.int_const, z.int_const, round_mode)); + if (!r.int_const->is_valid()) { + throw ParseError(where, *z.int_const == 0 ? "division by zero" : "integer overflow"); + } + x.unused(); + y.unused(); + z.unused(); + return push_const(r.int_const); + } + if (x.always_zero() || y.always_zero()) { + // dubious optimization for z=0... + x.unused(); + y.unused(); + z.unused(); + r.set_const(td::make_refint(0)); + return push_const(r.int_const); + } + char c = (round_mode < 0) ? 0 : (round_mode > 0 ? 'C' : 'R'); + r.val = emulate_div(emulate_mul(x.val, y.val), z.val); + if (z.is_int_const()) { + if (*z.int_const == 0) { + throw ParseError(where, "division by zero"); + } + if (*z.int_const == 1) { + z.unused(); + return compile_mul_internal(r, x, y, where); + } + } + if (y.is_int_const() && *y.int_const == 1) { + y.unused(); + return compile_div_internal(r, x, z, where, round_mode); + } + if (x.is_int_const() && *x.int_const == 1) { + x.unused(); + return compile_div_internal(r, y, z, where, round_mode); + } + if (z.is_int_const()) { + int k = is_pos_pow2(z.int_const); + if (k > 0) { + z.unused(); + std::string op = "MULRSHIFT"; + if (c) { + op += c; + } + return exec_arg_op(op + '#', k, 2); + } + } + if (y.is_int_const()) { + int k = is_pos_pow2(y.int_const); + if (k > 0) { + y.unused(); + std::string op = "LSHIFT#DIV"; + if (c) { + op += c; + } + return exec_arg_op(op, k, 2); + } + } + if (x.is_int_const()) { + int k = is_pos_pow2(x.int_const); + if (k > 0) { + x.unused(); + std::string op = "LSHIFT#DIV"; + if (c) { + op += c; + } + return exec_arg_op(op, k, 2); + } + } + std::string op = "MULDIV"; + if (c) { + op += c; + } + return exec_op(op, 3); +} + +int compute_compare(td::RefInt256 x, td::RefInt256 y, int mode) { + int s = td::cmp(x, y); + if (mode == 7) { + return s; + } else { + return -((mode >> (1 - s)) & 1); + } +} + +// return value: +// 4 -> constant 1 +// 2 -> constant 0 +// 1 -> constant -1 +// 3 -> 0 or -1 +int compute_compare(const VarDescr& x, const VarDescr& y, int mode) { + switch (mode) { + case 1: // > + return x.always_greater(y) ? 1 : (x.always_leq(y) ? 2 : 3); + case 2: // = + return x.always_equal(y) ? 1 : (x.always_neq(y) ? 2 : 3); + case 3: // >= + return x.always_geq(y) ? 1 : (x.always_less(y) ? 2 : 3); + case 4: // < + return x.always_less(y) ? 1 : (x.always_geq(y) ? 2 : 3); + case 5: // <> + return x.always_neq(y) ? 1 : (x.always_equal(y) ? 2 : 3); + case 6: // <= + return x.always_leq(y) ? 1 : (x.always_greater(y) ? 2 : 3); + case 7: // <=> + return x.always_less(y) + ? 1 + : (x.always_equal(y) + ? 2 + : (x.always_greater(y) + ? 4 + : (x.always_leq(y) ? 3 : (x.always_geq(y) ? 6 : (x.always_neq(y) ? 5 : 7))))); + default: + return 7; + } +} + +AsmOp compile_cmp_int(std::vector& res, std::vector& args, int mode) { + tolk_assert(mode >= 1 && mode <= 7); + tolk_assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (x.is_int_const() && y.is_int_const()) { + int v = compute_compare(x.int_const, y.int_const, mode); + r.set_const(v); + x.unused(); + y.unused(); + return mode == 7 ? push_const(r.int_const) : AsmOp::BoolConst(v != 0); + } + int v = compute_compare(x, y, mode); + // std::cerr << "compute_compare(" << x << ", " << y << ", " << mode << ") = " << v << std::endl; + tolk_assert(v); + if (!(v & (v - 1))) { + r.set_const(v - (v >> 2) - 2); + x.unused(); + y.unused(); + return mode == 7 ? push_const(r.int_const) : AsmOp::BoolConst(v & 1); + } + r.val = ~0; + if (v & 1) { + r.val &= VarDescr::ConstTrue; + } + if (v & 2) { + r.val &= VarDescr::ConstZero; + } + if (v & 4) { + r.val &= VarDescr::ConstOne; + } + // std::cerr << "result: " << r << std::endl; + static const char* cmp_int_names[] = {"", "GTINT", "EQINT", "GTINT", "LESSINT", "NEQINT", "LESSINT"}; + static const char* cmp_names[] = {"", "GREATER", "EQUAL", "GEQ", "LESS", "NEQ", "LEQ", "CMP"}; + static int cmp_int_delta[] = {0, 0, 0, -1, 0, 0, 1}; + if (mode != 7) { + if (y.is_int_const() && y.int_const >= -128 && y.int_const <= 127) { + y.unused(); + return exec_arg_op(cmp_int_names[mode], y.int_const + cmp_int_delta[mode], 1); + } + if (x.is_int_const() && x.int_const >= -128 && x.int_const <= 127) { + x.unused(); + mode = ((mode & 4) >> 2) | (mode & 2) | ((mode & 1) << 2); + return exec_arg_op(cmp_int_names[mode], x.int_const + cmp_int_delta[mode], 1); + } + } + return exec_op(cmp_names[mode], 2); +} + +AsmOp compile_throw(std::vector& res, std::vector& args, SrcLocation) { + tolk_assert(res.empty() && args.size() == 1); + VarDescr& x = args[0]; + if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { + x.unused(); + return exec_arg_op("THROW", x.int_const, 0, 0); + } else { + return exec_op("THROWANY", 1, 0); + } +} + +AsmOp compile_throw_if_unless(std::vector& res, std::vector& args, SrcLocation) { + tolk_assert(res.empty() && args.size() == 3); + VarDescr &x = args[0], &y = args[1], &z = args[2]; + if (!z.always_true() && !z.always_false()) { + throw Fatal("invalid usage of built-in symbol"); + } + bool mode = z.always_true(); + z.unused(); + std::string suff = (mode ? "IF" : "IFNOT"); + bool skip_cond = false; + if (y.always_true() || y.always_false()) { + y.unused(); + skip_cond = true; + if (y.always_true() != mode) { + x.unused(); + return AsmOp::Nop(); + } + } + if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { + x.unused(); + return skip_cond ? exec_arg_op("THROW", x.int_const, 0, 0) : exec_arg_op("THROW"s + suff, x.int_const, 1, 0); + } else { + return skip_cond ? exec_op("THROWANY", 1, 0) : exec_op("THROWANY"s + suff, 2, 0); + } +} + +AsmOp compile_throw_arg(std::vector& res, std::vector& args, SrcLocation) { + tolk_assert(res.empty() && args.size() == 2); + VarDescr &x = args[1]; + if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { + x.unused(); + return exec_arg_op("THROWARG", x.int_const, 1, 0); + } else { + return exec_op("THROWARGANY", 2, 0); + } +} + +AsmOp compile_bool_const(std::vector& res, std::vector& args, bool val) { + tolk_assert(res.size() == 1 && args.empty()); + VarDescr& r = res[0]; + r.set_const(val ? -1 : 0); + return AsmOp::Const(val ? "TRUE" : "FALSE"); +} + +// fun loadInt (mutate s: slice, len: int): int asm(s len -> 1 0) "LDIX"; +// fun loadUint (mutate s: slice, len: int): int asm( -> 1 0) "LDUX"; +// fun preloadInt (s: slice, len: int): int asm "PLDIX"; +// fun preloadUint(s: slice, len: int): int asm "PLDUX"; +AsmOp compile_fetch_int(std::vector& res, std::vector& args, bool fetch, bool sgnd) { + tolk_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); + auto &y = args[1], &r = res.back(); + r.val = (sgnd ? VarDescr::FiniteInt : VarDescr::FiniteUInt); + int v = -1; + if (y.is_int_const() && y.int_const >= 0 && y.int_const <= 256) { + v = (int)y.int_const->to_long(); + if (!v) { + r.val = VarDescr::ConstZero; + } + if (v == 1) { + r.val = (sgnd ? VarDescr::ValBool : VarDescr::ValBit); + } + if (v > 0) { + y.unused(); + return exec_arg_op((fetch ? "LD"s : "PLD"s) + (sgnd ? 'I' : 'U'), v, 1, 1 + (unsigned)fetch); + } + } + return exec_op((fetch ? "LD"s : "PLD"s) + (sgnd ? "IX" : "UX"), 2, 1 + (unsigned)fetch); +} + +// fun storeInt (mutate self: builder, x: int, len: int): self asm(x b len) "STIX"; +// fun storeUint (mutate self: builder, x: int, len: int): self asm(x b len) "STUX"; +AsmOp compile_store_int(std::vector& res, std::vector& args, bool sgnd) { + tolk_assert(args.size() == 3 && res.size() == 1); + auto& z = args[2]; + if (z.is_int_const() && z.int_const > 0 && z.int_const <= 256) { + z.unused(); + return exec_arg_op("ST"s + (sgnd ? 'I' : 'U'), z.int_const, 2, 1); + } + return exec_op("ST"s + (sgnd ? "IX" : "UX"), 3, 1); +} + +// fun loadBits (mutate self: slice, len: int): self asm(s len -> 1 0) "LDSLICEX" +// fun preloadBits(self: slice, len: int): slice asm(s len -> 1 0) "PLDSLICEX" +AsmOp compile_fetch_slice(std::vector& res, std::vector& args, bool fetch) { + tolk_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); + auto& y = args[1]; + int v = -1; + if (y.is_int_const() && y.int_const > 0 && y.int_const <= 256) { + v = (int)y.int_const->to_long(); + if (v > 0) { + y.unused(); + return exec_arg_op(fetch ? "LDSLICE" : "PLDSLICE", v, 1, 1 + (unsigned)fetch); + } + } + return exec_op(fetch ? "LDSLICEX" : "PLDSLICEX", 2, 1 + (unsigned)fetch); +} + +// fun tupleAt(t: tuple, index: int): X asm "INDEXVAR"; +AsmOp compile_tuple_at(std::vector& res, std::vector& args, SrcLocation) { + tolk_assert(args.size() == 2 && res.size() == 1); + auto& y = args[1]; + if (y.is_int_const() && y.int_const >= 0 && y.int_const < 16) { + y.unused(); + return exec_arg_op("INDEX", y.int_const, 1, 1); + } + return exec_op("INDEXVAR", 2, 1); +} + +// fun tupleSetAt(mutate self: tuple, value: X, index: int): void asm "SETINDEXVAR"; +AsmOp compile_tuple_set_at(std::vector& res, std::vector& args, SrcLocation) { + tolk_assert(args.size() == 3 && res.size() == 1); + auto& y = args[2]; + if (y.is_int_const() && y.int_const >= 0 && y.int_const < 16) { + y.unused(); + return exec_arg_op("SETINDEX", y.int_const, 1, 1); + } + return exec_op("SETINDEXVAR", 2, 1); +} + +// fun __isNull(X arg): bool +AsmOp compile_is_null(std::vector& res, std::vector& args, SrcLocation) { + tolk_assert(args.size() == 1 && res.size() == 1); + res[0].val = VarDescr::ValBool; + return exec_op("ISNULL", 1, 1); +} + + +void define_builtins() { + using namespace std::placeholders; + + TypePtr Unit = TypeDataVoid::create(); + TypePtr Int = TypeDataInt::create(); + TypePtr Bool = TypeDataBool::create(); + TypePtr Slice = TypeDataSlice::create(); + TypePtr Builder = TypeDataBuilder::create(); + TypePtr Tuple = TypeDataTuple::create(); + TypePtr Never = TypeDataNever::create(); + + std::vector itemsT; + itemsT.emplace_back("T"); + TypePtr typeT = TypeDataGenericT::create("T"); + const GenericsDeclaration* declGenericT = new GenericsDeclaration(std::move(itemsT)); + + std::vector ParamsInt1 = {Int}; + std::vector ParamsInt2 = {Int, Int}; + std::vector ParamsInt3 = {Int, Int, Int}; + std::vector ParamsSliceInt = {Slice, Int}; + + // builtin operators + // they are internally stored as functions, because at IR level, there is no difference + // between calling `userAdd(a,b)` and `_+_(a,b)` + // since they are registered in a global symtable, technically, they can even be referenced from Tolk code, + // though it's a "hidden feature" and won't work well for overloads (`==` for int and bool, for example) + + // unary operators + define_builtin_func("-_", ParamsInt1, Int, nullptr, + compile_unary_minus, + FunctionData::flagMarkedAsPure); + define_builtin_func("+_", ParamsInt1, Int, nullptr, + compile_unary_plus, + FunctionData::flagMarkedAsPure); + define_builtin_func("!_", ParamsInt1, Bool, nullptr, + std::bind(compile_logical_not, _1, _2, _3, true), + FunctionData::flagMarkedAsPure); + define_builtin_func("!b_", {Bool}, Bool, nullptr, // "overloaded" separate version for bool + std::bind(compile_logical_not, _1, _2, _3, false), + FunctionData::flagMarkedAsPure); + define_builtin_func("~_", ParamsInt1, Int, nullptr, + compile_bitwise_not, + FunctionData::flagMarkedAsPure); + + // binary operators + define_builtin_func("_+_", ParamsInt2, Int, nullptr, + compile_add, + FunctionData::flagMarkedAsPure); + define_builtin_func("_-_", ParamsInt2, Int, nullptr, + compile_sub, + FunctionData::flagMarkedAsPure); + define_builtin_func("_*_", ParamsInt2, Int, nullptr, + compile_mul, + FunctionData::flagMarkedAsPure); + define_builtin_func("_/_", ParamsInt2, Int, nullptr, + std::bind(compile_div, _1, _2, _3, -1), + FunctionData::flagMarkedAsPure); + define_builtin_func("_~/_", ParamsInt2, Int, nullptr, + std::bind(compile_div, _1, _2, _3, 0), + FunctionData::flagMarkedAsPure); + define_builtin_func("_^/_", ParamsInt2, Int, nullptr, + std::bind(compile_div, _1, _2, _3, 1), + FunctionData::flagMarkedAsPure); + define_builtin_func("_%_", ParamsInt2, Int, nullptr, + std::bind(compile_mod, _1, _2, _3, -1), + FunctionData::flagMarkedAsPure); + define_builtin_func("_<<_", ParamsInt2, Int, nullptr, + compile_lshift, + FunctionData::flagMarkedAsPure); + define_builtin_func("_>>_", ParamsInt2, Int, nullptr, + std::bind(compile_rshift, _1, _2, _3, -1), + FunctionData::flagMarkedAsPure); + define_builtin_func("_~>>_", ParamsInt2, Int, nullptr, + std::bind(compile_rshift, _1, _2, _3, 0), + FunctionData::flagMarkedAsPure); + define_builtin_func("_^>>_", ParamsInt2, Int, nullptr, + std::bind(compile_rshift, _1, _2, _3, 1), + FunctionData::flagMarkedAsPure); + define_builtin_func("_&_", ParamsInt2, Int, nullptr, // also works for bool + compile_bitwise_and, + FunctionData::flagMarkedAsPure); + define_builtin_func("_|_", ParamsInt2, Int, nullptr, // also works for bool + compile_bitwise_or, + FunctionData::flagMarkedAsPure); + define_builtin_func("_^_", ParamsInt2, Int, nullptr, // also works for bool + compile_bitwise_xor, + FunctionData::flagMarkedAsPure); + define_builtin_func("_==_", ParamsInt2, Int, nullptr, // also works for bool + std::bind(compile_cmp_int, _1, _2, 2), + FunctionData::flagMarkedAsPure); + define_builtin_func("_!=_", ParamsInt2, Int, nullptr, // also works for bool + std::bind(compile_cmp_int, _1, _2, 5), + FunctionData::flagMarkedAsPure); + define_builtin_func("_<_", ParamsInt2, Int, nullptr, + std::bind(compile_cmp_int, _1, _2, 4), + FunctionData::flagMarkedAsPure); + define_builtin_func("_>_", ParamsInt2, Int, nullptr, + std::bind(compile_cmp_int, _1, _2, 1), + FunctionData::flagMarkedAsPure); + define_builtin_func("_<=_", ParamsInt2, Int, nullptr, + std::bind(compile_cmp_int, _1, _2, 6), + FunctionData::flagMarkedAsPure); + define_builtin_func("_>=_", ParamsInt2, Int, nullptr, + std::bind(compile_cmp_int, _1, _2, 3), + FunctionData::flagMarkedAsPure); + define_builtin_func("_<=>_", ParamsInt2, Int, nullptr, + std::bind(compile_cmp_int, _1, _2, 7), + FunctionData::flagMarkedAsPure); + + // special function used for internal compilation of some lexical constructs + // for example, `throw 123;` is actually calling `__throw(123)` + define_builtin_func("__true", {}, Bool, nullptr, /* AsmOp::Const("TRUE") */ + std::bind(compile_bool_const, _1, _2, true), + FunctionData::flagMarkedAsPure); + define_builtin_func("__false", {}, Bool, nullptr, /* AsmOp::Const("FALSE") */ + std::bind(compile_bool_const, _1, _2, false), + FunctionData::flagMarkedAsPure); + define_builtin_func("__null", {}, typeT, declGenericT, + AsmOp::Const("PUSHNULL"), + FunctionData::flagMarkedAsPure); + define_builtin_func("__isNull", {typeT}, Bool, declGenericT, + compile_is_null, + FunctionData::flagMarkedAsPure); + define_builtin_func("__throw", ParamsInt1, Never, nullptr, + compile_throw, + 0); + define_builtin_func("__throw_arg", {typeT, Int}, Never, declGenericT, + compile_throw_arg, + 0); + define_builtin_func("__throw_if_unless", ParamsInt3, Unit, nullptr, + compile_throw_if_unless, + 0); + + // functions from stdlib marked as `builtin`, implemented at compiler level for optimizations + // (for example, `loadInt(1)` is `1 LDI`, but `loadInt(n)` for non-constant requires it be on a stack and `LDIX`) + define_builtin_func("mulDivFloor", ParamsInt3, Int, nullptr, + std::bind(compile_muldiv, _1, _2, _3, -1), + FunctionData::flagMarkedAsPure); + define_builtin_func("mulDivRound", ParamsInt3, Int, nullptr, + std::bind(compile_muldiv, _1, _2, _3, 0), + FunctionData::flagMarkedAsPure); + define_builtin_func("mulDivCeil", ParamsInt3, Int, nullptr, + std::bind(compile_muldiv, _1, _2, _3, 1), + FunctionData::flagMarkedAsPure); + define_builtin_func("mulDivMod", ParamsInt3, TypeDataTensor::create({Int, Int}), nullptr, + AsmOp::Custom("MULDIVMOD", 3, 2), + FunctionData::flagMarkedAsPure); + define_builtin_func("loadInt", ParamsSliceInt, Int, nullptr, + std::bind(compile_fetch_int, _1, _2, true, true), + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, + {}, {1, 0}); + define_builtin_func("loadUint", ParamsSliceInt, Int, nullptr, + std::bind(compile_fetch_int, _1, _2, true, false), + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, + {}, {1, 0}); + define_builtin_func("loadBits", ParamsSliceInt, Slice, nullptr, + std::bind(compile_fetch_slice, _1, _2, true), + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, + {}, {1, 0}); + define_builtin_func("preloadInt", ParamsSliceInt, Int, nullptr, + std::bind(compile_fetch_int, _1, _2, false, true), + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); + define_builtin_func("preloadUint", ParamsSliceInt, Int, nullptr, + std::bind(compile_fetch_int, _1, _2, false, false), + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); + define_builtin_func("preloadBits", ParamsSliceInt, Slice, nullptr, + std::bind(compile_fetch_slice, _1, _2, false), + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); + define_builtin_func("storeInt", {Builder, Int, Int}, Unit, nullptr, + std::bind(compile_store_int, _1, _2, true), + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf, + {1, 0, 2}, {}); + define_builtin_func("storeUint", {Builder, Int, Int}, Unit, nullptr, + std::bind(compile_store_int, _1, _2, false), + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf, + {1, 0, 2}, {}); + define_builtin_func("tupleAt", {Tuple, Int}, typeT, declGenericT, + compile_tuple_at, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); + define_builtin_func("tupleSetAt", {Tuple, typeT, Int}, Unit, declGenericT, + compile_tuple_set_at, + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf); + define_builtin_func("debugPrint", {typeT}, Unit, declGenericT, + AsmOp::Custom("s0 DUMP DROP", 1, 1), + 0); + define_builtin_func("debugPrintString", {typeT}, Unit, declGenericT, + AsmOp::Custom("STRDUMP DROP", 1, 1), + 0); + define_builtin_func("debugDumpStack", {}, Unit, nullptr, + AsmOp::Custom("DUMPSTK", 0, 0), + 0); + + // functions not presented in stdlib at all + // used in tolk-tester to check/expose internal compiler state + // each of them is handled in a special way, search by its name + define_builtin_func("__expect_type", {TypeDataUnknown::create(), Slice}, Unit, nullptr, + AsmOp::Nop(), + FunctionData::flagMarkedAsPure); +} + +} // namespace tolk diff --git a/tolk/codegen.cpp b/tolk/codegen.cpp new file mode 100644 index 00000000..ac1cf639 --- /dev/null +++ b/tolk/codegen.cpp @@ -0,0 +1,922 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" +#include "compiler-state.h" +#include "type-system.h" + +namespace tolk { + +/* + * + * GENERATE TVM STACK CODE + * + */ + +StackLayout Stack::vars() const { + StackLayout res; + res.reserve(s.size()); + for (auto x : s) { + res.push_back(x.first); + } + return res; +} + +int Stack::find(var_idx_t var, int from) const { + for (int i = from; i < depth(); i++) { + if (at(i).first == var) { + return i; + } + } + return -1; +} + +// finds var in [from .. to) +int Stack::find(var_idx_t var, int from, int to) const { + for (int i = from; i < depth() && i < to; i++) { + if (at(i).first == var) { + return i; + } + } + return -1; +} + +// finds var outside [from .. to) +int Stack::find_outside(var_idx_t var, int from, int to) const { + from = std::max(from, 0); + if (from >= to) { + return find(var); + } else { + int t = find(var, 0, from); + return t >= 0 ? t : find(var, to); + } +} + +int Stack::find_const(const_idx_t cst, int from) const { + for (int i = from; i < depth(); i++) { + if (at(i).second == cst) { + return i; + } + } + return -1; +} + +void Stack::forget_const() { + for (auto& vc : s) { + if (vc.second != not_const) { + vc.second = not_const; + } + } +} + +void Stack::issue_pop(int i) { + validate(i); + if (output_enabled()) { + o << AsmOp::Pop(i); + } + at(i) = get(0); + s.pop_back(); + modified(); +} + +void Stack::issue_push(int i) { + validate(i); + if (output_enabled()) { + o << AsmOp::Push(i); + } + s.push_back(get(i)); + modified(); +} + +void Stack::issue_xchg(int i, int j) { + validate(i); + validate(j); + if (i != j && get(i) != get(j)) { + if (output_enabled()) { + o << AsmOp::Xchg(i, j); + } + std::swap(at(i), at(j)); + modified(); + } +} + +int Stack::drop_vars_except(const VarDescrList& var_info, int excl_var) { + int dropped = 0, changes; + do { + changes = 0; + int n = depth(); + for (int i = 0; i < n; i++) { + var_idx_t idx = at(i).first; + if (((!var_info[idx] || var_info[idx]->is_unused()) && idx != excl_var) || find(idx, 0, i - 1) >= 0) { + // unneeded + issue_pop(i); + changes = 1; + break; + } + } + dropped += changes; + } while (changes); + return dropped; +} + +void Stack::show() { + std::ostringstream os; + for (auto i : s) { + os << ' '; + o.show_var_ext(os, i); + } + o << AsmOp::Comment(os.str()); + mode |= _Shown; +} + +void Stack::forget_var(var_idx_t idx) { + for (auto& x : s) { + if (x.first == idx) { + x = std::make_pair(_Garbage, not_const); + modified(); + } + } +} + +void Stack::push_new_var(var_idx_t idx) { + forget_var(idx); + s.emplace_back(idx, not_const); + modified(); +} + +void Stack::push_new_const(var_idx_t idx, const_idx_t cidx) { + forget_var(idx); + s.emplace_back(idx, cidx); + modified(); +} + +void Stack::assign_var(var_idx_t new_idx, var_idx_t old_idx) { + int i = find(old_idx); + tolk_assert(i >= 0 && "variable not found in stack"); + if (new_idx != old_idx) { + at(i).first = new_idx; + modified(); + } +} + +void Stack::do_copy_var(var_idx_t new_idx, var_idx_t old_idx) { + int i = find(old_idx); + tolk_assert(i >= 0 && "variable not found in stack"); + if (find(old_idx, i + 1) < 0) { + issue_push(i); + tolk_assert(at(0).first == old_idx); + } + assign_var(new_idx, old_idx); +} + +void Stack::enforce_state(const StackLayout& req_stack) { + int k = (int)req_stack.size(); + for (int i = 0; i < k; i++) { + var_idx_t x = req_stack[i]; + if (i < depth() && s[i].first == x) { + continue; + } + while (depth() > 0 && std::find(req_stack.cbegin(), req_stack.cend(), get(0).first) == req_stack.cend()) { + // current TOS entry is unused in req_stack, drop it + issue_pop(0); + } + int j = find(x); + if (j >= depth() - i) { + issue_push(j); + j = 0; + } + issue_xchg(j, depth() - i - 1); + tolk_assert(s[i].first == x); + } + while (depth() > k) { + issue_pop(0); + } + tolk_assert(depth() == k); + for (int i = 0; i < k; i++) { + tolk_assert(s[i].first == req_stack[i]); + } +} + +void Stack::merge_const(const Stack& req_stack) { + tolk_assert(s.size() == req_stack.s.size()); + for (std::size_t i = 0; i < s.size(); i++) { + tolk_assert(s[i].first == req_stack.s[i].first); + if (s[i].second != req_stack.s[i].second) { + s[i].second = not_const; + } + } +} + +void Stack::merge_state(const Stack& req_stack) { + enforce_state(req_stack.vars()); + merge_const(req_stack); +} + +void Stack::rearrange_top(const StackLayout& top, std::vector last) { + while (last.size() < top.size()) { + last.push_back(false); + } + int k = (int)top.size(); + for (int i = 0; i < k; i++) { + for (int j = i + 1; j < k; j++) { + if (top[i] == top[j]) { + last[i] = false; + break; + } + } + } + int ss = 0; + for (int i = 0; i < k; i++) { + if (last[i]) { + ++ss; + } + } + for (int i = 0; i < k; i++) { + var_idx_t x = top[i]; + // find s(j) containing x with j not in [ss, ss+i) + int j = find_outside(x, ss, ss + i); + if (last[i]) { + // rearrange x to be at s(ss-1) + issue_xchg(--ss, j); + tolk_assert(get(ss).first == x); + } else { + // create a new copy of x + issue_push(j); + issue_xchg(0, ss); + tolk_assert(get(ss).first == x); + } + } + tolk_assert(!ss); +} + +void Stack::rearrange_top(var_idx_t top, bool last) { + int i = find(top); + if (last) { + issue_xchg(0, i); + } else { + issue_push(i); + } + tolk_assert(get(0).first == top); +} + +bool Op::generate_code_step(Stack& stack) { + stack.opt_show(); + + // detect `throw 123` (actually _IntConst 123 + _Call __throw) + // don't clear the stack, since dropping unused elements make no sense, an exception is thrown anyway + bool will_now_immediate_throw = (cl == _Call && f_sym->is_builtin_function() && f_sym->name == "__throw") + || (cl == _IntConst && next->cl == _Call && next->f_sym->is_builtin_function() && next->f_sym->name == "__throw"); + if (!will_now_immediate_throw) { + stack.drop_vars_except(var_info); + stack.opt_show(); + } + + bool inline_func = stack.mode & Stack::_InlineFunc; + switch (cl) { + case _Nop: + case _Import: + return true; + case _Return: { + stack.enforce_state(left); + if (stack.o.retalt_ && (stack.mode & Stack::_NeedRetAlt)) { + stack.o << "RETALT"; + stack.o.retalt_inserted_ = true; + } + stack.opt_show(); + return false; + } + case _IntConst: { + auto p = next->var_info[left[0]]; + if (!p || p->is_unused()) { + return true; + } + auto cidx = stack.o.register_const(int_const); + int i = stack.find_const(cidx); + if (i < 0) { + stack.o << push_const(int_const); + stack.push_new_const(left[0], cidx); + } else { + tolk_assert(stack.at(i).second == cidx); + stack.do_copy_var(left[0], stack[i]); + } + return true; + } + case _SliceConst: { + auto p = next->var_info[left[0]]; + if (!p || p->is_unused()) { + return true; + } + stack.o << AsmOp::Const("x{" + str_const + "} PUSHSLICE"); + stack.push_new_var(left[0]); + return true; + } + case _GlobVar: + if (g_sym) { + bool used = false; + for (auto i : left) { + auto p = next->var_info[i]; + if (p && !p->is_unused()) { + used = true; + } + } + if (!used || disabled()) { + return true; + } + stack.o << AsmOp::Custom(g_sym->name + " GETGLOB", 0, 1); + if (left.size() != 1) { + tolk_assert(left.size() <= 15); + stack.o << AsmOp::UnTuple((int)left.size()); + } + for (auto i : left) { + stack.push_new_var(i); + } + return true; + } else { + tolk_assert(left.size() == 1); + auto p = next->var_info[left[0]]; + if (!p || p->is_unused() || disabled()) { + return true; + } + stack.o << "CONT:<{"; + stack.o.indent(); + if (f_sym->is_asm_function() || f_sym->is_builtin_function()) { + // TODO: create and compile a true lambda instead of this (so that arg_order and ret_order would work correctly) + std::vector args0, res; + int w_arg = 0; + for (const LocalVarData& param : f_sym->parameters) { + w_arg += param.declared_type->get_width_on_stack(); + } + int w_ret = f_sym->inferred_return_type->get_width_on_stack(); + tolk_assert(w_ret >= 0 && w_arg >= 0); + for (int i = 0; i < w_ret; i++) { + res.emplace_back(0); + } + for (int i = 0; i < w_arg; i++) { + args0.emplace_back(0); + } + if (f_sym->is_asm_function()) { + std::get(f_sym->body)->compile(stack.o); // compile res := f (args0) + } else { + std::get(f_sym->body)->compile(stack.o, res, args0, where); // compile res := f (args0) + } + } else { + stack.o << AsmOp::Custom(f_sym->name + " CALLDICT", (int)right.size(), (int)left.size()); + } + stack.o.undent(); + stack.o << "}>"; + stack.push_new_var(left.at(0)); + return true; + } + case _Let: { + tolk_assert(left.size() == right.size()); + int i = 0; + std::vector active; + active.reserve(left.size()); + for (std::size_t k = 0; k < left.size(); k++) { + var_idx_t y = left[k]; // "y" = "x" + auto p = next->var_info[y]; + active.push_back(p && !p->is_unused()); + } + for (std::size_t k = 0; k < left.size(); k++) { + if (!active[k]) { + continue; + } + var_idx_t x = right[k]; // "y" = "x" + bool is_last = true; + for (std::size_t l = k + 1; l < right.size(); l++) { + if (right[l] == x && active[l]) { + is_last = false; + } + } + if (is_last) { + auto info = var_info[x]; + is_last = (info && info->is_last()); + } + if (is_last) { + stack.assign_var(--i, x); + } else { + stack.do_copy_var(--i, x); + } + } + i = 0; + for (std::size_t k = 0; k < left.size(); k++) { + if (active[k]) { + stack.assign_var(left[k], --i); + } + } + return true; + } + case _Tuple: + case _UnTuple: { + if (disabled()) { + return true; + } + std::vector last; + for (var_idx_t x : right) { + last.push_back(var_info[x] && var_info[x]->is_last()); + } + stack.rearrange_top(right, std::move(last)); + stack.opt_show(); + int k = (int)stack.depth() - (int)right.size(); + tolk_assert(k >= 0); + if (cl == _Tuple) { + stack.o << AsmOp::Tuple((int)right.size()); + tolk_assert(left.size() == 1); + } else { + stack.o << AsmOp::UnTuple((int)left.size()); + tolk_assert(right.size() == 1); + } + stack.s.resize(k); + for (int i = 0; i < (int)left.size(); i++) { + stack.push_new_var(left.at(i)); + } + return true; + } + case _Call: + case _CallInd: { + if (disabled()) { + return true; + } + // f_sym can be nullptr for Op::_CallInd (invoke a variable, not a function) + const std::vector* arg_order = f_sym ? f_sym->get_arg_order() : nullptr; + const std::vector* ret_order = f_sym ? f_sym->get_ret_order() : nullptr; + tolk_assert(!arg_order || arg_order->size() == right.size()); + tolk_assert(!ret_order || ret_order->size() == left.size()); + std::vector right1; + if (args.size()) { + tolk_assert(args.size() == right.size()); + for (int i = 0; i < (int)right.size(); i++) { + int j = arg_order ? arg_order->at(i) : i; + const VarDescr& arg = args.at(j); + if (!arg.is_unused()) { + tolk_assert(var_info[arg.idx] && !var_info[arg.idx]->is_unused()); + right1.push_back(arg.idx); + } + } + } else { + tolk_assert(!arg_order); + right1 = right; + } + std::vector last; + last.reserve(right1.size()); + for (var_idx_t x : right1) { + last.push_back(var_info[x] && var_info[x]->is_last()); + } + stack.rearrange_top(right1, std::move(last)); + stack.opt_show(); + int k = (int)stack.depth() - (int)right1.size(); + tolk_assert(k >= 0); + for (int i = 0; i < (int)right1.size(); i++) { + if (stack.s[k + i].first != right1[i]) { + std::cerr << stack.o; + } + tolk_assert(stack.s[k + i].first == right1[i]); + } + auto exec_callxargs = [&](int args, int ret) { + if (args <= 15 && ret <= 15) { + stack.o << exec_arg2_op("CALLXARGS", args, ret, args + 1, ret); + } else { + tolk_assert(args <= 254 && ret <= 254); + stack.o << AsmOp::Const(PSTRING() << args << " PUSHINT"); + stack.o << AsmOp::Const(PSTRING() << ret << " PUSHINT"); + stack.o << AsmOp::Custom("CALLXVARARGS", args + 3, ret); + } + }; + if (cl == _CallInd) { + exec_callxargs((int)right.size() - 1, (int)left.size()); + } else if (!f_sym->is_code_function()) { + std::vector res; + res.reserve(left.size()); + for (var_idx_t i : left) { + res.emplace_back(i); + } + if (f_sym->is_asm_function()) { + std::get(f_sym->body)->compile(stack.o); // compile res := f (args) + } else { + std::get(f_sym->body)->compile(stack.o, res, args, where); // compile res := f (args) + } + } else { + if (f_sym->is_inline() || f_sym->is_inline_ref()) { + stack.o << AsmOp::Custom(f_sym->name + " INLINECALLDICT", (int)right.size(), (int)left.size()); + } else if (f_sym->is_code_function() && std::get(f_sym->body)->code->require_callxargs) { + stack.o << AsmOp::Custom(f_sym->name + (" PREPAREDICT"), 0, 2); + exec_callxargs((int)right.size() + 1, (int)left.size()); + } else { + stack.o << AsmOp::Custom(f_sym->name + " CALLDICT", (int)right.size(), (int)left.size()); + } + } + stack.s.resize(k); + for (int i = 0; i < (int)left.size(); i++) { + int j = ret_order ? ret_order->at(i) : i; + stack.push_new_var(left.at(j)); + } + return !f_sym || f_sym->declared_return_type != TypeDataNever::create(); + } + case _SetGlob: { + tolk_assert(g_sym); + std::vector last; + for (var_idx_t x : right) { + last.push_back(var_info[x] && var_info[x]->is_last()); + } + stack.rearrange_top(right, std::move(last)); + stack.opt_show(); + int k = (int)stack.depth() - (int)right.size(); + tolk_assert(k >= 0); + for (int i = 0; i < (int)right.size(); i++) { + if (stack.s[k + i].first != right[i]) { + std::cerr << stack.o; + } + tolk_assert(stack.s[k + i].first == right[i]); + } + if (right.size() > 1) { + stack.o << AsmOp::Tuple((int)right.size()); + } + if (!right.empty()) { + stack.o << AsmOp::Custom(g_sym->name + " SETGLOB", 1, 0); + } + stack.s.resize(k); + return true; + } + case _If: { + if (block0->is_empty() && block1->is_empty()) { + return true; + } + if (!next->noreturn() && (block0->noreturn() != block1->noreturn())) { + stack.o.retalt_ = true; + } + var_idx_t x = left[0]; + stack.rearrange_top(x, var_info[x] && var_info[x]->is_last()); + tolk_assert(stack[0] == x); + stack.opt_show(); + stack.s.pop_back(); + stack.modified(); + if (inline_func && (block0->noreturn() || block1->noreturn())) { + bool is0 = block0->noreturn(); + Op* block_noreturn = is0 ? block0.get() : block1.get(); + Op* block_other = is0 ? block1.get() : block0.get(); + stack.mode &= ~Stack::_InlineFunc; + stack.o << (is0 ? "IF:<{" : "IFNOT:<{"); + stack.o.indent(); + Stack stack_copy{stack}; + block_noreturn->generate_code_all(stack_copy); + stack.o.undent(); + stack.o << "}>ELSE<{"; + stack.o.indent(); + block_other->generate_code_all(stack); + if (!block_other->noreturn()) { + next->generate_code_all(stack); + } + stack.o.undent(); + stack.o << "}>"; + return false; + } + if (block1->is_empty() || block0->is_empty()) { + bool is0 = block1->is_empty(); + Op* block = is0 ? block0.get() : block1.get(); + // if (left) block0; ... + // if (!left) block1; ... + if (block->noreturn()) { + stack.o << (is0 ? "IFJMP:<{" : "IFNOTJMP:<{"); + stack.o.indent(); + Stack stack_copy{stack}; + stack_copy.mode &= ~Stack::_InlineFunc; + stack_copy.mode |= next->noreturn() ? 0 : Stack::_NeedRetAlt; + block->generate_code_all(stack_copy); + stack.o.undent(); + stack.o << "}>"; + return true; + } + stack.o << (is0 ? "IF:<{" : "IFNOT:<{"); + stack.o.indent(); + Stack stack_copy{stack}, stack_target{stack}; + stack_target.disable_output(); + stack_target.drop_vars_except(next->var_info); + stack_copy.mode &= ~Stack::_InlineFunc; + block->generate_code_all(stack_copy); + stack_copy.drop_vars_except(var_info); + stack_copy.opt_show(); + if ((is0 && stack_copy == stack) || (!is0 && stack_copy.vars() == stack.vars())) { + stack.o.undent(); + stack.o << "}>"; + if (!is0) { + stack.merge_const(stack_copy); + } + return true; + } + // stack_copy.drop_vars_except(next->var_info); + stack_copy.enforce_state(stack_target.vars()); + stack_copy.opt_show(); + if (stack_copy.vars() == stack.vars()) { + stack.o.undent(); + stack.o << "}>"; + stack.merge_const(stack_copy); + return true; + } + stack.o.undent(); + stack.o << "}>ELSE<{"; + stack.o.indent(); + stack.merge_state(stack_copy); + stack.opt_show(); + stack.o.undent(); + stack.o << "}>"; + return true; + } + if (block0->noreturn() || block1->noreturn()) { + bool is0 = block0->noreturn(); + Op* block_noreturn = is0 ? block0.get() : block1.get(); + Op* block_other = is0 ? block1.get() : block0.get(); + stack.o << (is0 ? "IFJMP:<{" : "IFNOTJMP:<{"); + stack.o.indent(); + Stack stack_copy{stack}; + stack_copy.mode &= ~Stack::_InlineFunc; + stack_copy.mode |= (block_other->noreturn() || next->noreturn()) ? 0 : Stack::_NeedRetAlt; + block_noreturn->generate_code_all(stack_copy); + stack.o.undent(); + stack.o << "}>"; + block_other->generate_code_all(stack); + return !block_other->noreturn(); + } + stack.o << "IF:<{"; + stack.o.indent(); + Stack stack_copy{stack}; + stack_copy.mode &= ~Stack::_InlineFunc; + block0->generate_code_all(stack_copy); + stack_copy.drop_vars_except(next->var_info); + stack_copy.opt_show(); + stack.o.undent(); + stack.o << "}>ELSE<{"; + stack.o.indent(); + stack.mode &= ~Stack::_InlineFunc; + block1->generate_code_all(stack); + stack.merge_state(stack_copy); + stack.opt_show(); + stack.o.undent(); + stack.o << "}>"; + return true; + } + case _Repeat: { + var_idx_t x = left[0]; + //stack.drop_vars_except(block0->var_info, x); + stack.rearrange_top(x, var_info[x] && var_info[x]->is_last()); + tolk_assert(stack[0] == x); + stack.opt_show(); + stack.s.pop_back(); + stack.modified(); + if (block0->noreturn()) { + stack.o.retalt_ = true; + } + if (true || !next->is_empty()) { + stack.o << "REPEAT:<{"; + stack.o.indent(); + stack.forget_const(); + if (block0->noreturn()) { + Stack stack_copy{stack}; + StackLayout layout1 = stack.vars(); + stack_copy.mode &= ~Stack::_InlineFunc; + stack_copy.mode |= Stack::_NeedRetAlt; + block0->generate_code_all(stack_copy); + } else { + StackLayout layout1 = stack.vars(); + stack.mode &= ~Stack::_InlineFunc; + stack.mode |= Stack::_NeedRetAlt; + block0->generate_code_all(stack); + stack.enforce_state(std::move(layout1)); + stack.opt_show(); + } + stack.o.undent(); + stack.o << "}>"; + return true; + } else { + stack.o << "REPEATEND"; + stack.forget_const(); + StackLayout layout1 = stack.vars(); + block0->generate_code_all(stack); + stack.enforce_state(std::move(layout1)); + stack.opt_show(); + return false; + } + } + case _Again: { + stack.drop_vars_except(block0->var_info); + stack.opt_show(); + if (block0->noreturn()) { + stack.o.retalt_ = true; + } + if (!next->is_empty() || inline_func) { + stack.o << "AGAIN:<{"; + stack.o.indent(); + stack.forget_const(); + StackLayout layout1 = stack.vars(); + stack.mode &= ~Stack::_InlineFunc; + stack.mode |= Stack::_NeedRetAlt; + block0->generate_code_all(stack); + stack.enforce_state(std::move(layout1)); + stack.opt_show(); + stack.o.undent(); + stack.o << "}>"; + return true; + } else { + stack.o << "AGAINEND"; + stack.forget_const(); + StackLayout layout1 = stack.vars(); + block0->generate_code_all(stack); + stack.enforce_state(std::move(layout1)); + stack.opt_show(); + return false; + } + } + case _Until: { + // stack.drop_vars_except(block0->var_info); + // stack.opt_show(); + if (block0->noreturn()) { + stack.o.retalt_ = true; + } + if (true || !next->is_empty()) { + stack.o << "UNTIL:<{"; + stack.o.indent(); + stack.forget_const(); + auto layout1 = stack.vars(); + stack.mode &= ~Stack::_InlineFunc; + stack.mode |= Stack::_NeedRetAlt; + block0->generate_code_all(stack); + layout1.push_back(left[0]); + stack.enforce_state(std::move(layout1)); + stack.opt_show(); + stack.o.undent(); + stack.o << "}>"; + stack.s.pop_back(); + stack.modified(); + return true; + } else { + stack.o << "UNTILEND"; + stack.forget_const(); + StackLayout layout1 = stack.vars(); + block0->generate_code_all(stack); + layout1.push_back(left[0]); + stack.enforce_state(std::move(layout1)); + stack.opt_show(); + return false; + } + } + case _While: { + // while (block0 | left) block1; ...next + var_idx_t x = left[0]; + stack.drop_vars_except(block0->var_info); + stack.opt_show(); + StackLayout layout1 = stack.vars(); + bool next_empty = false && next->is_empty(); + if (block0->noreturn()) { + stack.o.retalt_ = true; + } + stack.o << "WHILE:<{"; + stack.o.indent(); + stack.forget_const(); + stack.mode &= ~Stack::_InlineFunc; + stack.mode |= Stack::_NeedRetAlt; + block0->generate_code_all(stack); + stack.rearrange_top(x, !next->var_info[x] && !block1->var_info[x]); + stack.opt_show(); + stack.s.pop_back(); + stack.modified(); + stack.o.undent(); + Stack stack_copy{stack}; + stack.o << (next_empty ? "}>DO:" : "}>DO<{"); + if (!next_empty) { + stack.o.indent(); + } + stack_copy.opt_show(); + block1->generate_code_all(stack_copy); + stack_copy.enforce_state(std::move(layout1)); + stack_copy.opt_show(); + if (!next_empty) { + stack.o.undent(); + stack.o << "}>"; + return true; + } else { + return false; + } + } + case _TryCatch: { + if (block0->is_empty() && block1->is_empty()) { + return true; + } + if (block0->noreturn() || block1->noreturn()) { + stack.o.retalt_ = true; + } + Stack catch_stack{stack.o}; + std::vector catch_vars; + std::vector catch_last; + for (const VarDescr& var : block1->var_info.list) { + if (stack.find(var.idx) >= 0) { + catch_vars.push_back(var.idx); + catch_last.push_back(!block0->var_info[var.idx]); + } + } + const size_t block_size = 255; + for (size_t begin = catch_vars.size(), end = begin; end > 0; end = begin) { + begin = end >= block_size ? end - block_size : 0; + for (size_t i = begin; i < end; ++i) { + catch_stack.push_new_var(catch_vars[i]); + } + } + catch_stack.push_new_var(left[0]); + catch_stack.push_new_var(left[1]); + stack.rearrange_top(catch_vars, catch_last); + stack.opt_show(); + stack.o << "c1 PUSH"; + stack.o << "c3 PUSH"; + stack.o << "c4 PUSH"; + stack.o << "c5 PUSH"; + stack.o << "c7 PUSH"; + stack.o << "<{"; + stack.o.indent(); + if (block1->noreturn()) { + catch_stack.mode |= Stack::_NeedRetAlt; + } + block1->generate_code_all(catch_stack); + catch_stack.drop_vars_except(next->var_info); + catch_stack.opt_show(); + stack.o.undent(); + stack.o << "}>CONT"; + stack.o << "c7 SETCONT"; + stack.o << "c5 SETCONT"; + stack.o << "c4 SETCONT"; + stack.o << "c3 SETCONT"; + stack.o << "c1 SETCONT"; + for (size_t begin = catch_vars.size(), end = begin; end > 0; end = begin) { + begin = end >= block_size ? end - block_size : 0; + stack.o << std::to_string(end - begin) + " PUSHINT"; + stack.o << "-1 PUSHINT"; + stack.o << "SETCONTVARARGS"; + } + stack.s.erase(stack.s.end() - catch_vars.size(), stack.s.end()); + stack.modified(); + stack.o << "<{"; + stack.o.indent(); + if (block0->noreturn()) { + stack.mode |= Stack::_NeedRetAlt; + } + block0->generate_code_all(stack); + if (block0->noreturn()) { + stack.s = std::move(catch_stack.s); + } else if (!block1->noreturn()) { + stack.merge_state(catch_stack); + } + stack.opt_show(); + stack.o.undent(); + stack.o << "}>CONT"; + stack.o << "c1 PUSH"; + stack.o << "COMPOSALT"; + stack.o << "SWAP"; + stack.o << "TRY"; + return true; + } + default: + std::cerr << "fatal: unknown operation \n"; + throw ParseError{where, "unknown operation in generate_code()"}; + } +} + +void Op::generate_code_all(Stack& stack) { + int saved_mode = stack.mode; + auto cont = generate_code_step(stack); + stack.mode = (stack.mode & ~Stack::_ModeSave) | (saved_mode & Stack::_ModeSave); + if (cont && next) { + next->generate_code_all(stack); + } +} + +void CodeBlob::generate_code(AsmOpList& out, int mode) { + Stack stack{out, mode}; + tolk_assert(ops && ops->cl == Op::_Import); + auto args = (int)ops->left.size(); + for (var_idx_t x : ops->left) { + stack.push_new_var(x); + } + ops->generate_code_all(stack); + stack.apply_wrappers(require_callxargs && (mode & Stack::_InlineAny) ? args : -1); +} + +void CodeBlob::generate_code(std::ostream& os, int mode, int indent) { + AsmOpList out_list(indent, &vars); + generate_code(out_list, mode); + if (G.settings.optimization_level >= 2) { + optimize_code(out_list); + } + out_list.out(os, mode); +} + +} // namespace tolk diff --git a/tolk/compiler-state.cpp b/tolk/compiler-state.cpp new file mode 100644 index 00000000..95a7e6a5 --- /dev/null +++ b/tolk/compiler-state.cpp @@ -0,0 +1,73 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "compiler-state.h" +#include +#include + +namespace tolk { + +CompilerState G; // the only mutable global variable in tolk internals + +void ExperimentalOption::mark_deprecated(const char* deprecated_from_v, const char* deprecated_reason) { + this->deprecated_from_v = deprecated_from_v; + this->deprecated_reason = deprecated_reason; +} + +std::string_view PersistentHeapAllocator::copy_string_to_persistent_memory(std::string_view str_in_tmp_memory) { + size_t len = str_in_tmp_memory.size(); + char* allocated = new char[len]; + memcpy(allocated, str_in_tmp_memory.data(), str_in_tmp_memory.size()); + auto new_chunk = std::make_unique(allocated, std::move(head)); + head = std::move(new_chunk); + return {head->allocated, len}; +} + +void PersistentHeapAllocator::clear() { + head = nullptr; +} + +void CompilerSettings::enable_experimental_option(std::string_view name) { + ExperimentalOption* to_enable = nullptr; + + if (name == remove_unused_functions.name) { + to_enable = &remove_unused_functions; + } + + if (to_enable == nullptr) { + std::cerr << "unknown experimental option: " << name << std::endl; + } else if (to_enable->deprecated_from_v) { + std::cerr << "experimental option " << name << " " + << "is deprecated since Tolk v" << to_enable->deprecated_from_v + << ": " << to_enable->deprecated_reason << std::endl; + } else { + to_enable->enabled = true; + } +} + +void CompilerSettings::parse_experimental_options_cmd_arg(const std::string& cmd_arg) { + std::istringstream stream(cmd_arg); + std::string token; + while (std::getline(stream, token, ',')) { + enable_experimental_option(token); + } +} + +const std::vector& get_all_not_builtin_functions() { + return G.all_functions; +} + +} // namespace tolk diff --git a/tolk/compiler-state.h b/tolk/compiler-state.h new file mode 100644 index 00000000..1d166a3a --- /dev/null +++ b/tolk/compiler-state.h @@ -0,0 +1,109 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "src-file.h" +#include "symtable.h" +#include "td/utils/Status.h" +#include +#include +#include + +namespace tolk { + +// with cmd option -x, the user can pass experimental options to use +class ExperimentalOption { + friend struct CompilerSettings; + + const std::string_view name; + bool enabled = false; + const char* deprecated_from_v = nullptr; // when an option becomes deprecated (after the next compiler release), + const char* deprecated_reason = nullptr; // but the user still passes it, we'll warn to stderr + +public: + explicit ExperimentalOption(std::string_view name) : name(name) {} + + void mark_deprecated(const char* deprecated_from_v, const char* deprecated_reason); + + explicit operator bool() const { return enabled; } +}; + +// CompilerSettings contains settings that can be passed via cmd line or (partially) wasm envelope. +// They are filled once at start and are immutable since the compilation started. +struct CompilerSettings { + enum class FsReadCallbackKind { Realpath, ReadFile }; + + using FsReadCallback = std::function(FsReadCallbackKind, const char*)>; + + int verbosity = 0; + int optimization_level = 2; + bool stack_layout_comments = true; + + std::string output_filename; + std::string boc_output_filename; + std::string stdlib_folder; // a path to tolk-stdlib/; files imported via @stdlib/xxx are there + + FsReadCallback read_callback; + + ExperimentalOption remove_unused_functions{"remove-unused-functions"}; + + void enable_experimental_option(std::string_view name); + void parse_experimental_options_cmd_arg(const std::string& cmd_arg); +}; + +// AST nodes contain std::string_view referencing to contents of .tolk files (kept in memory after reading). +// It's more than enough, except a situation when we create new AST nodes inside the compiler +// and want some "persistent place" for std::string_view to point to. +// This class copies strings to heap, so that they remain valid after closing scope. +class PersistentHeapAllocator { + struct ChunkInHeap { + const char* allocated; + std::unique_ptr next; + + ChunkInHeap(const char* allocated, std::unique_ptr&& next) + : allocated(allocated), next(std::move(next)) {} + }; + + std::unique_ptr head = nullptr; + +public: + std::string_view copy_string_to_persistent_memory(std::string_view str_in_tmp_memory); + void clear(); +}; + +// CompilerState contains a mutable state that is changed while the compilation is going on. +// It's a "global state" of all compilation. +// Historically, in FunC, this global state was spread along many global C++ variables. +// Now, no global C++ variables except `CompilerState G` are present. +struct CompilerState { + CompilerSettings settings; + + GlobalSymbolTable symtable; + PersistentHeapAllocator persistent_mem; + + std::vector all_functions; // all user-defined (not built-in) functions, with generic instantiations + std::vector all_get_methods; + std::vector all_global_vars; + std::vector all_constants; + AllRegisteredSrcFiles all_src_files; + + bool is_verbosity(int gt_eq) const { return settings.verbosity >= gt_eq; } +}; + +extern CompilerState G; + +} // namespace tolk diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp new file mode 100644 index 00000000..4d11b922 --- /dev/null +++ b/tolk/constant-evaluator.cpp @@ -0,0 +1,317 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "constant-evaluator.h" +#include "ast.h" +#include "tolk.h" +#include "openssl/digest.hpp" +#include "crypto/common/util.h" +#include "td/utils/crypto.h" +#include "ton/ton-types.h" + +namespace tolk { + +// parse address like "EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5" +// based on unpack_std_smc_addr() from block.cpp +// (which is not included to avoid linking with ton_crypto) +static bool parse_friendly_address(const char packed[48], ton::WorkchainId& workchain, ton::StdSmcAddress& addr) { + unsigned char buffer[36]; + if (!td::buff_base64_decode(td::MutableSlice{buffer, 36}, td::Slice{packed, 48}, true)) { + return false; + } + td::uint16 crc = td::crc16(td::Slice{buffer, 34}); + if (buffer[34] != (crc >> 8) || buffer[35] != (crc & 0xff) || (buffer[0] & 0x3f) != 0x11) { + return false; + } + workchain = static_cast(buffer[1]); + std::memcpy(addr.data(), buffer + 2, 32); + return true; +} + +// parse address like "0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8" +// based on StdAddress::parse_addr() from block.cpp +// (which is not included to avoid linking with ton_crypto) +static bool parse_raw_address(const std::string& acc_string, int& workchain, ton::StdSmcAddress& addr) { + size_t pos = acc_string.find(':'); + if (pos != std::string::npos) { + td::Result r_wc = td::to_integer_safe(acc_string.substr(0, pos)); + if (r_wc.is_error()) { + return false; + } + workchain = r_wc.move_as_ok(); + pos++; + } else { + pos = 0; + } + if (acc_string.size() != pos + 64) { + return false; + } + + for (int i = 0; i < 64; ++i) { // loop through each hex digit + char c = acc_string[pos + i]; + int x; + if (c >= '0' && c <= '9') { + x = c - '0'; + } else if (c >= 'a' && c <= 'z') { + x = c - 'a' + 10; + } else if (c >= 'A' && c <= 'Z') { + x = c - 'A' + 10; + } else { + return false; + } + + if ((i & 1) == 0) { + addr.data()[i >> 1] = static_cast((addr.data()[i >> 1] & 0x0F) | (x << 4)); + } else { + addr.data()[i >> 1] = static_cast((addr.data()[i >> 1] & 0xF0) | x); + } + } + return true; +} + + +static std::string parse_vertex_string_const_as_slice(V v) { + std::string str = static_cast(v->str_val); + switch (v->modifier) { + case 0: { + return td::hex_encode(str); + } + case 's': { + unsigned char buff[128]; + long bits = td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.data(), str.data() + str.size()); + if (bits < 0) { + v->error("invalid hex bitstring constant '" + str + "'"); + } + return str; + } + case 'a': { // MsgAddress + ton::WorkchainId workchain; + ton::StdSmcAddress addr; + bool correct = (str.size() == 48 && parse_friendly_address(str.data(), workchain, addr)) || + (str.size() != 48 && parse_raw_address(str, workchain, addr)); + if (!correct) { + v->error("invalid standard address '" + str + "'"); + } + if (workchain < -128 || workchain >= 128) { + v->error("anycast addresses not supported"); + } + + unsigned char data[3 + 8 + 256]; // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; + td::bitstring::bits_store_long_top(data, 0, static_cast(4) << (64 - 3), 3); + td::bitstring::bits_store_long_top(data, 3, static_cast(workchain) << (64 - 8), 8); + td::bitstring::bits_memcpy(data, 3 + 8, addr.bits().ptr, 0, ton::StdSmcAddress::size()); + return td::BitSlice{data, sizeof(data)}.to_hex(); + } + default: + tolk_assert(false); + } +} + +static td::RefInt256 parse_vertex_string_const_as_int(V v) { + std::string str = static_cast(v->str_val); + switch (v->modifier) { + case 'u': { + td::RefInt256 intval = td::hex_string_to_int256(td::hex_encode(str)); + if (str.empty()) { + v->error("empty integer ascii-constant"); + } + if (intval.is_null()) { + v->error("too long integer ascii-constant"); + } + return intval; + } + case 'h': + case 'H': { + unsigned char hash[32]; + digest::hash_str(hash, str.data(), str.size()); + return td::bits_to_refint(hash, (v->modifier == 'h') ? 32 : 256, false); + } + case 'c': { + return td::make_refint(td::crc32(td::Slice{str})); + } + default: + tolk_assert(false); + } +} + + +struct ConstantEvaluator { + static bool is_overflow(const td::RefInt256& intval) { + return intval.is_null() || !intval->signed_fits_bits(257); + } + + static ConstantValue handle_unary_operator(V v, const ConstantValue& rhs) { + if (!rhs.is_int()) { + v->error("invalid operator, expecting integer"); + } + td::RefInt256 intval = std::get(rhs.value); + + switch (v->tok) { + case tok_minus: + intval = -intval; + break; + case tok_plus: + break; + case tok_bitwise_not: + intval = ~intval; + break; + case tok_logical_not: + intval = td::make_refint(intval == 0 ? -1 : 0); + break; + default: + v->error("not a constant expression"); + } + + if (is_overflow(intval)) { + v->error("integer overflow"); + } + return ConstantValue::from_int(std::move(intval)); + } + + static ConstantValue handle_binary_operator(V v, const ConstantValue& lhs, const ConstantValue& rhs) { + if (!lhs.is_int() || !rhs.is_int()) { + v->error("invalid operator, expecting integer"); + } + td::RefInt256 lhs_intval = std::get(lhs.value); + td::RefInt256 rhs_intval = std::get(rhs.value); + td::RefInt256 intval; + + switch (v->tok) { + case tok_minus: + intval = lhs_intval - rhs_intval; + break; + case tok_plus: + intval = lhs_intval + rhs_intval; + break; + case tok_mul: + intval = lhs_intval * rhs_intval; + break; + case tok_div: + intval = lhs_intval / rhs_intval; + break; + case tok_mod: + intval = lhs_intval % rhs_intval; + break; + case tok_lshift: + intval = lhs_intval << static_cast(rhs_intval->to_long()); + break; + case tok_rshift: + intval = lhs_intval >> static_cast(rhs_intval->to_long()); + break; + case tok_bitwise_and: + intval = lhs_intval & rhs_intval; + break; + case tok_bitwise_or: + intval = lhs_intval | rhs_intval; + break; + case tok_bitwise_xor: + intval = lhs_intval ^ rhs_intval; + break; + case tok_eq: + intval = td::make_refint(lhs_intval == rhs_intval ? -1 : 0); + break; + case tok_lt: + intval = td::make_refint(lhs_intval < rhs_intval ? -1 : 0); + break; + case tok_gt: + intval = td::make_refint(lhs_intval > rhs_intval ? -1 : 0); + break; + case tok_leq: + intval = td::make_refint(lhs_intval <= rhs_intval ? -1 : 0); + break; + case tok_geq: + intval = td::make_refint(lhs_intval >= rhs_intval ? -1 : 0); + break; + case tok_neq: + intval = td::make_refint(lhs_intval != rhs_intval ? -1 : 0); + break; + default: + v->error("unsupported binary operator in constant expression"); + } + + if (is_overflow(intval)) { + v->error("integer overflow"); + } + return ConstantValue::from_int(std::move(intval)); + } + + static ConstantValue handle_reference(V v) { + // todo better handle "appears, directly or indirectly, in its own initializer" + std::string_view name = v->get_name(); + const Symbol* sym = lookup_global_symbol(name); + if (!sym) { + v->error("undefined symbol `" + static_cast(name) + "`"); + } + GlobalConstPtr const_ref = sym->try_as(); + if (!const_ref) { + v->error("symbol `" + static_cast(name) + "` is not a constant"); + } + if (v->has_instantiationTs()) { // SOME_CONST + v->error("constant is not a generic"); + } + return {const_ref->value}; + } + + static ConstantValue visit(AnyExprV v) { + if (auto v_int = v->try_as()) { + return ConstantValue::from_int(v_int->intval); + } + if (auto v_bool = v->try_as()) { + return ConstantValue::from_int(v_bool->bool_val ? -1 : 0); + } + if (auto v_unop = v->try_as()) { + return handle_unary_operator(v_unop, visit(v_unop->get_rhs())); + } + if (auto v_binop = v->try_as()) { + return handle_binary_operator(v_binop, visit(v_binop->get_lhs()), visit(v_binop->get_rhs())); + } + if (auto v_ref = v->try_as()) { + return handle_reference(v_ref); + } + if (auto v_par = v->try_as()) { + return visit(v_par->get_expr()); + } + if (v->try_as()) { + return eval_const_init_value(v); + } + v->error("not a constant expression"); + } + + static ConstantValue eval_const_init_value(AnyExprV init_value) { + // it init_value is incorrect, an exception is thrown + return visit(init_value); + } +}; + +ConstantValue eval_const_init_value(AnyExprV init_value) { + // at first, handle most simple cases, not to launch heavy computation algorithm: just a number, just a string + // just `c = 1` or `c = 0xFF` + if (auto v_int = init_value->try_as()) { + return {v_int->intval}; + } + // just `c = "strval"`, probably with modifier (address, etc.) + if (auto v_string = init_value->try_as()) { + if (v_string->is_bitslice()) { + return {parse_vertex_string_const_as_slice(v_string)}; + } else { + return {parse_vertex_string_const_as_int(v_string)}; + } + } + // something more complex, like `c = anotherC` or `c = 1 << 8` + return ConstantEvaluator::eval_const_init_value(init_value); +} + +} // namespace tolk diff --git a/tonlib/tonlib/ClientActor.h b/tolk/constant-evaluator.h similarity index 50% rename from tonlib/tonlib/ClientActor.h rename to tolk/constant-evaluator.h index 592589cd..0f99867d 100644 --- a/tonlib/tonlib/ClientActor.h +++ b/tolk/constant-evaluator.h @@ -13,29 +13,33 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "tonlib/TonlibCallback.h" -#include "tonlib/TonlibClient.h" -#include "auto/tl/tonlib_api.h" +#include "fwd-declarations.h" +#include "crypto/common/refint.h" +#include -namespace tonlinb { -class ClientActor : public td::actor::Actor { - public: - explicit ClientActor(td::unique_ptr callback); - void request(td::uint64 id, tonlib_api::object_ptr request); - static tonlib_api::object_ptr execute(tonlib_api::object_ptr request); - ~ClientActor(); - ClientActor(ClientActor&& other); - ClientActor& operator=(ClientActor&& other); +namespace tolk { - ClientActor(const ClientActor& other) = delete; - ClientActor& operator=(const ClientActor& other) = delete; +struct ConstantValue { + std::variant value; - private: - td::actor::ActorOwn tonlib_; + bool is_int() const { return std::holds_alternative(value); } + bool is_slice() const { return std::holds_alternative(value); } + + td::RefInt256 as_int() const { return std::get(value); } + const std::string& as_slice() const { return std::get(value); } + + static ConstantValue from_int(int value) { + return {td::make_refint(value)}; + } + + static ConstantValue from_int(td::RefInt256 value) { + return {std::move(value)}; + } }; -} // namespace tonlinb + +ConstantValue eval_const_init_value(AnyExprV init_value); + +} // namespace tolk diff --git a/tolk/fwd-declarations.h b/tolk/fwd-declarations.h new file mode 100644 index 00000000..8d3b24a8 --- /dev/null +++ b/tolk/fwd-declarations.h @@ -0,0 +1,45 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +namespace tolk { + +struct ASTNodeBase; +struct ASTNodeExpressionBase; +struct ASTNodeStatementBase; + +using AnyV = const ASTNodeBase*; +using AnyExprV = const ASTNodeExpressionBase*; +using AnyStatementV = const ASTNodeStatementBase*; + +struct Symbol; +struct LocalVarData; +struct FunctionData; +struct GlobalVarData; +struct GlobalConstData; + +using LocalVarPtr = const LocalVarData*; +using FunctionPtr = const FunctionData*; +using GlobalVarPtr = const GlobalVarData*; +using GlobalConstPtr = const GlobalConstData*; + +class TypeData; +using TypePtr = const TypeData*; + +struct SrcFile; + +} // namespace tolk diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp new file mode 100644 index 00000000..9dae3f00 --- /dev/null +++ b/tolk/generics-helpers.cpp @@ -0,0 +1,271 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "generics-helpers.h" +#include "tolk.h" +#include "ast.h" +#include "ast-replicator.h" +#include "type-system.h" +#include "compiler-state.h" +#include "pipeline.h" + +namespace tolk { + +// given orig = "(int, T)" and substitutions = [slice], return "(int, slice)" +static TypePtr replace_genericT_with_deduced(TypePtr orig, const GenericsDeclaration* genericTs, const std::vector& substitutionTs) { + if (!orig || !orig->has_genericT_inside()) { + return orig; + } + tolk_assert(genericTs->size() == substitutionTs.size()); + + return orig->replace_children_custom([genericTs, substitutionTs](TypePtr child) { + if (const TypeDataGenericT* asT = child->try_as()) { + int idx = genericTs->find_nameT(asT->nameT); + if (idx == -1) { + throw Fatal("can not replace generic " + asT->nameT); + } + if (substitutionTs[idx] == nullptr) { + throw GenericDeduceError("can not deduce " + asT->nameT); + } + return substitutionTs[idx]; + } + return child; + }); +} + +GenericSubstitutionsDeduceForCall::GenericSubstitutionsDeduceForCall(FunctionPtr fun_ref) + : fun_ref(fun_ref) { + substitutionTs.resize(fun_ref->genericTs->size()); // filled with nullptr (nothing deduced) +} + +void GenericSubstitutionsDeduceForCall::provide_deducedT(const std::string& nameT, TypePtr deduced) { + if (deduced == TypeDataNullLiteral::create() || deduced->has_unknown_inside()) { + return; // just 'null' doesn't give sensible info + } + + int idx = fun_ref->genericTs->find_nameT(nameT); + if (substitutionTs[idx] == nullptr) { + substitutionTs[idx] = deduced; + } else if (substitutionTs[idx] != deduced) { + throw GenericDeduceError(nameT + " is both " + substitutionTs[idx]->as_human_readable() + " and " + deduced->as_human_readable()); + } +} + +void GenericSubstitutionsDeduceForCall::provide_manually_specified(std::vector&& substitutionTs) { + this->substitutionTs = std::move(substitutionTs); + this->manually_specified = true; +} + +// purpose: having `f(value: T)` and call `f(5)`, deduce T = int +// generally, there may be many generic Ts for declaration, and many arguments +// for every argument, `consider_next_condition()` is called +// example: `f(a: int, b: T1, c: (T1, T2))` and call `f(6, 7, (8, cs))` +// - `a` does not affect, it doesn't depend on generic Ts +// - next condition: param_type = `T1`, arg_type = `int`, deduce T1 = int +// - next condition: param_type = `(T1, T2)`, arg_type = `(int, slice)`, deduce T1 = int, T2 = slice +// for call `f(6, cs, (8, cs))` T1 will be both `slice` and `int`, fired an error +void GenericSubstitutionsDeduceForCall::consider_next_condition(TypePtr param_type, TypePtr arg_type) { + if (const auto* asT = param_type->try_as()) { + // `(arg: T)` called as `f([1, 2])` => T is [int, int] + provide_deducedT(asT->nameT, arg_type); + } else if (const auto* p_nullable = param_type->try_as()) { + // `arg: T?` called as `f(nullableInt)` => T is int + if (const auto* a_nullable = arg_type->try_as()) { + consider_next_condition(p_nullable->inner, a_nullable->inner); + } + // `arg: T?` called as `f(int)` => T is int + else { + consider_next_condition(p_nullable->inner, arg_type); + } + } else if (const auto* p_tensor = param_type->try_as()) { + // `arg: (int, T)` called as `f((5, cs))` => T is slice + if (const auto* a_tensor = arg_type->try_as(); a_tensor && a_tensor->size() == p_tensor->size()) { + for (int i = 0; i < a_tensor->size(); ++i) { + consider_next_condition(p_tensor->items[i], a_tensor->items[i]); + } + } + } else if (const auto* p_tuple = param_type->try_as()) { + // `arg: [int, T]` called as `f([5, cs])` => T is slice + if (const auto* a_tuple = arg_type->try_as(); a_tuple && a_tuple->size() == p_tuple->size()) { + for (int i = 0; i < a_tuple->size(); ++i) { + consider_next_condition(p_tuple->items[i], a_tuple->items[i]); + } + } + } else if (const auto* p_callable = param_type->try_as()) { + // `arg: fun(TArg) -> TResult` called as `f(calcTupleLen)` => TArg is tuple, TResult is int + if (const auto* a_callable = arg_type->try_as(); a_callable && a_callable->params_size() == p_callable->params_size()) { + for (int i = 0; i < a_callable->params_size(); ++i) { + consider_next_condition(p_callable->params_types[i], a_callable->params_types[i]); + } + consider_next_condition(p_callable->return_type, a_callable->return_type); + } + } +} + +TypePtr GenericSubstitutionsDeduceForCall::replace_by_manually_specified(TypePtr param_type) const { + return replace_genericT_with_deduced(param_type, fun_ref->genericTs, substitutionTs); +} + +TypePtr GenericSubstitutionsDeduceForCall::auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type) { + try { + if (!manually_specified) { + consider_next_condition(param_type, arg_type); + } + return replace_genericT_with_deduced(param_type, fun_ref->genericTs, substitutionTs); + } catch (const GenericDeduceError& ex) { + throw ParseError(cur_f, loc, ex.message + " for generic function `" + fun_ref->as_human_readable() + "`; instantiate it manually with `" + fun_ref->name + "<...>()`"); + } +} + +int GenericSubstitutionsDeduceForCall::get_first_not_deduced_idx() const { + for (int i = 0; i < static_cast(substitutionTs.size()); ++i) { + if (substitutionTs[i] == nullptr) { + return i; + } + } + return -1; +} + +// clone the body of `f` replacing T everywhere with a substitution +// before: `fun f(v: T) { var cp: [T] = [v]; }` +// after: `fun f(v: int) { var cp: [int] = [v]; }` +// an instantiated function becomes a deep copy, all AST nodes are copied, no previous pointers left +class GenericFunctionReplicator final : public ASTReplicatorFunction { + const GenericsDeclaration* genericTs; + const std::vector& substitutionTs; + +protected: + using ASTReplicatorFunction::clone; + + TypePtr clone(TypePtr t) override { + return replace_genericT_with_deduced(t, genericTs, substitutionTs); + } + +public: + GenericFunctionReplicator(const GenericsDeclaration* genericTs, const std::vector& substitutionTs) + : genericTs(genericTs) + , substitutionTs(substitutionTs) { + } + + V clone_function_body(V v_function) override { + return createV( + v_function->loc, + clone(v_function->get_identifier()), + clone(v_function->get_param_list()), + clone(v_function->get_body()), + clone(v_function->declared_return_type), + nullptr, // a newly-created function is not generic + v_function->method_id, + v_function->flags + ); + } +}; + +std::string GenericsDeclaration::as_human_readable() const { + std::string result = "<"; + for (const GenericsItem& item : itemsT) { + if (result.size() > 1) { + result += ","; + } + result += item.nameT; + } + result += ">"; + return result; +} + +int GenericsDeclaration::find_nameT(std::string_view nameT) const { + for (int i = 0; i < static_cast(itemsT.size()); ++i) { + if (itemsT[i].nameT == nameT) { + return i; + } + } + return -1; +} + +// after creating a deep copy of `f` like `f`, its new and fresh body needs the previous pipeline to run +// for example, all local vars need to be registered as symbols, etc. +static void run_pipeline_for_instantiated_function(FunctionPtr inst_fun_ref) { + // these pipes are exactly the same as in tolk.cpp — all preceding (and including) type inferring + pipeline_resolve_identifiers_and_assign_symbols(inst_fun_ref); + pipeline_calculate_rvalue_lvalue(inst_fun_ref); + pipeline_infer_types_and_calls_and_fields(inst_fun_ref); +} + +std::string generate_instantiated_name(const std::string& orig_name, const std::vector& substitutions) { + // an instantiated function name will be "{orig_name}<{T1,T2,...}>" + std::string name = orig_name; + name += "<"; + for (TypePtr subs : substitutions) { + if (name.size() > orig_name.size() + 1) { + name += ","; + } + name += subs->as_human_readable(); + } + name.erase(std::remove(name.begin(), name.end(), ' '), name.end()); + name += ">"; + return name; +} + +FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, const std::string& inst_name, std::vector&& substitutionTs) { + tolk_assert(fun_ref->genericTs); + + // if `f` was earlier instantiated, return it + if (const auto* existing = lookup_global_symbol(inst_name)) { + FunctionPtr inst_ref = existing->try_as(); + tolk_assert(inst_ref); + return inst_ref; + } + + std::vector parameters; + parameters.reserve(fun_ref->get_num_params()); + for (const LocalVarData& orig_p : fun_ref->parameters) { + parameters.emplace_back(orig_p.name, orig_p.loc, replace_genericT_with_deduced(orig_p.declared_type, fun_ref->genericTs, substitutionTs), orig_p.flags, orig_p.param_idx); + } + TypePtr declared_return_type = replace_genericT_with_deduced(fun_ref->declared_return_type, fun_ref->genericTs, substitutionTs); + const GenericsInstantiation* instantiationTs = new GenericsInstantiation(loc, std::move(substitutionTs)); + + if (fun_ref->is_asm_function()) { + FunctionData* inst_ref = new FunctionData(inst_name, fun_ref->loc, declared_return_type, std::move(parameters), fun_ref->flags, nullptr, instantiationTs, new FunctionBodyAsm, fun_ref->ast_root); + inst_ref->arg_order = fun_ref->arg_order; + inst_ref->ret_order = fun_ref->ret_order; + G.symtable.add_function(inst_ref); + G.all_functions.push_back(inst_ref); + run_pipeline_for_instantiated_function(inst_ref); + return inst_ref; + } + + if (fun_ref->is_builtin_function()) { + FunctionData* inst_ref = new FunctionData(inst_name, fun_ref->loc, declared_return_type, std::move(parameters), fun_ref->flags, nullptr, instantiationTs, fun_ref->body, fun_ref->ast_root); + inst_ref->arg_order = fun_ref->arg_order; + inst_ref->ret_order = fun_ref->ret_order; + G.symtable.add_function(inst_ref); + return inst_ref; + } + + GenericFunctionReplicator replicator(fun_ref->genericTs, instantiationTs->substitutions); + V inst_root = replicator.clone_function_body(fun_ref->ast_root->as()); + + FunctionData* inst_ref = new FunctionData(inst_name, fun_ref->loc, declared_return_type, std::move(parameters), fun_ref->flags, nullptr, instantiationTs, new FunctionBodyCode, inst_root); + inst_ref->arg_order = fun_ref->arg_order; + inst_ref->ret_order = fun_ref->ret_order; + inst_root->mutate()->assign_fun_ref(inst_ref); + G.symtable.add_function(inst_ref); + G.all_functions.push_back(inst_ref); + run_pipeline_for_instantiated_function(inst_ref); + return inst_ref; +} + +} // namespace tolk diff --git a/tolk/generics-helpers.h b/tolk/generics-helpers.h new file mode 100644 index 00000000..5ed245af --- /dev/null +++ b/tolk/generics-helpers.h @@ -0,0 +1,102 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "src-file.h" +#include "fwd-declarations.h" +#include "td/utils/Status.h" +#include + +namespace tolk { + +// when a function is declared `f`, this "" is represented as this class +// (not at AST, but at symbol storage level) +struct GenericsDeclaration { + struct GenericsItem { + std::string_view nameT; + + explicit GenericsItem(std::string_view nameT) + : nameT(nameT) {} + }; + + explicit GenericsDeclaration(std::vector&& itemsT) + : itemsT(std::move(itemsT)) {} + + const std::vector itemsT; + + std::string as_human_readable() const; + + size_t size() const { return itemsT.size(); } + bool has_nameT(std::string_view nameT) const { return find_nameT(nameT) != -1; } + int find_nameT(std::string_view nameT) const; + std::string get_nameT(int idx) const { return static_cast(itemsT[idx].nameT); } +}; + +// when a function call is `f()`, this "" is represented as this class +struct GenericsInstantiation { + const std::vector substitutions; // for genericTs + const SrcLocation loc; // first instantiation location + + explicit GenericsInstantiation(SrcLocation loc, std::vector&& substitutions) + : substitutions(std::move(substitutions)) + , loc(loc) { + } +}; + +// this class helps to deduce Ts on the fly +// purpose: having `f(value: T)` and call `f(5)`, deduce T = int +// while analyzing a call, arguments are handled one by one, by `auto_deduce_from_argument()` +// this class also handles manually specified substitutions like `f(5)` +class GenericSubstitutionsDeduceForCall { + FunctionPtr fun_ref; + std::vector substitutionTs; + bool manually_specified = false; + + void provide_deducedT(const std::string& nameT, TypePtr deduced); + void consider_next_condition(TypePtr param_type, TypePtr arg_type); + +public: + explicit GenericSubstitutionsDeduceForCall(FunctionPtr fun_ref); + + bool is_manually_specified() const { + return manually_specified; + } + + void provide_manually_specified(std::vector&& substitutionTs); + TypePtr replace_by_manually_specified(TypePtr param_type) const; + TypePtr auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type); + int get_first_not_deduced_idx() const; + + std::vector&& flush() { + return std::move(substitutionTs); + } +}; + +struct GenericDeduceError final : std::exception { + std::string message; + explicit GenericDeduceError(std::string message) + : message(std::move(message)) { } + + const char* what() const noexcept override { + return message.c_str(); + } +}; + +std::string generate_instantiated_name(const std::string& orig_name, const std::vector& substitutions); +FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, const std::string& inst_name, std::vector&& substitutionTs); + +} // namespace tolk diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp new file mode 100644 index 00000000..06913a5f --- /dev/null +++ b/tolk/lexer.cpp @@ -0,0 +1,616 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "lexer.h" +#include +#include +#include + +namespace tolk { + +// By 'chunk' in lexer I mean a token or a list of tokens parsed simultaneously. +// E.g., when we meet "str", ChunkString is called, it emits tok_string. +// E.g., when we meet "str"x, ChunkString emits not only tok_string, but tok_string_modifier. +// E.g., when we meet //, ChunkInlineComment is called, it emits nothing (just skips a line). +// We store all valid chunks lexers in a prefix tree (LexingTrie), see below. +struct ChunkLexerBase { + ChunkLexerBase(const ChunkLexerBase&) = delete; + ChunkLexerBase &operator=(const ChunkLexerBase&) = delete; + ChunkLexerBase() = default; + + virtual bool parse(Lexer* lex) const = 0; + virtual ~ChunkLexerBase() = default; +}; + +template +static T* singleton() { + static T obj; + return &obj; +} + +// LexingTrie is a prefix tree storing all available Tolk language constructs. +// It's effectively a map of a prefix to ChunkLexerBase. +class LexingTrie { + LexingTrie** next{nullptr}; // either nullptr or [256] + ChunkLexerBase* val{nullptr}; // non-null for leafs + + GNU_ATTRIBUTE_ALWAYS_INLINE void ensure_next_allocated() { + if (next == nullptr) { + next = new LexingTrie*[256]; + std::memset(next, 0, 256 * sizeof(LexingTrie*)); + } + } + + GNU_ATTRIBUTE_ALWAYS_INLINE void ensure_symbol_allocated(uint8_t symbol) const { + if (next[symbol] == nullptr) { + next[symbol] = new LexingTrie; + } + } + +public: + // Maps a prefix onto a chunk lexer. + // E.g. " -> ChunkString + // E.g. """ -> ChunkMultilineString + void add_prefix(const char* s, ChunkLexerBase* val) { + LexingTrie* cur = this; + + for (; *s; ++s) { + uint8_t symbol = static_cast(*s); + cur->ensure_next_allocated(); + cur->ensure_symbol_allocated(symbol); + cur = cur->next[symbol]; + } + +#ifdef TOLK_DEBUG + assert(!cur->val); +#endif + cur->val = val; + } + + // Maps a pattern onto a chunk lexer. + // E.g. -[0-9] -> ChunkNegativeNumber + // Internally, it expands the pattern to all possible prefixes: -0, -1, etc. + // (for example, [0-9][a-z_$] gives 10*28=280 prefixes) + void add_pattern(const char* pattern, ChunkLexerBase* val) { + std::vector all_possible_trie{this}; + + for (const char* c = pattern; *c; ++c) { + std::string to_append; + if (*c == '[') { + c++; + while (*c != ']') { // assume that input is corrent, no out-of-string checks + if (*(c + 1) == '-') { + char l = *c, r = *(c + 2); + for (char symbol = l; symbol <= r; ++symbol) { + to_append += symbol; + } + c += 3; + } else { + to_append += *c; + c++; + } + } + } else { + to_append += *c; + } + + std::vector next_all_possible_trie; + next_all_possible_trie.reserve(all_possible_trie.size() * to_append.size()); + for (LexingTrie* cur : all_possible_trie) { + cur->ensure_next_allocated(); + for (uint8_t symbol : to_append) { + cur->ensure_symbol_allocated(symbol); + next_all_possible_trie.emplace_back(cur->next[symbol]); + } + } + all_possible_trie = std::move(next_all_possible_trie); + } + + for (LexingTrie* trie : all_possible_trie) { + trie->val = val; + } + } + + // Looks up a chunk lexer given a string (in practice, s points to cur position in the middle of the file). + // It returns the deepest case: pointing to ", it will return ChunkMultilineString if """, or ChunkString otherwize. + ChunkLexerBase* get_deepest(const char* s) const { + const LexingTrie* best = this; + + for (const LexingTrie* cur = this; cur && cur->next; ++s) { + cur = cur->next[static_cast(*s)]; // if s reaches \0, cur will just become nullptr, and loop will end + if (cur && cur->val) { + best = cur; + } + } + + return best->val; + } +}; + +// +// ---------------------------------------------------------------------- +// A list of valid parsed chunks. +// + +// An inline comment, starting from '//' +struct ChunkInlineComment final : ChunkLexerBase { + bool parse(Lexer* lex) const override { + lex->skip_line(); + return true; + } +}; + +// A multiline comment, starting from '/*' +// Note, that nested comments are not supported. +struct ChunkMultilineComment final : ChunkLexerBase { + bool parse(Lexer* lex) const override { + while (!lex->is_eof()) { + if (lex->char_at() == '*' && lex->char_at(1) == '/') { + lex->skip_chars(2); + return true; + } + lex->skip_chars(1); + } + return true; // it's okay if comment extends past end of file + } +}; + +// A string, starting from " +// Note, that there are no escape symbols inside: the purpose of strings in Tolk just doesn't need it. +// After a closing quote, a string modifier may be present, like "Ef8zMzMzMzMzMzMzMzMzMzM0vF"a. +// If present, it emits a separate tok_string_modifier. +struct ChunkString final : ChunkLexerBase { + bool parse(Lexer* lex) const override { + const char* str_begin = lex->c_str(); + lex->skip_chars(1); + while (!lex->is_eof() && lex->char_at() != '"' && lex->char_at() != '\n') { + lex->skip_chars(1); + } + if (lex->char_at() != '"') { + lex->error("string extends past end of line"); + } + + std::string_view str_val(str_begin + 1, lex->c_str() - str_begin - 1); + lex->skip_chars(1); + lex->add_token(tok_string_const, str_val); + + if (std::isalpha(lex->char_at())) { + std::string_view modifier_val(lex->c_str(), 1); + lex->skip_chars(1); + lex->add_token(tok_string_modifier, modifier_val); + } + + return true; + } +}; + +// A string starting from """ +// Used for multiline asm constructions. Can not have a postfix modifier. +struct ChunkMultilineString final : ChunkLexerBase { + bool parse(Lexer* lex) const override { + const char* str_begin = lex->c_str(); + lex->skip_chars(3); + while (!lex->is_eof()) { + if (lex->char_at() == '"' && lex->char_at(1) == '"' && lex->char_at(2) == '"') { + break; + } + lex->skip_chars(1); + } + if (lex->is_eof()) { + lex->error("string extends past end of file"); + } + + std::string_view str_val(str_begin + 3, lex->c_str() - str_begin - 3); + lex->skip_chars(3); + lex->add_token(tok_string_const, str_val); + return true; + } +}; + +// An annotation for a function (in the future, for vars also): +// @inline and others +struct ChunkAnnotation final : ChunkLexerBase { + bool parse(Lexer* lex) const override { + const char* str_begin = lex->c_str(); + lex->skip_chars(1); + while (std::isalnum(lex->char_at()) || lex->char_at() == '_') { + lex->skip_chars(1); + } + + std::string_view str_val(str_begin, lex->c_str() - str_begin); + lex->add_token(tok_annotation_at, str_val); + return true; + } +}; + +// A number, may be a hex one. +struct ChunkNumber final : ChunkLexerBase { + bool parse(Lexer* lex) const override { + const char* str_begin = lex->c_str(); + bool hex = false; + if (lex->char_at() == '0' && lex->char_at(1) == 'x') { + lex->skip_chars(2); + hex = true; + } + if (lex->is_eof()) { + return false; + } + while (!lex->is_eof()) { + char c = lex->char_at(); + if (c >= '0' && c <= '9') { + lex->skip_chars(1); + continue; + } + if (!hex) { + break; + } + c |= 0x20; + if (c < 'a' || c > 'f') { + break; + } + lex->skip_chars(1); + } + + std::string_view str_val(str_begin, lex->c_str() - str_begin); + lex->add_token(tok_int_const, str_val); + return true; + } +}; + +// Tokens like !=, &, etc. emit just a simple TokenType. +// Since they are stored in trie, "parsing" them is just skipping len chars. +struct ChunkSimpleToken final : ChunkLexerBase { + TokenType tp; + int len; + + ChunkSimpleToken(TokenType tp, int len) : tp(tp), len(len) {} + + bool parse(Lexer* lex) const override { + std::string_view str_val(lex->c_str(), len); + lex->add_token(tp, str_val); + lex->skip_chars(len); + return true; + } +}; + +// Spaces and other space-like symbols are just skipped. +struct ChunkSkipWhitespace final : ChunkLexerBase { + bool parse(Lexer* lex) const override { + lex->skip_chars(1); + lex->skip_spaces(); + return true; + } +}; + +// Here we handle corner cases of grammar that are requested on demand. +// E.g., for 'tolk >0.5.0', '0.5.0' should be parsed specially to emit tok_semver. +// See TolkLanguageGrammar::parse_next_chunk_special(). +struct ChunkSpecialParsing { + static bool parse_semver(Lexer* lex) { + const char* str_begin = lex->c_str(); + while (std::isdigit(lex->char_at()) || lex->char_at() == '.') { + lex->skip_chars(1); + } + + std::string_view str_val(str_begin, lex->c_str() - str_begin); + if (str_val.empty()) { + return false; + } + lex->add_token(tok_semver, str_val); + return true; + } +}; + +// Anything starting from a valid identifier beginning symbol is parsed as an identifier. +// But if a resulting string is a keyword, a corresponding token is emitted instead of tok_identifier. +struct ChunkIdentifierOrKeyword final : ChunkLexerBase { + // having parsed str up to the valid end, look up whether it's a valid keyword + // in the future, this could be a bit more effective than just comparing strings (e.g. gperf), + // but nevertheless, performance of the naive code below is reasonably good + static TokenType maybe_keyword(std::string_view str) { + switch (str.size()) { + case 1: + if (str == "_") return tok_underscore; + break; + case 2: + if (str == "do") return tok_do; + if (str == "if") return tok_if; + if (str == "as") return tok_as; + break; + case 3: + if (str == "var") return tok_var; + if (str == "fun") return tok_fun; + if (str == "asm") return tok_asm; + if (str == "get") return tok_get; + if (str == "try") return tok_try; + if (str == "val") return tok_val; + break; + case 4: + if (str == "else") return tok_else; + if (str == "true") return tok_true; + if (str == "null") return tok_null; + if (str == "self") return tok_self; + if (str == "tolk") return tok_tolk; + if (str == "type") return tok_type; + if (str == "enum") return tok_enum; + break; + case 5: + if (str == "const") return tok_const; + if (str == "false") return tok_false; + if (str == "redef") return tok_redef; + if (str == "while") return tok_while; + if (str == "break") return tok_break; + if (str == "throw") return tok_throw; + if (str == "catch") return tok_catch; + if (str == "infix") return tok_infix; + break; + case 6: + if (str == "return") return tok_return; + if (str == "assert") return tok_assert; + if (str == "import") return tok_import; + if (str == "global") return tok_global; + if (str == "mutate") return tok_mutate; + if (str == "repeat") return tok_repeat; + if (str == "struct") return tok_struct; + if (str == "export") return tok_export; + break; + case 7: + if (str == "builtin") return tok_builtin; + break; + case 8: + if (str == "continue") return tok_continue; + if (str == "operator") return tok_operator; + break; + default: + break; + } + return tok_empty; + } + + bool parse(Lexer* lex) const override { + const char* sym_begin = lex->c_str(); + lex->skip_chars(1); + while (!lex->is_eof()) { + char c = lex->char_at(); + bool allowed_in_identifier = std::isalnum(c) || c == '_' || c == '$'; + if (!allowed_in_identifier) { + break; + } + lex->skip_chars(1); + } + + std::string_view str_val(sym_begin, lex->c_str() - sym_begin); + if (TokenType kw_tok = maybe_keyword(str_val)) { + lex->add_token(kw_tok, str_val); + } else { + lex->add_token(tok_identifier, str_val); + } + return true; + } +}; + +// Like in Kotlin, `backticks` can be used to wrap identifiers (both in declarations/usage, both for vars/functions). +// E.g.: function `do`() { var `with spaces` = 1; } +// This could be useful to use reserved names as identifiers (in a probable codegen from TL, for example). +struct ChunkIdentifierInBackticks final : ChunkLexerBase { + bool parse(Lexer* lex) const override { + const char* str_begin = lex->c_str(); + lex->skip_chars(1); + while (!lex->is_eof() && lex->char_at() != '`' && lex->char_at() != '\n') { + if (std::isspace(lex->char_at())) { + lex->error("an identifier can't have a space in its name (even inside backticks)"); + } + lex->skip_chars(1); + } + if (lex->char_at() != '`') { + lex->error("unclosed backtick `"); + } + + std::string_view str_val(str_begin + 1, lex->c_str() - str_begin - 1); + lex->skip_chars(1); + lex->add_token(tok_identifier, str_val); + return true; + } +}; + +// +// ---------------------------------------------------------------------- +// Here we define a grammar of Tolk. +// All valid chunks prefixes are stored in trie. +// + +struct TolkLanguageGrammar { + static LexingTrie trie; + + static bool parse_next_chunk(Lexer* lex) { + const ChunkLexerBase* best = trie.get_deepest(lex->c_str()); + return best && best->parse(lex); + } + + static bool parse_next_chunk_special(Lexer* lex, TokenType parse_next_as) { + switch (parse_next_as) { + case tok_semver: + return ChunkSpecialParsing::parse_semver(lex); + default: + assert(false); + return false; + } + } + + static void register_token(const char* str, int len, TokenType tp) { + trie.add_prefix(str, new ChunkSimpleToken(tp, len)); + } + + static void init() { + trie.add_prefix("//", singleton()); + trie.add_prefix("/*", singleton()); + trie.add_prefix(R"(")", singleton()); + trie.add_prefix(R"(""")", singleton()); + trie.add_prefix("@", singleton()); + trie.add_prefix(" ", singleton()); + trie.add_prefix("\t", singleton()); + trie.add_prefix("\r", singleton()); + trie.add_prefix("\n", singleton()); + + trie.add_pattern("[0-9]", singleton()); + trie.add_pattern("[a-zA-Z_$]", singleton()); + trie.add_prefix("`", singleton()); + + register_token("+", 1, tok_plus); + register_token("-", 1, tok_minus); + register_token("*", 1, tok_mul); + register_token("/", 1, tok_div); + register_token("%", 1, tok_mod); + register_token("?", 1, tok_question); + register_token(":", 1, tok_colon); + register_token(",", 1, tok_comma); + register_token(";", 1, tok_semicolon); + register_token("(", 1, tok_oppar); + register_token(")", 1, tok_clpar); + register_token("[", 1, tok_opbracket); + register_token("]", 1, tok_clbracket); + register_token("{", 1, tok_opbrace); + register_token("}", 1, tok_clbrace); + register_token("=", 1, tok_assign); + register_token("<", 1, tok_lt); + register_token(">", 1, tok_gt); + register_token("!", 1, tok_logical_not); + register_token("&", 1, tok_bitwise_and); + register_token("|", 1, tok_bitwise_or); + register_token("^", 1, tok_bitwise_xor); + register_token("~", 1, tok_bitwise_not); + register_token(".", 1, tok_dot); + register_token("==", 2, tok_eq); + register_token("!=", 2, tok_neq); + register_token("<=", 2, tok_leq); + register_token(">=", 2, tok_geq); + register_token("<<", 2, tok_lshift); + register_token(">>", 2, tok_rshift); + register_token("&&", 2, tok_logical_and); + register_token("||", 2, tok_logical_or); + register_token("~/", 2, tok_divR); + register_token("^/", 2, tok_divC); + register_token("+=", 2, tok_set_plus); + register_token("-=", 2, tok_set_minus); + register_token("*=", 2, tok_set_mul); + register_token("/=", 2, tok_set_div); + register_token("%=", 2, tok_set_mod); + register_token("&=", 2, tok_set_bitwise_and); + register_token("|=", 2, tok_set_bitwise_or); + register_token("^=", 2, tok_set_bitwise_xor); + register_token("->", 2, tok_arrow); + register_token("<=>", 3, tok_spaceship); + register_token("~>>", 3, tok_rshiftR); + register_token("^>>", 3, tok_rshiftC); + register_token("<<=", 3, tok_set_lshift); + register_token(">>=", 3, tok_set_rshift); + } +}; + +LexingTrie TolkLanguageGrammar::trie; + +// +// ---------------------------------------------------------------------- +// The Lexer class is to be used outside (by parser, which constructs AST from tokens). +// It's streaming. It means, that `next()` parses a next token on demand +// (instead of parsing all file contents to vector and iterating over it). +// Parsing on demand uses effectively less memory. +// Note, that chunks, being parsed, call `add_token()`, and a chunk may add multiple tokens at once. +// That's why a small cirlular buffer for tokens is used. +// `last_token_idx` actually means a number of total tokens added. +// `cur_token_idx` is a number of returned by `next()`. +// It's assumed that an input file has already been loaded, its contents is present and won't be deleted +// (`start`, `cur` and `end`, as well as every Token str_val, points inside file->text). +// + +Lexer::Lexer(const SrcFile* file) + : file(file) + , p_start(file->text.data()) + , p_end(p_start + file->text.size()) + , p_next(p_start) + , location(file) { + next(); +} + +Lexer::Lexer(std::string_view text) + : file(nullptr) + , p_start(text.data()) + , p_end(p_start + text.size()) + , p_next(p_start) + , location() { + next(); +} + +void Lexer::next() { + while (cur_token_idx == last_token_idx && !is_eof()) { + update_location(); + if (!TolkLanguageGrammar::parse_next_chunk(this)) { + error("failed to parse"); + } + } + if (is_eof()) { + add_token(tok_eof, ""); + } + cur_token = tokens_circularbuf[++cur_token_idx & 7]; +} + +void Lexer::next_special(TokenType parse_next_as, const char* str_expected) { + assert(cur_token_idx == last_token_idx); + skip_spaces(); + update_location(); + if (!TolkLanguageGrammar::parse_next_chunk_special(this, parse_next_as)) { + error(std::string(str_expected) + " expected"); + } + cur_token = tokens_circularbuf[++cur_token_idx & 7]; +} + +Lexer::SavedPositionForLookahead Lexer::save_parsing_position() const { + return {p_next, cur_token_idx, cur_token}; +} + +void Lexer::restore_position(SavedPositionForLookahead saved) { + p_next = saved.p_next; + cur_token_idx = last_token_idx = saved.cur_token_idx; + cur_token = saved.cur_token; +} + +void Lexer::error(const std::string& err_msg) const { + throw ParseError(cur_location(), err_msg); +} + +void Lexer::unexpected(const char* str_expected) const { + throw ParseError(cur_location(), "expected " + std::string(str_expected) + ", got `" + std::string(cur_str()) + "`"); +} + +void lexer_init() { + TolkLanguageGrammar::init(); +} + +// todo #ifdef TOLK_PROFILING +// As told above, `next()` produces tokens on demand, while AST is being generated. +// Hence, it's difficult to measure Lexer performance separately. +// This function can be called just to tick Lexer performance, it just scans all input files. +// There is no sense to use it in production, but when refactoring and optimizing Lexer, it's useful. +void lexer_measure_performance(const AllRegisteredSrcFiles& files_to_just_parse) { + for (const SrcFile* file : files_to_just_parse) { + Lexer lex(file); + while (!lex.is_eof()) { + lex.next(); + } + } +} + +} // namespace tolk diff --git a/tolk/lexer.h b/tolk/lexer.h new file mode 100644 index 00000000..58bc3640 --- /dev/null +++ b/tolk/lexer.h @@ -0,0 +1,237 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "platform-utils.h" +#include "src-file.h" +#include + +namespace tolk { + +enum TokenType { + tok_empty, + + tok_fun, + tok_get, + tok_type, + tok_enum, + tok_struct, + tok_operator, + tok_infix, + + tok_global, + tok_const, + tok_var, + tok_val, + tok_redef, + tok_mutate, + tok_self, + + tok_annotation_at, + tok_colon, + tok_asm, + tok_builtin, + + tok_int_const, + tok_string_const, + tok_string_modifier, + tok_true, + tok_false, + tok_null, + + tok_identifier, + tok_dot, + + tok_plus, + tok_set_plus, + tok_minus, + tok_set_minus, + tok_mul, + tok_set_mul, + tok_div, + tok_set_div, + tok_mod, + tok_set_mod, + tok_lshift, + tok_set_lshift, + tok_rshift, + tok_set_rshift, + tok_rshiftR, + tok_rshiftC, + tok_bitwise_and, + tok_set_bitwise_and, + tok_bitwise_or, + tok_set_bitwise_or, + tok_bitwise_xor, + tok_set_bitwise_xor, + tok_bitwise_not, + + tok_question, + tok_comma, + tok_semicolon, + tok_oppar, + tok_clpar, + tok_opbracket, + tok_clbracket, + tok_opbrace, + tok_clbrace, + tok_assign, + tok_underscore, + tok_lt, + tok_gt, + tok_logical_not, + tok_logical_and, + tok_logical_or, + + tok_eq, + tok_neq, + tok_leq, + tok_geq, + tok_spaceship, + tok_divR, + tok_divC, + + tok_return, + tok_repeat, + tok_do, + tok_while, + tok_break, + tok_continue, + tok_try, + tok_catch, + tok_throw, + tok_assert, + tok_if, + tok_else, + + tok_arrow, + tok_as, + + tok_tolk, + tok_semver, + tok_import, + tok_export, + + tok_eof +}; + +// All tolk language is parsed into tokens. +// Lexer::next() returns a Token. +struct Token { + TokenType type = tok_empty; + std::string_view str_val; + + Token() = default; + Token(TokenType type, std::string_view str_val): type(type), str_val(str_val) {} +}; + +// Lexer::next() is a method to be used externally (while parsing tolk file to AST). +// It's streaming: `next()` parses a token on demand. +// For comments, see lexer.cpp, a comment above Lexer constructor. +class Lexer { + Token tokens_circularbuf[8]{}; + int last_token_idx = -1; + int cur_token_idx = -1; + Token cur_token; // = tokens_circularbuf[cur_token_idx & 7] + + const SrcFile* file; + const char *p_start, *p_end, *p_next; + SrcLocation location; + + void update_location() { + location.char_offset = static_cast(p_next - p_start); + } + +public: + + struct SavedPositionForLookahead { + const char* p_next = nullptr; + int cur_token_idx = 0; + Token cur_token; + }; + + explicit Lexer(const SrcFile* file); + explicit Lexer(std::string_view text); + Lexer(const Lexer&) = delete; + Lexer &operator=(const Lexer&) = delete; + + void add_token(TokenType type, std::string_view str) { + tokens_circularbuf[++last_token_idx & 7] = Token(type, str); + } + + void skip_spaces() { + while (std::isspace(*p_next)) { + ++p_next; + } + } + + void skip_line() { + while (p_next < p_end && *p_next != '\n' && *p_next != '\r') { + ++p_next; + } + while (*p_next == '\n' || *p_next == '\r') { + ++p_next; + } + } + + void skip_chars(int n) { + p_next += n; + } + + bool is_eof() const { + return p_next >= p_end; + } + + char char_at() const { return *p_next; } + char char_at(int shift) const { return *(p_next + shift); } + const char* c_str() const { return p_next; } + + TokenType tok() const { return cur_token.type; } + std::string_view cur_str() const { return cur_token.str_val; } + SrcLocation cur_location() const { return location; } + const SrcFile* cur_file() const { return file; } + + void next(); + void next_special(TokenType parse_next_as, const char* str_expected); + + SavedPositionForLookahead save_parsing_position() const; + void restore_position(SavedPositionForLookahead saved); + + void check(TokenType next_tok, const char* str_expected) const { + if (cur_token.type != next_tok) { + unexpected(str_expected); // unlikely path, not inlined + } + } + void expect(TokenType next_tok, const char* str_expected) { + if (cur_token.type != next_tok) { + unexpected(str_expected); + } + next(); + } + + GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD + void unexpected(const char* str_expected) const; + GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD + void error(const std::string& err_msg) const; +}; + +void lexer_init(); + +// todo #ifdef TOLK_PROFILING +void lexer_measure_performance(const AllRegisteredSrcFiles& files_to_just_parse); + +} // namespace tolk diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp new file mode 100644 index 00000000..76d75638 --- /dev/null +++ b/tolk/optimize.cpp @@ -0,0 +1,652 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" + +namespace tolk { + +/* + * + * PEEPHOLE OPTIMIZER + * + */ + +void Optimizer::set_code(AsmOpConsList code) { + code_ = std::move(code); + unpack(); +} + +void Optimizer::unpack() { + int i = 0, j = 0; + for (AsmOpCons *p = code_.get(); p && i < optimize_depth; p = p->cdr.get(), ++j) { + if (p->car->is_very_custom()) { + break; + } + if (p->car->is_comment()) { + continue; + } + op_cons_[i] = p; + op_[i] = std::move(p->car); + offs_[i] = j; + ++i; + } + l_ = i; + indent_ = (i ? op_[0]->indent : 0); +} + +void Optimizer::pack() { + for (int i = 0; i < l_; i++) { + op_cons_[i]->car = std::move(op_[i]); + op_cons_[i] = nullptr; + } + l_ = 0; +} + +void Optimizer::apply() { + if (!p_ && !q_) { + return; + } + tolk_assert(p_ > 0 && p_ <= l_ && q_ >= 0 && q_ <= optimize_depth && l_ <= optimize_depth); + for (int i = p_; i < l_; i++) { + tolk_assert(op_[i]); + op_cons_[i]->car = std::move(op_[i]); + op_cons_[i] = nullptr; + } + for (int c = offs_[p_ - 1]; c >= 0; --c) { + code_ = std::move(code_->cdr); + } + for (int j = q_ - 1; j >= 0; j--) { + tolk_assert(oq_[j]); + oq_[j]->indent = indent_; + code_ = AsmOpCons::cons(std::move(oq_[j]), std::move(code_)); + } + l_ = 0; +} + +AsmOpConsList Optimizer::extract_code() { + pack(); + return std::move(code_); +} + +void Optimizer::show_head() const { + if (!debug_) { + return; + } + std::cerr << "optimizing"; + for (int i = 0; i < l_; i++) { + if (op_[i]) { + std::cerr << ' ' << *op_[i] << ' '; + } else { + std::cerr << " (null) "; + } + } + std::cerr << std::endl; +} + +void Optimizer::show_left() const { + if (!debug_) { + return; + } + std::cerr << "// *** rewriting"; + for (int i = 0; i < p_; i++) { + if (op_[i]) { + std::cerr << ' ' << *op_[i] << ' '; + } else { + std::cerr << " (null) "; + } + } +} + +void Optimizer::show_right() const { + if (!debug_) { + return; + } + std::cerr << "->"; + for (int i = 0; i < q_; i++) { + if (oq_[i]) { + std::cerr << ' ' << *oq_[i] << ' '; + } else { + std::cerr << " (null) "; + } + } + std::cerr << std::endl; +} + +bool Optimizer::say(std::string str) const { + if (debug_) { + std::cerr << str << std::endl; + } + return true; +} + +bool Optimizer::find_const_op(int* op_idx, int cst) { + for (int i = 0; i < l2_; i++) { + if (op_[i]->is_gconst() && tr_[i].get(0) == cst) { + *op_idx = i; + return true; + } + } + return false; +} + +bool Optimizer::is_push_const(int* i, int* c) const { + return pb_ >= 3 && pb_ <= l2_ && tr_[pb_ - 1].is_push_const(i, c); +} + +// PUSHCONST c ; PUSH s(i+1) ; SWAP -> PUSH s(i) ; PUSHCONST c +bool Optimizer::rewrite_push_const(int i, int c) { + p_ = pb_; + q_ = 2; + int idx = -1; + if (!(p_ >= 2 && find_const_op(&idx, c) && idx < p_)) { + return false; + } + show_left(); + oq_[1] = std::move(op_[idx]); + oq_[0] = std::move(op_[!idx]); + *oq_[0] = AsmOp::Push(i); + show_right(); + return true; +} + +bool Optimizer::is_const_rot(int* c) const { + return pb_ >= 3 && pb_ <= l2_ && tr_[pb_ - 1].is_const_rot(c); +} + +bool Optimizer::rewrite_const_rot(int c) { + p_ = pb_; + q_ = 2; + int idx = -1; + if (!(p_ >= 2 && find_const_op(&idx, c) && idx < p_)) { + return false; + } + show_left(); + oq_[0] = std::move(op_[idx]); + oq_[1] = std::move(op_[!idx]); + *oq_[1] = AsmOp::Custom("ROT", 3, 3); + show_right(); + return true; +} + +bool Optimizer::is_const_pop(int* c, int* i) const { + return pb_ >= 3 && pb_ <= l2_ && tr_[pb_ - 1].is_const_pop(c, i); +} + +bool Optimizer::rewrite_const_pop(int c, int i) { + p_ = pb_; + q_ = 2; + int idx = -1; + if (!(p_ >= 2 && find_const_op(&idx, c) && idx < p_)) { + return false; + } + show_left(); + oq_[0] = std::move(op_[idx]); + oq_[1] = std::move(op_[!idx]); + *oq_[1] = AsmOp::Pop(i); + show_right(); + return true; +} + +bool Optimizer::is_const_push_xchgs() { + if (!(pb_ >= 2 && pb_ <= l2_ && op_[0]->is_gconst())) { + return false; + } + StackTransform t; + int pos = 0, i; + for (i = 1; i < pb_; i++) { + int a, b; + if (op_[i]->is_xchg(&a, &b)) { + if (pos == a) { + pos = b; + } else if (pos == b) { + pos = a; + } else { + t.apply_xchg(a - (a > pos), b - (b > pos)); + } + } else if (op_[i]->is_push(&a)) { + if (pos == a) { + return false; + } + t.apply_push(a - (a > pos)); + ++pos; + } else { + return false; + } + } + if (pos) { + return false; + } + t.apply_push_newconst(); + if (t <= tr_[i - 1]) { + p_ = i; + return true; + } else { + return false; + } +} + +bool Optimizer::rewrite_const_push_xchgs() { + if (!p_) { + return false; + } + show_left(); + auto c_op = std::move(op_[0]); + tolk_assert(c_op->is_gconst()); + StackTransform t; + q_ = 0; + int pos = 0; + for (int i = 1; i < p_; i++) { + int a, b; + if (op_[i]->is_xchg(&a, &b)) { + if (a == pos) { + pos = b; + } else if (b == pos) { + pos = a; + } else { + oq_[q_] = std::move(op_[i]); + if (a > pos) { + oq_[q_]->a = a - 1; + } + if (b > pos) { + oq_[q_]->b = b - 1; + } + tolk_assert(apply_op(t, *oq_[q_])); + ++q_; + } + } else { + tolk_assert(op_[i]->is_push(&a)); + tolk_assert(a != pos); + oq_[q_] = std::move(op_[i]); + if (a > pos) { + oq_[q_]->a = a - 1; + } + tolk_assert(apply_op(t, *oq_[q_])); + ++q_; + ++pos; + } + } + tolk_assert(!pos); + t.apply_push_newconst(); + tolk_assert(t <= tr_[p_ - 1]); + oq_[q_++] = std::move(c_op); + show_right(); + return true; +} + +bool Optimizer::rewrite(int p, AsmOp&& new_op) { + tolk_assert(p > 0 && p <= l_); + p_ = p; + q_ = 1; + show_left(); + oq_[0] = std::move(op_[0]); + *oq_[0] = new_op; + show_right(); + return true; +} + +bool Optimizer::rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2) { + tolk_assert(p > 1 && p <= l_); + p_ = p; + q_ = 2; + show_left(); + oq_[0] = std::move(op_[0]); + *oq_[0] = new_op1; + oq_[1] = std::move(op_[1]); + *oq_[1] = new_op2; + show_right(); + return true; +} + +bool Optimizer::rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2, AsmOp&& new_op3) { + tolk_assert(p > 2 && p <= l_); + p_ = p; + q_ = 3; + show_left(); + oq_[0] = std::move(op_[0]); + *oq_[0] = new_op1; + oq_[1] = std::move(op_[1]); + *oq_[1] = new_op2; + oq_[2] = std::move(op_[2]); + *oq_[2] = new_op3; + show_right(); + return true; +} + +bool Optimizer::rewrite_nop() { + tolk_assert(p_ > 0 && p_ <= l_); + q_ = 0; + show_left(); + show_right(); + return true; +} + +bool Optimizer::is_pred(const std::function& pred, int min_p) { + min_p = std::max(min_p, pb_); + for (int p = l2_; p >= min_p; p--) { + if (pred(tr_[p - 1])) { + p_ = p; + return true; + } + } + return false; +} + +bool Optimizer::is_same_as(const StackTransform& trans, int min_p) { + return is_pred([&trans](const auto& t) { return t >= trans; }, min_p); +} + +// s1 s3 XCHG ; s0 s2 XCHG -> 2SWAP +bool Optimizer::is_2swap() { + static const StackTransform t_2swap{2, 3, 0, 1, 4}; + return is_same_as(t_2swap); +} + +// s3 PUSH ; s3 PUSH -> 2OVER +bool Optimizer::is_2over() { + static const StackTransform t_2over{2, 3, 0}; + return is_same_as(t_2over); +} + +bool Optimizer::is_2dup() { + static const StackTransform t_2dup{0, 1, 0}; + return is_same_as(t_2dup); +} + +bool Optimizer::is_tuck() { + static const StackTransform t_tuck{0, 1, 0, 2}; + return is_same_as(t_tuck); +} + +bool Optimizer::is_2drop() { + static const StackTransform t_2drop{2}; + return is_same_as(t_2drop); +} + +bool Optimizer::is_rot() { + return is_pred([](const auto& t) { return t.is_rot(); }); +} + +bool Optimizer::is_rotrev() { + return is_pred([](const auto& t) { return t.is_rotrev(); }); +} + +bool Optimizer::is_nop() { + return is_pred([](const auto& t) { return t.is_id(); }, 1); +} + +bool Optimizer::is_xchg(int* i, int* j) { + return is_pred([i, j](const auto& t) { return t.is_xchg(i, j) && ((*i < 16 && *j < 16) || (!*i && *j < 256)); }); +} + +bool Optimizer::is_xchg_xchg(int* i, int* j, int* k, int* l) { + return is_pred([i, j, k, l](const auto& t) { + return t.is_xchg_xchg(i, j, k, l) && (*i < 2 && *j < (*i ? 16 : 256) && *k < 2 && *l < (*k ? 16 : 256)); + }) && + (!(p_ == 2 && op_[0]->is_xchg(*i, *j) && op_[1]->is_xchg(*k, *l))); +} + +bool Optimizer::is_push(int* i) { + return is_pred([i](const auto& t) { return t.is_push(i) && *i < 256; }); +} + +bool Optimizer::is_pop(int* i) { + return is_pred([i](const auto& t) { return t.is_pop(i) && *i < 256; }); +} + +bool Optimizer::is_pop_pop(int* i, int* j) { + return is_pred([i, j](const auto& t) { return t.is_pop_pop(i, j) && *i < 256 && *j < 256; }, 3); +} + +bool Optimizer::is_push_rot(int* i) { + return is_pred([i](const auto& t) { return t.is_push_rot(i) && *i < 16; }, 3); +} + +bool Optimizer::is_push_rotrev(int* i) { + return is_pred([i](const auto& t) { return t.is_push_rotrev(i) && *i < 16; }, 3); +} + +bool Optimizer::is_push_xchg(int* i, int* j, int* k) { + return is_pred([i, j, k](const auto& t) { return t.is_push_xchg(i, j, k) && *i < 16 && *j < 16 && *k < 16; }) && + !(p_ == 2 && op_[0]->is_push() && op_[1]->is_xchg()); +} + +bool Optimizer::is_xchg2(int* i, int* j) { + return is_pred([i, j](const auto& t) { return t.is_xchg2(i, j) && *i < 16 && *j < 16; }); +} + +bool Optimizer::is_xcpu(int* i, int* j) { + return is_pred([i, j](const auto& t) { return t.is_xcpu(i, j) && *i < 16 && *j < 16; }); +} + +bool Optimizer::is_puxc(int* i, int* j) { + return is_pred([i, j](const auto& t) { return t.is_puxc(i, j) && *i < 16 && *j < 15; }); +} + +bool Optimizer::is_push2(int* i, int* j) { + return is_pred([i, j](const auto& t) { return t.is_push2(i, j) && *i < 16 && *j < 16; }); +} + +bool Optimizer::is_xchg3(int* i, int* j, int* k) { + return is_pred([i, j, k](const auto& t) { return t.is_xchg3(i, j, k) && *i < 16 && *j < 16 && *k < 16; }); +} + +bool Optimizer::is_xc2pu(int* i, int* j, int* k) { + return is_pred([i, j, k](const auto& t) { return t.is_xc2pu(i, j, k) && *i < 16 && *j < 16 && *k < 16; }); +} + +bool Optimizer::is_xcpuxc(int* i, int* j, int* k) { + return is_pred([i, j, k](const auto& t) { return t.is_xcpuxc(i, j, k) && *i < 16 && *j < 16 && *k < 15; }); +} + +bool Optimizer::is_xcpu2(int* i, int* j, int* k) { + return is_pred([i, j, k](const auto& t) { return t.is_xcpu2(i, j, k) && *i < 16 && *j < 16 && *k < 16; }); +} + +bool Optimizer::is_puxc2(int* i, int* j, int* k) { + return is_pred( + [i, j, k](const auto& t) { return t.is_puxc2(i, j, k) && *i < 16 && *j < 15 && *k < 15 && *j + *k != -1; }); +} + +bool Optimizer::is_puxcpu(int* i, int* j, int* k) { + return is_pred([i, j, k](const auto& t) { return t.is_puxcpu(i, j, k) && *i < 16 && *j < 15 && *k < 15; }); +} + +bool Optimizer::is_pu2xc(int* i, int* j, int* k) { + return is_pred([i, j, k](const auto& t) { return t.is_pu2xc(i, j, k) && *i < 16 && *j < 15 && *k < 14; }); +} + +bool Optimizer::is_push3(int* i, int* j, int* k) { + return is_pred([i, j, k](const auto& t) { return t.is_push3(i, j, k) && *i < 16 && *j < 16 && *k < 16; }); +} + +bool Optimizer::is_blkswap(int* i, int* j) { + return is_pred([i, j](const auto& t) { return t.is_blkswap(i, j) && *i > 0 && *j > 0 && *i <= 16 && *j <= 16; }); +} + +bool Optimizer::is_blkpush(int* i, int* j) { + return is_pred([i, j](const auto& t) { return t.is_blkpush(i, j) && *i > 0 && *i < 16 && *j < 16; }); +} + +bool Optimizer::is_blkdrop(int* i) { + return is_pred([i](const auto& t) { return t.is_blkdrop(i) && *i > 0 && *i < 16; }); +} + +bool Optimizer::is_blkdrop2(int* i, int* j) { + return is_pred([i, j](const auto& t) { return t.is_blkdrop2(i, j) && *i > 0 && *i < 16 && *j > 0 && *j < 16; }); +} + +bool Optimizer::is_reverse(int* i, int* j) { + return is_pred([i, j](const auto& t) { return t.is_reverse(i, j) && *i >= 2 && *i <= 17 && *j < 16; }); +} + +bool Optimizer::is_nip_seq(int* i, int* j) { + return is_pred([i, j](const auto& t) { return t.is_nip_seq(i, j) && *i >= 3 && *i <= 15; }); +} + +bool Optimizer::is_pop_blkdrop(int* i, int* k) { + return is_pred([i, k](const auto& t) { return t.is_pop_blkdrop(i, k) && *i >= *k && *k >= 2 && *k <= 15; }, 3); +} + +bool Optimizer::is_2pop_blkdrop(int* i, int* j, int* k) { + return is_pred( + [i, j, k](const auto& t) { return t.is_2pop_blkdrop(i, j, k) && *i >= *k && *j >= *k && *k >= 2 && *k <= 15; }, + 3); +} + +bool Optimizer::compute_stack_transforms() { + StackTransform trans; + for (int i = 0; i < l_; i++) { + if (!apply_op(trans, *op_[i])) { + l2_ = i; + return true; + } + tr_[i] = trans; + } + l2_ = l_; + return true; +} + +bool Optimizer::show_stack_transforms() const { + show_head(); + // slow version + /* + StackTransform trans2; + std::cerr << "id = " << trans2 << std::endl; + for (int i = 0; i < l_; i++) { + StackTransform op; + if (!apply_op(op, *op_[i])) { + std::cerr << "* (" << *op_[i] << " = invalid)\n"; + break; + } + trans2 *= op; + std::cerr << "* " << *op_[i] << " = " << op << " -> " << trans2 << std::endl; + } + */ + // fast version + StackTransform trans; + for (int i = 0; i < l_; i++) { + std::cerr << trans << std::endl << *op_[i] << " -> "; + if (!apply_op(trans, *op_[i])) { + std::cerr << " " << std::endl; + return true; + } + } + std::cerr << trans << std::endl; + return true; +} + +bool Optimizer::find_at_least(int pb) { + p_ = q_ = 0; + pb_ = pb; + // show_stack_transforms(); + int i, j, k, l, c; + return (is_push_const(&i, &c) && rewrite_push_const(i, c)) || (is_nop() && rewrite_nop()) || + (!(mode_ & 1) && is_const_rot(&c) && rewrite_const_rot(c)) || + (is_const_push_xchgs() && rewrite_const_push_xchgs()) || (is_const_pop(&c, &i) && rewrite_const_pop(c, i)) || + (is_xchg(&i, &j) && rewrite(AsmOp::Xchg(i, j))) || (is_push(&i) && rewrite(AsmOp::Push(i))) || + (is_pop(&i) && rewrite(AsmOp::Pop(i))) || (is_pop_pop(&i, &j) && rewrite(AsmOp::Pop(i), AsmOp::Pop(j))) || + (is_xchg_xchg(&i, &j, &k, &l) && rewrite(AsmOp::Xchg(i, j), AsmOp::Xchg(k, l))) || + (!(mode_ & 1) && + ((is_rot() && rewrite(AsmOp::Custom("ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom("-ROT", 3, 3))) || + (is_2dup() && rewrite(AsmOp::Custom("2DUP", 2, 4))) || + (is_2swap() && rewrite(AsmOp::Custom("2SWAP", 2, 4))) || + (is_2over() && rewrite(AsmOp::Custom("2OVER", 2, 4))) || + (is_tuck() && rewrite(AsmOp::Custom("TUCK", 2, 3))) || + (is_2drop() && rewrite(AsmOp::Custom("2DROP", 2, 0))) || (is_xchg2(&i, &j) && rewrite(AsmOp::Xchg2(i, j))) || + (is_xcpu(&i, &j) && rewrite(AsmOp::XcPu(i, j))) || (is_puxc(&i, &j) && rewrite(AsmOp::PuXc(i, j))) || + (is_push2(&i, &j) && rewrite(AsmOp::Push2(i, j))) || (is_blkswap(&i, &j) && rewrite(AsmOp::BlkSwap(i, j))) || + (is_blkpush(&i, &j) && rewrite(AsmOp::BlkPush(i, j))) || (is_blkdrop(&i) && rewrite(AsmOp::BlkDrop(i))) || + (is_push_rot(&i) && rewrite(AsmOp::Push(i), AsmOp::Custom("ROT"))) || + (is_push_rotrev(&i) && rewrite(AsmOp::Push(i), AsmOp::Custom("-ROT"))) || + (is_push_xchg(&i, &j, &k) && rewrite(AsmOp::Push(i), AsmOp::Xchg(j, k))) || + (is_reverse(&i, &j) && rewrite(AsmOp::BlkReverse(i, j))) || + (is_blkdrop2(&i, &j) && rewrite(AsmOp::BlkDrop2(i, j))) || + (is_nip_seq(&i, &j) && rewrite(AsmOp::Xchg(i, j), AsmOp::BlkDrop(i))) || + (is_pop_blkdrop(&i, &k) && rewrite(AsmOp::Pop(i), AsmOp::BlkDrop(k))) || + (is_2pop_blkdrop(&i, &j, &k) && (k >= 3 && k <= 13 && i != j + 1 && i <= 15 && j <= 14 + ? rewrite(AsmOp::Xchg2(j + 1, i), AsmOp::BlkDrop(k + 2)) + : rewrite(AsmOp::Pop(i), AsmOp::Pop(j), AsmOp::BlkDrop(k)))) || + (is_xchg3(&i, &j, &k) && rewrite(AsmOp::Xchg3(i, j, k))) || + (is_xc2pu(&i, &j, &k) && rewrite(AsmOp::Xc2Pu(i, j, k))) || + (is_xcpuxc(&i, &j, &k) && rewrite(AsmOp::XcPuXc(i, j, k))) || + (is_xcpu2(&i, &j, &k) && rewrite(AsmOp::XcPu2(i, j, k))) || + (is_puxc2(&i, &j, &k) && rewrite(AsmOp::PuXc2(i, j, k))) || + (is_puxcpu(&i, &j, &k) && rewrite(AsmOp::PuXcPu(i, j, k))) || + (is_pu2xc(&i, &j, &k) && rewrite(AsmOp::Pu2Xc(i, j, k))) || + (is_push3(&i, &j, &k) && rewrite(AsmOp::Push3(i, j, k))))); +} + +bool Optimizer::find() { + if (!compute_stack_transforms()) { + return false; + } + for (int pb = l_; pb > 0; --pb) { + if (find_at_least(pb)) { + return true; + } + } + return false; +} + +bool Optimizer::optimize() { + bool f = false; + while (find()) { + f = true; + apply(); + unpack(); + } + return f; +} + +AsmOpConsList optimize_code_head(AsmOpConsList op_list, int mode) { + Optimizer opt(std::move(op_list), false, mode); + opt.optimize(); + return opt.extract_code(); +} + +AsmOpConsList optimize_code(AsmOpConsList op_list, int mode) { + std::vector> v; + while (op_list) { + if (!op_list->car->is_comment()) { + op_list = optimize_code_head(std::move(op_list), mode); + } + if (op_list) { + v.push_back(std::move(op_list->car)); + op_list = std::move(op_list->cdr); + } + } + for (auto it = v.rbegin(); it < v.rend(); ++it) { + op_list = AsmOpCons::cons(std::move(*it), std::move(op_list)); + } + return std::move(op_list); +} + +void optimize_code(AsmOpList& ops) { + AsmOpConsList op_list; + for (auto it = ops.list_.rbegin(); it < ops.list_.rend(); ++it) { + op_list = AsmOpCons::cons(std::make_unique(std::move(*it)), std::move(op_list)); + } + for (int mode : {1, 1, 1, 1, 0, 0, 0, 0}) { + op_list = optimize_code(std::move(op_list), mode); + } + ops.list_.clear(); + while (op_list) { + ops.list_.push_back(std::move(*(op_list->car))); + op_list = std::move(op_list->cdr); + } +} + +} // namespace tolk diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp new file mode 100644 index 00000000..1561aa40 --- /dev/null +++ b/tolk/pipe-ast-to-legacy.cpp @@ -0,0 +1,1437 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" +#include "src-file.h" +#include "ast.h" +#include "ast-visitor.h" +#include "type-system.h" +#include "common/refint.h" +#include "constant-evaluator.h" +#include + +/* + * This pipe is the last one operating AST: it transforms AST to IR. + * IR is described as "Op" struct. So, here AST is transformed to Ops, and then all the rest "legacy" + * kernel (initially forked from FunC) comes into play. + * Up to this point, all types have been inferred, all validity checks have been passed, etc. + * All properties in AST nodes are assigned and can be safely used (fun_ref, etc.). + * So, if execution reaches this pass, the input is (almost) correct, and code generation should succeed. + * (previously, there was a check for one variable modified twice like `(t.0, t.0) = rhs`, but after changing + * execution order of assignment to "first lhs, then lhs", it was removed for several reasons) +* + * A noticeable property for IR generation is "target_type" used to extend/shrink stack. + * Example: `var a: (int,int)? = null`. This `null` has inferred_type "null literal", but target_type "nullable tensor", + * and when it's assigned, it's "expanded" from 1 stack slot to 3 (int + int + null flag). + * Example: `fun analyze(t: (int,int)?)` and a call `analyze((1,2))`. `(1,2)` is `(int,int)` (2 stack slots), + * and when passed to target (3 slots, one for null flag), this null flag is implicitly added (zero value). + * Example: `nullableInt!`; for `nullableInt` inferred_type is `int?`, and target_type is `int` + * (this doesn't lead to stack reorganization, but in case `nullableTensor!` does) + * (inferred_type of `nullableInt!` is `int`, and its target_type depends on its usage). + * The same mechanism will work for union types in the future. + */ + +namespace tolk { + +class LValContext; +std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr target_type = nullptr, LValContext* lval_ctx = nullptr); +std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, CodeBlob& code, LValContext* lval_ctx); +void process_any_statement(AnyV v, CodeBlob& code); + + +// The goal of VarsModificationWatcher is to detect such cases: `return (x, x += y, x)`. +// Without any changes, ops will be { _Call $2 = +($0_x, $1_y); _Return $0_x, $2, $0_x } - incorrect +// Correct will be to introduce tmp var: { _Let $3 = $0_x; _Call $2 = ...; _Return $3, $2, $0_x } +// This "introducing" is done when compiling tensors, whereas this class allows to watch vars for modification. +class VarsModificationWatcher { + struct WatchedVar { + var_idx_t ir_idx; + std::function on_modification_callback; + + WatchedVar(var_idx_t ir_idx, std::function on_modification_callback) + : ir_idx(ir_idx), on_modification_callback(std::move(on_modification_callback)) {} + }; + + std::vector all_callbacks; + +public: + + bool empty() const { return all_callbacks.empty(); } + + void push_callback(var_idx_t ir_idx, std::function callback) { + all_callbacks.emplace_back(ir_idx, std::move(callback)); + } + + void pop_callback(var_idx_t ir_idx) { + for (auto it = all_callbacks.rbegin(); it != all_callbacks.rend(); ++it) { + if (it->ir_idx == ir_idx) { + all_callbacks.erase((it + 1).base()); + return; + } + } + tolk_assert(false); + } + + void trigger_callbacks(const std::vector& left_lval_indices, SrcLocation loc) const { + for (const WatchedVar& w : all_callbacks) { + for (var_idx_t changed_var : left_lval_indices) { + if (w.ir_idx == changed_var) { + w.on_modification_callback(loc, w.ir_idx); + } + } + } + } +}; + +static VarsModificationWatcher vars_modification_watcher; + + +// Main goal of LValContext is to handle non-primitive lvalues. At IR level, a usual local variable +// exists, but on its change, something non-trivial should happen. +// Example: `globalVar = 9` actually does `Const $5 = 9` + `Let $6 = $5` + `SetGlob "globVar" = $6` +// Example: `tupleVar.0 = 9` actually does `Const $5 = 9` + `Let $6 = $5` + `Const $7 = 0` + `Call tupleSetAt($4, $6, $7)` +// Of course, mixing globals with tuples should also be supported. +// To achieve this, treat tupleObj inside "tupleObj.i" like "rvalue inside lvalue". +// For instance, `globalTuple.0 = 9` reads global (like rvalue), assigns 9 to tmp var, modifies tuple, writes global. +// Note, that tensors (not tuples) `tensorVar.0 = 9` do not emit anything special (unless global). +class LValContext { + // every global variable used as lvalue is registered here + // example: `globalInt = 9`, implicit var is created `$tmp = 9`, and `SetGlob "globalInt" $tmp` is done after + struct ModifiedGlobal { + GlobalVarPtr glob_ref; + std::vector lval_ir_idx; // typically 1, generally get_width_on_stack() of global var (tensors) + + // for 1-slot globals int/cell/slice, assigning to them is just SETGLOB + // same for tensors, if they are fully rewritten in an expression: `gTensor = (5,6)` + void apply_fully_rewrite(CodeBlob& code, SrcLocation loc) const { + Op& op = code.emplace_back(loc, Op::_SetGlob, std::vector{}, lval_ir_idx, glob_ref); + op.set_impure_flag(); + } + + // for N-slot globals tensor/struct/union, assigning to their parts, like `gTensor.1 = 6` + // we need to read gTensor as a whole (0-th and 1-th component), rewrite 1-th component, and SETGLOB a whole back + void apply_partially_rewrite(CodeBlob& code, SrcLocation loc, std::vector&& was_modified_by_let) const { + LValContext local_lval; + local_lval.enter_rval_inside_lval(); + std::vector local_ir_idx = pre_compile_symbol(loc, glob_ref, code, &local_lval); + for (size_t i = 0; i < local_ir_idx.size(); ++i) { + if (was_modified_by_let[i]) { + code.emplace_back(loc, Op::_Let, std::vector{local_ir_idx[i]}, std::vector{lval_ir_idx[i]}); + } + } + + Op& op = code.emplace_back(loc, Op::_SetGlob, std::vector{}, local_ir_idx, glob_ref); + op.set_impure_flag(); + } + }; + + // every tensor index, when a tensor is a global, is registered here (same for structs and fields) + // example: `global v: (int, int); v.1 = 5`, implicit var is created `$tmp = 5`, and when it's modified, + // we need to partially update w; essentially, apply_partially_rewrite() above will be called + struct ModifiedFieldOfGlobal { + AnyExprV tensor_obj; + int index_at; + std::vector lval_ir_idx; + + void apply(CodeBlob& code, SrcLocation loc) const { + LValContext local_lval; + local_lval.enter_rval_inside_lval(); + std::vector obj_ir_idx = pre_compile_expr(tensor_obj, code, nullptr, &local_lval); + const TypeDataTensor* t_tensor = tensor_obj->inferred_type->try_as(); + tolk_assert(t_tensor); + int stack_width = t_tensor->items[index_at]->get_width_on_stack(); + int stack_offset = 0; + for (int i = 0; i < index_at; ++i) { + stack_offset += t_tensor->items[i]->get_width_on_stack(); + } + std::vector field_ir_idx = {obj_ir_idx.begin() + stack_offset, obj_ir_idx.begin() + stack_offset + stack_width}; + tolk_assert(field_ir_idx.size() == lval_ir_idx.size()); + + vars_modification_watcher.trigger_callbacks(field_ir_idx, loc); + code.emplace_back(loc, Op::_Let, field_ir_idx, lval_ir_idx); + local_lval.after_let(std::move(field_ir_idx), code, loc); + } + }; + + // every tuple index used as lvalue is registered here + // example: `t.0 = 9`, implicit var is created `$tmp = 9`, as well as `$tmp_idx = 0` and `tupleSetAt()` is done after + // for `t.0.0` if t is `[[int, ...]]`, `tupleAt()` for it is done since it's rvalue, and `tupleSetAt()` is done 2 times + struct ModifiedTupleIndex { + AnyExprV tuple_obj; + int index_at; + std::vector lval_ir_idx; + + void apply(CodeBlob& code, SrcLocation loc) const { + LValContext local_lval; + local_lval.enter_rval_inside_lval(); + std::vector tuple_ir_idx = pre_compile_expr(tuple_obj, code, nullptr, &local_lval); + std::vector index_ir_idx = code.create_tmp_var(TypeDataInt::create(), loc, "(tuple-idx)"); + code.emplace_back(loc, Op::_IntConst, index_ir_idx, td::make_refint(index_at)); + + vars_modification_watcher.trigger_callbacks(tuple_ir_idx, loc); + FunctionPtr builtin_sym = lookup_global_symbol("tupleSetAt")->try_as(); + code.emplace_back(loc, Op::_Call, std::vector{tuple_ir_idx}, std::vector{tuple_ir_idx[0], lval_ir_idx[0], index_ir_idx[0]}, builtin_sym); + local_lval.after_let(std::move(tuple_ir_idx), code, loc); + } + }; + + int level_rval_inside_lval = 0; + std::vector> modifications; + + static bool vector_contains(const std::vector& ir_vars, var_idx_t ir_idx) { + for (var_idx_t var_in_vector : ir_vars) { + if (var_in_vector == ir_idx) { + return true; + } + } + return false; + } + +public: + void enter_rval_inside_lval() { level_rval_inside_lval++; } + void exit_rval_inside_lval() { level_rval_inside_lval--; } + bool is_rval_inside_lval() const { return level_rval_inside_lval > 0; } + + void capture_global_modification(GlobalVarPtr glob_ref, std::vector lval_ir_idx) { + modifications.emplace_back(ModifiedGlobal{glob_ref, std::move(lval_ir_idx)}); + } + + void capture_field_of_global_modification(AnyExprV tensor_obj, int index_at, std::vector lval_ir_idx) { + modifications.emplace_back(ModifiedFieldOfGlobal{tensor_obj, index_at, std::move(lval_ir_idx)}); + } + + void capture_tuple_index_modification(AnyExprV tuple_obj, int index_at, std::vector lval_ir_idx) { + modifications.emplace_back(ModifiedTupleIndex{tuple_obj, index_at, std::move(lval_ir_idx)}); + } + + void after_let(std::vector&& let_left_vars, CodeBlob& code, SrcLocation loc) const { + for (const auto& modification : modifications) { + if (const auto* m_glob = std::get_if(&modification)) { + int n_modified_by_let = 0; + std::vector was_modified_by_let; + was_modified_by_let.resize(m_glob->lval_ir_idx.size()); + for (size_t i = 0; i < m_glob->lval_ir_idx.size(); ++i) { + if (vector_contains(let_left_vars, m_glob->lval_ir_idx[i])) { + was_modified_by_let[i] = true; + n_modified_by_let++; + } + } + if (n_modified_by_let == static_cast(m_glob->lval_ir_idx.size())) { + m_glob->apply_fully_rewrite(code, loc); + } else if (n_modified_by_let > 0) { + m_glob->apply_partially_rewrite(code, loc, std::move(was_modified_by_let)); + } + } else if (const auto* m_tup = std::get_if(&modification)) { + bool was_tuple_index_modified = false; + for (var_idx_t field_ir_idx : m_tup->lval_ir_idx) { + was_tuple_index_modified |= vector_contains(let_left_vars, field_ir_idx); + } + if (was_tuple_index_modified) { + m_tup->apply(code, loc); + } + } else if (const auto* m_tens = std::get_if(&modification)) { + bool was_tensor_index_modified = false; + for (var_idx_t field_ir_idx : m_tens->lval_ir_idx) { + was_tensor_index_modified |= vector_contains(let_left_vars, field_ir_idx); + } + if (was_tensor_index_modified) { + m_tens->apply(code, loc); + } + } + } + } +}; + +// given `{some_expr}!`, return some_expr +static AnyExprV unwrap_not_null_operator(AnyExprV v) { + while (auto v_notnull = v->try_as()) { + v = v_notnull->get_expr(); + } + return v; +} + +// given `{some_expr}.{i}`, check it for pattern `some_var.0` / `some_var.0.1` / etc. +// return some_var if satisfies (it may be a local or a global var, a tensor or a tuple) +// return nullptr otherwise: `f().0` / `(v = rhs).0` / `some_var.method().0` / etc. +static V calc_sink_leftmost_obj(V v) { + AnyExprV leftmost_obj = unwrap_not_null_operator(v->get_obj()); + while (auto v_dot = leftmost_obj->try_as()) { + if (!v_dot->is_target_indexed_access()) { + break; + } + leftmost_obj = unwrap_not_null_operator(v_dot->get_obj()); + } + return leftmost_obj->type == ast_reference ? leftmost_obj->as() : nullptr; +} + + +static std::vector> pre_compile_tensor_inner(CodeBlob& code, const std::vector& args, + const TypeDataTensor* tensor_target_type, LValContext* lval_ctx) { + const int n = static_cast(args.size()); + if (n == 0) { // just `()` + return {}; + } + tolk_assert(!tensor_target_type || tensor_target_type->size() == n); + if (n == 1) { // just `(x)`: even if x is modified (e.g. `f(x=x+2)`), there are no next arguments + TypePtr child_target_type = tensor_target_type ? tensor_target_type->items[0] : nullptr; + return {pre_compile_expr(args[0], code, child_target_type, lval_ctx)}; + } + + // the purpose is to handle such cases: `return (x, x += y, x)` + // without this, ops will be { _Call $2 = +($0_x, $1_y); _Return $0_x, $2, $0_x } - invalid + // with this, ops will be { _Let $3 = $0_x; _Call $2 = ...; _Return $3, $2, $0_x } - valid, tmp var for x + // how it works: for every arg, after transforming to ops, start tracking ir_idx inside it + // on modification attempt, create Op::_Let to a tmp var and replace old ir_idx with tmp_idx in result + struct WatchingVarList { + std::vector watched_vars; + std::vector> res_lists; + + explicit WatchingVarList(int n_args) { + res_lists.reserve(n_args); + } + + bool is_watched(var_idx_t ir_idx) const { + return std::find(watched_vars.begin(), watched_vars.end(), ir_idx) != watched_vars.end(); + } + + void add_and_watch_modifications(std::vector&& vars_of_ith_arg, CodeBlob& code) { + for (var_idx_t ir_idx : vars_of_ith_arg) { + if (!code.vars[ir_idx].name.empty() && !is_watched(ir_idx)) { + watched_vars.emplace_back(ir_idx); + vars_modification_watcher.push_callback(ir_idx, [this, &code](SrcLocation loc, var_idx_t ir_idx) { + on_var_modified(ir_idx, loc, code); + }); + } + } + res_lists.emplace_back(std::move(vars_of_ith_arg)); + } + + void on_var_modified(var_idx_t ir_idx, SrcLocation loc, CodeBlob& code) { + tolk_assert(is_watched(ir_idx)); + std::vector tmp_idx_arr = code.create_tmp_var(code.vars[ir_idx].v_type, loc, "(pre-modified)"); + tolk_assert(tmp_idx_arr.size() == 1); + var_idx_t tmp_idx = tmp_idx_arr[0]; + code.emplace_back(loc, Op::_Let, std::vector{tmp_idx}, std::vector{ir_idx}); + for (std::vector& prev_vars : res_lists) { + std::replace(prev_vars.begin(), prev_vars.end(), ir_idx, tmp_idx); + } + } + + std::vector> clear_and_stop_watching() { + for (var_idx_t ir_idx : watched_vars) { + vars_modification_watcher.pop_callback(ir_idx); + } + watched_vars.clear(); + return std::move(res_lists); + } + }; + + WatchingVarList watched_vars(n); + for (int arg_idx = 0; arg_idx < n; ++arg_idx) { + TypePtr child_target_type = tensor_target_type ? tensor_target_type->items[arg_idx] : nullptr; + std::vector vars_of_ith_arg = pre_compile_expr(args[arg_idx], code, child_target_type, lval_ctx); + watched_vars.add_and_watch_modifications(std::move(vars_of_ith_arg), code); + } + return watched_vars.clear_and_stop_watching(); +} + +static std::vector pre_compile_tensor(CodeBlob& code, const std::vector& args, + LValContext* lval_ctx = nullptr) { + std::vector types_list; + types_list.reserve(args.size()); + for (AnyExprV item : args) { + types_list.push_back(item->inferred_type); + } + const TypeDataTensor* tensor_target_type = TypeDataTensor::create(std::move(types_list))->try_as(); + std::vector> res_lists = pre_compile_tensor_inner(code, args, tensor_target_type, lval_ctx); + std::vector res; + for (const std::vector& list : res_lists) { + res.insert(res.end(), list.cbegin(), list.cend()); + } + return res; +} + +static std::vector pre_compile_let(CodeBlob& code, AnyExprV lhs, AnyExprV rhs, SrcLocation loc) { + // [lhs] = [rhs]; since type checking is ok, it's the same as "lhs = rhs" + if (lhs->type == ast_typed_tuple && rhs->type == ast_typed_tuple) { + // note: there are no type transitions (adding nullability flag, etc.), since only 1-slot elements allowed in tuples + LValContext local_lval; + std::vector left = pre_compile_tensor(code, lhs->as()->get_items(), &local_lval); + vars_modification_watcher.trigger_callbacks(left, loc); + std::vector rvect = pre_compile_tensor(code, rhs->as()->get_items()); + code.emplace_back(loc, Op::_Let, left, rvect); + local_lval.after_let(std::move(left), code, loc); + std::vector right = code.create_tmp_var(TypeDataTuple::create(), loc, "(tuple)"); + code.emplace_back(lhs->loc, Op::_Tuple, right, std::move(rvect)); + return right; + } + // [lhs] = rhs; it's un-tuple to N left vars + if (lhs->type == ast_typed_tuple) { + LValContext local_lval; + std::vector left = pre_compile_tensor(code, lhs->as()->get_items(), &local_lval); + vars_modification_watcher.trigger_callbacks(left, loc); + std::vector right = pre_compile_expr(rhs, code, nullptr); + const TypeDataTypedTuple* inferred_tuple = rhs->inferred_type->try_as(); + std::vector types_list = inferred_tuple->items; + std::vector rvect = code.create_tmp_var(TypeDataTensor::create(std::move(types_list)), rhs->loc, "(unpack-tuple)"); + code.emplace_back(lhs->loc, Op::_UnTuple, rvect, std::move(right)); + code.emplace_back(loc, Op::_Let, left, rvect); + local_lval.after_let(std::move(left), code, loc); + return right; + } + // small optimization: `var x = rhs` or `local_var = rhs` (90% cases), LValContext not needed actually + if (lhs->type == ast_local_var_lhs || (lhs->type == ast_reference && lhs->as()->sym->try_as())) { + std::vector left = pre_compile_expr(lhs, code, nullptr); // effectively, local_var->ir_idx + vars_modification_watcher.trigger_callbacks(left, loc); + std::vector right = pre_compile_expr(rhs, code, lhs->inferred_type); + code.emplace_back(loc, Op::_Let, std::move(left), right); + return right; + } + // lhs = rhs + LValContext local_lval; + std::vector left = pre_compile_expr(lhs, code, nullptr, &local_lval); + vars_modification_watcher.trigger_callbacks(left, loc); + std::vector right = pre_compile_expr(rhs, code, lhs->inferred_type); + code.emplace_back(loc, Op::_Let, left, right); + local_lval.after_let(std::move(left), code, loc); + return right; +} + +static std::vector gen_op_call(CodeBlob& code, TypePtr ret_type, SrcLocation loc, + std::vector&& args_vars, FunctionPtr fun_ref, const char* debug_desc) { + std::vector rvect = code.create_tmp_var(ret_type, loc, debug_desc); + Op& op = code.emplace_back(loc, Op::_Call, rvect, std::move(args_vars), fun_ref); + if (!fun_ref->is_marked_as_pure()) { + op.set_impure_flag(); + } + return rvect; +} + +// "Transition to target (runtime) type" is the following process. +// Imagine `fun analyze(t: (int,int)?)` and a call `analyze((1,2))`. +// `(1,2)` (inferred_type) is 2 stack slots, but `t` (target_type) is 3 (one for null-flag). +// So, this null flag should be implicitly added (non-zero, since a variable is not null). +// Another example: `var t: (int, int)? = null`. +// `null` (inferred_type) is 1 stack slots, but target_type is 3, we should add 2 nulls. +// Another example: `var t1 = (1, null); var t2: (int, (int,int)?) = t1;`. +// Then t1's rvect is 2 vars (1 and null), but t1's `null` should be converted to 3 stack slots (resulting in 4 total). +// The same mechanism will work for union types in the future. +// Here rvect is a list of IR vars for inferred_type, probably patched due to target_type. +GNU_ATTRIBUTE_NOINLINE +static std::vector transition_expr_to_runtime_type_impl(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc) { + // pass `T` to `T` + // could occur for passing tensor `(..., T, ...)` to `(..., T, ...)` while traversing tensor's components + if (target_type == original_type) { + return rvect; + } + + int target_w = target_type->get_width_on_stack(); + const TypeDataNullable* t_nullable = target_type->try_as(); + const TypeDataNullable* o_nullable = original_type->try_as(); + + // handle `never` + // it may occur due to smart cast and in unreachable branches + // we can't do anything reasonable here, but (hopefully) execution will never reach this point, and stack won't be polluted + if (original_type == TypeDataNever::create()) { + std::vector dummy_rvect; + dummy_rvect.reserve(target_w); + for (int i = 0; i < target_w; ++i) { + dummy_rvect.push_back(code.create_tmp_var(TypeDataUnknown::create(), loc, "(never)")[0]); + } + return dummy_rvect; + } + if (target_type == TypeDataNever::create()) { + return {}; + } + + // pass `null` to `T?` + // for primitives like `int?`, no changes in rvect, null occupies the same TVM slot + // for tensors like `(int,int)?`, `null` is represented as N nulls + 1 null flag, insert N nulls + if (t_nullable && original_type == TypeDataNullLiteral::create()) { + tolk_assert(rvect.size() == 1); + if (target_w == 1 && !t_nullable->is_primitive_nullable()) { // `null` to `()?` + rvect = code.create_tmp_var(TypeDataInt::create(), loc, "(NNFlag)"); + code.emplace_back(loc, Op::_IntConst, rvect, td::make_refint(0)); + } + if (target_w > 1) { + FunctionPtr builtin_sym = lookup_global_symbol("__null")->try_as(); + rvect.reserve(target_w + 1); + for (int i = 1; i < target_w - 1; ++i) { + std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); + code.emplace_back(loc, Op::_Call, ith_null, std::vector{}, builtin_sym); + rvect.push_back(ith_null[0]); + } + std::vector null_flag_ir = code.create_tmp_var(TypeDataInt::create(), loc, "(NNFlag)"); + var_idx_t null_flag_ir_idx = null_flag_ir[0]; + code.emplace_back(loc, Op::_IntConst, std::move(null_flag_ir), td::make_refint(0)); + rvect.push_back(null_flag_ir_idx); + } + return rvect; + } + // pass `T` to `T?` + // for primitives like `int?`, no changes in rvect: `int` and `int?` occupy the same TVM slot (null is represented as NULL TVM value) + // for passing `(int, int)` to `(int, int)?` / `(int, null)` to `(int, (int,int)?)?`, add a null flag equals to 0 + if (t_nullable && !o_nullable) { + if (!t_nullable->is_primitive_nullable()) { + rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type, t_nullable->inner, loc); + tolk_assert(target_w == static_cast(rvect.size() + 1)); + std::vector null_flag_ir = code.create_tmp_var(TypeDataInt::create(), loc, "(NNFlag)"); + var_idx_t null_flag_ir_idx = null_flag_ir[0]; + code.emplace_back(loc, Op::_IntConst, std::move(null_flag_ir), td::make_refint(-1)); + rvect.push_back(null_flag_ir_idx); + } + return rvect; + } + // pass `T1?` to `T2?` + // for example, `int8?` to `int16?` + // transition inner types, leaving nullable flag unchanged for tensors + if (t_nullable && o_nullable) { + if (target_w > 1) { + var_idx_t null_flag_ir_idx = rvect.back(); + rvect.pop_back(); + rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, o_nullable->inner, t_nullable->inner, loc); + rvect.push_back(null_flag_ir_idx); + } + return rvect; + } + // pass `T?` to `null` + // it may occur due to smart cast, when a `T?` variable is guaranteed to be always null + // (for instance, always-null `(int,int)?` will be represented as 1 TVM NULL value, not 3) + if (target_type == TypeDataNullLiteral::create() && original_type->can_rhs_be_assigned(target_type)) { + tolk_assert(o_nullable || original_type == TypeDataUnknown::create()); + if (o_nullable && !o_nullable->is_primitive_nullable()) { + FunctionPtr builtin_sym = lookup_global_symbol("__null")->try_as(); + rvect = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); + code.emplace_back(loc, Op::_Call, rvect, std::vector{}, builtin_sym); + } + return rvect; + } + // pass `T?` to `T` (or, more generally, `T1?` to `T2`) + // it may occur due to operator `!` or smart cast + // for primitives like `int?`, no changes in rvect + // for passing `(int, int)?` to `(int, int)`, drop the null flag from the tail + // for complex scenarios like passing `(int, (int,int)?)?` to `(int, null)`, recurse the call + // (it may occur on `someF(t = (3,null))` when `(3,null)` at first targeted to lhs, but actually its result is rhs) + if (!t_nullable && o_nullable) { + if (!o_nullable->is_primitive_nullable()) { + rvect.pop_back(); + rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type->try_as()->inner, target_type, loc); + } + return rvect; + } + // pass `bool` to `int` + // in code, it's done via `as` operator, like `boolVar as int` + // no changes in rvect, boolVar is guaranteed to be -1 or 0 at TVM level + if (target_type == TypeDataInt::create() && original_type == TypeDataBool::create()) { + return rvect; + } + // pass something to `unknown` + // probably, it comes from `_ = rhs`, type of `_` is unknown, it's target_type of rhs + // no changes in rvect + if (target_type == TypeDataUnknown::create()) { + return rvect; + } + // pass `unknown` to something + // probably, it comes from `arg` in exception, it's inferred as `unknown` and could be cast to any value + if (original_type == TypeDataUnknown::create()) { + tolk_assert(rvect.size() == 1); + return rvect; + } + // pass tensor to tensor, e.g. `(1, null)` to `(int, slice?)` / `(1, null)` to `(int, (int,int)?)` + // every element of rhs tensor should be transitioned + if (target_type->try_as() && original_type->try_as()) { + const TypeDataTensor* target_tensor = target_type->try_as(); + const TypeDataTensor* inferred_tensor = original_type->try_as(); + tolk_assert(target_tensor->size() == inferred_tensor->size()); + tolk_assert(inferred_tensor->get_width_on_stack() == static_cast(rvect.size())); + std::vector result_rvect; + result_rvect.reserve(target_w); + int stack_offset = 0; + for (int i = 0; i < inferred_tensor->size(); ++i) { + int ith_w = inferred_tensor->items[i]->get_width_on_stack(); + std::vector rvect_i{rvect.begin() + stack_offset, rvect.begin() + stack_offset + ith_w}; + std::vector result_i = transition_expr_to_runtime_type_impl(std::move(rvect_i), code, inferred_tensor->items[i], target_tensor->items[i], loc); + result_rvect.insert(result_rvect.end(), result_i.begin(), result_i.end()); + stack_offset += ith_w; + } + return result_rvect; + } + // pass tuple to tuple, e.g. `[1, null]` to `[int, int?]` / `[1, null]` to `[int, [int?,int?]?]` + // to changes to rvect, since tuples contain only 1-slot elements + if (target_type->try_as() && original_type->try_as()) { + tolk_assert(target_type->get_width_on_stack() == original_type->get_width_on_stack()); + return rvect; + } + + throw Fatal("unhandled transition_expr_to_runtime_type_impl() combination"); +} + +// invoke the function above only if potentially needed to +// (if an expression is targeted to another type) +#ifndef TOLK_DEBUG +GNU_ATTRIBUTE_ALWAYS_INLINE +#endif +static std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr target_type, AnyExprV v) { + if (target_type != nullptr && target_type != v->inferred_type) { + rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, v->inferred_type, target_type, v->loc); + } + return rvect; +} + +// the second overload of the same function, invoke impl only when original and target differ +#ifndef TOLK_DEBUG +GNU_ATTRIBUTE_ALWAYS_INLINE +#endif +static std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc) { + if (target_type != original_type) { + rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type, target_type, loc); + } + return rvect; +} + + +std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, CodeBlob& code, LValContext* lval_ctx) { + if (GlobalVarPtr glob_ref = sym->try_as()) { + // handle `globalVar = rhs` / `mutate globalVar` + if (lval_ctx && !lval_ctx->is_rval_inside_lval()) { + std::vector lval_ir_idx = code.create_tmp_var(glob_ref->declared_type, loc, "(lval-glob)"); + lval_ctx->capture_global_modification(glob_ref, lval_ir_idx); + return lval_ir_idx; + } + // `globalVar` is used for reading, just create local IR var to represent its value, Op GlobVar will fill it + // note, that global tensors are stored as a tuple an unpacked to N vars on read, N determined by declared_type + std::vector local_ir_idx = code.create_var(glob_ref->declared_type, loc, "g_" + glob_ref->name); + code.emplace_back(loc, Op::_GlobVar, local_ir_idx, std::vector{}, glob_ref); + if (lval_ctx) { // `globalVar.0 = rhs`, globalVar is rval inside lval + lval_ctx->capture_global_modification(glob_ref, local_ir_idx); + } + return local_ir_idx; + } + if (GlobalConstPtr const_ref = sym->try_as()) { + if (const_ref->is_int_const()) { + std::vector rvect = code.create_tmp_var(TypeDataInt::create(), loc, "(glob-const)"); + code.emplace_back(loc, Op::_IntConst, rvect, const_ref->as_int_const()); + return rvect; + } else { + std::vector rvect = code.create_tmp_var(TypeDataSlice::create(), loc, "(glob-const)"); + code.emplace_back(loc, Op::_SliceConst, rvect, const_ref->as_slice_const()); + return rvect; + } + } + if (FunctionPtr fun_ref = sym->try_as()) { + std::vector rvect = code.create_tmp_var(fun_ref->inferred_full_type, loc, "(glob-var-fun)"); + code.emplace_back(loc, Op::_GlobVar, rvect, std::vector{}, fun_ref); + return rvect; + } + if (LocalVarPtr var_ref = sym->try_as()) { +#ifdef TOLK_DEBUG + tolk_assert(static_cast(var_ref->ir_idx.size()) == var_ref->declared_type->get_width_on_stack()); +#endif + return var_ref->ir_idx; + } + throw Fatal("pre_compile_symbol"); +} + +static std::vector process_reference(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + std::vector rvect = pre_compile_symbol(v->loc, v->sym, code, lval_ctx); + + // a local variable might be smart cast at this point, for example we're in `if (v != null)` + // it means that we must drop the null flag (if it's a tensor), or maybe perform other stack transformations + // (from original var_ref->ir_idx to fit smart cast) + if (LocalVarPtr var_ref = v->sym->try_as()) { + // note, inside `if (v != null)` when `v` is used for writing, v->inferred_type is an original (declared_type) + // (smart casts apply only for rvalue, not for lvalue, we don't check it here, it's a property of inferring) + rvect = transition_to_target_type(std::move(rvect), code, var_ref->declared_type, v->inferred_type, v->loc); + } + + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_assignment(V v, CodeBlob& code, TypePtr target_type) { + AnyExprV lhs = v->get_lhs(); + AnyExprV rhs = v->get_rhs(); + + if (auto lhs_decl = lhs->try_as()) { + std::vector rvect = pre_compile_let(code, lhs_decl->get_expr(), rhs, v->loc); + return transition_to_target_type(std::move(rvect), code, target_type, v); + } else { + std::vector rvect = pre_compile_let(code, lhs, rhs, v->loc); + // now rvect contains rhs IR vars constructed to fit lhs (for correct assignment, lhs type was target_type for rhs) + // but the type of `lhs = rhs` is RHS (see type inferring), so rvect now should fit rhs->inferred_type (= v->inferred_type) + // example: `t1 = t2 = null`, we're at `t2 = null`, earlier declared t1: `int?`, t2: `(int,int)?` + // currently "null" matches t2 (3 null slots), but type of this assignment is "plain null" (1 slot) assigned later to t1 + rvect = transition_to_target_type(std::move(rvect), code, lhs->inferred_type, v->inferred_type, v->loc); + return transition_to_target_type(std::move(rvect), code, target_type, v); + } +} + +static std::vector process_set_assign(V v, CodeBlob& code, TypePtr target_type) { + // for "a += b", emulate "a = a + b" + // seems not beautiful, but it works; probably, this transformation should be done at AST level in advance + std::string_view calc_operator = v->operator_name; // "+" for operator += + auto v_apply = createV(v->loc, calc_operator, static_cast(v->tok - 1), v->get_lhs(), v->get_rhs()); + v_apply->assign_inferred_type(v->inferred_type); + v_apply->assign_fun_ref(v->fun_ref); + + std::vector rvect = pre_compile_let(code, v->get_lhs(), v_apply, v->loc); + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_binary_operator(V v, CodeBlob& code, TypePtr target_type) { + TokenType t = v->tok; + + if (v->fun_ref) { // almost all operators, fun_ref was assigned at type inferring + std::vector args_vars = pre_compile_tensor(code, {v->get_lhs(), v->get_rhs()}); + std::vector rvect = gen_op_call(code, v->inferred_type, v->loc, std::move(args_vars), v->fun_ref, "(binary-op)"); + return transition_to_target_type(std::move(rvect), code, target_type, v); + } + if (t == tok_logical_and || t == tok_logical_or) { + // do the following transformations: + // a && b -> a ? (b != 0) : 0 + // a || b -> a ? 1 : (b != 0) + AnyExprV v_0 = createV(v->loc, td::make_refint(0), "0"); + v_0->mutate()->assign_inferred_type(TypeDataInt::create()); + AnyExprV v_1 = createV(v->loc, td::make_refint(-1), "-1"); + v_1->mutate()->assign_inferred_type(TypeDataInt::create()); + auto v_b_ne_0 = createV(v->loc, "!=", tok_neq, v->get_rhs(), v_0); + v_b_ne_0->mutate()->assign_inferred_type(TypeDataInt::create()); + v_b_ne_0->mutate()->assign_fun_ref(lookup_global_symbol("_!=_")->try_as()); + std::vector cond = pre_compile_expr(v->get_lhs(), code, nullptr); + tolk_assert(cond.size() == 1); + std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(ternary)"); + Op& if_op = code.emplace_back(v->loc, Op::_If, cond); + code.push_set_cur(if_op.block0); + code.emplace_back(v->loc, Op::_Let, rvect, pre_compile_expr(t == tok_logical_and ? v_b_ne_0 : v_1, code, nullptr)); + code.close_pop_cur(v->loc); + code.push_set_cur(if_op.block1); + code.emplace_back(v->loc, Op::_Let, rvect, pre_compile_expr(t == tok_logical_and ? v_0 : v_b_ne_0, code, nullptr)); + code.close_pop_cur(v->loc); + return transition_to_target_type(std::move(rvect), code, target_type, v); + } + + throw UnexpectedASTNodeType(v, "process_binary_operator"); +} + +static std::vector process_unary_operator(V v, CodeBlob& code, TypePtr target_type) { + std::vector rhs_vars = pre_compile_expr(v->get_rhs(), code, nullptr); + std::vector rvect = gen_op_call(code, v->inferred_type, v->loc, std::move(rhs_vars), v->fun_ref, "(unary-op)"); + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_ternary_operator(V v, CodeBlob& code, TypePtr target_type) { + std::vector cond = pre_compile_expr(v->get_cond(), code, nullptr); + tolk_assert(cond.size() == 1); + std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(cond)"); + + if (v->get_cond()->is_always_true) { + code.emplace_back(v->get_when_true()->loc, Op::_Let, rvect, pre_compile_expr(v->get_when_true(), code, v->inferred_type)); + } else if (v->get_cond()->is_always_false) { + code.emplace_back(v->get_when_false()->loc, Op::_Let, rvect, pre_compile_expr(v->get_when_false(), code, v->inferred_type)); + } else { + Op& if_op = code.emplace_back(v->loc, Op::_If, cond); + code.push_set_cur(if_op.block0); + code.emplace_back(v->get_when_true()->loc, Op::_Let, rvect, pre_compile_expr(v->get_when_true(), code, v->inferred_type)); + code.close_pop_cur(v->get_when_true()->loc); + code.push_set_cur(if_op.block1); + code.emplace_back(v->get_when_false()->loc, Op::_Let, rvect, pre_compile_expr(v->get_when_false(), code, v->inferred_type)); + code.close_pop_cur(v->get_when_false()->loc); + } + + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_cast_as_operator(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + TypePtr child_target_type = v->cast_to_type; + std::vector rvect = pre_compile_expr(v->get_expr(), code, child_target_type, lval_ctx); + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_not_null_operator(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + TypePtr child_target_type = v->get_expr()->inferred_type; + if (const auto* as_nullable = child_target_type->try_as()) { + child_target_type = as_nullable->inner; + } + std::vector rvect = pre_compile_expr(v->get_expr(), code, child_target_type, lval_ctx); + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_is_null_check(V v, CodeBlob& code, TypePtr target_type) { + std::vector expr_ir_idx = pre_compile_expr(v->get_expr(), code, nullptr); + std::vector isnull_ir_idx = code.create_tmp_var(TypeDataBool::create(), v->loc, "(is-null)"); + TypePtr expr_type = v->get_expr()->inferred_type; + + if (const TypeDataNullable* t_nullable = expr_type->try_as()) { + if (!t_nullable->is_primitive_nullable()) { + std::vector zero_ir_idx = code.create_tmp_var(TypeDataInt::create(), v->loc, "(zero)"); + code.emplace_back(v->loc, Op::_IntConst, zero_ir_idx, td::make_refint(0)); + FunctionPtr eq_sym = lookup_global_symbol("_==_")->try_as(); + code.emplace_back(v->loc, Op::_Call, isnull_ir_idx, std::vector{expr_ir_idx.back(), zero_ir_idx[0]}, eq_sym); + } else { + FunctionPtr builtin_sym = lookup_global_symbol("__isNull")->try_as(); + code.emplace_back(v->loc, Op::_Call, isnull_ir_idx, expr_ir_idx, builtin_sym); + } + } else { + bool always_null = expr_type == TypeDataNullLiteral::create(); + code.emplace_back(v->loc, Op::_IntConst, isnull_ir_idx, td::make_refint(always_null ? -1 : 0)); + } + + if (v->is_negated) { + FunctionPtr not_sym = lookup_global_symbol("!b_")->try_as(); + code.emplace_back(v->loc, Op::_Call, isnull_ir_idx, std::vector{isnull_ir_idx}, not_sym); + } + return transition_to_target_type(std::move(isnull_ir_idx), code, target_type, v); +} + +static std::vector process_dot_access(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + // it's NOT a method call `t.tupleSize()` (since such cases are handled by process_function_call) + // it's `t.0`, `getUser().id`, and `t.tupleSize` (as a reference, not as a call) + if (!v->is_target_fun_ref()) { + TypePtr obj_type = v->get_obj()->inferred_type; + int index_at = std::get(v->target); + // `tensorVar.0` + if (const auto* t_tensor = obj_type->try_as()) { + // handle `tensorVar.0 = rhs` if tensors is a global, special case, then the global will be read on demand + if (lval_ctx && !lval_ctx->is_rval_inside_lval()) { + if (auto sink = calc_sink_leftmost_obj(v); sink && sink->sym->try_as()) { + std::vector lval_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(lval-global-tensor)"); + lval_ctx->capture_field_of_global_modification(v->get_obj(), index_at, lval_ir_idx); + return lval_ir_idx; + } + } + // since a tensor of N elems are N vars on a stack actually, calculate offset + std::vector lhs_vars = pre_compile_expr(v->get_obj(), code, nullptr, lval_ctx); + int stack_width = t_tensor->items[index_at]->get_width_on_stack(); + int stack_offset = 0; + for (int i = 0; i < index_at; ++i) { + stack_offset += t_tensor->items[i]->get_width_on_stack(); + } + std::vector rvect{lhs_vars.begin() + stack_offset, lhs_vars.begin() + stack_offset + stack_width}; + // a tensor index might be smart cast at this point, for example we're in `if (t.1 != null)` + // it means that we must drop the null flag (if `t.1` is a tensor), or maybe perform other stack transformations + // (from original rvect = (vars of t.1) to fit smart cast) + rvect = transition_to_target_type(std::move(rvect), code, t_tensor->items[index_at], v->inferred_type, v->loc); + return transition_to_target_type(std::move(rvect), code, target_type, v); + } + // `tupleVar.0` + if (obj_type->try_as() || obj_type->try_as()) { + // handle `tupleVar.0 = rhs`, "0 SETINDEX" will be called when this was is modified + if (lval_ctx && !lval_ctx->is_rval_inside_lval() && calc_sink_leftmost_obj(v)) { + std::vector lval_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(lval-tuple-field)"); + lval_ctx->capture_tuple_index_modification(v->get_obj(), index_at, lval_ir_idx); + return lval_ir_idx; + } + // `tupleVar.0` as rvalue: the same as "tupleAt(tupleVar, 0)" written in terms of IR vars + std::vector tuple_ir_idx = pre_compile_expr(v->get_obj(), code); + std::vector index_ir_idx = code.create_tmp_var(TypeDataInt::create(), v->get_identifier()->loc, "(tuple-idx)"); + code.emplace_back(v->loc, Op::_IntConst, index_ir_idx, td::make_refint(index_at)); + std::vector field_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(tuple-field)"); + tolk_assert(tuple_ir_idx.size() == 1 && field_ir_idx.size() == 1); // tuples contain only 1-slot values + FunctionPtr builtin_sym = lookup_global_symbol("tupleAt")->try_as(); + code.emplace_back(v->loc, Op::_Call, field_ir_idx, std::vector{tuple_ir_idx[0], index_ir_idx[0]}, builtin_sym); + if (lval_ctx && calc_sink_leftmost_obj(v)) { // `tupleVar.0.1 = rhs`, then `tupleVar.0` is rval inside lval + lval_ctx->capture_tuple_index_modification(v->get_obj(), index_at, field_ir_idx); + } + // like tensor index, `tupleVar.1` also might be smart cast, for example we're in `if (tupleVar.1 != null)` + // but since tuple's elements are only 1-slot width (no tensors and unions), no stack transformations required + return transition_to_target_type(std::move(field_ir_idx), code, target_type, v); + } + tolk_assert(false); + } + + // okay, v->target refs a function, like `obj.method`, filled at type inferring + // (currently, nothing except a global function can be referenced, no object-scope methods exist) + FunctionPtr fun_ref = std::get(v->target); + tolk_assert(fun_ref); + std::vector rvect = pre_compile_symbol(v->loc, fun_ref, code, lval_ctx); + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_function_call(V v, CodeBlob& code, TypePtr target_type) { + // v is `globalF(args)` / `globalF(args)` / `obj.method(args)` / `local_var(args)` / `getF()(args)` + FunctionPtr fun_ref = v->fun_maybe; + if (!fun_ref) { + // it's `local_var(args)`, treat args like a tensor: + // 1) when variables are modified like `local_var(x, x += 2, x)`, regular mechanism of watching automatically works + // 2) when `null` is passed to `(int, int)?`, or any other type transitions, it automatically works + std::vector args; + args.reserve(v->get_num_args()); + for (int i = 0; i < v->get_num_args(); ++i) { + args.push_back(v->get_arg(i)->get_expr()); + } + std::vector params_types = v->get_callee()->inferred_type->try_as()->params_types; + const TypeDataTensor* tensor_tt = TypeDataTensor::create(std::move(params_types))->try_as(); + std::vector> vars_per_arg = pre_compile_tensor_inner(code, args, tensor_tt, nullptr); + std::vector args_vars; + for (const std::vector& list : vars_per_arg) { + args_vars.insert(args_vars.end(), list.cbegin(), list.cend()); + } + std::vector tfunc = pre_compile_expr(v->get_callee(), code, nullptr); + tolk_assert(tfunc.size() == 1); + args_vars.push_back(tfunc[0]); + std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(call-ind)"); + Op& op = code.emplace_back(v->loc, Op::_CallInd, rvect, std::move(args_vars)); + op.set_impure_flag(); + return transition_to_target_type(std::move(rvect), code, target_type, v); + } + + int delta_self = v->is_dot_call(); + AnyExprV obj_leftmost = nullptr; + std::vector args; + args.reserve(delta_self + v->get_num_args()); + if (delta_self) { + args.push_back(v->get_dot_obj()); + obj_leftmost = v->get_dot_obj(); + while (obj_leftmost->type == ast_function_call && obj_leftmost->as()->is_dot_call() && obj_leftmost->as()->fun_maybe && obj_leftmost->as()->fun_maybe->does_return_self()) { + obj_leftmost = obj_leftmost->as()->get_dot_obj(); + } + } + for (int i = 0; i < v->get_num_args(); ++i) { + args.push_back(v->get_arg(i)->get_expr()); + } + // the purpose of tensor_tt ("tensor target type") is to transition `null` to `(int, int)?` and so on + // the purpose of calling `pre_compile_tensor_inner` is to have 0-th IR vars to handle return self + std::vector params_types = fun_ref->inferred_full_type->try_as()->params_types; + const TypeDataTensor* tensor_tt = TypeDataTensor::create(std::move(params_types))->try_as(); + std::vector> vars_per_arg = pre_compile_tensor_inner(code, args, tensor_tt, nullptr); + + TypePtr op_call_type = v->inferred_type; + TypePtr real_ret_type = v->inferred_type; + if (delta_self && fun_ref->does_return_self()) { + real_ret_type = TypeDataVoid::create(); + if (!fun_ref->parameters[0].is_mutate_parameter()) { + op_call_type = TypeDataVoid::create(); + } + } + if (fun_ref->has_mutate_params()) { + std::vector types_list; + for (int i = 0; i < delta_self + v->get_num_args(); ++i) { + if (fun_ref->parameters[i].is_mutate_parameter()) { + types_list.push_back(fun_ref->parameters[i].declared_type); + } + } + types_list.push_back(real_ret_type); + op_call_type = TypeDataTensor::create(std::move(types_list)); + } + + std::vector args_vars; + for (const std::vector& list : vars_per_arg) { + args_vars.insert(args_vars.end(), list.cbegin(), list.cend()); + } + std::vector rvect_apply = gen_op_call(code, op_call_type, v->loc, std::move(args_vars), fun_ref, "(fun-call)"); + + if (fun_ref->has_mutate_params()) { + LValContext local_lval; + std::vector left; + for (int i = 0; i < delta_self + v->get_num_args(); ++i) { + if (fun_ref->parameters[i].is_mutate_parameter()) { + AnyExprV arg_i = obj_leftmost && i == 0 ? obj_leftmost : args[i]; + tolk_assert(arg_i->is_lvalue || i == 0); + if (arg_i->is_lvalue) { + std::vector ith_var_idx = pre_compile_expr(arg_i, code, nullptr, &local_lval); + left.insert(left.end(), ith_var_idx.begin(), ith_var_idx.end()); + } else { + left.insert(left.end(), vars_per_arg[0].begin(), vars_per_arg[0].end()); + } + } + } + std::vector rvect = code.create_tmp_var(real_ret_type, v->loc, "(fun-call)"); + left.insert(left.end(), rvect.begin(), rvect.end()); + vars_modification_watcher.trigger_callbacks(left, v->loc); + code.emplace_back(v->loc, Op::_Let, left, rvect_apply); + local_lval.after_let(std::move(left), code, v->loc); + rvect_apply = rvect; + } + + if (obj_leftmost && fun_ref->does_return_self()) { + if (obj_leftmost->is_lvalue) { // to handle if obj is global var, potentially re-assigned inside a chain + rvect_apply = pre_compile_expr(obj_leftmost, code, nullptr); + } else { // temporary object, not lvalue, pre_compile_expr + rvect_apply = vars_per_arg[0]; + } + } + + return transition_to_target_type(std::move(rvect_apply), code, target_type, v); +} + +static std::vector process_tensor(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + // tensor is compiled "as is", for example `(1, null)` occupies 2 slots + // and if assigned/passed to something other, like `(int, (int,int)?)`, a whole tensor is transitioned, it works + std::vector rvect = pre_compile_tensor(code, v->get_items(), lval_ctx); + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_typed_tuple(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + if (lval_ctx) { // todo some time, make "var (a, [b,c]) = (1, [2,3])" work + v->error("[...] can not be used as lvalue here"); + } + std::vector left = code.create_tmp_var(v->inferred_type, v->loc, "(pack-tuple)"); + std::vector right = pre_compile_tensor(code, v->get_items(), lval_ctx); + code.emplace_back(v->loc, Op::_Tuple, left, std::move(right)); + return transition_to_target_type(std::move(left), code, target_type, v); +} + +static std::vector process_int_const(V v, CodeBlob& code, TypePtr target_type) { + std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(int-const)"); + code.emplace_back(v->loc, Op::_IntConst, rvect, v->intval); + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_string_const(V v, CodeBlob& code, TypePtr target_type) { + ConstantValue value = eval_const_init_value(v); + std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(str-const)"); + if (value.is_int()) { + code.emplace_back(v->loc, Op::_IntConst, rvect, value.as_int()); + } else { + code.emplace_back(v->loc, Op::_SliceConst, rvect, value.as_slice()); + } + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_bool_const(V v, CodeBlob& code, TypePtr target_type) { + FunctionPtr builtin_sym = lookup_global_symbol(v->bool_val ? "__true" : "__false")->try_as(); + std::vector rvect = gen_op_call(code, v->inferred_type, v->loc, {}, builtin_sym, "(bool-const)"); + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_null_keyword(V v, CodeBlob& code, TypePtr target_type) { + FunctionPtr builtin_sym = lookup_global_symbol("__null")->try_as(); + std::vector rvect = gen_op_call(code, v->inferred_type, v->loc, {}, builtin_sym, "(null-literal)"); + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_local_var(V v, CodeBlob& code, TypePtr target_type) { + if (v->marked_as_redef) { + std::vector rvect = pre_compile_symbol(v->loc, v->var_ref, code, nullptr); + return transition_to_target_type(std::move(rvect), code, target_type, v); + } + + tolk_assert(v->var_ref->ir_idx.empty()); + v->var_ref->mutate()->assign_ir_idx(code.create_var(v->inferred_type, v->loc, v->var_ref->name)); + std::vector rvect = v->var_ref->ir_idx; + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_local_vars_declaration(V, CodeBlob&) { + // it can not appear as a standalone expression + // `var ... = rhs` is handled by ast_assign + tolk_assert(false); +} + +static std::vector process_underscore(V v, CodeBlob& code) { + // when _ is used as left side of assignment, like `(cs, _) = cs.loadAndReturn()` + return code.create_tmp_var(v->inferred_type, v->loc, "(underscore)"); +} + +std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + switch (v->type) { + case ast_reference: + return process_reference(v->as(), code, target_type, lval_ctx); + case ast_assign: + return process_assignment(v->as(), code, target_type); + case ast_set_assign: + return process_set_assign(v->as(), code, target_type); + case ast_binary_operator: + return process_binary_operator(v->as(), code, target_type); + case ast_unary_operator: + return process_unary_operator(v->as(), code, target_type); + case ast_ternary_operator: + return process_ternary_operator(v->as(), code, target_type); + case ast_cast_as_operator: + return process_cast_as_operator(v->as(), code, target_type, lval_ctx); + case ast_not_null_operator: + return process_not_null_operator(v->as(), code, target_type, lval_ctx); + case ast_is_null_check: + return process_is_null_check(v->as(), code, target_type); + case ast_dot_access: + return process_dot_access(v->as(), code, target_type, lval_ctx); + case ast_function_call: + return process_function_call(v->as(), code, target_type); + case ast_parenthesized_expression: + return pre_compile_expr(v->as()->get_expr(), code, target_type, lval_ctx); + case ast_tensor: + return process_tensor(v->as(), code, target_type, lval_ctx); + case ast_typed_tuple: + return process_typed_tuple(v->as(), code, target_type, lval_ctx); + case ast_int_const: + return process_int_const(v->as(), code, target_type); + case ast_string_const: + return process_string_const(v->as(), code, target_type); + case ast_bool_const: + return process_bool_const(v->as(), code, target_type); + case ast_null_keyword: + return process_null_keyword(v->as(), code, target_type); + case ast_local_var_lhs: + return process_local_var(v->as(), code, target_type); + case ast_local_vars_declaration: + return process_local_vars_declaration(v->as(), code); + case ast_underscore: + return process_underscore(v->as(), code); + default: + throw UnexpectedASTNodeType(v, "pre_compile_expr"); + } +} + + +static void process_sequence(V v, CodeBlob& code) { + for (AnyV item : v->get_items()) { + process_any_statement(item, code); + } +} + +static void process_assert_statement(V v, CodeBlob& code) { + std::vector args(3); + if (auto v_not = v->get_cond()->try_as(); v_not && v_not->tok == tok_logical_not) { + args[0] = v->get_thrown_code(); + args[1] = v->get_cond()->as()->get_rhs(); + args[2] = createV(v->loc, true); + args[2]->mutate()->assign_inferred_type(TypeDataInt::create()); + } else { + args[0] = v->get_thrown_code(); + args[1] = v->get_cond(); + args[2] = createV(v->loc, false); + args[2]->mutate()->assign_inferred_type(TypeDataInt::create()); + } + + FunctionPtr builtin_sym = lookup_global_symbol("__throw_if_unless")->try_as(); + std::vector args_vars = pre_compile_tensor(code, args); + gen_op_call(code, TypeDataVoid::create(), v->loc, std::move(args_vars), builtin_sym, "(throw-call)"); +} + +static void process_catch_variable(AnyExprV v_catch_var, CodeBlob& code) { + if (auto v_ref = v_catch_var->try_as(); v_ref && v_ref->sym) { // not underscore + LocalVarPtr var_ref = v_ref->sym->try_as(); + tolk_assert(var_ref->ir_idx.empty()); + var_ref->mutate()->assign_ir_idx(code.create_var(v_catch_var->inferred_type, v_catch_var->loc, var_ref->name)); + } +} + +static void process_try_catch_statement(V v, CodeBlob& code) { + code.require_callxargs = true; + Op& try_catch_op = code.emplace_back(v->loc, Op::_TryCatch); + code.push_set_cur(try_catch_op.block0); + process_any_statement(v->get_try_body(), code); + code.close_pop_cur(v->get_try_body()->loc_end); + code.push_set_cur(try_catch_op.block1); + + // transform catch (excNo, arg) into TVM-catch (arg, excNo), where arg is untyped and thus almost useless now + const std::vector& catch_vars = v->get_catch_expr()->get_items(); + tolk_assert(catch_vars.size() == 2); + process_catch_variable(catch_vars[0], code); + process_catch_variable(catch_vars[1], code); + try_catch_op.left = pre_compile_tensor(code, {catch_vars[1], catch_vars[0]}); + process_any_statement(v->get_catch_body(), code); + code.close_pop_cur(v->get_catch_body()->loc_end); +} + +static void process_repeat_statement(V v, CodeBlob& code) { + std::vector tmp_vars = pre_compile_expr(v->get_cond(), code, nullptr); + Op& repeat_op = code.emplace_back(v->loc, Op::_Repeat, tmp_vars); + code.push_set_cur(repeat_op.block0); + process_any_statement(v->get_body(), code); + code.close_pop_cur(v->get_body()->loc_end); +} + +static void process_if_statement(V v, CodeBlob& code) { + std::vector cond = pre_compile_expr(v->get_cond(), code, nullptr); + tolk_assert(cond.size() == 1); + + if (v->get_cond()->is_always_true) { + process_any_statement(v->get_if_body(), code); // v->is_ifnot does not matter here + return; + } + if (v->get_cond()->is_always_false) { + process_any_statement(v->get_else_body(), code); + return; + } + + Op& if_op = code.emplace_back(v->loc, Op::_If, std::move(cond)); + code.push_set_cur(if_op.block0); + process_any_statement(v->get_if_body(), code); + code.close_pop_cur(v->get_if_body()->loc_end); + code.push_set_cur(if_op.block1); + process_any_statement(v->get_else_body(), code); + code.close_pop_cur(v->get_else_body()->loc_end); + if (v->is_ifnot) { + std::swap(if_op.block0, if_op.block1); + } +} + +static void process_do_while_statement(V v, CodeBlob& code) { + Op& until_op = code.emplace_back(v->loc, Op::_Until); + code.push_set_cur(until_op.block0); + process_any_statement(v->get_body(), code); + + // in TVM, there is only "do until", but in Tolk, we want "do while" + // here we negate condition to pass it forward to legacy to Op::_Until + // also, handle common situations as a hardcoded "optimization": replace (a<0) with (a>=0) and so on + // todo these hardcoded conditions should be removed from this place in the future + AnyExprV cond = v->get_cond(); + AnyExprV until_cond; + if (auto v_not = cond->try_as(); v_not && v_not->tok == tok_logical_not) { + until_cond = v_not->get_rhs(); + } else if (auto v_eq = cond->try_as(); v_eq && v_eq->tok == tok_eq) { + until_cond = createV(cond->loc, "!=", tok_neq, v_eq->get_lhs(), v_eq->get_rhs()); + } else if (auto v_neq = cond->try_as(); v_neq && v_neq->tok == tok_neq) { + until_cond = createV(cond->loc, "==", tok_eq, v_neq->get_lhs(), v_neq->get_rhs()); + } else if (auto v_leq = cond->try_as(); v_leq && v_leq->tok == tok_leq) { + until_cond = createV(cond->loc, ">", tok_gt, v_leq->get_lhs(), v_leq->get_rhs()); + } else if (auto v_lt = cond->try_as(); v_lt && v_lt->tok == tok_lt) { + until_cond = createV(cond->loc, ">=", tok_geq, v_lt->get_lhs(), v_lt->get_rhs()); + } else if (auto v_geq = cond->try_as(); v_geq && v_geq->tok == tok_geq) { + until_cond = createV(cond->loc, "<", tok_lt, v_geq->get_lhs(), v_geq->get_rhs()); + } else if (auto v_gt = cond->try_as(); v_gt && v_gt->tok == tok_gt) { + until_cond = createV(cond->loc, "<=", tok_geq, v_gt->get_lhs(), v_gt->get_rhs()); + } else if (cond->inferred_type == TypeDataBool::create()) { + until_cond = createV(cond->loc, "!b", tok_logical_not, cond); + } else { + until_cond = createV(cond->loc, "!", tok_logical_not, cond); + } + until_cond->mutate()->assign_inferred_type(TypeDataInt::create()); + if (auto v_bin = until_cond->try_as(); v_bin && !v_bin->fun_ref) { + v_bin->mutate()->assign_fun_ref(lookup_global_symbol("_" + static_cast(v_bin->operator_name) + "_")->try_as()); + } else if (auto v_un = until_cond->try_as(); v_un && !v_un->fun_ref) { + v_un->mutate()->assign_fun_ref(lookup_global_symbol(static_cast(v_un->operator_name) + "_")->try_as()); + } + + until_op.left = pre_compile_expr(until_cond, code, nullptr); + tolk_assert(until_op.left.size() == 1); + code.close_pop_cur(v->get_body()->loc_end); +} + +static void process_while_statement(V v, CodeBlob& code) { + Op& while_op = code.emplace_back(v->loc, Op::_While); + code.push_set_cur(while_op.block0); + while_op.left = pre_compile_expr(v->get_cond(), code, nullptr); + tolk_assert(while_op.left.size() == 1); + code.close_pop_cur(v->get_body()->loc); + code.push_set_cur(while_op.block1); + process_any_statement(v->get_body(), code); + code.close_pop_cur(v->get_body()->loc_end); +} + +static void process_throw_statement(V v, CodeBlob& code) { + if (v->has_thrown_arg()) { + FunctionPtr builtin_sym = lookup_global_symbol("__throw_arg")->try_as(); + std::vector args_vars = pre_compile_tensor(code, {v->get_thrown_arg(), v->get_thrown_code()}); + gen_op_call(code, TypeDataVoid::create(), v->loc, std::move(args_vars), builtin_sym, "(throw-call)"); + } else { + FunctionPtr builtin_sym = lookup_global_symbol("__throw")->try_as(); + std::vector args_vars = pre_compile_tensor(code, {v->get_thrown_code()}); + gen_op_call(code, TypeDataVoid::create(), v->loc, std::move(args_vars), builtin_sym, "(throw-call)"); + } +} + +static void process_return_statement(V v, CodeBlob& code) { + std::vector return_vars; + if (v->has_return_value()) { + TypePtr child_target_type = code.fun_ref->inferred_return_type; + if (code.fun_ref->does_return_self()) { + child_target_type = code.fun_ref->parameters[0].declared_type; + } + return_vars = pre_compile_expr(v->get_return_value(), code, child_target_type); + } + if (code.fun_ref->does_return_self()) { + return_vars = {}; + } + if (code.fun_ref->has_mutate_params()) { + std::vector mutated_vars; + for (const LocalVarData& p_sym: code.fun_ref->parameters) { + if (p_sym.is_mutate_parameter()) { + mutated_vars.insert(mutated_vars.end(), p_sym.ir_idx.begin(), p_sym.ir_idx.end()); + } + } + return_vars.insert(return_vars.begin(), mutated_vars.begin(), mutated_vars.end()); + } + code.emplace_back(v->loc, Op::_Return, std::move(return_vars)); +} + +// append "return" (void) to the end of the function +// if it's not reachable, it will be dropped +// (IR cfg reachability may differ from FlowContext in case of "never" types, so there may be situations, +// when IR will consider this "return" reachable and leave it, but actually execution will never reach it) +static void append_implicit_return_statement(SrcLocation loc_end, CodeBlob& code) { + std::vector mutated_vars; + if (code.fun_ref->has_mutate_params()) { + for (const LocalVarData& p_sym: code.fun_ref->parameters) { + if (p_sym.is_mutate_parameter()) { + mutated_vars.insert(mutated_vars.end(), p_sym.ir_idx.begin(), p_sym.ir_idx.end()); + } + } + } + code.emplace_back(loc_end, Op::_Return, std::move(mutated_vars)); +} + + +void process_any_statement(AnyV v, CodeBlob& code) { + switch (v->type) { + case ast_sequence: + return process_sequence(v->as(), code); + case ast_return_statement: + return process_return_statement(v->as(), code); + case ast_repeat_statement: + return process_repeat_statement(v->as(), code); + case ast_if_statement: + return process_if_statement(v->as(), code); + case ast_do_while_statement: + return process_do_while_statement(v->as(), code); + case ast_while_statement: + return process_while_statement(v->as(), code); + case ast_throw_statement: + return process_throw_statement(v->as(), code); + case ast_assert_statement: + return process_assert_statement(v->as(), code); + case ast_try_catch_statement: + return process_try_catch_statement(v->as(), code); + case ast_empty_statement: + return; + default: + pre_compile_expr(reinterpret_cast(v), code, nullptr); + } +} + +static void convert_function_body_to_CodeBlob(FunctionPtr fun_ref, FunctionBodyCode* code_body) { + auto v_body = fun_ref->ast_root->as()->get_body()->as(); + CodeBlob* blob = new CodeBlob{fun_ref->name, fun_ref->loc, fun_ref}; + + std::vector rvect_import; + int total_arg_width = 0; + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + total_arg_width += fun_ref->parameters[i].declared_type->get_width_on_stack(); + } + rvect_import.reserve(total_arg_width); + + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + const LocalVarData& param_i = fun_ref->parameters[i]; + std::vector ir_idx = blob->create_var(param_i.declared_type, param_i.loc, param_i.name); + rvect_import.insert(rvect_import.end(), ir_idx.begin(), ir_idx.end()); + param_i.mutate()->assign_ir_idx(std::move(ir_idx)); + } + blob->emplace_back(fun_ref->loc, Op::_Import, rvect_import); + blob->in_var_cnt = blob->var_cnt; + tolk_assert(blob->var_cnt == total_arg_width); + + for (AnyV item : v_body->get_items()) { + process_any_statement(item, *blob); + } + append_implicit_return_statement(v_body->loc_end, *blob); + + blob->close_blk(v_body->loc_end); + code_body->set_code(blob); + tolk_assert(vars_modification_watcher.empty()); +} + +static void convert_asm_body_to_AsmOp(FunctionPtr fun_ref, FunctionBodyAsm* asm_body) { + int cnt = fun_ref->get_num_params(); + int width = fun_ref->inferred_return_type->get_width_on_stack(); + std::vector asm_ops; + for (AnyV v_child : fun_ref->ast_root->as()->get_body()->as()->get_asm_commands()) { + std::string_view ops = v_child->as()->str_val; // \n\n... + std::string op; + for (char c : ops) { + if (c == '\n' || c == '\r') { + if (!op.empty()) { + asm_ops.push_back(AsmOp::Parse(op, cnt, width)); + if (asm_ops.back().is_custom()) { + cnt = width; + } + op.clear(); + } + } else { + op.push_back(c); + } + } + if (!op.empty()) { + asm_ops.push_back(AsmOp::Parse(op, cnt, width)); + if (asm_ops.back().is_custom()) { + cnt = width; + } + } + } + + asm_body->set_code(std::move(asm_ops)); +} + +class UpdateArgRetOrderConsideringStackWidth final { +public: + static bool should_visit_function(FunctionPtr fun_ref) { + return !fun_ref->is_generic_function() && (!fun_ref->ret_order.empty() || !fun_ref->arg_order.empty()); + } + + static void start_visiting_function(FunctionPtr fun_ref, V v_function) { + int total_arg_mutate_width = 0; + bool has_arg_width_not_1 = false; + for (const LocalVarData& param : fun_ref->parameters) { + int arg_width = param.declared_type->get_width_on_stack(); + has_arg_width_not_1 |= arg_width != 1; + total_arg_mutate_width += param.is_mutate_parameter() * arg_width; + } + + // example: `fun f(a: int, b: (int, (int, int)), c: int)` with `asm (b a c)` + // current arg_order is [1 0 2] + // needs to be converted to [1 2 3 0 4] because b width is 3 + if (has_arg_width_not_1) { + int total_arg_width = 0; + std::vector cum_arg_width; + cum_arg_width.reserve(1 + fun_ref->get_num_params()); + cum_arg_width.push_back(0); + for (const LocalVarData& param : fun_ref->parameters) { + cum_arg_width.push_back(total_arg_width += param.declared_type->get_width_on_stack()); + } + std::vector arg_order; + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + int j = fun_ref->arg_order[i]; + int c1 = cum_arg_width[j], c2 = cum_arg_width[j + 1]; + while (c1 < c2) { + arg_order.push_back(c1++); + } + } + fun_ref->mutate()->assign_arg_order(std::move(arg_order)); + } + + // example: `fun f(mutate self: slice): slice` with `asm(-> 1 0)` + // ret_order is a shuffled range 0...N + // validate N: a function should return value and mutated arguments onto a stack + if (!fun_ref->ret_order.empty()) { + size_t expected_width = fun_ref->inferred_return_type->get_width_on_stack() + total_arg_mutate_width; + if (expected_width != fun_ref->ret_order.size()) { + v_function->get_body()->error("ret_order (after ->) expected to contain " + std::to_string(expected_width) + " numbers"); + } + } + } +}; + +class ConvertASTToLegacyOpVisitor final { +public: + static bool should_visit_function(FunctionPtr fun_ref) { + return !fun_ref->is_generic_function(); + } + + static void start_visiting_function(FunctionPtr fun_ref, V) { + tolk_assert(fun_ref->is_type_inferring_done()); + if (fun_ref->is_code_function()) { + convert_function_body_to_CodeBlob(fun_ref, std::get(fun_ref->body)); + } else if (fun_ref->is_asm_function()) { + convert_asm_body_to_AsmOp(fun_ref, std::get(fun_ref->body)); + } + } +}; + +void pipeline_convert_ast_to_legacy_Expr_Op() { + visit_ast_of_all_functions(); + visit_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipe-calc-rvalue-lvalue.cpp b/tolk/pipe-calc-rvalue-lvalue.cpp new file mode 100644 index 00000000..1f374bc8 --- /dev/null +++ b/tolk/pipe-calc-rvalue-lvalue.cpp @@ -0,0 +1,229 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-visitor.h" + +/* + * This pipe assigns lvalue/rvalue flags for AST expressions. + * It happens after identifiers have been resolved, but before type inferring (before methods binding). + * + * Example: `a = b`, `a` is lvalue, `b` is rvalue. + * Example: `a + b`, both are rvalue. + * + * Note, that this pass only assigns, not checks. So, for `f() = 4`, expr `f()` is lvalue. + * Checking (firing this as incorrect later) is performed after type inferring, see pipe-check-rvalue-lvalue. + */ + +namespace tolk { + +enum class MarkingState { + None, + LValue, + RValue, + LValueAndRValue +}; + +class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { + MarkingState cur_state = MarkingState::None; + + MarkingState enter_state(MarkingState activated) { + MarkingState saved = cur_state; + cur_state = activated; + return saved; + } + + void restore_state(MarkingState saved) { + cur_state = saved; + } + + void mark_vertex_cur_or_rvalue(AnyExprV v) const { + if (cur_state == MarkingState::LValue || cur_state == MarkingState::LValueAndRValue) { + v->mutate()->assign_lvalue_true(); + } + if (cur_state == MarkingState::RValue || cur_state == MarkingState::LValueAndRValue || cur_state == MarkingState::None) { + v->mutate()->assign_rvalue_true(); + } + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + parent::visit(v); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + parent::visit(v); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + parent::visit(v); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + MarkingState saved = enter_state(v->passed_as_mutate ? MarkingState::LValueAndRValue : MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + parent::visit(v); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v->get_obj()); + restore_state(saved); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + // underscore is a placeholder to ignore left side of assignment: `(a, _) = get2params()` + // so, if current state is "lvalue", `_` will be marked as lvalue, and ok + // but if used incorrectly, like `f(_)` or just `_;`, it will be marked rvalue + // and will fire an error later, in pipe lvalue/rvalue check + mark_vertex_cur_or_rvalue(v); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + MarkingState saved = enter_state(MarkingState::LValue); + parent::visit(v->get_lhs()); + enter_state(MarkingState::RValue); + parent::visit(v->get_rhs()); + restore_state(saved); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + MarkingState saved = enter_state(MarkingState::LValueAndRValue); + parent::visit(v->get_lhs()); + enter_state(MarkingState::RValue); + parent::visit(v->get_rhs()); + restore_state(saved); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); // both cond, when_true and when_false are rvalue, `(cond ? a : b) = 5` prohibited + restore_state(saved); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + parent::visit(v->get_expr()); // leave lvalue state unchanged, for `mutate (t.0 as int)` both `t.0 as int` and `t.0` are lvalue + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + parent::visit(v->get_expr()); // leave lvalue state unchanged, for `mutate x!` both `x!` and `x` are lvalue + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v->get_expr()); + restore_state(saved); + } + + void visit(V v) override { + tolk_assert(cur_state == MarkingState::LValue); + mark_vertex_cur_or_rvalue(v); + parent::visit(v); + } + + void visit(V v) override { + tolk_assert(cur_state == MarkingState::LValue); + mark_vertex_cur_or_rvalue(v); + parent::visit(v); + } + + void visit(V v) override { + parent::visit(v->get_try_body()); + MarkingState saved = enter_state(MarkingState::LValue); + parent::visit(v->get_catch_expr()); + restore_state(saved); + parent::visit(v->get_catch_body()); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } +}; + +void pipeline_calculate_rvalue_lvalue() { + visit_ast_of_all_functions(); +} + +void pipeline_calculate_rvalue_lvalue(FunctionPtr fun_ref) { + CalculateRvalueLvalueVisitor visitor; + if (visitor.should_visit_function(fun_ref)) { + visitor.start_visiting_function(fun_ref, fun_ref->ast_root->as()); + } +} + +} // namespace tolk diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp new file mode 100644 index 00000000..bae67c5f --- /dev/null +++ b/tolk/pipe-check-inferred-types.cpp @@ -0,0 +1,586 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-visitor.h" +#include "type-system.h" + +namespace tolk { + +GNU_ATTRIBUTE_NOINLINE +static std::string to_string(TypePtr type) { + return "`" + type->as_human_readable() + "`"; +} + +GNU_ATTRIBUTE_NOINLINE +static std::string to_string(AnyExprV v_with_type) { + return "`" + v_with_type->inferred_type->as_human_readable() + "`"; +} + +GNU_ATTRIBUTE_NOINLINE +static std::string expression_as_string(AnyExprV v) { + if (auto v_ref = v->try_as()) { + if (v_ref->sym->try_as() || v_ref->sym->try_as()) { + return "variable `" + static_cast(v_ref->get_identifier()->name) + "`"; + } + } + if (auto v_par = v->try_as()) { + return expression_as_string(v_par->get_expr()); + } + return "expression"; +} + +// fire a general "type mismatch" error, just a wrapper over `throw` +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire(FunctionPtr cur_f, SrcLocation loc, const std::string& message) { + throw ParseError(cur_f, loc, message); +} + +// fire an error on `!cell` / `+slice` +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_cannot_apply_operator(FunctionPtr cur_f, SrcLocation loc, std::string_view operator_name, AnyExprV unary_expr) { + std::string op = static_cast(operator_name); + fire(cur_f, loc, "can not apply operator `" + op + "` to " + to_string(unary_expr->inferred_type)); +} + +// fire an error on `int + cell` / `slice & int` +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_cannot_apply_operator(FunctionPtr cur_f, SrcLocation loc, std::string_view operator_name, AnyExprV lhs, AnyExprV rhs) { + std::string op = static_cast(operator_name); + fire(cur_f, loc, "can not apply operator `" + op + "` to " + to_string(lhs->inferred_type) + " and " + to_string(rhs->inferred_type)); +} + +GNU_ATTRIBUTE_NOINLINE +static void warning_condition_always_true_or_false(FunctionPtr cur_f, SrcLocation loc, AnyExprV cond, const char* operator_name) { + loc.show_warning("condition of " + static_cast(operator_name) + " is always " + (cond->is_always_true ? "true" : "false")); +} + +// given `f(x: int)` and a call `f(expr)`, check that expr_type is assignable to `int` +static void check_function_argument_passed(FunctionPtr cur_f, TypePtr param_type, AnyExprV ith_arg, bool is_obj_of_dot_call) { + if (!param_type->can_rhs_be_assigned(ith_arg->inferred_type)) { + if (is_obj_of_dot_call) { + fire(cur_f, ith_arg->loc, "can not call method for " + to_string(param_type) + " with object of type " + to_string(ith_arg)); + } else { + fire(cur_f, ith_arg->loc, "can not pass " + to_string(ith_arg) + " to " + to_string(param_type)); + } + } +} + +// given `f(x: mutate int?)` and a call `f(expr)`, check that `int?` is assignable to expr_type +// (for instance, can't call `f(mutate intVal)`, since f can potentially assign null to it) +static void check_function_argument_mutate_back(FunctionPtr cur_f, TypePtr param_type, AnyExprV ith_arg, bool is_obj_of_dot_call) { + if (!ith_arg->inferred_type->can_rhs_be_assigned(param_type)) { + if (is_obj_of_dot_call) { + fire(cur_f, ith_arg->loc,"can not call method for mutate " + to_string(param_type) + " with object of type " + to_string(ith_arg) + ", because mutation is not type compatible"); + } else { + fire(cur_f, ith_arg->loc,"can not pass " + to_string(ith_arg) + " to mutate " + to_string(param_type) + ", because mutation is not type compatible"); + } + } +} + +// fire an error on `var n = null` +// technically it's correct, type of `n` is TypeDataNullLiteral, but it's not what the user wanted +// so, it's better to see an error on assignment, that later, on `n` usage and types mismatch +// (most common is situation above, but generally, `var (x,n) = xn` where xn is a tensor with 2-nd always-null, can be) +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_assign_always_null_to_variable(FunctionPtr cur_f, SrcLocation loc, LocalVarPtr assigned_var, bool is_assigned_null_literal) { + std::string var_name = assigned_var->name; + fire(cur_f, loc, "can not infer type of `" + var_name + "`, it's always null; specify its type with `" + var_name + ": `" + (is_assigned_null_literal ? " or use `null as `" : "")); +} + +// fire an error on `untypedTupleVar.0` when inferred as (int,int), or `[int, (int,int)]`, or other non-1 width in a tuple +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_cannot_put_non1_stack_width_arg_to_tuple(FunctionPtr cur_f, SrcLocation loc, TypePtr inferred_type) { + fire(cur_f, loc, "a tuple can not have " + to_string(inferred_type) + " inside, because it occupies " + std::to_string(inferred_type->get_width_on_stack()) + " stack slots in TVM, not 1"); +} + +// handle __expect_type(expr, "type") call +// this is used in compiler tests +GNU_ATTRIBUTE_NOINLINE GNU_ATTRIBUTE_COLD +static void handle_possible_compiler_internal_call(FunctionPtr cur_f, V v) { + FunctionPtr fun_ref = v->fun_maybe; + tolk_assert(fun_ref && fun_ref->is_builtin_function()); + + if (fun_ref->name == "__expect_type") { + tolk_assert(v->get_num_args() == 2); + TypePtr expected_type = parse_type_from_string(v->get_arg(1)->get_expr()->as()->str_val); + TypePtr expr_type = v->get_arg(0)->inferred_type; + if (expected_type != expr_type) { + fire(cur_f, v->loc, "__expect_type failed: expected " + to_string(expected_type) + ", got " + to_string(expr_type)); + } + } +} + +static bool expect_integer(AnyExprV v_inferred) { + return v_inferred->inferred_type == TypeDataInt::create(); +} + +static bool expect_boolean(AnyExprV v_inferred) { + return v_inferred->inferred_type == TypeDataBool::create(); +} + + +class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { + FunctionPtr cur_f = nullptr; // may be nullptr if checking `const a = ...` init_value + +protected: + void visit(V v) override { + AnyExprV lhs = v->get_lhs(); + AnyExprV rhs = v->get_rhs(); + parent::visit(lhs); + parent::visit(rhs); + + // all operators (+=, etc.) can work for integers (if both sides are integers) + bool types_ok = expect_integer(lhs) && expect_integer(rhs); + // bitwise operators &= |= ^= are "overloaded" for booleans also (if both sides are booleans) + if (!types_ok && (v->tok == tok_set_bitwise_and || v->tok == tok_set_bitwise_or || v->tok == tok_set_bitwise_xor)) { + types_ok = expect_boolean(lhs) && expect_boolean(rhs); + } + // using += for other types (e.g. `tensorVar += tensorVar`) is not allowed + if (!types_ok) { + fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); + } + } + + void visit(V v) override { + AnyExprV rhs = v->get_rhs(); + parent::visit(rhs); + + switch (v->tok) { + case tok_logical_not: + if (!expect_integer(rhs) && !expect_boolean(rhs)) { + fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, rhs); + } + break; + default: + if (!expect_integer(rhs)) { + fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, rhs); + } + } + } + + void visit(V v) override { + AnyExprV lhs = v->get_lhs(); + AnyExprV rhs = v->get_rhs(); + parent::visit(lhs); + parent::visit(rhs); + + switch (v->tok) { + // == != can compare both integers and booleans, (int == bool) is NOT allowed + // note, that `int?` and `int?` can't be compared, since Fift `EQUAL` works with integers only + // (if to allow `int?` in the future, `==` must be expressed in a complicated Fift code considering TVM NULL) + case tok_eq: + case tok_neq: { + bool both_int = expect_integer(lhs) && expect_integer(rhs); + bool both_bool = expect_boolean(lhs) && expect_boolean(rhs); + if (!both_int && !both_bool) { + if (lhs->inferred_type == rhs->inferred_type) { // compare slice with slice, int? with int? + fire(cur_f, v->loc, "type " + to_string(lhs) + " can not be compared with `== !=`"); + } else { + fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); + } + } + break; + } + // < > can compare only strict integers + case tok_lt: + case tok_gt: + case tok_leq: + case tok_geq: + case tok_spaceship: + if (!expect_integer(lhs) || !expect_integer(rhs)) { + fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); + } + break; + // & | ^ are "overloaded" both for integers and booleans, (int & bool) is NOT allowed + case tok_bitwise_and: + case tok_bitwise_or: + case tok_bitwise_xor: { + bool both_int = expect_integer(lhs) && expect_integer(rhs); + bool both_bool = expect_boolean(lhs) && expect_boolean(rhs); + if (!both_int && !both_bool) { + fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); + } + break; + } + // && || can work with integers and booleans, (int && bool) is allowed, (int16 && int32) also + case tok_logical_and: + case tok_logical_or: { + bool lhs_ok = expect_integer(lhs) || expect_boolean(lhs); + bool rhs_ok = expect_integer(rhs) || expect_boolean(rhs); + if (!lhs_ok || !rhs_ok) { + fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); + } + break; + } + // others are mathematical: + * ... + default: + if (!expect_integer(lhs) || !expect_integer(rhs)) { + fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); + } + } + } + + void visit(V v) override { + parent::visit(v->get_expr()); + + if (!v->get_expr()->inferred_type->can_be_casted_with_as_operator(v->cast_to_type)) { + fire(cur_f, v->loc, "type " + to_string(v->get_expr()) + " can not be cast to " + to_string(v->cast_to_type)); + } + } + + void visit(V v) override { + parent::visit(v->get_expr()); + + if (v->get_expr()->inferred_type == TypeDataNullLiteral::create()) { + // operator `!` used for always-null (proven by smart casts, for example), it's an error + fire(cur_f, v->loc, "operator `!` used for always null expression"); + } + // if operator `!` used for non-nullable, probably a warning should be printed + } + + void visit(V v) override { + parent::visit(v->get_expr()); + + if ((v->is_always_true && !v->is_negated) || (v->is_always_false && v->is_negated)) { + v->loc.show_warning(expression_as_string(v->get_expr()) + " is always null, this condition is always " + (v->is_always_true ? "true" : "false")); + } + if ((v->is_always_false && !v->is_negated) || (v->is_always_true && v->is_negated)) { + v->loc.show_warning(expression_as_string(v->get_expr()) + " of type " + to_string(v->get_expr()) + " is always not null, this condition is always " + (v->is_always_true ? "true" : "false")); + } + } + + void visit(V v) override { + parent::visit(v); + + for (int i = 0; i < v->size(); ++i) { + AnyExprV item = v->get_item(i); + if (item->inferred_type->get_width_on_stack() != 1) { + fire_error_cannot_put_non1_stack_width_arg_to_tuple(cur_f, v->get_item(i)->loc, item->inferred_type); + } + } + } + + void visit(V v) override { + parent::visit(v); + + TypePtr obj_type = v->get_obj()->inferred_type; + if (v->is_target_indexed_access()) { + if (obj_type->try_as() && v->inferred_type->get_width_on_stack() != 1) { + fire_error_cannot_put_non1_stack_width_arg_to_tuple(cur_f, v->loc, v->inferred_type); + } + } + } + + void visit(V v) override { + parent::visit(v); // check against type mismatch inside nested arguments + + FunctionPtr fun_ref = v->fun_maybe; + if (!fun_ref) { + // `local_var(args)` and similar + const TypeDataFunCallable* f_callable = v->get_callee()->inferred_type->try_as(); + tolk_assert(f_callable && f_callable->params_size() == v->get_num_args()); + for (int i = 0; i < v->get_num_args(); ++i) { + auto arg_i = v->get_arg(i)->get_expr(); + TypePtr param_type = f_callable->params_types[i]; + if (!param_type->can_rhs_be_assigned(arg_i->inferred_type)) { + fire(cur_f, arg_i->loc, "can not pass " + to_string(arg_i) + " to " + to_string(param_type)); + } + } + return; + } + + // so, we have a call `f(args)` or `obj.f(args)`, f is a global function (fun_ref) (code / asm / builtin) + int delta_self = 0; + AnyExprV dot_obj = nullptr; + if (auto v_dot = v->get_callee()->try_as()) { + delta_self = 1; + dot_obj = v_dot->get_obj(); + } + + if (dot_obj) { + const LocalVarData& param_0 = fun_ref->parameters[0]; + TypePtr param_type = param_0.declared_type; + check_function_argument_passed(cur_f, param_type, dot_obj, true); + if (param_0.is_mutate_parameter()) { + check_function_argument_mutate_back(cur_f, param_type, dot_obj, true); + } + } + for (int i = 0; i < v->get_num_args(); ++i) { + const LocalVarData& param_i = fun_ref->parameters[delta_self + i]; + AnyExprV arg_i = v->get_arg(i)->get_expr(); + TypePtr param_type = param_i.declared_type; + check_function_argument_passed(cur_f, param_type, arg_i, false); + if (param_i.is_mutate_parameter()) { + check_function_argument_mutate_back(cur_f, param_type, arg_i, false); + } + } + + if (fun_ref->is_builtin_function() && fun_ref->name[0] == '_') { + handle_possible_compiler_internal_call(cur_f, v); + } + } + + void visit(V v) override { + parent::visit(v->get_lhs()); + parent::visit(v->get_rhs()); + + process_assignment_lhs(v->get_lhs(), v->get_rhs()->inferred_type, v->get_rhs()); + } + + // handle (and dig recursively) into `var lhs = rhs` + // examples: `var z = 5`, `var (x, [y]) = (2, [3])`, `var (x, [y]) = xy` + // while recursing, keep track of rhs if lhs and rhs have common shape (5 for z, 2 for x, [3] for [y], 3 for y) + // (so that on type mismatch, point to corresponding rhs, example: `var (x, y:slice) = (1, 2)` point to 2 + void process_assignment_lhs(AnyExprV lhs, TypePtr rhs_type, AnyExprV corresponding_maybe_rhs) { + AnyExprV err_loc = corresponding_maybe_rhs ? corresponding_maybe_rhs : lhs; + + // `var ... = rhs` - dig into left part + if (auto lhs_decl = lhs->try_as()) { + process_assignment_lhs(lhs_decl->get_expr(), rhs_type, corresponding_maybe_rhs); + return; + } + + // inside `var v: int = rhs` / `var _ = rhs` / `var v redef = rhs` (lhs is "v" / "_" / "v") + if (auto lhs_var = lhs->try_as()) { + TypePtr declared_type = lhs_var->declared_type; // `var v: int = rhs` (otherwise, nullptr) + if (lhs_var->marked_as_redef) { + tolk_assert(lhs_var->var_ref && lhs_var->var_ref->declared_type); + declared_type = lhs_var->var_ref->declared_type; + } + if (declared_type) { + if (!declared_type->can_rhs_be_assigned(rhs_type)) { + fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to variable of type " + to_string(declared_type)); + } + } else { + if (rhs_type == TypeDataNullLiteral::create()) { + fire_error_assign_always_null_to_variable(cur_f, err_loc->loc, lhs_var->var_ref->try_as(), corresponding_maybe_rhs && corresponding_maybe_rhs->type == ast_null_keyword); + } + } + return; + } + + // `(v1, v2) = rhs` / `var (v1, v2) = rhs` (rhs may be `(1,2)` or `tensorVar` or `someF()`, doesn't matter) + // dig recursively into v1 and v2 with corresponding rhs i-th item of a tensor + if (auto lhs_tensor = lhs->try_as()) { + const TypeDataTensor* rhs_type_tensor = rhs_type->try_as(); + if (!rhs_type_tensor) { + fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to a tensor"); + } + if (lhs_tensor->size() != rhs_type_tensor->size()) { + fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + ", sizes mismatch"); + } + V rhs_tensor_maybe = corresponding_maybe_rhs ? corresponding_maybe_rhs->try_as() : nullptr; + for (int i = 0; i < lhs_tensor->size(); ++i) { + process_assignment_lhs(lhs_tensor->get_item(i), rhs_type_tensor->items[i], rhs_tensor_maybe ? rhs_tensor_maybe->get_item(i) : nullptr); + } + return; + } + + // `[v1, v2] = rhs` / `var [v1, v2] = rhs` (rhs may be `[1,2]` or `tupleVar` or `someF()`, doesn't matter) + // dig recursively into v1 and v2 with corresponding rhs i-th item of a tuple + if (auto lhs_tuple = lhs->try_as()) { + const TypeDataTypedTuple* rhs_type_tuple = rhs_type->try_as(); + if (!rhs_type_tuple) { + fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to a tuple"); + } + if (lhs_tuple->size() != rhs_type_tuple->size()) { + fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + ", sizes mismatch"); + } + V rhs_tuple_maybe = corresponding_maybe_rhs ? corresponding_maybe_rhs->try_as() : nullptr; + for (int i = 0; i < lhs_tuple->size(); ++i) { + process_assignment_lhs(lhs_tuple->get_item(i), rhs_type_tuple->items[i], rhs_tuple_maybe ? rhs_tuple_maybe->get_item(i) : nullptr); + } + return; + } + + // check `untypedTuple.0 = rhs_tensor` and other non-1 width elements + if (auto lhs_dot = lhs->try_as()) { + if (lhs_dot->is_target_indexed_access() && lhs_dot->get_obj()->inferred_type == TypeDataTuple::create()) { + if (rhs_type->get_width_on_stack() != 1) { + fire_error_cannot_put_non1_stack_width_arg_to_tuple(cur_f, err_loc->loc, rhs_type); + } + } + } + + // here is `v = rhs` (just assignment, not `var v = rhs`) / `a.0 = rhs` / `getObj(z=f()).0 = rhs` etc. + // types were already inferred, so just check their compatibility + // for strange lhs like `f() = rhs` type checking will pass, but will fail lvalue check later + if (!lhs->inferred_type->can_rhs_be_assigned(rhs_type)) { + if (lhs->try_as()) { + fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to variable of type " + to_string(lhs)); + } else { + fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to " + to_string(lhs)); + } + } + } + + void visit(V v) override { + parent::visit(v->get_return_value()); + + if (cur_f->does_return_self()) { + if (!is_expr_valid_as_return_self(v->get_return_value())) { + fire(cur_f, v->loc, "invalid return from `self` function"); + } + return; + } + + TypePtr expr_type = v->get_return_value()->inferred_type; + if (!cur_f->inferred_return_type->can_rhs_be_assigned(expr_type)) { + fire(cur_f, v->get_return_value()->loc, "can not convert type " + to_string(expr_type) + " to return type " + to_string(cur_f->inferred_return_type)); + } + } + + static bool is_expr_valid_as_return_self(AnyExprV return_expr) { + // `return self` + if (return_expr->type == ast_reference && return_expr->as()->get_name() == "self") { + return true; + } + // `return self.someMethod()` + if (auto v_call = return_expr->try_as(); v_call && v_call->is_dot_call()) { + return v_call->fun_maybe && v_call->fun_maybe->does_return_self() && is_expr_valid_as_return_self(v_call->get_dot_obj()); + } + // `return cond ? ... : ...` + if (auto v_ternary = return_expr->try_as()) { + return is_expr_valid_as_return_self(v_ternary->get_when_true()) && is_expr_valid_as_return_self(v_ternary->get_when_false()); + } + return false; + } + + void visit(V v) override { + parent::visit(v); + + AnyExprV cond = v->get_cond(); + if (!expect_integer(cond) && !expect_boolean(cond)) { + fire(cur_f, cond->loc, "can not use " + to_string(cond) + " as a boolean condition"); + } + + if (cond->is_always_true || cond->is_always_false) { + warning_condition_always_true_or_false(cur_f, v->loc, cond, "ternary operator"); + } + } + + void visit(V v) override { + parent::visit(v); + + AnyExprV cond = v->get_cond(); + if (!expect_integer(cond) && !expect_boolean(cond)) { + fire(cur_f, cond->loc, "can not use " + to_string(cond) + " as a boolean condition"); + } + + if (cond->is_always_true || cond->is_always_false) { + warning_condition_always_true_or_false(cur_f, v->loc, cond, "`if`"); + } + } + + void visit(V v) override { + parent::visit(v); + + AnyExprV cond = v->get_cond(); + if (!expect_integer(cond)) { + fire(cur_f, cond->loc, "condition of `repeat` must be an integer, got " + to_string(cond)); + } + } + + void visit(V v) override { + parent::visit(v); + + AnyExprV cond = v->get_cond(); + if (!expect_integer(cond) && !expect_boolean(cond)) { + fire(cur_f, cond->loc, "can not use " + to_string(cond) + " as a boolean condition"); + } + + if (cond->is_always_true || cond->is_always_false) { + warning_condition_always_true_or_false(cur_f, v->loc, cond, "`while`"); + } + } + + void visit(V v) override { + parent::visit(v); + + AnyExprV cond = v->get_cond(); + if (!expect_integer(cond) && !expect_boolean(cond)) { + fire(cur_f, cond->loc, "can not use " + to_string(cond) + " as a boolean condition"); + } + + if (cond->is_always_true || cond->is_always_false) { + warning_condition_always_true_or_false(cur_f, v->loc, cond, "`do while`"); + } + } + + void visit(V v) override { + parent::visit(v); + + if (!expect_integer(v->get_thrown_code())) { + fire(cur_f, v->get_thrown_code()->loc, "excNo of `throw` must be an integer, got " + to_string(v->get_thrown_code())); + } + if (v->has_thrown_arg() && v->get_thrown_arg()->inferred_type->get_width_on_stack() != 1) { + fire(cur_f, v->get_thrown_arg()->loc, "can not throw " + to_string(v->get_thrown_arg()) + ", exception arg must occupy exactly 1 stack slot"); + } + } + + void visit(V v) override { + parent::visit(v); + + AnyExprV cond = v->get_cond(); + if (!expect_integer(cond) && !expect_boolean(cond)) { + fire(cur_f, cond->loc, "can not use " + to_string(cond) + " as a boolean condition"); + } + if (!expect_integer(v->get_thrown_code())) { + fire(cur_f, v->get_thrown_code()->loc, "thrown excNo of `assert` must be an integer, got " + to_string(v->get_thrown_code())); + } + + if (cond->is_always_true || cond->is_always_false) { + warning_condition_always_true_or_false(cur_f, v->loc, cond, "`assert`"); + } + } + + void visit(V v) override { + parent::visit(v); + + if (v->first_unreachable) { + // it's essential to print "unreachable code" warning AFTER type checking + // (printing it while inferring might be a false positive if types are incorrect, due to smart casts for example) + // a more correct approach would be to access cfg here somehow, but since cfg is now available only while inferring, + // a special v->first_unreachable was set specifically for this warning (again, which is correct if types match) + v->first_unreachable->loc.show_warning("unreachable code"); + } + } + + public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + cur_f = nullptr; + + if (fun_ref->is_implicit_return() && fun_ref->declared_return_type) { + if (!fun_ref->declared_return_type->can_rhs_be_assigned(TypeDataVoid::create()) || fun_ref->does_return_self()) { + fire(fun_ref, v_function->get_body()->as()->loc_end, "missing return"); + } + } + } +}; + +void pipeline_check_inferred_types() { + visit_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipe-check-pure-impure.cpp b/tolk/pipe-check-pure-impure.cpp new file mode 100644 index 00000000..366ff160 --- /dev/null +++ b/tolk/pipe-check-pure-impure.cpp @@ -0,0 +1,93 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-visitor.h" +#include "platform-utils.h" + +/* + * This pipe checks for impure operations inside pure functions. + * It happens after type inferring (after methods binding) since it operates fun_ref of calls. + */ + +namespace tolk { + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_impure_operation_inside_pure_function(AnyV v) { + v->error("an impure operation in a pure function"); +} + +class CheckImpureOperationsInPureFunctionVisitor final : public ASTVisitorFunctionBody { + static void fire_if_global_var(AnyExprV v) { + if (auto v_ident = v->try_as()) { + if (v_ident->sym->try_as()) { + fire_error_impure_operation_inside_pure_function(v); + } + } + } + + void visit(V v) override { + fire_if_global_var(v->get_lhs()); + parent::visit(v); + } + + void visit(V v) override { + fire_if_global_var(v->get_lhs()); + parent::visit(v); + } + + void visit(V v) override { + // v is `globalF(args)` / `globalF(args)` / `obj.method(args)` / `local_var(args)` / `getF()(args)` + if (!v->fun_maybe) { + // `local_var(args)` is always impure, no considerations about what's there at runtime + fire_error_impure_operation_inside_pure_function(v); + } + + if (!v->fun_maybe->is_marked_as_pure()) { + fire_error_impure_operation_inside_pure_function(v); + } + + parent::visit(v); + } + + void visit(V v) override { + if (v->passed_as_mutate) { + fire_if_global_var(v->get_expr()); + } + + parent::visit(v); + } + + void visit(V v) override { + fire_error_impure_operation_inside_pure_function(v); + } + + void visit(V v) override { + fire_error_impure_operation_inside_pure_function(v); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function() && fun_ref->is_marked_as_pure(); + } +}; + +void pipeline_check_pure_impure_operations() { + visit_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipe-check-rvalue-lvalue.cpp b/tolk/pipe-check-rvalue-lvalue.cpp new file mode 100644 index 00000000..3ec47a16 --- /dev/null +++ b/tolk/pipe-check-rvalue-lvalue.cpp @@ -0,0 +1,210 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-visitor.h" +#include "platform-utils.h" + +/* + * This pipe checks lvalue/rvalue for validity. + * It happens after type inferring (after methods binding) and after lvalue/rvalue are refined based on fun_ref. + * + * Example: `f() = 4`, `f()` was earlier marked as lvalue, it's incorrect. + * Example: `f(mutate 5)`, `5` was marked also, it's incorrect. + */ + +namespace tolk { + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_cannot_be_used_as_lvalue(AnyV v, const std::string& details) { + // example: `f() = 32` + // example: `loadUint(c.beginParse(), 32)` (since `loadUint()` mutates the first argument) + v->error(details + " can not be used as lvalue"); +} + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_modifying_immutable_variable(AnyExprV v, LocalVarPtr var_ref) { + if (var_ref->param_idx == 0 && var_ref->name == "self") { + v->error("modifying `self`, which is immutable by default; probably, you want to declare `mutate self`"); + } else { + v->error("modifying immutable variable `" + var_ref->name + "`"); + } +} + +// validate a function used as rvalue, like `var cb = f` +// it's not a generic function (ensured earlier at type inferring) and has some more restrictions +static void validate_function_used_as_noncall(AnyExprV v, FunctionPtr fun_ref) { + if (!fun_ref->arg_order.empty() || !fun_ref->ret_order.empty()) { + v->error("saving `" + fun_ref->name + "` into a variable will most likely lead to invalid usage, since it changes the order of variables on the stack"); + } + if (fun_ref->has_mutate_params()) { + v->error("saving `" + fun_ref->name + "` into a variable is impossible, since it has `mutate` parameters and thus can only be called directly"); + } +} + +class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(v, "assignment"); + } + parent::visit(v); + } + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(v, "assignment"); + } + parent::visit(v); + } + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(v, "operator " + static_cast(v->operator_name)); + } + parent::visit(v); + } + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(v, "operator " + static_cast(v->operator_name)); + } + parent::visit(v); + } + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(v, "operator ?:"); + } + parent::visit(v); + } + + void visit(V v) override { + // if `x as int` is lvalue, then `x` is also lvalue, so check that `x` is ok + parent::visit(v->get_expr()); + } + + void visit(V v) override { + // if `x!` is lvalue, then `x` is also lvalue, so check that `x` is ok + parent::visit(v->get_expr()); + } + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(v, v->is_negated ? "operator !=" : "operator =="); + } + parent::visit(v->get_expr()); + } + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(v, "literal"); + } + } + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(v, "literal"); + } + } + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(v, "literal"); + } + } + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(v, "literal"); + } + } + + void visit(V v) override { + // a reference to a method used as rvalue, like `var v = t.tupleAt` + if (v->is_rvalue && v->is_target_fun_ref()) { + validate_function_used_as_noncall(v, std::get(v->target)); + } + } + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(v, "function call"); + } + if (!v->fun_maybe) { + parent::visit(v->get_callee()); + } + // for `f()` don't visit ast_reference `f`, to detect `f` usage as non-call, like `var cb = f` + // same for `obj.method()`, don't visit ast_reference method, visit only obj + if (v->is_dot_call()) { + parent::visit(v->get_dot_obj()); + } + + for (int i = 0; i < v->get_num_args(); ++i) { + parent::visit(v->get_arg(i)); + } + } + + void visit(V v) override { + if (v->marked_as_redef) { + tolk_assert(v->var_ref); + if (v->var_ref->is_immutable()) { + v->error("`redef` for immutable variable"); + } + } + } + + void visit(V v) override { + if (v->is_lvalue) { + tolk_assert(v->sym); + if (LocalVarPtr var_ref = v->sym->try_as(); var_ref && var_ref->is_immutable()) { + fire_error_modifying_immutable_variable(v, var_ref); + } else if (v->sym->try_as()) { + v->error("modifying immutable constant"); + } else if (v->sym->try_as()) { + v->error("function can't be used as lvalue"); + } + } + + // a reference to a function used as rvalue, like `var v = someFunction` + if (FunctionPtr fun_ref = v->sym->try_as(); fun_ref && v->is_rvalue) { + validate_function_used_as_noncall(v, fun_ref); + } + } + + void visit(V v) override { + if (v->is_rvalue) { + v->error("`_` can't be used as a value; it's a placeholder for a left side of assignment"); + } + } + + void visit(V v) override { + parent::visit(v->get_try_body()); + // skip catch(_,excNo), there are always vars due to grammar, lvalue/rvalue aren't set to them + parent::visit(v->get_catch_body()); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } +}; + +void pipeline_check_rvalue_lvalue() { + visit_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp new file mode 100644 index 00000000..9c27029b --- /dev/null +++ b/tolk/pipe-constant-folding.cpp @@ -0,0 +1,112 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-replacer.h" +#include "type-system.h" + +/* + * This pipe is supposed to do constant folding, like replacing `2 + 3` with `5`. + * It happens after type inferring and validity checks, one of the last ones. + * + * Currently, it just replaces `-1` (ast_unary_operator ast_int_const) with a number -1 + * and `!true` with false. + * Also, all parenthesized `((expr))` are replaced with `expr`, it's a constant transformation. + * (not to handle parenthesized in optimization passes, like `((x)) == true`) + * More rich constant folding should be done some day, but even without this, IR optimizations + * (operating low-level stack variables) pretty manage to do all related optimizations. + * Constant folding in the future, done at AST level, just would slightly reduce amount of work for optimizer. + */ + +namespace tolk { + +class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { + static V create_int_const(SrcLocation loc, td::RefInt256&& intval) { + auto v_int = createV(loc, std::move(intval), {}); + v_int->assign_inferred_type(TypeDataInt::create()); + v_int->assign_rvalue_true(); + return v_int; + } + + static V create_bool_const(SrcLocation loc, bool bool_val) { + auto v_bool = createV(loc, bool_val); + v_bool->assign_inferred_type(TypeDataBool::create()); + v_bool->assign_rvalue_true(); + return v_bool; + } + + AnyExprV replace(V v) override { + AnyExprV inner = parent::replace(v->get_expr()); + if (v->is_lvalue) { + inner->mutate()->assign_lvalue_true(); + } + return inner; + } + + AnyExprV replace(V v) override { + parent::replace(v); + + TokenType t = v->tok; + // convert "-1" (tok_minus tok_int_const) to a const -1 + if (t == tok_minus && v->get_rhs()->type == ast_int_const) { + td::RefInt256 intval = v->get_rhs()->as()->intval; + tolk_assert(!intval.is_null()); + intval = -intval; + if (intval.is_null() || !intval->signed_fits_bits(257)) { + v->error("integer overflow"); + } + return create_int_const(v->loc, std::move(intval)); + } + // same for "+1" + if (t == tok_plus && v->get_rhs()->type == ast_int_const) { + return v->get_rhs(); + } + + // `!true` / `!false` + if (t == tok_logical_not && v->get_rhs()->type == ast_bool_const) { + return create_bool_const(v->loc, !v->get_rhs()->as()->bool_val); + } + // `!0` + if (t == tok_logical_not && v->get_rhs()->type == ast_int_const) { + return create_bool_const(v->loc, v->get_rhs()->as()->intval == 0); + } + + return v; + } + + AnyExprV replace(V v) override { + parent::replace(v); + + // `null == null` / `null != null` + if (v->get_expr()->type == ast_null_keyword) { + return create_bool_const(v->loc, !v->is_negated); + } + + return v; + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } +}; + +void pipeline_constant_folding() { + replace_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipe-discover-parse-sources.cpp b/tolk/pipe-discover-parse-sources.cpp new file mode 100644 index 00000000..d31348ba --- /dev/null +++ b/tolk/pipe-discover-parse-sources.cpp @@ -0,0 +1,70 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-from-tokens.h" +#include "compiler-state.h" + +/* + * This is the starting point of compilation pipeline. + * It parses Tolk files to AST, analyzes `import` statements and loads/parses imported files. + * + * When it finishes, all files have been parsed to AST, and no more files will later be added. + * If a parsing error happens (invalid syntax), an exception is thrown immediately from ast-from-tokens.cpp. + */ + +namespace tolk { + +void pipeline_discover_and_parse_sources(const std::string& stdlib_filename, const std::string& entrypoint_filename) { + G.all_src_files.locate_and_register_source_file(stdlib_filename, {}); + G.all_src_files.locate_and_register_source_file(entrypoint_filename, {}); + + while (SrcFile* file = G.all_src_files.get_next_unparsed_file()) { + tolk_assert(!file->ast); + + file->ast = parse_src_file_to_ast(file); + // if (!file->is_stdlib_file()) file->ast->debug_print(); + + for (AnyV v_toplevel : file->ast->as()->get_toplevel_declarations()) { + if (auto v_import = v_toplevel->try_as()) { + std::string imported_str = v_import->get_file_name(); + size_t cur_slash_pos = file->rel_filename.rfind('/'); + std::string rel_filename = cur_slash_pos == std::string::npos || imported_str[0] == '@' + ? std::move(imported_str) + : file->rel_filename.substr(0, cur_slash_pos + 1) + imported_str; + + const SrcFile* imported = G.all_src_files.locate_and_register_source_file(rel_filename, v_import->loc); + file->imports.push_back(SrcFile::ImportDirective{imported}); + v_import->mutate()->assign_src_file(imported); + } + } + } + + // todo #ifdef TOLK_PROFILING + lexer_measure_performance(G.all_src_files); +} + +} // namespace tolk diff --git a/tolk/pipe-find-unused-symbols.cpp b/tolk/pipe-find-unused-symbols.cpp new file mode 100644 index 00000000..2b7e5557 --- /dev/null +++ b/tolk/pipe-find-unused-symbols.cpp @@ -0,0 +1,76 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#include "tolk.h" +#include "compiler-state.h" + +/* + * This pipe finds unused symbols (global functions and variables) to strip them off codegen. + * It happens after converting AST to Op, so it does not traverse AST. + * In the future, when control flow graph is introduced, this should be done at AST level. + */ + +namespace tolk { + +static void mark_function_used_dfs(const std::unique_ptr& op); + +static void mark_function_used(FunctionPtr fun_ref) { + if (!fun_ref->is_code_function() || fun_ref->is_really_used()) { // already handled + return; + } + + fun_ref->mutate()->assign_is_really_used(); + mark_function_used_dfs(std::get(fun_ref->body)->code->ops); +} + +static void mark_global_var_used(GlobalVarPtr glob_ref) { + glob_ref->mutate()->assign_is_really_used(); +} + +static void mark_function_used_dfs(const std::unique_ptr& op) { + if (!op) { + return; + } + + if (op->f_sym) { // for Op::_Call + mark_function_used(op->f_sym); + } + if (op->g_sym) { // for Op::_GlobVar + mark_global_var_used(op->g_sym); + } + mark_function_used_dfs(op->next); + mark_function_used_dfs(op->block0); + mark_function_used_dfs(op->block1); +} + +void pipeline_find_unused_symbols() { + for (FunctionPtr fun_ref : G.all_functions) { + if (fun_ref->is_method_id_not_empty()) { // get methods, main and other entrypoints, regular functions with @method_id + mark_function_used(fun_ref); + } + } +} + +} // namespace tolk diff --git a/tolk/pipe-generate-fif-output.cpp b/tolk/pipe-generate-fif-output.cpp new file mode 100644 index 00000000..57f481f0 --- /dev/null +++ b/tolk/pipe-generate-fif-output.cpp @@ -0,0 +1,170 @@ +/* + This file is part of TON Blockchain source code-> + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#include "tolk.h" +#include "src-file.h" +#include "ast.h" +#include "compiler-state.h" + +namespace tolk { + +void FunctionBodyCode::set_code(CodeBlob* code) { + this->code = code; +} + +void FunctionBodyAsm::set_code(std::vector&& code) { + this->ops = std::move(code); +} + + +static void generate_output_func(FunctionPtr fun_ref) { + tolk_assert(fun_ref->is_code_function()); + if (G.is_verbosity(2)) { + std::cerr << "\n\n=========================\nfunction " << fun_ref->name << " : " << fun_ref->inferred_return_type << std::endl; + } + + CodeBlob* code = std::get(fun_ref->body)->code; + if (G.is_verbosity(3)) { + code->print(std::cerr, 9); + } + code->prune_unreachable_code(); + if (G.is_verbosity(5)) { + std::cerr << "after prune_unreachable: \n"; + code->print(std::cerr, 0); + } + for (int i = 0; i < 8; i++) { + code->compute_used_code_vars(); + if (G.is_verbosity(4)) { + std::cerr << "after compute_used_vars: \n"; + code->print(std::cerr, 6); + } + code->fwd_analyze(); + if (G.is_verbosity(5)) { + std::cerr << "after fwd_analyze: \n"; + code->print(std::cerr, 6); + } + code->prune_unreachable_code(); + if (G.is_verbosity(5)) { + std::cerr << "after prune_unreachable: \n"; + code->print(std::cerr, 6); + } + } + code->mark_noreturn(); + if (G.is_verbosity(3)) { + code->print(std::cerr, 15); + } + if (G.is_verbosity(2)) { + std::cerr << "\n---------- resulting code for " << fun_ref->name << " -------------\n"; + } + const char* modifier = ""; + if (fun_ref->is_inline()) { + modifier = "INLINE"; + } else if (fun_ref->is_inline_ref()) { + modifier = "REF"; + } + std::cout << std::string(2, ' ') << fun_ref->name << " PROC" << modifier << ":<{\n"; + int mode = 0; + if (G.settings.stack_layout_comments) { + mode |= Stack::_StkCmt | Stack::_CptStkCmt; + } + if (fun_ref->is_inline() && code->ops->noreturn()) { + mode |= Stack::_InlineFunc; + } + if (fun_ref->is_inline() || fun_ref->is_inline_ref()) { + mode |= Stack::_InlineAny; + } + code->generate_code(std::cout, mode, 2); + std::cout << std::string(2, ' ') << "}>\n"; + if (G.is_verbosity(2)) { + std::cerr << "--------------\n"; + } +} + +void pipeline_generate_fif_output_to_std_cout() { + std::cout << "\"Asm.fif\" include\n"; + std::cout << "// automatically generated from "; + bool need_comma = false; + for (const SrcFile* file : G.all_src_files) { + if (!file->is_stdlib_file()) { + if (need_comma) { + std::cout << ", "; + } + std::cout << file->rel_filename; + need_comma = true; + } + } + std::cout << std::endl; + std::cout << "PROGRAM{\n"; + + bool has_main_procedure = false; + for (FunctionPtr fun_ref : G.all_functions) { + if (!fun_ref->does_need_codegen()) { + if (G.is_verbosity(2) && fun_ref->is_code_function()) { + std::cerr << fun_ref->name << ": code not generated, function does not need codegen\n"; + } + continue; + } + + if (fun_ref->is_entrypoint() && (fun_ref->name == "main" || fun_ref->name == "onInternalMessage")) { + has_main_procedure = true; + } + + std::cout << std::string(2, ' '); + if (fun_ref->is_method_id_not_empty()) { + std::cout << fun_ref->method_id << " DECLMETHOD " << fun_ref->name << "\n"; + } else { + std::cout << "DECLPROC " << fun_ref->name << "\n"; + } + } + + if (!has_main_procedure) { + throw Fatal("the contract has no entrypoint; forgot `fun onInternalMessage(...)`?"); + } + + for (GlobalVarPtr var_ref : G.all_global_vars) { + if (!var_ref->is_really_used() && G.settings.remove_unused_functions) { + if (G.is_verbosity(2)) { + std::cerr << var_ref->name << ": variable not generated, it's unused\n"; + } + continue; + } + + std::cout << std::string(2, ' ') << "DECLGLOBVAR " << var_ref->name << "\n"; + } + + for (FunctionPtr fun_ref : G.all_functions) { + if (!fun_ref->does_need_codegen()) { + continue; + } + generate_output_func(fun_ref); + } + + std::cout << "}END>c\n"; + if (!G.settings.boc_output_filename.empty()) { + std::cout << "boc>B \"" << G.settings.boc_output_filename << "\" B>file\n"; + } +} + +} // namespace tolk diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp new file mode 100644 index 00000000..5fb12059 --- /dev/null +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -0,0 +1,1301 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" +#include "src-file.h" +#include "ast.h" +#include "ast-visitor.h" +#include "generics-helpers.h" +#include "type-system.h" +#include "smart-casts-cfg.h" + +/* + * This is a complicated and crucial part of the pipeline. It simultaneously does the following: + * * infers types of all expressions; example: `2 + 3` both are TypeDataInt, result is also + * * AND binds function/method calls (assigns fun_ref); example: `globalF()`, fun_ref is assigned to `globalF` (unless generic) + * * AND instantiates generic functions; example: `t.tuplePush(2)` creates `tuplePush` and assigns fun_ref to dot field + * * AND infers return type of functions if it's omitted (`fun f() { ... }` means "auto infer", not "void") + * * AND builds data flow graph, mostly used for smart casts (right at the time of inferring) + * Note, that type checking (errors about types mismatch) is a later compilation step, due to loops. + * + * It's important to do all these parts simultaneously, they can't be split or separated. + * For example, we can't bind `f(2)` earlier, because if `f` is a generic `f`, we should instantiate it, + * and in order to do it, we need to know argument types. + * For example, we can't bind `c.cellHash()` earlier, because in order to bind it, we need to know object type. + * For example, we can't infer `var y = x` without smart casts, because if x's type is refined, it affects y. + * And vice versa, to infer type of expression in the middle, we need to have inferred all expressions preceding it, + * which may also include generics, etc. + * + * About generics. They are more like "C++ templates". If `f` and `f` called from somewhere, + * there will be TWO new functions, inserted into symtable, and both will be code generated to Fift. + * Body of a generic function is NOT analyzed. Hence, `fun f(v: T) { v.method(); }` we don't know + * whether `v.method()` is a valid call until instantiate it with `f` for example. + * Same for `v + 2`, we don't know whether + operator can be applied until instantiation. + * In other words, we have a closed type system, not open. + * That's why generic functions' bodies aren't traversed here (and in most following pipes). + * Instead, when an instantiated function is created, it follows all the preceding pipeline (registering symbols, etc.), + * and type inferring is done inside instantiated functions (which can recursively instantiate another, etc.). + * + * A noticeable part of inferring is "hints". + * Example: `var a: User = { id: 3, name: "" }`. To infer type of `{...}` we need to know it's `User`. This hint is taken from lhs. + * Example: `fun tupleAt(t: tuple, idx: int):T`, just `t.tupleGet(2)` can't be deduced (T left unspecified), + * but for assignment with left-defined type, or a call to `fInt(t.tupleGet(2))` hint "int" helps deduce T. + * + * Control flow is represented NOT as a "graph with edges". Instead, it's a "structured DFS" for the AST: + * 1) at every point of inferring, we have "current flow facts" (FlowContext) + * 2) when we see an `if (...)`, we create two derived contexts (by cloning current) + * 3) after `if`, finalize them at the end and unify + * 4) if we detect unreachable code, we mark that path's context as "unreachable" + * In other words, we get the effect of a CFG but in a more direct approach. That's enough for AST-level data-flow. + * FlowContext contains "data-flow facts that are definitely known". + * // current facts: x is int?, t is (int, int) + * if (x != null && t.0 > 0) + * // current facts: x is int, t is (int, int), t.0 is positive + * else + * // current facts: x is null, t is (int, int), t.0 is not positive + * When branches rejoin, facts are merged back (int+null = int? and so on, here they would be equal to before if). + * See smart-casts-cfg.cpp for detailed comments. + * + * About loops and partial re-entering. Consider the following: + * var x: int? = 5; + * // <- here x is `int` (smart cast) + * while (true) { + * // <- but here x is `int?` (not `int`) due to assignment in a loop + * if (...) { x = getNullableInt(); } + * } + * When building control flow, loops are inferred twice. In the above, at first iteration, x will be `int`, + * but at the second, x will be `int?` (after merged with loop end). + * That's why type checking is done later, not to make false errors on the first iteration. + * Note, that it would also be better to postpone generics "materialization" also: here only to infer type arguments, + * but to instantiate and re-assign fun_ref later. But it complicates the architecture significantly. + * For now, generics may encounter problems within loops on first iteration, though it's unlikely to face this + * in practice. (example: in the loop above, `genericFn(x)` will at first instantiate and then ) + * + * Unlike other pipes, inferring can dig recursively on demand. + * Example: + * fun getInt() { return 1; } + * fun main() { var i = getInt(); } + * If `main` is handled the first, it should know the return type if `getInt`. It's not declared, so we need + * to launch type inferring for `getInt` and then proceed back to `main`. + * When a generic function is instantiated, type inferring inside it is also run. + */ + +namespace tolk { + +static void infer_and_save_return_type_of_function(FunctionPtr fun_ref); + +static TypePtr get_or_infer_return_type(FunctionPtr fun_ref) { + if (!fun_ref->inferred_return_type) { + infer_and_save_return_type_of_function(fun_ref); + } + return fun_ref->inferred_return_type; +} + +GNU_ATTRIBUTE_NOINLINE +static std::string to_string(TypePtr type) { + return "`" + type->as_human_readable() + "`"; +} + +GNU_ATTRIBUTE_NOINLINE +static std::string to_string(AnyExprV v_with_type) { + return "`" + v_with_type->inferred_type->as_human_readable() + "`"; +} + +GNU_ATTRIBUTE_NOINLINE +static std::string to_string(FunctionPtr fun_ref) { + return "`" + fun_ref->as_human_readable() + "`"; +} + +// fire a general error, just a wrapper over `throw` +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire(FunctionPtr cur_f, SrcLocation loc, const std::string& message) { + throw ParseError(cur_f, loc, message); +} + +// fire an error when `fun f(...) asm ...` is called with T=(int,int) or other non-1 width on stack +// asm functions generally can't handle it, they expect T to be a TVM primitive +// (in FunC, `forall` type just couldn't be unified with non-primitives; in Tolk, generic T is expectedly inferred) +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_calling_asm_function_with_non1_stack_width_arg(FunctionPtr cur_f, SrcLocation loc, FunctionPtr fun_ref, const std::vector& substitutions, int arg_idx) { + fire(cur_f, loc, "can not call `" + fun_ref->as_human_readable() + "` with " + fun_ref->genericTs->get_nameT(arg_idx) + "=" + substitutions[arg_idx]->as_human_readable() + ", because it occupies " + std::to_string(substitutions[arg_idx]->get_width_on_stack()) + " stack slots in TVM, not 1"); +} + +// fire an error on `untypedTupleVar.0` when used without a hint +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_cannot_deduce_untyped_tuple_access(FunctionPtr cur_f, SrcLocation loc, int index) { + std::string idx_access = "." + std::to_string(index); + fire(cur_f, loc, "can not deduce type of `" + idx_access + "`; either assign it to variable like `var c: int = " + idx_access + "` or cast the result like `" + idx_access + " as int`"); +} + + +/* + * This class handles all types of AST vertices and traverses them, filling all AnyExprV::inferred_type. + * Note, that it isn't derived from ASTVisitor, it has manual `switch` over all existing vertex types. + * There are two reasons for this: + * 1) when a new AST node type is introduced, I want it to fail here, not to be left un-inferred with UB at next steps + * 2) easy to maintain a hint (see comments at the top of the file) + */ +class InferTypesAndCallsAndFieldsVisitor final { + FunctionPtr cur_f = nullptr; + std::vector return_statements; + + GNU_ATTRIBUTE_ALWAYS_INLINE + static void assign_inferred_type(AnyExprV dst, AnyExprV src) { +#ifdef TOLK_DEBUG + tolk_assert(src->inferred_type != nullptr && !src->inferred_type->has_unresolved_inside() && !src->inferred_type->has_genericT_inside()); +#endif + dst->mutate()->assign_inferred_type(src->inferred_type); + } + + GNU_ATTRIBUTE_ALWAYS_INLINE + static void assign_inferred_type(AnyExprV dst, TypePtr inferred_type) { +#ifdef TOLK_DEBUG + tolk_assert(inferred_type != nullptr && !inferred_type->has_unresolved_inside() && !inferred_type->has_genericT_inside()); +#endif + dst->mutate()->assign_inferred_type(inferred_type); + } + + static void assign_inferred_type(LocalVarPtr local_var_or_param, TypePtr inferred_type) { +#ifdef TOLK_DEBUG + tolk_assert(inferred_type != nullptr && !inferred_type->has_unresolved_inside() && !inferred_type->has_genericT_inside()); +#endif + local_var_or_param->mutate()->assign_inferred_type(inferred_type); + } + + static void assign_inferred_type(FunctionPtr fun_ref, TypePtr inferred_return_type, TypePtr inferred_full_type) { +#ifdef TOLK_DEBUG + tolk_assert(inferred_return_type != nullptr && !inferred_return_type->has_unresolved_inside() && !inferred_return_type->has_genericT_inside()); +#endif + fun_ref->mutate()->assign_inferred_type(inferred_return_type, inferred_full_type); + } + + // traverse children in any statement + FlowContext process_any_statement(AnyV v, FlowContext&& flow) { + switch (v->type) { + case ast_sequence: + return process_sequence(v->as(), std::move(flow)); + case ast_return_statement: + return process_return_statement(v->as(), std::move(flow)); + case ast_if_statement: + return process_if_statement(v->as(), std::move(flow)); + case ast_repeat_statement: + return process_repeat_statement(v->as(), std::move(flow)); + case ast_while_statement: + return process_while_statement(v->as(), std::move(flow)); + case ast_do_while_statement: + return process_do_while_statement(v->as(), std::move(flow)); + case ast_throw_statement: + return process_throw_statement(v->as(), std::move(flow)); + case ast_assert_statement: + return process_assert_statement(v->as(), std::move(flow)); + case ast_try_catch_statement: + return process_try_catch_statement(v->as(), std::move(flow)); + case ast_empty_statement: + return flow; + default: + return process_expression_statement(reinterpret_cast(v), std::move(flow)); + } + } + + // assigns inferred_type for any expression (by calling assign_inferred_type) + // returns ExprFlow: out_facts that are "definitely known" after evaluating the whole expression + // if used_as_condition, true_facts/false_facts are also calculated (don't calculate them always for optimization) + ExprFlow infer_any_expr(AnyExprV v, FlowContext&& flow, bool used_as_condition, TypePtr hint = nullptr) { + switch (v->type) { + case ast_int_const: + return infer_int_const(v->as(), std::move(flow), used_as_condition); + case ast_string_const: + return infer_string_const(v->as(), std::move(flow), used_as_condition); + case ast_bool_const: + return infer_bool_const(v->as(), std::move(flow), used_as_condition); + case ast_local_vars_declaration: + return infer_local_vars_declaration(v->as(), std::move(flow), used_as_condition); + case ast_local_var_lhs: + return infer_local_var_lhs(v->as(), std::move(flow), used_as_condition); + case ast_assign: + return infer_assignment(v->as(), std::move(flow), used_as_condition); + case ast_set_assign: + return infer_set_assign(v->as(), std::move(flow), used_as_condition); + case ast_unary_operator: + return infer_unary_operator(v->as(), std::move(flow), used_as_condition); + case ast_binary_operator: + return infer_binary_operator(v->as(), std::move(flow), used_as_condition); + case ast_ternary_operator: + return infer_ternary_operator(v->as(), std::move(flow), used_as_condition, hint); + case ast_cast_as_operator: + return infer_cast_as_operator(v->as(), std::move(flow), used_as_condition); + case ast_not_null_operator: + return infer_not_null_operator(v->as(), std::move(flow), used_as_condition); + case ast_is_null_check: + return infer_is_null_check(v->as(), std::move(flow), used_as_condition); + case ast_parenthesized_expression: + return infer_parenthesized(v->as(), std::move(flow), used_as_condition, hint); + case ast_reference: + return infer_reference(v->as(), std::move(flow), used_as_condition); + case ast_dot_access: + return infer_dot_access(v->as(), std::move(flow), used_as_condition, hint); + case ast_function_call: + return infer_function_call(v->as(), std::move(flow), used_as_condition, hint); + case ast_tensor: + return infer_tensor(v->as(), std::move(flow), used_as_condition, hint); + case ast_typed_tuple: + return infer_typed_tuple(v->as(), std::move(flow), used_as_condition, hint); + case ast_null_keyword: + return infer_null_keyword(v->as(), std::move(flow), used_as_condition); + case ast_underscore: + return infer_underscore(v->as(), std::move(flow), used_as_condition, hint); + case ast_empty_expression: + return infer_empty_expression(v->as(), std::move(flow), used_as_condition); + default: + throw UnexpectedASTNodeType(v, "infer_any_expr"); + } + } + + static ExprFlow infer_int_const(V v, FlowContext&& flow, bool used_as_condition) { + assign_inferred_type(v, TypeDataInt::create()); + + ExprFlow after_v(std::move(flow), used_as_condition); + if (used_as_condition) { // `if (0)` always false + if (v->intval == 0) { + after_v.true_flow.mark_unreachable(UnreachableKind::CantHappen); + } else { + after_v.false_flow.mark_unreachable(UnreachableKind::CantHappen); + } + } + return after_v; + } + + static ExprFlow infer_string_const(V v, FlowContext&& flow, bool used_as_condition) { + if (v->is_bitslice()) { + assign_inferred_type(v, TypeDataSlice::create()); + } else { + assign_inferred_type(v, TypeDataInt::create()); + } + + return ExprFlow(std::move(flow), used_as_condition); + } + + static ExprFlow infer_bool_const(V v, FlowContext&& flow, bool used_as_condition) { + assign_inferred_type(v, TypeDataBool::create()); + + ExprFlow after_v(std::move(flow), used_as_condition); + if (used_as_condition) { // `if (false)` always false + if (v->bool_val == false) { + after_v.true_flow.mark_unreachable(UnreachableKind::CantHappen); + } else { + after_v.false_flow.mark_unreachable(UnreachableKind::CantHappen); + } + } + return after_v; + } + + ExprFlow infer_local_vars_declaration(V v, FlowContext&& flow, bool used_as_condition) { + flow = infer_any_expr(v->get_expr(), std::move(flow), used_as_condition).out_flow; + assign_inferred_type(v, v->get_expr()); + return ExprFlow(std::move(flow), used_as_condition); + } + + static ExprFlow infer_local_var_lhs(V v, FlowContext&& flow, bool used_as_condition) { + // `var v = rhs`, inferring is called for `v` + // at the moment of inferring left side of assignment, we don't know type of rhs (since lhs is executed first) + // so, mark `v` as unknown + // later, v's inferred_type will be reassigned; see process_assignment_lhs_after_infer_rhs() + if (v->marked_as_redef) { + assign_inferred_type(v, v->var_ref->declared_type); + } else { + assign_inferred_type(v, v->declared_type ? v->declared_type : TypeDataUnknown::create()); + } + return ExprFlow(std::move(flow), used_as_condition); + } + + ExprFlow infer_assignment(V v, FlowContext&& flow, bool used_as_condition) { + // v is assignment: `x = 5` / `var x = 5` / `var x: slice = 5` / `(cs,_) = f()` / `val (a,[b],_) = (a,t,0)` + // execution flow is: lhs first, rhs second (at IR generation, also lhs is evaluated first, unlike FunC) + // after inferring lhs, use it for hint when inferring rhs + // example: `var i: int = t.tupleAt(0)` is ok (hint=int, T=int), but `var i = t.tupleAt(0)` not, since `tupleAt(t,i): T` + AnyExprV lhs = v->get_lhs(); + AnyExprV rhs = v->get_rhs(); + flow = infer_left_side_of_assignment(lhs, std::move(flow)); + flow = infer_any_expr(rhs, std::move(flow), false, lhs->inferred_type).out_flow; + process_assignment_lhs_after_infer_rhs(lhs, rhs->inferred_type, flow); + assign_inferred_type(v, rhs); // note, that the resulting type is rhs, not lhs + + return ExprFlow(std::move(flow), used_as_condition); + } + + // for `v = rhs` (NOT `var v = lhs`), variable `v` may be smart cast at this point + // the purpose of this function is to drop smart casts from expressions used as left side of assignments + // another example: `x.0 = rhs`, smart cast is dropped for `x.0` (not for `x`) + // the goal of dropping smart casts is to have lhs->inferred_type as actually declared, used as hint to infer rhs + FlowContext infer_left_side_of_assignment(AnyExprV lhs, FlowContext&& flow) { + if (auto lhs_tensor = lhs->try_as()) { + std::vector types_list; + types_list.reserve(lhs_tensor->size()); + for (int i = 0; i < lhs_tensor->size(); ++i) { + flow = infer_left_side_of_assignment(lhs_tensor->get_item(i), std::move(flow)); + types_list.push_back(lhs_tensor->get_item(i)->inferred_type); + } + assign_inferred_type(lhs, TypeDataTensor::create(std::move(types_list))); + + } else if (auto lhs_tuple = lhs->try_as()) { + std::vector types_list; + types_list.reserve(lhs_tuple->size()); + for (int i = 0; i < lhs_tuple->size(); ++i) { + flow = infer_left_side_of_assignment(lhs_tuple->get_item(i), std::move(flow)); + types_list.push_back(lhs_tuple->get_item(i)->inferred_type); + } + assign_inferred_type(lhs, TypeDataTypedTuple::create(std::move(types_list))); + + } else if (auto lhs_par = lhs->try_as()) { + flow = infer_left_side_of_assignment(lhs_par->get_expr(), std::move(flow)); + assign_inferred_type(lhs, lhs_par->get_expr()->inferred_type); + + } else { + flow = infer_any_expr(lhs, std::move(flow), false).out_flow; + if (extract_sink_expression_from_vertex(lhs)) { + TypePtr lhs_declared_type = calc_declared_type_before_smart_cast(lhs); + assign_inferred_type(lhs, lhs_declared_type); + } + } + + return flow; + } + + // handle (and dig recursively) into `var lhs = rhs` + // at this point, both lhs and rhs are already inferred, but lhs newly-declared vars are unknown (unless have declared_type) + // examples: `var z = 5`, `var (x, [y]) = (2, [3])`, `var (x, [y]) = xy` + // the purpose is to update inferred_type of lhs vars (z, x, y) + // and to re-assign types of tensors/tuples inside: `var (x,[y]) = ...` was `(unknown,[unknown])`, becomes `(int,[int])` + // while recursing, keep track of rhs if lhs and rhs have common shape (5 for z, 2 for x, [3] for [y], 3 for y) + // (so that on type mismatch, point to corresponding rhs, example: `var (x, y:slice) = (1, 2)` point to 2 + static void process_assignment_lhs_after_infer_rhs(AnyExprV lhs, TypePtr rhs_type, FlowContext& out_flow) { + tolk_assert(lhs->inferred_type != nullptr); + + // `var ... = rhs` - dig into left part + if (auto lhs_decl = lhs->try_as()) { + process_assignment_lhs_after_infer_rhs(lhs_decl->get_expr(), rhs_type, out_flow); + return; + } + + // inside `var v: int = rhs` / `var _ = rhs` / `var v redef = rhs` (lhs is "v" / "_" / "v") + if (auto lhs_var = lhs->try_as()) { + TypePtr declared_type = lhs_var->marked_as_redef ? lhs_var->var_ref->declared_type : lhs_var->declared_type; + if (lhs_var->inferred_type == TypeDataUnknown::create()) { + assign_inferred_type(lhs_var, rhs_type); + assign_inferred_type(lhs_var->var_ref, rhs_type); + } + TypePtr smart_casted_type = declared_type ? calc_smart_cast_type_on_assignment(declared_type, rhs_type) : rhs_type; + out_flow.register_known_type(SinkExpression(lhs_var->var_ref), smart_casted_type); + return; + } + + // `(v1, v2) = rhs` / `var (v1, v2) = rhs` (rhs may be `(1,2)` or `tensorVar` or `someF()`, doesn't matter) + // dig recursively into v1 and v2 with corresponding rhs i-th item of a tensor + if (auto lhs_tensor = lhs->try_as()) { + const TypeDataTensor* rhs_type_tensor = rhs_type->try_as(); + std::vector types_list; + types_list.reserve(lhs_tensor->size()); + for (int i = 0; i < lhs_tensor->size(); ++i) { + TypePtr ith_rhs_type = rhs_type_tensor && i < rhs_type_tensor->size() ? rhs_type_tensor->items[i] : TypeDataUnknown::create(); + process_assignment_lhs_after_infer_rhs(lhs_tensor->get_item(i), ith_rhs_type, out_flow); + types_list.push_back(lhs_tensor->get_item(i)->inferred_type); + } + assign_inferred_type(lhs, TypeDataTensor::create(std::move(types_list))); + return; + } + + // `[v1, v2] = rhs` / `var [v1, v2] = rhs` (rhs may be `[1,2]` or `tupleVar` or `someF()`, doesn't matter) + // dig recursively into v1 and v2 with corresponding rhs i-th item of a tuple + if (auto lhs_tuple = lhs->try_as()) { + const TypeDataTypedTuple* rhs_type_tuple = rhs_type->try_as(); + std::vector types_list; + types_list.reserve(lhs_tuple->size()); + for (int i = 0; i < lhs_tuple->size(); ++i) { + TypePtr ith_rhs_type = rhs_type_tuple && i < rhs_type_tuple->size() ? rhs_type_tuple->items[i] : TypeDataUnknown::create(); + process_assignment_lhs_after_infer_rhs(lhs_tuple->get_item(i), ith_rhs_type, out_flow); + types_list.push_back(lhs_tuple->get_item(i)->inferred_type); + } + assign_inferred_type(lhs, TypeDataTypedTuple::create(std::move(types_list))); + return; + } + + // `(v) = (rhs)`, just surrounded by parenthesis + if (auto lhs_par = lhs->try_as()) { + process_assignment_lhs_after_infer_rhs(lhs_par->get_expr(), rhs_type, out_flow); + assign_inferred_type(lhs, lhs_par->get_expr()); + return; + } + + // here is `v = rhs` (just assignment, not `var v = rhs`) / `a.0 = rhs` / `getObj(z=f()).0 = rhs` etc. + // for instance, `tensorVar.0 = rhs` / `obj.field = rhs` has already checked index correctness while inferring lhs + // for strange lhs like `f() = rhs` type inferring (and later checking) will pass, but will fail lvalue check later + if (SinkExpression s_expr = extract_sink_expression_from_vertex(lhs)) { + TypePtr lhs_declared_type = calc_declared_type_before_smart_cast(lhs); + TypePtr smart_casted_type = calc_smart_cast_type_on_assignment(lhs_declared_type, rhs_type); + out_flow.register_known_type(s_expr, smart_casted_type); + assign_inferred_type(lhs, lhs_declared_type); + } + } + + ExprFlow infer_set_assign(V v, FlowContext&& flow, bool used_as_condition) { + AnyExprV lhs = v->get_lhs(); + AnyExprV rhs = v->get_rhs(); + ExprFlow after_lhs = infer_any_expr(lhs, std::move(flow), false); + FlowContext rhs_flow = std::move(after_lhs.out_flow); + ExprFlow after_rhs = infer_any_expr(rhs, std::move(rhs_flow), false, lhs->inferred_type); + + // almost all operators implementation is hardcoded by built-in functions `_+_` and similar + std::string_view builtin_func = v->operator_name; // "+" for operator += + + assign_inferred_type(v, lhs); + if (!builtin_func.empty()) { + FunctionPtr builtin_sym = lookup_global_symbol("_" + static_cast(builtin_func) + "_")->try_as(); + v->mutate()->assign_fun_ref(builtin_sym); + } + + return ExprFlow(std::move(after_rhs.out_flow), used_as_condition); + } + + ExprFlow infer_unary_operator(V v, FlowContext&& flow, bool used_as_condition) { + AnyExprV rhs = v->get_rhs(); + ExprFlow after_rhs = infer_any_expr(rhs, std::move(flow), used_as_condition); + + // all operators implementation is hardcoded by built-in functions `~_` and similar + std::string_view builtin_func = v->operator_name; + + switch (v->tok) { + case tok_minus: + case tok_plus: + case tok_bitwise_not: + assign_inferred_type(v, TypeDataInt::create()); + break; + case tok_logical_not: + if (rhs->inferred_type == TypeDataBool::create()) { + builtin_func = "!b"; // "overloaded" for bool + } + assign_inferred_type(v, TypeDataBool::create()); + std::swap(after_rhs.false_flow, after_rhs.true_flow); + break; + default: + tolk_assert(false); + } + + FunctionPtr builtin_sym = lookup_global_symbol(static_cast(builtin_func) + "_")->try_as(); + v->mutate()->assign_fun_ref(builtin_sym); + + return after_rhs; + } + + ExprFlow infer_binary_operator(V v, FlowContext&& flow, bool used_as_condition) { + AnyExprV lhs = v->get_lhs(); + AnyExprV rhs = v->get_rhs(); + + // almost all operators implementation is hardcoded by built-in functions `_+_` and similar + std::string_view builtin_func = v->operator_name; + + switch (v->tok) { + // comparison operators, returning bool + case tok_eq: + case tok_neq: + case tok_lt: + case tok_gt: + case tok_leq: + case tok_geq: + case tok_spaceship: + flow = infer_any_expr(lhs, std::move(flow), false).out_flow; + flow = infer_any_expr(rhs, std::move(flow), false).out_flow; + assign_inferred_type(v, TypeDataBool::create()); + break; + // & | ^ are "overloaded" both for integers and booleans + case tok_bitwise_and: + case tok_bitwise_or: + case tok_bitwise_xor: + flow = infer_any_expr(lhs, std::move(flow), false).out_flow; + flow = infer_any_expr(rhs, std::move(flow), false).out_flow; + if (lhs->inferred_type == TypeDataBool::create() && rhs->inferred_type == TypeDataBool::create()) { + assign_inferred_type(v, TypeDataBool::create()); + } else { + assign_inferred_type(v, TypeDataInt::create()); + } + assign_inferred_type(v, rhs); // (int & int) is int, (bool & bool) is bool + break; + // && || result in booleans, but building flow facts is tricky due to short-circuit + case tok_logical_and: { + ExprFlow after_lhs = infer_any_expr(lhs, std::move(flow), true); + ExprFlow after_rhs = infer_any_expr(rhs, std::move(after_lhs.true_flow), true); + assign_inferred_type(v, TypeDataBool::create()); + if (!used_as_condition) { + FlowContext out_flow = FlowContext::merge_flow(std::move(after_lhs.false_flow), std::move(after_rhs.out_flow)); + return ExprFlow(std::move(out_flow), false); + } + FlowContext out_flow = FlowContext::merge_flow(std::move(after_lhs.out_flow), std::move(after_rhs.out_flow)); + FlowContext true_flow = std::move(after_rhs.true_flow); + FlowContext false_flow = FlowContext::merge_flow(std::move(after_lhs.false_flow), std::move(after_rhs.false_flow)); + return ExprFlow(std::move(out_flow), std::move(true_flow), std::move(false_flow)); + } + case tok_logical_or: { + ExprFlow after_lhs = infer_any_expr(lhs, std::move(flow), true); + ExprFlow after_rhs = infer_any_expr(rhs, std::move(after_lhs.false_flow), true); + assign_inferred_type(v, TypeDataBool::create()); + if (!used_as_condition) { + FlowContext out_flow = FlowContext::merge_flow(std::move(after_lhs.true_flow), std::move(after_rhs.out_flow)); + return ExprFlow(std::move(after_rhs.out_flow), false); + } + FlowContext out_flow = FlowContext::merge_flow(std::move(after_lhs.out_flow), std::move(after_rhs.out_flow)); + FlowContext true_flow = FlowContext::merge_flow(std::move(after_lhs.true_flow), std::move(after_rhs.true_flow)); + FlowContext false_flow = std::move(after_rhs.false_flow); + return ExprFlow(std::move(out_flow), std::move(true_flow), std::move(false_flow)); + } + // others are mathematical: + * ... + default: + flow = infer_any_expr(lhs, std::move(flow), false).out_flow; + flow = infer_any_expr(rhs, std::move(flow), false).out_flow; + assign_inferred_type(v, TypeDataInt::create()); + } + + if (!builtin_func.empty()) { + FunctionPtr builtin_sym = lookup_global_symbol("_" + static_cast(builtin_func) + "_")->try_as(); + v->mutate()->assign_fun_ref(builtin_sym); + } + + return ExprFlow(std::move(flow), used_as_condition); + } + + ExprFlow infer_ternary_operator(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + ExprFlow after_cond = infer_any_expr(v->get_cond(), std::move(flow), true); + v->get_cond()->mutate()->assign_always_true_or_false(after_cond.get_always_true_false_state()); + + ExprFlow after_true = infer_any_expr(v->get_when_true(), std::move(after_cond.true_flow), used_as_condition, hint); + ExprFlow after_false = infer_any_expr(v->get_when_false(), std::move(after_cond.false_flow), used_as_condition, hint); + + if (v->get_cond()->is_always_true) { + assign_inferred_type(v, v->get_when_true()); + return after_true; + } + if (v->get_cond()->is_always_false) { + assign_inferred_type(v, v->get_when_false()); + return after_false; + } + + TypeInferringUnifyStrategy tern_type; + tern_type.unify_with(v->get_when_true()->inferred_type); + if (!tern_type.unify_with(v->get_when_false()->inferred_type)) { + fire(cur_f, v->loc, "types of ternary branches are incompatible: " + to_string(v->get_when_true()) + " and " + to_string(v->get_when_false())); + } + assign_inferred_type(v, tern_type.get_result()); + + FlowContext out_flow = FlowContext::merge_flow(std::move(after_true.out_flow), std::move(after_false.out_flow)); + return ExprFlow(std::move(out_flow), std::move(after_true.true_flow), std::move(after_false.false_flow)); + } + + ExprFlow infer_cast_as_operator(V v, FlowContext&& flow, bool used_as_condition) { + // for `expr as `, use this type for hint, so that `t.tupleAt(0) as int` is ok + ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false, v->cast_to_type); + assign_inferred_type(v, v->cast_to_type); + + if (!used_as_condition) { + return after_expr; + } + return ExprFlow(std::move(after_expr.out_flow), true); + } + + ExprFlow infer_is_null_check(V v, FlowContext&& flow, bool used_as_condition) { + ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false); + assign_inferred_type(v, TypeDataBool::create()); + + TypePtr expr_type = v->get_expr()->inferred_type; + TypePtr non_null_type = calculate_type_subtract_null(expr_type); + if (expr_type == TypeDataNullLiteral::create()) { // `expr == null` is always true + v->mutate()->assign_always_true_or_false(v->is_negated ? 2 : 1); + } else if (non_null_type == TypeDataNever::create()) { // `expr == null` is always false + v->mutate()->assign_always_true_or_false(v->is_negated ? 1 : 2); + } else { + v->mutate()->assign_always_true_or_false(0); + } + + if (!used_as_condition) { + return after_expr; + } + + FlowContext true_flow = after_expr.out_flow.clone(); + FlowContext false_flow = after_expr.out_flow.clone(); + if (SinkExpression s_expr = extract_sink_expression_from_vertex(v->get_expr())) { + if (v->is_always_true) { + false_flow.mark_unreachable(UnreachableKind::CantHappen); + false_flow.register_known_type(s_expr, TypeDataNever::create()); + } else if (v->is_always_false) { + true_flow.mark_unreachable(UnreachableKind::CantHappen); + true_flow.register_known_type(s_expr, TypeDataNever::create()); + } else if (!v->is_negated) { + true_flow.register_known_type(s_expr, TypeDataNullLiteral::create()); + false_flow.register_known_type(s_expr, non_null_type); + } else { + true_flow.register_known_type(s_expr, non_null_type); + false_flow.register_known_type(s_expr, TypeDataNullLiteral::create()); + } + } + return ExprFlow(std::move(after_expr.out_flow), std::move(true_flow), std::move(false_flow)); + } + + ExprFlow infer_not_null_operator(V v, FlowContext&& flow, bool used_as_condition) { + ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false); + + if (const auto* as_nullable = v->get_expr()->inferred_type->try_as()) { + assign_inferred_type(v, as_nullable->inner); + } else { + assign_inferred_type(v, v->get_expr()); + } + + if (!used_as_condition) { + return after_expr; + } + return ExprFlow(std::move(after_expr.out_flow), true); + } + + ExprFlow infer_parenthesized(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), used_as_condition, hint); + assign_inferred_type(v, v->get_expr()); + return after_expr; + } + + ExprFlow infer_reference(V v, FlowContext&& flow, bool used_as_condition) { + if (LocalVarPtr var_ref = v->sym->try_as()) { + TypePtr declared_or_smart_casted = flow.smart_cast_if_exists(SinkExpression(var_ref)); + tolk_assert(declared_or_smart_casted != nullptr); // all local vars are presented in flow + assign_inferred_type(v, declared_or_smart_casted); + + } else if (GlobalConstPtr const_ref = v->sym->try_as()) { + assign_inferred_type(v, const_ref->is_int_const() ? TypeDataInt::create() : TypeDataSlice::create()); + + } else if (GlobalVarPtr glob_ref = v->sym->try_as()) { + // there are no smart casts for globals, it's a way of preventing reading one global multiple times, it costs gas + assign_inferred_type(v, glob_ref->declared_type); + + } else if (FunctionPtr fun_ref = v->sym->try_as()) { + // it's `globalF` / `globalF` - references to functions used as non-call + V v_instantiationTs = v->get_instantiationTs(); + + if (fun_ref->is_generic_function() && !v_instantiationTs) { + // `genericFn` is invalid as non-call, can't be used without + fire(cur_f, v->loc, "can not use a generic function " + to_string(fun_ref) + " as non-call"); + + } else if (fun_ref->is_generic_function()) { + // `genericFn` is valid, it's a reference to instantiation + std::vector substitutions = collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref, v_instantiationTs); + fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutions)); + v->mutate()->assign_sym(fun_ref); + + } else if (v_instantiationTs != nullptr && !fun_ref->is_instantiation_of_generic_function()) { + // non-generic function referenced like `return beginCell;` + fire(cur_f, v_instantiationTs->loc, "not generic function used with generic T"); + } + + fun_ref->mutate()->assign_is_used_as_noncall(); + get_or_infer_return_type(fun_ref); + assign_inferred_type(v, fun_ref->inferred_full_type); + return ExprFlow(std::move(flow), used_as_condition); + + } else { + tolk_assert(false); + } + + // for non-functions: `local_var` and similar not allowed + if (UNLIKELY(v->has_instantiationTs())) { + fire(cur_f, v->get_instantiationTs()->loc, "generic T not expected here"); + } + return ExprFlow(std::move(flow), used_as_condition); + } + + // given `genericF` / `t.tupleFirst` (the user manually specified instantiation Ts), + // validate and collect them + // returns: [int, slice] / [cell] + std::vector collect_fun_generic_substitutions_from_manually_specified(SrcLocation loc, FunctionPtr fun_ref, V instantiationT_list) const { + if (fun_ref->genericTs->size() != instantiationT_list->get_items().size()) { + fire(cur_f, loc, "wrong count of generic T: expected " + std::to_string(fun_ref->genericTs->size()) + ", got " + std::to_string(instantiationT_list->size())); + } + + std::vector substitutions; + substitutions.reserve(instantiationT_list->size()); + for (int i = 0; i < instantiationT_list->size(); ++i) { + substitutions.push_back(instantiationT_list->get_item(i)->substituted_type); + } + + return substitutions; + } + + // when generic Ts have been collected from user-specified or deduced from arguments, + // instantiate a generic function + // example: was `t.tuplePush(2)`, deduced , instantiate `tuplePush` + // example: was `t.tuplePush(2)`, read , instantiate `tuplePush` (will later fail type check) + // example: was `var cb = t.tupleFirst;` (used as reference, as non-call), instantiate `tupleFirst` + // returns fun_ref to instantiated function + FunctionPtr check_and_instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, std::vector&& substitutionTs) const { + // T for asm function must be a TVM primitive (width 1), otherwise, asm would act incorrectly + if (fun_ref->is_asm_function() || fun_ref->is_builtin_function()) { + for (int i = 0; i < static_cast(substitutionTs.size()); ++i) { + if (substitutionTs[i]->get_width_on_stack() != 1) { + fire_error_calling_asm_function_with_non1_stack_width_arg(cur_f, loc, fun_ref, substitutionTs, i); + } + } + } + + std::string inst_name = generate_instantiated_name(fun_ref->name, substitutionTs); + // make deep clone of `f` with substitutionTs + // (if `f` was already instantiated, it will be immediately returned from a symbol table) + return instantiate_generic_function(loc, fun_ref, inst_name, std::move(substitutionTs)); + } + + ExprFlow infer_dot_access(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + // it's NOT a method call `t.tupleSize()` (since such cases are handled by infer_function_call) + // it's `t.0`, `getUser().id`, and `t.tupleSize` (as a reference, not as a call) + flow = infer_any_expr(v->get_obj(), std::move(flow), false).out_flow; + + TypePtr obj_type = v->get_obj()->inferred_type; + // our goal is to fill v->target knowing type of obj + V v_ident = v->get_identifier(); // field/method name vertex + V v_instantiationTs = v->get_instantiationTs(); + std::string_view field_name = v_ident->name; + + // it can be indexed access (`tensorVar.0`, `tupleVar.1`) or a method (`t.tupleSize`) + // at first, check for indexed access + if (field_name[0] >= '0' && field_name[0] <= '9') { + int index_at = std::stoi(std::string(field_name)); + if (const auto* t_tensor = obj_type->try_as()) { + if (index_at >= t_tensor->size()) { + fire(cur_f, v_ident->loc, "invalid tensor index, expected 0.." + std::to_string(t_tensor->items.size() - 1)); + } + v->mutate()->assign_target(index_at); + TypePtr inferred_type = t_tensor->items[index_at]; + if (SinkExpression s_expr = extract_sink_expression_from_vertex(v)) { + if (TypePtr smart_casted = flow.smart_cast_if_exists(s_expr)) { + inferred_type = smart_casted; + } + } + assign_inferred_type(v, inferred_type); + return ExprFlow(std::move(flow), used_as_condition); + } + if (const auto* t_tuple = obj_type->try_as()) { + if (index_at >= t_tuple->size()) { + fire(cur_f, v_ident->loc, "invalid tuple index, expected 0.." + std::to_string(t_tuple->items.size() - 1)); + } + v->mutate()->assign_target(index_at); + TypePtr inferred_type = t_tuple->items[index_at]; + if (SinkExpression s_expr = extract_sink_expression_from_vertex(v)) { + if (TypePtr smart_casted = flow.smart_cast_if_exists(s_expr)) { + inferred_type = smart_casted; + } + } + assign_inferred_type(v, inferred_type); + return ExprFlow(std::move(flow), used_as_condition); + } + if (obj_type->try_as()) { + TypePtr item_type = nullptr; + if (v->is_lvalue && !hint) { // left side of assignment + item_type = TypeDataUnknown::create(); + } else { + if (hint == nullptr) { + fire_error_cannot_deduce_untyped_tuple_access(cur_f, v->loc, index_at); + } + item_type = hint; + } + v->mutate()->assign_target(index_at); + assign_inferred_type(v, item_type); + return ExprFlow(std::move(flow), used_as_condition); + } + fire(cur_f, v_ident->loc, "type " + to_string(obj_type) + " is not indexable"); + } + + // for now, Tolk doesn't have fields and object-scoped methods; `t.tupleSize` is a global function `tupleSize` + const Symbol* sym = lookup_global_symbol(field_name); + FunctionPtr fun_ref = sym ? sym->try_as() : nullptr; + if (!fun_ref) { + fire(cur_f, v_ident->loc, "non-existing field `" + static_cast(field_name) + "` of type " + to_string(obj_type)); + } + + // `t.tupleSize` is ok, `cs.tupleSize` not + if (!fun_ref->parameters[0].declared_type->can_rhs_be_assigned(obj_type)) { + fire(cur_f, v_ident->loc, "referencing a method for " + to_string(fun_ref->parameters[0].declared_type) + " with object of type " + to_string(obj_type)); + } + + if (fun_ref->is_generic_function() && !v_instantiationTs) { + // `genericFn` and `t.tupleAt` are invalid as non-call, they can't be used without + fire(cur_f, v->loc, "can not use a generic function " + to_string(fun_ref) + " as non-call"); + + } else if (fun_ref->is_generic_function()) { + // `t.tupleAt` is valid, it's a reference to instantiation + std::vector substitutions = collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref, v_instantiationTs); + fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutions)); + + } else if (UNLIKELY(v_instantiationTs != nullptr)) { + // non-generic method referenced like `var cb = c.cellHash;` + fire(cur_f, v_instantiationTs->loc, "not generic function used with generic T"); + } + + fun_ref->mutate()->assign_is_used_as_noncall(); + v->mutate()->assign_target(fun_ref); + get_or_infer_return_type(fun_ref); + assign_inferred_type(v, fun_ref->inferred_full_type); // type of `t.tupleSize` is TypeDataFunCallable + return ExprFlow(std::move(flow), used_as_condition); + } + + ExprFlow infer_function_call(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + AnyExprV callee = v->get_callee(); + + // v is `globalF(args)` / `globalF(args)` / `obj.method(args)` / `local_var(args)` / `getF()(args)` + int delta_self = 0; + AnyExprV dot_obj = nullptr; + FunctionPtr fun_ref = nullptr; + V v_instantiationTs = nullptr; + + if (auto v_ref = callee->try_as()) { + // `globalF()` / `globalF()` / `local_var()` / `SOME_CONST()` + fun_ref = v_ref->sym->try_as(); // not null for `globalF` + v_instantiationTs = v_ref->get_instantiationTs(); // present for `globalF()` + + } else if (auto v_dot = callee->try_as()) { + // `obj.someMethod()` / `obj.someMethod()` / `getF().someMethod()` / `obj.SOME_CONST()` + // note, that dot_obj->target is not filled yet, since callee was not inferred yet + delta_self = 1; + dot_obj = v_dot->get_obj(); + v_instantiationTs = v_dot->get_instantiationTs(); // present for `obj.someMethod()` + flow = infer_any_expr(dot_obj, std::move(flow), false).out_flow; + + // it can be indexed access (`tensorVar.0()`, `tupleVar.1()`) or a method (`t.tupleSize()`) + std::string_view field_name = v_dot->get_field_name(); + if (field_name[0] >= '0' && field_name[0] <= '9') { + // indexed access `ab.2()`, then treat `ab.2` just like an expression, fun_ref remains nullptr + // infer_dot_access() will be called for a callee, it will check index correctness + } else { + // for now, Tolk doesn't have fields and object-scoped methods; `t.tupleSize` is a global function `tupleSize` + const Symbol* sym = lookup_global_symbol(field_name); + fun_ref = sym ? sym->try_as() : nullptr; + if (!fun_ref) { + fire(cur_f, v_dot->get_identifier()->loc, "non-existing method `" + static_cast(field_name) + "` of type " + to_string(dot_obj)); + } + } + + } else { + // `getF()()` / `5()` + // fun_ref remains nullptr + } + + // handle `local_var()` / `getF()()` / `5()` / `SOME_CONST()` / `obj.method()()()` / `tensorVar.0()` + if (!fun_ref) { + // treat callee like a usual expression + flow = infer_any_expr(callee, std::move(flow), false).out_flow; + // it must have "callable" inferred type + const TypeDataFunCallable* f_callable = callee->inferred_type->try_as(); + if (!f_callable) { // `5()` / `SOME_CONST()` / `null()` + fire(cur_f, v->loc, "calling a non-function " + to_string(callee->inferred_type)); + } + // check arguments count (their types will be checked in a later pipe) + if (v->get_num_args() != static_cast(f_callable->params_types.size())) { + fire(cur_f, v->loc, "expected " + std::to_string(f_callable->params_types.size()) + " arguments, got " + std::to_string(v->get_arg_list()->size())); + } + for (int i = 0; i < v->get_num_args(); ++i) { + auto arg_i = v->get_arg(i)->get_expr(); + flow = infer_any_expr(arg_i, std::move(flow), false, f_callable->params_types[i]).out_flow; + assign_inferred_type(v->get_arg(i), arg_i); + } + v->mutate()->assign_fun_ref(nullptr); // no fun_ref to a global function + assign_inferred_type(v, f_callable->return_type); + return ExprFlow(std::move(flow), used_as_condition); + } + + // so, we have a call `f(args)` or `obj.f(args)`, f is a global function (fun_ref) (code / asm / builtin) + // we're going to iterate over passed arguments, and (if generic) infer substitutionTs + // at first, check arguments count (Tolk doesn't have optional parameters, so just compare counts) + int n_arguments = v->get_num_args() + delta_self; + int n_parameters = fun_ref->get_num_params(); + if (!n_parameters && dot_obj) { + fire(cur_f, v->loc, "`" + fun_ref->name + "` has no parameters and can not be called as method"); + } + if (n_parameters < n_arguments) { + fire(cur_f, v->loc, "too many arguments in call to `" + fun_ref->name + "`, expected " + std::to_string(n_parameters - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); + } + if (n_arguments < n_parameters) { + fire(cur_f, v->loc, "too few arguments in call to `" + fun_ref->name + "`, expected " + std::to_string(n_parameters - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); + } + + // now, for every passed argument, we need to infer its type + // for regular functions, it's obvious + // but for generic functions, we need to infer type arguments (substitutionTs) on the fly + // (unless Ts are specified by a user like `f(args)` / `t.tupleAt()`, take them) + GenericSubstitutionsDeduceForCall* deducingTs = fun_ref->is_generic_function() ? new GenericSubstitutionsDeduceForCall(fun_ref) : nullptr; + if (deducingTs && v_instantiationTs) { + deducingTs->provide_manually_specified(collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref, v_instantiationTs)); + } + + // loop over every argument, for `obj.method()` obj is the first one + // if genericT deducing has a conflict, ParseError is thrown + // note, that deducing Ts one by one is important to manage control flow (mutate params work like assignments) + // a corner case, e.g. `f(v1:T?, v2:T?)` and `f(null,2)` will fail on first argument, won't try the second one + if (dot_obj) { + const LocalVarData& param_0 = fun_ref->parameters[0]; + TypePtr param_type = param_0.declared_type; + if (param_type->has_genericT_inside()) { + param_type = deducingTs->auto_deduce_from_argument(cur_f, dot_obj->loc, param_type, dot_obj->inferred_type); + } + if (param_0.is_mutate_parameter() && dot_obj->inferred_type != param_type) { + if (SinkExpression s_expr = extract_sink_expression_from_vertex(dot_obj)) { + assign_inferred_type(dot_obj, calc_declared_type_before_smart_cast(dot_obj)); + flow.register_known_type(s_expr, param_type); + } + } + } + for (int i = 0; i < v->get_num_args(); ++i) { + const LocalVarData& param_i = fun_ref->parameters[delta_self + i]; + AnyExprV arg_i = v->get_arg(i)->get_expr(); + TypePtr param_type = param_i.declared_type; + if (param_type->has_genericT_inside() && deducingTs->is_manually_specified()) { // `f(a)` + param_type = deducingTs->replace_by_manually_specified(param_type); + } + if (param_type->has_genericT_inside()) { // `f(a)` where f is generic: use `a` to infer param type + // then arg_i is inferred without any hint + flow = infer_any_expr(arg_i, std::move(flow), false).out_flow; + param_type = deducingTs->auto_deduce_from_argument(cur_f, arg_i->loc, param_type, arg_i->inferred_type); + } else { + // param_type is hint, helps infer arg_i + flow = infer_any_expr(arg_i, std::move(flow), false, param_type).out_flow; + } + assign_inferred_type(v->get_arg(i), arg_i); // arg itself is an expression + if (param_i.is_mutate_parameter() && arg_i->inferred_type != param_type) { + if (SinkExpression s_expr = extract_sink_expression_from_vertex(arg_i)) { + assign_inferred_type(arg_i, calc_declared_type_before_smart_cast(arg_i)); + flow.register_known_type(s_expr, param_type); + } + } + } + + // if it's a generic function `f`, we need to instantiate it, like `f` + // same for generic methods `t.tupleAt`, need to achieve `t.tupleAt` + + if (fun_ref->is_generic_function()) { + // if `f(args)` was called, Ts were inferred; check that all of them are known + int idx = deducingTs->get_first_not_deduced_idx(); + if (idx != -1 && hint && fun_ref->declared_return_type->has_genericT_inside()) { + // example: `t.tupleFirst()`, T doesn't depend on arguments, but is determined by return type + // if used like `var x: int = t.tupleFirst()` / `t.tupleFirst() as int` / etc., use hint + deducingTs->auto_deduce_from_argument(cur_f, v->loc, fun_ref->declared_return_type, hint); + idx = deducingTs->get_first_not_deduced_idx(); + } + if (idx != -1) { + fire(cur_f, v->loc, "can not deduce " + fun_ref->genericTs->get_nameT(idx)); + } + fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, deducingTs->flush()); + delete deducingTs; + + } else if (UNLIKELY(v_instantiationTs != nullptr)) { + // non-generic function/method called with type arguments, like `c.cellHash()` / `beginCell()` + fire(cur_f, v_instantiationTs->loc, "calling a not generic function with generic T"); + } + + v->mutate()->assign_fun_ref(fun_ref); + // since for `t.tupleAt()`, infer_dot_access() not called for callee = "t.tupleAt", assign its target here + if (v->is_dot_call()) { + v->get_callee()->as()->mutate()->assign_target(fun_ref); + } + // get return type either from user-specified declaration or infer here on demand traversing its body + get_or_infer_return_type(fun_ref); + TypePtr inferred_type = dot_obj && fun_ref->does_return_self() ? dot_obj->inferred_type : fun_ref->inferred_return_type; + assign_inferred_type(v, inferred_type); + assign_inferred_type(callee, fun_ref->inferred_full_type); + if (inferred_type == TypeDataNever::create()) { + flow.mark_unreachable(UnreachableKind::CallNeverReturnFunction); + } + // note, that mutate params don't affect typing, they are handled when converting to IR + return ExprFlow(std::move(flow), used_as_condition); + } + + ExprFlow infer_tensor(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + const TypeDataTensor* tensor_hint = hint ? hint->try_as() : nullptr; + std::vector types_list; + types_list.reserve(v->get_items().size()); + for (int i = 0; i < v->size(); ++i) { + AnyExprV item = v->get_item(i); + flow = infer_any_expr(item, std::move(flow), false, tensor_hint && i < tensor_hint->size() ? tensor_hint->items[i] : nullptr).out_flow; + types_list.emplace_back(item->inferred_type); + } + assign_inferred_type(v, TypeDataTensor::create(std::move(types_list))); + return ExprFlow(std::move(flow), used_as_condition); + } + + ExprFlow infer_typed_tuple(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + const TypeDataTypedTuple* tuple_hint = hint ? hint->try_as() : nullptr; + std::vector types_list; + types_list.reserve(v->get_items().size()); + for (int i = 0; i < v->size(); ++i) { + AnyExprV item = v->get_item(i); + flow = infer_any_expr(item, std::move(flow), false, tuple_hint && i < tuple_hint->size() ? tuple_hint->items[i] : nullptr).out_flow; + types_list.emplace_back(item->inferred_type); + } + assign_inferred_type(v, TypeDataTypedTuple::create(std::move(types_list))); + return ExprFlow(std::move(flow), used_as_condition); + } + + static ExprFlow infer_null_keyword(V v, FlowContext&& flow, bool used_as_condition) { + assign_inferred_type(v, TypeDataNullLiteral::create()); + + return ExprFlow(std::move(flow), used_as_condition); + } + + static ExprFlow infer_underscore(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + // if execution is here, underscore is either used as lhs of assignment, or incorrectly, like `f(_)` + // more precise is to always set unknown here, but for incorrect usages, instead of an error + // "can not pass unknown to X" would better be an error it can't be used as a value, at later steps + assign_inferred_type(v, hint ? hint : TypeDataUnknown::create()); + return ExprFlow(std::move(flow), used_as_condition); + } + + static ExprFlow infer_empty_expression(V v, FlowContext&& flow, bool used_as_condition) { + assign_inferred_type(v, TypeDataUnknown::create()); + return ExprFlow(std::move(flow), used_as_condition); + } + + FlowContext process_sequence(V v, FlowContext&& flow) { + // we'll print a warning if after some statement, control flow became unreachable + // (but don't print a warning if it's already unreachable, for example we're inside always-false if) + bool initially_unreachable = flow.is_unreachable(); + for (AnyV item : v->get_items()) { + if (flow.is_unreachable() && !initially_unreachable && !v->first_unreachable && item->type != ast_empty_statement) { + v->mutate()->assign_first_unreachable(item); // a warning will be printed later, after type checking + } + flow = process_any_statement(item, std::move(flow)); + } + return flow; + } + + FlowContext process_return_statement(V v, FlowContext&& flow) { + if (v->has_return_value()) { + flow = infer_any_expr(v->get_return_value(), std::move(flow), false, cur_f->declared_return_type).out_flow; + } else { + assign_inferred_type(v->get_return_value(), TypeDataVoid::create()); + } + flow.mark_unreachable(UnreachableKind::ReturnStatement); + + if (!cur_f->declared_return_type) { + return_statements.push_back(v->get_return_value()); // for future unification + } + return flow; + } + + FlowContext process_if_statement(V v, FlowContext&& flow) { + ExprFlow after_cond = infer_any_expr(v->get_cond(), std::move(flow), true); + v->get_cond()->mutate()->assign_always_true_or_false(after_cond.get_always_true_false_state()); + + FlowContext true_flow = process_any_statement(v->get_if_body(), std::move(after_cond.true_flow)); + FlowContext false_flow = process_any_statement(v->get_else_body(), std::move(after_cond.false_flow)); + + return FlowContext::merge_flow(std::move(true_flow), std::move(false_flow)); + } + + FlowContext process_repeat_statement(V v, FlowContext&& flow) { + ExprFlow after_cond = infer_any_expr(v->get_cond(), std::move(flow), false); + + return process_any_statement(v->get_body(), std::move(after_cond.out_flow)); + } + + FlowContext process_while_statement(V v, FlowContext&& flow) { + // loops are inferred twice, to merge body outcome with the state before the loop + // (a more correct approach would be not "twice", but "find a fixed point when state stop changing") + // also remember, we don't have a `break` statement, that's why when loop exits, condition became false + FlowContext loop_entry_facts = flow.clone(); + ExprFlow after_cond = infer_any_expr(v->get_cond(), std::move(flow), true); + FlowContext body_out = process_any_statement(v->get_body(), std::move(after_cond.true_flow)); + // second time, to refine all types + flow = FlowContext::merge_flow(std::move(loop_entry_facts), std::move(body_out)); + ExprFlow after_cond2 = infer_any_expr(v->get_cond(), std::move(flow), true); + v->get_cond()->mutate()->assign_always_true_or_false(after_cond2.get_always_true_false_state()); + + process_any_statement(v->get_body(), std::move(after_cond2.true_flow)); + + return std::move(after_cond2.false_flow); + } + + FlowContext process_do_while_statement(V v, FlowContext&& flow) { + // do while is also handled twice; read comments above + FlowContext loop_entry_facts = flow.clone(); + flow = process_any_statement(v->get_body(), std::move(flow)); + ExprFlow after_cond = infer_any_expr(v->get_cond(), std::move(flow), true); + // second time + flow = FlowContext::merge_flow(std::move(loop_entry_facts), std::move(after_cond.true_flow)); + flow = process_any_statement(v->get_body(), std::move(flow)); + ExprFlow after_cond2 = infer_any_expr(v->get_cond(), std::move(flow), true); + v->get_cond()->mutate()->assign_always_true_or_false(after_cond2.get_always_true_false_state()); + + return std::move(after_cond2.false_flow); + } + + FlowContext process_throw_statement(V v, FlowContext&& flow) { + flow = infer_any_expr(v->get_thrown_code(), std::move(flow), false).out_flow; + flow = infer_any_expr(v->get_thrown_arg(), std::move(flow), false).out_flow; + flow.mark_unreachable(UnreachableKind::ThrowStatement); + return flow; + } + + FlowContext process_assert_statement(V v, FlowContext&& flow) { + ExprFlow after_cond = infer_any_expr(v->get_cond(), std::move(flow), true); + v->get_cond()->mutate()->assign_always_true_or_false(after_cond.get_always_true_false_state()); + + ExprFlow after_throw = infer_any_expr(v->get_thrown_code(), std::move(after_cond.false_flow), false); + return std::move(after_cond.true_flow); + } + + static FlowContext process_catch_variable(AnyExprV catch_var, TypePtr catch_var_type, FlowContext&& flow) { + if (auto v_ref = catch_var->try_as(); v_ref && v_ref->sym) { // not underscore + LocalVarPtr var_ref = v_ref->sym->try_as(); + assign_inferred_type(var_ref, catch_var_type); + flow.register_known_type(SinkExpression(var_ref), catch_var_type); + } + assign_inferred_type(catch_var, catch_var_type); + return flow; + } + + FlowContext process_try_catch_statement(V v, FlowContext&& flow) { + FlowContext before_try = flow.clone(); + FlowContext try_end = process_any_statement(v->get_try_body(), std::move(flow)); + + // `catch` has exactly 2 variables: excNo and arg (when missing, they are implicit underscores) + // `arg` is a curious thing, it can be any TVM primitive, so assign unknown to it + // hence, using `fInt(arg)` (int from parameter is a target type) or `arg as slice` works well + // it's not truly correct, because `arg as (int,int)` also compiles, but can never happen, but let it be user responsibility + FlowContext catch_flow = std::move(before_try); + tolk_assert(v->get_catch_expr()->size() == 2); + std::vector types_list = {TypeDataInt::create(), TypeDataUnknown::create()}; + catch_flow = process_catch_variable(v->get_catch_expr()->get_item(0), types_list[0], std::move(catch_flow)); + catch_flow = process_catch_variable(v->get_catch_expr()->get_item(1), types_list[1], std::move(catch_flow)); + assign_inferred_type(v->get_catch_expr(), TypeDataTensor::create(std::move(types_list))); + + FlowContext catch_end = process_any_statement(v->get_catch_body(), std::move(catch_flow)); + return FlowContext::merge_flow(std::move(try_end), std::move(catch_end)); + } + + FlowContext process_expression_statement(AnyExprV v, FlowContext&& flow) { + ExprFlow after_v = infer_any_expr(v, std::move(flow), false); + return std::move(after_v.out_flow); + } + +public: + static void assign_fun_full_type(FunctionPtr fun_ref, TypePtr inferred_return_type) { + // calculate function full type `(params) -> ret_type` + std::vector params_types; + params_types.reserve(fun_ref->get_num_params()); + for (const LocalVarData& param : fun_ref->parameters) { + params_types.push_back(param.declared_type); + } + assign_inferred_type(fun_ref, inferred_return_type, TypeDataFunCallable::create(std::move(params_types), inferred_return_type)); + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) { + TypePtr inferred_return_type = fun_ref->declared_return_type; + if (fun_ref->is_code_function()) { + FlowContext body_start; + for (const LocalVarData& param : fun_ref->parameters) { + body_start.register_known_type(SinkExpression(¶m), param.declared_type); + } + + cur_f = fun_ref; + FlowContext body_end = process_any_statement(v_function->get_body(), std::move(body_start)); + cur_f = nullptr; + + if (!body_end.is_unreachable()) { + fun_ref->mutate()->assign_is_implicit_return(); + if (fun_ref->declared_return_type == TypeDataNever::create()) { // `never` can only be declared, it can't be inferred + fire(fun_ref, v_function->get_body()->as()->loc_end, "a function returning `never` can not have a reachable endpoint"); + } + } + + if (!fun_ref->declared_return_type) { + TypeInferringUnifyStrategy return_unifier; + if (fun_ref->does_return_self()) { + return_unifier.unify_with(fun_ref->parameters[0].declared_type); + } + for (AnyExprV return_value : return_statements) { + if (!return_unifier.unify_with(return_value->inferred_type)) { + fire(cur_f, return_value->loc, "can not unify type " + to_string(return_value) + " with previous return type " + to_string(return_unifier.get_result())); + } + } + if (!body_end.is_unreachable()) { + if (!return_unifier.unify_with_implicit_return_void()) { + fire(cur_f, v_function->get_body()->as()->loc_end, "missing return"); + } + } + inferred_return_type = return_unifier.get_result(); + if (inferred_return_type == nullptr && body_end.is_unreachable()) { + inferred_return_type = TypeDataVoid::create(); + } + } + } else { + // asm functions should be strictly typed, this was checked earlier + tolk_assert(fun_ref->declared_return_type); + } + + assign_fun_full_type(fun_ref, inferred_return_type); + fun_ref->mutate()->assign_is_type_inferring_done(); + } +}; + +class LaunchInferTypesAndMethodsOnce final { +public: + static bool should_visit_function(FunctionPtr fun_ref) { + // since inferring can be requested on demand, prevent second execution from a regular pipeline launcher + return !fun_ref->is_type_inferring_done() && !fun_ref->is_generic_function(); + } + + static void start_visiting_function(FunctionPtr fun_ref, V v_function) { + InferTypesAndCallsAndFieldsVisitor visitor; + visitor.start_visiting_function(fun_ref, v_function); + } +}; + +// infer return type "on demand" +// example: `fun f() { return g(); } fun g() { ... }` +// when analyzing `f()`, we need to infer what fun_ref=g returns +// (if `g` is generic, it was already instantiated, so fun_ref=g is here) +static void infer_and_save_return_type_of_function(FunctionPtr fun_ref) { + static std::vector called_stack; + + tolk_assert(!fun_ref->is_generic_function() && !fun_ref->is_type_inferring_done()); + // if `g` has return type declared, like `fun g(): int { ... }`, don't traverse its body + if (fun_ref->declared_return_type) { + InferTypesAndCallsAndFieldsVisitor::assign_fun_full_type(fun_ref, fun_ref->declared_return_type); + return; + } + + // prevent recursion of untyped functions, like `fun f() { return g(); } fun g() { return f(); }` + bool contains = std::find(called_stack.begin(), called_stack.end(), fun_ref) != called_stack.end(); + if (contains) { + fire(fun_ref, fun_ref->loc, "could not infer return type of " + to_string(fun_ref) + ", because it appears in a recursive call chain; specify `: ` manually"); + } + + // dig into g's body; it's safe, since the compiler is single-threaded + // on finish, fun_ref->inferred_return_type is filled, and won't be called anymore + called_stack.push_back(fun_ref); + InferTypesAndCallsAndFieldsVisitor visitor; + visitor.start_visiting_function(fun_ref, fun_ref->ast_root->as()); + called_stack.pop_back(); +} + +void pipeline_infer_types_and_calls_and_fields() { + visit_ast_of_all_functions(); +} + +void pipeline_infer_types_and_calls_and_fields(FunctionPtr fun_ref) { + InferTypesAndCallsAndFieldsVisitor visitor; + visitor.start_visiting_function(fun_ref, fun_ref->ast_root->as()); +} + +} // namespace tolk diff --git a/tolk/pipe-optimize-boolean-expr.cpp b/tolk/pipe-optimize-boolean-expr.cpp new file mode 100644 index 00000000..c4c5d1dc --- /dev/null +++ b/tolk/pipe-optimize-boolean-expr.cpp @@ -0,0 +1,139 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-replacer.h" +#include "type-system.h" + +/* + * This pipe does some optimizations related to booleans. + * It happens after type inferring, when we know types of all expressions. + * + * Example: `boolVar == true` -> `boolVar`. + * Example: `!!boolVar` -> `boolVar`. + * + * todo some day, replace && || with & | when it's safe (currently, && always produces IFs in Fift) + * It's tricky to implement whether replacing is safe. + * For example, safe: `a > 0 && a < 10` / `a != 3 && a != 5` + * For example, unsafe: `cached && calc()` / `a > 0 && log(a)` / `b != 0 && a / b > 1` / `i >= 0 && arr[idx]` / `f != null && close(f)` + */ + +namespace tolk { + +struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { + static V create_int_const(SrcLocation loc, td::RefInt256&& intval) { + auto v_int = createV(loc, std::move(intval), {}); + v_int->assign_inferred_type(TypeDataInt::create()); + v_int->assign_rvalue_true(); + return v_int; + } + + static V create_bool_const(SrcLocation loc, bool bool_val) { + auto v_bool = createV(loc, bool_val); + v_bool->assign_inferred_type(TypeDataInt::create()); + v_bool->assign_rvalue_true(); + return v_bool; + } + + static V create_logical_not_for_bool(SrcLocation loc, AnyExprV rhs) { + auto v_not = createV(loc, "!", tok_logical_not, rhs); + v_not->assign_inferred_type(TypeDataBool::create()); + v_not->assign_rvalue_true(); + v_not->assign_fun_ref(lookup_global_symbol("!b_")->try_as()); + return v_not; + } + +protected: + + AnyExprV replace(V v) override { + parent::replace(v); + + if (v->tok == tok_logical_not) { + if (auto inner_not = v->get_rhs()->try_as(); inner_not && inner_not->tok == tok_logical_not) { + AnyExprV cond_not_not = inner_not->get_rhs(); + // `!!boolVar` => `boolVar` + if (cond_not_not->inferred_type == TypeDataBool::create()) { + return cond_not_not; + } + // `!!intVar` => `intVar != 0` + if (cond_not_not->inferred_type == TypeDataInt::create()) { + auto v_zero = create_int_const(v->loc, td::make_refint(0)); + auto v_neq = createV(v->loc, "!=", tok_neq, cond_not_not, v_zero); + v_neq->mutate()->assign_rvalue_true(); + v_neq->mutate()->assign_inferred_type(TypeDataBool::create()); + v_neq->mutate()->assign_fun_ref(lookup_global_symbol("_!=_")->try_as()); + return v_neq; + } + } + if (auto inner_bool = v->get_rhs()->try_as()) { + // `!true` / `!false` + return create_bool_const(v->loc, !inner_bool->bool_val); + } + } + + return v; + } + + AnyExprV replace(V v) override { + parent::replace(v); + + if (v->tok == tok_eq || v->tok == tok_neq) { + AnyExprV lhs = v->get_lhs(); + AnyExprV rhs = v->get_rhs(); + if (lhs->inferred_type == TypeDataBool::create() && rhs->type == ast_bool_const) { + // `boolVar == true` / `boolVar != false` + if (rhs->as()->bool_val ^ (v->tok == tok_neq)) { + return lhs; + } + // `boolVar != true` / `boolVar == false` + return create_logical_not_for_bool(v->loc, lhs); + } + } + + return v; + } + + AnyV replace(V v) override { + parent::replace(v); + + // `if (!x)` -> ifnot(x) + while (auto v_cond_unary = v->get_cond()->try_as()) { + if (v_cond_unary->tok != tok_logical_not) { + break; + } + v = createV(v->loc, !v->is_ifnot, v_cond_unary->get_rhs(), v->get_if_body(), v->get_else_body()); + } + // `if (x != null)` -> ifnot(x == null) + if (auto v_cond_isnull = v->get_cond()->try_as(); v_cond_isnull && v_cond_isnull->is_negated) { + v_cond_isnull->mutate()->assign_is_negated(!v_cond_isnull->is_negated); + v = createV(v->loc, !v->is_ifnot, v_cond_isnull, v->get_if_body(), v->get_else_body()); + } + + return v; + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } +}; + +void pipeline_optimize_boolean_expressions() { + replace_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipe-refine-lvalue-for-mutate.cpp b/tolk/pipe-refine-lvalue-for-mutate.cpp new file mode 100644 index 00000000..a8b4f1ae --- /dev/null +++ b/tolk/pipe-refine-lvalue-for-mutate.cpp @@ -0,0 +1,128 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-visitor.h" + +/* + * This pipe refines rvalue/lvalue and checks `mutate` arguments validity. + * It happens after type inferring (after methods binding), because it uses fun_ref of calls. + * + * Example: `a.increment().increment()`, the first `a.increment()` becomes lvalue (assume that increment mutates self). + * Example: `increment(a)` is invalid, should be `increment(mutate a)`. + * + * Note, that explicitly specifying `mutate` for arguments, like `increment(mutate a)` is on purpose. + * If we wished `increment(a)` to be valid (to work and mutate `a`, like passing by ref), it would also be done here, + * refining `a` to be lvalue. But to avoid unexpected mutations, `mutate` keyword for an argument is required. + * So, for mutated arguments, instead of setting lvalue, we check its presence. + */ + +namespace tolk { + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_invalid_mutate_arg_passed(AnyExprV v, FunctionPtr fun_ref, const LocalVarData& p_sym, bool called_as_method, bool arg_passed_as_mutate, AnyV arg_expr) { + std::string arg_str(arg_expr->type == ast_reference ? arg_expr->as()->get_name() : "obj"); + + // case: `loadInt(cs, 32)`; suggest: `cs.loadInt(32)` + if (p_sym.is_mutate_parameter() && !arg_passed_as_mutate && !called_as_method && p_sym.param_idx == 0 && fun_ref->does_accept_self()) { + v->error("`" + fun_ref->name + "` is a mutating method; consider calling `" + arg_str + "." + fun_ref->name + "()`, not `" + fun_ref->name + "(" + arg_str + ")`"); + } + // case: `cs.mutating_function()`; suggest: `mutating_function(mutate cs)` or make it a method + if (p_sym.is_mutate_parameter() && called_as_method && p_sym.param_idx == 0 && !fun_ref->does_accept_self()) { + v->error("function `" + fun_ref->name + "` mutates parameter `" + p_sym.name + "`; consider calling `" + fun_ref->name + "(mutate " + arg_str + ")`, not `" + arg_str + "." + fun_ref->name + "`(); alternatively, rename parameter to `self` to make it a method"); + } + // case: `mutating_function(arg)`; suggest: `mutate arg` + if (p_sym.is_mutate_parameter() && !arg_passed_as_mutate) { + v->error("function `" + fun_ref->name + "` mutates parameter `" + p_sym.name + "`; you need to specify `mutate` when passing an argument, like `mutate " + arg_str + "`"); + } + // case: `usual_function(mutate arg)` + if (!p_sym.is_mutate_parameter() && arg_passed_as_mutate) { + v->error("incorrect `mutate`, since `" + fun_ref->name + "` does not mutate this parameter"); + } + throw Fatal("unreachable"); +} + + +class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBody { + void visit(V v) override { + // v is `globalF(args)` / `globalF(args)` / `obj.method(args)` / `local_var(args)` / `getF()(args)` + FunctionPtr fun_ref = v->fun_maybe; + if (!fun_ref) { + parent::visit(v); + for (int i = 0; i < v->get_num_args(); ++i) { + auto v_arg = v->get_arg(i); + if (v_arg->passed_as_mutate) { + v_arg->error("`mutate` used for non-mutate argument"); + } + } + return; + } + + int delta_self = v->is_dot_call(); + tolk_assert(fun_ref->get_num_params() == delta_self + v->get_num_args()); + + if (v->is_dot_call()) { + if (fun_ref->does_mutate_self()) { + // for `b.storeInt()`, `b` should become lvalue, since `storeInt` is a method mutating self + // but: `beginCell().storeInt()`, then `beginCell()` is not lvalue + // (it will be extracted as tmp var when transforming AST to IR) + AnyExprV leftmost_obj = v->get_dot_obj(); + while (true) { + if (auto as_par = leftmost_obj->try_as()) { + leftmost_obj = as_par->get_expr(); + } else if (auto as_cast = leftmost_obj->try_as()) { + leftmost_obj = as_cast->get_expr(); + } else if (auto as_nn = leftmost_obj->try_as()) { + leftmost_obj = as_nn->get_expr(); + } else { + break; + } + } + bool will_be_extracted_as_tmp_var = leftmost_obj->type == ast_function_call; + if (!will_be_extracted_as_tmp_var) { + leftmost_obj->mutate()->assign_lvalue_true(); + v->get_dot_obj()->mutate()->assign_lvalue_true(); + } + } + + if (!fun_ref->does_accept_self() && fun_ref->parameters[0].is_mutate_parameter()) { + fire_error_invalid_mutate_arg_passed(v, fun_ref, fun_ref->parameters[0], true, false, v->get_dot_obj()); + } + } + + for (int i = 0; i < v->get_num_args(); ++i) { + const LocalVarData& p_sym = fun_ref->parameters[delta_self + i]; + auto arg_i = v->get_arg(i); + if (p_sym.is_mutate_parameter() != arg_i->passed_as_mutate) { + fire_error_invalid_mutate_arg_passed(arg_i, fun_ref, p_sym, false, arg_i->passed_as_mutate, arg_i->get_expr()); + } + parent::visit(arg_i); + } + parent::visit(v->get_callee()); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } +}; + +void pipeline_refine_lvalue_for_mutate_arguments() { + visit_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp new file mode 100644 index 00000000..45246d6b --- /dev/null +++ b/tolk/pipe-register-symbols.cpp @@ -0,0 +1,262 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "platform-utils.h" +#include "src-file.h" +#include "ast.h" +#include "compiler-state.h" +#include "constant-evaluator.h" +#include "generics-helpers.h" +#include "td/utils/crypto.h" +#include "type-system.h" +#include + +/* + * This pipe registers global symbols: functions, constants, global vars, etc. + * It happens just after all files have been parsed to AST. + * + * "Registering" means adding symbols to a global symbol table. + * After this pass, any global symbol can be looked up. + * Note, that local variables are not analyzed here, it's a later step. + * Before digging into locals, we need a global symtable to be filled, exactly done here. + */ + +namespace tolk { + +static int calculate_method_id_for_entrypoint(std::string_view func_name) { + if (func_name == "main" || func_name == "onInternalMessage") { + return 0; + } + if (func_name == "onExternalMessage") { + return -1; + } + if (func_name == "onRunTickTock") { + return -2; + } + if (func_name == "onSplitPrepare") { + return -3; + } + if (func_name == "onSplitInstall") { + return -4; + } + tolk_assert(false); +} + +static int calculate_method_id_by_func_name(std::string_view func_name) { + unsigned int crc = td::crc16(static_cast(func_name)); + return static_cast(crc & 0xffff) | 0x10000; +} + +static void validate_arg_ret_order_of_asm_function(V v_body, int n_params, TypePtr ret_type) { + if (!ret_type) { + v_body->error("asm function must declare return type (before asm instructions)"); + } + if (n_params > 16) { + v_body->error("asm function can have at most 16 parameters"); + } + + // asm(param1 ... paramN), param names were previously mapped into indices + if (!v_body->arg_order.empty()) { + if (static_cast(v_body->arg_order.size()) != n_params) { + v_body->error("arg_order of asm function must specify all parameters"); + } + std::vector visited(v_body->arg_order.size(), false); + for (int j : v_body->arg_order) { + if (visited[j]) { + v_body->error("arg_order of asm function contains duplicates"); + } + visited[j] = true; + } + } + + // asm(-> 0 2 1 3), check for a shuffled range 0...N + // correctness of N (actual return width onto a stack) will be checked after type inferring and generics instantiation + if (!v_body->ret_order.empty()) { + std::vector visited(v_body->ret_order.size(), false); + for (int j : v_body->ret_order) { + if (j < 0 || j >= static_cast(v_body->ret_order.size()) || visited[j]) { + v_body->error("ret_order contains invalid integer, not in range 0 .. N"); + } + visited[j] = true; + } + } +} + +static const GenericsDeclaration* construct_genericTs(V v_list) { + std::vector itemsT; + itemsT.reserve(v_list->size()); + + for (int i = 0; i < v_list->size(); ++i) { + auto v_item = v_list->get_item(i); + auto it_existing = std::find_if(itemsT.begin(), itemsT.end(), [v_item](const GenericsDeclaration::GenericsItem& prev) { + return prev.nameT == v_item->nameT; + }); + if (it_existing != itemsT.end()) { + v_item->error("duplicate generic parameter `" + static_cast(v_item->nameT) + "`"); + } + itemsT.emplace_back(v_item->nameT); + } + + return new GenericsDeclaration(std::move(itemsT)); +} + +static void register_constant(V v) { + ConstantValue init_value = eval_const_init_value(v->get_init_value()); + GlobalConstData* c_sym = new GlobalConstData(static_cast(v->get_identifier()->name), v->loc, v->declared_type, std::move(init_value)); + + if (v->declared_type) { + bool ok = (c_sym->is_int_const() && (v->declared_type == TypeDataInt::create())) + || (c_sym->is_slice_const() && (v->declared_type == TypeDataSlice::create())); + if (!ok) { + v->error("expression type does not match declared type"); + } + } + + G.symtable.add_global_const(c_sym); + G.all_constants.push_back(c_sym); + v->mutate()->assign_const_ref(c_sym); +} + +static void register_global_var(V v) { + GlobalVarData* g_sym = new GlobalVarData(static_cast(v->get_identifier()->name), v->loc, v->declared_type); + + G.symtable.add_global_var(g_sym); + G.all_global_vars.push_back(g_sym); + v->mutate()->assign_var_ref(g_sym); +} + +static LocalVarData register_parameter(V v, int idx) { + if (v->is_underscore()) { + return {"", v->loc, v->declared_type, 0, idx}; + } + + int flags = 0; + if (v->declared_as_mutate) { + flags |= LocalVarData::flagMutateParameter; + } + if (!v->declared_as_mutate && idx == 0 && v->param_name == "self") { + flags |= LocalVarData::flagImmutable; + } + return LocalVarData(static_cast(v->param_name), v->loc, v->declared_type, flags, idx); +} + +static void register_function(V v) { + std::string_view func_name = v->get_identifier()->name; + + // calculate TypeData of a function + std::vector arg_types; + std::vector parameters; + int n_params = v->get_num_params(); + int n_mutate_params = 0; + arg_types.reserve(n_params); + parameters.reserve(n_params); + for (int i = 0; i < n_params; ++i) { + auto v_param = v->get_param(i); + arg_types.emplace_back(v_param->declared_type); + parameters.emplace_back(register_parameter(v_param, i)); + n_mutate_params += static_cast(v_param->declared_as_mutate); + } + + const GenericsDeclaration* genericTs = nullptr; + if (v->genericsT_list) { + genericTs = construct_genericTs(v->genericsT_list); + } + if (v->is_builtin_function()) { + const Symbol* sym = lookup_global_symbol(func_name); + FunctionPtr fun_ref = sym ? sym->try_as() : nullptr; + if (!fun_ref || !fun_ref->is_builtin_function()) { + v->error("`builtin` used for non-builtin function"); + } + v->mutate()->assign_fun_ref(fun_ref); + return; + } + + if (G.is_verbosity(1) && v->is_code_function()) { + std::cerr << "fun " << func_name << " : " << v->declared_return_type << std::endl; + } + + FunctionBody f_body = v->get_body()->type == ast_sequence ? static_cast(new FunctionBodyCode) : static_cast(new FunctionBodyAsm); + FunctionData* f_sym = new FunctionData(static_cast(func_name), v->loc, v->declared_return_type, std::move(parameters), 0, genericTs, nullptr, f_body, v); + + if (const auto* v_asm = v->get_body()->try_as()) { + validate_arg_ret_order_of_asm_function(v_asm, v->get_num_params(), v->declared_return_type); + f_sym->arg_order = v_asm->arg_order; + f_sym->ret_order = v_asm->ret_order; + } + + if (v->method_id.not_null()) { + f_sym->method_id = static_cast(v->method_id->to_long()); + } else if (v->flags & FunctionData::flagGetMethod) { + f_sym->method_id = calculate_method_id_by_func_name(func_name); + for (FunctionPtr other : G.all_get_methods) { + if (other->method_id == f_sym->method_id) { + v->error(PSTRING() << "GET methods hash collision: `" << other->name << "` and `" << f_sym->name << "` produce the same hash. Consider renaming one of these functions."); + } + } + } else if (v->flags & FunctionData::flagIsEntrypoint) { + f_sym->method_id = calculate_method_id_for_entrypoint(func_name); + } + f_sym->flags |= v->flags; + if (n_mutate_params) { + f_sym->flags |= FunctionData::flagHasMutateParams; + } + + G.symtable.add_function(f_sym); + G.all_functions.push_back(f_sym); + if (f_sym->is_get_method()) { + G.all_get_methods.push_back(f_sym); + } + v->mutate()->assign_fun_ref(f_sym); +} + +static void iterate_through_file_symbols(const SrcFile* file) { + static std::unordered_set seen; + if (!seen.insert(file).second) { + return; + } + tolk_assert(file && file->ast); + + for (AnyV v : file->ast->as()->get_toplevel_declarations()) { + switch (v->type) { + case ast_import_directive: + // on `import "another-file.tolk"`, register symbols from that file at first + // (for instance, it can calculate constants, which are used in init_val of constants in current file below import) + iterate_through_file_symbols(v->as()->file); + break; + + case ast_constant_declaration: + register_constant(v->as()); + break; + case ast_global_var_declaration: + register_global_var(v->as()); + break; + case ast_function_declaration: + register_function(v->as()); + break; + default: + break; + } + } +} + +void pipeline_register_global_symbols() { + for (const SrcFile* file : G.all_src_files) { + iterate_through_file_symbols(file); + } +} + +} // namespace tolk diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp new file mode 100644 index 00000000..5a735885 --- /dev/null +++ b/tolk/pipe-resolve-identifiers.cpp @@ -0,0 +1,353 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" +#include "platform-utils.h" +#include "compiler-state.h" +#include "src-file.h" +#include "generics-helpers.h" +#include "ast.h" +#include "ast-visitor.h" +#include "type-system.h" +#include + +/* + * This pipe resolves identifiers (local variables and types) in all functions bodies. + * It happens before type inferring, but after all global symbols are registered. + * It means, that for any symbol `x` we can look up whether it's a global name or not. + * + * About resolving variables. + * Example: `var x = 10; x = 20;` both `x` point to one LocalVarData. + * Example: `x = 20` undefined symbol `x` is also here (unless it's a global) + * Variables scoping and redeclaration are also here. + * Note, that `x` is stored as `ast_reference (ast_identifier "x")`. More formally, "references" are resolved. + * "Reference" in AST, besides the identifier, stores optional generics instantiation. `x` is grammar-valid. + * + * About resolving types. At the moment of parsing, `int`, `cell` and other predefined are parsed as TypeDataInt, etc. + * All the others are stored as TypeDataUnresolved, to be resolved here, after global symtable is filled. + * Example: `var x: T = 0` unresolved "T" is replaced by TypeDataGenericT inside `f`. + * Example: `f()` unresolved "MyAlias" is replaced by TypeDataAlias inside the reference. + * Example: `fun f(): KKK` unresolved "KKK" fires an error "unknown type name". + * When structures and type aliases are implemented, their resolving will also be done here. + * See finalize_type_data(). + * + * Note, that functions/methods binding is NOT here. + * In other words, for ast_function_call `beginCell()` and `t.tupleAt(0)`, their fun_ref is NOT filled here. + * Functions/methods binding is done later, simultaneously with type inferring and generics instantiation. + * For instance, to call a generic function `t.tuplePush(1)`, we need types of `t` and `1` to be inferred, + * as well as `tuplePush` to be instantiated, and fun_ref to point at that exact instantiations. + * + * As a result of this step, + * * every V::sym is filled, pointing either to a local var/parameter, or to a global symbol + * (exceptional for function calls and methods, their references are bound later) + * * all TypeData in all symbols is ready for analyzing, TypeDataUnresolved won't occur later in pipeline + */ + +namespace tolk { + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_undefined_symbol(FunctionPtr cur_f, V v) { + if (v->name == "self") { + throw ParseError(cur_f, v->loc, "using `self` in a non-member function (it does not accept the first `self` parameter)"); + } else { + throw ParseError(cur_f, v->loc, "undefined symbol `" + static_cast(v->name) + "`"); + } +} + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_unknown_type_name(FunctionPtr cur_f, SrcLocation loc, const std::string &text) { + throw ParseError(cur_f, loc, "unknown type name `" + text + "`"); +} + +static void check_import_exists_when_using_sym(FunctionPtr cur_f, AnyV v_usage, const Symbol* used_sym) { + SrcLocation sym_loc = used_sym->loc; + if (!v_usage->loc.is_symbol_from_same_or_builtin_file(sym_loc)) { + const SrcFile* declared_in = sym_loc.get_src_file(); + bool has_import = false; + for (const SrcFile::ImportDirective& import : v_usage->loc.get_src_file()->imports) { + if (import.imported_file == declared_in) { + has_import = true; + } + } + if (!has_import) { + throw ParseError(cur_f, v_usage->loc, "Using a non-imported symbol `" + used_sym->name + "`. Forgot to import \"" + declared_in->rel_filename + "\"?"); + } + } +} + +struct NameAndScopeResolver { + std::vector> scopes; + + static uint64_t key_hash(std::string_view name_key) { + return std::hash{}(name_key); + } + + void open_scope([[maybe_unused]] SrcLocation loc) { + // std::cerr << "open_scope " << scopes.size() + 1 << " at " << loc << std::endl; + scopes.emplace_back(); + } + + void close_scope([[maybe_unused]] SrcLocation loc) { + // std::cerr << "close_scope " << scopes.size() << " at " << loc << std::endl; + if (UNLIKELY(scopes.empty())) { + throw Fatal{"cannot close the outer scope"}; + } + scopes.pop_back(); + } + + const Symbol* lookup_symbol(std::string_view name) const { + uint64_t key = key_hash(name); + for (auto it = scopes.rbegin(); it != scopes.rend(); ++it) { // NOLINT(*-loop-convert) + const auto& scope = *it; + if (auto it_sym = scope.find(key); it_sym != scope.end()) { + return it_sym->second; + } + } + return G.symtable.lookup(name); + } + + void add_local_var(LocalVarPtr v_sym) { + if (UNLIKELY(scopes.empty())) { + throw Fatal("unexpected scope_level = 0"); + } + if (v_sym->name.empty()) { // underscore + return; + } + + uint64_t key = key_hash(v_sym->name); + const auto& [_, inserted] = scopes.rbegin()->emplace(key, v_sym); + if (UNLIKELY(!inserted)) { + throw ParseError(v_sym->loc, "redeclaration of local variable `" + v_sym->name + "`"); + } + } +}; + +struct TypeDataResolver { + GNU_ATTRIBUTE_NOINLINE + static TypePtr resolve_identifiers_in_type_data(FunctionPtr cur_f, TypePtr type_data, const GenericsDeclaration* genericTs) { + return type_data->replace_children_custom([cur_f, genericTs](TypePtr child) { + if (const TypeDataUnresolved* un = child->try_as()) { + if (genericTs && genericTs->has_nameT(un->text)) { + std::string nameT = un->text; + return TypeDataGenericT::create(std::move(nameT)); + } + if (un->text == "auto") { + throw ParseError(cur_f, un->loc, "`auto` type does not exist; just omit a type for local variable (will be inferred from assignment); parameters should always be typed"); + } + if (un->text == "self") { + throw ParseError(cur_f, un->loc, "`self` type can be used only as a return type of a function (enforcing it to be chainable)"); + } + fire_error_unknown_type_name(cur_f, un->loc, un->text); + } + return child; + }); + } +}; + +static TypePtr finalize_type_data(FunctionPtr cur_f, TypePtr type_data, const GenericsDeclaration* genericTs) { + if (!type_data || !type_data->has_unresolved_inside()) { + return type_data; + } + return TypeDataResolver::resolve_identifiers_in_type_data(cur_f, type_data, genericTs); +} + + +class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { + // more correctly this field shouldn't be static, but currently there is no need to make it a part of state + static NameAndScopeResolver current_scope; + static FunctionPtr cur_f; + static const GenericsDeclaration* current_genericTs; + + static LocalVarPtr create_local_var_sym(std::string_view name, SrcLocation loc, TypePtr declared_type, bool immutable) { + LocalVarData* v_sym = new LocalVarData(static_cast(name), loc, declared_type, immutable * LocalVarData::flagImmutable, -1); + current_scope.add_local_var(v_sym); + return v_sym; + } + + static void process_catch_variable(AnyExprV catch_var) { + if (auto v_ref = catch_var->try_as()) { + LocalVarPtr var_ref = create_local_var_sym(v_ref->get_name(), catch_var->loc, nullptr, true); + v_ref->mutate()->assign_sym(var_ref); + } + } + +protected: + void visit(V v) override { + if (v->marked_as_redef) { + const Symbol* sym = current_scope.lookup_symbol(v->get_name()); + if (sym == nullptr) { + throw ParseError(cur_f, v->loc, "`redef` for unknown variable"); + } + LocalVarPtr var_ref = sym->try_as(); + if (!var_ref) { + throw ParseError(cur_f, v->loc, "`redef` for unknown variable"); + } + v->mutate()->assign_var_ref(var_ref); + } else { + TypePtr declared_type = finalize_type_data(cur_f, v->declared_type, current_genericTs); + LocalVarPtr var_ref = create_local_var_sym(v->get_name(), v->loc, declared_type, v->is_immutable); + v->mutate()->assign_resolved_type(declared_type); + v->mutate()->assign_var_ref(var_ref); + } + } + + void visit(V v) override { + parent::visit(v->get_rhs()); // in this order, so that `var x = x` is invalid, "x" on the right unknown + parent::visit(v->get_lhs()); + } + + void visit(V v) override { + const Symbol* sym = current_scope.lookup_symbol(v->get_name()); + if (!sym) { + fire_error_undefined_symbol(cur_f, v->get_identifier()); + } + v->mutate()->assign_sym(sym); + + // for global functions, global vars and constants, `import` must exist + if (!sym->try_as()) { + check_import_exists_when_using_sym(cur_f, v, sym); + } + + // for `f` / `f`, resolve "MyAlias" and "T" + // (for function call `f()`, this v (ast_reference `f`) is callee) + if (auto v_instantiationTs = v->get_instantiationTs()) { + for (int i = 0; i < v_instantiationTs->size(); ++i) { + TypePtr substituted_type = finalize_type_data(cur_f, v_instantiationTs->get_item(i)->substituted_type, current_genericTs); + v_instantiationTs->get_item(i)->mutate()->assign_resolved_type(substituted_type); + } + } + } + + void visit(V v) override { + // for `t.tupleAt` / `obj.method`, resolve "MyAlias" and "T" + // (for function call `t.tupleAt()`, this v (ast_dot_access `t.tupleAt`) is callee) + if (auto v_instantiationTs = v->get_instantiationTs()) { + for (int i = 0; i < v_instantiationTs->size(); ++i) { + TypePtr substituted_type = finalize_type_data(cur_f, v_instantiationTs->get_item(i)->substituted_type, current_genericTs); + v_instantiationTs->get_item(i)->mutate()->assign_resolved_type(substituted_type); + } + } + parent::visit(v->get_obj()); + } + + void visit(V v) override { + TypePtr cast_to_type = finalize_type_data(cur_f, v->cast_to_type, current_genericTs); + v->mutate()->assign_resolved_type(cast_to_type); + parent::visit(v->get_expr()); + } + + void visit(V v) override { + if (v->empty()) { + return; + } + current_scope.open_scope(v->loc); + parent::visit(v); + current_scope.close_scope(v->loc_end); + } + + void visit(V v) override { + current_scope.open_scope(v->loc); + parent::visit(v->get_body()); + parent::visit(v->get_cond()); // in 'while' condition it's ok to use variables declared inside do + current_scope.close_scope(v->get_body()->loc_end); + } + + void visit(V v) override { + visit(v->get_try_body()); + current_scope.open_scope(v->get_catch_body()->loc); + const std::vector& catch_items = v->get_catch_expr()->get_items(); + tolk_assert(catch_items.size() == 2); + process_catch_variable(catch_items[1]); + process_catch_variable(catch_items[0]); + parent::visit(v->get_catch_body()); + current_scope.close_scope(v->get_catch_body()->loc_end); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + // this pipe is done just after parsing + // visit both asm and code functions, resolve identifiers in parameter/return types everywhere + // for generic functions, unresolved "T" will be replaced by TypeDataGenericT + return true; + } + + void start_visiting_function(FunctionPtr fun_ref, V v) override { + cur_f = fun_ref; + current_genericTs = fun_ref->genericTs; + + for (int i = 0; i < v->get_num_params(); ++i) { + const LocalVarData& param_var = fun_ref->parameters[i]; + TypePtr declared_type = finalize_type_data(cur_f, param_var.declared_type, fun_ref->genericTs); + v->get_param(i)->mutate()->assign_param_ref(¶m_var); + v->get_param(i)->mutate()->assign_resolved_type(declared_type); + param_var.mutate()->assign_resolved_type(declared_type); + } + TypePtr return_type = finalize_type_data(cur_f, fun_ref->declared_return_type, fun_ref->genericTs); + v->mutate()->assign_resolved_type(return_type); + fun_ref->mutate()->assign_resolved_type(return_type); + + if (fun_ref->is_code_function()) { + auto v_seq = v->get_body()->as(); + current_scope.open_scope(v->loc); + for (int i = 0; i < v->get_num_params(); ++i) { + current_scope.add_local_var(&fun_ref->parameters[i]); + } + parent::visit(v_seq); + current_scope.close_scope(v_seq->loc_end); + tolk_assert(current_scope.scopes.empty()); + } + + current_genericTs = nullptr; + cur_f = nullptr; + } +}; + +NameAndScopeResolver AssignSymInsideFunctionVisitor::current_scope; +FunctionPtr AssignSymInsideFunctionVisitor::cur_f = nullptr; +const GenericsDeclaration* AssignSymInsideFunctionVisitor::current_genericTs = nullptr; + +void pipeline_resolve_identifiers_and_assign_symbols() { + AssignSymInsideFunctionVisitor visitor; + for (const SrcFile* file : G.all_src_files) { + for (AnyV v : file->ast->as()->get_toplevel_declarations()) { + if (auto v_func = v->try_as()) { + tolk_assert(v_func->fun_ref); + visitor.start_visiting_function(v_func->fun_ref, v_func); + + } else if (auto v_global = v->try_as()) { + TypePtr declared_type = finalize_type_data(nullptr, v_global->var_ref->declared_type, nullptr); + v_global->mutate()->assign_resolved_type(declared_type); + v_global->var_ref->mutate()->assign_resolved_type(declared_type); + + } else if (auto v_const = v->try_as()) { + if (v_const->declared_type) { + TypePtr declared_type = finalize_type_data(nullptr, v_const->const_ref->declared_type, nullptr); + v_const->mutate()->assign_resolved_type(declared_type); + v_const->const_ref->mutate()->assign_resolved_type(declared_type); + } + } + } + } +} + +void pipeline_resolve_identifiers_and_assign_symbols(FunctionPtr fun_ref) { + AssignSymInsideFunctionVisitor visitor; + if (visitor.should_visit_function(fun_ref)) { + visitor.start_visiting_function(fun_ref, fun_ref->ast_root->as()); + } +} + +} // namespace tolk diff --git a/tolk/pipeline.h b/tolk/pipeline.h new file mode 100644 index 00000000..0a71d751 --- /dev/null +++ b/tolk/pipeline.h @@ -0,0 +1,58 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#pragma once + +#include "fwd-declarations.h" +#include + +namespace tolk { + +void pipeline_discover_and_parse_sources(const std::string& stdlib_filename, const std::string& entrypoint_filename); + +void pipeline_register_global_symbols(); +void pipeline_resolve_identifiers_and_assign_symbols(); +void pipeline_calculate_rvalue_lvalue(); +void pipeline_infer_types_and_calls_and_fields(); +void pipeline_check_inferred_types(); +void pipeline_refine_lvalue_for_mutate_arguments(); +void pipeline_check_rvalue_lvalue(); +void pipeline_check_pure_impure_operations(); +void pipeline_constant_folding(); +void pipeline_optimize_boolean_expressions(); +void pipeline_convert_ast_to_legacy_Expr_Op(); + +void pipeline_find_unused_symbols(); +void pipeline_generate_fif_output_to_std_cout(); + +// these pipes also can be called per-function individually +// they are called for instantiated generics functions, when `f` is deeply cloned as `f` +void pipeline_resolve_identifiers_and_assign_symbols(FunctionPtr); +void pipeline_calculate_rvalue_lvalue(FunctionPtr); +void pipeline_detect_unreachable_statements(FunctionPtr); +void pipeline_infer_types_and_calls_and_fields(FunctionPtr); + + +} // namespace tolk diff --git a/tolk/platform-utils.h b/tolk/platform-utils.h new file mode 100644 index 00000000..5ab01220 --- /dev/null +++ b/tolk/platform-utils.h @@ -0,0 +1,48 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#pragma once + +#if __GNUC__ +#define GNU_ATTRIBUTE_COLD [[gnu::cold]] +#define GNU_ATTRIBUTE_FLATTEN [[gnu::flatten]] +#define GNU_ATTRIBUTE_NORETURN [[gnu::noreturn]] +#define GNU_ATTRIBUTE_NOINLINE [[gnu::noinline]] +#define GNU_ATTRIBUTE_ALWAYS_INLINE [[gnu::always_inline]] +#else +#define GNU_ATTRIBUTE_COLD +#define GNU_ATTRIBUTE_FLATTEN +#define GNU_ATTRIBUTE_NORETURN [[noreturn]] +#define GNU_ATTRIBUTE_NOINLINE [[noinline]] +#define GNU_ATTRIBUTE_ALWAYS_INLINE +#endif + +#if defined(__GNUC__) +#define LIKELY(x) __builtin_expect(x, true) +#define UNLIKELY(x) __builtin_expect(x, false) +#else +#define LIKELY(x) (x) +#define UNLIKELY(x) (x) +#endif diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp new file mode 100644 index 00000000..7b86f519 --- /dev/null +++ b/tolk/smart-casts-cfg.cpp @@ -0,0 +1,472 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "smart-casts-cfg.h" +#include "ast.h" +#include "tolk.h" + +/* + * This file represents internals of AST-level control flow and data flow analysis. + * Data flow is mostly used for smart casts and is calculated AT THE TIME of type inferring. + * Not before, not after, but simultaneously with type inferring, because any local variable can be smart cast, + * which affects other expressions/variables types, generics instantiation, return auto-infer, etc. + * Though it's a part of type inferring, it's extracted as a separate file to keep inferring a bit clearer. + * + * Control flow is represented NOT as a "graph with edges". Instead, it's a "structured DFS" for the AST: + * 1) at every point of inferring, we have "current flow facts" (FlowContext) + * 2) when we see an `if (...)`, we create two derived contexts (by cloning current) + * 3) after `if`, finalize them at the end and unify + * 4) if we detect unreachable code, we mark that path's context as "unreachable" + * In other words, we get the effect of a CFG but in a more direct approach. That's enough for AST-level data-flow. + * + * FlowContext contains "data-flow facts that are definitely known": variables types (original or refined), + * sign state (definitely positive, definitely zero, etc.), boolean state (definitely true, definitely false). + * Each local variable is contained there, and possibly sub-fields of tensors/objects if definitely known: + * // current facts: x is int?, t is (int, int) + * if (x != null && t.0 > 0) + * // current facts: x is int, t is (int, int), t.0 is positive + * else + * // current facts: x is null, t is (int, int), t.0 is not positive + * When branches rejoin, facts are merged back (int+null = int? and so on, here they would be equal to before if). + * Another example: + * // current facts: x is int? + * if (x == null) { + * // current facts: x is null + * x = 1; + * // current facts: x is int + * } // else branch is empty, its facts are: x is int + * // current facts (after rejoin): x is int + * + * Every expression analysis result (performed along with type inferring) returns ExprFlow: + * 1) out_flow: facts after evaluating the whole expression, no matter how it evaluates (true or false) + * 2) true_flow: the environment if expression is definitely true + * 3) false_flow: the environment if expression is definitely false + * + * Note, that globals are NOT analyzed (smart casts work for locals only). The explanation is simple: + * don't encourage to use a global twice, it costs gas, better assign it to a local. + * See SinkExpression. + * + * An important highlight about internal structure of tensors / tuples / objects and `t.1` sink expressions. + * When a tensor/object is assigned, its fields are NOT tracked individually. + * For better understanding, I'll give some examples in TypeScript (having the same behavior): + * interface User { id: number | string, ... } + * var u: User = { id: 123, ... } + * u.id // it's number|string, not number + * u = { id: 'asdf', ... } + * u.id // it's number|string, not string + * if (typeof u.id === 'string') { + * // here `u.id` is string (smart cast) + * } + * u.id = 123; + * u.id // now it's number (smart cast) (until `u.id` or `u` are reassigned) + * // but `u` still has type `{ id: number | string, ... }`, not `{ id: number, ... }`; only `u.id` is refined + * The same example, but with nullable tensor in Tolk: + * var t: (int?, ...) = (123, ...) + * t.0 // it's int?, not int + * t = (null, ...) + * t.0 // it's int?, not null + * if (t.0 == null) { + * // here `t.0` is null (smart cast) + * } + * t.0 = 123; + * t.0 // now it's int (smart cast) (until `t.0` or `t` are reassigned) + * // but `t` still has type `(int?, ...)`, not `(int, ...)`; only `t.0` is refined + * + * In the future, not only smart casts, but other data-flow analysis can be implemented. + * 1) detect signs: `if (x > 0) { ... if (x < 0)` to warn always false + * 2) detect always true/false: `if (x) { return; } ... if (!x)` to warn always true + * These potential improvements are SignState and BoolState. Now they are NOT IMPLEMENTED, though declared. + * Their purpose is to show, that data flow is not only about smart casts, but eventually for other facts also. + * (though it's not obvious whether they should be analyzed at AST level or at IR level, like constants now) + */ + +namespace tolk { + +std::string SinkExpression::to_string() const { + std::string result = var_ref->name; + uint64_t cur_path = index_path; + while (cur_path != 0) { + result += "."; + result += std::to_string((cur_path & 0xFF) - 1); + cur_path >>= 8; + } + return result; +} + +static std::string to_string(SignState s) { + static const char* txt[6 + 1] = {"sign=unknown", ">0", "<0", "=0", ">=0", "<=0", "sign=never"}; + return txt[static_cast(s)]; +} + +static std::string to_string(BoolState s) { + static const char* txt[4 + 1] = {"unknown", "always_true", "always_false", "bool=never"}; + return txt[static_cast(s)]; +} + +// from `expr!` get `expr` +static AnyExprV unwrap_not_null_operator(AnyExprV expr) { + while (auto v_not_null = expr->try_as()) { + expr = v_not_null->get_expr(); + } + return expr; +} + +// "type lca" for a and b is T, so that both are assignable to T +// it's used +// 1) for auto-infer return type of the function if not specified +// example: `fun f(x: int?) { ... return 1; ... return x; }`; lca(`int`,`int?`) = `int?` +// 2) for auto-infer type of ternary and `match` expressions +// example: `cond ? beginCell() : null`; lca(`builder`,`null`) = `builder?` +// 3) when two data flows rejoin +// example: `if (tensorVar != null) ... else ...` rejoin `(int,int)` and `null` into `(int,int)?` +// when lca can't be calculated (example: `(int,int)` and `(int,int,int)`), nullptr is returned +static TypePtr calculate_type_lca(TypePtr a, TypePtr b) { + if (a == b) { + return a; + } + if (a == TypeDataNever::create()) { + return b; + } + if (b == TypeDataNever::create()) { + return a; + } + + if (a->can_rhs_be_assigned(b)) { + return a; + } + if (b->can_rhs_be_assigned(a)) { + return b; + } + + if (a == TypeDataUnknown::create() || b == TypeDataUnknown::create()) { + return TypeDataUnknown::create(); + } + + if (a == TypeDataNullLiteral::create()) { + return TypeDataNullable::create(b); + } + if (b == TypeDataNullLiteral::create()) { + return TypeDataNullable::create(a); + } + + const auto* tensor1 = a->try_as(); + const auto* tensor2 = b->try_as(); + if (tensor1 && tensor2 && tensor1->size() == tensor2->size()) { + std::vector types_lca; + types_lca.reserve(tensor1->size()); + for (int i = 0; i < tensor1->size(); ++i) { + TypePtr next = calculate_type_lca(tensor1->items[i], tensor2->items[i]); + if (next == nullptr) { + return nullptr; + } + types_lca.push_back(next); + } + return TypeDataTensor::create(std::move(types_lca)); + } + + const auto* tuple1 = a->try_as(); + const auto* tuple2 = b->try_as(); + if (tuple1 && tuple2 && tuple1->size() == tuple2->size()) { + std::vector types_lca; + types_lca.reserve(tuple1->size()); + for (int i = 0; i < tuple1->size(); ++i) { + TypePtr next = calculate_type_lca(tuple1->items[i], tuple2->items[i]); + if (next == nullptr) { + return nullptr; + } + types_lca.push_back(next); + } + return TypeDataTypedTuple::create(std::move(types_lca)); + } + + return nullptr; +} + +// merge (unify) of two sign states: what sign do we definitely have +// it's used on data flow rejoin +// example: `if (x > 0) ... else ...`; lca(Positive, NonPositive) = Unknown +SignState calculate_sign_lca(SignState a, SignState b) { + using s = SignState; + // a transformation lookup table, using the following rules: + // 1) if one is Unknown, the result is Unknown ("no definite constraints") + // 2) if one is Never (can't happen), the result is the other + // example: x is known > 0 already, given code `if (x > 0) {} else {}` merges Positive (always true) and Never + // 3) handle all other combinations carefully + static constexpr SignState transformations[7][7] = { + // b= Unknown | Positive | Negative | Zero | NonNegative | NonPositive | Never | + /* a=Unknown */ {s::Unknown, s::Unknown, s::Unknown, s::Unknown, s::Unknown, s::Unknown, s::Unknown }, + /* a=Positive */ {s::Unknown, s::Positive, s::Unknown, s::NonNegative, s::NonNegative, s::Unknown, s::Positive }, + /* a=Negative */ {s::Unknown, s::Unknown, s::Negative, s::NonPositive, s::Unknown, s::NonPositive, s::Negative }, + /* a=Zero */ {s::Unknown, s::NonNegative, s::NonPositive, s::Zero, s::NonNegative, s::NonPositive, s::Zero }, + /* a=NonNegative */ {s::Unknown, s::NonNegative, s::Unknown, s::NonNegative, s::NonNegative, s::Unknown, s::NonNegative}, + /* a=NonPositive */ {s::Unknown, s::Unknown, s::NonPositive, s::NonPositive, s::Unknown, s::NonPositive, s::NonPositive}, + /* a=Never */ {s::Unknown, s::Positive, s::Negative, s::Zero, s::NonNegative, s::NonPositive, s::Never } + }; + + return transformations[static_cast(a)][static_cast(b)]; +} + +// merge (unify) two bool state: what state do we definitely have +// it's used on data flow rejoin +// example: `if (x) ... else ...`; lca(AlwaysTrue, AlwaysFalse) = Unknown +BoolState calculate_bool_lca(BoolState a, BoolState b) { + using s = BoolState; + static constexpr BoolState transformations[4][4] = { + // b= Unknown | AlwaysTrue | AlwaysFalse | Never | + /* a=Unknown */ {s::Unknown, s::Unknown, s::Unknown, s::Unknown }, + /* a=AlwaysTrue */ {s::Unknown, s::AlwaysTrue, s::Unknown, s::AlwaysTrue }, + /* a=AlwaysFalse */ {s::Unknown, s::Unknown, s::AlwaysFalse, s::AlwaysFalse}, + /* a=Never */ {s::Unknown, s::AlwaysTrue, s::AlwaysFalse, s::Never } + }; + + return transformations[static_cast(a)][static_cast(b)]; +} + +// see comments above TypeInferringUnifyStrategy +// this function calculates lca or currently stored result and next +bool TypeInferringUnifyStrategy::unify_with(TypePtr next) { + if (unified_result == nullptr) { + unified_result = next; + return true; + } + if (unified_result == next) { + return true; + } + + TypePtr combined = calculate_type_lca(unified_result, next); + if (!combined) { + return false; + } + + unified_result = combined; + return true; +} + +bool TypeInferringUnifyStrategy::unify_with_implicit_return_void() { + if (unified_result == nullptr) { + unified_result = TypeDataVoid::create(); + return true; + } + + return unified_result == TypeDataVoid::create(); +} + +// invalidate knowledge about sub-fields of a variable or its field +// example: `tensorVar = 2`, invalidate facts about `tensorVar`, `tensorVar.0`, `tensorVar.1.2`, and all others +// example: `user.id = rhs`, invalidate facts about `user.id` (sign, etc.) and `user.id.*` if exist +void FlowContext::invalidate_all_subfields(LocalVarPtr var_ref, uint64_t parent_path, uint64_t parent_mask) { + for (auto it = known_facts.begin(); it != known_facts.end();) { + bool is_self_or_field = it->first.var_ref == var_ref && (it->first.index_path & parent_mask) == parent_path; + if (is_self_or_field) { + it = known_facts.erase(it); + } else { + ++it; + } + } +} + +// update current type of `local_var` / `tensorVar.0` / `obj.field` +// example: `local_var = rhs` +// example: `f(mutate obj.field)` +// example: `if (t.0 != null)`, in true_flow `t.0` assigned to "not-null of current", in false_flow to null +void FlowContext::register_known_type(SinkExpression s_expr, TypePtr assigned_type) { + // having index_path = (some bytes filled in the end), + // calc index_mask: replace every filled byte with 0xFF + // example: `t.0.1`, index_path = (1<<8) + 2, index_mask = 0xFFFF + uint64_t index_path = s_expr.index_path; + uint64_t index_mask = 0; + while (index_path > 0) { + index_mask = index_mask << 8 | 0xFF; + index_path >>= 8; + } + invalidate_all_subfields(s_expr.var_ref, s_expr.index_path, index_mask); + + // if just `int` assigned, we have no considerations about its sign + // so, even if something existed by the key s_expr, drop all knowledge + known_facts[s_expr] = FactsAboutExpr(assigned_type, SignState::Unknown, BoolState::Unknown); +} + +// mark control flow unreachable / interrupted +void FlowContext::mark_unreachable(UnreachableKind reason) { + unreachable = true; + // currently we don't save why control flow became unreachable (it's not obvious how, there may be consequent reasons), + // but it helps debugging and reading outer code + static_cast(reason); +} + + +// "merge" two data-flow contexts occurs on control flow rejoins (if/else branches merging, for example) +// it's generating a new context that describes "knowledge that definitely outcomes from these two" +// example: in one branch x is `int`, in x is `null`, result is `int?` unless any of them is unreachable +FlowContext FlowContext::merge_flow(FlowContext&& c1, FlowContext&& c2) { + if (!c1.unreachable && c2.unreachable) { + return merge_flow(std::move(c2), std::move(c1)); + } + + std::map unified; + + if (c1.unreachable && !c2.unreachable) { + // `if (...) return; else ...;` — copy facts about common variables only from else (c2) + for (const auto& [s_expr, i2] : c2.known_facts) { + auto it1 = c1.known_facts.find(s_expr); + bool need_add = it1 != c1.known_facts.end() || s_expr.index_path != 0; + if (need_add) { + unified.emplace(s_expr, i2); + } + } + + } else { + // either both reachable, or both not — merge types and restrictions of common variables and fields + for (const auto& [s_expr, i1] : c1.known_facts) { + if (auto it2 = c2.known_facts.find(s_expr); it2 != c2.known_facts.end()) { + const FactsAboutExpr& i2 = it2->second; + unified.emplace(s_expr, i1 == i2 ? i1 : FactsAboutExpr( + calculate_type_lca(i1.expr_type, i2.expr_type), + calculate_sign_lca(i1.sign_state, i2.sign_state), + calculate_bool_lca(i1.bool_state, i2.bool_state) + )); + } + } + } + + return FlowContext(std::move(unified), c1.unreachable && c2.unreachable); +} + +// return `T`, so that `T?` = type +// what for: `if (x != null)`, to smart cast x inside if +TypePtr calculate_type_subtract_null(TypePtr type) { + if (const auto* as_nullable = type->try_as()) { + return as_nullable->inner; + } + // union types will be handled here + return TypeDataNever::create(); +} + +// given any expression vertex, extract SinkExpression is possible +// example: `x.0` is { var_ref: x, index_path: 1 } +// example: `x.1` is { var_ref: x, index_path: 2 } +// example: `x!.1` is the same +// example: `x.1.2` is { var_ref: x, index_path: 2<<8 + 3 } +// example: `x!.1!.2` is the same +// not SinkExpressions: `globalVar` / `f()` / `obj.method().1` +SinkExpression extract_sink_expression_from_vertex(AnyExprV v) { + if (auto as_ref = v->try_as()) { + if (LocalVarPtr var_ref = as_ref->sym->try_as()) { + return SinkExpression(var_ref); + } + } + + if (auto as_dot = v->try_as(); as_dot && as_dot->is_target_indexed_access()) { + V cur_dot = as_dot; + uint64_t index_path = 0; + while (cur_dot->is_target_indexed_access()) { + int index_at = std::get(cur_dot->target); + index_path = (index_path << 8) + index_at + 1; + if (auto parent_dot = unwrap_not_null_operator(cur_dot->get_obj())->try_as()) { + cur_dot = parent_dot; + } else { + break; + } + } + if (auto as_ref = unwrap_not_null_operator(cur_dot->get_obj())->try_as()) { + if (LocalVarPtr var_ref = as_ref->sym->try_as()) { + return SinkExpression(var_ref, index_path); + } + } + } + + if (auto as_par = v->try_as()) { + return extract_sink_expression_from_vertex(as_par->get_expr()); + } + + if (auto as_assign = v->try_as()) { + return extract_sink_expression_from_vertex(as_assign->get_lhs()); + } + + return {}; +} + +// given `lhs = rhs`, calculate "original" type of `lhs` +// example: `var x: int? = ...; if (x != null) { x (here) = null; }` +// "(here)" x is `int` (smart cast), but originally declared as `int?` +// example: `if (x is (int,int)?) { x!.0 = rhs }`, here `x!.0` is `int` +TypePtr calc_declared_type_before_smart_cast(AnyExprV v) { + if (auto as_ref = v->try_as()) { + if (LocalVarPtr var_ref = as_ref->sym->try_as()) { + return var_ref->declared_type; + } + } + + if (auto as_dot = v->try_as(); as_dot && as_dot->is_target_indexed_access()) { + TypePtr obj_type = as_dot->get_obj()->inferred_type; // v already inferred; hence, index_at is correct + int index_at = std::get(as_dot->target); + if (const auto* t_tensor = obj_type->try_as()) { + return t_tensor->items[index_at]; + } + if (const auto* t_tuple = obj_type->try_as()) { + return t_tuple->items[index_at]; + } + } + + return v->inferred_type; +} + +// given `lhs = rhs` (and `var x = rhs`), calculate probable smart cast for lhs +// it's NOT directly type of rhs! see comment at the top of the file about internal structure of tensors/tuples. +// obvious example: `var x: int? = 5`, it's `int` (most cases are like this) +// obvious example: `var x: (int,int)? = null`, it's `null` (`x == null` is always true, `x` can be passed to any `T?`) +// not obvious example: `var x: (int?, int?)? = (3,null)`, result is `(int?,int?)`, whereas type of rhs is `(int,null)` +TypePtr calc_smart_cast_type_on_assignment(TypePtr lhs_declared_type, TypePtr rhs_inferred_type) { + // assign `T` to `T?` (or at least "assignable-to-T" to "T?") + // smart cast to `T` + if (const auto* lhs_nullable = lhs_declared_type->try_as()) { + if (lhs_nullable->inner->can_rhs_be_assigned(rhs_inferred_type)) { + return lhs_nullable->inner; + } + } + + // assign `null` to `T?` + // smart cast to `null` + if (lhs_declared_type->try_as() && rhs_inferred_type == TypeDataNullLiteral::create()) { + return TypeDataNullLiteral::create(); + } + + // no smart cast, type is the same as declared + // example: `var x: (int?,slice?) = (1, null)`, it's `(int?,slice?)`, not `(int,null)` + return lhs_declared_type; +} + + +std::ostream& operator<<(std::ostream& os, const FlowContext& flow) { + os << "(" << flow.known_facts.size() << " facts) " << (flow.unreachable ? "(unreachable) " : ""); + for (const auto& [s_expr, facts] : flow.known_facts) { + os << ", " << s_expr.to_string() << ": " << facts; + } + return os; +} + +std::ostream& operator<<(std::ostream& os, const FactsAboutExpr& facts) { + os << facts.expr_type; + if (facts.sign_state != SignState::Unknown) { + os << " " << to_string(facts.sign_state); + } + if (facts.bool_state != BoolState::Unknown) { + os << " " << to_string(facts.bool_state); + } + return os; +} + +} // namespace tolk diff --git a/tolk/smart-casts-cfg.h b/tolk/smart-casts-cfg.h new file mode 100644 index 00000000..b97c8864 --- /dev/null +++ b/tolk/smart-casts-cfg.h @@ -0,0 +1,208 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "fwd-declarations.h" +#include "type-system.h" +#include +#include + +namespace tolk { + +/* + * TypeInferringUnifyStrategy unifies types from various branches to a common result (lca). + * It's used to auto infer function return type based on return statements, like in TypeScript. + * Example: `fun f() { ... return 1; ... return null; }` inferred as `int?`. + * + * Besides function returns, it's also used for ternary `return cond ? 1 : null` and `match` expression. + * If types can't be unified (a function returns int and cell, for example), `unify()` returns false, handled outside. + * BTW, don't confuse this way of inferring with Hindley-Milner, they have nothing in common. + */ +class TypeInferringUnifyStrategy { + TypePtr unified_result = nullptr; + +public: + bool unify_with(TypePtr next); + bool unify_with_implicit_return_void(); + + TypePtr get_result() const { return unified_result; } +}; + +/* + * SinkExpression is an expression that can be smart cast like `if (x != null)` (x is int inside) + * or analyzed by data flow is some other way like `if (x > 0) ... else ...` (x <= 0 inside else). + * In other words, it "absorbs" data flow facts. + * Examples: `localVar`, `localTensor.1`, `localTuple.1.2.3`, `localObj.field` + * These are NOT sink expressions: `globalVar`, `f()`, `f().1` + * Note, that globals are NOT sink: don't encourage to use a global twice, it costs gas, better assign it to a local. + */ +struct SinkExpression { + LocalVarPtr const var_ref; // smart casts and data flow applies only to locals + const uint64_t index_path; // 0 for just `v`; for `v.N` it's (N+1), for `v.N.M` it's (N+1) + (M+1)<<8, etc. + + SinkExpression() + : var_ref(nullptr), index_path(0) {} + explicit SinkExpression(LocalVarPtr var_ref) + : var_ref(var_ref), index_path(0) {} + explicit SinkExpression(LocalVarPtr var_ref, uint64_t index_path) + : var_ref(var_ref), index_path(index_path) {} + + SinkExpression(const SinkExpression&) = default; + SinkExpression& operator=(const SinkExpression&) = delete; + + bool operator==(const SinkExpression& rhs) const { return var_ref == rhs.var_ref && index_path == rhs.index_path; } + bool operator<(const SinkExpression& rhs) const { return var_ref == rhs.var_ref ? index_path < rhs.index_path : var_ref < rhs.var_ref; } + explicit operator bool() const { return var_ref != nullptr; } + + std::string to_string() const; +}; + +// UnreachableKind is a reason of why control flow is unreachable or interrupted +// example: `return;` interrupts control flow +// example: `if (true) ... else ...` inside "else" flow is unreachable because it can't happen +enum class UnreachableKind { + Unknown, // no definite info or not unreachable + CantHappen, + ThrowStatement, + ReturnStatement, + CallNeverReturnFunction, +}; + +// SignState is "definitely positive", etc. +// example: inside `if (x > 0)`, x is Positive, in `else` it's NonPositive (if x is local, until reassigned) +enum class SignState { + Unknown, // no definite info + Positive, + Negative, + Zero, + NonNegative, + NonPositive, + Never // can't happen, like "never" type +}; + +// BoolState is "definitely true" or "definitely false" +// example: inside `if (x)`, x is AlwaysTrue, in `else` it's AlwaysFalse +enum class BoolState { + Unknown, // no definite info + AlwaysTrue, + AlwaysFalse, + Never // can't happen, like "never" type +}; + +// FactsAboutExpr represents "everything known about SinkExpression at a given execution point" +// example: after `var x = getNullableInt()`, x is `int?`, sign/bool is Unknown +// example: after `x = 2;`, x is `int`, sign is Positive, bool is AlwaysTrue +// example: inside `if (x != null && x > 0)`, x is `int`, sign is Positive (in else, no definite knowledge) +// remember, that indices/fields are also expressions, `t.1 = 2` or `u.id = 2` also store such facts +// WARNING! Detecting data-flow facts about sign state and bool state is NOT IMPLEMENTED +// (e.g. `if (x > 0)` / `if (!t.1)` is NOT analysed, therefore not updated, always Unknown now) +// it's a potential improvement for the future, for example `if (x > 0) { ... if (x < 0)` to warn always false +// their purpose for now is to show, that data flow is not only about smart casts, but eventually for other facts also +struct FactsAboutExpr { + TypePtr expr_type; // originally declared type or smart cast (Unknown if no info) + SignState sign_state; // definitely positive, etc. (Unknown if no info) + BoolState bool_state; // definitely true/false (Unknown if no info) + + FactsAboutExpr() + : expr_type(nullptr), sign_state(SignState::Unknown), bool_state(BoolState::Unknown) {} + FactsAboutExpr(TypePtr smart_cast_type, SignState sign_state, BoolState bool_state) + : expr_type(smart_cast_type), sign_state(sign_state), bool_state(bool_state) {} + + bool operator==(const FactsAboutExpr& rhs) const = default; +}; + +// FlowContext represents "everything known about control flow at a given execution point" +// while traversing AST, each statement node gets "in" FlowContext (prior knowledge) +// and returns "output" FlowContext (representing a state AFTER execution of a statement) +// on branching, like if/else, input context is cloned, two contexts for each branch calculated, and merged to a result +class FlowContext { + // std::map, not std::unordered_map, because LLDB visualises it better, for debugging + std::map known_facts; // all local vars plus (optionally) indices/fields of tensors/tuples/objects + bool unreachable = false; // if execution can't reach this point (after `return`, for example) + + FlowContext(std::map&& known_facts, bool unreachable) + : known_facts(std::move(known_facts)), unreachable(unreachable) {} + + void invalidate_all_subfields(LocalVarPtr var_ref, uint64_t parent_path, uint64_t parent_mask); + + friend std::ostream& operator<<(std::ostream& os, const FlowContext& flow); + +public: + FlowContext() = default; + FlowContext(FlowContext&&) noexcept = default; + FlowContext(const FlowContext&) = delete; + FlowContext& operator=(FlowContext&&) = default; + FlowContext& operator=(const FlowContext&) = delete; + + FlowContext clone() const { + std::map copy = known_facts; + return FlowContext(std::move(copy), unreachable); + } + + bool is_unreachable() const { return unreachable; } + + TypePtr smart_cast_if_exists(SinkExpression s_expr) const { + auto it = known_facts.find(s_expr); + return it == known_facts.end() ? nullptr : it->second.expr_type; + } + + void register_known_type(SinkExpression s_expr, TypePtr assigned_type); + void mark_unreachable(UnreachableKind reason); + + static FlowContext merge_flow(FlowContext&& c1, FlowContext&& c2); +}; + +struct ExprFlow { + FlowContext out_flow; + + // only calculated inside `if`, left of `&&`, etc. — there this expression is immediate condition, empty otherwise + FlowContext true_flow; + FlowContext false_flow; + + ExprFlow(FlowContext&& out_flow, FlowContext&& true_flow, FlowContext&& false_flow) + : out_flow(std::move(out_flow)) + , true_flow(std::move(true_flow)) + , false_flow(std::move(false_flow)) {} + ExprFlow(FlowContext&& out_flow, const bool clone_flow_for_condition) + : out_flow(std::move(out_flow)) { + if (clone_flow_for_condition) { + true_flow = this->out_flow.clone(); + false_flow = this->out_flow.clone(); + } + } + + ExprFlow(ExprFlow&&) noexcept = default; + ExprFlow(const ExprFlow&) = delete; + ExprFlow& operator=(ExprFlow&&) = delete; + ExprFlow& operator=(const ExprFlow&) = delete; + + int get_always_true_false_state() const { + if (true_flow.is_unreachable() != false_flow.is_unreachable()) { + return false_flow.is_unreachable() ? 1 : 2; // 1 is "always true" + } + return 0; + } +}; + +std::ostream& operator<<(std::ostream& os, const FactsAboutExpr& facts); +std::ostream& operator<<(std::ostream& os, const FlowContext& flow); +TypePtr calculate_type_subtract_null(TypePtr type); +SinkExpression extract_sink_expression_from_vertex(AnyExprV v); +TypePtr calc_declared_type_before_smart_cast(AnyExprV v); +TypePtr calc_smart_cast_type_on_assignment(TypePtr lhs_declared_type, TypePtr rhs_inferred_type); + +} // namespace tolk diff --git a/tolk/src-file.cpp b/tolk/src-file.cpp new file mode 100644 index 00000000..1286c1f9 --- /dev/null +++ b/tolk/src-file.cpp @@ -0,0 +1,205 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "src-file.h" +#include "compiler-state.h" +#include +#include +#include + +namespace tolk { + +static_assert(sizeof(SrcLocation) == 8); + +const SrcFile* AllRegisteredSrcFiles::find_file(int file_id) const { + for (const SrcFile* file : all_src_files) { + if (file->file_id == file_id) { + return file; + } + } + return nullptr; +} + +const SrcFile* AllRegisteredSrcFiles::find_file(const std::string& abs_filename) const { + for (const SrcFile* file : all_src_files) { + if (file->abs_filename == abs_filename) { + return file; + } + } + return nullptr; +} + +const SrcFile* AllRegisteredSrcFiles::locate_and_register_source_file(const std::string& rel_filename, SrcLocation included_from) { + td::Result path = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::Realpath, rel_filename.c_str()); + if (path.is_error()) { + if (included_from.is_defined()) { + throw ParseError(included_from, "Failed to import: " + path.move_as_error().message().str()); + } + throw Fatal("Failed to locate " + rel_filename + ": " + path.move_as_error().message().str()); + } + + std::string abs_filename = path.move_as_ok(); + if (const SrcFile* file = find_file(abs_filename)) { + return file; + } + + td::Result text = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::ReadFile, abs_filename.c_str()); + if (text.is_error()) { + if (included_from.is_defined()) { + throw ParseError(included_from, "Failed to import: " + text.move_as_error().message().str()); + } + throw Fatal("Failed to read " + rel_filename + ": " + text.move_as_error().message().str()); + } + + SrcFile* created = new SrcFile(++last_registered_file_id, rel_filename, std::move(abs_filename), text.move_as_ok()); + if (G.is_verbosity(1)) { + std::cerr << "register file_id " << created->file_id << " " << created->abs_filename << std::endl; + } + all_src_files.push_back(created); + return created; +} + +SrcFile* AllRegisteredSrcFiles::get_next_unparsed_file() { + if (last_parsed_file_id >= last_registered_file_id) { + return nullptr; + } + return const_cast(all_src_files[++last_parsed_file_id]); +} + +bool SrcFile::is_stdlib_file() const { + std::string_view rel(rel_filename); + return rel.size() > 10 && rel.substr(0, 8) == "@stdlib/"; // common.tolk, tvm-dicts.tolk, etc +} + +bool SrcFile::is_offset_valid(int offset) const { + return offset >= 0 && offset < static_cast(text.size()); +} + +SrcFile::SrcPosition SrcFile::convert_offset(int offset) const { + if (!is_offset_valid(offset)) { + return SrcPosition{offset, -1, -1, "invalid offset"}; + } + + int line_idx = 0; + int char_idx = 0; + int line_offset = 0; + for (int i = 0; i < offset; ++i) { + char c = text[i]; + if (c == '\n') { + line_idx++; + char_idx = 0; + line_offset = i + 1; + } else { + char_idx++; + } + } + + size_t line_len = text.size() - line_offset; + for (int i = line_offset; i < static_cast(text.size()); ++i) { + if (text[i] == '\n') { + line_len = i - line_offset; + break; + } + } + + std::string_view line_str(text.data() + line_offset, line_len); + return SrcPosition{offset, line_idx + 1, char_idx + 1, line_str}; +} + + +std::ostream& operator<<(std::ostream& os, const SrcFile* src_file) { + return os << (src_file ? src_file->rel_filename : "unknown-location"); +} + +std::ostream& operator<<(std::ostream& os, const Fatal& fatal) { + return os << fatal.what(); +} + +const SrcFile* SrcLocation::get_src_file() const { + return G.all_src_files.find_file(file_id); +} + +void SrcLocation::show(std::ostream& os) const { + const SrcFile* src_file = get_src_file(); + os << src_file; + if (src_file && src_file->is_offset_valid(char_offset)) { + SrcFile::SrcPosition pos = src_file->convert_offset(char_offset); + os << ':' << pos.line_no << ':' << pos.char_no; + } +} + +void SrcLocation::show_context(std::ostream& os) const { + const SrcFile* src_file = get_src_file(); + if (!src_file || !src_file->is_offset_valid(char_offset)) { + return; + } + SrcFile::SrcPosition pos = src_file->convert_offset(char_offset); + os << std::right << std::setw(4) << pos.line_no << " | "; + os << pos.line_str << "\n"; + + os << " " << " | "; + for (int i = 1; i < pos.char_no; ++i) { + os << ' '; + } + os << '^' << "\n"; +} + +std::string SrcLocation::to_string() const { + std::ostringstream os; + show(os); + return os.str(); +} + +std::ostream& operator<<(std::ostream& os, SrcLocation loc) { + loc.show(os); + return os; +} + +void SrcLocation::show_general_error(std::ostream& os, const std::string& message, const std::string& err_type) const { + show(os); + if (!err_type.empty()) { + os << ": " << err_type; + } + os << ": " << message << std::endl; + show_context(os); +} + +void SrcLocation::show_note(const std::string& err_msg) const { + show_general_error(std::cerr, err_msg, "note"); +} + +void SrcLocation::show_warning(const std::string& err_msg) const { + show_general_error(std::cerr, err_msg, "warning"); +} + +void SrcLocation::show_error(const std::string& err_msg) const { + show_general_error(std::cerr, err_msg, "error"); +} + +std::ostream& operator<<(std::ostream& os, const ParseError& error) { + error.show(os); + return os; +} + +void ParseError::show(std::ostream& os) const { + os << loc << ": error: " << message << std::endl; + if (current_function) { + os << " // in function `" << current_function->as_human_readable() << "`" << std::endl; + } + loc.show_context(os); +} + +} // namespace tolk diff --git a/tolk/src-file.h b/tolk/src-file.h new file mode 100644 index 00000000..b0f9cba3 --- /dev/null +++ b/tolk/src-file.h @@ -0,0 +1,144 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include +#include +#include "fwd-declarations.h" + +namespace tolk { + +struct SrcFile { + struct SrcPosition { + int offset; + int line_no; + int char_no; + std::string_view line_str; + }; + + struct ImportDirective { + const SrcFile* imported_file; + }; + + int file_id; // an incremental counter through all parsed files + std::string rel_filename; // relative to cwd + std::string abs_filename; // absolute from root + std::string text; // file contents loaded into memory, every Token::str_val points inside it + AnyV ast = nullptr; // when a file has been parsed, its ast_tolk_file is kept here + std::vector imports; // to check strictness (can't use a symbol without importing its file) + + SrcFile(int file_id, std::string rel_filename, std::string abs_filename, std::string&& text) + : file_id(file_id) + , rel_filename(std::move(rel_filename)) + , abs_filename(std::move(abs_filename)) + , text(std::move(text)) { } + + SrcFile(const SrcFile& other) = delete; + SrcFile &operator=(const SrcFile&) = delete; + + bool is_stdlib_file() const; + + bool is_offset_valid(int offset) const; + SrcPosition convert_offset(int offset) const; +}; + + +// SrcLocation points to a location (line, column) in some loaded .tolk source SrcFile. +// Note, that instead of storing src_file, line_no, etc., only 2 ints are stored. +// The purpose is: sizeof(SrcLocation) == 8, so it's just passed/stored without pointers/refs, just like int64_t. +// When decoding SrcLocation into human-readable format, it's converted to SrcFile::SrcPosition via offset. +class SrcLocation { + friend class Lexer; + + int file_id = -1; // = SrcFile::file_id (note, that get_src_file() does linear search) + int char_offset = -1; // offset from SrcFile::text + +public: + + SrcLocation() = default; + explicit SrcLocation(const SrcFile* src_file) : file_id(src_file->file_id) { + } + + bool is_defined() const { return file_id != -1; } + bool is_stdlib() const { return file_id == 0; } + const SrcFile* get_src_file() const; + + // similar to `this->get_src_file() == symbol->get_src_file() || symbol->get_src_file()->is_stdlib()` + // (but effectively, avoiding linear search) + bool is_symbol_from_same_or_builtin_file(SrcLocation symbol_loc) const { + return file_id == symbol_loc.file_id || symbol_loc.file_id < 1; + } + + void show(std::ostream& os) const; + void show_context(std::ostream& os) const; + std::string to_string() const; + + void show_general_error(std::ostream& os, const std::string& message, const std::string& err_type) const; + void show_note(const std::string& err_msg) const; + void show_warning(const std::string& err_msg) const; + void show_error(const std::string& err_msg) const; +}; + +std::ostream& operator<<(std::ostream& os, SrcLocation loc); + +class AllRegisteredSrcFiles { + std::vector all_src_files; + int last_registered_file_id = -1; + int last_parsed_file_id = -1; + +public: + const SrcFile* find_file(int file_id) const; + const SrcFile* find_file(const std::string& abs_filename) const; + + const SrcFile* locate_and_register_source_file(const std::string& rel_filename, SrcLocation included_from); + SrcFile* get_next_unparsed_file(); + + auto begin() const { return all_src_files.begin(); } + auto end() const { return all_src_files.end(); } +}; + +struct Fatal final : std::exception { + std::string message; + + explicit Fatal(std::string _msg) : message(std::move(_msg)) { + } + const char* what() const noexcept override { + return message.c_str(); + } +}; + +std::ostream& operator<<(std::ostream& os, const Fatal& fatal); + +struct ParseError : std::exception { + FunctionPtr current_function; + SrcLocation loc; + std::string message; + + ParseError(SrcLocation loc, std::string message) + : current_function(nullptr), loc(loc), message(std::move(message)) {} + ParseError(FunctionPtr current_function, SrcLocation loc, std::string message) + : current_function(current_function), loc(loc), message(std::move(message)) {} + + const char* what() const noexcept override { + return message.c_str(); + } + void show(std::ostream& os) const; +}; + +std::ostream& operator<<(std::ostream& os, const ParseError& error); + +} // namespace tolk diff --git a/tolk/stack-transform.cpp b/tolk/stack-transform.cpp new file mode 100644 index 00000000..fe5735e5 --- /dev/null +++ b/tolk/stack-transform.cpp @@ -0,0 +1,1054 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" + +namespace tolk { + +/* + * + * GENERIC STACK TRANSFORMATIONS + * + */ + +StackTransform::StackTransform(std::initializer_list list) { + *this = list; +} + +StackTransform &StackTransform::operator=(std::initializer_list list) { + if (list.size() > 255) { + invalidate(); + return *this; + } + set_id(); + if (!list.size()) { + return *this; + } + int m = (int)list.size(); + d = list.begin()[m - 1] - (m - 1); + if (d >= 128 || d < -128) { + invalidate(); + return *this; + } + for (int i = 0; i < m - 1; i++) { + int x = d + i; + int y = list.begin()[i]; + if (y != x) { + if (x != (short)x || y != (short)y || n == max_n) { + invalidate(); + return *this; + } + dp = std::max(dp, std::max(x, y) + 1); + A[n++] = std::make_pair((short)x, (short)y); + } + } + return *this; +} + +bool StackTransform::assign(const StackTransform &other) { + if (!other.is_valid() || (unsigned)other.n > max_n) { + return invalidate(); + } + d = other.d; + n = other.n; + dp = other.dp; + c = other.c; + invalid = false; + for (int i = 0; i < n; i++) { + A[i] = other.A[i]; + } + return true; +} + +int StackTransform::get(int x) const { + if (!is_valid()) { + return -1; + } + if (x <= c_start) { + return x - c; + } + x += d; + int i; + for (i = 0; i < n && A[i].first < x; i++) { + } + if (i < n && A[i].first == x) { + return A[i].second; + } else { + return x; + } +} + +bool StackTransform::set(int x, int y, bool relaxed) { + if (!is_valid()) { + return false; + } + if (x < 0) { + return (relaxed && y == x + d) || invalidate(); + } + if (!relaxed) { + touch(x); + } + x += d; + int i; + for (i = 0; i < n && A[i].first < x; i++) { + } + if (i < n && A[i].first == x) { + if (x != y) { + if (y != (short)y) { + return invalidate(); + } + A[i].second = (short)y; + } else { + --n; + for (; i < n; i++) { + A[i] = A[i + 1]; + } + } + } else { + if (x != y) { + if (x != (short)x || y != (short)y || n == max_n) { + return invalidate(); + } + for (int j = n++; j > i; j--) { + A[j] = A[j - 1]; + } + A[i].first = (short)x; + A[i].second = (short)y; + touch(x - d); + touch(y); + } + } + return true; +} + +// f(x') = x' + d for all x' >= x ? +bool StackTransform::is_trivial_after(int x) const { + return is_valid() && (!n || A[n - 1].first < x + d); +} + +// card f^{-1}(y) +int StackTransform::preimage_count(int y) const { + if (!is_valid()) { + return -1; + } + int count = (y >= d); + for (const auto &pair : A) { + if (pair.second == y) { + ++count; + } else if (pair.first == y) { + --count; + } + } + return count; +} + +// f^{-1}(y) +std::vector StackTransform::preimage(int y) const { + if (!is_valid()) { + return {}; + } + std::vector res; + bool f = (y >= d); + for (const auto &pair : A) { + if (pair.first > y && f) { + res.push_back(y - d); + f = false; + } + if (pair.first == y) { + f = false; + } else if (pair.second == y) { + res.push_back(pair.first - d); + } + } + return res; +} + +// is f:N->N bijective ? +bool StackTransform::is_permutation() const { + if (!is_valid() || d) { + return false; + } + tolk_assert(n <= max_n); + std::array X, Y; + for (int i = 0; i < n; i++) { + X[i] = A[i].first; + Y[i] = A[i].second; + if (Y[i] < 0) { + return false; + } + } + std::sort(Y.begin(), Y.begin() + n); + for (int i = 0; i < n; i++) { + if (X[i] != Y[i]) { + return false; + } + } + return true; +} + +bool StackTransform::remove_negative() { + int s = 0; + while (s < n && A[s].first < d) { + ++s; + } + if (s) { + n -= s; + for (int i = 0; i < n; i++) { + A[i] = A[i + s]; + } + } + return true; +} + +int StackTransform::try_load(int &i, int offs) const { + return i < n ? A[i++].first + offs : inf_x; +} + +bool StackTransform::try_store(int x, int y) { + if (x == y || x < d) { + return true; + } + if (n == max_n || x != (short)x || y != (short)y) { + return invalidate(); + } + A[n].first = (short)x; + A[n++].second = (short)y; + return true; +} + +// c := a * b +bool StackTransform::compose(const StackTransform &a, const StackTransform &b, StackTransform &c) { + if (!a.is_valid() || !b.is_valid()) { + return c.invalidate(); + } + c.d = a.d + b.d; + c.n = 0; + c.dp = std::max(a.dp, b.dp + a.d); + c.c = a.c + b.c; + c.invalid = false; + int i = 0, j = 0; + int x1 = a.try_load(i); + int x2 = b.try_load(j, a.d); + while (true) { + if (x1 < x2) { + int y = a.A[i - 1].second; + if (!c.try_store(x1, y)) { + return false; + } + x1 = a.try_load(i); + } else if (x2 < inf_x) { + if (x1 == x2) { + x1 = a.try_load(i); + } + int y = b.A[j - 1].second; + if (!c.try_store(x2, a(y))) { + return false; + } + x2 = b.try_load(j, a.d); + } else { + return true; + } + } +} + +// this = this * other +bool StackTransform::apply(const StackTransform &other) { + StackTransform res; + if (!compose(*this, other, res)) { + return invalidate(); + } + return assign(res); +} + +// this = other * this +bool StackTransform::preapply(const StackTransform &other) { + StackTransform res; + if (!compose(other, *this, res)) { + return invalidate(); + } + return assign(res); +} + +StackTransform StackTransform::operator*(const StackTransform &b) const & { + StackTransform res; + compose(*this, b, res); + return res; +} + +// this = this * other +StackTransform &StackTransform::operator*=(const StackTransform &other) { + StackTransform res; + (compose(*this, other, res) && assign(res)) || invalidate(); + return *this; +} + +bool StackTransform::apply_xchg(int i, int j, bool relaxed) { + if (!is_valid() || i < 0 || j < 0) { + return invalidate(); + } + if (i == j) { + return relaxed || touch(i); + } + int u = touch_get(i), v = touch_get(j); + return set(i, v) && set(j, u); +} + +bool StackTransform::apply_push(int i) { + if (!is_valid() || i < 0) { + return invalidate(); + } + int u = touch_get(i); + return shift(-1) && set(0, u); +} + +bool StackTransform::apply_push_newconst() { + if (!is_valid()) { + return false; + } + return shift(-1) && set(0, c_start - c++); +} + +bool StackTransform::apply_pop(int i) { + if (!is_valid() || i < 0) { + return invalidate(); + } + if (!i) { + return touch(0) && shift(1); + } else { + return set(i, get(0)) && shift(1); + } +} + +bool StackTransform::apply_blkpop(int k) { + if (!is_valid() || k < 0) { + return invalidate(); + } + return !k || (touch(k - 1) && shift(k)); +} + +bool StackTransform::equal(const StackTransform &other, bool relaxed) const { + if (!is_valid() || !other.is_valid()) { + return false; + } + if (!(n == other.n && d == other.d)) { + return false; + } + for (int i = 0; i < n; i++) { + if (A[i] != other.A[i]) { + return false; + } + } + return relaxed || dp == other.dp; +} + +StackTransform StackTransform::Xchg(int i, int j, bool relaxed) { + StackTransform t; + t.apply_xchg(i, j, relaxed); + return t; +} + +StackTransform StackTransform::Push(int i) { + StackTransform t; + t.apply_push(i); + return t; +} + +StackTransform StackTransform::Pop(int i) { + StackTransform t; + t.apply_pop(i); + return t; +} + +bool StackTransform::is_xchg(int i, int j) const { + if (i == j) { + return is_id(); + } + return is_valid() && !d && n == 2 && i >= 0 && j >= 0 && get(i) == j && get(j) == i; +} + +bool StackTransform::is_xchg(int *i, int *j) const { + if (!is_valid() || d || n > 2 || !dp) { + return false; + } + if (!n) { + *i = *j = 0; + return true; + } + if (n != 2) { + return false; + } + int a = A[0].first, b = A[1].first; + if (A[0].second != b || A[1].second != a) { + return false; + } + *i = std::min(a, b); + *j = std::max(a, b); + return true; +} + +bool StackTransform::is_xchg_xchg(int i, int j, int k, int l) const { + if (is_valid() && !d && n <= 4 && (i | j | k | l) >= 0) { + StackTransform t; + return t.apply_xchg(i, j) && t.apply_xchg(k, l) && t <= *this; + } else { + return false; + } +} + +bool StackTransform::is_xchg_xchg(int *i, int *j, int *k, int *l) const { + if (!is_valid() || d || n > 4 || !dp || !is_permutation()) { + return false; + } + if (!n) { + *i = *j = *k = *l = 0; + return true; + } + if (n <= 2) { + *k = *l = 0; + return is_xchg(i, j); + } + if (n == 3) { + // rotation: a -> b -> c -> a + int a = A[0].first; + int b = A[0].second; + int s = (b == A[2].first ? 2 : 1); + int c = A[s].second; + if (b != A[s].first || c != A[3 - s].first || a != A[3 - s].second) { + return false; + } + // implement as XCHG s(a),s(c) ; XCHG s(a),s(b) + *i = *k = a; + *j = c; + *l = b; + return is_xchg_xchg(*i, *j, *k, *l); + } + *i = A[0].first; + *j = A[0].second; + if (get(*j) != *i) { + return false; + } + for (int s = 1; s < 4; s++) { + if (A[s].first != *j) { + *k = A[s].first; + *l = A[s].second; + return get(*l) == *k && is_xchg_xchg(*i, *j, *k, *l); + } + } + return false; +} + +bool StackTransform::is_push(int i) const { + return is_valid() && d == -1 && n == 1 && A[0].first == -1 && A[0].second == i; +} + +bool StackTransform::is_push(int *i) const { + if (is_valid() && d == -1 && n == 1 && A[0].first == -1 && A[0].second >= 0) { + *i = A[0].second; + return true; + } else { + return false; + } +} + +// 1 2 3 4 .. = pop0 +// 0 2 3 4 .. = pop1 +// 1 0 3 4 .. = pop2 +// 1 2 0 4 .. = pop3 +// POP s(i) : 1 2 ... i-1 0 i+1 ... ; d=1, n=1, {(i,0)} +bool StackTransform::is_pop(int i) const { + if (!is_valid() || d != 1 || n > 1 || i < 0) { + return false; + } + if (!i) { + return !n; + } + return n == 1 && A[0].first == i && !A[0].second; +} + +bool StackTransform::is_pop(int *i) const { + if (!is_valid() || d != 1 || n > 1) { + return false; + } + if (!n) { + *i = 0; + return true; + } + if (n == 1 && !A[0].second) { + *i = A[0].first; + return true; + } + return false; +} + +// POP s(i) ; POP s(j) : 2 ... i-1 0 i+1 ... j 1 j+2 ... ; d=2, n=2, {(i,0),(j+1,1)} if i <> j+1 +bool StackTransform::is_pop_pop(int i, int j) const { + if (is_valid() && d == 2 && n <= 2 && i >= 0 && j >= 0) { + StackTransform t; + return t.apply_pop(i) && t.apply_pop(j) && t <= *this; + } else { + return false; + } +} + +bool StackTransform::is_pop_pop(int *i, int *j) const { + if (!is_valid() || d != 2 || n > 2) { + return false; + } + if (!n) { + *i = *j = 0; // 2DROP + } else if (n == 2) { + *i = A[0].first - A[0].second; + *j = A[1].first - A[1].second; + if (A[0].second > A[1].second) { + std::swap(*i, *j); + } + } else if (!A[0].second) { + *i = A[0].first; + *j = 0; + } else { + *i = 0; + *j = A[0].first - 1; + } + return is_pop_pop(*i, *j); +} + +const StackTransform StackTransform::rot{2, 0, 1, 3}; +const StackTransform StackTransform::rot_rev{1, 2, 0, 3}; + +bool StackTransform::is_rot() const { + return equal(rot, true); +} + +bool StackTransform::is_rotrev() const { + return equal(rot_rev, true); +} + +// PUSH i ; ROT == 1 i 0 2 3 +bool StackTransform::is_push_rot(int i) const { + return is_valid() && d == -1 && i >= 0 && is_trivial_after(3) && get(0) == 1 && get(1) == i && get(2) == 0; +} + +bool StackTransform::is_push_rot(int *i) const { + return is_valid() && (*i = get(1)) >= 0 && is_push_rot(*i); +} + +// PUSH i ; -ROT == 0 1 i 2 3 +bool StackTransform::is_push_rotrev(int i) const { + return is_valid() && d == -1 && i >= 0 && is_trivial_after(3) && get(0) == 0 && get(1) == 1 && get(2) == i; +} + +bool StackTransform::is_push_rotrev(int *i) const { + return is_valid() && (*i = get(2)) >= 0 && is_push_rotrev(*i); +} + +// PUSH s(i) ; XCHG s(j),s(k) --> i 0 1 .. i .. +// PUSH s(i) ; XCHG s(0),s(k) --> k-1 0 1 .. k-2 i k .. +bool StackTransform::is_push_xchg(int i, int j, int k) const { + StackTransform t; + return is_valid() && d == -1 && n <= 3 && t.apply_push(i) && t.apply_xchg(j, k) && t <= *this; +} + +bool StackTransform::is_push_xchg(int *i, int *j, int *k) const { + if (!(is_valid() && d == -1 && n <= 3 && n > 0)) { + return false; + } + int s = get(0); + if (s < 0) { + return false; + } + *i = s; + *j = 0; + if (n == 1) { + *k = 0; + } else if (n == 2) { + *k = s + 1; + *i = get(s + 1); + } else { + *j = A[1].first + 1; + *k = A[2].first + 1; + } + return is_push_xchg(*i, *j, *k); +} + +// XCHG s1,s(i) ; XCHG s0,s(j) +bool StackTransform::is_xchg2(int i, int j) const { + StackTransform t; + return is_valid() && !d && t.apply_xchg(1, i) && t.apply_xchg(0, j) && t <= *this; +} + +bool StackTransform::is_xchg2(int *i, int *j) const { + if (!is_valid() || d || n > 4 || n == 1 || dp < 2) { + return false; + } + *i = get(1); + *j = get(0); + if (!n) { + return true; + } + if (*i < 0 || *j < 0) { + return false; + } + if (n == 2 && !*i) { + *j = *i; // XCHG s0,s1 = XCHG2 s0,s0 + } else if (n == 3 && *i) { + // XCHG2 s(i),s(i) = XCHG s1,s(i) ; XCHG s0,s(i) : 0->1, 1->i + *j = *i; + } // XCHG2 s0,s(i) = XCHG s0,s1 ; XCHG s0,s(i) : 0->i, 1->0 + return is_xchg2(*i, *j); +} + +// XCHG s0,s(i) ; PUSH s(j) = PUSH s(j') ; XCHG s1,s(i+1) +// j'=j if j!=0, j!=i +// j'=0 if j=i +// j'=i if j=0 +bool StackTransform::is_xcpu(int i, int j) const { + StackTransform t; + return is_valid() && d == -1 && t.apply_xchg(0, i) && t.apply_push(j) && t <= *this; +} + +bool StackTransform::is_xcpu(int *i, int *j) const { + if (!is_valid() || d != -1 || n > 3 || dp < 1) { + return false; + } + *i = get(1); + *j = get(0); + if (!*j) { + *j = *i; + } else if (*j == *i) { + *j = 0; + } + return is_xcpu(*i, *j); +} + +// PUSH s(i) ; XCHG s0, s1 ; XCHG s0, s(j+1) +bool StackTransform::is_puxc(int i, int j) const { + StackTransform t; + return is_valid() && d == -1 && t.apply_push(i) && t.apply_xchg(0, 1) && t.apply_xchg(0, j + 1) && t <= *this; +} + +// j > 0 : 0 -> j, 1 -> i +// j = 0 : 0 -> i, 1 -> 0 ( PUSH s(i) ) +// j = -1 : 0 -> 0, 1 -> i ( PUSH s(i) ; XCHG s0, s1 ) +bool StackTransform::is_puxc(int *i, int *j) const { + if (!is_valid() || d != -1 || n > 3) { + return false; + } + *i = get(1); + *j = get(0); + if (!*i && is_push(*j)) { + std::swap(*i, *j); + return is_puxc(*i, *j); + } + if (!*j) { + --*j; + } + return is_puxc(*i, *j); +} + +// PUSH s(i) ; PUSH s(j+1) +bool StackTransform::is_push2(int i, int j) const { + StackTransform t; + return is_valid() && d == -2 && t.apply_push(i) && t.apply_push(j + 1) && t <= *this; +} + +bool StackTransform::is_push2(int *i, int *j) const { + if (!is_valid() || d != -2 || n > 2) { + return false; + } + *i = get(1); + *j = get(0); + return is_push2(*i, *j); +} + +// XCHG s2,s(i) ; XCHG s1,s(j) ; XCHG s0,s(k) +bool StackTransform::is_xchg3(int *i, int *j, int *k) const { + if (!is_valid() || d || dp < 3 || !is_permutation()) { + return false; + } + for (int s = 2; s >= 0; s--) { + *i = get(s); + StackTransform t = Xchg(2, *i) * *this; + if (t.is_xchg2(j, k)) { + return true; + } + } + return false; +} + +// XCHG s1,s(i) ; XCHG s0,s(j) ; PUSH s(k) +bool StackTransform::is_xc2pu(int *i, int *j, int *k) const { + if (!is_valid() || d != -1 || dp < 2) { + return false; + } + for (int s = 2; s >= 1; s--) { + *i = get(s); + StackTransform t = Xchg(1, *i) * *this; + if (t.is_xcpu(j, k)) { + return true; + } + } + return false; +} + +// XCHG s1,s(i) ; PUSH s(j) ; XCHG s0,s1 ; XCHG s0,s(k+1) +bool StackTransform::is_xcpuxc(int *i, int *j, int *k) const { + if (!is_valid() || d != -1 || dp < 2) { + return false; + } + for (int s = 2; s >= 0; s--) { + *i = get(s); + StackTransform t = Xchg(1, *i) * *this; + if (t.is_puxc(j, k)) { + return true; + } + } + return false; +} + +// XCHG s0,s(i) ; PUSH s(j) ; PUSH s(k+1) +bool StackTransform::is_xcpu2(int *i, int *j, int *k) const { + if (!is_valid() || d != -2 || dp < 1) { + return false; + } + *i = get(2); + StackTransform t = Xchg(0, *i) * *this; + return t.is_push2(j, k); +} + +// PUSH s(i) ; XCHG s0,s2 ; XCHG s1,s(j+1) ; XCHG s0,s(k+1) +// 0 -> i or 1 -> i or 2 -> i ; i has two preimages +// 0 -> k if k >= 2, k != j +// 1 -> j=k if j = k >= 2 +// 1 -> j if j >= 2, k != 0 +// 0 -> j if j >= 2, k = 0 +// => i in {f(0), f(1), f(2)} ; j in {-1, 0, 1, f(0), f(1)} ; k in {-1, 0, 1, f(0), f(1)} +bool StackTransform::is_puxc2(int *i, int *j, int *k) const { + if (!is_valid() || d != -1 || dp < 2) { + return false; + } + for (int s = 2; s >= 0; s--) { + *i = get(s); + if (preimage_count(*i) != 2) { + continue; + } + for (int u = -1; u <= 3; u++) { + *j = (u >= 2 ? get(u - 2) : u); + for (int v = -1; v <= 3; v++) { + *k = (v >= 2 ? get(v - 2) : v); + if (is_puxc2(*i, *j, *k)) { + return true; + } + } + } + } + return false; +} + +// PUSH s(i) ; XCHG s0,s2 ; XCHG s1,s(j+1) ; XCHG s0,s(k+1) +bool StackTransform::is_puxc2(int i, int j, int k) const { + StackTransform t; + return is_valid() && d == -1 && dp >= 2 // basic checks + && t.apply_push(i) && t.apply_xchg(0, 2) // PUSH s(i) ; XCHG s0,s2 + && t.apply_xchg(1, j + 1) // XCHG s1,s(j+1) + && t.apply_xchg(0, k + 1) && t <= *this; // XCHG s0,s(k+2) +} + +// PUSH s(i) ; XCHG s0,s1 ; XCHG s0,s(j+1) ; PUSH s(k+1) +bool StackTransform::is_puxcpu(int *i, int *j, int *k) const { + if (!is_valid() || d != -2 || dp < 1) { + return false; + } + StackTransform t = *this; + if (t.apply_pop() && t.is_puxc(i, j)) { + int y = get(0); + auto v = t.preimage(y); + if (!v.empty()) { + *k = v[0] - 1; + t.apply_push(*k + 1); + return t <= *this; + } + } + return false; +} + +// PUSH s(i) ; XCHG s0,s1 ; PUSH s(j+1) ; XCHG s0,s1 ; XCHG s0,s(k+2) +// 2 -> i; 1 -> j (if j >= 1, k != -1), 1 -> i (if j = 0, k != -1), 1 -> 0 (if j = -1, k != -1) +// 0 -> k (if k >= 1), 0 -> i (if k = 0), 0 -> j (if k = -1, j >= 1) +bool StackTransform::is_pu2xc(int *i, int *j, int *k) const { + if (!is_valid() || d != -2 || dp < 1) { + return false; + } + *i = get(2); + for (int v = -2; v <= 1; v++) { + *k = (v <= 0 ? v : get(0)); // one of -2, -1, 0, get(0) + for (int u = -1; u <= 1; u++) { + *j = (u <= 0 ? u : get(v != -1)); // one of -1, 0, get(0), get(1) + if (is_pu2xc(*i, *j, *k)) { + return true; + } + } + } + return false; +} + +bool StackTransform::is_pu2xc(int i, int j, int k) const { + StackTransform t; + return is_valid() && d == -2 && dp >= 1 // basic checks + && t.apply_push(i) && t.apply_xchg(0, 1) // PUSH s(i) ; XCHG s0,s1 + && t.apply_push(j + 1) && t.apply_xchg(0, 1) // PUSH s(j+1) ; XCHG s0,s1 + && t.apply_xchg(0, k + 2) && t <= *this; // XCHG s0,s(k+2) +} + +// PUSH s(i) ; PUSH s(j+1) ; PUSH s(k+2) +bool StackTransform::is_push3(int i, int j, int k) const { + StackTransform t; + return is_valid() && d == -3 && t.apply_push(i) && t.apply_push(j + 1) && t.apply_push(k + 2) && t <= *this; +} + +bool StackTransform::is_push3(int *i, int *j, int *k) const { + if (!is_valid() || d != -3 || n > 3) { + return false; + } + *i = get(2); + *j = get(1); + *k = get(0); + return is_push3(*i, *j, *k); +} + +bool StackTransform::is_blkswap(int *i, int *j) const { + if (!is_valid() || d || !is_permutation()) { + return false; + } + *j = get(0); + if (*j <= 0) { + return false; + } + auto v = preimage(0); + if (v.size() != 1) { + return false; + } + *i = v[0]; + return *i > 0 && is_blkswap(*i, *j); +} + +bool StackTransform::is_blkswap(int i, int j) const { + if (!is_valid() || d || i <= 0 || j <= 0 || dp < i + j || !is_trivial_after(i + j)) { + return false; + } + for (int s = 0; s < i; s++) { + if (get(s) != s + j) { + return false; + } + } + for (int s = 0; s < j; s++) { + if (get(s + i) != s) { + return false; + } + } + return true; +} + +// equivalent to i times DROP +bool StackTransform::is_blkdrop(int *i) const { + if (is_valid() && d > 0 && !n) { + *i = d; + return true; + } + return false; +} + +// 0 1 .. j-1 j+i j+i+1 ... +bool StackTransform::is_blkdrop2(int i, int j) const { + if (!is_valid() || d != i || i <= 0 || j < 0 || dp < i + j || n != j || !is_trivial_after(j)) { + return false; + } + for (int s = 0; s < j; s++) { + if (get(s) != s) { + return false; + } + } + return true; +} + +bool StackTransform::is_blkdrop2(int *i, int *j) const { + if (is_valid() && is_blkdrop2(d, n)) { + *i = d; + *j = n; + return true; + } + return false; +} + +// equivalent to i times PUSH s(j) +bool StackTransform::is_blkpush(int *i, int *j) const { + if (!is_valid() || d >= 0) { + return false; + } + *i = -d; + *j = get(*i - 1); + return is_blkpush(*i, *j); +} + +bool StackTransform::is_blkpush(int i, int j) const { + if (!is_valid() || d >= 0 || d != -i || j < 0 || dp < i + j || !is_trivial_after(i)) { + return false; + } + StackTransform t; + for (int s = 0; s < i; s++) { + if (!t.apply_push(j)) { + return false; + } + } + return t <= *this; +} + +bool StackTransform::is_reverse(int *i, int *j) const { + if (!is_valid() || d || !is_permutation() || n < 2) { + return false; + } + *j = A[0].first; + *i = A[n - 1].first - A[0].first + 1; + return is_reverse(*i, *j); +} + +bool StackTransform::is_reverse(int i, int j) const { + if (!is_valid() || d || !is_trivial_after(i + j) || n < 2 || A[0].first != j || A[n - 1].first != j + i - 1) { + return false; + } + for (int s = 0; s < i; s++) { + if (get(j + s) != j + i - 1 - s) { + return false; + } + } + return true; +} + +// 0 i+1 i+2 ... == i*NIP +// j i+1 i+2 ... == XCHG s(i),s(j) ; BLKDROP i +bool StackTransform::is_nip_seq(int i, int j) const { + return is_valid() && d == i && i > j && j >= 0 && n == 1 && A[0].first == i && A[0].second == j; +} + +bool StackTransform::is_nip_seq(int *i) const { + *i = d; + return is_nip_seq(*i); +} + +bool StackTransform::is_nip_seq(int *i, int *j) const { + if (is_valid() && n > 0) { + *i = d; + *j = A[0].second; + return is_nip_seq(*i, *j); + } else { + return false; + } +} + +// POP s(i); BLKDROP k (usually for i >= k >= 0) +bool StackTransform::is_pop_blkdrop(int i, int k) const { + StackTransform t; + return is_valid() && d == k + 1 && t.apply_pop(i) && t.apply_blkpop(k) && t <= *this; +} + +// POP s(i); BLKDROP k == XCHG s0,s(i); BLKDROP k+1 for i >= k >= 0 +// k+1 k+2 .. i-1 0 i+1 .. +bool StackTransform::is_pop_blkdrop(int *i, int *k) const { + if (is_valid() && n == 1 && d > 0 && !A[0].second) { + *k = d - 1; + *i = A[0].first; + return is_pop_blkdrop(*i, *k); + } else { + return false; + } +} + +// POP s(i); POP s(j); BLKDROP k (usually for i<>j >= k >= 0) +bool StackTransform::is_2pop_blkdrop(int i, int j, int k) const { + StackTransform t; + return is_valid() && d == k + 2 && t.apply_pop(i) && t.apply_pop(j) && t.apply_blkpop(k) && t <= *this; +} + +// POP s(i); POP s(j); BLKDROP k == XCHG s0,s(i); XCHG s1,s(j+1); BLKDROP k+2 (usually for i<>j >= k >= 2) +// k+2 k+3 .. i-1 0 i+1 ... j 1 j+2 ... +bool StackTransform::is_2pop_blkdrop(int *i, int *j, int *k) const { + if (is_valid() && n == 2 && d >= 2 && A[0].second + A[1].second == 1) { + *k = d - 2; + int t = (A[0].second > 0); + *i = A[t].first; + *j = A[1 - t].first - 1; + return is_2pop_blkdrop(*i, *j, *k); + } else { + return false; + } +} + +// PUSHCONST c ; ROT == 1 -1000 0 2 3 +bool StackTransform::is_const_rot(int c) const { + return is_valid() && d == -1 && is_trivial_after(3) && get(0) == 1 && c <= c_start && get(1) == c && get(2) == 0; +} + +bool StackTransform::is_const_rot(int *c) const { + return is_valid() && (*c = get(1)) <= c_start && is_const_rot(*c); +} + +// PUSHCONST c ; POP s(i) == 0 1 .. i-1 -1000 i+1 ... +bool StackTransform::is_const_pop(int c, int i) const { + return is_valid() && !d && n == 1 && i > 0 && c <= c_start && get(i - 1) == c; +} + +bool StackTransform::is_const_pop(int *c, int *i) const { + if (is_valid() && !d && n == 1 && A[0].second <= c_start) { + *i = A[0].first + 1; + *c = A[0].second; + return is_const_pop(*c, *i); + } else { + return false; + } +} + +// PUSH i ; PUSHCONST c == c i 0 1 2 ... +bool StackTransform::is_push_const(int i, int c) const { + return is_valid() && d == -2 && c <= c_start && i >= 0 && is_trivial_after(2) && get(0) == c && get(1) == i; +} + +bool StackTransform::is_push_const(int *i, int *c) const { + return is_valid() && d == -2 && n == 2 && is_push_const(*i = get(1), *c = get(0)); +} + +void StackTransform::show(std::ostream &os, int mode) const { + if (!is_valid()) { + os << ""; + return; + } + int mi = 0, ma = 0; + if (n > 0 && A[0].first < d) { + mi = A[0].first - d; + } + if (n > 0) { + ma = std::max(ma, A[n - 1].first - d + 1); + } + ma = std::max(ma + 1, dp - d); + os << '{'; + if (dp == d) { + os << '|'; + } + for (int i = mi; i < ma; i++) { + os << get(i) << (i == -1 ? '?' : (i == dp - d - 1 ? '|' : ' ')); + } + os << get(ma) << "..}"; +} + +} // namespace tolk diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp new file mode 100644 index 00000000..51dc3440 --- /dev/null +++ b/tolk/symtable.cpp @@ -0,0 +1,148 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "symtable.h" +#include "compiler-state.h" +#include "platform-utils.h" +#include "generics-helpers.h" + +namespace tolk { + +std::string FunctionData::as_human_readable() const { + if (!genericTs) { + return name; // if it's generic instantiation like `f`, its name is "f", not "f" + } + return name + genericTs->as_human_readable(); +} + +bool FunctionData::does_need_codegen() const { + // when a function is declared, but not referenced from code in any way, don't generate its body + if (!is_really_used() && G.settings.remove_unused_functions) { + return false; + } + // functions with asm body don't need code generation + // (even if used as non-call: `var a = beginCell;` inserts TVM continuation inline) + if (is_asm_function() || is_builtin_function()) { + return false; + } + // when a function is referenced like `var a = some_fn;` (or in some other non-call way), its continuation should exist + if (is_used_as_noncall()) { + return true; + } + // generic functions also don't need code generation, only generic instantiations do + if (is_generic_function()) { + return false; + } + // currently, there is no inlining, all functions are codegenerated + // (but actually, unused ones are later removed by Fift) + // in the future, we may want to implement a true AST inlining for "simple" functions + return true; +} + +void FunctionData::assign_resolved_type(TypePtr declared_return_type) { + this->declared_return_type = declared_return_type; +} + +void FunctionData::assign_inferred_type(TypePtr inferred_return_type, TypePtr inferred_full_type) { + this->inferred_return_type = inferred_return_type; + this->inferred_full_type = inferred_full_type; +} + +void FunctionData::assign_is_used_as_noncall() { + this->flags |= flagUsedAsNonCall; +} + +void FunctionData::assign_is_implicit_return() { + this->flags |= flagImplicitReturn; +} + +void FunctionData::assign_is_type_inferring_done() { + this->flags |= flagTypeInferringDone; +} + +void FunctionData::assign_is_really_used() { + this->flags |= flagReallyUsed; +} + +void FunctionData::assign_arg_order(std::vector&& arg_order) { + this->arg_order = std::move(arg_order); +} + +void GlobalVarData::assign_resolved_type(TypePtr declared_type) { + this->declared_type = declared_type; +} + +void GlobalVarData::assign_is_really_used() { + this->flags |= flagReallyUsed; +} + +void GlobalConstData::assign_resolved_type(TypePtr declared_type) { + this->declared_type = declared_type; +} + +void LocalVarData::assign_ir_idx(std::vector&& ir_idx) { + this->ir_idx = std::move(ir_idx); +} + +void LocalVarData::assign_resolved_type(TypePtr declared_type) { + this->declared_type = declared_type; +} + +void LocalVarData::assign_inferred_type(TypePtr inferred_type) { + this->declared_type = inferred_type; +} + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_redefinition_of_symbol(SrcLocation loc, const Symbol* previous) { + SrcLocation prev_loc = previous->loc; + if (prev_loc.is_stdlib()) { + throw ParseError(loc, "redefinition of a symbol from stdlib"); + } + if (prev_loc.is_defined()) { + throw ParseError(loc, "redefinition of symbol, previous was at: " + prev_loc.to_string()); + } + throw ParseError(loc, "redefinition of built-in symbol"); +} + +void GlobalSymbolTable::add_function(FunctionPtr f_sym) { + auto key = key_hash(f_sym->name); + auto [it, inserted] = entries.emplace(key, f_sym); + if (!inserted) { + fire_error_redefinition_of_symbol(f_sym->loc, it->second); + } +} + +void GlobalSymbolTable::add_global_var(GlobalVarPtr g_sym) { + auto key = key_hash(g_sym->name); + auto [it, inserted] = entries.emplace(key, g_sym); + if (!inserted) { + fire_error_redefinition_of_symbol(g_sym->loc, it->second); + } +} + +void GlobalSymbolTable::add_global_const(GlobalConstPtr c_sym) { + auto key = key_hash(c_sym->name); + auto [it, inserted] = entries.emplace(key, c_sym); + if (!inserted) { + fire_error_redefinition_of_symbol(c_sym->loc, it->second); + } +} + +const Symbol* lookup_global_symbol(std::string_view name) { + return G.symtable.lookup(name); +} + +} // namespace tolk diff --git a/tolk/symtable.h b/tolk/symtable.h new file mode 100644 index 00000000..9419afce --- /dev/null +++ b/tolk/symtable.h @@ -0,0 +1,239 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "src-file.h" +#include "fwd-declarations.h" +#include "constant-evaluator.h" +#include "crypto/common/refint.h" +#include +#include +#include + +namespace tolk { + +struct Symbol { + std::string name; + SrcLocation loc; + + Symbol(std::string name, SrcLocation loc) + : name(std::move(name)) + , loc(loc) { + } + + virtual ~Symbol() = default; + + template + ConstTPtr try_as() const { +#ifdef TOLK_DEBUG + assert(this != nullptr); +#endif + return dynamic_cast(this); + } +}; + +struct LocalVarData final : Symbol { + enum { + flagMutateParameter = 1, // parameter was declared with `mutate` keyword + flagImmutable = 2, // variable was declared via `val` (not `var`) + }; + + TypePtr declared_type; // either at declaration `var x:int`, or if omitted, from assigned value `var x=2` + int flags; + int param_idx; // 0...N for function parameters, -1 for local vars + std::vector ir_idx; + + LocalVarData(std::string name, SrcLocation loc, TypePtr declared_type, int flags, int param_idx) + : Symbol(std::move(name), loc) + , declared_type(declared_type) + , flags(flags) + , param_idx(param_idx) { + } + + bool is_parameter() const { return param_idx >= 0; } + + bool is_immutable() const { return flags & flagImmutable; } + bool is_mutate_parameter() const { return flags & flagMutateParameter; } + + LocalVarData* mutate() const { return const_cast(this); } + void assign_ir_idx(std::vector&& ir_idx); + void assign_resolved_type(TypePtr declared_type); + void assign_inferred_type(TypePtr inferred_type); +}; + +struct FunctionBodyCode; +struct FunctionBodyAsm; +struct FunctionBodyBuiltin; +struct GenericsDeclaration; +struct GenericsInstantiation; + +typedef std::variant< + FunctionBodyCode*, + FunctionBodyAsm*, + FunctionBodyBuiltin* +> FunctionBody; + +struct FunctionData final : Symbol { + static constexpr int EMPTY_METHOD_ID = -10; + + enum { + flagInline = 1, // marked `@inline` + flagInlineRef = 2, // marked `@inline_ref` + flagTypeInferringDone = 4, // type inferring step of function's body (all AST nodes assigning v->inferred_type) is done + flagUsedAsNonCall = 8, // used not only as `f()`, but as a 1-st class function (assigned to var, pushed to tuple, etc.) + flagMarkedAsPure = 16, // declared as `pure`, can't call impure and access globals, unused invocations are optimized out + flagImplicitReturn = 32, // control flow reaches end of function, so it needs implicit return at the end + flagGetMethod = 64, // was declared via `get func(): T`, method_id is auto-assigned + flagIsEntrypoint = 128, // it's `main` / `onExternalMessage` / etc. + flagHasMutateParams = 256, // has parameters declared as `mutate` + flagAcceptsSelf = 512, // is a member function (has `self` first parameter) + flagReturnsSelf = 1024, // return type is `self` (returns the mutated 1st argument), calls can be chainable + flagReallyUsed = 2048, // calculated via dfs from used functions; declared but unused functions are not codegenerated + }; + + int method_id = EMPTY_METHOD_ID; + int flags; + + std::vector parameters; + std::vector arg_order, ret_order; + TypePtr declared_return_type; // may be nullptr, meaning "auto infer" + TypePtr inferred_return_type = nullptr; // assigned on type inferring + TypePtr inferred_full_type = nullptr; // assigned on type inferring, it's TypeDataFunCallable(params -> return) + + const GenericsDeclaration* genericTs; + const GenericsInstantiation* instantiationTs; + FunctionBody body; + AnyV ast_root; // V for user-defined (not builtin) + + FunctionData(std::string name, SrcLocation loc, TypePtr declared_return_type, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsInstantiation* instantiationTs, FunctionBody body, AnyV ast_root) + : Symbol(std::move(name), loc) + , flags(initial_flags) + , parameters(std::move(parameters)) + , declared_return_type(declared_return_type) + , genericTs(genericTs) + , instantiationTs(instantiationTs) + , body(body) + , ast_root(ast_root) { + } + + std::string as_human_readable() const; + + const std::vector* get_arg_order() const { + return arg_order.empty() ? nullptr : &arg_order; + } + const std::vector* get_ret_order() const { + return ret_order.empty() ? nullptr : &ret_order; + } + + int get_num_params() const { return static_cast(parameters.size()); } + const LocalVarData& get_param(int idx) const { return parameters[idx]; } + + bool is_code_function() const { return std::holds_alternative(body); } + bool is_asm_function() const { return std::holds_alternative(body); } + bool is_builtin_function() const { return ast_root == nullptr; } + + bool is_generic_function() const { return genericTs != nullptr; } + bool is_instantiation_of_generic_function() const { return instantiationTs != nullptr; } + + bool is_inline() const { return flags & flagInline; } + bool is_inline_ref() const { return flags & flagInlineRef; } + bool is_type_inferring_done() const { return flags & flagTypeInferringDone; } + bool is_used_as_noncall() const { return flags & flagUsedAsNonCall; } + bool is_marked_as_pure() const { return flags & flagMarkedAsPure; } + bool is_implicit_return() const { return flags & flagImplicitReturn; } + bool is_get_method() const { return flags & flagGetMethod; } + bool is_method_id_not_empty() const { return method_id != EMPTY_METHOD_ID; } + bool is_entrypoint() const { return flags & flagIsEntrypoint; } + bool has_mutate_params() const { return flags & flagHasMutateParams; } + bool does_accept_self() const { return flags & flagAcceptsSelf; } + bool does_return_self() const { return flags & flagReturnsSelf; } + bool does_mutate_self() const { return (flags & flagAcceptsSelf) && parameters[0].is_mutate_parameter(); } + bool is_really_used() const { return flags & flagReallyUsed; } + + bool does_need_codegen() const; + + FunctionData* mutate() const { return const_cast(this); } + void assign_resolved_type(TypePtr declared_return_type); + void assign_inferred_type(TypePtr inferred_return_type, TypePtr inferred_full_type); + void assign_is_used_as_noncall(); + void assign_is_implicit_return(); + void assign_is_type_inferring_done(); + void assign_is_really_used(); + void assign_arg_order(std::vector&& arg_order); +}; + +struct GlobalVarData final : Symbol { + enum { + flagReallyUsed = 1, // calculated via dfs from used functions; unused globals are not codegenerated + }; + + TypePtr declared_type; // always exists, declaring globals without type is prohibited + int flags = 0; + + GlobalVarData(std::string name, SrcLocation loc, TypePtr declared_type) + : Symbol(std::move(name), loc) + , declared_type(declared_type) { + } + + bool is_really_used() const { return flags & flagReallyUsed; } + + GlobalVarData* mutate() const { return const_cast(this); } + void assign_resolved_type(TypePtr declared_type); + void assign_is_really_used(); +}; + +struct GlobalConstData final : Symbol { + ConstantValue value; + TypePtr declared_type; // may be nullptr + + GlobalConstData(std::string name, SrcLocation loc, TypePtr declared_type, ConstantValue&& value) + : Symbol(std::move(name), loc) + , value(std::move(value)) + , declared_type(declared_type) { + } + + bool is_int_const() const { return value.is_int(); } + bool is_slice_const() const { return value.is_slice(); } + + td::RefInt256 as_int_const() const { return value.as_int(); } + const std::string& as_slice_const() const { return value.as_slice(); } + + GlobalConstData* mutate() const { return const_cast(this); } + void assign_resolved_type(TypePtr declared_type); +}; + +class GlobalSymbolTable { + std::unordered_map entries; + + static uint64_t key_hash(std::string_view name_key) { + return std::hash{}(name_key); + } + +public: + void add_function(FunctionPtr f_sym); + void add_global_var(GlobalVarPtr g_sym); + void add_global_const(GlobalConstPtr c_sym); + + const Symbol* lookup(std::string_view name) const { + const auto it = entries.find(key_hash(name)); + return it == entries.end() ? nullptr : it->second; + } +}; + +const Symbol* lookup_global_symbol(std::string_view name); + +} // namespace tolk diff --git a/tolk/tolk-main.cpp b/tolk/tolk-main.cpp new file mode 100644 index 00000000..7f939670 --- /dev/null +++ b/tolk/tolk-main.cpp @@ -0,0 +1,285 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#include "tolk.h" +#include "tolk-version.h" +#include "compiler-state.h" +#include "td/utils/port/path.h" +#include +#include +#include +#ifdef TD_DARWIN +#include +#elif TD_WINDOWS +#include +#else // linux +#include +#endif +#include "git.h" + +using namespace tolk; + +void usage(const char* progname) { + std::cerr + << "usage: " << progname << " [options] \n" + "\tGenerates Fift TVM assembler code from a .tolk file\n" + "-o\tWrites generated code into specified .fif file instead of stdout\n" + "-b\tGenerate Fift instructions to save TVM bytecode into .boc file\n" + "-O\tSets optimization level (2 by default)\n" + "-x\tEnables experimental options, comma-separated\n" + "-S\tDon't include stack layout comments into Fift output\n" + "-e\tIncreases verbosity level (extra output into stderr)\n" + "-v\tOutput version of Tolk and exit\n"; + std::exit(2); +} + +static bool stdlib_folder_exists(const char* stdlib_folder) { + struct stat f_stat; + int res = stat(stdlib_folder, &f_stat); + return res == 0 && (f_stat.st_mode & S_IFMT) == S_IFDIR; +} + +// getting current executable path is a complicated and not cross-platform task +// for instance, we can't just use argv[0] or even filesystem::canonical +// https://stackoverflow.com/questions/1023306/finding-current-executables-path-without-proc-self-exe/1024937 +static bool get_current_executable_filename(std::string& out) { +#ifdef TD_DARWIN + char name_buf[1024]; + unsigned int size = 1024; + if (0 == _NSGetExecutablePath(name_buf, &size)) { // may contain ../, so normalize it + char *exe_path = realpath(name_buf, nullptr); + if (exe_path != nullptr) { + out = exe_path; + return true; + } + } +#elif TD_WINDOWS + char exe_path[1024]; + if (GetModuleFileNameA(nullptr, exe_path, 1024)) { + out = exe_path; + std::replace(out.begin(), out.end(), '\\', '/'); // modern Windows correctly deals with / separator + return true; + } +#else // linux + char exe_path[1024]; + ssize_t res = readlink("/proc/self/exe", exe_path, 1024 - 1); + if (res >= 0) { + exe_path[res] = 0; + out = exe_path; + return true; + } +#endif + return false; +} + +// simple join "/some/folder/" (guaranteed to end with /) and "../relative/path" +static std::string join_path(std::string dir, const char* relative) { + while (relative[0] == '.' && relative[1] == '.' && relative[2] == '/') { + size_t slash_pos = dir.find_last_of('/', dir.size() - 2); // last symbol is slash, find before it + if (slash_pos != std::string::npos) { + dir = dir.substr(0, slash_pos + 1); + } + relative += 3; + } + + return dir + relative; +} + +static std::string auto_discover_stdlib_folder() { + // if the user launches tolk compiler from a package installed (e.g. /usr/bin/tolk), + // locate stdlib in /usr/share/ton/smartcont (this folder exists on package installation) + // (note, that paths are not absolute, they are relative to the launched binary) + // consider https://github.com/ton-blockchain/packages for actual paths + std::string executable_filename; + if (!get_current_executable_filename(executable_filename)) { + return {}; + } + + // extract dirname to concatenate with relative paths (separator / is ok even for windows) + size_t slash_pos = executable_filename.find_last_of('/'); + std::string executable_dir = executable_filename.substr(0, slash_pos + 1); + +#ifdef TD_DARWIN + std::string def_location = join_path(executable_dir, "../share/ton/ton/smartcont/tolk-stdlib"); +#elif TD_WINDOWS + std::string def_location = join_path(executable_dir, "smartcont/tolk-stdlib"); +#else // linux + std::string def_location = join_path(executable_dir, "../share/ton/smartcont/tolk-stdlib"); +#endif + + if (stdlib_folder_exists(def_location.c_str())) { + return def_location; + } + + // so, the binary is not from a system package + // maybe it's just built from sources? e.g. ~/ton/cmake-build-debug/tolk/tolk + // then, check the ~/ton/crypto/smartcont folder + std::string near_when_built_from_sources = join_path(executable_dir, "../../crypto/smartcont/tolk-stdlib"); + if (stdlib_folder_exists(near_when_built_from_sources.c_str())) { + return near_when_built_from_sources; + } + + // no idea of where to find stdlib; let's show an error for the user, he should provide env var above + return {}; +} + +td::Result fs_read_callback(CompilerSettings::FsReadCallbackKind kind, const char* query) { + switch (kind) { + case CompilerSettings::FsReadCallbackKind::Realpath: { + td::Result res_realpath; + if (query[0] == '@' && strlen(query) > 8 && !strncmp(query, "@stdlib/", 8)) { + // import "@stdlib/filename" or import "@stdlib/filename.tolk" + std::string path = G.settings.stdlib_folder + static_cast(query + 7); + if (strncmp(path.c_str() + path.size() - 5, ".tolk", 5) != 0) { + path += ".tolk"; + } + res_realpath = td::realpath(td::CSlice(path.c_str())); + } else { + // import "relative/to/cwd/path.tolk" + res_realpath = td::realpath(td::CSlice(query)); + } + + if (res_realpath.is_error()) { + // note, that for non-existing files, `realpath()` on Linux/Mac returns an error, + // whereas on Windows, it returns okay, but fails after, on reading, with a message "cannot open file" + return td::Status::Error(std::string{"cannot find file "} + query); + } + return res_realpath; + } + case CompilerSettings::FsReadCallbackKind::ReadFile: { + struct stat f_stat; + int res = stat(query, &f_stat); // query here is already resolved realpath + if (res != 0 || (f_stat.st_mode & S_IFMT) != S_IFREG) { + return td::Status::Error(std::string{"cannot open file "} + query); + } + + size_t file_size = static_cast(f_stat.st_size); + std::string str; + str.resize(file_size); + FILE* f = fopen(query, "rb"); + fread(str.data(), file_size, 1, f); + fclose(f); + return std::move(str); + } + default: { + return td::Status::Error("unknown query kind"); + } + } +} + +class StdCoutRedirectToFile { + std::unique_ptr output_file; + std::streambuf* backup_sbuf = nullptr; + +public: + explicit StdCoutRedirectToFile(const std::string& output_filename) { + if (!output_filename.empty()) { + output_file = std::make_unique(output_filename, std::fstream::trunc | std::fstream::out); + if (output_file->is_open()) { + backup_sbuf = std::cout.rdbuf(output_file->rdbuf()); + } + } + } + + ~StdCoutRedirectToFile() { + if (backup_sbuf) { + std::cout.rdbuf(backup_sbuf); + } + } + + bool is_failed() const { return output_file && !output_file->is_open(); } +}; + +int main(int argc, char* const argv[]) { + int i; + while ((i = getopt(argc, argv, "o:b:O:x:Sevh")) != -1) { + switch (i) { + case 'o': + G.settings.output_filename = optarg; + break; + case 'b': + G.settings.boc_output_filename = optarg; + break; + case 'O': + G.settings.optimization_level = std::max(0, atoi(optarg)); + break; + case 'x': + G.settings.parse_experimental_options_cmd_arg(optarg); + break; + case 'S': + G.settings.stack_layout_comments = false; + break; + case 'e': + G.settings.verbosity++; + break; + case 'v': + std::cout << "Tolk compiler v" << TOLK_VERSION << std::endl; + std::cout << "Build commit: " << GitMetadata::CommitSHA1() << std::endl; + std::cout << "Build date: " << GitMetadata::CommitDate() << std::endl; + std::exit(0); + case 'h': + default: + usage(argv[0]); + } + } + + StdCoutRedirectToFile redirect_cout(G.settings.output_filename); + if (redirect_cout.is_failed()) { + std::cerr << "Failed to create output file " << G.settings.output_filename << std::endl; + return 2; + } + + // locate tolk-stdlib/ based on env or default system paths + if (const char* env_var = getenv("TOLK_STDLIB")) { + std::string stdlib_filename = static_cast(env_var) + "/common.tolk"; + td::Result res = td::realpath(td::CSlice(stdlib_filename.c_str())); + if (res.is_error()) { + std::cerr << "Environment variable TOLK_STDLIB is invalid: " << res.move_as_error().message().c_str() << std::endl; + return 2; + } + G.settings.stdlib_folder = env_var; + } else { + G.settings.stdlib_folder = auto_discover_stdlib_folder(); + } + if (G.settings.stdlib_folder.empty()) { + std::cerr << "Failed to discover Tolk stdlib.\n" + "Probably, you have a non-standard Tolk installation.\n" + "Please, provide env variable TOLK_STDLIB referencing to tolk-stdlib/ folder.\n"; + return 2; + } + if (G.is_verbosity(2)) { + std::cerr << "stdlib folder: " << G.settings.stdlib_folder << std::endl; + } + + if (optind != argc - 1) { + std::cerr << "invalid usage: should specify exactly one input file.tolk" << std::endl; + return 2; + } + + G.settings.read_callback = fs_read_callback; + + int exit_code = tolk_proceed(argv[optind]); + return exit_code; +} diff --git a/tonlib/tonlib/ClientActor.cpp b/tolk/tolk-version.h similarity index 88% rename from tonlib/tonlib/ClientActor.cpp rename to tolk/tolk-version.h index ab2bc125..bbea63ff 100644 --- a/tonlib/tonlib/ClientActor.cpp +++ b/tolk/tolk-version.h @@ -13,6 +13,11 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP */ +#pragma once + +namespace tolk { + +constexpr const char* TOLK_VERSION = "0.9.0"; + +} // namespace tolk diff --git a/tolk/tolk-wasm.cpp b/tolk/tolk-wasm.cpp new file mode 100644 index 00000000..e74589ce --- /dev/null +++ b/tolk/tolk-wasm.cpp @@ -0,0 +1,127 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#include "tolk.h" +#include "tolk-version.h" +#include "compiler-state.h" +#include "git.h" +#include "td/utils/JsonBuilder.h" +#include "fift/utils.h" +#include "td/utils/Status.h" +#include + +using namespace tolk; + +static td::Result compile_internal(char *config_json) { + TRY_RESULT(input_json, td::json_decode(td::MutableSlice(config_json))) + td::JsonObject& config = input_json.get_object(); + + TRY_RESULT(opt_level, td::get_json_object_int_field(config, "optimizationLevel", true, 2)); + TRY_RESULT(stack_comments, td::get_json_object_bool_field(config, "withStackComments", true, false)); + TRY_RESULT(entrypoint_filename, td::get_json_object_string_field(config, "entrypointFileName", false)); + TRY_RESULT(experimental_options, td::get_json_object_string_field(config, "experimentalOptions", true)); + + G.settings.verbosity = 0; + G.settings.optimization_level = std::max(0, opt_level); + G.settings.stack_layout_comments = stack_comments; + if (!experimental_options.empty()) { + G.settings.parse_experimental_options_cmd_arg(experimental_options.c_str()); + } + + std::ostringstream outs, errs; + std::cout.rdbuf(outs.rdbuf()); + std::cerr.rdbuf(errs.rdbuf()); + int exit_code = tolk_proceed(entrypoint_filename); + if (exit_code != 0) { + return td::Status::Error("Tolk compilation error: " + errs.str()); + } + + TRY_RESULT(fift_res, fift::compile_asm_program(outs.str(), "/fiftlib/")); + + td::JsonBuilder result_json; + auto obj = result_json.enter_object(); + obj("status", "ok"); + obj("fiftCode", fift_res.fiftCode); + obj("codeBoc64", fift_res.codeBoc64); + obj("codeHashHex", fift_res.codeHashHex); + obj("stderr", errs.str().c_str()); + obj.leave(); + + return result_json.string_builder().as_cslice().str(); +} + +/// Callback used to retrieve file contents from a "not file system". See tolk-js for implementation. +/// The callback must fill either destContents or destError. +/// The implementor must use malloc() for them and use free() after tolk_compile returns. +typedef void (*WasmFsReadCallback)(int kind, char const* data, char** destContents, char** destError); + +static CompilerSettings::FsReadCallback wrap_wasm_read_callback(WasmFsReadCallback _readCallback) { + return [_readCallback](CompilerSettings::FsReadCallbackKind kind, char const* data) -> td::Result { + char* destContents = nullptr; + char* destError = nullptr; + if (_readCallback) { + _readCallback(static_cast(kind), data, &destContents, &destError); + } + if (destContents) { + return destContents; + } + if (destError) { + return td::Status::Error(std::string(destError)); + } + return td::Status::Error("Invalid callback from wasm"); + }; +} + +extern "C" { + +const char* version() { + td::JsonBuilder version_json = td::JsonBuilder(); + auto obj = version_json.enter_object(); + obj("tolkVersion", TOLK_VERSION); + obj("tolkFiftLibCommitHash", GitMetadata::CommitSHA1()); + obj("tolkFiftLibCommitDate", GitMetadata::CommitDate()); + obj.leave(); + return strdup(version_json.string_builder().as_cslice().c_str()); +} + +const char *tolk_compile(char *config_json, WasmFsReadCallback callback) { + G.settings.read_callback = wrap_wasm_read_callback(callback); + + td::Result res = compile_internal(config_json); + + if (res.is_error()) { + td::JsonBuilder error_res = td::JsonBuilder(); + auto obj = error_res.enter_object(); + obj("status", "error"); + obj("message", res.move_as_error().message().str()); + obj.leave(); + return strdup(error_res.string_builder().as_cslice().c_str()); + } + + std::string res_string = res.move_as_ok(); + return strdup(res_string.c_str()); +} + +} // extern "C" diff --git a/tolk/tolk.cpp b/tolk/tolk.cpp new file mode 100644 index 00000000..71d1969d --- /dev/null +++ b/tolk/tolk.cpp @@ -0,0 +1,87 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#include "tolk.h" +#include "pipeline.h" +#include "compiler-state.h" +#include "lexer.h" +#include "ast.h" +#include "type-system.h" + +namespace tolk { + + +void on_assertion_failed(const char *description, const char *file_name, int line_number) { + std::string message = static_cast("Assertion failed at ") + file_name + ":" + std::to_string(line_number) + ": " + description; +#ifdef TOLK_DEBUG +#ifdef __arm64__ + // when developing, it's handy when the debugger stops on assertion failure (stacktraces and watches are available) + std::cerr << message << std::endl; + __builtin_debugtrap(); +#endif +#endif + throw Fatal(std::move(message)); +} + +int tolk_proceed(const std::string &entrypoint_filename) { + type_system_init(); + define_builtins(); + lexer_init(); + + // on any error, an exception is thrown, and the message is printed out below + // (currently, only a single error can be printed) + try { + pipeline_discover_and_parse_sources("@stdlib/common.tolk", entrypoint_filename); + + pipeline_register_global_symbols(); + pipeline_resolve_identifiers_and_assign_symbols(); + pipeline_calculate_rvalue_lvalue(); + pipeline_infer_types_and_calls_and_fields(); + pipeline_check_inferred_types(); + pipeline_refine_lvalue_for_mutate_arguments(); + pipeline_check_rvalue_lvalue(); + pipeline_check_pure_impure_operations(); + pipeline_constant_folding(); + pipeline_optimize_boolean_expressions(); + pipeline_convert_ast_to_legacy_Expr_Op(); + + pipeline_find_unused_symbols(); + pipeline_generate_fif_output_to_std_cout(); + + return 0; + } catch (Fatal& fatal) { + std::cerr << "fatal: " << fatal << std::endl; + return 2; + } catch (ParseError& error) { + std::cerr << error << std::endl; + return 2; + } catch (UnexpectedASTNodeType& error) { + std::cerr << "fatal: " << error.what() << std::endl; + std::cerr << "It's a compiler bug, please report to developers" << std::endl; + return 2; + } +} + +} // namespace tolk diff --git a/tolk/tolk.h b/tolk/tolk.h new file mode 100644 index 00000000..3f00d0d4 --- /dev/null +++ b/tolk/tolk.h @@ -0,0 +1,1166 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "platform-utils.h" +#include "src-file.h" +#include "symtable.h" +#include "crypto/common/refint.h" +#include "td/utils/Status.h" +#include +#include +#include +#include +#include + +#define tolk_assert(expr) if(UNLIKELY(!(expr))) on_assertion_failed(#expr, __FILE__, __LINE__); + +namespace tolk { + +GNU_ATTRIBUTE_COLD GNU_ATTRIBUTE_NORETURN +void on_assertion_failed(const char *description, const char *file_name, int line_number); + +/* + * + * ABSTRACT CODE + * + */ + +typedef int var_idx_t; +typedef int const_idx_t; + +struct TmpVar { + var_idx_t ir_idx; // every var in IR represents 1 stack slot + TypePtr v_type; // get_width_on_stack() is 1 + std::string name; // "x" for vars originated from user sources; "x.0" for tensor components; empty for implicitly created tmp vars + SrcLocation loc; // location of var declaration in sources or where a tmp var was originated +#ifdef TOLK_DEBUG + const char* desc = nullptr; // "origin" of tmp var, for debug output like `'15 (binary-op) '16 (glob-var)` +#endif + + TmpVar(var_idx_t ir_idx, TypePtr v_type, std::string name, SrcLocation loc) + : ir_idx(ir_idx) + , v_type(v_type) + , name(std::move(name)) + , loc(loc) { + } + + void show_as_stack_comment(std::ostream& os) const; + void show(std::ostream& os) const; +}; + +struct VarDescr { + var_idx_t idx; + enum { _Last = 1, _Unused = 2 }; + int flags; + enum { + _Const = 16, + _Int = 32, + _Zero = 64, + _NonZero = 128, + _Pos = 256, + _Neg = 512, + _Finite = 4096, + _Nan = 8192, + _Even = 16384, + _Odd = 32768, + }; + static constexpr int ConstZero = _Const | _Int | _Zero | _Pos | _Neg | _Finite | _Even; + static constexpr int ConstOne = _Const | _Int | _NonZero | _Pos | _Finite | _Odd; + static constexpr int ConstTrue = _Const | _Int | _NonZero | _Neg | _Finite | _Odd; + static constexpr int ValBit = _Int | _Pos | _Finite; + static constexpr int ValBool = _Int | _Neg | _Finite; + static constexpr int FiniteInt = _Int | _Finite; + static constexpr int FiniteUInt = _Int | _Finite | _Pos; + int val; + td::RefInt256 int_const; + std::string str_const; + + explicit VarDescr(var_idx_t _idx = -1, int _flags = 0, int _val = 0) : idx(_idx), flags(_flags), val(_val) { + } + bool operator<(var_idx_t other_idx) const { + return idx < other_idx; + } + bool is_unused() const { + return flags & _Unused; + } + bool is_last() const { + return flags & _Last; + } + bool always_true() const { + return val & _NonZero; + } + bool always_false() const { + return val & _Zero; + } + bool always_nonzero() const { + return val & _NonZero; + } + bool always_zero() const { + return val & _Zero; + } + bool always_even() const { + return val & _Even; + } + bool always_odd() const { + return val & _Odd; + } + bool is_int_const() const { + return (val & (_Int | _Const)) == (_Int | _Const) && int_const.not_null(); + } + bool always_nonpos() const { + return val & _Neg; + } + bool always_nonneg() const { + return val & _Pos; + } + bool always_pos() const { + return (val & (_Pos | _NonZero)) == (_Pos | _NonZero); + } + bool always_neg() const { + return (val & (_Neg | _NonZero)) == (_Neg | _NonZero); + } + bool always_finite() const { + return val & _Finite; + } + bool always_less(const VarDescr& other) const; + bool always_leq(const VarDescr& other) const; + bool always_greater(const VarDescr& other) const; + bool always_geq(const VarDescr& other) const; + bool always_equal(const VarDescr& other) const; + bool always_neq(const VarDescr& other) const; + void unused() { + flags |= _Unused; + } + void clear_unused() { + flags &= ~_Unused; + } + void set_const(long long value); + void set_const(td::RefInt256 value); + void set_const(std::string value); + void operator+=(const VarDescr& y) { + flags &= y.flags; + } + void operator|=(const VarDescr& y); + void operator&=(const VarDescr& y); + void set_value(const VarDescr& y); + void set_value(VarDescr&& y); + void set_value(const VarDescr* y) { + if (y) { + set_value(*y); + } + } + void clear_value(); + void show_value(std::ostream& os) const; + void show(std::ostream& os, const char* var_name = nullptr) const; +}; + +inline std::ostream& operator<<(std::ostream& os, const VarDescr& vd) { + vd.show(os); + return os; +} + +struct VarDescrList { + std::vector list; + bool unreachable{false}; + VarDescrList() : list() { + } + VarDescrList(const std::vector& _list) : list(_list) { + } + VarDescrList(std::vector&& _list) : list(std::move(_list)) { + } + std::size_t size() const { + return list.size(); + } + VarDescr* operator[](var_idx_t idx); + const VarDescr* operator[](var_idx_t idx) const; + VarDescrList operator+(const VarDescrList& y) const; + VarDescrList& operator+=(const VarDescrList& y); + VarDescrList& clear_last(); + VarDescrList& operator+=(var_idx_t idx) { + return add_var(idx); + } + VarDescrList& operator+=(const std::vector& idx_list) { + return add_vars(idx_list); + } + VarDescrList& add_var(var_idx_t idx, bool unused = false); + VarDescrList& add_vars(const std::vector& idx_list, bool unused = false); + VarDescrList& operator-=(const std::vector& idx_list); + VarDescrList& operator-=(var_idx_t idx); + std::size_t count(const std::vector idx_list) const; + std::size_t count_used(const std::vector idx_list) const; + VarDescr& add(var_idx_t idx); + VarDescr& add_newval(var_idx_t idx); + VarDescrList& import_values(const VarDescrList& values); + VarDescrList operator|(const VarDescrList& y) const; + VarDescrList& operator|=(const VarDescrList& values); + void show(std::ostream& os) const; + void set_unreachable() { + list.clear(); + unreachable = true; + } +}; + +inline std::ostream& operator<<(std::ostream& os, const VarDescrList& values) { + values.show(os); + return os; +} + +struct CodeBlob; + +template +class ListIterator { + T* ptr; + + public: + ListIterator() : ptr(nullptr) { + } + explicit ListIterator(T* _ptr) : ptr(_ptr) { + } + ListIterator& operator++() { + ptr = ptr->next.get(); + return *this; + } + ListIterator operator++(int) { + T* z = ptr; + ptr = ptr->next.get(); + return ListIterator{z}; + } + T& operator*() const { + return *ptr; + } + T* operator->() const { + return ptr; + } + bool operator==(const ListIterator& y) const { + return ptr == y.ptr; + } + bool operator!=(const ListIterator& y) const { + return ptr != y.ptr; + } +}; + +struct Stack; + +struct Op { + enum OpKind { + _Undef, + _Nop, + _Call, + _CallInd, + _Let, + _IntConst, + _GlobVar, + _SetGlob, + _Import, + _Return, + _Tuple, + _UnTuple, + _If, + _While, + _Until, + _Repeat, + _Again, + _TryCatch, + _SliceConst, + }; + OpKind cl; + enum { _Disabled = 1, _NoReturn = 4, _Impure = 24 }; + int flags; + std::unique_ptr next; + FunctionPtr f_sym = nullptr; + GlobalVarPtr g_sym = nullptr; + SrcLocation where; + VarDescrList var_info; + std::vector args; + std::vector left, right; + std::unique_ptr block0, block1; + td::RefInt256 int_const; + std::string str_const; + Op(SrcLocation _where = {}, OpKind _cl = _Undef) : cl(_cl), flags(0), f_sym(nullptr), where(_where) { + } + Op(SrcLocation _where, OpKind _cl, const std::vector& _left) + : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(_left) { + } + Op(SrcLocation _where, OpKind _cl, std::vector&& _left) + : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(std::move(_left)) { + } + Op(SrcLocation _where, OpKind _cl, const std::vector& _left, td::RefInt256 _const) + : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(_left), int_const(_const) { + } + Op(SrcLocation _where, OpKind _cl, const std::vector& _left, std::string _const) + : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(_left), str_const(_const) { + } + Op(SrcLocation _where, OpKind _cl, const std::vector& _left, const std::vector& _right) + : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(_left), right(_right) { + } + Op(SrcLocation _where, OpKind _cl, std::vector&& _left, std::vector&& _right) + : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(std::move(_left)), right(std::move(_right)) { + } + Op(SrcLocation _where, OpKind _cl, const std::vector& _left, const std::vector& _right, + FunctionPtr _fun) + : cl(_cl), flags(0), f_sym(_fun), where(_where), left(_left), right(_right) { + } + Op(SrcLocation _where, OpKind _cl, std::vector&& _left, std::vector&& _right, + FunctionPtr _fun) + : cl(_cl), flags(0), f_sym(_fun), where(_where), left(std::move(_left)), right(std::move(_right)) { + } + Op(SrcLocation _where, OpKind _cl, const std::vector& _left, const std::vector& _right, + GlobalVarPtr _gvar) + : cl(_cl), flags(0), g_sym(_gvar), where(_where), left(_left), right(_right) { + } + Op(SrcLocation _where, OpKind _cl, std::vector&& _left, std::vector&& _right, + GlobalVarPtr _gvar) + : cl(_cl), flags(0), g_sym(_gvar), where(_where), left(std::move(_left)), right(std::move(_right)) { + } + + bool disabled() const { return flags & _Disabled; } + void set_disabled() { flags |= _Disabled; } + void set_disabled(bool flag); + + bool noreturn() const { return flags & _NoReturn; } + bool set_noreturn() { flags |= _NoReturn; return true; } + bool set_noreturn(bool flag); + + bool impure() const { return flags & _Impure; } + void set_impure_flag(); + + void show(std::ostream& os, const std::vector& vars, std::string pfx = "", int mode = 0) const; + void show_var_list(std::ostream& os, const std::vector& idx_list, const std::vector& vars) const; + void show_var_list(std::ostream& os, const std::vector& list, const std::vector& vars) const; + static void show_block(std::ostream& os, const Op* block, const std::vector& vars, std::string pfx = "", + int mode = 0); + bool compute_used_vars(const CodeBlob& code, bool edit); + bool std_compute_used_vars(bool disabled = false); + bool set_var_info(const VarDescrList& new_var_info); + bool set_var_info(VarDescrList&& new_var_info); + bool set_var_info_except(const VarDescrList& new_var_info, const std::vector& var_list); + bool set_var_info_except(VarDescrList&& new_var_info, const std::vector& var_list); + void prepare_args(VarDescrList values); + VarDescrList fwd_analyze(VarDescrList values); + bool mark_noreturn(); + bool is_empty() const { + return cl == _Nop && !next; + } + bool generate_code_step(Stack& stack); + void generate_code_all(Stack& stack); + Op& last() { + return next ? next->last() : *this; + } + const Op& last() const { + return next ? next->last() : *this; + } +}; + +inline ListIterator begin(const std::unique_ptr& op_list) { + return ListIterator{op_list.get()}; +} + +inline ListIterator end(const std::unique_ptr& op_list) { + return ListIterator{}; +} + +inline ListIterator begin(const Op* op_list) { + return ListIterator{op_list}; +} + +inline ListIterator end(const Op* op_list) { + return ListIterator{}; +} + +struct AsmOpList; + +struct FunctionBodyCode { + CodeBlob* code = nullptr; + void set_code(CodeBlob* code); +}; + +/* + * + * GENERATE CODE + * + */ + +typedef std::vector StackLayout; +typedef std::pair var_const_idx_t; +typedef std::vector StackLayoutExt; +constexpr const_idx_t not_const = -1; +using Const = td::RefInt256; + +struct AsmOp { + enum Type { a_none, a_xchg, a_push, a_pop, a_const, a_custom, a_magic }; + Type t{a_none}; + int indent{0}; + int a, b; + bool gconst{false}; + std::string op; + struct SReg { + int idx; + SReg(int _idx) : idx(_idx) { + } + }; + AsmOp() = default; + AsmOp(Type _t) : t(_t) { + } + AsmOp(Type _t, std::string _op) : t(_t), op(std::move(_op)) { + } + AsmOp(Type _t, int _a) : t(_t), a(_a) { + } + AsmOp(Type _t, int _a, std::string _op) : t(_t), a(_a), op(std::move(_op)) { + } + AsmOp(Type _t, int _a, int _b) : t(_t), a(_a), b(_b) { + } + AsmOp(Type _t, int _a, int _b, std::string _op) : t(_t), a(_a), b(_b), op(std::move(_op)) { + compute_gconst(); + } + void out(std::ostream& os) const; + void out_indent_nl(std::ostream& os, bool no_nl = false) const; + std::string to_string() const; + void compute_gconst() { + gconst = (is_custom() && (op == "PUSHNULL" || op == "NEWC" || op == "NEWB" || op == "TRUE" || op == "FALSE" || op == "NOW")); + } + bool is_nop() const { + return t == a_none && op.empty(); + } + bool is_comment() const { + return t == a_none && !op.empty(); + } + bool is_custom() const { + return t == a_custom; + } + bool is_very_custom() const { + return is_custom() && a >= 255; + } + bool is_push() const { + return t == a_push; + } + bool is_push(int x) const { + return is_push() && a == x; + } + bool is_push(int* x) const { + *x = a; + return is_push(); + } + bool is_pop() const { + return t == a_pop; + } + bool is_pop(int x) const { + return is_pop() && a == x; + } + bool is_xchg() const { + return t == a_xchg; + } + bool is_xchg(int x, int y) const { + return is_xchg() && b == y && a == x; + } + bool is_xchg(int* x, int* y) const { + *x = a; + *y = b; + return is_xchg(); + } + bool is_xchg_short() const { + return is_xchg() && (a <= 1 || b <= 1); + } + bool is_swap() const { + return is_xchg(0, 1); + } + bool is_const() const { + return t == a_const && !a && b == 1; + } + bool is_gconst() const { + return !a && b == 1 && (t == a_const || gconst); + } + static AsmOp Nop() { + return AsmOp(a_none); + } + static AsmOp Xchg(int a, int b = 0) { + return a == b ? AsmOp(a_none) : (a < b ? AsmOp(a_xchg, a, b) : AsmOp(a_xchg, b, a)); + } + static AsmOp Push(int a) { + return AsmOp(a_push, a); + } + static AsmOp Pop(int a = 0) { + return AsmOp(a_pop, a); + } + static AsmOp Xchg2(int a, int b) { + return make_stk2(a, b, "XCHG2", 0); + } + static AsmOp XcPu(int a, int b) { + return make_stk2(a, b, "XCPU", 1); + } + static AsmOp PuXc(int a, int b) { + return make_stk2(a, b, "PUXC", 1); + } + static AsmOp Push2(int a, int b) { + return make_stk2(a, b, "PUSH2", 2); + } + static AsmOp Xchg3(int a, int b, int c) { + return make_stk3(a, b, c, "XCHG3", 0); + } + static AsmOp Xc2Pu(int a, int b, int c) { + return make_stk3(a, b, c, "XC2PU", 1); + } + static AsmOp XcPuXc(int a, int b, int c) { + return make_stk3(a, b, c, "XCPUXC", 1); + } + static AsmOp XcPu2(int a, int b, int c) { + return make_stk3(a, b, c, "XCPU2", 3); + } + static AsmOp PuXc2(int a, int b, int c) { + return make_stk3(a, b, c, "PUXC2", 3); + } + static AsmOp PuXcPu(int a, int b, int c) { + return make_stk3(a, b, c, "PUXCPU", 3); + } + static AsmOp Pu2Xc(int a, int b, int c) { + return make_stk3(a, b, c, "PU2XC", 3); + } + static AsmOp Push3(int a, int b, int c) { + return make_stk3(a, b, c, "PUSH3", 3); + } + static AsmOp BlkSwap(int a, int b); + static AsmOp BlkPush(int a, int b); + static AsmOp BlkDrop(int a); + static AsmOp BlkDrop2(int a, int b); + static AsmOp BlkReverse(int a, int b); + static AsmOp make_stk2(int a, int b, const char* str, int delta); + static AsmOp make_stk3(int a, int b, int c, const char* str, int delta); + static AsmOp IntConst(const td::RefInt256& x); + static AsmOp BoolConst(bool f); + static AsmOp Const(std::string push_op) { + return AsmOp(a_const, 0, 1, std::move(push_op)); + } + static AsmOp Const(int arg, const std::string& push_op); + static AsmOp Comment(const std::string& comment) { + return AsmOp(a_none, std::string{"// "} + comment); + } + static AsmOp Custom(const std::string& custom_op) { + return AsmOp(a_custom, 255, 255, custom_op); + } + static AsmOp Parse(const std::string& custom_op); + static AsmOp Custom(const std::string& custom_op, int args, int retv = 1) { + return AsmOp(a_custom, args, retv, custom_op); + } + static AsmOp Parse(std::string custom_op, int args, int retv = 1); + static AsmOp Tuple(int a); + static AsmOp UnTuple(int a); +}; + +inline std::ostream& operator<<(std::ostream& os, const AsmOp& op) { + op.out(os); + return os; +} + +std::ostream& operator<<(std::ostream& os, AsmOp::SReg stack_reg); +std::ostream& operator<<(std::ostream& os, TypePtr type_data); + +struct AsmOpList { + std::vector list_; + int indent_{0}; + const std::vector* var_names_{nullptr}; + std::vector constants_; + bool retalt_{false}; + bool retalt_inserted_{false}; + void out(std::ostream& os, int mode = 0) const; + AsmOpList(int indent = 0, const std::vector* var_names = nullptr) : indent_(indent), var_names_(var_names) { + } + template + AsmOpList& add(Args&&... args) { + append(AsmOp(std::forward(args)...)); + adjust_last(); + return *this; + } + bool append(const AsmOp& op) { + list_.push_back(op); + adjust_last(); + return true; + } + bool append(const std::vector& ops); + bool append(std::initializer_list ops) { + return append(std::vector(std::move(ops))); + } + AsmOpList& operator<<(const AsmOp& op) { + return add(op); + } + AsmOpList& operator<<(AsmOp&& op) { + return add(std::move(op)); + } + AsmOpList& operator<<(std::string str) { + return add(AsmOp::Type::a_custom, 255, 255, str); + } + const_idx_t register_const(Const new_const); + Const get_const(const_idx_t idx); + void show_var_ext(std::ostream& os, std::pair idx_pair) const; + void adjust_last() { + if (list_.back().is_nop()) { + list_.pop_back(); + } else { + list_.back().indent = indent_; + } + } + void indent() { + ++indent_; + } + void undent() { + --indent_; + } + void set_indent(int new_indent) { + indent_ = new_indent; + } + void insert(size_t pos, std::string str) { + insert(pos, AsmOp(AsmOp::a_custom, 255, 255, str)); + } + void insert(size_t pos, const AsmOp& op) { + auto ip = list_.begin() + pos; + ip = list_.insert(ip, op); + ip->indent = (ip == list_.begin()) ? indent_ : (ip - 1)->indent; + } + void indent_all() { + for (auto &op : list_) { + ++op.indent; + } + } +}; + +inline std::ostream& operator<<(std::ostream& os, const AsmOpList& op_list) { + op_list.out(os); + return os; +} + +struct AsmOpCons { + std::unique_ptr car; + std::unique_ptr cdr; + AsmOpCons(std::unique_ptr head, std::unique_ptr tail) : car(std::move(head)), cdr(std::move(tail)) { + } + static std::unique_ptr cons(std::unique_ptr head, std::unique_ptr tail) { + return std::make_unique(std::move(head), std::move(tail)); + } +}; + +using AsmOpConsList = std::unique_ptr; + +int is_pos_pow2(td::RefInt256 x); +int is_neg_pow2(td::RefInt256 x); + +/* + * + * STACK TRANSFORMS + * + */ + +/* +A stack transform is a map f:N={0,1,...} -> N, such that f(x) = x + d_f for almost all x:N and for a fixed d_f:N. +They form a monoid under composition: (fg)(x)=f(g(x)). +They act on stacks S on the right: Sf=S', such that S'[n]=S[f(n)]. + +A stack transform f is determined by d_f and the finite set A of all pairs (x,y), such that x>=d_f, f(x-d_f) = y and y<>x. They are listed in increasing order by x. +*/ +struct StackTransform { + enum { max_n = 16, inf_x = 0x7fffffff, c_start = -1000 }; + int d{0}, n{0}, dp{0}, c{0}; + bool invalid{false}; + std::array, max_n> A; + StackTransform() = default; + // list of f(0),f(1),...,f(s); assumes next values are f(s)+1,f(s)+2,... + StackTransform(std::initializer_list list); + StackTransform& operator=(std::initializer_list list); + bool assign(const StackTransform& other); + static StackTransform id() { + return {}; + } + bool invalidate() { + invalid = true; + return false; + } + bool is_valid() const { + return !invalid; + } + bool set_id() { + d = n = dp = c = 0; + invalid = false; + return true; + } + bool shift(int offs) { // post-composes with x -> x + offs + d += offs; + return offs <= 0 || remove_negative(); + } + bool remove_negative(); + bool touch(int i) { + dp = std::max(dp, i + d + 1); + return true; + } + bool is_permutation() const; // is f:N->N bijective ? + bool is_trivial_after(int x) const; // f(x') = x' + d for all x' >= x + int preimage_count(int y) const; // card f^{-1}(y) + std::vector preimage(int y) const; + bool apply_xchg(int i, int j, bool relaxed = false); + bool apply_push(int i); + bool apply_pop(int i = 0); + bool apply_push_newconst(); + bool apply_blkpop(int k); + bool apply(const StackTransform& other); // this = this * other + bool preapply(const StackTransform& other); // this = other * this + // c := a * b + static bool compose(const StackTransform& a, const StackTransform& b, StackTransform& c); + StackTransform& operator*=(const StackTransform& other); + StackTransform operator*(const StackTransform& b) const &; + bool equal(const StackTransform& other, bool relaxed = false) const; + bool almost_equal(const StackTransform& other) const { + return equal(other, true); + } + bool operator==(const StackTransform& other) const { + return dp == other.dp && almost_equal(other); + } + bool operator<=(const StackTransform& other) const { + return dp <= other.dp && almost_equal(other); + } + bool operator>=(const StackTransform& other) const { + return dp >= other.dp && almost_equal(other); + } + int get(int i) const; + int touch_get(int i, bool relaxed = false) { + if (!relaxed) { + touch(i); + } + return get(i); + } + bool set(int i, int v, bool relaxed = false); + int operator()(int i) const { + return get(i); + } + class Pos { + StackTransform& t_; + int p_; + + public: + Pos(StackTransform& t, int p) : t_(t), p_(p) { + } + Pos& operator=(const Pos& other) = delete; + operator int() const { + return t_.get(p_); + } + const Pos& operator=(int v) const { + t_.set(p_, v); + return *this; + } + }; + Pos operator[](int i) { + return Pos(*this, i); + } + static const StackTransform rot; + static const StackTransform rot_rev; + bool is_id() const { + return is_valid() && !d && !n; + } + bool is_xchg(int i, int j) const; + bool is_xchg(int* i, int* j) const; + bool is_xchg_xchg(int i, int j, int k, int l) const; + bool is_xchg_xchg(int* i, int* j, int* k, int* l) const; + bool is_push(int i) const; + bool is_push(int* i) const; + bool is_pop(int i) const; + bool is_pop(int* i) const; + bool is_pop_pop(int i, int j) const; + bool is_pop_pop(int* i, int* j) const; + bool is_rot() const; + bool is_rotrev() const; + bool is_push_rot(int i) const; + bool is_push_rot(int* i) const; + bool is_push_rotrev(int i) const; + bool is_push_rotrev(int* i) const; + bool is_push_xchg(int i, int j, int k) const; + bool is_push_xchg(int* i, int* j, int* k) const; + bool is_xchg2(int i, int j) const; + bool is_xchg2(int* i, int* j) const; + bool is_xcpu(int i, int j) const; + bool is_xcpu(int* i, int* j) const; + bool is_puxc(int i, int j) const; + bool is_puxc(int* i, int* j) const; + bool is_push2(int i, int j) const; + bool is_push2(int* i, int* j) const; + bool is_xchg3(int* i, int* j, int* k) const; + bool is_xc2pu(int* i, int* j, int* k) const; + bool is_xcpuxc(int* i, int* j, int* k) const; + bool is_xcpu2(int* i, int* j, int* k) const; + bool is_puxc2(int i, int j, int k) const; + bool is_puxc2(int* i, int* j, int* k) const; + bool is_puxcpu(int* i, int* j, int* k) const; + bool is_pu2xc(int i, int j, int k) const; + bool is_pu2xc(int* i, int* j, int* k) const; + bool is_push3(int i, int j, int k) const; + bool is_push3(int* i, int* j, int* k) const; + bool is_blkswap(int i, int j) const; + bool is_blkswap(int* i, int* j) const; + bool is_blkpush(int i, int j) const; + bool is_blkpush(int* i, int* j) const; + bool is_blkdrop(int* i) const; + bool is_blkdrop2(int i, int j) const; + bool is_blkdrop2(int* i, int* j) const; + bool is_reverse(int i, int j) const; + bool is_reverse(int* i, int* j) const; + bool is_nip_seq(int i, int j = 0) const; + bool is_nip_seq(int* i) const; + bool is_nip_seq(int* i, int* j) const; + bool is_pop_blkdrop(int i, int k) const; + bool is_pop_blkdrop(int* i, int* k) const; + bool is_2pop_blkdrop(int i, int j, int k) const; + bool is_2pop_blkdrop(int* i, int* j, int* k) const; + bool is_const_rot(int c) const; + bool is_const_rot(int* c) const; + bool is_const_pop(int c, int i) const; + bool is_const_pop(int* c, int* i) const; + bool is_push_const(int i, int c) const; + bool is_push_const(int* i, int* c) const; + + void show(std::ostream& os, int mode = 0) const; + + static StackTransform Xchg(int i, int j, bool relaxed = false); + static StackTransform Push(int i); + static StackTransform Pop(int i); + + private: + int try_load(int& i, int offs = 0) const; // returns A[i++].first + offs or inf_x + bool try_store(int x, int y); // appends (x,y) to A +}; + +inline std::ostream& operator<<(std::ostream& os, const StackTransform& trans) { + trans.show(os); + return os; +} + +bool apply_op(StackTransform& trans, const AsmOp& op); + +/* + * + * STACK OPERATION OPTIMIZER + * + */ + +struct Optimizer { + static constexpr int optimize_depth = 20; + AsmOpConsList code_; + int l_{0}, l2_{0}, p_, pb_, q_, indent_; + bool debug_{false}; + std::unique_ptr op_[optimize_depth], oq_[optimize_depth]; + AsmOpCons* op_cons_[optimize_depth]; + int offs_[optimize_depth]; + StackTransform tr_[optimize_depth]; + int mode_{0}; + Optimizer() { + } + Optimizer(bool debug, int mode = 0) : debug_(debug), mode_(mode) { + } + Optimizer(AsmOpConsList code, bool debug = false, int mode = 0) : Optimizer(debug, mode) { + set_code(std::move(code)); + } + void set_code(AsmOpConsList code_); + void unpack(); + void pack(); + void apply(); + bool find_at_least(int pb); + bool find(); + bool optimize(); + bool compute_stack_transforms(); + bool say(std::string str) const; + bool show_stack_transforms() const; + void show_head() const; + void show_left() const; + void show_right() const; + bool find_const_op(int* op_idx, int cst); + bool is_push_const(int* i, int* c) const; + bool rewrite_push_const(int i, int c); + bool is_const_push_xchgs(); + bool rewrite_const_push_xchgs(); + bool is_const_rot(int* c) const; + bool rewrite_const_rot(int c); + bool is_const_pop(int* c, int* i) const; + bool rewrite_const_pop(int c, int i); + bool rewrite(int p, AsmOp&& new_op); + bool rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2); + bool rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2, AsmOp&& new_op3); + bool rewrite(AsmOp&& new_op) { + return rewrite(p_, std::move(new_op)); + } + bool rewrite(AsmOp&& new_op1, AsmOp&& new_op2) { + return rewrite(p_, std::move(new_op1), std::move(new_op2)); + } + bool rewrite(AsmOp&& new_op1, AsmOp&& new_op2, AsmOp&& new_op3) { + return rewrite(p_, std::move(new_op1), std::move(new_op2), std::move(new_op3)); + } + bool rewrite_nop(); + bool is_pred(const std::function& pred, int min_p = 2); + bool is_same_as(const StackTransform& trans, int min_p = 2); + bool is_rot(); + bool is_rotrev(); + bool is_tuck(); + bool is_2dup(); + bool is_2drop(); + bool is_2swap(); + bool is_2over(); + bool is_xchg(int* i, int* j); + bool is_xchg_xchg(int* i, int* j, int* k, int* l); + bool is_push(int* i); + bool is_pop(int* i); + bool is_pop_pop(int* i, int* j); + bool is_nop(); + bool is_push_rot(int* i); + bool is_push_rotrev(int* i); + bool is_push_xchg(int* i, int* j, int* k); + bool is_xchg2(int* i, int* j); + bool is_xcpu(int* i, int* j); + bool is_puxc(int* i, int* j); + bool is_push2(int* i, int* j); + bool is_xchg3(int* i, int* j, int* k); + bool is_xc2pu(int* i, int* j, int* k); + bool is_xcpuxc(int* i, int* j, int* k); + bool is_xcpu2(int* i, int* j, int* k); + bool is_puxc2(int* i, int* j, int* k); + bool is_puxcpu(int* i, int* j, int* k); + bool is_pu2xc(int* i, int* j, int* k); + bool is_push3(int* i, int* j, int* k); + bool is_blkswap(int* i, int* j); + bool is_blkpush(int* i, int* j); + bool is_blkdrop(int* i); + bool is_blkdrop2(int* i, int* j); + bool is_reverse(int* i, int* j); + bool is_nip_seq(int* i, int* j); + bool is_pop_blkdrop(int* i, int* k); + bool is_2pop_blkdrop(int* i, int* j, int* k); + AsmOpConsList extract_code(); +}; + +AsmOpConsList optimize_code_head(AsmOpConsList op_list, int mode = 0); +AsmOpConsList optimize_code(AsmOpConsList op_list, int mode); +void optimize_code(AsmOpList& ops); + +struct Stack { + StackLayoutExt s; + AsmOpList& o; + enum { + _StkCmt = 1, _CptStkCmt = 2, _DisableOut = 128, _Shown = 256, + _InlineFunc = 512, _NeedRetAlt = 1024, _InlineAny = 2048, + _ModeSave = _InlineFunc | _NeedRetAlt | _InlineAny, + _Garbage = -0x10000 + }; + int mode; + Stack(AsmOpList& _o, int _mode = 0) : o(_o), mode(_mode) { + } + Stack(AsmOpList& _o, const StackLayoutExt& _s, int _mode = 0) : s(_s), o(_o), mode(_mode) { + } + Stack(AsmOpList& _o, StackLayoutExt&& _s, int _mode = 0) : s(std::move(_s)), o(_o), mode(_mode) { + } + int depth() const { + return (int)s.size(); + } + var_idx_t operator[](int i) const { + validate(i); + return s[depth() - i - 1].first; + } + var_const_idx_t& at(int i) { + validate(i); + return s[depth() - i - 1]; + } + var_const_idx_t at(int i) const { + validate(i); + return s[depth() - i - 1]; + } + var_const_idx_t get(int i) const { + return at(i); + } + bool output_disabled() const { + return mode & _DisableOut; + } + bool output_enabled() const { + return !output_disabled(); + } + void disable_output() { + mode |= _DisableOut; + } + StackLayout vars() const; + int find(var_idx_t var, int from = 0) const; + int find(var_idx_t var, int from, int to) const; + int find_const(const_idx_t cst, int from = 0) const; + int find_outside(var_idx_t var, int from, int to) const; + void forget_const(); + void validate(int i) const { + if (i > 255) { + throw Fatal{"Too deep stack"}; + } + tolk_assert(i >= 0 && i < depth() && "invalid stack reference"); + } + void modified() { + mode &= ~_Shown; + } + void issue_pop(int i); + void issue_push(int i); + void issue_xchg(int i, int j); + int drop_vars_except(const VarDescrList& var_info, int excl_var = 0x80000000); + void forget_var(var_idx_t idx); + void push_new_var(var_idx_t idx); + void push_new_const(var_idx_t idx, const_idx_t cidx); + void assign_var(var_idx_t new_idx, var_idx_t old_idx); + void do_copy_var(var_idx_t new_idx, var_idx_t old_idx); + void enforce_state(const StackLayout& req_stack); + void rearrange_top(const StackLayout& top, std::vector last); + void rearrange_top(var_idx_t top, bool last); + void merge_const(const Stack& req_stack); + void merge_state(const Stack& req_stack); + void show(); + void opt_show() { + if ((mode & (_StkCmt | _Shown)) == _StkCmt) { + show(); + } + } + bool operator==(const Stack& y) const & { + return s == y.s; + } + void apply_wrappers(int callxargs_count) { + bool is_inline = mode & _InlineFunc; + if (o.retalt_inserted_) { + o.insert(0, "SAMEALTSAVE"); + o.insert(0, "c2 SAVE"); + } + if (callxargs_count != -1 || (is_inline && o.retalt_)) { + o.indent_all(); + o.insert(0, "CONT:<{"); + o << "}>"; + if (callxargs_count != -1) { + if (callxargs_count <= 15) { + o << AsmOp::Custom(PSTRING() << callxargs_count << " -1 CALLXARGS"); + } else { + tolk_assert(callxargs_count <= 254); + o << AsmOp::Custom(PSTRING() << callxargs_count << " PUSHINT -1 PUSHINT CALLXVARARGS"); + } + } else { + o << "EXECUTE"; + } + } + } +}; + +/* + * + * SPECIFIC SYMBOL VALUES, + * BUILT-IN FUNCTIONS AND OPERATIONS + * + */ + +typedef std::function&, std::vector&, SrcLocation)> simple_compile_func_t; + +inline simple_compile_func_t make_simple_compile(AsmOp op) { + return [op](std::vector& out, std::vector& in, SrcLocation) -> AsmOp { return op; }; +} + +struct FunctionBodyBuiltin { + simple_compile_func_t simple_compile; + + explicit FunctionBodyBuiltin(simple_compile_func_t compile) + : simple_compile(std::move(compile)) {} + + void compile(AsmOpList& dest, std::vector& out, std::vector& in, SrcLocation where) const; +}; + +struct FunctionBodyAsm { + std::vector ops; + + void set_code(std::vector&& code); + void compile(AsmOpList& dest) const; +}; + +struct CodeBlob { + int var_cnt, in_var_cnt; + FunctionPtr fun_ref; + std::string name; + SrcLocation loc; + std::vector vars; + std::unique_ptr ops; + std::unique_ptr* cur_ops; +#ifdef TOLK_DEBUG + std::vector _vector_of_ops; // to see it in debugger instead of nested pointers +#endif + std::stack*> cur_ops_stack; + bool require_callxargs = false; + CodeBlob(std::string name, SrcLocation loc, FunctionPtr fun_ref) + : var_cnt(0), in_var_cnt(0), fun_ref(fun_ref), name(std::move(name)), loc(loc), cur_ops(&ops) { + } + template + Op& emplace_back(Args&&... args) { + Op& res = *(*cur_ops = std::make_unique(args...)); + cur_ops = &(res.next); +#ifdef TOLK_DEBUG + _vector_of_ops.push_back(&res); +#endif + return res; + } + std::vector create_var(TypePtr var_type, SrcLocation loc, std::string name); + std::vector create_tmp_var(TypePtr var_type, SrcLocation loc, const char* desc) { + std::vector ir_idx = create_var(var_type, loc, {}); +#ifdef TOLK_DEBUG + for (var_idx_t v : ir_idx) { + vars[v].desc = desc; + } +#endif + return ir_idx; + } + bool compute_used_code_vars(); + bool compute_used_code_vars(std::unique_ptr& ops, const VarDescrList& var_info, bool edit) const; + void print(std::ostream& os, int flags = 0) const; + void push_set_cur(std::unique_ptr& new_cur_ops) { + cur_ops_stack.push(cur_ops); + cur_ops = &new_cur_ops; + } + void close_blk(SrcLocation location) { + *cur_ops = std::make_unique(location, Op::_Nop); + } + void pop_cur() { + cur_ops = cur_ops_stack.top(); + cur_ops_stack.pop(); + } + void close_pop_cur(SrcLocation location) { + close_blk(location); + pop_cur(); + } + void prune_unreachable_code(); + void fwd_analyze(); + void mark_noreturn(); + void generate_code(AsmOpList& out_list, int mode = 0); + void generate_code(std::ostream& os, int mode = 0, int indent = 0); +}; + +// defined in builtins.cpp +AsmOp exec_arg_op(std::string op, long long arg); +AsmOp exec_arg_op(std::string op, long long arg, int args, int retv = 1); +AsmOp exec_arg_op(std::string op, td::RefInt256 arg); +AsmOp exec_arg_op(std::string op, td::RefInt256 arg, int args, int retv = 1); +AsmOp exec_arg2_op(std::string op, long long imm1, long long imm2, int args, int retv = 1); +AsmOp push_const(td::RefInt256 x); + +void define_builtins(); + + + +/* + * + * OUTPUT CODE GENERATOR + * + */ + +int tolk_proceed(const std::string &entrypoint_filename); + +} // namespace tolk + + diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp new file mode 100644 index 00000000..d73625c2 --- /dev/null +++ b/tolk/type-system.cpp @@ -0,0 +1,756 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "type-system.h" +#include "lexer.h" +#include "platform-utils.h" +#include "compiler-state.h" +#include + +namespace tolk { + +/* + * This class stores a big hashtable [hash => TypeData] + * Every non-trivial TypeData*::create() method at first looks here, and allocates an object only if not found. + * That's why all allocated TypeData objects are unique, storing unique type_id. + */ +class TypeDataTypeIdCalculation { + uint64_t cur_hash; + int children_flags_mask = 0; + + static std::unordered_map all_unique_occurred_types; + +public: + explicit TypeDataTypeIdCalculation(uint64_t initial_arbitrary_unique_number) + : cur_hash(initial_arbitrary_unique_number) {} + + void feed_hash(uint64_t val) { + cur_hash = cur_hash * 56235515617499ULL + val; + } + + void feed_string(const std::string& s) { + feed_hash(std::hash{}(s)); + } + + void feed_child(TypePtr inner) { + feed_hash(inner->type_id); + children_flags_mask |= inner->flags; + } + + uint64_t type_id() const { + return cur_hash; + } + + int children_flags() const { + return children_flags_mask; + } + + GNU_ATTRIBUTE_FLATTEN + TypePtr get_existing() const { + auto it = all_unique_occurred_types.find(cur_hash); + return it != all_unique_occurred_types.end() ? it->second : nullptr; + } + + GNU_ATTRIBUTE_NOINLINE + TypePtr register_unique(TypePtr newly_created) const { +#ifdef TOLK_DEBUG + assert(newly_created->type_id == cur_hash); +#endif + all_unique_occurred_types[cur_hash] = newly_created; + return newly_created; + } +}; + +std::unordered_map TypeDataTypeIdCalculation::all_unique_occurred_types; +TypePtr TypeDataInt::singleton; +TypePtr TypeDataBool::singleton; +TypePtr TypeDataCell::singleton; +TypePtr TypeDataSlice::singleton; +TypePtr TypeDataBuilder::singleton; +TypePtr TypeDataTuple::singleton; +TypePtr TypeDataContinuation::singleton; +TypePtr TypeDataNullLiteral::singleton; +TypePtr TypeDataUnknown::singleton; +TypePtr TypeDataNever::singleton; +TypePtr TypeDataVoid::singleton; + +void type_system_init() { + TypeDataInt::singleton = new TypeDataInt; + TypeDataBool::singleton = new TypeDataBool; + TypeDataCell::singleton = new TypeDataCell; + TypeDataSlice::singleton = new TypeDataSlice; + TypeDataBuilder::singleton = new TypeDataBuilder; + TypeDataTuple::singleton = new TypeDataTuple; + TypeDataContinuation::singleton = new TypeDataContinuation; + TypeDataNullLiteral::singleton = new TypeDataNullLiteral; + TypeDataUnknown::singleton = new TypeDataUnknown; + TypeDataNever::singleton = new TypeDataNever; + TypeDataVoid::singleton = new TypeDataVoid; +} + + +// -------------------------------------------- +// create() +// +// all constructors of TypeData classes are private, only TypeData*::create() is allowed +// each non-trivial create() method calculates hash (type_id) +// and creates an object only if it isn't found in a global hashtable +// + +TypePtr TypeDataNullable::create(TypePtr inner) { + TypeDataTypeIdCalculation hash(1774084920039440885ULL); + hash.feed_child(inner); + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + // most types (int?, slice?, etc.), when nullable, still occupy 1 stack slot (holding TVM NULL at runtime) + // but for example for `(int, int)` we need an extra stack slot "null flag" + int width_on_stack = inner->can_hold_tvm_null_instead() ? 1 : inner->get_width_on_stack() + 1; + return hash.register_unique(new TypeDataNullable(hash.type_id(), hash.children_flags(), width_on_stack, inner)); +} + +TypePtr TypeDataFunCallable::create(std::vector&& params_types, TypePtr return_type) { + TypeDataTypeIdCalculation hash(3184039965511020991ULL); + for (TypePtr param : params_types) { + hash.feed_child(param); + hash.feed_hash(767721); + } + hash.feed_child(return_type); + hash.feed_hash(767722); + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + return hash.register_unique(new TypeDataFunCallable(hash.type_id(), hash.children_flags(), std::move(params_types), return_type)); +} + +TypePtr TypeDataGenericT::create(std::string&& nameT) { + TypeDataTypeIdCalculation hash(9145033724911680012ULL); + hash.feed_string(nameT); + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + return hash.register_unique(new TypeDataGenericT(hash.type_id(), std::move(nameT))); +} + +TypePtr TypeDataTensor::create(std::vector&& items) { + TypeDataTypeIdCalculation hash(3159238551239480381ULL); + for (TypePtr item : items) { + hash.feed_child(item); + hash.feed_hash(819613); + } + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + int width_on_stack = 0; + for (TypePtr item : items) { + width_on_stack += item->get_width_on_stack(); + } + return hash.register_unique(new TypeDataTensor(hash.type_id(), hash.children_flags(), width_on_stack, std::move(items))); +} + +TypePtr TypeDataTypedTuple::create(std::vector&& items) { + TypeDataTypeIdCalculation hash(9189266157349499320ULL); + for (TypePtr item : items) { + hash.feed_child(item); + hash.feed_hash(735911); + } + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + return hash.register_unique(new TypeDataTypedTuple(hash.type_id(), hash.children_flags(), std::move(items))); +} + +TypePtr TypeDataUnresolved::create(std::string&& text, SrcLocation loc) { + TypeDataTypeIdCalculation hash(3680147223540048162ULL); + hash.feed_string(text); + // hash.feed_hash(*reinterpret_cast(&loc)); + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + return hash.register_unique(new TypeDataUnresolved(hash.type_id(), std::move(text), loc)); +} + + +// -------------------------------------------- +// as_human_readable() +// +// is used only for error messages and debugging, therefore no optimizations for simplicity +// only non-trivial implementations are here; trivial are defined in .h file +// + +std::string TypeDataNullable::as_human_readable() const { + std::string nested = inner->as_human_readable(); + bool embrace = inner->try_as(); + return embrace ? "(" + nested + ")?" : nested + "?"; +} + +std::string TypeDataFunCallable::as_human_readable() const { + std::string result = "("; + for (TypePtr param : params_types) { + if (result.size() > 1) { + result += ", "; + } + result += param->as_human_readable(); + } + result += ") -> "; + result += return_type->as_human_readable(); + return result; +} + +std::string TypeDataTensor::as_human_readable() const { + std::string result = "("; + for (TypePtr item : items) { + if (result.size() > 1) { + result += ", "; + } + result += item->as_human_readable(); + } + result += ')'; + return result; +} + +std::string TypeDataTypedTuple::as_human_readable() const { + std::string result = "["; + for (TypePtr item : items) { + if (result.size() > 1) { + result += ", "; + } + result += item->as_human_readable(); + } + result += ']'; + return result; +} + + +// -------------------------------------------- +// traverse() +// +// invokes a callback for TypeData itself and all its children +// only non-trivial implementations are here; by default (no children), `callback(this)` is executed +// + +void TypeDataNullable::traverse(const TraverserCallbackT& callback) const { + callback(this); + inner->traverse(callback); +} + +void TypeDataFunCallable::traverse(const TraverserCallbackT& callback) const { + callback(this); + for (TypePtr param : params_types) { + param->traverse(callback); + } + return_type->traverse(callback); +} + +void TypeDataTensor::traverse(const TraverserCallbackT& callback) const { + callback(this); + for (TypePtr item : items) { + item->traverse(callback); + } +} + +void TypeDataTypedTuple::traverse(const TraverserCallbackT& callback) const { + callback(this); + for (TypePtr item : items) { + item->traverse(callback); + } +} + + +// -------------------------------------------- +// replace_children_custom() +// +// returns new TypeData with children replaced by a custom callback +// used to replace generic T on generics expansion — to convert `f` to `f` +// only non-trivial implementations are here; by default (no children), `return callback(this)` is executed +// + +TypePtr TypeDataNullable::replace_children_custom(const ReplacerCallbackT& callback) const { + return callback(create(inner->replace_children_custom(callback))); +} + +TypePtr TypeDataFunCallable::replace_children_custom(const ReplacerCallbackT& callback) const { + std::vector mapped; + mapped.reserve(params_types.size()); + for (TypePtr param : params_types) { + mapped.push_back(param->replace_children_custom(callback)); + } + return callback(create(std::move(mapped), return_type->replace_children_custom(callback))); +} + +TypePtr TypeDataTensor::replace_children_custom(const ReplacerCallbackT& callback) const { + std::vector mapped; + mapped.reserve(items.size()); + for (TypePtr item : items) { + mapped.push_back(item->replace_children_custom(callback)); + } + return callback(create(std::move(mapped))); +} + +TypePtr TypeDataTypedTuple::replace_children_custom(const ReplacerCallbackT& callback) const { + std::vector mapped; + mapped.reserve(items.size()); + for (TypePtr item : items) { + mapped.push_back(item->replace_children_custom(callback)); + } + return callback(create(std::move(mapped))); +} + + +// -------------------------------------------- +// can_rhs_be_assigned() +// +// on `var lhs: = rhs`, having inferred rhs_type, check that it can be assigned without any casts +// the same goes for passing arguments, returning values, etc. — where the "receiver" (lhs) checks "applier" (rhs) +// + +bool TypeDataInt::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataBool::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataCell::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataSlice::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataBuilder::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataTuple::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataContinuation::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataNullLiteral::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataNullable::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + if (rhs == TypeDataNullLiteral::create()) { + return true; + } + if (const TypeDataNullable* rhs_nullable = rhs->try_as()) { + return inner->can_rhs_be_assigned(rhs_nullable->inner); + } + if (inner->can_rhs_be_assigned(rhs)) { + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataFunCallable::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataGenericT::can_rhs_be_assigned(TypePtr rhs) const { + assert(false); + return false; +} + +bool TypeDataTensor::can_rhs_be_assigned(TypePtr rhs) const { + if (const auto* as_tensor = rhs->try_as(); as_tensor && as_tensor->size() == size()) { + for (int i = 0; i < size(); ++i) { + if (!items[i]->can_rhs_be_assigned(as_tensor->items[i])) { + return false; + } + } + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataTypedTuple::can_rhs_be_assigned(TypePtr rhs) const { + if (const auto* as_tuple = rhs->try_as(); as_tuple && as_tuple->size() == size()) { + for (int i = 0; i < size(); ++i) { + if (!items[i]->can_rhs_be_assigned(as_tuple->items[i])) { + return false; + } + } + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataUnknown::can_rhs_be_assigned(TypePtr rhs) const { + return true; +} + +bool TypeDataUnresolved::can_rhs_be_assigned(TypePtr rhs) const { + assert(false); + return false; +} + +bool TypeDataNever::can_rhs_be_assigned(TypePtr rhs) const { + return true; +} + +bool TypeDataVoid::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return rhs == TypeDataNever::create(); +} + + +// -------------------------------------------- +// can_be_casted_with_as_operator() +// +// on `expr as `, check whether casting is applicable +// note, that it's not auto-casts `var lhs: = rhs`, it's an expression `rhs as ` +// + +bool TypeDataInt::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_nullable = cast_to->try_as()) { // `int` as `int?` + return can_be_casted_with_as_operator(to_nullable->inner); + } + return cast_to == this; +} + +bool TypeDataBool::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_nullable = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_nullable->inner); + } + return cast_to == this || cast_to == TypeDataInt::create(); +} + +bool TypeDataCell::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_nullable = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_nullable->inner); + } + return cast_to == this; +} + +bool TypeDataSlice::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_nullable = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_nullable->inner); + } + return cast_to == this; +} + +bool TypeDataBuilder::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_nullable = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_nullable->inner); + } + return cast_to == this; +} + +bool TypeDataTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_nullable = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_nullable->inner); + } + return cast_to == this; +} + +bool TypeDataContinuation::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_nullable = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_nullable->inner); + } + return cast_to == this; +} + +bool TypeDataNullLiteral::can_be_casted_with_as_operator(TypePtr cast_to) const { + return cast_to == this || cast_to->try_as(); +} + +bool TypeDataNullable::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_nullable = cast_to->try_as()) { + return inner->can_be_casted_with_as_operator(to_nullable->inner); + } + return false; +} + +bool TypeDataFunCallable::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_nullable = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_nullable->inner); + } + return this == cast_to; +} + +bool TypeDataGenericT::can_be_casted_with_as_operator(TypePtr cast_to) const { + return true; +} + +bool TypeDataTensor::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_tensor = cast_to->try_as(); to_tensor && to_tensor->size() == size()) { + for (int i = 0; i < size(); ++i) { + if (!items[i]->can_be_casted_with_as_operator(to_tensor->items[i])) { + return false; + } + } + return true; + } + if (const auto* to_nullable = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_nullable->inner); + } + return false; +} + +bool TypeDataTypedTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_tuple = cast_to->try_as(); to_tuple && to_tuple->size() == size()) { + for (int i = 0; i < size(); ++i) { + if (!items[i]->can_be_casted_with_as_operator(to_tuple->items[i])) { + return false; + } + } + return true; + } + if (const auto* to_nullable = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_nullable->inner); + } + return false; +} + +bool TypeDataUnknown::can_be_casted_with_as_operator(TypePtr cast_to) const { + // 'unknown' can be cast to any TVM value + return cast_to->get_width_on_stack() == 1; +} + +bool TypeDataUnresolved::can_be_casted_with_as_operator(TypePtr cast_to) const { + return false; +} + +bool TypeDataNever::can_be_casted_with_as_operator(TypePtr cast_to) const { + return true; +} + +bool TypeDataVoid::can_be_casted_with_as_operator(TypePtr cast_to) const { + return cast_to == this; +} + + +// -------------------------------------------- +// can_hold_tvm_null_instead() +// +// assigning `null` to a primitive variable like `int?` / `cell?` can store TVM NULL inside the same slot +// (that's why the default implementation is just "return true", and most of types occupy 1 slot) +// but for complex variables, like `(int, int)?`, "null presence" is kept in a separate slot (UTag for union types) +// though still, tricky situations like `(int, ())?` can still "embed" TVM NULL in parallel with original value +// + +bool TypeDataNullable::can_hold_tvm_null_instead() const { + if (get_width_on_stack() != 1) { // `(int, int)?` / `()?` can not hold null instead + return false; // only `int?` / `cell?` / `StructWith1IntField?` can + } // and some tricky situations like `(int, ())?`, but not `(int?, ())?` + return !inner->can_hold_tvm_null_instead(); +} + +bool TypeDataTensor::can_hold_tvm_null_instead() const { + if (get_width_on_stack() != 1) { // `(int, int)` / `()` can not hold null instead, since null is 1 slot + return false; // only `((), int)` and similar can: + } // one item is width 1 (and not nullable), others are 0 + for (TypePtr item : items) { + if (item->get_width_on_stack() == 1 && !item->can_hold_tvm_null_instead()) { + return false; + } + } + return true; +} + +bool TypeDataNever::can_hold_tvm_null_instead() const { + return false; +} + +bool TypeDataVoid::can_hold_tvm_null_instead() const { + return false; +} + + +// -------------------------------------------- +// parsing type from tokens +// +// here we implement parsing types (mostly after colon) to TypeData +// example: `var v: int` is TypeDataInt +// example: `var v: (builder?, [cell])` is TypeDataTensor(TypeDataNullable(TypeDataBuilder), TypeDataTypedTuple(TypeDataCell)) +// example: `fun f(): ()` is TypeDataTensor() (an empty one) +// +// note, that unrecognized type names (MyEnum, MyStruct, T) are parsed as TypeDataUnresolved, +// and later, when all files are parsed and all symbols registered, such identifiers are resolved +// example: `fun f(v: T)` at first v is TypeDataUnresolved("T"), later becomes TypeDataGenericT +// see finalize_type_data() +// +// note, that `self` does not name a type, it can appear only as a return value of a function (parsed specially) +// when `self` appears as a type, it's parsed as TypeDataUnresolved, and later an error is emitted +// + +static TypePtr parse_type_expression(Lexer& lex); + +std::vector parse_nested_type_list(Lexer& lex, TokenType tok_op, const char* s_op, TokenType tok_cl, const char* s_cl) { + lex.expect(tok_op, s_op); + std::vector sub_types; + while (true) { + if (lex.tok() == tok_cl) { // empty lists allowed + lex.next(); + break; + } + + sub_types.emplace_back(parse_type_expression(lex)); + if (lex.tok() == tok_comma) { + lex.next(); + } else if (lex.tok() != tok_cl) { + lex.unexpected(s_cl); + } + } + return sub_types; +} + +std::vector parse_nested_type_list_in_parenthesis(Lexer& lex) { + return parse_nested_type_list(lex, tok_oppar, "`(`", tok_clpar, "`)` or `,`"); +} + +static TypePtr parse_simple_type(Lexer& lex) { + switch (lex.tok()) { + case tok_self: + case tok_identifier: { + SrcLocation loc = lex.cur_location(); + std::string_view str = lex.cur_str(); + lex.next(); + switch (str.size()) { + case 3: + if (str == "int") return TypeDataInt::create(); + break; + case 4: + if (str == "cell") return TypeDataCell::create(); + if (str == "void") return TypeDataVoid::create(); + if (str == "bool") return TypeDataBool::create(); + break; + case 5: + if (str == "slice") return TypeDataSlice::create(); + if (str == "tuple") return TypeDataTuple::create(); + if (str == "never") return TypeDataNever::create(); + break; + case 7: + if (str == "builder") return TypeDataBuilder::create(); + break; + case 12: + if (str == "continuation") return TypeDataContinuation::create(); + break; + default: + break; + } + return TypeDataUnresolved::create(std::string(str), loc); + } + case tok_null: + lex.next(); + return TypeDataNullLiteral::create(); + case tok_oppar: { + std::vector items = parse_nested_type_list_in_parenthesis(lex); + if (items.size() == 1) { + return items.front(); + } + return TypeDataTensor::create(std::move(items)); + } + case tok_opbracket: { + std::vector items = parse_nested_type_list(lex, tok_opbracket, "`[`", tok_clbracket, "`]` or `,`"); + return TypeDataTypedTuple::create(std::move(items)); + } + default: + lex.unexpected(""); + } +} + +static TypePtr parse_type_nullable(Lexer& lex) { + TypePtr result = parse_simple_type(lex); + + if (lex.tok() == tok_question) { + lex.next(); + result = TypeDataNullable::create(result); + } + + return result; +} + +static TypePtr parse_type_expression(Lexer& lex) { + TypePtr result = parse_type_nullable(lex); + + if (lex.tok() == tok_arrow) { // `int -> int`, `(cell, slice) -> void` + lex.next(); + TypePtr return_type = parse_type_expression(lex); + std::vector params_types = {result}; + if (const auto* as_tensor = result->try_as()) { + params_types = as_tensor->items; + } + return TypeDataFunCallable::create(std::move(params_types), return_type); + } + + if (lex.tok() != tok_bitwise_or) { + return result; + } + + lex.error("union types are not supported yet"); +} + +TypePtr parse_type_from_tokens(Lexer& lex) { + return parse_type_expression(lex); +} + +// for internal usage only +TypePtr parse_type_from_string(std::string_view text) { + Lexer lex(text); + return parse_type_expression(lex); +} + +std::ostream& operator<<(std::ostream& os, TypePtr type_data) { + return os << (type_data ? type_data->as_human_readable() : "(nullptr-type)"); +} + +} // namespace tolk diff --git a/tolk/type-system.h b/tolk/type-system.h new file mode 100644 index 00000000..4b671e30 --- /dev/null +++ b/tolk/type-system.h @@ -0,0 +1,463 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "src-file.h" +#include +#include +#include + +namespace tolk { + +/* + * TypeData is both a user-given and an inferred type representation. + * `int`, `cell`, `T`, `(int, [tuple])` are instances of TypeData. + * Every unique TypeData is created only once, so for example TypeDataTensor::create(int, int) + * returns one and the same pointer always. This "uniqueness" is called type_id, calculated before creation. + * + * In Tolk code, types after colon `var v: (int, T)` are parsed to TypeData. + * See parse_type_from_tokens(). + * So, AST nodes which can have declared types (local/global variables and others) store a pointer to TypeData. + * + * Type inferring also creates TypeData for inferred expressions. All AST expression nodes have inferred_type. + * For example, `1 + 2`, both operands are TypeDataInt, its result is also TypeDataInt. + * Type checking also uses TypeData. For example, `var i: slice = 1 + 2`, at first rhs (TypeDataInt) is inferred, + * then lhs (TypeDataSlice from declaration) is checked whether rhs can be assigned. + * See can_rhs_be_assigned(). + * + * Note, that while initial parsing Tolk files to AST, known types (`int`, `cell`, etc.) are created as-is, + * but user-defined types (`T`, `MyStruct`, `MyAlias`) are saved as TypeDataUnresolved. + * After all symbols have been registered, resolving identifiers step is executed, where particularly + * all TypeDataUnresolved instances are converted to a resolved type. At inferring, no unresolved remain. + * For instance, `fun f(v: T)`, at first "T" of `v` is unresolved, and then converted to TypeDataGenericT. + */ +class TypeData { + // all unique types have unique type_id; it's used both for allocating memory once and for tagged unions + const uint64_t type_id; + // bits of flag_mask, to store often-used properties and return them without tree traversing + const int flags; + // how many slots on a stack this type occupies (calculated on creation), e.g. `int`=1, `(int,int)`=2, `(int,int)?`=3 + const int width_on_stack; + + friend class TypeDataTypeIdCalculation; + +protected: + enum flag_mask { + flag_contains_unknown_inside = 1 << 1, + flag_contains_genericT_inside = 1 << 2, + flag_contains_unresolved_inside = 1 << 3, + }; + + explicit TypeData(uint64_t type_id, int flags_with_children, int width_on_stack) + : type_id(type_id) + , flags(flags_with_children) + , width_on_stack(width_on_stack) { + } + +public: + virtual ~TypeData() = default; + + template + const Derived* try_as() const { + return dynamic_cast(this); + } + + uint64_t get_type_id() const { return type_id; } + int get_width_on_stack() const { return width_on_stack; } + + bool has_unknown_inside() const { return flags & flag_contains_unknown_inside; } + bool has_genericT_inside() const { return flags & flag_contains_genericT_inside; } + bool has_unresolved_inside() const { return flags & flag_contains_unresolved_inside; } + + using TraverserCallbackT = std::function; + using ReplacerCallbackT = std::function; + + virtual std::string as_human_readable() const = 0; + virtual bool can_rhs_be_assigned(TypePtr rhs) const = 0; + virtual bool can_be_casted_with_as_operator(TypePtr cast_to) const = 0; + + virtual bool can_hold_tvm_null_instead() const { + return true; + } + + virtual void traverse(const TraverserCallbackT& callback) const { + callback(this); + } + + virtual TypePtr replace_children_custom(const ReplacerCallbackT& callback) const { + return callback(this); + } +}; + +/* + * `int` is TypeDataInt, representation of TVM int. + */ +class TypeDataInt final : public TypeData { + TypeDataInt() : TypeData(1ULL, 0, 1) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "int"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * `bool` is TypeDataBool. TVM has no bool, only integers. Under the hood, -1 is true, 0 is false. + * From the type system point of view, int and bool are different, not-autocastable types. + */ +class TypeDataBool final : public TypeData { + TypeDataBool() : TypeData(2ULL, 0, 1) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "bool"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * `cell` is TypeDataCell, representation of TVM cell. + */ +class TypeDataCell final : public TypeData { + TypeDataCell() : TypeData(3ULL, 0, 1) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "cell"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * `slice` is TypeDataSlice, representation of TVM slice. + */ +class TypeDataSlice final : public TypeData { + TypeDataSlice() : TypeData(4ULL, 0, 1) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "slice"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * `builder` is TypeDataBuilder, representation of TVM builder. + */ +class TypeDataBuilder final : public TypeData { + TypeDataBuilder() : TypeData(5ULL, 0, 1) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "builder"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * `tuple` is TypeDataTuple, representation of TVM tuple. + * Note, that it's UNTYPED tuple. It occupies 1 stack slot in TVM. Its elements are any TVM values at runtime, + * so getting its element results in TypeDataUnknown (which must be assigned/cast explicitly). + */ +class TypeDataTuple final : public TypeData { + TypeDataTuple() : TypeData(6ULL, 0, 1) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "tuple"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * `continuation` is TypeDataContinuation, representation of TVM continuation. + * It's like "untyped callable", not compatible with other types. + */ +class TypeDataContinuation final : public TypeData { + TypeDataContinuation() : TypeData(7ULL, 0, 1) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "continuation"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * `null` has TypeDataNullLiteral type. + * It can be assigned only to nullable types (`int?`, etc.), to ensure null safety. + * Note, that `var i = null`, though valid (i would be constant null), fires an "always-null" compilation error + * (it's much better for user to see an error here than when he passes this variable somewhere). + */ +class TypeDataNullLiteral final : public TypeData { + TypeDataNullLiteral() : TypeData(8ULL, 0, 1) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "null"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * `T?` is "nullable T". + * It can be converted to T either with ! (non-null assertion operator) or with smart casts. + */ +class TypeDataNullable final : public TypeData { + TypeDataNullable(uint64_t type_id, int children_flags, int width_on_stack, TypePtr inner) + : TypeData(type_id, children_flags, width_on_stack) + , inner(inner) {} + +public: + const TypePtr inner; + + static TypePtr create(TypePtr inner); + + bool is_primitive_nullable() const { return get_width_on_stack() == 1 && inner->get_width_on_stack() == 1; } + + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + void traverse(const TraverserCallbackT& callback) const override; + TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; + bool can_hold_tvm_null_instead() const override; +}; + +/* + * `fun(int, int) -> void` is TypeDataFunCallable, think of is as a typed continuation. + * A type of function `fun f(x: int) { return x; }` is actually `fun(int) -> int`. + * So, when assigning it to a variable `var cb = f`, this variable also has this type. + */ +class TypeDataFunCallable final : public TypeData { + TypeDataFunCallable(uint64_t type_id, int children_flags, std::vector&& params_types, TypePtr return_type) + : TypeData(type_id, children_flags, 1) + , params_types(std::move(params_types)) + , return_type(return_type) {} + +public: + const std::vector params_types; + const TypePtr return_type; + + static TypePtr create(std::vector&& params_types, TypePtr return_type); + + int params_size() const { return static_cast(params_types.size()); } + + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + void traverse(const TraverserCallbackT& callback) const override; + TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; +}; + +/* + * `T` inside generic functions is TypeDataGenericT. + * Example: `fun f(a: X, b: Y): [X, Y]` (here X and Y are). + * On instantiation like `f(1,"")`, a new function `f` is created with type `fun(int,slice)->[int,slice]`. + */ +class TypeDataGenericT final : public TypeData { + TypeDataGenericT(uint64_t type_id, std::string&& nameT) + : TypeData(type_id, flag_contains_genericT_inside, -999999) // width undefined until instantiated + , nameT(std::move(nameT)) {} + +public: + const std::string nameT; + + static TypePtr create(std::string&& nameT); + + std::string as_human_readable() const override { return nameT; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * `(int, slice)` is TypeDataTensor of 2 elements. Tensor of N elements occupies N stack slots. + * Of course, there may be nested tensors, like `(int, (int, slice), cell)`. + * Arguments, variables, globals, return values, etc. can be tensors. + * A tensor can be empty. + */ +class TypeDataTensor final : public TypeData { + TypeDataTensor(uint64_t type_id, int children_flags, int width_on_stack, std::vector&& items) + : TypeData(type_id, children_flags, width_on_stack) + , items(std::move(items)) {} + +public: + const std::vector items; + + static TypePtr create(std::vector&& items); + + int size() const { return static_cast(items.size()); } + + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + void traverse(const TraverserCallbackT& callback) const override; + TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; + bool can_hold_tvm_null_instead() const override; +}; + +/* + * `[int, slice]` is TypeDataTypedTuple, a TVM 'tuple' under the hood, contained in 1 stack slot. + * Unlike TypeDataTuple (untyped tuples), it has a predefined inner structure and can be assigned as + * `var [i, cs] = [0, ""]` (where a and b become two separate variables on a stack, int and slice). + */ +class TypeDataTypedTuple final : public TypeData { + TypeDataTypedTuple(uint64_t type_id, int children_flags, std::vector&& items) + : TypeData(type_id, children_flags, 1) + , items(std::move(items)) {} + +public: + const std::vector items; + + static TypePtr create(std::vector&& items); + + int size() const { return static_cast(items.size()); } + + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + void traverse(const TraverserCallbackT& callback) const override; + TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; +}; + +/* + * `unknown` is a special type, which can appear in corner cases. + * The type of exception argument (which can hold any TVM value at runtime) is unknown. + * The type of `_` used as rvalue is unknown. + * The only thing available to do with unknown is to cast it: `catch (excNo, arg) { var i = arg as int; }` + */ +class TypeDataUnknown final : public TypeData { + TypeDataUnknown() : TypeData(20ULL, flag_contains_unknown_inside, 1) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "unknown"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * "Unresolved" is not actually a type — it's an intermediate state between parsing and resolving. + * At parsing to AST, unrecognized type names (MyEnum, MyStruct, T) are parsed as TypeDataUnresolved, + * and after all source files parsed and global symbols registered, they are replaced by actual ones. + * Example: `fun f(v: T)` at first v is TypeDataUnresolved("T"), later becomes TypeDataGenericT. + */ +class TypeDataUnresolved final : public TypeData { + TypeDataUnresolved(uint64_t type_id, std::string&& text, SrcLocation loc) + : TypeData(type_id, flag_contains_unresolved_inside, -999999) + , text(std::move(text)) + , loc(loc) {} + +public: + const std::string text; + const SrcLocation loc; + + static TypePtr create(std::string&& text, SrcLocation loc); + + std::string as_human_readable() const override { return text + "*"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * `never` is a special type meaning "no value can be hold". + * Is may appear due to smart casts, for example `if (x == null && x != null)` makes x "never". + * Functions returning "never" assume to never exit, calling them interrupts control flow. + * Such variables can not be cast to any other types, all their usage will trigger type mismatch errors. + */ +class TypeDataNever final : public TypeData { + TypeDataNever() : TypeData(19ULL, 0, 0) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "never"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + bool can_hold_tvm_null_instead() const override; +}; + +/* + * `void` is TypeDataVoid. + * From the type system point of view, `void` functions return nothing. + * Empty tensor is not compatible with void, although at IR level they are similar, 0 stack slots. + */ +class TypeDataVoid final : public TypeData { + TypeDataVoid() : TypeData(10ULL, 0, 0) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "void"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + bool can_hold_tvm_null_instead() const override; +}; + + +// -------------------------------------------- + + +class Lexer; +TypePtr parse_type_from_tokens(Lexer& lex); +TypePtr parse_type_from_string(std::string_view text); + +void type_system_init(); + +} // namespace tolk diff --git a/ton/ton-tl.hpp b/ton/ton-tl.hpp index 6d676ea8..1bc9a28c 100644 --- a/ton/ton-tl.hpp +++ b/ton/ton-tl.hpp @@ -19,7 +19,8 @@ #pragma once #include "ton-types.h" -#include "auto/tl/ton_api.h" +#include "auto/tl/ton_api.hpp" +#include "td/utils/overloaded.h" namespace ton { @@ -53,4 +54,12 @@ inline ZeroStateIdExt create_zero_state_id(tl_object_ptrworkchain_, B->root_hash_, B->file_hash_}; } +inline ShardIdFull create_shard_id(const tl_object_ptr &s) { + return ShardIdFull{s->workchain_, static_cast(s->shard_)}; +} + +inline tl_object_ptr create_tl_shard_id(const ShardIdFull &s) { + return create_tl_object(s.workchain, s.shard); +} + } // namespace ton diff --git a/ton/ton-types.h b/ton/ton-types.h index 867584fa..c7aff644 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -51,13 +51,18 @@ using ValidatorSessionId = td::Bits256; constexpr WorkchainId masterchainId = -1, basechainId = 0, workchainInvalid = 0x80000000; constexpr ShardId shardIdAll = (1ULL << 63); +constexpr int max_shard_pfx_len = 60; + enum GlobalCapabilities { capIhrEnabled = 1, capCreateStatsEnabled = 2, capBounceMsgBody = 4, capReportVersion = 8, capSplitMergeTransactions = 16, - capShortDequeue = 32 + capShortDequeue = 32, + capStoreOutMsgQueueSize = 64, + capMsgMetadata = 128, + capDeferMessages = 256 }; inline int shard_pfx_len(ShardId shard) { @@ -115,6 +120,26 @@ struct ShardIdFull { char buffer[64]; return std::string{buffer, (unsigned)snprintf(buffer, 63, "(%d,%016llx)", workchain, (unsigned long long)shard)}; } + static td::Result parse(td::Slice s) { + // Formats: (0,2000000000000000) (0:2000000000000000) 0,2000000000000000 0:2000000000000000 + if (s.empty()) { + return td::Status::Error("empty string"); + } + if (s[0] == '(' && s.back() == ')') { + s = s.substr(1, s.size() - 2); + } + auto sep = s.find(':'); + if (sep == td::Slice::npos) { + sep = s.find(','); + } + if (sep == td::Slice::npos || s.size() - sep - 1 != 16) { + return td::Status::Error(PSTRING() << "invalid shard " << s); + } + ShardIdFull shard; + TRY_RESULT_ASSIGN(shard.workchain, td::to_integer_safe(s.substr(0, sep))); + TRY_RESULT_ASSIGN(shard.shard, td::hex_to_integer_safe(s.substr(sep + 1))); + return shard; + } }; struct AccountIdPrefixFull { @@ -344,6 +369,10 @@ struct BlockSignature { struct ReceivedBlock { BlockIdExt id; td::BufferSlice data; + + ReceivedBlock clone() const { + return ReceivedBlock{id, data.clone()}; + } }; struct BlockBroadcast { @@ -353,6 +382,14 @@ struct BlockBroadcast { td::uint32 validator_set_hash; td::BufferSlice data; td::BufferSlice proof; + + BlockBroadcast clone() const { + std::vector new_signatures; + for (const BlockSignature& s : signatures) { + new_signatures.emplace_back(s.node, s.signature.clone()); + } + return {block_id, std::move(new_signatures), catchain_seqno, validator_set_hash, data.clone(), proof.clone()}; + } }; struct Ed25519_PrivateKey { @@ -391,6 +428,9 @@ struct Ed25519_PublicKey { bool operator==(const Ed25519_PublicKey& other) const { return _pubkey == other._pubkey; } + bool operator!=(const Ed25519_PublicKey& other) const { + return _pubkey != other._pubkey; + } bool clear() { _pubkey.set_zero(); return true; @@ -453,6 +493,7 @@ struct CatChainOptions { td::uint64 max_block_height_coeff = 0; bool debug_disable_db = false; + double broadcast_speed_multiplier = 1.0; }; struct ValidatorSessionConfig { @@ -473,4 +514,14 @@ struct ValidatorSessionConfig { static const td::uint32 BLOCK_HASH_COVERS_DATA_FROM_VERSION = 2; }; +struct PersistentStateDescription : public td::CntObject { + BlockIdExt masterchain_id; + std::vector shard_blocks; + UnixTime start_time, end_time; + + virtual CntObject* make_copy() const { + return new PersistentStateDescription(*this); + } +}; + } // namespace ton diff --git a/tonlib/CMakeLists.txt b/tonlib/CMakeLists.txt index f27e00f2..3dbd628d 100644 --- a/tonlib/CMakeLists.txt +++ b/tonlib/CMakeLists.txt @@ -1,4 +1,6 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) + +option(TONLIBJSON_STATIC "Build tonlibjson as static library" OFF) if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) @@ -8,7 +10,6 @@ set(TONLIB_SOURCE tonlib/Client.cpp tonlib/Config.cpp tonlib/ExtClient.cpp - tonlib/ExtClientLazy.cpp tonlib/ExtClientOutbound.cpp tonlib/KeyStorage.cpp tonlib/KeyValue.cpp @@ -23,7 +24,6 @@ set(TONLIB_SOURCE tonlib/Client.h tonlib/Config.h tonlib/ExtClient.h - tonlib/ExtClientLazy.h tonlib/ExtClientOutbound.h tonlib/KeyStorage.h tonlib/KeyValue.h @@ -60,7 +60,7 @@ target_include_directories(tonlib PUBLIC $/.. $ ) -target_link_libraries(tonlib PRIVATE tdactor adnllite tl_lite_api tl-lite-utils ton_crypto ton_block lite-client-common smc-envelope) +target_link_libraries(tonlib PRIVATE tdactor adnllite tl_lite_api tl-lite-utils ton_crypto lite-client-common smc-envelope emulator_static) target_link_libraries(tonlib PUBLIC tdutils tl_tonlib_api) if (TONLIB_ENABLE_JNI AND NOT ANDROID) # jni is available by default on Android @@ -92,14 +92,22 @@ set(TONLIB_JSON_HEADERS tonlib/tonlib_client_json.h) set(TONLIB_JSON_SOURCE tonlib/tonlib_client_json.cpp) include(GenerateExportHeader) -if (NOT USE_EMSCRIPTEN) - add_library(tonlibjson SHARED ${TONLIB_JSON_SOURCE} ${TONLIB_JSON_HEADERS}) +if (TONLIBJSON_STATIC OR USE_EMSCRIPTEN) + add_library(tonlibjson STATIC ${TONLIB_JSON_SOURCE}) else() - add_library(tonlibjson STATIC ${TONLIB_JSON_SOURCE} ${TONLIB_JSON_HEADERS}) + add_library(tonlibjson SHARED ${TONLIB_JSON_SOURCE}) +endif() + +if (PORTABLE AND NOT APPLE) + target_link_libraries(tonlibjson PRIVATE tonlibjson_private -static-libgcc -static-libstdc++) +else() + target_link_libraries(tonlibjson PRIVATE tonlibjson_private) endif() -target_link_libraries(tonlibjson PRIVATE tonlibjson_private) generate_export_header(tonlibjson EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/tonlib/tonlibjson_export.h) +if (TONLIBJSON_STATIC OR USE_EMSCRIPTEN) + target_compile_definitions(tonlibjson PUBLIC TONLIBJSON_STATIC_DEFINE) +endif() target_include_directories(tonlibjson PUBLIC $ $) @@ -107,13 +115,6 @@ if (APPLE) set_target_properties(tonlibjson PROPERTIES LINK_FLAGS "-Wl,-exported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/tonlibclientjson_export_list") endif() -add_library(tonlibjson_static STATIC ${TONLIB_JSON_SOURCE} ${TONLIB_JSON_HEADERS}) -target_link_libraries(tonlibjson_static PRIVATE tonlibjson_private) -target_compile_definitions(tonlibjson_static PUBLIC TONLIBJSON_STATIC_DEFINE) -target_include_directories(tonlibjson_static PUBLIC - $ - $) - add_library(TonlibJson INTERFACE) target_link_libraries(TonlibJson INTERFACE tonlibjson) @@ -135,8 +136,8 @@ if (NOT TON_USE_ABSEIL) if (WIN32) set(WINGETOPT_TARGET wingetopt) endif() -install(TARGETS tdnet keys crc32c tdactor adnllite tl_api tl-utils tl_lite_api tl-lite-utils ton_crypto ton_block smc-envelope ${WINGETOPT_TARGET} - tdutils tl_tonlib_api tonlib lite-client-common tddb_utils Tonlib EXPORT Tonlib +install(TARGETS tdnet keys crc32c tdactor adnllite tl_api tl-utils tl_lite_api tl-lite-utils ton_crypto ton_crypto_core ton_block smc-envelope ${WINGETOPT_TARGET} + tdutils tl_tonlib_api tonlib lite-client-common tddb_utils emulator_static Tonlib EXPORT Tonlib LIBRARY DESTINATION lib ARCHIVE DESTINATION lib RUNTIME DESTINATION bin @@ -158,13 +159,17 @@ endif() install(FILES ${TONLIB_JSON_HEADERS} ${CMAKE_CURRENT_BINARY_DIR}/tonlib/tonlibjson_export.h DESTINATION include/tonlib/) -if (NOT USE_EMSCRIPTEN) +if (NOT USE_EMSCRIPTEN AND NOT TONLIBJSON_STATIC) install(EXPORT Tonlib FILE TonlibTargets.cmake NAMESPACE Tonlib:: DESTINATION lib/cmake/Tonlib ) + + # Add SOVERSION to shared libraries + set_property(TARGET tonlibjson PROPERTY SOVERSION ${TON_VERSION}) endif() + include(CMakePackageConfigHelpers) write_basic_package_version_file("TonlibConfigVersion.cmake" VERSION ${TON_VERSION} @@ -174,6 +179,4 @@ install(FILES "TonlibConfig.cmake" "${CMAKE_CURRENT_BINARY_DIR}/TonlibConfigVers DESTINATION lib/cmake/Tonlib ) -# Add SOVERSION to shared libraries -set_property(TARGET tonlibjson PROPERTY SOVERSION ${TON_VERSION}) install(TARGETS tonlib-cli RUNTIME DESTINATION bin) diff --git a/tonlib/test/offline.cpp b/tonlib/test/offline.cpp index a1e5a0f6..47d5d6a2 100644 --- a/tonlib/test/offline.cpp +++ b/tonlib/test/offline.cpp @@ -83,7 +83,7 @@ using namespace tonlib; TEST(Tonlib, PublicKey) { block::PublicKey::parse("pubjns2gp7DGCnEH7EOWeCnb6Lw1akm538YYaz6sdLVHfRB2").ensure_error(); auto key = block::PublicKey::parse("Pubjns2gp7DGCnEH7EOWeCnb6Lw1akm538YYaz6sdLVHfRB2").move_as_ok(); - CHECK(td::buffer_to_hex(key.key) == "3EE9DC0A7A0B6CA01770CE34698792BD8ECB53A6949BFD6C81B6E3CA475B74D7"); + CHECK(td::buffer_to_hex(key.key) == "E39ECDA0A7B0C60A7107EC43967829DBE8BC356A49B9DFC6186B3EAC74B5477D"); CHECK(key.serialize() == "Pubjns2gp7DGCnEH7EOWeCnb6Lw1akm538YYaz6sdLVHfRB2"); } @@ -333,8 +333,8 @@ TEST(Tonlib, ConfigParseBug) { unsigned char buff[128]; int bits = (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), literal.begin(), literal.end()); CHECK(bits >= 0); - auto slice = vm::CellBuilder().store_bits(td::ConstBitPtr{buff}, bits).finalize(); - block::Config::do_get_gas_limits_prices(std::move(slice), 21).ensure(); + auto cell = vm::CellBuilder().store_bits(td::ConstBitPtr{buff}, bits).finalize(); + block::Config::do_get_gas_limits_prices(vm::load_cell_slice(cell), 21).ensure(); } TEST(Tonlib, EncryptionApi) { @@ -467,15 +467,20 @@ TEST(Tonlib, KeysApi) { make_object(make_object(key->public_key_, key->secret_.copy()))) .move_as_ok(); - auto err1 = sync_send(client, make_object( - new_local_password.copy(), td::SecureString("wrong password"), - make_object(copy_word_list()))) - .move_as_error(); + auto err1 = sync_send( + client, make_object(new_local_password.copy(), td::SecureString("wrong password"), + make_object(copy_word_list()))); + if (err1.is_ok()) { + if (err1.ok()->public_key_ != key->public_key_) { + err1 = td::Status::Error("imported key successfully, but the public key is different"); + } + } + err1.ensure_error(); auto err2 = sync_send(client, make_object(new_local_password.copy(), td::SecureString(), - make_object(copy_word_list()))) - .move_as_error(); - LOG(INFO) << err1 << " | " << err2; + make_object(copy_word_list()))); + err2.ensure_error(); + LOG(INFO) << err1.move_as_error() << " | " << err2.move_as_error(); auto imported_key = sync_send(client, make_object(new_local_password.copy(), mnemonic_password.copy(), make_object(copy_word_list()))) @@ -609,7 +614,30 @@ TEST(Tonlib, ConfigCache) { "seqno": 0, "root_hash": "gj+B8wb/AmlPk1z1AhVI484rhrUpgSr2oSFIh56VoSg=", "file_hash": "Z+IKwYS54DmmJmesw/nAD5DzWadnOCMzee+kdgSYDOg=" - } + }, + "hardforks": [ + { + "file_hash": "jF3RTD+OyOoP+OI9oIjdV6M8EaOh9E+8+c3m5JkPYdg=", + "seqno": 5141579, + "root_hash": "6JSqIYIkW7y8IorxfbQBoXiuY3kXjcoYgQOxTJpjXXA=", + "workchain": -1, + "shard": -9223372036854775808 + }, + { + "file_hash": "WrNoMrn5UIVPDV/ug/VPjYatvde8TPvz5v1VYHCLPh8=", + "seqno": 5172980, + "root_hash": "054VCNNtUEwYGoRe1zjH+9b1q21/MeM+3fOo76Vcjes=", + "workchain": -1, + "shard": -9223372036854775808 + }, + { + "file_hash": "xRaxgUwgTXYFb16YnR+Q+VVsczLl6jmYwvzhQ/ncrh4=", + "seqno": 5176527, + "root_hash": "SoPLqMe9Dz26YJPOGDOHApTSe5i0kXFtRmRh/zPMGuI=", + "workchain": -1, + "shard": -9223372036854775808 + } + ] } })abc"; auto custom = R"abc({ @@ -631,11 +659,12 @@ TEST(Tonlib, ConfigCache) { ], "validator": { "@type": "validator.config.global", - "zero_state": { + "init_block": { "workchain": -1, "shard": -9223372036854775808, "seqno": 0, "file_hash": "eh9yveSz1qMdJ7mOsO+I+H77jkLr9NpAuEkoJuseXBo=" + "root_hash": "ZXSXxDHhTALFxReyTZRd8E4Ya3ySOmpOWAS4rBX9XBY=", } } })abc"; diff --git a/tonlib/test/online.cpp b/tonlib/test/online.cpp index 52f51720..cf892c29 100644 --- a/tonlib/test/online.cpp +++ b/tonlib/test/online.cpp @@ -264,7 +264,8 @@ td::Result create_send_grams_query(Client& client, const Wallet& source data = tonlib_api::make_object(message.raw.unwrap(), message.init_state.unwrap()); } msgs.push_back(tonlib_api::make_object( - tonlib_api::make_object(destination), "", amount, std::move(data), -1)); + tonlib_api::make_object(destination), "", amount, + std::vector>{}, std::move(data), -1)); auto r_id = sync_send(client, tonlib_api::make_object( @@ -566,7 +567,7 @@ void test_multisig(Client& client, const Wallet& giver_wallet) { for (int i = 0; i < 2; i++) { // Just transfer all (some) money back in one query vm::CellBuilder icb; - ton::GenericAccount::store_int_message(icb, block::StdAddress::parse(giver_wallet.address).move_as_ok(), 1); + ton::GenericAccount::store_int_message(icb, block::StdAddress::parse(giver_wallet.address).move_as_ok(), 1, {}); icb.store_bytes("\0\0\0\0", 4); vm::CellString::store(icb, "Greatings from multisig", 35 * 8).ensure(); ton::MultisigWallet::QueryBuilder qb(wallet_id, -1 - i, icb.finalize()); diff --git a/tonlib/tonlib/CellString.cpp b/tonlib/tonlib/CellString.cpp deleted file mode 100644 index ad2cbf5f..00000000 --- a/tonlib/tonlib/CellString.cpp +++ /dev/null @@ -1,64 +0,0 @@ -#include "CellString.h" -#include "td/utils/misc.h" - -#include "vm/cells/CellSlice.h" - -namespace vm { -td::Status CellString::store(CellBuilder &cb, td::Slice slice, unsigned int top_bits) { - td::uint32 size = td::narrow_cast(slice.size() * 8); - return store(cb, td::BitSlice(slice.ubegin(), size), top_bits); -} - -td::Status CellString::store(CellBuilder &cb, td::BitSlice slice, unsigned int top_bits) { - if (slice.size() > max_bytes * 8) { - return td::Status::Error("String is too long (1)"); - } - unsigned int head = td::min(slice.size(), td::min(cb.remaining_bits(), top_bits)) / 8 * 8; - auto max_bits = vm::Cell::max_bits / 8 * 8; - auto depth = 1 + (slice.size() - head + max_bits - 1) / max_bits; - if (depth > max_chain_length) { - return td::Status::Error("String is too long (2)"); - } - cb.append_bitslice(slice.subslice(0, head)); - slice.advance(head); - if (slice.size() == 0) { - return td::Status::OK(); - } - CellBuilder child_cb; - store(child_cb, std::move(slice)); - cb.store_ref(child_cb.finalize()); - return td::Status::OK(); -} - -template -void CellString::for_each(F &&f, CellSlice &cs, unsigned int top_bits) { - unsigned int head = td::min(cs.size(), top_bits); - f(cs.prefetch_bits(head)); - if (!cs.have_refs()) { - return; - } - auto ref = cs.prefetch_ref(); - while (true) { - auto cs = vm::load_cell_slice(ref); - f(cs.prefetch_bits(cs.size())); - if (!cs.have_refs()) { - return; - } - ref = cs.prefetch_ref(); - } -} - -td::Result CellString::load(CellSlice &cs, unsigned int top_bits) { - unsigned int size = 0; - for_each([&](auto slice) { size += slice.size(); }, cs, top_bits); - if (size % 8 != 0) { - return td::Status::Error("Size is not divisible by 8"); - } - std::string res(size / 8, 0); - - td::BitPtr to(td::MutableSlice(res).ubegin()); - for_each([&](auto slice) { to.concat(slice); }, cs, top_bits); - CHECK(to.offs == (int)size); - return res; -} -} // namespace vm diff --git a/tonlib/tonlib/CellString.h b/tonlib/tonlib/CellString.h deleted file mode 100644 index 89c933d8..00000000 --- a/tonlib/tonlib/CellString.h +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -#include "td/utils/Status.h" - -#include "vm/cells/CellBuilder.h" - -namespace vm { -class CellString { - public: - static constexpr unsigned int max_bytes = 1024; - static constexpr unsigned int max_chain_length = 16; - - static td::Status store(CellBuilder &cb, td::Slice slice, unsigned int top_bits = Cell::max_bits); - static td::Status store(CellBuilder &cb, td::BitSlice slice, unsigned int top_bits = Cell::max_bits); - static td::Result load(CellSlice &cs, unsigned int top_bits = Cell::max_bits); - - private: - template - static void for_each(F &&f, CellSlice &cs, unsigned int top_bits = Cell::max_bits); -}; - -} // namespace vm diff --git a/tonlib/tonlib/Config.cpp b/tonlib/tonlib/Config.cpp index 3cd9d50e..063b34bb 100644 --- a/tonlib/tonlib/Config.cpp +++ b/tonlib/tonlib/Config.cpp @@ -19,6 +19,8 @@ #include "Config.h" #include "adnl/adnl-node-id.hpp" #include "td/utils/JsonBuilder.h" +#include "auto/tl/ton_api_json.h" +#include "ton/ton-tl.hpp" namespace tonlib { td::Result parse_block_id_ext(td::JsonObject &obj) { @@ -63,75 +65,26 @@ td::Result parse_block_id_ext(td::JsonObject &obj) { td::Result Config::parse(std::string str) { TRY_RESULT(json, td::json_decode(str)); if (json.type() != td::JsonValue::Type::Object) { - return td::Status::Error("Invalid config (1)"); + return td::Status::Error("Invalid config: json is not an object"); } - //TRY_RESULT(main_type, td::get_json_object_string_field(json.get_object(), "@type", false)); - //if (main_type != "config.global") { - //return td::Status::Error("Invalid config (3)"); - //} - TRY_RESULT(lite_clients_obj, - td::get_json_object_field(json.get_object(), "liteservers", td::JsonValue::Type::Array, false)); - auto &lite_clients = lite_clients_obj.get_array(); - Config res; - for (auto &value : lite_clients) { - if (value.type() != td::JsonValue::Type::Object) { - return td::Status::Error("Invalid config (2)"); - } - auto &object = value.get_object(); - //TRY_RESULT(value_type, td::get_json_object_string_field(object, "@type", false)); - //if (value_type != "liteclient.config.global") { - //return td::Status::Error("Invalid config (4)"); - //} + ton::ton_api::liteclient_config_global conf; + TRY_STATUS(ton::ton_api::from_json(conf, json.get_object())); + TRY_RESULT_ASSIGN(res.lite_servers, liteclient::LiteServerConfig::parse_global_config(conf)); - TRY_RESULT(ip, td::get_json_object_long_field(object, "ip", false)); - TRY_RESULT(port, td::get_json_object_int_field(object, "port", false)); - Config::LiteClient client; - TRY_STATUS(client.address.init_host_port(td::IPAddress::ipv4_to_str(static_cast(ip)), port)); - - TRY_RESULT(id_obj, td::get_json_object_field(object, "id", td::JsonValue::Type::Object, false)); - auto &id = id_obj.get_object(); - TRY_RESULT(id_type, td::get_json_object_string_field(id, "@type", false)); - if (id_type != "pub.ed25519") { - return td::Status::Error("Invalid config (5)"); - } - TRY_RESULT(key_base64, td::get_json_object_string_field(id, "key", false)); - TRY_RESULT(key, td::base64_decode(key_base64)); - if (key.size() != 32) { - return td::Status::Error("Invalid config (6)"); - } - - client.adnl_id = ton::adnl::AdnlNodeIdFull(ton::pubkeys::Ed25519(td::Bits256(td::Slice(key).ubegin()))); - res.lite_clients.push_back(std::move(client)); + if (!conf.validator_) { + return td::Status::Error("Invalid config: no 'validator' section"); + } + if (!conf.validator_->zero_state_) { + return td::Status::Error("Invalid config: no zerostate"); + } + res.zero_state_id = ton::create_block_id(conf.validator_->zero_state_); + if (conf.validator_->init_block_) { + res.init_block_id = ton::create_block_id(conf.validator_->init_block_); } - TRY_RESULT(validator_obj, - td::get_json_object_field(json.get_object(), "validator", td::JsonValue::Type::Object, false)); - auto &validator = validator_obj.get_object(); - TRY_RESULT(validator_type, td::get_json_object_string_field(validator, "@type", false)); - if (validator_type != "validator.config.global") { - return td::Status::Error("Invalid config (7)"); - } - TRY_RESULT(zero_state_obj, td::get_json_object_field(validator, "zero_state", td::JsonValue::Type::Object, false)); - TRY_RESULT(zero_state_id, parse_block_id_ext(zero_state_obj.get_object())); - res.zero_state_id = zero_state_id; - auto r_init_block_obj = td::get_json_object_field(validator, "init_block", td::JsonValue::Type::Object, false); - if (r_init_block_obj.is_ok()) { - TRY_RESULT(init_block_id, parse_block_id_ext(r_init_block_obj.move_as_ok().get_object())); - res.init_block_id = init_block_id; - } - - auto r_hardforks = td::get_json_object_field(validator, "hardforks", td::JsonValue::Type::Array, false); - if (r_hardforks.is_ok()) { - auto hardforks_obj = r_hardforks.move_as_ok(); - auto &hardforks = hardforks_obj.get_array(); - for (auto &fork : hardforks) { - if (fork.type() != td::JsonValue::Type::Object) { - return td::Status::Error("Invalid config (8)"); - } - TRY_RESULT(fork_block, parse_block_id_ext(fork.get_object())); - res.hardforks.push_back(std::move(fork_block)); - } + for (auto &fork : conf.validator_->hardforks_) { + res.hardforks.push_back(ton::create_block_id(fork)); } for (auto hardfork : res.hardforks) { diff --git a/tonlib/tonlib/Config.h b/tonlib/tonlib/Config.h index 3902c341..28f23881 100644 --- a/tonlib/tonlib/Config.h +++ b/tonlib/tonlib/Config.h @@ -20,17 +20,14 @@ #include "adnl/adnl-node-id.hpp" #include "td/utils/port/IPAddress.h" #include "ton/ton-types.h" +#include "lite-client/ext-client.h" namespace tonlib { struct Config { - struct LiteClient { - ton::adnl::AdnlNodeIdFull adnl_id; - td::IPAddress address; - }; ton::BlockIdExt zero_state_id; ton::BlockIdExt init_block_id; std::vector hardforks; - std::vector lite_clients; + std::vector lite_servers; std::string name; static td::Result parse(std::string str); }; diff --git a/tonlib/tonlib/ExtClient.cpp b/tonlib/tonlib/ExtClient.cpp index 30a29b59..b66ca25c 100644 --- a/tonlib/tonlib/ExtClient.cpp +++ b/tonlib/tonlib/ExtClient.cpp @@ -65,7 +65,7 @@ void ExtClient::send_raw_query(td::BufferSlice query, td::Promise adnl_ext_client_; + td::actor::ActorId adnl_ext_client_; td::actor::ActorId last_block_actor_; td::actor::ActorId last_config_actor_; }; @@ -97,7 +97,7 @@ class ExtClient { void force_change_liteserver() { if (!client_.adnl_ext_client_.empty()) { - td::actor::send_closure(client_.adnl_ext_client_, &ExtClientLazy::force_change_liteserver); + td::actor::send_closure(client_.adnl_ext_client_, &liteclient::ExtClient::reset_servers); } } diff --git a/tonlib/tonlib/ExtClientLazy.cpp b/tonlib/tonlib/ExtClientLazy.cpp deleted file mode 100644 index 335a0ff9..00000000 --- a/tonlib/tonlib/ExtClientLazy.cpp +++ /dev/null @@ -1,152 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "ExtClientLazy.h" -#include "TonlibError.h" -#include "td/utils/Random.h" -namespace tonlib { - -class ExtClientLazyImp : public ExtClientLazy { - public: - ExtClientLazyImp(std::vector> servers, - td::unique_ptr callback) - : servers_(std::move(servers)), callback_(std::move(callback)) { - CHECK(!servers_.empty()); - } - - void start_up() override { - td::Random::Fast rnd; - td::random_shuffle(td::as_mutable_span(servers_), rnd); - } - - void check_ready(td::Promise promise) override { - before_query(); - if (client_.empty()) { - return promise.set_error(TonlibError::Cancelled()); - } - send_closure(client_, &ton::adnl::AdnlExtClient::check_ready, std::move(promise)); - } - - void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, - td::Promise promise) override { - before_query(); - if (client_.empty()) { - return promise.set_error(TonlibError::Cancelled()); - } - td::Promise P = [SelfId = actor_id(this), idx = cur_server_idx_, - promise = std::move(promise)](td::Result R) mutable { - if (R.is_error() && - (R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) { - td::actor::send_closure(SelfId, &ExtClientLazyImp::set_server_bad, idx, true); - } - promise.set_result(std::move(R)); - }; - send_closure(client_, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout, - std::move(P)); - } - - void force_change_liteserver() override { - if (servers_.size() == 1) { - return; - } - cur_server_bad_ = cur_server_bad_force_ = true; - } - - private: - void before_query() { - if (is_closing_) { - return; - } - alarm_timestamp() = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT); - if (cur_server_bad_) { - ++cur_server_idx_; - } else if (!client_.empty()) { - return; - } - class Callback : public ton::adnl::AdnlExtClient::Callback { - public: - explicit Callback(td::actor::ActorShared parent, size_t idx) - : parent_(std::move(parent)), idx_(idx) { - } - void on_ready() override { - td::actor::send_closure(parent_, &ExtClientLazyImp::set_server_bad, idx_, false); - } - void on_stop_ready() override { - td::actor::send_closure(parent_, &ExtClientLazyImp::set_server_bad, idx_, true); - } - - private: - td::actor::ActorShared parent_; - size_t idx_; - }; - ref_cnt_++; - cur_server_bad_ = false; - cur_server_bad_force_ = false; - const auto& s = servers_[cur_server_idx_ % servers_.size()]; - LOG(INFO) << "Connecting to liteserver " << s.second; - client_ = ton::adnl::AdnlExtClient::create( - s.first, s.second, std::make_unique(td::actor::actor_shared(this), cur_server_idx_)); - } - - std::vector> servers_; - size_t cur_server_idx_ = 0; - bool cur_server_bad_ = false; - bool cur_server_bad_force_ = false; - - td::actor::ActorOwn client_; - td::unique_ptr callback_; - static constexpr double MAX_NO_QUERIES_TIMEOUT = 100; - - bool is_closing_{false}; - td::uint32 ref_cnt_{1}; - - void set_server_bad(size_t idx, bool bad) { - if (idx == cur_server_idx_ && servers_.size() > 1 && !cur_server_bad_force_) { - cur_server_bad_ = bad; - } - } - void alarm() override { - client_.reset(); - } - void hangup_shared() override { - ref_cnt_--; - try_stop(); - } - void hangup() override { - is_closing_ = true; - ref_cnt_--; - client_.reset(); - try_stop(); - } - void try_stop() { - if (is_closing_ && ref_cnt_ == 0) { - stop(); - } - } -}; - -td::actor::ActorOwn ExtClientLazy::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, - td::unique_ptr callback) { - return create({std::make_pair(dst, dst_addr)}, std::move(callback)); -} - -td::actor::ActorOwn ExtClientLazy::create( - std::vector> servers, td::unique_ptr callback) { - return td::actor::create_actor("ExtClientLazy", std::move(servers), std::move(callback)); -} -} // namespace tonlib diff --git a/tonlib/tonlib/ExtClientOutbound.cpp b/tonlib/tonlib/ExtClientOutbound.cpp index 025ba848..e5fac8b4 100644 --- a/tonlib/tonlib/ExtClientOutbound.cpp +++ b/tonlib/tonlib/ExtClientOutbound.cpp @@ -20,15 +20,12 @@ #include "ExtClientOutbound.h" #include "TonlibError.h" #include + namespace tonlib { -class ExtClientOutboundImp : public ExtClientOutbound { +class ExtClientOutboundImpl : public ExtClientOutbound { public: - ExtClientOutboundImp(td::unique_ptr callback) : callback_(std::move(callback)) { - } - - void check_ready(td::Promise promise) override { - promise.set_error(td::Status::Error("Not supported")); + ExtClientOutboundImpl(td::unique_ptr callback) : callback_(std::move(callback)) { } void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, @@ -38,9 +35,6 @@ class ExtClientOutboundImp : public ExtClientOutbound { callback_->request(query_id, data.as_slice().str()); } - void force_change_liteserver() override { - } - void on_query_result(td::int64 id, td::Result r_data, td::Promise promise) override { auto it = queries_.find(id); if (it == queries_.end()) { @@ -66,6 +60,6 @@ class ExtClientOutboundImp : public ExtClientOutbound { }; td::actor::ActorOwn ExtClientOutbound::create(td::unique_ptr callback) { - return td::actor::create_actor("ExtClientOutbound", std::move(callback)); + return td::actor::create_actor("ExtClientOutbound", std::move(callback)); } } // namespace tonlib diff --git a/tonlib/tonlib/ExtClientOutbound.h b/tonlib/tonlib/ExtClientOutbound.h index 87b73b9e..bf52c8c2 100644 --- a/tonlib/tonlib/ExtClientOutbound.h +++ b/tonlib/tonlib/ExtClientOutbound.h @@ -18,11 +18,10 @@ */ #pragma once #include "td/actor/actor.h" - -#include "ExtClientLazy.h" +#include "lite-client/ext-client.h" namespace tonlib { -class ExtClientOutbound : public ExtClientLazy { +class ExtClientOutbound : public liteclient::ExtClient { public: class Callback { public: diff --git a/tonlib/tonlib/GenericAccount.cpp b/tonlib/tonlib/GenericAccount.cpp deleted file mode 100644 index 9f44ede2..00000000 --- a/tonlib/tonlib/GenericAccount.cpp +++ /dev/null @@ -1,80 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2019 Telegram Systems LLP -*/ -#include "tonlib/GenericAccount.h" -#include "tonlib/utils.h" -#include "block/block-auto.h" -namespace tonlib { -td::Ref GenericAccount::get_init_state(td::Ref code, td::Ref data) noexcept { - return vm::CellBuilder() - .append_cellslice(binary_bitstring_to_cellslice("b{00110}").move_as_ok()) - .store_ref(std::move(code)) - .store_ref(std::move(data)) - .finalize(); -} -block::StdAddress GenericAccount::get_address(ton::WorkchainId workchain_id, - const td::Ref& init_state) noexcept { - return block::StdAddress(workchain_id, init_state->get_hash().bits(), true /*bounce*/); -} -td::Ref GenericAccount::create_ext_message(const block::StdAddress& address, td::Ref new_state, - td::Ref body) noexcept { - block::gen::Message::Record message; - /*info*/ { - block::gen::CommonMsgInfo::Record_ext_in_msg_info info; - /* src */ - tlb::csr_pack(info.src, block::gen::MsgAddressExt::Record_addr_none{}); - /* dest */ { - block::gen::MsgAddressInt::Record_addr_std dest; - dest.anycast = vm::CellBuilder().store_zeroes(1).as_cellslice_ref(); - dest.workchain_id = address.workchain; - dest.address = address.addr; - - tlb::csr_pack(info.dest, dest); - } - /* import_fee */ { - vm::CellBuilder cb; - block::tlb::t_Grams.store_integer_value(cb, td::BigInt256(0)); - info.import_fee = cb.as_cellslice_ref(); - } - - tlb::csr_pack(message.info, info); - } - /* init */ { - if (new_state.not_null()) { - // Just(Left(new_state)) - message.init = vm::CellBuilder() - .store_ones(1) - .store_zeroes(1) - .append_cellslice(vm::load_cell_slice(new_state)) - .as_cellslice_ref(); - } else { - message.init = vm::CellBuilder().store_zeroes(1).as_cellslice_ref(); - CHECK(message.init.not_null()); - } - } - /* body */ { - message.body = vm::CellBuilder().store_zeroes(1).append_cellslice(vm::load_cell_slice_ref(body)).as_cellslice_ref(); - } - - td::Ref res; - tlb::type_pack_cell(res, block::gen::t_Message_Any, message); - CHECK(res.not_null()); - - return res; -} -} // namespace tonlib diff --git a/tonlib/tonlib/KeyValue.cpp b/tonlib/tonlib/KeyValue.cpp index cdb0f5ee..b62ce7c0 100644 --- a/tonlib/tonlib/KeyValue.cpp +++ b/tonlib/tonlib/KeyValue.cpp @@ -22,6 +22,7 @@ #include "td/utils/port/path.h" #include "td/utils/PathView.h" +#include #include #include @@ -42,7 +43,11 @@ class KeyValueDir : public KeyValue { } td::Status add(td::Slice key, td::Slice value) override { - auto path = to_file_path(key.str()); + auto key_str = key.str(); + if (!is_valid_key(key_str)) { + return td::Status::Error("Invalid key"); + } + auto path = to_file_path(key_str); if (td::stat(path).is_ok()) { return td::Status::Error(PSLICE() << "File " << path << "already exists"); } @@ -50,15 +55,27 @@ class KeyValueDir : public KeyValue { } td::Status set(td::Slice key, td::Slice value) override { - return td::atomic_write_file(to_file_path(key.str()), value); + auto key_str = key.str(); + if (!is_valid_key(key_str)) { + return td::Status::Error("Invalid key"); + } + return td::atomic_write_file(to_file_path(key_str), value); } td::Result get(td::Slice key) override { - return td::read_file_secure(to_file_path(key.str())); + auto key_str = key.str(); + if (!is_valid_key(key_str)) { + return td::Status::Error("Invalid key"); + } + return td::read_file_secure(to_file_path(key_str)); } td::Status erase(td::Slice key) override { - return td::unlink(to_file_path(key.str())); + auto key_str = key.str(); + if (!is_valid_key(key_str)) { + return td::Status::Error("Invalid key"); + } + return td::unlink(to_file_path(key_str)); } void foreach_key(std::function f) override { @@ -83,6 +100,20 @@ class KeyValueDir : public KeyValue { std::string to_file_path(std::string key) { return directory_ + TD_DIR_SLASH + key; } + + bool is_valid_key(const std::string& key) { + if (key.empty()) { + return false; + } + + if (key.find(TD_DIR_SLASH) != std::string::npos || key.find("..") != std::string::npos) { + return false; + } + + return std::all_of(key.begin(), key.end(), [](char c) { + return std::isalnum(c) || c == '_' || c == '-' || c == '.'; + }); + } }; class KeyValueInmemory : public KeyValue { diff --git a/tonlib/tonlib/LastBlockStorage.cpp b/tonlib/tonlib/LastBlockStorage.cpp index 53c4456e..6e5b3f0f 100644 --- a/tonlib/tonlib/LastBlockStorage.cpp +++ b/tonlib/tonlib/LastBlockStorage.cpp @@ -32,13 +32,43 @@ void LastBlockStorage::set_key_value(std::shared_ptr kv) { } namespace { +std::string buffer_to_hex_nibbles_reversed(td::Slice buffer) { + const char *hex = "0123456789ABCDEF"; + std::string res(2 * buffer.size(), '\0'); + for (std::size_t i = 0; i < buffer.size(); i++) { + unsigned char c = buffer[i]; + res[2 * i + 1] = hex[c >> 4]; + res[2 * i] = hex[c & 15]; + } + return res; +} + +std::string get_file_name_depr(td::Slice name) { + return buffer_to_hex_nibbles_reversed(name) + ".blkstate"; +} + std::string get_file_name(td::Slice name) { return td::buffer_to_hex(name) + ".blkstate"; } } // namespace td::Result LastBlockStorage::get_state(td::Slice name) { - TRY_RESULT(data, kv_->get(get_file_name(name))); + // This migration addresses an issue in the old version of Tonlib, where the td::buffer_to_hex + // incorrectly reversed the order of nibbles in hex representation. + auto data_r = kv_->get(get_file_name(name)); + if (data_r.is_error()) { + auto key_depr = get_file_name_depr(name); + auto data_depr = kv_->get(key_depr); + if (data_depr.is_ok()) { + kv_->set(get_file_name(name), data_depr.move_as_ok()); + kv_->erase(key_depr); + data_r = std::move(data_depr); + } else { + return td::Status::Error("not found"); + } + } + + auto data = data_r.move_as_ok(); if (data.size() < 8) { return td::Status::Error("too short"); } diff --git a/tonlib/tonlib/LastConfig.cpp b/tonlib/tonlib/LastConfig.cpp index 960d5994..e972d84e 100644 --- a/tonlib/tonlib/LastConfig.cpp +++ b/tonlib/tonlib/LastConfig.cpp @@ -62,7 +62,8 @@ void LastConfig::with_last_block(td::Result r_last_block) { } auto last_block = r_last_block.move_as_ok(); - client_.send_query(ton::lite_api::liteServer_getConfigAll(0, create_tl_lite_block_id(last_block.last_block_id)), + client_.send_query(ton::lite_api::liteServer_getConfigAll(block::ConfigInfo::needPrevBlocks, + create_tl_lite_block_id(last_block.last_block_id)), [this](auto r_config) { this->on_config(std::move(r_config)); }); } @@ -92,7 +93,8 @@ td::Status LastConfig::process_config_proof(ton::ton_api::object_ptrstate_proof_.as_slice(), raw_config->config_proof_.as_slice())); - TRY_RESULT(config, block::Config::extract_from_state(std::move(state), 0)); + TRY_RESULT(config, block::ConfigInfo::extract_config( + std::move(state), block::ConfigInfo::needPrevBlocks | block::ConfigInfo::needCapabilities)); for (auto i : params_) { VLOG(last_config) << "ConfigParam(" << i << ") = "; @@ -109,6 +111,7 @@ td::Status LastConfig::process_config_proof(ton::ton_api::object_ptrget_prev_blocks_info()); state_.config.reset(config.release()); return td::Status::OK(); } diff --git a/tonlib/tonlib/LastConfig.h b/tonlib/tonlib/LastConfig.h index 514b4a59..901733dc 100644 --- a/tonlib/tonlib/LastConfig.h +++ b/tonlib/tonlib/LastConfig.h @@ -30,6 +30,7 @@ namespace tonlib { struct LastConfigState { std::shared_ptr config; + td::Ref prev_blocks_info; }; td::StringBuilder& operator<<(td::StringBuilder& sb, const LastConfigState& state); diff --git a/tonlib/tonlib/TestGiver.cpp b/tonlib/tonlib/TestGiver.cpp deleted file mode 100644 index b906193d..00000000 --- a/tonlib/tonlib/TestGiver.cpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2019 Telegram Systems LLP -*/ -#include "tonlib/TestGiver.h" -#include "tonlib/utils.h" - -#include "td/utils/base64.h" - -namespace tonlib { -const block::StdAddress& TestGiver::address() noexcept { - static block::StdAddress res = - block::StdAddress::parse("kf_8uRo6OBbQ97jCx2EIuKm8Wmt6Vb15-KsQHFLbKSMiYIny").move_as_ok(); - return res; -} - -vm::CellHash TestGiver::get_init_code_hash() noexcept { - return vm::CellHash::from_slice(td::base64_decode("wDkZp0yR4xo+9+BnuAPfGVjBzK6FPzqdv2DwRq3z3KE=").move_as_ok()); -} - -td::Ref TestGiver::make_a_gift_message(td::uint32 seqno, td::uint64 gramms, td::Slice message, - const block::StdAddress& dest_address) noexcept { - td::BigInt256 dest_addr; - dest_addr.import_bits(dest_address.addr.as_bitslice()); - vm::CellBuilder cb; - cb.append_cellslice(binary_bitstring_to_cellslice("b{01}").move_as_ok()) - .store_long(dest_address.bounceable, 1) - .append_cellslice(binary_bitstring_to_cellslice("b{000100}").move_as_ok()) - .store_long(dest_address.workchain, 8) - .store_int256(dest_addr, 256); - block::tlb::t_Grams.store_integer_value(cb, td::BigInt256(gramms)); - - cb.store_zeroes(9 + 64 + 32 + 1 + 1).store_bytes("\0\0\0\0", 4); - vm::CellString::store(cb, message, 35 * 8).ensure(); - auto message_inner = cb.finalize(); - return vm::CellBuilder().store_long(seqno, 32).store_long(1, 8).store_ref(message_inner).finalize(); -} -} // namespace tonlib diff --git a/tonlib/tonlib/TestWallet.cpp b/tonlib/tonlib/TestWallet.cpp deleted file mode 100644 index 8bdf78c9..00000000 --- a/tonlib/tonlib/TestWallet.cpp +++ /dev/null @@ -1,84 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2019 Telegram Systems LLP -*/ -#include "tonlib/TestWallet.h" -#include "tonlib/GenericAccount.h" -#include "tonlib/utils.h" - -#include "vm/boc.h" -#include "td/utils/base64.h" - -namespace tonlib { -td::Ref TestWallet::get_init_state(const td::Ed25519::PublicKey& public_key) noexcept { - auto code = get_init_code(); - auto data = get_init_data(public_key); - return GenericAccount::get_init_state(std::move(code), std::move(data)); -} - -td::Ref TestWallet::get_init_message(const td::Ed25519::PrivateKey& private_key) noexcept { - std::string seq_no(4, 0); - auto signature = - private_key.sign(vm::CellBuilder().store_bytes(seq_no).finalize()->get_hash().as_slice()).move_as_ok(); - return vm::CellBuilder().store_bytes(signature).store_bytes(seq_no).finalize(); -} - -td::Ref TestWallet::make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 seqno, - td::int64 gramms, td::Slice message, - const block::StdAddress& dest_address) noexcept { - td::BigInt256 dest_addr; - dest_addr.import_bits(dest_address.addr.as_bitslice()); - vm::CellBuilder cb; - cb.append_cellslice(binary_bitstring_to_cellslice("b{01}").move_as_ok()) - .store_long(dest_address.bounceable, 1) - .append_cellslice(binary_bitstring_to_cellslice("b{000100}").move_as_ok()) - .store_long(dest_address.workchain, 8) - .store_int256(dest_addr, 256); - td::int32 send_mode = 3; - if (gramms == -1) { - gramms = 0; - send_mode += 128; - } - block::tlb::t_Grams.store_integer_value(cb, td::BigInt256(gramms)); - cb.store_zeroes(9 + 64 + 32 + 1 + 1).store_bytes("\0\0\0\0", 4); - vm::CellString::store(cb, message, 35 * 8).ensure(); - auto message_inner = cb.finalize(); - auto message_outer = - vm::CellBuilder().store_long(seqno, 32).store_long(send_mode, 8).store_ref(message_inner).finalize(); - auto signature = private_key.sign(message_outer->get_hash().as_slice()).move_as_ok(); - return vm::CellBuilder().store_bytes(signature).append_cellslice(vm::load_cell_slice(message_outer)).finalize(); -} - -td::Ref TestWallet::get_init_code() noexcept { - static auto res = [] { - auto serialized_code = td::base64_decode( - "te6ccgEEAQEAAAAAUwAAov8AIN0gggFMl7qXMO1E0NcLH+Ck8mCBAgDXGCDXCx/tRNDTH9P/" - "0VESuvKhIvkBVBBE+RDyovgAAdMfMSDXSpbTB9QC+wDe0aTIyx/L/8ntVA==") - .move_as_ok(); - return vm::std_boc_deserialize(serialized_code).move_as_ok(); - }(); - return res; -} - -vm::CellHash TestWallet::get_init_code_hash() noexcept { - return get_init_code()->get_hash(); -} - -td::Ref TestWallet::get_init_data(const td::Ed25519::PublicKey& public_key) noexcept { - return vm::CellBuilder().store_long(0, 32).store_bytes(public_key.as_octet_string()).finalize(); -} -} // namespace tonlib diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index bdfea9f5..d73e715c 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -18,7 +18,6 @@ */ #include "TonlibClient.h" -#include "tonlib/ExtClientLazy.h" #include "tonlib/ExtClientOutbound.h" #include "tonlib/LastBlock.h" #include "tonlib/LastConfig.h" @@ -31,11 +30,14 @@ #include "smc-envelope/GenericAccount.h" #include "smc-envelope/ManualDns.h" #include "smc-envelope/WalletV3.h" +#include "smc-envelope/WalletV4.h" #include "smc-envelope/HighloadWallet.h" #include "smc-envelope/HighloadWalletV2.h" #include "smc-envelope/PaymentChannel.h" #include "smc-envelope/SmartContractCode.h" +#include "emulator/transaction-emulator.h" + #include "auto/tl/tonlib_api.hpp" #include "block/block-auto.h" #include "block/check-proof.h" @@ -59,6 +61,7 @@ #include "td/utils/port/path.h" #include "common/util.h" +#include "td/actor/MultiPromise.h" template using lite_api_ptr = ton::lite_api::object_ptr; @@ -74,6 +77,14 @@ struct GetAccountState { using ReturnType = td::unique_ptr; }; +struct GetAccountStateByTransaction { + block::StdAddress address; + std::int64_t lt; + td::Bits256 hash; + //td::optional public_key; + using ReturnType = td::unique_ptr; +}; + struct RemoteRunSmcMethod { block::StdAddress address; td::optional block_id; @@ -91,6 +102,11 @@ struct RemoteRunSmcMethodReturnType { // libs }; +struct ScanAndLoadGlobalLibs { + td::Ref root; + using ReturnType = vm::Dictionary; +}; + struct GetPrivateKey { KeyStorage::InputKey input_key; using ReturnType = KeyStorage::PrivateKey; @@ -161,6 +177,7 @@ static block::AccountState create_account_state(ton::tl_object_ptr extra_currencies; ton::UnixTime storage_last_paid{0}; vm::CellStorageStat storage_stat; @@ -189,6 +206,74 @@ std::string to_bytes(td::Ref cell) { return vm::std_boc_serialize(cell, vm::BagOfCells::Mode::WithCRC32C).move_as_ok().as_slice().str(); } +td::Result>> parse_extra_currencies_or_throw( + const td::Ref dict_root) { + std::vector> result; + vm::Dictionary dict{dict_root, 32}; + if (!dict.check_for_each([&](td::Ref value, td::ConstBitPtr key, int n) { + CHECK(n == 32); + int id = (int)key.get_int(n); + auto amount_ref = block::tlb::t_VarUIntegerPos_32.as_integer_skip(value.write()); + if (amount_ref.is_null() || !value->empty_ext()) { + return false; + } + td::int64 amount = amount_ref->to_long(); + if (amount == td::int64(~0ULL << 63)) { + return false; + } + result.push_back(tonlib_api::make_object(id, amount)); + return true; + })) { + return td::Status::Error("Failed to parse extra currencies dict"); + } + return result; +} + +td::Result>> parse_extra_currencies( + const td::Ref& dict_root) { + return TRY_VM(parse_extra_currencies_or_throw(dict_root)); +} + +td::Result> to_extra_currenctes_dict( + const std::vector>& extra_currencies) { + vm::Dictionary dict{32}; + for (const auto &f : extra_currencies) { + if (f->amount_ == 0) { + continue; + } + if (f->amount_ < 0) { + return td::Status::Error("Negative extra currency amount"); + } + vm::CellBuilder cb2; + block::tlb::t_VarUInteger_32.store_integer_value(cb2, *td::make_refint(f->amount_)); + if (!dict.set_builder(td::BitArray<32>(f->id_), cb2, vm::DictionaryBase::SetMode::Add)) { + return td::Status::Error("Duplicate extra currency id"); + } + } + return std::move(dict).extract_root_cell(); +} + +td::Status check_enough_extra_currencies(const td::Ref &balance, const td::Ref &amount) { + block::CurrencyCollection c1{td::zero_refint(), balance}; + block::CurrencyCollection c2{td::zero_refint(), amount}; + auto res = TRY_VM(td::Result{c1 >= c2}); + TRY_RESULT(v, std::move(res)); + if (!v) { + return TonlibError::NotEnoughFunds(); + } + return td::Status::OK(); +} + +td::Result> add_extra_currencies(const td::Ref &e1, const td::Ref &e2) { + block::CurrencyCollection c1{td::zero_refint(), e1}; + block::CurrencyCollection c2{td::zero_refint(), e2}; + TRY_RESULT_ASSIGN(c1, TRY_VM(td::Result{c1 + c2})); + if (c1.is_valid()) { + return td::Status::Error("Failed to add extra currencies"); + } + return c1.extra; +} + td::Result get_public_key(td::Slice public_key) { TRY_RESULT_PREFIX(address, block::PublicKey::parse(public_key), TonlibError::InvalidPublicKey()); return address; @@ -212,6 +297,14 @@ td::Result to_init_data(const tonlib_api::wallet_v3_ini return std::move(init_data); } +td::Result to_init_data(const tonlib_api::wallet_v4_initialAccountState& wallet_state) { + TRY_RESULT(key_bytes, get_public_key(wallet_state.public_key_)); + ton::WalletV4::InitData init_data; + init_data.public_key = td::SecureString(key_bytes.key); + init_data.wallet_id = static_cast(wallet_state.wallet_id_); + return std::move(init_data); +} + td::Result to_init_data(const tonlib_api::rwallet_initialAccountState& rwallet_state) { TRY_RESULT(init_key_bytes, get_public_key(rwallet_state.init_public_key_)); TRY_RESULT(key_bytes, get_public_key(rwallet_state.public_key_)); @@ -288,9 +381,10 @@ class AccountState { if (state.data.not_null()) { data = to_bytes(state.data); } + TRY_RESULT(extra_currencies, parse_extra_currencies(get_extra_currencies())); return tonlib_api::make_object( - get_balance(), std::move(code), std::move(data), to_transaction_id(raw().info), to_tonlib_api(raw().block_id), - raw().frozen_hash, get_sync_time()); + get_balance(), std::move(extra_currencies), std::move(code), std::move(data), to_transaction_id(raw().info), + to_tonlib_api(raw().block_id), raw().frozen_hash, get_sync_time()); } td::Result> to_wallet_v3_accountState() const { @@ -303,6 +397,16 @@ class AccountState { return tonlib_api::make_object(static_cast(wallet_id), static_cast(seqno)); } + td::Result> to_wallet_v4_accountState() const { + if (wallet_type_ != WalletV4) { + return TonlibError::AccountTypeUnexpected("WalletV4"); + } + auto wallet = ton::WalletV4(get_smc_state()); + TRY_RESULT(seqno, wallet.get_seqno()); + TRY_RESULT(wallet_id, wallet.get_wallet_id()); + return tonlib_api::make_object(static_cast(wallet_id), + static_cast(seqno)); + } td::Result> to_wallet_highload_v1_accountState() const { if (wallet_type_ != HighloadWalletV1) { @@ -404,16 +508,36 @@ class AccountState { return f(to_dns_accountState()); case PaymentChannel: return f(to_payment_channel_accountState()); + case WalletV4: + return f(to_wallet_v4_accountState()); } UNREACHABLE(); } td::Result> to_fullAccountState() const { TRY_RESULT(account_state, to_accountState()); + TRY_RESULT(extra_currencies, parse_extra_currencies(get_extra_currencies())); return tonlib_api::make_object( tonlib_api::make_object(get_address().rserialize(true)), get_balance(), - to_transaction_id(raw().info), to_tonlib_api(raw().block_id), get_sync_time(), std::move(account_state), - get_wallet_revision()); + std::move(extra_currencies), to_transaction_id(raw().info), to_tonlib_api(raw().block_id), get_sync_time(), + std::move(account_state), get_wallet_revision()); + } + + td::Result> to_shardAccountCell() const { + auto account_root = raw_.info.root; + if (account_root.is_null()) { + block::gen::Account().cell_pack_account_none(account_root); + } + auto cell = vm::CellBuilder().store_ref(account_root).store_bits(raw_.info.last_trans_hash.as_bitslice()).store_long(raw_.info.last_trans_lt).finalize(); + return tonlib_api::make_object(to_bytes(cell)); + } + + td::Result> to_shardAccountCellSlice() const { + auto account_root = raw_.info.root; + if (account_root.is_null()) { + block::gen::Account().cell_pack_account_none(account_root); + } + return vm::CellBuilder().store_ref(account_root).store_bits(raw_.info.last_trans_hash.as_bitslice()).store_long(raw_.info.last_trans_lt).as_cellslice_ref(); } //NB: Order is important! Used during guessAccountRevision @@ -425,7 +549,8 @@ class AccountState { HighloadWalletV2, ManualDns, PaymentChannel, - RestrictedWallet + RestrictedWallet, + WalletV4 }; WalletType get_wallet_type() const { return wallet_type_; @@ -444,6 +569,7 @@ class AccountState { case AccountState::HighloadWalletV1: case AccountState::HighloadWalletV2: case AccountState::RestrictedWallet: + case AccountState::WalletV4: return true; } UNREACHABLE(); @@ -464,6 +590,8 @@ class AccountState { return td::make_unique(get_smc_state()); case AccountState::RestrictedWallet: return td::make_unique(get_smc_state()); + case AccountState::WalletV4: + return td::make_unique(get_smc_state()); } UNREACHABLE(); return {}; @@ -492,6 +620,10 @@ class AccountState { return raw_.balance; } + td::Ref get_extra_currencies() const { + return raw_.extra_currencies; + } + const RawAccountState& raw() const { return raw_; } @@ -521,6 +653,23 @@ class AccountState { break; } }, + [&](tonlib_api::wallet_v4_initialAccountState& v4wallet) { + for (auto revision : ton::SmartContractCode::get_revisions(ton::SmartContractCode::WalletV4)) { + auto init_data = to_init_data(v4wallet); + if (init_data.is_error()) { + continue; + } + auto wallet = ton::WalletV4::create(init_data.move_as_ok(), revision); + if (!(wallet->get_address(ton::masterchainId) == address_ || + wallet->get_address(ton::basechainId) == address_)) { + continue; + } + wallet_type_ = WalletType::WalletV4; + wallet_revision_ = revision; + set_new_state(wallet->get_state()); + break; + } + }, [&](tonlib_api::rwallet_initialAccountState& rwallet) { for (auto revision : ton::SmartContractCode::get_revisions(ton::SmartContractCode::RestrictedWallet)) { auto r_init_data = to_init_data(rwallet); @@ -564,7 +713,7 @@ class AccountState { return wallet_type_; } auto wallet_id = static_cast(address_.workchain + wallet_id_); - ton::WalletV3::InitData init_data{key.as_octet_string(), wallet_id}; + ton::WalletInterface::DefaultInitData init_data{key.as_octet_string(), wallet_id}; auto o_revision = ton::WalletV3::guess_revision(address_, init_data); if (o_revision) { wallet_type_ = WalletType::WalletV3; @@ -572,6 +721,13 @@ class AccountState { set_new_state(ton::WalletV3::get_init_state(wallet_revision_, init_data)); return wallet_type_; } + o_revision = ton::WalletV4::guess_revision(address_, init_data); + if (o_revision) { + wallet_type_ = WalletType::WalletV4; + wallet_revision_ = o_revision.value(); + set_new_state(ton::WalletV4::get_init_state(wallet_revision_, init_data)); + return wallet_type_; + } o_revision = ton::HighloadWalletV2::guess_revision(address_, init_data); if (o_revision) { wallet_type_ = WalletType::HighloadWalletV2; @@ -649,6 +805,12 @@ class AccountState { wallet_revision_ = o_revision.value(); return wallet_type_; } + o_revision = ton::WalletV4::guess_revision(code_hash); + if (o_revision) { + wallet_type_ = WalletType::WalletV4; + wallet_revision_ = o_revision.value(); + return wallet_type_; + } o_revision = ton::HighloadWalletV2::guess_revision(code_hash); if (o_revision) { wallet_type_ = WalletType::HighloadWalletV2; @@ -820,8 +982,9 @@ class Query { return td::Status::Error("estimate_fee: action_set_code unsupported"); case block::gen::OutAction::action_send_msg: { block::gen::OutAction::Record_action_send_msg act_rec; - // mode: +128 = attach all remaining balance, +64 = attach all remaining balance of the inbound message, +1 = pay message fees, +2 = skip if message cannot be sent - if (!tlb::unpack_exact(cs, act_rec) || (act_rec.mode & ~0xe3) || (act_rec.mode & 0xc0) == 0xc0) { + // mode: +128 = attach all remaining balance, +64 = attach all remaining balance of the inbound message, + // +1 = pay message fees, +2 = skip if message cannot be sent, +16 = bounce if action fails + if (!tlb::unpack_exact(cs, act_rec) || (act_rec.mode & ~0xf3) || (act_rec.mode & 0xc0) == 0xc0) { return td::Status::Error("estimate_fee: can't parse send_msg"); } block::gen::MessageRelaxed::Record msg; @@ -862,8 +1025,10 @@ class Query { } return res; } - td::Result>> estimate_fees(bool ignore_chksig, std::shared_ptr& cfg, vm::Dictionary& libraries) { + td::Result>> estimate_fees(bool ignore_chksig, const LastConfigState& state, + vm::Dictionary& libraries) { // gas fees + const auto& cfg = state.config; bool is_masterchain = raw_.source->get_address().workchain == ton::masterchainId; TRY_RESULT(gas_limits_prices, cfg->get_gas_limits_prices(is_masterchain)); TRY_RESULT(storage_prices, cfg->get_storage_prices()); @@ -885,13 +1050,17 @@ class Query { } vm::GasLimits gas_limits = compute_gas_limits(td::make_refint(raw_.source->get_balance()), gas_limits_prices); - auto res = smc.write().send_external_message(raw_.message_body, ton::SmartContract::Args() - .set_limits(gas_limits) - .set_balance(raw_.source->get_balance()) - .set_now(raw_.source->get_sync_time()) - .set_ignore_chksig(ignore_chksig) - .set_address(raw_.source->get_address()) - .set_config(cfg).set_libraries(libraries)); + auto res = smc.write().send_external_message(raw_.message_body, + ton::SmartContract::Args() + .set_limits(gas_limits) + .set_balance(raw_.source->get_balance()) + .set_extra_currencies(raw_.source->get_extra_currencies()) + .set_now(raw_.source->get_sync_time()) + .set_ignore_chksig(ignore_chksig) + .set_address(raw_.source->get_address()) + .set_config(cfg) + .set_prev_blocks_info(state.prev_blocks_info) + .set_libraries(libraries)); td::int64 fwd_fee = 0; if (res.success) { LOG(DEBUG) << "output actions:\n" @@ -940,7 +1109,7 @@ class Query { } while (list.not_null()); return i; } -}; // namespace tonlib +}; td::Result to_balance_or_throw(td::Ref balance_ref) { vm::CellSlice balance_slice = *balance_ref; @@ -1123,7 +1292,7 @@ class RemoteRunSmcMethod : public td::actor::Actor { client_.send_query( //liteServer.runSmcMethod mode:# id:tonNode.blockIdExt account:liteServer.accountId method_id:long params:bytes = liteServer.RunMethodResult; ton::lite_api::liteServer_runSmcMethod( - 0x1f, ton::create_tl_lite_block_id(query_.block_id.value()), + 0x17, ton::create_tl_lite_block_id(query_.block_id.value()), ton::create_tl_object(query_.address.workchain, query_.address.addr), method_id, std::move(serialized_stack)), [self = this](auto r_state) { self->with_run_method_result(std::move(r_state)); }, @@ -1134,8 +1303,7 @@ class RemoteRunSmcMethod : public td::actor::Actor { td::Status do_with_last_block(td::Result r_last_block) { TRY_RESULT(last_block, std::move(r_last_block)); query_.block_id = std::move(last_block.last_block_id); - with_block_id(); - return td::Status::OK(); + return with_block_id(); } void start_up() override { @@ -1250,6 +1418,7 @@ class GetRawAccountState : public td::actor::Actor { } TRY_RESULT(balance, to_balance(storage.balance)); res.balance = balance; + res.extra_currencies = storage.balance->prefetch_ref(); auto state_tag = block::gen::t_AccountState.get_tag(*storage.state); if (state_tag < 0) { return td::Status::Error("Failed to parse AccountState tag"); @@ -1288,7 +1457,7 @@ class GetRawAccountState : public td::actor::Actor { ton::lite_api::liteServer_getAccountState( ton::create_tl_lite_block_id(block_id_.value()), ton::create_tl_object(address_.workchain, address_.addr)), - [self = this](auto r_state) { self->with_account_state(std::move(r_state)); }, block_id_.value().id.seqno); + [self = this](auto r_state) { self->with_account_state(std::move(r_state)); }); } td::Status do_with_last_block(td::Result r_last_block) { @@ -1633,6 +1802,339 @@ class GetShardBlockProof : public td::actor::Actor { std::vector> links_; }; +auto to_lite_api(const tonlib_api::ton_blockIdExt& blk) -> td::Result>; +auto to_tonlib_api(const ton::lite_api::liteServer_transactionId& txid) -> tonlib_api_ptr; + +td::Status check_block_transactions_proof(lite_api_ptr& bTxes, int32_t mode, + ton::LogicalTime start_lt, td::Bits256 start_addr, td::Bits256 root_hash, int req_count); + +class RunEmulator : public TonlibQueryActor { + public: + RunEmulator(ExtClientRef ext_client_ref, int_api::GetAccountStateByTransaction request, + td::actor::ActorShared parent, td::Promise>&& promise) + : TonlibQueryActor(std::move(parent)), request_(std::move(request)), promise_(std::move(promise)) { + client_.set_client(ext_client_ref); + } + + private: + struct FullBlockId { + ton::BlockIdExt id; + ton::BlockIdExt mc; + ton::BlockIdExt prev; + ton::Bits256 rand_seed; + }; + + ExtClient client_; + int_api::GetAccountStateByTransaction request_; + td::Promise> promise_; + + std::map> actors_; + td::int64 actor_id_{1}; + + FullBlockId block_id_; + td::Ref mc_state_root_; // ^ShardStateUnsplit + td::unique_ptr account_state_; + vm::Dictionary global_libraries_{256}; + std::vector> transactions_; // std::vector<^Transaction> + + size_t count_{0}; + size_t count_transactions_{0}; + bool incomplete_{true}; + bool stopped_{false}; + + void get_block_id(td::Promise&& promise) { + auto shard_id = ton::shard_prefix(request_.address.addr, 60); + auto query = ton::lite_api::liteServer_lookupBlock(0b111111010, ton::create_tl_lite_block_id_simple({request_.address.workchain, shard_id, 0}), request_.lt, 0); + client_.send_query(std::move(query), promise.wrap([shard_id](td::Result> header_r) -> td::Result { + + TRY_RESULT(header, std::move(header_r)); + ton::BlockIdExt block_id = ton::create_block_id(header->id_); + TRY_RESULT(root, vm::std_boc_deserialize(std::move(header->header_proof_))); + + try { + auto virt_root = vm::MerkleProof::virtualize(root, 1); + if (virt_root.is_null()) { + return td::Status::Error("block header proof is not a valid Merkle proof"); + } + + if (ton::RootHash{virt_root->get_hash().bits()} != block_id.root_hash) { + return td::Status::Error("block header has incorrect root hash"); + } + + std::vector prev_blocks; + ton::BlockIdExt mc_block_id; + bool after_split; + td::Status status = block::unpack_block_prev_blk_ext(virt_root, block_id, prev_blocks, mc_block_id, after_split); + if (status.is_error()) { + return status.move_as_error(); + } + + ton::BlockIdExt prev_block; + if (prev_blocks.size() == 1 || ton::shard_is_ancestor(prev_blocks[0].id.shard, shard_id)) { + prev_block = std::move(prev_blocks[0]); + } else { + prev_block = std::move(prev_blocks[1]); + } + + block::gen::Block::Record block; + block::gen::BlockExtra::Record extra; + if (!tlb::unpack_cell(virt_root, block) || !tlb::unpack_cell(block.extra, extra)) { + return td::Status::Error("cannot unpack block header"); + } + + return FullBlockId{std::move(block_id), std::move(mc_block_id), std::move(prev_block), std::move(extra.rand_seed)}; + } catch (vm::VmError& err) { + return err.as_status("error processing header"); + } catch (vm::VmVirtError& err) { + return err.as_status("error processing header"); + } + })); + } + + void get_mc_state_root(td::Promise>&& promise) { + TRY_RESULT_PROMISE(promise, lite_block, to_lite_api(*to_tonlib_api(block_id_.mc))); + auto block = ton::create_block_id(lite_block); + client_.send_query(ton::lite_api::liteServer_getConfigAll(0b11'11111111, std::move(lite_block)), promise.wrap([block](auto r_config) -> td::Result> { + + TRY_RESULT(state, block::check_extract_state_proof(block, r_config->state_proof_.as_slice(), r_config->config_proof_.as_slice())); + + return std::move(state); + })); + } + + void get_account_state(td::Promise>&& promise) { + auto actor_id = actor_id_++; + actors_[actor_id] = td::actor::create_actor( + "GetAccountState", client_.get_client(), request_.address, block_id_.prev, + actor_shared(this, actor_id), + promise.wrap([address = request_.address](auto&& state) { + return td::make_unique(std::move(address), std::move(state), 0); + })); + } + + td::Status get_transactions(std::int64_t lt) { + TRY_RESULT(lite_block, to_lite_api(*to_tonlib_api(block_id_.id))); + auto after = ton::lite_api::make_object(request_.address.addr, lt); + auto mode = 0b10100111; + constexpr int req_count = 256; + auto query = ton::lite_api::liteServer_listBlockTransactions(std::move(lite_block), mode, req_count, std::move(after), false, true); + + client_.send_query(std::move(query), [self = this, mode, lt, root_hash = block_id_.id.root_hash](lite_api_ptr&& bTxes) { + if (!bTxes) { + self->check(td::Status::Error("liteServer.blockTransactions is null")); + return; + } + + self->check(check_block_transactions_proof(bTxes, mode, lt, self->request_.address.addr, root_hash, req_count)); + + std::int64_t last_lt = 0; + for (auto& id : bTxes->ids_) { + last_lt = id->lt_; + if (id->account_ != self->request_.address.addr) { + continue; + } + + if (id->lt_ == self->request_.lt && id->hash_ == self->request_.hash) { + self->incomplete_ = false; + } + + self->transactions_.push_back({}); + self->get_transaction(id->lt_, id->hash_, [self, i = self->transactions_.size() - 1](auto transaction) { self->set_transaction(i, std::move(transaction)); }); + + if (!self->incomplete_) { + return; + } + } + + if (bTxes->incomplete_) { + self->check(self->get_transactions(last_lt)); + } else { + self->check(td::Status::Error("Transaction not found")); + } + }); + return td::Status::OK(); + } + + void get_transaction(std::int64_t lt, td::Bits256 hash, td::Promise>&& promise) { + auto actor_id = actor_id_++; + actors_[actor_id] = td::actor::create_actor( + "GetTransactionHistory", client_.get_client(), request_.address, lt, hash, 1, actor_shared(this, actor_id), + promise.wrap([](auto&& transactions) mutable { + return std::move(transactions.transactions.front().transaction); + })); + } + + void start_up() override { + if (stopped_) { + return; + } + get_block_id([SelfId = actor_id(this)](td::Result&& block_id) { + td::actor::send_closure(SelfId, &RunEmulator::set_block_id, std::move(block_id)); + }); + } + + void set_block_id(td::Result&& block_id) { + if (block_id.is_error()) { + check(block_id.move_as_error()); + } else { + block_id_ = block_id.move_as_ok(); + + get_mc_state_root([SelfId = actor_id(this)](td::Result>&& mc_state_root) { + td::actor::send_closure(SelfId, &RunEmulator::set_mc_state_root, std::move(mc_state_root)); + }); + get_account_state([SelfId = actor_id(this)](td::Result>&& state) { + td::actor::send_closure(SelfId, &RunEmulator::set_account_state, std::move(state)); + }); + check(get_transactions(0)); + + inc(); + } + } + + void set_mc_state_root(td::Result>&& mc_state_root) { + if (mc_state_root.is_error()) { + check(mc_state_root.move_as_error()); + } else { + mc_state_root_ = mc_state_root.move_as_ok(); + inc(); + } + } + + void set_account_state(td::Result>&& account_state) { + if (account_state.is_error()) { + check(account_state.move_as_error()); + } else { + account_state_ = account_state.move_as_ok(); + send_query(int_api::ScanAndLoadGlobalLibs{account_state_->get_raw_state()}, + [SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &RunEmulator::set_global_libraries, std::move(R)); + }); + } + } + + void set_global_libraries(td::Result R) { + if (R.is_error()) { + check(R.move_as_error()); + } else { + global_libraries_ = R.move_as_ok(); + inc(); + } + } + + void set_transaction(size_t i, td::Result>&& transaction) { + if (transaction.is_error()) { + check(transaction.move_as_error()); + } else { + transactions_[i] = transaction.move_as_ok(); + inc_transactions(); + } + } + + void inc_transactions() { + if (stopped_ || ++count_transactions_ != transactions_.size() || incomplete_) { + return; + } + inc(); + } + + void inc() { + if (stopped_ || ++count_ != 4) { // 4 -- block_id + mc_state_root + account_state + transactions + return; + } + + try { + auto r_config = block::ConfigInfo::extract_config(mc_state_root_, 0b11'11111111); + if (r_config.is_error()) { + check(r_config.move_as_error()); + return; + } + std::shared_ptr config = r_config.move_as_ok(); + + auto r_shard_account = account_state_->to_shardAccountCellSlice(); + if (r_shard_account.is_error()) { + check(r_shard_account.move_as_error()); + return; + } + td::Ref shard_account = r_shard_account.move_as_ok(); + + const block::StdAddress& address = account_state_->get_address(); + ton::UnixTime now = account_state_->get_sync_time(); + bool is_special = address.workchain == ton::masterchainId && config->is_special_smartcontract(address.addr); + block::Account account(address.workchain, address.addr.bits()); + if (!account.unpack(std::move(shard_account), now, is_special)) { + check(td::Status::Error("Can't unpack shard account")); + return; + } + + auto prev_blocks_info = config->get_prev_blocks_info(); + if (prev_blocks_info.is_error()) { + check(prev_blocks_info.move_as_error()); + return; + } + vm::Dictionary libraries = global_libraries_; + emulator::TransactionEmulator trans_emulator(config); + trans_emulator.set_prev_blocks_info(prev_blocks_info.move_as_ok()); + trans_emulator.set_libs(std::move(libraries)); + trans_emulator.set_rand_seed(block_id_.rand_seed); + td::Result emulation_result = + trans_emulator.emulate_transactions_chain(std::move(account), std::move(transactions_)); + + if (emulation_result.is_error()) { + promise_.set_error(emulation_result.move_as_error()); + } else { + account = std::move(emulation_result.move_as_ok().account); + RawAccountState raw = std::move(account_state_->raw()); + raw.block_id = block_id_.id; + block::CurrencyCollection balance = account.get_balance(); + raw.balance = balance.grams->to_long(); + raw.extra_currencies = balance.extra; + raw.storage_last_paid = std::move(account.last_paid); + raw.storage_stat = std::move(account.storage_stat); + raw.code = std::move(account.code); + raw.data = std::move(account.data); + raw.state = std::move(account.total_state); + raw.info.last_trans_lt = account.last_trans_lt_; + raw.info.last_trans_hash = account.last_trans_hash_; + raw.info.gen_utime = account.now_; + + if (account.status == block::Account::acc_frozen) { + raw.frozen_hash = (char*)account.state_hash.data(); + } + + promise_.set_value(td::make_unique(address, std::move(raw), 0)); + } + } catch (vm::VmVirtError& err) { + check(td::Status::Error(PSLICE() << "virtualization error while emulating transaction: " << err.get_msg())); + return; + } + stopped_ = true; + try_stop(); + } + + void check(td::Status status) { + if (status.is_error()) { + promise_.set_error(std::move(status)); + stopped_ = true; + try_stop(); + } + } + + void try_stop() { + if (stopped_ && actors_.empty()) { + stop(); + } + } + + void hangup_shared() override { + actors_.erase(get_link_token()); + try_stop(); + } + + void hangup() override { + check(TonlibError::Cancelled()); + } +}; + TonlibClient::TonlibClient(td::unique_ptr callback) : callback_(std::move(callback)) { } TonlibClient::~TonlibClient() = default; @@ -1683,21 +2185,8 @@ void TonlibClient::init_ext_client() { ext_client_outbound_ = client.get(); raw_client_ = std::move(client); } else { - std::vector> servers; - for (const auto& s : config_.lite_clients) { - servers.emplace_back(s.adnl_id, s.address); - } - class Callback : public ExtClientLazy::Callback { - public: - explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { - } - - private: - td::actor::ActorShared<> parent_; - }; ext_client_outbound_ = {}; - ref_cnt_++; - raw_client_ = ExtClientLazy::create(std::move(servers), td::make_unique(td::actor::actor_shared())); + raw_client_ = liteclient::ExtClient::create(config_.lite_servers, nullptr); } } @@ -1901,6 +2390,13 @@ td::Result get_account_address(const tonlib_api::wallet_v3_in ->get_address(workchain_id); } +td::Result get_account_address(const tonlib_api::wallet_v4_initialAccountState& test_wallet_state, + td::int32 revision, ton::WorkchainId workchain_id) { + TRY_RESULT(key_bytes, get_public_key(test_wallet_state.public_key_)); + return ton::WalletV4::create({key_bytes.key, static_cast(test_wallet_state.wallet_id_)}, revision) + ->get_address(workchain_id); +} + td::Result get_account_address( const tonlib_api::wallet_highload_v1_initialAccountState& test_wallet_state, td::int32 revision, ton::WorkchainId workchain_id) { @@ -1948,6 +2444,7 @@ static td::optional get_wallet_type(tonlib_api::In td::overloaded( [](const tonlib_api::raw_initialAccountState&) { return td::optional(); }, [](const tonlib_api::wallet_v3_initialAccountState&) { return ton::SmartContractCode::WalletV3; }, + [](const tonlib_api::wallet_v4_initialAccountState&) { return ton::SmartContractCode::WalletV4; }, [](const tonlib_api::wallet_highload_v1_initialAccountState&) { return ton::SmartContractCode::HighloadWalletV1; }, @@ -2037,6 +2534,12 @@ td::Status TonlibClient::do_request(tonlib_api::guessAccount& request, sources.push_back(Source{tonlib_api::make_object( request.public_key_, wallet_id_ + ton::basechainId), ton::basechainId}); + sources.push_back(Source{tonlib_api::make_object( + request.public_key_, wallet_id_ + ton::masterchainId), + ton::masterchainId}); + sources.push_back(Source{tonlib_api::make_object( + request.public_key_, wallet_id_ + ton::basechainId), + ton::basechainId}); for (Source& source : sources) { auto o_type = get_wallet_type(*source.init_state); if (!o_type) { @@ -2268,9 +2771,9 @@ const MasterConfig& get_default_master_config() { "file_hash": "XplPz01CXAps5qeSWUtxcyBfdAo5zVb1N979KLSKD24=" }, "init_block" : { - "root_hash": "irEt9whDfgaYwD+8AzBlYzrMZHhrkhSVp3PU1s4DOz4=", - "seqno": 10171687, - "file_hash": "lay/bUKUUFDJXU9S6gx9GACQFl+uK+zX8SqHWS9oLZc=", + "root_hash": "YRkrcmZMvLBvjanwKCyL3w4oceGPtFfgx8ym1QKCK/4=", + "seqno": 27747086, + "file_hash": "N42xzPnJjDlE3hxPXOb+pNzXomgRtpX5AZzMPnIA41s=", "workchain": -1, "shard": -9223372036854775808 }, @@ -2345,7 +2848,7 @@ td::Result TonlibClient::validate_config(tonlib_api::o TRY_RESULT_PREFIX(new_config, Config::parse(std::move(config->config_)), TonlibError::InvalidConfig("can't parse config")); - if (new_config.lite_clients.empty() && !config->use_callbacks_for_network_) { + if (new_config.lite_servers.empty() && !config->use_callbacks_for_network_) { return TonlibError::InvalidConfig("no lite clients"); } td::optional o_master_config; @@ -2531,6 +3034,7 @@ struct ToRawTransactions { } auto body_cell = vm::CellBuilder().append_cellslice(*body).finalize(); auto body_hash = body_cell->get_hash().as_slice().str(); + auto msg_hash = cell->get_hash().as_slice().str(); td::Ref init_state_cell; auto& init_state_cs = message.init.write(); @@ -2545,29 +3049,31 @@ struct ToRawTransactions { auto get_data = [body = std::move(body), body_cell = std::move(body_cell), init_state_cell = std::move(init_state_cell), this](td::Slice salt) mutable { tonlib_api::object_ptr data; - if (try_decode_messages_ && body->size() >= 32 && static_cast(body->prefetch_long(32)) <= 1) { - auto type = body.write().fetch_long(32); - td::Status status; + if (try_decode_messages_ && body->size() >= 32) { + auto type = static_cast(body.write().fetch_long(32)); + if (type == 0 || type == ton::WalletInterface::EncryptedCommentOp) { + td::Status status; - auto r_body_message = vm::CellString::load(body.write()); - LOG_IF(WARNING, r_body_message.is_error()) << "Failed to parse a message: " << r_body_message.error(); + auto r_body_message = TRY_VM(vm::CellString::load(body.write())); + LOG_IF(WARNING, r_body_message.is_error()) << "Failed to parse a message: " << r_body_message.error(); - if (r_body_message.is_ok()) { - if (type == 0) { - data = tonlib_api::make_object(r_body_message.move_as_ok()); - } else { - auto encrypted_message = r_body_message.move_as_ok(); - auto r_decrypted_message = [&]() -> td::Result { - if (!private_key_) { - return TonlibError::EmptyField("private_key"); - } - TRY_RESULT(decrypted, SimpleEncryptionV2::decrypt_data(encrypted_message, private_key_.value(), salt)); - return decrypted.data.as_slice().str(); - }(); - if (r_decrypted_message.is_ok()) { - data = tonlib_api::make_object(r_decrypted_message.move_as_ok()); + if (r_body_message.is_ok()) { + if (type == 0) { + data = tonlib_api::make_object(r_body_message.move_as_ok()); } else { - data = tonlib_api::make_object(encrypted_message); + auto encrypted_message = r_body_message.move_as_ok(); + auto r_decrypted_message = [&]() -> td::Result { + if (!private_key_) { + return TonlibError::EmptyField("private_key"); + } + TRY_RESULT(decrypted, SimpleEncryptionV2::decrypt_data(encrypted_message, private_key_.value(), salt)); + return decrypted.data.as_slice().str(); + }(); + if (r_decrypted_message.is_ok()) { + data = tonlib_api::make_object(r_decrypted_message.move_as_ok()); + } else { + data = tonlib_api::make_object(encrypted_message); + } } } } @@ -2590,6 +3096,7 @@ struct ToRawTransactions { } TRY_RESULT(balance, to_balance(msg_info.value)); + TRY_RESULT(extra_currencies, parse_extra_currencies(msg_info.value->prefetch_ref())); TRY_RESULT(src, to_std_address(msg_info.src)); TRY_RESULT(dest, to_std_address(msg_info.dest)); TRY_RESULT(fwd_fee, to_balance(msg_info.fwd_fee)); @@ -2597,9 +3104,11 @@ struct ToRawTransactions { auto created_lt = static_cast(msg_info.created_lt); return tonlib_api::make_object( + msg_hash, tonlib_api::make_object(src), - tonlib_api::make_object(std::move(dest)), balance, fwd_fee, ihr_fee, created_lt, - std::move(body_hash), get_data(src)); + tonlib_api::make_object(std::move(dest)), balance, + std::move(extra_currencies), fwd_fee, ihr_fee, created_lt, std::move(body_hash), + get_data(src)); } case block::gen::CommonMsgInfo::ext_in_msg_info: { block::gen::CommonMsgInfo::Record_ext_in_msg_info msg_info; @@ -2608,8 +3117,10 @@ struct ToRawTransactions { } TRY_RESULT(dest, to_std_address(msg_info.dest)); return tonlib_api::make_object( + msg_hash, tonlib_api::make_object(), - tonlib_api::make_object(std::move(dest)), 0, 0, 0, 0, std::move(body_hash), + tonlib_api::make_object(std::move(dest)), 0, + std::vector>{}, 0, 0, 0, std::move(body_hash), get_data("")); } case block::gen::CommonMsgInfo::ext_out_msg_info: { @@ -2620,8 +3131,11 @@ struct ToRawTransactions { TRY_RESULT(src, to_std_address(msg_info.src)); auto created_lt = static_cast(msg_info.created_lt); return tonlib_api::make_object( + msg_hash, tonlib_api::make_object(src), - tonlib_api::make_object(), 0, 0, 0, created_lt, std::move(body_hash), get_data(src)); + tonlib_api::make_object(), 0, + std::vector>{}, 0, 0, created_lt, std::move(body_hash), + get_data(src)); } } @@ -2640,6 +3154,7 @@ struct ToRawTransactions { std::vector> out_msgs; td::int64 fees = 0; td::int64 storage_fee = 0; + td::string address; if (info.transaction.not_null()) { data = to_bytes(info.transaction); block::gen::Transaction::Record trans; @@ -2648,11 +3163,6 @@ struct ToRawTransactions { } TRY_RESULT_ASSIGN(fees, to_balance(trans.total_fees)); - //LOG(ERROR) << fees; - - //std::ostringstream outp; - //block::gen::t_Transaction.print_ref(outp, info.transaction); - //LOG(INFO) << outp.str(); auto is_just = trans.r1.in_msg->prefetch_long(1); if (is_just == trans.r1.in_msg->fetch_long_eof) { @@ -2678,8 +3188,11 @@ struct ToRawTransactions { return td::Status::Error("Failed to fetch storage fee from transaction"); } storage_fee = storage_fees->to_long(); + auto std_address = block::StdAddress(info.blkid.id.workchain, trans.account_addr); + address = std_address.rserialize(true); } return tonlib_api::make_object( + tonlib_api::make_object(std::move(address)), info.now, data, tonlib_api::make_object(info.prev_trans_lt, info.prev_trans_hash.as_slice().str()), @@ -2706,6 +3219,74 @@ struct ToRawTransactions { return tonlib_api::make_object(std::move(transactions), std::move(transaction_id)); } + + td::Result> to_raw_transaction_or_throw( + block::BlockTransaction::Info&& info) { + std::string data; + + tonlib_api::object_ptr in_msg; + std::vector> out_msgs; + td::int64 fees = 0; + td::int64 storage_fee = 0; + td::string address; + if (info.transaction.not_null()) { + data = to_bytes(info.transaction); + block::gen::Transaction::Record trans; + if (!tlb::unpack_cell(info.transaction, trans)) { + return td::Status::Error("Failed to unpack Transaction"); + } + + TRY_RESULT_ASSIGN(fees, to_balance(trans.total_fees)); + + auto is_just = trans.r1.in_msg->prefetch_long(1); + if (is_just == trans.r1.in_msg->fetch_long_eof) { + return td::Status::Error("Failed to parse long"); + } + if (is_just == -1) { + auto msg = trans.r1.in_msg->prefetch_ref(); + TRY_RESULT(in_msg_copy, to_raw_message(trans.r1.in_msg->prefetch_ref())); + in_msg = std::move(in_msg_copy); + } + + if (trans.outmsg_cnt != 0) { + vm::Dictionary dict{trans.r1.out_msgs, 15}; + for (int x = 0; x < trans.outmsg_cnt; x++) { + TRY_RESULT(out_msg, to_raw_message(dict.lookup_ref(td::BitArray<15>{x}))); + fees += out_msg->fwd_fee_; + fees += out_msg->ihr_fee_; + out_msgs.push_back(std::move(out_msg)); + } + } + td::RefInt256 storage_fees; + if (!block::tlb::t_TransactionDescr.get_storage_fees(trans.description, storage_fees)) { + return td::Status::Error("Failed to fetch storage fee from transaction"); + } + storage_fee = storage_fees->to_long(); + auto std_address = block::StdAddress(info.blkid.id.workchain, trans.account_addr); + address = std_address.rserialize(true); + } + return tonlib_api::make_object( + tonlib_api::make_object(std::move(address)), + info.now, data, + tonlib_api::make_object(info.lt, + info.hash.as_slice().str()), + fees, storage_fee, fees - storage_fee, std::move(in_msg), std::move(out_msgs)); + } + + td::Result> to_raw_transaction(block::BlockTransaction::Info&& info) { + return TRY_VM(to_raw_transaction_or_throw(std::move(info))); + } + + td::Result>> to_raw_transactions( + block::BlockTransactionList::Info&& info) { + std::vector> transactions; + for (auto& transaction : info.transactions) { + TRY_RESULT(raw_transaction, to_raw_transaction(std::move(transaction))); + transactions.push_back(std::move(raw_transaction)); + } + + return std::move(transactions); + } }; // Raw @@ -2765,6 +3346,27 @@ td::Status TonlibClient::do_request(tonlib_api::raw_getAccountState& request, return td::Status::OK(); } +td::Status TonlibClient::do_request(tonlib_api::raw_getAccountStateByTransaction& request, + td::Promise>&& promise) { + if (!request.account_address_) { + return TonlibError::EmptyField("account_address"); + } + if (!request.transaction_id_) { + return TonlibError::EmptyField("transaction_id"); + } + TRY_RESULT(account_address, get_account_address(request.account_address_->account_address_)); + auto lt = request.transaction_id_->lt_; + auto hash_str = request.transaction_id_->hash_; + if (hash_str.size() != 32) { + return td::Status::Error(400, "Invalid transaction id hash size"); + } + td::Bits256 hash; + hash.as_slice().copy_from(hash_str); + make_request(int_api::GetAccountStateByTransaction{account_address, lt, hash}, + promise.wrap([](auto&& res) { return res->to_raw_fullAccountState(); })); + return td::Status::OK(); +} + td::Result from_tonlib(tonlib_api::inputKeyRegular& input_key) { if (!input_key.key_) { return TonlibError::EmptyField("key"); @@ -2877,6 +3479,59 @@ td::Status TonlibClient::do_request(const tonlib_api::getAccountState& request, return td::Status::OK(); } +td::Status TonlibClient::do_request(const tonlib_api::getAccountStateByTransaction& request, + td::Promise>&& promise) { + if (!request.account_address_) { + return TonlibError::EmptyField("account_address"); + } + if (!request.transaction_id_) { + return TonlibError::EmptyField("transaction_id"); + } + TRY_RESULT(account_address, get_account_address(request.account_address_->account_address_)); + auto lt = request.transaction_id_->lt_; + auto hash_str = request.transaction_id_->hash_; + if (hash_str.size() != 32) { + return td::Status::Error(400, "Invalid transaction id hash size"); + } + td::Bits256 hash; + hash.as_slice().copy_from(hash_str); + make_request(int_api::GetAccountStateByTransaction{account_address, lt, hash}, + promise.wrap([](auto&& res) { return res->to_fullAccountState(); })); + return td::Status::OK(); +} + +td::Status TonlibClient::do_request(const tonlib_api::getShardAccountCell& request, + td::Promise>&& promise) { + if (!request.account_address_) { + return TonlibError::EmptyField("account_address"); + } + TRY_RESULT(account_address, get_account_address(request.account_address_->account_address_)); + make_request(int_api::GetAccountState{std::move(account_address), query_context_.block_id.copy(), {}}, + promise.wrap([](auto&& res) { return res->to_shardAccountCell(); })); + return td::Status::OK(); +} + +td::Status TonlibClient::do_request(const tonlib_api::getShardAccountCellByTransaction& request, + td::Promise>&& promise) { + if (!request.account_address_) { + return TonlibError::EmptyField("account_address"); + } + if (!request.transaction_id_) { + return TonlibError::EmptyField("transaction_id"); + } + TRY_RESULT(account_address, get_account_address(request.account_address_->account_address_)); + auto lt = request.transaction_id_->lt_; + auto hash_str = request.transaction_id_->hash_; + if (hash_str.size() != 32) { + return td::Status::Error(400, "Invalid transaction id hash size"); + } + td::Bits256 hash; + hash.as_slice().copy_from(hash_str); + make_request(int_api::GetAccountStateByTransaction{account_address, lt, hash}, + promise.wrap([](auto&& res) { return res->to_shardAccountCell(); })); + return td::Status::OK(); +} + td::Result to_dns_entry_data(tonlib_api::dns_EntryData& entry_data) { using R = td::Result; return downcast_call2( @@ -2936,6 +3591,7 @@ class GenericCreateSendGrams : public TonlibQueryActor { struct Action { block::StdAddress destination; td::int64 amount; + td::Ref extra_currencies; td::int32 send_mode{-1}; bool is_encrypted{false}; @@ -2983,6 +3639,7 @@ class GenericCreateSendGrams : public TonlibQueryActor { return TonlibError::InvalidField("amount", "can't be negative"); } res.amount = message.amount_; + TRY_RESULT_ASSIGN(res.extra_currencies, to_extra_currenctes_dict(message.extra_currencies_)); if (!message.public_key_.empty()) { TRY_RESULT(public_key, get_public_key(message.public_key_)); auto key = td::Ed25519::PublicKey(td::SecureString(public_key.key)); @@ -3393,8 +4050,10 @@ class GenericCreateSendGrams : public TonlibQueryActor { } td::int64 amount = 0; + td::Ref extra_currencies; for (auto& action : actions_) { amount += action.amount; + TRY_RESULT_ASSIGN(extra_currencies, add_extra_currencies(extra_currencies, action.extra_currencies)); } if (amount > source_->get_balance()) { @@ -3406,6 +4065,8 @@ class GenericCreateSendGrams : public TonlibQueryActor { return TonlibError::NotEnoughFunds(); } + TRY_STATUS(check_enough_extra_currencies(source_->get_extra_currencies(), extra_currencies)); + if (source_->get_wallet_type() == AccountState::RestrictedWallet) { auto r_unlocked_balance = ton::RestrictedWallet::create(source_->get_smc_state()) ->get_balance(source_->get_balance(), source_->get_sync_time()); @@ -3423,12 +4084,13 @@ class GenericCreateSendGrams : public TonlibQueryActor { auto& destination = destinations_[i]; gift.destination = destinations_[i]->get_address(); gift.gramms = action.amount; + gift.extra_currencies = action.extra_currencies; gift.send_mode = action.send_mode; // Temporary turn off this dangerous transfer - if (false && action.amount == source_->get_balance()) { - gift.gramms = -1; - } + // if (action.amount == source_->get_balance()) { + // gift.gramms = -1; + // } if (action.body.not_null()) { gift.body = action.body; @@ -3456,9 +4118,9 @@ class GenericCreateSendGrams : public TonlibQueryActor { } } -// if (!o_public_key) { // todo: (tolya-yanot) temporary disable msg comment encryption (The exchanges/payment services needs to read the comment of incoming messages). This will be uncommented when a general standard is developed. - return TonlibError::MessageEncryption("Get public key (in destination)"); -// } + if (!o_public_key) { + return TonlibError::MessageEncryption("Cannot get public key of destination (possibly unknown wallet type)"); + } auto addr = source_->get_address(); addr.bounceable = true; @@ -3502,7 +4164,7 @@ class GenericCreateSendGrams : public TonlibQueryActor { return with_wallet(*source_->get_wallet()); } -}; // namespace tonlib +}; td::int64 TonlibClient::register_query(td::unique_ptr query) { auto query_id = ++next_query_id_; @@ -3635,7 +4297,7 @@ void TonlibClient::query_estimate_fees(td::int64 id, bool ignore_chksig, td::Res return; } TRY_RESULT_PROMISE(promise, state, std::move(r_state)); - TRY_RESULT_PROMISE_PREFIX(promise, fees, TRY_VM(it->second->estimate_fees(ignore_chksig, state.config, libraries)), + TRY_RESULT_PROMISE_PREFIX(promise, fees, TRY_VM(it->second->estimate_fees(ignore_chksig, state, libraries)), TonlibError::Internal()); promise.set_value(tonlib_api::make_object( fees.first.to_tonlib_api(), td::transform(fees.second, [](auto& x) { return x.to_tonlib_api(); }))); @@ -3674,8 +4336,7 @@ td::Status TonlibClient::do_request(const tonlib_api::query_send& request, td::Status TonlibClient::do_request(tonlib_api::query_forget& request, td::Promise>&& promise) { - auto it = queries_.find(request.id_); - if (it == queries_.end()) { + if (queries_.erase(request.id_) == 0) { return TonlibError::InvalidQueryId(); } promise.set_value(tonlib_api::make_object()); @@ -3713,6 +4374,27 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_load& request, return td::Status::OK(); } +td::Status TonlibClient::do_request(const tonlib_api::smc_loadByTransaction& request, + td::Promise>&& promise) { + if (!request.account_address_) { + return TonlibError::EmptyField("account_address"); + } + if (!request.transaction_id_) { + return TonlibError::EmptyField("transaction_id"); + } + TRY_RESULT(account_address, get_account_address(request.account_address_->account_address_)); + auto lt = request.transaction_id_->lt_; + auto hash_str = request.transaction_id_->hash_; + if (hash_str.size() != 32) { + return td::Status::Error(400, "Invalid transaction id hash size"); + } + td::Bits256 hash; + hash.as_slice().copy_from(hash_str); + make_request(int_api::GetAccountStateByTransaction{account_address, lt, hash}, + promise.send_closure(actor_id(this), &TonlibClient::finish_load_smc)); + return td::Status::OK(); +} + td::Status TonlibClient::do_request(const tonlib_api::smc_forget& request, td::Promise>&& promise) { auto it = smcs_.find(request.id_); @@ -3760,6 +4442,17 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_getState& request, return td::Status::OK(); } +td::Status TonlibClient::do_request(const tonlib_api::smc_getRawFullAccountState& request, + td::Promise>&& promise) { + auto it = smcs_.find(request.id_); + if (it == smcs_.end()) { + return TonlibError::InvalidSmcId(); + } + auto& acc = it->second; + promise.set_result(acc->to_raw_fullAccountState()); + return td::Status::OK(); +} + bool is_list(vm::StackEntry entry) { while (true) { if (entry.type() == vm::StackEntry::Type::t_null) { @@ -3774,13 +4467,16 @@ bool is_list(vm::StackEntry entry) { entry = entry.as_tuple()->at(1); } }; -auto to_tonlib_api(const vm::StackEntry& entry) -> tonlib_api::object_ptr { +auto to_tonlib_api(const vm::StackEntry& entry, int& limit) -> td::Result> { + if (limit <= 0) { + return td::Status::Error(PSLICE() << "TVM stack size exceeds limit"); + } switch (entry.type()) { case vm::StackEntry::Type::t_int: return tonlib_api::make_object( tonlib_api::make_object(dec_string(entry.as_int()))); case vm::StackEntry::Type::t_slice: - return tonlib_api::make_object(tonlib_api::make_object( + return tonlib_api::make_object(tonlib_api::make_object( to_bytes(vm::CellBuilder().append_cellslice(entry.as_slice()).finalize()))); case vm::StackEntry::Type::t_cell: return tonlib_api::make_object( @@ -3791,7 +4487,8 @@ auto to_tonlib_api(const vm::StackEntry& entry) -> tonlib_api::object_ptrat(0))); + TRY_RESULT(tl_entry, to_tonlib_api(node.as_tuple()->at(0), --limit)); + elements.push_back(std::move(tl_entry)); node = node.as_tuple()->at(1); } return tonlib_api::make_object( @@ -3799,7 +4496,8 @@ auto to_tonlib_api(const vm::StackEntry& entry) -> tonlib_api::object_ptr( tonlib_api::make_object(std::move(elements))); @@ -3811,6 +4509,16 @@ auto to_tonlib_api(const vm::StackEntry& entry) -> tonlib_api::object_ptr& stack) -> td::Result>> { + int stack_limit = 1000; + std::vector> tl_stack; + for (auto& entry: stack->as_span()) { + TRY_RESULT(tl_entry, to_tonlib_api(entry, --stack_limit)); + tl_stack.push_back(std::move(tl_entry)); + } + return tl_stack; +} + td::Result from_tonlib_api(tonlib_api::tvm_StackEntry& entry) { // TODO: error codes // downcast_call @@ -3854,8 +4562,8 @@ td::Result from_tonlib_api(tonlib_api::tvm_StackEntry& entry) { } void deep_library_search(std::set& set, std::set& visited, - vm::Dictionary& libs, td::Ref cell, int depth) { - if (depth <= 0 || set.size() >= 16 || visited.size() >= 256) { + vm::Dictionary& libs, td::Ref cell, int depth, size_t max_libs = 16) { + if (depth <= 0 || set.size() >= max_libs || visited.size() >= 256) { return; } auto ins = visited.insert(cell->get_hash()); @@ -3881,16 +4589,39 @@ void deep_library_search(std::set& set, std::set& v return; } for (unsigned int i=0; iget_refs_cnt(); i++) { - deep_library_search(set, visited, libs, loaded_cell.data_cell->get_ref(i), depth - 1); + deep_library_search(set, visited, libs, loaded_cell.data_cell->get_ref(i), depth - 1, max_libs); } } td::Status TonlibClient::do_request(const tonlib_api::smc_getLibraries& request, td::Promise>&& promise) { + if (request.library_list_.size() > 16) { + promise.set_error(TonlibError::InvalidField("library_list", ": too many libraries requested, 16 maximum")); + } + if (query_context_.block_id) { + get_libraries(query_context_.block_id.value(), request.library_list_, std::move(promise)); + } else { + client_.with_last_block([this, promise = std::move(promise), library_list = request.library_list_](td::Result r_last_block) mutable { + if (r_last_block.is_error()) { + promise.set_error(r_last_block.move_as_error_prefix(TonlibError::Internal("get last block failed "))); + } else { + this->get_libraries(r_last_block.move_as_ok().last_block_id, library_list, std::move(promise)); + } + }); + } + return td::Status::OK(); +} + +void TonlibClient::get_libraries(ton::BlockIdExt blkid, std::vector library_list, td::Promise>&& promise) { + sort(library_list.begin(), library_list.end()); + library_list.erase(unique(library_list.begin(), library_list.end()), library_list.end()); + std::vector> result_entries; - result_entries.reserve(request.library_list_.size()); + result_entries.reserve(library_list.size()); std::vector not_cached_hashes; - for (auto& library_hash : request.library_list_) { + not_cached_hashes.reserve(library_list.size()); + + for (auto& library_hash : library_list) { if (libraries.key_exists(library_hash)) { auto library_content = vm::std_boc_serialize(libraries.lookup_ref(library_hash)).move_as_ok().as_slice().str(); result_entries.push_back(tonlib_api::make_object(library_hash, library_content)); @@ -3901,39 +4632,146 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_getLibraries& request, if (not_cached_hashes.empty()) { promise.set_value(tonlib_api::make_object(std::move(result_entries))); - return td::Status::OK(); + return; } - client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(not_cached_hashes)), - promise.wrap([self=this, result_entries = std::move(result_entries)] - (td::Result> r_libraries) mutable - { - if (r_libraries.is_error()) { - LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); - } else { - auto libraries = r_libraries.move_as_ok(); - bool updated = false; - for (auto& lr : libraries->result_) { - auto contents = vm::std_boc_deserialize(lr->data_); - if (contents.is_ok() && contents.ok().not_null()) { - if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { - LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); - continue; - } - result_entries.push_back(tonlib_api::make_object(lr->hash_, lr->data_.as_slice().str())); - self->libraries.set_ref(lr->hash_, contents.move_as_ok()); - updated = true; - LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); - } else { - LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); - } - if (updated) { - self->store_libs_to_disk(); - } - } + auto missed_lib_ids = not_cached_hashes; + client_.send_query(ton::lite_api::liteServer_getLibrariesWithProof(ton::create_tl_lite_block_id(blkid), 1, std::move(missed_lib_ids)), + promise.wrap([self=this, blkid, result_entries = std::move(result_entries), not_cached_hashes] + (td::Result> r_libraries) mutable + -> td::Result> { + if (r_libraries.is_error()) { + LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); + return r_libraries.move_as_error(); + } + + auto libraries = r_libraries.move_as_ok(); + auto state = block::check_extract_state_proof(blkid, libraries->state_proof_.as_slice(), + libraries->data_proof_.as_slice()); + if (state.is_error()) { + LOG(WARNING) << "cannot check state proof: " << state.move_as_error().to_string(); + return state.move_as_error(); + } + auto state_root = state.move_as_ok(); + + try { + block::gen::ShardStateUnsplit::Record state_record; + if (!tlb::unpack_cell(state_root, state_record)) { + return td::Status::Error("cannot unpack shardchain state"); } + auto libraries_dict = vm::Dictionary(state_record.r1.libraries->prefetch_ref(), 256); + + for (auto& hash : not_cached_hashes) { + auto csr = libraries_dict.lookup(hash.bits(), 256); + if (csr.is_null()) { + LOG(WARNING) << "library " << hash.to_hex() << " not found in config"; + if (std::any_of(libraries->result_.begin(), libraries->result_.end(), + [&hash](const auto& lib) { return lib->hash_.bits().equals(hash.cbits(), 256); })) { + return TonlibError::Internal("library is included in response but it's not found in proof"); + } + continue; + } + block::gen::LibDescr::Record libdescr; + if (!tlb::csr_unpack(csr, libdescr)) { + return TonlibError::Internal("cannot unpack LibDescr record"); + } + + auto lib_it = std::find_if(libraries->result_.begin(), libraries->result_.end(), + [&hash](const auto& lib) { return lib->hash_.bits().equals(hash.cbits(), 256); }); + if (lib_it == libraries->result_.end()) { + return TonlibError::Internal("library is found in proof but not in response"); + } + auto& lib = *lib_it; + auto contents = vm::std_boc_deserialize(lib->data_); + if (!contents.is_ok() || contents.ok().is_null()) { + return TonlibError::Internal(PSLICE() << "cannot deserialize library cell " << lib->hash_.to_hex()); + } + + if (!contents.ok()->get_hash().bits().equals(hash.cbits(), 256)) { + return TonlibError::Internal(PSLICE() << "library hash mismatch data " << contents.ok()->get_hash().to_hex() << " != requested " << hash.to_hex()); + } + + if (contents.ok()->get_hash() != libdescr.lib->get_hash()) { + return TonlibError::Internal(PSLICE() << "library hash mismatch data " << lib->hash_.to_hex() << " != proof " << libdescr.lib->get_hash().to_hex()); + } + + result_entries.push_back(tonlib_api::make_object(lib->hash_, lib->data_.as_slice().str())); + self->libraries.set_ref(lib->hash_, contents.move_as_ok()); + LOG(DEBUG) << "registered library " << lib->hash_.to_hex(); + } + self->store_libs_to_disk(); return tonlib_api::make_object(std::move(result_entries)); - })); + } catch (vm::VmError& err) { + return TonlibError::Internal(PSLICE() << "error while checking getLibrariesWithProof proof: " << err.get_msg()); + } catch (vm::VmVirtError& err) { + return TonlibError::Internal(PSLICE() << "virtualization error while checking getLibrariesWithProof proof: " << err.get_msg()); + } + })); +} + +td::Status TonlibClient::do_request(const tonlib_api::smc_getLibrariesExt& request, + td::Promise>&& promise) { + std::set request_libs; + for (auto& x : request.list_) { + td::Status status = td::Status::OK(); + downcast_call(*x, td::overloaded([&](tonlib_api::smc_libraryQueryExt_one& one) { request_libs.insert(one.hash_); }, + [&](tonlib_api::smc_libraryQueryExt_scanBoc& scan) { + std::set visited; + vm::Dictionary empty{256}; + td::Result> r_cell = vm::std_boc_deserialize(scan.boc_); + if (r_cell.is_error()) { + status = r_cell.move_as_error(); + return; + } + size_t max_libs = scan.max_libs_ < 0 ? (1 << 30) : (size_t)scan.max_libs_; + std::set new_libs; + deep_library_search(new_libs, visited, empty, r_cell.move_as_ok(), 1024, + max_libs); + request_libs.insert(new_libs.begin(), new_libs.end()); + })); + TRY_STATUS(std::move(status)); + } + std::vector not_cached; + for (const td::Bits256& h : request_libs) { + if (libraries.lookup(h).is_null()) { + not_cached.push_back(h); + } + } + td::MultiPromise mp; + auto ig = mp.init_guard(); + LOG(DEBUG) << "Requesting " << not_cached.size() << " libraries"; + for (size_t i = 0; i < not_cached.size(); i += 16) { + size_t r = std::min(i + 16, not_cached.size()); + client_.send_query( + ton::lite_api::liteServer_getLibraries( + std::vector(not_cached.begin() + i, not_cached.begin() + r)), + [self = this, promise = ig.get_promise()]( + td::Result> r_libraries) mutable { + self->process_new_libraries(std::move(r_libraries)); + promise.set_result(td::Unit()); + }); + } + + ig.add_promise(promise.wrap([self = this, libs = std::move(request_libs)](td::Unit&&) { + vm::Dictionary dict{256}; + std::vector libs_ok, libs_not_found; + for (const auto& h : libs) { + auto lib = self->libraries.lookup_ref(h); + if (lib.is_null()) { + libs_not_found.push_back(h); + } else { + libs_ok.push_back(h); + dict.set_ref(h, lib); + } + } + td::BufferSlice dict_boc; + if (!dict.is_empty()) { + dict_boc = vm::std_boc_serialize(dict.get_root_cell()).move_as_ok(); + } + return ton::create_tl_object(dict_boc.as_slice().str(), std::move(libs_ok), + std::move(libs_not_found)); + })); + return td::Status::OK(); } @@ -3957,6 +4795,7 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, } args.set_stack(std::move(stack)); args.set_balance(it->second->get_balance()); + args.set_extra_currencies(it->second->get_extra_currencies()); args.set_now(it->second->get_sync_time()); args.set_address(it->second->get_address()); @@ -3964,6 +4803,7 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, ](td::Result r_state) mutable { TRY_RESULT_PROMISE(promise, state, std::move(r_state)); args.set_config(state.config); + args.set_prev_blocks_info(state.prev_blocks_info); auto code = smc->get_state().code; if (code.not_null()) { @@ -3973,37 +4813,14 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, std::vector libraryList{librarySet.begin(), librarySet.end()}; if (libraryList.size() > 0) { LOG(DEBUG) << "Requesting found libraries in code (" << libraryList.size() << ")"; - self->client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(libraryList)), - [self, smc = std::move(smc), args = std::move(args), promise = std::move(promise)] - (td::Result> r_libraries) mutable - { - if (r_libraries.is_error()) { - LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); - } else { - auto libraries = r_libraries.move_as_ok(); - bool updated = false; - for (auto& lr : libraries->result_) { - auto contents = vm::std_boc_deserialize(lr->data_); - if (contents.is_ok() && contents.ok().not_null()) { - if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { - LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); - continue; - } - self->libraries.set_ref(lr->hash_, contents.move_as_ok()); - updated = true; - LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); - } else { - LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); - } - } - if (updated) { - self->store_libs_to_disk(); - } - } - self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); - }); - } - else { + self->client_.send_query( + ton::lite_api::liteServer_getLibraries(std::move(libraryList)), + [self, smc = std::move(smc), args = std::move(args), promise = std::move(promise)]( + td::Result> r_libraries) mutable { + self->process_new_libraries(std::move(r_libraries)); + self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); + }); + } else { self->perform_smc_execution(std::move(smc), std::move(args), std::move(promise)); } } @@ -4014,6 +4831,33 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, return td::Status::OK(); } +void TonlibClient::process_new_libraries( + td::Result> r_libraries) { + if (r_libraries.is_error()) { + LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); + } else { + auto new_libraries = r_libraries.move_as_ok(); + bool updated = false; + for (auto& lr : new_libraries->result_) { + auto contents = vm::std_boc_deserialize(lr->data_); + if (contents.is_ok() && contents.ok().not_null()) { + if (contents.ok()->get_hash().bits().compare(lr->hash_.cbits(), 256)) { + LOG(WARNING) << "hash mismatch for library " << lr->hash_.to_hex(); + continue; + } + libraries.set_ref(lr->hash_, contents.move_as_ok()); + updated = true; + LOG(DEBUG) << "registered library " << lr->hash_.to_hex(); + } else { + LOG(WARNING) << "failed to deserialize library: " << lr->hash_.to_hex(); + } + } + if (updated) { + store_libs_to_disk(); + } + } +} + void TonlibClient::perform_smc_execution(td::Ref smc, ton::SmartContract::Args args, td::Promise>&& promise) { @@ -4022,17 +4866,19 @@ void TonlibClient::perform_smc_execution(td::Ref smc, ton::S auto res = smc->run_get_method(args); // smc.runResult gas_used:int53 stack:vector exit_code:int32 = smc.RunResult; - std::vector> res_stack; - for (auto& entry : res.stack->as_span()) { - res_stack.push_back(to_tonlib_api(entry)); + auto R = to_tonlib_api(res.stack); + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; } + auto res_stack = R.move_as_ok(); - if (res.missing_library.not_null()) { - td::Bits256 hash = res.missing_library; + if (res.missing_library) { + td::Bits256 hash = res.missing_library.value(); LOG(DEBUG) << "Requesting missing library: " << hash.to_hex(); - std::vector req = {std::move(hash)}; + std::vector req = {hash}; client_.send_query(ton::lite_api::liteServer_getLibraries(std::move(req)), - [self = this, res = std::move(res), res_stack = std::move(res_stack), hash = std::move(hash), + [self = this, res = std::move(res), res_stack = std::move(res_stack), hash, smc = std::move(smc), args = std::move(args), promise = std::move(promise)] (td::Result> r_libraries) mutable { @@ -4319,6 +5165,8 @@ td::Status TonlibClient::do_request(const tonlib_api::importKey& request, if (!request.exported_key_) { return TonlibError::EmptyField("exported_key"); } + // Note: the mnemonic is considered valid if a certain hash starts with zero byte (see Mnemonic::is_basic_seed()) + // Therefore, importKey with invalid password has 1/256 chance to return OK TRY_RESULT(key, key_storage_.import_key(std::move(request.local_password_), std::move(request.mnemonic_password_), KeyStorage::ExportedKey{std::move(request.exported_key_->word_list_)})); TRY_RESULT(key_bytes, public_key_from_bytes(key.public_key.as_slice())); @@ -4555,6 +5403,17 @@ td::Status TonlibClient::do_request(int_api::GetAccountState request, return td::Status::OK(); } +td::Status TonlibClient::do_request(int_api::GetAccountStateByTransaction request, + td::Promise>&& promise) { + auto actor_id = actor_id_++; + actors_[actor_id] = td::actor::create_actor( + "RunEmulator", client_.get_client(), request, actor_shared(this, actor_id), + promise.wrap([](auto&& state) { + return std::move(state); + })); + return td::Status::OK(); +} + td::Status TonlibClient::do_request(int_api::RemoteRunSmcMethod request, td::Promise&& promise) { auto actor_id = actor_id_++; @@ -4641,29 +5500,72 @@ td::Result to_block_id(const tonlib_api::ton_blockIdExt& blk) { return ton::BlockIdExt(blk.workchain_, blk.shard_, blk.seqno_, root_hash, file_hash); } -td::Status TonlibClient::do_request(const tonlib_api::getConfigParam& request, - td::Promise>&& promise) { - TRY_RESULT(lite_block, to_lite_api(*request.id_)) - auto block = create_block_id(std::move(lite_block)); - auto param = request.param_; +void TonlibClient::get_config_param(int32_t param, int32_t mode, ton::BlockIdExt block, td::Promise>&& promise) { std::vector params = { param }; - - client_.send_query(ton::lite_api::liteServer_getConfigParams(0, std::move(lite_block), std::move(params)), - promise.wrap([block, param](auto r_config) { + client_.send_query(ton::lite_api::liteServer_getConfigParams(mode, ton::create_tl_lite_block_id(block), std::move(params)), + promise.wrap([param, block](auto r_config) -> td::Result> { auto state = block::check_extract_state_proof(block, r_config->state_proof_.as_slice(), r_config->config_proof_.as_slice()); if (state.is_error()) { - LOG(ERROR) << "block::check_extract_state_proof failed: " << state.error(); + return state.move_as_error_prefix(TonlibError::ValidateConfig()); } auto config = block::Config::extract_from_state(std::move(state.move_as_ok()), 0); if (config.is_error()) { - LOG(ERROR) << "block::Config::extract_from_state failed: " << config.error(); + return config.move_as_error_prefix(TonlibError::ValidateConfig()); } tonlib_api::configInfo config_result; config_result.config_ = tonlib_api::make_object(to_bytes(config.move_as_ok()->get_config_param(param))); return tonlib_api::make_object(std::move(config_result)); })); +} +td::Status TonlibClient::do_request(const tonlib_api::getConfigParam& request, + td::Promise>&& promise) { + if (query_context_.block_id) { + get_config_param(request.param_, request.mode_, query_context_.block_id.value(), std::move(promise)); + } else { + client_.with_last_block([this, promise = std::move(promise), param = request.param_, mode = request.mode_](td::Result r_last_block) mutable { + if (r_last_block.is_error()) { + promise.set_error(r_last_block.move_as_error_prefix(TonlibError::Internal("get last block failed "))); + } else { + this->get_config_param(param, mode, r_last_block.move_as_ok().last_block_id, std::move(promise)); + } + }); + } + return td::Status::OK(); +} + +void TonlibClient::get_config_all(int32_t mode, ton::BlockIdExt block, td::Promise>&& promise) { + client_.send_query(ton::lite_api::liteServer_getConfigAll(mode, ton::create_tl_lite_block_id(block)), + promise.wrap([block](auto r_config) -> td::Result> { + auto state = block::check_extract_state_proof(block, r_config->state_proof_.as_slice(), + r_config->config_proof_.as_slice()); + if (state.is_error()) { + return state.move_as_error_prefix(TonlibError::ValidateConfig()); + } + auto config = block::Config::extract_from_state(std::move(state.move_as_ok()), 0); + if (config.is_error()) { + return config.move_as_error_prefix(TonlibError::ValidateConfig()); + } + tonlib_api::configInfo config_result; + config_result.config_ = tonlib_api::make_object(to_bytes(config.move_as_ok()->get_root_cell())); + return tonlib_api::make_object(std::move(config_result)); + })); +} + +td::Status TonlibClient::do_request(const tonlib_api::getConfigAll& request, + td::Promise>&& promise) { + if (query_context_.block_id) { + get_config_all(request.mode_, query_context_.block_id.value(), std::move(promise)); + } else { + client_.with_last_block([this, promise = std::move(promise), mode = request.mode_](td::Result r_last_block) mutable { + if (r_last_block.is_error()) { + promise.set_error(r_last_block.move_as_error_prefix(TonlibError::Internal("get last block failed "))); + } else { + this->get_config_all(mode, r_last_block.move_as_ok().last_block_id, std::move(promise)); + } + }); + } return td::Status::OK(); } @@ -4681,53 +5583,257 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getMasterchainInfo& td::Status TonlibClient::do_request(const tonlib_api::blocks_getShards& request, td::Promise>&& promise) { TRY_RESULT(block, to_lite_api(*request.id_)) + TRY_RESULT(req_blk_id, to_block_id(*request.id_)); client_.send_query(ton::lite_api::liteServer_getAllShardsInfo(std::move(block)), - promise.wrap([](lite_api_ptr&& all_shards_info) + promise.wrap([req_blk_id](lite_api_ptr&& all_shards_info) -> td::Result> { - td::BufferSlice proof = std::move((*all_shards_info).proof_); - td::BufferSlice data = std::move((*all_shards_info).data_); - if (data.empty()) { - return td::Status::Error("shard configuration is empty"); - } else { - auto R = vm::std_boc_deserialize(data.clone()); - if (R.is_error()) { - return R.move_as_error_prefix("cannot deserialize shard configuration: "); - } - auto root = R.move_as_ok(); - block::ShardConfig sh_conf; - if (!sh_conf.unpack(vm::load_cell_slice_ref(root))) { - return td::Status::Error("cannot extract shard block list from shard configuration"); - } else { - auto ids = sh_conf.get_shard_hash_ids(true); - tonlib_api::blocks_shards shards; - for (auto id : ids) { - auto ref = sh_conf.get_shard_hash(ton::ShardIdFull(id)); - if (ref.not_null()) { - shards.shards_.push_back(to_tonlib_api(ref->top_block_id())); - } - } - return tonlib_api::make_object(std::move(shards)); - } - } + auto blk_id = ton::create_block_id(all_shards_info->id_); + if (blk_id != req_blk_id) { + return td::Status::Error("Liteserver responded with wrong block"); + } + td::BufferSlice proof = std::move((*all_shards_info).proof_); + td::BufferSlice data = std::move((*all_shards_info).data_); + if (data.empty() || proof.empty()) { + return td::Status::Error("Shard configuration or proof is empty"); + } + auto proof_cell = vm::std_boc_deserialize(std::move(proof)); + if (proof_cell.is_error()) { + return proof_cell.move_as_error_prefix("Couldn't deserialize shards proof: "); + } + auto data_cell = vm::std_boc_deserialize(std::move(data)); + if (data_cell.is_error()) { + return data_cell.move_as_error_prefix("Couldn't deserialize shards data: "); + } + try { + auto virt_root = vm::MerkleProof::virtualize(proof_cell.move_as_ok(), 1); + if (virt_root.is_null()) { + return td::Status::Error("Virt root is null"); + } + if (ton::RootHash{virt_root->get_hash().bits()} != blk_id.root_hash) { + return td::Status::Error("Block shards merkle proof has incorrect root hash"); + } + + block::gen::Block::Record blk; + block::gen::BlockExtra::Record extra; + block::gen::McBlockExtra::Record mc_extra; + if (!tlb::unpack_cell(virt_root, blk) || !tlb::unpack_cell(blk.extra, extra) || !extra.custom->have_refs() || + !tlb::unpack_cell(extra.custom->prefetch_ref(), mc_extra)) { + return td::Status::Error("cannot unpack block extra of block " + blk_id.to_str()); + } + auto data_csr = vm::load_cell_slice_ref(data_cell.move_as_ok()); + if (data_csr->prefetch_ref()->get_hash() != mc_extra.shard_hashes->prefetch_ref()->get_hash()) { + return td::Status::Error("Block shards data and proof hashes don't match"); + } + + block::ShardConfig sh_conf; + if (!sh_conf.unpack(data_csr)) { + return td::Status::Error("cannot extract shard block list from shard configuration"); + } + auto ids = sh_conf.get_shard_hash_ids(true); + tonlib_api::blocks_shards shards; + for (auto& id : ids) { + auto ref = sh_conf.get_shard_hash(ton::ShardIdFull(id)); + if (ref.not_null()) { + shards.shards_.push_back(to_tonlib_api(ref->top_block_id())); + } + } + return tonlib_api::make_object(std::move(shards)); + } catch (vm::VmError& err) { + return err.as_status("Couldn't verify proof: "); + } catch (vm::VmVirtError& err) { + return err.as_status("Couldn't verify proof: "); + } catch (...) { + return td::Status::Error("Unknown exception raised while verifying proof"); + } })); return td::Status::OK(); } +td::Status check_lookup_block_proof(lite_api_ptr& result, int mode, + ton::BlockId blkid, ton::BlockIdExt client_mc_blkid, td::uint64 lt, + td::uint32 utime); td::Status TonlibClient::do_request(const tonlib_api::blocks_lookupBlock& request, td::Promise>&& promise) { - client_.send_query(ton::lite_api::liteServer_lookupBlock( - request.mode_, - ton::lite_api::make_object((*request.id_).workchain_, (*request.id_).shard_, (*request.id_).seqno_), - (td::uint64)(request.lt_), - (td::uint32)(request.utime_)), - promise.wrap([](lite_api_ptr&& header) { - const auto& id = header->id_; - return to_tonlib_api(*id); - //tonlib_api::make_object( - // ton::tonlib_api::ton_blockIdExt(id->workchain_, id->) - //); - })); + auto lite_block = ton::lite_api::make_object((*request.id_).workchain_, (*request.id_).shard_, (*request.id_).seqno_); + auto blkid = ton::BlockId(request.id_->workchain_, request.id_->shard_, request.id_->seqno_); + client_.with_last_block( + [self = this, blkid, lite_block = std::move(lite_block), mode = request.mode_, lt = (td::uint64)request.lt_, + utime = (td::uint32)request.utime_, promise = std::move(promise)](td::Result r_last_block) mutable { + if (r_last_block.is_error()) { + promise.set_error(r_last_block.move_as_error_prefix(TonlibError::Internal("get last block failed "))); + return; + } + + self->client_.send_query(ton::lite_api::liteServer_lookupBlockWithProof(mode, std::move(lite_block), ton::create_tl_lite_block_id(r_last_block.ok().last_block_id), lt, utime), + promise.wrap([blkid, mode, utime, lt, last_block = r_last_block.ok().last_block_id](lite_api_ptr&& result) + -> td::Result> { + TRY_STATUS(check_lookup_block_proof(result, mode, blkid, last_block, lt, utime)); + return to_tonlib_api(*result->id_); + }) + ); + }); + return td::Status::OK(); +} + +td::Status check_lookup_block_proof(lite_api_ptr& result, int mode, ton::BlockId blkid, ton::BlockIdExt client_mc_blkid, td::uint64 lt, td::uint32 utime) { + try { + ton::BlockIdExt cur_id = ton::create_block_id(result->mc_block_id_); + if (!cur_id.is_masterchain_ext()) { + return td::Status::Error("invalid response: mc block id is not from masterchain"); + } + if (client_mc_blkid != cur_id) { + auto state = block::check_extract_state_proof(client_mc_blkid, result->client_mc_state_proof_.as_slice(), + result->mc_block_proof_.as_slice()); + if (state.is_error()) { + LOG(WARNING) << "cannot check state proof: " << state.move_as_error().to_string(); + return state.move_as_error(); + } + auto state_root = state.move_as_ok(); + auto prev_blocks_dict = block::get_prev_blocks_dict(state_root); + if (!prev_blocks_dict) { + return td::Status::Error("cannot extract prev blocks dict from state"); + } + + if (!block::check_old_mc_block_id(*prev_blocks_dict, cur_id)) { + return td::Status::Error("couldn't check old mc block id"); + } + } + try { + for (auto& link : result->shard_links_) { + ton::BlockIdExt prev_id = create_block_id(link->id_); + td::BufferSlice proof = std::move(link->proof_); + auto R = vm::std_boc_deserialize(proof); + if (R.is_error()) { + return TonlibError::InvalidBagOfCells("proof"); + } + auto block_root = vm::MerkleProof::virtualize(R.move_as_ok(), 1); + if (cur_id.root_hash != block_root->get_hash().bits()) { + return td::Status::Error("invalid block hash in proof"); + } + if (cur_id.is_masterchain()) { + block::gen::Block::Record blk; + block::gen::BlockExtra::Record extra; + block::gen::McBlockExtra::Record mc_extra; + if (!tlb::unpack_cell(block_root, blk) || !tlb::unpack_cell(blk.extra, extra) || !extra.custom->have_refs() || + !tlb::unpack_cell(extra.custom->prefetch_ref(), mc_extra)) { + return td::Status::Error("cannot unpack block header"); + } + block::ShardConfig shards(mc_extra.shard_hashes->prefetch_ref()); + td::Ref shard_hash = shards.get_shard_hash(prev_id.shard_full(), true); + if (shard_hash.is_null() || shard_hash->top_block_id() != prev_id) { + return td::Status::Error("invalid proof chain: prev block is not in mc shard list"); + } + } else { + std::vector prev; + ton::BlockIdExt mc_blkid; + bool after_split; + td::Status S = block::unpack_block_prev_blk_try(block_root, cur_id, prev, mc_blkid, after_split); + if (S.is_error()) { + return S; + } + CHECK(prev.size() == 1 || prev.size() == 2); + bool found = prev_id == prev[0] || (prev.size() == 2 && prev_id == prev[1]); + if (!found) { + return td::Status::Error("invalid proof chain: prev block is not in prev blocks list"); + } + } + cur_id = prev_id; + } + } catch (vm::VmVirtError& err) { + return err.as_status(); + } + if (cur_id.id.workchain != blkid.workchain || !ton::shard_contains(cur_id.id.shard, blkid.shard)) { + return td::Status::Error("response block has incorrect workchain/shard"); + } + + auto header_r = vm::std_boc_deserialize(std::move(result->header_)); + if (header_r.is_error()) { + return TonlibError::InvalidBagOfCells("header"); + } + auto header_root = vm::MerkleProof::virtualize(header_r.move_as_ok(), 1); + if (header_root.is_null()) { + return td::Status::Error("header_root is null"); + } + if (cur_id.root_hash != header_root->get_hash().bits()) { + return td::Status::Error("invalid header hash in proof"); + } + + std::vector prev; + ton::BlockIdExt mc_blkid; + bool after_split; + auto R = block::unpack_block_prev_blk_try(header_root, cur_id, prev, mc_blkid, after_split); + if (R.is_error()) { + return R; + } + if (cur_id != ton::create_block_id(result->id_)) { + return td::Status::Error("response blkid doesn't match header"); + } + + block::gen::Block::Record blk; + block::gen::BlockInfo::Record info; + if (!(tlb::unpack_cell(header_root, blk) && tlb::unpack_cell(blk.info, info))) { + return td::Status::Error("block header unpack failed"); + } + + if (mode & 1) { + if (cur_id.seqno() != blkid.seqno) { + return td::Status::Error("invalid seqno in proof"); + } + } else if (mode & 6) { + auto prev_header_r = vm::std_boc_deserialize(std::move(result->prev_header_)); + if (prev_header_r.is_error()) { + return TonlibError::InvalidBagOfCells("prev_headers"); + } + auto prev_header = prev_header_r.move_as_ok(); + auto prev_root = vm::MerkleProof::virtualize(prev_header, 1); + if (prev_root.is_null()) { + return td::Status::Error("prev_root is null"); + } + + bool prev_valid = false; + int prev_idx = -1; + for (size_t i = 0; i < prev.size(); i++) { + if (prev[i].root_hash == prev_root->get_hash().bits()) { + prev_valid = true; + prev_idx = i; + } + } + if (!prev_valid) { + return td::Status::Error("invalid prev header hash in proof"); + } + if (!ton::shard_contains(prev[prev_idx].id.shard, blkid.shard)) { + return td::Status::Error("invalid prev header shard in proof"); + } + + block::gen::Block::Record prev_blk; + block::gen::BlockInfo::Record prev_info; + if (!(tlb::unpack_cell(prev_root, prev_blk) && tlb::unpack_cell(prev_blk.info, prev_info))) { + return td::Status::Error("prev header unpack failed"); + } + + if (mode & 2) { + if (prev_info.end_lt > lt) { + return td::Status::Error("prev header end_lt > lt"); + } + if (info.end_lt < lt) { + return td::Status::Error("header end_lt < lt"); + } + } else if (mode & 4) { + if (prev_info.gen_utime > utime) { + return td::Status::Error("prev header end_lt > lt"); + } + if (info.gen_utime < utime) { + return td::Status::Error("header end_lt < lt"); + } + } + } + } catch (vm::VmError& err) { + return td::Status::Error(PSLICE() << "error while checking lookupBlock proof: " << err.get_msg()); + } catch (vm::VmVirtError& err) { + return td::Status::Error(PSLICE() << "virtualization error while checking lookupBlock proof: " << err.get_msg()); + } + return td::Status::OK(); } @@ -4737,28 +5843,128 @@ auto to_tonlib_api(const ton::lite_api::liteServer_transactionId& txid) txid.mode_, txid.account_.as_slice().str(), txid.lt_, txid.hash_.as_slice().str()); } +td::Status check_block_transactions_proof(lite_api_ptr& bTxes, int32_t mode, + ton::LogicalTime start_lt, td::Bits256 start_addr, td::Bits256 root_hash, int req_count) { + if ((mode & ton::lite_api::liteServer_listBlockTransactions::WANT_PROOF_MASK) == 0) { + return td::Status::OK(); + } + constexpr int max_answer_transactions = 256; + bool reverse_mode = mode & ton::lite_api::liteServer_listBlockTransactions::REVERSE_ORDER_MASK; + + try { + TRY_RESULT(proof_cell, vm::std_boc_deserialize(std::move(bTxes->proof_))); + auto virt_root = vm::MerkleProof::virtualize(proof_cell, 1); + + if (root_hash != virt_root->get_hash().bits()) { + return td::Status::Error("Invalid block proof root hash"); + } + block::gen::Block::Record blk; + block::gen::BlockExtra::Record extra; + if (!(tlb::unpack_cell(virt_root, blk) && tlb::unpack_cell(std::move(blk.extra), extra))) { + return td::Status::Error("Error unpacking proof cell"); + } + vm::AugmentedDictionary acc_dict{vm::load_cell_slice_ref(extra.account_blocks), 256, + block::tlb::aug_ShardAccountBlocks}; + + bool eof = false; + ton::LogicalTime reverse = reverse_mode ? ~0ULL : 0; + ton::LogicalTime trans_lt = static_cast(start_lt); + td::Bits256 cur_addr = start_addr; + bool allow_same = true; + int count = 0; + while (!eof && count < req_count && count < max_answer_transactions) { + auto value = acc_dict.extract_value( + acc_dict.vm::DictionaryFixed::lookup_nearest_key(cur_addr.bits(), 256, !reverse, allow_same)); + if (value.is_null()) { + eof = true; + break; + } + allow_same = false; + if (cur_addr != start_addr) { + trans_lt = reverse; + } + + block::gen::AccountBlock::Record acc_blk; + if (!tlb::csr_unpack(std::move(value), acc_blk) || acc_blk.account_addr != cur_addr) { + return td::Status::Error("Error unpacking proof account block"); + } + vm::AugmentedDictionary trans_dict{vm::DictNonEmpty(), std::move(acc_blk.transactions), 64, + block::tlb::aug_AccountTransactions}; + td::BitArray<64> cur_trans{(long long)trans_lt}; + while (count < req_count && count < max_answer_transactions) { + auto tvalue = trans_dict.extract_value_ref( + trans_dict.vm::DictionaryFixed::lookup_nearest_key(cur_trans.bits(), 64, !reverse)); + if (tvalue.is_null()) { + trans_lt = reverse; + break; + } + if (static_cast(count) < bTxes->ids_.size()) { + if (mode & 4 && !tvalue->get_hash().bits().equals(bTxes->ids_[count]->hash_.bits(), 256)) { + return td::Status::Error("Couldn't verify proof (hash)"); + } + if (mode & 2 && cur_trans != td::BitArray<64>(bTxes->ids_[count]->lt_)) { + return td::Status::Error("Couldn't verify proof (lt)"); + } + if (mode & 1 && cur_addr != bTxes->ids_[count]->account_) { + return td::Status::Error("Couldn't verify proof (account)"); + } + } + count++; + } + } + if (static_cast(count) != bTxes->ids_.size()) { + return td::Status::Error(PSLICE() << "Txs count mismatch in proof (" << count << ") and response (" << bTxes->ids_.size() << ")"); + } + } catch (vm::VmError& err) { + return err.as_status("Couldn't verify proof: "); + } catch (vm::VmVirtError& err) { + return err.as_status("Couldn't verify proof: "); + } catch (...) { + return td::Status::Error("Unknown exception raised while verifying proof"); + } + return td::Status::OK(); +} + td::Status TonlibClient::do_request(const tonlib_api::blocks_getTransactions& request, td::Promise>&& promise) { TRY_RESULT(block, to_lite_api(*request.id_)) - TRY_RESULT(account, to_bits256((*request.after_).account_, "account")); - auto after = ton::lite_api::make_object(account, (*request.after_).lt_); + auto root_hash = block->root_hash_; + bool check_proof = request.mode_ & ton::lite_api::liteServer_listBlockTransactions::WANT_PROOF_MASK; + bool reverse_mode = request.mode_ & ton::lite_api::liteServer_listBlockTransactions::REVERSE_ORDER_MASK; + bool has_starting_tx = request.mode_ & ton::lite_api::liteServer_listBlockTransactions::AFTER_MASK; + + td::Bits256 start_addr; + ton::LogicalTime start_lt; + ton::lite_api::object_ptr after; + if (has_starting_tx) { + if (!request.after_) { + return td::Status::Error("Missing field `after`"); + } + TRY_RESULT_ASSIGN(start_addr, to_bits256(request.after_->account_, "account")); + start_lt = request.after_->lt_; + after = ton::lite_api::make_object(start_addr, start_lt); + } else { + start_addr = reverse_mode ? td::Bits256::ones() : td::Bits256::zero(); + start_lt = reverse_mode ? ~0ULL : 0; + after = nullptr; + } + client_.send_query(ton::lite_api::liteServer_listBlockTransactions( std::move(block), request.mode_, request.count_, std::move(after), - false, - false), - promise.wrap([](lite_api_ptr&& bTxes) { - const auto& id = bTxes->id_; - //for (auto id : ids) { + reverse_mode, + check_proof), + promise.wrap([root_hash, req_count = request.count_, start_addr, start_lt, mode = request.mode_] + (lite_api_ptr&& bTxes) -> td::Result> { + TRY_STATUS(check_block_transactions_proof(bTxes, mode, start_lt, start_addr, root_hash, req_count)); + tonlib_api::blocks_transactions r; - r.id_ = to_tonlib_api(*id); + r.id_ = to_tonlib_api(*bTxes->id_); r.req_count_ = bTxes->req_count_; r.incomplete_ = bTxes->incomplete_; for (auto& id: bTxes->ids_) { - //tonlib_api::blocks_shortTxId txid = tonlib_api::blocks_shortTxId(id->mode_, id->account_.as_slice().str(), id->lt_, id->hash_.as_slice().str()); - //r.transactions_.push_back(txid); r.transactions_.push_back(to_tonlib_api(*id)); } return tonlib_api::make_object(std::move(r)); @@ -4766,79 +5972,144 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getTransactions& re return td::Status::OK(); } +td::Status TonlibClient::do_request(const tonlib_api::blocks_getTransactionsExt& request, + td::Promise>&& promise) { + TRY_RESULT(block, to_lite_api(*request.id_)) + bool check_proof = request.mode_ & ton::lite_api::liteServer_listBlockTransactionsExt::WANT_PROOF_MASK; + bool reverse_mode = request.mode_ & ton::lite_api::liteServer_listBlockTransactionsExt::REVERSE_ORDER_MASK; + bool has_starting_tx = request.mode_ & ton::lite_api::liteServer_listBlockTransactionsExt::AFTER_MASK; + + td::Bits256 start_addr; + ton::LogicalTime start_lt; + ton::lite_api::object_ptr after; + if (has_starting_tx) { + if (!request.after_) { + return td::Status::Error("Missing field `after`"); + } + TRY_RESULT_ASSIGN(start_addr, to_bits256(request.after_->account_, "account")); + start_lt = request.after_->lt_; + after = ton::lite_api::make_object(start_addr, start_lt); + } else { + start_addr = reverse_mode ? td::Bits256::ones() : td::Bits256::zero(); + start_lt = reverse_mode ? ~0ULL : 0; + after = nullptr; + } + auto block_id = ton::create_block_id(block); + client_.send_query(ton::lite_api::liteServer_listBlockTransactionsExt( + std::move(block), + request.mode_, + request.count_, + std::move(after), + reverse_mode, + check_proof), + promise.wrap([block_id, check_proof, reverse_mode, start_addr, start_lt, req_count = request.count_] + (lite_api_ptr&& bTxes) -> td::Result> { + if (block_id != create_block_id(bTxes->id_)) { + return td::Status::Error("Liteserver responded with wrong block"); + } + + block::BlockTransactionList list; + list.blkid = block_id; + list.transactions_boc = std::move(bTxes->transactions_); + list.proof_boc = std::move(bTxes->proof_); + list.reverse_mode = reverse_mode; + list.start_lt = start_lt; + list.start_addr = start_addr; + list.req_count = req_count; + auto info = list.validate(check_proof); + if (info.is_error()) { + return info.move_as_error_prefix("Validation of block::BlockTransactionList failed: "); + } + + auto raw_transactions = ToRawTransactions(td::optional()).to_raw_transactions(info.move_as_ok()); + if (raw_transactions.is_error()) { + return raw_transactions.move_as_error_prefix("Error occured while creating tonlib_api::raw_transaction: "); + } + + tonlib_api::blocks_transactionsExt r; + r.id_ = to_tonlib_api(*bTxes->id_); + r.req_count_ = bTxes->req_count_; + r.incomplete_ = bTxes->incomplete_; + r.transactions_ = raw_transactions.move_as_ok(); + return tonlib_api::make_object(std::move(r)); + })); + return td::Status::OK(); +} + td::Status TonlibClient::do_request(const tonlib_api::blocks_getBlockHeader& request, td::Promise>&& promise) { - TRY_RESULT(block, to_lite_api(*request.id_)) + TRY_RESULT(lite_block, to_lite_api(*request.id_)) + TRY_RESULT(req_blk_id, to_block_id(*request.id_)); client_.send_query(ton::lite_api::liteServer_getBlockHeader( - std::move(block), + std::move(lite_block), 0xffff), - promise.wrap([](lite_api_ptr&& hdr) { + promise.wrap([req_blk_id](lite_api_ptr&& hdr) -> td::Result> { auto blk_id = ton::create_block_id(hdr->id_); - auto R = vm::std_boc_deserialize(std::move(hdr->header_proof_)); - tonlib_api::blocks_header header; - if (R.is_error()) { - LOG(WARNING) << "R.is_error() "; - } else { - auto root = R.move_as_ok(); - try { - ton::RootHash vhash{root->get_hash().bits()}; - auto virt_root = vm::MerkleProof::virtualize(root, 1); - if (virt_root.is_null()) { - LOG(WARNING) << "virt root is null"; - } else { - std::vector prev; - ton::BlockIdExt mc_blkid; - bool after_split; - auto res = block::unpack_block_prev_blk_ext(virt_root, blk_id, prev, mc_blkid, after_split); - if (res.is_error()) { - LOG(WARNING) << "res.is_error() "; - } else { - block::gen::Block::Record blk; - block::gen::BlockInfo::Record info; - if (!(tlb::unpack_cell(virt_root, blk) && tlb::unpack_cell(blk.info, info))) { - LOG(WARNING) << "unpack failed"; - } else { - header.id_ = to_tonlib_api(blk_id); - header.global_id_ = blk.global_id; - header.version_ = info.version; - header.flags_ = info.flags; - header.after_merge_ = info.after_merge; - header.after_split_ = info.after_split; - header.before_split_ = info.before_split; - header.want_merge_ = info.want_merge; - header.want_split_ = info.want_split; - header.validator_list_hash_short_ = info.gen_validator_list_hash_short; - header.catchain_seqno_ = info.gen_catchain_seqno; - header.min_ref_mc_seqno_ = info.min_ref_mc_seqno; - header.start_lt_ = info.start_lt; - header.end_lt_ = info.end_lt; - header.gen_utime_ = info.gen_utime; - header.is_key_block_ = info.key_block; - header.vert_seqno_ = info.vert_seq_no; - if(!info.not_master) { - header.prev_key_block_seqno_ = info.prev_key_block_seqno; - } - for (auto id : prev) { - header.prev_blocks_.push_back(to_tonlib_api(id)); - } - //if(info.before_split) { - //} else { - //} - return tonlib_api::make_object(std::move(header)); - } - } - } - } catch (vm::VmError& err) { - auto E = err.as_status(PSLICE() << "error processing header for " << blk_id.to_str() << " :"); - LOG(ERROR) << std::move(E); - } catch (vm::VmVirtError& err) { - auto E = err.as_status(PSLICE() << "error processing header for " << blk_id.to_str() << " :"); - LOG(ERROR) << std::move(E); - } catch (...) { - LOG(WARNING) << "exception catched "; - } + if (blk_id != req_blk_id) { + return td::Status::Error("Liteserver responded with wrong block"); + } + auto R = vm::std_boc_deserialize(std::move(hdr->header_proof_)); + if (R.is_error()) { + return R.move_as_error_prefix("Couldn't deserialize header proof: "); + } else { + auto root = R.move_as_ok(); + try { + auto virt_root = vm::MerkleProof::virtualize(root, 1); + if (virt_root.is_null()) { + return td::Status::Error("Virt root is null"); + } else { + if (ton::RootHash{virt_root->get_hash().bits()} != blk_id.root_hash) { + return td::Status::Error("Block header merkle proof has incorrect root hash"); + } + std::vector prev; + ton::BlockIdExt mc_blkid; + bool after_split; + auto res = + block::unpack_block_prev_blk_ext(virt_root, blk_id, prev, mc_blkid, after_split); + if (res.is_error()) { + return td::Status::Error("Unpack failed"); + } else { + block::gen::Block::Record blk; + block::gen::BlockInfo::Record info; + if (!(tlb::unpack_cell(virt_root, blk) && tlb::unpack_cell(blk.info, info))) { + return td::Status::Error("Unpack failed"); + } else { + tonlib_api::blocks_header header; + header.id_ = to_tonlib_api(blk_id); + header.global_id_ = blk.global_id; + header.version_ = info.version; + header.flags_ = info.flags; + header.after_merge_ = info.after_merge; + header.after_split_ = info.after_split; + header.before_split_ = info.before_split; + header.want_merge_ = info.want_merge; + header.want_split_ = info.want_split; + header.validator_list_hash_short_ = info.gen_validator_list_hash_short; + header.catchain_seqno_ = info.gen_catchain_seqno; + header.min_ref_mc_seqno_ = info.min_ref_mc_seqno; + header.start_lt_ = info.start_lt; + header.end_lt_ = info.end_lt; + header.gen_utime_ = info.gen_utime; + header.is_key_block_ = info.key_block; + header.vert_seqno_ = info.vert_seq_no; + if (!info.not_master) { + header.prev_key_block_seqno_ = info.prev_key_block_seqno; + } + for (auto& id : prev) { + header.prev_blocks_.push_back(to_tonlib_api(id)); + } + return tonlib_api::make_object(std::move(header)); + } + } + } + } catch (vm::VmError& err) { + return err.as_status(PSLICE() << "error processing header for " << blk_id.to_str() << " :"); + } catch (vm::VmVirtError& err) { + return err.as_status(PSLICE() << "error processing header for " << blk_id.to_str() << " :"); + } catch (...) { + return td::Status::Error("Unhandled exception catched while processing header"); + } } - return tonlib_api::make_object(std::move(header)); })); return td::Status::OK(); } @@ -4865,6 +6136,24 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getShardBlockProof& return td::Status::OK(); } +td::Status TonlibClient::do_request(const tonlib_api::blocks_getOutMsgQueueSizes& request, + td::Promise>&& promise) { + client_.send_query(ton::lite_api::liteServer_getOutMsgQueueSizes(request.mode_, request.wc_, request.shard_), + promise.wrap([](lite_api_ptr&& queue_sizes) { + tonlib_api::blocks_outMsgQueueSizes result; + result.ext_msg_queue_size_limit_ = queue_sizes->ext_msg_queue_size_limit_; + for (auto &x : queue_sizes->shards_) { + tonlib_api::blocks_outMsgQueueSize shard; + shard.id_ = to_tonlib_api(*x->id_); + shard.size_ = x->size_; + result.shards_.push_back(tonlib_api::make_object(std::move(shard))); + } + return tonlib_api::make_object(std::move(result)); + })); + + return td::Status::OK(); +} + void TonlibClient::load_libs_from_disk() { LOG(DEBUG) << "loading libraries from disk cache"; auto r_data = kv_->get("tonlib.libcache"); @@ -4877,17 +6166,41 @@ void TonlibClient::load_libs_from_disk() { } libraries = vm::Dictionary(vm::load_cell_slice(vm::CellBuilder().append_cellslice(vm::load_cell_slice( r_dict.move_as_ok())).finalize()), 256); - // int n = 0; for (auto&& lr : libraries) n++; + LOG(DEBUG) << "loaded libraries from disk cache"; } void TonlibClient::store_libs_to_disk() { // NB: Dictionary.get_root_cell does not compute_root, and it is protected kv_->set("tonlib.libcache", vm::std_boc_serialize(vm::CellBuilder().append_cellslice(libraries.get_root()) .finalize()).move_as_ok().as_slice()); - // int n = 0; for (auto&& lr : libraries) n++; + LOG(DEBUG) << "stored libraries to disk cache"; } +td::Status TonlibClient::do_request(const int_api::ScanAndLoadGlobalLibs& request, td::Promise promise) { + if (request.root.is_null()) { + promise.set_value(vm::Dictionary{256}); + return td::Status::OK(); + } + std::set to_load; + std::set visited; + deep_library_search(to_load, visited, libraries, request.root, 24); + if (to_load.empty()) { + promise.set_result(libraries); + return td::Status::OK(); + } + std::vector to_load_list(to_load.begin(), to_load.end()); + LOG(DEBUG) << "Requesting found libraries in account state (" << to_load_list.size() << ")"; + client_.send_query( + ton::lite_api::liteServer_getLibraries(std::move(to_load_list)), + [self = this, promise = std::move(promise)]( + td::Result> r_libraries) mutable { + self->process_new_libraries(std::move(r_libraries)); + promise.set_result(self->libraries); + }); + return td::Status::OK(); +} + template td::Status TonlibClient::do_request(const tonlib_api::runTests& request, P&&) { UNREACHABLE(); diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index 0bb7aadc..28239a8a 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -33,17 +33,20 @@ #include "td/utils/optional.h" #include "smc-envelope/ManualDns.h" +#include "lite-client/ext-client.h" #include namespace tonlib { namespace int_api { struct GetAccountState; +struct GetAccountStateByTransaction; struct GetPrivateKey; struct GetDnsResolver; struct SendMessage; struct RemoteRunSmcMethod; struct RemoteRunSmcMethodReturnType; +struct ScanAndLoadGlobalLibs; inline std::string to_string(const int_api::SendMessage&) { return "Send message"; @@ -51,6 +54,7 @@ inline std::string to_string(const int_api::SendMessage&) { } // namespace int_api class AccountState; class Query; +class RunEmulator; td::Result> to_tonlib_api( const ton::ManualDns::EntryData& entry_data); @@ -110,7 +114,7 @@ class TonlibClient : public td::actor::Actor { vm::Dictionary libraries{256}; // network - td::actor::ActorOwn raw_client_; + td::actor::ActorOwn raw_client_; td::actor::ActorId ext_client_outbound_; td::actor::ActorOwn raw_last_block_; td::actor::ActorOwn raw_last_config_; @@ -234,6 +238,8 @@ class TonlibClient : public td::actor::Actor { td::Status do_request(tonlib_api::raw_getAccountState& request, td::Promise>&& promise); + td::Status do_request(tonlib_api::raw_getAccountStateByTransaction& request, + td::Promise>&& promise); td::Status do_request(tonlib_api::raw_getTransactions& request, td::Promise>&& promise); td::Status do_request(tonlib_api::raw_getTransactionsV2& request, @@ -241,6 +247,12 @@ class TonlibClient : public td::actor::Actor { td::Status do_request(const tonlib_api::getAccountState& request, td::Promise>&& promise); + td::Status do_request(const tonlib_api::getAccountStateByTransaction& request, + td::Promise>&& promise); + td::Status do_request(const tonlib_api::getShardAccountCell& request, + td::Promise>&& promise); + td::Status do_request(const tonlib_api::getShardAccountCellByTransaction& request, + td::Promise>&& promise); td::Status do_request(tonlib_api::guessAccountRevision& request, td::Promise>&& promise); td::Status do_request(tonlib_api::guessAccount& request, @@ -305,6 +317,7 @@ class TonlibClient : public td::actor::Actor { td::Result> get_smc_info(td::int64 id); void finish_load_smc(td::unique_ptr query, td::Promise>&& promise); td::Status do_request(const tonlib_api::smc_load& request, td::Promise>&& promise); + td::Status do_request(const tonlib_api::smc_loadByTransaction& request, td::Promise>&& promise); td::Status do_request(const tonlib_api::smc_forget& request, td::Promise>&& promise); td::Status do_request(const tonlib_api::smc_getCode& request, td::Promise>&& promise); @@ -312,12 +325,18 @@ class TonlibClient : public td::actor::Actor { td::Promise>&& promise); td::Status do_request(const tonlib_api::smc_getState& request, td::Promise>&& promise); + td::Status do_request(const tonlib_api::smc_getRawFullAccountState& request, + td::Promise>&& promise); td::Status do_request(const tonlib_api::smc_runGetMethod& request, td::Promise>&& promise); td::Status do_request(const tonlib_api::smc_getLibraries& request, td::Promise>&& promise); + void get_libraries(ton::BlockIdExt blkid, std::vector library_list_, td::Promise>&& promise); + + td::Status do_request(const tonlib_api::smc_getLibrariesExt& request, + td::Promise>&& promise); td::Status do_request(const tonlib_api::dns_resolve& request, td::Promise>&& promise); @@ -330,6 +349,8 @@ class TonlibClient : public td::actor::Actor { td::Status do_request(tonlib_api::pchan_unpackPromise& request, td::Promise>&& promise); + void process_new_libraries( + td::Result> r_libraries); void perform_smc_execution(td::Ref smc, ton::SmartContract::Args args, td::Promise>&& promise); @@ -344,6 +365,7 @@ class TonlibClient : public td::actor::Actor { td::Promise>&& promise); td::Status do_request(int_api::GetAccountState request, td::Promise>&&); + td::Status do_request(int_api::GetAccountStateByTransaction request, td::Promise>&&); td::Status do_request(int_api::GetPrivateKey request, td::Promise&&); td::Status do_request(int_api::GetDnsResolver request, td::Promise&&); td::Status do_request(int_api::RemoteRunSmcMethod request, @@ -363,15 +385,25 @@ class TonlibClient : public td::actor::Actor { td::Promise>&& promise); td::Status do_request(const tonlib_api::blocks_getTransactions& block_data, td::Promise>&& promise); + td::Status do_request(const tonlib_api::blocks_getTransactionsExt& request, + td::Promise>&& promise); td::Status do_request(const tonlib_api::blocks_getBlockHeader& request, td::Promise>&& promise); td::Status do_request(const tonlib_api::blocks_getMasterchainBlockSignatures& request, td::Promise>&& promise); td::Status do_request(const tonlib_api::blocks_getShardBlockProof& request, td::Promise>&& promise); + td::Status do_request(const tonlib_api::blocks_getOutMsgQueueSizes& request, + td::Promise>&& promise); + void get_config_param(int32_t param, int32_t mode, ton::BlockIdExt block, + td::Promise>&& promise); td::Status do_request(const tonlib_api::getConfigParam& request, td::Promise>&& promise); + void get_config_all(int32_t mode, ton::BlockIdExt block, + td::Promise>&& promise); + td::Status do_request(const tonlib_api::getConfigAll& request, + td::Promise>&& promise); void proxy_request(td::int64 query_id, std::string data); @@ -388,5 +420,7 @@ class TonlibClient : public td::actor::Actor { td::Status guess_revisions(std::vector targets, td::Promise>&& promise); + + td::Status do_request(const int_api::ScanAndLoadGlobalLibs& request, td::Promise promise); }; } // namespace tonlib diff --git a/tonlib/tonlib/Wallet.cpp b/tonlib/tonlib/Wallet.cpp deleted file mode 100644 index b4682823..00000000 --- a/tonlib/tonlib/Wallet.cpp +++ /dev/null @@ -1,95 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2019 Telegram Systems LLP -*/ -#include "tonlib/Wallet.h" -#include "tonlib/CellString.h" -#include "tonlib/GenericAccount.h" -#include "tonlib/utils.h" - -#include "vm/boc.h" -#include "td/utils/base64.h" - -#include - -namespace tonlib { -td::Ref Wallet::get_init_state(const td::Ed25519::PublicKey& public_key) noexcept { - auto code = get_init_code(); - auto data = get_init_data(public_key); - return GenericAccount::get_init_state(std::move(code), std::move(data)); -} - -td::Ref Wallet::get_init_message(const td::Ed25519::PrivateKey& private_key) noexcept { - td::uint32 seqno = 0; - td::uint32 valid_until = std::numeric_limits::max(); - auto signature = - private_key - .sign(vm::CellBuilder().store_long(seqno, 32).store_long(valid_until, 32).finalize()->get_hash().as_slice()) - .move_as_ok(); - return vm::CellBuilder().store_bytes(signature).store_long(seqno, 32).store_long(valid_until, 32).finalize(); -} - -td::Ref Wallet::make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 seqno, - td::uint32 valid_until, td::int64 gramms, td::Slice message, - const block::StdAddress& dest_address) noexcept { - td::BigInt256 dest_addr; - dest_addr.import_bits(dest_address.addr.as_bitslice()); - vm::CellBuilder cb; - cb.append_cellslice(binary_bitstring_to_cellslice("b{01}").move_as_ok()) - .store_long(dest_address.bounceable, 1) - .append_cellslice(binary_bitstring_to_cellslice("b{000100}").move_as_ok()) - .store_long(dest_address.workchain, 8) - .store_int256(dest_addr, 256); - td::int32 send_mode = 3; - if (gramms == -1) { - gramms = 0; - send_mode += 128; - } - block::tlb::t_Grams.store_integer_value(cb, td::BigInt256(gramms)); - cb.store_zeroes(9 + 64 + 32 + 1 + 1).store_bytes("\0\0\0\0", 4); - vm::CellString::store(cb, message, 35 * 8).ensure(); - auto message_inner = cb.finalize(); - auto message_outer = vm::CellBuilder() - .store_long(seqno, 32) - .store_long(valid_until, 32) - .store_long(send_mode, 8) - .store_ref(message_inner) - .finalize(); - std::string seq_no(4, 0); - auto signature = private_key.sign(message_outer->get_hash().as_slice()).move_as_ok(); - return vm::CellBuilder().store_bytes(signature).append_cellslice(vm::load_cell_slice(message_outer)).finalize(); -} - -td::Ref Wallet::get_init_code() noexcept { - static auto res = [] { - auto serialized_code = td::base64_decode( - "te6ccgEEAQEAAAAAVwAAqv8AIN0gggFMl7qXMO1E0NcLH+Ck8mCDCNcYINMf0x8B+CO78mPtRNDTH9P/" - "0VExuvKhA/kBVBBC+RDyovgAApMg10qW0wfUAvsA6NGkyMsfy//J7VQ=") - .move_as_ok(); - return vm::std_boc_deserialize(serialized_code).move_as_ok(); - }(); - return res; -} - -vm::CellHash Wallet::get_init_code_hash() noexcept { - return get_init_code()->get_hash(); -} - -td::Ref Wallet::get_init_data(const td::Ed25519::PublicKey& public_key) noexcept { - return vm::CellBuilder().store_long(0, 32).store_bytes(public_key.as_octet_string()).finalize(); -} -} // namespace tonlib diff --git a/tonlib/tonlib/Wallet.h b/tonlib/tonlib/Wallet.h deleted file mode 100644 index dd114cce..00000000 --- a/tonlib/tonlib/Wallet.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2019 Telegram Systems LLP -*/ -#pragma once - -#include "vm/cells.h" -#include "Ed25519.h" -#include "block/block.h" -#include "CellString.h" - -namespace tonlib { -class Wallet { - public: - static constexpr unsigned max_message_size = vm::CellString::max_bytes; - static td::Ref get_init_state(const td::Ed25519::PublicKey& public_key) noexcept; - static td::Ref get_init_message(const td::Ed25519::PrivateKey& private_key) noexcept; - static td::Ref make_a_gift_message(const td::Ed25519::PrivateKey& private_key, td::uint32 seqno, - td::uint32 valid_until, td::int64 gramms, td::Slice message, - const block::StdAddress& dest_address) noexcept; - - static td::Ref get_init_code() noexcept; - static vm::CellHash get_init_code_hash() noexcept; - static td::Ref get_init_data(const td::Ed25519::PublicKey& public_key) noexcept; -}; -} // namespace tonlib diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index db7497b0..2c7100f2 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -47,8 +47,6 @@ #include "tonlib/TonlibClient.h" #include "tonlib/TonlibCallback.h" -#include "tonlib/ExtClientLazy.h" - #include "smc-envelope/ManualDns.h" #include "smc-envelope/PaymentChannel.h" @@ -57,6 +55,7 @@ #include "crypto/util/Miner.h" #include "vm/boc.h" #include "vm/cells/CellBuilder.h" +#include "lite-client/ext-client.h" #include #include @@ -174,7 +173,7 @@ class TonlibCli : public td::actor::Actor { std::map>> query_handlers_; - td::actor::ActorOwn raw_client_; + td::actor::ActorOwn raw_client_; bool is_closing_{false}; td::uint32 ref_cnt_{1}; @@ -223,11 +222,7 @@ class TonlibCli : public td::actor::Actor { if (options_.use_callbacks_for_network) { auto config = tonlib::Config::parse(options_.config).move_as_ok(); - auto lite_clients_size = config.lite_clients.size(); - CHECK(lite_clients_size != 0); - auto lite_client_id = td::Random::fast(0, td::narrow_cast(lite_clients_size) - 1); - auto& lite_client = config.lite_clients[lite_client_id]; - class Callback : public tonlib::ExtClientLazy::Callback { + class Callback : public liteclient::ExtClient::Callback { public: explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { } @@ -236,14 +231,14 @@ class TonlibCli : public td::actor::Actor { td::actor::ActorShared<> parent_; }; ref_cnt_++; - raw_client_ = tonlib::ExtClientLazy::create(lite_client.adnl_id, lite_client.address, + raw_client_ = liteclient::ExtClient::create(config.lite_servers, td::make_unique(td::actor::actor_shared())); } auto config = !options_.config.empty() - ? make_object(options_.config, options_.name, - options_.use_callbacks_for_network, options_.ignore_cache) - : nullptr; + ? make_object(options_.config, options_.name, + options_.use_callbacks_for_network, options_.ignore_cache) + : nullptr; tonlib_api::object_ptr ks_type; if (options_.in_memory) { @@ -386,9 +381,11 @@ class TonlibCli : public td::actor::Actor { td::TerminalIO::out() << "sendfile \tLoad a serialized message from and send it to server\n"; td::TerminalIO::out() << "setconfig|validateconfig [] [] [] - set or validate " "lite server config\n"; - td::TerminalIO::out() << "runmethod ...\tRuns GET method of account " + td::TerminalIO::out() << "runmethod ...\tRuns GET method of account " " with specified parameters\n"; td::TerminalIO::out() << "getstate \tget state of wallet with requested key\n"; + td::TerminalIO::out() << "getstatebytransaction \tget state of wallet with requested key after transaction with local time and hash (base64url)\n"; + td::TerminalIO::out() << "getconfig \tshow specified configuration parameter from the latest masterchain state\n"; td::TerminalIO::out() << "guessrevision \tsearch of existing accounts corresponding to the given key\n"; td::TerminalIO::out() << "guessaccount \tsearch of existing accounts corresponding to the given key\n"; td::TerminalIO::out() << "getaddress \tget address of wallet with requested key\n"; @@ -421,6 +418,8 @@ class TonlibCli : public td::actor::Actor { td::TerminalIO::out() << "exportkeypem [] - export key\n"; td::TerminalIO::out() << "gethistory - get history fo simple wallet with requested key (last 10 transactions)\n"; + td::TerminalIO::out() << "showtransactions [] - show transaction on account " + "with given and (in base64) and previous transactions (up to ).\n"; td::TerminalIO::out() << "init - init simple wallet with requested key\n"; td::TerminalIO::out() << "transfer[f][F][e][k][c] ( |) - " "make transfer from \n" @@ -489,6 +488,10 @@ class TonlibCli : public td::actor::Actor { transfer(parser, cmd, std::move(cmd_promise)); } else if (cmd == "getstate") { get_state(parser.read_word(), std::move(cmd_promise)); + } else if (cmd == "getstatebytransaction") { + get_state_by_transaction(parser, std::move(cmd_promise)); + } else if (cmd == "getconfig") { + get_config_param(parser, std::move(cmd_promise)); } else if (cmd == "getaddress") { get_address(parser.read_word(), std::move(cmd_promise)); } else if (cmd == "importkeypem") { @@ -514,6 +517,8 @@ class TonlibCli : public td::actor::Actor { } else if (cmd == "getmasterchainsignatures") { auto seqno = parser.read_word(); run_get_masterchain_block_signatures(seqno, std::move(cmd_promise)); + } else if (cmd == "showtransactions") { + run_show_transactions(parser, std::move(cmd_promise)); } else { cmd_promise.set_error(td::Status::Error(PSLICE() << "Unkwnown query `" << cmd << "`")); } @@ -1098,9 +1103,9 @@ class TonlibCli : public td::actor::Actor { void pchan_init_2(Address addr, td::int32 pchan_id, td::int64 value, tonlib_api::object_ptr query, td::Promise promise) { std::vector> messages; - messages.push_back( - make_object(channels_[pchan_id].to_address(), "", value, - make_object(query->body_, query->init_state_), -1)); + messages.push_back(make_object( + channels_[pchan_id].to_address(), "", value, std::vector>{}, + make_object(query->body_, query->init_state_), -1)); auto action = make_object(std::move(messages), true); send_query( make_object(addr.input_key(), std::move(addr.address), 60, std::move(action), nullptr), @@ -1351,10 +1356,10 @@ class TonlibCli : public td::actor::Actor { } if (l >= 3 && (str[0] == 'x' || str[0] == 'b') && str[1] == '{' && str.back() == '}') { unsigned char buff[128]; - int bits = - (str[0] == 'x') - ? (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.begin() + 2, str.end() - 1) - : (int)td::bitstring::parse_bitstring_binary_literal(buff, sizeof(buff), str.begin() + 2, str.end() - 1); + int bits = (str[0] == 'x') ? (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.begin() + 2, + str.end() - 1) + : (int)td::bitstring::parse_bitstring_binary_literal(buff, sizeof(buff) * 8, + str.begin() + 2, str.end() - 1); if (bits < 0) { return td::Status::Error("Failed to parse slice"); } @@ -1413,7 +1418,8 @@ class TonlibCli : public td::actor::Actor { if (r_cell.is_error()) { sb << ""; } - auto cs = vm::load_cell_slice(r_cell.move_as_ok()); + bool spec = true; + auto cs = vm::load_cell_slice_special(r_cell.move_as_ok(), spec); std::stringstream ss; cs.print_rec(ss); sb << ss.str(); @@ -1534,7 +1540,7 @@ class TonlibCli : public td::actor::Actor { auto update = tonlib_api::move_object_as(std::move(result)); CHECK(!raw_client_.empty()); snd_bytes_ += update->data_.size(); - send_closure(raw_client_, &ton::adnl::AdnlExtClient::send_query, "query", td::BufferSlice(update->data_), + send_closure(raw_client_, &liteclient::ExtClient::send_query, "query", td::BufferSlice(update->data_), td::Timestamp::in(5), [actor_id = actor_id(this), id = update->id_](td::Result res) { send_closure(actor_id, &TonlibCli::on_adnl_result, id, std::move(res)); @@ -2047,6 +2053,19 @@ class TonlibCli : public td::actor::Actor { }); } + static void print_full_account_state(const ton::tl_object_ptr& state) { + td::StringBuilder balance_str; + balance_str << "Balance: " << Grams{td::narrow_cast(state->balance_ * (state->balance_ > 0))}; + for (const auto& extra : state->extra_currencies_) { + balance_str << " + " << extra->amount_ << ".$" << extra->id_; + } + td::TerminalIO::out() << balance_str.as_cslice() << "\n"; + td::TerminalIO::out() << "Sync utime: " << state->sync_utime_ << "\n"; + td::TerminalIO::out() << "transaction.LT: " << state->last_transaction_id_->lt_ << "\n"; + td::TerminalIO::out() << "transaction.Hash: " << td::base64_encode(state->last_transaction_id_->hash_) << "\n"; + td::TerminalIO::out() << to_string(state->account_state_); + } + void get_state(td::Slice key, td::Promise promise) { TRY_RESULT_PROMISE(promise, address, to_account_address(key, false)); @@ -2055,14 +2074,44 @@ class TonlibCli : public td::actor::Actor { ton::move_tl_object_as(std::move(address.address))), promise.wrap([address_str](auto&& state) { td::TerminalIO::out() << "Address: " << address_str << "\n"; - td::TerminalIO::out() << "Balance: " - << Grams{td::narrow_cast(state->balance_ * (state->balance_ > 0))} - << "\n"; - td::TerminalIO::out() << "Sync utime: " << state->sync_utime_ << "\n"; - td::TerminalIO::out() << "transaction.LT: " << state->last_transaction_id_->lt_ << "\n"; - td::TerminalIO::out() << "transaction.Hash: " << td::base64_encode(state->last_transaction_id_->hash_) - << "\n"; - td::TerminalIO::out() << to_string(state->account_state_); + print_full_account_state(state); + return td::Unit(); + })); + } + + void get_state_by_transaction(td::ConstParser& parser, td::Promise promise) { + TRY_RESULT_PROMISE(promise, address, to_account_address(parser.read_word(), false)); + TRY_RESULT_PROMISE(promise, lt, td::to_integer_safe(parser.read_word())); + TRY_RESULT_PROMISE(promise, hash, td::base64url_decode(parser.read_word())); + + auto address_str = address.address->account_address_; + auto transaction_id = std::make_unique(lt, std::move(hash)); + send_query(make_object( + ton::move_tl_object_as(std::move(address.address)), + ton::move_tl_object_as(std::move(transaction_id))), + promise.wrap([address_str](auto&& state) { + td::TerminalIO::out() << "Address: " << address_str << "\n"; + print_full_account_state(state); + return td::Unit(); + })); + } + + void get_config_param(td::ConstParser& parser, td::Promise promise) { + TRY_RESULT_PROMISE(promise, param, td::to_integer_safe(parser.read_word())); + send_query(make_object(0, param), + promise.wrap([param](auto&& result) -> td::Result { + TRY_RESULT(cell, vm::std_boc_deserialize(result->config_->bytes_, true)); + if (cell.is_null()) { + td::TerminalIO::out() << "ConfigParam(" << param << ") = (null)\n"; + return td::Unit(); + } + std::ostringstream os; + if (param >= 0) { + block::gen::ConfigParam{param}.print_ref(4096, os, cell); + os << "\n"; + } + vm::load_cell_slice(cell).print_rec(4096, os); + td::TerminalIO::out() << "ConfigParam(" << param << ") = " << os.str() << "\n"; return td::Unit(); })); } @@ -2112,6 +2161,45 @@ class TonlibCli : public td::actor::Actor { })); } + void run_show_transactions(td::ConstParser& parser, td::Promise promise) { + TRY_RESULT_PROMISE(promise, address, to_account_address(parser.read_word(), false)); + TRY_RESULT_PROMISE(promise, lt, td::to_integer_safe(parser.read_word())); + TRY_RESULT_PROMISE(promise, hash, td::base64_decode(parser.read_word())); + int count = 1; + if (!parser.empty()) { + TRY_RESULT_PROMISE_ASSIGN(promise, count, td::to_integer_safe(parser.read_word())); + } + auto id = make_object(lt, hash); + send_query(make_object( + nullptr, ton::move_tl_object_as(std::move(address.address)), + std::move(id), count, false), + promise.wrap([](ton::tl_object_ptr&& result) -> td::Result { + td::TerminalIO::out() << "Found " << result->transactions_.size() << " transactions\n"; + for (size_t i = 0; i < result->transactions_.size(); ++i) { + td::TerminalIO::out() << "Transaction #" << i << "\n"; + auto& tr = result->transactions_[i]; + TRY_RESULT(root, vm::std_boc_deserialize(tr->data_)); + block::gen::Transaction::Record trans; + if (!tlb::unpack_cell(root, trans)) { + return td::Status::Error("cannot unpack transaction"); + } + td::TerminalIO::out() << "Transaction Account: " << tr->address_->account_address_ << "\n"; + td::TerminalIO::out() << "Transaction LT: " << tr->transaction_id_->lt_ << "\n"; + td::TerminalIO::out() << "Transaction Hash: " << td::base64_encode(tr->transaction_id_->hash_) + << "\n"; + td::TerminalIO::out() << "Transaction Timestamp: " << tr->utime_ << "\n"; + td::TerminalIO::out() << "Transaction Out messages: " << tr->out_msgs_.size() << "\n"; + td::TerminalIO::out() << "Previous transaction LT: " << trans.prev_trans_lt << "\n"; + td::TerminalIO::out() << "Previous transaction Hash: " + << td::base64_encode(trans.prev_trans_hash.as_slice()) << "\n"; + std::ostringstream ss; + block::gen::t_Transaction.print_ref(2048, ss, root); + td::TerminalIO::out() << "Transaction dump: " << ss.str() << "\n"; + } + return td::Unit(); + })); + } + void get_history2(td::Slice key, td::Result> r_state, td::Promise promise) { TRY_RESULT_PROMISE(promise, state, std::move(r_state)); @@ -2131,15 +2219,29 @@ class TonlibCli : public td::actor::Actor { td::StringBuilder sb; for (tonlib_api::object_ptr& t : res->transactions_) { td::int64 balance = 0; + std::map extra_currencies; balance += t->in_msg_->value_; + for (const auto& extra : t->in_msg_->extra_currencies_) { + extra_currencies[extra->id_] += extra->amount_; + } for (auto& ot : t->out_msgs_) { balance -= ot->value_; + for (const auto& extra : ot->extra_currencies_) { + extra_currencies[extra->id_] -= extra->amount_; + } } if (balance >= 0) { sb << Grams{td::uint64(balance)}; } else { sb << "-" << Grams{td::uint64(-balance)}; } + for (const auto& [id, amount] : extra_currencies) { + if (amount > 0) { + sb << " + " << amount << ".$" << id; + } else if (amount < 0) { + sb << " - " << -amount << ".$" << id; + } + } sb << " Fee: " << Grams{td::uint64(t->fee_)}; if (t->in_msg_->source_->account_address_.empty()) { sb << " External "; @@ -2169,6 +2271,9 @@ class TonlibCli : public td::actor::Actor { sb << " To " << ot->destination_->account_address_; } sb << " " << Grams{td::uint64(ot->value_)}; + for (const auto& extra : ot->extra_currencies_) { + sb << " + " << extra->amount_ << ".$" << extra->id_; + } print_msg_data(sb, ot->msg_data_); } sb << "\n"; @@ -2233,8 +2338,9 @@ class TonlibCli : public td::actor::Actor { } else { data = make_object(message.str()); } - messages.push_back( - make_object(std::move(address.address), "", amount.nano, std::move(data), -1)); + messages.push_back(make_object( + std::move(address.address), "", amount.nano, std::vector>{}, + std::move(data), -1)); return td::Status::OK(); }; diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 3d029c93..e9ffd2a2 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) @@ -7,22 +7,21 @@ endif() add_executable(generate-random-id generate-random-id.cpp ) target_link_libraries(generate-random-id tl_api ton_crypto keys adnl git) -target_include_directories(generate-random-id PUBLIC - $/..) +target_include_directories(generate-random-id PUBLIC $/..) add_executable(json2tlo json2tlo.cpp ) target_link_libraries(json2tlo tl_api ton_crypto keys git) -target_include_directories(json2tlo PUBLIC - $/..) +target_include_directories(json2tlo PUBLIC $/..) add_executable(pack-viewer pack-viewer.cpp ) target_link_libraries(pack-viewer tl_api ton_crypto keys validator tddb) -target_include_directories(pack-viewer PUBLIC - $/..) +target_include_directories(pack-viewer PUBLIC $/..) add_executable(opcode-timing opcode-timing.cpp ) target_link_libraries(opcode-timing ton_crypto) -target_include_directories(pack-viewer PUBLIC - $/..) +target_include_directories(pack-viewer PUBLIC $/..) -install(TARGETS generate-random-id RUNTIME DESTINATION bin) +add_executable(proxy-liteserver proxy-liteserver.cpp) +target_link_libraries(proxy-liteserver tdutils tdactor adnl dht tl_api ton_crypto git lite-client-common) + +install(TARGETS generate-random-id proxy-liteserver RUNTIME DESTINATION bin) diff --git a/utils/generate-random-id.cpp b/utils/generate-random-id.cpp index 3727f291..a487ac17 100644 --- a/utils/generate-random-id.cpp +++ b/utils/generate-random-id.cpp @@ -28,9 +28,6 @@ #include #include #include -#include -#include -#include "crypto/ellcurve/Ed25519.h" #include "adnl/utils.hpp" #include "auto/tl/ton_api.h" #include "auto/tl/ton_api_json.h" @@ -38,12 +35,13 @@ #include "td/utils/OptionParser.h" #include "td/utils/filesystem.h" #include "keys/encryptor.h" -#include "keys/keys.hpp" #include "git.h" +#include "dht/dht-node.hpp" int main(int argc, char *argv[]) { ton::PrivateKey pk; - ton::tl_object_ptr addr_list; + td::optional addr_list; + td::optional network_id_opt; td::OptionParser p; p.set_description("generate random id"); @@ -78,11 +76,32 @@ int main(int argc, char *argv[]) { if (addr_list) { return td::Status::Error("duplicate '-a' option"); } - CHECK(!addr_list); td::BufferSlice bs(key); TRY_RESULT_PREFIX(as_json_value, td::json_decode(bs.as_slice()), "bad addr list JSON: "); - TRY_STATUS_PREFIX(td::from_json(addr_list, std::move(as_json_value)), "bad addr list TL: "); + ton::tl_object_ptr addr_list_tl; + TRY_STATUS_PREFIX(td::from_json(addr_list_tl, std::move(as_json_value)), "bad addr list TL: "); + TRY_RESULT_PREFIX_ASSIGN(addr_list, ton::adnl::AdnlAddressList::create(addr_list_tl), "bad addr list: "); + return td::Status::OK(); + }); + p.add_checked_option('f', "path to file with addr-list", "addr list to sign", [&](td::Slice key) { + if (addr_list) { + return td::Status::Error("duplicate '-f' option"); + } + + td::BufferSlice bs(key); + TRY_RESULT_PREFIX(data, td::read_file(key.str()), "failed to read addr-list: "); + TRY_RESULT_PREFIX(as_json_value, td::json_decode(data.as_slice()), "bad addr list JSON: "); + ton::tl_object_ptr addr_list_tl; + TRY_STATUS_PREFIX(td::from_json(addr_list_tl, std::move(as_json_value)), "bad addr list TL: "); + TRY_RESULT_PREFIX_ASSIGN(addr_list, ton::adnl::AdnlAddressList::create(addr_list_tl), "bad addr list: "); + return td::Status::OK(); + }); + p.add_checked_option('i', "network-id", "dht network id (default: -1)", [&](td::Slice key) { + if (network_id_opt) { + return td::Status::Error("duplicate '-i' option"); + } + TRY_RESULT_PREFIX_ASSIGN(network_id_opt, td::to_integer_safe(key), "bad network id: "); return td::Status::OK(); }); @@ -118,7 +137,7 @@ int main(int argc, char *argv[]) { std::cerr << "'-a' option missing" << std::endl; return 2; } - auto x = ton::create_tl_object(pub_key.tl(), std::move(addr_list)); + auto x = ton::create_tl_object(pub_key.tl(), addr_list.value().tl()); auto e = pk.create_decryptor().move_as_ok(); auto r = e->sign(ton::serialize_tl_object(x, true).as_slice()).move_as_ok(); @@ -129,12 +148,17 @@ int main(int argc, char *argv[]) { std::cerr << "'-a' option missing" << std::endl; return 2; } - auto x = ton::create_tl_object(pub_key.tl(), std::move(addr_list), -1, td::BufferSlice()); + td::int32 network_id = network_id_opt ? network_id_opt.value() : -1; + td::BufferSlice to_sign = ton::serialize_tl_object( + ton::dht::DhtNode{ton::adnl::AdnlNodeIdFull{pub_key}, addr_list.value(), -1, network_id, td::BufferSlice{}} + .tl(), + true); auto e = pk.create_decryptor().move_as_ok(); - auto r = e->sign(ton::serialize_tl_object(x, true).as_slice()).move_as_ok(); - x->signature_ = std::move(r); + auto signature = e->sign(to_sign.as_slice()).move_as_ok(); + auto node = + ton::dht::DhtNode{ton::adnl::AdnlNodeIdFull{pub_key}, addr_list.value(), -1, network_id, std::move(signature)}; - auto v = td::json_encode(td::ToJson(x)); + auto v = td::json_encode(td::ToJson(node.tl())); std::cout << v << "\n"; } else if (mode == "keys") { td::write_file(name, pk.export_as_slice()).ensure(); diff --git a/utils/opcode-timing.cpp b/utils/opcode-timing.cpp index dc6ac75f..47171eec 100644 --- a/utils/opcode-timing.cpp +++ b/utils/opcode-timing.cpp @@ -8,15 +8,71 @@ #include "common/bigint.hpp" #include "td/utils/base64.h" -#include "td/utils/tests.h" #include "td/utils/ScopeGuard.h" #include "td/utils/StringBuilder.h" +#include "td/utils/Timer.h" +#include "block.h" +#include "td/utils/filesystem.h" +#include "mc-config.h" -td::Ref to_cell(const unsigned char *buff, int bits) { - return vm::CellBuilder().store_bits(buff, bits, 0).finalize(); +td::Ref c7; + +void prepare_c7() { + auto now = (td::uint32)td::Clocks::system(); + td::Ref config_root; + auto config_data = td::read_file("config.boc"); + if (config_data.is_ok()) { + LOG(WARNING) << "Reading config from config.boc"; + auto r_cell = vm::std_boc_deserialize(config_data.move_as_ok()); + r_cell.ensure(); + config_root = r_cell.move_as_ok(); + } + + vm::CellBuilder addr; + addr.store_long(4, 3); + addr.store_long(0, 8); + addr.store_ones(256); + std::vector tuple = { + td::make_refint(0x076ef1ea), // [ magic:0x076ef1ea + td::zero_refint(), // actions:Integer + td::zero_refint(), // msgs_sent:Integer + td::make_refint(now), // unixtime:Integer + td::make_refint(0), // block_lt:Integer + td::make_refint(0), // trans_lt:Integer + td::make_refint(123), // rand_seed:Integer + block::CurrencyCollection(td::make_refint(10000LL * 1000000000)) + .as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)] + addr.as_cellslice_ref(), // myself:MsgAddressInt + vm::StackEntry::maybe(config_root) // global_config:(Maybe Cell) ] = SmartContractInfo; + }; + tuple.push_back({}); // code:Cell + tuple.push_back(block::CurrencyCollection(td::make_refint(2000LL * 1000000000)) + .as_vm_tuple()); // in_msg_value:[Integer (Maybe Cell)] + tuple.push_back(td::make_refint(0)); // storage_fees:Integer + tuple.push_back(vm::StackEntry()); // prev_blocks_info + if (config_root.not_null()) { + block::Config config{config_root}; + config.unpack().ensure(); + tuple.push_back(config.get_unpacked_config_tuple(now)); // unpacked_config_tuple + } else { + tuple.push_back(vm::StackEntry()); + } + tuple.push_back(td::zero_refint()); + auto tuple_ref = td::make_cnt_ref>(std::move(tuple)); + c7 = vm::make_tuple_ref(std::move(tuple_ref)); } -long double timingBaseline; +td::Ref to_cell(td::Slice s) { + if (s.size() >= 4 && s.substr(0, 4) == "boc:") { + s.remove_prefix(4); + auto boc = td::base64_decode(s).move_as_ok(); + return vm::std_boc_deserialize(boc).move_as_ok(); + } + unsigned char buff[128]; + const int bits = (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), s.begin(), s.end()); + CHECK(bits >= 0); + return vm::CellBuilder().store_bits(buff, bits, 0).finalize(); +} typedef struct { long double mean; @@ -28,9 +84,11 @@ struct runInfo { long long gasUsage; int vmReturnCode; - runInfo() : runtime(0.0), gasUsage(0), vmReturnCode(0) {} - runInfo(long double runtime, long long gasUsage, int vmReturnCode) : - runtime(runtime), gasUsage(gasUsage), vmReturnCode(vmReturnCode) {} + runInfo() : runtime(0.0), gasUsage(0), vmReturnCode(0) { + } + runInfo(long double runtime, long long gasUsage, int vmReturnCode) + : runtime(runtime), gasUsage(gasUsage), vmReturnCode(vmReturnCode) { + } runInfo operator+(const runInfo& addend) const { return {runtime + addend.runtime, gasUsage + addend.gasUsage, vmReturnCode ? vmReturnCode : addend.vmReturnCode}; @@ -39,7 +97,7 @@ struct runInfo { runInfo& operator+=(const runInfo& addend) { runtime += addend.runtime; gasUsage += addend.gasUsage; - if(!vmReturnCode && addend.vmReturnCode) { + if (!vmReturnCode && addend.vmReturnCode) { vmReturnCode = addend.vmReturnCode; } return *this; @@ -56,116 +114,120 @@ typedef struct { bool errored; } runtimeStats; -runInfo time_run_vm(td::Slice command) { - unsigned char buff[128]; - const int bits = (int)td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), command.begin(), command.end()); - CHECK(bits >= 0); - - const auto cell = to_cell(buff, bits); - - vm::init_op_cp0(); +vm::Stack prepare_stack(td::Slice command) { + const auto cell = to_cell(command); vm::DictionaryBase::get_empty_dictionary(); - - class Logger : public td::LogInterface { - public: - void append(td::CSlice slice) override { - res.append(slice.data(), slice.size()); - } - std::string res; - }; - static Logger logger; - logger.res = ""; - td::set_log_fatal_error_callback([](td::CSlice message) { td::default_log_interface->append(logger.res); }); - vm::VmLog log{&logger, td::LogOptions::plain()}; - log.log_options.level = 4; - log.log_options.fix_newlines = true; - log.log_mask |= vm::VmLog::DumpStack; - vm::Stack stack; try { - vm::GasLimits gas_limit(10000, 10000); + vm::GasLimits gas_limit; + int ret = vm::run_vm_code(vm::load_cell_slice_ref(cell), stack, 0 /*flags*/, nullptr /*data*/, vm::VmLog{}, nullptr, + &gas_limit, {}, c7, nullptr, ton::SUPPORTED_VERSION); + CHECK(ret == 0); + } catch (...) { + LOG(FATAL) << "catch unhandled exception"; + } + return stack; +} +runInfo time_run_vm(td::Slice command, td::Ref stack) { + const auto cell = to_cell(command); + vm::DictionaryBase::get_empty_dictionary(); + CHECK(stack.is_unique()); + try { + vm::GasLimits gas_limit; + vm::VmState vm{ + vm::load_cell_slice_ref(cell), ton::SUPPORTED_VERSION, std::move(stack), gas_limit, 0, {}, vm::VmLog{}, {}, c7}; std::clock_t cStart = std::clock(); - int ret = vm::run_vm_code(vm::load_cell_slice_ref(cell), stack, 0 /*flags*/, nullptr /*data*/, - std::move(log) /*VmLog*/, nullptr, &gas_limit); + int ret = ~vm.run(); std::clock_t cEnd = std::clock(); - const auto time = (1000.0 * static_cast(cEnd - cStart) / CLOCKS_PER_SEC) - timingBaseline; - return {time >= 0 ? time : 0, gas_limit.gas_consumed(), ret}; + const auto time = (1000.0 * static_cast(cEnd - cStart) / CLOCKS_PER_SEC); + return {time >= 0 ? time : 0, vm.gas_consumed(), ret}; } catch (...) { LOG(FATAL) << "catch unhandled exception"; return {-1, -1, 1}; } } -runtimeStats averageRuntime(td::Slice command) { - const size_t samples = 5000; +runtimeStats averageRuntime(td::Slice command, const vm::Stack& stack) { + size_t samples = 100000; runInfo total; std::vector values; values.reserve(samples); - for(size_t i=0; i(true, stack)); + const auto value_code = time_run_vm(command, td::Ref(true, stack)); + runInfo value{value_code.runtime - value_empty.runtime, value_code.gasUsage - value_empty.gasUsage, + value_code.vmReturnCode}; values.push_back(value); total += value; + if (t0.elapsed() > 2.0 && i + 1 >= 20) { + samples = i + 1; + values.resize(samples); + break; + } } const auto runtimeMean = total.runtime / static_cast(samples); const auto gasMean = static_cast(total.gasUsage) / static_cast(samples); long double runtimeDiffSum = 0.0; long double gasDiffSum = 0.0; bool errored = false; - for(const auto value : values) { + for (const auto value : values) { const auto runtime = value.runtime - runtimeMean; const auto gasUsage = static_cast(value.gasUsage) - gasMean; runtimeDiffSum += runtime * runtime; gasDiffSum += gasUsage * gasUsage; errored = errored || value.errored(); } - return { - {runtimeMean, sqrt(runtimeDiffSum / static_cast(samples))}, - {gasMean, sqrt(gasDiffSum / static_cast(samples))}, - errored - }; + return {{runtimeMean, sqrtl(runtimeDiffSum / static_cast(samples))}, + {gasMean, sqrtl(gasDiffSum / static_cast(samples))}, + errored}; } runtimeStats timeInstruction(const std::string& setupCode, const std::string& toMeasure) { - const auto setupCodeTime = averageRuntime(setupCode); - const auto totalCodeTime = averageRuntime(setupCode + toMeasure); - return { - {totalCodeTime.runtime.mean - setupCodeTime.runtime.mean, totalCodeTime.runtime.stddev}, - {totalCodeTime.gasUsage.mean - setupCodeTime.gasUsage.mean, totalCodeTime.gasUsage.stddev}, - false - }; + vm::Stack stack = prepare_stack(setupCode); + return averageRuntime(toMeasure, stack); } int main(int argc, char** argv) { - if(argc != 2 && argc != 3) { - std::cerr << - "This utility compares the timing of VM execution against the gas used.\n" - "It can be used to discover opcodes or opcode sequences that consume an " - "inordinate amount of computational resources relative to their gas cost.\n" - "\n" - "The utility expects two command line arguments, each a hex string: \n" - "The TVM code used to set up the stack and VM state followed by the TVM code to measure.\n" - "For example, to test the DIVMODC opcode:\n" - "\t$ " << argv[0] << " 80FF801C A90E 2>/dev/null\n" - "\tOPCODE,runtime mean,runtime stddev,gas mean,gas stddev\n" - "\tA90E,0.0066416,0.00233496,26,0\n" - "\n" - "Usage: " << argv[0] << - " [TVM_SETUP_BYTECODE_HEX] TVM_BYTECODE_HEX" << std::endl << std::endl; + SET_VERBOSITY_LEVEL(verbosity_ERROR); + if (argc != 2 && argc != 3) { + std::cerr << "This utility compares the timing of VM execution against the gas used.\n" + "It can be used to discover opcodes or opcode sequences that consume an " + "inordinate amount of computational resources relative to their gas cost.\n" + "\n" + "The utility expects two command line arguments: \n" + "The TVM code used to set up the stack and VM state followed by the TVM code to measure.\n" + "For example, to test the DIVMODC opcode:\n" + "\t$ " + << argv[0] + << " 80FF801C A90E 2>/dev/null\n" + "\tOPCODE,runtime mean,runtime stddev,gas mean,gas stddev\n" + "\tA90E,0.0066416,0.00233496,26,0\n" + "\n" + "Usage: " + << argv[0] + << " [TVM_SETUP_BYTECODE] TVM_BYTECODE\n" + "\tBYTECODE is either:\n" + "\t1. hex-encoded string (e.g. A90E for DIVMODC)\n" + "\t2. boc: (e.g. boc:te6ccgEBAgEABwABAogBAAJ7)" + << std::endl + << std::endl; return 1; } - std::cout << "OPCODE,runtime mean,runtime stddev,gas mean,gas stddev" << std::endl; - timingBaseline = averageRuntime("").runtime.mean; + std::cout << "OPCODE,runtime mean,runtime stddev,gas mean,gas stddev,error" << std::endl; std::string setup, code; - if(argc == 2) { + if (argc == 2) { setup = ""; code = argv[1]; } else { setup = argv[1]; code = argv[2]; } + vm::init_vm().ensure(); + prepare_c7(); const auto time = timeInstruction(setup, code); - std::cout << code << "," << time.runtime.mean << "," << time.runtime.stddev << "," << - time.gasUsage.mean << "," << time.gasUsage.stddev << std::endl; + std::cout << std::fixed << std::setprecision(9) << code << "," << time.runtime.mean << "," << time.runtime.stddev + << "," << time.gasUsage.mean << "," << time.gasUsage.stddev << "," << (int)time.errored << std::endl; return 0; } diff --git a/utils/proxy-liteserver.cpp b/utils/proxy-liteserver.cpp new file mode 100644 index 00000000..a9baa759 --- /dev/null +++ b/utils/proxy-liteserver.cpp @@ -0,0 +1,473 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#include "td/utils/filesystem.h" +#include "td/actor/actor.h" +#include "td/actor/MultiPromise.h" +#include "td/utils/OptionParser.h" +#include "td/utils/port/path.h" +#include "td/utils/port/signals.h" +#include "td/utils/port/IPAddress.h" +#include "td/utils/Random.h" +#include "td/utils/FileLog.h" +#include "git.h" +#include "auto/tl/ton_api.h" +#include "auto/tl/lite_api.h" +#include "tl-utils/lite-utils.hpp" +#include "auto/tl/ton_api_json.h" +#include "adnl/adnl.h" +#include "lite-client/ext-client.h" + +#if TD_DARWIN || TD_LINUX +#include +#endif +#include "td/utils/overloaded.h" + +#include +#include +#include +#include "td/utils/tl_storers.h" + +using namespace ton; + +class ProxyLiteserver : public td::actor::Actor { + public: + ProxyLiteserver(std::string global_config, std::string db_root, td::uint16 port, PublicKeyHash public_key_hash) + : global_config_(std::move(global_config)) + , db_root_(std::move(db_root)) + , port_(port) + , public_key_hash_(public_key_hash) { + } + + void start_up() override { + LOG_CHECK(!db_root_.empty()) << "db root is not set"; + td::mkdir(db_root_).ensure(); + db_root_ = td::realpath(db_root_).move_as_ok(); + keyring_ = keyring::Keyring::create(db_root_ + "/keyring"); + + if (public_key_hash_.is_zero()) { + id_ = {}; + run(); + } else { + td::actor::send_closure(keyring_, &keyring::Keyring::get_public_key, public_key_hash_, + [SelfId = actor_id(this)](td::Result R) mutable { + if (R.is_error()) { + LOG(FATAL) << "Failed to load public key: " << R.move_as_error(); + } + td::actor::send_closure(SelfId, &ProxyLiteserver::got_public_key, R.move_as_ok()); + }); + } + } + + void got_public_key(PublicKey pub) { + id_ = adnl::AdnlNodeIdFull{pub}; + run(); + } + + void run() { + td::Status S = prepare_local_config(); + if (S.is_error()) { + LOG(FATAL) << "Local config error: " << S; + } + + S = parse_global_config(); + if (S.is_error()) { + LOG(FATAL) << S; + } + + run_clients(); + create_ext_server(); + } + + td::Status prepare_local_config() { + auto r_conf_data = td::read_file(config_file()); + if (r_conf_data.is_ok()) { + auto conf_data = r_conf_data.move_as_ok(); + TRY_RESULT_PREFIX(conf_json, td::json_decode(conf_data.as_slice()), "failed to parse json: "); + TRY_STATUS_PREFIX(ton_api::from_json(*config_, conf_json.get_object()), "json does not fit TL scheme: "); + TRY_RESULT_PREFIX(cfg_port, td::narrow_cast_safe(config_->port_), "invalid port: "); + TRY_RESULT_PREFIX(cfg_id, adnl::AdnlNodeIdFull::create(config_->id_), "invalid id: "); + bool rewrite_config = false; + if (port_ == 0) { + port_ = cfg_port; + } else { + rewrite_config |= (port_ != cfg_port); + } + if (id_.empty()) { + id_ = std::move(cfg_id); + } else { + rewrite_config |= (id_ != cfg_id); + } + if (!rewrite_config) { + return td::Status::OK(); + } + } else { + LOG(WARNING) << "First launch, creating local config"; + } + if (port_ == 0) { + return td::Status::Error("port is not set"); + } + config_->port_ = port_; + if (id_.empty()) { + auto pk = PrivateKey{privkeys::Ed25519::random()}; + id_ = adnl::AdnlNodeIdFull{pk.compute_public_key()}; + td::actor::send_closure(keyring_, &keyring::Keyring::add_key, std::move(pk), false, [](td::Result R) { + if (R.is_error()) { + LOG(FATAL) << "Failed to store private key"; + } + }); + } + config_->id_ = id_.tl(); + + auto s = td::json_encode(td::ToJson(*config_), true); + TRY_STATUS_PREFIX(td::write_file(config_file(), s), "failed to write file: "); + LOG(WARNING) << "Writing config.json"; + return td::Status::OK(); + } + + td::Status parse_global_config() { + TRY_RESULT_PREFIX(global_config_data, td::read_file(global_config_), "Failed to read global config: "); + TRY_RESULT_PREFIX(global_config_json, td::json_decode(global_config_data.as_slice()), + "Failed to parse global config: "); + ton_api::liteclient_config_global gc; + TRY_STATUS_PREFIX(ton_api::from_json(gc, global_config_json.get_object()), "Failed to parse global config: "); + TRY_RESULT_PREFIX(servers, liteclient::LiteServerConfig::parse_global_config(gc), + "Falied to parse liteservers in global config: "); + if (servers.empty()) { + return td::Status::Error("No liteservers in global config"); + } + for (auto& s : servers) { + servers_.emplace_back(); + servers_.back().config = std::move(s); + } + return td::Status::OK(); + } + + void run_clients() { + class Callback : public adnl::AdnlExtClient::Callback { + public: + explicit Callback(td::actor::ActorId id, size_t idx) : id_(std::move(id)), idx_(idx) { + } + void on_ready() override { + td::actor::send_closure(id_, &ProxyLiteserver::on_client_status, idx_, true); + } + void on_stop_ready() override { + td::actor::send_closure(id_, &ProxyLiteserver::on_client_status, idx_, false); + } + + private: + td::actor::ActorId id_; + size_t idx_; + }; + + for (size_t i = 0; i < servers_.size(); ++i) { + Server& server = servers_[i]; + server.client = adnl::AdnlExtClient::create(server.config.adnl_id, server.config.addr, + std::make_unique(actor_id(this), i)); + server.alive = false; + } + } + + void on_client_status(size_t idx, bool ready) { + Server& server = servers_[idx]; + if (server.alive == ready) { + return; + } + server.alive = ready; + LOG(WARNING) << (ready ? "Connected to" : "Disconnected from") << " server #" << idx << " (" + << server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")"; + } + + void create_ext_server() { + adnl_ = adnl::Adnl::create("", keyring_.get()); + td::actor::send_closure(adnl_, &adnl::Adnl::add_id, id_, adnl::AdnlAddressList{}, (td::uint8)255); + + class AdnlCallback : public adnl::Adnl::Callback { + public: + explicit AdnlCallback(td::actor::ActorId id) : id_(id) { + } + + void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(id_, &ProxyLiteserver::receive_query, std::move(data), std::move(promise)); + } + + private: + td::actor::ActorId id_; + }; + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, id_.compute_short_id(), + adnl::Adnl::int_to_bytestring(lite_api::liteServer_query::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_, &adnl::Adnl::create_ext_server, std::vector{id_.compute_short_id()}, + std::vector{port_}, + [SelfId = actor_id(this)](td::Result> R) { + R.ensure(); + td::actor::send_closure(SelfId, &ProxyLiteserver::created_ext_server, R.move_as_ok()); + }); + } + + void created_ext_server(td::actor::ActorOwn s) { + ext_server_ = std::move(s); + LOG(WARNING) << "Started proxy liteserver on port " << port_; + alarm(); + } + + td::Result select_server(const liteclient::QueryInfo& query_info) { + size_t best_idx = servers_.size(); + int cnt = 0; + for (size_t i = 0; i < servers_.size(); ++i) { + Server& server = servers_[i]; + if (!server.alive || !server.config.accepts_query(query_info)) { + continue; + } + ++cnt; + if (td::Random::fast(1, cnt) == 1) { + best_idx = i; + } + } + if (best_idx == servers_.size()) { + return td::Status::Error(PSTRING() << "no liteserver for query " << query_info.to_str()); + } + return best_idx; + } + + void receive_query(td::BufferSlice data, td::Promise promise) { + // Like in ValidatorManagerImpl::run_ext_query + auto F = fetch_tl_object(data, true); + if (F.is_ok()) { + data = std::move(F.move_as_ok()->data_); + } else { + auto G = fetch_tl_prefix(data, true); + if (G.is_error()) { + promise.set_error(G.move_as_error()); + return; + } + } + + tl_object_ptr wait_mc_seqno_obj; + auto E = fetch_tl_prefix(data, true); + if (E.is_ok()) { + wait_mc_seqno_obj = E.move_as_ok(); + } + liteclient::QueryInfo query_info = liteclient::get_query_info(data); + ++ls_stats_[query_info.query_id]; + promise = [promise = std::move(promise), query_info, timer = td::Timer(), + wait_mc_seqno = + (wait_mc_seqno_obj ? wait_mc_seqno_obj->seqno_ : 0)](td::Result R) mutable { + if (R.is_ok()) { + LOG(INFO) << "Query " << query_info.to_str() + << (wait_mc_seqno ? PSTRING() << " (wait seqno " << wait_mc_seqno << ")" : "") + << ": OK, time=" << timer.elapsed() << ", response_size=" << R.ok().size(); + promise.set_value(R.move_as_ok()); + return; + } + LOG(INFO) << "Query " << query_info.to_str() + << (wait_mc_seqno ? PSTRING() << " (wait seqno " << wait_mc_seqno << ")" : "") << ": " << R.error(); + promise.set_value(create_serialize_tl_object( + R.error().code(), "Gateway error: " + R.error().message().str())); + }; + + TRY_RESULT_PROMISE(promise, server_idx, select_server(query_info)); + Server& server = servers_[server_idx]; + LOG(INFO) << "Sending query " << query_info.to_str() + << (wait_mc_seqno_obj ? PSTRING() << " (wait seqno " << wait_mc_seqno_obj->seqno_ << ")" : "") + << ", size=" << data.size() << ", to server #" << server_idx << " (" << server.config.addr.get_ip_str() + << ":" << server.config.addr.get_port() << ")"; + + BlockSeqno wait_mc_seqno = wait_mc_seqno_obj ? wait_mc_seqno_obj->seqno_ : 0; + wait_mc_seqno = std::max(wait_mc_seqno, last_known_masterchain_seqno_); + if (server.last_known_masterchain_seqno < wait_mc_seqno) { + int timeout_ms = wait_mc_seqno_obj ? wait_mc_seqno_obj->timeout_ms_ : 8000; + data = serialize_tl_object(create_tl_object(wait_mc_seqno, timeout_ms), + true, std::move(data)); + } + data = create_serialize_tl_object(std::move(data)); + td::actor::send_closure(server.client, &adnl::AdnlExtClient::send_query, "q", std::move(data), + td::Timestamp::in(8.0), + [SelfId = actor_id(this), promise = std::move(promise), server_idx, + wait_mc_seqno](td::Result R) mutable { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &ProxyLiteserver::process_query_response, + R.ok().clone(), server_idx, wait_mc_seqno); + } + promise.set_result(std::move(R)); + }); + } + + void process_query_response(td::BufferSlice data, size_t server_idx, BlockSeqno wait_mc_seqno) { + auto F = fetch_tl_object(data, true); + if (F.is_error() || F.ok()->get_id() == lite_api::liteServer_error::ID) { + return; + } + BlockSeqno new_seqno = wait_mc_seqno; + lite_api::downcast_call(*F.ok_ref(), td::overloaded( + [&](lite_api::liteServer_masterchainInfo& f) { + new_seqno = std::max(new_seqno, f.last_->seqno_); + }, + [&](lite_api::liteServer_masterchainInfoExt& f) { + new_seqno = std::max(new_seqno, f.last_->seqno_); + }, + [&](lite_api::liteServer_accountState& f) { + if (f.id_->workchain_ == masterchainId) { + new_seqno = std::max(new_seqno, f.id_->seqno_); + } + }, + [&](auto& obj) {})); + servers_[server_idx].last_known_masterchain_seqno = + std::max(servers_[server_idx].last_known_masterchain_seqno, new_seqno); + if (new_seqno > last_known_masterchain_seqno_) { + last_known_masterchain_seqno_ = new_seqno; + LOG(INFO) << "Last known masterchain seqno = " << new_seqno; + } + } + + void alarm() override { + alarm_timestamp() = td::Timestamp::in(60.0); + if (!ls_stats_.empty()) { + td::StringBuilder sb; + sb << "Liteserver stats (1 minute):"; + td::uint32 total = 0; + for (const auto& p : ls_stats_) { + sb << " " << lite_query_name_by_id(p.first) << ":" << p.second; + total += p.second; + } + sb << " TOTAL:" << total; + LOG(WARNING) << sb.as_cslice(); + ls_stats_.clear(); + } + } + + private: + std::string global_config_; + std::string db_root_; + td::uint16 port_; + PublicKeyHash public_key_hash_; + + tl_object_ptr config_ = create_tl_object(); + adnl::AdnlNodeIdFull id_; + + td::actor::ActorOwn keyring_; + td::actor::ActorOwn adnl_; + td::actor::ActorOwn ext_server_; + + struct Server { + liteclient::LiteServerConfig config; + td::actor::ActorOwn client; + bool alive = false; + BlockSeqno last_known_masterchain_seqno = 0; + }; + std::vector servers_; + + std::map ls_stats_; // lite_api ID -> count, 0 for unknown + + BlockSeqno last_known_masterchain_seqno_ = 0; + tl_object_ptr last_masterchain_info_; + + std::string config_file() const { + return db_root_ + "/config.json"; + } +}; + +int main(int argc, char* argv[]) { + SET_VERBOSITY_LEVEL(verbosity_WARNING); + td::set_default_failure_signal_handler().ensure(); + + td::unique_ptr logger_; + SCOPE_EXIT { + td::log_interface = td::default_log_interface; + }; + + std::string global_config, db_root; + td::uint16 port = 0; + PublicKeyHash public_key_hash = PublicKeyHash::zero(); + td::uint32 threads = 4; + + td::OptionParser p; + p.set_description("Proxy liteserver: distributes incoming queries to servers in global config\n"); + p.add_option('v', "verbosity", "set verbosity level", [&](td::Slice arg) { + int v = VERBOSITY_NAME(FATAL) + (td::to_integer(arg)); + SET_VERBOSITY_LEVEL(v); + }); + p.add_option('V', "version", "show build information", [&]() { + std::cout << "proxy-liteserver build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::exit(0); + }); + p.add_option('h', "help", "print help", [&]() { + char b[10240]; + td::StringBuilder sb(td::MutableSlice{b, 10000}); + sb << p; + std::cout << sb.as_cslice().c_str(); + std::exit(2); + }); + p.add_checked_option('p', "port", "liteserver port (required only on first launch)", + [&](td::Slice arg) -> td::Status { + TRY_RESULT_ASSIGN(port, td::to_integer_safe(arg)); + return td::Status::OK(); + }); + p.add_checked_option( + 'A', "adnl-id", + "liteserver public key hash in hex (optional). The corresponding private key is required in /keyring/", + [&](td::Slice arg) -> td::Status { + td::Bits256 value; + if (value.from_hex(arg) != 256) { + return td::Status::Error("invalid adnl-id"); + } + public_key_hash = PublicKeyHash{value}; + return td::Status::OK(); + }); + p.add_option('C', "global-config", "global TON configuration file", + [&](td::Slice arg) { global_config = arg.str(); }); + p.add_option('D', "db", "db root", [&](td::Slice arg) { db_root = arg.str(); }); + p.add_option('d', "daemonize", "set SIGHUP", [&]() { + td::set_signal_handler(td::SignalType::HangUp, [](int) { +#if TD_DARWIN || TD_LINUX + close(0); + setsid(); +#endif + }).ensure(); + }); + p.add_option('l', "logname", "log to file", [&](td::Slice fname) { + logger_ = td::FileLog::create(fname.str()).move_as_ok(); + td::log_interface = logger_.get(); + }); + p.add_checked_option('t', "threads", PSTRING() << "number of threads (default=" << 4 << ")", + [&](td::Slice arg) -> td::Status { + TRY_RESULT_ASSIGN(threads, td::to_integer_safe(arg)); + return td::Status::OK(); + }); + + p.run(argc, argv).ensure(); + td::actor::Scheduler scheduler({threads}); + + scheduler.run_in_context([&] { + td::actor::create_actor("proxy-liteserver", global_config, db_root, port, public_key_hash) + .release(); + }); + while (scheduler.run(1)) { + } +} diff --git a/validator-engine-console/CMakeLists.txt b/validator-engine-console/CMakeLists.txt index 48716960..d87d8a6f 100644 --- a/validator-engine-console/CMakeLists.txt +++ b/validator-engine-console/CMakeLists.txt @@ -1,9 +1,9 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) add_executable (validator-engine-console validator-engine-console.cpp validator-engine-console.h validator-engine-console-query.cpp validator-engine-console-query.h ) -target_link_libraries(validator-engine-console tdutils tdactor adnllite tl_api tl_lite_api tl-lite-utils ton_crypto ton_block terminal git) +target_link_libraries(validator-engine-console tdactor adnllite tl_api tl_lite_api tl-lite-utils ton_crypto terminal git) install(TARGETS validator-engine-console RUNTIME DESTINATION bin) diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 1aee7cc4..d1110019 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -32,6 +32,12 @@ #include "terminal/terminal.h" #include "td/utils/filesystem.h" #include "overlay/overlays.h" +#include "ton/ton-tl.hpp" +#include "td/utils/JsonBuilder.h" +#include "auto/tl/ton_api_json.h" +#include "keys/encryptor.h" +#include "td/utils/port/path.h" +#include "tl/tl_json.h" #include #include @@ -279,6 +285,66 @@ td::Status SignFileQuery::receive(td::BufferSlice data) { return td::Status::OK(); } +td::Status ExportAllPrivateKeysQuery::run() { + TRY_RESULT_ASSIGN(directory_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + client_pk_ = ton::privkeys::Ed25519::random(); + return td::Status::OK(); +} + +td::Status ExportAllPrivateKeysQuery::send() { + auto b = ton::create_serialize_tl_object( + client_pk_.compute_public_key().tl()); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ExportAllPrivateKeysQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + // Private keys are encrypted using client-provided public key to avoid storing them in + // non-secure buffers (not td::SecureString) + TRY_RESULT_PREFIX(decryptor, client_pk_.create_decryptor(), "cannot create decryptor: "); + TRY_RESULT_PREFIX(keys_data, decryptor->decrypt(f->encrypted_data_.as_slice()), "cannot decrypt data: "); + SCOPE_EXIT { + keys_data.as_slice().fill_zero_secure(); + }; + td::Slice slice = keys_data.as_slice(); + if (slice.size() < 32) { + return td::Status::Error("data is too small"); + } + slice.remove_suffix(32); + std::vector private_keys; + while (!slice.empty()) { + if (slice.size() < 4) { + return td::Status::Error("unexpected end of data"); + } + td::uint32 size; + td::MutableSlice{reinterpret_cast(&size), 4}.copy_from(slice.substr(0, 4)); + if (size > slice.size()) { + return td::Status::Error("unexpected end of data"); + } + slice.remove_prefix(4); + TRY_RESULT_PREFIX(private_key, ton::PrivateKey::import(slice.substr(0, size)), "cannot parse private key: "); + if (!private_key.exportable()) { + return td::Status::Error("private key is not exportable"); + } + private_keys.push_back(std::move(private_key)); + slice.remove_prefix(size); + } + + TRY_STATUS_PREFIX(td::mkpath(directory_ + "/"), "cannot create directory " + directory_ + ": "); + td::TerminalIO::out() << "exported " << private_keys.size() << " private keys" << "\n"; + for (const ton::PrivateKey &private_key : private_keys) { + std::string hash_hex = private_key.compute_short_id().bits256_value().to_hex(); + TRY_STATUS_PREFIX(td::write_file(directory_ + "/" + hash_hex, private_key.export_as_slice()), + "failed to write file: "); + td::TerminalIO::out() << "pubkey_hash " << hash_hex << "\n"; + } + td::TerminalIO::out() << "written all files to " << directory_ << "\n"; + return td::Status::OK(); +} + td::Status AddAdnlAddrQuery::run() { TRY_RESULT_ASSIGN(key_hash_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(category_, tokenizer_.get_token()); @@ -839,23 +905,27 @@ td::Status GetOverlaysStatsQuery::receive(td::BufferSlice data) { td::StringBuilder sb; sb << "overlay_id: " << s->overlay_id_ << " adnl_id: " << s->adnl_id_ << " scope: " << s->scope_ << "\n"; sb << " nodes:\n"; - - td::uint32 overlay_t_out_bytes = 0; - td::uint32 overlay_t_out_pckts = 0; - td::uint32 overlay_t_in_bytes = 0; - td::uint32 overlay_t_in_pckts = 0; - + + auto print_traffic = [&](const char *name, const char *indent, + ton::tl_object_ptr &t) { + sb << indent << name << ":\n" + << indent << " out: " << t->t_out_bytes_ << " bytes/sec, " << t->t_out_pckts_ << " pckts/sec\n" + << indent << " in: " << t->t_in_bytes_ << " bytes/sec, " << t->t_in_pckts_ << " pckts/sec\n"; + }; for (auto &n : s->nodes_) { - sb << " adnl_id: " << n->adnl_id_ << " ip_addr: " << n->ip_addr_ << " broadcast_errors: " << n->bdcst_errors_ << " fec_broadcast_errors: " << n->fec_bdcst_errors_ << " last_in_query: " << n->last_in_query_ << " (" << time_to_human(n->last_in_query_) << ")" << " last_out_query: " << n->last_out_query_ << " (" << time_to_human(n->last_out_query_) << ")" << "\n throughput:\n out: " << n->t_out_bytes_ << " bytes/sec, " << n->t_out_pckts_ << " pckts/sec\n in: " << n->t_in_bytes_ << " bytes/sec, " << n->t_in_pckts_ << " pckts/sec\n"; - - overlay_t_out_bytes += n->t_out_bytes_; - overlay_t_out_pckts += n->t_out_pckts_; - - overlay_t_in_bytes += n->t_in_bytes_; - overlay_t_in_pckts += n->t_in_pckts_; + sb << " adnl_id: " << n->adnl_id_ << " ip_addr: " << n->ip_addr_ << " broadcast_errors: " << n->bdcst_errors_ + << " fec_broadcast_errors: " << n->fec_bdcst_errors_ << " last_in_query: " << n->last_in_query_ << " (" + << time_to_human(n->last_in_query_) << ")" + << " last_out_query: " << n->last_out_query_ << " (" << time_to_human(n->last_out_query_) << ")" + << "\n"; + sb << " is_neighbour: " << n->is_neighbour_ << " is_alive: " << n->is_alive_ + << " node_flags: " << n->node_flags_ << "\n"; + print_traffic("throughput", " ", n->traffic_); + print_traffic("throughput (responses only)", " ", n->traffic_responses_); } - sb << " total_throughput:\n out: " << overlay_t_out_bytes << " bytes/sec, " << overlay_t_out_pckts << " pckts/sec\n in: " << overlay_t_in_bytes << " bytes/sec, " << overlay_t_in_pckts << " pckts/sec\n"; - + print_traffic("total_throughput", " ", s->total_traffic_); + print_traffic("total_throughput (responses only)", " ", s->total_traffic_responses_); + sb << " stats:\n"; for (auto &t : s->stats_) { sb << " " << t->key_ << "\t" << t->value_ << "\n"; @@ -881,62 +951,82 @@ td::Status GetOverlaysStatsJsonQuery::receive(td::BufferSlice data) { TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), "received incorrect answer: "); std::ofstream sb(file_name_); - + sb << "[\n"; bool rtail = false; for (auto &s : f->overlays_) { - if(rtail) { + if (rtail) { sb << ",\n"; } else { rtail = true; } - - sb << "{\n \"overlay_id\": \"" << s->overlay_id_ << "\",\n \"adnl_id\": \"" << s->adnl_id_ << "\",\n \"scope\": " << s->scope_ << ",\n"; + + sb << "{\n \"overlay_id\": \"" << s->overlay_id_ << "\",\n \"adnl_id\": \"" << s->adnl_id_ + << "\",\n \"scope\": " << s->scope_ << ",\n"; sb << " \"nodes\": [\n"; - - td::uint32 overlay_t_out_bytes = 0; - td::uint32 overlay_t_out_pckts = 0; - td::uint32 overlay_t_in_bytes = 0; - td::uint32 overlay_t_in_pckts = 0; - + + auto print_traffic = [&](const char *name, + ton::tl_object_ptr &t) { + sb << "\"" << name << "\": { \"out_bytes_sec\": " << t->t_out_bytes_ << ", \"out_pckts_sec\": " << t->t_out_pckts_ + << ", \"in_bytes_sec\": " << t->t_in_bytes_ << ", \"in_pckts_sec\": " << t->t_in_pckts_ << " }"; + }; + bool tail = false; for (auto &n : s->nodes_) { - if(tail) { + if (tail) { sb << ",\n"; } else { tail = true; } - - sb << " {\n \"adnl_id\": \"" << n->adnl_id_ << "\",\n \"ip_addr\": \"" << n->ip_addr_ << "\",\n \"broadcast_errors\": " << n->bdcst_errors_ << ",\n \"fec_broadcast_errors\": " << n->fec_bdcst_errors_ << ",\n \"last_in_query_unix\": " << n->last_in_query_ << ",\n \"last_in_query_human\": \"" << time_to_human(n->last_in_query_) << "\",\n" << " \"last_out_query_unix\": " << n->last_out_query_ << ",\n \"last_out_query_human\": \"" << time_to_human(n->last_out_query_) << "\",\n" << "\n \"throughput\": { \"out_bytes_sec\": " << n->t_out_bytes_ << ", \"out_pckts_sec\": " << n->t_out_pckts_ << ", \"in_bytes_sec\": " << n->t_in_bytes_ << ", \"in_pckts_sec\": " << n->t_in_pckts_ << " }\n }"; - - overlay_t_out_bytes += n->t_out_bytes_; - overlay_t_out_pckts += n->t_out_pckts_; - - overlay_t_in_bytes += n->t_in_bytes_; - overlay_t_in_pckts += n->t_in_pckts_; + + sb << " {\n \"adnl_id\": \"" << n->adnl_id_ << "\",\n \"ip_addr\": \"" << n->ip_addr_ + << "\",\n \"broadcast_errors\": " << n->bdcst_errors_ + << ",\n \"fec_broadcast_errors\": " << n->fec_bdcst_errors_ + << ",\n \"last_in_query_unix\": " << n->last_in_query_ << ",\n \"last_in_query_human\": \"" + << time_to_human(n->last_in_query_) << "\",\n" + << " \"last_out_query_unix\": " << n->last_out_query_ << ",\n \"last_out_query_human\": \"" + << time_to_human(n->last_out_query_) << "\",\n" + << "\n "; + print_traffic("throughput", n->traffic_); + sb << ",\n "; + print_traffic("throughput_responses", n->traffic_responses_); + sb << "\n }"; } - sb << " ],\n"; - - sb << " \"total_throughput\": { \"out_bytes_sec\": " << overlay_t_out_bytes << ", \"out_pckts_sec\": " << overlay_t_out_pckts << ", \"in_bytes_sec\": " << overlay_t_in_bytes << ", \"in_pckts_sec\": " << overlay_t_in_pckts << " },\n"; - + sb << " ],\n "; + + print_traffic("total_throughput", s->total_traffic_); + sb << ",\n "; + print_traffic("total_throughput_responses", s->total_traffic_responses_); + sb << ",\n"; + sb << " \"stats\": {\n"; - + tail = false; for (auto &t : s->stats_) { - if(tail) { + if (tail) { sb << ",\n"; } else { tail = true; } - + sb << " \"" << t->key_ << "\": \"" << t->value_ << "\""; } - sb << "\n }\n"; - sb << "}\n"; + sb << "\n }"; + if (!s->extra_.empty()) { + sb << ",\n \"extra\": "; + for (char c : s->extra_) { + if (c == '\n') { + sb << "\n "; + } else { + sb << c; + } + } + } + sb << "\n}\n"; } sb << "]\n"; sb << std::flush; - + td::TerminalIO::output(std::string("wrote stats to " + file_name_ + "\n")); return td::Status::OK(); } @@ -951,8 +1041,7 @@ td::Status ImportCertificateQuery::receive(td::BufferSlice data) { td::Status SignShardOverlayCertificateQuery::run() { - TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); - TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); TRY_RESULT_ASSIGN(key_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(expire_at_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(max_size_, tokenizer_.get_token()); @@ -962,8 +1051,9 @@ td::Status SignShardOverlayCertificateQuery::run() { } td::Status SignShardOverlayCertificateQuery::send() { - auto b = ton::create_serialize_tl_object - (wc_, shard_, ton::create_tl_object(key_.tl()), expire_at_, max_size_); + auto b = ton::create_serialize_tl_object( + shard_.workchain, shard_.shard, ton::create_tl_object(key_.tl()), + expire_at_, max_size_); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -981,8 +1071,7 @@ td::Status SignShardOverlayCertificateQuery::receive(td::BufferSlice data) { } td::Status ImportShardOverlayCertificateQuery::run() { - TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); - TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(key_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(in_file_, tokenizer_.get_token()); @@ -993,8 +1082,9 @@ td::Status ImportShardOverlayCertificateQuery::send() { TRY_RESULT(data, td::read_file(in_file_)); TRY_RESULT_PREFIX(cert, ton::fetch_tl_object(data.as_slice(), true), "incorrect certificate"); - auto b = ton::create_serialize_tl_object - (wc_, shard_, ton::create_tl_object(key_.tl()), std::move(cert)); + auto b = ton::create_serialize_tl_object( + shard_.workchain, shard_.shard, ton::create_tl_object(key_.tl()), + std::move(cert)); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1005,6 +1095,32 @@ td::Status ImportShardOverlayCertificateQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "successfully sent certificate to overlay manager\n"; return td::Status::OK(); } +td::Status GetActorStatsQuery::run() { + auto r_file_name = tokenizer_.get_token(); + if (r_file_name.is_ok()) { + file_name_ = r_file_name.move_as_ok(); + } + return td::Status::OK(); +} +td::Status GetActorStatsQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetActorStatsQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + if (file_name_.empty()) { + td::TerminalIO::out() << f->data_; + } else { + std::ofstream sb(file_name_); + sb << f->data_; + sb << std::flush; + td::TerminalIO::output(std::string("wrote stats to " + file_name_ + "\n")); + } + return td::Status::OK(); +} td::Status GetPerfTimerStatsJsonQuery::run() { TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); @@ -1055,3 +1171,428 @@ td::Status GetPerfTimerStatsJsonQuery::receive(td::BufferSlice data) { td::TerminalIO::output(std::string("wrote stats to " + file_name_ + "\n")); return td::Status::OK(); } + +td::Status GetShardOutQueueSizeQuery::run() { + TRY_RESULT(shard, tokenizer_.get_token()); + block_id_.workchain = shard.workchain; + block_id_.shard = shard.shard; + TRY_RESULT_ASSIGN(block_id_.seqno, tokenizer_.get_token()); + if (!tokenizer_.endl()) { + TRY_RESULT_ASSIGN(dest_, tokenizer_.get_token()); + } + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status GetShardOutQueueSizeQuery::send() { + auto b = ton::create_serialize_tl_object( + dest_.is_valid() ? 1 : 0, ton::create_tl_block_id_simple(block_id_), dest_.workchain, dest_.shard); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetShardOutQueueSizeQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "Queue_size: " << f->size_ << "\n"; + return td::Status::OK(); +} + +td::Status SetExtMessagesBroadcastDisabledQuery::run() { + TRY_RESULT(x, tokenizer_.get_token()); + if (x < 0 || x > 1) { + return td::Status::Error("value should be 0 or 1"); + } + value = x; + return td::Status::OK(); +} + +td::Status SetExtMessagesBroadcastDisabledQuery::send() { + auto b = ton::create_serialize_tl_object(value); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status SetExtMessagesBroadcastDisabledQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status AddCustomOverlayQuery::run() { + TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status AddCustomOverlayQuery::send() { + TRY_RESULT(data, td::read_file(file_name_)); + TRY_RESULT(json, td::json_decode(data.as_slice())); + auto overlay = ton::create_tl_object(); + TRY_STATUS(ton::ton_api::from_json(*overlay, json.get_object())); + auto b = ton::create_serialize_tl_object(std::move(overlay)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status AddCustomOverlayQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status DelCustomOverlayQuery::run() { + TRY_RESULT_ASSIGN(name_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status DelCustomOverlayQuery::send() { + auto b = ton::create_serialize_tl_object(name_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status DelCustomOverlayQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status ShowCustomOverlaysQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status ShowCustomOverlaysQuery::send() { + auto b = ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ShowCustomOverlaysQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << f->overlays_.size() << " custom overlays:\n\n"; + for (const auto &overlay : f->overlays_) { + td::TerminalIO::out() << "Overlay \"" << overlay->name_ << "\": " << overlay->nodes_.size() << " nodes\n"; + for (const auto &node : overlay->nodes_) { + td::TerminalIO::out() << " " << node->adnl_id_ + << (node->msg_sender_ + ? (PSTRING() << " (msg sender, p=" << node->msg_sender_priority_ << ")") + : "") + << (node->block_sender_ ? " (block sender)" : "") << "\n"; + } + if (!overlay->sender_shards_.empty()) { + td::TerminalIO::out() << "Sender shards:\n"; + for (const auto &shard : overlay->sender_shards_) { + td::TerminalIO::out() << " " << ton::create_shard_id(shard).to_str() << "\n"; + } + } + td::TerminalIO::out() << "\n"; + } + return td::Status::OK(); +} + +td::Status SetStateSerializerEnabledQuery::run() { + TRY_RESULT(value, tokenizer_.get_token()); + if (value != 0 && value != 1) { + return td::Status::Error("expected 0 or 1"); + } + TRY_STATUS(tokenizer_.check_endl()); + enabled_ = value; + return td::Status::OK(); +} + +td::Status SetStateSerializerEnabledQuery::send() { + auto b = ton::create_serialize_tl_object(enabled_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status SetStateSerializerEnabledQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status SetCollatorOptionsJsonQuery::run() { + TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status SetCollatorOptionsJsonQuery::send() { + TRY_RESULT(data, td::read_file(file_name_)); + auto b = + ton::create_serialize_tl_object(data.as_slice().str()); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status SetCollatorOptionsJsonQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status ResetCollatorOptionsQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status ResetCollatorOptionsQuery::send() { + auto b = ton::create_serialize_tl_object("{}"); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ResetCollatorOptionsQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + +td::Status GetCollatorOptionsJsonQuery::run() { + TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status GetCollatorOptionsJsonQuery::send() { + auto b = + ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetCollatorOptionsJsonQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + TRY_STATUS(td::write_file(file_name_, f->data_)); + td::TerminalIO::out() << "saved config to " << file_name_ << "\n"; + return td::Status::OK(); +} + +td::Status GetAdnlStatsJsonQuery::run() { + TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); + if (!tokenizer_.endl()) { + TRY_RESULT(s, tokenizer_.get_token()); + if (s == "all") { + all_ = true; + } else { + return td::Status::Error(PSTRING() << "unexpected token " << s); + } + } + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status GetAdnlStatsJsonQuery::send() { + auto b = + ton::create_serialize_tl_object(all_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetAdnlStatsJsonQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + auto s = td::json_encode(td::ToJson(*f), true); + TRY_STATUS(td::write_file(file_name_, s)); + td::TerminalIO::out() << "saved adnl stats to " << file_name_ << "\n"; + return td::Status::OK(); +} + +td::Status GetAdnlStatsQuery::run() { + if (!tokenizer_.endl()) { + TRY_RESULT(s, tokenizer_.get_token()); + if (s == "all") { + all_ = true; + } else { + return td::Status::Error(PSTRING() << "unexpected token " << s); + } + } + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status GetAdnlStatsQuery::send() { + auto b = + ton::create_serialize_tl_object(all_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetAdnlStatsQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(stats, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::StringBuilder sb; + sb << "================================= ADNL STATS =================================\n"; + bool first = true; + double now = td::Clocks::system(); + for (auto &local_id : stats->local_ids_) { + if (first) { + first = false; + } else { + sb << "\n"; + } + sb << "LOCAL ID " << local_id->short_id_ << "\n"; + if (!local_id->current_decrypt_.empty()) { + std::sort(local_id->current_decrypt_.begin(), local_id->current_decrypt_.end(), + [](const ton::tl_object_ptr &a, + const ton::tl_object_ptr &b) { + return a->packets_ > b->packets_; + }); + td::uint64 total = 0; + for (auto &x : local_id->current_decrypt_) { + total += x->packets_; + } + sb << " Packets in decryptor: total=" << total; + for (auto &x : local_id->current_decrypt_) { + sb << " " << (x->ip_str_.empty() ? "unknown" : x->ip_str_) << "=" << x->packets_; + } + sb << "\n"; + } + auto print_local_id_packets = [&](const std::string &name, + std::vector> &vec) { + if (vec.empty()) { + return; + } + std::sort(vec.begin(), vec.end(), + [](const ton::tl_object_ptr &a, + const ton::tl_object_ptr &b) { + return a->packets_ > b->packets_; + }); + td::uint64 total = 0; + for (auto &x : vec) { + total += x->packets_; + } + sb << " " << name << ": total=" << total; + int cnt = 0; + for (auto &x : vec) { + ++cnt; + if (cnt >= 8) { + sb << " ..."; + break; + } + sb << " " << (x->ip_str_.empty() ? "unknown" : x->ip_str_) << "=" << x->packets_; + } + sb << "\n"; + }; + print_local_id_packets("Decrypted packets (recent)", local_id->packets_recent_->decrypted_packets_); + print_local_id_packets("Dropped packets (recent)", local_id->packets_recent_->dropped_packets_); + print_local_id_packets("Decrypted packets (total)", local_id->packets_total_->decrypted_packets_); + print_local_id_packets("Dropped packets (total)", local_id->packets_total_->dropped_packets_); + sb << " PEERS (" << local_id->peers_.size() << "):\n"; + std::sort(local_id->peers_.begin(), local_id->peers_.end(), + [](const ton::tl_object_ptr &a, + const ton::tl_object_ptr &b) { + return a->packets_recent_->in_bytes_ + a->packets_recent_->out_bytes_ > + b->packets_recent_->in_bytes_ + b->packets_recent_->out_bytes_; + }); + for (auto &peer : local_id->peers_) { + sb << " PEER " << peer->peer_id_ << "\n"; + sb << " Address: " << (peer->ip_str_.empty() ? "unknown" : peer->ip_str_) << "\n"; + sb << " Connection " << (peer->connection_ready_ ? "ready" : "not ready") << ", "; + switch (peer->channel_status_) { + case 0: + sb << "channel: none\n"; + break; + case 1: + sb << "channel: inited\n"; + break; + case 2: + sb << "channel: ready\n"; + break; + default: + sb << "\n"; + } + + auto print_packets = [&](const std::string &name, + const ton::tl_object_ptr &obj) { + if (obj->in_packets_) { + sb << " In (" << name << "): " << obj->in_packets_ << " packets (" + << td::format::as_size(obj->in_bytes_) << "), channel: " << obj->in_packets_channel_ << " packets (" + << td::format::as_size(obj->in_bytes_channel_) << ")\n"; + } + if (obj->out_packets_) { + sb << " Out (" << name << "): " << obj->out_packets_ << " packets (" + << td::format::as_size(obj->out_bytes_) << "), channel: " << obj->out_packets_channel_ << " packets (" + << td::format::as_size(obj->out_bytes_channel_) << ")\n"; + } + if (obj->out_expired_messages_) { + sb << " Out expired (" << name << "): " << obj->out_expired_messages_ << " messages (" + << td::format::as_size(obj->out_expired_bytes_) << ")\n"; + } + }; + print_packets("recent", peer->packets_recent_); + print_packets("total", peer->packets_total_); + + sb << " Last in packet: "; + if (peer->last_in_packet_ts_) { + sb << now - peer->last_in_packet_ts_ << " s ago"; + } else { + sb << "never"; + } + sb << " Last out packet: "; + if (peer->last_out_packet_ts_) { + sb << now - peer->last_out_packet_ts_ << " s ago"; + } else { + sb << "never"; + } + sb << "\n"; + if (peer->out_queue_messages_) { + sb << " Out message queue: " << peer->out_queue_messages_ << " messages (" + << td::format::as_size(peer->out_queue_bytes_) << ")\n"; + } + } + } + sb << "==============================================================================\n"; + td::TerminalIO::out() << sb.as_cslice(); + return td::Status::OK(); +} + +td::Status AddShardQuery::run() { + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status AddShardQuery::send() { + auto b = ton::create_serialize_tl_object(ton::create_tl_shard_id(shard_)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status AddShardQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully added shard\n"; + return td::Status::OK(); +} + +td::Status DelShardQuery::run() { + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status DelShardQuery::send() { + auto b = ton::create_serialize_tl_object(ton::create_tl_shard_id(shard_)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status DelShardQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully removed shard\n"; + return td::Status::OK(); +} \ No newline at end of file diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index ab2141dd..817d70c9 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -33,8 +33,10 @@ #include "td/utils/SharedSlice.h" #include "td/utils/port/IPAddress.h" #include "td/actor/actor.h" +#include "ton/ton-types.h" #include "keys/keys.hpp" +#include "td/utils/base64.h" class ValidatorEngineConsole; @@ -94,27 +96,25 @@ inline td::Result Tokenizer::get_token() { } template <> -inline td::Result Tokenizer::get_token() { - TRY_RESULT(S, get_raw_token()); - TRY_RESULT(F, td::hex_decode(S)); - if (F.size() == 32) { - return ton::PublicKeyHash{td::Slice{F}}; +inline td::Result Tokenizer::get_token() { + TRY_RESULT(word, get_raw_token()); + std::string data; + if (word.size() == 64) { + TRY_RESULT_ASSIGN(data, td::hex_decode(word)); + } else if (word.size() == 44) { + TRY_RESULT_ASSIGN(data, td::base64_decode(word)); } else { return td::Status::Error("cannot parse keyhash: bad length"); } + td::Bits256 v; + v.as_slice().copy_from(data); + return v; } template <> -inline td::Result Tokenizer::get_token() { - TRY_RESULT(S, get_raw_token()); - TRY_RESULT(F, td::hex_decode(S)); - if (F.size() == 32) { - td::Bits256 v; - v.as_slice().copy_from(F); - return v; - } else { - return td::Status::Error("cannot parse keyhash: bad length"); - } +inline td::Result Tokenizer::get_token() { + TRY_RESULT(x, get_token()); + return ton::PublicKeyHash{x}; } template <> @@ -145,6 +145,18 @@ inline td::Result> Tokenizer::get_token_vector() { } } +template <> +inline td::Result Tokenizer::get_token() { + TRY_RESULT(word, get_raw_token()); + auto r_wc = td::to_integer_safe(word); + if (r_wc.is_ok()) { + TRY_RESULT_ASSIGN(word, get_raw_token()); + TRY_RESULT(shard, td::to_integer_safe(word)); + return ton::ShardIdFull{r_wc.move_as_ok(), shard}; + } + return ton::ShardIdFull::parse(word); +} + class QueryRunner { public: virtual ~QueryRunner() = default; @@ -221,10 +233,10 @@ class GetTimeQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice R) override; static std::string get_name() { - return "gettime"; + return "get-time"; } static std::string get_help() { - return "gettime\tshows current server unixtime"; + return "get-time\tshows current server unixtime"; } std::string name() const override { return get_name(); @@ -286,10 +298,10 @@ class NewKeyQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice R) override; static std::string get_name() { - return "newkey"; + return "new-key"; } static std::string get_help() { - return "newkey\tgenerates new key pair on server"; + return "new-key\tgenerates new key pair on server"; } std::string name() const override { return get_name(); @@ -307,10 +319,10 @@ class ImportPrivateKeyFileQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice R) override; static std::string get_name() { - return "importf"; + return "import-f"; } static std::string get_help() { - return "importf \timport private key"; + return "import-f \timport private key"; } std::string name() const override { return get_name(); @@ -329,10 +341,10 @@ class ExportPublicKeyQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice R) override; static std::string get_name() { - return "exportpub"; + return "export-pub"; } static std::string get_help() { - return "exportpub \texports public key by key hash"; + return "export-pub \texports public key by key hash"; } std::string name() const override { return get_name(); @@ -351,10 +363,10 @@ class ExportPublicKeyFileQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice R) override; static std::string get_name() { - return "exportpubf"; + return "export-pubf"; } static std::string get_help() { - return "exportpubf \texports public key by key hash"; + return "export-pub-f \texports public key by key hash"; } std::string name() const override { return get_name(); @@ -397,10 +409,10 @@ class SignFileQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "signf"; + return "sign-f"; } static std::string get_help() { - return "signf \tsigns bytestring with privkey"; + return "sign-f \tsigns bytestring with privkey"; } std::string name() const override { return get_name(); @@ -412,6 +424,30 @@ class SignFileQuery : public Query { std::string out_file_; }; +class ExportAllPrivateKeysQuery : public Query { + public: + ExportAllPrivateKeysQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice R) override; + static std::string get_name() { + return "export-all-private-keys"; + } + static std::string get_help() { + return "export-all-private-keys \texports all private keys from validator engine and stores them to " + ""; + } + std::string name() const override { + return get_name(); + } + + private: + std::string directory_; + ton::PrivateKey client_pk_; +}; + class AddAdnlAddrQuery : public Query { public: AddAdnlAddrQuery(td::actor::ActorId console, Tokenizer tokenizer) @@ -421,10 +457,10 @@ class AddAdnlAddrQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addadnl"; + return "add-adnl"; } static std::string get_help() { - return "addadnl \tuse key as ADNL addr"; + return "add-adnl \tuse key as ADNL addr"; } std::string name() const override { return get_name(); @@ -444,10 +480,10 @@ class AddDhtIdQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "adddht"; + return "add-dht"; } static std::string get_help() { - return "adddht \tcreate DHT node with specified ADNL addr"; + return "add-dht \tcreate DHT node with specified ADNL addr"; } std::string name() const override { return get_name(); @@ -466,10 +502,10 @@ class AddValidatorPermanentKeyQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addpermkey"; + return "add-perm-key"; } static std::string get_help() { - return "addpermkey \tadd validator permanent key"; + return "add-perm-key \tadd validator permanent key"; } std::string name() const override { return get_name(); @@ -490,10 +526,10 @@ class AddValidatorTempKeyQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addtempkey"; + return "add-temp-key"; } static std::string get_help() { - return "addtempkey \tadd validator temp key"; + return "add-temp-key \tadd validator temp key"; } std::string name() const override { return get_name(); @@ -514,10 +550,10 @@ class AddValidatorAdnlAddrQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addvalidatoraddr"; + return "add-validator-addr"; } static std::string get_help() { - return "addvalidatoraddr \tadd validator ADNL addr"; + return "add-validator-addr \tadd validator ADNL addr"; } std::string name() const override { return get_name(); @@ -538,10 +574,10 @@ class ChangeFullNodeAdnlAddrQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "changefullnodeaddr"; + return "change-full-node-addr"; } static std::string get_help() { - return "changefullnodeaddr \tchanges fullnode ADNL address"; + return "change-full-node-addr \tchanges fullnode ADNL address"; } std::string name() const override { return get_name(); @@ -560,10 +596,10 @@ class AddLiteServerQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addliteserver"; + return "add-liteserver"; } static std::string get_help() { - return "addliteserver \tadd liteserver"; + return "add-liteserver \tadd liteserver"; } std::string name() const override { return get_name(); @@ -583,10 +619,10 @@ class DelAdnlAddrQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "deladnl"; + return "del-adnl"; } static std::string get_help() { - return "deladnl \tdel unused ADNL addr"; + return "del-adnl \tdel unused ADNL addr"; } std::string name() const override { return get_name(); @@ -605,10 +641,10 @@ class DelDhtIdQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "deldht"; + return "del-dht"; } static std::string get_help() { - return "deldht \tdel unused DHT node"; + return "del-dht \tdel unused DHT node"; } std::string name() const override { return get_name(); @@ -627,10 +663,10 @@ class DelValidatorPermanentKeyQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "delpermkey"; + return "del-perm-key"; } static std::string get_help() { - return "delpermkey \tforce del unused validator permanent key"; + return "del-perm-key \tforce del unused validator permanent key"; } std::string name() const override { return get_name(); @@ -649,10 +685,10 @@ class DelValidatorTempKeyQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "deltempkey"; + return "del-temp-key"; } static std::string get_help() { - return "deltempkey \tforce del unused validator temp key"; + return "del-temp-key \tforce del unused validator temp key"; } std::string name() const override { return get_name(); @@ -672,10 +708,10 @@ class DelValidatorAdnlAddrQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "delvalidatoraddr"; + return "del-validator-addr"; } static std::string get_help() { - return "delvalidatoraddr \tforce del unused validator ADNL addr"; + return "del-validator-addr \tforce del unused validator ADNL addr"; } std::string name() const override { return get_name(); @@ -695,10 +731,10 @@ class GetConfigQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getconfig"; + return "get-config"; } static std::string get_help() { - return "getconfig\tdownloads current config"; + return "get-config\tdownloads current config"; } std::string name() const override { return get_name(); @@ -716,10 +752,10 @@ class SetVerbosityQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "setverbosity"; + return "set-verbosity"; } static std::string get_help() { - return "setverbosity \tchanges verbosity level"; + return "set-verbosity \tchanges verbosity level"; } std::string name() const override { return get_name(); @@ -738,10 +774,10 @@ class GetStatsQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getstats"; + return "get-stats"; } static std::string get_help() { - return "getstats\tprints stats"; + return "get-stats\tprints stats"; } std::string name() const override { return get_name(); @@ -782,10 +818,10 @@ class AddNetworkAddressQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addaddr"; + return "add-addr"; } static std::string get_help() { - return "addaddr {cats...} {priocats...}\tadds ip address to address list"; + return "add-addr {cats...} {priocats...}\tadds ip address to address list"; } std::string name() const override { return get_name(); @@ -806,10 +842,10 @@ class AddNetworkProxyAddressQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addproxyaddr"; + return "add-proxy-addr"; } static std::string get_help() { - return "addproxyaddr {cats...} {priocats...}\tadds ip address to address list"; + return "add-proxy-addr {cats...} {priocats...}\tadds ip address to address list"; } std::string name() const override { return get_name(); @@ -833,10 +869,10 @@ class CreateElectionBidQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "createelectionbid"; + return "create-election-bid"; } static std::string get_help() { - return "createelectionbid \tcreate election bid"; + return "create-election-bid \tcreate election bid"; } std::string name() const override { return get_name(); @@ -858,10 +894,10 @@ class CreateProposalVoteQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "createproposalvote"; + return "create-proposal-vote"; } static std::string get_help() { - return "createproposalvote \tcreate proposal vote"; + return "create-proposal-vote \tcreate proposal vote"; } std::string name() const override { return get_name(); @@ -881,10 +917,10 @@ class CreateComplaintVoteQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "createcomplaintvote"; + return "create-complaint-vote"; } static std::string get_help() { - return "createcomplaintvote \tcreate proposal vote"; + return "create-complaint-vote \tcreate proposal vote"; } std::string name() const override { return get_name(); @@ -905,10 +941,10 @@ class CheckDhtServersQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "checkdht"; + return "check-dht"; } static std::string get_help() { - return "checkdht \tchecks, which root DHT servers are accessible from this ADNL addr"; + return "check-dht \tchecks, which root DHT servers are accessible from this ADNL addr"; } std::string name() const override { return get_name(); @@ -927,10 +963,10 @@ class GetOverlaysStatsQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getoverlaysstats"; + return "get-overlays-stats"; } static std::string get_help() { - return "getoverlaysstats\tgets stats for all overlays"; + return "get-overlays-stats\tgets stats for all overlays"; } std::string name() const override { return get_name(); @@ -946,10 +982,10 @@ class GetOverlaysStatsJsonQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getoverlaysstatsjson"; + return "get-overlays-stats-json"; } static std::string get_help() { - return "getoverlaysstatsjson \tgets stats for all overlays and writes to json file"; + return "get-overlays-stats-json \tgets stats for all overlays and writes to json file"; } std::string name() const override { return get_name(); @@ -968,10 +1004,11 @@ class SignCertificateQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "signcert"; + return "sign-cert"; } static std::string get_help() { - return "signcert \tsign overlay certificate by key"; + return "sign-cert \tsign overlay certificate by " + " key"; } std::string name() const override { return get_name(); @@ -1004,10 +1041,10 @@ class ImportCertificateQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "importcert"; + return "import-cert"; } static std::string get_help() { - return "importcert \timport overlay certificate for specific key"; + return "import-cert \timport overlay certificate for specific key"; } std::string name() const override { return get_name(); @@ -1029,10 +1066,11 @@ class SignShardOverlayCertificateQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "signshardoverlaycert"; + return "sign-shard-overlay-cert"; } static std::string get_help() { - return "signshardoverlaycert \tsign certificate for in currently active shard overlay"; + return "sign-shard-overlay-cert : \tsign certificate " + "for in currently active shard overlay"; } std::string name() const override { return get_name(); @@ -1040,8 +1078,7 @@ class SignShardOverlayCertificateQuery : public Query { private: - td::int32 wc_; - td::int64 shard_; + ton::ShardIdFull shard_; td::int32 expire_at_; ton::PublicKeyHash key_; td::uint32 max_size_; @@ -1058,10 +1095,11 @@ class ImportShardOverlayCertificateQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "importshardoverlaycert"; + return "import-shard-overlay-cert"; } static std::string get_help() { - return "importshardoverlaycert \timport certificate for in currently active shard overlay"; + return "import-shard-overlay-cert : \timport certificate for in " + "currently active shard overlay"; } std::string name() const override { return get_name(); @@ -1069,12 +1107,33 @@ class ImportShardOverlayCertificateQuery : public Query { private: - td::int32 wc_; - td::int64 shard_; + ton::ShardIdFull shard_; ton::PublicKeyHash key_; std::string in_file_; }; +class GetActorStatsQuery : public Query { + public: + GetActorStatsQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "get-actor-stats"; + } + static std::string get_help() { + return "get-actor-stats []\tget actor stats and print it either in stdout or in "; + } + std::string name() const override { + return get_name(); + } + + private: + std::string file_name_; +}; + class GetPerfTimerStatsJsonQuery : public Query { public: GetPerfTimerStatsJsonQuery(td::actor::ActorId console, Tokenizer tokenizer) @@ -1084,10 +1143,11 @@ class GetPerfTimerStatsJsonQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getperftimerstatsjson"; + return "get-perf-timer-stats-json"; } static std::string get_help() { - return "getperftimerstatsjson \tgets min, average and max event processing time for last 60, 300 and 3600 seconds and writes to json file"; + return "get-perf-timer-stats-json \tgets min, average and max event processing time for last 60, 300 and " + "3600 seconds and writes to json file"; } std::string name() const override { return get_name(); @@ -1096,3 +1156,290 @@ class GetPerfTimerStatsJsonQuery : public Query { private: std::string file_name_; }; + +class GetShardOutQueueSizeQuery : public Query { + public: + GetShardOutQueueSizeQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "get-shard-out-queue-size"; + } + static std::string get_help() { + return "get-shard-out-queue-size : [:]\treturns number of messages in the " + "queue of the given shard. Destination shard is optional."; + } + std::string name() const override { + return get_name(); + } + + private: + ton::BlockId block_id_; + ton::ShardIdFull dest_ = ton::ShardIdFull{ton::workchainInvalid}; +}; + +class SetExtMessagesBroadcastDisabledQuery : public Query { + public: + SetExtMessagesBroadcastDisabledQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "set-ext-messages-broadcast-disabled"; + } + static std::string get_help() { + return "set-ext-messages-broadcast-disabled \tdisable broadcasting and rebroadcasting ext messages; value " + "is 0 or 1."; + } + std::string name() const override { + return get_name(); + } + + private: + bool value; +}; + +class AddCustomOverlayQuery : public Query { + public: + AddCustomOverlayQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "add-custom-overlay"; + } + static std::string get_help() { + return "add-custom-overlay \tadd custom overlay with config from file "; + } + std::string name() const override { + return get_name(); + } + + private: + std::string file_name_; +}; + +class DelCustomOverlayQuery : public Query { + public: + DelCustomOverlayQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "del-custom-overlay"; + } + static std::string get_help() { + return "del-custom-overlay \tdelete custom overlay with name "; + } + std::string name() const override { + return get_name(); + } + + private: + std::string name_; +}; + +class ShowCustomOverlaysQuery : public Query { + public: + ShowCustomOverlaysQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "show-custom-overlays"; + } + static std::string get_help() { + return "show-custom-overlays\tshow all custom overlays"; + } + std::string name() const override { + return get_name(); + } +}; + +class SetStateSerializerEnabledQuery : public Query { + public: + SetStateSerializerEnabledQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "set-state-serializer-enabled"; + } + static std::string get_help() { + return "set-state-serializer-enabled \tdisable or enable persistent state serializer; value is 0 or 1"; + } + std::string name() const override { + return get_name(); + } + + private: + bool enabled_; +}; + +class SetCollatorOptionsJsonQuery : public Query { + public: + SetCollatorOptionsJsonQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "set-collator-options-json"; + } + static std::string get_help() { + return "set-collator-options-json \tset collator options from file "; + } + std::string name() const override { + return get_name(); + } + + private: + std::string file_name_; +}; + +class ResetCollatorOptionsQuery : public Query { + public: + ResetCollatorOptionsQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "reset-collator-options"; + } + static std::string get_help() { + return "reset-collator-options\tset collator options to default values"; + } + std::string name() const override { + return get_name(); + } +}; + +class GetCollatorOptionsJsonQuery : public Query { + public: + GetCollatorOptionsJsonQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "get-collator-options-json"; + } + static std::string get_help() { + return "get-collator-options-json \tsave current collator options to file "; + } + std::string name() const override { + return get_name(); + } + + private: + std::string file_name_; +}; + +class GetAdnlStatsJsonQuery : public Query { + public: + GetAdnlStatsJsonQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "get-adnl-stats-json"; + } + static std::string get_help() { + return "get-adnl-stats-json [all]\tsave adnl stats to . all - returns all peers (default - " + "only peers with traffic in the last 10 minutes)"; + } + std::string name() const override { + return get_name(); + } + + private: + std::string file_name_; + bool all_ = false; +}; + +class GetAdnlStatsQuery : public Query { + public: + GetAdnlStatsQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "get-adnl-stats"; + } + static std::string get_help() { + return "get-adnl-stats [all]\tdisplay adnl stats. all - returns all peers (default - only peers with traffic in " + "the last 10 minutes)"; + } + std::string name() const override { + return get_name(); + } + + private: + std::string file_name_; + bool all_ = false; +}; + +class AddShardQuery : public Query { + public: + AddShardQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "add-shard"; + } + static std::string get_help() { + return "add-shard :\tstart monitoring shard"; + } + std::string name() const override { + return get_name(); + } + + private: + ton::ShardIdFull shard_; +}; + +class DelShardQuery : public Query { + public: + DelShardQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "del-shard"; + } + static std::string get_help() { + return "del-shard :\tstop monitoring shard"; + } + std::string name() const override { + return get_name(); + } + + private: + ton::ShardIdFull shard_; +}; \ No newline at end of file diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 5ce8526b..234cd6a5 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -112,6 +112,7 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); @@ -140,7 +141,21 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { @@ -191,9 +206,8 @@ void ValidatorEngineConsole::show_help(std::string command, td::Promisehelp() << "\n"; } } else { - auto it = query_runners_.find(command); - if (it != query_runners_.end()) { - td::TerminalIO::out() << it->second->help() << "\n"; + if (auto query = get_query(command)) { + td::TerminalIO::out() << query->help() << "\n"; } else { td::TerminalIO::out() << "unknown command '" << command << "'\n"; } @@ -217,10 +231,9 @@ void ValidatorEngineConsole::parse_line(td::BufferSlice data) { } auto name = tokenizer.get_token().move_as_ok(); - auto it = query_runners_.find(name); - if (it != query_runners_.end()) { + if (auto query = get_query(name)) { running_queries_++; - it->second->run(actor_id(this), std::move(tokenizer)); + query->run(actor_id(this), std::move(tokenizer)); } else { td::TerminalIO::out() << "unknown command '" << name << "'\n"; } diff --git a/validator-engine-console/validator-engine-console.h b/validator-engine-console/validator-engine-console.h index 7a384276..c802794e 100644 --- a/validator-engine-console/validator-engine-console.h +++ b/validator-engine-console/validator-engine-console.h @@ -57,9 +57,23 @@ class ValidatorEngineConsole : public td::actor::Actor { std::unique_ptr make_callback(); std::map> query_runners_; + std::map alternate_names_; + static std::string simplify_name(std::string name) { + std::erase_if(name, [](char c) { return c == '-'; }); + return name; + } void add_query_runner(std::unique_ptr runner) { auto name = runner->name(); query_runners_[name] = std::move(runner); + alternate_names_[simplify_name(name)] = name; + } + QueryRunner* get_query(std::string name) { + auto it = alternate_names_.find(name); + if (it != alternate_names_.end()) { + name = it->second; + } + auto it2 = query_runners_.find(name); + return it2 == query_runners_.end() ? nullptr : it2->second.get(); } public: diff --git a/validator-engine/CMakeLists.txt b/validator-engine/CMakeLists.txt index 6c1ea7e2..73949d80 100644 --- a/validator-engine/CMakeLists.txt +++ b/validator-engine/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) @@ -12,7 +12,11 @@ set(VALIDATOR_ENGINE_SOURCE add_executable(validator-engine ${VALIDATOR_ENGINE_SOURCE}) target_link_libraries(validator-engine overlay tdutils tdactor adnl tl_api dht - rldp catchain validatorsession full-node validator ton_validator validator + rldp rldp2 catchain validatorsession full-node validator ton_validator validator fift-lib memprof git ${JEMALLOC_LIBRARIES}) +if (JEMALLOC_FOUND) + target_include_directories(validator-engine PRIVATE ${JEMALLOC_INCLUDE_DIR}) + target_compile_definitions(validator-engine PRIVATE -DTON_USE_JEMALLOC=1) +endif() install(TARGETS validator-engine RUNTIME DESTINATION bin) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index b6eb8b26..2ea04e18 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -38,7 +38,7 @@ #include "common/errorlog.h" -#include "crypto/vm/cp0.h" +#include "crypto/vm/vm.h" #include "crypto/fift/utils.h" #include "td/utils/filesystem.h" @@ -54,6 +54,7 @@ #include "td/utils/Random.h" #include "auto/tl/lite_api.h" +#include "tl/tl_json.h" #include "memprof/memprof.h" @@ -64,19 +65,27 @@ #endif #include #include -#include #include #include #include +#include #include "git.h" +#include "block-auto.h" +#include "block-parse.h" +#include "common/delay.h" +#include "block/precompiled-smc/PrecompiledSmartContract.h" +#include "interfaces/validator-manager.h" +#if TON_USE_JEMALLOC +#include +#endif Config::Config() { out_port = 3278; full_node = ton::PublicKeyHash::zero(); } -Config::Config(ton::ton_api::engine_validator_config &config) { +Config::Config(const ton::ton_api::engine_validator_config &config) { full_node = ton::PublicKeyHash::zero(); out_port = static_cast(config.out_port_); if (!out_port) { @@ -89,7 +98,7 @@ Config::Config(ton::ton_api::engine_validator_config &config) { std::vector categories; std::vector priority_categories; ton::ton_api::downcast_call( - *addr.get(), + *addr, td::overloaded( [&](const ton::ton_api::engine_addr &obj) { in_ip.init_ipv4_port(td::IPAddress::ipv4_to_str(obj.ip_), static_cast(obj.port_)).ensure(); @@ -149,6 +158,15 @@ Config::Config(ton::ton_api::engine_validator_config &config) { config_add_full_node_master(s->port_, ton::PublicKeyHash{s->adnl_}).ensure(); } + if (config.fullnodeconfig_) { + full_node_config = ton::validator::fullnode::FullNodeConfig(config.fullnodeconfig_); + } + if (config.extraconfig_) { + state_serializer_enabled = config.extraconfig_->state_serializer_enabled_; + } else { + state_serializer_enabled = true; + } + for (auto &serv : config.liteservers_) { config_add_lite_server(ton::PublicKeyHash{serv->id_}, serv->port_).ensure(); } @@ -162,6 +180,10 @@ Config::Config(ton::ton_api::engine_validator_config &config) { } } + for (auto &shard : config.shards_to_monitor_) { + config_add_shard(ton::create_shard_id(shard)).ensure(); + } + if (config.gc_) { for (auto &gc : config.gc_->ids_) { config_add_gc(ton::PublicKeyHash{gc}).ensure(); @@ -219,6 +241,17 @@ ton::tl_object_ptr Config::tl() const { ton::create_tl_object(x.first, x.second.tl())); } + ton::tl_object_ptr full_node_config_obj = {}; + if (full_node_config != ton::validator::fullnode::FullNodeConfig()) { + full_node_config_obj = full_node_config.tl(); + } + + ton::tl_object_ptr extra_config_obj = {}; + if (!state_serializer_enabled) { + // Non-default values + extra_config_obj = ton::create_tl_object(state_serializer_enabled); + } + std::vector> liteserver_vec; for (auto &x : liteservers) { liteserver_vec.push_back(ton::create_tl_object(x.second.tl(), x.first)); @@ -234,14 +267,21 @@ ton::tl_object_ptr Config::tl() const { std::move(control_proc_vec))); } + std::vector> shards_vec; + for (auto &shard : shards_to_monitor) { + shards_vec.push_back(ton::create_tl_shard_id(shard)); + } + auto gc_vec = ton::create_tl_object(std::vector{}); for (auto &id : gc) { gc_vec->ids_.push_back(id.tl()); } + return ton::create_tl_object( - out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), full_node.tl(), - std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(liteserver_vec), - std::move(control_vec), std::move(gc_vec)); + out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), + full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), + std::move(full_node_config_obj), std::move(extra_config_obj), std::move(liteserver_vec), std::move(control_vec), + std::move(shards_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, @@ -504,6 +544,32 @@ td::Result Config::config_add_control_process(ton::PublicKeyHash key, td:: } } +td::Result Config::config_add_shard(ton::ShardIdFull shard) { + if (shard.is_masterchain()) { + return td::Status::Error("masterchain is monitored by default"); + } + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << shard.to_str()); + } + if (std::find(shards_to_monitor.begin(), shards_to_monitor.end(), shard) != shards_to_monitor.end()) { + return false; + } + shards_to_monitor.push_back(shard); + return true; +} + +td::Result Config::config_del_shard(ton::ShardIdFull shard) { + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << shard.to_str()); + } + auto it = std::find(shards_to_monitor.begin(), shards_to_monitor.end(), shard); + if (it == shards_to_monitor.end()) { + return false; + } + shards_to_monitor.erase(it); + return true; +} + td::Result Config::config_add_gc(ton::PublicKeyHash key) { return gc.insert(key).second; } @@ -1155,6 +1221,55 @@ class CheckDhtServerStatusQuery : public td::actor::Actor { td::Promise promise_; }; +#if TON_USE_JEMALLOC +class JemallocStatsWriter : public td::actor::Actor { + public: + void start_up() override { + alarm(); + } + + void alarm() override { + alarm_timestamp() = td::Timestamp::in(60.0); + auto r_stats = get_stats(); + if (r_stats.is_error()) { + LOG(WARNING) << "Jemalloc stats error : " << r_stats.move_as_error(); + } else { + auto s = r_stats.move_as_ok(); + LOG(WARNING) << "JEMALLOC_STATS : [ timestamp=" << (ton::UnixTime)td::Clocks::system() + << " allocated=" << s.allocated << " active=" << s.active << " metadata=" << s.metadata + << " resident=" << s.resident << " ]"; + } + } + + private: + struct JemallocStats { + size_t allocated, active, metadata, resident; + }; + + static td::Result get_stats() { + size_t sz = sizeof(size_t); + static size_t epoch = 1; + if (mallctl("epoch", &epoch, &sz, &epoch, sz)) { + return td::Status::Error("Failed to refrash stats"); + } + JemallocStats stats; + if (mallctl("stats.allocated", &stats.allocated, &sz, nullptr, 0)) { + return td::Status::Error("Cannot get stats.allocated"); + } + if (mallctl("stats.active", &stats.active, &sz, nullptr, 0)) { + return td::Status::Error("Cannot get stats.active"); + } + if (mallctl("stats.metadata", &stats.metadata, &sz, nullptr, 0)) { + return td::Status::Error("Cannot get stats.metadata"); + } + if (mallctl("stats.resident", &stats.resident, &sz, nullptr, 0)) { + return td::Status::Error("Cannot get stats.resident"); + } + return stats; + } +}; +#endif + void ValidatorEngine::set_local_config(std::string str) { local_config_ = str; } @@ -1164,8 +1279,23 @@ void ValidatorEngine::set_global_config(std::string str) { void ValidatorEngine::set_db_root(std::string db_root) { db_root_ = db_root; } +void ValidatorEngine::schedule_shutdown(double at) { + td::Timestamp ts = td::Timestamp::at_unix(at); + if (ts.is_in_past()) { + LOG(DEBUG) << "Scheduled shutdown is in past (" << at << ")"; + } else { + LOG(INFO) << "Schedule shutdown for " << at << " (in " << ts.in() << "s)"; + ton::delay_action([]() { + LOG(WARNING) << "Shutting down as scheduled"; + std::_Exit(0); + }, ts); + } +} void ValidatorEngine::start_up() { alarm_timestamp() = td::Timestamp::in(1.0 + td::Random::fast(0, 100) * 0.01); +#if TON_USE_JEMALLOC + td::actor::create_actor("mem-stat").release(); +#endif } void ValidatorEngine::alarm() { @@ -1301,18 +1431,6 @@ td::Status ValidatorEngine::load_global_config() { } validator_options_ = ton::validator::ValidatorManagerOptions::create(zero_state, init_block); - validator_options_.write().set_shard_check_function( - [](ton::ShardIdFull shard, ton::CatchainSeqno cc_seqno, - ton::validator::ValidatorManagerOptions::ShardCheckMode mode) -> bool { - if (mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_monitor) { - return true; - } - CHECK(mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_validate); - return true; - /*ton::ShardIdFull p{ton::basechainId, ((cc_seqno * 1ull % 4) << 62) + 1}; - auto s = ton::shard_prefix(p, 2); - return shard.is_masterchain() || ton::shard_intersects(shard, s);*/ - }); if (state_ttl_ != 0) { validator_options_.write().set_state_ttl(state_ttl_); } @@ -1343,6 +1461,29 @@ td::Status ValidatorEngine::load_global_config() { if (!session_logs_file_.empty()) { validator_options_.write().set_session_logs_file(session_logs_file_); } + if (celldb_in_memory_) { + celldb_compress_depth_ = 0; + } + validator_options_.write().set_celldb_compress_depth(celldb_compress_depth_); + validator_options_.write().set_celldb_in_memory(celldb_in_memory_); + validator_options_.write().set_max_open_archive_files(max_open_archive_files_); + validator_options_.write().set_archive_preload_period(archive_preload_period_); + validator_options_.write().set_disable_rocksdb_stats(disable_rocksdb_stats_); + validator_options_.write().set_nonfinal_ls_queries_enabled(nonfinal_ls_queries_enabled_); + if (celldb_cache_size_) { + validator_options_.write().set_celldb_cache_size(celldb_cache_size_.value()); + } + if (!celldb_cache_size_ || celldb_cache_size_.value() < (30ULL << 30)) { + celldb_direct_io_ = false; + } + validator_options_.write().set_celldb_direct_io(celldb_direct_io_); + validator_options_.write().set_celldb_preload_all(celldb_preload_all_); + if (catchain_max_block_delay_) { + validator_options_.write().set_catchain_max_block_delay(catchain_max_block_delay_.value()); + } + if (catchain_max_block_delay_slow_) { + validator_options_.write().set_catchain_max_block_delay_slow(catchain_max_block_delay_slow_.value()); + } std::vector h; for (auto &x : conf.validator_->hardforks_) { @@ -1362,10 +1503,34 @@ td::Status ValidatorEngine::load_global_config() { h.push_back(b); } validator_options_.write().set_hardforks(std::move(h)); + validator_options_.write().set_fast_state_serializer_enabled(fast_state_serializer_enabled_); + validator_options_.write().set_catchain_broadcast_speed_multiplier(broadcast_speed_multiplier_catchain_); return td::Status::OK(); } +void ValidatorEngine::set_shard_check_function() { + if (!not_all_shards_) { + validator_options_.write().set_shard_check_function([](ton::ShardIdFull shard) -> bool { return true; }); + } else { + std::vector shards = {ton::ShardIdFull(ton::masterchainId)}; + for (const auto& s : config_.shards_to_monitor) { + shards.push_back(s); + } + std::sort(shards.begin(), shards.end()); + shards.erase(std::unique(shards.begin(), shards.end()), shards.end()); + validator_options_.write().set_shard_check_function( + [shards = std::move(shards)](ton::ShardIdFull shard) -> bool { + for (auto s : shards) { + if (shard_intersects(shard, s)) { + return true; + } + } + return false; + }); + } +} + void ValidatorEngine::load_empty_local_config(td::Promise promise) { auto ret_promise = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { @@ -1407,6 +1572,14 @@ void ValidatorEngine::load_empty_local_config(td::Promise promise) { } void ValidatorEngine::load_local_config(td::Promise promise) { + for (ton::ShardIdFull shard : add_shard_cmds_) { + auto R = config_.config_add_shard(shard); + if (R.is_error()) { + LOG(WARNING) << "Cannot add shard " << shard.to_str() << " : " << R.move_as_error(); + } else if (R.ok()) { + LOG(WARNING) << "Adding shard to monitor " << shard.to_str(); + } + } if (local_config_.size() == 0) { load_empty_local_config(std::move(promise)); return; @@ -1574,6 +1747,12 @@ void ValidatorEngine::load_config(td::Promise promise) { config_file_ = db_root_ + "/config.json"; } auto conf_data_R = td::read_file(config_file_); + if (conf_data_R.is_error()) { + conf_data_R = td::read_file(temp_config_file()); + if (conf_data_R.is_ok()) { + td::rename(temp_config_file(), config_file_).ensure(); + } + } if (conf_data_R.is_error()) { auto P = td::PromiseCreator::lambda( [name = local_config_, new_name = config_file_, promise = std::move(promise)](td::Result R) { @@ -1616,18 +1795,30 @@ void ValidatorEngine::load_config(td::Promise promise) { td::actor::send_closure(keyring_, &ton::keyring::Keyring::add_key_short, key.first, get_key_promise(ig)); } + for (ton::ShardIdFull shard : add_shard_cmds_) { + auto R = config_.config_add_shard(shard); + if (R.is_error()) { + LOG(WARNING) << "Cannot add shard " << shard.to_str() << " : " << R.move_as_error(); + } else if (R.ok()) { + LOG(WARNING) << "Adding shard to monitor " << shard.to_str(); + } + } + write_config(ig.get_promise()); } void ValidatorEngine::write_config(td::Promise promise) { auto s = td::json_encode(td::ToJson(*config_.tl().get()), true); - auto S = td::write_file(config_file_, s); - if (S.is_ok()) { - promise.set_value(td::Unit()); - } else { + auto S = td::write_file(temp_config_file(), s); + if (S.is_error()) { + td::unlink(temp_config_file()).ignore(); promise.set_error(std::move(S)); + return; } + td::unlink(config_file_).ignore(); + TRY_STATUS_PROMISE(promise, td::rename(temp_config_file(), config_file_)); + promise.set_value(td::Unit()); } td::Promise ValidatorEngine::get_key_promise(td::MultiPromise::InitGuard &ig) { @@ -1648,6 +1839,7 @@ void ValidatorEngine::got_key(ton::PublicKey key) { } void ValidatorEngine::start() { + set_shard_check_function(); read_config_ = true; start_adnl(); } @@ -1742,6 +1934,9 @@ void ValidatorEngine::started_dht() { void ValidatorEngine::start_rldp() { rldp_ = ton::rldp::Rldp::create(adnl_.get()); + rldp2_ = ton::rldp2::Rldp::create(adnl_.get()); + td::actor::send_closure(rldp_, &ton::rldp::Rldp::set_default_mtu, 2048); + td::actor::send_closure(rldp2_, &ton::rldp2::Rldp::set_default_mtu, 2048); started_rldp(); } @@ -1763,6 +1958,10 @@ void ValidatorEngine::started_overlays() { void ValidatorEngine::start_validator() { validator_options_.write().set_allow_blockchain_init(config_.validators.size() > 0); + validator_options_.write().set_state_serializer_enabled(config_.state_serializer_enabled && + !state_serializer_disabled_flag_); + load_collator_options(); + validator_manager_ = ton::validator::ValidatorManagerFactory::create( validator_options_, db_root_, keyring_.get(), adnl_.get(), rldp_.get(), overlay_manager_.get()); @@ -1802,19 +2001,31 @@ void ValidatorEngine::start_full_node() { }; full_node_client_ = ton::adnl::AdnlExtMultiClient::create(std::move(vec), std::make_unique()); } + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &ValidatorEngine::started_full_node); + }); + ton::validator::fullnode::FullNodeOptions full_node_options{ + .config_ = config_.full_node_config, + .public_broadcast_speed_multiplier_ = broadcast_speed_multiplier_public_, + .private_broadcast_speed_multiplier_ = broadcast_speed_multiplier_private_}; full_node_ = ton::validator::fullnode::FullNode::create( short_id, ton::adnl::AdnlNodeIdShort{config_.full_node}, validator_options_->zero_block_id().file_hash, - keyring_.get(), adnl_.get(), rldp_.get(), + full_node_options, keyring_.get(), adnl_.get(), rldp_.get(), rldp2_.get(), default_dht_node_.is_zero() ? td::actor::ActorId{} : dht_nodes_[default_dht_node_].get(), - overlay_manager_.get(), validator_manager_.get(), full_node_client_.get(), db_root_); + overlay_manager_.get(), validator_manager_.get(), full_node_client_.get(), db_root_, std::move(P)); + for (auto &v : config_.validators) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_permanent_key, v.first, + [](td::Unit) {}); + } + load_custom_overlays_config(); + if (!validator_telemetry_filename_.empty()) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::set_validator_telemetry_filename, + validator_telemetry_filename_); + } + } else { + started_full_node(); } - - for (auto &v : config_.validators) { - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_permanent_key, v.first, - [](td::Unit) {}); - } - - started_full_node(); } void ValidatorEngine::started_full_node() { @@ -2299,6 +2510,130 @@ void ValidatorEngine::try_del_proxy(td::uint32 ip, td::int32 port, std::vector(); + auto data_R = td::read_file(custom_overlays_config_file()); + if (data_R.is_error()) { + return; + } + auto data = data_R.move_as_ok(); + auto json_R = td::json_decode(data.as_slice()); + if (json_R.is_error()) { + LOG(ERROR) << "Failed to parse custom overlays config: " << json_R.move_as_error(); + return; + } + auto json = json_R.move_as_ok(); + auto S = ton::ton_api::from_json(*custom_overlays_config_, json.get_object()); + if (S.is_error()) { + LOG(ERROR) << "Failed to parse custom overlays config: " << S; + return; + } + + for (auto &overlay : custom_overlays_config_->overlays_) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_custom_overlay, + ton::validator::fullnode::CustomOverlayParams::fetch(*overlay), + [](td::Result R) { R.ensure(); }); + } +} + +td::Status ValidatorEngine::write_custom_overlays_config() { + auto s = td::json_encode(td::ToJson(*custom_overlays_config_), true); + TRY_STATUS_PREFIX(td::write_file(custom_overlays_config_file(), s), "failed to write config: "); + return td::Status::OK(); +} + +void ValidatorEngine::add_custom_overlay_to_config( + ton::tl_object_ptr overlay, td::Promise promise) { + custom_overlays_config_->overlays_.push_back(std::move(overlay)); + TRY_STATUS_PROMISE(promise, write_custom_overlays_config()); + promise.set_result(td::Unit()); +} + +void ValidatorEngine::del_custom_overlay_from_config(std::string name, td::Promise promise) { + auto &overlays = custom_overlays_config_->overlays_; + for (size_t i = 0; i < overlays.size(); ++i) { + if (overlays[i]->name_ == name) { + overlays.erase(overlays.begin() + i); + TRY_STATUS_PROMISE(promise, write_custom_overlays_config()); + promise.set_result(td::Unit()); + return; + } + } + promise.set_error(td::Status::Error(PSTRING() << "no overlay \"" << name << "\" in config")); +} + +static td::Result> parse_collator_options(td::MutableSlice json_str) { + td::Ref ref{true}; + ton::validator::CollatorOptions& opts = ref.write(); + + // Set default values (from_json leaves missing fields as is) + ton::ton_api::engine_validator_collatorOptions f; + f.deferring_enabled_ = opts.deferring_enabled; + f.defer_out_queue_size_limit_ = opts.defer_out_queue_size_limit; + f.defer_messages_after_ = opts.defer_messages_after; + f.dispatch_phase_2_max_total_ = opts.dispatch_phase_2_max_total; + f.dispatch_phase_3_max_total_ = opts.dispatch_phase_3_max_total; + f.dispatch_phase_2_max_per_initiator_ = opts.dispatch_phase_2_max_per_initiator; + f.dispatch_phase_3_max_per_initiator_ = + opts.dispatch_phase_3_max_per_initiator ? opts.dispatch_phase_3_max_per_initiator.value() : -1; + + TRY_RESULT_PREFIX(json, td::json_decode(json_str), "failed to parse json: "); + TRY_STATUS_PREFIX(ton::ton_api::from_json(f, json.get_object()), "json does not fit TL scheme: "); + + if (f.defer_messages_after_ <= 0) { + return td::Status::Error("defer_messages_after should be positive"); + } + if (f.defer_out_queue_size_limit_ < 0) { + return td::Status::Error("defer_out_queue_size_limit should be non-negative"); + } + if (f.dispatch_phase_2_max_total_ < 0) { + return td::Status::Error("dispatch_phase_2_max_total should be non-negative"); + } + if (f.dispatch_phase_3_max_total_ < 0) { + return td::Status::Error("dispatch_phase_3_max_total should be non-negative"); + } + if (f.dispatch_phase_2_max_per_initiator_ < 0) { + return td::Status::Error("dispatch_phase_2_max_per_initiator should be non-negative"); + } + + opts.deferring_enabled = f.deferring_enabled_; + opts.defer_messages_after = f.defer_messages_after_; + opts.defer_out_queue_size_limit = f.defer_out_queue_size_limit_; + opts.dispatch_phase_2_max_total = f.dispatch_phase_2_max_total_; + opts.dispatch_phase_3_max_total = f.dispatch_phase_3_max_total_; + opts.dispatch_phase_2_max_per_initiator = f.dispatch_phase_2_max_per_initiator_; + if (f.dispatch_phase_3_max_per_initiator_ >= 0) { + opts.dispatch_phase_3_max_per_initiator = f.dispatch_phase_3_max_per_initiator_; + } else { + opts.dispatch_phase_3_max_per_initiator = {}; + } + for (const std::string& s : f.whitelist_) { + TRY_RESULT(addr, block::StdAddress::parse(s)); + opts.whitelist.emplace(addr.workchain, addr.addr); + } + for (const std::string& s : f.prioritylist_) { + TRY_RESULT(addr, block::StdAddress::parse(s)); + opts.prioritylist.emplace(addr.workchain, addr.addr); + } + + return ref; +} + +void ValidatorEngine::load_collator_options() { + auto r_data = td::read_file(collator_options_file()); + if (r_data.is_error()) { + return; + } + td::BufferSlice data = r_data.move_as_ok(); + auto r_collator_options = parse_collator_options(data.as_slice()); + if (r_collator_options.is_error()) { + LOG(ERROR) << "Failed to read collator options from file: " << r_collator_options.move_as_error(); + return; + } + validator_options_.write().set_collator_options(r_collator_options.move_as_ok()); +} + void ValidatorEngine::check_key(ton::PublicKeyHash id, td::Promise promise) { if (keys_.count(id) == 1) { promise.set_value(td::Unit()); @@ -2993,6 +3328,70 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_sign &que std::move(query.data_), std::move(P)); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_exportAllPrivateKeys &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_unsafe)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (keyring_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started keyring"))); + return; + } + + ton::PublicKey client_pubkey = ton::PublicKey{query.encryption_key_}; + if (!client_pubkey.is_ed25519()) { + promise.set_value( + create_control_query_error(td::Status::Error(ton::ErrorCode::protoviolation, "encryption key is not Ed25519"))); + return; + } + + td::actor::send_closure( + keyring_, &ton::keyring::Keyring::export_all_private_keys, + [promise = std::move(promise), + client_pubkey = std::move(client_pubkey)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + // Private keys are encrypted using client-provided public key to avoid storing them in + // non-secure buffers (not td::SecureString) + std::vector serialized_keys; + size_t data_size = 32; + for (const ton::PrivateKey &key : R.ok()) { + serialized_keys.push_back(key.export_as_slice()); + data_size += serialized_keys.back().size() + 4; + } + td::SecureString data{data_size}; + td::MutableSlice slice = data.as_mutable_slice(); + for (const td::SecureString &s : serialized_keys) { + td::uint32 size = td::narrow_cast_safe(s.size()).move_as_ok(); + CHECK(slice.size() >= size + 4); + slice.copy_from(td::Slice{reinterpret_cast(&size), 4}); + slice.remove_prefix(4); + slice.copy_from(s.as_slice()); + slice.remove_prefix(s.size()); + } + CHECK(slice.size() == 32); + td::Random::secure_bytes(slice); + + auto r_encryptor = client_pubkey.create_encryptor(); + if (r_encryptor.is_error()) { + promise.set_value(create_control_query_error(r_encryptor.move_as_error_prefix("cannot create encryptor: "))); + return; + } + auto encryptor = r_encryptor.move_as_ok(); + auto r_encrypted = encryptor->encrypt(data.as_slice()); + if (r_encryptor.is_error()) { + promise.set_value(create_control_query_error(r_encrypted.move_as_error_prefix("cannot encrypt data: "))); + return; + } + promise.set_value(ton::create_serialize_tl_object( + r_encrypted.move_as_ok())); + }); +} + void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setVerbosity &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_default)) { @@ -3282,6 +3681,31 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getOverla }); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getActorTextStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + + if (validator_manager_.empty()) { + promise.set_value( + create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "validator manager not started"))); + return; + } + + auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + auto r = R.move_as_ok(); + promise.set_value(ton::create_serialize_tl_object(std::move(r))); + } + }); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::prepare_actor_stats, + std::move(P)); +} + void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getPerfTimerStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_default)) { @@ -3333,6 +3757,378 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getPerfTi td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::prepare_perf_timer_stats, std::move(P)); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getShardOutQueueSize &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + + if (validator_manager_.empty()) { + promise.set_value( + create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "validator manager not started"))); + return; + } + + ton::BlockId block_id = ton::create_block_id_simple(query.block_id_); + if (!block_id.is_valid_ext()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "invalid block id"))); + return; + } + td::optional dest; + if (query.flags_ & 1) { + dest = ton::ShardIdFull{query.dest_wc_, (ton::ShardId)query.dest_shard_}; + if (!dest.value().is_valid_ext()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "invalid shard"))); + return; + } + } + + td::actor::send_closure( + validator_manager_, &ton::validator::ValidatorManagerInterface::get_block_by_seqno_from_db, + ton::AccountIdPrefixFull{block_id.workchain, block_id.shard}, block_id.seqno, + [=, promise = std::move(promise), + manager = validator_manager_.get()](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + auto handle = R.move_as_ok(); + if (handle->id().id != block_id) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "no such block"))); + return; + } + if (!dest) { + td::actor::send_closure( + manager, &ton::validator::ValidatorManagerInterface::get_out_msg_queue_size, handle->id(), + [promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error_prefix("failed to get queue size: "))); + } else { + promise.set_value(ton::create_serialize_tl_object( + R.move_as_ok())); + } + }); + return; + } + td::actor::send_closure( + manager, &ton::validator::ValidatorManagerInterface::get_shard_state_from_db, handle, + [=, promise = std::move(promise)](td::Result> R) mutable { + auto res = [&]() -> td::Result { + TRY_RESULT(state, std::move(R)); + TRY_RESULT(outq_descr, state->message_queue()); + block::gen::OutMsgQueueInfo::Record qinfo; + if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { + return td::Status::Error(ton::ErrorCode::error, "invalid message queue"); + } + auto queue = std::make_unique(qinfo.out_queue->prefetch_ref(0), 352, + block::tlb::aug_OutMsgQueue); + if (dest) { + td::BitArray<96> prefix; + td::BitPtr ptr = prefix.bits(); + ptr.store_int(dest.value().workchain, 32); + ptr.advance(32); + ptr.store_uint(dest.value().shard, 64); + if (!queue->cut_prefix_subdict(prefix.bits(), 32 + dest.value().pfx_len())) { + return td::Status::Error(ton::ErrorCode::error, "invalid message queue"); + } + } + int size = 0; + queue->check_for_each([&](td::Ref, td::ConstBitPtr, int) -> bool { + ++size; + return true; + }); + return ton::create_serialize_tl_object(size); + }(); + if (res.is_error()) { + promise.set_value(create_control_query_error(res.move_as_error())); + } else { + promise.set_value(res.move_as_ok()); + } + }); + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setExtMessagesBroadcastDisabled &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + if (config_.full_node_config.ext_messages_broadcast_disabled_ == query.disabled_) { + promise.set_value(ton::create_serialize_tl_object()); + return; + } + config_.full_node_config.ext_messages_broadcast_disabled_ = query.disabled_; + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::set_config, config_.full_node_config); + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value(ton::create_serialize_tl_object()); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCustomOverlay &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_ || full_node_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + auto &overlay = query.overlay_; + std::vector nodes; + std::map senders; + for (const auto &node : overlay->nodes_) { + nodes.emplace_back(node->adnl_id_); + if (node->msg_sender_) { + senders[ton::adnl::AdnlNodeIdShort{node->adnl_id_}] = node->msg_sender_priority_; + } + } + auto params = ton::validator::fullnode::CustomOverlayParams::fetch(*query.overlay_); + td::actor::send_closure( + full_node_, &ton::validator::fullnode::FullNode::add_custom_overlay, std::move(params), + [SelfId = actor_id(this), overlay = std::move(query.overlay_), + promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + td::actor::send_closure( + SelfId, &ValidatorEngine::add_custom_overlay_to_config, std::move(overlay), + [promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + promise.set_value(ton::create_serialize_tl_object()); + }); + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCustomOverlay &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_ || full_node_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + td::actor::send_closure( + full_node_, &ton::validator::fullnode::FullNode::del_custom_overlay, query.name_, + [SelfId = actor_id(this), name = query.name_, promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + td::actor::send_closure( + SelfId, &ValidatorEngine::del_custom_overlay_from_config, std::move(name), + [promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + promise.set_value(ton::create_serialize_tl_object()); + }); + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCustomOverlays &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_ || full_node_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + promise.set_value( + ton::serialize_tl_object(custom_overlays_config_, true)); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setStateSerializerEnabled &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + if (query.enabled_ == validator_options_->get_state_serializer_enabled()) { + promise.set_value(ton::create_serialize_tl_object()); + return; + } + validator_options_.write().set_state_serializer_enabled(query.enabled_ && !state_serializer_disabled_flag_); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + config_.state_serializer_enabled = query.enabled_; + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value(ton::create_serialize_tl_object()); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setCollatorOptionsJson &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + auto r_collator_options = parse_collator_options(query.json_); + if (r_collator_options.is_error()) { + promise.set_value(create_control_query_error(r_collator_options.move_as_error_prefix("failed to parse json: "))); + return; + } + auto S = td::write_file(collator_options_file(), query.json_); + if (S.is_error()) { + promise.set_value(create_control_query_error(r_collator_options.move_as_error_prefix("failed to write file: "))); + return; + } + validator_options_.write().set_collator_options(r_collator_options.move_as_ok()); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + promise.set_value(ton::create_serialize_tl_object()); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getCollatorOptionsJson &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + auto r_data = td::read_file(collator_options_file()); + if (r_data.is_error()) { + promise.set_value(ton::create_serialize_tl_object("{}")); + } else { + promise.set_value( + ton::create_serialize_tl_object(r_data.ok().as_slice().str())); + } +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getAdnlStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (adnl_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + td::actor::send_closure( + adnl_, &ton::adnl::Adnl::get_stats, query.all_, + [promise = std::move(promise)](td::Result> R) mutable { + if (R.is_ok()) { + promise.set_value(ton::serialize_tl_object(R.move_as_ok(), true)); + } else { + promise.set_value( + create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "failed to get adnl stats"))); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addShard &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + auto shard = ton::create_shard_id(query.shard_); + auto R = config_.config_add_shard(shard); + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + set_shard_check_function(); + if (!validator_manager_.empty()) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + } + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delShard &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + auto shard = ton::create_shard_id(query.shard_); + auto R = config_.config_del_shard(shard); + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + if (!R.move_as_ok()) { + promise.set_value(create_control_query_error(td::Status::Error("No such shard"))); + return; + } + set_shard_check_function(); + if (!validator_manager_.empty()) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + } + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { @@ -3366,7 +4162,7 @@ void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNode } auto f = F.move_as_ok(); - ton::ton_api::downcast_call(*f.get(), [&](auto &obj) { + ton::ton_api::downcast_call(*f, [&](auto &obj) { run_control_query(obj, std::move(data), src.pubkey_hash(), it->second, std::move(promise)); }); } @@ -3432,7 +4228,7 @@ void need_scheduler_status(int sig) { need_scheduler_status_flag.store(true); } -void dump_memory_stats() { +void dump_memprof_stats() { if (!is_memprof_on()) { return; } @@ -3457,8 +4253,20 @@ void dump_memory_stats() { LOG(WARNING) << td::tag("fast_backtrace_success_rate", get_fast_backtrace_success_rate()); } +void dump_jemalloc_prof() { +#if TON_USE_JEMALLOC + const char *filename = "/tmp/validator-jemalloc.dump"; + if (mallctl("prof.dump", nullptr, nullptr, &filename, sizeof(const char *)) == 0) { + LOG(ERROR) << "Written jemalloc dump to " << filename; + } else { + LOG(ERROR) << "Failed to write jemalloc dump to " << filename; + } +#endif +} + void dump_stats() { - dump_memory_stats(); + dump_memprof_stats(); + dump_jemalloc_prof(); LOG(WARNING) << td::NamedThreadSafeCounter::get_default(); } @@ -3530,34 +4338,61 @@ int main(int argc, char *argv[]) { logger_ = td::TsFileLog::create(fname.str()).move_as_ok(); td::log_interface = logger_.get(); }); - p.add_option('s', "state-ttl", "state will be gc'd after this time (in seconds) default=3600", [&](td::Slice fname) { - auto v = td::to_double(fname); - acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_state_ttl, v); }); - }); - p.add_option('m', "mempool-num", "Maximal number of mempool external message", [&](td::Slice fname) { + p.add_checked_option('s', "state-ttl", "state will be gc'd after this time (in seconds) default=86400", + [&](td::Slice fname) { + auto v = td::to_double(fname); + if (v <= 0) { + return td::Status::Error("state-ttl should be positive"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_state_ttl, v); }); + return td::Status::OK(); + }); + p.add_checked_option('m', "mempool-num", "Maximal number of mempool external message", [&](td::Slice fname) { auto v = td::to_double(fname); + if (v < 0) { + return td::Status::Error("mempool-num should be non-negative"); + } acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_mempool_num, v); }); + return td::Status::OK(); }); - p.add_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=7*86400", - [&](td::Slice fname) { - auto v = td::to_double(fname); - acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_block_ttl, v); }); - }); - p.add_option('A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=365*86400", - [&](td::Slice fname) { - auto v = td::to_double(fname); - acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_archive_ttl, v); }); - }); - p.add_option('K', "key-proof-ttl", "key blocks will be deleted after this time (in seconds) default=365*86400*10", - [&](td::Slice fname) { - auto v = td::to_double(fname); - acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_key_proof_ttl, v); }); - }); - p.add_option('S', "sync-before", "in initial sync download all blocks for last given seconds default=3600", - [&](td::Slice fname) { - auto v = td::to_double(fname); - acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_sync_ttl, v); }); - }); + p.add_checked_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=86400", + [&](td::Slice fname) { + auto v = td::to_double(fname); + if (v <= 0) { + return td::Status::Error("block-ttl should be positive"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_block_ttl, v); }); + return td::Status::OK(); + }); + p.add_checked_option( + 'A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=7*86400", + [&](td::Slice fname) { + auto v = td::to_double(fname); + if (v <= 0) { + return td::Status::Error("archive-ttl should be positive"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_archive_ttl, v); }); + return td::Status::OK(); + }); + p.add_checked_option( + 'K', "key-proof-ttl", "key blocks will be deleted after this time (in seconds) default=365*86400*10", + [&](td::Slice fname) { + auto v = td::to_double(fname); + if (v <= 0) { + return td::Status::Error("key-proof-ttl should be positive"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_key_proof_ttl, v); }); + return td::Status::OK(); + }); + p.add_checked_option('S', "sync-before", "in initial sync download all blocks for last given seconds default=3600", + [&](td::Slice fname) { + auto v = td::to_double(fname); + if (v <= 0) { + return td::Status::Error("sync-before should be positive"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_sync_ttl, v); }); + return td::Status::OK(); + }); p.add_option('T', "truncate-db", "truncate db (with specified seqno as new top masterchain block seqno)", [&](td::Slice fname) { auto v = td::to_integer(fname); @@ -3586,22 +4421,188 @@ int main(int argc, char *argv[]) { }); return td::Status::OK(); }); + p.add_option('M', "not-all-shards", "monitor only a necessary set of shards instead of all", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_not_all_shards); }); + }); + p.add_checked_option( + '\0', "add-shard", "add shard to monitor (same as addshard in validator console), format: 0:8000000000000000", + [&](td::Slice arg) -> td::Status { + std::string str = arg.str(); + int wc; + unsigned long long shard; + if (sscanf(str.c_str(), "%d:%016llx", &wc, &shard) != 2) { + return td::Status::Error(PSTRING() << "invalid shard " << str); + } + acts.push_back([=, &x]() { + td::actor::send_closure(x, &ValidatorEngine::add_shard_cmd, ton::ShardIdFull{wc, (ton::ShardId)shard}); + }); + return td::Status::OK(); + }); td::uint32 threads = 7; p.add_checked_option( - 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice fname) { + 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice arg) { td::int32 v; try { - v = std::stoi(fname.str()); + v = std::stoi(arg.str()); } catch (...) { return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: not a number"); } - if (v < 1 || v > 256) { - return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: should be in range [1..256]"); + if (v <= 0) { + return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: should be > 0"); + } + if (v > 127) { + LOG(WARNING) << "`--threads " << v << "` is too big, effective value will be 127"; + v = 127; } threads = v; return td::Status::OK(); }); p.add_checked_option('u', "user", "change user", [&](td::Slice user) { return td::change_user(user.str()); }); + p.add_checked_option('\0', "shutdown-at", "stop validator at the given time (unix timestamp)", [&](td::Slice arg) { + TRY_RESULT(at, td::to_integer_safe(arg)); + acts.push_back([&x, at]() { td::actor::send_closure(x, &ValidatorEngine::schedule_shutdown, (double)at); }); + return td::Status::OK(); + }); + p.add_checked_option('\0', "celldb-compress-depth", + "optimize celldb by storing cells of depth X with whole subtrees (experimental, default: 0)", + [&](td::Slice arg) { + TRY_RESULT(value, td::to_integer_safe(arg)); + acts.push_back([&x, value]() { + td::actor::send_closure(x, &ValidatorEngine::set_celldb_compress_depth, value); + }); + return td::Status::OK(); + }); + p.add_checked_option( + '\0', "max-archive-fd", + "limit for a number of open file descriptirs in archive manager. 0 is unlimited (default)", + [&](td::Slice s) -> td::Status { + TRY_RESULT(v, td::to_integer_safe(s)); + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_open_archive_files, v); }); + return td::Status::OK(); + }); + p.add_checked_option( + '\0', "archive-preload-period", "open archive slices for the past X second on startup (default: 0)", + [&](td::Slice s) -> td::Status { + auto v = td::to_double(s); + if (v < 0) { + return td::Status::Error("sync-before should be non-negative"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_archive_preload_period, v); }); + return td::Status::OK(); + }); + p.add_option('\0', "enable-precompiled-smc", + "enable exectuion of precompiled contracts (experimental, disabled by default)", + []() { block::precompiled::set_precompiled_execution_enabled(true); }); + p.add_option('\0', "disable-rocksdb-stats", "disable gathering rocksdb statistics (enabled by default)", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_disable_rocksdb_stats, true); }); + }); + p.add_option('\0', "nonfinal-ls", "enable special LS queries to non-finalized blocks", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_nonfinal_ls_queries_enabled); }); + }); + p.add_checked_option( + '\0', "celldb-cache-size", "block cache size for RocksDb in CellDb, in bytes (default: 1G)", + [&](td::Slice s) -> td::Status { + TRY_RESULT(v, td::to_integer_safe(s)); + if (v == 0) { + return td::Status::Error("celldb-cache-size should be positive"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_cache_size, v); }); + return td::Status::OK(); + }); + p.add_option('\0', "celldb-direct-io", + "enable direct I/O mode for RocksDb in CellDb (doesn't apply when celldb cache is < 30G)", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io, true); }); + }); + p.add_option('\0', "celldb-preload-all", + "preload all cells from CellDb on startup (recommended to use with big enough celldb-cache-size and " + "celldb-direct-io)", + [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_preload_all, true); }); + }); + + p.add_option( + '\0', "celldb-in-memory", + "store all cells in-memory, much faster but requires a lot of RAM. RocksDb is still used as persistent storage", + [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_in_memory, true); }); + }); + p.add_checked_option( + '\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.4)", + [&](td::Slice s) -> td::Status { + auto v = td::to_double(s); + if (v < 0) { + return td::Status::Error("catchain-max-block-delay should be non-negative"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_catchain_max_block_delay, v); }); + return td::Status::OK(); + }); + p.add_checked_option( + '\0', "catchain-max-block-delay-slow", "max extended catchain block delay (for too long rounds), (default: 1.0)", + [&](td::Slice s) -> td::Status { + auto v = td::to_double(s); + if (v < 0) { + return td::Status::Error("catchain-max-block-delay-slow should be non-negative"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_catchain_max_block_delay_slow, v); }); + return td::Status::OK(); + }); + p.add_option( + '\0', "fast-state-serializer", + "faster persistent state serializer, but requires more RAM", + [&]() { + acts.push_back( + [&x]() { td::actor::send_closure(x, &ValidatorEngine::set_fast_state_serializer_enabled, true); }); + }); + p.add_option( + '\0', "collect-validator-telemetry", + "store validator telemetry from private block overlay to a given file (json format)", + [&](td::Slice s) { + acts.push_back( + [&x, s = s.str()]() { + td::actor::send_closure(x, &ValidatorEngine::set_validator_telemetry_filename, s); + }); + }); + p.add_option( + '\0', "disable-state-serializer", + "disable persistent state serializer (similar to set-state-serializer-enabled 0 in validator console)", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_state_serializer_disabled_flag); }); + }); + p.add_checked_option( + '\0', "broadcast-speed-catchain", + "multiplier for broadcast speed in catchain overlays (experimental, default is 1.0, which is ~300 KB/s)", + [&](td::Slice s) -> td::Status { + auto v = td::to_double(s); + if (v <= 0.0) { + return td::Status::Error("broadcast-speed-catchain should be positive"); + } + acts.push_back( + [&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_broadcast_speed_multiplier_catchain, v); }); + return td::Status::OK(); + }); + p.add_checked_option( + '\0', "broadcast-speed-public", + "multiplier for broadcast speed in public shard overlays (experimental, default is 1.0, which is ~300 KB/s)", + [&](td::Slice s) -> td::Status { + auto v = td::to_double(s); + if (v <= 0.0) { + return td::Status::Error("broadcast-speed-public should be positive"); + } + acts.push_back( + [&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_broadcast_speed_multiplier_public, v); }); + return td::Status::OK(); + }); + p.add_checked_option( + '\0', "broadcast-speed-private", + "multiplier for broadcast speed in private block overlays (experimental, default is 1.0, which is ~300 KB/s)", + [&](td::Slice s) -> td::Status { + auto v = td::to_double(s); + if (v <= 0.0) { + return td::Status::Error("broadcast-speed-private should be positive"); + } + acts.push_back( + [&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_broadcast_speed_multiplier_private, v); }); + return td::Status::OK(); + }); auto S = p.run(argc, argv); if (S.is_error()) { LOG(ERROR) << "failed to parse options: " << S.move_as_error(); @@ -3615,7 +4616,7 @@ int main(int argc, char *argv[]) { td::actor::Scheduler scheduler({threads}); scheduler.run_in_context([&] { - CHECK(vm::init_op_cp0()); + vm::init_vm().ensure(); x = td::actor::create_actor("validator-engine"); for (auto &act : acts) { act(); @@ -3629,7 +4630,9 @@ int main(int argc, char *argv[]) { } if (need_scheduler_status_flag.exchange(false)) { LOG(ERROR) << "DUMPING SCHEDULER STATISTICS"; - scheduler.get_debug().dump(); + td::StringBuilder sb; + scheduler.get_debug().dump(sb); + LOG(ERROR) << "GOT SCHEDULER STATISTICS\n" << sb.as_cslice(); } if (rotate_logs_flags.exchange(false)) { if (td::log_interface) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 7284a5be..e0dc91f1 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -30,6 +30,7 @@ #include "adnl/adnl.h" #include "auto/tl/ton_api.h" #include "rldp/rldp.h" +#include "rldp2/rldp.h" #include "dht/dht.h" #include "validator/manager.h" #include "validator/validator.h" @@ -85,8 +86,12 @@ struct Config { std::vector full_node_slaves; std::map full_node_masters; std::map liteservers; + ton::validator::fullnode::FullNodeConfig full_node_config; std::map controls; std::set gc; + std::vector shards_to_monitor; + + bool state_serializer_enabled = true; void decref(ton::PublicKeyHash key); void incref(ton::PublicKeyHash key) { @@ -111,6 +116,8 @@ struct Config { td::Result config_add_control_interface(ton::PublicKeyHash key, td::int32 port); td::Result config_add_control_process(ton::PublicKeyHash key, td::int32 port, ton::PublicKeyHash id, td::uint32 permissions); + td::Result config_add_shard(ton::ShardIdFull shard); + td::Result config_del_shard(ton::ShardIdFull shard); td::Result config_add_gc(ton::PublicKeyHash key); td::Result config_del_network_addr(td::IPAddress addr, std::vector cats, std::vector prio_cats); @@ -128,7 +135,7 @@ struct Config { ton::tl_object_ptr tl() const; Config(); - Config(ton::ton_api::engine_validator_config &config); + Config(const ton::ton_api::engine_validator_config &config); }; class ValidatorEngine : public td::actor::Actor { @@ -137,6 +144,7 @@ class ValidatorEngine : public td::actor::Actor { td::actor::ActorOwn adnl_network_manager_; td::actor::ActorOwn adnl_; td::actor::ActorOwn rldp_; + td::actor::ActorOwn rldp2_; std::map> dht_nodes_; ton::PublicKeyHash default_dht_node_ = ton::PublicKeyHash::zero(); td::actor::ActorOwn overlay_manager_; @@ -149,6 +157,9 @@ class ValidatorEngine : public td::actor::Actor { std::string local_config_ = ""; std::string global_config_ = "ton-global.config"; std::string config_file_; + std::string temp_config_file() const { + return config_file_ + ".tmp"; + } std::string fift_dir_ = ""; @@ -161,6 +172,7 @@ class ValidatorEngine : public td::actor::Actor { std::shared_ptr dht_config_; td::Ref validator_options_; Config config_; + ton::tl_object_ptr custom_overlays_config_; std::set running_gc_; @@ -197,11 +209,29 @@ class ValidatorEngine : public td::actor::Actor { double sync_ttl_ = 0; double archive_ttl_ = 0; double key_proof_ttl_ = 0; + td::uint32 celldb_compress_depth_ = 0; + size_t max_open_archive_files_ = 0; + double archive_preload_period_ = 0.0; + bool disable_rocksdb_stats_ = false; + bool nonfinal_ls_queries_enabled_ = false; + td::optional celldb_cache_size_ = 1LL << 30; + bool celldb_direct_io_ = false; + bool celldb_preload_all_ = false; + bool celldb_in_memory_ = false; + td::optional catchain_max_block_delay_, catchain_max_block_delay_slow_; bool read_config_ = false; bool started_keyring_ = false; bool started_ = false; ton::BlockSeqno truncate_seqno_{0}; std::string session_logs_file_; + bool fast_state_serializer_enabled_ = false; + std::string validator_telemetry_filename_; + bool not_all_shards_ = false; + std::vector add_shard_cmds_; + bool state_serializer_disabled_flag_ = false; + double broadcast_speed_multiplier_catchain_ = 1.0; + double broadcast_speed_multiplier_public_ = 1.0; + double broadcast_speed_multiplier_private_ = 1.0; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -253,6 +283,65 @@ class ValidatorEngine : public td::actor::Actor { void add_key_to_set(ton::PublicKey key) { keys_[key.compute_short_id()] = key; } + void schedule_shutdown(double at); + void set_celldb_compress_depth(td::uint32 value) { + celldb_compress_depth_ = value; + } + void set_max_open_archive_files(size_t value) { + max_open_archive_files_ = value; + } + void set_archive_preload_period(double value) { + archive_preload_period_ = value; + } + void set_disable_rocksdb_stats(bool value) { + disable_rocksdb_stats_ = value; + } + void set_nonfinal_ls_queries_enabled() { + nonfinal_ls_queries_enabled_ = true; + } + void set_celldb_cache_size(td::uint64 value) { + celldb_cache_size_ = value; + } + void set_celldb_direct_io(bool value) { + celldb_direct_io_ = value; + } + void set_celldb_preload_all(bool value) { + celldb_preload_all_ = value; + } + void set_celldb_in_memory(bool value) { + celldb_in_memory_ = value; + } + void set_catchain_max_block_delay(double value) { + catchain_max_block_delay_ = value; + } + void set_catchain_max_block_delay_slow(double value) { + catchain_max_block_delay_slow_ = value; + } + void set_fast_state_serializer_enabled(bool value) { + fast_state_serializer_enabled_ = value; + } + void set_validator_telemetry_filename(std::string value) { + validator_telemetry_filename_ = std::move(value); + } + void set_not_all_shards() { + not_all_shards_ = true; + } + void add_shard_cmd(ton::ShardIdFull shard) { + add_shard_cmds_.push_back(shard); + } + void set_state_serializer_disabled_flag() { + state_serializer_disabled_flag_ = true; + } + void set_broadcast_speed_multiplier_catchain(double value) { + broadcast_speed_multiplier_catchain_ = value; + } + void set_broadcast_speed_multiplier_public(double value) { + broadcast_speed_multiplier_public_ = value; + } + void set_broadcast_speed_multiplier_private(double value) { + broadcast_speed_multiplier_private_ = value; + } + void start_up() override; ValidatorEngine() { } @@ -262,6 +351,7 @@ class ValidatorEngine : public td::actor::Actor { void load_empty_local_config(td::Promise promise); void load_local_config(td::Promise promise); void load_config(td::Promise promise); + void set_shard_check_function(); void start(); @@ -335,6 +425,20 @@ class ValidatorEngine : public td::actor::Actor { void try_del_proxy(td::uint32 ip, td::int32 port, std::vector cats, std::vector prio_cats, td::Promise promise); + std::string custom_overlays_config_file() const { + return db_root_ + "/custom-overlays.json"; + } + std::string collator_options_file() const { + return db_root_ + "/collator-options.json"; + } + + void load_custom_overlays_config(); + td::Status write_custom_overlays_config(); + void add_custom_overlay_to_config( + ton::tl_object_ptr overlay, td::Promise promise); + void del_custom_overlay_from_config(std::string name, td::Promise promise); + void load_collator_options(); + void check_key(ton::PublicKeyHash id, td::Promise promise); static td::BufferSlice create_control_query_error(td::Status error); @@ -387,6 +491,8 @@ class ValidatorEngine : public td::actor::Actor { td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_sign &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_exportAllPrivateKeys &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_setVerbosity &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getStats &query, td::BufferSlice data, ton::PublicKeyHash src, @@ -407,8 +513,32 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getOverlaysStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_getActorTextStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_addShard &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_delShard &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getPerfTimerStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_getShardOutQueueSize &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_setExtMessagesBroadcastDisabled &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_addCustomOverlay &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_delCustomOverlay &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_showCustomOverlays &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_setStateSerializerEnabled &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_setCollatorOptionsJson &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_getCollatorOptionsJson &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_getAdnlStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator-session/CMakeLists.txt b/validator-session/CMakeLists.txt index dc837219..6b4e8f68 100644 --- a/validator-session/CMakeLists.txt +++ b/validator-session/CMakeLists.txt @@ -1,22 +1,25 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) endif() set(VALIDATOR_SESSION_SOURCE + candidate-serializer.cpp persistent-vector.cpp validator-session-description.cpp validator-session-state.cpp validator-session.cpp + validator-session-round-attempt-state.cpp + candidate-serializer.h persistent-vector.h validator-session-description.h validator-session-description.hpp validator-session-state.h validator-session.h validator-session.hpp -) + validator-session-round-attempt-state.h) add_library(validatorsession STATIC ${VALIDATOR_SESSION_SOURCE}) @@ -25,5 +28,4 @@ target_include_directories(validatorsession PUBLIC $/.. ${OPENSSL_INCLUDE_DIR} ) -target_link_libraries(validatorsession PRIVATE tdutils tdactor adnl rldp tl_api dht tdfec - overlay catchain) +target_link_libraries(validatorsession PRIVATE tdutils tdactor adnl rldp tl_api dht tdfec overlay catchain) diff --git a/validator-session/candidate-serializer.cpp b/validator-session/candidate-serializer.cpp new file mode 100644 index 00000000..4442504e --- /dev/null +++ b/validator-session/candidate-serializer.cpp @@ -0,0 +1,88 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "candidate-serializer.h" +#include "tl-utils/tl-utils.hpp" +#include "vm/boc.h" +#include "td/utils/lz4.h" +#include "validator-session-types.h" + +namespace ton::validatorsession { + +td::Result serialize_candidate(const tl_object_ptr& block, + bool compression_enabled) { + if (!compression_enabled) { + return serialize_tl_object(block, true); + } + size_t decompressed_size; + TRY_RESULT(compressed, compress_candidate_data(block->data_, block->collated_data_, decompressed_size)) + return create_serialize_tl_object( + 0, block->src_, block->round_, block->root_hash_, (int)decompressed_size, std::move(compressed)); +} + +td::Result> deserialize_candidate(td::Slice data, + bool compression_enabled, + int max_decompressed_data_size) { + if (!compression_enabled) { + return fetch_tl_object(data, true); + } + TRY_RESULT(f, fetch_tl_object(data, true)); + if (f->decompressed_size_ > max_decompressed_data_size) { + return td::Status::Error("decompressed size is too big"); + } + TRY_RESULT(p, decompress_candidate_data(f->data_, f->decompressed_size_)); + return create_tl_object(f->src_, f->round_, f->root_hash_, std::move(p.first), + std::move(p.second)); +} + +td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, + size_t& decompressed_size) { + vm::BagOfCells boc1, boc2; + TRY_STATUS(boc1.deserialize(block)); + if (boc1.get_root_count() != 1) { + return td::Status::Error("block candidate should have exactly one root"); + } + std::vector> roots = {boc1.get_root_cell()}; + TRY_STATUS(boc2.deserialize(collated_data)); + for (int i = 0; i < boc2.get_root_count(); ++i) { + roots.push_back(boc2.get_root_cell(i)); + } + TRY_RESULT(data, vm::std_boc_serialize_multi(std::move(roots), 2)); + decompressed_size = data.size(); + td::BufferSlice compressed = td::lz4_compress(data); + LOG(DEBUG) << "Compressing block candidate: " << block.size() + collated_data.size() << " -> " << compressed.size(); + return compressed; +} + +td::Result> decompress_candidate_data(td::Slice compressed, + int decompressed_size) { + TRY_RESULT(decompressed, td::lz4_decompress(compressed, decompressed_size)); + if (decompressed.size() != (size_t)decompressed_size) { + return td::Status::Error("decompressed size mismatch"); + } + TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed)); + if (roots.empty()) { + return td::Status::Error("boc is empty"); + } + TRY_RESULT(block_data, vm::std_boc_serialize(roots[0], 31)); + roots.erase(roots.begin()); + TRY_RESULT(collated_data, vm::std_boc_serialize_multi(std::move(roots), 31)); + LOG(DEBUG) << "Decompressing block candidate: " << compressed.size() << " -> " + << block_data.size() + collated_data.size(); + return std::make_pair(std::move(block_data), std::move(collated_data)); +} + +} // namespace ton::validatorsession diff --git a/validator-session/candidate-serializer.h b/validator-session/candidate-serializer.h new file mode 100644 index 00000000..e88376cd --- /dev/null +++ b/validator-session/candidate-serializer.h @@ -0,0 +1,34 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "ton/ton-types.h" +#include "auto/tl/ton_api.h" + +namespace ton::validatorsession { + +td::Result serialize_candidate(const tl_object_ptr& block, + bool compression_enabled); +td::Result> deserialize_candidate(td::Slice data, + bool compression_enabled, + int max_decompressed_data_size); + +td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, + size_t& decompressed_size); +td::Result> decompress_candidate_data(td::Slice compressed, + int decompressed_size); + +} // namespace ton::validatorsession diff --git a/validator-session/persistent-vector.h b/validator-session/persistent-vector.h index 77852f80..3491164c 100644 --- a/validator-session/persistent-vector.h +++ b/validator-session/persistent-vector.h @@ -322,7 +322,6 @@ class CntVector : public ValidatorSessionDescription::RootObject { CHECK(idx < size()); return data_[idx]; } - //const T& at(size_t idx) const; private: const td::uint32 data_size_; @@ -546,7 +545,6 @@ class CntVector : public ValidatorSessionDescription::RootObject { CHECK(idx < max_size()); return get_bit(data_, idx); } - //const T& at(size_t idx) const; private: const td::uint32 data_size_; @@ -734,10 +732,6 @@ class CntSortedVector : public ValidatorSessionDescription::RootObject { return CntSortedVector::create(desc, std::move(v)); } - /*static const CntSortedVector* merge(ValidatorSessionDescription& desc, const CntSortedVector* l, - const CntSortedVector* r) { - return merge(desc, l, r, [](T l, T r) { return l; }); - }*/ static const CntSortedVector* push(ValidatorSessionDescription& desc, const CntSortedVector* v, T value) { if (!v) { return create(desc, std::vector{value}); @@ -795,7 +789,6 @@ class CntSortedVector : public ValidatorSessionDescription::RootObject { CHECK(idx < size()); return data_[idx]; } - //const T& at(size_t idx) const; private: const td::uint32 data_size_; diff --git a/validator-session/validator-session-description.cpp b/validator-session/validator-session-description.cpp index eb4c3a8e..c747e26e 100644 --- a/validator-session/validator-session-description.cpp +++ b/validator-session/validator-session-description.cpp @@ -51,13 +51,6 @@ ValidatorSessionDescriptionImpl::ValidatorSessionDescriptionImpl(ValidatorSessio CHECK(it != rev_sources_.end()); self_idx_ = it->second; - pdata_temp_ptr_ = 0; - pdata_temp_size_ = 1 << 27; - pdata_temp_ = new td::uint8[pdata_temp_size_]; - - pdata_perm_size_ = 1ull << 27; - pdata_perm_ptr_ = 0; - for (auto &el : cache_) { Cached v{nullptr}; el.store(v, std::memory_order_relaxed); @@ -79,6 +72,11 @@ td::uint32 ValidatorSessionDescriptionImpl::get_max_priority() const { return opts_.round_candidates - 1; } +td::uint32 ValidatorSessionDescriptionImpl::get_node_by_priority(td::uint32 round, td::uint32 priority) const { + CHECK(priority <= get_max_priority()); + return (round + priority) % get_total_nodes(); +} + ValidatorSessionCandidateId ValidatorSessionDescriptionImpl::candidate_id( td::uint32 src_idx, ValidatorSessionRootHash root_hash, ValidatorSessionFileHash file_hash, ValidatorSessionCollatedDataFileHash collated_data_file_hash) const { @@ -162,43 +160,11 @@ void ValidatorSessionDescriptionImpl::update_hash(const RootObject *obj, HashTyp } void *ValidatorSessionDescriptionImpl::alloc(size_t size, size_t align, bool temp) { - CHECK(align && !(align & (align - 1))); // align should be a power of 2 - auto get_padding = [&](const uint8_t* ptr) { - return (-(size_t)ptr) & (align - 1); - }; - if (temp) { - pdata_temp_ptr_ += get_padding(pdata_temp_ + pdata_temp_ptr_); - auto s = pdata_temp_ptr_; - pdata_temp_ptr_ += size; - CHECK(s + size <= pdata_temp_size_); - return static_cast(pdata_temp_ + s); - } else { - while (true) { - size_t idx = pdata_perm_ptr_ / pdata_perm_size_; - if (idx < pdata_perm_.size()) { - auto ptr = pdata_perm_[idx] + (pdata_perm_ptr_ % pdata_perm_size_); - pdata_perm_ptr_ += get_padding(ptr); - ptr += get_padding(ptr); - pdata_perm_ptr_ += size; - if (pdata_perm_ptr_ <= pdata_perm_.size() * pdata_perm_size_) { - return static_cast(ptr); - } - } - pdata_perm_.push_back(new td::uint8[pdata_perm_size_]); - } - } + return (temp ? mem_temp_ : mem_perm_).alloc(size, align); } bool ValidatorSessionDescriptionImpl::is_persistent(const void *ptr) const { - if (ptr == nullptr) { - return true; - } - for (auto &v : pdata_perm_) { - if (ptr >= v && ptr <= v + pdata_perm_size_) { - return true; - } - } - return false; + return mem_perm_.contains(ptr); } std::unique_ptr ValidatorSessionDescription::create( @@ -206,6 +172,58 @@ std::unique_ptr ValidatorSessionDescription::create return std::make_unique(std::move(opts), nodes, local_id); } +ValidatorSessionDescriptionImpl::MemPool::MemPool(size_t chunk_size) : chunk_size_(chunk_size) { +} + +ValidatorSessionDescriptionImpl::MemPool::~MemPool() { + for (auto &v : data_) { + delete[] v; + } +} + +void *ValidatorSessionDescriptionImpl::MemPool::alloc(size_t size, size_t align) { + CHECK(align && !(align & (align - 1))); // align should be a power of 2 + CHECK(size + align <= chunk_size_); + auto get_padding = [&](const uint8_t* ptr) { + return (-(size_t)ptr) & (align - 1); + }; + while (true) { + size_t idx = ptr_ / chunk_size_; + if (idx < data_.size()) { + auto ptr = data_[idx] + (ptr_ % chunk_size_); + ptr_ += get_padding(ptr); + ptr += get_padding(ptr); + ptr_ += size; + if (ptr_ <= data_.size() * chunk_size_) { + return static_cast(ptr); + } else { + ptr_ = data_.size() * chunk_size_; + } + } + data_.push_back(new td::uint8[chunk_size_]); + } +} + +void ValidatorSessionDescriptionImpl::MemPool::clear() { + while (data_.size() > 1) { + delete[] data_.back(); + data_.pop_back(); + } + ptr_ = 0; +} + +bool ValidatorSessionDescriptionImpl::MemPool::contains(const void* ptr) const { + if (ptr == nullptr) { + return true; + } + for (auto &v : data_) { + if (ptr >= v && ptr <= v + chunk_size_) { + return true; + } + } + return false; +} + } // namespace validatorsession } // namespace ton diff --git a/validator-session/validator-session-description.h b/validator-session/validator-session-description.h index 60e006d0..ac504312 100644 --- a/validator-session/validator-session-description.h +++ b/validator-session/validator-session-description.h @@ -85,6 +85,7 @@ class ValidatorSessionDescription { virtual ValidatorWeight get_total_weight() const = 0; virtual td::int32 get_node_priority(td::uint32 src_idx, td::uint32 round) const = 0; virtual td::uint32 get_max_priority() const = 0; + virtual td::uint32 get_node_by_priority(td::uint32 round, td::uint32 priority) const = 0; virtual td::uint32 get_unixtime(td::uint64 t) const = 0; virtual td::uint32 get_attempt_seqno(td::uint64 t) const = 0; virtual td::uint32 get_self_idx() const = 0; diff --git a/validator-session/validator-session-description.hpp b/validator-session/validator-session-description.hpp index d65369ac..0fbb4ede 100644 --- a/validator-session/validator-session-description.hpp +++ b/validator-session/validator-session-description.hpp @@ -50,20 +50,33 @@ class ValidatorSessionDescriptionImpl : public ValidatorSessionDescription { td::uint32 self_idx_; static constexpr td::uint32 cache_size = (1 << 20); + static constexpr size_t mem_chunk_size_perm = (1 << 27); + static constexpr size_t mem_chunk_size_temp = (1 << 27); struct Cached { const RootObject *ptr; }; std::array, cache_size> cache_; - //std::array, cache_size> temp_cache_; - td::uint8 *pdata_temp_; - size_t pdata_temp_ptr_; - size_t pdata_temp_size_; + public: + class MemPool { + public: + explicit MemPool(size_t chunk_size); + ~MemPool(); + void *alloc(size_t size, size_t align); + void clear(); + bool contains(const void* ptr) const; + + private: + size_t chunk_size_; + std::vector data_; + size_t ptr_ = 0; + }; + + private: + MemPool mem_perm_ = MemPool(mem_chunk_size_perm); + MemPool mem_temp_ = MemPool(mem_chunk_size_temp); - size_t pdata_perm_size_; - std::vector pdata_perm_; - size_t pdata_perm_ptr_; std::atomic reuse_{0}; public: @@ -101,6 +114,7 @@ class ValidatorSessionDescriptionImpl : public ValidatorSessionDescription { } td::int32 get_node_priority(td::uint32 src_idx, td::uint32 round) const override; td::uint32 get_max_priority() const override; + td::uint32 get_node_by_priority(td::uint32 round, td::uint32 priority) const override; td::uint32 get_unixtime(td::uint64 ts) const override { return static_cast(ts >> 32); } @@ -116,7 +130,7 @@ class ValidatorSessionDescriptionImpl : public ValidatorSessionDescription { void update_hash(const RootObject *obj, HashType hash) override; void *alloc(size_t size, size_t align, bool temp) override; void clear_temp_memory() override { - pdata_temp_ptr_ = 0; + mem_temp_.clear(); } bool is_persistent(const void *ptr) const override; HashType compute_hash(td::Slice data) const override; @@ -152,12 +166,6 @@ class ValidatorSessionDescriptionImpl : public ValidatorSessionDescription { const ValidatorSessionOptions &opts() const override { return opts_; } - ~ValidatorSessionDescriptionImpl() { - delete[] pdata_temp_; - for (auto &x : pdata_perm_) { - delete[] x; - } - } }; } // namespace validatorsession diff --git a/validator-session/validator-session-round-attempt-state.cpp b/validator-session/validator-session-round-attempt-state.cpp new file mode 100644 index 00000000..33c3d804 --- /dev/null +++ b/validator-session/validator-session-round-attempt-state.cpp @@ -0,0 +1,486 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . + + Copyright 2017-2020 Telegram Systems LLP +*/ +#include "validator-session-state.h" +#include "td/utils/Random.h" +#include "auto/tl/ton_api.hpp" + +#include + +namespace ton { + +namespace validatorsession { + +static const SessionVoteCandidate* get_candidate(const VoteVector* vec, ValidatorSessionCandidateId id) { + if (!vec) { + return nullptr; + } + auto size = vec->size(); + auto v = vec->data(); + for (td::uint32 i = 0; i < size; i++) { + if (v[i]->get_id() == id) { + return v[i]; + } + } + return nullptr; +} + +// +// +// SessionBlockCandidateSignature +// +// + +const SessionBlockCandidateSignature* SessionBlockCandidateSignature::merge(ValidatorSessionDescription& desc, + const SessionBlockCandidateSignature* l, + const SessionBlockCandidateSignature* r) { + if (!l) { + return r; + } + if (!r) { + return l; + } + if (l == r) { + return l; + } + if (l->as_slice() < r->as_slice()) { + return l; + } else { + return r; + } +} + +// +// +// SessionBlockCandidate +// +// + +bool SessionBlockCandidate::check_block_is_approved(ValidatorSessionDescription& desc) const { + ValidatorWeight w = 0; + for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { + if (approved_by_->at(i)) { + w += desc.get_node_weight(i); + if (w >= desc.get_cutoff_weight()) { + return true; + } + } + } + return false; +} + +const SessionBlockCandidate* SessionBlockCandidate::merge(ValidatorSessionDescription& desc, + const SessionBlockCandidate* l, + const SessionBlockCandidate* r) { + if (!l) { + return r; + } + if (!r) { + return l; + } + if (l == r) { + return l; + } + CHECK(l->get_id() == r->get_id()); + auto v = SessionBlockCandidateSignatureVector::merge( + desc, l->approved_by_, r->approved_by_, + [&](const SessionBlockCandidateSignature* l, const SessionBlockCandidateSignature* r) { + return SessionBlockCandidateSignature::merge(desc, l, r); + }); + return SessionBlockCandidate::create(desc, l->block_, std::move(v)); +} + +// +// +// SessionVoteCandidate +// +// + +bool SessionVoteCandidate::check_block_is_voted(ValidatorSessionDescription& desc) const { + ValidatorWeight w = 0; + for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { + if (voted_by_->at(i)) { + w += desc.get_node_weight(i); + if (w >= desc.get_cutoff_weight()) { + return true; + } + } + } + return false; +} + +const SessionVoteCandidate* SessionVoteCandidate::merge(ValidatorSessionDescription& desc, + const SessionVoteCandidate* l, const SessionVoteCandidate* r) { + if (!l) { + return r; + } + if (!r) { + return l; + } + if (l == r) { + return l; + } + CHECK(l->get_id() == r->get_id()); + auto v = CntVector::merge(desc, l->voted_by_, r->voted_by_); + return SessionVoteCandidate::create(desc, l->block_, std::move(v)); +} + +// +// +// ATTEMPT STATE +// +// + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::merge( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* left, + const ValidatorSessionRoundAttemptState* right) { + if (!left) { + return right; + } + if (!right) { + return left; + } + if (left == right) { + return left; + } + CHECK(left->seqno_ == right->seqno_); + + const SentBlock* vote_for = nullptr; + bool vote_for_inited = false; + if (!left->vote_for_inited_) { + vote_for = right->vote_for_; + vote_for_inited = right->vote_for_inited_; + } else if (!right->vote_for_inited_) { + vote_for = left->vote_for_; + vote_for_inited = left->vote_for_inited_; + } else if (left->vote_for_ == right->vote_for_) { + vote_for_inited = true; + vote_for = left->vote_for_; + } else { + auto l = SentBlock::get_block_id(left->vote_for_); + auto r = SentBlock::get_block_id(right->vote_for_); + + vote_for_inited = true; + if (l < r) { + vote_for = left->vote_for_; + } else { + vote_for = right->vote_for_; + } + } + + auto precommitted = CntVector::merge(desc, left->precommitted_, right->precommitted_); + auto votes = VoteVector::merge(desc, left->votes_, right->votes_, + [&](const SessionVoteCandidate* l, const SessionVoteCandidate* r) { + return SessionVoteCandidate::merge(desc, l, r); + }); + + return ValidatorSessionRoundAttemptState::create(desc, left->seqno_, std::move(votes), std::move(precommitted), + vote_for, vote_for_inited); +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ton_api::validatorSession_message_voteFor& act, const ValidatorSessionRoundState* round) { + if (state->vote_for_inited_) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: invalid message: duplicate VOTEFOR"; + return state; + } + if (src_idx != desc.get_vote_for_author(att)) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: invalid message: bad VOTEFOR author"; + return state; + } + if (round->get_first_attempt(src_idx) == 0 && desc.opts().max_round_attempts > 0) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: invalid message: too early for VOTEFOR"; + return state; + } + if (round->get_first_attempt(src_idx) + desc.opts().max_round_attempts > att && desc.opts().max_round_attempts > 0) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: invalid message: too early for VOTEFOR"; + return state; + } + + auto x = round->get_block(act.candidate_); + + if (!x) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: invalid message: VOTEFOR for not submitted block"; + return state; + } + if (!x->check_block_is_approved(desc)) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: invalid message: VOTEFOR for not approved block"; + return state; + } + + return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, state->votes_, state->precommitted_, + x->get_block(), true); +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ton_api::validatorSession_message_vote& act, const ValidatorSessionRoundState* round) { + bool made; + return make_one(desc, state, src_idx, att, round, &act, made); +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ton_api::validatorSession_message_precommit& act, const ValidatorSessionRoundState* round) { + bool made; + return make_one(desc, state, src_idx, att, round, &act, made); +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ton_api::validatorSession_message_empty& act, const ValidatorSessionRoundState* round) { + bool made; + return make_one(desc, state, src_idx, att, round, &act, made); +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::try_vote( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act, + bool& made) { + made = false; + if (state->check_vote_received_from(src_idx)) { + return state; + } + auto found = false; + auto block = round->choose_block_to_vote(desc, src_idx, att, state->vote_for_, state->vote_for_inited_, found); + if (!found) { + return state; + } + auto block_id = SentBlock::get_block_id(block); + made = true; + + if (act) { + if (act->get_id() != ton_api::validatorSession_message_vote::ID) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: expected VOTE(" << block_id << ")"; + } else { + auto x = static_cast(act); + if (x->candidate_ != block_id) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: expected VOTE(" << block_id << ")"; + } + } + } else { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) + << "]: making implicit VOTE(" << block_id << ")"; + } + + auto candidate = get_candidate(state->votes_, block_id); + if (!candidate) { + candidate = SessionVoteCandidate::create(desc, block); + } + candidate = SessionVoteCandidate::push(desc, candidate, src_idx); + auto v = VoteVector::push(desc, state->votes_, candidate); + return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, std::move(v), state->precommitted_, + state->vote_for_, state->vote_for_inited_); +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::try_precommit( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act, + bool& made) { + made = false; + if (state->check_precommit_received_from(src_idx)) { + return state; + } + bool found; + auto block = state->get_voted_block(desc, found); + if (!found) { + return state; + } + made = true; + auto block_id = SentBlock::get_block_id(block); + + if (act) { + if (act->get_id() != ton_api::validatorSession_message_precommit::ID) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: expected PRECOMMIT(" << block_id << ")"; + } else { + auto x = static_cast(act); + if (x->candidate_ != block_id) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: expected PRECOMMIT(" << block_id << ")"; + } + } + } else { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) + << "]: making implicit PRECOMMIT(" << block_id << ")"; + } + + auto v = CntVector::change(desc, state->precommitted_, src_idx, true); + return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, state->votes_, std::move(v), state->vote_for_, + state->vote_for_inited_); +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::make_one( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act, + bool& made) { + made = false; + state = try_vote(desc, state, src_idx, att, round, act, made); + if (made) { + return state; + } + state = try_precommit(desc, state, src_idx, att, round, act, made); + if (made) { + return state; + } + if (act && act->get_id() != ton_api::validatorSession_message_empty::ID) { + VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act + << "]: invalid message: expected EMPTY"; + } + return state; +} + +const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( + ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, + td::uint32 att, const ton_api::validatorSession_round_Message* act, const ValidatorSessionRoundState* round) { + ton_api::downcast_call(*const_cast(act), + [&](auto& obj) { state = action(desc, state, src_idx, att, obj, round); }); + return state; +} + +bool ValidatorSessionRoundAttemptState::check_vote_received_from(td::uint32 src_idx) const { + if (!votes_) { + return false; + } + auto size = votes_->size(); + auto v = votes_->data(); + for (td::uint32 i = 0; i < size; i++) { + if (v[i]->check_block_is_voted_by(src_idx)) { + return true; + } + } + return false; +} + +bool ValidatorSessionRoundAttemptState::check_precommit_received_from(td::uint32 src_idx) const { + return precommitted_->at(src_idx); +} + +const SentBlock* ValidatorSessionRoundAttemptState::get_voted_block(ValidatorSessionDescription& desc, bool& f) const { + f = false; + if (!votes_) { + return nullptr; + } + + auto size = votes_->size(); + auto v = votes_->data(); + for (td::uint32 i = 0; i < size; i++) { + if (v[i]->check_block_is_voted(desc)) { + f = true; + return v[i]->get_block(); + } + } + return nullptr; +} + +bool ValidatorSessionRoundAttemptState::check_attempt_is_precommitted(ValidatorSessionDescription& desc) const { + ValidatorWeight weight = 0; + for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { + if (precommitted_->at(i)) { + weight += desc.get_node_weight(i); + if (weight >= desc.get_cutoff_weight()) { + return true; + } + } + } + return false; +} + +tl_object_ptr ValidatorSessionRoundAttemptState::create_action( + ValidatorSessionDescription& desc, const ValidatorSessionRoundState* round, td::uint32 src_idx, + td::uint32 att) const { + if (!check_vote_received_from(src_idx)) { + auto found = false; + auto B = round->choose_block_to_vote(desc, src_idx, att, vote_for_, vote_for_inited_, found); + if (found) { + auto block_id = SentBlock::get_block_id(B); + return create_tl_object(round->get_seqno(), seqno_, block_id); + } + } + if (!check_precommit_received_from(src_idx)) { + bool f = false; + auto B = get_voted_block(desc, f); + + if (f) { + auto block_id = SentBlock::get_block_id(B); + + return create_tl_object(round->get_seqno(), seqno_, block_id); + } + } + + return create_tl_object(round->get_seqno(), seqno_); +} + +void ValidatorSessionRoundAttemptState::dump(ValidatorSessionDescription& desc, td::StringBuilder& sb) const { + sb << "attempt=" << seqno_ << "\n"; + sb << ">>>>\n"; + + if (vote_for_inited_) { + sb << "vote_for=" << (vote_for_ ? vote_for_->get_src_idx() : std::numeric_limits::max()) << "\n"; + } else { + sb << "vote_for=NONE\n"; + } + + if (votes_) { + auto s = votes_->size(); + sb << "votes: "; + std::vector R; + R.resize(desc.get_total_nodes(), -1); + for (td::uint32 i = 0; i < s; i++) { + const auto e = votes_->at(i); + const auto& x = e->get_voters_list(); + for (td::uint32 j = 0; j < desc.get_total_nodes(); j++) { + if (x->at(j)) { + R[j] = e->get_src_idx(); + } + } + } + for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { + sb << R[i] << " "; + } + sb << "\n"; + } else { + sb << "votes: EMPTY\n"; + } + + sb << "precommits: "; + for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { + const auto e = precommitted_->at(i); + if (e) { + sb << "+ "; + } else { + sb << "- "; + } + } + sb << "\n"; + sb << "<<<<\n"; +} + + +} // namespace validatorsession + +} // namespace ton diff --git a/validator-session/validator-session-round-attempt-state.h b/validator-session/validator-session-round-attempt-state.h new file mode 100644 index 00000000..a4a6b68b --- /dev/null +++ b/validator-session/validator-session-round-attempt-state.h @@ -0,0 +1,620 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ + +#pragma once + +#include "td/utils/int_types.h" +#include "td/utils/buffer.h" +#include "adnl/utils.hpp" +#include "common/io.hpp" + +#include "persistent-vector.h" + +#include "validator-session-description.h" + +#include "validator-session-types.h" + +#include + + +namespace ton { + +namespace validatorsession { + +using HashType = ValidatorSessionDescription::HashType; + +struct SessionBlockCandidateSignature : public ValidatorSessionDescription::RootObject { + public: + static auto create_hash(ValidatorSessionDescription& desc, td::Slice data) { + auto obj = create_tl_object(desc.compute_hash(data)); + return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); + } + + static bool compare(const RootObject* r, td::Slice data, HashType hash) { + if (!r || r->get_size() < sizeof(SessionBlockCandidateSignature)) { + return false; + } + auto R = static_cast(r); + return R->hash_ == hash && R->data_.ubegin() == data.ubegin() && R->data_.size() == data.size(); + } + + static auto lookup(ValidatorSessionDescription& desc, td::Slice data, HashType hash, bool temp) { + auto r = desc.get_by_hash(hash, temp); + if (compare(r, data, hash)) { + desc.on_reuse(); + return static_cast(r); + } + return static_cast(nullptr); + } + static SessionBlockCandidateSignature* create(ValidatorSessionDescription& desc, td::BufferSlice value) { + auto hash = create_hash(desc, value.as_slice()); + auto d = static_cast(desc.alloc(value.size(), 8, false)); + td::MutableSlice s{d, value.size()}; + s.copy_from(value.as_slice()); + return new (desc, true) SessionBlockCandidateSignature{desc, s, hash}; + } + static const SessionBlockCandidateSignature* move_to_persistent(ValidatorSessionDescription& desc, + const SessionBlockCandidateSignature* b) { + if (desc.is_persistent(b)) { + return b; + } + CHECK(desc.is_persistent(b->data_.ubegin())); + auto r = lookup(desc, b->data_, b->hash_, false); + if (r) { + return r; + } + return new (desc, false) SessionBlockCandidateSignature{desc, b->data_, b->hash_}; + } + static const SessionBlockCandidateSignature* merge(ValidatorSessionDescription& desc, + const SessionBlockCandidateSignature* l, + const SessionBlockCandidateSignature* r); + SessionBlockCandidateSignature(ValidatorSessionDescription& desc, td::Slice data, HashType hash) + : RootObject(sizeof(SessionBlockCandidateSignature)), data_{data}, hash_(std::move(hash)) { + desc.update_hash(this, hash_); + } + td::BufferSlice value() const { + return td::BufferSlice{data_}; + } + td::Slice as_slice() const { + return data_; + } + auto get_hash(ValidatorSessionDescription& desc) const { + return hash_; + } + + private: + const td::Slice data_; + const HashType hash_; +}; + +using SessionBlockCandidateSignatureVector = CntVector; + +class SentBlock : public ValidatorSessionDescription::RootObject { + public: + static HashType create_hash(ValidatorSessionDescription& desc, td::uint32 src_idx, ValidatorSessionRootHash root_hash, + ValidatorSessionFileHash file_hash, + ValidatorSessionCollatedDataFileHash collated_data_file_hash) { + auto obj = create_tl_object(src_idx, get_vs_hash(desc, root_hash), + get_vs_hash(desc, file_hash), + get_vs_hash(desc, collated_data_file_hash)); + return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); + } + static bool compare(const RootObject* root_object, td::uint32 src_idx, const ValidatorSessionRootHash& root_hash, + const ValidatorSessionFileHash& file_hash, + const ValidatorSessionCollatedDataFileHash& collated_data_file_hash, HashType hash) { + if (!root_object || root_object->get_size() < sizeof(SentBlock)) { + return false; + } + auto obj = static_cast(root_object); + return obj->src_idx_ == src_idx && obj->root_hash_ == root_hash && obj->file_hash_ == file_hash && + obj->collated_data_file_hash_ == collated_data_file_hash && obj->hash_ == hash; + } + static auto lookup(ValidatorSessionDescription& desc, td::uint32 src_idx, const ValidatorSessionRootHash& root_hash, + const ValidatorSessionFileHash& file_hash, + const ValidatorSessionCollatedDataFileHash& collated_data_file_hash, HashType hash, bool temp) { + auto r = desc.get_by_hash(hash, temp); + if (compare(r, src_idx, root_hash, file_hash, collated_data_file_hash, hash)) { + desc.on_reuse(); + return static_cast(r); + } + return static_cast(nullptr); + } + static const SentBlock* create(ValidatorSessionDescription& desc, td::uint32 src_idx, + const ValidatorSessionRootHash& root_hash, const ValidatorSessionFileHash& file_hash, + const ValidatorSessionCollatedDataFileHash& collated_data_file_hash) { + auto hash = create_hash(desc, src_idx, root_hash, file_hash, collated_data_file_hash); + auto r = lookup(desc, src_idx, root_hash, file_hash, collated_data_file_hash, hash, true); + if (r) { + return r; + } + + auto candidate_id = desc.candidate_id(src_idx, root_hash, file_hash, collated_data_file_hash); + + return new (desc, true) SentBlock{desc, src_idx, root_hash, file_hash, collated_data_file_hash, candidate_id, hash}; + } + static const SentBlock* create(ValidatorSessionDescription& desc, const ValidatorSessionCandidateId& zero) { + CHECK(zero.is_zero()); + auto hash = create_hash(desc, 0, ValidatorSessionRootHash::zero(), ValidatorSessionFileHash::zero(), + ValidatorSessionCollatedDataFileHash::zero()); + + return new (desc, true) SentBlock{desc, + 0, + ValidatorSessionRootHash::zero(), + ValidatorSessionFileHash::zero(), + ValidatorSessionCollatedDataFileHash::zero(), + zero, + hash}; + } + static const SentBlock* move_to_persistent(ValidatorSessionDescription& desc, const SentBlock* b) { + if (desc.is_persistent(b)) { + return b; + } + auto r = lookup(desc, b->src_idx_, b->root_hash_, b->file_hash_, b->collated_data_file_hash_, b->hash_, false); + if (r) { + return r; + } + + return new (desc, false) SentBlock{ + desc, b->src_idx_, b->root_hash_, b->file_hash_, b->collated_data_file_hash_, b->candidate_id_, b->hash_}; + } + SentBlock(ValidatorSessionDescription& desc, td::uint32 src_idx, ValidatorSessionRootHash root_hash, + ValidatorSessionFileHash file_hash, ValidatorSessionCollatedDataFileHash collated_data_file_hash, + ValidatorSessionCandidateId candidate_id, HashType hash) + : RootObject(sizeof(SentBlock)) + , src_idx_(src_idx) + , root_hash_(std::move(root_hash)) + , file_hash_(std::move(file_hash)) + , collated_data_file_hash_(std::move(collated_data_file_hash)) + , candidate_id_(candidate_id) + , hash_(std::move(hash)) { + desc.update_hash(this, hash_); + } + auto get_src_idx() const { + return src_idx_; + } + auto get_root_hash() const { + return root_hash_; + } + auto get_file_hash() const { + return file_hash_; + } + auto get_collated_data_file_hash() const { + return collated_data_file_hash_; + } + static ValidatorSessionCandidateId get_block_id(const SentBlock* block) { + return block ? block->candidate_id_ : skip_round_candidate_id(); + } + HashType get_hash(ValidatorSessionDescription& desc) const { + return hash_; + } + bool operator<(const SentBlock& block) const { + if (src_idx_ < block.src_idx_) { + return true; + } + if (src_idx_ > block.src_idx_) { + return false; + } + if (candidate_id_ < block.candidate_id_) { + return true; + } + return false; + } + struct Compare { + bool operator()(const SentBlock* a, const SentBlock* b) const { + return *a < *b; + } + }; + + private: + const td::uint32 src_idx_; + const ValidatorSessionRootHash root_hash_; + const ValidatorSessionFileHash file_hash_; + const ValidatorSessionCollatedDataFileHash collated_data_file_hash_; + const ValidatorSessionCandidateId candidate_id_; + const HashType hash_; +}; + +class SessionBlockCandidate : public ValidatorSessionDescription::RootObject { + public: + static HashType create_hash(ValidatorSessionDescription& desc, HashType block, HashType approved) { + auto obj = create_tl_object(block, approved); + return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); + } + static bool compare(const RootObject* r, const SentBlock* block, const SessionBlockCandidateSignatureVector* approved, + HashType hash) { + if (!r || r->get_size() < sizeof(SessionBlockCandidate)) { + return false; + } + auto R = static_cast(r); + return R->block_ == block && R->approved_by_ == approved && R->hash_ == hash; + } + static auto lookup(ValidatorSessionDescription& desc, const SentBlock* block, + const SessionBlockCandidateSignatureVector* approved, HashType hash, bool temp) { + auto r = desc.get_by_hash(hash, temp); + if (compare(r, block, approved, hash)) { + desc.on_reuse(); + return static_cast(r); + } + return static_cast(nullptr); + } + static const SessionBlockCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block, + const SessionBlockCandidateSignatureVector* approved) { + auto hash = create_hash(desc, get_vs_hash(desc, block), get_vs_hash(desc, approved)); + + auto r = lookup(desc, block, approved, hash, true); + if (r) { + return r; + } + + return new (desc, true) SessionBlockCandidate(desc, block, approved, hash); + } + static const SessionBlockCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block) { + std::vector v; + v.resize(desc.get_total_nodes(), nullptr); + auto vec = SessionBlockCandidateSignatureVector::create(desc, std::move(v)); + return create(desc, block, vec); + } + static const SessionBlockCandidate* move_to_persistent(ValidatorSessionDescription& desc, + const SessionBlockCandidate* b) { + if (desc.is_persistent(b)) { + return b; + } + auto block = SentBlock::move_to_persistent(desc, b->block_); + auto approved = SessionBlockCandidateSignatureVector::move_to_persistent(desc, b->approved_by_); + auto r = lookup(desc, block, approved, b->hash_, false); + if (r) { + return r; + } + + return new (desc, false) SessionBlockCandidate{desc, block, approved, b->hash_}; + } + SessionBlockCandidate(ValidatorSessionDescription& desc, const SentBlock* block, + const SessionBlockCandidateSignatureVector* approved, HashType hash) + : RootObject{sizeof(SessionBlockCandidate)}, block_(block), approved_by_(approved), hash_(hash) { + desc.update_hash(this, hash_); + } + static const SessionBlockCandidate* merge(ValidatorSessionDescription& desc, const SessionBlockCandidate* l, + const SessionBlockCandidate* r); + auto get_block() const { + return block_; + } + auto get_id() const { + return SentBlock::get_block_id(block_); + } + auto get_src_idx() const { + return block_ ? block_->get_src_idx() : std::numeric_limits::max(); + } + bool check_block_is_approved_by(td::uint32 src_idx) const { + return approved_by_->at(src_idx); + } + bool check_block_is_approved(ValidatorSessionDescription& desc) const; + auto get_hash(ValidatorSessionDescription& desc) const { + return hash_; + } + auto get_approvers_list() const { + return approved_by_; + } + static const SessionBlockCandidate* push(ValidatorSessionDescription& desc, const SessionBlockCandidate* state, + td::uint32 src_idx, const SessionBlockCandidateSignature* sig) { + CHECK(state); + if (state->approved_by_->at(src_idx)) { + return state; + } + return create(desc, state->block_, + SessionBlockCandidateSignatureVector::change(desc, state->approved_by_, src_idx, sig)); + } + class Compare { + public: + bool operator()(const SessionBlockCandidate* l, const SessionBlockCandidate* r) { + return l->get_id() < r->get_id(); + } + }; + + private: + const SentBlock* block_; + const SessionBlockCandidateSignatureVector* approved_by_; + const HashType hash_; +}; + +class SessionVoteCandidate : public ValidatorSessionDescription::RootObject { + public: + static HashType create_hash(ValidatorSessionDescription& desc, HashType block, HashType voted) { + auto obj = create_tl_object(block, voted); + return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); + } + static bool compare(const RootObject* r, const SentBlock* block, const CntVector* voted, HashType hash) { + if (!r || r->get_size() < sizeof(SessionVoteCandidate)) { + return false; + } + auto R = static_cast(r); + return R->block_ == block && R->voted_by_ == voted && R->hash_ == hash; + } + static auto lookup(ValidatorSessionDescription& desc, const SentBlock* block, const CntVector* voted, + HashType hash, bool temp) { + auto r = desc.get_by_hash(hash, temp); + if (compare(r, block, voted, hash)) { + desc.on_reuse(); + return static_cast(r); + } + return static_cast(nullptr); + } + static const SessionVoteCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block, + const CntVector* voted) { + auto hash = create_hash(desc, get_vs_hash(desc, block), get_vs_hash(desc, voted)); + + auto r = lookup(desc, block, voted, hash, true); + if (r) { + return r; + } + + return new (desc, true) SessionVoteCandidate(desc, block, voted, hash); + } + static const SessionVoteCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block) { + std::vector v; + v.resize(desc.get_total_nodes(), false); + auto vec = CntVector::create(desc, std::move(v)); + return create(desc, block, vec); + } + static const SessionVoteCandidate* move_to_persistent(ValidatorSessionDescription& desc, + const SessionVoteCandidate* b) { + if (desc.is_persistent(b)) { + return b; + } + auto block = SentBlock::move_to_persistent(desc, b->block_); + auto voted = CntVector::move_to_persistent(desc, b->voted_by_); + auto r = lookup(desc, block, voted, b->hash_, false); + if (r) { + return r; + } + + return new (desc, false) SessionVoteCandidate{desc, block, voted, b->hash_}; + } + SessionVoteCandidate(ValidatorSessionDescription& desc, const SentBlock* block, const CntVector* voted, + HashType hash) + : RootObject{sizeof(SessionVoteCandidate)}, block_(block), voted_by_(voted), hash_(hash) { + desc.update_hash(this, hash_); + } + static const SessionVoteCandidate* merge(ValidatorSessionDescription& desc, const SessionVoteCandidate* l, + const SessionVoteCandidate* r); + auto get_block() const { + return block_; + } + auto get_id() const { + return SentBlock::get_block_id(block_); + } + auto get_src_idx() const { + return block_ ? block_->get_src_idx() : std::numeric_limits::max(); + } + bool check_block_is_voted_by(td::uint32 src_idx) const { + return voted_by_->at(src_idx); + } + bool check_block_is_voted(ValidatorSessionDescription& desc) const; + auto get_hash(ValidatorSessionDescription& desc) const { + return hash_; + } + auto get_voters_list() const { + return voted_by_; + } + static const SessionVoteCandidate* push(ValidatorSessionDescription& desc, const SessionVoteCandidate* state, + td::uint32 src_idx) { + CHECK(state); + if (state->voted_by_->at(src_idx)) { + return state; + } + return create(desc, state->block_, CntVector::change(desc, state->voted_by_, src_idx, true)); + } + class Compare { + public: + bool operator()(const SessionVoteCandidate* l, const SessionVoteCandidate* r) { + return l->get_id() < r->get_id(); + } + }; + + private: + const SentBlock* block_; + const CntVector* voted_by_; + const HashType hash_; +}; + +using VoteVector = CntSortedVector; +using ApproveVector = CntSortedVector; +class ValidatorSessionRoundState; + +class ValidatorSessionRoundAttemptState : public ValidatorSessionDescription::RootObject { + public: + static HashType create_hash(ValidatorSessionDescription& desc, td::uint32 seqno, HashType votes, + HashType precommitted, bool vote_for_inited, HashType vote_for) { + auto obj = create_tl_object(seqno, votes, precommitted, + vote_for_inited, vote_for); + return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); + } + static bool compare(const RootObject* r, td::uint32 seqno, const VoteVector* votes, + const CntVector* precommitted, const SentBlock* vote_for, bool vote_for_inited, + HashType hash) { + if (!r || r->get_size() < sizeof(ValidatorSessionRoundAttemptState)) { + return false; + } + auto R = static_cast(r); + return R->seqno_ == seqno && R->votes_ == votes && R->precommitted_ == precommitted && R->vote_for_ == vote_for && + R->vote_for_inited_ == vote_for_inited && R->hash_ == hash; + } + static auto lookup(ValidatorSessionDescription& desc, td::uint32 seqno, const VoteVector* votes, + const CntVector* precommitted, const SentBlock* vote_for, bool vote_for_inited, + HashType hash, bool temp) { + auto r = desc.get_by_hash(hash, temp); + if (compare(r, seqno, votes, precommitted, vote_for, vote_for_inited, hash)) { + desc.on_reuse(); + return static_cast(r); + } + return static_cast(nullptr); + } + static const ValidatorSessionRoundAttemptState* move_to_persistent(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* b) { + if (desc.is_persistent(b)) { + return b; + } + auto votes = VoteVector::move_to_persistent(desc, b->votes_); + auto precommitted = CntVector::move_to_persistent(desc, b->precommitted_); + auto vote_for = SentBlock::move_to_persistent(desc, b->vote_for_); + + auto r = lookup(desc, b->seqno_, votes, precommitted, vote_for, b->vote_for_inited_, b->hash_, false); + if (r) { + return r; + } + + return new (desc, false) ValidatorSessionRoundAttemptState{desc, b->seqno_, votes, precommitted, + vote_for, b->vote_for_inited_, b->hash_}; + } + + static const ValidatorSessionRoundAttemptState* create(ValidatorSessionDescription& desc, td::uint32 seqno, + const VoteVector* votes, const CntVector* precommitted, + const SentBlock* vote_for, bool vote_for_inited) { + auto hash = create_hash(desc, seqno, get_vs_hash(desc, votes), get_vs_hash(desc, precommitted), + get_vs_hash(desc, vote_for), vote_for_inited); + + auto r = lookup(desc, seqno, votes, precommitted, vote_for, vote_for_inited, hash, true); + if (r) { + return r; + } + + return new (desc, true) + ValidatorSessionRoundAttemptState(desc, seqno, votes, precommitted, vote_for, vote_for_inited, hash); + } + static const ValidatorSessionRoundAttemptState* create(ValidatorSessionDescription& desc, td::uint32 seqno) { + std::vector x; + x.resize(desc.get_total_nodes(), false); + auto p = CntVector::create(desc, std::move(x)); + + return create(desc, seqno, nullptr, p, nullptr, false); + } + + ValidatorSessionRoundAttemptState(ValidatorSessionDescription& desc, td::uint32 seqno, const VoteVector* votes, + const CntVector* precommitted, const SentBlock* vote_for, + bool vote_for_inited, HashType hash) + : RootObject{sizeof(ValidatorSessionRoundAttemptState)} + , seqno_(seqno) + , votes_(votes) + , precommitted_(precommitted) + , vote_for_(vote_for) + , vote_for_inited_(vote_for_inited) + , hash_(std::move(hash)) { + desc.update_hash(this, hash_); + } + auto get_hash(ValidatorSessionDescription& desc) const { + return hash_; + } + auto get_seqno() const { + return seqno_; + } + auto get_votes() const { + return votes_; + } + auto get_precommits() const { + return precommitted_; + } + const SentBlock* get_voted_block(ValidatorSessionDescription& desc, bool& f) const; + const SentBlock* get_vote_for_block(ValidatorSessionDescription& desc, bool& f) const { + f = vote_for_inited_; + return vote_for_; + } + bool check_attempt_is_precommitted(ValidatorSessionDescription& desc) const; + bool check_vote_received_from(td::uint32 src_idx) const; + bool check_precommit_received_from(td::uint32 src_idx) const; + + bool operator<(const ValidatorSessionRoundAttemptState& right) const { + return seqno_ < right.seqno_; + } + struct Compare { + bool operator()(const ValidatorSessionRoundAttemptState* a, const ValidatorSessionRoundAttemptState* b) const { + return *a < *b; + } + }; + + static const ValidatorSessionRoundAttemptState* merge(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* left, + const ValidatorSessionRoundAttemptState* right); + static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ton_api::validatorSession_message_voteFor& act, + const ValidatorSessionRoundState* round); + static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ton_api::validatorSession_message_vote& act, + const ValidatorSessionRoundState* round); + static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ton_api::validatorSession_message_precommit& act, + const ValidatorSessionRoundState* round); + static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ton_api::validatorSession_message_empty& act, + const ValidatorSessionRoundState* round); + static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ton_api::validatorSession_round_Message* action, + const ValidatorSessionRoundState* round); + template + static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, const T& action, + const ValidatorSessionRoundState* round) { + UNREACHABLE(); + } + static const ValidatorSessionRoundAttemptState* try_vote(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ValidatorSessionRoundState* round, + const ton_api::validatorSession_round_Message* cmp, + bool& made); + static const ValidatorSessionRoundAttemptState* try_precommit(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ValidatorSessionRoundState* round, + const ton_api::validatorSession_round_Message* cmp, + bool& made); + static const ValidatorSessionRoundAttemptState* make_one(ValidatorSessionDescription& desc, + const ValidatorSessionRoundAttemptState* state, + td::uint32 src_idx, td::uint32 att, + const ValidatorSessionRoundState* round, + const ton_api::validatorSession_round_Message* cmp, + bool& made); + tl_object_ptr create_action(ValidatorSessionDescription& desc, + const ValidatorSessionRoundState* round, + td::uint32 src_idx, td::uint32 att) const; + void dump(ValidatorSessionDescription& desc, td::StringBuilder& sb) const; + + private: + const td::uint32 seqno_; + const VoteVector* votes_; + const CntVector* precommitted_; + const SentBlock* vote_for_; + const bool vote_for_inited_; + const HashType hash_; +}; + +using AttemptVector = + CntSortedVector; + + +} // namespace validatorsession + +} // namespace ton diff --git a/validator-session/validator-session-state.cpp b/validator-session/validator-session-state.cpp index 62a054d5..97b91895 100644 --- a/validator-session/validator-session-state.cpp +++ b/validator-session/validator-session-state.cpp @@ -67,14 +67,6 @@ namespace ton { namespace validatorsession { -static td::uint32 get_round_id(const ton_api::validatorSession_round_Message* message) { - td::uint32 round = 0; - bool is_called = ton_api::downcast_call(*const_cast(message), - [&](auto& obj) { round = obj.round_; }); - CHECK(is_called); - return round; -} - static const ValidatorSessionRoundAttemptState* get_attempt(const AttemptVector* vec, td::uint32 seqno) { if (!vec) { return nullptr; @@ -108,118 +100,12 @@ static const SessionBlockCandidate* get_candidate(const ApproveVector* vec, Vali return nullptr; } -static const SessionVoteCandidate* get_candidate(const VoteVector* vec, ValidatorSessionCandidateId id) { - if (!vec) { - return nullptr; - } - auto size = vec->size(); - auto v = vec->data(); - for (td::uint32 i = 0; i < size; i++) { - if (v[i]->get_id() == id) { - return v[i]; - } - } - return nullptr; -} - -// -// -// SessionBlockCandidateSignature -// -// - -const SessionBlockCandidateSignature* SessionBlockCandidateSignature::merge(ValidatorSessionDescription& desc, - const SessionBlockCandidateSignature* l, - const SessionBlockCandidateSignature* r) { - if (!l) { - return r; - } - if (!r) { - return l; - } - if (l == r) { - return l; - } - if (l->as_slice() < r->as_slice()) { - return l; - } else { - return r; - } -} - -// -// -// SessionBlockCandidate -// -// - -bool SessionBlockCandidate::check_block_is_approved(ValidatorSessionDescription& desc) const { - ValidatorWeight w = 0; - for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { - if (approved_by_->at(i)) { - w += desc.get_node_weight(i); - if (w >= desc.get_cutoff_weight()) { - return true; - } - } - } - return false; -} - -const SessionBlockCandidate* SessionBlockCandidate::merge(ValidatorSessionDescription& desc, - const SessionBlockCandidate* l, - const SessionBlockCandidate* r) { - if (!l) { - return r; - } - if (!r) { - return l; - } - if (l == r) { - return l; - } - CHECK(l->get_id() == r->get_id()); - auto v = SessionBlockCandidateSignatureVector::merge( - desc, l->approved_by_, r->approved_by_, - [&](const SessionBlockCandidateSignature* l, const SessionBlockCandidateSignature* r) { - return SessionBlockCandidateSignature::merge(desc, l, r); - }); - return SessionBlockCandidate::create(desc, l->block_, std::move(v)); -} - -// -// -// SessionVoteCandidate -// -// - -bool SessionVoteCandidate::check_block_is_voted(ValidatorSessionDescription& desc) const { - ValidatorWeight w = 0; - for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { - if (voted_by_->at(i)) { - w += desc.get_node_weight(i); - if (w >= desc.get_cutoff_weight()) { - return true; - } - } - } - return false; -} - -const SessionVoteCandidate* SessionVoteCandidate::merge(ValidatorSessionDescription& desc, - const SessionVoteCandidate* l, const SessionVoteCandidate* r) { - if (!l) { - return r; - } - if (!r) { - return l; - } - if (l == r) { - return l; - } - CHECK(l->get_id() == r->get_id()); - auto v = CntVector::merge(desc, l->voted_by_, r->voted_by_); - return SessionVoteCandidate::create(desc, l->block_, std::move(v)); +static td::uint32 get_round_id(const ton_api::validatorSession_round_Message* message) { + td::uint32 round = 0; + bool is_called = ton_api::downcast_call(*const_cast(message), + [&](auto& obj) { round = obj.round_; }); + CHECK(is_called); + return round; } // @@ -374,346 +260,6 @@ const ValidatorSessionOldRoundState* ValidatorSessionOldRoundState::action( return state; } -// -// -// ATTEMPT STATE -// -// - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::merge( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* left, - const ValidatorSessionRoundAttemptState* right) { - if (!left) { - return right; - } - if (!right) { - return left; - } - if (left == right) { - return left; - } - CHECK(left->seqno_ == right->seqno_); - - const SentBlock* vote_for = nullptr; - bool vote_for_inited = false; - if (!left->vote_for_inited_) { - vote_for = right->vote_for_; - vote_for_inited = right->vote_for_inited_; - } else if (!right->vote_for_inited_) { - vote_for = left->vote_for_; - vote_for_inited = left->vote_for_inited_; - } else if (left->vote_for_ == right->vote_for_) { - vote_for_inited = true; - vote_for = left->vote_for_; - } else { - auto l = SentBlock::get_block_id(left->vote_for_); - auto r = SentBlock::get_block_id(right->vote_for_); - - vote_for_inited = true; - if (l < r) { - vote_for = left->vote_for_; - } else { - vote_for = right->vote_for_; - } - } - - auto precommitted = CntVector::merge(desc, left->precommitted_, right->precommitted_); - auto votes = VoteVector::merge(desc, left->votes_, right->votes_, - [&](const SessionVoteCandidate* l, const SessionVoteCandidate* r) { - return SessionVoteCandidate::merge(desc, l, r); - }); - - return ValidatorSessionRoundAttemptState::create(desc, left->seqno_, std::move(votes), std::move(precommitted), - vote_for, vote_for_inited); -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ton_api::validatorSession_message_voteFor& act, const ValidatorSessionRoundState* round) { - if (state->vote_for_inited_) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: invalid message: duplicate VOTEFOR"; - return state; - } - if (src_idx != desc.get_vote_for_author(att)) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: invalid message: bad VOTEFOR author"; - return state; - } - if (round->get_first_attempt(src_idx) == 0 && desc.opts().max_round_attempts > 0) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: invalid message: too early for VOTEFOR"; - return state; - } - if (round->get_first_attempt(src_idx) + desc.opts().max_round_attempts > att && desc.opts().max_round_attempts == 0) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: invalid message: too early for VOTEFOR"; - return state; - } - - auto x = round->get_block(act.candidate_); - - if (!x) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: invalid message: VOTEFOR for not approved block"; - return state; - } - if (!x->check_block_is_approved(desc)) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: invalid message: VOTEFOR for not approved block"; - return state; - } - - return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, state->votes_, state->precommitted_, - x->get_block(), true); -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ton_api::validatorSession_message_vote& act, const ValidatorSessionRoundState* round) { - bool made; - return make_one(desc, state, src_idx, att, round, &act, made); -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ton_api::validatorSession_message_precommit& act, const ValidatorSessionRoundState* round) { - bool made; - return make_one(desc, state, src_idx, att, round, &act, made); -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ton_api::validatorSession_message_empty& act, const ValidatorSessionRoundState* round) { - bool made; - return make_one(desc, state, src_idx, att, round, &act, made); -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::try_vote( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act, - bool& made) { - made = false; - if (state->check_vote_received_from(src_idx)) { - return state; - } - auto found = false; - auto block = round->choose_block_to_vote(desc, src_idx, att, state->vote_for_, state->vote_for_inited_, found); - if (!found) { - return state; - } - auto block_id = SentBlock::get_block_id(block); - made = true; - - if (act) { - if (act->get_id() != ton_api::validatorSession_message_vote::ID) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: expected VOTE(" << block_id << ")"; - } else { - auto x = static_cast(act); - if (x->candidate_ != block_id) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: expected VOTE(" << block_id << ")"; - } - } - } else { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) - << "]: making implicit VOTE(" << block_id << ")"; - } - - auto candidate = get_candidate(state->votes_, block_id); - if (!candidate) { - candidate = SessionVoteCandidate::create(desc, block); - } - candidate = SessionVoteCandidate::push(desc, candidate, src_idx); - auto v = VoteVector::push(desc, state->votes_, candidate); - return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, std::move(v), state->precommitted_, - state->vote_for_, state->vote_for_inited_); -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::try_precommit( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act, - bool& made) { - made = false; - if (state->check_precommit_received_from(src_idx)) { - return state; - } - bool found; - auto block = state->get_voted_block(desc, found); - if (!found) { - return state; - } - made = true; - auto block_id = SentBlock::get_block_id(block); - - if (act) { - if (act->get_id() != ton_api::validatorSession_message_precommit::ID) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: expected PRECOMMIT(" << block_id << ")"; - } else { - auto x = static_cast(act); - if (x->candidate_ != block_id) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: expected PRECOMMIT(" << block_id << ")"; - } - } - } else { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) - << "]: making implicit PRECOMMIT(" << block_id << ")"; - } - - auto v = CntVector::change(desc, state->precommitted_, src_idx, true); - return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, state->votes_, std::move(v), state->vote_for_, - state->vote_for_inited_); -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::make_one( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act, - bool& made) { - made = false; - state = try_vote(desc, state, src_idx, att, round, act, made); - if (made) { - return state; - } - state = try_precommit(desc, state, src_idx, att, round, act, made); - if (made) { - return state; - } - if (act && act->get_id() != ton_api::validatorSession_message_empty::ID) { - VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act - << "]: invalid message: expected EMPTY"; - } - return state; -} - -const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action( - ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx, - td::uint32 att, const ton_api::validatorSession_round_Message* act, const ValidatorSessionRoundState* round) { - ton_api::downcast_call(*const_cast(act), - [&](auto& obj) { state = action(desc, state, src_idx, att, obj, round); }); - return state; -} - -bool ValidatorSessionRoundAttemptState::check_vote_received_from(td::uint32 src_idx) const { - if (!votes_) { - return false; - } - auto size = votes_->size(); - auto v = votes_->data(); - for (td::uint32 i = 0; i < size; i++) { - if (v[i]->check_block_is_voted_by(src_idx)) { - return true; - } - } - return false; -} - -bool ValidatorSessionRoundAttemptState::check_precommit_received_from(td::uint32 src_idx) const { - return precommitted_->at(src_idx); -} - -const SentBlock* ValidatorSessionRoundAttemptState::get_voted_block(ValidatorSessionDescription& desc, bool& f) const { - f = false; - if (!votes_) { - return nullptr; - } - - auto size = votes_->size(); - auto v = votes_->data(); - for (td::uint32 i = 0; i < size; i++) { - if (v[i]->check_block_is_voted(desc)) { - f = true; - return v[i]->get_block(); - } - } - return nullptr; -} - -bool ValidatorSessionRoundAttemptState::check_attempt_is_precommitted(ValidatorSessionDescription& desc) const { - ValidatorWeight weight = 0; - for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { - if (precommitted_->at(i)) { - weight += desc.get_node_weight(i); - if (weight >= desc.get_cutoff_weight()) { - return true; - } - } - } - return false; -} - -tl_object_ptr ValidatorSessionRoundAttemptState::create_action( - ValidatorSessionDescription& desc, const ValidatorSessionRoundState* round, td::uint32 src_idx, - td::uint32 att) const { - if (!check_vote_received_from(src_idx)) { - auto found = false; - auto B = round->choose_block_to_vote(desc, src_idx, att, vote_for_, vote_for_inited_, found); - if (found) { - auto block_id = SentBlock::get_block_id(B); - return create_tl_object(round->get_seqno(), seqno_, block_id); - } - } - if (!check_precommit_received_from(src_idx)) { - bool f = false; - auto B = get_voted_block(desc, f); - - if (f) { - auto block_id = SentBlock::get_block_id(B); - - return create_tl_object(round->get_seqno(), seqno_, block_id); - } - } - - return create_tl_object(round->get_seqno(), seqno_); -} - -void ValidatorSessionRoundAttemptState::dump(ValidatorSessionDescription& desc, td::StringBuilder& sb) const { - sb << "attempt=" << seqno_ << "\n"; - sb << ">>>>\n"; - - if (vote_for_inited_) { - sb << "vote_for=" << (vote_for_ ? vote_for_->get_src_idx() : std::numeric_limits::max()) << "\n"; - } else { - sb << "vote_for=NONE\n"; - } - - if (votes_) { - auto s = votes_->size(); - sb << "votes: "; - std::vector R; - R.resize(desc.get_total_nodes(), -1); - for (td::uint32 i = 0; i < s; i++) { - const auto e = votes_->at(i); - const auto& x = e->get_voters_list(); - for (td::uint32 j = 0; j < desc.get_total_nodes(); j++) { - if (x->at(j)) { - R[j] = e->get_src_idx(); - } - } - } - for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { - sb << R[i] << " "; - } - sb << "\n"; - } else { - sb << "votes: EMPTY\n"; - } - - sb << "precommits: "; - for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) { - const auto e = precommitted_->at(i); - if (e) { - sb << "+ "; - } else { - sb << "- "; - } - } - sb << "\n"; - sb << "<<<<\n"; -} - // // // ROUND STATE @@ -821,8 +367,6 @@ const ValidatorSessionRoundState* ValidatorSessionRoundState::merge(ValidatorSes return SessionBlockCandidateSignature::merge(desc, l, r); }); - //auto sent_vec = SentBlockVector::merge(desc, left->sent_blocks_, right->sent_blocks_); - auto sent = ApproveVector::merge(desc, left->sent_blocks_, right->sent_blocks_, [&](const SessionBlockCandidate* l, const SessionBlockCandidate* r) { return SessionBlockCandidate::merge(desc, l, r); @@ -985,9 +529,6 @@ const ValidatorSessionRoundState* ValidatorSessionRoundState::forward_action_to_ attempt = ValidatorSessionRoundAttemptState::create(desc, att); } - bool had_voted_block; - attempt->get_voted_block(desc, had_voted_block); - ton_api::downcast_call(*const_cast(act), [&](auto& obj) { attempt = ValidatorSessionRoundAttemptState::action(desc, attempt, src_idx, att, obj, state); }); @@ -1302,6 +843,8 @@ std::vector ValidatorSessionRoundState::choose_blocks_to_appro CHECK(prio >= 0); td::uint32 blk_src_idx = B->get_src_idx(); if (was_source.count(blk_src_idx) > 0) { + // Any honest validator submits at most one block in a round + // Therefore, we can ignore all blocks from a node if it submits more than one x[prio] = nullptr; } else { was_source.insert(blk_src_idx); diff --git a/validator-session/validator-session-state.h b/validator-session/validator-session-state.h index 37708cba..4efaf77f 100644 --- a/validator-session/validator-session-state.h +++ b/validator-session/validator-session-state.h @@ -24,15 +24,15 @@ #include "common/io.hpp" #include "persistent-vector.h" - #include "validator-session-description.h" - #include "validator-session-types.h" +#include "validator-session-round-attempt-state.h" #include namespace td { +td::StringBuilder& operator<<(td::StringBuilder& sb, const ton::ton_api::validatorSession_round_Message& message); td::StringBuilder& operator<<(td::StringBuilder& sb, const ton::ton_api::validatorSession_round_Message* message); } @@ -41,410 +41,6 @@ namespace ton { namespace validatorsession { -using HashType = ValidatorSessionDescription::HashType; - -constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_WARNING) = verbosity_WARNING; -constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_NOTICE) = verbosity_DEBUG; -constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_INFO) = verbosity_DEBUG; -constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_DEBUG) = verbosity_DEBUG; -constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_EXTRA_DEBUG) = verbosity_DEBUG + 1; - -struct SessionBlockCandidateSignature : public ValidatorSessionDescription::RootObject { - public: - static auto create_hash(ValidatorSessionDescription& desc, td::Slice data) { - auto obj = create_tl_object(desc.compute_hash(data)); - return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); - } - - static bool compare(const RootObject* r, td::Slice data, HashType hash) { - if (!r || r->get_size() < sizeof(SessionBlockCandidateSignature)) { - return false; - } - auto R = static_cast(r); - return R->hash_ == hash && R->data_.ubegin() == data.ubegin() && R->data_.size() == data.size(); - } - - static auto lookup(ValidatorSessionDescription& desc, td::Slice data, HashType hash, bool temp) { - auto r = desc.get_by_hash(hash, temp); - if (compare(r, data, hash)) { - desc.on_reuse(); - return static_cast(r); - } - return static_cast(nullptr); - } - static SessionBlockCandidateSignature* create(ValidatorSessionDescription& desc, td::BufferSlice value) { - auto hash = create_hash(desc, value.as_slice()); - auto d = static_cast(desc.alloc(value.size(), 8, false)); - td::MutableSlice s{d, value.size()}; - s.copy_from(value.as_slice()); - return new (desc, true) SessionBlockCandidateSignature{desc, s, hash}; - } - static const SessionBlockCandidateSignature* move_to_persistent(ValidatorSessionDescription& desc, - const SessionBlockCandidateSignature* b) { - if (desc.is_persistent(b)) { - return b; - } - CHECK(desc.is_persistent(b->data_.ubegin())); - auto r = lookup(desc, b->data_, b->hash_, false); - if (r) { - return r; - } - return new (desc, false) SessionBlockCandidateSignature{desc, b->data_, b->hash_}; - } - static const SessionBlockCandidateSignature* merge(ValidatorSessionDescription& desc, - const SessionBlockCandidateSignature* l, - const SessionBlockCandidateSignature* r); - SessionBlockCandidateSignature(ValidatorSessionDescription& desc, td::Slice data, HashType hash) - : RootObject(sizeof(SessionBlockCandidateSignature)), data_{data}, hash_(std::move(hash)) { - desc.update_hash(this, hash_); - } - td::BufferSlice value() const { - return td::BufferSlice{data_}; - } - td::Slice as_slice() const { - return data_; - } - auto get_hash(ValidatorSessionDescription& desc) const { - return hash_; - } - - private: - const td::Slice data_; - const HashType hash_; -}; - -using SessionBlockCandidateSignatureVector = CntVector; - -class SentBlock : public ValidatorSessionDescription::RootObject { - public: - static HashType create_hash(ValidatorSessionDescription& desc, td::uint32 src_idx, ValidatorSessionRootHash root_hash, - ValidatorSessionFileHash file_hash, - ValidatorSessionCollatedDataFileHash collated_data_file_hash) { - auto obj = create_tl_object(src_idx, get_vs_hash(desc, root_hash), - get_vs_hash(desc, file_hash), - get_vs_hash(desc, collated_data_file_hash)); - return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); - } - static bool compare(const RootObject* root_object, td::uint32 src_idx, const ValidatorSessionRootHash& root_hash, - const ValidatorSessionFileHash& file_hash, - const ValidatorSessionCollatedDataFileHash& collated_data_file_hash, HashType hash) { - if (!root_object || root_object->get_size() < sizeof(SentBlock)) { - return false; - } - auto obj = static_cast(root_object); - return obj->src_idx_ == src_idx && obj->root_hash_ == root_hash && obj->file_hash_ == file_hash && - obj->collated_data_file_hash_ == collated_data_file_hash && obj->hash_ == hash; - } - static auto lookup(ValidatorSessionDescription& desc, td::uint32 src_idx, const ValidatorSessionRootHash& root_hash, - const ValidatorSessionFileHash& file_hash, - const ValidatorSessionCollatedDataFileHash& collated_data_file_hash, HashType hash, bool temp) { - auto r = desc.get_by_hash(hash, temp); - if (compare(r, src_idx, root_hash, file_hash, collated_data_file_hash, hash)) { - desc.on_reuse(); - return static_cast(r); - } - return static_cast(nullptr); - } - static const SentBlock* create(ValidatorSessionDescription& desc, td::uint32 src_idx, - const ValidatorSessionRootHash& root_hash, const ValidatorSessionFileHash& file_hash, - const ValidatorSessionCollatedDataFileHash& collated_data_file_hash) { - auto hash = create_hash(desc, src_idx, root_hash, file_hash, collated_data_file_hash); - auto r = lookup(desc, src_idx, root_hash, file_hash, collated_data_file_hash, hash, true); - if (r) { - return r; - } - - auto candidate_id = desc.candidate_id(src_idx, root_hash, file_hash, collated_data_file_hash); - - return new (desc, true) SentBlock{desc, src_idx, root_hash, file_hash, collated_data_file_hash, candidate_id, hash}; - } - static const SentBlock* create(ValidatorSessionDescription& desc, const ValidatorSessionCandidateId& zero) { - CHECK(zero.is_zero()); - auto hash = create_hash(desc, 0, ValidatorSessionRootHash::zero(), ValidatorSessionFileHash::zero(), - ValidatorSessionCollatedDataFileHash::zero()); - - return new (desc, true) SentBlock{desc, - 0, - ValidatorSessionRootHash::zero(), - ValidatorSessionFileHash::zero(), - ValidatorSessionCollatedDataFileHash::zero(), - zero, - hash}; - } - static const SentBlock* move_to_persistent(ValidatorSessionDescription& desc, const SentBlock* b) { - if (desc.is_persistent(b)) { - return b; - } - auto r = lookup(desc, b->src_idx_, b->root_hash_, b->file_hash_, b->collated_data_file_hash_, b->hash_, false); - if (r) { - return r; - } - - return new (desc, false) SentBlock{ - desc, b->src_idx_, b->root_hash_, b->file_hash_, b->collated_data_file_hash_, b->candidate_id_, b->hash_}; - } - SentBlock(ValidatorSessionDescription& desc, td::uint32 src_idx, ValidatorSessionRootHash root_hash, - ValidatorSessionFileHash file_hash, ValidatorSessionCollatedDataFileHash collated_data_file_hash, - ValidatorSessionCandidateId candidate_id, HashType hash) - : RootObject(sizeof(SentBlock)) - , src_idx_(src_idx) - , root_hash_(std::move(root_hash)) - , file_hash_(std::move(file_hash)) - , collated_data_file_hash_(std::move(collated_data_file_hash)) - , candidate_id_(candidate_id) - , hash_(std::move(hash)) { - desc.update_hash(this, hash_); - } - auto get_src_idx() const { - return src_idx_; - } - auto get_root_hash() const { - return root_hash_; - } - auto get_file_hash() const { - return file_hash_; - } - auto get_collated_data_file_hash() const { - return collated_data_file_hash_; - } - static ValidatorSessionCandidateId get_block_id(const SentBlock* block) { - return block ? block->candidate_id_ : skip_round_candidate_id(); - } - HashType get_hash(ValidatorSessionDescription& desc) const { - return hash_; - } - bool operator<(const SentBlock& block) const { - if (src_idx_ < block.src_idx_) { - return true; - } - if (src_idx_ > block.src_idx_) { - return false; - } - if (candidate_id_ < block.candidate_id_) { - return true; - } - return false; - } - struct Compare { - bool operator()(const SentBlock* a, const SentBlock* b) const { - return *a < *b; - } - }; - - private: - const td::uint32 src_idx_; - const ValidatorSessionRootHash root_hash_; - const ValidatorSessionFileHash file_hash_; - const ValidatorSessionCollatedDataFileHash collated_data_file_hash_; - const ValidatorSessionCandidateId candidate_id_; - const HashType hash_; -}; - -class SessionBlockCandidate : public ValidatorSessionDescription::RootObject { - public: - static HashType create_hash(ValidatorSessionDescription& desc, HashType block, HashType approved) { - auto obj = create_tl_object(block, approved); - return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); - } - static bool compare(const RootObject* r, const SentBlock* block, const SessionBlockCandidateSignatureVector* approved, - HashType hash) { - if (!r || r->get_size() < sizeof(SessionBlockCandidate)) { - return false; - } - auto R = static_cast(r); - return R->block_ == block && R->approved_by_ == approved && R->hash_ == hash; - } - static auto lookup(ValidatorSessionDescription& desc, const SentBlock* block, - const SessionBlockCandidateSignatureVector* approved, HashType hash, bool temp) { - auto r = desc.get_by_hash(hash, temp); - if (compare(r, block, approved, hash)) { - desc.on_reuse(); - return static_cast(r); - } - return static_cast(nullptr); - } - static const SessionBlockCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block, - const SessionBlockCandidateSignatureVector* approved) { - auto hash = create_hash(desc, get_vs_hash(desc, block), get_vs_hash(desc, approved)); - - auto r = lookup(desc, block, approved, hash, true); - if (r) { - return r; - } - - return new (desc, true) SessionBlockCandidate(desc, block, approved, hash); - } - static const SessionBlockCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block) { - std::vector v; - v.resize(desc.get_total_nodes(), nullptr); - auto vec = SessionBlockCandidateSignatureVector::create(desc, std::move(v)); - return create(desc, block, vec); - } - static const SessionBlockCandidate* move_to_persistent(ValidatorSessionDescription& desc, - const SessionBlockCandidate* b) { - if (desc.is_persistent(b)) { - return b; - } - auto block = SentBlock::move_to_persistent(desc, b->block_); - auto approved = SessionBlockCandidateSignatureVector::move_to_persistent(desc, b->approved_by_); - auto r = lookup(desc, block, approved, b->hash_, false); - if (r) { - return r; - } - - return new (desc, false) SessionBlockCandidate{desc, block, approved, b->hash_}; - } - SessionBlockCandidate(ValidatorSessionDescription& desc, const SentBlock* block, - const SessionBlockCandidateSignatureVector* approved, HashType hash) - : RootObject{sizeof(SessionBlockCandidate)}, block_(block), approved_by_(approved), hash_(hash) { - desc.update_hash(this, hash_); - } - static const SessionBlockCandidate* merge(ValidatorSessionDescription& desc, const SessionBlockCandidate* l, - const SessionBlockCandidate* r); - auto get_block() const { - return block_; - } - auto get_id() const { - return SentBlock::get_block_id(block_); - } - auto get_src_idx() const { - return block_ ? block_->get_src_idx() : std::numeric_limits::max(); - } - bool check_block_is_approved_by(td::uint32 src_idx) const { - return approved_by_->at(src_idx); - } - bool check_block_is_approved(ValidatorSessionDescription& desc) const; - auto get_hash(ValidatorSessionDescription& desc) const { - return hash_; - } - auto get_approvers_list() const { - return approved_by_; - } - static const SessionBlockCandidate* push(ValidatorSessionDescription& desc, const SessionBlockCandidate* state, - td::uint32 src_idx, const SessionBlockCandidateSignature* sig) { - CHECK(state); - if (state->approved_by_->at(src_idx)) { - return state; - } - return create(desc, state->block_, - SessionBlockCandidateSignatureVector::change(desc, state->approved_by_, src_idx, sig)); - } - class Compare { - public: - bool operator()(const SessionBlockCandidate* l, const SessionBlockCandidate* r) { - return l->get_id() < r->get_id(); - } - }; - - private: - const SentBlock* block_; - const SessionBlockCandidateSignatureVector* approved_by_; - const HashType hash_; -}; - -class SessionVoteCandidate : public ValidatorSessionDescription::RootObject { - public: - static HashType create_hash(ValidatorSessionDescription& desc, HashType block, HashType voted) { - auto obj = create_tl_object(block, voted); - return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); - } - static bool compare(const RootObject* r, const SentBlock* block, const CntVector* voted, HashType hash) { - if (!r || r->get_size() < sizeof(SessionVoteCandidate)) { - return false; - } - auto R = static_cast(r); - return R->block_ == block && R->voted_by_ == voted && R->hash_ == hash; - } - static auto lookup(ValidatorSessionDescription& desc, const SentBlock* block, const CntVector* voted, - HashType hash, bool temp) { - auto r = desc.get_by_hash(hash, temp); - if (compare(r, block, voted, hash)) { - desc.on_reuse(); - return static_cast(r); - } - return static_cast(nullptr); - } - static const SessionVoteCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block, - const CntVector* voted) { - auto hash = create_hash(desc, get_vs_hash(desc, block), get_vs_hash(desc, voted)); - - auto r = lookup(desc, block, voted, hash, true); - if (r) { - return r; - } - - return new (desc, true) SessionVoteCandidate(desc, block, voted, hash); - } - static const SessionVoteCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block) { - std::vector v; - v.resize(desc.get_total_nodes(), false); - auto vec = CntVector::create(desc, std::move(v)); - return create(desc, block, vec); - } - static const SessionVoteCandidate* move_to_persistent(ValidatorSessionDescription& desc, - const SessionVoteCandidate* b) { - if (desc.is_persistent(b)) { - return b; - } - auto block = SentBlock::move_to_persistent(desc, b->block_); - auto voted = CntVector::move_to_persistent(desc, b->voted_by_); - auto r = lookup(desc, block, voted, b->hash_, false); - if (r) { - return r; - } - - return new (desc, false) SessionVoteCandidate{desc, block, voted, b->hash_}; - } - SessionVoteCandidate(ValidatorSessionDescription& desc, const SentBlock* block, const CntVector* voted, - HashType hash) - : RootObject{sizeof(SessionVoteCandidate)}, block_(block), voted_by_(voted), hash_(hash) { - desc.update_hash(this, hash_); - } - static const SessionVoteCandidate* merge(ValidatorSessionDescription& desc, const SessionVoteCandidate* l, - const SessionVoteCandidate* r); - auto get_block() const { - return block_; - } - auto get_id() const { - return SentBlock::get_block_id(block_); - } - auto get_src_idx() const { - return block_ ? block_->get_src_idx() : std::numeric_limits::max(); - } - bool check_block_is_voted_by(td::uint32 src_idx) const { - return voted_by_->at(src_idx); - } - bool check_block_is_voted(ValidatorSessionDescription& desc) const; - auto get_hash(ValidatorSessionDescription& desc) const { - return hash_; - } - auto get_voters_list() const { - return voted_by_; - } - static const SessionVoteCandidate* push(ValidatorSessionDescription& desc, const SessionVoteCandidate* state, - td::uint32 src_idx) { - CHECK(state); - if (state->voted_by_->at(src_idx)) { - return state; - } - return create(desc, state->block_, CntVector::change(desc, state->voted_by_, src_idx, true)); - } - class Compare { - public: - bool operator()(const SessionVoteCandidate* l, const SessionVoteCandidate* r) { - return l->get_id() < r->get_id(); - } - }; - - private: - const SentBlock* block_; - const CntVector* voted_by_; - const HashType hash_; -}; - -//using SentBlockVector = CntSortedVector; - -class ValidatorSessionRoundState; class ValidatorSessionOldRoundState : public ValidatorSessionDescription::RootObject { public: static HashType create_hash(ValidatorSessionDescription& desc, td::uint32 seqno, HashType block, HashType signatures, @@ -582,188 +178,6 @@ class ValidatorSessionOldRoundState : public ValidatorSessionDescription::RootOb const HashType hash_; }; -using VoteVector = CntSortedVector; -using ApproveVector = CntSortedVector; - -class ValidatorSessionRoundAttemptState : public ValidatorSessionDescription::RootObject { - public: - static HashType create_hash(ValidatorSessionDescription& desc, td::uint32 seqno, HashType votes, - HashType precommitted, bool vote_for_inited, HashType vote_for) { - auto obj = create_tl_object(seqno, votes, precommitted, - vote_for_inited, vote_for); - return desc.compute_hash(serialize_tl_object(obj, true).as_slice()); - } - static bool compare(const RootObject* r, td::uint32 seqno, const VoteVector* votes, - const CntVector* precommitted, const SentBlock* vote_for, bool vote_for_inited, - HashType hash) { - if (!r || r->get_size() < sizeof(ValidatorSessionRoundAttemptState)) { - return false; - } - auto R = static_cast(r); - return R->seqno_ == seqno && R->votes_ == votes && R->precommitted_ == precommitted && R->vote_for_ == vote_for && - R->vote_for_inited_ == vote_for_inited && R->hash_ == hash; - } - static auto lookup(ValidatorSessionDescription& desc, td::uint32 seqno, const VoteVector* votes, - const CntVector* precommitted, const SentBlock* vote_for, bool vote_for_inited, - HashType hash, bool temp) { - auto r = desc.get_by_hash(hash, temp); - if (compare(r, seqno, votes, precommitted, vote_for, vote_for_inited, hash)) { - desc.on_reuse(); - return static_cast(r); - } - return static_cast(nullptr); - } - static const ValidatorSessionRoundAttemptState* move_to_persistent(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* b) { - if (desc.is_persistent(b)) { - return b; - } - auto votes = VoteVector::move_to_persistent(desc, b->votes_); - auto precommitted = CntVector::move_to_persistent(desc, b->precommitted_); - auto vote_for = SentBlock::move_to_persistent(desc, b->vote_for_); - - auto r = lookup(desc, b->seqno_, votes, precommitted, vote_for, b->vote_for_inited_, b->hash_, false); - if (r) { - return r; - } - - return new (desc, false) ValidatorSessionRoundAttemptState{desc, b->seqno_, votes, precommitted, - vote_for, b->vote_for_inited_, b->hash_}; - } - - static const ValidatorSessionRoundAttemptState* create(ValidatorSessionDescription& desc, td::uint32 seqno, - const VoteVector* votes, const CntVector* precommitted, - const SentBlock* vote_for, bool vote_for_inited) { - auto hash = create_hash(desc, seqno, get_vs_hash(desc, votes), get_vs_hash(desc, precommitted), - get_vs_hash(desc, vote_for), vote_for_inited); - - auto r = lookup(desc, seqno, votes, precommitted, vote_for, vote_for_inited, hash, true); - if (r) { - return r; - } - - return new (desc, true) - ValidatorSessionRoundAttemptState(desc, seqno, votes, precommitted, vote_for, vote_for_inited, hash); - } - static const ValidatorSessionRoundAttemptState* create(ValidatorSessionDescription& desc, td::uint32 seqno) { - std::vector x; - x.resize(desc.get_total_nodes(), false); - auto p = CntVector::create(desc, std::move(x)); - - return create(desc, seqno, nullptr, p, nullptr, false); - } - - ValidatorSessionRoundAttemptState(ValidatorSessionDescription& desc, td::uint32 seqno, const VoteVector* votes, - const CntVector* precommitted, const SentBlock* vote_for, - bool vote_for_inited, HashType hash) - : RootObject{sizeof(ValidatorSessionRoundAttemptState)} - , seqno_(seqno) - , votes_(votes) - , precommitted_(precommitted) - , vote_for_(vote_for) - , vote_for_inited_(vote_for_inited) - , hash_(std::move(hash)) { - desc.update_hash(this, hash_); - } - auto get_hash(ValidatorSessionDescription& desc) const { - return hash_; - } - auto get_seqno() const { - return seqno_; - } - auto get_votes() const { - return votes_; - } - auto get_precommits() const { - return precommitted_; - } - const SentBlock* get_voted_block(ValidatorSessionDescription& desc, bool& f) const; - const SentBlock* get_vote_for_block(ValidatorSessionDescription& desc, bool& f) const { - f = vote_for_inited_; - return vote_for_; - } - bool check_attempt_is_precommitted(ValidatorSessionDescription& desc) const; - bool check_vote_received_from(td::uint32 src_idx) const; - bool check_precommit_received_from(td::uint32 src_idx) const; - - bool operator<(const ValidatorSessionRoundAttemptState& right) const { - return seqno_ < right.seqno_; - } - struct Compare { - bool operator()(const ValidatorSessionRoundAttemptState* a, const ValidatorSessionRoundAttemptState* b) const { - return *a < *b; - } - }; - - static const ValidatorSessionRoundAttemptState* merge(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* left, - const ValidatorSessionRoundAttemptState* right); - static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ton_api::validatorSession_message_voteFor& act, - const ValidatorSessionRoundState* round); - static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ton_api::validatorSession_message_vote& act, - const ValidatorSessionRoundState* round); - static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ton_api::validatorSession_message_precommit& act, - const ValidatorSessionRoundState* round); - static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ton_api::validatorSession_message_empty& act, - const ValidatorSessionRoundState* round); - static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ton_api::validatorSession_round_Message* action, - const ValidatorSessionRoundState* round); - template - static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, const T& action, - const ValidatorSessionRoundState* round) { - UNREACHABLE(); - } - static const ValidatorSessionRoundAttemptState* try_vote(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ValidatorSessionRoundState* round, - const ton_api::validatorSession_round_Message* cmp, - bool& made); - static const ValidatorSessionRoundAttemptState* try_precommit(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ValidatorSessionRoundState* round, - const ton_api::validatorSession_round_Message* cmp, - bool& made); - static const ValidatorSessionRoundAttemptState* make_one(ValidatorSessionDescription& desc, - const ValidatorSessionRoundAttemptState* state, - td::uint32 src_idx, td::uint32 att, - const ValidatorSessionRoundState* round, - const ton_api::validatorSession_round_Message* cmp, - bool& made); - tl_object_ptr create_action(ValidatorSessionDescription& desc, - const ValidatorSessionRoundState* round, - td::uint32 src_idx, td::uint32 att) const; - void dump(ValidatorSessionDescription& desc, td::StringBuilder& sb) const; - - private: - const td::uint32 seqno_; - const VoteVector* votes_; - const CntVector* precommitted_; - const SentBlock* vote_for_; - const bool vote_for_inited_; - const HashType hash_; -}; - -using AttemptVector = - CntSortedVector; class ValidatorSessionRoundState : public ValidatorSessionDescription::RootObject { public: @@ -962,6 +376,15 @@ class ValidatorSessionRoundState : public ValidatorSessionDescription::RootObjec void dump(ValidatorSessionDescription& desc, td::StringBuilder& sb, td::uint32 att) const; void dump_cur_attempt(ValidatorSessionDescription& desc, td::StringBuilder& sb) const; + void for_each_sent_block(std::function foo) const { + if (!sent_blocks_) { + return; + } + for (td::uint32 i = 0; i < sent_blocks_->size(); ++i) { + foo(sent_blocks_->at(i)); + } + } + private: const SentBlock* precommitted_block_; const td::uint32 seqno_; @@ -1055,6 +478,14 @@ class ValidatorSessionState : public ValidatorSessionDescription::RootObject { auto get_ts(td::uint32 src_idx) const { return att_->at(src_idx); } + td::uint32 cur_attempt_in_round(const ValidatorSessionDescription& desc) const { + td::uint32 first_attempt = cur_round_->get_first_attempt(desc.get_self_idx()); + td::uint32 cur_attempt = desc.get_attempt_seqno(desc.get_ts()); + if (cur_attempt < first_attempt || first_attempt == 0) { + return 0; + } + return cur_attempt - first_attempt; + } const SentBlock* choose_block_to_sign(ValidatorSessionDescription& desc, td::uint32 src_idx, bool& found) const; const SentBlock* get_committed_block(ValidatorSessionDescription& desc, td::uint32 seqno) const; @@ -1102,6 +533,19 @@ class ValidatorSessionState : public ValidatorSessionDescription::RootObject { cur_round_->dump_cur_attempt(desc, sb); } + void for_each_cur_round_sent_block(std::function foo) const { + cur_round_->for_each_sent_block(std::move(foo)); + } + + const SentBlock* get_cur_round_precommitted_block() const { + bool found; + return cur_round_->get_precommitted_block(found); + } + + const CntVector* get_cur_round_signatures() const { + return cur_round_->get_signatures(); + } + static const ValidatorSessionState* make_one(ValidatorSessionDescription& desc, const ValidatorSessionState* state, td::uint32 src_idx, td::uint32 att, bool& made); static const ValidatorSessionState* make_all(ValidatorSessionDescription& desc, const ValidatorSessionState* state, diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index 357f6217..7147bf2d 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -27,6 +27,12 @@ namespace ton { namespace validatorsession { +constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_WARNING) = verbosity_WARNING; +constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_NOTICE) = verbosity_DEBUG; +constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_INFO) = verbosity_DEBUG; +constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_DEBUG) = verbosity_DEBUG; +constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_EXTRA_DEBUG) = verbosity_DEBUG + 1; + using ValidatorSessionRootHash = td::Bits256; using ValidatorSessionFileHash = td::Bits256; using ValidatorSessionCollatedDataFileHash = td::Bits256; @@ -68,18 +74,74 @@ struct ValidatorSessionStats { struct Producer { PublicKeyHash id = PublicKeyHash::zero(); + ValidatorSessionCandidateId candidate_id = ValidatorSessionCandidateId::zero(); int block_status = status_none; - td::uint64 block_timestamp = 0; + double block_timestamp = -1.0; + td::Bits256 root_hash = td::Bits256::zero(); + td::Bits256 file_hash = td::Bits256::zero(); + std::string comment; + + bool is_accepted = false; + bool is_ours = false; + double got_submit_at = -1.0; + double collation_time = -1.0; + double validation_time = -1.0; + double collated_at = -1.0; + double validated_at = -1.0; + bool collation_cached = false; + bool validation_cached = false; + double gen_utime = -1.0; + + std::vector approvers, signers; + ValidatorWeight approved_weight = 0; + ValidatorWeight signed_weight = 0; + double approved_33pct_at = -1.0; + double approved_66pct_at = -1.0; + double signed_33pct_at = -1.0; + double signed_66pct_at = -1.0; + + double serialize_time = -1.0; + double deserialize_time = -1.0; + td::int32 serialized_size = -1; + + void set_approved_by(td::uint32 id, ValidatorWeight weight, ValidatorWeight total_weight) { + if (!approvers.at(id)) { + approvers.at(id) = true; + approved_weight += weight; + if (approved_33pct_at <= 0.0 && approved_weight >= total_weight / 3 + 1) { + approved_33pct_at = td::Clocks::system(); + } + if (approved_66pct_at <= 0.0 && approved_weight >= (total_weight * 2) / 3 + 1) { + approved_66pct_at = td::Clocks::system(); + } + } + } + + void set_signed_by(td::uint32 id, ValidatorWeight weight, ValidatorWeight total_weight) { + if (!signers.at(id)) { + signers.at(id) = true; + signed_weight += weight; + if (signed_33pct_at <= 0.0 && signed_weight >= total_weight / 3 + 1) { + signed_33pct_at = td::Clocks::system(); + } + if (signed_66pct_at <= 0.0 && signed_weight >= (total_weight * 2) / 3 + 1) { + signed_66pct_at = td::Clocks::system(); + } + } + } }; struct Round { - td::uint64 timestamp = 0; + double timestamp = -1.0; std::vector producers; }; td::uint32 first_round; std::vector rounds; - td::uint64 timestamp = 0; + bool success = false; + ValidatorSessionId session_id = ValidatorSessionId::zero(); + CatchainSeqno cc_seqno = 0; + double timestamp = -1.0; PublicKeyHash self = PublicKeyHash::zero(); PublicKeyHash creator = PublicKeyHash::zero(); td::uint32 total_validators = 0; @@ -90,6 +152,38 @@ struct ValidatorSessionStats { ValidatorWeight approve_signatures_weight = 0; }; +struct NewValidatorGroupStats { + struct Node { + PublicKeyHash id = PublicKeyHash::zero(); + ValidatorWeight weight = 0; + }; + + ValidatorSessionId session_id = ValidatorSessionId::zero(); + ShardIdFull shard{masterchainId}; + CatchainSeqno cc_seqno = 0; + BlockSeqno last_key_block_seqno = 0; + double timestamp = -1.0; + td::uint32 self_idx = 0; + std::vector nodes; +}; + +struct EndValidatorGroupStats { + struct Node { + PublicKeyHash id = PublicKeyHash::zero(); + td::uint32 catchain_blocks = 0; + }; + + ValidatorSessionId session_id = ValidatorSessionId::zero(); + double timestamp = -1.0; + std::vector nodes; +}; + +struct BlockSourceInfo { + td::uint32 round, first_block_round; + PublicKey source; + td::int32 source_priority; +}; + } // namespace validatorsession } // namespace ton diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 3f56e3a3..3a913990 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -19,6 +19,9 @@ #include "validator-session.hpp" #include "td/utils/Random.h" #include "td/utils/crypto.h" +#include "candidate-serializer.h" +#include "td/utils/overloaded.h" +#include "ton/ton-tl.hpp" namespace ton { @@ -39,14 +42,14 @@ void ValidatorSessionImpl::process_blocks(std::vector on_new_round(real_state_->cur_round_seqno()); } - td::uint32 cnt = 0; + [[maybe_unused]] td::uint32 cnt = 0; auto ts = description().get_ts(); auto att = description().get_attempt_seqno(ts); std::vector> msgs; if (generated_ && !sent_generated_) { - auto it = blocks_[0].find(generated_block_); - CHECK(it != blocks_[0].end()); + auto it = blocks_.find(generated_block_); + CHECK(it != blocks_.end()); auto &B = it->second; auto file_hash = sha256_bits256(B->data_); @@ -85,6 +88,7 @@ void ValidatorSessionImpl::process_blocks(std::vector for (auto &msg : msgs) { VLOG(VALIDATOR_SESSION_INFO) << this << ": applying action: " << msg.get(); + stats_process_action(local_idx(), *msg); real_state_ = ValidatorSessionState::action(description(), real_state_, local_idx(), att, msg.get()); } @@ -166,6 +170,7 @@ void ValidatorSessionImpl::preprocess_block(catchain::CatChainBlock *block) { for (auto &msg : B->actions_) { VLOG(VALIDATOR_SESSION_INFO) << this << "[node " << description().get_source_id(block->source()) << "][block " << block->hash() << "]: applying action " << msg.get(); + stats_process_action(block->source(), *msg); state = ValidatorSessionState::action(description(), state, block->source(), att, msg.get()); } state = ValidatorSessionState::make_all(description(), state, block->source(), att); @@ -203,9 +208,29 @@ void ValidatorSessionImpl::preprocess_block(catchain::CatChainBlock *block) { << "ms: state=" << state->get_hash(description()); } -void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice data) { +bool ValidatorSessionImpl::ensure_candidate_unique(td::uint32 src_idx, td::uint32 round, + ValidatorSessionCandidateId block_id) { + auto it = src_round_candidate_[src_idx].find(round); + if (it != src_round_candidate_[src_idx].end() && it->second != block_id) { + VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << description_->get_source_adnl_id(src_idx) << "][candidate " + << block_id << "]: this node already has candidate in round " << round; + return false; + } + src_round_candidate_[src_idx][round] = block_id; + return true; +} + +void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice data, + td::optional expected_id, + bool is_overlay_broadcast) { + // Note: src is not necessarily equal to the sender of this message: + // If requested using get_broadcast_p2p, src is the creator of the block, sender possibly is some other node. auto src_idx = description().get_source_idx(src); - auto R = fetch_tl_object(data.clone(), true); + td::Timer deserialize_timer; + auto R = + deserialize_candidate(data, compress_block_candidates_, + description().opts().max_block_size + description().opts().max_collated_data_size + 1024); + double deserialize_time = deserialize_timer.elapsed(); if (R.is_error()) { VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) << "]: failed to parse: " << R.move_as_error(); @@ -230,13 +255,35 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice auto block_round = static_cast(candidate->round_); auto block_id = description().candidate_id(src_idx, candidate->root_hash_, file_hash, collated_data_file_hash); - if (block_round < cur_round_ || block_round >= cur_round_ + blocks_.size()) { + if (expected_id && expected_id.value() != block_id) { + VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) + << "]: id mismatch"; + return; + } + + auto stat = stats_get_candidate_stat(block_round, src, block_id); + if (stat) { + if (stat->block_status == ValidatorSessionStats::status_none) { + stat->block_status = ValidatorSessionStats::status_received; + } + if (stat->block_timestamp <= 0.0) { + stat->block_timestamp = td::Clocks::system(); + } + stat->deserialize_time = deserialize_time; + stat->serialized_size = data.size(); + stat->root_hash = candidate->root_hash_; + stat->file_hash = file_hash; + } + + if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK || + block_round >= cur_round_ + MAX_FUTURE_ROUND_BLOCK) { VLOG(VALIDATOR_SESSION_NOTICE) << this << "[node " << src << "][broadcast " << block_id << "]: bad round=" << block_round << " cur_round" << cur_round_; return; } - auto it = blocks_[block_round - cur_round_].find(block_id); - if (it != blocks_[block_round - cur_round_].end()) { + auto it = blocks_.find(block_id); + if (it != blocks_.end()) { + it->second->round_ = std::max(it->second->round_, block_round); VLOG(VALIDATOR_SESSION_INFO) << this << "[node " << src << "][broadcast " << block_id << "]: duplicate"; return; } @@ -248,7 +295,11 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice return; } - blocks_[block_round - cur_round_][block_id] = std::move(candidate); + if (is_overlay_broadcast && !ensure_candidate_unique(src_idx, block_round, block_id)) { + return; + } + + blocks_[block_id] = std::move(candidate); VLOG(VALIDATOR_SESSION_WARNING) << this << ": received broadcast " << block_id; if (block_round != cur_round_) { @@ -260,7 +311,6 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice CHECK(!pending_reject_.count(block_id)); CHECK(!rejected_.count(block_id)); - stats_set_candidate_status(cur_round_, src, ValidatorSessionStats::status_received); auto v = virtual_state_->choose_blocks_to_approve(description(), local_idx()); for (auto &b : v) { if (b && SentBlock::get_block_id(b) == block_id) { @@ -315,25 +365,33 @@ void ValidatorSessionImpl::process_query(PublicKeyHash src, td::BufferSlice data } CHECK(block); - auto P = td::PromiseCreator::lambda( - [promise = std::move(promise), src = f->id_->src_, round_id](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error_prefix("failed to get candidate: ")); - } else { - auto c = R.move_as_ok(); - auto obj = create_tl_object( - src, round_id, c.id.root_hash, std::move(c.data), std::move(c.collated_data)); - promise.set_value(serialize_tl_object(obj, true)); - } - }); + auto P = td::PromiseCreator::lambda([promise = std::move(promise), src = f->id_->src_, round_id, + compress = compress_block_candidates_](td::Result R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix("failed to get candidate: ")); + } else { + auto c = R.move_as_ok(); + auto obj = create_tl_object(src, round_id, c.id.root_hash, std::move(c.data), + std::move(c.collated_data)); + promise.set_result(serialize_candidate(obj, compress)); + } + }); callback_->get_approved_candidate(description().get_source_public_key(block->get_src_idx()), f->id_->root_hash_, f->id_->file_hash_, f->id_->collated_data_file_hash_, std::move(P)); } void ValidatorSessionImpl::candidate_decision_fail(td::uint32 round, ValidatorSessionCandidateId hash, - std::string result, td::uint32 src, td::BufferSlice proof) { - stats_set_candidate_status(round, description().get_source_id(src), ValidatorSessionStats::status_rejected); + std::string result, td::uint32 src, td::BufferSlice proof, + double validation_time, bool validation_cached) { + auto stat = stats_get_candidate_stat(round, description().get_source_id(src), hash); + if (stat) { + stat->block_status = ValidatorSessionStats::status_rejected; + stat->comment = result; + stat->validation_time = validation_time; + stat->validated_at = td::Clocks::system(); + stat->validation_cached = validation_cached; + } if (round != cur_round_) { return; } @@ -347,8 +405,17 @@ void ValidatorSessionImpl::candidate_decision_fail(td::uint32 round, ValidatorSe } void ValidatorSessionImpl::candidate_decision_ok(td::uint32 round, ValidatorSessionCandidateId hash, RootHash root_hash, - FileHash file_hash, td::uint32 src, td::uint32 ok_from) { - stats_set_candidate_status(round, description().get_source_id(src), ValidatorSessionStats::status_approved); + FileHash file_hash, td::uint32 src, td::uint32 ok_from, + double validation_time, bool validation_cached) { + auto stat = stats_get_candidate_stat(round, description().get_source_id(src), hash); + if (stat) { + stat->block_status = ValidatorSessionStats::status_approved; + stat->comment = PSTRING() << "ts=" << ok_from; + stat->validation_time = validation_time; + stat->gen_utime = (double)ok_from; + stat->validated_at = td::Clocks::system(); + stat->validation_cached = validation_cached; + } if (round != cur_round_) { return; } @@ -385,10 +452,8 @@ void ValidatorSessionImpl::candidate_approved_signed(td::uint32 round, Validator } void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCandidateId root_hash, - td::BufferSlice data, td::BufferSlice collated_data) { - if (round != cur_round_) { - return; - } + td::BufferSlice data, td::BufferSlice collated_data, double collation_time, + bool collation_cached) { if (data.size() > description().opts().max_block_size || collated_data.size() > description().opts().max_collated_data_size) { LOG(ERROR) << this << ": generated candidate is too big. Dropping. size=" << data.size() << " " @@ -397,17 +462,33 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCan } auto file_hash = sha256_bits256(data.as_slice()); auto collated_data_file_hash = sha256_bits256(collated_data.as_slice()); + auto block_id = description().candidate_id(local_idx(), root_hash, file_hash, collated_data_file_hash); + auto stat = stats_get_candidate_stat(round, local_id(), block_id); + if (stat) { + stat->block_status = ValidatorSessionStats::status_received; + stat->collation_time = collation_time; + stat->collated_at = td::Clocks::system(); + stat->block_timestamp = td::Clocks::system(); + stat->collation_cached = collation_cached; + stat->root_hash = root_hash; + stat->file_hash = file_hash; + } + if (round != cur_round_) { + return; + } + td::Timer serialize_timer; auto b = create_tl_object(local_id().tl(), round, root_hash, std::move(data), std::move(collated_data)); - - auto B = serialize_tl_object(b, true); - - auto block_id = description().candidate_id(local_idx(), root_hash, file_hash, collated_data_file_hash); + auto B = serialize_candidate(b, compress_block_candidates_).move_as_ok(); + if (stat) { + stat->serialize_time = serialize_timer.elapsed(); + stat->serialized_size = B.size(); + } td::actor::send_closure(catchain_, &catchain::CatChain::send_broadcast, std::move(B)); - blocks_[0].emplace(block_id, std::move(b)); + blocks_.emplace(block_id, std::move(b)); pending_generate_ = false; generated_ = true; generated_block_ = block_id; @@ -463,16 +544,18 @@ void ValidatorSessionImpl::check_generate_slot() { td::PerfWarningTimer timer{"too long block generation", 1.0}; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), print_id = print_id(), timer = std::move(timer), - round = cur_round_](td::Result R) { + round = cur_round_](td::Result R) { if (R.is_ok()) { - auto c = R.move_as_ok(); + auto c = std::move(R.ok_ref().candidate); td::actor::send_closure(SelfId, &ValidatorSessionImpl::generated_block, round, c.id.root_hash, - c.data.clone(), c.collated_data.clone()); + c.data.clone(), c.collated_data.clone(), timer.elapsed(), R.ok().is_cached); } else { LOG(WARNING) << print_id << ": failed to generate block candidate: " << R.move_as_error(); } }); - callback_->on_generate_slot(cur_round_, std::move(P)); + callback_->on_generate_slot( + BlockSourceInfo{cur_round_, first_block_round_, description().get_source_public_key(local_idx()), priority}, + std::move(P)); } else { alarm_timestamp().relax(t); } @@ -506,12 +589,29 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { } if (block) { - auto T = td::Timestamp::at(round_started_at_.at() + description().get_delay(block->get_src_idx()) + 2.0); - auto it = blocks_[0].find(block_id); + if (!ensure_candidate_unique(block->get_src_idx(), cur_round_, SentBlock::get_block_id(block))) { + return; + } + auto T = td::Timestamp::at(round_started_at_.at() + description().get_delay(block->get_src_idx()) + + REQUEST_BROADCAST_P2P_DELAY); + auto it = blocks_.find(block_id); - if (it != blocks_[0].end()) { + if (it != blocks_.end()) { + it->second->round_ = std::max(it->second->round_, cur_round_); td::PerfWarningTimer timer{"too long block validation", 1.0}; auto &B = it->second; + auto stat = stats_get_candidate_stat(B->round_, PublicKeyHash{B->src_}); + if (stat) { + // Can happen if block is cached from previous round + if (stat->block_status == ValidatorSessionStats::status_none) { + stat->block_status = ValidatorSessionStats::status_received; + } + if (stat->block_timestamp <= 0.0) { + stat->block_timestamp = td::Clocks::system(); + } + stat->root_hash = B->root_hash_; + stat->file_hash = td::sha256_bits256(B->data_); + } auto P = td::PromiseCreator::lambda([round = cur_round_, hash = block_id, root_hash = block->get_root_hash(), file_hash = block->get_file_hash(), timer = std::move(timer), @@ -525,17 +625,18 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { auto R = res.move_as_ok(); if (R.is_ok()) { td::actor::send_closure(SelfId, &ValidatorSessionImpl::candidate_decision_ok, round, hash, root_hash, - file_hash, src, R.ok_from()); + file_hash, src, R.ok_from(), timer.elapsed(), R.is_cached()); } else { td::actor::send_closure(SelfId, &ValidatorSessionImpl::candidate_decision_fail, round, hash, R.reason(), - src, R.proof()); + src, R.proof(), timer.elapsed(), R.is_cached()); } }); pending_approve_.insert(block_id); - CHECK(static_cast(cur_round_) == B->round_); - callback_->on_candidate(cur_round_, description().get_source_public_key(block->get_src_idx()), B->root_hash_, - B->data_.clone(), B->collated_data_.clone(), std::move(P)); + callback_->on_candidate( + BlockSourceInfo{cur_round_, first_block_round_, description().get_source_public_key(block->get_src_idx()), + description().get_node_priority(block->get_src_idx(), cur_round_)}, + B->root_hash_, B->data_.clone(), B->collated_data_.clone(), std::move(P)); } else if (T.is_in_past()) { if (!active_requests_.count(block_id)) { auto v = virtual_state_->get_block_approvers(description(), block_id); @@ -543,20 +644,22 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { auto id = description().get_source_id(v[td::Random::fast(0, static_cast(v.size() - 1))]); auto src_id = description().get_source_id(block->get_src_idx()); active_requests_.insert(block_id); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id, src_id, print_id = print_id(), - hash = block_id, round = cur_round_](td::Result R) { - td::actor::send_closure(SelfId, &ValidatorSessionImpl::end_request, round, hash); - if (R.is_error()) { - VLOG(VALIDATOR_SESSION_WARNING) - << print_id << ": failed to get candidate " << hash << " from " << id << ": " << R.move_as_error(); - } else { - td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src_id, R.move_as_ok()); - } - }); + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), id, src_id, print_id = print_id(), hash = block_id, round = cur_round_, + candidate_id = SentBlock::get_block_id(block)](td::Result R) { + td::actor::send_closure(SelfId, &ValidatorSessionImpl::end_request, round, hash); + if (R.is_error()) { + VLOG(VALIDATOR_SESSION_WARNING) << print_id << ": failed to get candidate " << hash << " from " << id + << ": " << R.move_as_error(); + } else { + td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src_id, R.move_as_ok(), + candidate_id, false); + } + }); get_broadcast_p2p(id, block->get_file_hash(), block->get_collated_data_file_hash(), description().get_source_id(block->get_src_idx()), cur_round_, block->get_root_hash(), - std::move(P), td::Timestamp::in(2.0)); + std::move(P), td::Timestamp::in(15.0)); } else { LOG(VALIDATOR_SESSION_DEBUG) << this << ": no nodes to download candidate " << block << " from"; } @@ -584,10 +687,11 @@ void ValidatorSessionImpl::get_broadcast_p2p(PublicKeyHash node, ValidatorSessio round, create_tl_object(src.tl(), root_hash, file_hash, collated_data_file_hash)); - td::actor::send_closure(catchain_, &catchain::CatChain::send_query_via, node, "download candidate", - std::move(promise), timeout, serialize_tl_object(obj, true), - description().opts().max_block_size + description().opts().max_collated_data_size + 1024, - rldp_); + td::actor::send_closure( + catchain_, &catchain::CatChain::send_query_via, node, "download candidate", std::move(promise), timeout, + serialize_tl_object(obj, true), + description().opts().max_block_size + description().opts().max_collated_data_size + MAX_CANDIDATE_EXTRA_SIZE, + rldp_); } void ValidatorSessionImpl::check_sign_slot() { @@ -714,13 +818,25 @@ void ValidatorSessionImpl::request_new_block(bool now) { } else { double lambda = 10.0 / description().get_total_nodes(); double x = -1 / lambda * log(td::Random::fast(1, 999) * 0.001); - if (x > 0.5) { - x = 0.5; - } + x = std::min(x, get_current_max_block_delay()); // default = 0.4 td::actor::send_closure(catchain_, &catchain::CatChain::need_new_block, td::Timestamp::in(x)); } } +double ValidatorSessionImpl::get_current_max_block_delay() const { + td::uint32 att = real_state_->cur_attempt_in_round(*description_); + td::uint32 att1 = description_->opts().max_round_attempts; + if (att <= att1) { + return catchain_max_block_delay_; + } + td::uint32 att2 = att1 + 4; + if (att >= att2) { + return catchain_max_block_delay_slow_; + } + return catchain_max_block_delay_ + + (catchain_max_block_delay_slow_ - catchain_max_block_delay_) * (double)(att - att1) / (double)(att2 - att1); +} + void ValidatorSessionImpl::on_new_round(td::uint32 round) { if (round != 0) { CHECK(cur_round_ < round); @@ -743,7 +859,6 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { while (cur_round_ < round) { auto block = real_state_->get_committed_block(description(), cur_round_); - //CHECK(block); auto sigs = real_state_->get_committed_block_signatures(description(), cur_round_); CHECK(sigs); auto approve_sigs = real_state_->get_committed_block_approve_signatures(description(), cur_round_); @@ -773,41 +888,59 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { } } - auto it = blocks_[0].find(SentBlock::get_block_id(block)); + auto it = blocks_.find(SentBlock::get_block_id(block)); bool have_block = (bool)block; if (!have_block) { callback_->on_block_skipped(cur_round_); } else { - cur_stats_.timestamp = (td::uint64)td::Clocks::system(); - cur_stats_.total_validators = description().get_total_nodes(); - cur_stats_.total_weight = description().get_total_weight(); + cur_stats_.success = true; + cur_stats_.timestamp = td::Clocks::system(); cur_stats_.signatures = (td::uint32)export_sigs.size(); cur_stats_.signatures_weight = signatures_weight; cur_stats_.approve_signatures = (td::uint32)export_approve_sigs.size(); cur_stats_.approve_signatures_weight = approve_signatures_weight; cur_stats_.creator = description().get_source_id(block->get_src_idx()); - cur_stats_.self = description().get_source_id(local_idx()); - - if (it == blocks_[0].end()) { - callback_->on_block_committed(cur_round_, description().get_source_public_key(block->get_src_idx()), - block->get_root_hash(), block->get_file_hash(), td::BufferSlice(), - std::move(export_sigs), std::move(export_approve_sigs), std::move(cur_stats_)); - } else { - callback_->on_block_committed(cur_round_, description().get_source_public_key(block->get_src_idx()), - block->get_root_hash(), block->get_file_hash(), it->second->data_.clone(), - std::move(export_sigs), std::move(export_approve_sigs), std::move(cur_stats_)); + auto stat = stats_get_candidate_stat(cur_round_, cur_stats_.creator); + if (stat) { + stat->is_accepted = true; } + auto stats = cur_stats_; + while (!stats.rounds.empty() && stats.rounds.size() + stats.first_round - 1 > cur_round_) { + stats.rounds.pop_back(); + } + + BlockSourceInfo source_info{cur_round_, first_block_round_, + description().get_source_public_key(block->get_src_idx()), + description().get_node_priority(block->get_src_idx(), cur_round_)}; + if (it == blocks_.end()) { + callback_->on_block_committed(std::move(source_info), block->get_root_hash(), block->get_file_hash(), + td::BufferSlice(), std::move(export_sigs), std::move(export_approve_sigs), + std::move(stats)); + } else { + callback_->on_block_committed(std::move(source_info), block->get_root_hash(), block->get_file_hash(), + it->second->data_.clone(), std::move(export_sigs), std::move(export_approve_sigs), + std::move(stats)); + } + first_block_round_ = cur_round_ + 1; } cur_round_++; if (have_block) { stats_init(); } else { - stats_add_round(); + size_t round_idx = cur_round_ - cur_stats_.first_round; + while (round_idx >= cur_stats_.rounds.size()) { + stats_add_round(); + } + cur_stats_.rounds[round_idx].timestamp = td::Clocks::system(); } - for (size_t i = 0; i < blocks_.size() - 1; i++) { - blocks_[i] = std::move(blocks_[i + 1]); + auto it2 = blocks_.begin(); + while (it2 != blocks_.end()) { + if (it2->second->round_ < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK) { + it2 = blocks_.erase(it2); + } else { + ++it2; + } } - blocks_[blocks_.size() - 1].clear(); } round_started_at_ = td::Timestamp::now(); @@ -824,7 +957,8 @@ void ValidatorSessionImpl::on_catchain_started() { if (x) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), round = virtual_state_->cur_round_seqno(), src = description().get_source_id(x->get_src_idx()), - root_hash = x->get_root_hash()](td::Result R) { + root_hash = x->get_root_hash(), + compress = compress_block_candidates_](td::Result R) { if (R.is_error()) { LOG(ERROR) << "failed to get candidate: " << R.move_as_error(); } else { @@ -832,7 +966,8 @@ void ValidatorSessionImpl::on_catchain_started() { auto broadcast = create_tl_object( src.tl(), round, root_hash, std::move(B.data), std::move(B.collated_data)); td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src, - serialize_tl_object(broadcast, true)); + serialize_candidate(broadcast, compress).move_as_ok(), td::optional(), + false); } }); callback_->get_approved_candidate(description().get_source_public_key(x->get_src_idx()), x->get_root_hash(), @@ -859,10 +994,15 @@ ValidatorSessionImpl::ValidatorSessionImpl(catchain::CatChainSessionId session_i , rldp_(rldp) , overlay_manager_(overlays) , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) { + compress_block_candidates_ = opts.proto_version >= 4; description_ = ValidatorSessionDescription::create(std::move(opts), nodes, local_id); + src_round_candidate_.resize(description_->get_total_nodes()); } void ValidatorSessionImpl::start() { + round_started_at_ = td::Timestamp::now(); + round_debug_at_ = td::Timestamp::in(60.0); + stats_init(); started_ = true; VLOG(VALIDATOR_SESSION_NOTICE) << this << ": started"; @@ -883,6 +1023,80 @@ void ValidatorSessionImpl::destroy() { stop(); } +void ValidatorSessionImpl::get_current_stats(td::Promise promise) { + promise.set_result(cur_stats_); +} + +void ValidatorSessionImpl::get_end_stats(td::Promise promise) { + if (!started_) { + promise.set_error(td::Status::Error(ErrorCode::notready, "not started")); + return; + } + EndValidatorGroupStats stats; + stats.session_id = unique_hash_; + stats.timestamp = td::Clocks::system(); + stats.nodes.resize(description().get_total_nodes()); + for (size_t i = 0; i < stats.nodes.size(); ++i) { + stats.nodes[i].id = description().get_source_id(i); + } + td::actor::send_closure(catchain_, &catchain::CatChain::get_source_heights, + [promise = std::move(promise), + stats = std::move(stats)](td::Result> R) mutable { + TRY_RESULT_PROMISE(promise, heights, std::move(R)); + for (size_t i = 0; i < std::min(heights.size(), stats.nodes.size()); ++i) { + stats.nodes[i].catchain_blocks = heights[i]; + } + promise.set_result(std::move(stats)); + }); +} + +void ValidatorSessionImpl::get_validator_group_info_for_litequery( + td::uint32 cur_round, + td::Promise>> promise) { + if (cur_round != cur_round_ || real_state_->cur_round_seqno() != cur_round) { + promise.set_value({}); + return; + } + std::vector> result; + real_state_->for_each_cur_round_sent_block([&](const SessionBlockCandidate *block) { + if (block->get_block() == nullptr) { + return; + } + auto candidate = create_tl_object(); + + candidate->id_ = create_tl_object(); + candidate->id_->block_id_ = create_tl_object(); + candidate->id_->block_id_->root_hash_ = + block->get_block()->get_root_hash(); // other fields will be filled in validator-group.cpp + candidate->id_->block_id_->file_hash_ = block->get_block()->get_file_hash(); + candidate->id_->creator_ = + description().get_source_public_key(block->get_block()->get_src_idx()).ed25519_value().raw(); + candidate->id_->collated_data_hash_ = block->get_block()->get_collated_data_file_hash(); + + candidate->total_weight_ = description().get_total_weight(); + candidate->approved_weight_ = 0; + candidate->signed_weight_ = 0; + for (td::uint32 i = 0; i < description().get_total_nodes(); ++i) { + if (real_state_->check_block_is_approved_by(description(), i, block->get_id())) { + candidate->approved_weight_ += description().get_node_weight(i); + } + } + auto precommited = real_state_->get_cur_round_precommitted_block(); + if (SentBlock::get_block_id(precommited) == SentBlock::get_block_id(block->get_block())) { + auto signatures = real_state_->get_cur_round_signatures(); + if (signatures) { + for (td::uint32 i = 0; i < description().get_total_nodes(); ++i) { + if (signatures->at(i)) { + candidate->signed_weight_ += description().get_node_weight(i); + } + } + } + } + result.push_back(std::move(candidate)); + }); + promise.set_result(std::move(result)); +} + void ValidatorSessionImpl::start_up() { CHECK(!rldp_.empty()); cur_round_ = 0; @@ -894,47 +1108,152 @@ void ValidatorSessionImpl::start_up() { check_all(); td::actor::send_closure(rldp_, &rldp::Rldp::add_id, description().get_source_adnl_id(local_idx())); - - stats_init(); } void ValidatorSessionImpl::stats_init() { + auto old_rounds = std::move(cur_stats_.rounds); + if (stats_inited_ && cur_stats_.first_round + old_rounds.size() > cur_round_) { + old_rounds.erase(old_rounds.begin(), old_rounds.end() - (cur_stats_.first_round + old_rounds.size() - cur_round_)); + } else { + old_rounds.clear(); + } cur_stats_ = ValidatorSessionStats(); + cur_stats_.rounds = std::move(old_rounds); cur_stats_.first_round = cur_round_; - stats_add_round(); + cur_stats_.session_id = unique_hash_; + cur_stats_.total_validators = description().get_total_nodes(); + cur_stats_.total_weight = description().get_total_weight(); + cur_stats_.self = description().get_source_id(local_idx()); + + for (auto it = stats_pending_approve_.begin(); it != stats_pending_approve_.end(); ) { + if (it->first.first < cur_round_) { + it = stats_pending_approve_.erase(it); + } else { + ++it; + } + } + for (auto it = stats_pending_sign_.begin(); it != stats_pending_sign_.end(); ) { + if (it->first.first < cur_round_) { + it = stats_pending_sign_.erase(it); + } else { + ++it; + } + } + + if (cur_stats_.rounds.empty()) { + stats_add_round(); + } + cur_stats_.rounds[0].timestamp = td::Clocks::system(); + stats_inited_ = true; } void ValidatorSessionImpl::stats_add_round() { + td::uint32 round = cur_stats_.first_round + cur_stats_.rounds.size(); cur_stats_.rounds.emplace_back(); - auto& round = cur_stats_.rounds.back(); - round.timestamp = (td::uint64)td::Clocks::system(); - round.producers.resize(description().get_max_priority() + 1); + auto& stat = cur_stats_.rounds.back(); + stat.producers.resize(description().get_max_priority() + 1); for (td::uint32 i = 0; i < description().get_total_nodes(); i++) { - td::int32 priority = description().get_node_priority(i, cur_round_); + td::int32 priority = description().get_node_priority(i, round); if (priority >= 0) { - CHECK((size_t)priority < round.producers.size()); - round.producers[priority].id = description().get_source_id(i); + CHECK((size_t)priority < stat.producers.size()); + stat.producers[priority].id = description().get_source_id(i); + stat.producers[priority].is_ours = (local_idx() == i); + stat.producers[priority].approvers.resize(description().get_total_nodes(), false); + stat.producers[priority].signers.resize(description().get_total_nodes(), false); } } - while (!round.producers.empty() && round.producers.back().id.is_zero()) { - round.producers.pop_back(); + while (!stat.producers.empty() && stat.producers.back().id.is_zero()) { + stat.producers.pop_back(); } } -void ValidatorSessionImpl::stats_set_candidate_status(td::uint32 round, PublicKeyHash src, int status) { - if (round < cur_stats_.first_round || round - cur_stats_.first_round >= cur_stats_.rounds.size()) { - return; +ValidatorSessionStats::Producer *ValidatorSessionImpl::stats_get_candidate_stat( + td::uint32 round, PublicKeyHash src, ValidatorSessionCandidateId candidate_id) { + if (round < cur_stats_.first_round || round > cur_round_ + 5) { + return nullptr; } - auto& stats_round = cur_stats_.rounds[round - cur_stats_.first_round]; + while (round - cur_stats_.first_round >= cur_stats_.rounds.size()) { + stats_add_round(); + } + auto &stats_round = cur_stats_.rounds[round - cur_stats_.first_round]; auto it = std::find_if(stats_round.producers.begin(), stats_round.producers.end(), - [&](const ValidatorSessionStats::Producer& p) { return p.id == src; }); + [&](const ValidatorSessionStats::Producer &p) { return p.id == src; }); if (it == stats_round.producers.end()) { - return; + return nullptr; } - if (it->block_status == ValidatorSessionStats::status_none) { - it->block_timestamp = (td::uint64)td::Clocks::system(); + if (!candidate_id.is_zero()) { + it->candidate_id = candidate_id; } - it->block_status = status; + auto it2 = stats_pending_approve_.find({round, it->candidate_id}); + if (it2 != stats_pending_approve_.end()) { + for (td::uint32 node_id : it2->second) { + it->set_approved_by(node_id, description().get_node_weight(node_id), description().get_total_weight()); + } + stats_pending_approve_.erase(it2); + } + it2 = stats_pending_sign_.find({round, it->candidate_id}); + if (it2 != stats_pending_sign_.end()) { + for (td::uint32 node_id : it2->second) { + it->set_signed_by(node_id, description().get_node_weight(node_id), description().get_total_weight()); + } + stats_pending_sign_.erase(it2); + } + return &*it; +} + +ValidatorSessionStats::Producer *ValidatorSessionImpl::stats_get_candidate_stat_by_id( + td::uint32 round, ValidatorSessionCandidateId candidate_id) { + if (round < cur_stats_.first_round || round > cur_round_ + 5) { + return nullptr; + } + while (round - cur_stats_.first_round >= cur_stats_.rounds.size()) { + stats_add_round(); + } + auto &stats_round = cur_stats_.rounds[round - cur_stats_.first_round]; + auto it = std::find_if(stats_round.producers.begin(), stats_round.producers.end(), + [&](const ValidatorSessionStats::Producer &p) { return p.candidate_id == candidate_id; }); + if (it == stats_round.producers.end()) { + return nullptr; + } + return &*it; +} + +void ValidatorSessionImpl::stats_process_action(td::uint32 node_id, ton_api::validatorSession_round_Message &action) { + ton_api::downcast_call(action, td::overloaded( + [&](const ton_api::validatorSession_message_submittedBlock &obj) { + auto candidate_id = description().candidate_id( + node_id, obj.root_hash_, obj.file_hash_, obj.collated_data_file_hash_); + auto stat = stats_get_candidate_stat( + obj.round_, description().get_source_id(node_id), candidate_id); + if (stat && stat->got_submit_at <= 0.0) { + stat->got_submit_at = td::Clocks::system(); + } + }, + [&](const ton_api::validatorSession_message_approvedBlock &obj) { + if (obj.candidate_ == skip_round_candidate_id()) { + return; + } + auto stat = stats_get_candidate_stat_by_id(obj.round_, obj.candidate_); + if (stat) { + stat->set_approved_by(node_id, description().get_node_weight(node_id), + description().get_total_weight()); + } else { + stats_pending_approve_[{obj.round_, obj.candidate_}].push_back(node_id); + } + }, + [&](const ton_api::validatorSession_message_commit &obj) { + if (obj.candidate_ == skip_round_candidate_id()) { + return; + } + auto stat = stats_get_candidate_stat_by_id(obj.round_, obj.candidate_); + if (stat) { + stat->set_signed_by(node_id, description().get_node_weight(node_id), + description().get_total_weight()); + } else { + stats_pending_sign_[{obj.round_, obj.candidate_}].push_back(node_id); + } + }, + [](const auto &) {})); } td::actor::ActorOwn ValidatorSession::create( diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index 376cac45..b099d65e 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -28,6 +28,7 @@ #include "catchain/catchain-types.h" #include "validator-session-types.h" +#include "auto/tl/lite_api.h" namespace ton { @@ -55,6 +56,12 @@ class ValidatorSession : public td::actor::Actor { td::BufferSlice proof() const { return proof_.clone(); } + bool is_cached() const { + return is_cached_; + } + void set_is_cached(bool value = true) { + is_cached_ = value; + } CandidateDecision(td::uint32 ok_from) { ok_ = true; ok_from_ = ok_from; @@ -68,15 +75,20 @@ class ValidatorSession : public td::actor::Actor { td::uint32 ok_from_ = 0; std::string reason_; td::BufferSlice proof_; + bool is_cached_ = false; + }; + + struct GeneratedCandidate { + BlockCandidate candidate; + bool is_cached = false; }; class Callback { public: - virtual void on_candidate(td::uint32 round, PublicKey source, ValidatorSessionRootHash root_hash, - td::BufferSlice data, td::BufferSlice collated_data, - td::Promise promise) = 0; - virtual void on_generate_slot(td::uint32 round, td::Promise promise) = 0; - virtual void on_block_committed(td::uint32 round, PublicKey source, ValidatorSessionRootHash root_hash, + virtual void on_candidate(BlockSourceInfo source_info, ValidatorSessionRootHash root_hash, td::BufferSlice data, + td::BufferSlice collated_data, td::Promise promise) = 0; + virtual void on_generate_slot(BlockSourceInfo source_info, td::Promise promise) = 0; + virtual void on_block_committed(BlockSourceInfo source_info, ValidatorSessionRootHash root_hash, ValidatorSessionFileHash file_hash, td::BufferSlice data, std::vector> signatures, std::vector> approve_signatures, @@ -91,6 +103,12 @@ class ValidatorSession : public td::actor::Actor { virtual void start() = 0; virtual void destroy() = 0; + virtual void get_current_stats(td::Promise promise) = 0; + virtual void get_end_stats(td::Promise promise) = 0; + virtual void get_validator_group_info_for_litequery( + td::uint32 cur_round, + td::Promise>> promise) = 0; + virtual void set_catchain_max_block_delay(double delay, double delay_slow) = 0; static td::actor::ActorOwn create( catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 54bba33d..690346f7 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -53,7 +53,7 @@ class ValidatorSessionImpl : public ValidatorSession { const ValidatorSessionState *real_state_ = nullptr; const ValidatorSessionState *virtual_state_ = nullptr; - td::uint32 cur_round_ = 0; + td::uint32 cur_round_ = 0, first_block_round_ = 0; td::Timestamp round_started_at_ = td::Timestamp::never(); td::Timestamp round_debug_at_ = td::Timestamp::never(); std::set pending_approve_; @@ -73,7 +73,9 @@ class ValidatorSessionImpl : public ValidatorSession { ValidatorSessionCandidateId signed_block_; td::BufferSlice signature_; - std::array>, 100> blocks_; + std::map> blocks_; + // src_round_candidate_[src_id][round] -> candidate id + std::vector> src_round_candidate_; catchain::CatChainSessionId unique_hash_; @@ -88,6 +90,9 @@ class ValidatorSessionImpl : public ValidatorSession { td::actor::ActorOwn catchain_; std::unique_ptr description_; + double catchain_max_block_delay_ = 0.4; + double catchain_max_block_delay_slow_ = 1.0; + void on_new_round(td::uint32 round); void on_catchain_started(); void check_vote_for_slot(td::uint32 att); @@ -110,7 +115,8 @@ class ValidatorSessionImpl : public ValidatorSession { td::actor::send_closure(id_, &ValidatorSessionImpl::preprocess_block, block); } void process_broadcast(const PublicKeyHash &src, td::BufferSlice data) override { - td::actor::send_closure(id_, &ValidatorSessionImpl::process_broadcast, src, std::move(data)); + td::actor::send_closure(id_, &ValidatorSessionImpl::process_broadcast, src, std::move(data), + td::optional(), true); } void process_message(const PublicKeyHash &src, td::BufferSlice data) override { td::actor::send_closure(id_, &ValidatorSessionImpl::process_message, src, std::move(data)); @@ -145,6 +151,7 @@ class ValidatorSessionImpl : public ValidatorSession { } void request_new_block(bool now); + double get_current_max_block_delay() const; void get_broadcast_p2p(PublicKeyHash node, ValidatorSessionFileHash file_hash, ValidatorSessionCollatedDataFileHash collated_data_file_hash, PublicKeyHash src, td::uint32 round, ValidatorSessionRootHash root_hash, td::Promise promise, @@ -153,11 +160,22 @@ class ValidatorSessionImpl : public ValidatorSession { bool started_ = false; bool catchain_started_ = false; bool allow_unsafe_self_blocks_resync_; + bool compress_block_candidates_ = false; ValidatorSessionStats cur_stats_; + bool stats_inited_ = false; + std::map, std::vector> + stats_pending_approve_; // round, candidate_id -> approvers + std::map, std::vector> + stats_pending_sign_; // round, candidate_id -> signers void stats_init(); void stats_add_round(); - void stats_set_candidate_status(td::uint32 round, PublicKeyHash src, int status); + ValidatorSessionStats::Producer *stats_get_candidate_stat( + td::uint32 round, PublicKeyHash src, + ValidatorSessionCandidateId candidate_id = ValidatorSessionCandidateId::zero()); + ValidatorSessionStats::Producer *stats_get_candidate_stat_by_id(td::uint32 round, + ValidatorSessionCandidateId candidate_id); + void stats_process_action(td::uint32 node_id, ton_api::validatorSession_round_Message &action); public: ValidatorSessionImpl(catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, @@ -170,26 +188,37 @@ class ValidatorSessionImpl : public ValidatorSession { void start() override; void destroy() override; + void get_current_stats(td::Promise promise) override; + void get_end_stats(td::Promise promise) override; + void get_validator_group_info_for_litequery( + td::uint32 cur_round, + td::Promise>> promise) override; + + void set_catchain_max_block_delay(double delay, double delay_slow) override { + catchain_max_block_delay_ = delay; + catchain_max_block_delay_slow_ = delay_slow; + } void process_blocks(std::vector blocks); void finished_processing(); void preprocess_block(catchain::CatChainBlock *block); - void process_broadcast(PublicKeyHash src, td::BufferSlice data); + bool ensure_candidate_unique(td::uint32 src_idx, td::uint32 round, ValidatorSessionCandidateId block_id); + void process_broadcast(PublicKeyHash src, td::BufferSlice data, td::optional expected_id, + bool is_overlay_broadcast); void process_message(PublicKeyHash src, td::BufferSlice data); void process_query(PublicKeyHash src, td::BufferSlice data, td::Promise promise); void try_approve_block(const SentBlock *block); - void try_sign(); - void candidate_decision_fail(td::uint32 round, ValidatorSessionCandidateId hash, std::string result, - td::uint32 src, td::BufferSlice proof); + void candidate_decision_fail(td::uint32 round, ValidatorSessionCandidateId hash, std::string result, td::uint32 src, + td::BufferSlice proof, double validation_time, bool validation_cached); void candidate_decision_ok(td::uint32 round, ValidatorSessionCandidateId hash, RootHash root_hash, FileHash file_hash, - td::uint32 src, td::uint32 ok_from); + td::uint32 src, td::uint32 ok_from, double validation_time, bool validation_cached); void candidate_approved_signed(td::uint32 round, ValidatorSessionCandidateId hash, td::uint32 ok_from, td::BufferSlice signature); void generated_block(td::uint32 round, ValidatorSessionRootHash root_hash, td::BufferSlice data, - td::BufferSlice collated); + td::BufferSlice collated, double collation_time, bool collation_cached); void signed_block(td::uint32 round, ValidatorSessionCandidateId hash, td::BufferSlice signature); void end_request(td::uint32 round, ValidatorSessionCandidateId block_id) { @@ -204,6 +233,10 @@ class ValidatorSessionImpl : public ValidatorSession { private: static const size_t MAX_REJECT_REASON_SIZE = 1024; + static const td::int32 MAX_FUTURE_ROUND_BLOCK = 100; + static const td::int32 MAX_PAST_ROUND_BLOCK = 20; + constexpr static const double REQUEST_BROADCAST_P2P_DELAY = 2.0; + static const td::uint32 MAX_CANDIDATE_EXTRA_SIZE = 1024; }; } // namespace validatorsession diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index 65d7b348..5f5544b2 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) @@ -25,6 +25,8 @@ set(VALIDATOR_DB_SOURCE db/statedb.cpp db/staticfilesdb.cpp db/staticfilesdb.hpp + db/db-utils.cpp + db/db-utils.h db/package.hpp db/package.cpp @@ -43,6 +45,8 @@ set(VALIDATOR_HEADERS fabric.h interfaces/db.h interfaces/external-message.h + interfaces/liteserver.h + interfaces/out-msg-queue-proof.h interfaces/proof.h interfaces/shard.h interfaces/signature-set.h @@ -52,6 +56,8 @@ set(VALIDATOR_HEADERS invariants.hpp import-db-slice.hpp + queue-size-counter.hpp + validator-telemetry.hpp manager-disk.h manager-disk.hpp @@ -77,6 +83,8 @@ set(VALIDATOR_SOURCE validator-full-id.cpp validator-group.cpp validator-options.cpp + queue-size-counter.cpp + validator-telemetry.cpp downloaders/wait-block-data.cpp downloaders/wait-block-state.cpp @@ -98,7 +106,8 @@ set(DISK_VALIDATOR_SOURCE validator-full-id.cpp validator-group.cpp validator-options.cpp - + queue-size-counter.cpp + downloaders/wait-block-data-disk.cpp downloaders/wait-block-state.cpp downloaders/wait-block-state-merge.cpp @@ -117,7 +126,8 @@ set(HARDFORK_VALIDATOR_SOURCE validator-full-id.cpp validator-group.cpp validator-options.cpp - + queue-size-counter.cpp + downloaders/wait-block-data-disk.cpp downloaders/wait-block-state.cpp downloaders/wait-block-state-merge.cpp @@ -139,7 +149,11 @@ set(FULL_NODE_SOURCE full-node-master.h full-node-master.hpp full-node-master.cpp - + full-node-private-overlay.hpp + full-node-private-overlay.cpp + full-node-serializer.hpp + full-node-serializer.cpp + net/download-block.hpp net/download-block.cpp net/download-block-new.hpp @@ -189,14 +203,10 @@ target_include_directories(full-node PUBLIC ${OPENSSL_INCLUDE_DIR} ) -target_link_libraries(validator PRIVATE tdutils tdactor adnl rldp tl_api dht tdfec - overlay catchain validatorsession ton_crypto ton_block ton_db) +target_link_libraries(validator PRIVATE tdactor adnl rldp tl_api dht tdfec overlay catchain validatorsession ton_db) -target_link_libraries(validator-disk PRIVATE tdutils tdactor adnl rldp tl_api dht tdfec - overlay catchain validatorsession ton_crypto ton_block ton_db) +target_link_libraries(validator-disk PRIVATE tdactor adnl rldp tl_api dht tdfec overlay catchain validatorsession ton_db) -target_link_libraries(validator-hardfork PRIVATE tdutils tdactor adnl rldp tl_api dht tdfec - overlay catchain validatorsession ton_crypto ton_block ton_db) +target_link_libraries(validator-hardfork PRIVATE tdactor adnl rldp tl_api dht tdfec overlay catchain validatorsession ton_db) -target_link_libraries(full-node PRIVATE tdutils tdactor adnl rldp tl_api dht tdfec - overlay catchain validatorsession ton_crypto ton_block ton_db) +target_link_libraries(full-node PRIVATE tdactor adnl rldp rldp2 tl_api dht tdfec overlay catchain validatorsession ton_db) diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 6bea5db6..8c7cde17 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -32,11 +32,11 @@ std::string PackageId::path() const { return "/files/packages/"; } else if (key) { char s[24]; - sprintf(s, "key%03d", id / 1000000); + snprintf(s, sizeof(s), "key%03d", id / 1000000); return PSTRING() << "/archive/packages/" << s << "/"; } else { char s[20]; - sprintf(s, "arch%04d", id / 100000); + snprintf(s, sizeof(s), "arch%04d", id / 100000); return PSTRING() << "/archive/packages/" << s << "/"; } } @@ -46,16 +46,18 @@ std::string PackageId::name() const { return PSTRING() << "temp.archive." << id; } else if (key) { char s[20]; - sprintf(s, "%06d", id); + snprintf(s, sizeof(s), "%06d", id); return PSTRING() << "key.archive." << s; } else { char s[10]; - sprintf(s, "%05d", id); + snprintf(s, sizeof(s), "%05d", id); return PSTRING() << "archive." << s; } } -ArchiveManager::ArchiveManager(td::actor::ActorId root, std::string db_root) : db_root_(db_root) { +ArchiveManager::ArchiveManager(td::actor::ActorId root, std::string db_root, + td::Ref opts) + : db_root_(db_root), opts_(opts) { } void ArchiveManager::add_handle(BlockHandle handle, td::Promise promise) { @@ -74,7 +76,7 @@ void ArchiveManager::add_handle(BlockHandle handle, td::Promise promis } void ArchiveManager::update_handle(BlockHandle handle, td::Promise promise) { - FileDescription *f; + const FileDescription *f; if (handle->handle_moved_to_archive()) { CHECK(handle->inited_unix_time()); if (!handle->need_flush()) { @@ -308,14 +310,17 @@ void ArchiveManager::get_file(ConstBlockHandle handle, FileReference ref_id, td: get_file_short_cont(std::move(ref_id), get_max_temp_file_desc_idx(), std::move(promise)); } -void ArchiveManager::written_perm_state(FileReferenceShort id) { - perm_states_.emplace(id.hash(), id); +void ArchiveManager::register_perm_state(FileReferenceShort id) { + BlockSeqno masterchain_seqno = 0; + id.ref().visit(td::overloaded( + [&](const fileref::PersistentStateShort &x) { masterchain_seqno = x.masterchain_seqno; }, [&](const auto &) {})); + perm_states_[{masterchain_seqno, id.hash()}] = id; } void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise promise) { auto id = FileReference{fileref::ZeroState{block_id}}; auto hash = id.hash(); - if (perm_states_.find(hash) != perm_states_.end()) { + if (perm_states_.find({0, hash}) != perm_states_.end()) { promise.set_value(td::Unit()); return; } @@ -326,7 +331,7 @@ void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, t if (R.is_error()) { promise.set_error(R.move_as_error()); } else { - td::actor::send_closure(SelfId, &ArchiveManager::written_perm_state, id); + td::actor::send_closure(SelfId, &ArchiveManager::register_perm_state, id); promise.set_value(td::Unit()); } }); @@ -337,30 +342,31 @@ void ArchiveManager::add_zero_state(BlockIdExt block_id, td::BufferSlice data, t void ArchiveManager::add_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice data, td::Promise promise) { auto create_writer = [&](std::string path, td::Promise P) { - td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", - std::move(path), std::move(data), std::move(P)) + td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", std::move(path), std::move(data), + std::move(P)) .release(); }; add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer)); } void ArchiveManager::add_persistent_state_gen(BlockIdExt block_id, BlockIdExt masterchain_block_id, - std::function write_state, + std::function write_state, td::Promise promise) { auto create_writer = [&](std::string path, td::Promise P) { - td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", - std::move(path), std::move(write_state), std::move(P)) + td::actor::create_actor("writefile", db_root_ + "/archive/tmp/", std::move(path), + std::move(write_state), std::move(P)) .release(); }; add_persistent_state_impl(block_id, masterchain_block_id, std::move(promise), std::move(create_writer)); } -void ArchiveManager::add_persistent_state_impl(BlockIdExt block_id, BlockIdExt masterchain_block_id, - td::Promise promise, - std::function)> create_writer) { +void ArchiveManager::add_persistent_state_impl( + BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise, + std::function)> create_writer) { auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; + BlockSeqno masterchain_seqno = masterchain_block_id.seqno(); auto hash = id.hash(); - if (perm_states_.find(hash) != perm_states_.end()) { + if (perm_states_.find({masterchain_seqno, hash}) != perm_states_.end()) { promise.set_value(td::Unit()); return; } @@ -371,7 +377,7 @@ void ArchiveManager::add_persistent_state_impl(BlockIdExt block_id, BlockIdExt m if (R.is_error()) { promise.set_error(R.move_as_error()); } else { - td::actor::send_closure(SelfId, &ArchiveManager::written_perm_state, id); + td::actor::send_closure(SelfId, &ArchiveManager::register_perm_state, id); promise.set_value(td::Unit()); } }); @@ -381,7 +387,7 @@ void ArchiveManager::add_persistent_state_impl(BlockIdExt block_id, BlockIdExt m void ArchiveManager::get_zero_state(BlockIdExt block_id, td::Promise promise) { auto id = FileReference{fileref::ZeroState{block_id}}; auto hash = id.hash(); - if (perm_states_.find(hash) == perm_states_.end()) { + if (perm_states_.find({0, hash}) == perm_states_.end()) { promise.set_error(td::Status::Error(ErrorCode::notready, "zerostate not in db")); return; } @@ -393,18 +399,38 @@ void ArchiveManager::get_zero_state(BlockIdExt block_id, td::Promise promise) { auto id = FileReference{fileref::ZeroState{block_id}}; auto hash = id.hash(); - if (perm_states_.find(hash) == perm_states_.end()) { + if (perm_states_.find({0, hash}) == perm_states_.end()) { promise.set_result(false); return; } promise.set_result(true); } +void ArchiveManager::get_previous_persistent_state_files( + BlockSeqno cur_mc_seqno, td::Promise>> promise) { + auto it = perm_states_.lower_bound({cur_mc_seqno, FileHash::zero()}); + if (it == perm_states_.begin()) { + promise.set_value({}); + return; + } + --it; + BlockSeqno mc_seqno = it->first.first; + std::vector> files; + while (it->first.first == mc_seqno) { + files.emplace_back(db_root_ + "/archive/states/" + it->second.filename_short(), it->second.shard()); + if (it == perm_states_.begin()) { + break; + } + --it; + } + promise.set_value(std::move(files)); +} + void ArchiveManager::get_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) { auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; auto hash = id.hash(); - if (perm_states_.find(hash) == perm_states_.end()) { + if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) { promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db")); return; } @@ -417,7 +443,7 @@ void ArchiveManager::get_persistent_state_slice(BlockIdExt block_id, BlockIdExt td::int64 max_size, td::Promise promise) { auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; auto hash = id.hash(); - if (perm_states_.find(hash) == perm_states_.end()) { + if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) { promise.set_error(td::Status::Error(ErrorCode::notready, "state file not in db")); return; } @@ -430,7 +456,7 @@ void ArchiveManager::check_persistent_state(BlockIdExt block_id, BlockIdExt mast td::Promise promise) { auto id = FileReference{fileref::PersistentState{block_id, masterchain_block_id}}; auto hash = id.hash(); - if (perm_states_.find(hash) == perm_states_.end()) { + if (perm_states_.find({masterchain_block_id.seqno(), hash}) == perm_states_.end()) { promise.set_result(false); return; } @@ -439,15 +465,15 @@ void ArchiveManager::check_persistent_state(BlockIdExt block_id, BlockIdExt mast void ArchiveManager::get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise promise) { - auto f = get_file_desc_by_unix_time(account_id, ts, false); - if (f) { - auto n = f; - do { - n = get_next_file_desc(n); - } while (n != nullptr && !n->has_account_prefix(account_id)); + auto f1 = get_file_desc_by_unix_time(account_id, ts, false); + auto f2 = get_next_file_desc(f1, account_id, false); + if (!f1) { + std::swap(f1, f2); + } + if (f1) { td::actor::ActorId aid; - if (n) { - aid = n->file_actor_id(); + if (f2) { + aid = f2->file_actor_id(); } auto P = td::PromiseCreator::lambda( [aid, account_id, ts, promise = std::move(promise)](td::Result R) mutable { @@ -457,7 +483,7 @@ void ArchiveManager::get_block_by_unix_time(AccountIdPrefixFull account_id, Unix td::actor::send_closure(aid, &ArchiveSlice::get_block_by_unix_time, account_id, ts, std::move(promise)); } }); - td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_block_by_unix_time, account_id, ts, std::move(P)); + td::actor::send_closure(f1->file_actor_id(), &ArchiveSlice::get_block_by_unix_time, account_id, ts, std::move(P)); } else { promise.set_error(td::Status::Error(ErrorCode::notready, "ts not in db")); } @@ -465,15 +491,15 @@ void ArchiveManager::get_block_by_unix_time(AccountIdPrefixFull account_id, Unix void ArchiveManager::get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise promise) { - auto f = get_file_desc_by_lt(account_id, lt, false); - if (f) { - auto n = f; - do { - n = get_next_file_desc(n); - } while (n != nullptr && !n->has_account_prefix(account_id)); + auto f1 = get_file_desc_by_lt(account_id, lt, false); + auto f2 = get_next_file_desc(f1, account_id, false); + if (!f1) { + std::swap(f1, f2); + } + if (f1) { td::actor::ActorId aid; - if (n) { - aid = n->file_actor_id(); + if (f2) { + aid = f2->file_actor_id(); } auto P = td::PromiseCreator::lambda( [aid, account_id, lt, promise = std::move(promise)](td::Result R) mutable { @@ -483,7 +509,7 @@ void ArchiveManager::get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime td::actor::send_closure(aid, &ArchiveSlice::get_block_by_lt, account_id, lt, std::move(promise)); } }); - td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_block_by_lt, account_id, lt, std::move(P)); + td::actor::send_closure(f1->file_actor_id(), &ArchiveSlice::get_block_by_lt, account_id, lt, std::move(P)); } else { promise.set_error(td::Status::Error(ErrorCode::notready, "lt not in db")); } @@ -558,11 +584,16 @@ void ArchiveManager::deleted_package(PackageId id, td::Promise promise auto it = m.find(id); CHECK(it != m.end()); CHECK(it->second.deleted); - it->second.clear_actor_id(); + it->second.file.reset(); promise.set_value(td::Unit()); } void ArchiveManager::load_package(PackageId id) { + auto &m = get_file_map(id); + if (m.count(id)) { + LOG(WARNING) << "Duplicate id " << id.name(); + return; + } auto key = create_serialize_tl_object(id.id, id.key, id.temp); std::string value; @@ -593,13 +624,15 @@ void ArchiveManager::load_package(PackageId id) { } } - desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_); + desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, 0, db_root_, + archive_lru_.get(), statistics_); - get_file_map(id).emplace(id, std::move(desc)); + m.emplace(id, std::move(desc)); + update_permanent_slices(); } -ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, - UnixTime ts, LogicalTime lt, bool force) { +const ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, + UnixTime ts, LogicalTime lt, bool force) { auto &f = get_file_map(id); auto it = f.find(id); if (it != f.end()) { @@ -607,7 +640,7 @@ ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard return nullptr; } if (force && !id.temp) { - update_desc(it->second, shard, seqno, ts, lt); + update_desc(f, it->second, shard, seqno, ts, lt); } return &it->second; } @@ -618,17 +651,20 @@ ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard return add_file_desc(shard, id, seqno, ts, lt); } -ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, - UnixTime ts, LogicalTime lt) { +const ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, + UnixTime ts, LogicalTime lt) { auto &f = get_file_map(id); CHECK(f.count(id) == 0); - FileDescription desc{id, false}; + FileDescription new_desc{id, false}; td::mkdir(db_root_ + id.path()).ensure(); std::string prefix = PSTRING() << db_root_ << id.path() << id.name(); - desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_); + new_desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, + id.key || id.temp ? 0 : cur_shard_split_depth_, db_root_, + archive_lru_.get(), statistics_); + const FileDescription &desc = f.emplace(id, std::move(new_desc)); if (!id.temp) { - update_desc(desc, shard, seqno, ts, lt); + update_desc(f, desc, shard, seqno, ts, lt); } std::vector> vec; @@ -652,7 +688,6 @@ ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull shard for (auto &e : temp_files_) { tt.push_back(e.first.id); } - (id.temp ? tt : (id.key ? tk : t)).push_back(id.id); index_ ->set(create_serialize_tl_object().as_slice(), create_serialize_tl_object(std::move(t), std::move(tk), std::move(tt)) @@ -668,17 +703,17 @@ ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull shard .ensure(); } index_->commit_transaction().ensure(); - - return &f.emplace(id, std::move(desc)).first->second; + update_permanent_slices(); + return &desc; } -void ArchiveManager::update_desc(FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, UnixTime ts, - LogicalTime lt) { +void ArchiveManager::update_desc(FileMap &f, const FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, + UnixTime ts, LogicalTime lt) { auto it = desc.first_blocks.find(shard); if (it != desc.first_blocks.end() && it->second.seqno <= seqno) { return; } - desc.first_blocks[shard] = FileDescription::Desc{seqno, ts, lt}; + f.set_shard_first_block(desc, shard, FileDescription::Desc{seqno, ts, lt}); std::vector> vec; for (auto &e : desc.first_blocks) { vec.push_back(create_tl_object(e.first.workchain, e.first.shard, @@ -694,150 +729,91 @@ void ArchiveManager::update_desc(FileDescription &desc, ShardIdFull shard, Block index_->commit_transaction().ensure(); } -ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno, - bool key_block) { - auto &f = get_file_map(PackageId{0, key_block, false}); - for (auto it = f.rbegin(); it != f.rend(); it++) { - auto i = it->second.first_blocks.find(shard); - if (i != it->second.first_blocks.end() && i->second.seqno <= seqno) { - if (it->second.deleted) { - return nullptr; - } else { - return &it->second; - } - } - } - return nullptr; +const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno, + bool key_block) { + return get_file_map(PackageId{0, key_block, false}).get_file_desc_by_seqno(shard, seqno); } -ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts, - bool key_block) { - auto &f = get_file_map(PackageId{0, key_block, false}); - for (auto it = f.rbegin(); it != f.rend(); it++) { - auto i = it->second.first_blocks.find(shard); - if (i != it->second.first_blocks.end() && i->second.ts <= ts) { - if (it->second.deleted) { - return nullptr; - } else { - return &it->second; - } - } - } - return nullptr; +const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts, + bool key_block) { + return get_file_map(PackageId{0, key_block, false}).get_file_desc_by_unix_time(shard, ts); } -ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt, - bool key_block) { - auto &f = get_file_map(PackageId{0, key_block, false}); - for (auto it = f.rbegin(); it != f.rend(); it++) { - auto i = it->second.first_blocks.find(shard); - if (i != it->second.first_blocks.end() && i->second.lt <= lt) { - if (it->second.deleted) { - return nullptr; - } else { - return &it->second; - } - } - } - return nullptr; +const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt, + bool key_block) { + return get_file_map(PackageId{0, key_block, false}).get_file_desc_by_lt(shard, lt); } -ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno, - bool key_block) { - auto &f = get_file_map(PackageId{0, key_block, false}); +const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(AccountIdPrefixFull account, + BlockSeqno seqno, bool key_block) { if (account.is_masterchain()) { return get_file_desc_by_seqno(ShardIdFull{masterchainId}, seqno, key_block); } - for (auto it = f.rbegin(); it != f.rend(); it++) { - if (it->second.deleted) { - continue; - } - bool found = false; - for (int i = 0; i < 60; i++) { - auto shard = shard_prefix(account, i); - auto it2 = it->second.first_blocks.find(shard); - if (it2 != it->second.first_blocks.end()) { - if (it2->second.seqno <= seqno) { - return &it->second; - } - found = true; - } else if (found) { - break; - } + auto &f = get_file_map(PackageId{0, key_block, false}); + const FileDescription *result = nullptr; + for (int i = 0; i <= 60; i++) { + const FileDescription *desc = f.get_file_desc_by_seqno(shard_prefix(account, i), seqno); + if (desc && (!result || result->id < desc->id)) { + result = desc; + } else if (result && (!desc || desc->id < result->id)) { + break; } } - return nullptr; + return result; } -ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(AccountIdPrefixFull account, UnixTime ts, - bool key_block) { - auto &f = get_file_map(PackageId{0, key_block, false}); +const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(AccountIdPrefixFull account, + UnixTime ts, bool key_block) { if (account.is_masterchain()) { return get_file_desc_by_unix_time(ShardIdFull{masterchainId}, ts, key_block); } - for (auto it = f.rbegin(); it != f.rend(); it++) { - if (it->second.deleted) { - continue; - } - bool found = false; - for (int i = 0; i < 60; i++) { - auto shard = shard_prefix(account, i); - auto it2 = it->second.first_blocks.find(shard); - if (it2 != it->second.first_blocks.end()) { - if (it2->second.ts <= ts) { - return &it->second; - } - found = true; - } else if (found) { - break; - } + auto &f = get_file_map(PackageId{0, key_block, false}); + const FileDescription *result = nullptr; + for (int i = 0; i <= 60; i++) { + const FileDescription *desc = f.get_file_desc_by_unix_time(shard_prefix(account, i), ts); + if (desc && (!result || result->id < desc->id)) { + result = desc; + } else if (result && (!desc || desc->id < result->id)) { + break; } } - return nullptr; + return result; } -ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(AccountIdPrefixFull account, LogicalTime lt, - bool key_block) { - auto &f = get_file_map(PackageId{0, key_block, false}); +const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(AccountIdPrefixFull account, LogicalTime lt, + bool key_block) { if (account.is_masterchain()) { return get_file_desc_by_lt(ShardIdFull{masterchainId}, lt, key_block); } - for (auto it = f.rbegin(); it != f.rend(); it++) { - if (it->second.deleted) { - continue; - } - bool found = false; - for (int i = 0; i < 60; i++) { - auto shard = shard_prefix(account, i); - auto it2 = it->second.first_blocks.find(shard); - if (it2 != it->second.first_blocks.end()) { - if (it2->second.lt <= lt) { - return &it->second; - } - found = true; - } else if (found) { - break; - } + auto &f = get_file_map(PackageId{0, key_block, false}); + const FileDescription *result = nullptr; + for (int i = 0; i <= 60; i++) { + const FileDescription *desc = f.get_file_desc_by_lt(shard_prefix(account, i), lt); + if (desc && (!result || result->id < desc->id)) { + result = desc; + } else if (result && (!desc || desc->id < result->id)) { + break; } } - return nullptr; + return result; } -ArchiveManager::FileDescription *ArchiveManager::get_next_file_desc(FileDescription *f) { - auto &m = get_file_map(f->id); - auto it = m.find(f->id); - CHECK(it != m.end()); - while (true) { - it++; - if (it == m.end()) { - return nullptr; - } else if (!it->second.deleted) { - return &it->second; +const ArchiveManager::FileDescription *ArchiveManager::get_next_file_desc(const FileDescription *f, + AccountIdPrefixFull shard, bool key_block) { + auto &m = get_file_map(PackageId{0, key_block, false}); + const FileDescription *result = nullptr; + for (int i = 0; i <= 60; i++) { + const FileDescription *desc = m.get_next_file_desc(shard_prefix(shard, i), f); + if (desc && (!result || desc->id < result->id)) { + result = desc; + } else if (result && (!desc || result->id < desc->id)) { + break; } } + return result; } -ArchiveManager::FileDescription *ArchiveManager::get_temp_file_desc_by_idx(PackageId idx) { +const ArchiveManager::FileDescription *ArchiveManager::get_temp_file_desc_by_idx(PackageId idx) { auto it = temp_files_.find(idx); if (it != temp_files_.end()) { if (it->second.deleted) { @@ -875,7 +851,16 @@ void ArchiveManager::start_up() { td::mkdir(db_root_ + "/archive/states/").ensure(); td::mkdir(db_root_ + "/files/").ensure(); td::mkdir(db_root_ + "/files/packages/").ensure(); - index_ = std::make_shared(td::RocksDb::open(db_root_ + "/files/globalindex").move_as_ok()); + if (opts_->get_max_open_archive_files() > 0) { + archive_lru_ = td::actor::create_actor("archive_lru", opts_->get_max_open_archive_files()); + } + if (!opts_->get_disable_rocksdb_stats()) { + statistics_.init(); + } + td::RocksDbOptions db_options; + db_options.statistics = statistics_.rocksdb_statistics; + index_ = std::make_shared( + td::RocksDb::open(db_root_ + "/files/globalindex", std::move(db_options)).move_as_ok()); std::string value; auto v = index_->get(create_serialize_tl_object().as_slice(), value); v.ensure(); @@ -906,7 +891,7 @@ void ArchiveManager::start_up() { td::WalkPath::run(db_root_ + "/archive/states/", [&](td::CSlice fname, td::WalkPath::Type t) -> void { if (t == td::WalkPath::Type::NotDir) { LOG(ERROR) << "checking file " << fname; - auto pos = fname.rfind('/'); + auto pos = fname.rfind(TD_DIR_SLASH); if (pos != td::Slice::npos) { fname.remove_prefix(pos + 1); } @@ -924,17 +909,55 @@ void ArchiveManager::start_up() { R = FileReferenceShort::create(newfname); R.ensure(); } - auto f = R.move_as_ok(); - auto hash = f.hash(); - perm_states_[hash] = std::move(f); + register_perm_state(R.move_as_ok()); } }).ensure(); - persistent_state_gc(FileHash::zero()); + persistent_state_gc({0, FileHash::zero()}); + + double open_since = td::Clocks::system() - opts_->get_archive_preload_period(); + for (auto it = files_.rbegin(); it != files_.rend(); ++it) { + if (it->second.file_actor_id().empty()) { + continue; + } + td::actor::send_closure(it->second.file_actor_id(), &ArchiveSlice::open_files); + bool stop = true; + for (const auto &first_block : it->second.first_blocks) { + if ((double)first_block.second.ts >= open_since) { + stop = false; + break; + } + } + if (stop) { + break; + } + } + + if (!opts_->get_disable_rocksdb_stats()) { + alarm_timestamp() = td::Timestamp::in(60.0); + } } -void ArchiveManager::run_gc(UnixTime ts, UnixTime archive_ttl) { - auto p = get_temp_package_id_by_unixtime(ts); +void ArchiveManager::alarm() { + alarm_timestamp() = td::Timestamp::in(60.0); + auto stats = statistics_.to_string_and_reset(); + auto to_file_r = + td::FileFd::open(db_root_ + "/db_stats.txt", td::FileFd::Truncate | td::FileFd::Create | td::FileFd::Write, 0644); + if (to_file_r.is_error()) { + LOG(ERROR) << "Failed to open db_stats.txt: " << to_file_r.move_as_error(); + return; + } + auto to_file = to_file_r.move_as_ok(); + auto res = to_file.write(stats); + to_file.close(); + if (res.is_error()) { + LOG(ERROR) << "Failed to write to db_stats.txt: " << res.move_as_error(); + return; + } +} + +void ArchiveManager::run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) { + auto p = get_temp_package_id_by_unixtime(mc_ts - TEMP_PACKAGES_TTL); std::vector vec; for (auto &x : temp_files_) { if (x.first < p) { @@ -962,7 +985,7 @@ void ArchiveManager::run_gc(UnixTime ts, UnixTime archive_ttl) { if (it == desc.first_blocks.end()) { continue; } - if (it->second.ts < ts - archive_ttl) { + if (it->second.ts < gc_ts - archive_ttl) { vec.push_back(f.first); } } @@ -977,11 +1000,12 @@ void ArchiveManager::run_gc(UnixTime ts, UnixTime archive_ttl) { } } -void ArchiveManager::persistent_state_gc(FileHash last) { - if (perm_states_.size() == 0) { +void ArchiveManager::persistent_state_gc(std::pair last) { + if (perm_states_.empty()) { delay_action( - [hash = FileHash::zero(), SelfId = actor_id(this)]() { - td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, hash); + [SelfId = actor_id(this)]() { + td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, + std::pair{0, FileHash::zero()}); }, td::Timestamp::in(1.0)); return; @@ -994,12 +1018,12 @@ void ArchiveManager::persistent_state_gc(FileHash last) { it = perm_states_.begin(); } + auto key = it->first; auto &F = it->second; - auto hash = F.hash(); int res = 0; BlockSeqno seqno = 0; - F.ref().visit(td::overloaded([&](const fileref::ZeroStateShort &x) { res = 1; }, + F.ref().visit(td::overloaded([&](const fileref::ZeroStateShort &) { res = 1; }, [&](const fileref::PersistentStateShort &x) { res = 0; seqno = x.masterchain_seqno; @@ -1011,24 +1035,41 @@ void ArchiveManager::persistent_state_gc(FileHash last) { perm_states_.erase(it); } if (res != 0) { - delay_action([hash, SelfId = actor_id( - this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, hash); }, + delay_action([key, SelfId = actor_id( + this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); }, + td::Timestamp::in(1.0)); + return; + } + CHECK(seqno == key.first); + + // Do not delete the most recent fully serialized state + bool allow_delete = false; + auto it2 = perm_states_.lower_bound({seqno + 1, FileHash::zero()}); + if (it2 != perm_states_.end()) { + it2 = perm_states_.lower_bound({it2->first.first + 1, FileHash::zero()}); + if (it2 != perm_states_.end()) { + allow_delete = true; + } + } + if (!allow_delete) { + delay_action([key, SelfId = actor_id( + this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); }, td::Timestamp::in(1.0)); return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), hash](td::Result R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), key](td::Result R) { if (R.is_error()) { - td::actor::send_closure(SelfId, &ArchiveManager::got_gc_masterchain_handle, nullptr, hash); + td::actor::send_closure(SelfId, &ArchiveManager::got_gc_masterchain_handle, nullptr, key); } else { - td::actor::send_closure(SelfId, &ArchiveManager::got_gc_masterchain_handle, R.move_as_ok(), hash); + td::actor::send_closure(SelfId, &ArchiveManager::got_gc_masterchain_handle, R.move_as_ok(), key); } }); get_block_by_seqno(AccountIdPrefixFull{masterchainId, 0}, seqno, std::move(P)); } -void ArchiveManager::got_gc_masterchain_handle(ConstBlockHandle handle, FileHash hash) { +void ArchiveManager::got_gc_masterchain_handle(ConstBlockHandle handle, std::pair key) { bool to_del = false; if (!handle || !handle->inited_unix_time() || !handle->unix_time()) { to_del = true; @@ -1036,16 +1077,16 @@ void ArchiveManager::got_gc_masterchain_handle(ConstBlockHandle handle, FileHash auto ttl = ValidatorManager::persistent_state_ttl(handle->unix_time()); to_del = ttl < td::Clocks::system(); } - auto it = perm_states_.find(hash); + auto it = perm_states_.find(key); CHECK(it != perm_states_.end()); auto &F = it->second; if (to_del) { td::unlink(db_root_ + "/archive/states/" + F.filename_short()).ignore(); perm_states_.erase(it); } - delay_action([hash, SelfId = actor_id( - this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, hash); }, - td::Timestamp::in(1.0)); + delay_action( + [key, SelfId = actor_id(this)]() { td::actor::send_closure(SelfId, &ArchiveManager::persistent_state_gc, key); }, + td::Timestamp::in(1.0)); } PackageId ArchiveManager::get_temp_package_id() const { @@ -1092,14 +1133,16 @@ PackageId ArchiveManager::get_package_id_force(BlockSeqno masterchain_seqno, Sha return it->first; } -void ArchiveManager::get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) { +void ArchiveManager::get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) { auto F = get_file_desc_by_seqno(ShardIdFull{masterchainId}, masterchain_seqno, false); if (!F) { promise.set_error(td::Status::Error(ErrorCode::notready, "archive not found")); return; } - td::actor::send_closure(F->file_actor_id(), &ArchiveSlice::get_archive_id, masterchain_seqno, std::move(promise)); + td::actor::send_closure(F->file_actor_id(), &ArchiveSlice::get_archive_id, masterchain_seqno, shard_prefix, + std::move(promise)); } void ArchiveManager::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, @@ -1153,6 +1196,30 @@ void ArchiveManager::set_async_mode(bool mode, td::Promise promise) { } } +void ArchiveManager::prepare_stats(td::Promise>> promise) { + std::vector> stats; + { + std::map states; + for (auto &[key, file] : perm_states_) { + BlockSeqno seqno = key.first; + auto r_stat = td::stat(db_root_ + "/archive/states/" + file.filename_short()); + if (r_stat.is_error()) { + LOG(WARNING) << "Cannot stat persistent state file " << file.filename_short() << " : " << r_stat.move_as_error(); + } else { + states[seqno] += r_stat.move_as_ok().size_; + } + } + td::StringBuilder sb; + for (auto &[seqno, size] : states) { + sb << seqno << ":" << td::format::as_size(size) << " "; + } + if (!sb.as_cslice().empty()) { + stats.emplace_back("persistent_states", sb.as_cslice().str()); + } + } + promise.set_value(std::move(stats)); +} + void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise) { index_->begin_transaction().ensure(); td::MultiPromise mp; @@ -1255,16 +1322,120 @@ void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle han } } } + update_permanent_slices(); } -bool ArchiveManager::FileDescription::has_account_prefix(AccountIdPrefixFull account_id) const { - for (int i = 0; i < 60; i++) { - auto shard = shard_prefix(account_id, i); - if (first_blocks.count(shard)) { - return true; - } +void ArchiveManager::FileMap::shard_index_add(const FileDescription &desc) { + for (const auto &p : desc.first_blocks) { + ShardIndex &s = shards_[p.first]; + s.seqno_index_[p.second.seqno] = &desc; + s.lt_index_[p.second.lt] = &desc; + s.unix_time_index_[p.second.ts] = &desc; + s.packages_index_[desc.id] = &desc; } - return false; +} + +void ArchiveManager::FileMap::shard_index_del(const FileDescription &desc) { + for (const auto &p : desc.first_blocks) { + ShardIndex &s = shards_[p.first]; + s.seqno_index_.erase(p.second.seqno); + s.lt_index_.erase(p.second.lt); + s.unix_time_index_.erase(p.second.ts); + s.packages_index_.erase(desc.id); + } +} + +void ArchiveManager::FileMap::set_shard_first_block(const FileDescription &desc, ShardIdFull shard, + FileDescription::Desc v) { + ShardIndex &s = shards_[shard]; + auto &d = const_cast(desc); + auto it = d.first_blocks.find(shard); + if (it != d.first_blocks.end()) { + s.seqno_index_.erase(it->second.seqno); + s.lt_index_.erase(it->second.lt); + s.unix_time_index_.erase(it->second.ts); + } + d.first_blocks[shard] = v; + s.seqno_index_[v.seqno] = &d; + s.lt_index_[v.lt] = &d; + s.unix_time_index_[v.ts] = &d; + s.packages_index_[d.id] = &d; +} + +const ArchiveManager::FileDescription *ArchiveManager::FileMap::get_file_desc_by_seqno(ShardIdFull shard, + BlockSeqno seqno) const { + auto it = shards_.find(shard); + if (it == shards_.end()) { + return nullptr; + } + const auto &map = it->second.seqno_index_; + auto it2 = map.upper_bound(seqno); + if (it2 == map.begin()) { + return nullptr; + } + --it2; + return it2->second->deleted ? nullptr : it2->second; +} + +const ArchiveManager::FileDescription *ArchiveManager::FileMap::get_file_desc_by_lt(ShardIdFull shard, + LogicalTime lt) const { + auto it = shards_.find(shard); + if (it == shards_.end()) { + return nullptr; + } + const auto &map = it->second.lt_index_; + auto it2 = map.upper_bound(lt); + if (it2 == map.begin()) { + return nullptr; + } + --it2; + return it2->second->deleted ? nullptr : it2->second; +} + +const ArchiveManager::FileDescription *ArchiveManager::FileMap::get_file_desc_by_unix_time(ShardIdFull shard, + UnixTime ts) const { + auto it = shards_.find(shard); + if (it == shards_.end()) { + return nullptr; + } + const auto &map = it->second.unix_time_index_; + auto it2 = map.upper_bound(ts); + if (it2 == map.begin()) { + return nullptr; + } + --it2; + return it2->second->deleted ? nullptr : it2->second; +} + +const ArchiveManager::FileDescription *ArchiveManager::FileMap::get_next_file_desc(ShardIdFull shard, + const FileDescription *desc) const { + auto it = shards_.find(shard); + if (it == shards_.end()) { + return nullptr; + } + const auto &map = it->second.packages_index_; + auto it2 = desc ? map.upper_bound(desc->id) : map.begin(); + if (it2 == map.end()) { + return nullptr; + } + return it2->second->deleted ? nullptr : it2->second; +} + +void ArchiveManager::update_permanent_slices() { + if (archive_lru_.empty()) { + return; + } + std::vector ids; + if (!files_.empty()) { + ids.push_back(files_.rbegin()->first); + } + if (!key_files_.empty()) { + ids.push_back(key_files_.rbegin()->first); + } + if (!temp_files_.empty()) { + ids.push_back(temp_files_.rbegin()->first); + } + td::actor::send_closure(archive_lru_, &ArchiveLru::set_permanent_slices, std::move(ids)); } } // namespace validator diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index 79e6a2d7..d919e32e 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -28,7 +28,7 @@ class RootDb; class ArchiveManager : public td::actor::Actor { public: - ArchiveManager(td::actor::ActorId root, std::string db_root); + ArchiveManager(td::actor::ActorId root, std::string db_root, td::Ref opts); void add_handle(BlockHandle handle, td::Promise promise); void update_handle(BlockHandle handle, td::Promise promise); @@ -54,27 +54,35 @@ class ArchiveManager : public td::actor::Actor { td::int64 max_size, td::Promise promise); void check_persistent_state(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise); void check_zero_state(BlockIdExt block_id, td::Promise promise); + void get_previous_persistent_state_files(BlockSeqno cur_mc_seqno, + td::Promise>> promise); void truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise); //void truncate_continue(BlockSeqno masterchain_seqno, td::Promise promise); - void run_gc(UnixTime ts, UnixTime archive_ttl); + void run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl); /* from LTDB */ void get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts, td::Promise promise); void get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise promise); void get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno, td::Promise promise); - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise); + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::Promise promise); void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise); void start_up() override; + void alarm() override; - void begin_transaction(); void commit_transaction(); void set_async_mode(bool mode, td::Promise promise); + void set_current_shard_split_depth(td::uint32 value) { + cur_shard_split_depth_ = value; + } + + void prepare_stats(td::Promise>> promise); + static constexpr td::uint32 archive_size() { return 20000; } @@ -94,30 +102,94 @@ class ArchiveManager : public td::actor::Actor { auto file_actor_id() const { return file.get(); } - void clear_actor_id() { - file.reset(); - } - bool has_account_prefix(AccountIdPrefixFull account_id) const; PackageId id; - bool deleted; + mutable bool deleted; std::map first_blocks; - td::actor::ActorOwn file; + mutable td::actor::ActorOwn file; }; - std::map files_; - std::map key_files_; - std::map temp_files_; + class FileMap { + public: + std::map::const_iterator begin() const { + return files_.cbegin(); + } + std::map::const_iterator end() const { + return files_.cend(); + } + std::map::const_reverse_iterator rbegin() const { + return files_.crbegin(); + } + std::map::const_reverse_iterator rend() const { + return files_.crend(); + } + std::map::const_iterator find(PackageId x) const { + return files_.find(x); + } + size_t count(const PackageId &x) const { + return files_.count(x); + } + size_t size() const { + return files_.size(); + } + bool empty() const { + return files_.empty(); + } + std::map::const_iterator lower_bound(const PackageId &x) const { + return files_.lower_bound(x); + } + std::map::const_iterator upper_bound(const PackageId &x) const { + return files_.upper_bound(x); + } + void clear() { + files_.clear(); + shards_.clear(); + } + const FileDescription &emplace(const PackageId &id, FileDescription desc) { + auto it = files_.emplace(id, std::move(desc)); + if (it.second) { + shard_index_add(it.first->second); + } + return it.first->second; + } + void erase(std::map::const_iterator it) { + shard_index_del(it->second); + files_.erase(it); + } + void set_shard_first_block(const FileDescription &desc, ShardIdFull shard, FileDescription::Desc v); + const FileDescription *get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno) const; + const FileDescription *get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt) const; + const FileDescription *get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts) const; + const FileDescription *get_next_file_desc(ShardIdFull shard, const FileDescription *desc) const; + + private: + std::map files_; + struct ShardIndex { + std::map seqno_index_; + std::map lt_index_; + std::map unix_time_index_; + std::map packages_index_; + }; + std::map shards_; + + void shard_index_add(const FileDescription &desc); + void shard_index_del(const FileDescription &desc); + }; + FileMap files_, key_files_, temp_files_; + td::actor::ActorOwn archive_lru_; BlockSeqno finalized_up_to_{0}; bool async_mode_ = false; bool huge_transaction_started_ = false; td::uint32 huge_transaction_size_ = 0; + td::uint32 cur_shard_split_depth_ = 0; - auto &get_file_map(const PackageId &p) { + DbStatistics statistics_; + + FileMap &get_file_map(const PackageId &p) { return p.key ? key_files_ : p.temp ? temp_files_ : files_; } - std::map perm_states_; + std::map, FileReferenceShort> perm_states_; // Mc block seqno, hash -> state void load_package(PackageId seqno); void delete_package(PackageId seqno, td::Promise promise); @@ -126,29 +198,31 @@ class ArchiveManager : public td::actor::Actor { void get_handle_finish(BlockHandle handle, td::Promise promise); void get_file_short_cont(FileReference ref_id, PackageId idx, td::Promise promise); - FileDescription *get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt, - bool force); - FileDescription *add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt); - void update_desc(FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, UnixTime ts, LogicalTime lt); - FileDescription *get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno, bool key_block); - FileDescription *get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt, bool key_block); - FileDescription *get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts, bool key_block); - FileDescription *get_file_desc_by_seqno(AccountIdPrefixFull shard, BlockSeqno seqno, bool key_block); - FileDescription *get_file_desc_by_lt(AccountIdPrefixFull shard, LogicalTime lt, bool key_block); - FileDescription *get_file_desc_by_unix_time(AccountIdPrefixFull shard, UnixTime ts, bool key_block); - FileDescription *get_next_file_desc(FileDescription *f); - FileDescription *get_temp_file_desc_by_idx(PackageId idx); + const FileDescription *get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt, + bool force); + const FileDescription *add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt); + void update_desc(FileMap &f, const FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, UnixTime ts, + LogicalTime lt); + const FileDescription *get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno, bool key_block); + const FileDescription *get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt, bool key_block); + const FileDescription *get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts, bool key_block); + const FileDescription *get_file_desc_by_seqno(AccountIdPrefixFull shard, BlockSeqno seqno, bool key_block); + const FileDescription *get_file_desc_by_lt(AccountIdPrefixFull shard, LogicalTime lt, bool key_block); + const FileDescription *get_file_desc_by_unix_time(AccountIdPrefixFull shard, UnixTime ts, bool key_block); + const FileDescription *get_next_file_desc(const FileDescription *f, AccountIdPrefixFull shard, bool key_block); + const FileDescription *get_temp_file_desc_by_idx(PackageId idx); PackageId get_max_temp_file_desc_idx(); PackageId get_prev_temp_file_desc_idx(PackageId id); void add_persistent_state_impl(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise, std::function)> create_writer); - void written_perm_state(FileReferenceShort id); + void register_perm_state(FileReferenceShort id); - void persistent_state_gc(FileHash last); - void got_gc_masterchain_handle(ConstBlockHandle handle, FileHash hash); + void persistent_state_gc(std::pair last); + void got_gc_masterchain_handle(ConstBlockHandle handle, std::pair key); std::string db_root_; + td::Ref opts_; std::shared_ptr index_; @@ -158,6 +232,10 @@ class ArchiveManager : public td::actor::Actor { PackageId get_temp_package_id() const; PackageId get_key_package_id(BlockSeqno seqno) const; PackageId get_temp_package_id_by_unixtime(UnixTime ts) const; + + void update_permanent_slices(); + + static const td::uint32 TEMP_PACKAGES_TTL = 3600; }; } // namespace validator diff --git a/validator/db/archive-slice.cpp b/validator/db/archive-slice.cpp index 5f1bc708..41ed5099 100644 --- a/validator/db/archive-slice.cpp +++ b/validator/db/archive-slice.cpp @@ -21,31 +21,127 @@ #include "td/actor/MultiPromise.h" #include "validator/fabric.h" #include "td/db/RocksDb.h" -#include "ton/ton-io.hpp" #include "td/utils/port/path.h" #include "common/delay.h" #include "files-async.hpp" +#include "db-utils.h" namespace ton { namespace validator { +class PackageStatistics { + public: + void record_open(uint64_t count = 1) { + open_count.fetch_add(count, std::memory_order_relaxed); + } + + void record_close(uint64_t count = 1) { + close_count.fetch_add(count, std::memory_order_relaxed); + } + + void record_read(double time, uint64_t bytes) { + read_bytes.fetch_add(bytes, std::memory_order_relaxed); + std::lock_guard guard(read_mutex); + read_time.insert(time); + } + + void record_write(double time, uint64_t bytes) { + write_bytes.fetch_add(bytes, std::memory_order_relaxed); + std::lock_guard guard(write_mutex); + write_time.insert(time); + } + + std::string to_string_and_reset() { + std::stringstream ss; + ss.setf(std::ios::fixed); + ss.precision(6); + + ss << "ton.pack.open COUNT : " << open_count.exchange(0, std::memory_order_relaxed) << "\n"; + ss << "ton.pack.close COUNT : " << close_count.exchange(0, std::memory_order_relaxed) << "\n"; + + ss << "ton.pack.read.bytes COUNT : " << read_bytes.exchange(0, std::memory_order_relaxed) << "\n"; + ss << "ton.pack.write.bytes COUNT : " << write_bytes.exchange(0, std::memory_order_relaxed) << "\n"; + + PercentileStats temp_read_time; + { + std::lock_guard guard(read_mutex); + temp_read_time = std::move(read_time); + read_time.clear(); + } + ss << "ton.pack.read.micros " << temp_read_time.to_string() << "\n"; + + PercentileStats temp_write_time; + { + std::lock_guard guard(write_mutex); + temp_write_time = std::move(write_time); + write_time.clear(); + } + ss << "ton.pack.write.micros " << temp_write_time.to_string() << "\n"; + + return ss.str(); + } + + private: + std::atomic_uint64_t open_count{0}; + std::atomic_uint64_t close_count{0}; + PercentileStats read_time; + std::atomic_uint64_t read_bytes{0}; + PercentileStats write_time; + std::atomic_uint64_t write_bytes{0}; + + mutable std::mutex read_mutex; + mutable std::mutex write_mutex; +}; + +void DbStatistics::init() { + rocksdb_statistics = td::RocksDb::create_statistics(); + pack_statistics = std::make_shared(); +} + +std::string DbStatistics::to_string_and_reset() { + std::stringstream ss; + ss << td::RocksDb::statistics_to_string(rocksdb_statistics) << pack_statistics->to_string_and_reset(); + td::RocksDb::reset_statistics(rocksdb_statistics); + return ss.str(); +} + void PackageWriter::append(std::string filename, td::BufferSlice data, td::Promise> promise) { - auto offset = package_->append(std::move(filename), std::move(data), !async_mode_); - auto size = package_->size(); - + td::uint64 offset, size; + auto data_size = data.size(); + td::Timestamp start, end; + { + auto p = package_.lock(); + if (!p) { + promise.set_error(td::Status::Error("Package is closed")); + return; + } + start = td::Timestamp::now(); + offset = p->append(std::move(filename), std::move(data), !async_mode_); + end = td::Timestamp::now(); + size = p->size(); + } + if (statistics_) { + statistics_->record_write((end.at() - start.at()) * 1e6, data_size); + } promise.set_value(std::pair{offset, size}); } class PackageReader : public td::actor::Actor { public: PackageReader(std::shared_ptr package, td::uint64 offset, - td::Promise> promise) - : package_(std::move(package)), offset_(offset), promise_(std::move(promise)) { + td::Promise> promise, std::shared_ptr statistics) + : package_(std::move(package)), offset_(offset), promise_(std::move(promise)), statistics_(std::move(statistics)) { } - void start_up() { - promise_.set_result(package_->read(offset_)); + void start_up() override { + auto start = td::Timestamp::now(); + auto result = package_->read(offset_); + if (statistics_ && result.is_ok()) { + statistics_->record_read((td::Timestamp::now().at() - start.at()) * 1e6, result.ok_ref().second.size()); + } + package_ = {}; + promise_.set_result(std::move(result)); stop(); } @@ -53,8 +149,24 @@ class PackageReader : public td::actor::Actor { std::shared_ptr package_; td::uint64 offset_; td::Promise> promise_; + std::shared_ptr statistics_; }; +static std::string get_package_file_name(PackageId p_id, ShardIdFull shard_prefix) { + td::StringBuilder sb; + sb << p_id.name(); + if (!shard_prefix.is_masterchain()) { + sb << "."; + sb << shard_prefix.workchain << ":" << shard_to_str(shard_prefix.shard); + } + sb << ".pack"; + return sb.as_cslice().str(); +} + +static std::string package_info_to_str(BlockSeqno seqno, ShardIdFull shard_prefix) { + return PSTRING() << seqno << "." << shard_prefix.workchain << ":" << shard_to_str(shard_prefix.shard); +} + void ArchiveSlice::add_handle(BlockHandle handle, td::Promise promise) { if (destroyed_) { promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd")); @@ -64,6 +176,7 @@ void ArchiveSlice::add_handle(BlockHandle handle, td::Promise promise) update_handle(std::move(handle), std::move(promise)); return; } + before_query(); CHECK(!key_blocks_only_); CHECK(!temp_); CHECK(handle->inited_unix_time()); @@ -146,6 +259,7 @@ void ArchiveSlice::update_handle(BlockHandle handle, td::Promise promi promise.set_value(td::Unit()); return; } + before_query(); CHECK(!key_blocks_only_); begin_transaction(); @@ -168,10 +282,12 @@ void ArchiveSlice::add_file(BlockHandle handle, FileReference ref_id, td::Buffer promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd")); return; } + before_query(); TRY_RESULT_PROMISE( promise, p, choose_package( - handle ? handle->id().is_masterchain() ? handle->id().seqno() : handle->masterchain_ref_block() : 0, true)); + handle ? handle->id().is_masterchain() ? handle->id().seqno() : handle->masterchain_ref_block() : 0, + handle ? handle->id().shard_full() : ShardIdFull{masterchainId}, true)); std::string value; auto R = kv_->get(ref_id.hash().to_hex(), value); R.ensure(); @@ -179,6 +295,7 @@ void ArchiveSlice::add_file(BlockHandle handle, FileReference ref_id, td::Buffer promise.set_value(td::Unit()); return; } + promise = begin_async_query(std::move(promise)); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), idx = p->idx, ref_id, promise = std::move(promise)]( td::Result> R) mutable { if (R.is_error()) { @@ -217,6 +334,7 @@ void ArchiveSlice::get_handle(BlockIdExt block_id, td::Promise prom promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd")); return; } + before_query(); CHECK(!key_blocks_only_); std::string value; auto R = kv_->get(get_db_key_block_info(block_id), value); @@ -239,6 +357,7 @@ void ArchiveSlice::get_temp_handle(BlockIdExt block_id, td::Promiseget(get_db_key_block_info(block_id), value); @@ -261,6 +380,7 @@ void ArchiveSlice::get_file(ConstBlockHandle handle, FileReference ref_id, td::P promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd")); return; } + before_query(); std::string value; auto R = kv_->get(ref_id.hash().to_hex(), value); R.ensure(); @@ -272,7 +392,9 @@ void ArchiveSlice::get_file(ConstBlockHandle handle, FileReference ref_id, td::P TRY_RESULT_PROMISE( promise, p, choose_package( - handle ? handle->id().is_masterchain() ? handle->id().seqno() : handle->masterchain_ref_block() : 0, false)); + handle ? handle->id().is_masterchain() ? handle->id().seqno() : handle->masterchain_ref_block() : 0, + handle ? handle->id().shard_full() : ShardIdFull{masterchainId}, false)); + promise = begin_async_query(std::move(promise)); auto P = td::PromiseCreator::lambda( [promise = std::move(promise)](td::Result> R) mutable { if (R.is_error()) { @@ -281,7 +403,7 @@ void ArchiveSlice::get_file(ConstBlockHandle handle, FileReference ref_id, td::P promise.set_value(std::move(R.move_as_ok().second)); } }); - td::actor::create_actor("reader", p->package, offset, std::move(P)).release(); + td::actor::create_actor("reader", p->package, offset, std::move(P), statistics_.pack_statistics).release(); } void ArchiveSlice::get_block_common(AccountIdPrefixFull account_id, @@ -292,6 +414,7 @@ void ArchiveSlice::get_block_common(AccountIdPrefixFull account_id, promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd")); return; } + before_query(); bool f = false; BlockIdExt block_id; td::uint32 ls = 0; @@ -312,7 +435,7 @@ void ArchiveSlice::get_block_common(AccountIdPrefixFull account_id, auto G = fetch_tl_object(value, true); G.ensure(); auto g = G.move_as_ok(); - if (compare_desc(*g.get()) > 0) { + if (compare_desc(*g) > 0) { continue; } td::uint32 l = g->first_idx_ - 1; @@ -328,7 +451,7 @@ void ArchiveSlice::get_block_common(AccountIdPrefixFull account_id, auto E = fetch_tl_object(td::BufferSlice{value}, true); E.ensure(); auto e = E.move_as_ok(); - int cmp_val = compare(*e.get()); + int cmp_val = compare(*e); if (cmp_val < 0) { rseq = create_block_id(e->id_); @@ -342,9 +465,7 @@ void ArchiveSlice::get_block_common(AccountIdPrefixFull account_id, } } if (rseq.is_valid()) { - if (!block_id.is_valid()) { - block_id = rseq; - } else if (block_id.id.seqno > rseq.id.seqno) { + if (!block_id.is_valid() || block_id.id.seqno > rseq.id.seqno) { block_id = rseq; } } @@ -430,76 +551,173 @@ void ArchiveSlice::get_slice(td::uint64 archive_id, td::uint64 offset, td::uint3 promise.set_error(td::Status::Error(ErrorCode::error, "bad archive id")); return; } + before_query(); auto value = static_cast(archive_id >> 32); - TRY_RESULT_PROMISE(promise, p, choose_package(value, false)); + PackageInfo *p; + if (shard_split_depth_ == 0) { + TRY_RESULT_PROMISE_ASSIGN(promise, p, choose_package(value, ShardIdFull{masterchainId}, false)); + } else { + if (value >= packages_.size()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "no such package")); + return; + } + p = &packages_[value]; + } + promise = begin_async_query(std::move(promise)); td::actor::create_actor("readfile", p->path, offset, limit, 0, std::move(promise)).release(); } -void ArchiveSlice::get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) { +void ArchiveSlice::get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) { + before_query(); if (!sliced_mode_) { promise.set_result(archive_id_); } else { - TRY_RESULT_PROMISE(promise, p, choose_package(masterchain_seqno, false)); - promise.set_result(p->id * (1ull << 32) + archive_id_); + TRY_RESULT_PROMISE(promise, p, choose_package(masterchain_seqno, shard_prefix, false)); + if (shard_split_depth_ == 0) { + promise.set_result(p->seqno * (1ull << 32) + archive_id_); + } else { + promise.set_result(p->idx * (1ull << 32) + archive_id_); + } } } -void ArchiveSlice::start_up() { - PackageId p_id{archive_id_, key_blocks_only_, temp_}; - std::string db_path = PSTRING() << db_root_ << p_id.path() << p_id.name() << ".index"; - kv_ = std::make_shared(td::RocksDb::open(db_path).move_as_ok()); +void ArchiveSlice::before_query() { + if (status_ == st_closed) { + LOG(DEBUG) << "Opening archive slice " << db_path_; + td::RocksDbOptions db_options; + db_options.statistics = statistics_.rocksdb_statistics; + kv_ = std::make_unique(td::RocksDb::open(db_path_, std::move(db_options)).move_as_ok()); + std::string value; + auto R2 = kv_->get("status", value); + R2.ensure(); + sliced_mode_ = false; + slice_size_ = 100; - std::string value; - auto R2 = kv_->get("status", value); - R2.ensure(); - - if (R2.move_as_ok() == td::KeyValue::GetStatus::Ok) { - if (value == "sliced") { - sliced_mode_ = true; - R2 = kv_->get("slices", value); - R2.ensure(); - auto tot = td::to_integer(value); - R2 = kv_->get("slice_size", value); - R2.ensure(); - slice_size_ = td::to_integer(value); - CHECK(slice_size_ > 0); - for (td::uint32 i = 0; i < tot; i++) { - R2 = kv_->get(PSTRING() << "status." << i, value); + if (R2.move_as_ok() == td::KeyValue::GetStatus::Ok) { + if (value == "sliced") { + sliced_mode_ = true; + R2 = kv_->get("slices", value); R2.ensure(); - auto len = td::to_integer(value); - R2 = kv_->get(PSTRING() << "version." << i, value); + auto tot = td::to_integer(value); + R2 = kv_->get("slice_size", value); + R2.ensure(); + slice_size_ = td::to_integer(value); + CHECK(slice_size_ > 0); + R2 = kv_->get("shard_split_depth", value); R2.ensure(); - td::uint32 ver = 0; if (R2.move_as_ok() == td::KeyValue::GetStatus::Ok) { - ver = td::to_integer(value); + shard_split_depth_ = td::to_integer(value); + CHECK(shard_split_depth_ <= 60); + } else { + shard_split_depth_ = 0; } - auto v = archive_id_ + slice_size_ * i; - add_package(v, len, ver); + for (td::uint32 i = 0; i < tot; i++) { + R2 = kv_->get(PSTRING() << "status." << i, value); + R2.ensure(); + CHECK(R2.move_as_ok() == td::KeyValue::GetStatus::Ok); + auto len = td::to_integer(value); + R2 = kv_->get(PSTRING() << "version." << i, value); + R2.ensure(); + td::uint32 ver = 0; + if (R2.move_as_ok() == td::KeyValue::GetStatus::Ok) { + ver = td::to_integer(value); + } + td::uint32 seqno; + ShardIdFull shard_prefix; + if (shard_split_depth_ == 0) { + seqno = archive_id_ + slice_size_ * i; + shard_prefix = ShardIdFull{masterchainId}; + } else { + R2 = kv_->get(PSTRING() << "info." << i, value); + R2.ensure(); + CHECK(R2.move_as_ok() == td::KeyValue::GetStatus::Ok); + unsigned long long shard; + CHECK(sscanf(value.c_str(), "%u.%d:%016llx", &seqno, &shard_prefix.workchain, &shard) == 3); + shard_prefix.shard = shard; + } + add_package(seqno, shard_prefix, len, ver); + } + } else { + auto len = td::to_integer(value); + add_package(archive_id_, ShardIdFull{masterchainId}, len, 0); } } else { - auto len = td::to_integer(value); - add_package(archive_id_, len, 0); + if (!temp_ && !key_blocks_only_) { + sliced_mode_ = true; + kv_->begin_transaction().ensure(); + kv_->set("status", "sliced").ensure(); + kv_->set("slices", "1").ensure(); + kv_->set("slice_size", td::to_string(slice_size_)).ensure(); + kv_->set("status.0", "0").ensure(); + kv_->set("version.0", td::to_string(default_package_version())).ensure(); + if (shard_split_depth_ > 0) { + kv_->set("info.0", package_info_to_str(archive_id_, ShardIdFull{masterchainId})).ensure(); + kv_->set("shard_split_depth", td::to_string(shard_split_depth_)).ensure(); + } + kv_->commit_transaction().ensure(); + add_package(archive_id_, ShardIdFull{masterchainId}, 0, default_package_version()); + } else { + kv_->begin_transaction().ensure(); + kv_->set("status", "0").ensure(); + kv_->commit_transaction().ensure(); + add_package(archive_id_, ShardIdFull{masterchainId}, 0, 0); + } } - } else { - if (!temp_ && !key_blocks_only_) { - sliced_mode_ = true; - kv_->begin_transaction().ensure(); - kv_->set("status", "sliced").ensure(); - kv_->set("slices", "1").ensure(); - kv_->set("slice_size", td::to_string(slice_size_)).ensure(); - kv_->set("status.0", "0").ensure(); - kv_->set("version.0", td::to_string(default_package_version())).ensure(); - kv_->commit_transaction().ensure(); - add_package(archive_id_, 0, default_package_version()); + } + status_ = st_open; + if (!archive_lru_.empty()) { + td::actor::send_closure(archive_lru_, &ArchiveLru::on_query, actor_id(this), p_id_, + packages_.size() + ESTIMATED_DB_OPEN_FILES); + } +} + +void ArchiveSlice::open_files() { + before_query(); +} + +void ArchiveSlice::close_files() { + if (status_ == st_open) { + if (active_queries_ == 0) { + do_close(); } else { - kv_->begin_transaction().ensure(); - kv_->set("status", "0").ensure(); - kv_->commit_transaction().ensure(); - add_package(archive_id_, 0, 0); + status_ = st_want_close; } } } +void ArchiveSlice::do_close() { + if (destroyed_) { + return; + } + CHECK(status_ != st_closed && active_queries_ == 0); + LOG(DEBUG) << "Closing archive slice " << db_path_; + status_ = st_closed; + kv_ = {}; + if (statistics_.pack_statistics) { + statistics_.pack_statistics->record_close(packages_.size()); + } + packages_.clear(); + id_to_package_.clear(); +} + +template +td::Promise ArchiveSlice::begin_async_query(td::Promise promise) { + ++active_queries_; + return [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + td::actor::send_closure(SelfId, &ArchiveSlice::end_async_query); + promise.set_result(std::move(R)); + }; +} + +void ArchiveSlice::end_async_query() { + CHECK(active_queries_ > 0); + --active_queries_; + if (active_queries_ == 0 && status_ == st_want_close) { + do_close(); + } +} + void ArchiveSlice::begin_transaction() { if (!async_mode_ || !huge_transaction_started_) { kv_->begin_transaction().ensure(); @@ -521,7 +739,7 @@ void ArchiveSlice::commit_transaction() { void ArchiveSlice::set_async_mode(bool mode, td::Promise promise) { async_mode_ = mode; - if (!async_mode_ && huge_transaction_started_) { + if (!async_mode_ && huge_transaction_started_ && kv_) { kv_->commit_transaction().ensure(); huge_transaction_size_ = 0; huge_transaction_started_ = false; @@ -532,63 +750,85 @@ void ArchiveSlice::set_async_mode(bool mode, td::Promise promise) { ig.add_promise(std::move(promise)); for (auto &p : packages_) { - td::actor::send_closure(p.writer, &PackageWriter::set_async_mode, mode, std::move(promise)); + td::actor::send_closure(p.writer, &PackageWriter::set_async_mode, mode, ig.get_promise()); } } -ArchiveSlice::ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, std::string db_root) +ArchiveSlice::ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, + td::uint32 shard_split_depth, std::string db_root, + td::actor::ActorId archive_lru, DbStatistics statistics) : archive_id_(archive_id) , key_blocks_only_(key_blocks_only) , temp_(temp) , finalized_(finalized) - , db_root_(std::move(db_root)) { + , p_id_(archive_id_, key_blocks_only_, temp_) + , shard_split_depth_(temp || key_blocks_only ? 0 : shard_split_depth) + , db_root_(std::move(db_root)) + , archive_lru_(std::move(archive_lru)) + , statistics_(statistics) { + db_path_ = PSTRING() << db_root_ << p_id_.path() << p_id_.name() << ".index"; } -td::Result ArchiveSlice::choose_package(BlockSeqno masterchain_seqno, bool force) { +td::Result ArchiveSlice::choose_package(BlockSeqno masterchain_seqno, + ShardIdFull shard_prefix, bool force) { if (temp_ || key_blocks_only_ || !sliced_mode_) { return &packages_[0]; } if (masterchain_seqno < archive_id_) { return td::Status::Error(ErrorCode::notready, "too small masterchain seqno"); } - auto v = (masterchain_seqno - archive_id_) / slice_size_; - if (v >= packages_.size()) { + masterchain_seqno -= (masterchain_seqno - archive_id_) % slice_size_; + CHECK((masterchain_seqno - archive_id_) % slice_size_ == 0); + if (shard_split_depth_ == 0) { + shard_prefix = ShardIdFull{masterchainId}; + } else if (!shard_prefix.is_masterchain()) { + shard_prefix.shard |= 1; // In case length is < split depth + shard_prefix = ton::shard_prefix(shard_prefix, shard_split_depth_); + } + auto it = id_to_package_.find({masterchain_seqno, shard_prefix}); + if (it == id_to_package_.end()) { if (!force) { - return td::Status::Error(ErrorCode::notready, "too big masterchain seqno"); + return td::Status::Error(ErrorCode::notready, "no such package"); } - CHECK(v == packages_.size()); begin_transaction(); + size_t v = packages_.size(); kv_->set("slices", td::to_string(v + 1)).ensure(); kv_->set(PSTRING() << "status." << v, "0").ensure(); kv_->set(PSTRING() << "version." << v, td::to_string(default_package_version())).ensure(); + if (shard_split_depth_ > 0) { + kv_->set(PSTRING() << "info." << v, package_info_to_str(masterchain_seqno, shard_prefix)).ensure(); + } commit_transaction(); - CHECK((masterchain_seqno - archive_id_) % slice_size_ == 0); - add_package(masterchain_seqno, 0, default_package_version()); + add_package(masterchain_seqno, shard_prefix, 0, default_package_version()); return &packages_[v]; } else { - return &packages_[v]; + return &packages_[it->second]; } } -void ArchiveSlice::add_package(td::uint32 seqno, td::uint64 size, td::uint32 version) { +void ArchiveSlice::add_package(td::uint32 seqno, ShardIdFull shard_prefix, td::uint64 size, td::uint32 version) { PackageId p_id{seqno, key_blocks_only_, temp_}; - std::string path = PSTRING() << db_root_ << p_id.path() << p_id.name() << ".pack"; + std::string path = PSTRING() << db_root_ << p_id.path() << get_package_file_name(p_id, shard_prefix); auto R = Package::open(path, false, true); if (R.is_error()) { LOG(FATAL) << "failed to open/create archive '" << path << "': " << R.move_as_error(); return; } + if (statistics_.pack_statistics) { + statistics_.pack_statistics->record_open(); + } auto idx = td::narrow_cast(packages_.size()); + id_to_package_[{seqno, shard_prefix}] = idx; if (finalized_) { - packages_.emplace_back(nullptr, td::actor::ActorOwn(), seqno, path, idx, version); + packages_.emplace_back(nullptr, td::actor::ActorOwn(), seqno, shard_prefix, path, idx, version); return; } auto pack = std::make_shared(R.move_as_ok()); if (version >= 1) { pack->truncate(size).ensure(); } - auto writer = td::actor::create_actor("writer", pack); - packages_.emplace_back(std::move(pack), std::move(writer), seqno, path, idx, version); + auto writer = td::actor::create_actor("writer", pack, async_mode_, statistics_.pack_statistics); + packages_.emplace_back(std::move(pack), std::move(writer), seqno, shard_prefix, path, idx, version); } namespace { @@ -599,34 +839,33 @@ void destroy_db(std::string name, td::uint32 attempt, td::Promise prom promise.set_value(td::Unit()); return; } - if (S.is_error() && attempt > 0 && attempt % 64 == 0) { + if (attempt > 0 && attempt % 64 == 0) { LOG(ERROR) << "failed to destroy index " << name << ": " << S; } else { LOG(DEBUG) << "failed to destroy index " << name << ": " << S; } delay_action( - [name, attempt, promise = std::move(promise)]() mutable { destroy_db(name, attempt, std::move(promise)); }, + [name, attempt, promise = std::move(promise)]() mutable { destroy_db(name, attempt + 1, std::move(promise)); }, td::Timestamp::in(1.0)); } } // namespace void ArchiveSlice::destroy(td::Promise promise) { - td::MultiPromise mp; - auto ig = mp.init_guard(); - ig.add_promise(std::move(promise)); + before_query(); destroyed_ = true; for (auto &p : packages_) { td::unlink(p.path).ensure(); } - + if (statistics_.pack_statistics) { + statistics_.pack_statistics->record_close(packages_.size()); + } packages_.clear(); + id_to_package_.clear(); kv_ = nullptr; - PackageId p_id{archive_id_, key_blocks_only_, temp_}; - std::string db_path = PSTRING() << db_root_ << p_id.path() << p_id.name() << ".index"; - delay_action([name = db_path, attempt = 0, - promise = ig.get_promise()]() mutable { destroy_db(name, attempt, std::move(promise)); }, + delay_action([name = db_path_, attempt = 0, + promise = std::move(promise)]() mutable { destroy_db(name, attempt, std::move(promise)); }, td::Timestamp::in(0.0)); } @@ -694,7 +933,7 @@ void ArchiveSlice::move_handle(ConstBlockHandle handle, Package *old_pack, Packa move_file(fileref::Block{handle->id()}, old_pack, pack); } -bool ArchiveSlice::truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block_id, td::uint32 cutoff_idx, +bool ArchiveSlice::truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block_id, td::uint32 cutoff_seqno, Package *pack) { std::string value; auto R = kv_->get(get_db_key_block_info(block_id), value); @@ -709,18 +948,18 @@ bool ArchiveSlice::truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block return false; } - auto S = choose_package(seqno, false); + auto S = choose_package(seqno, block_id.shard_full(), false); S.ensure(); auto p = S.move_as_ok(); - CHECK(p->idx <= cutoff_idx); - if (p->idx == cutoff_idx) { + CHECK(p->seqno <= cutoff_seqno); + if (p->seqno == cutoff_seqno) { move_handle(std::move(handle), p->package.get(), pack); } return true; } -void ArchiveSlice::truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shard, td::uint32 cutoff_idx, +void ArchiveSlice::truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shard, td::uint32 cutoff_seqno, Package *pack) { auto key = get_db_key_lt_desc(shard); std::string value; @@ -746,7 +985,7 @@ void ArchiveSlice::truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shar E.ensure(); auto e = E.move_as_ok(); - if (truncate_block(masterchain_seqno, create_block_id(e->id_), cutoff_idx, pack)) { + if (truncate_block(masterchain_seqno, create_block_id(e->id_), cutoff_seqno, pack)) { CHECK(new_last_idx == i); new_last_idx = i + 1; } @@ -758,11 +997,12 @@ void ArchiveSlice::truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shar } } -void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise) { +void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle, td::Promise promise) { if (temp_ || archive_id_ > masterchain_seqno) { destroy(std::move(promise)); return; } + before_query(); LOG(INFO) << "TRUNCATE: slice " << archive_id_ << " maxseqno= " << max_masterchain_seqno() << " truncate_upto=" << masterchain_seqno; if (max_masterchain_seqno() <= masterchain_seqno) { @@ -770,15 +1010,8 @@ void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handl return; } - auto cutoff = choose_package(masterchain_seqno, false); - cutoff.ensure(); - auto pack = cutoff.move_as_ok(); - CHECK(pack); - - auto pack_r = Package::open(pack->path + ".new", false, true); - pack_r.ensure(); - auto new_package = std::make_shared(pack_r.move_as_ok()); - new_package->truncate(0).ensure(); + std::map old_packages; + std::map> new_packages; std::string value; auto status_key = create_serialize_tl_object(); @@ -799,38 +1032,129 @@ void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handl auto G = fetch_tl_object(value, true); G.ensure(); auto g = G.move_as_ok(); + ShardIdFull shard{g->workchain_, static_cast(g->shard_)}; - truncate_shard(masterchain_seqno, ShardIdFull{g->workchain_, static_cast(g->shard_)}, pack->idx, - new_package.get()); + auto package_r = choose_package(masterchain_seqno, shard, false); + if (package_r.is_error()) { + continue; + } + auto package = package_r.move_as_ok(); + CHECK(package); + if (!old_packages.count(package->shard_prefix)) { + old_packages[package->shard_prefix] = package; + auto new_package_r = Package::open(package->path + ".new", false, true); + new_package_r.ensure(); + auto new_package = std::make_shared(new_package_r.move_as_ok()); + new_package->truncate(0).ensure(); + new_packages[package->shard_prefix] = std::move(new_package); + } + truncate_shard(masterchain_seqno, shard, package->seqno, new_packages[package->shard_prefix].get()); } + for (auto& [shard_prefix, package] : old_packages) { + auto new_package = new_packages[shard_prefix]; + CHECK(new_package); + package->package = new_package; + package->writer.reset(); + td::unlink(package->path).ensure(); + td::rename(package->path + ".new", package->path).ensure(); + package->writer = td::actor::create_actor("writer", new_package, async_mode_); + } + + std::vector new_packages_info; + if (!sliced_mode_) { - kv_->set("status", td::to_string(new_package->size())).ensure(); + kv_->set("status", td::to_string(packages_.at(0).package->size())).ensure(); } else { - kv_->set(PSTRING() << "status." << pack->idx, td::to_string(new_package->size())).ensure(); - for (size_t i = pack->idx + 1; i < packages_.size(); i++) { + for (PackageInfo &package : packages_) { + if (package.seqno <= masterchain_seqno) { + new_packages_info.push_back(std::move(package)); + } else { + td::unlink(package.path).ensure(); + } + } + id_to_package_.clear(); + for (td::uint32 i = 0; i < new_packages_info.size(); ++i) { + PackageInfo &package = new_packages_info[i]; + package.idx = i; + kv_->set(PSTRING() << "status." << i, td::to_string(package.package->size())).ensure(); + kv_->set(PSTRING() << "version." << i, td::to_string(package.version)).ensure(); + if (shard_split_depth_ > 0) { + kv_->set(PSTRING() << "info." << i, package_info_to_str(package.seqno, package.shard_prefix)).ensure(); + } + id_to_package_[{package.seqno, package.shard_prefix}] = i; + } + for (size_t i = new_packages_info.size(); i < packages_.size(); i++) { kv_->erase(PSTRING() << "status." << i); kv_->erase(PSTRING() << "version." << i); + kv_->erase(PSTRING() << "info." << i); } - kv_->set("slices", td::to_string(pack->idx + 1)); + kv_->set("slices", td::to_string(new_packages_info.size())); + if (statistics_.pack_statistics) { + statistics_.pack_statistics->record_close(packages_.size() - new_packages_info.size()); + } + packages_ = std::move(new_packages_info); } - pack->package = new_package; - pack->writer.reset(); - td::unlink(pack->path).ensure(); - td::rename(pack->path + ".new", pack->path).ensure(); - pack->writer = td::actor::create_actor("writer", new_package); - - for (auto idx = pack->idx + 1; idx < packages_.size(); idx++) { - td::unlink(packages_[idx].path).ensure(); - } - packages_.erase(packages_.begin() + pack->idx + 1); - kv_->commit_transaction().ensure(); - promise.set_value(td::Unit()); } +static std::tuple to_tuple(const PackageId &id) { + return {id.id, id.temp, id.key}; +} + +void ArchiveLru::on_query(td::actor::ActorId slice, PackageId id, size_t files_count) { + SliceInfo &info = slices_[to_tuple(id)]; + if (info.opened_idx != 0) { + total_files_ -= info.files_count; + lru_.erase(info.opened_idx); + } + info.actor = std::move(slice); + total_files_ += (info.files_count = files_count); + info.opened_idx = current_idx_++; + if (!info.is_permanent) { + lru_.emplace(info.opened_idx, id); + } + enforce_limit(); +} + +void ArchiveLru::set_permanent_slices(std::vector ids) { + for (auto id : permanent_slices_) { + SliceInfo &info = slices_[to_tuple(id)]; + if (!info.is_permanent) { + continue; + } + info.is_permanent = false; + if (info.opened_idx) { + lru_.emplace(info.opened_idx, id); + } + } + permanent_slices_ = std::move(ids); + for (auto id : permanent_slices_) { + SliceInfo &info = slices_[to_tuple(id)]; + if (info.is_permanent) { + continue; + } + info.is_permanent = true; + if (info.opened_idx) { + lru_.erase(info.opened_idx); + } + } + enforce_limit(); +} + +void ArchiveLru::enforce_limit() { + while (total_files_ > max_total_files_ && lru_.size() > 1) { + auto it = lru_.begin(); + auto it2 = slices_.find(to_tuple(it->second)); + lru_.erase(it); + total_files_ -= it2->second.files_count; + td::actor::send_closure(it2->second.actor, &ArchiveSlice::close_files); + it2->second.opened_idx = 0; + } +} + } // namespace validator } // namespace ton diff --git a/validator/db/archive-slice.hpp b/validator/db/archive-slice.hpp index 487115fe..a027ec0f 100644 --- a/validator/db/archive-slice.hpp +++ b/validator/db/archive-slice.hpp @@ -21,6 +21,12 @@ #include "validator/interfaces/db.h" #include "package.hpp" #include "fileref.hpp" +#include "td/db/RocksDb.h" +#include + +namespace rocksdb { +class Statistics; +} namespace ton { @@ -44,7 +50,7 @@ struct PackageId { std::string path() const; std::string name() const; - bool is_empty() { + bool is_empty() const { return id == std::numeric_limits::max(); } static PackageId empty(bool key, bool temp) { @@ -52,30 +58,48 @@ struct PackageId { } }; +class PackageStatistics; + +struct DbStatistics { + void init(); + std::string to_string_and_reset(); + + std::shared_ptr pack_statistics; + std::shared_ptr rocksdb_statistics; +}; + class PackageWriter : public td::actor::Actor { public: - PackageWriter(std::shared_ptr package) : package_(std::move(package)) { + PackageWriter(std::weak_ptr package, bool async_mode = false, std::shared_ptr statistics = nullptr) + : package_(std::move(package)), async_mode_(async_mode), statistics_(std::move(statistics)) { } void append(std::string filename, td::BufferSlice data, td::Promise> promise); void set_async_mode(bool mode, td::Promise promise) { async_mode_ = mode; if (!async_mode_) { - package_->sync(); + auto p = package_.lock(); + if (p) { + p->sync(); + } } promise.set_value(td::Unit()); } private: - std::shared_ptr package_; + std::weak_ptr package_; bool async_mode_ = false; + std::shared_ptr statistics_; }; +class ArchiveLru; + class ArchiveSlice : public td::actor::Actor { public: - ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, std::string db_root); + ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, td::uint32 shard_split_depth, + std::string db_root, td::actor::ActorId archive_lru, DbStatistics statistics = {}); - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise); + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::Promise promise); void add_handle(BlockHandle handle, td::Promise promise); void update_handle(BlockHandle handle, td::Promise promise); @@ -95,16 +119,24 @@ class ArchiveSlice : public td::actor::Actor { void get_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise); - void start_up() override; void destroy(td::Promise promise); void truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise); - void begin_transaction(); - void commit_transaction(); void set_async_mode(bool mode, td::Promise promise); + void open_files(); + void close_files(); + private: - void written_data(BlockHandle handle, td::Promise promise); + void before_query(); + void do_close(); + template + td::Promise begin_async_query(td::Promise promise); + void end_async_query(); + + void begin_transaction(); + void commit_transaction(); + void add_file_cont(size_t idx, FileReference ref_id, td::uint64 offset, td::uint64 size, td::Promise promise); @@ -112,13 +144,14 @@ class ArchiveSlice : public td::actor::Actor { td::BufferSlice get_db_key_lt_desc(ShardIdFull shard); td::BufferSlice get_db_key_lt_el(ShardIdFull shard, td::uint32 idx); td::BufferSlice get_db_key_block_info(BlockIdExt block_id); - td::BufferSlice get_lt_from_db(ShardIdFull shard, td::uint32 idx); td::uint32 archive_id_; bool key_blocks_only_; bool temp_; bool finalized_; + PackageId p_id_; + std::string db_path_; bool destroyed_ = false; bool async_mode_ = false; @@ -126,33 +159,44 @@ class ArchiveSlice : public td::actor::Actor { bool sliced_mode_{false}; td::uint32 huge_transaction_size_ = 0; td::uint32 slice_size_{100}; + td::uint32 shard_split_depth_ = 0; + + enum Status { + st_closed, st_open, st_want_close + } status_ = st_closed; + size_t active_queries_ = 0; std::string db_root_; - std::shared_ptr kv_; + td::actor::ActorId archive_lru_; + DbStatistics statistics_; + std::unique_ptr kv_; struct PackageInfo { - PackageInfo(std::shared_ptr package, td::actor::ActorOwn writer, BlockSeqno id, + PackageInfo(std::shared_ptr package, td::actor::ActorOwn writer, BlockSeqno seqno, ShardIdFull shard_prefix, std::string path, td::uint32 idx, td::uint32 version) : package(std::move(package)) , writer(std ::move(writer)) - , id(id) + , seqno(seqno) + , shard_prefix(shard_prefix) , path(std::move(path)) , idx(idx) , version(version) { } std::shared_ptr package; td::actor::ActorOwn writer; - BlockSeqno id; + BlockSeqno seqno; + ShardIdFull shard_prefix; std::string path; td::uint32 idx; td::uint32 version; }; std::vector packages_; + std::map, td::uint32> id_to_package_; - td::Result choose_package(BlockSeqno masterchain_seqno, bool force); - void add_package(BlockSeqno masterchain_seqno, td::uint64 size, td::uint32 version); - void truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shard, td::uint32 cutoff_idx, Package *pack); - bool truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block_id, td::uint32 cutoff_idx, Package *pack); + td::Result choose_package(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, bool force); + void add_package(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::uint64 size, td::uint32 version); + void truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shard, td::uint32 cutoff_seqno, Package *pack); + bool truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block_id, td::uint32 cutoff_seqno, Package *pack); void delete_handle(ConstBlockHandle handle); void delete_file(FileReference ref_id); @@ -164,6 +208,32 @@ class ArchiveSlice : public td::actor::Actor { static constexpr td::uint32 default_package_version() { return 1; } + + static const size_t ESTIMATED_DB_OPEN_FILES = 5; +}; + +class ArchiveLru : public td::actor::Actor { + public: + explicit ArchiveLru(size_t max_total_files) : max_total_files_(max_total_files) { + CHECK(max_total_files_ > 0); + } + void on_query(td::actor::ActorId slice, PackageId id, size_t files_count); + void set_permanent_slices(std::vector ids); + private: + size_t current_idx_ = 1; + struct SliceInfo { + td::actor::ActorId actor; + size_t files_count = 0; + size_t opened_idx = 0; // 0 - not opened + bool is_permanent = false; + }; + std::map, SliceInfo> slices_; + std::map lru_; + size_t total_files_ = 0; + size_t max_total_files_ = 0; + std::vector permanent_slices_; + + void enforce_limit(); }; } // namespace validator diff --git a/validator/db/archiver.cpp b/validator/db/archiver.cpp index c8cbd7b9..93ba18ca 100644 --- a/validator/db/archiver.cpp +++ b/validator/db/archiver.cpp @@ -25,11 +25,27 @@ namespace ton { namespace validator { BlockArchiver::BlockArchiver(BlockHandle handle, td::actor::ActorId archive_db, - td::Promise promise) - : handle_(std::move(handle)), archive_(archive_db), promise_(std::move(promise)) { + td::actor::ActorId db, td::Promise promise) + : handle_(std::move(handle)), archive_(archive_db), db_(std::move(db)), promise_(std::move(promise)) { } void BlockArchiver::start_up() { + if (handle_->id().is_masterchain()) { + td::actor::send_closure(db_, &Db::get_block_state, handle_, + [SelfId = actor_id(this), archive = archive_](td::Result> R) { + R.ensure(); + td::Ref state{R.move_as_ok()}; + td::uint32 monitor_min_split = state->monitor_min_split_depth(basechainId); + td::actor::send_closure(archive, &ArchiveManager::set_current_shard_split_depth, + monitor_min_split); + td::actor::send_closure(SelfId, &BlockArchiver::move_handle); + }); + } else { + move_handle(); + } +} + +void BlockArchiver::move_handle() { if (handle_->handle_moved_to_archive()) { moved_handle(); } else { diff --git a/validator/db/archiver.hpp b/validator/db/archiver.hpp index 859f269c..9498977f 100644 --- a/validator/db/archiver.hpp +++ b/validator/db/archiver.hpp @@ -33,11 +33,13 @@ class FileDb; class BlockArchiver : public td::actor::Actor { public: - BlockArchiver(BlockHandle handle, td::actor::ActorId archive_db, td::Promise promise); + BlockArchiver(BlockHandle handle, td::actor::ActorId archive_db, td::actor::ActorId db, + td::Promise promise); void abort_query(td::Status error); void start_up() override; + void move_handle(); void moved_handle(); void got_proof(td::BufferSlice data); void written_proof(); @@ -50,6 +52,7 @@ class BlockArchiver : public td::actor::Actor { private: BlockHandle handle_; td::actor::ActorId archive_; + td::actor::ActorId db_; td::Promise promise_; }; diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index caefe8e4..e86a373d 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -17,39 +17,45 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "celldb.hpp" + +#include "files-async.hpp" #include "rootdb.hpp" #include "td/db/RocksDb.h" +#include "rocksdb/utilities/optimistic_transaction_db.h" #include "ton/ton-tl.hpp" #include "ton/ton-io.hpp" +#include "common/delay.h" namespace ton { namespace validator { - class CellDbAsyncExecutor : public vm::DynamicBagOfCellsDb::AsyncExecutor { public: explicit CellDbAsyncExecutor(td::actor::ActorId cell_db) : cell_db_(std::move(cell_db)) { } - void execute_async(std::function f) { + void execute_async(std::function f) override { class Runner : public td::actor::Actor { public: - explicit Runner(std::function f) : f_(std::move(f)) {} - void start_up() { + explicit Runner(std::function f) : f_(std::move(f)) { + } + void start_up() override { f_(); stop(); } + private: std::function f_; }; td::actor::create_actor("executeasync", std::move(f)).release(); } - void execute_sync(std::function f) { + void execute_sync(std::function f) override { td::actor::send_closure(cell_db_, &CellDbBase::execute_sync, std::move(f)); } + private: td::actor::ActorId cell_db_; }; @@ -62,17 +68,63 @@ void CellDbBase::execute_sync(std::function f) { f(); } -CellDbIn::CellDbIn(td::actor::ActorId root_db, td::actor::ActorId parent, std::string path) - : root_db_(root_db), parent_(parent), path_(std::move(path)) { +CellDbIn::CellDbIn(td::actor::ActorId root_db, td::actor::ActorId parent, std::string path, + td::Ref opts) + : root_db_(root_db), parent_(parent), path_(std::move(path)), opts_(opts) { } void CellDbIn::start_up() { - CellDbBase::start_up(); - cell_db_ = std::make_shared(td::RocksDb::open(path_).move_as_ok()); + on_load_callback_ = [actor = std::make_shared>( + td::actor::create_actor("celldbmigration", actor_id(this))), + compress_depth = opts_->get_celldb_compress_depth()](const vm::CellLoader::LoadResult& res) { + if (res.cell_.is_null()) { + return; + } + bool expected_stored_boc = res.cell_->get_depth() == compress_depth && compress_depth != 0; + if (expected_stored_boc != res.stored_boc_) { + td::actor::send_closure(*actor, &CellDbIn::MigrationProxy::migrate_cell, + td::Bits256{res.cell_->get_hash().bits()}); + } + }; - boc_ = vm::DynamicBagOfCellsDb::create(); - boc_->set_loader(std::make_unique(cell_db_->snapshot())).ensure(); - td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); + CellDbBase::start_up(); + td::RocksDbOptions db_options; + if (!opts_->get_disable_rocksdb_stats()) { + statistics_ = td::RocksDb::create_statistics(); + statistics_flush_at_ = td::Timestamp::in(60.0); + snapshot_statistics_ = std::make_shared(); + db_options.snapshot_statistics = snapshot_statistics_; + } + db_options.statistics = statistics_; + if (opts_->get_celldb_cache_size()) { + db_options.block_cache = td::RocksDb::create_cache(opts_->get_celldb_cache_size().value()); + LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(opts_->get_celldb_cache_size().value()); + } + db_options.use_direct_reads = opts_->get_celldb_direct_io(); + + if (opts_->get_celldb_in_memory()) { + td::RocksDbOptions read_db_options; + read_db_options.use_direct_reads = true; + read_db_options.no_block_cache = true; + read_db_options.block_cache = {}; + LOG(WARNING) << "Loading all cells in memory (because of --celldb-in-memory)"; + td::Timer timer; + auto read_cell_db = + std::make_shared(td::RocksDb::open(path_, std::move(read_db_options)).move_as_ok()); + boc_ = vm::DynamicBagOfCellsDb::create_in_memory(read_cell_db.get(), {}); + in_memory_load_time_ = timer.elapsed(); + td::actor::send_closure(parent_, &CellDb::set_in_memory_boc, boc_); + } + + auto rocks_db = std::make_shared(td::RocksDb::open(path_, std::move(db_options)).move_as_ok()); + rocks_db_ = rocks_db->raw_db(); + cell_db_ = std::move(rocks_db); + if (!opts_->get_celldb_in_memory()) { + boc_ = vm::DynamicBagOfCellsDb::create(); + boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth()); + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); + } alarm_timestamp() = td::Timestamp::in(10.0); @@ -83,15 +135,71 @@ void CellDbIn::start_up() { set_block(empty, std::move(e)); cell_db_->commit_write_batch().ensure(); } - last_gc_ = empty; + + if (opts_->get_celldb_preload_all()) { + // Iterate whole DB in a separate thread + delay_action( + [snapshot = cell_db_->snapshot()]() { + LOG(WARNING) << "CellDb: pre-loading all keys"; + td::uint64 total = 0; + td::Timer timer; + auto S = snapshot->for_each([&](td::Slice, td::Slice) { + ++total; + if (total % 1000000 == 0) { + LOG(INFO) << "CellDb: iterated " << total << " keys"; + } + return td::Status::OK(); + }); + if (S.is_error()) { + LOG(ERROR) << "CellDb: pre-load failed: " << S.move_as_error(); + } else { + LOG(WARNING) << "CellDb: iterated " << total << " keys in " << timer.elapsed() << "s"; + } + }, + td::Timestamp::now()); + } + + { + std::string key = "stats.last_deleted_mc_seqno", value; + auto R = cell_db_->get(td::as_slice(key), value); + R.ensure(); + if (R.ok() == td::KeyValue::GetStatus::Ok) { + auto r_value = td::to_integer_safe(value); + r_value.ensure(); + last_deleted_mc_state_ = r_value.move_as_ok(); + } + } } void CellDbIn::load_cell(RootHash hash, td::Promise> promise) { - boc_->load_cell_async(hash.as_slice(), async_executor, std::move(promise)); + if (db_busy_) { + action_queue_.push([self = this, hash, promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + self->load_cell(hash, std::move(promise)); + }); + return; + } + if (opts_->get_celldb_in_memory()) { + auto result = boc_->load_root(hash.as_slice()); + async_apply("load_cell_result", std::move(promise), std::move(result)); + return; + } + auto cell = boc_->load_cell(hash.as_slice()); + delay_action( + [cell = std::move(cell), promise = std::move(promise)]() mutable { promise.set_result(std::move(cell)); }, + td::Timestamp::now()); } void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise) { - td::PerfWarningTimer{"storecell", 0.1}; + if (db_busy_) { + action_queue_.push( + [self = this, block_id, cell = std::move(cell), promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + self->store_cell(block_id, std::move(cell), std::move(promise)); + }); + return; + } + td::PerfWarningTimer timer{"storecell", 0.1}; auto key_hash = get_key_hash(block_id); auto R = get_block(key_hash); // duplicate @@ -100,58 +208,168 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi return; } - auto empty = get_empty_key_hash(); - auto ER = get_block(empty); - ER.ensure(); - auto E = ER.move_as_ok(); - - auto PR = get_block(E.prev); - PR.ensure(); - auto P = PR.move_as_ok(); - CHECK(P.next == empty); - - DbEntry D{block_id, E.prev, empty, cell->get_hash().bits()}; - - E.prev = key_hash; - P.next = key_hash; - - if (P.is_empty()) { - E.next = key_hash; - P.prev = key_hash; - } - boc_->inc(cell); - boc_->prepare_commit().ensure(); - vm::CellStorer stor{*cell_db_.get()}; - cell_db_->begin_write_batch().ensure(); - boc_->commit(stor).ensure(); - set_block(empty, std::move(E)); - set_block(D.prev, std::move(P)); - set_block(key_hash, std::move(D)); - cell_db_->commit_write_batch().ensure(); + db_busy_ = true; + boc_->prepare_commit_async(async_executor, [=, this, SelfId = actor_id(this), timer = std::move(timer), + timer_prepare = td::Timer{}, promise = std::move(promise), + cell = std::move(cell)](td::Result Res) mutable { + Res.ensure(); + timer_prepare.pause(); + td::actor::send_lambda( + SelfId, [=, this, timer = std::move(timer), promise = std::move(promise), cell = std::move(cell)]() mutable { + TD_PERF_COUNTER(celldb_store_cell); + auto empty = get_empty_key_hash(); + auto ER = get_block(empty); + ER.ensure(); + auto E = ER.move_as_ok(); - boc_->set_loader(std::make_unique(cell_db_->snapshot())).ensure(); - td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); + auto PR = get_block(E.prev); + PR.ensure(); + auto P = PR.move_as_ok(); + CHECK(P.next == empty); - promise.set_result(boc_->load_cell(cell->get_hash().as_slice())); + DbEntry D{block_id, E.prev, empty, cell->get_hash().bits()}; + + E.prev = key_hash; + P.next = key_hash; + + if (P.is_empty()) { + E.next = key_hash; + P.prev = key_hash; + } + td::Timer timer_write; + vm::CellStorer stor{*cell_db_}; + cell_db_->begin_write_batch().ensure(); + boc_->commit(stor).ensure(); + set_block(get_empty_key_hash(), std::move(E)); + set_block(D.prev, std::move(P)); + set_block(key_hash, std::move(D)); + cell_db_->commit_write_batch().ensure(); + timer_write.pause(); + + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); + } + + promise.set_result(boc_->load_cell(cell->get_hash().as_slice())); + if (!opts_->get_disable_rocksdb_stats()) { + cell_db_statistics_.store_cell_time_.insert(timer.elapsed() * 1e6); + cell_db_statistics_.store_cell_prepare_time_.insert(timer_prepare.elapsed() * 1e6); + cell_db_statistics_.store_cell_write_time_.insert(timer_write.elapsed() * 1e6); + } + LOG(DEBUG) << "Stored state " << block_id.to_str(); + release_db(); + }); + }); } void CellDbIn::get_cell_db_reader(td::Promise> promise) { + if (db_busy_) { + action_queue_.push( + [self = this, promise = std::move(promise)](td::Result R) mutable { + R.ensure(); + self->get_cell_db_reader(std::move(promise)); + }); + return; + } promise.set_result(boc_->get_cell_db_reader()); } -void CellDbIn::alarm() { - auto R = get_block(last_gc_); - R.ensure(); +std::vector> CellDbIn::prepare_stats() { + TD_PERF_COUNTER(celldb_prepare_stats); + auto r_boc_stats = boc_->get_stats(); + if (r_boc_stats.is_ok()) { + cell_db_statistics_.boc_stats_ = r_boc_stats.move_as_ok(); + } + cell_db_statistics_.in_memory_load_time_ = in_memory_load_time_; + auto stats = cell_db_statistics_.prepare_stats(); + auto add_stat = [&](const auto& key, const auto& value) { stats.emplace_back(key, PSTRING() << value); }; - auto N = R.move_as_ok(); + add_stat("started", "true"); + auto r_mem_stat = td::mem_stat(); + auto r_total_mem_stat = td::get_total_mem_stat(); + td::uint64 celldb_size = 0; + bool ok_celldb_size = rocks_db_->GetIntProperty("rocksdb.total-sst-files-size", &celldb_size); + if (celldb_size > 0 && r_mem_stat.is_ok() && r_total_mem_stat.is_ok() && ok_celldb_size) { + auto mem_stat = r_mem_stat.move_as_ok(); + auto total_mem_stat = r_total_mem_stat.move_as_ok(); + add_stat("rss", td::format::as_size(mem_stat.resident_size_)); + add_stat("available_ram", td::format::as_size(total_mem_stat.available_ram)); + add_stat("total_ram", td::format::as_size(total_mem_stat.total_ram)); + add_stat("actual_ram_to_celldb_ratio", double(total_mem_stat.available_ram) / double(celldb_size)); + add_stat("if_restarted_ram_to_celldb_ratio", + double(total_mem_stat.available_ram + mem_stat.resident_size_ - 10 * (td::uint64(1) << 30)) / + double(celldb_size)); + add_stat("max_possible_ram_to_celldb_ratio", double(total_mem_stat.total_ram) / double(celldb_size)); + } + stats.emplace_back("last_deleted_mc_state", td::to_string(last_deleted_mc_state_)); + + return stats; + // do not clear statistics, it is needed for flush_db_stats +} +void CellDbIn::flush_db_stats() { + if (opts_->get_disable_rocksdb_stats()) { + return; + } + if (db_busy_) { + action_queue_.push([self = this](td::Result R) mutable { + R.ensure(); + self->flush_db_stats(); + }); + return; + } + + auto celldb_stats = prepare_stats(); + td::StringBuilder ss; + for (auto& [key, value] : celldb_stats) { + ss << "ton.celldb." << key << " " << value << "\n"; + } + + auto stats = + td::RocksDb::statistics_to_string(statistics_) + snapshot_statistics_->to_string() + ss.as_cslice().str(); + auto to_file_r = + td::FileFd::open(path_ + "/db_stats.txt", td::FileFd::Truncate | td::FileFd::Create | td::FileFd::Write, 0644); + if (to_file_r.is_error()) { + LOG(ERROR) << "Failed to open db_stats.txt: " << to_file_r.move_as_error(); + return; + } + auto to_file = to_file_r.move_as_ok(); + auto res = to_file.write(stats); + to_file.close(); + if (res.is_error()) { + LOG(ERROR) << "Failed to write to db_stats.txt: " << res.move_as_error(); + return; + } + td::RocksDb::reset_statistics(statistics_); + cell_db_statistics_.clear(); +} + +void CellDbIn::alarm() { + if (statistics_flush_at_ && statistics_flush_at_.is_in_past()) { + statistics_flush_at_ = td::Timestamp::in(60.0); + flush_db_stats(); + } + + if (migrate_after_ && migrate_after_.is_in_past()) { + migrate_cells(); + } + if (migration_stats_ && migration_stats_->end_at_.is_in_past()) { + LOG(INFO) << "CellDb migration, " << migration_stats_->start_.elapsed() + << "s stats: batches=" << migration_stats_->batches_ << " migrated=" << migration_stats_->migrated_cells_ + << " checked=" << migration_stats_->checked_cells_ << " time=" << migration_stats_->total_time_ + << " queue_size=" << cells_to_migrate_.size(); + migration_stats_ = {}; + } + auto E = get_block(get_empty_key_hash()).move_as_ok(); + auto N = get_block(E.next).move_as_ok(); if (N.is_empty()) { - last_gc_ = N.next; alarm_timestamp() = td::Timestamp::in(0.1); return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + auto block_id = N.block_id; + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &CellDbIn::skip_gc); } else { @@ -159,24 +377,19 @@ void CellDbIn::alarm() { if (!value) { td::actor::send_closure(SelfId, &CellDbIn::skip_gc); } else { - td::actor::send_closure(SelfId, &CellDbIn::gc); + td::actor::send_closure(SelfId, &CellDbIn::gc, block_id); } } }); - td::actor::send_closure(root_db_, &RootDb::allow_state_gc, N.block_id, std::move(P)); + td::actor::send_closure(root_db_, &RootDb::allow_state_gc, block_id, std::move(P)); } -void CellDbIn::gc() { - auto R = get_block(last_gc_); - R.ensure(); - - auto N = R.move_as_ok(); - +void CellDbIn::gc(BlockIdExt block_id) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); td::actor::send_closure(SelfId, &CellDbIn::gc_cont, R.move_as_ok()); }); - td::actor::send_closure(root_db_, &RootDb::get_block_handle_external, N.block_id, false, std::move(P)); + td::actor::send_closure(root_db_, &RootDb::get_block_handle_external, block_id, false, std::move(P)); } void CellDbIn::gc_cont(BlockHandle handle) { @@ -194,9 +407,20 @@ void CellDbIn::gc_cont(BlockHandle handle) { } void CellDbIn::gc_cont2(BlockHandle handle) { - td::PerfWarningTimer{"gccell", 0.1}; + if (db_busy_) { + action_queue_.push([self = this, handle = std::move(handle)](td::Result R) mutable { + R.ensure(); + self->gc_cont2(handle); + }); + return; + } - auto FR = get_block(last_gc_); + td::PerfWarningTimer timer{"gccell", 0.1}; + td::PerfWarningTimer timer_all{"gccell_all", 0.05}; + + td::PerfWarningTimer timer_get_keys{"gccell_get_keys", 0.05}; + auto key_hash = get_key_hash(handle->id()); + auto FR = get_block(key_hash); FR.ensure(); auto F = FR.move_as_ok(); @@ -213,33 +437,70 @@ void CellDbIn::gc_cont2(BlockHandle handle) { P.prev = P.next; N.next = N.prev; } + timer_get_keys.reset(); + td::PerfWarningTimer timer_boc{"gccell_boc", 0.05}; auto cell = boc_->load_cell(F.root_hash.as_slice()).move_as_ok(); boc_->dec(cell); - boc_->prepare_commit().ensure(); - vm::CellStorer stor{*cell_db_.get()}; - cell_db_->begin_write_batch().ensure(); - boc_->commit(stor).ensure(); - cell_db_->erase(get_key(last_gc_)).ensure(); - set_block(F.prev, std::move(P)); - set_block(F.next, std::move(N)); - cell_db_->commit_write_batch().ensure(); - alarm_timestamp() = td::Timestamp::now(); + db_busy_ = true; + boc_->prepare_commit_async( + async_executor, [this, SelfId = actor_id(this), timer_boc = std::move(timer_boc), F = std::move(F), key_hash, + P = std::move(P), N = std::move(N), cell = std::move(cell), timer = std::move(timer), + timer_all = std::move(timer_all), handle](td::Result R) mutable { + R.ensure(); + td::actor::send_lambda(SelfId, [this, timer_boc = std::move(timer_boc), F = std::move(F), key_hash, + P = std::move(P), N = std::move(N), cell = std::move(cell), + timer = std::move(timer), timer_all = std::move(timer_all), handle]() mutable { + TD_PERF_COUNTER(celldb_gc_cell); + vm::CellStorer stor{*cell_db_}; + timer_boc.reset(); - boc_->set_loader(std::make_unique(cell_db_->snapshot())).ensure(); - td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); + td::PerfWarningTimer timer_write_batch{"gccell_write_batch", 0.05}; + cell_db_->begin_write_batch().ensure(); + boc_->commit(stor).ensure(); - DCHECK(get_block(last_gc_).is_error()); - last_gc_ = F.next; + cell_db_->erase(get_key(key_hash)).ensure(); + set_block(F.prev, std::move(P)); + set_block(F.next, std::move(N)); + if (handle->id().is_masterchain()) { + last_deleted_mc_state_ = handle->id().seqno(); + std::string key = "stats.last_deleted_mc_seqno", value = td::to_string(last_deleted_mc_state_); + cell_db_->set(td::as_slice(key), td::as_slice(value)); + } + cell_db_->commit_write_batch().ensure(); + alarm_timestamp() = td::Timestamp::now(); + timer_write_batch.reset(); + + td::PerfWarningTimer timer_free_cells{"gccell_free_cells", 0.05}; + auto before = td::ref_get_delete_count(); + cell = {}; + auto after = td::ref_get_delete_count(); + if (timer_free_cells.elapsed() > 0.04) { + LOG(ERROR) << "deleted " << after - before << " cells"; + } + timer_free_cells.reset(); + + td::PerfWarningTimer timer_finish{"gccell_finish", 0.05}; + if (!opts_->get_celldb_in_memory()) { + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); + } + + DCHECK(get_block(key_hash).is_error()); + if (!opts_->get_disable_rocksdb_stats()) { + cell_db_statistics_.gc_cell_time_.insert(timer.elapsed() * 1e6); + } + LOG(DEBUG) << "Deleted state " << handle->id().to_str(); + timer_finish.reset(); + timer_all.reset(); + release_db(); + }); + }); } void CellDbIn::skip_gc() { - auto FR = get_block(last_gc_); - FR.ensure(); - auto F = FR.move_as_ok(); - last_gc_ = F.next; - alarm_timestamp() = td::Timestamp::in(0.01); + alarm_timestamp() = td::Timestamp::in(1.0); } std::string CellDbIn::get_key(KeyHash key_hash) { @@ -285,7 +546,99 @@ void CellDbIn::set_block(KeyHash key_hash, DbEntry e) { cell_db_->set(td::as_slice(key), e.release()).ensure(); } +void CellDbIn::migrate_cell(td::Bits256 hash) { + cells_to_migrate_.insert(hash); + if (!migration_active_) { + migration_active_ = true; + migrate_after_ = td::Timestamp::in(10.0); + } +} + +void CellDbIn::migrate_cells() { + migrate_after_ = td::Timestamp::never(); + if (db_busy_) { + action_queue_.push([self = this](td::Result R) mutable { + R.ensure(); + self->migrate_cells(); + }); + return; + } + if (cells_to_migrate_.empty()) { + migration_active_ = false; + return; + } + td::Timer timer; + if (!migration_stats_) { + migration_stats_ = std::make_unique(); + } + vm::CellStorer stor{*cell_db_}; + auto loader = std::make_unique(cell_db_->snapshot()); + boc_->set_loader(std::make_unique(*loader)).ensure(); + cell_db_->begin_write_batch().ensure(); + td::uint32 checked = 0, migrated = 0; + for (auto it = cells_to_migrate_.begin(); it != cells_to_migrate_.end() && checked < 128;) { + ++checked; + td::Bits256 hash = *it; + it = cells_to_migrate_.erase(it); + auto R = loader->load(hash.as_slice(), true, boc_->as_ext_cell_creator()); + if (R.is_error()) { + continue; + } + if (R.ok().status == vm::CellLoader::LoadResult::NotFound) { + continue; + } + bool expected_stored_boc = + R.ok().cell_->get_depth() == opts_->get_celldb_compress_depth() && opts_->get_celldb_compress_depth() != 0; + if (expected_stored_boc != R.ok().stored_boc_) { + ++migrated; + stor.set(R.ok().refcnt(), R.ok().cell_, expected_stored_boc).ensure(); + } + } + cell_db_->commit_write_batch().ensure(); + boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); + td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); + + double time = timer.elapsed(); + LOG(DEBUG) << "CellDb migration: migrated=" << migrated << " checked=" << checked << " time=" << time; + ++migration_stats_->batches_; + migration_stats_->migrated_cells_ += migrated; + migration_stats_->checked_cells_ += checked; + migration_stats_->total_time_ += time; + + if (cells_to_migrate_.empty()) { + migration_active_ = false; + } else { + delay_action([SelfId = actor_id(this)] { td::actor::send_closure(SelfId, &CellDbIn::migrate_cells); }, + td::Timestamp::in(time * 2)); + } +} + +void CellDb::prepare_stats(td::Promise>> promise) { + promise.set_value(decltype(prepared_stats_)(prepared_stats_)); +} + +void CellDb::update_stats(td::Result>> r_stats) { + if (r_stats.is_error()) { + LOG(ERROR) << "error updating stats: " << r_stats.error(); + } else { + prepared_stats_ = r_stats.move_as_ok(); + } + alarm_timestamp() = td::Timestamp::in(2.0); +} + +void CellDb::alarm() { + send_closure(cell_db_, &CellDbIn::prepare_stats, td::promise_send_closure(actor_id(this), &CellDb::update_stats)); +} + void CellDb::load_cell(RootHash hash, td::Promise> promise) { + if (in_memory_boc_) { + auto result = in_memory_boc_->load_root_thread_safe(hash.as_slice()); + if (result.is_ok()) { + return async_apply("load_cell_result", std::move(promise), std::move(result)); + } else { + LOG(ERROR) << "load_root_thread_safe failed - this is suspicious"; + } + } if (!started_) { td::actor::send_closure(cell_db_, &CellDbIn::load_cell, hash, std::move(promise)); } else { @@ -296,7 +649,7 @@ void CellDb::load_cell(RootHash hash, td::Promise> promise } else { promise.set_result(R.move_as_ok()); } - }); + }); boc_->load_cell_async(hash.as_slice(), async_executor, std::move(P)); } } @@ -312,7 +665,20 @@ void CellDb::get_cell_db_reader(td::Promise> p void CellDb::start_up() { CellDbBase::start_up(); boc_ = vm::DynamicBagOfCellsDb::create(); - cell_db_ = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_); + boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth()); + cell_db_ = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_); + on_load_callback_ = [actor = std::make_shared>( + td::actor::create_actor("celldbmigration", cell_db_.get())), + compress_depth = opts_->get_celldb_compress_depth()](const vm::CellLoader::LoadResult& res) { + if (res.cell_.is_null()) { + return; + } + bool expected_stored_boc = res.cell_->get_depth() == compress_depth && compress_depth != 0; + if (expected_stored_boc != res.stored_boc_) { + td::actor::send_closure(*actor, &CellDbIn::MigrationProxy::migrate_cell, + td::Bits256{res.cell_->get_hash().bits()}); + } + }; } CellDbIn::DbEntry::DbEntry(tl_object_ptr entry) @@ -326,6 +692,28 @@ td::BufferSlice CellDbIn::DbEntry::release() { return create_serialize_tl_object(create_tl_block_id(block_id), prev, next, root_hash); } +std::vector> CellDbIn::CellDbStatistics::prepare_stats() { + std::vector> stats; + stats.emplace_back("store_cell.micros", PSTRING() << store_cell_time_.to_string()); + stats.emplace_back("store_cell.prepare.micros", PSTRING() << store_cell_prepare_time_.to_string()); + stats.emplace_back("store_cell.write.micros", PSTRING() << store_cell_write_time_.to_string()); + stats.emplace_back("gc_cell.micros", PSTRING() << gc_cell_time_.to_string()); + stats.emplace_back("total_time.micros", PSTRING() << (td::Timestamp::now().at() - stats_start_time_.at()) * 1e6); + stats.emplace_back("in_memory", PSTRING() << bool(in_memory_load_time_)); + if (in_memory_load_time_) { + stats.emplace_back("in_memory_load_time", PSTRING() << in_memory_load_time_.value()); + } + if (boc_stats_) { + stats.emplace_back("cells_count", PSTRING() << boc_stats_->cells_total_count); + stats.emplace_back("cells_size", PSTRING() << boc_stats_->cells_total_size); + stats.emplace_back("roots_count", PSTRING() << boc_stats_->roots_total_count); + for (auto& [key, value] : boc_stats_->custom_stats) { + stats.emplace_back(key, value); + } + } + return stats; +} + } // namespace validator } // namespace ton diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index e54526b9..5639b974 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -25,6 +25,17 @@ #include "ton/ton-types.h" #include "interfaces/block-handle.h" #include "auto/tl/ton_api.h" +#include "validator.h" +#include "db-utils.h" +#include "td/db/RocksDb.h" + +#include +#include + +namespace rocksdb { +class Statistics; +class DB; +} // namespace rocksdb namespace ton { @@ -37,9 +48,11 @@ class CellDbAsyncExecutor; class CellDbBase : public td::actor::Actor { public: - virtual void start_up(); + void start_up() override; + protected: std::shared_ptr async_executor; + private: void execute_sync(std::function f); friend CellDbAsyncExecutor; @@ -49,11 +62,17 @@ class CellDbIn : public CellDbBase { public: using KeyHash = td::Bits256; + std::vector> prepare_stats(); void load_cell(RootHash hash, td::Promise> promise); void store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise); void get_cell_db_reader(td::Promise> promise); - CellDbIn(td::actor::ActorId root_db, td::actor::ActorId parent, std::string path); + void migrate_cell(td::Bits256 hash); + + void flush_db_stats(); + + CellDbIn(td::actor::ActorId root_db, td::actor::ActorId parent, std::string path, + td::Ref opts); void start_up() override; void alarm() override; @@ -66,8 +85,7 @@ class CellDbIn : public CellDbBase { RootHash root_hash; DbEntry(tl_object_ptr entry); - DbEntry() { - } + DbEntry() = default; DbEntry(BlockIdExt block_id, KeyHash prev, KeyHash next, RootHash root_hash) : block_id(block_id), prev(prev), next(next), root_hash(root_hash) { } @@ -84,33 +102,111 @@ class CellDbIn : public CellDbBase { static BlockIdExt get_empty_key(); KeyHash get_empty_key_hash(); - void gc(); + void gc(BlockIdExt block_id); void gc_cont(BlockHandle handle); void gc_cont2(BlockHandle handle); void skip_gc(); + void migrate_cells(); + td::actor::ActorId root_db_; td::actor::ActorId parent_; std::string path_; + td::Ref opts_; - std::unique_ptr boc_; + std::shared_ptr boc_; std::shared_ptr cell_db_; + std::shared_ptr rocks_db_; - KeyHash last_gc_; + std::function on_load_callback_; + std::set cells_to_migrate_; + td::Timestamp migrate_after_ = td::Timestamp::never(); + bool migration_active_ = false; + std::optional in_memory_load_time_; + + struct MigrationStats { + td::Timer start_; + td::Timestamp end_at_ = td::Timestamp::in(60.0); + size_t batches_ = 0; + size_t migrated_cells_ = 0; + size_t checked_cells_ = 0; + double total_time_ = 0.0; + }; + std::unique_ptr migration_stats_; + + struct CellDbStatistics { + PercentileStats store_cell_time_; + PercentileStats store_cell_prepare_time_; + PercentileStats store_cell_write_time_; + PercentileStats gc_cell_time_; + td::Timestamp stats_start_time_ = td::Timestamp::now(); + std::optional in_memory_load_time_; + std::optional boc_stats_; + + std::vector> prepare_stats(); + void clear() { + *this = CellDbStatistics{}; + } + }; + + std::shared_ptr statistics_; + std::shared_ptr snapshot_statistics_; + CellDbStatistics cell_db_statistics_; + td::Timestamp statistics_flush_at_ = td::Timestamp::never(); + BlockSeqno last_deleted_mc_state_ = 0; + + bool db_busy_ = false; + std::queue> action_queue_; + + void release_db() { + db_busy_ = false; + while (!db_busy_ && !action_queue_.empty()) { + auto action = std::move(action_queue_.front()); + action_queue_.pop(); + action.set_value(td::Unit()); + } + } + + public: + class MigrationProxy : public td::actor::Actor { + public: + explicit MigrationProxy(td::actor::ActorId cell_db) : cell_db_(cell_db) { + } + void migrate_cell(td::Bits256 hash) { + td::actor::send_closure(cell_db_, &CellDbIn::migrate_cell, hash); + } + + private: + td::actor::ActorId cell_db_; + }; }; class CellDb : public CellDbBase { public: + void prepare_stats(td::Promise>> promise); void load_cell(RootHash hash, td::Promise> promise); void store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise); void update_snapshot(std::unique_ptr snapshot) { + CHECK(!opts_->get_celldb_in_memory()); + if (!started_) { + alarm(); + } started_ = true; - boc_->set_loader(std::make_unique(std::move(snapshot))).ensure(); + boc_->set_loader(std::make_unique(std::move(snapshot), on_load_callback_)).ensure(); + } + void set_in_memory_boc(std::shared_ptr in_memory_boc) { + CHECK(opts_->get_celldb_in_memory()); + if (!started_) { + alarm(); + } + started_ = true; + in_memory_boc_ = std::move(in_memory_boc); } void get_cell_db_reader(td::Promise> promise); - CellDb(td::actor::ActorId root_db, std::string path) : root_db_(root_db), path_(path) { + CellDb(td::actor::ActorId root_db, std::string path, td::Ref opts) + : root_db_(root_db), path_(path), opts_(opts) { } void start_up() override; @@ -118,11 +214,19 @@ class CellDb : public CellDbBase { private: td::actor::ActorId root_db_; std::string path_; + td::Ref opts_; td::actor::ActorOwn cell_db_; std::unique_ptr boc_; + std::shared_ptr in_memory_boc_; bool started_ = false; + std::vector> prepared_stats_{{"started", "false"}}; + + std::function on_load_callback_; + + void update_stats(td::Result>> stats); + void alarm() override; }; } // namespace validator diff --git a/validator/db/db-utils.cpp b/validator/db/db-utils.cpp new file mode 100644 index 00000000..9375f97e --- /dev/null +++ b/validator/db/db-utils.cpp @@ -0,0 +1,54 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "db-utils.h" + +#include "td/utils/logging.h" + +#include + +namespace ton::validator { + +void PercentileStats::insert(double value) { + values_.insert(value); +} + +std::string PercentileStats::to_string() const { + double percentiles[4] = {0.0, 0.0, 0.0, 0.0}; + double sum = 0.0; + size_t size = values_.size(); + if (!values_.empty()) { + size_t indices[4] = {(size_t)std::ceil(0.5 * (double)size) - 1, (size_t)std::ceil(0.95 * (double)size) - 1, + (size_t)std::ceil(0.99 * (double)size) - 1, size - 1}; + size_t i = 0; + for (auto it = values_.begin(); it != values_.end(); ++it, ++i) { + for (size_t j = 0; j < 4; ++j) { + if (indices[j] == i) { + percentiles[j] = *it; + } + } + sum += *it; + } + } + return PSTRING() << "P50 : " << percentiles[0] << " P95 : " << percentiles[1] << " P99 : " << percentiles[2] + << " P100 : " << percentiles[3] << " COUNT : " << size << " SUM : " << sum; +} + +void PercentileStats::clear() { + values_.clear(); +} + +} // namespace ton::validator \ No newline at end of file diff --git a/tonlib/tonlib/TestGiver.h b/validator/db/db-utils.h similarity index 59% rename from tonlib/tonlib/TestGiver.h rename to validator/db/db-utils.h index f8b62599..8ffbf5c9 100644 --- a/tonlib/tonlib/TestGiver.h +++ b/validator/db/db-utils.h @@ -13,19 +13,21 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . - - Copyright 2017-2019 Telegram Systems LLP */ #pragma once -#include "block/block.h" -#include "CellString.h" -namespace tonlib { -class TestGiver { +#include +#include + +namespace ton::validator { + +class PercentileStats { public: - static constexpr unsigned max_message_size = vm::CellString::max_bytes; - static const block::StdAddress& address() noexcept; - static vm::CellHash get_init_code_hash() noexcept; - static td::Ref make_a_gift_message(td::uint32 seqno, td::uint64 gramms, td::Slice message, - const block::StdAddress& dest_address) noexcept; + void insert(double value); + std::string to_string() const; + void clear(); + + private: + std::multiset values_; }; -} // namespace tonlib + +} // namespace ton::validator \ No newline at end of file diff --git a/validator/db/fileref.cpp b/validator/db/fileref.cpp index 2e2a3b6b..feefd1f9 100644 --- a/validator/db/fileref.cpp +++ b/validator/db/fileref.cpp @@ -79,14 +79,14 @@ std::string Block::filename() const { std::string Block::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.id.shard)); return PSTRING() << "block_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" << hash().to_hex(); } std::string BlockShort::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.shard)); return PSTRING() << "block_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex(); } @@ -116,14 +116,14 @@ std::string PersistentState::filename() const { std::string PersistentState::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.id.shard)); return PSTRING() << "state_" << masterchain_block_id.seqno() << "_" << block_id.id.workchain << "_" << s << "_" << hash().to_hex(); } std::string PersistentStateShort::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(shard_id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(shard_id.shard)); return PSTRING() << "state_" << masterchain_seqno << "_" << shard_id.workchain << "_" << s << "_" << hash().to_hex(); } @@ -137,14 +137,14 @@ std::string Proof::filename() const { std::string Proof::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.id.shard)); return PSTRING() << "proof_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" << hash().to_hex(); } std::string ProofShort::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.shard)); return PSTRING() << "proof_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex(); } @@ -158,14 +158,14 @@ std::string ProofLink::filename() const { std::string ProofLink::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.id.shard)); return PSTRING() << "prooflink_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" << hash().to_hex(); } std::string ProofLinkShort::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.shard)); return PSTRING() << "prooflink_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex(); } @@ -179,14 +179,14 @@ std::string Signatures::filename() const { std::string Signatures::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.id.shard)); return PSTRING() << "signatures_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" << hash().to_hex(); } std::string SignaturesShort::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.shard)); return PSTRING() << "signatures_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex(); } @@ -202,17 +202,39 @@ std::string Candidate::filename() const { std::string Candidate::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.id.shard)); return PSTRING() << "candidate_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" << hash().to_hex(); } std::string CandidateShort::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.shard)); return PSTRING() << "candidate_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex(); } +CandidateRefShort CandidateRef::shortref() const { + return CandidateRefShort{block_id.id, hash()}; +} + +std::string CandidateRef::filename() const { + return PSTRING() << "candidateref_" << block_id.to_str(); +} + +std::string CandidateRef::filename_short() const { + char s[33]; + snprintf(s, sizeof(s), "%llx", static_cast(block_id.id.shard)); + return PSTRING() << "candidateref_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" + << hash().to_hex(); +} + +std::string CandidateRefShort::filename_short() const { + char s[33]; + snprintf(s, sizeof(s), "%llx", static_cast(block_id.shard)); + return PSTRING() << "candidateref_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" + << hash().to_hex(); +} + BlockInfoShort BlockInfo::shortref() const { return BlockInfoShort{block_id.id, hash()}; } @@ -223,14 +245,14 @@ std::string BlockInfo::filename() const { std::string BlockInfo::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.id.shard)); return PSTRING() << "info_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" << hash().to_hex(); } std::string BlockInfoShort::filename_short() const { char s[33]; - sprintf(s, "%llx", static_cast(block_id.shard)); + snprintf(s, sizeof(s), "%llx", static_cast(block_id.shard)); return PSTRING() << "info_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex(); } @@ -259,6 +281,9 @@ FileReference::FileReference(tl_object_ptr key) { ref_ = fileref::Candidate{PublicKey{key.id_->source_}, create_block_id(key.id_->id_), key.id_->collated_data_file_hash_}; }, + [&](const ton_api::db_filedb_key_candidateRef& key) { + ref_ = fileref::CandidateRef{create_block_id(key.id_)}; + }, [&](const ton_api::db_filedb_key_blockInfo& key) { ref_ = fileref::BlockInfo{create_block_id(key.block_id_)}; })); diff --git a/validator/db/fileref.hpp b/validator/db/fileref.hpp index 14424a65..6710cc06 100644 --- a/validator/db/fileref.hpp +++ b/validator/db/fileref.hpp @@ -278,6 +278,38 @@ class Candidate { FileHash collated_data_file_hash; }; +class CandidateRefShort { + public: + FileHash hash() const { + return hashv; + } + ShardIdFull shard() const { + return block_id.shard_full(); + } + std::string filename_short() const; + + BlockId block_id; + FileHash hashv; +}; + +class CandidateRef { + public: + tl_object_ptr tl() const { + return create_tl_object(create_tl_block_id(block_id)); + } + FileHash hash() const { + return create_hash_tl_object(create_tl_block_id(block_id)); + } + ShardIdFull shard() const { + return block_id.shard_full(); + } + CandidateRefShort shortref() const; + std::string filename() const; + std::string filename_short() const; + + BlockIdExt block_id; +}; + class BlockInfoShort { public: FileHash hash() const { @@ -316,7 +348,7 @@ class FileReferenceShort { private: td::Variant + fileref::CandidateShort, fileref::CandidateRefShort, fileref::BlockInfoShort> ref_; public: @@ -340,7 +372,8 @@ class FileReferenceShort { class FileReference { private: td::Variant + fileref::Proof, fileref::ProofLink, fileref::Signatures, fileref::Candidate, fileref::CandidateRef, + fileref::BlockInfo> ref_; public: diff --git a/validator/db/files-async.hpp b/validator/db/files-async.hpp index 2da534bf..c8508996 100644 --- a/validator/db/files-async.hpp +++ b/validator/db/files-async.hpp @@ -57,7 +57,7 @@ class WriteFile : public td::actor::Actor { status = file.sync(); } if (status.is_error()) { - td::unlink(old_name); + td::unlink(old_name).ignore(); promise_.set_error(std::move(status)); stop(); return; @@ -82,10 +82,9 @@ class WriteFile : public td::actor::Actor { : tmp_dir_(tmp_dir), new_name_(new_name), promise_(std::move(promise)) { write_data_ = [data_ptr = std::make_shared(std::move(data))] (td::FileFd& fd) { auto data = std::move(*data_ptr); - td::uint64 offset = 0; while (data.size() > 0) { - TRY_RESULT(s, fd.pwrite(data.as_slice(), offset)); - offset += s; + auto piece_size = std::min(data.size(), 1 << 30); + TRY_RESULT(s, fd.write(data.as_slice().substr(0, piece_size))); data.confirm_read(s); } return td::Status::OK(); diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index a7a1becf..8d83e7a7 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -177,21 +177,21 @@ void RootDb::get_block_proof_link(ConstBlockHandle handle, td::Promise promise) { + auto source = PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}; auto obj = create_serialize_tl_object( - PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}.tl(), create_tl_block_id(candidate.id), - std::move(candidate.data), std::move(candidate.collated_data)); - - auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - } else { - promise.set_value(td::Unit()); - } - }); + source.tl(), create_tl_block_id(candidate.id), std::move(candidate.data), std::move(candidate.collated_data)); + auto P = td::PromiseCreator::lambda( + [archive_db = archive_db_.get(), promise = std::move(promise), block_id = candidate.id, source, + collated_file_hash = candidate.collated_file_hash](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, _, std::move(R)); + td::actor::send_closure(archive_db, &ArchiveManager::add_temp_file_short, fileref::CandidateRef{block_id}, + create_serialize_tl_object( + source.tl(), create_tl_block_id(block_id), collated_file_hash), + std::move(promise)); + }); td::actor::send_closure(archive_db_, &ArchiveManager::add_temp_file_short, - fileref::Candidate{PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}, candidate.id, - candidate.collated_file_hash}, - std::move(obj), std::move(P)); + fileref::Candidate{source, candidate.id, candidate.collated_file_hash}, std::move(obj), + std::move(P)); } void RootDb::get_block_candidate(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, @@ -215,6 +215,17 @@ void RootDb::get_block_candidate(PublicKey source, BlockIdExt id, FileHash colla fileref::Candidate{source, id, collated_data_file_hash}, std::move(P)); } +void RootDb::get_block_candidate_by_block_id(BlockIdExt id, td::Promise promise) { + td::actor::send_closure( + archive_db_, &ArchiveManager::get_temp_file_short, fileref::CandidateRef{id}, + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, data, std::move(R)); + TRY_RESULT_PROMISE(promise, f, fetch_tl_object(data, true)); + td::actor::send_closure(SelfId, &RootDb::get_block_candidate, PublicKey{f->source_}, create_block_id(f->id_), + f->collated_data_file_hash_, std::move(promise)); + }); +} + void RootDb::store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) { if (handle->moved_to_archive()) { @@ -317,6 +328,12 @@ void RootDb::check_zero_state_file_exists(BlockIdExt block_id, td::Promise td::actor::send_closure(archive_db_, &ArchiveManager::check_zero_state, block_id, std::move(promise)); } +void RootDb::get_previous_persistent_state_files( + BlockSeqno cur_mc_seqno, td::Promise>> promise) { + td::actor::send_closure(archive_db_, &ArchiveManager::get_previous_persistent_state_files, cur_mc_seqno, + std::move(promise)); +} + void RootDb::store_block_handle(BlockHandle handle, td::Promise promise) { td::actor::send_closure(archive_db_, &ArchiveManager::update_handle, std::move(handle), std::move(promise)); } @@ -330,7 +347,8 @@ void RootDb::try_get_static_file(FileHash file_hash, td::Promise promise) { - td::actor::create_actor("archiver", std::move(handle), archive_db_.get(), std::move(promise)) + td::actor::create_actor("archiver", std::move(handle), archive_db_.get(), actor_id(this), + std::move(promise)) .release(); } @@ -397,14 +415,15 @@ void RootDb::get_hardforks(td::Promise> promise) { } void RootDb::start_up() { - cell_db_ = td::actor::create_actor("celldb", actor_id(this), root_path_ + "/celldb/"); + cell_db_ = td::actor::create_actor("celldb", actor_id(this), root_path_ + "/celldb/", opts_); state_db_ = td::actor::create_actor("statedb", actor_id(this), root_path_ + "/state/"); static_files_db_ = td::actor::create_actor("staticfilesdb", actor_id(this), root_path_ + "/static/"); - archive_db_ = td::actor::create_actor("archive", actor_id(this), root_path_); + archive_db_ = td::actor::create_actor("archive", actor_id(this), root_path_, opts_); } void RootDb::archive(BlockHandle handle, td::Promise promise) { - td::actor::create_actor("archiveblock", std::move(handle), archive_db_.get(), std::move(promise)) + td::actor::create_actor("archiveblock", std::move(handle), archive_db_.get(), actor_id(this), + std::move(promise)) .release(); } @@ -418,6 +437,8 @@ void RootDb::allow_block_gc(BlockIdExt block_id, td::Promise promise) { void RootDb::prepare_stats(td::Promise>> promise) { auto merger = StatsMerger::create(std::move(promise)); + td::actor::send_closure(cell_db_, &CellDb::prepare_stats, merger.make_promise("celldb.")); + td::actor::send_closure(archive_db_, &ArchiveManager::prepare_stats, merger.make_promise("archive.")); } void RootDb::truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise promise) { @@ -483,8 +504,9 @@ void RootDb::check_key_block_proof_link_exists(BlockIdExt block_id, td::Promise< std::move(P)); } -void RootDb::get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) { - td::actor::send_closure(archive_db_, &ArchiveManager::get_archive_id, masterchain_seqno, std::move(promise)); +void RootDb::get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::Promise promise) { + td::actor::send_closure(archive_db_, &ArchiveManager::get_archive_id, masterchain_seqno, shard_prefix, + std::move(promise)); } void RootDb::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, @@ -497,8 +519,16 @@ void RootDb::set_async_mode(bool mode, td::Promise promise) { td::actor::send_closure(archive_db_, &ArchiveManager::set_async_mode, mode, std::move(promise)); } -void RootDb::run_gc(UnixTime ts, UnixTime archive_ttl) { - td::actor::send_closure(archive_db_, &ArchiveManager::run_gc, ts, archive_ttl); +void RootDb::run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) { + td::actor::send_closure(archive_db_, &ArchiveManager::run_gc, mc_ts, gc_ts, archive_ttl); +} + +void RootDb::add_persistent_state_description(td::Ref desc, td::Promise promise) { + td::actor::send_closure(state_db_, &StateDb::add_persistent_state_description, std::move(desc), std::move(promise)); +} + +void RootDb::get_persistent_state_descriptions(td::Promise>> promise) { + td::actor::send_closure(state_db_, &StateDb::get_persistent_state_descriptions, std::move(promise)); } } // namespace validator diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 9b0d52a6..52f6098e 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -26,6 +26,7 @@ #include "statedb.hpp" #include "staticfilesdb.hpp" #include "archive-manager.hpp" +#include "validator.h" namespace ton { @@ -34,8 +35,9 @@ namespace validator { class RootDb : public Db { public: enum class Flags : td::uint32 { f_started = 1, f_ready = 2, f_switched = 4, f_archived = 8 }; - RootDb(td::actor::ActorId validator_manager, std::string root_path) - : validator_manager_(validator_manager), root_path_(std::move(root_path)) { + RootDb(td::actor::ActorId validator_manager, std::string root_path, + td::Ref opts) + : validator_manager_(validator_manager), root_path_(std::move(root_path)), opts_(opts) { } void start_up() override; @@ -56,6 +58,7 @@ class RootDb : public Db { void store_block_candidate(BlockCandidate candidate, td::Promise promise) override; void get_block_candidate(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) override; + void get_block_candidate_by_block_id(BlockIdExt id, td::Promise promise) override; void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; @@ -82,6 +85,8 @@ class RootDb : public Db { void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) override; void get_zero_state_file(BlockIdExt block_id, td::Promise promise) override; void check_zero_state_file_exists(BlockIdExt block_id, td::Promise promise) override; + void get_previous_persistent_state_files( + BlockSeqno cur_mc_seqno, td::Promise>> promise) override; void try_get_static_file(FileHash file_hash, td::Promise promise) override; @@ -127,17 +132,20 @@ class RootDb : public Db { void check_key_block_proof_exists(BlockIdExt block_id, td::Promise promise) override; void check_key_block_proof_link_exists(BlockIdExt block_id, td::Promise promise) override; - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) override; + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::Promise promise) override; void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise) override; void set_async_mode(bool mode, td::Promise promise) override; - void run_gc(UnixTime ts, UnixTime archive_ttl) override; + void run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) override; + void add_persistent_state_description(td::Ref desc, td::Promise promise) override; + void get_persistent_state_descriptions(td::Promise>> promise) override; + private: td::actor::ActorId validator_manager_; - std::string root_path_; + td::Ref opts_; td::actor::ActorOwn cell_db_; td::actor::ActorOwn state_db_; diff --git a/validator/db/statedb.cpp b/validator/db/statedb.cpp index 5d49ae2b..fe3b9d73 100644 --- a/validator/db/statedb.cpp +++ b/validator/db/statedb.cpp @@ -240,6 +240,101 @@ void StateDb::start_up() { } } +void StateDb::add_persistent_state_description(td::Ref desc, + td::Promise promise) { + std::string value; + auto list_key = create_hash_tl_object(); + auto R = kv_->get(list_key.as_slice(), value); + R.ensure(); + tl_object_ptr list; + if (R.ok() == td::KeyValue::GetStatus::Ok) { + auto F = fetch_tl_object(value, true); + F.ensure(); + list = F.move_as_ok(); + } else { + list = create_tl_object( + std::vector>()); + } + for (const auto& obj : list->list_) { + if ((BlockSeqno)obj->masterchain_id_->seqno_ == desc->masterchain_id.seqno()) { + promise.set_error(td::Status::Error("duplicate masterchain seqno")); + return; + } + } + + auto now = (UnixTime)td::Clocks::system(); + size_t new_size = 0; + kv_->begin_write_batch().ensure(); + for (auto& obj : list->list_) { + auto end_time = (UnixTime)obj->end_time_; + if (end_time <= now) { + auto key = + create_hash_tl_object(obj->masterchain_id_->seqno_); + kv_->erase(key.as_slice()).ensure(); + } else { + list->list_[new_size++] = std::move(obj); + } + } + list->list_.resize(new_size); + + std::vector> shard_blocks; + for (const BlockIdExt& block_id : desc->shard_blocks) { + shard_blocks.push_back(create_tl_block_id(block_id)); + } + auto key = + create_hash_tl_object(desc->masterchain_id.seqno()); + kv_->set(key.as_slice(), + create_serialize_tl_object(std::move(shard_blocks)) + .as_slice()) + .ensure(); + + list->list_.push_back(create_tl_object( + create_tl_block_id(desc->masterchain_id), desc->start_time, desc->end_time)); + kv_->set(list_key.as_slice(), serialize_tl_object(list, true).as_slice()).ensure(); + + kv_->commit_write_batch().ensure(); + + promise.set_result(td::Unit()); +} + +void StateDb::get_persistent_state_descriptions(td::Promise>> promise) { + std::string value; + auto R = kv_->get(create_hash_tl_object().as_slice(), value); + R.ensure(); + if (R.ok() == td::KeyValue::GetStatus::NotFound) { + promise.set_value({}); + return; + } + auto F = fetch_tl_object(value, true); + F.ensure(); + std::vector> result; + auto now = (UnixTime)td::Clocks::system(); + for (const auto& obj : F.ok()->list_) { + auto end_time = (UnixTime)obj->end_time_; + if (end_time <= now) { + continue; + } + PersistentStateDescription desc; + desc.start_time = (UnixTime)obj->start_time_; + desc.end_time = end_time; + desc.masterchain_id = create_block_id(obj->masterchain_id_); + auto key = + create_hash_tl_object(desc.masterchain_id.seqno()); + auto R2 = kv_->get(key.as_slice(), value); + R2.ensure(); + if (R2.ok() == td::KeyValue::GetStatus::NotFound) { + continue; + } + auto F2 = fetch_tl_object(value, true); + F2.ensure(); + for (const auto& block_id : F2.ok()->shard_blocks_) { + desc.shard_blocks.push_back(create_block_id(block_id)); + } + result.push_back(td::Ref(true, std::move(desc))); + } + promise.set_result(std::move(result)); +} + void StateDb::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise) { { auto key = create_hash_tl_object(); diff --git a/validator/db/statedb.hpp b/validator/db/statedb.hpp index 75382d61..fe23898f 100644 --- a/validator/db/statedb.hpp +++ b/validator/db/statedb.hpp @@ -50,8 +50,8 @@ class StateDb : public td::actor::Actor { void update_hardforks(std::vector blocks, td::Promise promise); void get_hardforks(td::Promise> promise); - void update_db_version(td::uint32 version, td::Promise promise); - void get_db_version(td::Promise promise); + void add_persistent_state_description(td::Ref desc, td::Promise promise); + void get_persistent_state_descriptions(td::Promise>> promise); StateDb(td::actor::ActorId root_db, std::string path); diff --git a/validator/downloaders/download-state.cpp b/validator/downloaders/download-state.cpp index ccce8f77..8473cb22 100644 --- a/validator/downloaders/download-state.cpp +++ b/validator/downloaders/download-state.cpp @@ -38,6 +38,7 @@ DownloadShardState::DownloadShardState(BlockIdExt block_id, BlockIdExt mastercha } void DownloadShardState::start_up() { + status_ = ProcessStatus(manager_, "process.download_state"); alarm_timestamp() = timeout_; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { @@ -50,7 +51,16 @@ void DownloadShardState::start_up() { void DownloadShardState::got_block_handle(BlockHandle handle) { handle_ = std::move(handle); - download_state(); + if (handle_->received_state()) { + LOG(WARNING) << "shard state " << block_id_.to_str() << " already stored in db"; + td::actor::send_closure(manager_, &ValidatorManagerInterface::get_shard_state_from_db, handle_, + [SelfId = actor_id(this)](td::Result> R) { + R.ensure(); + td::actor::send_closure(SelfId, &DownloadShardState::written_shard_state, R.move_as_ok()); + }); + } else { + download_state(); + } } void DownloadShardState::retry() { @@ -72,6 +82,7 @@ void DownloadShardState::download_state() { }); td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, block_id_, priority_, std::move(P)); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading proof"); } void DownloadShardState::downloaded_proof_link(td::BufferSlice data) { @@ -114,6 +125,7 @@ void DownloadShardState::checked_proof_link() { td::actor::send_closure(manager_, &ValidatorManager::send_get_persistent_state_request, block_id_, masterchain_block_id_, priority_, std::move(P)); } + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading state"); } void DownloadShardState::download_zero_state() { @@ -143,6 +155,7 @@ void DownloadShardState::downloaded_zero_state(td::BufferSlice data) { } void DownloadShardState::downloaded_shard_state(td::BufferSlice data) { + status_.set_status(PSTRING() << block_id_.id.to_str() << " : processing downloaded state"); auto S = create_shard_state(block_id_, data.clone()); if (S.is_error()) { fail_handler(actor_id(this), S.move_as_error()); @@ -165,6 +178,8 @@ void DownloadShardState::downloaded_shard_state(td::BufferSlice data) { } void DownloadShardState::checked_shard_state() { + status_.set_status(PSTRING() << block_id_.id.to_str() << " : storing state file"); + LOG(WARNING) << "checked shard state " << block_id_.to_str(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); td::actor::send_closure(SelfId, &DownloadShardState::written_shard_state_file); @@ -179,6 +194,8 @@ void DownloadShardState::checked_shard_state() { } void DownloadShardState::written_shard_state_file() { + status_.set_status(PSTRING() << block_id_.id.to_str() << " : storing state to celldb"); + LOG(WARNING) << "written shard state file " << block_id_.to_str(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); td::actor::send_closure(SelfId, &DownloadShardState::written_shard_state, R.move_as_ok()); @@ -187,6 +204,7 @@ void DownloadShardState::written_shard_state_file() { } void DownloadShardState::written_shard_state(td::Ref state) { + status_.set_status(PSTRING() << block_id_.id.to_str() << " : finishing"); state_ = std::move(state); handle_->set_unix_time(state_->get_unix_time()); handle_->set_is_key_block(block_id_.is_masterchain()); @@ -207,6 +225,7 @@ void DownloadShardState::written_shard_state(td::Ref state) { } void DownloadShardState::written_block_handle() { + LOG(WARNING) << "finished downloading and storing shard state " << block_id_.to_str(); finish_query(); } diff --git a/validator/downloaders/download-state.hpp b/validator/downloaders/download-state.hpp index 02984c53..bde80aae 100644 --- a/validator/downloaders/download-state.hpp +++ b/validator/downloaders/download-state.hpp @@ -19,6 +19,7 @@ #pragma once #include "validator/interfaces/validator-manager.h" +#include "stats-provider.h" namespace ton { @@ -67,6 +68,8 @@ class DownloadShardState : public td::actor::Actor { td::BufferSlice data_; td::Ref state_; + + ProcessStatus status_; }; } // namespace validator diff --git a/validator/downloaders/wait-block-data.cpp b/validator/downloaders/wait-block-data.cpp index c40dba48..53a3d351 100644 --- a/validator/downloaders/wait-block-data.cpp +++ b/validator/downloaders/wait-block-data.cpp @@ -17,10 +17,14 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "wait-block-data.hpp" + +#include "block-parse.h" +#include "block-auto.h" #include "fabric.h" #include "adnl/utils.hpp" #include "ton/ton-io.hpp" #include "common/delay.h" +#include "vm/cells/MerkleProof.h" namespace ton { @@ -102,13 +106,24 @@ void WaitBlockData::start() { }); td::actor::send_closure(manager_, &ValidatorManager::try_get_static_file, handle_->id().file_hash, std::move(P)); + } else if (try_get_candidate_) { + try_get_candidate_ = false; + td::actor::send_closure( + manager_, &ValidatorManager::get_candidate_data_by_block_id_from_db, handle_->id(), + [SelfId = actor_id(this), id = handle_->id()](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &WaitBlockData::start); + } else { + td::actor::send_closure(SelfId, &WaitBlockData::loaded_data, ReceivedBlock{id, R.move_as_ok()}); + } + }); } else { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &WaitBlockData::failed_to_get_block_data_from_net, R.move_as_error_prefix("net error: ")); } else { - td::actor::send_closure(SelfId, &WaitBlockData::got_block_data_from_net, R.move_as_ok()); + td::actor::send_closure(SelfId, &WaitBlockData::loaded_data, R.move_as_ok()); } }); @@ -133,13 +148,49 @@ void WaitBlockData::failed_to_get_block_data_from_net(td::Status reason) { td::Timestamp::in(0.1)); } -void WaitBlockData::got_block_data_from_net(ReceivedBlock block) { +void WaitBlockData::loaded_data(ReceivedBlock block) { auto X = create_block(std::move(block)); if (X.is_error()) { failed_to_get_block_data_from_net(X.move_as_error_prefix("bad block from net: ")); return; } - data_ = X.move_as_ok(); + loaded_block_data(X.move_as_ok()); +} + +void WaitBlockData::loaded_block_data(td::Ref block) { + if (data_.not_null()) { + return; + } + data_ = std::move(block); + if (handle_->received()) { + finish_query(); + return; + } + if (!handle_->id().is_masterchain() && !handle_->inited_proof_link()) { + // This can happen if we get block from candidates cache. + // Proof link can be derived from the block (but not for masterchain block). + auto r_proof_link = generate_proof_link(handle_->id(), data_->root_cell()); + if (r_proof_link.is_error()) { + abort_query(r_proof_link.move_as_error_prefix("failed to create proof link for block: ")); + return; + } + td::actor::send_closure(manager_, &ValidatorManager::validate_block_proof_link, handle_->id(), + r_proof_link.move_as_ok(), + [id = handle_->id().id, SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &WaitBlockData::abort_query, + R.move_as_error_prefix("validate proof link error: ")); + return; + } + LOG(DEBUG) << "Created and validated proof link for " << id.to_str(); + td::actor::send_closure(SelfId, &WaitBlockData::checked_proof_link); + }); + return; + } + checked_proof_link(); +} + +void WaitBlockData::checked_proof_link() { CHECK(handle_->id().is_masterchain() ? handle_->inited_proof() : handle_->inited_proof_link()); if (!handle_->received()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { @@ -198,6 +249,41 @@ void WaitBlockData::got_static_file(td::BufferSlice data) { run_hardfork_accept_block_query(handle_->id(), data_, manager_, std::move(P)); } +td::Result WaitBlockData::generate_proof_link(BlockIdExt id, td::Ref block_root) { + // Creating proof link. Similar to accept-block.cpp + if (id.is_masterchain()) { + return td::Status::Error("cannot create proof link for masterchain block"); + } + auto usage_tree = std::make_shared(); + auto usage_cell = vm::UsageCell::create(block_root, usage_tree->root_ptr()); + + block::gen::Block::Record blk; + block::gen::BlockInfo::Record info; + block::gen::BlockExtra::Record extra; + block::gen::ExtBlkRef::Record mcref{}; // _ ExtBlkRef = BlkMasterInfo; + ShardIdFull shard; + if (!(tlb::unpack_cell(usage_cell, blk) && tlb::unpack_cell(blk.info, info) && !info.version && + block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) && + block::gen::BlkPrevInfo{info.after_merge}.validate_ref(info.prev_ref) && + tlb::unpack_cell(std::move(blk.extra), extra) && block::gen::t_ValueFlow.force_validate_ref(blk.value_flow) && + (!info.not_master || tlb::unpack_cell(info.master_ref, mcref)))) { + return td::Status::Error("cannot unpack block header"); + } + vm::CellSlice upd_cs{vm::NoVmSpec(), blk.state_update}; + + auto proof = vm::MerkleProof::generate(block_root, usage_tree.get()); + vm::CellBuilder cb; + td::Ref bs_cell; + if (!(cb.store_long_bool(0xc3, 8) // block_proof#c3 + && block::tlb::t_BlockIdExt.pack(cb, id) // proof_for:BlockIdExt + && cb.store_ref_bool(std::move(proof)) // proof:^Cell + && cb.store_bool_bool(false) // signatures:(Maybe ^BlockSignatures) + && cb.finalize_to(bs_cell))) { + return td::Status::Error("cannot serialize BlockProof"); + } + return std_boc_serialize(bs_cell, 0); +} + } // namespace validator } // namespace ton diff --git a/validator/downloaders/wait-block-data.hpp b/validator/downloaders/wait-block-data.hpp index 9a03b1cb..f3b367d5 100644 --- a/validator/downloaders/wait-block-data.hpp +++ b/validator/downloaders/wait-block-data.hpp @@ -30,15 +30,16 @@ class ValidatorManager; class WaitBlockData : public td::actor::Actor { public: WaitBlockData(BlockHandle handle, td::uint32 priority, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise> promise) + td::Timestamp timeout, bool try_get_candidate, td::Promise> promise) : handle_(std::move(handle)) , priority_(priority) , manager_(manager) , timeout_(timeout) + , try_get_candidate_(try_get_candidate) , promise_(std::move(promise)) , perf_timer_("waitdata", 1.0, [manager](double duration) { - send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitdata", duration); - }) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitdata", duration); + }) { } void update_timeout(td::Timestamp timeout, td::uint32 priority) { @@ -57,11 +58,15 @@ class WaitBlockData : public td::actor::Actor { void set_is_hardfork(bool value); void start(); void got_block_data_from_db(td::Ref data); - void got_block_data_from_net(ReceivedBlock data); + void loaded_data(ReceivedBlock data); + void loaded_block_data(td::Ref block); + void checked_proof_link(); void failed_to_get_block_data_from_net(td::Status reason); void got_static_file(td::BufferSlice data); + static td::Result generate_proof_link(BlockIdExt id, td::Ref block_root); + private: BlockHandle handle_; @@ -69,6 +74,7 @@ class WaitBlockData : public td::actor::Actor { td::actor::ActorId manager_; td::Timestamp timeout_; + bool try_get_candidate_; td::Promise> promise_; td::Ref data_; diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index 56137fc3..c80e7d89 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -21,6 +21,7 @@ #include "ton/ton-io.hpp" #include "common/checksum.h" #include "common/delay.h" +#include "validator/downloaders/download-state.hpp" namespace ton { @@ -66,7 +67,8 @@ void WaitBlockState::start() { if (reading_from_db_) { return; } - if (handle_->received_state()) { + bool inited_proof = handle_->id().is_masterchain() ? handle_->inited_proof() : handle_->inited_proof_link(); + if (handle_->received_state() && inited_proof) { reading_from_db_ = true; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { @@ -106,15 +108,30 @@ void WaitBlockState::start() { }); td::actor::send_closure(manager_, &ValidatorManager::send_get_zero_state_request, handle_->id(), priority_, std::move(P)); + } else if (check_persistent_state_desc() && !handle_->received_state()) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + LOG(WARNING) << "failed to get persistent state: " << R.move_as_error(); + td::actor::send_closure(SelfId, &WaitBlockState::start); + } else { + td::actor::send_closure(SelfId, &WaitBlockState::written_state, R.move_as_ok()); + } + }); + BlockIdExt masterchain_id = persistent_state_desc_->masterchain_id; + td::actor::create_actor("downloadstate", handle_->id(), masterchain_id, priority_, manager_, + timeout_, std::move(P)) + .release(); } else if (!handle_->inited_prev() || (!handle_->inited_proof() && !handle_->inited_proof_link())) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result R) { if (R.is_error()) { - delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::start); }, td::Timestamp::in(0.1)); + delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); }, + td::Timestamp::in(0.1)); } else { td::actor::send_closure(SelfId, &WaitBlockState::got_proof_link, R.move_as_ok()); } }); + waiting_proof_link_ = true; td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, handle_->id(), priority_, std::move(P)); } else if (prev_state_.is_null()) { @@ -133,12 +150,14 @@ void WaitBlockState::start() { } else if (handle_->id().is_masterchain() && !handle_->inited_proof()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result R) { if (R.is_error()) { - delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::start); }, td::Timestamp::in(0.1)); + delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof); }, + td::Timestamp::in(0.1)); } else { td::actor::send_closure(SelfId, &WaitBlockState::got_proof, R.move_as_ok()); } }); + waiting_proof_ = true; td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_request, handle_->id(), priority_, std::move(P)); } else if (block_.is_null()) { @@ -172,6 +191,9 @@ void WaitBlockState::got_prev_state(td::Ref state) { } void WaitBlockState::got_proof_link(td::BufferSlice data) { + if (!waiting_proof_link_) { + return; + } auto R = create_proof_link(handle_->id(), std::move(data)); if (R.is_error()) { LOG(INFO) << "received bad proof link: " << R.move_as_error(); @@ -182,22 +204,26 @@ void WaitBlockState::got_proof_link(td::BufferSlice data) { if (R.is_ok()) { auto h = R.move_as_ok(); CHECK(h->inited_prev()); - td::actor::send_closure(SelfId, &WaitBlockState::start); + td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); } else { LOG(INFO) << "received bad proof link: " << R.move_as_error(); - td::actor::send_closure(SelfId, &WaitBlockState::start); + delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); }, + td::Timestamp::in(0.1)); } }); run_check_proof_link_query(handle_->id(), R.move_as_ok(), manager_, timeout_, std::move(P)); } void WaitBlockState::got_proof(td::BufferSlice data) { + if (!waiting_proof_) { + return; + } auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_ok()) { - td::actor::send_closure(SelfId, &WaitBlockState::start); + td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof); } else { LOG(INFO) << "received bad proof link: " << R.move_as_error(); - td::actor::send_closure(SelfId, &WaitBlockState::start); + td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof); } }); td::actor::send_closure(manager_, &ValidatorManager::validate_block_proof, handle_->id(), std::move(data), @@ -219,6 +245,7 @@ void WaitBlockState::got_block_data(td::Ref data) { } void WaitBlockState::apply() { + TD_PERF_COUNTER(apply_block_to_state); td::PerfWarningTimer t{"applyblocktostate", 0.1}; auto S = prev_state_.write().apply_block(handle_->id(), block_); if (S.is_error()) { diff --git a/validator/downloaders/wait-block-state.hpp b/validator/downloaders/wait-block-state.hpp index 7cdc0699..6a14d909 100644 --- a/validator/downloaders/wait-block-state.hpp +++ b/validator/downloaders/wait-block-state.hpp @@ -27,12 +27,14 @@ namespace validator { class WaitBlockState : public td::actor::Actor { public: WaitBlockState(BlockHandle handle, td::uint32 priority, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise> promise) + td::Timestamp timeout, td::Promise> promise, + td::Ref persistent_state_desc = {}) : handle_(std::move(handle)) , priority_(priority) , manager_(manager) , timeout_(timeout) , promise_(std::move(promise)) + , persistent_state_desc_(std::move(persistent_state_desc)) , perf_timer_("waitstate", 1.0, [manager](double duration) { send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitstate", duration); }) { @@ -45,11 +47,9 @@ class WaitBlockState : public td::actor::Actor { void force_read_from_db(); void start_up() override; - void got_block_handle(BlockHandle handle); void start(); void got_state_from_db(td::Ref data); void got_state_from_static_file(td::Ref state, td::BufferSlice data); - void failed_to_get_state_from_db(td::Status reason); void got_prev_state(td::Ref state); void failed_to_get_prev_state(td::Status reason); void got_block_data(td::Ref data); @@ -68,6 +68,22 @@ class WaitBlockState : public td::actor::Actor { priority_ = priority; } + // These two methods can be called from ValidatorManagerImpl::written_handle + void after_get_proof_link() { + if (!waiting_proof_link_) { + return; + } + waiting_proof_link_ = false; + start(); + } + void after_get_proof() { + if (!waiting_proof_) { + return; + } + waiting_proof_ = false; + start(); + } + private: BlockHandle handle_; @@ -76,14 +92,25 @@ class WaitBlockState : public td::actor::Actor { td::actor::ActorId manager_; td::Timestamp timeout_; td::Promise> promise_; + td::Ref persistent_state_desc_; td::Ref prev_state_; td::Ref block_; bool reading_from_db_ = false; + bool waiting_proof_link_ = false; + bool waiting_proof_ = false; td::Timestamp next_static_file_attempt_; - td::PerfWarningTimer perf_timer_; + td::PerfWarningTimer perf_timer_{"waitstate", 1.0}; + + bool check_persistent_state_desc() const { + if (persistent_state_desc_.is_null()) { + return false; + } + auto now = (UnixTime)td::Clocks::system(); + return persistent_state_desc_->end_time > now + 3600 && persistent_state_desc_->start_time < now - 6 * 3600; + } }; } // namespace validator diff --git a/validator/fabric.h b/validator/fabric.h index 58f0647b..2c39aceb 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -20,12 +20,16 @@ #include "interfaces/validator-manager.h" #include "interfaces/db.h" +#include "validator.h" namespace ton { namespace validator { -td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_); +enum CollateMode { skip_store_candidate = 1 }; + +td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_, + td::Ref opts); td::actor::ActorOwn create_liteserver_cache_actor(td::actor::ActorId manager, std::string db_root); @@ -47,12 +51,12 @@ td::Result>> create_new_shard_bloc td::Ref create_signature_set(std::vector sig_set); -void run_check_external_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits, - td::actor::ActorId manager, td::Promise> promise); +void run_check_external_message(td::Ref message, td::actor::ActorId manager, + td::Promise> promise); void run_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, bool send_broadcast, + td::Ref approve_signatures, int send_broadcast_mode, td::actor::ActorId manager, td::Promise promise); void run_fake_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::actor::ActorId manager, @@ -72,14 +76,15 @@ void run_check_proof_query(BlockIdExt id, td::Ref proof, td::actor::Actor td::Ref rel_key_block_proof, bool skip_check_signatures = false); void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); -void run_validate_query(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, - std::vector prev, BlockCandidate candidate, td::Ref validator_set, +void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, + BlockCandidate candidate, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, bool is_fake = false); -void run_collate_query(ShardIdFull shard, td::uint32 min_ts, const BlockIdExt& min_masterchain_block_id, - std::vector prev, Ed25519_PublicKey local_id, td::Ref validator_set, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise); +void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, + Ed25519_PublicKey creator, td::Ref validator_set, + td::Ref collator_opts, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise promise, + td::CancellationToken cancellation_token, unsigned mode, int attempt_idx = 0); void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); diff --git a/validator/full-node-master.cpp b/validator/full-node-master.cpp index f26b1198..da49f0e2 100644 --- a/validator/full-node-master.cpp +++ b/validator/full-node-master.cpp @@ -371,7 +371,8 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise) { - promise.set_value(create_serialize_tl_object(proto_version(), proto_capabilities())); + promise.set_value( + create_serialize_tl_object(proto_version_major(), proto_version_minor(), 0)); } void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, @@ -385,7 +386,7 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo } }); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_, - std::move(P)); + ShardIdFull{masterchainId}, std::move(P)); } void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, diff --git a/validator/full-node-master.hpp b/validator/full-node-master.hpp index 5e9d22e1..ce0aedd3 100644 --- a/validator/full-node-master.hpp +++ b/validator/full-node-master.hpp @@ -28,10 +28,10 @@ namespace fullnode { class FullNodeMasterImpl : public FullNodeMaster { public: - static constexpr td::uint32 proto_version() { + static constexpr td::uint32 proto_version_major() { return 1; } - static constexpr td::uint64 proto_capabilities() { + static constexpr td::uint32 proto_version_minor() { return 0; } void start_up() override; diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp new file mode 100644 index 00000000..f86323fc --- /dev/null +++ b/validator/full-node-private-overlay.cpp @@ -0,0 +1,496 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "full-node-private-overlay.hpp" +#include "ton/ton-tl.hpp" +#include "common/delay.h" +#include "common/checksum.h" +#include "full-node-serializer.hpp" +#include "auto/tl/ton_api_json.h" +#include "td/utils/JsonBuilder.h" +#include "tl/tl_json.h" + +namespace ton::validator::fullnode { + +void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { + process_block_broadcast(src, query); +} + +void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_blockBroadcastCompressed &query) { + process_block_broadcast(src, query); +} + +void FullNodePrivateBlockOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); + if (B.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << B.move_as_error(); + return; + } + VLOG(FULL_NODE_DEBUG) << "Received block broadcast in private overlay from " << src << ": " + << B.ok().block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); +} + +void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { + BlockIdExt block_id = create_block_id(query.block_->block_); + VLOG(FULL_NODE_DEBUG) << "Received newShardBlockBroadcast in private overlay from " << src << ": " + << block_id.to_str(); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block, block_id, + query.block_->cc_seqno_, std::move(query.block_->data_)); +} + +void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcast &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressed &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodePrivateBlockOverlay::process_block_candidate_broadcast(PublicKeyHash src, + ton_api::tonNode_Broadcast &query) { + BlockIdExt block_id; + CatchainSeqno cc_seqno; + td::uint32 validator_set_hash; + td::BufferSlice data; + auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, + overlay::Overlays::max_fec_broadcast_size()); + if (S.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << S; + return; + } + if (data.size() > FullNode::max_block_size()) { + VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; + return; + } + if (td::sha256_bits256(data.as_slice()) != block_id.file_hash) { + VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; + return; + } + VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in private overlay from " << src << ": " << block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, + validator_set_hash, std::move(data)); +} + +void FullNodePrivateBlockOverlay::process_telemetry_broadcast( + PublicKeyHash src, const tl_object_ptr& telemetry) { + if (telemetry->adnl_id_ != src.bits256_value()) { + VLOG(FULL_NODE_WARNING) << "Invalid telemetry broadcast from " << src << ": adnl_id mismatch"; + return; + } + auto now = (td::int32)td::Clocks::system(); + if (telemetry->timestamp_ < now - 60) { + VLOG(FULL_NODE_WARNING) << "Invalid telemetry broadcast from " << src << ": too old (" + << now - telemetry->timestamp_ << "s ago)"; + return; + } + if (telemetry->timestamp_ > now + 60) { + VLOG(FULL_NODE_WARNING) << "Invalid telemetry broadcast from " << src << ": too new (" + << telemetry->timestamp_ - now << "s in the future)"; + return; + } + VLOG(FULL_NODE_DEBUG) << "Got telemetry broadcast from " << src; + auto s = td::json_encode(td::ToJson(*telemetry), false); + std::erase_if(s, [](char c) { + return c == '\n' || c == '\r'; + }); + telemetry_file_ << s << "\n"; + telemetry_file_.flush(); + if (telemetry_file_.fail()) { + VLOG(FULL_NODE_WARNING) << "Failed to write telemetry to file"; + } +} + +void FullNodePrivateBlockOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { + if (adnl::AdnlNodeIdShort{src} == local_id_) { + return; + } + auto B = fetch_tl_object(std::move(broadcast), true); + if (B.is_error()) { + if (collect_telemetry_ && src != local_id_.pubkey_hash()) { + auto R = fetch_tl_prefix(broadcast, true); + if (R.is_ok()) { + process_telemetry_broadcast(src, R.ok()); + } + } + return; + } + ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto& obj) { + Self->process_broadcast(src, obj); + }); +} + +void FullNodePrivateBlockOverlay::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::BufferSlice data) { + if (!inited_) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newShardBlockBroadcast in private overlay: " << block_id.to_str(); + auto B = create_serialize_tl_object( + create_tl_object(create_tl_block_id(block_id), cc_seqno, std::move(data))); + if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), 0, std::move(B)); + } else { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); + } +} + +void FullNodePrivateBlockOverlay::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) { + if (!inited_) { + return; + } + auto B = + serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in private overlay: " << block_id.to_str(); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); +} + +void FullNodePrivateBlockOverlay::send_broadcast(BlockBroadcast broadcast) { + if (!inited_) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending block broadcast in private overlay" + << (enable_compression_ ? " (with compression)" : "") << ": " << broadcast.block_id.to_str(); + auto B = serialize_block_broadcast(broadcast, enable_compression_); + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); + return; + } + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); +} + +void FullNodePrivateBlockOverlay::send_validator_telemetry(tl_object_ptr telemetry) { + process_telemetry_broadcast(local_id_.pubkey_hash(), telemetry); + auto data = serialize_tl_object(telemetry, true); + if (data.size() <= overlay::Overlays::max_simple_broadcast_size()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), 0, std::move(data)); + } else { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), 0, std::move(data)); + } +} + +void FullNodePrivateBlockOverlay::collect_validator_telemetry(std::string filename) { + if (collect_telemetry_) { + telemetry_file_.close(); + } + collect_telemetry_ = true; + LOG(FULL_NODE_WARNING) << "Collecting validator telemetry to " << filename << " (local id: " << local_id_ << ")"; + telemetry_file_.open(filename, std::ios_base::app); + if (!telemetry_file_.is_open()) { + LOG(WARNING) << "Cannot open file " << filename << " for validator telemetry"; + } +} + +void FullNodePrivateBlockOverlay::start_up() { + std::sort(nodes_.begin(), nodes_.end()); + nodes_.erase(std::unique(nodes_.begin(), nodes_.end()), nodes_.end()); + + std::vector nodes; + for (const adnl::AdnlNodeIdShort &id : nodes_) { + nodes.push_back(id.bits256_value()); + } + auto X = create_hash_tl_object(zero_state_file_hash_, std::move(nodes)); + td::BufferSlice b{32}; + b.as_slice().copy_from(as_slice(X)); + overlay_id_full_ = overlay::OverlayIdFull{std::move(b)}; + overlay_id_ = overlay_id_full_.compute_short_id(); + + try_init(); +} + +void FullNodePrivateBlockOverlay::try_init() { + // Sometimes adnl id is added to validator engine later (or not at all) + td::actor::send_closure( + adnl_, &adnl::Adnl::check_id_exists, local_id_, [SelfId = actor_id(this)](td::Result R) { + if (R.is_ok() && R.ok()) { + td::actor::send_closure(SelfId, &FullNodePrivateBlockOverlay::init); + } else { + delay_action([SelfId]() { td::actor::send_closure(SelfId, &FullNodePrivateBlockOverlay::try_init); }, + td::Timestamp::in(30.0)); + } + }); +} + +void FullNodePrivateBlockOverlay::init() { + LOG(FULL_NODE_WARNING) << "Creating private block overlay for adnl id " << local_id_ << " : " << nodes_.size() + << " nodes, overlay_id=" << overlay_id_; + class Callback : public overlay::Overlays::Callback { + public: + void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + } + void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { + td::actor::send_closure(node_, &FullNodePrivateBlockOverlay::receive_broadcast, src, std::move(data)); + } + void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + } + Callback(td::actor::ActorId node) : node_(node) { + } + + private: + td::actor::ActorId node_; + }; + + overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(), + overlay::CertificateFlags::AllowFec | overlay::CertificateFlags::Trusted, + {}}; + overlay::OverlayOptions overlay_options; + overlay_options.broadcast_speed_multiplier_ = opts_.private_broadcast_speed_multiplier_; + td::actor::send_closure(overlays_, &overlay::Overlays::create_private_overlay_ex, local_id_, overlay_id_full_.clone(), + nodes_, std::make_unique(actor_id(this)), rules, R"({ "type": "private-blocks" })", + overlay_options); + + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); + td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_); + inited_ = true; +} + +void FullNodePrivateBlockOverlay::tear_down() { + if (inited_) { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, local_id_, overlay_id_); + } +} + +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { + process_block_broadcast(src, query); +} + +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query) { + process_block_broadcast(src, query); +} + +void FullNodeCustomOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { + VLOG(FULL_NODE_DEBUG) << "Dropping block broadcast in private overlay \"" << name_ << "\" from unauthorized sender " + << src; + return; + } + auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); + if (B.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << B.move_as_error(); + return; + } + VLOG(FULL_NODE_DEBUG) << "Received block broadcast in custom overlay \"" << name_ << "\" from " << src << ": " + << B.ok().block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); +} + +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query) { + auto it = msg_senders_.find(adnl::AdnlNodeIdShort{src}); + if (it == msg_senders_.end()) { + VLOG(FULL_NODE_DEBUG) << "Dropping external message broadcast in custom overlay \"" << name_ + << "\" from unauthorized sender " << src; + return; + } + VLOG(FULL_NODE_DEBUG) << "Got external message in custom overlay \"" << name_ << "\" from " << src + << " (priority=" << it->second << ")"; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_external_message, + std::move(query.message_->data_), it->second); +} + +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressed &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeCustomOverlay::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { + VLOG(FULL_NODE_DEBUG) << "Dropping block candidate broadcast in private overlay \"" << name_ + << "\" from unauthorized sender " << src; + return; + } + BlockIdExt block_id; + CatchainSeqno cc_seqno; + td::uint32 validator_set_hash; + td::BufferSlice data; + auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, + overlay::Overlays::max_fec_broadcast_size()); + if (S.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << S; + return; + } + if (data.size() > FullNode::max_block_size()) { + VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; + return; + } + if (td::sha256_bits256(data.as_slice()) != block_id.file_hash) { + VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; + return; + } + VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in custom overlay \"" << name_ << "\" from " << src << ": " + << block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, + validator_set_hash, std::move(data)); +} + +void FullNodeCustomOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { + if (adnl::AdnlNodeIdShort{src} == local_id_) { + return; + } + auto B = fetch_tl_object(std::move(broadcast), true); + if (B.is_error()) { + return; + } + ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto &obj) { Self->process_broadcast(src, obj); }); +} + +void FullNodeCustomOverlay::send_external_message(td::BufferSlice data) { + if (!inited_ || opts_.config_.ext_messages_broadcast_disabled_) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending external message to custom overlay \"" << name_ << "\""; + auto B = create_serialize_tl_object( + create_tl_object(std::move(data))); + if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), 0, std::move(B)); + } else { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), 0, std::move(B)); + } +} + +void FullNodeCustomOverlay::send_broadcast(BlockBroadcast broadcast) { + if (!inited_) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending block broadcast to custom overlay \"" << name_ + << "\": " << broadcast.block_id.to_str(); + auto B = serialize_block_broadcast(broadcast, true); // compression_enabled = true + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); + return; + } + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); +} + +void FullNodeCustomOverlay::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) { + if (!inited_) { + return; + } + auto B = + serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in custom overlay \"" << name_ << "\": " << block_id.to_str(); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); +} + +void FullNodeCustomOverlay::start_up() { + std::sort(nodes_.begin(), nodes_.end()); + nodes_.erase(std::unique(nodes_.begin(), nodes_.end()), nodes_.end()); + std::vector nodes; + for (const adnl::AdnlNodeIdShort &id : nodes_) { + nodes.push_back(id.bits256_value()); + } + auto X = create_hash_tl_object(zero_state_file_hash_, name_, std::move(nodes)); + td::BufferSlice b{32}; + b.as_slice().copy_from(as_slice(X)); + overlay_id_full_ = overlay::OverlayIdFull{std::move(b)}; + overlay_id_ = overlay_id_full_.compute_short_id(); + try_init(); +} + +void FullNodeCustomOverlay::try_init() { + // Sometimes adnl id is added to validator engine later (or not at all) + td::actor::send_closure( + adnl_, &adnl::Adnl::check_id_exists, local_id_, [SelfId = actor_id(this)](td::Result R) { + if (R.is_ok() && R.ok()) { + td::actor::send_closure(SelfId, &FullNodeCustomOverlay::init); + } else { + delay_action([SelfId]() { td::actor::send_closure(SelfId, &FullNodeCustomOverlay::try_init); }, + td::Timestamp::in(30.0)); + } + }); +} + +void FullNodeCustomOverlay::init() { + LOG(FULL_NODE_WARNING) << "Creating custom overlay \"" << name_ << "\" for adnl id " << local_id_ << " : " + << nodes_.size() << " nodes, " << msg_senders_.size() << " msg senders, " + << block_senders_.size() << " block senders, overlay_id=" << overlay_id_; + class Callback : public overlay::Overlays::Callback { + public: + void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + } + void receive_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { + td::actor::send_closure(node_, &FullNodeCustomOverlay::receive_broadcast, src, std::move(data)); + } + void check_broadcast(PublicKeyHash src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + } + Callback(td::actor::ActorId node) : node_(node) { + } + + private: + td::actor::ActorId node_; + }; + + std::map authorized_keys; + for (const auto &sender : msg_senders_) { + authorized_keys[sender.first.pubkey_hash()] = overlay::Overlays::max_fec_broadcast_size(); + } + for (const auto &sender : block_senders_) { + authorized_keys[sender.pubkey_hash()] = overlay::Overlays::max_fec_broadcast_size(); + } + overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(), 0, std::move(authorized_keys)}; + overlay::OverlayOptions overlay_options; + overlay_options.broadcast_speed_multiplier_ = opts_.private_broadcast_speed_multiplier_; + td::actor::send_closure( + overlays_, &overlay::Overlays::create_private_overlay_ex, local_id_, overlay_id_full_.clone(), nodes_, + std::make_unique(actor_id(this)), rules, + PSTRING() << R"({ "type": "custom-overlay", "name": ")" << td::format::Escaped{name_} << R"(" })", + overlay_options); + + td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); + td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_); + inited_ = true; +} + +void FullNodeCustomOverlay::tear_down() { + LOG(FULL_NODE_WARNING) << "Destroying custom overlay \"" << name_ << "\" for adnl id " << local_id_; + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, local_id_, overlay_id_); +} + +} // namespace ton::validator::fullnode diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp new file mode 100644 index 00000000..70e196ea --- /dev/null +++ b/validator/full-node-private-overlay.hpp @@ -0,0 +1,182 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "full-node.h" +#include + +namespace ton::validator::fullnode { + +class FullNodePrivateBlockOverlay : public td::actor::Actor { + public: + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); + void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); + + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + + void process_telemetry_broadcast(PublicKeyHash src, const tl_object_ptr& telemetry); + + template + void process_broadcast(PublicKeyHash, T &) { + VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast"; + } + void receive_broadcast(PublicKeyHash src, td::BufferSlice query); + + void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data); + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data); + void send_broadcast(BlockBroadcast broadcast); + void send_validator_telemetry(tl_object_ptr telemetry); + + void collect_validator_telemetry(std::string filename); + + void set_config(FullNodeConfig config) { + opts_.config_ = std::move(config); + } + + void start_up() override; + void tear_down() override; + + FullNodePrivateBlockOverlay(adnl::AdnlNodeIdShort local_id, std::vector nodes, + FileHash zero_state_file_hash, FullNodeOptions opts, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId rldp2, + td::actor::ActorId overlays, + td::actor::ActorId validator_manager, + td::actor::ActorId full_node) + : local_id_(local_id) + , nodes_(std::move(nodes)) + , zero_state_file_hash_(zero_state_file_hash) + , opts_(opts) + , keyring_(keyring) + , adnl_(adnl) + , rldp_(rldp) + , rldp2_(rldp2) + , overlays_(overlays) + , validator_manager_(validator_manager) + , full_node_(full_node) { + } + + private: + adnl::AdnlNodeIdShort local_id_; + std::vector nodes_; + FileHash zero_state_file_hash_; + FullNodeOptions opts_; + bool enable_compression_ = true; + + td::actor::ActorId keyring_; + td::actor::ActorId adnl_; + td::actor::ActorId rldp_; + td::actor::ActorId rldp2_; + td::actor::ActorId overlays_; + td::actor::ActorId validator_manager_; + td::actor::ActorId full_node_; + + bool inited_ = false; + overlay::OverlayIdFull overlay_id_full_; + overlay::OverlayIdShort overlay_id_; + + void try_init(); + void init(); + + bool collect_telemetry_ = false; + std::ofstream telemetry_file_; +}; + +class FullNodeCustomOverlay : public td::actor::Actor { + public: + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); + void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + + void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); + + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + + template + void process_broadcast(PublicKeyHash, T &) { + VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast"; + } + void receive_broadcast(PublicKeyHash src, td::BufferSlice query); + + void send_external_message(td::BufferSlice data); + void send_broadcast(BlockBroadcast broadcast); + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data); + + void set_config(FullNodeConfig config) { + opts_.config_ = std::move(config); + } + + void start_up() override; + void tear_down() override; + + FullNodeCustomOverlay(adnl::AdnlNodeIdShort local_id, CustomOverlayParams params, FileHash zero_state_file_hash, + FullNodeOptions opts, td::actor::ActorId keyring, + td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId rldp2, td::actor::ActorId overlays, + td::actor::ActorId validator_manager, + td::actor::ActorId full_node) + : local_id_(local_id) + , name_(std::move(params.name_)) + , nodes_(std::move(params.nodes_)) + , msg_senders_(std::move(params.msg_senders_)) + , block_senders_(std::move(params.block_senders_)) + , zero_state_file_hash_(zero_state_file_hash) + , opts_(opts) + , keyring_(keyring) + , adnl_(adnl) + , rldp_(rldp) + , rldp2_(rldp2) + , overlays_(overlays) + , validator_manager_(validator_manager) + , full_node_(full_node) { + } + + private: + adnl::AdnlNodeIdShort local_id_; + std::string name_; + std::vector nodes_; + std::map msg_senders_; + std::set block_senders_; + FileHash zero_state_file_hash_; + FullNodeOptions opts_; + + td::actor::ActorId keyring_; + td::actor::ActorId adnl_; + td::actor::ActorId rldp_; + td::actor::ActorId rldp2_; + td::actor::ActorId overlays_; + td::actor::ActorId validator_manager_; + td::actor::ActorId full_node_; + + bool inited_ = false; + overlay::OverlayIdFull overlay_id_full_; + overlay::OverlayIdShort overlay_id_; + + void try_init(); + void init(); +}; + +} // namespace ton::validator::fullnode diff --git a/validator/full-node-serializer.cpp b/validator/full-node-serializer.cpp new file mode 100644 index 00000000..94dc2155 --- /dev/null +++ b/validator/full-node-serializer.cpp @@ -0,0 +1,214 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "full-node-serializer.hpp" +#include "ton/ton-tl.hpp" +#include "tl-utils/common-utils.hpp" +#include "auto/tl/ton_api.hpp" +#include "tl-utils/tl-utils.hpp" +#include "vm/boc.h" +#include "td/utils/lz4.h" +#include "full-node.h" +#include "td/utils/overloaded.h" + +namespace ton::validator::fullnode { + +td::Result serialize_block_broadcast(const BlockBroadcast& broadcast, bool compression_enabled) { + std::vector> sigs; + for (auto& sig : broadcast.signatures) { + sigs.emplace_back(create_tl_object(sig.node, sig.signature.clone())); + } + if (!compression_enabled) { + return create_serialize_tl_object( + create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs), + broadcast.proof.clone(), broadcast.data.clone()); + } + + TRY_RESULT(proof_root, vm::std_boc_deserialize(broadcast.proof)); + TRY_RESULT(data_root, vm::std_boc_deserialize(broadcast.data)); + TRY_RESULT(boc, vm::std_boc_serialize_multi({proof_root, data_root}, 2)); + td::BufferSlice data = + create_serialize_tl_object(std::move(sigs), std::move(boc)); + td::BufferSlice compressed = td::lz4_compress(data); + VLOG(FULL_NODE_DEBUG) << "Compressing block broadcast: " + << broadcast.data.size() + broadcast.proof.size() + broadcast.signatures.size() * 96 << " -> " + << compressed.size(); + return create_serialize_tl_object( + create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, 0, + std::move(compressed)); +} + +static td::Result deserialize_block_broadcast(ton_api::tonNode_blockBroadcast& f) { + std::vector signatures; + for (auto& sig : f.signatures_) { + signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)}); + } + return BlockBroadcast{create_block_id(f.id_), + std::move(signatures), + static_cast(f.catchain_seqno_), + static_cast(f.validator_set_hash_), + std::move(f.data_), + std::move(f.proof_)}; +} + +static td::Result deserialize_block_broadcast(ton_api::tonNode_blockBroadcastCompressed& f, + int max_decompressed_size) { + TRY_RESULT(decompressed, td::lz4_decompress(f.compressed_, max_decompressed_size)); + TRY_RESULT(f2, fetch_tl_object(decompressed, true)); + std::vector signatures; + for (auto& sig : f2->signatures_) { + signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)}); + } + TRY_RESULT(roots, vm::std_boc_deserialize_multi(f2->proof_data_, 2)); + if (roots.size() != 2) { + return td::Status::Error("expected 2 roots in boc"); + } + TRY_RESULT(proof, vm::std_boc_serialize(roots[0], 0)); + TRY_RESULT(data, vm::std_boc_serialize(roots[1], 31)); + VLOG(FULL_NODE_DEBUG) << "Decompressing block broadcast: " << f.compressed_.size() << " -> " + << data.size() + proof.size() + signatures.size() * 96; + return BlockBroadcast{create_block_id(f.id_), + std::move(signatures), + static_cast(f.catchain_seqno_), + static_cast(f.validator_set_hash_), + std::move(data), + std::move(proof)}; +} + +td::Result deserialize_block_broadcast(ton_api::tonNode_Broadcast& obj, + int max_decompressed_data_size) { + td::Result B; + ton_api::downcast_call(obj, + td::overloaded([&](ton_api::tonNode_blockBroadcast& f) { B = deserialize_block_broadcast(f); }, + [&](ton_api::tonNode_blockBroadcastCompressed& f) { + B = deserialize_block_broadcast(f, max_decompressed_data_size); + }, + [&](auto&) { B = td::Status::Error("unknown broadcast type"); })); + return B; +} + +td::Result serialize_block_full(const BlockIdExt& id, td::Slice proof, td::Slice data, + bool is_proof_link, bool compression_enabled) { + if (!compression_enabled) { + return create_serialize_tl_object(create_tl_block_id(id), td::BufferSlice(proof), + td::BufferSlice(data), is_proof_link); + } + TRY_RESULT(proof_root, vm::std_boc_deserialize(proof)); + TRY_RESULT(data_root, vm::std_boc_deserialize(data)); + TRY_RESULT(boc, vm::std_boc_serialize_multi({proof_root, data_root}, 2)); + td::BufferSlice compressed = td::lz4_compress(boc); + VLOG(FULL_NODE_DEBUG) << "Compressing block full: " << data.size() + proof.size() << " -> " << compressed.size(); + return create_serialize_tl_object(create_tl_block_id(id), 0, + std::move(compressed), is_proof_link); +} + +static td::Status deserialize_block_full(ton_api::tonNode_dataFull& f, BlockIdExt& id, td::BufferSlice& proof, + td::BufferSlice& data, bool& is_proof_link) { + id = create_block_id(f.id_); + proof = std::move(f.proof_); + data = std::move(f.block_); + is_proof_link = f.is_link_; + return td::Status::OK(); +} + +static td::Status deserialize_block_full(ton_api::tonNode_dataFullCompressed& f, BlockIdExt& id, td::BufferSlice& proof, + td::BufferSlice& data, bool& is_proof_link, int max_decompressed_size) { + TRY_RESULT(decompressed, td::lz4_decompress(f.compressed_, max_decompressed_size)); + TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed, 2)); + if (roots.size() != 2) { + return td::Status::Error("expected 2 roots in boc"); + } + TRY_RESULT_ASSIGN(proof, vm::std_boc_serialize(roots[0], 0)); + TRY_RESULT_ASSIGN(data, vm::std_boc_serialize(roots[1], 31)); + VLOG(FULL_NODE_DEBUG) << "Decompressing block full: " << f.compressed_.size() << " -> " << data.size() + proof.size(); + id = create_block_id(f.id_); + is_proof_link = f.is_link_; + return td::Status::OK(); +} + +td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id, td::BufferSlice& proof, + td::BufferSlice& data, bool& is_proof_link, int max_decompressed_data_size) { + td::Status S; + ton_api::downcast_call( + obj, td::overloaded( + [&](ton_api::tonNode_dataFull& f) { S = deserialize_block_full(f, id, proof, data, is_proof_link); }, + [&](ton_api::tonNode_dataFullCompressed& f) { + S = deserialize_block_full(f, id, proof, data, is_proof_link, max_decompressed_data_size); + }, + [&](auto&) { S = td::Status::Error("unknown data type"); })); + return S; +} + +td::Result serialize_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Slice data, + bool compression_enabled) { + if (!compression_enabled) { + return create_serialize_tl_object( + create_tl_block_id(block_id), cc_seqno, validator_set_hash, + create_tl_object(Bits256::zero(), td::BufferSlice()), td::BufferSlice(data)); + } + TRY_RESULT(root, vm::std_boc_deserialize(data)); + TRY_RESULT(data_new, vm::std_boc_serialize(root, 2)); + td::BufferSlice compressed = td::lz4_compress(data_new); + VLOG(FULL_NODE_DEBUG) << "Compressing block candidate broadcast: " << data.size() << " -> " << compressed.size(); + return create_serialize_tl_object( + create_tl_block_id(block_id), cc_seqno, validator_set_hash, + create_tl_object(Bits256::zero(), td::BufferSlice()), 0, std::move(compressed)); +} + +static td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_newBlockCandidateBroadcast& obj, + BlockIdExt& block_id, CatchainSeqno& cc_seqno, + td::uint32& validator_set_hash, td::BufferSlice& data) { + block_id = create_block_id(obj.id_); + cc_seqno = obj.catchain_seqno_; + validator_set_hash = obj.validator_set_hash_; + data = std::move(obj.data_); + return td::Status::OK(); +} + +static td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_newBlockCandidateBroadcastCompressed& obj, + BlockIdExt& block_id, CatchainSeqno& cc_seqno, + td::uint32& validator_set_hash, td::BufferSlice& data, + int max_decompressed_data_size) { + block_id = create_block_id(obj.id_); + cc_seqno = obj.catchain_seqno_; + validator_set_hash = obj.validator_set_hash_; + TRY_RESULT(decompressed, td::lz4_decompress(obj.compressed_, max_decompressed_data_size)); + TRY_RESULT(root, vm::std_boc_deserialize(decompressed)); + TRY_RESULT_ASSIGN(data, vm::std_boc_serialize(root, 31)); + VLOG(FULL_NODE_DEBUG) << "Decompressing block candidate broadcast: " << obj.compressed_.size() << " -> " + << data.size(); + return td::Status::OK(); +} + +td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_Broadcast& obj, BlockIdExt& block_id, + CatchainSeqno& cc_seqno, td::uint32& validator_set_hash, + td::BufferSlice& data, int max_decompressed_data_size) { + td::Status S; + ton_api::downcast_call(obj, td::overloaded( + [&](ton_api::tonNode_newBlockCandidateBroadcast& f) { + S = deserialize_block_candidate_broadcast(f, block_id, cc_seqno, validator_set_hash, + data); + }, + [&](ton_api::tonNode_newBlockCandidateBroadcastCompressed& f) { + S = deserialize_block_candidate_broadcast(f, block_id, cc_seqno, validator_set_hash, + data, max_decompressed_data_size); + }, + [&](auto&) { S = td::Status::Error("unknown data type"); })); + return S; +} + +} // namespace ton::validator::fullnode diff --git a/validator/full-node-serializer.hpp b/validator/full-node-serializer.hpp new file mode 100644 index 00000000..f6751689 --- /dev/null +++ b/validator/full-node-serializer.hpp @@ -0,0 +1,38 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "ton/ton-types.h" +#include "auto/tl/ton_api.h" + +namespace ton::validator::fullnode { + +td::Result serialize_block_broadcast(const BlockBroadcast& broadcast, bool compression_enabled); +td::Result deserialize_block_broadcast(ton_api::tonNode_Broadcast& obj, int max_decompressed_data_size); + +td::Result serialize_block_full(const BlockIdExt& id, td::Slice proof, td::Slice data, + bool is_proof_link, bool compression_enabled); +td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id, td::BufferSlice& proof, + td::BufferSlice& data, bool& is_proof_link, int max_decompressed_data_size); + +td::Result serialize_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Slice data, + bool compression_enabled); +td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_Broadcast& obj, BlockIdExt& block_id, + CatchainSeqno& cc_seqno, td::uint32& validator_set_hash, + td::BufferSlice& data, int max_decompressed_data_size); + +} // namespace ton::validator::fullnode diff --git a/validator/full-node-shard-queries.hpp b/validator/full-node-shard-queries.hpp index 1a2cd716..17068229 100644 --- a/validator/full-node-shard-queries.hpp +++ b/validator/full-node-shard-queries.hpp @@ -20,6 +20,7 @@ #include "validator/validator.h" #include "ton/ton-tl.hpp" +#include "full-node-serializer.hpp" namespace ton { @@ -38,8 +39,8 @@ class BlockFullSender : public td::actor::Actor { stop(); } void finish_query() { - promise_.set_value(create_serialize_tl_object( - create_tl_block_id(block_id_), std::move(proof_), std::move(data_), is_proof_link_)); + promise_.set_result( + serialize_block_full(block_id_, proof_, data_, is_proof_link_, false)); // compression_enabled = false stop(); } void start_up() override { diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 6ed0ae85..ac0eb768 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -17,14 +17,17 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "auto/tl/ton_api.h" +#include "checksum.h" #include "overlays.h" #include "td/utils/SharedSlice.h" +#include "td/utils/overloaded.h" #include "full-node-shard.hpp" #include "full-node-shard-queries.hpp" +#include "full-node-serializer.hpp" +#include "td/utils/buffer.h" #include "ton/ton-shard.h" #include "ton/ton-tl.hpp" -#include "ton/ton-io.hpp" #include "adnl/utils.hpp" #include "net/download-block-new.hpp" @@ -34,10 +37,14 @@ #include "net/download-proof.hpp" #include "net/get-next-key-blocks.hpp" #include "net/download-archive-slice.hpp" +#include "impl/out-msg-queue-proof.hpp" #include "td/utils/Random.h" #include "common/delay.h" +#include "td/utils/JsonBuilder.h" +#include "tl/tl_json.h" +#include "auto/tl/ton_api_json.h" namespace ton { @@ -47,9 +54,10 @@ namespace fullnode { Neighbour Neighbour::zero = Neighbour{adnl::AdnlNodeIdShort::zero()}; -void Neighbour::update_proto_version(const ton_api::tonNode_capabilities &q) { - proto_version = q.version_; - capabilities = q.capabilities_; +void Neighbour::update_proto_version(ton_api::tonNode_capabilities &q) { + version_major = q.version_major_; + version_minor = q.version_minor_; + flags = q.flags_; } void Neighbour::query_success(double t) { @@ -71,8 +79,9 @@ void Neighbour::update_roundtrip(double t) { void FullNodeShardImpl::create_overlay() { class Callback : public overlay::Overlays::Callback { public: - void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { - // just ignore + void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, + td::BufferSlice data) override { + td::actor::send_closure(node_, &FullNodeShardImpl::receive_message, src, std::move(data)); } void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, td::Promise promise) override { @@ -85,38 +94,65 @@ void FullNodeShardImpl::create_overlay() { td::Promise promise) override { td::actor::send_closure(node_, &FullNodeShardImpl::check_broadcast, src, std::move(data), std::move(promise)); } + void get_stats_extra(td::Promise promise) override { + td::actor::send_closure(node_, &FullNodeShardImpl::get_stats_extra, std::move(promise)); + } Callback(td::actor::ActorId node) : node_(node) { } private: td::actor::ActorId node_; }; - - td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay, adnl_id_, overlay_id_full_.clone(), - std::make_unique(actor_id(this)), rules_, PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() << ", \"workchain_id\": " << get_workchain() << " }"); + overlay::OverlayOptions opts; + opts.announce_self_ = active_; + opts.broadcast_speed_multiplier_ = opts_.public_broadcast_speed_multiplier_; + td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay_ex, adnl_id_, overlay_id_full_.clone(), + std::make_unique(actor_id(this)), rules_, + PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() + << ", \"workchain_id\": " << get_workchain() << " }", + opts); td::actor::send_closure(rldp_, &rldp::Rldp::add_id, adnl_id_); + td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, adnl_id_); if (cert_) { td::actor::send_closure(overlays_, &overlay::Overlays::update_certificate, adnl_id_, overlay_id_, local_id_, cert_); } } void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broadcast, td::Promise promise) { + if (!active_) { + return promise.set_error(td::Status::Error("cannot check broadcast: shard is not active")); + } auto B = fetch_tl_object(std::move(broadcast), true); if (B.is_error()) { return promise.set_error(B.move_as_error_prefix("failed to parse external message broadcast: ")); } auto q = B.move_as_ok(); + auto hash = td::sha256_bits256(q->message_->data_); + if (!processed_ext_msg_broadcasts_.insert(hash).second) { + return promise.set_error(td::Status::Error("duplicate external message broadcast")); + } + if (opts_.config_.ext_messages_broadcast_disabled_) { + promise.set_error(td::Status::Error("rebroadcasting external messages is disabled")); + promise = [manager = validator_manager_, message = q->message_->data_.clone()](td::Result R) mutable { + if (R.is_ok()) { + td::actor::send_closure(manager, &ValidatorManagerInterface::new_external_message, std::move(message), 0); + } + }; + } + if (my_ext_msg_broadcasts_.count(hash)) { + // Don't re-check messages that were sent by us + promise.set_result(td::Unit()); + return; + } td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::check_external_message, std::move(q->message_->data_), - [promise = std::move(promise)](td::Result> R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - } else { - promise.set_result(td::Unit()); - } - }); + promise.wrap([](td::Ref) { return td::Unit(); })); +} + +void FullNodeShardImpl::remove_neighbour(adnl::AdnlNodeIdShort id) { + neighbours_.erase(id); } void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) { @@ -126,6 +162,18 @@ void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promis create_overlay(); } +void FullNodeShardImpl::set_active(bool active) { + if (shard_.is_masterchain()) { + return; + } + if (active_ == active) { + return; + } + active_ = active; + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); + create_overlay(); +} + void FullNodeShardImpl::try_get_next_block(td::Timestamp timeout, td::Promise promise) { if (timeout.is_in_past()) { promise.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); @@ -133,7 +181,7 @@ void FullNodeShardImpl::try_get_next_block(td::Timestamp timeout, td::Promise= 1) { + if (!b.adnl_id.is_zero() && b.version_major >= 1) { VLOG(FULL_NODE_DEBUG) << "using new download method with adnlid=" << b.adnl_id; td::actor::create_actor("downloadnext", adnl_id_, overlay_id_, handle_->id(), b.adnl_id, download_next_priority(), timeout, validator_manager_, rldp_, overlays_, @@ -172,7 +220,6 @@ void FullNodeShardImpl::got_next_block(td::Result R) { } void FullNodeShardImpl::get_next_block() { - //return; attempt_++; auto P = td::PromiseCreator::lambda([validator_manager = validator_manager_, attempt = attempt_, block_id = handle_->id(), SelfId = actor_id(this)](td::Result R) { @@ -221,8 +268,9 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod } } }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_next_block, - create_block_id(query.prev_block_), std::move(P)); + BlockIdExt block_id = create_block_id(query.prev_block_); + VLOG(FULL_NODE_DEBUG) << "Got query getNextBlockDescription " << block_id.to_str() << " from " << src; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_next_block, block_id, std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareBlock &query, @@ -242,8 +290,10 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod } } }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_block_handle, - create_block_id(query.block_), false, std::move(P)); + BlockIdExt block_id = create_block_id(query.block_); + VLOG(FULL_NODE_DEBUG) << "Got query prepareBlock " << block_id.to_str() << " from " << src; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_block_handle, block_id, false, + std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadBlock &query, @@ -261,22 +311,24 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod } } }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_block_handle, - create_block_id(query.block_), false, std::move(P)); + BlockIdExt block_id = create_block_id(query.block_); + VLOG(FULL_NODE_DEBUG) << "Got query downloadBlock " << block_id.to_str() << " from " << src; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_block_handle, block_id, false, + std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadBlockFull &query, td::Promise promise) { - td::actor::create_actor("sender", ton::create_block_id(query.block_), false, validator_manager_, - std::move(promise)) - .release(); + BlockIdExt block_id = create_block_id(query.block_); + VLOG(FULL_NODE_DEBUG) << "Got query downloadBlockFull " << block_id.to_str() << " from " << src; + td::actor::create_actor("sender", block_id, false, validator_manager_, std::move(promise)).release(); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadNextBlockFull &query, td::Promise promise) { - td::actor::create_actor("sender", ton::create_block_id(query.prev_block_), true, validator_manager_, - std::move(promise)) - .release(); + BlockIdExt block_id = create_block_id(query.prev_block_); + VLOG(FULL_NODE_DEBUG) << "Got query downloadNextBlockFull " << block_id.to_str() << " from " << src; + td::actor::create_actor("sender", block_id, true, validator_manager_, std::move(promise)).release(); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareBlockProof &query, @@ -308,8 +360,10 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod } }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_block_handle, - create_block_id(query.block_), false, std::move(P)); + BlockIdExt block_id = create_block_id(query.block_); + VLOG(FULL_NODE_DEBUG) << "Got query prepareBlockProof " << block_id.to_str() << " from " << src; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_block_handle, block_id, false, + std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareKeyBlockProof &query, @@ -332,12 +386,15 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod } }); + BlockIdExt block_id = create_block_id(query.block_); + VLOG(FULL_NODE_DEBUG) << "Got query prepareKeyBlockProof " << block_id.to_str() << " " << query.allow_partial_ + << " from " << src; if (query.allow_partial_) { - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_key_block_proof_link, - create_block_id(query.block_), std::move(P)); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_key_block_proof_link, block_id, + std::move(P)); } else { - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_key_block_proof, - create_block_id(query.block_), std::move(P)); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_key_block_proof, block_id, + std::move(P)); } } @@ -360,8 +417,10 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod } }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_block_handle, - create_block_id(query.block_), false, std::move(P)); + BlockIdExt block_id = create_block_id(query.block_); + VLOG(FULL_NODE_DEBUG) << "Got query downloadBlockProof " << block_id.to_str() << " from " << src; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_block_handle, block_id, false, + std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadBlockProofLink &query, @@ -383,8 +442,10 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod } }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_block_handle, - create_block_id(query.block_), false, std::move(P)); + BlockIdExt block_id = create_block_id(query.block_); + VLOG(FULL_NODE_DEBUG) << "Got query downloadBlockProofLink " << block_id.to_str() << " from " << src; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_block_handle, block_id, false, + std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadKeyBlockProof &query, @@ -401,8 +462,9 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod } }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_key_block_proof, - create_block_id(query.block_), std::move(P)); + BlockIdExt block_id = create_block_id(query.block_); + VLOG(FULL_NODE_DEBUG) << "Got query downloadKeyBlockProof " << block_id.to_str() << " from " << src; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_key_block_proof, block_id, std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadKeyBlockProofLink &query, @@ -419,8 +481,10 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod } }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_key_block_proof_link, - create_block_id(query.block_), std::move(P)); + BlockIdExt block_id = create_block_id(query.block_); + VLOG(FULL_NODE_DEBUG) << "Got query downloadKeyBlockProofLink " << block_id.to_str() << " from " << src; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_key_block_proof_link, block_id, + std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareZeroState &query, @@ -437,6 +501,7 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod promise.set_value(std::move(x)); }); auto block_id = create_block_id(query.block_); + VLOG(FULL_NODE_DEBUG) << "Got query prepareZeroState " << block_id.to_str() << " from " << src; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::check_zero_state_exists, block_id, std::move(P)); } @@ -456,6 +521,8 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod }); auto block_id = create_block_id(query.block_); auto masterchain_block_id = create_block_id(query.masterchain_block_); + VLOG(FULL_NODE_DEBUG) << "Got query preparePersistentState " << block_id.to_str() << " " + << masterchain_block_id.to_str() << " from " << src; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::check_persistent_state_exists, block_id, masterchain_block_id, std::move(P)); } @@ -484,6 +551,7 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod promise.set_value(std::move(x)); }); auto block_id = create_block_id(query.block_); + VLOG(FULL_NODE_DEBUG) << "Got query getNextKeyBlockIds " << block_id.to_str() << " " << cnt << " from " << src; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_next_key_blocks, block_id, cnt, std::move(P)); } @@ -500,28 +568,45 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod promise.set_value(R.move_as_ok()); }); auto block_id = create_block_id(query.block_); + VLOG(FULL_NODE_DEBUG) << "Got query downloadZeroState " << block_id.to_str() << " from " << src; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_zero_state, block_id, std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentState &query, td::Promise promise) { + td::uint64 max_size = 1 << 24; auto P = td::PromiseCreator::lambda( - [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + [SelfId = actor_id(this), promise = std::move(promise), max_size](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error_prefix("failed to get state from db: ")); return; } - - promise.set_value(R.move_as_ok()); + td::BufferSlice s = R.move_as_ok(); + if (s.size() > max_size) { + promise.set_error(td::Status::Error("state is too big")); + return; + } + promise.set_value(std::move(s)); }); auto block_id = create_block_id(query.block_); auto masterchain_block_id = create_block_id(query.masterchain_block_); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id, - masterchain_block_id, std::move(P)); + VLOG(FULL_NODE_DEBUG) << "Got query downloadPersistentState " << block_id.to_str() << " " + << masterchain_block_id.to_str() << " from " << src; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, + masterchain_block_id, 0, max_size + 1, std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_downloadPersistentStateSlice &query, td::Promise promise) { + auto block_id = create_block_id(query.block_); + auto masterchain_block_id = create_block_id(query.masterchain_block_); + VLOG(FULL_NODE_DEBUG) << "Got query downloadPersistentStateSlice " << block_id.to_str() << " " + << masterchain_block_id.to_str() << " " << query.offset_ << " " << query.max_size_ << " from " + << src; + if (query.max_size_ < 0 || query.max_size_ > (1 << 24)) { + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "invalid max_size")); + return; + } auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { @@ -531,15 +616,15 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod promise.set_value(R.move_as_ok()); }); - auto block_id = create_block_id(query.block_); - auto masterchain_block_id = create_block_id(query.masterchain_block_); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state_slice, block_id, masterchain_block_id, query.offset_, query.max_size_, std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise) { - promise.set_value(create_serialize_tl_object(proto_version(), proto_capabilities())); + VLOG(FULL_NODE_DEBUG) << "Got query getCapabilities from " << src; + promise.set_value( + create_serialize_tl_object(proto_version_major(), proto_version_minor(), 0)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, @@ -552,18 +637,104 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod promise.set_value(create_serialize_tl_object(R.move_as_ok())); } }); + VLOG(FULL_NODE_DEBUG) << "Got query getArchiveInfo " << query.masterchain_seqno_ << " from " << src; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_, - std::move(P)); + ShardIdFull{masterchainId}, std::move(P)); +} + +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getShardArchiveInfo &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_serialize_tl_object()); + } else { + promise.set_value(create_serialize_tl_object(R.move_as_ok())); + } + }); + ShardIdFull shard_prefix = create_shard_id(query.shard_prefix_); + VLOG(FULL_NODE_DEBUG) << "Got query getShardArchiveInfo " << query.masterchain_seqno_ << " " << shard_prefix.to_str() + << " from " << src; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_, + shard_prefix, std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, td::Promise promise) { + VLOG(FULL_NODE_DEBUG) << "Got query getArchiveSlice " << query.archive_id_ << " " << query.offset_ << " " + << query.max_size_ << " from " << src; + if (query.max_size_ < 0 || query.max_size_ > (1 << 24)) { + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "invalid max_size")); + return; + } td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_slice, query.archive_id_, query.offset_, query.max_size_, std::move(promise)); } +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getOutMsgQueueProof &query, + td::Promise promise) { + std::vector blocks; + for (const auto &x : query.blocks_) { + BlockIdExt id = create_block_id(x); + if (!id.is_valid_ext()) { + promise.set_error(td::Status::Error("invalid block_id")); + return; + } + if (!shard_is_ancestor(shard_, id.shard_full())) { + promise.set_error(td::Status::Error("query in wrong overlay")); + return; + } + blocks.push_back(create_block_id(x)); + } + ShardIdFull dst_shard = create_shard_id(query.dst_shard_); + if (!dst_shard.is_valid_ext()) { + promise.set_error(td::Status::Error("invalid shard")); + return; + } + block::ImportedMsgQueueLimits limits{(td::uint32)query.limits_->max_bytes_, (td::uint32)query.limits_->max_msgs_}; + if (limits.max_msgs > 512) { + promise.set_error(td::Status::Error("max_msgs is too big")); + return; + } + if (limits.max_bytes > (1 << 21)) { + promise.set_error(td::Status::Error("max_bytes is too big")); + return; + } + FLOG(DEBUG) { + sb << "Got query getOutMsgQueueProof to shard " << dst_shard.to_str() << " from blocks"; + for (const BlockIdExt &id : blocks) { + sb << " " << id.id.to_str(); + } + sb << " from " << src; + }; + td::actor::send_closure( + full_node_, &FullNode::get_out_msg_queue_query_token, + [=, manager = validator_manager_, blocks = std::move(blocks), + promise = std::move(promise)](td::Result> R) mutable { + TRY_RESULT_PROMISE(promise, token, std::move(R)); + auto P = + td::PromiseCreator::lambda([promise = std::move(promise), token = std::move(token)]( + td::Result> R) mutable { + if (R.is_error()) { + promise.set_result(create_serialize_tl_object()); + } else { + promise.set_result(serialize_tl_object(R.move_as_ok(), true)); + } + }); + td::actor::create_actor("buildqueueproof", dst_shard, std::move(blocks), limits, manager, + std::move(P)) + .release(); + }); +} + void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise) { + if (!active_) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_message, src, adnl_id_, overlay_id_, + create_serialize_tl_object()); + promise.set_error(td::Status::Error("shard is inactive")); + return; + } auto B = fetch_tl_object(std::move(query), true); if (B.is_error()) { promise.set_error(td::Status::Error(ErrorCode::protoviolation, "cannot parse tonnode query")); @@ -572,6 +743,16 @@ void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice ton_api::downcast_call(*B.move_as_ok().get(), [&](auto &obj) { this->process_query(src, obj, std::move(promise)); }); } +void FullNodeShardImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) { + auto B = fetch_tl_object(std::move(data), true); + if (B.is_error()) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Got tonNode.forgetPeer from " << src; + neighbours_.erase(src); + td::actor::send_closure(overlays_, &overlay::Overlays::forget_peer, adnl_id_, overlay_id_, src); +} + void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query) { td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_ihr_message, std::move(query.message_->data_)); @@ -579,43 +760,72 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ih void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query) { td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_external_message, - std::move(query.message_->data_)); + std::move(query.message_->data_), 0); } void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block, - create_block_id(query.block_->block_), query.block_->cc_seqno_, - std::move(query.block_->data_)); + BlockIdExt block_id = create_block_id(query.block_->block_); + VLOG(FULL_NODE_DEBUG) << "Received newShardBlockBroadcast from " << src << ": " << block_id.to_str(); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_shard_block, block_id, + query.block_->cc_seqno_, std::move(query.block_->data_)); +} + +void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeShardImpl::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressed &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeShardImpl::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + BlockIdExt block_id; + CatchainSeqno cc_seqno; + td::uint32 validator_set_hash; + td::BufferSlice data; + auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, + overlay::Overlays::max_fec_broadcast_size()); + if (data.size() > FullNode::max_block_size()) { + VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; + return; + } + if (td::sha256_bits256(data.as_slice()) != block_id.file_hash) { + VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; + return; + } + VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate from " << src << ": " << block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, + validator_set_hash, std::move(data)); } void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { - std::vector signatures; - for (auto &sig : query.signatures_) { - signatures.emplace_back(BlockSignature{sig->who_, std::move(sig->signature_)}); + process_block_broadcast(src, query); +} + +void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query) { + process_block_broadcast(src, query); +} + +void FullNodeShardImpl::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); + if (B.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << B.move_as_error(); + return; } - - BlockIdExt block_id = create_block_id(query.id_); - BlockBroadcast B{block_id, - std::move(signatures), - static_cast(query.catchain_seqno_), - static_cast(query.validator_set_hash_), - std::move(query.data_), - std::move(query.proof_)}; - - auto P = td::PromiseCreator::lambda([](td::Result R) { - if (R.is_error()) { - if (R.error().code() == ErrorCode::notready) { - LOG(DEBUG) << "dropped broadcast: " << R.move_as_error(); - } else { - LOG(INFO) << "dropped broadcast: " << R.move_as_error(); - } - } - }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(B), - std::move(P)); + //if (!shard_is_ancestor(shard_, block_id.shard_full())) { + // LOG(FULL_NODE_WARNING) << "dropping block broadcast: shard mismatch. overlay=" << shard_.to_str() + // << " block=" << block_id.to_str(); + // return; + //} + VLOG(FULL_NODE_DEBUG) << "Received block broadcast from " << src << ": " << B.ok().block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); } void FullNodeShardImpl::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { + if (!active_) { + return; + } auto B = fetch_tl_object(std::move(broadcast), true); if (B.is_error()) { return; @@ -641,6 +851,9 @@ void FullNodeShardImpl::send_ihr_message(td::BufferSlice data) { } void FullNodeShardImpl::send_external_message(td::BufferSlice data) { + if (opts_.config_.ext_messages_broadcast_disabled_) { + return; + } if (!client_.empty()) { td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "send_ext_query", create_serialize_tl_object_suffix( @@ -653,6 +866,11 @@ void FullNodeShardImpl::send_external_message(td::BufferSlice data) { }); return; } + td::Bits256 hash = td::sha256_bits256(data); + if (processed_ext_msg_broadcasts_.count(hash)) { + return; + } + my_ext_msg_broadcasts_.insert(hash); auto B = create_serialize_tl_object( create_tl_object(std::move(data))); if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) { @@ -669,6 +887,7 @@ void FullNodeShardImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno UNREACHABLE(); return; } + VLOG(FULL_NODE_DEBUG) << "Sending newShardBlockBroadcast: " << block_id.to_str(); auto B = create_serialize_tl_object( create_tl_object(create_tl_block_id(block_id), cc_seqno, std::move(data))); if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) { @@ -680,26 +899,42 @@ void FullNodeShardImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno } } +void FullNodeShardImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) { + if (!client_.empty()) { + UNREACHABLE(); + return; + } + auto B = + serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate: " << block_id.to_str(); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, adnl_id_, overlay_id_, local_id_, + overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); +} + void FullNodeShardImpl::send_broadcast(BlockBroadcast broadcast) { if (!client_.empty()) { UNREACHABLE(); return; } - std::vector> sigs; - for (auto &sig : broadcast.signatures) { - sigs.emplace_back(create_tl_object(sig.node, sig.signature.clone())); + VLOG(FULL_NODE_DEBUG) << "Sending block broadcast in private overlay: " << broadcast.block_id.to_str(); + auto B = serialize_block_broadcast(broadcast, false); // compression_enabled = false + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); + return; } - auto B = create_serialize_tl_object( - create_tl_block_id(broadcast.block_id), broadcast.catchain_seqno, broadcast.validator_set_hash, std::move(sigs), - broadcast.proof.clone(), broadcast.data.clone()); td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, adnl_id_, overlay_id_, local_id_, - overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); + overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); } void FullNodeShardImpl::download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { auto &b = choose_neighbour(); - if (!b.adnl_id.is_zero() && b.proto_version >= 1) { + if (!b.adnl_id.is_zero() && b.version_major >= 1) { VLOG(FULL_NODE_DEBUG) << "new block download"; td::actor::create_actor("downloadreq", id, adnl_id_, overlay_id_, b.adnl_id, priority, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, @@ -724,9 +959,10 @@ void FullNodeShardImpl::download_zero_state(BlockIdExt id, td::uint32 priority, void FullNodeShardImpl::download_persistent_state(BlockIdExt id, BlockIdExt masterchain_block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { + auto &b = choose_neighbour(); td::actor::create_actor(PSTRING() << "downloadstatereq" << id.id.to_str(), id, masterchain_block_id, - adnl_id_, overlay_id_, adnl::AdnlNodeIdShort::zero(), priority, timeout, - validator_manager_, rldp_, overlays_, adnl_, client_, std::move(promise)) + adnl_id_, overlay_id_, b.adnl_id, priority, timeout, validator_manager_, + rldp2_, overlays_, adnl_, client_, std::move(promise)) .release(); } @@ -743,7 +979,7 @@ void FullNodeShardImpl::download_block_proof_link(BlockIdExt block_id, td::uint3 td::Promise promise) { auto &b = choose_neighbour(); td::actor::create_actor("downloadproofreq", block_id, true, false, adnl_id_, overlay_id_, - adnl::AdnlNodeIdShort::zero(), priority, timeout, validator_manager_, rldp_, + b.adnl_id, priority, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise))) .release(); } @@ -751,21 +987,62 @@ void FullNodeShardImpl::download_block_proof_link(BlockIdExt block_id, td::uint3 void FullNodeShardImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) { auto &b = choose_neighbour(); - td::actor::create_actor("next", block_id, 16, adnl_id_, overlay_id_, adnl::AdnlNodeIdShort::zero(), + td::actor::create_actor("next", block_id, 16, adnl_id_, overlay_id_, b.adnl_id, 1, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise))) .release(); } -void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) { +void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) { auto &b = choose_neighbour(); td::actor::create_actor( - "archive", masterchain_seqno, std::move(tmp_dir), adnl_id_, overlay_id_, adnl::AdnlNodeIdShort::zero(), timeout, - validator_manager_, rldp_, overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise))) + "archive", masterchain_seqno, shard_prefix, std::move(tmp_dir), adnl_id_, overlay_id_, b.adnl_id, timeout, + validator_manager_, rldp2_, overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise))) .release(); } +void FullNodeShardImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise) { + // TODO: maybe more complex download (like other requests here) + auto &b = choose_neighbour(); + if (b.adnl_id == adnl::AdnlNodeIdShort::zero()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "no nodes")); + return; + } + std::vector> blocks_tl; + for (const BlockIdExt &id : blocks) { + blocks_tl.push_back(create_tl_block_id(id)); + } + td::BufferSlice query = create_serialize_tl_object( + create_tl_shard_id(dst_shard), std::move(blocks_tl), + create_tl_object(limits.max_bytes, limits.max_msgs)); + + auto P = td::PromiseCreator::lambda( + [=, promise = std::move(promise), blocks = std::move(blocks)](td::Result R) mutable { + if (R.is_error()) { + promise.set_result(R.move_as_error()); + return; + } + TRY_RESULT_PROMISE(promise, f, fetch_tl_object(R.move_as_ok(), true)); + ton_api::downcast_call( + *f, td::overloaded( + [&](ton_api::tonNode_outMsgQueueProofEmpty &x) { + promise.set_error(td::Status::Error("node doesn't have this block")); + }, + [&](ton_api::tonNode_outMsgQueueProof &x) { + delay_action( + [=, promise = std::move(promise), blocks = std::move(blocks), x = std::move(x)]() mutable { + promise.set_result(OutMsgQueueProof::fetch(dst_shard, blocks, limits, x)); + }, + td::Timestamp::now()); + })); + }); + td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, b.adnl_id, adnl_id_, overlay_id_, + "get_msg_queue", std::move(P), timeout, std::move(query), 1 << 22, rldp_); +} + void FullNodeShardImpl::set_handle(BlockHandle handle, td::Promise promise) { CHECK(!handle_); handle_ = std::move(handle); @@ -799,10 +1076,16 @@ void FullNodeShardImpl::alarm() { update_certificate_at_ = td::Timestamp::never(); } } + if (cleanup_processed_ext_msg_at_ && cleanup_processed_ext_msg_at_.is_in_past()) { + processed_ext_msg_broadcasts_.clear(); + my_ext_msg_broadcasts_.clear(); + cleanup_processed_ext_msg_at_ = td::Timestamp::in(60.0); + } alarm_timestamp().relax(sync_completed_at_); alarm_timestamp().relax(update_certificate_at_); alarm_timestamp().relax(reload_neighbours_at_); alarm_timestamp().relax(ping_neighbours_at_); + alarm_timestamp().relax(cleanup_processed_ext_msg_at_); } void FullNodeShardImpl::start_up() { @@ -819,11 +1102,15 @@ void FullNodeShardImpl::start_up() { reload_neighbours_at_ = td::Timestamp::now(); ping_neighbours_at_ = td::Timestamp::now(); - alarm_timestamp().relax(reload_neighbours_at_); - alarm_timestamp().relax(ping_neighbours_at_); + cleanup_processed_ext_msg_at_ = td::Timestamp::now(); + alarm_timestamp().relax(td::Timestamp::now()); } } +void FullNodeShardImpl::tear_down() { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); +} + void FullNodeShardImpl::sign_new_certificate(PublicKeyHash sign_by) { if (sign_by.is_zero()) { return; @@ -971,15 +1258,19 @@ const Neighbour &FullNodeShardImpl::choose_neighbour() const { return Neighbour::zero; } + double min_unreliability = 1e9; + for (auto &x : neighbours_) { + min_unreliability = std::min(min_unreliability, x.second.unreliability); + } const Neighbour *best = nullptr; td::uint32 sum = 0; for (auto &x : neighbours_) { - td::uint32 unr = static_cast(x.second.unreliability); + auto unr = static_cast(x.second.unreliability - min_unreliability); - if (x.second.proto_version < proto_version()) { + if (x.second.version_major < proto_version_major()) { unr += 4; - } else if (x.second.proto_version == proto_version() && x.second.capabilities < proto_capabilities()) { + } else if (x.second.version_major == proto_version_major() && x.second.version_minor < proto_version_minor()) { unr += 2; } @@ -993,7 +1284,10 @@ const Neighbour &FullNodeShardImpl::choose_neighbour() const { } } } - return best ? *best : Neighbour::zero; + if (best) { + return *best; + } + return Neighbour::zero; } void FullNodeShardImpl::update_neighbour_stats(adnl::AdnlNodeIdShort adnl_id, double t, bool success) { @@ -1016,7 +1310,7 @@ void FullNodeShardImpl::got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id if (F.is_error()) { it->second.query_failed(); } else { - it->second.update_proto_version(*F.move_as_ok().get()); + it->second.update_proto_version(*F.ok()); it->second.query_success(t); } } @@ -1045,7 +1339,7 @@ void FullNodeShardImpl::ping_neighbours() { td::Time::now() - start_time, R.move_as_ok()); } }); - auto q = create_serialize_tl_object(); + td::BufferSlice q = create_serialize_tl_object(); td::actor::send_closure(overlays_, &overlay::Overlays::send_query, it->first, adnl_id_, overlay_id_, "get_prepare_block", std::move(P), td::Timestamp::in(1.0), std::move(q)); @@ -1055,12 +1349,32 @@ void FullNodeShardImpl::ping_neighbours() { } } +void FullNodeShardImpl::get_stats_extra(td::Promise promise) { + auto res = create_tl_object(); + res->shard_ = shard_.to_str(); + res->active_ = active_; + for (const auto &p : neighbours_) { + const auto &n = p.second; + auto f = create_tl_object(); + f->id_ = n.adnl_id.bits256_value().to_hex(); + f->verison_major_ = n.version_major; + f->version_minor_ = n.version_minor; + f->flags_ = n.flags; + f->roundtrip_ = n.roundtrip; + f->unreliability_ = n.unreliability; + res->neighbours_.push_back(std::move(f)); + } + promise.set_result(td::json_encode(td::ToJson(*res), true)); +} + FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId rldp, + FileHash zero_state_file_hash, FullNodeOptions opts, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client) + td::actor::ActorId client, + td::actor::ActorId full_node, bool active) : shard_(shard) , local_id_(local_id) , adnl_id_(adnl_id) @@ -1068,18 +1382,24 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, , keyring_(keyring) , adnl_(adnl) , rldp_(rldp) + , rldp2_(rldp2) , overlays_(overlays) , validator_manager_(validator_manager) - , client_(client) { + , client_(client) + , full_node_(full_node) + , active_(active) + , opts_(opts) { } td::actor::ActorOwn FullNodeShard::create( ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, - td::actor::ActorId validator_manager, td::actor::ActorId client) { - return td::actor::create_actor("tonnode", shard, local_id, adnl_id, zero_state_file_hash, keyring, - adnl, rldp, overlays, validator_manager, client); + FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId rldp2, + td::actor::ActorId overlays, td::actor::ActorId validator_manager, + td::actor::ActorId client, td::actor::ActorId full_node, bool active) { + return td::actor::create_actor(PSTRING() << "tonnode" << shard.to_str(), shard, local_id, adnl_id, + zero_state_file_hash, opts, keyring, adnl, rldp, rldp2, overlays, + validator_manager, client, full_node, active); } } // namespace fullnode diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index c1712baf..5898db80 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -36,15 +36,20 @@ class FullNodeShard : public td::actor::Actor { virtual ShardIdFull get_shard_full() const = 0; virtual void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; + virtual void set_active(bool active) = 0; + virtual void set_config(FullNodeConfig config) = 0; virtual void send_ihr_message(td::BufferSlice data) = 0; virtual void send_external_message(td::BufferSlice data) = 0; virtual void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; + virtual void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) = 0; virtual void send_broadcast(BlockBroadcast broadcast) = 0; - virtual void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) = 0; - virtual void import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) = 0; - + virtual void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, + td::Promise promise) = 0; + virtual void import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, + td::Promise promise) = 0; virtual void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; @@ -59,8 +64,11 @@ class FullNodeShard : public td::actor::Actor { td::Promise promise) = 0; virtual void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) = 0; - virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) = 0; + virtual void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) = 0; + virtual void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise) = 0; virtual void set_handle(BlockHandle handle, td::Promise promise) = 0; @@ -68,9 +76,10 @@ class FullNodeShard : public td::actor::Actor { static td::actor::ActorOwn create( ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, - td::actor::ActorId validator_manager, td::actor::ActorId client); + FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId rldp2, + td::actor::ActorId overlays, td::actor::ActorId validator_manager, + td::actor::ActorId client, td::actor::ActorId full_node, bool active); }; } // namespace fullnode diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index a2dd5cc4..fb3eef76 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -18,9 +18,11 @@ */ #pragma once +#include "auto/tl/ton_api.h" #include "full-node-shard.h" #include "td/actor/PromiseFuture.h" #include "td/utils/port/Poll.h" +#include namespace ton { @@ -30,16 +32,17 @@ namespace fullnode { struct Neighbour { adnl::AdnlNodeIdShort adnl_id; - td::uint32 proto_version = 0; - td::uint64 capabilities = 0; + td::uint32 version_major = 0; + td::uint32 version_minor = 0; + td::uint32 flags = 0; double roundtrip = 0; double roundtrip_relax_at = 0; double roundtrip_weight = 0; double unreliability = 0; - Neighbour(adnl::AdnlNodeIdShort adnl_id) : adnl_id(std::move(adnl_id)) { + explicit Neighbour(adnl::AdnlNodeIdShort adnl_id) : adnl_id(std::move(adnl_id)) { } - void update_proto_version(const ton_api::tonNode_capabilities &q); + void update_proto_version(ton_api::tonNode_capabilities &q); void query_success(double t); void query_failed(); void update_roundtrip(double t); @@ -62,11 +65,11 @@ class FullNodeShardImpl : public FullNodeShard { static constexpr td::uint32 download_next_priority() { return 1; } - static constexpr td::uint32 proto_version() { - return 2; + static constexpr td::uint32 proto_version_major() { + return 3; } - static constexpr td::uint64 proto_capabilities() { - return 1; + static constexpr td::uint32 proto_version_minor() { + return 0; } static constexpr td::uint32 max_neighbours() { return 16; @@ -80,10 +83,12 @@ class FullNodeShardImpl : public FullNodeShard { void create_overlay(); void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; + void set_active(bool active) override; + + void set_config(FullNodeConfig config) override { + opts_.config_ = config; + } - //td::Result fetch_block(td::BufferSlice data); - void prevalidate_block(BlockIdExt block_id, td::BufferSlice data, td::BufferSlice proof, - td::Promise promise); void try_get_next_block(td::Timestamp timestamp, td::Promise promise); void got_next_block(td::Result block); void get_next_block(); @@ -130,22 +135,37 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getShardArchiveInfo &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, td::Promise promise); - // void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query, - // td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getOutMsgQueueProof &query, + td::Promise promise); void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise); + void receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); + void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); + + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + void receive_broadcast(PublicKeyHash src, td::BufferSlice query); void check_broadcast(PublicKeyHash src, td::BufferSlice query, td::Promise promise); + void get_stats_extra(td::Promise promise); + void remove_neighbour(adnl::AdnlNodeIdShort id); void send_ihr_message(td::BufferSlice data) override; void send_external_message(td::BufferSlice data) override; void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override; void send_broadcast(BlockBroadcast broadcast) override; void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, @@ -161,12 +181,16 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise) override; void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) override; - void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) override; + void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override; + void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise) override; void set_handle(BlockHandle handle, td::Promise promise) override; void start_up() override; + void tear_down() override; void alarm() override; void update_validators(std::vector public_key_hashes, PublicKeyHash local_hash) override; @@ -198,11 +222,12 @@ class FullNodeShardImpl : public FullNodeShard { } FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, td::actor::ActorId keyring, + FileHash zero_state_file_hash, FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, - td::actor::ActorId overlays, + td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client); + td::actor::ActorId client, td::actor::ActorId full_node, + bool active); private: bool use_new_download() const { @@ -220,9 +245,11 @@ class FullNodeShardImpl : public FullNodeShard { td::actor::ActorId keyring_; td::actor::ActorId adnl_; td::actor::ActorId rldp_; + td::actor::ActorId rldp2_; td::actor::ActorId overlays_; td::actor::ActorId validator_manager_; td::actor::ActorId client_; + td::actor::ActorId full_node_; td::uint32 attempt_ = 0; @@ -239,6 +266,14 @@ class FullNodeShardImpl : public FullNodeShard { td::Timestamp reload_neighbours_at_; td::Timestamp ping_neighbours_at_; adnl::AdnlNodeIdShort last_pinged_neighbour_ = adnl::AdnlNodeIdShort::zero(); + + bool active_; + + FullNodeOptions opts_; + + std::set my_ext_msg_broadcasts_; + std::set processed_ext_msg_broadcasts_; + td::Timestamp cleanup_processed_ext_msg_at_; }; } // namespace fullnode diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 6606f215..e1951c36 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -17,9 +17,13 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "full-node.hpp" -#include "ton/ton-shard.h" #include "ton/ton-io.hpp" #include "td/actor/MultiPromise.h" +#include "full-node.h" +#include "common/delay.h" +#include "impl/out-msg-queue-proof.hpp" +#include "td/utils/Random.h" +#include "ton/ton-tl.hpp" namespace ton { @@ -27,6 +31,8 @@ namespace validator { namespace fullnode { +static const double INACTIVE_SHARD_TTL = (double)overlay::Overlays::overlay_peer_ttl() + 60.0; + void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise promise) { if (local_keys_.count(key)) { promise.set_value(td::Unit()); @@ -34,6 +40,10 @@ void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise pr } local_keys_.insert(key); + create_private_block_overlay(key); + for (auto &p : custom_overlays_) { + update_custom_overlay(p.second); + } if (!sign_cert_by_.is_zero()) { promise.set_value(td::Unit()); @@ -47,7 +57,9 @@ void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise pr } for (auto &shard : shards_) { - td::actor::send_closure(shard.second, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + if (!shard.second.actor.empty()) { + td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + } } promise.set_value(td::Unit()); } @@ -58,6 +70,12 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise pr return; } local_keys_.erase(key); + private_block_overlays_.erase(key); + update_validator_telemetry_collector(); + for (auto &p : custom_overlays_) { + update_custom_overlay(p.second); + } + if (sign_cert_by_ != key) { promise.set_value(td::Unit()); return; @@ -71,30 +89,34 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise pr } for (auto &shard : shards_) { - td::actor::send_closure(shard.second, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + if (!shard.second.actor.empty()) { + td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + } } promise.set_value(td::Unit()); } -void FullNodeImpl::sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, - td::uint32 expiry_at, td::uint32 max_size, - td::Promise promise) { - auto it = shards_.find(shard_id); - if(it == shards_.end()) { - promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); - return; - } - td::actor::send_closure(it->second, &FullNodeShard::sign_overlay_certificate, signed_key, expiry_at, max_size, std::move(promise)); +void FullNodeImpl::sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, + td::uint32 max_size, td::Promise promise) { + auto it = shards_.find(shard_id); + if(it == shards_.end() || it->second.actor.empty()) { + promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); + return; + } + td::actor::send_closure(it->second.actor, &FullNodeShard::sign_overlay_certificate, signed_key, expiry_at, max_size, + std::move(promise)); } void FullNodeImpl::import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) { - auto it = shards_.find(shard_id); - if(it == shards_.end()) { - promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); - } - td::actor::send_closure(it->second, &FullNodeShard::import_overlay_certificate, signed_key, cert, std::move(promise)); + auto it = shards_.find(shard_id); + if(it == shards_.end() || it->second.actor.empty()) { + promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); + return; + } + td::actor::send_closure(it->second.actor, &FullNodeShard::import_overlay_certificate, signed_key, cert, + std::move(promise)); } void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) { @@ -105,9 +127,59 @@ void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) { + if (params.nodes_.empty()) { + promise.set_error(td::Status::Error("list of nodes is empty")); + return; + } + std::string name = params.name_; + if (custom_overlays_.count(name)) { + promise.set_error(td::Status::Error(PSTRING() << "duplicate custom overlay name \"" << name << "\"")); + return; + } + VLOG(FULL_NODE_WARNING) << "Adding custom overlay \"" << name << "\", " << params.nodes_.size() << " nodes"; + auto &p = custom_overlays_[name]; + p.params_ = std::move(params); + update_custom_overlay(p); + promise.set_result(td::Unit()); +} + +void FullNodeImpl::del_custom_overlay(std::string name, td::Promise promise) { + auto it = custom_overlays_.find(name); + if (it == custom_overlays_.end()) { + promise.set_error(td::Status::Error(PSTRING() << "no such overlay \"" << name << "\"")); + return; + } + custom_overlays_.erase(it); + promise.set_result(td::Unit()); } void FullNodeImpl::initial_read_complete(BlockHandle top_handle) { @@ -116,31 +188,84 @@ void FullNodeImpl::initial_read_complete(BlockHandle top_handle) { td::actor::send_closure(SelfId, &FullNodeImpl::sync_completed); }); auto it = shards_.find(ShardIdFull{masterchainId}); - CHECK(it != shards_.end()); - td::actor::send_closure(it->second, &FullNodeShard::set_handle, top_handle, std::move(P)); + CHECK(it != shards_.end() && !it->second.actor.empty()); + td::actor::send_closure(it->second.actor, &FullNodeShard::set_handle, top_handle, std::move(P)); } -void FullNodeImpl::add_shard(ShardIdFull shard) { - while (true) { - if (shards_.count(shard) == 0) { - shards_.emplace(shard, FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, keyring_, adnl_, - rldp_, overlays_, validator_manager_, client_)); - if (all_validators_.size() > 0) { - td::actor::send_closure(shards_[shard], &FullNodeShard::update_validators, all_validators_, sign_cert_by_); +void FullNodeImpl::on_new_masterchain_block(td::Ref state, std::set shards_to_monitor) { + CHECK(shards_to_monitor.count(ShardIdFull(masterchainId))); + bool join_all_overlays = !sign_cert_by_.is_zero(); + std::set all_shards; + std::set new_active; + all_shards.insert(ShardIdFull(masterchainId)); + std::set workchains; + wc_monitor_min_split_ = state->monitor_min_split_depth(basechainId); + auto cut_shard = [&](ShardIdFull shard) -> ShardIdFull { + return wc_monitor_min_split_ < shard.pfx_len() ? shard_prefix(shard, wc_monitor_min_split_) : shard; + }; + for (auto &info : state->get_shards()) { + workchains.insert(info->shard().workchain); + ShardIdFull shard = cut_shard(info->shard()); + while (true) { + all_shards.insert(shard); + if (shard.pfx_len() == 0) { + break; } + shard = shard_parent(shard); + } + } + for (const auto &[wc, winfo] : state->get_workchain_list()) { + if (!workchains.contains(wc) && winfo->active && winfo->enabled_since <= state->get_unix_time()) { + all_shards.insert(ShardIdFull(wc)); + } + } + for (ShardIdFull shard : shards_to_monitor) { + shard = cut_shard(shard); + while (true) { + new_active.insert(shard); + if (shard.pfx_len() == 0) { + break; + } + shard = shard_parent(shard); + } + } + + for (auto it = shards_.begin(); it != shards_.end(); ) { + if (all_shards.contains(it->first)) { + ++it; } else { - break; + it = shards_.erase(it); } - if (shard.shard == shardIdAll) { - break; + } + for (ShardIdFull shard : all_shards) { + bool active = new_active.contains(shard); + bool overlay_exists = !shards_[shard].actor.empty(); + if (active || join_all_overlays || overlay_exists) { + update_shard_actor(shard, active); + } + } + + for (auto &[_, shard_info] : shards_) { + if (!shard_info.active && shard_info.delete_at && shard_info.delete_at.is_in_past() && !join_all_overlays) { + shard_info.actor = {}; + shard_info.delete_at = td::Timestamp::never(); } - shard = shard_parent(shard); } } -void FullNodeImpl::del_shard(ShardIdFull shard) { - LOG(FATAL) << "deleting shards not implemented: shard=" << shard; - shards_.erase(shard); +void FullNodeImpl::update_shard_actor(ShardIdFull shard, bool active) { + ShardInfo &info = shards_[shard]; + if (info.actor.empty()) { + info.actor = FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, opts_, keyring_, adnl_, rldp_, + rldp2_, overlays_, validator_manager_, client_, actor_id(this), active); + if (!all_validators_.empty()) { + td::actor::send_closure(info.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + } + } else if (info.active != active) { + td::actor::send_closure(info.actor, &FullNodeShard::set_active, active); + } + info.active = active; + info.delete_at = active ? td::Timestamp::never() : td::Timestamp::in(INACTIVE_SHARD_TTL); } void FullNodeImpl::sync_completed() { @@ -148,7 +273,7 @@ void FullNodeImpl::sync_completed() { } void FullNodeImpl::send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) { - auto shard = get_shard(ShardIdFull{masterchainId}); + auto shard = get_shard(dst); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping OUT ihr message to unknown shard"; return; @@ -162,25 +287,67 @@ void FullNodeImpl::send_ext_message(AccountIdPrefixFull dst, td::BufferSlice dat VLOG(FULL_NODE_WARNING) << "dropping OUT ext message to unknown shard"; return; } + for (auto &[_, private_overlay] : custom_overlays_) { + if (private_overlay.params_.send_shard(dst.as_leaf_shard())) { + for (auto &[local_id, actor] : private_overlay.actors_) { + if (private_overlay.params_.msg_senders_.contains(local_id)) { + td::actor::send_closure(actor, &FullNodeCustomOverlay::send_external_message, data.clone()); + } + } + } + } td::actor::send_closure(shard, &FullNodeShard::send_external_message, std::move(data)); } void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { + auto shard = get_shard(ShardIdFull{masterchainId}); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; + return; + } + if (!private_block_overlays_.empty()) { + td::actor::send_closure(private_block_overlays_.begin()->second, + &FullNodePrivateBlockOverlay::send_shard_block_info, block_id, cc_seqno, data.clone()); + } + td::actor::send_closure(shard, &FullNodeShard::send_shard_block_info, block_id, cc_seqno, std::move(data)); +} + +void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) { + send_block_candidate_broadcast_to_custom_overlays(block_id, cc_seqno, validator_set_hash, data); auto shard = get_shard(ShardIdFull{masterchainId, shardIdAll}); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; return; } - td::actor::send_closure(shard, &FullNodeShard::send_shard_block_info, block_id, cc_seqno, std::move(data)); + if (!private_block_overlays_.empty()) { + td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_block_candidate, + block_id, cc_seqno, validator_set_hash, data.clone()); + } + if (broadcast_block_candidates_in_public_overlay_) { + td::actor::send_closure(shard, &FullNodeShard::send_block_candidate, block_id, cc_seqno, validator_set_hash, + std::move(data)); + } } -void FullNodeImpl::send_broadcast(BlockBroadcast broadcast) { - auto shard = get_shard(ShardIdFull{masterchainId}); +void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, int mode) { + if (mode & broadcast_mode_custom) { + send_block_broadcast_to_custom_overlays(broadcast); + } + auto shard = get_shard(broadcast.block_id.shard_full()); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; return; } - td::actor::send_closure(shard, &FullNodeShard::send_broadcast, std::move(broadcast)); + if (mode & broadcast_mode_private_block) { + if (!private_block_overlays_.empty()) { + td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast, + broadcast.clone()); + } + } + if (mode & broadcast_mode_public) { + td::actor::send_closure(shard, &FullNodeShard::send_broadcast, std::move(broadcast)); + } } void FullNodeImpl::download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, @@ -251,36 +418,67 @@ void FullNodeImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeou td::actor::send_closure(shard, &FullNodeShard::get_next_key_blocks, block_id, timeout, std::move(promise)); } -void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) { - auto shard = get_shard(ShardIdFull{masterchainId}); +void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) { + auto shard = get_shard(shard_prefix); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping download archive query to unknown shard"; + promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); + return; + } CHECK(!shard.empty()); - td::actor::send_closure(shard, &FullNodeShard::download_archive, masterchain_seqno, std::move(tmp_dir), timeout, - std::move(promise)); + td::actor::send_closure(shard, &FullNodeShard::download_archive, masterchain_seqno, shard_prefix, std::move(tmp_dir), + timeout, std::move(promise)); +} + +void FullNodeImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise) { + if (blocks.empty()) { + promise.set_value({}); + return; + } + // All blocks are expected to have the same minsplit shard prefix + auto shard = get_shard(blocks[0].shard_full()); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping download msg queue query to unknown shard"; + promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); + return; + } + td::actor::send_closure(shard, &FullNodeShard::download_out_msg_queue_proof, dst_shard, std::move(blocks), limits, + timeout, std::move(promise)); } td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { - add_shard(ShardIdFull{shard.workchain, shardIdAll}); - while (shards_.count(shard) == 0) { - if (shard.shard == shardIdAll) { - return td::actor::ActorId{}; - } - shard = shard_parent(shard); + if (shard.is_masterchain()) { + return shards_[ShardIdFull{masterchainId}].actor.get(); } - return shards_[shard].get(); + if (shard.workchain != basechainId) { + return {}; + } + int pfx_len = shard.pfx_len(); + if (pfx_len > wc_monitor_min_split_) { + shard = shard_prefix(shard, wc_monitor_min_split_); + } + auto it = shards_.find(shard); + if (it != shards_.end()) { + update_shard_actor(shard, it->second.active); + return it->second.actor.get(); + } + + // Special case if shards_ was not yet initialized. + // This can happen briefly on node startup. + return shards_[ShardIdFull{masterchainId}].actor.get(); } td::actor::ActorId FullNodeImpl::get_shard(AccountIdPrefixFull dst) { - return get_shard(shard_prefix(dst, 60)); + return get_shard(shard_prefix(dst, max_shard_pfx_len)); } -void FullNodeImpl::got_key_block_proof(td::Ref proof) { - auto R = proof->get_key_block_config(); - R.ensure(); - auto config = R.move_as_ok(); - +void FullNodeImpl::got_key_block_config(td::Ref config) { PublicKeyHash l = PublicKeyHash::zero(); std::vector keys; + std::map current_validators; for (td::int32 i = -1; i <= 1; i++) { auto r = config->get_total_validator_set(i < 0 ? i : 1 - i); if (r.not_null()) { @@ -291,52 +489,31 @@ void FullNodeImpl::got_key_block_proof(td::Ref proof) { if (local_keys_.count(key)) { l = key; } - } - } - } - - if (keys == all_validators_) { - return; - } - - all_validators_ = keys; - sign_cert_by_ = l; - CHECK(all_validators_.size() > 0); - - for (auto &shard : shards_) { - td::actor::send_closure(shard.second, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); - } -} - -void FullNodeImpl::got_zero_block_state(td::Ref state) { - auto m = td::Ref{std::move(state)}; - - PublicKeyHash l = PublicKeyHash::zero(); - std::vector keys; - for (td::int32 i = -1; i <= 1; i++) { - auto r = m->get_total_validator_set(i < 0 ? i : 1 - i); - if (r.not_null()) { - auto vec = r->export_vector(); - for (auto &el : vec) { - auto key = ValidatorFullId{el.key}.compute_short_id(); - keys.push_back(key); - if (local_keys_.count(key)) { - l = key; + if (i == 1) { + current_validators[key] = adnl::AdnlNodeIdShort{el.addr.is_zero() ? key.bits256_value() : el.addr}; } } } } - if (keys == all_validators_) { - return; + if (current_validators != current_validators_) { + current_validators_ = std::move(current_validators); + update_private_overlays(); } + // Let's turn off this optimization, since keyblocks are rare enough to update on each keyblock + // if (keys == all_validators_) { + // return; + // } + all_validators_ = keys; sign_cert_by_ = l; CHECK(all_validators_.size() > 0); for (auto &shard : shards_) { - td::actor::send_closure(shard.second, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + if (!shard.second.actor.empty()) { + td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + } } } @@ -346,7 +523,9 @@ void FullNodeImpl::new_key_block(BlockHandle handle) { if (R.is_error()) { VLOG(FULL_NODE_WARNING) << "failed to get zero state: " << R.move_as_error(); } else { - td::actor::send_closure(SelfId, &FullNodeImpl::got_zero_block_state, R.move_as_ok()); + auto s = td::Ref{R.move_as_ok()}; + CHECK(s.not_null()); + td::actor::send_closure(SelfId, &FullNodeImpl::got_key_block_config, s->get_config_holder().move_as_ok()); } }); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_shard_state_from_db, handle, @@ -357,7 +536,8 @@ void FullNodeImpl::new_key_block(BlockHandle handle) { if (R.is_error()) { VLOG(FULL_NODE_WARNING) << "failed to get key block proof: " << R.move_as_error(); } else { - td::actor::send_closure(SelfId, &FullNodeImpl::got_key_block_proof, R.move_as_ok()); + td::actor::send_closure(SelfId, &FullNodeImpl::got_key_block_config, + R.ok()->get_key_block_config().move_as_ok()); } }); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_block_proof_link_from_db, handle, @@ -365,9 +545,64 @@ void FullNodeImpl::new_key_block(BlockHandle handle) { } } +void FullNodeImpl::send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) { + auto it = private_block_overlays_.find(key); + if (it == private_block_overlays_.end()) { + VLOG(FULL_NODE_INFO) << "Cannot send validator telemetry for " << key << " : no private block overlay"; + return; + } + td::actor::send_closure(it->second, &FullNodePrivateBlockOverlay::send_validator_telemetry, std::move(telemetry)); +} + +void FullNodeImpl::process_block_broadcast(BlockBroadcast broadcast) { + send_block_broadcast_to_custom_overlays(broadcast); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(broadcast), + [](td::Result R) { + if (R.is_error()) { + if (R.error().code() == ErrorCode::notready) { + LOG(DEBUG) << "dropped broadcast: " << R.move_as_error(); + } else { + LOG(INFO) << "dropped broadcast: " << R.move_as_error(); + } + } + }); +} + +void FullNodeImpl::process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) { + send_block_candidate_broadcast_to_custom_overlays(block_id, cc_seqno, validator_set_hash, data); + // ignore cc_seqno and validator_hash for now + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_block_candidate, block_id, + std::move(data)); +} + +void FullNodeImpl::get_out_msg_queue_query_token(td::Promise> promise) { + td::actor::send_closure(out_msg_queue_query_token_manager_, &TokenManager::get_token, 1, 0, td::Timestamp::in(10.0), + std::move(promise)); +} + +void FullNodeImpl::set_validator_telemetry_filename(std::string value) { + validator_telemetry_filename_ = std::move(value); + update_validator_telemetry_collector(); +} + +void FullNodeImpl::update_validator_telemetry_collector() { + if (validator_telemetry_filename_.empty() || private_block_overlays_.empty()) { + validator_telemetry_collector_key_ = PublicKeyHash::zero(); + return; + } + if (!private_block_overlays_.contains(validator_telemetry_collector_key_)) { + auto it = private_block_overlays_.begin(); + validator_telemetry_collector_key_ = it->first; + td::actor::send_closure(it->second, &FullNodePrivateBlockOverlay::collect_validator_telemetry, + validator_telemetry_filename_); + } +} + void FullNodeImpl::start_up() { + update_shard_actor(ShardIdFull{masterchainId}, true); if (local_id_.is_zero()) { - if(adnl_id_.is_zero()) { + if (adnl_id_.is_zero()) { auto pk = ton::PrivateKey{ton::privkeys::Ed25519::random()}; local_id_ = pk.compute_short_id(); @@ -381,11 +616,9 @@ void FullNodeImpl::start_up() { void initial_read_complete(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::initial_read_complete, handle); } - void add_shard(ShardIdFull shard) override { - td::actor::send_closure(id_, &FullNodeImpl::add_shard, shard); - } - void del_shard(ShardIdFull shard) override { - td::actor::send_closure(id_, &FullNodeImpl::del_shard, shard); + void on_new_masterchain_block(td::Ref state, std::set shards_to_monitor) override { + td::actor::send_closure(id_, &FullNodeImpl::on_new_masterchain_block, std::move(state), + std::move(shards_to_monitor)); } void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) override { td::actor::send_closure(id_, &FullNodeImpl::send_ihr_message, dst, std::move(data)); @@ -396,8 +629,13 @@ void FullNodeImpl::start_up() { void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override { td::actor::send_closure(id_, &FullNodeImpl::send_shard_block_info, block_id, cc_seqno, std::move(data)); } - void send_broadcast(BlockBroadcast broadcast) override { - td::actor::send_closure(id_, &FullNodeImpl::send_broadcast, std::move(broadcast)); + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override { + td::actor::send_closure(id_, &FullNodeImpl::send_block_candidate, block_id, cc_seqno, validator_set_hash, + std::move(data)); + } + void send_broadcast(BlockBroadcast broadcast, int mode) override { + td::actor::send_closure(id_, &FullNodeImpl::send_broadcast, std::move(broadcast), mode); } void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override { @@ -426,58 +664,206 @@ void FullNodeImpl::start_up() { td::Promise> promise) override { td::actor::send_closure(id_, &FullNodeImpl::get_next_key_blocks, block_id, timeout, std::move(promise)); } - void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) override { - td::actor::send_closure(id_, &FullNodeImpl::download_archive, masterchain_seqno, std::move(tmp_dir), timeout, - std::move(promise)); + void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override { + td::actor::send_closure(id_, &FullNodeImpl::download_archive, masterchain_seqno, shard_prefix, std::move(tmp_dir), + timeout, std::move(promise)); + } + void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise) override { + td::actor::send_closure(id_, &FullNodeImpl::download_out_msg_queue_proof, dst_shard, std::move(blocks), limits, + timeout, std::move(promise)); } void new_key_block(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::new_key_block, std::move(handle)); } + void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override { + td::actor::send_closure(id_, &FullNodeImpl::send_validator_telemetry, key, std::move(telemetry)); + } - Callback(td::actor::ActorId id) : id_(id) { + explicit Callback(td::actor::ActorId id) : id_(id) { } private: td::actor::ActorId id_; }; - auto P = td::PromiseCreator::lambda([](td::Unit R) {}); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::install_callback, - std::make_unique(actor_id(this)), std::move(P)); + std::make_unique(actor_id(this)), std::move(started_promise_)); +} + +void FullNodeImpl::update_private_overlays() { + for (auto &p : custom_overlays_) { + update_custom_overlay(p.second); + } + + private_block_overlays_.clear(); + update_validator_telemetry_collector(); + if (local_keys_.empty()) { + return; + } + for (const auto &key : local_keys_) { + create_private_block_overlay(key); + } +} + +void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { + CHECK(local_keys_.count(key)); + if (current_validators_.count(key)) { + std::vector nodes; + for (const auto &p : current_validators_) { + nodes.push_back(p.second); + } + private_block_overlays_[key] = td::actor::create_actor( + "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, opts_, keyring_, + adnl_, rldp_, rldp2_, overlays_, validator_manager_, actor_id(this)); + update_validator_telemetry_collector(); + } +} + +void FullNodeImpl::update_custom_overlay(CustomOverlayInfo &overlay) { + auto old_actors = std::move(overlay.actors_); + overlay.actors_.clear(); + CustomOverlayParams ¶ms = overlay.params_; + auto try_local_id = [&](const adnl::AdnlNodeIdShort &local_id) { + if (std::find(params.nodes_.begin(), params.nodes_.end(), local_id) != params.nodes_.end()) { + auto it = old_actors.find(local_id); + if (it != old_actors.end()) { + overlay.actors_[local_id] = std::move(it->second); + old_actors.erase(it); + } else { + overlay.actors_[local_id] = td::actor::create_actor( + "CustomOverlay", local_id, params, zero_state_file_hash_, opts_, keyring_, adnl_, rldp_, rldp2_, + overlays_, validator_manager_, actor_id(this)); + } + } + }; + try_local_id(adnl_id_); + for (const PublicKeyHash &local_key : local_keys_) { + auto it = current_validators_.find(local_key); + if (it != current_validators_.end()) { + try_local_id(it->second); + } + } +} + +void FullNodeImpl::send_block_broadcast_to_custom_overlays(const BlockBroadcast &broadcast) { + if (!custom_overlays_sent_broadcasts_.insert(broadcast.block_id).second) { + return; + } + custom_overlays_sent_broadcasts_lru_.push(broadcast.block_id); + if (custom_overlays_sent_broadcasts_lru_.size() > 256) { + custom_overlays_sent_broadcasts_.erase(custom_overlays_sent_broadcasts_lru_.front()); + custom_overlays_sent_broadcasts_lru_.pop(); + } + for (auto &[_, private_overlay] : custom_overlays_) { + if (private_overlay.params_.send_shard(broadcast.block_id.shard_full())) { + for (auto &[local_id, actor] : private_overlay.actors_) { + if (private_overlay.params_.block_senders_.contains(local_id)) { + td::actor::send_closure(actor, &FullNodeCustomOverlay::send_broadcast, broadcast.clone()); + } + } + } + } +} + +void FullNodeImpl::send_block_candidate_broadcast_to_custom_overlays(const BlockIdExt &block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, + const td::BufferSlice &data) { + // Same cache of sent broadcasts as in send_block_broadcast_to_custom_overlays + if (!custom_overlays_sent_broadcasts_.insert(block_id).second) { + return; + } + custom_overlays_sent_broadcasts_lru_.push(block_id); + if (custom_overlays_sent_broadcasts_lru_.size() > 256) { + custom_overlays_sent_broadcasts_.erase(custom_overlays_sent_broadcasts_lru_.front()); + custom_overlays_sent_broadcasts_lru_.pop(); + } + for (auto &[_, private_overlay] : custom_overlays_) { + if (private_overlay.params_.send_shard(block_id.shard_full())) { + for (auto &[local_id, actor] : private_overlay.actors_) { + if (private_overlay.params_.block_senders_.contains(local_id)) { + td::actor::send_closure(actor, &FullNodeCustomOverlay::send_block_candidate, block_id, cc_seqno, + validator_set_hash, data.clone()); + } + } + } + } } FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId dht, + FullNodeOptions opts, td::actor::ActorId keyring, + td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId rldp2, td::actor::ActorId dht, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, std::string db_root) + td::actor::ActorId client, std::string db_root, + td::Promise started_promise) : local_id_(local_id) , adnl_id_(adnl_id) , zero_state_file_hash_(zero_state_file_hash) , keyring_(keyring) , adnl_(adnl) , rldp_(rldp) + , rldp2_(rldp2) , dht_(dht) , overlays_(overlays) , validator_manager_(validator_manager) , client_(client) - , db_root_(db_root) { - add_shard(ShardIdFull{masterchainId}); + , db_root_(db_root) + , started_promise_(std::move(started_promise)) + , opts_(opts) { } -td::actor::ActorOwn FullNode::create(ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, - td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId rldp, - td::actor::ActorId dht, - td::actor::ActorId overlays, - td::actor::ActorId validator_manager, - td::actor::ActorId client, std::string db_root) { - return td::actor::create_actor("fullnode", local_id, adnl_id, zero_state_file_hash, keyring, adnl, rldp, - dht, overlays, validator_manager, client, db_root); +td::actor::ActorOwn FullNode::create( + ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeOptions opts, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId dht, + td::actor::ActorId overlays, td::actor::ActorId validator_manager, + td::actor::ActorId client, std::string db_root, td::Promise started_promise) { + return td::actor::create_actor("fullnode", local_id, adnl_id, zero_state_file_hash, opts, keyring, + adnl, rldp, rldp2, dht, overlays, validator_manager, client, db_root, + std::move(started_promise)); +} + +FullNodeConfig::FullNodeConfig(const tl_object_ptr &obj) + : ext_messages_broadcast_disabled_(obj->ext_messages_broadcast_disabled_) { +} + +tl_object_ptr FullNodeConfig::tl() const { + return create_tl_object(ext_messages_broadcast_disabled_); +} +bool FullNodeConfig::operator==(const FullNodeConfig &rhs) const { + return ext_messages_broadcast_disabled_ == rhs.ext_messages_broadcast_disabled_; +} +bool FullNodeConfig::operator!=(const FullNodeConfig &rhs) const { + return !(*this == rhs); +} + +bool CustomOverlayParams::send_shard(const ShardIdFull &shard) const { + return sender_shards_.empty() || + std::any_of(sender_shards_.begin(), sender_shards_.end(), + [&](const ShardIdFull &our_shard) { return shard_intersects(shard, our_shard); }); +} + +CustomOverlayParams CustomOverlayParams::fetch(const ton_api::engine_validator_customOverlay& f) { + CustomOverlayParams c; + c.name_ = f.name_; + for (const auto &node : f.nodes_) { + c.nodes_.emplace_back(node->adnl_id_); + if (node->msg_sender_) { + c.msg_senders_[adnl::AdnlNodeIdShort{node->adnl_id_}] = node->msg_sender_priority_; + } + if (node->block_sender_) { + c.block_senders_.emplace(node->adnl_id_); + } + } + for (const auto &shard : f.sender_shards_) { + c.sender_shards_.push_back(create_shard_id(shard)); + } + return c; } } // namespace fullnode diff --git a/validator/full-node.h b/validator/full-node.h index cdf39d6f..555082dc 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -27,6 +27,7 @@ #include "adnl/adnl.h" #include "rldp/rldp.h" +#include "rldp2/rldp.h" #include "dht/dht.h" #include "overlay/overlays.h" #include "validator/validator.h" @@ -44,6 +45,33 @@ constexpr int VERBOSITY_NAME(FULL_NODE_INFO) = verbosity_DEBUG; constexpr int VERBOSITY_NAME(FULL_NODE_DEBUG) = verbosity_DEBUG; constexpr int VERBOSITY_NAME(FULL_NODE_EXTRA_DEBUG) = verbosity_DEBUG + 1; +struct FullNodeConfig { + FullNodeConfig() = default; + FullNodeConfig(const tl_object_ptr& obj); + tl_object_ptr tl() const; + bool operator==(const FullNodeConfig& rhs) const; + bool operator!=(const FullNodeConfig& rhs) const; + + bool ext_messages_broadcast_disabled_ = false; +}; + +struct FullNodeOptions { + FullNodeConfig config_; + double public_broadcast_speed_multiplier_ = 1.0; + double private_broadcast_speed_multiplier_ = 1.0; +}; + +struct CustomOverlayParams { + std::string name_; + std::vector nodes_; + std::map msg_senders_; + std::set block_senders_; + std::vector sender_shards_; + + bool send_shard(const ShardIdFull& shard) const; + static CustomOverlayParams fetch(const ton_api::engine_validator_customOverlay& f); +}; + class FullNode : public td::actor::Actor { public: virtual ~FullNode() = default; @@ -61,6 +89,17 @@ class FullNode : public td::actor::Actor { td::Promise promise) = 0; virtual void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; + virtual void set_config(FullNodeConfig config) = 0; + + virtual void add_custom_overlay(CustomOverlayParams params, td::Promise promise) = 0; + virtual void del_custom_overlay(std::string name, td::Promise promise) = 0; + + virtual void process_block_broadcast(BlockBroadcast broadcast) = 0; + virtual void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) = 0; + virtual void get_out_msg_queue_query_token(td::Promise> promise) = 0; + + virtual void set_validator_telemetry_filename(std::string value) = 0; static constexpr td::uint32 max_block_size() { return 4 << 20; @@ -71,15 +110,14 @@ class FullNode : public td::actor::Actor { static constexpr td::uint64 max_state_size() { return 4ull << 30; } + enum { broadcast_mode_public = 1, broadcast_mode_private_block = 2, broadcast_mode_custom = 4 }; - static td::actor::ActorOwn create(ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, - td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId rldp, - td::actor::ActorId dht, - td::actor::ActorId overlays, - td::actor::ActorId validator_manager, - td::actor::ActorId client, std::string db_root); + static td::actor::ActorOwn create( + ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeOptions opts, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId dht, + td::actor::ActorId overlays, td::actor::ActorId validator_manager, + td::actor::ActorId client, std::string db_root, td::Promise started_promise); }; } // namespace fullnode diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 6d57f4a8..b4c79363 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -23,9 +23,12 @@ //#include "ton-node-slave.h" #include "interfaces/proof.h" #include "interfaces/shard.h" +#include "full-node-private-overlay.hpp" #include #include +#include +#include namespace ton { @@ -42,18 +45,19 @@ class FullNodeImpl : public FullNode { void add_permanent_key(PublicKeyHash key, td::Promise promise) override; void del_permanent_key(PublicKeyHash key, td::Promise promise) override; - void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, - td::uint32 expiry_at, td::uint32 max_size, - td::Promise promise) override; + void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, + td::uint32 max_size, td::Promise promise) override; void import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) override; - void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; + void set_config(FullNodeConfig config) override; - void add_shard(ShardIdFull shard); - void del_shard(ShardIdFull shard); + void add_custom_overlay(CustomOverlayParams params, td::Promise promise) override; + void del_custom_overlay(std::string name, td::Promise promise) override; + + void on_new_masterchain_block(td::Ref state, std::set shards_to_monitor); void sync_completed(); @@ -61,7 +65,9 @@ class FullNodeImpl : public FullNode { void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data); void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data); void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqnp, td::BufferSlice data); - void send_broadcast(BlockBroadcast broadcast); + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data); + void send_broadcast(BlockBroadcast broadcast, int mode); void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); @@ -72,35 +78,55 @@ class FullNodeImpl : public FullNode { void download_block_proof_link(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise); - void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise); + void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise); + void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise); - void got_key_block_proof(td::Ref proof); - void got_zero_block_state(td::Ref state); + void got_key_block_config(td::Ref config); void new_key_block(BlockHandle handle); + void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry); + + void process_block_broadcast(BlockBroadcast broadcast) override; + void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override; + void get_out_msg_queue_query_token(td::Promise> promise) override; + + void set_validator_telemetry_filename(std::string value) override; void start_up() override; FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId dht, - td::actor::ActorId overlays, + FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId rldp2, + td::actor::ActorId dht, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, std::string db_root); + td::actor::ActorId client, std::string db_root, + td::Promise started_promise); private: + struct ShardInfo { + td::actor::ActorOwn actor; + bool active = false; + td::Timestamp delete_at = td::Timestamp::never(); + }; + + void update_shard_actor(ShardIdFull shard, bool active); + PublicKeyHash local_id_; adnl::AdnlNodeIdShort adnl_id_; FileHash zero_state_file_hash_; td::actor::ActorId get_shard(AccountIdPrefixFull dst); - td::actor::ActorId get_shard(ShardIdFull dst); - - std::map> shards_; + td::actor::ActorId get_shard(ShardIdFull shard); + std::map shards_; + int wc_monitor_min_split_ = 0; td::actor::ActorId keyring_; td::actor::ActorId adnl_; td::actor::ActorId rldp_; + td::actor::ActorId rldp2_; td::actor::ActorId dht_; td::actor::ActorId overlays_; td::actor::ActorId validator_manager_; @@ -110,8 +136,38 @@ class FullNodeImpl : public FullNode { PublicKeyHash sign_cert_by_; std::vector all_validators_; + std::map current_validators_; std::set local_keys_; + + td::Promise started_promise_; + FullNodeOptions opts_; + + std::map> private_block_overlays_; + bool broadcast_block_candidates_in_public_overlay_ = false; + + struct CustomOverlayInfo { + CustomOverlayParams params_; + std::map> actors_; // our local id -> actor + }; + std::map custom_overlays_; + std::set custom_overlays_sent_broadcasts_; + std::queue custom_overlays_sent_broadcasts_lru_; + + void update_private_overlays(); + void create_private_block_overlay(PublicKeyHash key); + void update_custom_overlay(CustomOverlayInfo& overlay); + void send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast); + void send_block_candidate_broadcast_to_custom_overlays(const BlockIdExt& block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, const td::BufferSlice& data); + + std::string validator_telemetry_filename_; + PublicKeyHash validator_telemetry_collector_key_ = PublicKeyHash::zero(); + + void update_validator_telemetry_collector(); + + td::actor::ActorOwn out_msg_queue_query_token_manager_ = + td::actor::create_actor("tokens", /* max_tokens = */ 1); }; } // namespace fullnode diff --git a/validator/impl/CMakeLists.txt b/validator/impl/CMakeLists.txt index 459e7724..978cf859 100644 --- a/validator/impl/CMakeLists.txt +++ b/validator/impl/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) @@ -7,6 +7,7 @@ endif() set(TON_VALIDATOR_SOURCE accept-block.cpp block.cpp + candidates-buffer.cpp check-proof.cpp collator.cpp config.cpp @@ -15,6 +16,7 @@ set(TON_VALIDATOR_SOURCE ihr-message.cpp liteserver.cpp message-queue.cpp + out-msg-queue-proof.cpp proof.cpp shard.cpp signature-set.cpp @@ -24,22 +26,23 @@ set(TON_VALIDATOR_SOURCE accept-block.hpp block.hpp + candidates-buffer.hpp check-proof.hpp - collate-query-impl.h collator-impl.h collator.h config.hpp external-message.hpp ihr-message.hpp liteserver.hpp + liteserver-cache.hpp message-queue.hpp + out-msg-queue-proof.hpp proof.hpp shard.hpp signature-set.hpp top-shard-descr.hpp validate-query.hpp - validator-set.hpp -) + validator-set.hpp) add_library(ton_validator STATIC ${TON_VALIDATOR_SOURCE}) diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index ff953b66..de48626d 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -41,7 +41,7 @@ using namespace std::literals::string_literals; AcceptBlockQuery::AcceptBlockQuery(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, bool send_broadcast, + td::Ref approve_signatures, int send_broadcast_mode, td::actor::ActorId manager, td::Promise promise) : id_(id) , data_(std::move(data)) @@ -51,7 +51,7 @@ AcceptBlockQuery::AcceptBlockQuery(BlockIdExt id, td::Ref data, std:: , approve_signatures_(std::move(approve_signatures)) , is_fake_(false) , is_fork_(false) - , send_broadcast_(send_broadcast) + , send_broadcast_mode_(send_broadcast_mode) , manager_(manager) , promise_(std::move(promise)) , perf_timer_("acceptblock", 0.1, [manager](double duration) { @@ -72,7 +72,6 @@ AcceptBlockQuery::AcceptBlockQuery(AcceptBlockQuery::IsFake fake, BlockIdExt id, , validator_set_(std::move(validator_set)) , is_fake_(true) , is_fork_(false) - , send_broadcast_(false) , manager_(manager) , promise_(std::move(promise)) , perf_timer_("acceptblock", 0.1, [manager](double duration) { @@ -90,7 +89,6 @@ AcceptBlockQuery::AcceptBlockQuery(ForceFork ffork, BlockIdExt id, td::Refget_hash(0).bits(); @@ -307,8 +308,11 @@ bool AcceptBlockQuery::create_new_proof() { } // 10. check resulting object if (!block::gen::t_BlockProof.validate_ref(bs_cell)) { - block::gen::t_BlockProof.print_ref(std::cerr, bs_cell); - vm::load_cell_slice(bs_cell).print_rec(std::cerr); + FLOG(WARNING) { + sb << "BlockProof object just created failed to pass automated consistency checks: "; + block::gen::t_BlockProof.print_ref(sb, bs_cell); + vm::load_cell_slice(bs_cell).print_rec(sb); + }; return fatal_error("BlockProof object just created failed to pass automated consistency checks"); } // 11. create a proof object from this cell @@ -410,7 +414,36 @@ void AcceptBlockQuery::got_block_handle(BlockHandle handle) { : handle_->inited_proof_link())) { finish_query(); return; + } + if (data_.is_null()) { + td::actor::send_closure(manager_, &ValidatorManager::get_candidate_data_by_block_id_from_db, id_, [SelfId = actor_id(this)](td::Result R) { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &AcceptBlockQuery::got_block_candidate_data, R.move_as_ok()); + } else { + td::actor::send_closure(SelfId, &AcceptBlockQuery::got_block_handle_cont); + } + }); + } else { + got_block_handle_cont(); } +} + +void AcceptBlockQuery::got_block_candidate_data(td::BufferSlice data) { + auto r_block = create_block(id_, std::move(data)); + if (r_block.is_error()) { + fatal_error("invalid block candidate data in db: " + r_block.error().to_string()); + return; + } + data_ = r_block.move_as_ok(); + VLOG(VALIDATOR_DEBUG) << "got block candidate data from db"; + if (data_.not_null() && !precheck_header()) { + fatal_error("invalid block header in AcceptBlock"); + return; + } + got_block_handle_cont(); +} + +void AcceptBlockQuery::got_block_handle_cont() { if (data_.not_null() && !handle_->received()) { td::actor::send_closure( manager_, &ValidatorManager::set_block_data, handle_, data_, [SelfId = actor_id(this)](td::Result R) { @@ -821,15 +854,12 @@ bool AcceptBlockQuery::create_top_shard_block_description() { && (root.is_null() || cb.store_ref_bool(std::move(root))) && cb.finalize_to(td_cell))) { return fatal_error("cannot serialize ShardTopBlockDescription for the newly-accepted block "s + id_.to_str()); } - if (false) { - // debug output - std::cerr << "new ShardTopBlockDescription: "; - block::gen::t_TopBlockDescr.print_ref(std::cerr, td_cell); - vm::load_cell_slice(td_cell).print_rec(std::cerr); - } if (!block::gen::t_TopBlockDescr.validate_ref(td_cell)) { - block::gen::t_TopBlockDescr.print_ref(std::cerr, td_cell); - vm::load_cell_slice(td_cell).print_rec(std::cerr); + FLOG(WARNING) { + sb << "just created ShardTopBlockDescription is invalid: "; + block::gen::t_TopBlockDescr.print_ref(sb, td_cell); + vm::load_cell_slice(td_cell).print_rec(sb); + }; return fatal_error("just created ShardTopBlockDescription for "s + id_.to_str() + " is invalid"); } auto res = vm::std_boc_serialize(td_cell, 0); @@ -896,11 +926,10 @@ void AcceptBlockQuery::written_block_info_2() { } void AcceptBlockQuery::applied() { - if (!send_broadcast_) { + if (send_broadcast_mode_ == 0) { finish_query(); return; } - BlockBroadcast b; b.data = data_->data(); b.block_id = id_; @@ -920,7 +949,7 @@ void AcceptBlockQuery::applied() { } // do not wait for answer - td::actor::send_closure_later(manager_, &ValidatorManager::send_block_broadcast, std::move(b)); + td::actor::send_closure_later(manager_, &ValidatorManager::send_block_broadcast, std::move(b), send_broadcast_mode_); finish_query(); } diff --git a/validator/impl/accept-block.hpp b/validator/impl/accept-block.hpp index 2600b258..d1c0baa6 100644 --- a/validator/impl/accept-block.hpp +++ b/validator/impl/accept-block.hpp @@ -50,7 +50,7 @@ class AcceptBlockQuery : public td::actor::Actor { struct ForceFork {}; AcceptBlockQuery(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, bool send_broadcast, + td::Ref approve_signatures, int send_broadcast_mode, td::actor::ActorId manager, td::Promise promise); AcceptBlockQuery(IsFake fake, BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::actor::ActorId manager, @@ -71,6 +71,8 @@ class AcceptBlockQuery : public td::actor::Actor { void written_block_data(); void written_block_signatures(); void got_block_handle(BlockHandle handle); + void got_block_candidate_data(td::BufferSlice data); + void got_block_handle_cont(); void written_block_info(); void got_block_data(td::Ref data); void got_prev_state(td::Ref state); @@ -97,7 +99,7 @@ class AcceptBlockQuery : public td::actor::Actor { Ref approve_signatures_; bool is_fake_; bool is_fork_; - bool send_broadcast_; + int send_broadcast_mode_{0}; bool ancestors_split_{false}, is_key_block_{false}; td::Timestamp timeout_ = td::Timestamp::in(600.0); td::actor::ActorId manager_; diff --git a/validator/impl/candidates-buffer.cpp b/validator/impl/candidates-buffer.cpp new file mode 100644 index 00000000..e24913b6 --- /dev/null +++ b/validator/impl/candidates-buffer.cpp @@ -0,0 +1,213 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "candidates-buffer.hpp" +#include "fabric.h" + +namespace ton::validator { + +void CandidatesBuffer::start_up() { + alarm_timestamp() = td::Timestamp::in(60.0); +} + +void CandidatesBuffer::alarm() { + alarm_timestamp() = td::Timestamp::in(60.0); + for (auto it = candidates_.begin(); it != candidates_.end();) { + Candidate &entry = it->second; + if (entry.ttl_.is_in_past()) { + for (auto &p : entry.data_waiters_) { + p.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); + } + for (auto &p : entry.state_waiters_) { + p.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); + } + it = candidates_.erase(it); + } else { + ++it; + } + } +} + +void CandidatesBuffer::add_new_candidate(BlockIdExt id, PublicKey source, FileHash collated_data_file_hash) { + auto it = candidates_.emplace(id, Candidate{}); + Candidate &entry = it.first->second; + entry.ttl_ = td::Timestamp::in(120.0); + if (!it.second) { // not inserted + return; + } + LOG(DEBUG) << "New block candidate " << id.to_str(); + entry.source_ = source; + entry.collated_data_file_hash_ = collated_data_file_hash; +} + +void CandidatesBuffer::get_block_data(BlockIdExt id, td::Promise> promise) { + auto it = candidates_.find(id); + if (it == candidates_.end()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "unknown block candidate")); + return; + } + Candidate &entry = it->second; + if (entry.data_.not_null()) { + promise.set_result(entry.data_); + return; + } + entry.data_waiters_.push_back(std::move(promise)); + if (entry.data_requested_) { + return; + } + entry.data_requested_ = true; + td::actor::send_closure(manager_, &ValidatorManager::get_block_candidate_from_db, entry.source_, id, + entry.collated_data_file_hash_, [SelfId = actor_id(this), id](td::Result R) { + td::actor::send_closure(SelfId, &CandidatesBuffer::got_block_candidate, id, std::move(R)); + }); +} + +void CandidatesBuffer::got_block_candidate(BlockIdExt id, td::Result R) { + if (R.is_error()) { + finish_get_block_data(id, R.move_as_error()); + return; + } + BlockCandidate cand = R.move_as_ok(); + CHECK(cand.id == id); + finish_get_block_data(id, create_block(id, std::move(cand.data))); +} + +void CandidatesBuffer::get_block_state(BlockIdExt id, td::Promise> promise) { + auto it = candidates_.find(id); + if (it == candidates_.end()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "unknown block candidate")); + return; + } + Candidate &entry = it->second; + if (entry.state_.not_null()) { + promise.set_result(entry.state_); + return; + } + entry.state_waiters_.push_back(std::move(promise)); + if (entry.state_requested_) { + return; + } + entry.state_requested_ = true; + get_block_data(id, [SelfId = actor_id(this), id](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &CandidatesBuffer::finish_get_block_state, id, R.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &CandidatesBuffer::get_block_state_cont, id, R.move_as_ok()); + }); +} + +void CandidatesBuffer::get_block_state_cont(BlockIdExt id, td::Ref data) { + CHECK(id == data->block_id()); + std::vector prev; + BlockIdExt mc_blkid; + bool after_split; + auto S = block::unpack_block_prev_blk_ext(data->root_cell(), id, prev, mc_blkid, after_split); + if (S.is_error()) { + finish_get_block_state(id, std::move(S)); + return; + } + get_block_state_cont2(std::move(data), std::move(prev), {}); +} + +void CandidatesBuffer::get_block_state_cont2(td::Ref block, std::vector prev, + std::vector> prev_states) { + if (prev_states.size() < prev.size()) { + BlockIdExt prev_id = prev[prev_states.size()]; + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db_short, prev_id, + [SelfId = actor_id(this), block = std::move(block), prev = std::move(prev), + prev_states = std::move(prev_states)](td::Result> R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &CandidatesBuffer::finish_get_block_state, + block->block_id(), R.move_as_error()); + return; + } + prev_states.push_back(R.move_as_ok()); + td::actor::send_closure(SelfId, &CandidatesBuffer::get_block_state_cont2, + std::move(block), std::move(prev), std::move(prev_states)); + }); + return; + } + + BlockIdExt id = block->block_id(); + td::Ref state; + CHECK(prev_states.size() == 1 || prev_states.size() == 2); + if (prev_states.size() == 2) { // after merge + auto R = prev_states[0]->merge_with(*prev_states[1]); + if (R.is_error()) { + finish_get_block_state(id, R.move_as_error()); + return; + } + state = R.move_as_ok(); + } else if (id.shard_full() != prev[0].shard_full()) { // after split + auto R = prev_states[0]->split(); + if (R.is_error()) { + finish_get_block_state(id, R.move_as_error()); + return; + } + auto s = R.move_as_ok(); + state = is_left_child(id.shard_full()) ? std::move(s.first) : std::move(s.second); + } else { // no split/merge + state = std::move(prev_states[0]); + } + + auto S = state.write().apply_block(id, std::move(block)); + if (S.is_error()) { + finish_get_block_state(id, std::move(S)); + return; + } + finish_get_block_state(id, std::move(state)); +} + +void CandidatesBuffer::finish_get_block_data(BlockIdExt id, td::Result> res) { + auto it = candidates_.find(id); + if (it == candidates_.end()) { + return; + } + Candidate &entry = it->second; + for (auto &p : entry.data_waiters_) { + p.set_result(res.clone()); + } + entry.data_waiters_.clear(); + entry.data_requested_ = false; + if (res.is_ok()) { + entry.data_ = res.move_as_ok(); + LOG(DEBUG) << "Loaded block data for " << id.to_str(); + } else { + LOG(DEBUG) << "Failed to load block data for " << id.to_str() << ": " << res.move_as_error(); + } +} + +void CandidatesBuffer::finish_get_block_state(BlockIdExt id, td::Result> res) { + auto it = candidates_.find(id); + if (it == candidates_.end()) { + return; + } + Candidate &entry = it->second; + for (auto &p : entry.state_waiters_) { + p.set_result(res.clone()); + } + entry.state_waiters_.clear(); + entry.state_requested_ = false; + if (res.is_ok()) { + entry.state_ = res.move_as_ok(); + LOG(DEBUG) << "Loaded block state for " << id.to_str(); + } else { + LOG(DEBUG) << "Failed to load block state for " << id.to_str() << ": " << res.move_as_error(); + } +} + +} // namespace ton::validator diff --git a/validator/impl/candidates-buffer.hpp b/validator/impl/candidates-buffer.hpp new file mode 100644 index 00000000..5db1e516 --- /dev/null +++ b/validator/impl/candidates-buffer.hpp @@ -0,0 +1,64 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "ton/ton-types.h" +#include "td/actor/actor.h" +#include "interfaces/validator-manager.h" + +namespace ton::validator { + +class CandidatesBuffer : public td::actor::Actor { + public: + explicit CandidatesBuffer(td::actor::ActorId manager) : manager_(std::move(manager)) { + } + + void start_up() override; + void alarm() override; + + void add_new_candidate(BlockIdExt id, PublicKey source, FileHash collated_data_file_hash); + void get_block_data(BlockIdExt id, td::Promise> promise); + void get_block_state(BlockIdExt id, td::Promise> promise); + + private: + td::actor::ActorId manager_; + + struct Candidate { + PublicKey source_; + FileHash collated_data_file_hash_; + td::Timestamp ttl_; + + td::Ref data_; + std::vector>> data_waiters_; + bool data_requested_{false}; + + td::Ref state_; + std::vector>> state_waiters_; + bool state_requested_{false}; + }; + std::map candidates_; + + void got_block_candidate(BlockIdExt id, td::Result R); + + void get_block_state_cont(BlockIdExt id, td::Ref data); + void get_block_state_cont2(td::Ref block, std::vector prev, + std::vector> prev_states); + + void finish_get_block_data(BlockIdExt id, td::Result> res); + void finish_get_block_state(BlockIdExt id, td::Result> res); +}; + +} // namespace ton::validator diff --git a/validator/impl/collate-query-impl.h b/validator/impl/collate-query-impl.h deleted file mode 100644 index 83832646..00000000 --- a/validator/impl/collate-query-impl.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once - -#include "validator/interfaces/validator-manager.h" - -namespace ton { - -namespace validator { - -class CollateQuery : public td::actor::Actor { - public: - CollateQuery(ShardIdFull shard, td::uint32 min_ts, BlockIdExt min_masterchain_block_id, std::vector prev, - td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise); - CollateQuery(ShardIdFull shard, td::uint32 min_ts, BlockIdExt min_masterchain_block_id, ZeroStateIdExt zero_state_id, - td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise); - - void alarm() override; - - void abort_query(td::Status reason); - void finish_query(); - - void start_up() override; - void got_prev_state(td::Ref state); - void written_block_data(); - void written_block_collated_data(); - - private: - ShardIdFull shard_; - UnixTime min_ts_; - BlockIdExt min_masterchain_block_id_; - std::vector prev_; - ZeroStateIdExt zero_state_id_; - td::Ref validator_set_; - td::actor::ActorId manager_; - td::Timestamp timeout_; - td::Promise promise_; - - BlockCandidate candidate_; - UnixTime ts_; -}; - -} // namespace validator - -} // namespace ton diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 8fd8dc0c..340e3a40 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -32,6 +32,7 @@ #include "vm/cells/MerkleUpdate.h" #include #include +#include "common/global-version.h" namespace ton { @@ -40,15 +41,16 @@ using td::Ref; class Collator final : public td::actor::Actor { static constexpr int supported_version() { - return 3; + return SUPPORTED_VERSION; } static constexpr long long supported_capabilities() { - return ton::capCreateStatsEnabled | ton::capBounceMsgBody | ton::capReportVersion | ton::capShortDequeue; + return ton::capCreateStatsEnabled | ton::capBounceMsgBody | ton::capReportVersion | ton::capShortDequeue | + ton::capStoreOutMsgQueueSize | ton::capMsgMetadata | ton::capDeferMessages; } using LtCellRef = block::LtCellRef; using NewOutMsg = block::NewOutMsg; const ShardIdFull shard_; - ton::BlockId new_id; + ton::BlockId new_id{workchainInvalid, 0, 0}; bool busy_{false}; bool before_split_{false}; bool after_split_{false}; @@ -59,22 +61,24 @@ class Collator final : public td::actor::Actor { bool preinit_complete{false}; bool is_key_block_{false}; bool block_full_{false}; - bool outq_cleanup_partial_{false}; bool inbound_queues_empty_{false}; bool libraries_changed_{false}; bool prev_key_block_exists_{false}; bool is_hardfork_{false}; - UnixTime min_ts; BlockIdExt min_mc_block_id; std::vector prev_blocks; std::vector> prev_states; std::vector> prev_block_data; Ed25519_PublicKey created_by_; + Ref collator_opts_; Ref validator_set_; td::actor::ActorId manager; td::Timestamp timeout; - td::Timestamp soft_timeout_, medium_timeout_; + td::Timestamp queue_cleanup_timeout_, soft_timeout_, medium_timeout_; td::Promise main_promise; + unsigned mode_ = 0; + int attempt_idx_; + bool allow_repeat_collation_ = false; ton::BlockSeqno last_block_seqno{0}; ton::BlockSeqno prev_mc_block_seqno{0}; ton::BlockSeqno new_block_seqno{0}; @@ -87,9 +91,10 @@ class Collator final : public td::actor::Actor { static constexpr bool shard_splitting_enabled = true; public: - Collator(ShardIdFull shard, bool is_hardfork, td::uint32 min_ts, BlockIdExt min_masterchain_block_id, - std::vector prev, Ref validator_set, Ed25519_PublicKey collator_id, - td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); + Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, + Ref validator_set, Ed25519_PublicKey collator_id, Ref collator_opts, + td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, + td::CancellationToken cancellation_token, unsigned mode, int attempt_idx); ~Collator() override = default; bool is_busy() const { return busy_; @@ -104,26 +109,11 @@ class Collator final : public td::actor::Actor { return 2; } - static td::Result> - impl_fetch_config_params(std::unique_ptr config, - Ref* old_mparams, - std::vector* storage_prices, - block::StoragePhaseConfig* storage_phase_cfg, - td::BitArray<256>* rand_seed, - block::ComputePhaseConfig* compute_phase_cfg, - block::ActionPhaseConfig* action_phase_cfg, - td::RefInt256* masterchain_create_fee, - td::RefInt256* basechain_create_fee, - WorkchainId wc, UnixTime now); - - static td::Result> - impl_create_ordinary_transaction(Ref msg_root, - block::Account* acc, - UnixTime utime, LogicalTime lt, - block::StoragePhaseConfig* storage_phase_cfg, - block::ComputePhaseConfig* compute_phase_cfg, - block::ActionPhaseConfig* action_phase_cfg, - bool external, LogicalTime after_lt); + static td::Result> impl_create_ordinary_transaction( + Ref msg_root, block::Account* acc, UnixTime utime, LogicalTime lt, + block::StoragePhaseConfig* storage_phase_cfg, block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, block::SerializeConfig* serialize_cfg, bool external, + LogicalTime after_lt); private: void start_up() override; @@ -140,7 +130,7 @@ class Collator final : public td::actor::Actor { BlockIdExt mc_block_id_; Ref mc_state_root; Ref mc_block_root; - td::BitArray<256> rand_seed_; + td::BitArray<256> rand_seed_ = td::Bits256::zero(); std::unique_ptr config_; std::unique_ptr shard_conf_; std::map> aux_mc_states_; @@ -184,9 +174,11 @@ class Collator final : public td::actor::Actor { block::StoragePhaseConfig storage_phase_cfg_{&storage_prices_}; block::ComputePhaseConfig compute_phase_cfg_; block::ActionPhaseConfig action_phase_cfg_; + block::SerializeConfig serialize_cfg_; td::RefInt256 masterchain_create_fee_, basechain_create_fee_; std::unique_ptr block_limits_; std::unique_ptr block_limit_status_; + int block_limit_class_ = 0; ton::LogicalTime min_new_msg_lt{std::numeric_limits::max()}; block::CurrencyCollection total_balance_, old_total_balance_, total_validator_fees_; block::CurrencyCollection global_balance_, old_global_balance_, import_created_{0}; @@ -195,10 +187,19 @@ class Collator final : public td::actor::Actor { block::ValueFlow value_flow_{block::ValueFlow::SetZero()}; std::unique_ptr fees_import_dict_; std::map ext_msg_map; - std::vector, ExtMessage::Hash>> ext_msg_list_; + struct ExtMsg { + Ref cell; + ExtMessage::Hash hash; + int priority; + }; + std::vector ext_msg_list_; std::priority_queue, std::greater> new_msgs; std::pair last_proc_int_msg_, first_unproc_int_msg_; std::unique_ptr in_msg_dict, out_msg_dict, out_msg_queue_, sibling_out_msg_queue_; + std::map unprocessed_deferred_messages_; // number of messages from dispatch queue in new_msgs + td::uint64 out_msg_queue_size_ = 0; + td::uint64 old_out_msg_queue_size_ = 0; + bool have_out_msg_queue_size_in_state_ = false; std::unique_ptr ihr_pending; std::shared_ptr processed_upto_, sibling_processed_upto_; std::unique_ptr block_create_stats_; @@ -209,11 +210,28 @@ class Collator final : public td::actor::Actor { std::vector> collated_roots_; std::unique_ptr block_candidate; + std::unique_ptr dispatch_queue_; + std::map sender_generated_messages_count_; + unsigned dispatch_queue_ops_{0}; + std::map last_dispatch_queue_emitted_lt_; + bool have_unprocessed_account_dispatch_queue_ = false; + bool dispatch_queue_total_limit_reached_ = false; + td::uint64 defer_out_queue_size_limit_; + td::uint64 hard_defer_out_queue_size_limit_; + + std::unique_ptr account_dict_estimator_; + std::set account_dict_estimator_added_accounts_; + unsigned account_dict_ops_{0}; + + bool msg_metadata_enabled_ = false; + bool deferring_messages_enabled_ = false; + bool store_out_msg_queue_size_ = false; + td::PerfWarningTimer perf_timer_; // block::Account* lookup_account(td::ConstBitPtr addr) const; std::unique_ptr make_account_from(td::ConstBitPtr addr, Ref account, - Ref extra, bool force_create = false); + bool force_create); td::Result make_account(td::ConstBitPtr addr, bool force_create = false); td::actor::ActorId get_self() { return actor_id(this); @@ -237,6 +255,7 @@ class Collator final : public td::actor::Actor { bool fix_one_processed_upto(block::MsgProcessedUpto& proc, const ton::ShardIdFull& owner); bool fix_processed_upto(block::MsgProcessedUptoCollection& upto); void got_neighbor_out_queue(int i, td::Result> res); + void got_out_queue_size(size_t i, td::Result res); bool adjust_shard_config(); bool store_shard_fees(ShardIdFull shard, const block::CurrencyCollection& fees, const block::CurrencyCollection& created); @@ -254,7 +273,8 @@ class Collator final : public td::actor::Actor { Ref& in_msg); bool create_ticktock_transactions(int mask); bool create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, ton::LogicalTime req_start_lt, int mask); - Ref create_ordinary_transaction(Ref msg_root); + Ref create_ordinary_transaction(Ref msg_root, td::optional msg_metadata, + LogicalTime after_lt, bool is_special_tx = false); bool check_cur_validator_set(); bool unpack_last_mc_state(); bool unpack_last_state(); @@ -270,6 +290,7 @@ class Collator final : public td::actor::Actor { bool check_prev_block_exact(const BlockIdExt& listed, const BlockIdExt& prev); bool check_this_shard_mc_info(); bool request_neighbor_msg_queues(); + bool request_out_msg_queue_size(); void update_max_lt(ton::LogicalTime lt); bool is_masterchain() const { return shard_.is_masterchain(); @@ -277,15 +298,12 @@ class Collator final : public td::actor::Actor { bool is_our_address(Ref addr_ref) const; bool is_our_address(ton::AccountIdPrefixFull addr_prefix) const; bool is_our_address(const ton::StdSmcAddress& addr) const; - void after_get_external_messages(td::Result>> res); - td::Result register_external_message_cell(Ref ext_msg, const ExtMessage::Hash& ext_hash); + void after_get_external_messages(td::Result, int>>> res); + td::Result register_external_message_cell(Ref ext_msg, const ExtMessage::Hash& ext_hash, + int priority); // td::Result register_external_message(td::Slice ext_msg_boc); - td::Result register_ihr_message_cell(Ref ihr_msg); - td::Result register_ihr_message(td::Slice ihr_msg_boc); - td::Result register_shard_signatures_cell(Ref shard_blk_signatures); - td::Result register_shard_signatures(td::Slice shard_blk_signatures_boc); void register_new_msg(block::NewOutMsg msg); - void register_new_msgs(block::Transaction& trans); + void register_new_msgs(block::transaction::Transaction& trans, td::optional msg_metadata); bool process_new_messages(bool enqueue_only = false); int process_one_new_message(block::NewOutMsg msg, bool enqueue_only = false, Ref* is_special = nullptr); bool process_inbound_internal_messages(); @@ -293,15 +311,22 @@ class Collator final : public td::actor::Actor { const block::McShardDescr& src_nb); bool process_inbound_external_messages(); int process_external_message(Ref msg); - bool enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_remaining, ton::LogicalTime enqueued_lt); + bool process_dispatch_queue(); + bool process_deferred_message(Ref enq_msg, StdSmcAddress src_addr, LogicalTime lt, + td::optional& msg_metadata); + bool enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_remaining, StdSmcAddress src_addr, + bool defer = false); bool enqueue_transit_message(Ref msg, Ref old_msg_env, ton::AccountIdPrefixFull prev_prefix, ton::AccountIdPrefixFull cur_prefix, ton::AccountIdPrefixFull dest_prefix, - td::RefInt256 fwd_fee_remaining, ton::LogicalTime enqueued_lt); + td::RefInt256 fwd_fee_remaining, td::optional msg_metadata, + td::optional emitted_lt = {}); bool delete_out_msg_queue_msg(td::ConstBitPtr key); bool insert_in_msg(Ref in_msg); bool insert_out_msg(Ref out_msg); bool insert_out_msg(Ref out_msg, td::ConstBitPtr msg_hash); bool register_out_msg_queue_op(bool force = false); + bool register_dispatch_queue_op(bool force = false); + bool update_account_dict_estimation(const block::transaction::Transaction& trans); bool update_min_mc_seqno(ton::BlockSeqno some_mc_seqno); bool combine_account_transactions(); bool update_public_libraries(); @@ -332,9 +357,21 @@ class Collator final : public td::actor::Actor { bool create_block(); Ref collate_shard_block_descr_set(); bool create_collated_data(); + bool create_block_candidate(); void return_block_candidate(td::Result saved); bool update_last_proc_int_msg(const std::pair& new_lt_hash); + + td::CancellationToken cancellation_token_; + bool check_cancelled(); + + public: + static td::uint32 get_skip_externals_queue_size(); + + private: + td::Timer work_timer_{true}; + td::ThreadCpuTimer cpu_work_timer_{true}; + CollationStats stats_; }; } // namespace validator diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index fb18d8be..2a6d7a2b 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -44,39 +44,74 @@ namespace validator { using td::Ref; using namespace std::literals::string_literals; -#define DBG(__n) dbg(__n)&& -#define DSTART int __dcnt = 0; -#define DEB DBG(++__dcnt) +// Don't increase MERGE_MAX_QUEUE_LIMIT too much: merging requires cleaning the whole queue in out_msg_queue_cleanup +static constexpr td::uint32 FORCE_SPLIT_QUEUE_SIZE = 4096; +static constexpr td::uint32 SPLIT_MAX_QUEUE_SIZE = 100000; +static constexpr td::uint32 MERGE_MAX_QUEUE_SIZE = 2047; +static constexpr td::uint32 SKIP_EXTERNALS_QUEUE_SIZE = 8000; +static constexpr int HIGH_PRIORITY_EXTERNAL = 10; // don't skip high priority externals when queue is big -static inline bool dbg(int c) TD_UNUSED; -static inline bool dbg(int c) { - std::cerr << '[' << (char)('0' + c / 10) << (char)('0' + c % 10) << ']'; - return true; -} +static constexpr int MAX_ATTEMPTS = 5; -Collator::Collator(ShardIdFull shard, bool is_hardfork, UnixTime min_ts, BlockIdExt min_masterchain_block_id, +/** + * Constructs a Collator object. + * + * @param shard The shard of the new block. + * @param is_hardfork A boolean indicating whether the new block is a hardfork. + * @param min_masterchain_block_id The the minimum reference masterchain block. + * @param prev A vector of BlockIdExt representing the previous blocks. + * @param validator_set A reference to the ValidatorSet. + * @param collator_id The public key of the block creator. + * @param collator_opts A reference to CollatorOptions. + * @param manager The ActorId of the ValidatorManager. + * @param timeout The timeout for the collator. + * @param promise The promise to return the result. + * @param cancellation_token Token to cancel collation. + * @param mode +1 - skip storing candidate to disk. + * @param attempt_idx The index of the attempt, starting from 0. On later attempts collator decreases block limits and skips some steps. + */ +Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, td::Ref validator_set, Ed25519_PublicKey collator_id, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise) + Ref collator_opts, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise promise, td::CancellationToken cancellation_token, + unsigned mode, int attempt_idx) : shard_(shard) , is_hardfork_(is_hardfork) - , min_ts(min_ts) , min_mc_block_id{min_masterchain_block_id} , prev_blocks(std::move(prev)) , created_by_(collator_id) + , collator_opts_(collator_opts) , validator_set_(std::move(validator_set)) , manager(manager) , timeout(timeout) + // default timeout is 10 seconds, declared in validator/validator-group.cpp:generate_block_candidate:run_collate_query + , queue_cleanup_timeout_(td::Timestamp::at(timeout.at() - 5.0)) , soft_timeout_(td::Timestamp::at(timeout.at() - 3.0)) , medium_timeout_(td::Timestamp::at(timeout.at() - 1.5)) , main_promise(std::move(promise)) - , perf_timer_("collate", 0.1, [manager](double duration) { - send_closure(manager, &ValidatorManager::add_perf_timer_stat, "collate", duration); - }) { + , mode_(mode) + , attempt_idx_(attempt_idx) + , perf_timer_("collate", 0.1, + [manager](double duration) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "collate", duration); + }) + , cancellation_token_(std::move(cancellation_token)) { } +/** + * Starts the Collator. + * + * This function initializes the Collator by performing various checks and queries to the ValidatorManager. + * It checks the validity of the shard, the previous blocks, and the workchain. + * If all checks pass, it proceeds to query the ValidatorManager for the top masterchain state block, shard states, block data, external messages, and shard blocks. + * The results of these queries are handled by corresponding callback functions. + */ void Collator::start_up() { - LOG(DEBUG) << "Collator for shard " << shard_.to_str() << " started"; + LOG(WARNING) << "Collator for shard " << shard_.to_str() << " started" + << (attempt_idx_ ? PSTRING() << " (attempt #" << attempt_idx_ << ")" : ""); + if (!check_cancelled()) { + return; + } LOG(DEBUG) << "Previous block #1 is " << prev_blocks.at(0).to_str(); if (prev_blocks.size() > 1) { LOG(DEBUG) << "Previous block #2 is " << prev_blocks.at(1).to_str(); @@ -226,11 +261,10 @@ void Collator::start_up() { LOG(DEBUG) << "sending get_external_messages() query to Manager"; ++pending; td::actor::send_closure_later(manager, &ValidatorManager::get_external_messages, shard_, - [self = get_self()](td::Result>> res) -> void { - LOG(DEBUG) << "got answer to get_external_messages() query"; - td::actor::send_closure_later( - std::move(self), &Collator::after_get_external_messages, std::move(res)); - }); + [self = get_self()](td::Result, int>>> res) -> void { + LOG(DEBUG) << "got answer to get_external_messages() query"; + td::actor::send_closure_later(std::move(self), &Collator::after_get_external_messages, std::move(res)); + }); } if (is_masterchain() && !is_hardfork_) { // 5. load shard block info messages @@ -248,10 +282,21 @@ void Collator::start_up() { CHECK(pending); } +/** + * Raises an error when timeout is reached. + */ void Collator::alarm() { fatal_error(ErrorCode::timeout, "timeout"); } +/** + * Generates a string representation of a shard. + * + * @param workchain The workchain ID of the shard. + * @param shard The shard ID. + * + * @returns A string representation of the shard. + */ std::string show_shard(ton::WorkchainId workchain, ton::ShardId shard) { char tmp[128]; char* ptr = tmp + snprintf(tmp, 31, "%d:", workchain); @@ -266,35 +311,94 @@ std::string show_shard(ton::WorkchainId workchain, ton::ShardId shard) { return {tmp, ptr}; } +/** + * Returns a string representation of the shard of the given block. + * + * @param blk_id The BlockId object. + * + * @returns A string representation of the shard. + */ std::string show_shard(const ton::BlockId blk_id) { return show_shard(blk_id.workchain, blk_id.shard); } +/** + * Converts a `ShardIdFull` object to a string representation. + * + * @param blk_id The `ShardIdFull` object to convert. + * + * @returns The string representation of the `ShardIdFull` object. + */ std::string show_shard(const ton::ShardIdFull blk_id) { return show_shard(blk_id.workchain, blk_id.shard); } +/** + * Handles a fatal error encountered during block candidate generation. + * + * @param error The error encountered. + * + * @returns False to indicate that a fatal error occurred. + */ bool Collator::fatal_error(td::Status error) { error.ensure_error(); LOG(ERROR) << "cannot generate block candidate for " << show_shard(shard_) << " : " << error.to_string(); if (busy_) { - main_promise(std::move(error)); + if (allow_repeat_collation_ && error.code() != ErrorCode::cancelled && attempt_idx_ + 1 < MAX_ATTEMPTS && + !is_hardfork_ && !timeout.is_in_past()) { + LOG(WARNING) << "Repeating collation (attempt #" << attempt_idx_ + 1 << ")"; + run_collate_query(shard_, min_mc_block_id, prev_blocks, created_by_, validator_set_, collator_opts_, manager, + td::Timestamp::in(10.0), std::move(main_promise), std::move(cancellation_token_), mode_, + attempt_idx_ + 1); + } else { + main_promise(std::move(error)); + td::actor::send_closure(manager, &ValidatorManager::record_collate_query_stats, BlockIdExt{new_id, RootHash::zero(), FileHash::zero()}, + work_timer_.elapsed(), cpu_work_timer_.elapsed(), td::optional{}); + } busy_ = false; } stop(); return false; } +/** + * Handles a fatal error encountered during block candidate generation. + * + * @param err_code The error code. + * @param err_msg The error message. + * + * @returns False to indicate that a fatal error occurred. + */ bool Collator::fatal_error(int err_code, std::string err_msg) { return fatal_error(td::Status::Error(err_code, err_msg)); } +/** + * Handles a fatal error encountered during block candidate generation. + * + * @param err_msg The error message. + * @param err_code The error code. + * + * @returns False to indicate that a fatal error occurred. + */ bool Collator::fatal_error(std::string err_msg, int err_code) { return fatal_error(td::Status::Error(err_code, err_msg)); } +/** + * Checks if there are any pending tasks. + * + * If there are no pending tasks, it continues collation. + * If collation fails, it raises a fatal error. + * If an exception is caught during collation, it raises a fatal error with the corresponding error message. + * + * @returns None + */ void Collator::check_pending() { // LOG(DEBUG) << "pending = " << pending; + if (!check_cancelled()) { + return; + } if (!pending) { step = 2; try { @@ -307,6 +411,13 @@ void Collator::check_pending() { } } +/** + * Registers a masterchain state. + * + * @param other_mc_state The masterchain state to register. + * + * @returns True if the registration is successful, false otherwise. + */ bool Collator::register_mc_state(Ref other_mc_state) { if (other_mc_state.is_null() || mc_state_.is_null()) { return false; @@ -332,6 +443,14 @@ bool Collator::register_mc_state(Ref other_mc_state) { return true; } +/** + * Requests the auxiliary masterchain state. + * + * @param seqno The seqno of the block. + * @param state A reference to the auxiliary masterchain state. + * + * @returns True if the auxiliary masterchain state is successfully requested, false otherwise. + */ bool Collator::request_aux_mc_state(BlockSeqno seqno, Ref& state) { if (mc_state_.is_null()) { return fatal_error(PSTRING() << "cannot find masterchain block with seqno " << seqno @@ -364,6 +483,13 @@ bool Collator::request_aux_mc_state(BlockSeqno seqno, Ref& st return true; } +/** + * Retrieves the auxiliary masterchain state for a given block sequence number. + * + * @param seqno The sequence number of the block. + * + * @returns A reference to the auxiliary masterchain state if found, otherwise an empty reference. + */ Ref Collator::get_aux_mc_state(BlockSeqno seqno) const { auto it = aux_mc_states_.find(seqno); if (it != aux_mc_states_.end()) { @@ -373,6 +499,13 @@ Ref Collator::get_aux_mc_state(BlockSeqno seqno) const { } } +/** + * Callback function called after retrieving the auxiliary shard state. + * Handles the retrieved shard state and performs necessary checks and registrations. + * + * @param blkid The BlockIdExt of the shard state. + * @param res The result of retrieving the shard state. + */ void Collator::after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res) { LOG(DEBUG) << "in Collator::after_get_aux_shard_state(" << blkid.to_str() << ")"; --pending; @@ -398,6 +531,14 @@ void Collator::after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result, BlockIdExt>> res) { - LOG(DEBUG) << "in Collator::after_get_mc_state()"; + LOG(WARNING) << "in Collator::after_get_mc_state()"; --pending; if (res.is_error()) { fatal_error(res.move_as_error()); @@ -453,8 +599,14 @@ void Collator::after_get_mc_state(td::Result, Bl check_pending(); } +/** + * Callback function called after retrieving the shard state for a previous block. + * + * @param idx The index of the previous shard block (0 or 1). + * @param res The retrieved shard state. + */ void Collator::after_get_shard_state(int idx, td::Result> res) { - LOG(DEBUG) << "in Collator::after_get_shard_state(" << idx << ")"; + LOG(WARNING) << "in Collator::after_get_shard_state(" << idx << ")"; --pending; if (res.is_error()) { fatal_error(res.move_as_error()); @@ -480,6 +632,12 @@ void Collator::after_get_shard_state(int idx, td::Result> res) { check_pending(); } +/** + * Callback function called after retrieving block data for a previous block. + * + * @param idx The index of the previous block (0 or 1). + * @param res The retreved block data. + */ void Collator::after_get_block_data(int idx, td::Result> res) { LOG(DEBUG) << "in Collator::after_get_block_data(" << idx << ")"; --pending; @@ -511,6 +669,11 @@ void Collator::after_get_block_data(int idx, td::Result> res) { check_pending(); } +/** + * Callback function called after retrieving shard block descriptions for masterchain. + * + * @param res The retrieved shard block descriptions. + */ void Collator::after_get_shard_blocks(td::Result>> res) { --pending; if (res.is_error()) { @@ -523,11 +686,17 @@ void Collator::after_get_shard_blocks(td::Resultcreate_stats_enabled(); report_version_ = config_->has_capability(ton::capReportVersion); short_dequeue_records_ = config_->has_capability(ton::capShortDequeue); + store_out_msg_queue_size_ = config_->has_capability(ton::capStoreOutMsgQueueSize); + msg_metadata_enabled_ = config_->has_capability(ton::capMsgMetadata); + deferring_messages_enabled_ = config_->has_capability(ton::capDeferMessages); shard_conf_ = std::make_unique(*config_); prev_key_block_exists_ = config_->get_last_key_block(prev_key_block_, prev_key_block_lt_); if (prev_key_block_exists_) { @@ -557,6 +729,15 @@ bool Collator::unpack_last_mc_state() { return fatal_error(limits.move_as_error()); } block_limits_ = limits.move_as_ok(); + if (attempt_idx_ == 3) { + LOG(INFO) << "Attempt #3: bytes, gas limits /= 2"; + block_limits_->bytes.multiply_by(0.5); + block_limits_->gas.multiply_by(0.5); + } else if (attempt_idx_ == 4) { + LOG(INFO) << "Attempt #4: bytes, gas limits /= 4"; + block_limits_->bytes.multiply_by(0.25); + block_limits_->gas.multiply_by(0.25); + } LOG(DEBUG) << "block limits: bytes [" << block_limits_->bytes.underload() << ", " << block_limits_->bytes.soft() << ", " << block_limits_->bytes.hard() << "]"; LOG(DEBUG) << "block limits: gas [" << block_limits_->gas.underload() << ", " << block_limits_->gas.soft() << ", " @@ -572,11 +753,14 @@ bool Collator::unpack_last_mc_state() { << " (upgrade validator software?)"; } // TODO: extract start_lt and end_lt from prev_mc_block as well - // std::cerr << " block::gen::ShardState::print_ref(mc_state_root) = "; - // block::gen::t_ShardState.print_ref(std::cerr, mc_state_root, 2); return true; } +/** + * Checks that the current validator set is entitled to create blocks in this shard and has a correct catchain seqno. + * + * @returns True if the current validator set is valid, false otherwise. + */ bool Collator::check_cur_validator_set() { if (is_hardfork_) { return true; @@ -603,6 +787,11 @@ bool Collator::check_cur_validator_set() { return true; } +/** + * Requests the message queues of neighboring shards. + * + * @returns True if the request for neighbor message queues was successful, false otherwise. + */ bool Collator::request_neighbor_msg_queues() { assert(config_ && shard_conf_); auto neighbor_list = shard_conf_->get_neighbor_shard_hash_ids(shard_); @@ -631,8 +820,34 @@ bool Collator::request_neighbor_msg_queues() { return true; } +/** + * Requests the size of the outbound message queue from the previous state(s) if needed. + * +* @returns True if the request was successful, false otherwise. + */ +bool Collator::request_out_msg_queue_size() { + if (have_out_msg_queue_size_in_state_) { + // if after_split then have_out_msg_queue_size_in_state_ is always true, since the size is calculated during split + return true; + } + out_msg_queue_size_ = 0; + for (size_t i = 0; i < prev_blocks.size(); ++i) { + ++pending; + send_closure_later(manager, &ValidatorManager::get_out_msg_queue_size, prev_blocks[i], + [self = get_self(), i](td::Result res) { + td::actor::send_closure(std::move(self), &Collator::got_out_queue_size, i, std::move(res)); + }); + } + return true; +} + +/** + * Handles the result of obtaining the outbound queue for a neighbor. + * + * @param i The index of the neighbor. + * @param res The obtained outbound queue. + */ void Collator::got_neighbor_out_queue(int i, td::Result> res) { - LOG(DEBUG) << "obtained outbound queue for neighbor #" << i; --pending; if (res.is_error()) { fatal_error(res.move_as_error()); @@ -640,6 +855,7 @@ void Collator::got_neighbor_out_queue(int i, td::Result> res) } Ref outq_descr = res.move_as_ok(); block::McShardDescr& descr = neighbors_.at(i); + LOG(WARNING) << "obtained outbound queue for neighbor #" << i << " : " << descr.shard().to_str(); if (outq_descr->get_block_id() != descr.blk_) { LOG(DEBUG) << "outq_descr->id = " << outq_descr->get_block_id().to_str() << " ; descr.id = " << descr.blk_.to_str(); fatal_error( @@ -662,8 +878,10 @@ void Collator::got_neighbor_out_queue(int i, td::Result> res) // unpack ProcessedUpto LOG(DEBUG) << "unpacking ProcessedUpto of neighbor " << descr.blk_.to_str(); if (verbosity >= 2) { - block::gen::t_ProcessedInfo.print(std::cerr, qinfo.proc_info); - qinfo.proc_info->print_rec(std::cerr); + FLOG(INFO) { + block::gen::t_ProcessedInfo.print(sb, qinfo.proc_info); + qinfo.proc_info->print_rec(sb); + }; } descr.processed_upto = block::MsgProcessedUptoCollection::unpack(descr.shard(), qinfo.proc_info); if (!descr.processed_upto) { @@ -689,6 +907,33 @@ void Collator::got_neighbor_out_queue(int i, td::Result> res) check_pending(); } +/** + * Handles the result of obtaining the size of the outbound message queue. + * + * If the block is after merge then the two sizes are added. + * + * @param i The index of the previous block (0 or 1). + * @param res The result object containing the size of the queue. + */ +void Collator::got_out_queue_size(size_t i, td::Result res) { + --pending; + if (res.is_error()) { + fatal_error( + res.move_as_error_prefix(PSTRING() << "failed to get message queue size from prev block #" << i << ": ")); + return; + } + td::uint64 size = res.move_as_ok(); + LOG(WARNING) << "got outbound queue size from prev block #" << i << ": " << size; + out_msg_queue_size_ += size; + check_pending(); +} + +/** + * Unpacks and merges the states of two previous blocks. + * Used if the block is after_merge. + * + * @returns True if the unpacking and merging was successful, false otherwise. + */ bool Collator::unpack_merge_last_state() { LOG(DEBUG) << "unpack/merge last states"; // 0. mechanically merge two ShardStateUnsplit into split_state constructor @@ -727,6 +972,12 @@ bool Collator::unpack_merge_last_state() { return import_shard_state_data(ss0); } +/** + * Unpacks the state of the previous block. + * Used if the block is not after_merge. + * + * @returns True if the unpacking is successful, false otherwise. + */ bool Collator::unpack_last_state() { if (after_merge_) { if (!unpack_merge_last_state()) { @@ -746,6 +997,15 @@ bool Collator::unpack_last_state() { import_shard_state_data(ss); } +/** + * Unpacks the state of a previous block and performs necessary checks. + * + * @param ss The ShardState object to unpack the state into. + * @param blkid The BlockIdExt of the previous block. + * @param prev_state_root The root of the state. + * + * @returns True if the unpacking and checks are successful, false otherwise. + */ bool Collator::unpack_one_last_state(block::ShardState& ss, BlockIdExt blkid, Ref prev_state_root) { auto res = ss.unpack_state_ext(blkid, std::move(prev_state_root), global_id_, prev_mc_block_seqno, after_split_, after_split_ | after_merge_, [self = this](ton::BlockSeqno mc_seqno) { @@ -764,6 +1024,14 @@ bool Collator::unpack_one_last_state(block::ShardState& ss, BlockIdExt blkid, Re return true; } +/** + * Splits the state of previous block. + * Used if the block is after_split. + * + * @param ss The ShardState object representing the previous state. The result is stored here. + * + * @returns True if the split operation is successful, false otherwise. + */ bool Collator::split_last_state(block::ShardState& ss) { LOG(INFO) << "Splitting previous state " << ss.id_.to_str() << " to subshard " << shard_.to_str(); CHECK(after_split_); @@ -785,13 +1053,22 @@ bool Collator::split_last_state(block::ShardState& ss) { return true; } -// SETS: account_dict, shard_libraries_, mc_state_extra -// total_balance_ = old_total_balance_, total_validator_fees_ -// SETS: overload_history_, underload_history_ -// SETS: prev_state_utime_, prev_state_lt_, prev_vert_seqno_ -// SETS: out_msg_queue, processed_upto_, ihr_pending +/** + * Imports the shard state data into the Collator object. + * + * SETS: account_dict = account_dict_estimator_, shard_libraries_, mc_state_extra + * total_balance_ = old_total_balance_, total_validator_fees_ + * SETS: overload_history_, underload_history_ + * SETS: prev_state_utime_, prev_state_lt_, prev_vert_seqno_ + * SETS: out_msg_queue, processed_upto_, ihr_pending + * + * @param ss The ShardState object containing the shard state data. + * + * @returns True if the import was successful, False otherwise. + */ bool Collator::import_shard_state_data(block::ShardState& ss) { account_dict = std::move(ss.account_dict_); + account_dict_estimator_ = std::make_unique(*account_dict); shard_libraries_ = std::move(ss.shard_libraries_); mc_state_extra_ = std::move(ss.mc_state_extra_); overload_history_ = ss.overload_history_; @@ -806,10 +1083,21 @@ bool Collator::import_shard_state_data(block::ShardState& ss) { out_msg_queue_ = std::move(ss.out_msg_queue_); processed_upto_ = std::move(ss.processed_upto_); ihr_pending = std::move(ss.ihr_pending_); + dispatch_queue_ = std::move(ss.dispatch_queue_); block_create_stats_ = std::move(ss.block_create_stats_); + if (ss.out_msg_queue_size_) { + have_out_msg_queue_size_in_state_ = true; + out_msg_queue_size_ = ss.out_msg_queue_size_.value(); + } return true; } +/** + * Adds trivials neighbor after merging two shards. + * Trivial neighbors are the two previous blocks. + * + * @returns True if the operation is successful, false otherwise. + */ bool Collator::add_trivial_neighbor_after_merge() { LOG(DEBUG) << "in add_trivial_neighbor_after_merge()"; CHECK(prev_blocks.size() == 2); @@ -844,6 +1132,12 @@ bool Collator::add_trivial_neighbor_after_merge() { return true; } +/** + * Adds a trivial neighbor. + * A trivial neighbor is the previous block. + * + * @returns True if the operation is successful, false otherwise. + */ bool Collator::add_trivial_neighbor() { LOG(DEBUG) << "in add_trivial_neighbor()"; if (after_merge_) { @@ -979,6 +1273,15 @@ bool Collator::add_trivial_neighbor() { return true; } +/** + * Checks the previous block against the block registered in the masterchain. + * + * @param listed The BlockIdExt of the top block of this shard registered in the masterchain. + * @param prev The BlockIdExt of the previous block. + * @param chk_chain_len Flag indicating whether to check the chain length. + * + * @returns True if the previous block is valid, false otherwise. + */ bool Collator::check_prev_block(const BlockIdExt& listed, const BlockIdExt& prev, bool chk_chain_len) { if (listed.seqno() > prev.seqno()) { return fatal_error(PSTRING() << "cannot generate a shardchain block after previous block " << prev.to_str() @@ -998,6 +1301,14 @@ bool Collator::check_prev_block(const BlockIdExt& listed, const BlockIdExt& prev return true; } +/** + * Checks the previous block against the block registered in the masterchain. + * + * @param listed The BlockIdExt of the top block of this shard registered in the masterchain. + * @param prev The BlockIdExt of the previous block. + * + * @returns True if the previous block is equal to the one registered in the masterchain, false otherwise. + */ bool Collator::check_prev_block_exact(const BlockIdExt& listed, const BlockIdExt& prev) { if (listed != prev) { return fatal_error(PSTRING() << "cannot generate shardchain block for shard " << shard_.to_str() @@ -1008,6 +1319,11 @@ bool Collator::check_prev_block_exact(const BlockIdExt& listed, const BlockIdExt return true; } +/** + * Checks the validity of the shard configuration of the current shard. + * + * @returns True if the shard configuration is valid, false otherwise. + */ bool Collator::check_this_shard_mc_info() { wc_info_ = config_->get_workchain_info(workchain()); if (wc_info_.is_null()) { @@ -1142,14 +1458,27 @@ bool Collator::check_this_shard_mc_info() { return true; } +/** + * Initializes the block limits for the collator. + * + * @returns True if the block limits were successfully initialized, false otherwise. + */ bool Collator::init_block_limits() { CHECK(block_limits_); CHECK(state_usage_tree_); + if (now_ > prev_now_ + 15 && block_limits_->lt_delta.hard() > 200) { + block_limits_->lt_delta = {20, 180, 200}; + } block_limits_->usage_tree = state_usage_tree_.get(); block_limit_status_ = std::make_unique(*block_limits_); return true; } +/** + * Performs pre-initialization steps for the Collator. + * + * @returns True if pre-initialization is successful, false otherwise. + */ bool Collator::do_preinit() { CHECK(prev_blocks.size() == 1U + after_merge_); last_block_seqno = prev_blocks[0].seqno(); @@ -1203,9 +1532,18 @@ bool Collator::do_preinit() { if (!request_neighbor_msg_queues()) { return false; } + if (!request_out_msg_queue_size()) { + return false; + } return true; } +/** + * Adjusts the shard configuration by adding new workchains to the shard configuration in the masterchain state. + * Used in masterchain collator. + * + * @returns True if the shard configuration was successfully adjusted, false otherwise. + */ bool Collator::adjust_shard_config() { CHECK(is_masterchain() && config_ && shard_conf_); const block::WorkchainSet& wset = config_->get_workchain_list(); @@ -1234,12 +1572,30 @@ bool Collator::adjust_shard_config() { return true; } +/** + * Compares two ShardTopBlockDescription references based on their block IDs. + * + * @param a The first ShardTopBlockDescription reference. + * @param b The second ShardTopBlockDescription reference. + * + * @returns True if a is considered less than b, false otherwise. + */ static bool cmp_shard_block_descr_ref(const Ref& a, const Ref& b) { BlockId x = a->block_id().id, y = b->block_id().id; return x.workchain < y.workchain || (x.workchain == y.workchain && (x.shard < y.shard || (x.shard == y.shard && x.seqno > y.seqno))); } +/** + * Stores the fees imported from a shard blocks to `fees_import_dict_`. + * Used in masterchain collator. + * + * @param shard The shard identifier. + * @param fees The fees imported from the block. + * @param created The fee for creating shard blocks. + * + * @returns True if the fees were successfully stored, false otherwise. + */ bool Collator::store_shard_fees(ShardIdFull shard, const block::CurrencyCollection& fees, const block::CurrencyCollection& created) { if (shard.is_valid() && fees.is_valid()) { @@ -1255,6 +1611,14 @@ bool Collator::store_shard_fees(ShardIdFull shard, const block::CurrencyCollecti } } +/** + * Stores the fees imported from a shard blocks to `fees_import_dict_`. + * Used in masterchain collator. + * + * @param descr A reference to the McShardHash object containing the shard information. + * + * @returns True if the shard fees and funds created were successfully stored, false otherwise. + */ bool Collator::store_shard_fees(Ref descr) { CHECK(descr.not_null()); CHECK(descr->fees_collected_.is_valid()); @@ -1263,6 +1627,11 @@ bool Collator::store_shard_fees(Ref descr) { return true; } +/** + * Imports new top shard blocks and updates the shard configuration. + * + * @returns True if the import was successful, false otherwise. + */ bool Collator::import_new_shard_top_blocks() { if (shard_block_descr_.empty()) { return true; @@ -1379,9 +1748,11 @@ bool Collator::import_new_shard_top_blocks() { shard_conf_adjusted_ = true; } if (tb_act && verbosity >= 0) { // DEBUG - LOG(INFO) << "updated shard block configuration to "; - auto csr = shard_conf_->get_root_csr(); - block::gen::t_ShardHashes.print(std::cerr, csr.write()); + FLOG(INFO) { + sb << "updated shard block configuration to "; + auto csr = shard_conf_->get_root_csr(); + block::gen::t_ShardHashes.print(sb, csr); + }; } block::gen::ShardFeeCreated::Record fc; if (!(tlb::csr_unpack(fees_import_dict_->get_root_extra(), @@ -1391,10 +1762,23 @@ bool Collator::import_new_shard_top_blocks() { } LOG(INFO) << "total fees_imported = " << value_flow_.fees_imported.to_str() << " ; out of them, total fees_created = " << import_created_.to_str(); - value_flow_.fees_collected += value_flow_.fees_imported; + block::CurrencyCollection burned = + config_->get_burning_config().calculate_burned_fees(value_flow_.fees_imported - import_created_); + if (!burned.is_valid()) { + return fatal_error("cannot calculate amount of burned imported fees"); + } + value_flow_.burned += burned; + value_flow_.fees_collected += value_flow_.fees_imported - burned; return true; } +/** + * Registers the shard block creators to block_create_count_ + * + * @param creator_list A vector of Bits256 representing the shard block creators. + * + * @returns True if the registration was successful, False otherwise. + */ bool Collator::register_shard_block_creators(std::vector creator_list) { for (const auto& x : creator_list) { LOG(DEBUG) << "registering block creator " << x.to_hex(); @@ -1409,9 +1793,20 @@ bool Collator::register_shard_block_creators(std::vector creator_li return true; } +/** + * Performs pre-initialization and collates the new block. + * + * @returns True if collation is successful, false otherwise. + */ bool Collator::try_collate() { + work_timer_.resume(); + cpu_work_timer_.resume(); + SCOPE_EXIT { + work_timer_.pause(); + cpu_work_timer_.pause(); + }; if (!preinit_complete) { - LOG(DEBUG) << "running do_preinit()"; + LOG(WARNING) << "running do_preinit()"; if (!do_preinit()) { return fatal_error(-667, "error preinitializing data required by collator"); } @@ -1425,6 +1820,7 @@ bool Collator::try_collate() { last_proc_int_msg_.second.set_zero(); first_unproc_int_msg_.first = ~0ULL; first_unproc_int_msg_.second.set_ones(); + old_out_msg_queue_size_ = out_msg_queue_size_; if (is_masterchain()) { LOG(DEBUG) << "getting the list of special smart contracts"; auto res = config_->get_special_smartcontracts(); @@ -1468,6 +1864,14 @@ bool Collator::try_collate() { return do_collate(); } +/** + * Adjusts one entry from the processed up to information using the masterchain state that is referenced in the entry. + * + * @param proc The MsgProcessedUpto object. + * @param owner The shard that the MsgProcessesUpto information is taken from. + * + * @returns True if the processed up to information was successfully adjusted, false otherwise. + */ bool Collator::fix_one_processed_upto(block::MsgProcessedUpto& proc, const ton::ShardIdFull& owner) { if (proc.compute_shard_end_lt) { return true; @@ -1484,6 +1888,13 @@ bool Collator::fix_one_processed_upto(block::MsgProcessedUpto& proc, const ton:: return (bool)proc.compute_shard_end_lt; } +/** + * Adjusts the processed up to collection using the using the auxilliary masterchain states. + * + * @param upto The MsgProcessedUptoCollection to be adjusted. + * + * @returns True if all entries were successfully adjusted, False otherwise. + */ bool Collator::fix_processed_upto(block::MsgProcessedUptoCollection& upto) { for (auto& entry : upto.list) { if (!fix_one_processed_upto(entry, upto.owner)) { @@ -1493,10 +1904,25 @@ bool Collator::fix_processed_upto(block::MsgProcessedUptoCollection& upto) { return true; } +/** + * Initializes the unix time for the new block. + * + * Unix time is set based on the current time, and the timestamps of the previous blocks. + * If the previous block has a timestamp too far in the past then skipping importing external messages and new shard blocks is allowed. + * + * @returns True if the initialization is successful, false otherwise. + */ bool Collator::init_utime() { CHECK(config_); // consider unixtime and lt from previous block(s) of the same shardchain prev_now_ = prev_state_utime_; + // Extend collator timeout if previous block is too old + td::Timestamp new_timeout = td::Timestamp::in(std::min(30.0, (td::Clocks::system() - (double)prev_now_) / 2)); + if (timeout < new_timeout) { + timeout = new_timeout; + alarm_timestamp() = timeout; + } + auto prev = std::max(config_->utime, prev_now_); now_ = std::max(prev + 1, (unsigned)std::time(nullptr)); if (now_ > now_upper_limit_) { @@ -1538,6 +1964,9 @@ bool Collator::init_utime() { return true; } +/** + * Initializes the logical time of the new block. + */ bool Collator::init_lt() { CHECK(config_); start_lt = config_->lt; @@ -1560,91 +1989,33 @@ bool Collator::init_lt() { return true; } +/** + * Fetches and initializes the configuration parameters using the masterchain configuration. + * + * @returns True if the configuration parameters were successfully fetched and initialized, false otherwise. + */ bool Collator::fetch_config_params() { - auto res = impl_fetch_config_params(std::move(config_), &old_mparams_, &storage_prices_, &storage_phase_cfg_, - &rand_seed_, &compute_phase_cfg_, &action_phase_cfg_, &masterchain_create_fee_, - &basechain_create_fee_, workchain(), now_); + auto res = block::FetchConfigParams::fetch_config_params( + *config_, &old_mparams_, &storage_prices_, &storage_phase_cfg_, &rand_seed_, &compute_phase_cfg_, + &action_phase_cfg_, &serialize_cfg_, &masterchain_create_fee_, &basechain_create_fee_, workchain(), now_); if (res.is_error()) { return fatal_error(res.move_as_error()); } - config_ = res.move_as_ok(); + compute_phase_cfg_.libraries = std::make_unique(config_->get_libraries_root(), 256); + defer_out_queue_size_limit_ = std::max(collator_opts_->defer_out_queue_size_limit, + compute_phase_cfg_.size_limits.defer_out_queue_size_limit); + // This one is checked in validate-query + hard_defer_out_queue_size_limit_ = compute_phase_cfg_.size_limits.defer_out_queue_size_limit; return true; } -td::Result> Collator::impl_fetch_config_params( - std::unique_ptr config, Ref* old_mparams, - std::vector* storage_prices, block::StoragePhaseConfig* storage_phase_cfg, - td::BitArray<256>* rand_seed, block::ComputePhaseConfig* compute_phase_cfg, - block::ActionPhaseConfig* action_phase_cfg, td::RefInt256* masterchain_create_fee, - td::RefInt256* basechain_create_fee, WorkchainId wc, UnixTime now) { - *old_mparams = config->get_config_param(9); - { - auto res = config->get_storage_prices(); - if (res.is_error()) { - return res.move_as_error(); - } - *storage_prices = res.move_as_ok(); - } - { - // generate rand seed - prng::rand_gen().strong_rand_bytes(rand_seed->data(), 32); - LOG(DEBUG) << "block random seed set to " << rand_seed->to_hex(); - } - TRY_RESULT(size_limits, config->get_size_limits_config()); - { - // compute compute_phase_cfg / storage_phase_cfg - auto cell = config->get_config_param(wc == ton::masterchainId ? 20 : 21); - if (cell.is_null()) { - return td::Status::Error(-668, "cannot fetch current gas prices and limits from masterchain configuration"); - } - if (!compute_phase_cfg->parse_GasLimitsPrices(std::move(cell), storage_phase_cfg->freeze_due_limit, - storage_phase_cfg->delete_due_limit)) { - return td::Status::Error(-668, "cannot unpack current gas prices and limits from masterchain configuration"); - } - compute_phase_cfg->block_rand_seed = *rand_seed; - compute_phase_cfg->libraries = std::make_unique(config->get_libraries_root(), 256); - compute_phase_cfg->max_vm_data_depth = size_limits.max_vm_data_depth; - compute_phase_cfg->global_config = config->get_root_cell(); - compute_phase_cfg->suspended_addresses = config->get_suspended_addresses(now); - } - { - // compute action_phase_cfg - block::gen::MsgForwardPrices::Record rec; - auto cell = config->get_config_param(24); - if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { - return td::Status::Error(-668, "cannot fetch masterchain message transfer prices from masterchain configuration"); - } - action_phase_cfg->fwd_mc = - block::MsgPrices{rec.lump_price, rec.bit_price, rec.cell_price, rec.ihr_price_factor, - (unsigned)rec.first_frac, (unsigned)rec.next_frac}; - cell = config->get_config_param(25); - if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) { - return td::Status::Error(-668, "cannot fetch standard message transfer prices from masterchain configuration"); - } - action_phase_cfg->fwd_std = - block::MsgPrices{rec.lump_price, rec.bit_price, rec.cell_price, rec.ihr_price_factor, - (unsigned)rec.first_frac, (unsigned)rec.next_frac}; - action_phase_cfg->workchains = &config->get_workchain_list(); - action_phase_cfg->bounce_msg_body = (config->has_capability(ton::capBounceMsgBody) ? 256 : 0); - action_phase_cfg->size_limits = size_limits; - } - { - // fetch block_grams_created - auto cell = config->get_config_param(14); - if (cell.is_null()) { - *basechain_create_fee = *masterchain_create_fee = td::zero_refint(); - } else { - block::gen::BlockCreateFees::Record create_fees; - if (!(tlb::unpack_cell(cell, create_fees) && - block::tlb::t_Grams.as_integer_to(create_fees.masterchain_block_fee, *masterchain_create_fee) && - block::tlb::t_Grams.as_integer_to(create_fees.basechain_block_fee, *basechain_create_fee))) { - return td::Status::Error(-668, "cannot unpack BlockCreateFees from configuration parameter #14"); - } - } - } - return std::move(config); -} - +/** + * Computes the amount of extra currencies to be minted. + * + * @param to_mint A reference to the CurrencyCollection object to store the minted amount. + * + * @returns True if the computation is successful, false otherwise. + */ bool Collator::compute_minted_amount(block::CurrencyCollection& to_mint) { if (!is_masterchain()) { return to_mint.set_zero(); @@ -1695,6 +2066,11 @@ bool Collator::compute_minted_amount(block::CurrencyCollection& to_mint) { return true; } +/** + * Initializes value_flow_ and computes fees for creating the new block. + * + * @returns True if the initialization is successful, false otherwise. + */ bool Collator::init_value_create() { value_flow_.created.set_zero(); value_flow_.minted.set_zero(); @@ -1725,11 +2101,14 @@ bool Collator::init_value_create() { return true; } +/** + * Performs the collation of the new block. + */ bool Collator::do_collate() { // After do_collate started it will not be interrupted by timeout alarm_timestamp() = td::Timestamp::never(); - LOG(DEBUG) << "do_collate() : start"; + LOG(WARNING) << "do_collate() : start"; if (!fetch_config_params()) { return fatal_error("cannot fetch required configuration parameters from masterchain state"); } @@ -1740,6 +2119,7 @@ bool Collator::do_collate() { if (max_lt == start_lt) { ++max_lt; } + allow_repeat_collation_ = true; // NB: interchanged 1.2 and 1.1 (is this always correct?) // 1.1. re-adjust neighbors' out_msg_queues (for oneself) if (!add_trivial_neighbor()) { @@ -1757,6 +2137,11 @@ bool Collator::do_collate() { if (!init_value_create()) { return fatal_error("cannot compute the value to be created / minted / recovered"); } + // 2-. take messages from dispatch queue + LOG(INFO) << "process dispatch queue"; + if (!process_dispatch_queue()) { + return fatal_error("cannot process dispatch queue"); + } // 2. tick transactions LOG(INFO) << "create tick transactions"; if (!create_ticktock_transactions(2)) { @@ -1848,6 +2233,14 @@ bool Collator::do_collate() { return true; } +/** + * Dequeues an outbound message from the message queue of this shard. + * + * @param msg_envelope The message envelope to dequeue. + * @param delivered_lt The logical time at which the message was delivered. + * + * @returns True if the message was successfully dequeued, false otherwise. + */ bool Collator::dequeue_message(Ref msg_envelope, ton::LogicalTime delivered_lt) { LOG(DEBUG) << "dequeueing outbound message"; vm::CellBuilder cb; @@ -1867,82 +2260,182 @@ bool Collator::dequeue_message(Ref msg_envelope, ton::LogicalTime deli } } +/** + * Cleans up the outbound message queue by removing messages that have already been imported by neighbors. + * + * Cleanup may be interrupted early if it takes too long. + * + * @returns True if the cleanup operation was successful, false otherwise. + */ bool Collator::out_msg_queue_cleanup() { LOG(INFO) << "cleaning outbound queue from messages already imported by neighbors"; if (verbosity >= 2) { - auto rt = out_msg_queue_->get_root(); - std::cerr << "old out_msg_queue is "; - block::gen::t_OutMsgQueue.print(std::cerr, *rt); - rt->print_rec(std::cerr); - } - for (const auto& nb : neighbors_) { - if (!nb.is_disabled() && (!nb.processed_upto || !nb.processed_upto->can_check_processed())) { - return fatal_error(-667, PSTRING() << "internal error: no info for checking processed messages from neighbor " - << nb.blk_.to_str()); - } + FLOG(INFO) { + auto rt = out_msg_queue_->get_root(); + sb << "old out_msg_queue is "; + block::gen::t_OutMsgQueue.print(sb, rt); + rt->print_rec(sb); + }; } - auto res = out_msg_queue_->filter([&](vm::CellSlice& cs, td::ConstBitPtr key, int n) -> int { - assert(n == 352); - // LOG(DEBUG) << "key is " << key.to_hex(n); - if (block_full_) { - LOG(WARNING) << "BLOCK FULL while cleaning up outbound queue, cleanup completed only partially"; - outq_cleanup_partial_ = true; - return (1 << 30) + 1; // retain all remaining outbound queue entries including this one without processing - } - block::EnqueuedMsgDescr enq_msg_descr; - unsigned long long created_lt; - if (!(cs.fetch_ulong_bool(64, created_lt) // augmentation - && enq_msg_descr.unpack(cs) // unpack EnqueuedMsg - && enq_msg_descr.check_key(key) // check key - && enq_msg_descr.lt_ == created_lt)) { - LOG(ERROR) << "cannot unpack EnqueuedMsg with key " << key.to_hex(n); - return -1; - } - LOG(DEBUG) << "scanning outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," - << enq_msg_descr.hash_.to_hex() << ") enqueued_lt=" << enq_msg_descr.enqueued_lt_; - bool delivered = false; - ton::LogicalTime deliver_lt = 0; - for (const auto& neighbor : neighbors_) { - // could look up neighbor with shard containing enq_msg_descr.next_prefix more efficiently - // (instead of checking all neighbors) - if (!neighbor.is_disabled() && neighbor.processed_upto->already_processed(enq_msg_descr)) { - delivered = true; - deliver_lt = neighbor.end_lt(); - break; + if (after_merge_) { + // We need to clean the whole queue after merge + // Queue is not too big, see const MERGE_MAX_QUEUE_SIZE + for (const auto& nb : neighbors_) { + if (!nb.is_disabled() && (!nb.processed_upto || !nb.processed_upto->can_check_processed())) { + return fatal_error(-667, PSTRING() << "internal error: no info for checking processed messages from neighbor " + << nb.blk_.to_str()); } } - if (delivered) { - LOG(DEBUG) << "outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," << enq_msg_descr.hash_.to_hex() - << ") enqueued_lt=" << enq_msg_descr.enqueued_lt_ << " has been already delivered, dequeueing"; - if (!dequeue_message(std::move(enq_msg_descr.msg_env_), deliver_lt)) { - fatal_error(PSTRING() << "cannot dequeue outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," - << enq_msg_descr.hash_.to_hex() << ") by inserting a msg_export_deq record"); + td::uint32 deleted = 0; + auto res = out_msg_queue_->filter([&](vm::CellSlice& cs, td::ConstBitPtr key, int n) -> int { + assert(n == 352); + block::EnqueuedMsgDescr enq_msg_descr; + unsigned long long created_lt; + if (!(cs.fetch_ulong_bool(64, created_lt) // augmentation + && enq_msg_descr.unpack(cs) // unpack EnqueuedMsg + && enq_msg_descr.check_key(key) // check key + && enq_msg_descr.lt_ == created_lt)) { + LOG(ERROR) << "cannot unpack EnqueuedMsg with key " << key.to_hex(n); return -1; } - register_out_msg_queue_op(); - if (!block_limit_status_->fits(block::ParamLimits::cl_normal)) { - block_full_ = true; + LOG(DEBUG) << "scanning outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," + << enq_msg_descr.hash_.to_hex() << ") enqueued_lt=" << enq_msg_descr.enqueued_lt_; + bool delivered = false; + ton::LogicalTime deliver_lt = 0; + for (const auto& neighbor : neighbors_) { + // could look up neighbor with shard containing enq_msg_descr.next_prefix more efficiently + // (instead of checking all neighbors) + if (!neighbor.is_disabled() && neighbor.processed_upto->already_processed(enq_msg_descr)) { + delivered = true; + deliver_lt = neighbor.end_lt(); + break; + } } + if (delivered) { + ++deleted; + CHECK(out_msg_queue_size_ > 0); + --out_msg_queue_size_; + LOG(DEBUG) << "outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," << enq_msg_descr.hash_.to_hex() + << ") enqueued_lt=" << enq_msg_descr.enqueued_lt_ << " has been already delivered, dequeueing"; + if (!dequeue_message(std::move(enq_msg_descr.msg_env_), deliver_lt)) { + fatal_error(PSTRING() << "cannot dequeue outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," + << enq_msg_descr.hash_.to_hex() << ") by inserting a msg_export_deq record"); + return -1; + } + register_out_msg_queue_op(); + if (!block_limit_status_->fits(block::ParamLimits::cl_normal)) { + block_full_ = true; + block_limit_class_ = std::max(block_limit_class_, block_limit_status_->classify()); + } + } + return !delivered; + }); + LOG(WARNING) << "deleted " << deleted << " messages from out_msg_queue after merge, remaining queue size is " + << out_msg_queue_size_; + if (res < 0) { + return fatal_error("error scanning/updating OutMsgQueue"); } - return !delivered; - }); - LOG(DEBUG) << "deleted " << res << " messages from out_msg_queue"; - if (res < 0) { - return fatal_error("error scanning/updating OutMsgQueue"); + } else { + std::vector> queue_parts; + + block::OutputQueueMerger::Neighbor this_queue{BlockIdExt{new_id} /* block id is only used for logs */, + out_msg_queue_->get_root_cell()}; + for (const auto& nb : neighbors_) { + if (nb.is_disabled()) { + continue; + } + if (!nb.processed_upto || !nb.processed_upto->can_check_processed()) { + return fatal_error(-667, PSTRING() << "internal error: no info for checking processed messages from neighbor " + << nb.blk_.to_str()); + } + queue_parts.emplace_back(block::OutputQueueMerger{nb.shard(), {this_queue}}, &nb); + } + + size_t i = 0; + td::uint32 deleted = 0; + while (!queue_parts.empty()) { + if (block_full_) { + LOG(WARNING) << "BLOCK FULL while cleaning up outbound queue, cleanup completed only partially"; + break; + } + if (queue_cleanup_timeout_.is_in_past(td::Timestamp::now())) { + LOG(WARNING) << "cleaning up outbound queue takes too long, ending"; + break; + } + if (!check_cancelled()) { + return false; + } + if (i == queue_parts.size()) { + i = 0; + } + auto& queue = queue_parts.at(i).first; + auto nb = queue_parts.at(i).second; + auto kv = queue.extract_cur(); + if (kv) { + block::EnqueuedMsgDescr enq_msg_descr; + if (!(enq_msg_descr.unpack(kv->msg.write()) // unpack EnqueuedMsg + && enq_msg_descr.check_key(kv->key.cbits()) // check key + )) { + return fatal_error(PSTRING() << "error scanning/updating OutMsgQueue: cannot unpack EnqueuedMsg with key " + << kv->key.to_hex()); + } + if (nb->processed_upto->already_processed(enq_msg_descr)) { + LOG(DEBUG) << "scanning outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," + << enq_msg_descr.hash_.to_hex() << ") enqueued_lt=" << enq_msg_descr.enqueued_lt_ + << ": message has been already delivered, dequeueing"; + ++deleted; + CHECK(out_msg_queue_size_ > 0); + --out_msg_queue_size_; + out_msg_queue_->lookup_delete_with_extra(kv->key.cbits(), kv->key_len); + if (!dequeue_message(std::move(enq_msg_descr.msg_env_), nb->end_lt())) { + return fatal_error(PSTRING() << "cannot dequeue outbound message with (lt,hash)=(" << enq_msg_descr.lt_ + << "," << enq_msg_descr.hash_.to_hex() + << ") by inserting a msg_export_deq record"); + } + register_out_msg_queue_op(); + if (!block_limit_status_->fits(block::ParamLimits::cl_normal)) { + block_full_ = true; + block_limit_class_ = std::max(block_limit_class_, block_limit_status_->classify()); + } + queue.next(); + ++i; + continue; + } else { + LOG(DEBUG) << "scanning outbound message with (lt,hash)=(" << enq_msg_descr.lt_ << "," + << enq_msg_descr.hash_.to_hex() << ") enqueued_lt=" << enq_msg_descr.enqueued_lt_ + << ": message has not been delivered"; + } + } + LOG(DEBUG) << "no more unprocessed messages to shard " << nb->shard().to_str(); + std::swap(queue_parts[i], queue_parts.back()); + queue_parts.pop_back(); + } + LOG(WARNING) << "deleted " << deleted << " messages from out_msg_queue, remaining queue size is " + << out_msg_queue_size_; } - auto rt = out_msg_queue_->get_root(); if (verbosity >= 2) { - std::cerr << "new out_msg_queue is "; - block::gen::t_OutMsgQueue.print(std::cerr, *rt); - rt->print_rec(std::cerr); + FLOG(INFO) { + auto rt = out_msg_queue_->get_root(); + sb << "new out_msg_queue is "; + block::gen::t_OutMsgQueue.print(sb, rt); + rt->print_rec(sb); + }; } - // CHECK(block::gen::t_OutMsgQueue.validate_upto(100000, *rt)); // DEBUG, comment later if SLOW return register_out_msg_queue_op(true); } +/** + * Creates a new Account object from the given address and serialized account data. + * + * @param addr A pointer to the 256-bit address of the account. + * @param account A cell slice with an account serialized using ShardAccount TLB-scheme. + * @param force_create A flag indicating whether to force the creation of a new account if `account` is null. + * + * @returns A unique pointer to the created Account object, or nullptr if the creation failed. + */ std::unique_ptr Collator::make_account_from(td::ConstBitPtr addr, Ref account, - Ref extra, bool force_create) { + bool force_create) { if (account.is_null() && !force_create) { return nullptr; } @@ -1951,19 +2444,35 @@ std::unique_ptr Collator::make_account_from(td::ConstBitPtr addr if (!ptr->init_new(now_)) { return nullptr; } - } else if (!ptr->unpack(std::move(account), std::move(extra), now_, - is_masterchain() && config_->is_special_smartcontract(addr))) { + } else if (!ptr->unpack(std::move(account), now_, is_masterchain() && config_->is_special_smartcontract(addr))) { return nullptr; } ptr->block_lt = start_lt; return ptr; } +/** + * Looks up an account in the Collator's account map. + * + * @param addr A pointer to the 256-bit address of the account to be looked up. + * + * @returns A pointer to the Account object if found, otherwise returns nullptr. + */ block::Account* Collator::lookup_account(td::ConstBitPtr addr) const { auto found = accounts.find(addr); return found != accounts.end() ? found->second.get() : nullptr; } +/** + * Retreives an Account object from the data in the shard state. + * Accounts are cached in the Collator's map. + * + * @param addr The 256-bit address of the account. + * @param force_create Flag indicating whether to create a new account if it does not exist. + * + * @returns A Result object containing a pointer to the account if found or created successfully, or an error status. + * Returns nullptr if account does not exist and not force_create. + */ td::Result Collator::make_account(td::ConstBitPtr addr, bool force_create) { auto found = lookup_account(addr); if (found) { @@ -1975,7 +2484,7 @@ td::Result Collator::make_account(td::ConstBitPtr addr, bool fo return nullptr; } } - auto new_acc = make_account_from(addr, std::move(dict_entry.first), std::move(dict_entry.second), force_create); + auto new_acc = make_account_from(addr, std::move(dict_entry.first), force_create); if (!new_acc) { return td::Status::Error(PSTRING() << "cannot load account " << addr.to_hex(256) << " from previous state"); } @@ -1991,6 +2500,11 @@ td::Result Collator::make_account(td::ConstBitPtr addr, bool fo return ins.first->second.get(); } +/** + * Combines account transactions and updates the ShardAccountBlocks and ShardAccounts. + * + * @returns True if the operation is successful, false otherwise. + */ bool Collator::combine_account_transactions() { vm::AugmentedDictionary dict{256, block::tlb::aug_ShardAccountBlocks}; for (auto& z : accounts) { @@ -2005,19 +2519,27 @@ bool Collator::combine_account_transactions() { auto cell = cb.finalize(); auto csr = vm::load_cell_slice_ref(cell); if (verbosity > 2) { - std::cerr << "new AccountBlock for " << z.first.to_hex() << ": "; - block::gen::t_AccountBlock.print_ref(std::cerr, cell); - csr->print_rec(std::cerr); + FLOG(INFO) { + sb << "new AccountBlock for " << z.first.to_hex() << ": "; + block::gen::t_AccountBlock.print_ref(sb, cell); + csr->print_rec(sb); + }; } if (!block::gen::t_AccountBlock.validate_ref(100000, cell)) { - block::gen::t_AccountBlock.print_ref(std::cerr, cell); - csr->print_rec(std::cerr); + FLOG(WARNING) { + sb << "AccountBlock failed to pass automatic validation tests: "; + block::gen::t_AccountBlock.print_ref(sb, cell); + csr->print_rec(sb); + }; return fatal_error(std::string{"new AccountBlock for "} + z.first.to_hex() + " failed to pass automatic validation tests"); } if (!block::tlb::t_AccountBlock.validate_ref(100000, cell)) { - block::gen::t_AccountBlock.print_ref(std::cerr, cell); - csr->print_rec(std::cerr); + FLOG(WARNING) { + sb << "AccountBlock failed to pass handwritten validation tests: "; + block::gen::t_AccountBlock.print_ref(sb, cell); + csr->print_rec(sb); + }; return fatal_error(std::string{"new AccountBlock for "} + z.first.to_hex() + " failed to pass handwritten validation tests"); } @@ -2042,8 +2564,10 @@ bool Collator::combine_account_transactions() { } else if (acc.status == block::Account::acc_nonexist) { // account deleted if (verbosity > 2) { - std::cerr << "deleting account " << acc.addr.to_hex() << " with empty new value "; - block::gen::t_Account.print_ref(std::cerr, acc.total_state); + FLOG(INFO) { + sb << "deleting account " << acc.addr.to_hex() << " with empty new value "; + block::gen::t_Account.print_ref(sb, acc.total_state); + }; } if (account_dict->lookup_delete(acc.addr).is_null()) { return fatal_error(std::string{"cannot delete account "} + acc.addr.to_hex() + " from ShardAccounts"); @@ -2051,8 +2575,10 @@ bool Collator::combine_account_transactions() { } else { // existing account modified if (verbosity > 4) { - std::cerr << "modifying account " << acc.addr.to_hex() << " to "; - block::gen::t_Account.print_ref(std::cerr, acc.total_state); + FLOG(INFO) { + sb << "modifying account " << acc.addr.to_hex() << " to "; + block::gen::t_Account.print_ref(sb, acc.total_state); + }; } if (!(cb.store_ref_bool(acc.total_state) // account_descr$_ account:^Account && cb.store_bits_bool(acc.last_trans_hash_) // last_trans_hash:bits256 @@ -2075,9 +2601,11 @@ bool Collator::combine_account_transactions() { return fatal_error("cannot serialize ShardAccountBlocks"); } if (verbosity > 2) { - std::cerr << "new ShardAccountBlocks: "; - block::gen::t_ShardAccountBlocks.print_ref(std::cerr, shard_account_blocks_); - vm::load_cell_slice(shard_account_blocks_).print_rec(std::cerr); + FLOG(INFO) { + sb << "new ShardAccountBlocks: "; + block::gen::t_ShardAccountBlocks.print_ref(sb, shard_account_blocks_); + vm::load_cell_slice(shard_account_blocks_).print_rec(sb); + }; } if (!block::gen::t_ShardAccountBlocks.validate_ref(100000, shard_account_blocks_)) { return fatal_error("new ShardAccountBlocks failed to pass automatic validity tests"); @@ -2087,9 +2615,11 @@ bool Collator::combine_account_transactions() { } auto shard_accounts = account_dict->get_root(); if (verbosity > 2) { - std::cerr << "new ShardAccounts: "; - block::gen::t_ShardAccounts.print(std::cerr, *shard_accounts); - shard_accounts->print_rec(std::cerr); + FLOG(INFO) { + sb << "new ShardAccounts: "; + block::gen::t_ShardAccounts.print(sb, shard_accounts); + shard_accounts->print_rec(sb); + }; } if (verify >= 2) { LOG(INFO) << "verifying new ShardAccounts"; @@ -2103,6 +2633,15 @@ bool Collator::combine_account_transactions() { return true; } +/** + * Creates a special transaction to recover a specified amount of currency to a destination address. + * + * @param amount The amount of currency to recover. + * @param dest_addr_cell The cell containing the destination address. + * @param in_msg The reference to the input message. + * + * @returns True if the special transaction was created successfully, false otherwise. + */ bool Collator::create_special_transaction(block::CurrencyCollection amount, Ref dest_addr_cell, Ref& in_msg) { if (amount.is_zero()) { @@ -2131,11 +2670,13 @@ bool Collator::create_special_transaction(block::CurrencyCollection amount, Ref< addr.to_hex()); } if (verbosity >= 4) { - block::gen::t_Message_Any.print_ref(std::cerr, msg); + FLOG(INFO) { + block::gen::t_Message_Any.print_ref(sb, msg); + }; } CHECK(block::gen::t_Message_Any.validate_ref(msg)); CHECK(block::tlb::t_Message.validate_ref(msg)); - if (process_one_new_message(block::NewOutMsg{lt, msg, Ref{}}, false, &in_msg) != 1) { + if (process_one_new_message(block::NewOutMsg{lt, msg, Ref{}, 0}, false, &in_msg) != 1) { return fatal_error("cannot generate special transaction for recovering "s + amount.to_str() + " to account " + addr.to_hex()); } @@ -2143,12 +2684,27 @@ bool Collator::create_special_transaction(block::CurrencyCollection amount, Ref< return true; } +/** + * Creates special transactions for retreiving fees and minted currencies. + * Used in masterchain collator. + * + * @returns True if both special transactions were + */ bool Collator::create_special_transactions() { CHECK(is_masterchain()); return create_special_transaction(value_flow_.recovered, config_->get_config_param(3, 1), recover_create_msg_) && create_special_transaction(value_flow_.minted, config_->get_config_param(2, 0), mint_msg_); } +/** + * Creates a tick-tock transaction for a given smart contract. + * + * @param smc_addr The address of the smart contract. + * @param req_start_lt The requested start logical time for the transaction. + * @param mask The value indicating whether the thansaction is tick (mask == 2) or tock (mask == 1). + * + * @returns True if the transaction was created successfully, false otherwise. + */ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, ton::LogicalTime req_start_lt, int mask) { auto acc_res = make_account(smc_addr.cbits(), false); @@ -2162,13 +2718,18 @@ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, t return true; } req_start_lt = std::max(req_start_lt, start_lt + 1); + auto it = last_dispatch_queue_emitted_lt_.find(acc->addr); + if (it != last_dispatch_queue_emitted_lt_.end()) { + req_start_lt = std::max(req_start_lt, it->second + 1); + } if (acc->last_trans_end_lt_ >= start_lt && acc->transactions.empty()) { return fatal_error(td::Status::Error(-666, PSTRING() << "last transaction time in the state of account " << workchain() << ":" << smc_addr.to_hex() << " is too large")); } - std::unique_ptr trans = std::make_unique( - *acc, mask == 2 ? block::Transaction::tr_tick : block::Transaction::tr_tock, req_start_lt, now_); + std::unique_ptr trans = std::make_unique( + *acc, mask == 2 ? block::transaction::Transaction::tr_tick : block::transaction::Transaction::tr_tock, + req_start_lt, now_); if (!trans->prepare_storage_phase(storage_phase_cfg_, true)) { return fatal_error(td::Status::Error( -666, std::string{"cannot create storage phase of a new transaction for smart contract "} + smc_addr.to_hex())); @@ -2186,23 +2747,39 @@ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, t return fatal_error(td::Status::Error( -666, std::string{"cannot create action phase of a new transaction for smart contract "} + smc_addr.to_hex())); } - if (!trans->serialize()) { + if (!trans->serialize(serialize_cfg_)) { return fatal_error(td::Status::Error( -666, std::string{"cannot serialize new transaction for smart contract "} + smc_addr.to_hex())); } - if (!trans->update_limits(*block_limit_status_)) { + if (!trans->update_limits(*block_limit_status_, /* with_gas = */ false)) { return fatal_error(-666, "cannot update block limit status to include the new transaction"); } if (trans->commit(*acc).is_null()) { return fatal_error( td::Status::Error(-666, std::string{"cannot commit new transaction for smart contract "} + smc_addr.to_hex())); } + if (!update_account_dict_estimation(*trans)) { + return fatal_error(-666, "cannot update account dict size estimation"); + } update_max_lt(acc->last_trans_end_lt_); - register_new_msgs(*trans); + block::MsgMetadata new_msg_metadata{0, acc->workchain, acc->addr, trans->start_lt}; + register_new_msgs(*trans, std::move(new_msg_metadata)); return true; } -Ref Collator::create_ordinary_transaction(Ref msg_root) { +/** + * Creates an ordinary transaction using a given message. + * + * @param msg_root The root of the message to be processed serialized using Message TLB-scheme. + * @param msg_metadata Metadata of the inbound message. + * @param after_lt Transaction lt will be grater than after_lt. Used for deferred messages. + * @param is_special_tx True if creating a special transaction (mint/recover), false otherwise. + * + * @returns The root of the serialized transaction, or an empty reference if the transaction creation fails. + */ +Ref Collator::create_ordinary_transaction(Ref msg_root, + td::optional msg_metadata, LogicalTime after_lt, + bool is_special_tx) { ton::StdSmcAddress addr; auto cs = vm::load_cell_slice(msg_root); bool external; @@ -2246,8 +2823,15 @@ Ref Collator::create_ordinary_transaction(Ref msg_root) { block::Account* acc = acc_res.move_as_ok(); assert(acc); + if (external) { + after_lt = std::max(after_lt, last_proc_int_msg_.first); + } + auto it = last_dispatch_queue_emitted_lt_.find(acc->addr); + if (it != last_dispatch_queue_emitted_lt_.end()) { + after_lt = std::max(after_lt, it->second); + } auto res = impl_create_ordinary_transaction(msg_root, acc, now_, start_lt, &storage_phase_cfg_, &compute_phase_cfg_, - &action_phase_cfg_, external, last_proc_int_msg_.first); + &action_phase_cfg_, &serialize_cfg_, external, after_lt); if (res.is_error()) { auto error = res.move_as_error(); if (error.code() == -701) { @@ -2258,9 +2842,10 @@ Ref Collator::create_ordinary_transaction(Ref msg_root) { fatal_error(std::move(error)); return {}; } - std::unique_ptr trans = res.move_as_ok(); + std::unique_ptr trans = res.move_as_ok(); - if (!trans->update_limits(*block_limit_status_)) { + if (!trans->update_limits(*block_limit_status_, + /* with_gas = */ !(is_special_tx && compute_phase_cfg_.special_gas_full))) { fatal_error("cannot update block limit status to include the new transaction"); return {}; } @@ -2269,30 +2854,61 @@ Ref Collator::create_ordinary_transaction(Ref msg_root) { fatal_error("cannot commit new transaction for smart contract "s + addr.to_hex()); return {}; } + if (!update_account_dict_estimation(*trans)) { + fatal_error("cannot update account dict size estimation"); + return {}; + } - register_new_msgs(*trans); + td::optional new_msg_metadata; + if (external || is_special_tx) { + new_msg_metadata = block::MsgMetadata{0, acc->workchain, acc->addr, trans->start_lt}; + } else if (msg_metadata) { + new_msg_metadata = std::move(msg_metadata); + ++new_msg_metadata.value().depth; + } + register_new_msgs(*trans, std::move(new_msg_metadata)); update_max_lt(acc->last_trans_end_lt_); + value_flow_.burned += trans->blackhole_burned; return trans_root; } -// If td::status::error_code == 669 - Fatal Error block can not be produced -// if td::status::error_code == 701 - Transaction can not be included into block, but it's ok (external or too early internal) -td::Result> Collator::impl_create_ordinary_transaction( - Ref msg_root, block::Account* acc, UnixTime utime, LogicalTime lt, - block::StoragePhaseConfig* storage_phase_cfg, block::ComputePhaseConfig* compute_phase_cfg, - block::ActionPhaseConfig* action_phase_cfg, bool external, LogicalTime after_lt) { +/** + * Creates an ordinary transaction using given parameters. + * + * @param msg_root The root of the message to be processed serialized using Message TLB-scheme. + * @param acc The account for which the transaction is being created. + * @param utime The Unix time of the transaction. + * @param lt The minimal logical time of the transaction. + * @param storage_phase_cfg The configuration for the storage phase of the transaction. + * @param compute_phase_cfg The configuration for the compute phase of the transaction. + * @param action_phase_cfg The configuration for the action phase of the transaction. + * @param serialize_cfg The configuration for the serialization of the transaction. + * @param external Flag indicating if the message is external. + * @param after_lt The logical time after which the transaction should occur. Used only for external messages. + * + * @returns A Result object containing the created transaction. + * Returns error_code == 669 if the error is fatal and the block can not be produced. + * Returns error_code == 701 if the transaction can not be included into block, but it's ok (external or too early internal). + */ +td::Result> Collator::impl_create_ordinary_transaction(Ref msg_root, + block::Account* acc, + UnixTime utime, LogicalTime lt, + block::StoragePhaseConfig* storage_phase_cfg, + block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, + block::SerializeConfig* serialize_cfg, + bool external, LogicalTime after_lt) { if (acc->last_trans_end_lt_ >= lt && acc->transactions.empty()) { return td::Status::Error(-669, PSTRING() << "last transaction time in the state of account " << acc->workchain << ":" << acc->addr.to_hex() << " is too large"); } auto trans_min_lt = lt; - if (external) { - // transactions processing external messages must have lt larger than all processed internal messages - trans_min_lt = std::max(trans_min_lt, after_lt); - } + // transactions processing external messages must have lt larger than all processed internal messages + // if account has deferred message processed in this block, the next transaction should have lt > emitted_lt + trans_min_lt = std::max(trans_min_lt, after_lt); - std::unique_ptr trans = - std::make_unique(*acc, block::Transaction::tr_ord, trans_min_lt + 1, utime, msg_root); + std::unique_ptr trans = std::make_unique( + *acc, block::transaction::Transaction::tr_ord, trans_min_lt + 1, utime, msg_root); bool ihr_delivered = false; // FIXME if (!trans->unpack_input_msg(ihr_delivered, action_phase_cfg)) { if (external) { @@ -2342,17 +2958,23 @@ td::Result> Collator::impl_create_ordinary_t return td::Status::Error( -669, "cannot create action phase of a new transaction for smart contract "s + acc->addr.to_hex()); } - if (trans->bounce_enabled && (!trans->compute_phase->success || trans->action_phase->state_size_too_big) && + if (trans->bounce_enabled && + (!trans->compute_phase->success || trans->action_phase->state_exceeds_limits || trans->action_phase->bounce) && !trans->prepare_bounce_phase(*action_phase_cfg)) { return td::Status::Error( -669, "cannot create bounce phase of a new transaction for smart contract "s + acc->addr.to_hex()); } - if (!trans->serialize()) { + if (!trans->serialize(*serialize_cfg)) { return td::Status::Error(-669, "cannot serialize new transaction for smart contract "s + acc->addr.to_hex()); } return std::move(trans); } +/** + * Updates the maximum logical time if the given logical time is greater than the current maximum logical time. + * + * @param lt The logical time to be compared. + */ void Collator::update_max_lt(ton::LogicalTime lt) { CHECK(lt >= start_lt); if (lt > max_lt) { @@ -2360,6 +2982,13 @@ void Collator::update_max_lt(ton::LogicalTime lt) { } } +/** + * Updates information on the last processed internal message with a new logical time and hash. + * + * @param new_lt_hash The new logical time and hash pair. + * + * @returns True if the last processed internal message was successfully updated, false otherwise. + */ bool Collator::update_last_proc_int_msg(const std::pair& new_lt_hash) { if (last_proc_int_msg_ < new_lt_hash) { last_proc_int_msg_ = new_lt_hash; @@ -2374,6 +3003,14 @@ bool Collator::update_last_proc_int_msg(const std::pair addr_ref) const { return is_our_address(block::tlb::t_MsgAddressInt.get_prefix(std::move(addr_ref))); } +/** + * Checks if the given account ID prefix belongs to the current shard. + * + * @param addr_pfx The account ID prefix to check. + * + * @returns True if the account ID prefix belongs to the current shard, False otherwise. + */ bool Collator::is_our_address(ton::AccountIdPrefixFull addr_pfx) const { return ton::shard_contains(shard_, addr_pfx); } +/** + * Checks if the given address belongs to the current shard. + * + * @param addr The address to check. + * + * @returns True if the address belongs to the current shard, False otherwise. + */ bool Collator::is_our_address(const ton::StdSmcAddress& addr) const { return ton::shard_contains(get_shard(), addr); } -// 1 = processed, 0 = enqueued, 3 = processed, all future messages must be enqueued +/** + * Processes a message generated in this block or a message from DispatchQueue. + * + * @param msg The new message to be processed. + * @param enqueue_only Flag indicating whether the message should only be enqueued. + * @param is_special New message if creating a special transaction, nullptr otherwise. + * + * @returns Returns: + * 0 - message was enqueued. + * 1 - message was processed. + * 3 - message was processed, all future messages must be enqueued. + * -1 - error occured. + */ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, Ref* is_special) { + bool from_dispatch_queue = msg.msg_env_from_dispatch_queue.not_null(); Ref src, dest; bool enqueue, external; auto cs = load_cell_slice(msg.msg); @@ -2413,7 +3084,7 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R if (!tlb::unpack(cs, info)) { return -1; } - CHECK(info.created_lt == msg.lt && info.created_at == now_); + CHECK(info.created_lt == msg.lt && info.created_at == now_ && !from_dispatch_queue); src = std::move(info.src); enqueue = external = true; break; @@ -2423,7 +3094,7 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R if (!tlb::unpack(cs, info)) { return -1; } - CHECK(info.created_lt == msg.lt && info.created_at == now_); + CHECK(from_dispatch_queue || (info.created_lt == msg.lt && info.created_at == now_)); src = std::move(info.src); dest = std::move(info.dest); fwd_fees = block::tlb::t_Grams.as_integer(info.fwd_fee); @@ -2435,7 +3106,7 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R default: return -1; } - CHECK(is_our_address(std::move(src))); + CHECK(is_our_address(src)); if (external) { // 1. construct a msg_export_ext OutMsg vm::CellBuilder cb; @@ -2447,9 +3118,46 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R // (if ever a structure in the block for listing all external outbound messages appears, insert this message there as well) return 0; } - if (enqueue) { - auto lt = msg.lt; - bool ok = enqueue_message(std::move(msg), std::move(fwd_fees), lt); + + WorkchainId src_wc; + StdSmcAddress src_addr; + CHECK(block::tlb::t_MsgAddressInt.extract_std_address(src, src_wc, src_addr)); + CHECK(src_wc == workchain()); + bool is_special_account = is_masterchain() && config_->is_special_smartcontract(src_addr); + bool defer = false; + if (!from_dispatch_queue) { + if (deferring_messages_enabled_ && collator_opts_->deferring_enabled && !is_special && !is_special_account && + !collator_opts_->whitelist.count({src_wc, src_addr}) && msg.msg_idx != 0) { + if (++sender_generated_messages_count_[src_addr] >= collator_opts_->defer_messages_after || + out_msg_queue_size_ > defer_out_queue_size_limit_) { + defer = true; + } + } + if (dispatch_queue_->lookup(src_addr).not_null() || unprocessed_deferred_messages_.count(src_addr)) { + defer = true; + } + } else { + auto &x = unprocessed_deferred_messages_[src_addr]; + CHECK(x > 0); + if (--x == 0) { + unprocessed_deferred_messages_.erase(src_addr); + } + } + + if (enqueue || defer) { + bool ok; + if (from_dispatch_queue) { + auto msg_env = msg.msg_env_from_dispatch_queue; + block::tlb::MsgEnvelope::Record_std env; + CHECK(block::tlb::unpack_cell(msg_env, env)); + auto src_prefix = block::tlb::MsgAddressInt::get_prefix(src); + auto dest_prefix = block::tlb::MsgAddressInt::get_prefix(dest); + CHECK(env.emitted_lt && env.emitted_lt.value() == msg.lt); + ok = enqueue_transit_message(std::move(msg.msg), std::move(msg_env), src_prefix, src_prefix, dest_prefix, + std::move(env.fwd_fee_remaining), std::move(env.metadata), msg.lt); + } else { + ok = enqueue_message(std::move(msg), std::move(fwd_fees), src_addr, defer); + } return ok ? 0 : -1; } // process message by a transaction in this block: @@ -2460,26 +3168,38 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R return -1; } // 1. create a Transaction processing this Message - auto trans_root = create_ordinary_transaction(msg.msg); + auto trans_root = create_ordinary_transaction(msg.msg, msg.metadata, msg.lt, is_special != nullptr); if (trans_root.is_null()) { fatal_error("cannot create transaction for re-processing output message"); return -1; } // 2. create a MsgEnvelope enveloping this Message - vm::CellBuilder cb; - CHECK(cb.store_long_bool(0x46060, 20) // msg_envelope#4 cur_addr:.. next_addr:.. - && block::tlb::t_Grams.store_integer_ref(cb, fwd_fees) // fwd_fee_remaining:t_Grams - && cb.store_ref_bool(msg.msg)); // msg:^(Message Any) - Ref msg_env = cb.finalize(); + block::tlb::MsgEnvelope::Record_std msg_env_rec{0x60, 0x60, fwd_fees, msg.msg, {}, msg.metadata}; + Ref msg_env; + CHECK(block::tlb::pack_cell(msg_env, msg_env_rec)); if (verbosity > 2) { - std::cerr << "new (processed outbound) message envelope: "; - block::gen::t_MsgEnvelope.print_ref(std::cerr, msg_env); + FLOG(INFO) { + sb << "new (processed outbound) message envelope: "; + block::gen::t_MsgEnvelope.print_ref(sb, msg_env); + }; } // 3. create InMsg, referring to this MsgEnvelope and this Transaction - CHECK(cb.store_long_bool(3, 3) // msg_import_imm$011 - && cb.store_ref_bool(msg_env) // in_msg:^MsgEnvelope - && cb.store_ref_bool(trans_root) // transaction:^Transaction - && block::tlb::t_Grams.store_integer_ref(cb, fwd_fees)); // fwd_fee:Grams + vm::CellBuilder cb; + if (from_dispatch_queue) { + auto msg_env = msg.msg_env_from_dispatch_queue; + block::tlb::MsgEnvelope::Record_std env; + CHECK(block::tlb::unpack_cell(msg_env, env)); + CHECK(env.emitted_lt && env.emitted_lt.value() == msg.lt); + CHECK(cb.store_long_bool(0b00100, 5) // msg_import_deferred_fin$00100 + && cb.store_ref_bool(msg_env) // in_msg:^MsgEnvelope + && cb.store_ref_bool(trans_root) // transaction:^Transaction + && block::tlb::t_Grams.store_integer_ref(cb, env.fwd_fee_remaining)); // fwd_fee:Grams + } else { + CHECK(cb.store_long_bool(3, 3) // msg_import_imm$011 + && cb.store_ref_bool(msg_env) // in_msg:^MsgEnvelope + && cb.store_ref_bool(trans_root) // transaction:^Transaction + && block::tlb::t_Grams.store_integer_ref(cb, fwd_fees)); // fwd_fee:Grams + } // 4. insert InMsg into InMsgDescr Ref in_msg = cb.finalize(); if (!insert_in_msg(in_msg)) { @@ -2490,18 +3210,21 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R *is_special = in_msg; return 1; } - // 5. create OutMsg, referring to this MsgEnvelope and InMsg - CHECK(cb.store_long_bool(2, 3) // msg_export_imm$010 - && cb.store_ref_bool(msg_env) // out_msg:^MsgEnvelope - && cb.store_ref_bool(msg.trans) // transaction:^Transaction - && cb.store_ref_bool(in_msg)); // reimport:^InMsg - // 6. insert OutMsg into OutMsgDescr - if (!insert_out_msg(cb.finalize())) { - return -1; + if (!from_dispatch_queue) { + // 5. create OutMsg, referring to this MsgEnvelope and InMsg + CHECK(cb.store_long_bool(2, 3) // msg_export_imm$010 + && cb.store_ref_bool(msg_env) // out_msg:^MsgEnvelope + && cb.store_ref_bool(msg.trans) // transaction:^Transaction + && cb.store_ref_bool(in_msg)); // reimport:^InMsg + // 6. insert OutMsg into OutMsgDescr + if (!insert_out_msg(cb.finalize())) { + return -1; + } } // 7. check whether the block is full now if (!block_limit_status_->fits(block::ParamLimits::cl_normal)) { block_full_ = true; + block_limit_class_ = std::max(block_limit_class_, block_limit_status_->classify()); return 3; } if (soft_timeout_.is_in_past(td::Timestamp::now())) { @@ -2512,60 +3235,95 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R return 1; } -// very similar to enqueue_message(), but for transit messages +/** + * Enqueues a transit message. + * Very similar to enqueue_message(), but for transit messages. + * + * @param msg The message to be enqueued. + * @param old_msg_env The previous message envelope. + * @param prev_prefix The account ID prefix for this shard. + * @param cur_prefix The account ID prefix for the next hop. + * @param dest_prefix The prefix of the destination account ID. + * @param fwd_fee_remaining The remaining forward fee. + * @param msg_metadata Metadata of the message. + * @param emitted_lt If present - the message was taken from DispatchQueue, and msg_env will have this emitted_lt. + * + * @returns True if the transit message is successfully enqueued, false otherwise. + */ bool Collator::enqueue_transit_message(Ref msg, Ref old_msg_env, ton::AccountIdPrefixFull prev_prefix, ton::AccountIdPrefixFull cur_prefix, ton::AccountIdPrefixFull dest_prefix, td::RefInt256 fwd_fee_remaining, - ton::LogicalTime enqueued_lt) { - LOG(DEBUG) << "enqueueing transit message " << msg->get_hash().bits().to_hex(256); - bool requeue = is_our_address(prev_prefix); + td::optional msg_metadata, + td::optional emitted_lt) { + bool from_dispatch_queue = (bool)emitted_lt; + if (from_dispatch_queue) { + LOG(DEBUG) << "enqueueing message from dispatch queue " << msg->get_hash().bits().to_hex(256) + << ", emitted_lt=" << emitted_lt.value(); + } else { + LOG(DEBUG) << "enqueueing transit message " << msg->get_hash().bits().to_hex(256); + } + bool requeue = !from_dispatch_queue && is_our_address(prev_prefix) && !from_dispatch_queue; // 1. perform hypercube routing auto route_info = block::perform_hypercube_routing(cur_prefix, dest_prefix, shard_); if ((unsigned)route_info.first > 96 || (unsigned)route_info.second > 96) { return fatal_error("cannot perform hypercube routing for a transit message"); } // 2. compute our part of transit fees - td::RefInt256 transit_fee = action_phase_cfg_.fwd_std.get_next_part(fwd_fee_remaining); + td::RefInt256 transit_fee = + from_dispatch_queue ? td::zero_refint() : action_phase_cfg_.fwd_std.get_next_part(fwd_fee_remaining); fwd_fee_remaining -= transit_fee; CHECK(td::sgn(transit_fee) >= 0 && td::sgn(fwd_fee_remaining) >= 0); // 3. create a new MsgEnvelope - vm::CellBuilder cb; - CHECK(cb.store_long_bool(4, 4) // msg_envelope#4 cur_addr:.. next_addr:.. - && cb.store_long_bool(route_info.first, 8) // cur_addr:IntermediateAddress - && cb.store_long_bool(route_info.second, 8) // next_addr:IntermediateAddress - && block::tlb::t_Grams.store_integer_ref(cb, fwd_fee_remaining) // fwd_fee_remaining:t_Grams - && cb.store_ref_bool(msg)); // msg:^(Message Any) - Ref msg_env = cb.finalize(); + block::tlb::MsgEnvelope::Record_std msg_env_rec{route_info.first, route_info.second, fwd_fee_remaining, msg, + emitted_lt, std::move(msg_metadata)}; + Ref msg_env; + CHECK(block::tlb::t_MsgEnvelope.pack_cell(msg_env, msg_env_rec)); // 4. create InMsg - CHECK(cb.store_long_bool(5, 3) // msg_import_tr$101 - && cb.store_ref_bool(old_msg_env) // in_msg:^MsgEnvelope - && cb.store_ref_bool(msg_env) // out_msg:^MsgEnvelope - && block::tlb::t_Grams.store_integer_ref(cb, transit_fee)); // transit_fee:Grams + vm::CellBuilder cb; + if (from_dispatch_queue) { + CHECK(cb.store_long_bool(0b00101, 5) // msg_import_deferred_tr$00101 + && cb.store_ref_bool(old_msg_env) // in_msg:^MsgEnvelope + && cb.store_ref_bool(msg_env)); // out_msg:^MsgEnvelope + } else { + CHECK(cb.store_long_bool(5, 3) // msg_import_tr$101 + && cb.store_ref_bool(old_msg_env) // in_msg:^MsgEnvelope + && cb.store_ref_bool(msg_env) // out_msg:^MsgEnvelope + && block::tlb::t_Grams.store_integer_ref(cb, transit_fee)); // transit_fee:Grams + } Ref in_msg = cb.finalize(); // 5. create a new OutMsg - CHECK(cb.store_long_bool(requeue ? 7 : 3, 3) // msg_export_tr$011 or msg_export_tr_req$111 - && cb.store_ref_bool(msg_env) // out_msg:^MsgEnvelope - && cb.store_ref_bool(in_msg)); // imported:^InMsg + // msg_export_tr$011 / msg_export_tr_req$111 / msg_export_deferred_tr$10101 + if (from_dispatch_queue) { + CHECK(cb.store_long_bool(0b10101, 5)); + } else { + CHECK(cb.store_long_bool(requeue ? 7 : 3, 3)); + } + CHECK(cb.store_ref_bool(msg_env) // out_msg:^MsgEnvelope + && cb.store_ref_bool(in_msg)); // imported:^InMsg Ref out_msg = cb.finalize(); // 4.1. insert OutMsg into OutMsgDescr if (verbosity > 2) { - std::cerr << "OutMsg for a transit message: "; - block::gen::t_OutMsg.print_ref(std::cerr, out_msg); + FLOG(INFO) { + sb << "OutMsg for a transit message: "; + block::gen::t_OutMsg.print_ref(sb, out_msg); + }; } if (!insert_out_msg(out_msg)) { return fatal_error("cannot insert a new OutMsg into OutMsgDescr"); } // 4.2. insert InMsg into InMsgDescr if (verbosity > 2) { - std::cerr << "InMsg for a transit message: "; - block::gen::t_InMsg.print_ref(std::cerr, in_msg); + FLOG(INFO) { + sb << "InMsg for a transit message: "; + block::gen::t_InMsg.print_ref(sb, in_msg); + }; } if (!insert_in_msg(in_msg)) { return fatal_error("cannot insert a new InMsg into InMsgDescr"); } // 5. create EnqueuedMsg - CHECK(cb.store_long_bool(start_lt) // _ enqueued_lt:uint64 - && cb.store_ref_bool(msg_env)); // out_msg:^MsgEnvelope = EnqueuedMsg; + CHECK(cb.store_long_bool(from_dispatch_queue ? emitted_lt.value() : start_lt) // _ enqueued_lt:uint64 + && cb.store_ref_bool(msg_env)); // out_msg:^MsgEnvelope = EnqueuedMsg; // 6. insert EnqueuedMsg into OutMsgQueue // NB: we use here cur_prefix instead of src_prefix; should we check that route_info.first >= next_addr.use_dest_bits of the old envelope? auto next_hop = block::interpolate_addr(cur_prefix, dest_prefix, route_info.second); @@ -2577,6 +3335,7 @@ bool Collator::enqueue_transit_message(Ref msg, Ref old_msg_ try { LOG(DEBUG) << "inserting into outbound queue message with (lt,key)=(" << start_lt << "," << key.to_hex() << ")"; ok = out_msg_queue_->set_builder(key.bits(), 352, cb, vm::Dictionary::SetMode::Add); + ++out_msg_queue_size_; } catch (vm::VmError) { ok = false; } @@ -2587,11 +3346,20 @@ bool Collator::enqueue_transit_message(Ref msg, Ref old_msg_ return register_out_msg_queue_op(); } +/** + * Deletes a message from the outbound message queue. + * + * @param key The key of the message to be deleted. + * + * @returns True if the message was successfully deleted, false otherwise. + */ bool Collator::delete_out_msg_queue_msg(td::ConstBitPtr key) { Ref queue_rec; try { LOG(DEBUG) << "deleting from outbound queue message with key=" << key.to_hex(352); queue_rec = out_msg_queue_->lookup_delete(key, 352); + CHECK(out_msg_queue_size_ > 0); + --out_msg_queue_size_; } catch (vm::VmError err) { LOG(ERROR) << "error deleting from out_msg_queue dictionary: " << err.get_msg(); } @@ -2602,13 +3370,27 @@ bool Collator::delete_out_msg_queue_msg(td::ConstBitPtr key) { return register_out_msg_queue_op(); } +/** + * Processes an inbound message from a neighbor's outbound queue. + * The message may create a transaction or be enqueued. + * + * @param enq_msg The inbound message serialized using EnqueuedMsg TLB-scheme. + * @param lt The logical time of the message. + * @param key The 32+64+256-bit key of the message. + * @param src_nb The description of the source neighbor shard. + * + * @returns True if the message was processed successfully, false otherwise. + */ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalTime lt, td::ConstBitPtr key, const block::McShardDescr& src_nb) { ton::LogicalTime enqueued_lt = 0; if (enq_msg.is_null() || enq_msg->size_ext() != 0x10040 || (enqueued_lt = enq_msg->prefetch_ulong(64)) < /* 0 */ 1 * lt) { // DEBUG if (enq_msg.not_null()) { - block::gen::t_EnqueuedMsg.print(std::cerr, *enq_msg); + FLOG(WARNING) { + sb << "inbound internal message is not a valid EnqueuedMsg: "; + block::gen::t_EnqueuedMsg.print(sb, enq_msg); + }; } LOG(ERROR) << "inbound internal message is not a valid EnqueuedMsg (created lt " << lt << ", enqueued " << enqueued_lt << ")"; @@ -2626,7 +3408,7 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT return false; } if (!block::tlb::t_MsgEnvelope.validate_ref(msg_env)) { - LOG(ERROR) << "inbound internal MsgEnvelope is invalid according to automated checks"; + LOG(ERROR) << "inbound internal MsgEnvelope is invalid according to hand-written checks"; return false; } // 1. unpack MsgEnvelope @@ -2646,9 +3428,18 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT LOG(ERROR) << "cannot unpack CommonMsgInfo of an inbound internal message"; return false; } - if (info.created_lt != lt) { + if (!env.emitted_lt && info.created_lt != lt) { LOG(ERROR) << "inbound internal message has an augmentation value in source OutMsgQueue distinct from the one in " - "its contents"; + "its contents (CommonMsgInfo)"; + return false; + } + if (env.emitted_lt && env.emitted_lt.value() != lt) { + LOG(ERROR) << "inbound internal message has an augmentation value in source OutMsgQueue distinct from the one in " + "its contents (deferred_it in MsgEnvelope)"; + return false; + } + if (!block::tlb::validate_message_libs(env.msg)) { + LOG(ERROR) << "inbound internal message has invalid StateInit"; return false; } // 2.0. update last_proc_int_msg @@ -2707,7 +3498,8 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT bool our = ton::shard_contains(shard_, cur_prefix); bool to_us = ton::shard_contains(shard_, dest_prefix); - block::EnqueuedMsgDescr enq_msg_descr{cur_prefix, next_prefix, info.created_lt, enqueued_lt, + block::EnqueuedMsgDescr enq_msg_descr{cur_prefix, next_prefix, + env.emitted_lt ? env.emitted_lt.value() : info.created_lt, enqueued_lt, env.msg->get_hash().bits()}; if (processed_upto_->already_processed(enq_msg_descr)) { LOG(DEBUG) << "inbound internal message with lt=" << enq_msg_descr.lt_ << " hash=" << enq_msg_descr.hash_.to_hex() @@ -2724,7 +3516,7 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT // destination is outside our shard, relay transit message // (very similar to enqueue_message()) if (!enqueue_transit_message(std::move(env.msg), std::move(msg_env), cur_prefix, next_prefix, dest_prefix, - std::move(env.fwd_fee_remaining), max_lt)) { + std::move(env.fwd_fee_remaining), std::move(env.metadata))) { return fatal_error("cannot enqueue transit internal message with key "s + key.to_hex(352)); } return !our || delete_out_msg_queue_msg(key); @@ -2733,7 +3525,7 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT // process the message by an ordinary transaction similarly to process_one_new_message() // // 8. create a Transaction processing this Message - auto trans_root = create_ordinary_transaction(env.msg); + auto trans_root = create_ordinary_transaction(env.msg, env.metadata, 0); if (trans_root.is_null()) { return fatal_error("cannot create transaction for processing inbound message"); } @@ -2766,31 +3558,74 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT return true; } +/** + * Creates a string that explains which limit is exceeded. Used for collator stats. + * + * @param block_limit_status Status of block limits. + * @param cls Which limit class is exceeded. + * + * @returns String for collator stats. + */ +static std::string block_full_comment(const block::BlockLimitStatus& block_limit_status, unsigned cls) { + auto bytes = block_limit_status.estimate_block_size(); + if (!block_limit_status.limits.bytes.fits(cls, bytes)) { + return PSTRING() << "block_full bytes " << bytes; + } + if (!block_limit_status.limits.gas.fits(cls, block_limit_status.gas_used)) { + return PSTRING() << "block_full gas " << block_limit_status.gas_used; + } + auto lt_delta = block_limit_status.cur_lt - block_limit_status.limits.start_lt; + if (!block_limit_status.limits.lt_delta.fits(cls, lt_delta)) { + return PSTRING() << "block_full lt_delta " << lt_delta; + } + return ""; +} + +/** + * Processes inbound internal messages from message queues of the neighbors. + * Messages are processed until the normal limit is reached, soft timeout is reached or there are no more messages. + * + * @returns True if the processing was successful, false otherwise. + */ bool Collator::process_inbound_internal_messages() { + if (have_unprocessed_account_dispatch_queue_) { + return true; + } while (!block_full_ && !nb_out_msgs_->is_eof()) { block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); if (block_full_) { LOG(INFO) << "BLOCK FULL, stop processing inbound internal messages"; + block_limit_class_ = std::max(block_limit_class_, block_limit_status_->classify()); + stats_.limits_log += PSTRING() << "INBOUND_INT_MESSAGES: " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) << "\n"; break; } if (soft_timeout_.is_in_past(td::Timestamp::now())) { block_full_ = true; LOG(WARNING) << "soft timeout reached, stop processing inbound internal messages"; + stats_.limits_log += PSTRING() << "INBOUND_INT_MESSAGES: timeout\n"; break; } + if (!check_cancelled()) { + return false; + } auto kv = nb_out_msgs_->extract_cur(); CHECK(kv && kv->msg.not_null()); LOG(DEBUG) << "processing inbound message with (lt,hash)=(" << kv->lt << "," << kv->key.to_hex() << ") from neighbor #" << kv->source; if (verbosity > 2) { - std::cerr << "inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() << " msg="; - block::gen::t_EnqueuedMsg.print(std::cerr, *(kv->msg)); + FLOG(INFO) { + sb << "inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() << " msg="; + block::gen::t_EnqueuedMsg.print(sb, kv->msg); + }; } if (!process_inbound_message(kv->msg, kv->lt, kv->key.cbits(), neighbors_.at(kv->source))) { if (verbosity > 1) { - std::cerr << "invalid inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() - << " msg="; - block::gen::t_EnqueuedMsg.print(std::cerr, *(kv->msg)); + FLOG(INFO) { + sb << "invalid inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() + << " msg="; + block::gen::t_EnqueuedMsg.print(sb, kv->msg); + }; } return fatal_error("error processing inbound internal message"); } @@ -2800,33 +3635,63 @@ bool Collator::process_inbound_internal_messages() { return true; } +/** + * Processes inbound external messages. + * Messages are processed until the soft limit is reached, medium timeout is reached or there are no more messages. + * + * @returns True if the processing was successful, false otherwise. + */ bool Collator::process_inbound_external_messages() { if (skip_extmsg_) { LOG(INFO) << "skipping processing of inbound external messages"; return true; } + if (attempt_idx_ >= 2) { + LOG(INFO) << "Attempt #" << attempt_idx_ << ": skip external messages"; + return true; + } + if (out_msg_queue_size_ > SKIP_EXTERNALS_QUEUE_SIZE) { + LOG(INFO) << "skipping processing of inbound external messages (except for high-priority) because out_msg_queue is " + "too big (" + << out_msg_queue_size_ << " > " << SKIP_EXTERNALS_QUEUE_SIZE << ")"; + } bool full = !block_limit_status_->fits(block::ParamLimits::cl_soft); - for (auto& ext_msg_pair : ext_msg_list_) { + for (auto& ext_msg_struct : ext_msg_list_) { + if (out_msg_queue_size_ > SKIP_EXTERNALS_QUEUE_SIZE && ext_msg_struct.priority < HIGH_PRIORITY_EXTERNAL) { + continue; + } if (full) { LOG(INFO) << "BLOCK FULL, stop processing external messages"; + stats_.limits_log += PSTRING() << "INBOUND_EXT_MESSAGES: " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_soft) << "\n"; break; } if (medium_timeout_.is_in_past(td::Timestamp::now())) { LOG(WARNING) << "medium timeout reached, stop processing inbound external messages"; + stats_.limits_log += PSTRING() << "INBOUND_EXT_MESSAGES: timeout\n"; break; } - auto ext_msg = ext_msg_pair.first; + if (!check_cancelled()) { + return false; + } + auto ext_msg = ext_msg_struct.cell; ton::Bits256 hash{ext_msg->get_hash().bits()}; int r = process_external_message(std::move(ext_msg)); + if (r > 0) { + ++stats_.ext_msgs_accepted; + } else { + ++stats_.ext_msgs_rejected; + } if (r < 0) { - bad_ext_msgs_.emplace_back(ext_msg_pair.second); + bad_ext_msgs_.emplace_back(ext_msg_struct.hash); return false; } if (!r) { - delay_ext_msgs_.emplace_back(ext_msg_pair.second); + delay_ext_msgs_.emplace_back(ext_msg_struct.hash); } if (r > 0) { full = !block_limit_status_->fits(block::ParamLimits::cl_soft); + block_limit_class_ = std::max(block_limit_class_, block_limit_status_->classify()); } auto it = ext_msg_map.find(hash); CHECK(it != ext_msg_map.end()); @@ -2838,7 +3703,17 @@ bool Collator::process_inbound_external_messages() { return true; } -// 1 = processed, 0 = skipped, 3 = processed, all future messages must be skipped (block overflown) +/** + * Processes an external message. + * + * @param msg The message to be processed serialized as Message TLB-scheme. + * + * @returns The result of processing the message: + * -1 if a fatal error occurred. + * 0 if the message is rejected. + * 1 if the message was processed. + * 3 if the message was processed and all future messages must be skipped (block overflown). + */ int Collator::process_external_message(Ref msg) { auto cs = load_cell_slice(msg); td::RefInt256 fwd_fees; @@ -2851,7 +3726,7 @@ int Collator::process_external_message(Ref msg) { } // process message by a transaction in this block: // 1. create a Transaction processing this Message - auto trans_root = create_ordinary_transaction(msg); + auto trans_root = create_ordinary_transaction(msg, /* metadata = */ {}, 0); if (trans_root.is_null()) { if (busy_) { // transaction rejected by account @@ -2875,19 +3750,286 @@ int Collator::process_external_message(Ref msg) { return 1; } -// inserts an InMsg into InMsgDescr +/** + * Processes messages from dispatch queue + * + * Messages from dispatch queue are taken in three steps: + * 1. Take one message from each account (in the order of lt) + * 2. Take up to 10 per account (including from p.1), up to 20 per initiator, up to 150 in total + * 3. Take up to X messages per initiator, up to 150 in total. X depends on out msg queue size + * + * @returns True if the processing was successful, false otherwise. + */ +bool Collator::process_dispatch_queue() { + if (out_msg_queue_size_ > defer_out_queue_size_limit_ && old_out_msg_queue_size_ > hard_defer_out_queue_size_limit_) { + return true; + } + have_unprocessed_account_dispatch_queue_ = true; + size_t max_total_count[3] = {1 << 30, collator_opts_->dispatch_phase_2_max_total, + collator_opts_->dispatch_phase_3_max_total}; + size_t max_per_initiator[3] = {1 << 30, collator_opts_->dispatch_phase_2_max_per_initiator, 0}; + if (collator_opts_->dispatch_phase_3_max_per_initiator) { + max_per_initiator[2] = collator_opts_->dispatch_phase_3_max_per_initiator.value(); + } else if (out_msg_queue_size_ <= 256) { + max_per_initiator[2] = 10; + } else if (out_msg_queue_size_ <= 512) { + max_per_initiator[2] = 2; + } else if (out_msg_queue_size_ <= 1500) { + max_per_initiator[2] = 1; + } + for (int iter = 0; iter < 3; ++iter) { + if (max_per_initiator[iter] == 0 || max_total_count[iter] == 0) { + continue; + } + if (iter > 0 && attempt_idx_ >= 1) { + LOG(INFO) << "Attempt #" << attempt_idx_ << ": skip process_dispatch_queue"; + break; + } + vm::AugmentedDictionary cur_dispatch_queue{dispatch_queue_->get_root(), 256, block::tlb::aug_DispatchQueue}; + std::map, size_t> count_per_initiator; + size_t total_count = 0; + auto prioritylist = collator_opts_->prioritylist; + auto prioritylist_iter = prioritylist.begin(); + while (!cur_dispatch_queue.is_empty()) { + block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); + if (block_full_) { + LOG(INFO) << "BLOCK FULL, stop processing dispatch queue"; + block_limit_class_ = std::max(block_limit_class_, block_limit_status_->classify()); + stats_.limits_log += PSTRING() << "DISPATCH_QUEUE_STAGE_" << iter << ": " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) + << "\n"; + return register_dispatch_queue_op(true); + } + if (soft_timeout_.is_in_past(td::Timestamp::now())) { + block_full_ = true; + LOG(WARNING) << "soft timeout reached, stop processing dispatch queue"; + stats_.limits_log += PSTRING() << "DISPATCH_QUEUE_STAGE_" << iter << ": timeout\n"; + return register_dispatch_queue_op(true); + } + StdSmcAddress src_addr; + td::Ref account_dispatch_queue; + while (!prioritylist.empty()) { + if (prioritylist_iter == prioritylist.end()) { + prioritylist_iter = prioritylist.begin(); + } + auto priority_addr = *prioritylist_iter; + if (priority_addr.first != workchain() || !is_our_address(priority_addr.second)) { + prioritylist_iter = prioritylist.erase(prioritylist_iter); + continue; + } + src_addr = priority_addr.second; + account_dispatch_queue = cur_dispatch_queue.lookup(src_addr); + if (account_dispatch_queue.is_null()) { + prioritylist_iter = prioritylist.erase(prioritylist_iter); + } else { + ++prioritylist_iter; + break; + } + } + if (account_dispatch_queue.is_null()) { + account_dispatch_queue = block::get_dispatch_queue_min_lt_account(cur_dispatch_queue, src_addr); + if (account_dispatch_queue.is_null()) { + return fatal_error("invalid dispatch queue in shard state"); + } + } + vm::Dictionary dict{64}; + td::uint64 dict_size; + if (!block::unpack_account_dispatch_queue(account_dispatch_queue, dict, dict_size)) { + return fatal_error(PSTRING() << "invalid account dispatch queue for account " << src_addr.to_hex()); + } + td::BitArray<64> key; + Ref enqueued_msg = dict.extract_minmax_key(key.bits(), 64, false, false); + LogicalTime lt = key.to_ulong(); + + td::optional msg_metadata; + if (!process_deferred_message(std::move(enqueued_msg), src_addr, lt, msg_metadata)) { + return fatal_error(PSTRING() << "error processing internal message from dispatch queue: account=" + << src_addr.to_hex() << ", lt=" << lt); + } + + // Remove message from DispatchQueue + bool ok; + if (iter == 0 || + (iter == 1 && sender_generated_messages_count_[src_addr] >= collator_opts_->defer_messages_after && + !collator_opts_->whitelist.count({workchain(), src_addr}))) { + ok = cur_dispatch_queue.lookup_delete(src_addr).not_null(); + } else { + dict.lookup_delete(key); + --dict_size; + account_dispatch_queue = block::pack_account_dispatch_queue(dict, dict_size); + ok = account_dispatch_queue.not_null() ? cur_dispatch_queue.set(src_addr, account_dispatch_queue) + : cur_dispatch_queue.lookup_delete(src_addr).not_null(); + } + if (!ok) { + return fatal_error(PSTRING() << "error processing internal message from dispatch queue: account=" + << src_addr.to_hex() << ", lt=" << lt); + } + if (msg_metadata) { + auto initiator = std::make_tuple(msg_metadata.value().initiator_wc, msg_metadata.value().initiator_addr, + msg_metadata.value().initiator_lt); + size_t initiator_count = ++count_per_initiator[initiator]; + if (initiator_count >= max_per_initiator[iter]) { + cur_dispatch_queue.lookup_delete(src_addr); + } + } + ++total_count; + if (total_count >= max_total_count[iter]) { + dispatch_queue_total_limit_reached_ = true; + stats_.limits_log += PSTRING() << "DISPATCH_QUEUE_STAGE_" << iter << ": total limit reached\n"; + break; + } + } + if (iter == 0) { + have_unprocessed_account_dispatch_queue_ = false; + } + register_dispatch_queue_op(true); + } + return true; +} + +/** + * Processes an internal message from DispatchQueue. + * The message may create a transaction or be enqueued. + * + * Similar to Collator::process_inbound_message. + * + * @param enq_msg The internal message serialized using EnqueuedMsg TLB-scheme. + * @param src_addr 256-bit address of the sender. + * @param lt The logical time of the message. + * @param msg_metadata Reference to store msg_metadata + * + * @returns True if the message was processed successfully, false otherwise. + */ +bool Collator::process_deferred_message(Ref enq_msg, StdSmcAddress src_addr, LogicalTime lt, + td::optional& msg_metadata) { + if (!block::remove_dispatch_queue_entry(*dispatch_queue_, src_addr, lt)) { + return fatal_error(PSTRING() << "failed to delete message from DispatchQueue: address=" << src_addr.to_hex() + << ", lt=" << lt); + } + register_dispatch_queue_op(); + ++sender_generated_messages_count_[src_addr]; + + LogicalTime enqueued_lt = 0; + if (enq_msg.is_null() || enq_msg->size_ext() != 0x10040 || (enqueued_lt = enq_msg->prefetch_ulong(64)) != lt) { + if (enq_msg.not_null()) { + FLOG(WARNING) { + sb << "internal message in DispatchQueue is not a valid EnqueuedMsg: "; + block::gen::t_EnqueuedMsg.print(sb, enq_msg); + }; + } + LOG(ERROR) << "internal message in DispatchQueue is not a valid EnqueuedMsg (created lt " << lt << ", enqueued " + << enqueued_lt << ")"; + return false; + } + auto msg_env = enq_msg->prefetch_ref(); + CHECK(msg_env.not_null()); + // 0. check MsgEnvelope + if (msg_env->get_level() != 0) { + LOG(ERROR) << "cannot import a message with non-zero level!"; + return false; + } + if (!block::gen::t_MsgEnvelope.validate_ref(msg_env)) { + LOG(ERROR) << "MsgEnvelope from DispatchQueue is invalid according to automated checks"; + return false; + } + if (!block::tlb::t_MsgEnvelope.validate_ref(msg_env)) { + LOG(ERROR) << "MsgEnvelope from DispatchQueue is invalid according to hand-written checks"; + return false; + } + // 1. unpack MsgEnvelope + block::tlb::MsgEnvelope::Record_std env; + if (!tlb::unpack_cell(msg_env, env)) { + LOG(ERROR) << "cannot unpack MsgEnvelope from DispatchQueue"; + return false; + } + // 2. unpack CommonMsgInfo of the message + vm::CellSlice cs{vm::NoVmOrd{}, env.msg}; + if (block::gen::t_CommonMsgInfo.get_tag(cs) != block::gen::CommonMsgInfo::int_msg_info) { + LOG(ERROR) << "internal message from DispatchQueue is not in fact internal!"; + return false; + } + block::gen::CommonMsgInfo::Record_int_msg_info info; + if (!tlb::unpack(cs, info)) { + LOG(ERROR) << "cannot unpack CommonMsgInfo of an internal message from DispatchQueue"; + return false; + } + if (info.created_lt != lt) { + LOG(ERROR) << "internal message has lt in DispatchQueue distinct from the one in " + "its contents"; + return false; + } + if (!block::tlb::validate_message_libs(env.msg)) { + LOG(ERROR) << "internal message in DispatchQueue has invalid StateInit"; + return false; + } + // 2.1. check fwd_fee and fwd_fee_remaining + td::RefInt256 orig_fwd_fee = block::tlb::t_Grams.as_integer(info.fwd_fee); + if (env.fwd_fee_remaining > orig_fwd_fee) { + LOG(ERROR) << "internal message if DispatchQueue has fwd_fee_remaining=" << td::dec_string(env.fwd_fee_remaining) + << " larger than original fwd_fee=" << td::dec_string(orig_fwd_fee); + return false; + } + // 3. extract source and destination shards + auto src_prefix = block::tlb::t_MsgAddressInt.get_prefix(info.src); + auto dest_prefix = block::tlb::t_MsgAddressInt.get_prefix(info.dest); + if (!(src_prefix.is_valid() && dest_prefix.is_valid())) { + LOG(ERROR) << "internal message in DispatchQueue has invalid source or destination address"; + return false; + } + // 4. chech current and next hop shards + if (env.cur_addr != 0 || env.next_addr != 0) { + LOG(ERROR) << "internal message in DispatchQueue is expected to have zero cur_addr and next_addr"; + return false; + } + // 5. calculate emitted_lt + LogicalTime emitted_lt = std::max(start_lt, last_dispatch_queue_emitted_lt_[src_addr]) + 1; + auto it = accounts.find(src_addr); + if (it != accounts.end()) { + emitted_lt = std::max(emitted_lt, it->second->last_trans_end_lt_ + 1); + } + last_dispatch_queue_emitted_lt_[src_addr] = emitted_lt; + update_max_lt(emitted_lt + 1); + + env.emitted_lt = emitted_lt; + if (!block::tlb::pack_cell(msg_env, env)) { + return fatal_error("cannot pack msg envelope"); + } + + // 6. create NewOutMsg + block::NewOutMsg new_msg{emitted_lt, env.msg, {}, 0}; + new_msg.metadata = env.metadata; + new_msg.msg_env_from_dispatch_queue = msg_env; + ++unprocessed_deferred_messages_[src_addr]; + LOG(INFO) << "delivering deferred message from account " << src_addr.to_hex() << ", lt=" << lt + << ", emitted_lt=" << emitted_lt; + block_limit_status_->add_cell(msg_env); + register_new_msg(std::move(new_msg)); + msg_metadata = std::move(env.metadata); + return true; +} + +/** + * Inserts an InMsg into the block's InMsgDescr. + * + * @param in_msg The input message to be inserted. + * + * @returns True if the insertion is successful, false otherwise. + */ bool Collator::insert_in_msg(Ref in_msg) { if (verbosity > 2) { - std::cerr << "InMsg being inserted into InMsgDescr: "; - block::gen::t_InMsg.print_ref(std::cerr, in_msg); + FLOG(INFO) { + sb << "InMsg being inserted into InMsgDescr: "; + block::gen::t_InMsg.print_ref(sb, in_msg); + }; } auto cs = load_cell_slice(in_msg); if (!cs.size_refs()) { return false; } Ref msg = cs.prefetch_ref(); - int tag = (int)cs.prefetch_ulong(3); - if (!(tag == 0 || tag == 2)) { // msg_import_ext$000 or msg_import_ihr$010 contain (Message Any) directly + int tag = block::gen::t_InMsg.get_tag(cs); + // msg_import_ext$000 or msg_import_ihr$010 contain (Message Any) directly + if (!(tag == block::gen::InMsg::msg_import_ext || tag == block::gen::InMsg::msg_import_ihr)) { // extract Message Any from MsgEnvelope to compute correct key auto cs2 = load_cell_slice(std::move(msg)); if (!cs2.size_refs()) { @@ -2910,11 +4052,19 @@ bool Collator::insert_in_msg(Ref in_msg) { ((in_descr_cnt_ & 63) || block_limit_status_->add_cell(in_msg_dict->get_root_cell())); } -// inserts an OutMsg into OutMsgDescr +/** + * Inserts an OutMsg into the block's OutMsgDescr. + * + * @param out_msg The outgoing message to be inserted. + * + * @returns True if the insertion was successful, false otherwise. + */ bool Collator::insert_out_msg(Ref out_msg) { if (verbosity > 2) { - std::cerr << "OutMsg being inserted into OutMsgDescr: "; - block::gen::t_OutMsg.print_ref(std::cerr, out_msg); + FLOG(INFO) { + sb << "OutMsg being inserted into OutMsgDescr: "; + block::gen::t_OutMsg.print_ref(sb, out_msg); + }; } auto cs = load_cell_slice(out_msg); if (!cs.size_refs()) { @@ -2933,6 +4083,14 @@ bool Collator::insert_out_msg(Ref out_msg) { return insert_out_msg(std::move(out_msg), msg->get_hash().bits()); } +/** + * Inserts an outgoing message into the block's OutMsgDescr dictionary. + * + * @param out_msg The outgoing message to be inserted. + * @param msg_hash The 256-bit hash of the outgoing message. + * + * @returns True if the insertion was successful, false otherwise. + */ bool Collator::insert_out_msg(Ref out_msg, td::ConstBitPtr msg_hash) { bool ok; try { @@ -2949,8 +4107,20 @@ bool Collator::insert_out_msg(Ref out_msg, td::ConstBitPtr msg_hash) { ((out_descr_cnt_ & 63) || block_limit_status_->add_cell(out_msg_dict->get_root_cell())); } -// enqueues a new Message into OutMsgDescr and OutMsgQueue -bool Collator::enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_remaining, ton::LogicalTime enqueued_lt) { +/** + * Enqueues a new message into the block's outbound message queue and OutMsgDescr. + * + * @param msg The new outbound message to enqueue. + * @param fwd_fees_remaining The remaining forward fees for the message. + * @param src_addr 256-bit address of the sender + * @param defer Put the message to DispatchQueue + * + * @returns True if the message was successfully enqueued, false otherwise. + */ +bool Collator::enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_remaining, StdSmcAddress src_addr, + bool defer) { + LogicalTime enqueued_lt = msg.lt; + CHECK(msg.msg_env_from_dispatch_queue.is_null()); // 0. unpack src_addr and dest_addr block::gen::CommonMsgInfo::Record_int_msg_info info; if (!tlb::unpack_cell_inexact(msg.msg, info)) { @@ -2970,22 +4140,30 @@ bool Collator::enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_rema return fatal_error("cannot perform hypercube routing for a new outbound message"); } // 2. create a new MsgEnvelope - vm::CellBuilder cb; - CHECK(cb.store_long_bool(4, 4) // msg_envelope#4 cur_addr:.. next_addr:.. - && cb.store_long_bool(route_info.first, 8) // cur_addr:IntermediateAddress - && cb.store_long_bool(route_info.second, 8) // next_addr:IntermediateAddress - && block::tlb::t_Grams.store_integer_ref(cb, fwd_fees_remaining) // fwd_fee_remaining:t_Grams - && cb.store_ref_bool(msg.msg)); // msg:^(Message Any) - Ref msg_env = cb.finalize(); + block::tlb::MsgEnvelope::Record_std msg_env_rec{ + defer ? 0 : route_info.first, defer ? 0 : route_info.second, fwd_fees_remaining, msg.msg, {}, msg.metadata}; + Ref msg_env; + CHECK(block::tlb::pack_cell(msg_env, msg_env_rec)); // 3. create a new OutMsg - CHECK(cb.store_long_bool(1, 3) // msg_export_new$001 - && cb.store_ref_bool(msg_env) // out_msg:^MsgEnvelope - && cb.store_ref_bool(msg.trans)); // transaction:^Transaction - Ref out_msg = cb.finalize(); + vm::CellBuilder cb; + Ref out_msg; + if (defer) { + CHECK(cb.store_long_bool(0b10100, 5) // msg_export_new_defer$10100 + && cb.store_ref_bool(msg_env) // out_msg:^MsgEnvelope + && cb.store_ref_bool(msg.trans)); // transaction:^Transaction + out_msg = cb.finalize(); + } else { + CHECK(cb.store_long_bool(1, 3) // msg_export_new$001 + && cb.store_ref_bool(msg_env) // out_msg:^MsgEnvelope + && cb.store_ref_bool(msg.trans)); // transaction:^Transaction + out_msg = cb.finalize(); + } // 4. insert OutMsg into OutMsgDescr if (verbosity > 2) { - std::cerr << "OutMsg for a newly-generated message: "; - block::gen::t_OutMsg.print_ref(std::cerr, out_msg); + FLOG(INFO) { + sb << "OutMsg for a newly-generated message: "; + block::gen::t_OutMsg.print_ref(sb, out_msg); + }; } if (!insert_out_msg(out_msg)) { return fatal_error("cannot insert a new OutMsg into OutMsgDescr"); @@ -2993,7 +4171,26 @@ bool Collator::enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_rema // 5. create EnqueuedMsg CHECK(cb.store_long_bool(enqueued_lt) // _ enqueued_lt:uint64 && cb.store_ref_bool(msg_env)); // out_msg:^MsgEnvelope = EnqueuedMsg; - // 6. insert EnqueuedMsg into OutMsgQueue + + // 6. insert EnqueuedMsg into OutMsgQueue (or DispatchQueue) + if (defer) { + LOG(INFO) << "deferring new message from account " << workchain() << ":" << src_addr.to_hex() << ", lt=" << msg.lt; + vm::Dictionary dispatch_dict{64}; + td::uint64 dispatch_dict_size; + if (!block::unpack_account_dispatch_queue(dispatch_queue_->lookup(src_addr), dispatch_dict, dispatch_dict_size)) { + return fatal_error(PSTRING() << "cannot unpack AccountDispatchQueue for account " << src_addr.to_hex()); + } + td::BitArray<64> key; + key.store_ulong(msg.lt); + if (!dispatch_dict.set_builder(key, cb, vm::Dictionary::SetMode::Add)) { + return fatal_error(PSTRING() << "cannot add message to AccountDispatchQueue for account " << src_addr.to_hex() + << ", lt=" << msg.lt); + } + ++dispatch_dict_size; + dispatch_queue_->set(src_addr, block::pack_account_dispatch_queue(dispatch_dict, dispatch_dict_size)); + return register_dispatch_queue_op(); + } + auto next_hop = block::interpolate_addr(src_prefix, dest_prefix, route_info.second); td::BitArray<32 + 64 + 256> key; key.bits().store_int(next_hop.workchain, 32); @@ -3004,6 +4201,7 @@ bool Collator::enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_rema LOG(DEBUG) << "inserting into outbound queue a new message with (lt,key)=(" << start_lt << "," << key.to_hex() << ")"; ok = out_msg_queue_->set_builder(key.bits(), 352, cb, vm::Dictionary::SetMode::Add); + ++out_msg_queue_size_; } catch (vm::VmError) { ok = false; } @@ -3014,13 +4212,26 @@ bool Collator::enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_rema return register_out_msg_queue_op(); } +/** + * Processes new messages that were generated in this block. + * + * @param enqueue_only If true, only enqueue the new messages without creating transactions. + * + * @returns True if all new messages were processed successfully, false otherwise. + */ bool Collator::process_new_messages(bool enqueue_only) { while (!new_msgs.empty()) { block::NewOutMsg msg = new_msgs.top(); new_msgs.pop(); - if (block_full_ && !enqueue_only) { + block_limit_status_->extra_out_msgs--; + if ((block_full_ || have_unprocessed_account_dispatch_queue_) && !enqueue_only) { LOG(INFO) << "BLOCK FULL, enqueue all remaining new messages"; enqueue_only = true; + stats_.limits_log += PSTRING() << "NEW_MESSAGES: " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) << "\n"; + } + if (!check_cancelled()) { + return false; } LOG(DEBUG) << "have message with lt=" << msg.lt; int res = process_one_new_message(std::move(msg), enqueue_only); @@ -3029,22 +4240,41 @@ bool Collator::process_new_messages(bool enqueue_only) { } else if (res == 3) { LOG(INFO) << "All remaining new messages must be enqueued (BLOCK FULL)"; enqueue_only = true; + stats_.limits_log += PSTRING() << "NEW_MESSAGES: " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) << "\n"; } } return true; } +/** + * Registers a new output message. + * + * @param new_msg The new output message to be registered. + */ void Collator::register_new_msg(block::NewOutMsg new_msg) { if (new_msg.lt < min_new_msg_lt) { min_new_msg_lt = new_msg.lt; } new_msgs.push(std::move(new_msg)); + block_limit_status_->extra_out_msgs++; } -void Collator::register_new_msgs(block::Transaction& trans) { +/** + * Registers new messages that were created in the transaction. + * + * @param trans The transaction containing the messages. + * @param msg_metadata Metadata of the new messages. + */ +void Collator::register_new_msgs(block::transaction::Transaction& trans, + td::optional msg_metadata) { CHECK(trans.root.not_null()); for (unsigned i = 0; i < trans.out_msgs.size(); i++) { - register_new_msg(trans.extract_out_msg_ext(i)); + block::NewOutMsg msg = trans.extract_out_msg_ext(i); + if (msg_metadata_enabled_) { + msg.metadata = msg_metadata; + } + register_new_msg(std::move(msg)); } } @@ -3054,6 +4284,15 @@ void Collator::register_new_msgs(block::Transaction& trans) { * */ +/** + * Stores an external block reference to a CellBuilder object. + * + * @param cb The CellBuilder object to store the reference in. + * @param id_ext The block ID. + * @param end_lt The end logical time of the block. + * + * @returns True if the reference was successfully stored, false otherwise. + */ bool store_ext_blk_ref_to(vm::CellBuilder& cb, const ton::BlockIdExt& id_ext, ton::LogicalTime end_lt) { return cb.store_long_bool(end_lt, 64) // end_lt:uint64 && cb.store_long_bool(id_ext.seqno(), 32) // seq_no:uint32 @@ -3061,6 +4300,15 @@ bool store_ext_blk_ref_to(vm::CellBuilder& cb, const ton::BlockIdExt& id_ext, to && cb.store_bits_bool(id_ext.file_hash); // file_hash:bits256 } +/** + * Stores an external block reference to a CellBuilder. + * + * @param cb The CellBuilder to store the reference in. + * @param id_ext The block ID. + * @param blk_root The root of the block. + * + * @returns True if the reference was successfully stored, false otherwise. + */ bool store_ext_blk_ref_to(vm::CellBuilder& cb, const ton::BlockIdExt& id_ext, Ref blk_root) { block::gen::Block::Record rec; block::gen::BlockInfo::Record info; @@ -3075,6 +4323,19 @@ bool store_ext_blk_ref_to(vm::CellBuilder& cb, const ton::BlockIdExt& id_ext, Re && store_ext_blk_ref_to(cb, id_ext, info.end_lt); // store } +/** + * Updates one shard description in the masterchain shard configuration. + * Used in masterchain collator. + * + * @param info The shard information to be updated. + * @param sibling The sibling shard information. + * @param wc_info The workchain information. + * @param now The current Unix time. + * @param ccvc The Catchain validators configuration. + * @param update_cc Flag indicating whether to update the Catchain seqno. + * + * @returns A boolean value indicating whether the shard description has changed. + */ static int update_one_shard(block::McShardHash& info, const block::McShardHash* sibling, const block::WorkchainInfo* wc_info, ton::UnixTime now, const block::CatchainValidatorsConfig& ccvc, bool update_cc) { @@ -3127,6 +4388,16 @@ static int update_one_shard(block::McShardHash& info, const block::McShardHash* return changed; } +/** + * Updates the shard configuration in the masterchain. + * Used in masterchain collator. + * + * @param wc_set The set of workchains. + * @param ccvc The Catchain validators configuration. + * @param update_cc A boolean indicating whether to update the Catchain seqno. + * + * @returns True if the shard configuration was successfully updated, false otherwise. + */ bool Collator::update_shard_config(const block::WorkchainSet& wc_set, const block::CatchainValidatorsConfig& ccvc, bool update_cc) { LOG(DEBUG) << "updating shard configuration (update_cc=" << update_cc << ")"; @@ -3153,6 +4424,12 @@ bool Collator::update_shard_config(const block::WorkchainSet& wc_set, const bloc }); } +/** + * Creates McStateExtra. + * Used in masterchain collator. + * + * @returns True if the creation is successful, false otherwise. + */ bool Collator::create_mc_state_extra() { if (!is_masterchain()) { CHECK(mc_state_extra_.is_null()); @@ -3179,9 +4456,12 @@ bool Collator::create_mc_state_extra() { bool ignore_cfg_changes = false; Ref cfg0; if (!block::valid_config_data(cfg_smc_config, config_addr, true, true, old_mparams_)) { - block::gen::t_Hashmap_32_Ref_Cell.print_ref(std::cerr, cfg_smc_config); LOG(ERROR) << "configuration smart contract "s + config_addr.to_hex() + " contains an invalid configuration in its data, IGNORING CHANGES"; + FLOG(WARNING) { + sb << "ignored configuration: "; + block::gen::t_Hashmap_32_Ref_Cell.print_ref(sb, cfg_smc_config); + }; ignore_cfg_changes = true; } else { cfg0 = cfg_dict.lookup_ref(td::BitArray<32>{(long long)0}); @@ -3219,34 +4499,26 @@ bool Collator::create_mc_state_extra() { return fatal_error(wset_res.move_as_error()); } bool update_shard_cc = is_key_block_ || (now_ / ccvc.shard_cc_lifetime > prev_now_ / ccvc.shard_cc_lifetime); - // temp debug - if (verbosity >= 3 * 1) { - auto csr = shard_conf_->get_root_csr(); - LOG(INFO) << "new shard configuration before post-processing is"; - std::ostringstream os; - csr->print_rec(os); - block::gen::t_ShardHashes.print(os, csr.write()); - LOG(INFO) << os.str(); - } - // end (temp debug) if (!update_shard_config(wset_res.move_as_ok(), ccvc, update_shard_cc)) { auto csr = shard_conf_->get_root_csr(); if (csr.is_null()) { LOG(WARNING) << "new shard configuration is null (!)"; } else { LOG(WARNING) << "invalid new shard configuration is"; - std::ostringstream os; - csr->print_rec(os); - block::gen::t_ShardHashes.print(os, csr.write()); - LOG(WARNING) << os.str(); + FLOG(WARNING) { + csr->print_rec(sb); + block::gen::t_ShardHashes.print(sb, csr); + }; } return fatal_error("cannot post-process shard configuration"); } // 3. save new shard_hashes state_extra.shard_hashes = shard_conf_->get_root_csr(); - if (verbosity >= 3 * 0) { // DEBUG - std::cerr << "updated shard configuration to "; - block::gen::t_ShardHashes.print(std::cerr, *state_extra.shard_hashes); + if (verbosity >= 3) { + FLOG(INFO) { + sb << "updated shard configuration to "; + block::gen::t_ShardHashes.print(sb, state_extra.shard_hashes); + }; } if (!block::gen::t_ShardHashes.validate_upto(10000, *state_extra.shard_hashes)) { return fatal_error("new ShardHashes is invalid"); @@ -3282,7 +4554,7 @@ bool Collator::create_mc_state_extra() { cc_updated = true; LOG(INFO) << "increased masterchain catchain seqno to " << val_info.catchain_seqno; } - auto nodes = block::Config::do_compute_validator_set(ccvc, shard_, *cur_validators, now_, val_info.catchain_seqno); + auto nodes = block::Config::do_compute_validator_set(ccvc, shard_, *cur_validators, val_info.catchain_seqno); LOG_CHECK(!nodes.empty()) << "validator node list in unpacked validator set is empty"; auto vlist_hash = block::compute_validator_set_hash(/* val_info.catchain_seqno */ 0, shard_, std::move(nodes)); @@ -3347,13 +4619,18 @@ bool Collator::create_mc_state_extra() { if (verify >= 2) { LOG(INFO) << "verifying new BlockCreateStats"; if (!block::gen::t_BlockCreateStats.validate_csr(100000, cs)) { - cs->print_rec(std::cerr); - block::gen::t_BlockCreateStats.print(std::cerr, *cs); + FLOG(WARNING) { + sb << "BlockCreateStats in the new masterchain state failed to pass automated validity checks: "; + cs->print_rec(sb); + block::gen::t_BlockCreateStats.print(sb, cs); + }; return fatal_error("BlockCreateStats in the new masterchain state failed to pass automated validity checks"); } } if (verbosity >= 4 * 1) { - block::gen::t_BlockCreateStats.print(std::cerr, *cs); + FLOG(INFO) { + block::gen::t_BlockCreateStats.print(sb, cs); + }; } } else { state_extra.r1.block_create_stats.clear(); @@ -3371,6 +4648,16 @@ bool Collator::create_mc_state_extra() { return true; } +/** + * Updates the `block_creator_stats_` for a given key. + * Used in masterchain collator. + * + * @param key The 256-bit key of the creator. + * @param shard_incr The increment value for the shardchain block counter. + * @param mc_incr The increment value for the masterchain block counter. + * + * @returns True if the block creator count was successfully updated, false otherwise. + */ bool Collator::update_block_creator_count(td::ConstBitPtr key, unsigned shard_incr, unsigned mc_incr) { LOG(DEBUG) << "increasing CreatorStats for " << key.to_hex(256) << " by (" << mc_incr << ", " << shard_incr << ")"; block::DiscountedCounter mc_cnt, shard_cnt; @@ -3378,7 +4665,6 @@ bool Collator::update_block_creator_count(td::ConstBitPtr key, unsigned shard_in if (!block::unpack_CreatorStats(std::move(cs), mc_cnt, shard_cnt)) { return fatal_error("cannot unpack CreatorStats for "s + key.to_hex(256) + " from previous masterchain state"); } - // std::cerr << mc_cnt.to_str() << " " << shard_cnt.to_str() << std::endl; if (mc_incr && !mc_cnt.increase_by(mc_incr, now_)) { return fatal_error(PSTRING() << "cannot increase masterchain block counter in CreatorStats for " << key.to_hex(256) << " by " << mc_incr << " (old value is " << mc_cnt.to_str() << ")"); @@ -3397,6 +4683,17 @@ bool Collator::update_block_creator_count(td::ConstBitPtr key, unsigned shard_in return true; } +/** + * Determines if the creator count is outdated for a given key. + * Used in masterchain collator. + * + * @param key The key of the creator. + * @param cs The CellSlice containing the CreatorStats. + * + * @returns -1 if there was a fatal error. + * 0 if the CreatorStats should be removed as they are stale, + * 1 if the CreatorStats are still valid. + */ int Collator::creator_count_outdated(td::ConstBitPtr key, vm::CellSlice& cs) { block::DiscountedCounter mc_cnt, shard_cnt; if (!(block::fetch_CreatorStats(cs, mc_cnt, shard_cnt) && cs.empty_ext())) { @@ -3415,6 +4712,11 @@ int Collator::creator_count_outdated(td::ConstBitPtr key, vm::CellSlice& cs) { } } +/** + * Updates `block_create_stats_` using information about creators of all new blocks. + * + * @returns True if the update was successful, false otherwise. + */ bool Collator::update_block_creator_stats() { if (!create_stats_enabled_) { return true; @@ -3465,10 +4767,25 @@ bool Collator::update_block_creator_stats() { return cnt >= 0; } +/** + * Retrieves the global masterchain config from the config contract. + * + * @param cfg_addr The address of the configuration smart contract. + * + * @returns A Result object containing a reference to the configuration data. + */ td::Result> Collator::get_config_data_from_smc(const ton::StdSmcAddress& cfg_addr) { return block::get_config_data_from_smc(account_dict->lookup_ref(cfg_addr)); } +/** + * Fetches and validates a new configuration from the configuration smart contract. + * + * @param cfg_addr The address of the configuration smart contract. + * @param new_config A reference to a vm::Cell object to store the new configuration. + * + * @returns True if the new configuration was successfully fetched, false otherwise. + */ bool Collator::try_fetch_new_config(const ton::StdSmcAddress& cfg_addr, Ref& new_config) { auto cfg_res = get_config_data_from_smc(cfg_addr); if (cfg_res.is_error()) { @@ -3486,28 +4803,65 @@ bool Collator::try_fetch_new_config(const ton::StdSmcAddress& cfg_addr, Refestimate_block_size(); LOG(INFO) << "block load statistics: gas=" << block_limit_status_->gas_used << " lt_delta=" << block_limit_status_->cur_lt - block_limit_status_->limits.start_lt << " size_estimate=" << block_size_estimate_; - auto cl = block_limit_status_->classify(); - if (cl <= block::ParamLimits::cl_underload) { - underload_history_ |= 1; - LOG(INFO) << "block is underloaded"; - } else if (cl >= block::ParamLimits::cl_soft) { - overload_history_ |= 1; - LOG(INFO) << "block is overloaded (category " << cl << ")"; + block_limit_class_ = std::max(block_limit_class_, block_limit_status_->classify()); + if (block_limit_class_ >= block::ParamLimits::cl_soft || dispatch_queue_total_limit_reached_) { + std::string message = "block is overloaded "; + if (block_limit_class_ >= block::ParamLimits::cl_soft) { + message += PSTRING() << "(category " << block_limit_class_ << ")"; + } else { + message += "(long dispatch queue processing)"; + } + if (out_msg_queue_size_ > SPLIT_MAX_QUEUE_SIZE) { + LOG(INFO) << message << ", but don't set overload history because out_msg_queue size is too big to split (" + << out_msg_queue_size_ << " > " << SPLIT_MAX_QUEUE_SIZE << ")"; + } else { + overload_history_ |= 1; + LOG(INFO) << message; + } + } else if (block_limit_class_ <= block::ParamLimits::cl_underload) { + if (out_msg_queue_size_ > MERGE_MAX_QUEUE_SIZE) { + LOG(INFO) + << "block is underloaded, but don't set underload history because out_msg_queue size is too big to merge (" + << out_msg_queue_size_ << " > " << MERGE_MAX_QUEUE_SIZE << ")"; + } else { + underload_history_ |= 1; + LOG(INFO) << "block is underloaded"; + } } else { LOG(INFO) << "block is loaded normally"; } + if (!(overload_history_ & 1) && out_msg_queue_size_ >= FORCE_SPLIT_QUEUE_SIZE && + out_msg_queue_size_ <= SPLIT_MAX_QUEUE_SIZE) { + overload_history_ |= 1; + LOG(INFO) << "setting overload history because out_msg_queue reached force split limit (" << out_msg_queue_size_ + << " >= " << FORCE_SPLIT_QUEUE_SIZE << ")"; + } if (collator_settings & 1) { LOG(INFO) << "want_split manually set"; want_split_ = true; @@ -3519,17 +4873,27 @@ bool Collator::check_block_overload() { } char buffer[17]; if (history_weight(overload_history_) >= 0) { - sprintf(buffer, "%016llx", (unsigned long long)overload_history_); + snprintf(buffer, sizeof(buffer), "%016llx", (unsigned long long)overload_history_); LOG(INFO) << "want_split set because of overload history " << buffer; want_split_ = true; } else if (history_weight(underload_history_) >= 0) { - sprintf(buffer, "%016llx", (unsigned long long)underload_history_); + snprintf(buffer, sizeof(buffer), "%016llx", (unsigned long long)underload_history_); LOG(INFO) << "want_merge set because of underload history " << buffer; want_merge_ = true; } return true; } +/** + * Processes removing a library from the collection of public libraries of an account. + * Updates the global collection of public libraries. + * Used in masterchain collator. + * + * @param key The 256-bit key of the public library to remove. + * @param addr The 256-bit address of the account where the library is removed. + * + * @returns True if the public library was successfully removed, false otherwise. + */ bool Collator::remove_public_library(td::ConstBitPtr key, td::ConstBitPtr addr) { LOG(INFO) << "Removing public library " << key.to_hex(256) << " of account " << addr.to_hex(256); auto val = shard_libraries_->lookup(key, 256); @@ -3573,6 +4937,17 @@ bool Collator::remove_public_library(td::ConstBitPtr key, td::ConstBitPtr addr) return true; } +/** + * Processes adding a library to the collection of public libraries of an account. + * Updates the global collection of public libraries. + * Used in masterchain collator. + * + * @param key The key of the public library. + * @param addr The address of the account where the library is added. + * @param library The root cell of the library. + * + * @returns True if the public library was successfully added, false otherwise. + */ bool Collator::add_public_library(td::ConstBitPtr key, td::ConstBitPtr addr, Ref library) { LOG(INFO) << "Adding public library " << key.to_hex(256) << " of account " << addr.to_hex(256); CHECK(library.not_null() && !library->get_hash().bits().compare(key, 256)); @@ -3611,6 +4986,17 @@ bool Collator::add_public_library(td::ConstBitPtr key, td::ConstBitPtr addr, Ref return true; } +/** + * Processes changes in libraries of an account. + * Updates the global collection of public libraries. + * Used in masterchain collator. + * + * @param orig_libs The original libraries of the account. + * @param final_libs The final libraries of the account. + * @param addr The address associated with the account. + * + * @returns True if the update was successful, false otherwise. + */ bool Collator::update_account_public_libraries(Ref orig_libs, Ref final_libs, const td::Bits256& addr) { vm::Dictionary dict1{std::move(orig_libs), 256}, dict2{std::move(final_libs), 256}; @@ -3628,6 +5014,13 @@ bool Collator::update_account_public_libraries(Ref orig_libs, Ref= 2 * 0) { - std::cerr << "New public libraries: "; - block::gen::t_HashmapE_256_LibDescr.print(std::cerr, shard_libraries_->get_root()); - shard_libraries_->get_root()->print_rec(std::cerr); + if (libraries_changed_ && verbosity >= 2) { + FLOG(INFO) { + sb << "New public libraries: "; + block::gen::t_HashmapE_256_LibDescr.print(sb, shard_libraries_->get_root()); + shard_libraries_->get_root()->print_rec(sb); + }; } return true; } +/** + * Updates the minimum reference masterchain seqno. + * + * @param some_mc_seqno The masterchain seqno to compare with the current minimum. + * + * @returns True if the minimum reference masterchain sequence number was updated successfully, false otherwise. + */ bool Collator::update_min_mc_seqno(ton::BlockSeqno some_mc_seqno) { min_ref_mc_seqno_ = std::min(min_ref_mc_seqno_, some_mc_seqno); return true; } +/** + * Registers an output message queue operation. + * Adds the proof to the block limit status every 64 operations. + * + * @param force If true, the proof will always be added to the block limit status. + * + * @returns True if the operation was successfully registered, false otherwise. + */ bool Collator::register_out_msg_queue_op(bool force) { ++out_msg_queue_ops_; if (force || !(out_msg_queue_ops_ & 63)) { @@ -3663,6 +5073,61 @@ bool Collator::register_out_msg_queue_op(bool force) { } } +/** + * Registers a dispatch queue message queue operation. + * Adds the proof to the block limit status every 64 operations. + * + * @param force If true, the proof will always be added to the block limit status. + * + * @returns True if the operation was successfully registered, false otherwise. + */ +bool Collator::register_dispatch_queue_op(bool force) { + ++dispatch_queue_ops_; + if (force || !(dispatch_queue_ops_ & 63)) { + return block_limit_status_->add_proof(dispatch_queue_->get_root_cell()); + } else { + return true; + } +} + +/** + * Update size estimation for the account dictionary. + * This is required to count the depth of the ShardAccounts dictionary in the block size estimation. + * account_dict_estimator_ is used for block limits only. + * + * @param trans Newly-created transaction. + * + * @returns True on success, false otherwise. + */ +bool Collator::update_account_dict_estimation(const block::transaction::Transaction& trans) { + const block::Account& acc = trans.account; + if (acc.orig_total_state->get_hash() != acc.total_state->get_hash() && + account_dict_estimator_added_accounts_.insert(acc.addr).second) { + // see combine_account_transactions + if (acc.status == block::Account::acc_nonexist) { + account_dict_estimator_->lookup_delete(acc.addr); + } else { + vm::CellBuilder cb; + if (!(cb.store_ref_bool(acc.total_state) // account_descr$_ account:^Account + && cb.store_bits_bool(acc.last_trans_hash_) // last_trans_hash:bits256 + && cb.store_long_bool(acc.last_trans_lt_, 64) // last_trans_lt:uint64 + && account_dict_estimator_->set_builder(acc.addr, cb))) { + return false; + } + } + } + ++account_dict_ops_; + if (!(account_dict_ops_ & 15)) { + return block_limit_status_->add_proof(account_dict_estimator_->get_root_cell()); + } + return true; +} + +/** + * Creates a new shard state and the Merkle update. + * + * @returns True if the shard state and Merkle update were successfully created, false otherwise. + */ bool Collator::create_shard_state() { Ref msg_q_info; vm::CellBuilder cb, cb2; @@ -3697,9 +5162,11 @@ bool Collator::create_shard_state() { } LOG(DEBUG) << "min_ref_mc_seqno is " << min_ref_mc_seqno_; if (verbosity > 2) { - std::cerr << "new ShardState: "; - block::gen::t_ShardState.print_ref(std::cerr, state_root); - vm::load_cell_slice(state_root).print_rec(std::cerr); + FLOG(INFO) { + sb << "new ShardState: "; + block::gen::t_ShardState.print_ref(sb, state_root); + vm::load_cell_slice(state_root).print_rec(sb); + }; } if (verify >= 2) { LOG(INFO) << "verifying new ShardState"; @@ -3708,10 +5175,15 @@ bool Collator::create_shard_state() { } LOG(INFO) << "creating Merkle update for the ShardState"; state_update = vm::MerkleUpdate::generate(prev_state_root_, state_root, state_usage_tree_.get()); + if (state_update.is_null()) { + return fatal_error("cannot create Merkle update for ShardState"); + } if (verbosity > 2) { - std::cerr << "Merkle Update for ShardState: "; - vm::CellSlice cs{vm::NoVm{}, state_update}; - cs.print_rec(std::cerr); + FLOG(INFO) { + sb << "Merkle Update for ShardState: "; + vm::CellSlice cs{vm::NoVm{}, state_update}; + cs.print_rec(sb); + }; } LOG(INFO) << "updating block profile statistics"; block_limit_status_->add_proof(state_root); @@ -3719,11 +5191,20 @@ bool Collator::create_shard_state() { return true; } -// stores BlkMasterInfo (for non-masterchain blocks) +/** + * Stores BlkMasterInfo (for non-masterchain blocks) in the provided CellBuilder. + * + * @param cb The CellBuilder to store the reference in. + * + * @returns True if the reference is successfully stored, false otherwise. + */ bool Collator::store_master_ref(vm::CellBuilder& cb) { return mc_block_root.not_null() && store_ext_blk_ref_to(cb, mc_block_id_, mc_block_root); } +/** + * Updates the processed_upto information for the new block based on the information on the last processed inbound message. + */ bool Collator::update_processed_upto() { auto ref_mc_seqno = is_masterchain() ? new_block_seqno : prev_mc_block_seqno; update_min_mc_seqno(ref_mc_seqno); @@ -3738,20 +5219,53 @@ bool Collator::update_processed_upto() { return processed_upto_->compactify(); } +/** + * Computes the outbound message queue. + * + * @param out_msg_queue_info A reference to a vm::Cell object to store the computed queue. + * + * @returns True if the computation is successful, False otherwise. + */ bool Collator::compute_out_msg_queue_info(Ref& out_msg_queue_info) { if (verbosity >= 2) { - auto rt = out_msg_queue_->get_root(); - std::cerr << "resulting out_msg_queue is "; - block::gen::t_OutMsgQueue.print(std::cerr, *rt); - rt->print_rec(std::cerr); + FLOG(INFO) { + auto rt = out_msg_queue_->get_root(); + sb << "resulting out_msg_queue is "; + block::gen::t_OutMsgQueue.print(sb, rt); + rt->print_rec(sb); + }; } vm::CellBuilder cb; - return register_out_msg_queue_op(true) && out_msg_queue_->append_dict_to_bool(cb) // _ out_queue:OutMsgQueue - && processed_upto_->pack(cb) // proc_info:ProcessedInfo - && ihr_pending->append_dict_to_bool(cb) // ihr_pending:IhrPendingInfo + // out_msg_queue_extra#0 dispatch_queue:DispatchQueue out_queue_size:(Maybe uint48) = OutMsgQueueExtra; + // ... extra:(Maybe OutMsgQueueExtra) + if (!dispatch_queue_->is_empty() || store_out_msg_queue_size_) { + if (!(cb.store_long_bool(1, 1) && cb.store_long_bool(0, 4) && dispatch_queue_->append_dict_to_bool(cb))) { + return false; + } + if (!(cb.store_bool_bool(store_out_msg_queue_size_) && + (!store_out_msg_queue_size_ || cb.store_long_bool(out_msg_queue_size_, 48)))) { + return false; + } + } else { + if (!cb.store_long_bool(0, 1)) { + return false; + } + } + vm::CellSlice maybe_extra = cb.as_cellslice(); + cb.reset(); + + return register_out_msg_queue_op(true) && register_dispatch_queue_op(true) && + out_msg_queue_->append_dict_to_bool(cb) // _ out_queue:OutMsgQueue + && processed_upto_->pack(cb) // proc_info:ProcessedInfo + && cb.append_cellslice_bool(maybe_extra) // extra:(Maybe OutMsgQueueExtra) && cb.finalize_to(out_msg_queue_info); } +/** + * Computes the total balance of the shard state. + * + * @returns True if the total balance computation is successful, false otherwise. + */ bool Collator::compute_total_balance() { // 1. compute total_balance_ from the augmentation value of ShardAccounts auto accounts_extra = account_dict->get_root_extra(); @@ -3769,8 +5283,10 @@ bool Collator::compute_total_balance() { } vm::CellSlice cs{*(in_msg_dict->get_root_extra())}; if (verbosity > 2) { - block::gen::t_ImportFees.print(std::cerr, vm::CellSlice{*(in_msg_dict->get_root_extra())}); - cs.print_rec(std::cerr); + FLOG(INFO) { + block::gen::t_ImportFees.print(sb, in_msg_dict->get_root_extra()); + cs.print_rec(sb); + }; } auto new_import_fees = block::tlb::t_Grams.as_integer_skip(cs); if (new_import_fees.is_null()) { @@ -3785,7 +5301,16 @@ bool Collator::compute_total_balance() { LOG(ERROR) << "cannot unpack CurrencyCollection from the root of OutMsgDescr"; return false; } - value_flow_.fees_collected += new_transaction_fees + new_import_fees; + block::CurrencyCollection total_fees = new_transaction_fees + new_import_fees; + value_flow_.fees_collected += total_fees; + if (is_masterchain()) { + block::CurrencyCollection burned = config_->get_burning_config().calculate_burned_fees(total_fees); + if (!burned.is_valid()) { + return fatal_error("cannot calculate amount of burned masterchain fees"); + } + value_flow_.fees_collected -= burned; + value_flow_.burned += burned; + } // 3. compute total_validator_fees total_validator_fees_ += value_flow_.fees_collected; total_validator_fees_ -= value_flow_.recovered; @@ -3793,6 +5318,13 @@ bool Collator::compute_total_balance() { return true; } +/** + * Creates BlockInfo of the new block. + * + * @param block_info A reference to the cell to put the serialized info to. + * + * @returns True if the block info cell was successfully created, false otherwise. + */ bool Collator::create_block_info(Ref& block_info) { vm::CellBuilder cb, cb2; bool mc = is_masterchain(); @@ -3830,10 +5362,24 @@ bool Collator::create_block_info(Ref& block_info) { && cb.finalize_to(block_info); } +/** + * Stores the version information in a CellBuilder. + * + * @param cb The CellBuilder object to store the version information. + * + * @returns True if the version information was successfully stored, false otherwise. + */ bool Collator::store_version(vm::CellBuilder& cb) const { return block::gen::t_GlobalVersion.pack_capabilities(cb, supported_version(), supported_capabilities()); } +/** + * Stores the zero state reference in the given CellBuilder. + * + * @param cb The CellBuilder to store the zero state reference in. + * + * @returns True if the zero state reference is successfully stored, false otherwise. + */ bool Collator::store_zero_state_ref(vm::CellBuilder& cb) { CHECK(prev_state_root_.not_null()); RootHash root_hash = prev_state_root_->get_hash().bits(); @@ -3846,6 +5392,14 @@ bool Collator::store_zero_state_ref(vm::CellBuilder& cb) { && cb.store_bits_bool(prev_blocks[0].file_hash); // file_hash:bits256 } +/** + * Stores the previous block references to the given CellBuilder. + * + * @param cb The CellBuilder object to store the references. + * @param is_after_merge A boolean indicating whether the new block after a merge. + * + * @returns True if the references are successfully stored, false otherwise. + */ bool Collator::store_prev_blk_ref(vm::CellBuilder& cb, bool is_after_merge) { if (is_after_merge) { auto root2 = prev_block_data.at(1)->root_cell(); @@ -3863,6 +5417,11 @@ bool Collator::store_prev_blk_ref(vm::CellBuilder& cb, bool is_after_merge) { } } +/** + * Validates the value flow of the block. + * + * @returns True if the value flow is correct, false otherwise. + */ bool Collator::check_value_flow() { if (!value_flow_.validate()) { LOG(ERROR) << "incorrect value flow in new block : " << value_flow_.to_str(); @@ -3872,6 +5431,13 @@ bool Collator::check_value_flow() { return true; } +/** + * Creates the BlockExtra of the new block. + * + * @param block_extra A reference to the cell to put the serialized info to. + * + * @returns True if the block extra data was successfully created, false otherwise. + */ bool Collator::create_block_extra(Ref& block_extra) { bool mc = is_masterchain(); Ref mc_block_extra; @@ -3887,6 +5453,14 @@ bool Collator::create_block_extra(Ref& block_extra) { && cb.finalize_to(block_extra); // = BlockExtra; } +/** + * Creates the McBlockExtra of the new masterchain block. + * Used in masterchain collator. + * + * @param mc_block_extra A reference to the cell to put the serialized info to. + * + * @returns True if the extra data was successfully created, false otherwise. + */ bool Collator::create_mc_block_extra(Ref& mc_block_extra) { if (!is_masterchain()) { return false; @@ -3904,6 +5478,18 @@ bool Collator::create_mc_block_extra(Ref& mc_block_extra) { && cb.finalize_to(mc_block_extra); // = McBlockExtra } +/** + * Serialized the new block. + * + * This function performs the following steps: + * 1. Creates a BlockInfo for the new block. + * 2. Checks the value flow for the new block. + * 3. Creates a BlockExtra for the new block. + * 4. Builds a new block using the created BlockInfo, value flow, state update, and BlockExtra. + * 5. Verifies the new block if the verification is enabled. + * + * @returns True if the new block is successfully created, false otherwise. + */ bool Collator::create_block() { Ref block_info, extra; if (!create_block_info(block_info)) { @@ -3928,13 +5514,15 @@ bool Collator::create_block() { return fatal_error("cannot create new Block"); } if (verbosity >= 3 * 1) { - std::cerr << "new Block: "; - block::gen::t_Block.print_ref(std::cerr, new_block); - vm::load_cell_slice(new_block).print_rec(std::cerr); + FLOG(INFO) { + sb << "new Block: "; + block::gen::t_Block.print_ref(sb, new_block); + vm::load_cell_slice(new_block).print_rec(sb); + }; } if (verify >= 1) { LOG(INFO) << "verifying new Block"; - if (!block::gen::t_Block.validate_ref(1000000, new_block)) { + if (!block::gen::t_Block.validate_ref(10000000, new_block)) { return fatal_error("new Block failed to pass automatic validity tests"); } } @@ -3942,6 +5530,15 @@ bool Collator::create_block() { return true; } +/** + * Collates the shard block description set. + * Used in masterchain collator. + * + * This function creates a dictionary and populates it with the shard block descriptions. + * + * @returns A `Ref` containing the serialized `TopBlockDescrSet` record. + * If serialization fails, an empty `Ref` is returned. + */ Ref Collator::collate_shard_block_descr_set() { vm::Dictionary dict{96}; for (const auto& descr : used_shard_block_descr_) { @@ -3959,13 +5556,20 @@ Ref Collator::collate_shard_block_descr_set() { return {}; } if (verbosity >= 4 * 1) { - std::cerr << "serialized TopBlockDescrSet for collated data is: "; - block::gen::t_TopBlockDescrSet.print_ref(std::cerr, cell); - vm::load_cell_slice(cell).print_rec(std::cerr); + FLOG(INFO) { + sb << "serialized TopBlockDescrSet for collated data is: "; + block::gen::t_TopBlockDescrSet.print_ref(sb, cell); + vm::load_cell_slice(cell).print_rec(sb); + }; } return cell; } +/** + * Creates collated data for the block. + * + * @returns True if the collated data was successfully created, false otherwise. + */ bool Collator::create_collated_data() { // TODO: store something into collated_roots_ // 1. store the set of used shard block descriptions @@ -3981,6 +5585,18 @@ bool Collator::create_collated_data() { return true; } +/** + * Creates a block candidate for the Collator. + * + * This function serializes the new block and collated data, and creates a BlockCandidate object + * with the necessary information. It then checks if the size of the block candidate exceeds the + * limits specified in the consensus configuration. + * + * Finally, the block candidate is saved to the disk. + * If there are any bad external messages or delayed external messages, the ValidatorManager is called to handle them. + * + * @returns True if the block candidate was created successfully, false otherwise. + */ bool Collator::create_block_candidate() { // 1. serialize block LOG(INFO) << "serializing new Block"; @@ -4026,23 +5642,57 @@ bool Collator::create_block_candidate() { ton::BlockIdExt{ton::BlockId{shard_, new_block_seqno}, new_block->get_hash().bits(), block::compute_file_hash(blk_slice.as_slice())}, block::compute_file_hash(cdata_slice.as_slice()), blk_slice.clone(), cdata_slice.clone()); + // 3.1 check block and collated data size + auto consensus_config = config_->get_consensus_config(); + if (block_candidate->data.size() > consensus_config.max_block_size) { + return fatal_error(PSTRING() << "block size (" << block_candidate->data.size() + << ") exceeds the limit in consensus config (" << consensus_config.max_block_size + << ")"); + } + if (block_candidate->collated_data.size() > consensus_config.max_collated_data_size) { + return fatal_error(PSTRING() << "collated data size (" << block_candidate->collated_data.size() + << ") exceeds the limit in consensus config (" + << consensus_config.max_collated_data_size << ")"); + } // 4. save block candidate - LOG(INFO) << "saving new BlockCandidate"; - td::actor::send_closure_later(manager, &ValidatorManager::set_block_candidate, block_candidate->id, - block_candidate->clone(), [self = get_self()](td::Result saved) -> void { - LOG(DEBUG) << "got answer to set_block_candidate"; - td::actor::send_closure_later(std::move(self), &Collator::return_block_candidate, - std::move(saved)); - }); + if (mode_ & CollateMode::skip_store_candidate) { + td::actor::send_closure_later(actor_id(this), &Collator::return_block_candidate, td::Unit()); + } else { + LOG(INFO) << "saving new BlockCandidate"; + td::actor::send_closure_later( + manager, &ValidatorManager::set_block_candidate, block_candidate->id, block_candidate->clone(), + validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), + [self = get_self()](td::Result saved) -> void { + LOG(DEBUG) << "got answer to set_block_candidate"; + td::actor::send_closure_later(std::move(self), &Collator::return_block_candidate, std::move(saved)); + }); + } // 5. communicate about bad and delayed external messages if (!bad_ext_msgs_.empty() || !delay_ext_msgs_.empty()) { LOG(INFO) << "sending complete_external_messages() to Manager"; td::actor::send_closure_later(manager, &ValidatorManager::complete_external_messages, std::move(delay_ext_msgs_), std::move(bad_ext_msgs_)); } + + double work_time = work_timer_.elapsed(); + double cpu_work_time = cpu_work_timer_.elapsed(); + LOG(WARNING) << "Collate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; + stats_.bytes = block_limit_status_->estimate_block_size(); + stats_.gas = block_limit_status_->gas_used; + stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt; + stats_.cat_bytes = block_limit_status_->limits.classify_size(stats_.bytes); + stats_.cat_gas = block_limit_status_->limits.classify_gas(stats_.gas); + stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); + td::actor::send_closure(manager, &ValidatorManager::record_collate_query_stats, block_candidate->id, work_time, + cpu_work_time, std::move(stats_)); return true; } +/** + * Returns a block candidate to the Promise. + * + * @param saved The result of saving the block candidate to the disk. + */ void Collator::return_block_candidate(td::Result saved) { // 6. return data to the original "caller" if (saved.is_error()) { @@ -4051,7 +5701,7 @@ void Collator::return_block_candidate(td::Result saved) { fatal_error(std::move(err)); } else { CHECK(block_candidate); - LOG(INFO) << "sending new BlockCandidate to Promise"; + LOG(WARNING) << "sending new BlockCandidate to Promise"; main_promise(block_candidate->clone()); busy_ = false; stop(); @@ -4064,7 +5714,20 @@ void Collator::return_block_candidate(td::Result saved) { * */ -td::Result Collator::register_external_message_cell(Ref ext_msg, const ExtMessage::Hash& ext_hash) { +/** + * Registers an external message to the list of external messages in the Collator. + * + * @param ext_msg The reference to the external message cell. + * @param ext_hash The hash of the external message. + * + * @returns Result indicating the success or failure of the registration. + * - If the external message is invalid, returns an error. + * - If the external message has been previously rejected, returns an error + * - If the external message has been previuosly registered and accepted, returns false. + * - Otherwise returns true. + */ +td::Result Collator::register_external_message_cell(Ref ext_msg, const ExtMessage::Hash& ext_hash, + int priority) { if (ext_msg->get_level() != 0) { return td::Status::Error("external message must have zero level"); } @@ -4088,6 +5751,9 @@ td::Result Collator::register_external_message_cell(Ref ext_msg, if (!block::tlb::t_Message.validate_ref(256, ext_msg)) { return td::Status::Error("external message is not a (Message Any) according to hand-written checks"); } + if (!block::tlb::validate_message_libs(ext_msg)) { + return td::Status::Error("external message has invalid libs in StateInit"); + } block::gen::CommonMsgInfo::Record_ext_in_msg_info info; if (!tlb::unpack_cell_inexact(ext_msg, info)) { return td::Status::Error("cannot unpack external message header"); @@ -4101,90 +5767,65 @@ td::Result Collator::register_external_message_cell(Ref ext_msg, return td::Status::Error("inbound external message has destination address not in this shard"); } if (verbosity > 2) { - std::cerr << "registered external message: "; - block::gen::t_Message_Any.print_ref(std::cerr, ext_msg); + FLOG(INFO) { + sb << "registered external message: "; + block::gen::t_Message_Any.print_ref(sb, ext_msg); + }; } ext_msg_map.emplace(hash, 1); - ext_msg_list_.emplace_back(std::move(ext_msg), ext_hash); + ext_msg_list_.push_back({std::move(ext_msg), ext_hash, priority}); return true; } -/* -td::Result Collator::register_external_message(td::Slice ext_msg_boc) { - if (ext_msg_boc.size() > max_ext_msg_size) { - return td::Status::Error("external message too large, rejecting"); - } - vm::BagOfCells boc; - auto res = boc.deserialize(ext_msg_boc); - if (res.is_error()) { - return res.move_as_error(); - } - if (boc.get_root_count() != 1) { - return td::Status::Error("external message is not a valid bag of cells"); // not a valid bag-of-Cells - } - return register_external_message_cell(boc.get_root_cell(0)); -} -*/ - -void Collator::after_get_external_messages(td::Result>> res) { +/** + * Callback function called after retrieving external messages. + * + * @param res The result of the external message retrieval operation. + */ +void Collator::after_get_external_messages(td::Result, int>>> res) { + // res: pair {ext msg, priority} --pending; if (res.is_error()) { fatal_error(res.move_as_error()); return; } auto vect = res.move_as_ok(); - for (auto&& ext_msg : vect) { + for (auto& p : vect) { + ++stats_.ext_msgs_total; + auto& ext_msg = p.first; + int priority = p.second; Ref ext_msg_cell = ext_msg->root_cell(); bool err = ext_msg_cell.is_null(); if (!err) { - auto reg_res = register_external_message_cell(std::move(ext_msg_cell), ext_msg->hash()); + auto reg_res = register_external_message_cell(std::move(ext_msg_cell), ext_msg->hash(), priority); if (reg_res.is_error() || !reg_res.move_as_ok()) { err = true; } } if (err) { + ++stats_.ext_msgs_filtered; bad_ext_msgs_.emplace_back(ext_msg->hash()); } } + LOG(WARNING) << "got " << vect.size() << " external messages from mempool, " << bad_ext_msgs_.size() + << " bad messages"; check_pending(); } -td::Result Collator::register_ihr_message_cell(Ref ihr_msg) { - return false; +/** + * Checks if collation was cancelled via cancellation token + * + * @returns false if the collation was cancelled, true otherwise + */ +bool Collator::check_cancelled() { + if (cancellation_token_) { + return fatal_error(td::Status::Error(ErrorCode::cancelled, "cancelled")); + } + return true; } -td::Result Collator::register_ihr_message(td::Slice ihr_msg_boc) { - if (ihr_msg_boc.size() > max_ihr_msg_size) { - return td::Status::Error("IHR message too large, rejecting"); - } - vm::BagOfCells boc; - auto res = boc.deserialize(ihr_msg_boc); - if (res.is_error()) { - return res.move_as_error(); - } - if (boc.get_root_count() != 1) { - return td::Status::Error("IHR message is not a valid bag of cells"); // not a valid bag-of-Cells - } - return register_ihr_message_cell(boc.get_root_cell(0)); -} - -td::Result Collator::register_shard_signatures_cell(Ref signatures) { - return false; -} - -td::Result Collator::register_shard_signatures(td::Slice signatures_boc) { - if (signatures_boc.size() > max_blk_sign_size) { - return td::Status::Error("Shardchain signatures block too large, rejecting"); - } - vm::BagOfCells boc; - auto res = boc.deserialize(signatures_boc); - if (res.is_error()) { - return res.move_as_error(); - } - if (boc.get_root_count() != 1) { - return td::Status::Error("Shardchain signatures block is not a valid bag of cells"); // not a valid bag-of-Cells - } - return register_shard_signatures_cell(boc.get_root_cell(0)); +td::uint32 Collator::get_skip_externals_queue_size() { + return SKIP_EXTERNALS_QUEUE_SIZE; } } // namespace validator diff --git a/validator/impl/collator.h b/validator/impl/collator.h index c92fa80e..5acc20e8 100644 --- a/validator/impl/collator.h +++ b/validator/impl/collator.h @@ -24,26 +24,7 @@ #include "vm/cells.h" namespace ton { -using td::Ref; extern int collator_settings; // +1 = force want_split, +2 = force want_merge -class Collator : public td::actor::Actor { - protected: - Collator() = default; - - public: - virtual ~Collator() = default; - static td::actor::ActorOwn create_collator( - td::actor::ActorId block_db, - ShardIdFull shard /* , td::actor::ActorId validator_manager */); - virtual void generate_block_candidate(ShardIdFull shard, td::Promise promise) = 0; - virtual td::Result register_external_message_cell(Ref ext_msg) = 0; - virtual td::Result register_external_message(td::Slice ext_msg_boc) = 0; - virtual td::Result register_ihr_message_cell(Ref ihr_msg) = 0; - virtual td::Result register_ihr_message(td::Slice ihr_msg_boc) = 0; - virtual td::Result register_shard_signatures_cell(Ref shard_blk_signatures) = 0; - virtual td::Result register_shard_signatures(td::Slice shard_blk_signatures_boc) = 0; -}; - } // namespace ton diff --git a/validator/impl/external-message.cpp b/validator/impl/external-message.cpp index 5d3028db..8b1f5eb7 100644 --- a/validator/impl/external-message.cpp +++ b/validator/impl/external-message.cpp @@ -86,24 +86,18 @@ td::Result> ExtMessageQ::create_ext_message(td::BufferSlice dat return Ref{true, std::move(data), std::move(ext_msg), dest_prefix, wc, addr}; } -void ExtMessageQ::run_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits, - td::actor::ActorId manager, +void ExtMessageQ::run_message(td::Ref message, td::actor::ActorId manager, td::Promise> promise) { - auto R = create_ext_message(std::move(data), limits); - if (R.is_error()) { - return promise.set_error(R.move_as_error_prefix("failed to parse external message ")); - } - auto M = R.move_as_ok(); - auto root = M->root_cell(); + auto root = message->root_cell(); block::gen::CommonMsgInfo::Record_ext_in_msg_info info; tlb::unpack_cell_inexact(root, info); // checked in create message - ton::StdSmcAddress addr = M->addr(); - ton::WorkchainId wc = M->wc(); + ton::StdSmcAddress addr = message->addr(); + ton::WorkchainId wc = message->wc(); run_fetch_account_state( wc, addr, manager, - [promise = std::move(promise), msg_root = root, wc, - M](td::Result, UnixTime, LogicalTime, std::unique_ptr>> + [promise = std::move(promise), msg_root = root, wc, addr, message]( + td::Result, UnixTime, LogicalTime, std::unique_ptr>> res) mutable { if (res.is_error()) { promise.set_error(td::Status::Error(PSLICE() << "Failed to get account state")); @@ -114,12 +108,13 @@ void ExtMessageQ::run_message(td::BufferSlice data, block::SizeLimitsConfig::Ext auto utime = std::get<1>(tuple); auto lt = std::get<2>(tuple); auto config = std::move(std::get<3>(tuple)); - if (!acc.unpack(shard_acc, {}, utime, false)) { + bool special = wc == masterchainId && config->is_special_smartcontract(addr); + if (!acc.unpack(shard_acc, utime, special)) { promise.set_error(td::Status::Error(PSLICE() << "Failed to unpack account state")); } else { auto status = run_message_on_account(wc, &acc, utime, lt + 1, msg_root, std::move(config)); if (status.is_ok()) { - promise.set_value(std::move(M)); + promise.set_value(std::move(message)); } else { promise.set_error(td::Status::Error(PSLICE() << "External message was not accepted\n" << status.message())); @@ -141,30 +136,30 @@ td::Status ExtMessageQ::run_message_on_account(ton::WorkchainId wc, td::BitArray<256> rand_seed_; block::ComputePhaseConfig compute_phase_cfg_; block::ActionPhaseConfig action_phase_cfg_; + block::SerializeConfig serialize_config_; td::RefInt256 masterchain_create_fee, basechain_create_fee; - auto fetch_res = Collator::impl_fetch_config_params(std::move(config), &old_mparams, - &storage_prices_, &storage_phase_cfg_, - &rand_seed_, &compute_phase_cfg_, - &action_phase_cfg_, &masterchain_create_fee, - &basechain_create_fee, wc, utime); + auto fetch_res = block::FetchConfigParams::fetch_config_params( + *config, &old_mparams, &storage_prices_, &storage_phase_cfg_, &rand_seed_, &compute_phase_cfg_, + &action_phase_cfg_, &serialize_config_, &masterchain_create_fee, &basechain_create_fee, wc, utime); if(fetch_res.is_error()) { auto error = fetch_res.move_as_error(); LOG(DEBUG) << "Cannot fetch config params: " << error.message(); return error.move_as_error_prefix("Cannot fetch config params: "); } + compute_phase_cfg_.libraries = std::make_unique(config->get_libraries_root(), 256); compute_phase_cfg_.with_vm_log = true; + compute_phase_cfg_.stop_on_accept_message = true; - auto res = Collator::impl_create_ordinary_transaction(msg_root, acc, utime, lt, - &storage_phase_cfg_, &compute_phase_cfg_, - &action_phase_cfg_, - true, lt); + auto res = + Collator::impl_create_ordinary_transaction(msg_root, acc, utime, lt, &storage_phase_cfg_, &compute_phase_cfg_, + &action_phase_cfg_, &serialize_config_, true, lt); if(res.is_error()) { auto error = res.move_as_error(); LOG(DEBUG) << "Cannot run message on account: " << error.message(); return error.move_as_error_prefix("Cannot run message on account: "); } - std::unique_ptr trans = res.move_as_ok(); + std::unique_ptr trans = res.move_as_ok(); auto trans_root = trans->commit(*acc); if (trans_root.is_null()) { diff --git a/validator/impl/external-message.hpp b/validator/impl/external-message.hpp index d5084761..ad7ecc74 100644 --- a/validator/impl/external-message.hpp +++ b/validator/impl/external-message.hpp @@ -61,8 +61,7 @@ class ExtMessageQ : public ExtMessage { ton::StdSmcAddress addr); static td::Result> create_ext_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits); - static void run_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits, - td::actor::ActorId manager, + static void run_message(td::Ref message, td::actor::ActorId manager, td::Promise> promise); static td::Status run_message_on_account(ton::WorkchainId wc, block::Account* acc, diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index 0131fff7..65b92262 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -34,18 +34,20 @@ #include "ton/ton-io.hpp" #include "liteserver.hpp" #include "validator/fabric.h" +#include "liteserver-cache.hpp" namespace ton { namespace validator { -td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_) { - return td::actor::create_actor("db", manager, db_root_); +td::actor::ActorOwn create_db_actor(td::actor::ActorId manager, std::string db_root_, + td::Ref opts) { + return td::actor::create_actor("db", manager, db_root_, opts); } td::actor::ActorOwn create_liteserver_cache_actor(td::actor::ActorId manager, std::string db_root) { - return td::actor::create_actor("cache"); + return td::actor::create_actor("cache"); } td::Result> create_block(BlockIdExt block_id, td::BufferSlice data) { @@ -117,10 +119,9 @@ td::Result> create_ext_message(td::BufferSlice data, return std::move(res); } -void run_check_external_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits, - td::actor::ActorId manager, +void run_check_external_message(Ref message, td::actor::ActorId manager, td::Promise> promise) { - ExtMessageQ::run_message(std::move(data), limits, std::move(manager), std::move(promise)); + ExtMessageQ::run_message(std::move(message), std::move(manager), std::move(promise)); } td::Result> create_ihr_message(td::BufferSlice data) { @@ -130,11 +131,11 @@ td::Result> create_ihr_message(td::BufferSlice data) { void run_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, - td::Ref approve_signatures, bool send_broadcast, + td::Ref approve_signatures, int send_broadcast_mode, td::actor::ActorId manager, td::Promise promise) { - td::actor::create_actor("accept", id, std::move(data), prev, std::move(validator_set), - std::move(signatures), std::move(approve_signatures), send_broadcast, - manager, std::move(promise)) + td::actor::create_actor( + PSTRING() << "accept" << id.id.to_str(), id, std::move(data), prev, std::move(validator_set), + std::move(signatures), std::move(approve_signatures), send_broadcast_mode, manager, std::move(promise)) .release(); } @@ -191,7 +192,7 @@ void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::act .release(); } -void run_validate_query(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, +void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, BlockCandidate candidate, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, bool is_fake) { @@ -201,26 +202,31 @@ void run_validate_query(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_maste seqno = p.seqno(); } } - td::actor::create_actor( - PSTRING() << (is_fake ? "fakevalidate" : "validateblock") << shard.to_str() << ":" << (seqno + 1), shard, min_ts, - min_masterchain_block_id, std::move(prev), std::move(candidate), std::move(validator_set), std::move(manager), - timeout, std::move(promise), is_fake) + static std::atomic idx; + td::actor::create_actor(PSTRING() << (is_fake ? "fakevalidate" : "validateblock") << shard.to_str() + << ":" << (seqno + 1) << "#" << idx.fetch_add(1), + shard, min_masterchain_block_id, std::move(prev), std::move(candidate), + std::move(validator_set), std::move(manager), timeout, std::move(promise), + is_fake) .release(); } -void run_collate_query(ShardIdFull shard, td::uint32 min_ts, const BlockIdExt& min_masterchain_block_id, - std::vector prev, Ed25519_PublicKey collator_id, td::Ref validator_set, - td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise) { +void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, + Ed25519_PublicKey creator, td::Ref validator_set, + td::Ref collator_opts, td::actor::ActorId manager, + td::Timestamp timeout, td::Promise promise, + td::CancellationToken cancellation_token, unsigned mode, int attempt_idx) { BlockSeqno seqno = 0; for (auto& p : prev) { if (p.seqno() > seqno) { seqno = p.seqno(); } } - td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, false, - min_ts, min_masterchain_block_id, std::move(prev), std::move(validator_set), - collator_id, std::move(manager), timeout, std::move(promise)) + td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1) + << (attempt_idx ? "_" + td::to_string(attempt_idx) : ""), + shard, false, min_masterchain_block_id, std::move(prev), std::move(validator_set), + creator, std::move(collator_opts), std::move(manager), timeout, std::move(promise), + std::move(cancellation_token), mode, attempt_idx) .release(); } @@ -233,15 +239,16 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b seqno = p.seqno(); } } - td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, 0, + td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, min_masterchain_block_id, std::move(prev), td::Ref{}, - Ed25519_PublicKey{Bits256::zero()}, std::move(manager), timeout, std::move(promise)) + Ed25519_PublicKey{Bits256::zero()}, td::Ref{true}, + std::move(manager), timeout, std::move(promise), td::CancellationToken{}, 0, 0) .release(); } void run_liteserver_query(td::BufferSlice data, td::actor::ActorId manager, td::actor::ActorId cache, td::Promise promise) { - LiteQuery::run_query(std::move(data), std::move(manager), std::move(promise)); + LiteQuery::run_query(std::move(data), std::move(manager), std::move(cache), std::move(promise)); } void run_fetch_account_state(WorkchainId wc, StdSmcAddress addr, td::actor::ActorId manager, diff --git a/validator/impl/liteserver-cache.hpp b/validator/impl/liteserver-cache.hpp new file mode 100644 index 00000000..20beca91 --- /dev/null +++ b/validator/impl/liteserver-cache.hpp @@ -0,0 +1,116 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "interfaces/liteserver.h" +#include + +namespace ton::validator { + +class LiteServerCacheImpl : public LiteServerCache { + public: + void start_up() override { + alarm(); + } + + void alarm() override { + alarm_timestamp() = td::Timestamp::in(60.0); + if (queries_cnt_ > 0 || !send_message_cache_.empty()) { + LOG(WARNING) << "LS Cache stats: " << queries_cnt_ << " queries, " << queries_hit_cnt_ << " hits; " + << cache_.size() << " entries, size=" << total_size_ << "/" << MAX_CACHE_SIZE << "; " + << send_message_cache_.size() << " different sendMessage queries, " << send_message_error_cnt_ + << " duplicates"; + queries_cnt_ = 0; + queries_hit_cnt_ = 0; + send_message_cache_.clear(); + send_message_error_cnt_ = 0; + } + } + + void lookup(td::Bits256 key, td::Promise promise) override { + ++queries_cnt_; + auto it = cache_.find(key); + if (it == cache_.end()) { + promise.set_error(td::Status::Error("not found")); + return; + } + ++queries_hit_cnt_; + auto entry = it->second.get(); + entry->remove(); + lru_.put(entry); + promise.set_value(entry->value_.clone()); + } + + void update(td::Bits256 key, td::BufferSlice value) override { + std::unique_ptr &entry = cache_[key]; + if (entry == nullptr) { + entry = std::make_unique(key, std::move(value)); + } else { + total_size_ -= entry->size(); + entry->value_ = std::move(value); + entry->remove(); + } + lru_.put(entry.get()); + total_size_ += entry->size(); + + while (total_size_ > MAX_CACHE_SIZE) { + auto to_remove = (CacheEntry *)lru_.get(); + CHECK(to_remove); + total_size_ -= to_remove->size(); + to_remove->remove(); + cache_.erase(to_remove->key_); + } + } + + void process_send_message(td::Bits256 key, td::Promise promise) override { + if (send_message_cache_.insert(key).second) { + promise.set_result(td::Unit()); + } else { + ++send_message_error_cnt_; + promise.set_error(td::Status::Error("duplicate message")); + } + } + + void drop_send_message_from_cache(td::Bits256 key) override { + send_message_cache_.erase(key); + } + + private: + struct CacheEntry : public td::ListNode { + explicit CacheEntry(td::Bits256 key, td::BufferSlice value) : key_(key), value_(std::move(value)) { + } + td::Bits256 key_; + td::BufferSlice value_; + + size_t size() const { + return value_.size() + 32 * 2; + } + }; + + std::map> cache_; + td::ListNode lru_; + size_t total_size_ = 0; + + size_t queries_cnt_ = 0, queries_hit_cnt_ = 0; + + std::set send_message_cache_; + size_t send_message_error_cnt_ = 0; + + static constexpr size_t MAX_CACHE_SIZE = 64 << 20; +}; + +} // namespace ton::validator diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 7f6811fb..06f40b8f 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -42,6 +42,8 @@ #include "signature-set.hpp" #include "fabric.h" #include +#include "td/actor/MultiPromise.h" +#include "collator-impl.h" namespace ton { @@ -54,8 +56,11 @@ td::int32 get_tl_tag(td::Slice slice) { } void LiteQuery::run_query(td::BufferSlice data, td::actor::ActorId manager, + td::actor::ActorId cache, td::Promise promise) { - td::actor::create_actor("litequery", std::move(data), std::move(manager), std::move(promise)).release(); + td::actor::create_actor("litequery", std::move(data), std::move(manager), std::move(cache), + std::move(promise)) + .release(); } void LiteQuery::fetch_account_state(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, @@ -64,8 +69,8 @@ void LiteQuery::fetch_account_state(WorkchainId wc, StdSmcAddress acc_addr, td: } LiteQuery::LiteQuery(td::BufferSlice data, td::actor::ActorId manager, - td::Promise promise) - : query_(std::move(data)), manager_(std::move(manager)), promise_(std::move(promise)) { + td::actor::ActorId cache, td::Promise promise) + : query_(std::move(data)), manager_(std::move(manager)), cache_(std::move(cache)), promise_(std::move(promise)) { timeout_ = td::Timestamp::in(default_timeout_msec * 0.001); } @@ -80,19 +85,13 @@ void LiteQuery::abort_query(td::Status reason) { if (acc_state_promise_) { acc_state_promise_.set_error(std::move(reason)); } else if (promise_) { + td::actor::send_closure(manager_, &ValidatorManager::add_lite_query_stats, query_obj_ ? query_obj_->get_id() : 0, + false); promise_.set_error(std::move(reason)); } stop(); } -void LiteQuery::abort_query_ext(td::Status reason, std::string comment) { - LOG(INFO) << "aborted liteserver query: " << comment << " : " << reason.to_string(); - if (promise_) { - promise_.set_error(reason.move_as_error_prefix(comment + " : ")); - } - stop(); -} - bool LiteQuery::fatal_error(td::Status error) { abort_query(std::move(error)); return false; @@ -110,8 +109,13 @@ void LiteQuery::alarm() { fatal_error(-503, "timeout"); } -bool LiteQuery::finish_query(td::BufferSlice result) { +bool LiteQuery::finish_query(td::BufferSlice result, bool skip_cache_update) { + if (use_cache_ && !skip_cache_update) { + td::actor::send_closure(cache_, &LiteServerCache::update, cache_key_, result.clone()); + } if (promise_) { + td::actor::send_closure(manager_, &ValidatorManager::add_lite_query_stats, query_obj_ ? query_obj_->get_id() : 0, + true); promise_.set_result(std::move(result)); stop(); return true; @@ -124,19 +128,67 @@ bool LiteQuery::finish_query(td::BufferSlice result) { void LiteQuery::start_up() { alarm_timestamp() = timeout_; - if(acc_state_promise_) { - td::actor::send_closure_later(actor_id(this),&LiteQuery::perform_fetchAccountState); + if (acc_state_promise_) { + td::actor::send_closure_later(actor_id(this), &LiteQuery::perform_fetchAccountState); return; } - auto F = fetch_tl_object(std::move(query_), true); + auto F = fetch_tl_object(query_, true); if (F.is_error()) { abort_query(F.move_as_error()); return; } + query_obj_ = F.move_as_ok(); + if (!cache_.empty() && query_obj_->get_id() == lite_api::liteServer_sendMessage::ID) { + // Dropping duplicate "sendMessage" + cache_key_ = td::sha256_bits256(query_); + td::actor::send_closure(cache_, &LiteServerCache::process_send_message, cache_key_, + [SelfId = actor_id(this)](td::Result R) { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &LiteQuery::perform); + } else { + td::actor::send_closure(SelfId, &LiteQuery::abort_query, + R.move_as_error_prefix("cannot send external message : ")); + } + }); + return; + } + use_cache_ = use_cache(); + if (use_cache_) { + cache_key_ = td::sha256_bits256(query_); + td::actor::send_closure( + cache_, &LiteServerCache::lookup, cache_key_, [SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &LiteQuery::perform); + } else { + td::actor::send_closure(SelfId, &LiteQuery::finish_query, R.move_as_ok(), true); + } + }); + } else { + perform(); + } +} + +bool LiteQuery::use_cache() { + if (cache_.empty()) { + return false; + } + bool use = false; lite_api::downcast_call( - *F.move_as_ok().get(), + *query_obj_, + td::overloaded( + [&](lite_api::liteServer_runSmcMethod& q) { + // wc=-1, seqno=-1 means "use latest mc block" + use = q.id_->workchain_ != masterchainId || q.id_->seqno_ != -1; + }, + [&](auto& obj) { use = false; })); + return use; +} + +void LiteQuery::perform() { + lite_api::downcast_call( + *query_obj_, td::overloaded( [&](lite_api::liteServer_getTime& q) { this->perform_getTime(); }, [&](lite_api::liteServer_getVersion& q) { this->perform_getVersion(); }, @@ -177,11 +229,19 @@ void LiteQuery::start_up() { [&](lite_api::liteServer_lookupBlock& q) { this->perform_lookupBlock(ton::create_block_id_simple(q.id_), q.mode_, q.lt_, q.utime_); }, + [&](lite_api::liteServer_lookupBlockWithProof& q) { + this->perform_lookupBlockWithProof(ton::create_block_id_simple(q.id_), ton::create_block_id(q.mc_block_id_), q.mode_, q.lt_, q.utime_); + }, [&](lite_api::liteServer_listBlockTransactions& q) { this->perform_listBlockTransactions(ton::create_block_id(q.id_), q.mode_, q.count_, (q.mode_ & 128) ? q.after_->account_ : td::Bits256::zero(), static_cast((q.mode_ & 128) ? (q.after_->lt_) : 0)); }, + [&](lite_api::liteServer_listBlockTransactionsExt& q) { + this->perform_listBlockTransactionsExt(ton::create_block_id(q.id_), q.mode_, q.count_, + (q.mode_ & 128) ? q.after_->account_ : td::Bits256::zero(), + static_cast((q.mode_ & 128) ? (q.after_->lt_) : 0)); + }, [&](lite_api::liteServer_getConfigParams& q) { this->perform_getConfigParams(ton::create_block_id(q.id_), (q.mode_ & 0xffff) | 0x10000, q.param_list_); }, @@ -205,9 +265,32 @@ void LiteQuery::start_up() { [&](lite_api::liteServer_getLibraries& q) { this->perform_getLibraries(q.library_list_); }, + [&](lite_api::liteServer_getLibrariesWithProof& q) { + this->perform_getLibrariesWithProof(ton::create_block_id(q.id_), q.mode_, q.library_list_); + }, [&](lite_api::liteServer_getShardBlockProof& q) { this->perform_getShardBlockProof(create_block_id(q.id_)); }, + [&](lite_api::liteServer_nonfinal_getCandidate& q) { + this->perform_nonfinal_getCandidate(q.id_->creator_, create_block_id(q.id_->block_id_), + q.id_->collated_data_hash_); + }, + [&](lite_api::liteServer_nonfinal_getValidatorGroups& q) { + this->perform_nonfinal_getValidatorGroups(q.mode_, ShardIdFull{q.wc_, (ShardId)q.shard_}); + }, + [&](lite_api::liteServer_getOutMsgQueueSizes& q) { + this->perform_getOutMsgQueueSizes(q.mode_ & 1 ? ShardIdFull(q.wc_, q.shard_) : td::optional()); + }, + [&](lite_api::liteServer_getBlockOutMsgQueueSize& q) { + this->perform_getBlockOutMsgQueueSize(q.mode_, create_block_id(q.id_)); + }, + [&](lite_api::liteServer_getDispatchQueueInfo& q) { + this->perform_getDispatchQueueInfo(q.mode_, create_block_id(q.id_), q.after_addr_, q.max_accounts_); + }, + [&](lite_api::liteServer_getDispatchQueueMessages& q) { + this->perform_getDispatchQueueMessages(q.mode_, create_block_id(q.id_), q.addr_, + std::max(q.after_lt_, 0), q.max_messages_); + }, [&](auto& obj) { this->abort_query(td::Status::Error(ErrorCode::protoviolation, "unknown query")); })); } @@ -276,21 +359,15 @@ void LiteQuery::perform_getBlock(BlockIdExt blkid) { fatal_error("invalid BlockIdExt"); return; } - get_block_handle_checked(blkid, [manager = manager_, Self = actor_id(this), blkid](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); - return; - } - td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(), - [=](td::Result> res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); - } else { - td::actor::send_closure_later(Self, &LiteQuery::continue_getBlock, blkid, - res.move_as_ok()); - } - }); - }); + td::actor::send_closure(manager_, &ValidatorManager::get_block_data_for_litequery, blkid, + [Self = actor_id(this), blkid](td::Result> res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::continue_getBlock, blkid, + res.move_as_ok()); + } + }); } void LiteQuery::continue_getBlock(BlockIdExt blkid, Ref block) { @@ -308,21 +385,15 @@ void LiteQuery::perform_getBlockHeader(BlockIdExt blkid, int mode) { fatal_error("invalid BlockIdExt"); return; } - get_block_handle_checked(blkid, [=, manager = manager_, Self = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); - return; - } - td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(), - [=](td::Result> res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); - } else { - td::actor::send_closure_later(Self, &LiteQuery::continue_getBlockHeader, blkid, - mode, res.move_as_ok()); - } - }); - }); + td::actor::send_closure(manager_, &ValidatorManager::get_block_data_for_litequery, blkid, + [Self = actor_id(this), blkid, mode](td::Result> res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::continue_getBlockHeader, blkid, mode, + res.move_as_ok()); + } + }); } static bool visit(Ref cell); @@ -428,33 +499,27 @@ void LiteQuery::perform_getState(BlockIdExt blkid) { fatal_error("cannot request total state: possibly too large"); return; } - get_block_handle_checked(blkid, [=, manager = manager_, Self = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); - return; - } - if (blkid.id.seqno) { - td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db, R.move_as_ok(), - [=](td::Result> res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); - } else { - td::actor::send_closure_later(Self, &LiteQuery::continue_getState, blkid, - res.move_as_ok()); - } - }); - } else { - td::actor::send_closure_later(manager, &ValidatorManager::get_zero_state, blkid, - [=](td::Result res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); - } else { - td::actor::send_closure_later(Self, &LiteQuery::continue_getZeroState, blkid, - res.move_as_ok()); - } - }); - } - }); + if (blkid.id.seqno) { + td::actor::send_closure(manager_, &ValidatorManager::get_block_state_for_litequery, blkid, + [Self = actor_id(this), blkid](td::Result> res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::continue_getState, blkid, + res.move_as_ok()); + } + }); + } else { + td::actor::send_closure_later(manager_, &ValidatorManager::get_zero_state, blkid, + [Self = actor_id(this), blkid](td::Result res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::continue_getZeroState, blkid, + res.move_as_ok()); + } + }); + } } void LiteQuery::continue_getState(BlockIdExt blkid, Ref state) { @@ -486,34 +551,24 @@ void LiteQuery::perform_sendMessage(td::BufferSlice data) { auto copy = data.clone(); td::actor::send_closure_later( manager_, &ValidatorManager::check_external_message, std::move(copy), - [Self = actor_id(this), data = std::move(data), manager = manager_](td::Result> res) mutable { - if(res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, - res.move_as_error_prefix("cannot apply external message to current state : "s)); + [Self = actor_id(this), data = std::move(data), manager = manager_, cache = cache_, + cache_key = cache_key_](td::Result> res) mutable { + if (res.is_error()) { + // Don't cache errors + td::actor::send_closure(cache, &LiteServerCache::drop_send_message_from_cache, cache_key); + td::actor::send_closure(Self, &LiteQuery::abort_query, + res.move_as_error_prefix("cannot apply external message to current state : "s)); } else { LOG(INFO) << "sending an external message to validator manager"; td::actor::send_closure_later(manager, &ValidatorManager::send_external_message, res.move_as_ok()); auto b = ton::create_serialize_tl_object(1); - td::actor::send_closure(Self, &LiteQuery::finish_query, std::move(b)); + td::actor::send_closure(Self, &LiteQuery::finish_query, std::move(b), false); } }); } void LiteQuery::get_block_handle_checked(BlockIdExt blkid, td::Promise promise) { - auto P = td::PromiseCreator::lambda( - [promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - } else { - auto handle = R.move_as_ok(); - if (handle->is_applied()) { - promise.set_result(std::move(handle)); - } else { - promise.set_error(td::Status::Error("block is not applied")); - } - } - }); - td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, blkid, false, std::move(P)); + td::actor::send_closure(manager_, &ValidatorManager::get_block_handle_for_litequery, blkid, std::move(promise)); } bool LiteQuery::request_mc_block_data(BlockIdExt blkid) { @@ -526,7 +581,7 @@ bool LiteQuery::request_mc_block_data(BlockIdExt blkid) { base_blk_id_ = blkid; ++pending_; td::actor::send_closure_later( - manager_, &ValidatorManager::get_block_data_from_db_short, blkid, + manager_, &ValidatorManager::get_block_data_for_litequery, blkid, [Self = actor_id(this), blkid](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, @@ -584,7 +639,7 @@ bool LiteQuery::request_mc_block_state(BlockIdExt blkid) { base_blk_id_ = blkid; ++pending_; td::actor::send_closure_later( - manager_, &ValidatorManager::get_shard_state_from_db_short, blkid, + manager_, &ValidatorManager::get_block_state_for_litequery, blkid, [Self = actor_id(this), blkid](td::Result> res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, @@ -614,22 +669,16 @@ bool LiteQuery::request_block_state(BlockIdExt blkid) { } blk_id_ = blkid; ++pending_; - get_block_handle_checked(blkid, [=, manager = manager_, Self = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); - return; - } - td::actor::send_closure_later( - manager, &ValidatorManager::get_shard_state_from_db, R.move_as_ok(), - [=](td::Result> res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, - res.move_as_error_prefix("cannot load state for "s + blkid.to_str() + " : ")); - } else { - td::actor::send_closure_later(Self, &LiteQuery::got_block_state, blkid, res.move_as_ok()); - } - }); - }); + td::actor::send_closure(manager_, &ValidatorManager::get_block_state_for_litequery, blkid, + [Self = actor_id(this), blkid](td::Result> res) { + if (res.is_error()) { + td::actor::send_closure( + Self, &LiteQuery::abort_query, + res.move_as_error_prefix("cannot load state for "s + blkid.to_str() + " : ")); + } else { + td::actor::send_closure_later(Self, &LiteQuery::got_block_state, blkid, res.move_as_ok()); + } + }); return true; } @@ -642,22 +691,16 @@ bool LiteQuery::request_block_data(BlockIdExt blkid) { } blk_id_ = blkid; ++pending_; - get_block_handle_checked(blkid, [=, manager = manager_, Self = actor_id(this)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); - return; - } - td::actor::send_closure_later( - manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(), - [=](td::Result> res) { - if (res.is_error()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, - res.move_as_error_prefix("cannot load block "s + blkid.to_str() + " : ")); - } else { - td::actor::send_closure_later(Self, &LiteQuery::got_block_data, blkid, res.move_as_ok()); - } - }); - }); + td::actor::send_closure(manager_, &ValidatorManager::get_block_data_for_litequery, blkid, + [Self = actor_id(this), blkid](td::Result> res) { + if (res.is_error()) { + td::actor::send_closure( + Self, &LiteQuery::abort_query, + res.move_as_error_prefix("cannot load block "s + blkid.to_str() + " : ")); + } else { + td::actor::send_closure_later(Self, &LiteQuery::got_block_data, blkid, res.move_as_ok()); + } + }); return true; } @@ -813,7 +856,7 @@ void LiteQuery::perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, St fatal_error("more than 64k parameter bytes passed"); return; } - if (mode & ~0x1f) { + if (mode & ~0x3f) { fatal_error("unsupported mode in runSmcMethod"); return; } @@ -828,7 +871,7 @@ void LiteQuery::perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, St vm::FakeVmStateLimits fstate(1000); // limit recursive (de)serialization calls vm::VmStateInterface::Guard guard(&fstate); auto cs = vm::load_cell_slice(res.move_as_ok()); - if (!(vm::Stack::deserialize_to(cs, stack_, 0) && cs.empty_ext())) { + if (!(vm::Stack::deserialize_to(cs, stack_, 2 /* no continuations */) && cs.empty_ext())) { fatal_error("parameter list boc cannot be deserialized as a VmStack"); return; } @@ -913,6 +956,100 @@ void LiteQuery::continue_getLibraries(Ref mc_s finish_query(std::move(b)); } +void LiteQuery::perform_getLibrariesWithProof(BlockIdExt blkid, int mode, std::vector library_list) { + LOG(INFO) << "started a getLibrariesWithProof() liteserver query"; + if (library_list.size() > 16) { + LOG(INFO) << "too many libraries requested, returning only first 16"; + library_list.resize(16); + } + sort( library_list.begin(), library_list.end() ); + library_list.erase( unique( library_list.begin(), library_list.end() ), library_list.end() ); + + set_continuation([this, library_list, mode]() -> void { continue_getLibrariesWithProof(library_list, mode); }); + request_mc_block_data_state(blkid); +} + +void LiteQuery::continue_getLibrariesWithProof(std::vector library_list, int mode) { + LOG(INFO) << "obtained masterchain block = " << base_blk_id_.to_str(); + CHECK(mc_state_.not_null()); + + Ref state_proof, data_proof; + if (!make_mc_state_root_proof(state_proof)) { + return; + } + + vm::MerkleProofBuilder pb{mc_state_->root_cell()}; + block::gen::ShardStateUnsplit::Record state; + if (!tlb::unpack_cell(pb.root(), state)) { + fatal_error("cannot unpack header of shardchain state "s + base_blk_id_.to_str()); + } + auto libraries_dict = vm::Dictionary(state.r1.libraries->prefetch_ref(), 256); + + std::vector> result; + std::vector result_hashes; + for (const auto& hash : library_list) { + LOG(INFO) << "looking for library " << hash.to_hex(); + + auto csr = libraries_dict.lookup(hash.bits(), 256); + if (csr.is_null() || csr->prefetch_ulong(2) != 0 || !csr->have_refs()) { // shared_lib_descr$00 lib:^Cell + continue; + } + block::gen::LibDescr::Record libdescr; + if (!tlb::csr_unpack(csr, libdescr)) { + fatal_error("cannot unpack LibDescr record "s + hash.to_hex()); + return; + } + if (mode & 1) { + // include first 16 publishers in the proof + auto publishers_dict = vm::Dictionary{vm::DictNonEmpty(), libdescr.publishers, 256}; + auto iter = publishers_dict.begin(); + constexpr int max_publishers = 15; // set to 15 because publishers_dict.begin() counts as the first visit + for (int i = 0; i < max_publishers && iter != publishers_dict.end(); ++i, ++iter) {} + } + + result_hashes.push_back(hash); + } + + auto data_proof_boc = pb.extract_proof_boc(); + if (data_proof_boc.is_error()) { + fatal_error(data_proof_boc.move_as_error()); + return; + } + auto state_proof_boc = vm::std_boc_serialize(std::move(state_proof)); + if (state_proof_boc.is_error()) { + fatal_error(state_proof_boc.move_as_error()); + return; + } + + for (const auto& hash : result_hashes) { + auto csr = libraries_dict.lookup(hash.bits(), 256); + block::gen::LibDescr::Record libdescr; + if (!tlb::csr_unpack(csr, libdescr)) { + fatal_error("cannot unpack LibDescr record "s + hash.to_hex()); + return; + } + if (!libdescr.lib->get_hash().bits().equals(hash.bits(), 256)) { + LOG(ERROR) << "public library hash mismatch: expected " << hash.to_hex() << " , found " + << libdescr.lib->get_hash().to_hex(); + continue; + } + td::BufferSlice libdata; + if (!(mode & 2)) { + auto data = vm::std_boc_serialize(libdescr.lib); + if (data.is_error()) { + LOG(WARNING) << "library serialization failed: " << data.move_as_error().to_string(); + continue; + } + libdata = data.move_as_ok(); + } + result.push_back(ton::create_tl_object(hash, std::move(libdata))); + } + + auto b = ton::create_serialize_tl_object(ton::create_tl_lite_block_id(base_blk_id_), mode, std::move(result), + state_proof_boc.move_as_ok(), data_proof_boc.move_as_ok()); + finish_query(std::move(b)); +} + void LiteQuery::perform_getOneTransaction(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, LogicalTime lt) { LOG(INFO) << "started a getOneTransaction(" << blkid.to_str() << ", " << workchain << ", " << addr.to_hex() << "," << lt << ") liteserver query"; @@ -1042,7 +1179,8 @@ bool LiteQuery::make_state_root_proof(Ref& proof, Ref state_ vm::MerkleProofBuilder pb{std::move(block_root)}; block::gen::Block::Record blk; block::gen::BlockInfo::Record info; - if (!(tlb::unpack_cell(pb.root(), blk) && tlb::unpack_cell(blk.info, info))) { + if (!(tlb::unpack_cell(pb.root(), blk) && tlb::unpack_cell(blk.info, info) && + block::gen::BlkPrevInfo(info.after_merge).validate_ref(info.prev_ref))) { return fatal_error("cannot unpack block header"); } vm::CellSlice upd_cs{vm::NoVmSpec(), blk.state_update}; @@ -1050,9 +1188,9 @@ bool LiteQuery::make_state_root_proof(Ref& proof, Ref state_ && upd_cs.size_ext() == 0x20228)) { return fatal_error("invalid Merkle update in block"); } - auto upd_hash = upd_cs.prefetch_ref(1)->get_hash(0).bits(); - auto state_hash = state_root->get_hash().bits(); - if (upd_hash.compare(state_hash, 256)) { + auto upd_hash = upd_cs.prefetch_ref(1)->get_hash(0); + auto state_hash = state_root->get_hash(); + if (upd_hash != state_hash) { return fatal_error("cannot construct Merkle proof for given masterchain state because of hash mismatch"); } if (!pb.extract_proof_to(proof)) { @@ -1168,7 +1306,7 @@ void LiteQuery::continue_getAccountState() { void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) { LOG(INFO) << "completing getAccountState() query"; - Ref proof1; + Ref proof1, proof2; if (!make_state_root_proof(proof1)) { return; } @@ -1187,6 +1325,7 @@ void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) { return; } auto rconfig = config.move_as_ok(); + rconfig->set_block_id_ext(mc_state_->get_block_id()); acc_state_promise_.set_value(std::make_tuple( std::move(acc_csr), sstate.gen_utime, sstate.gen_lt, std::move(rconfig) )); @@ -1197,31 +1336,50 @@ void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) { if (acc_csr.not_null()) { acc_root = acc_csr->prefetch_ref(); } - auto proof = vm::std_boc_serialize_multi({std::move(proof1), pb.extract_proof()}); + if (!pb.extract_proof_to(proof2)) { + fatal_error("unknown error creating Merkle proof"); + return; + } + auto proof = vm::std_boc_serialize_multi({std::move(proof1), std::move(proof2)}); pb.clear(); if (proof.is_error()) { fatal_error(proof.move_as_error()); return; } if (mode_ & 0x10000) { - finish_runSmcMethod(std::move(shard_proof), proof.move_as_ok(), std::move(acc_root), sstate.gen_utime, - sstate.gen_lt); + if (!mc_state_.is_null()) { + finish_runSmcMethod(std::move(shard_proof), proof.move_as_ok(), std::move(acc_root), sstate.gen_utime, + sstate.gen_lt); + return; + } + shard_proof_ = std::move(shard_proof); + proof_ = proof.move_as_ok(); + set_continuation( + [&, base_blk_id = base_blk_id_, acc_root, utime = sstate.gen_utime, lt = sstate.gen_lt]() mutable -> void { + base_blk_id_ = base_blk_id; // It gets overridden by request_mc_block_data_state + finish_runSmcMethod(std::move(shard_proof_), std::move(proof_), std::move(acc_root), utime, lt); + }); + td::optional master_ref = state_->get_master_ref(); + if (!master_ref) { + fatal_error("masterchain ref block is not available"); + return; + } + request_mc_block_data_state(master_ref.value()); return; } td::BufferSlice data; if (acc_root.not_null()) { if (mode_ & 0x40000000) { vm::MerkleProofBuilder mpb{acc_root}; - // account_none$0 = Account; - // account$1 addr:MsgAddressInt storage_stat:StorageInfo storage:AccountStorage = Account; - // account_storage$_ last_trans_lt:uint64 balance:CurrencyCollection state:AccountState = AccountStorage; - // account_active$1 _:StateInit = AccountState; - auto S = mpb.root()->load_cell(); - if (S.is_error()) { - fatal_error(S.move_as_error_prefix("Failed to load account: ")); + // This does not include code, data and libs into proof, but it includes extra currencies + if (!block::gen::t_Account.validate_ref(mpb.root())) { + fatal_error("failed to validate Account"); + return; + } + if (!mpb.extract_proof_to(acc_root)) { + fatal_error("unknown error creating Merkle proof"); return; } - acc_root = mpb.extract_proof(); } auto res = vm::std_boc_serialize(std::move(acc_root)); if (res.is_error()) { @@ -1239,25 +1397,51 @@ void LiteQuery::finish_getAccountState(td::BufferSlice shard_proof) { // same as in lite-client/lite-client-common.cpp static td::Ref prepare_vm_c7(ton::UnixTime now, ton::LogicalTime lt, td::Ref my_addr, - const block::CurrencyCollection& balance) { + const block::CurrencyCollection& balance, + const block::ConfigInfo* config = nullptr, td::Ref my_code = {}, + td::RefInt256 due_payment = td::zero_refint()) { td::BitArray<256> rand_seed; td::RefInt256 rand_seed_int{true}; td::Random::secure_bytes(rand_seed.as_slice()); if (!rand_seed_int.unique_write().import_bits(rand_seed.cbits(), 256, false)) { return {}; } - auto tuple = vm::make_tuple_ref(td::make_refint(0x076ef1ea), // [ magic:0x076ef1ea - td::make_refint(0), // actions:Integer - td::make_refint(0), // msgs_sent:Integer - td::make_refint(now), // unixtime:Integer - td::make_refint(lt), // block_lt:Integer - td::make_refint(lt), // trans_lt:Integer - std::move(rand_seed_int), // rand_seed:Integer - balance.as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)] - my_addr, // myself:MsgAddressInt - vm::StackEntry()); // global_config:(Maybe Cell) ] = SmartContractInfo; - LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple).to_string(); - return vm::make_tuple_ref(std::move(tuple)); + std::vector tuple = { + td::make_refint(0x076ef1ea), // [ magic:0x076ef1ea + td::make_refint(0), // actions:Integer + td::make_refint(0), // msgs_sent:Integer + td::make_refint(now), // unixtime:Integer + td::make_refint(lt), // block_lt:Integer + td::make_refint(lt), // trans_lt:Integer + std::move(rand_seed_int), // rand_seed:Integer + balance.as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)] + my_addr, // myself:MsgAddressInt + config ? config->get_root_cell() : vm::StackEntry() // global_config:(Maybe Cell) ] = SmartContractInfo; + }; + if (config && config->get_global_version() >= 4) { + tuple.push_back(vm::StackEntry::maybe(my_code)); // code:Cell + tuple.push_back(block::CurrencyCollection::zero().as_vm_tuple()); // in_msg_value:[Integer (Maybe Cell)] + tuple.push_back(td::zero_refint()); // storage_fees:Integer + + // [ wc:Integer shard:Integer seqno:Integer root_hash:Integer file_hash:Integer] = BlockId; + // [ last_mc_blocks:[BlockId...] + // prev_key_block:BlockId ] : PrevBlocksInfo + auto info = config->get_prev_blocks_info(); + tuple.push_back(info.is_ok() ? info.move_as_ok() : vm::StackEntry()); + } + if (config && config->get_global_version() >= 6) { + tuple.push_back(vm::StackEntry::maybe(config->get_unpacked_config_tuple(now))); // unpacked_config_tuple:[...] + tuple.push_back(due_payment); // due_payment:Integer + // precomiled_gas_usage:(Maybe Integer) + td::optional precompiled; + if (my_code.not_null()) { + precompiled = config->get_precompiled_contracts_config().get_contract(my_code->get_hash().bits()); + } + tuple.push_back(precompiled ? td::make_refint(precompiled.value().gas_usage) : vm::StackEntry()); + } + auto tuple_ref = td::make_cnt_ref>(std::move(tuple)); + LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple_ref).to_string(); + return vm::make_tuple_ref(std::move(tuple_ref)); } void LiteQuery::finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice state_proof, Ref acc_root, @@ -1276,28 +1460,70 @@ void LiteQuery::finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice } vm::MerkleProofBuilder pb{std::move(acc_root)}; block::gen::Account::Record_account acc; + block::gen::StorageInfo::Record storage_info; block::gen::AccountStorage::Record store; block::CurrencyCollection balance; block::gen::StateInit::Record state_init; if (!(tlb::unpack_cell(pb.root(), acc) && tlb::csr_unpack(std::move(acc.storage), store) && balance.validate_unpack(store.balance) && store.state->prefetch_ulong(1) == 1 && - store.state.write().advance(1) && tlb::csr_unpack(std::move(store.state), state_init))) { + store.state.write().advance(1) && tlb::csr_unpack(std::move(store.state), state_init) && + tlb::csr_unpack(std::move(acc.storage_stat), storage_info))) { LOG(INFO) << "error unpacking account state, or account is frozen or uninitialized"; + td::Result proof_boc; + if (mode & 2) { + proof_boc = pb.extract_proof_boc(); + if (proof_boc.is_error()) { + fatal_error(proof_boc.move_as_error()); + return; + } + } else { + proof_boc = td::BufferSlice(); + } auto b = ton::create_serialize_tl_object( mode, ton::create_tl_lite_block_id(base_blk_id_), ton::create_tl_lite_block_id(blk_id_), std::move(shard_proof), - std::move(state_proof), mode & 2 ? pb.extract_proof_boc().move_as_ok() : td::BufferSlice(), td::BufferSlice(), - td::BufferSlice(), -0x100, td::BufferSlice()); + std::move(state_proof), proof_boc.move_as_ok(), td::BufferSlice(), td::BufferSlice(), -0x100, + td::BufferSlice()); finish_query(std::move(b)); return; } auto code = state_init.code->prefetch_ref(); auto data = state_init.data->prefetch_ref(); + auto acc_libs = state_init.library->prefetch_ref(); long long gas_limit = client_method_gas_limit; + td::RefInt256 due_payment; + if (storage_info.due_payment.write().fetch_long(1)) { + due_payment = block::tlb::t_Grams.as_integer(storage_info.due_payment); + } else { + due_payment = td::zero_refint(); + } LOG(DEBUG) << "creating VM with gas limit " << gas_limit; // **** INIT VM **** + auto r_config = block::ConfigInfo::extract_config( + mc_state_->root_cell(), + block::ConfigInfo::needLibraries | block::ConfigInfo::needCapabilities | block::ConfigInfo::needPrevBlocks); + if (r_config.is_error()) { + fatal_error(r_config.move_as_error()); + return; + } + auto config = r_config.move_as_ok(); + std::vector> libraries; + if (config->get_libraries_root().not_null()) { + libraries.push_back(config->get_libraries_root()); + } + if (acc_libs.not_null()) { + libraries.push_back(acc_libs); + } vm::GasLimits gas{gas_limit, gas_limit}; - vm::VmState vm{std::move(code), std::move(stack_), gas, 1, std::move(data), vm::VmLog::Null()}; - auto c7 = prepare_vm_c7(gen_utime, gen_lt, td::make_ref(acc.addr->clone()), balance); + vm::VmState vm{code, + config->get_global_version(), + std::move(stack_), + gas, + 1, + std::move(data), + vm::VmLog::Null(), + std::move(libraries)}; + auto c7 = prepare_vm_c7(gen_utime, gen_lt, td::make_ref(acc.addr->clone()), balance, config.get(), + std::move(code), due_payment); vm.set_c7(c7); // tuple with SmartContractInfo // vm.incr_stack_trace(1); // enable stack dump after each step LOG(INFO) << "starting VM to run GET-method of smart contract " << acc_workchain_ << ":" << acc_addr_.to_hex(); @@ -1313,6 +1539,9 @@ void LiteQuery::finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice td::BufferSlice c7_info, result; if (mode & 8) { // serialize c7 + if (!(mode & 32)) { + c7 = prepare_vm_c7(gen_utime, gen_lt, td::make_ref(acc.addr->clone()), balance); + } vm::CellBuilder cb; if (!(vm::StackEntry{std::move(c7)}.serialize(cb) && cb.finalize_to(cell))) { fatal_error("cannot serialize c7"); @@ -1340,10 +1569,20 @@ void LiteQuery::finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice } result = res.move_as_ok(); } + td::Result proof_boc; + if (mode & 2) { + proof_boc = pb.extract_proof_boc(); + if (proof_boc.is_error()) { + fatal_error(proof_boc.move_as_error()); + return; + } + } else { + proof_boc = td::BufferSlice(); + } auto b = ton::create_serialize_tl_object( mode, ton::create_tl_lite_block_id(base_blk_id_), ton::create_tl_lite_block_id(blk_id_), std::move(shard_proof), - std::move(state_proof), mode & 2 ? pb.extract_proof_boc().move_as_ok() : td::BufferSlice(), std::move(c7_info), - td::BufferSlice(), exit_code, std::move(result)); + std::move(state_proof), proof_boc.move_as_ok(), std::move(c7_info), td::BufferSlice(), exit_code, + std::move(result)); finish_query(std::move(b)); } @@ -1464,17 +1703,12 @@ void LiteQuery::continue_getTransactions(unsigned remaining, bool exact) { LOG(DEBUG) << "sending get_block_by_lt_from_db() query to manager for " << acc_workchain_ << ":" << acc_addr_.to_hex() << " " << trans_lt_; td::actor::send_closure_later( - manager_, &ValidatorManager::get_block_by_lt_from_db, ton::extract_addr_prefix(acc_workchain_, acc_addr_), + manager_, &ValidatorManager::get_block_by_lt_for_litequery, ton::extract_addr_prefix(acc_workchain_, acc_addr_), trans_lt_, [Self = actor_id(this), remaining, manager = manager_](td::Result res) { if (res.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_getTransactions, res.move_as_error(), ton::BlockIdExt{}); } else { auto handle = res.move_as_ok(); - if (!handle->is_applied()) { - td::actor::send_closure(Self, &LiteQuery::abort_getTransactions, td::Status::Error("block is not applied"), - ton::BlockIdExt{}); - return; - } LOG(DEBUG) << "requesting data for block " << handle->id().to_str(); td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle, [Self, blkid = handle->id(), remaining](td::Result> res) { @@ -1622,7 +1856,7 @@ void LiteQuery::perform_getConfigParams(BlockIdExt blkid, int mode, std::vector< request_mc_block_data_state(blkid); } else { // get configuration from previous key block - load_prevKeyBlock(blkid, [this, blkid, mode, param_list = std::move(param_list)]( + load_prevKeyBlock(blkid, [this, mode, param_list = std::move(param_list)]( td::Result>> res) mutable { if (res.is_error()) { this->abort_query(res.move_as_error()); @@ -1655,13 +1889,26 @@ void LiteQuery::continue_getConfigParams(int mode, std::vector param_list) } } - auto res = keyblk ? block::Config::extract_from_key_block(mpb.root(), mode) - : block::Config::extract_from_state(mpb.root(), mode); - if (res.is_error()) { - fatal_error(res.move_as_error()); - return; + std::unique_ptr cfg; + if (keyblk || !(mode & block::ConfigInfo::needPrevBlocks)) { + auto res = keyblk ? block::Config::extract_from_key_block(mpb.root(), mode) + : block::Config::extract_from_state(mpb.root(), mode); + if (res.is_error()) { + fatal_error(res.move_as_error()); + return; + } + cfg = res.move_as_ok(); + } else { + if (mode & block::ConfigInfo::needPrevBlocks) { + mode |= block::ConfigInfo::needCapabilities; + } + auto res = block::ConfigInfo::extract_config(mpb.root(), mode); + if (res.is_error()) { + fatal_error(res.move_as_error()); + return; + } + cfg = res.move_as_ok(); } - auto cfg = res.move_as_ok(); if (!cfg) { fatal_error("cannot extract configuration from last mc state"); return; @@ -1674,6 +1921,9 @@ void LiteQuery::continue_getConfigParams(int mode, std::vector param_list) visit(cfg->get_config_param(i)); } } + if (!keyblk && mode & block::ConfigInfo::needPrevBlocks) { + ((block::ConfigInfo*)cfg.get())->get_prev_blocks_info(); + } } catch (vm::VmError& err) { fatal_error("error while traversing required configuration parameters: "s + err.get_msg()); return; @@ -1697,7 +1947,7 @@ void LiteQuery::continue_getConfigParams(int mode, std::vector param_list) void LiteQuery::perform_getAllShardsInfo(BlockIdExt blkid) { LOG(INFO) << "started a getAllShardsInfo(" << blkid.to_str() << ") liteserver query"; set_continuation([&]() -> void { continue_getAllShardsInfo(); }); - request_mc_block_data_state(blkid); + request_mc_block_data(blkid); } void LiteQuery::continue_getShardInfo(ShardIdFull shard, bool exact) { @@ -1744,30 +1994,30 @@ void LiteQuery::continue_getShardInfo(ShardIdFull shard, bool exact) { void LiteQuery::continue_getAllShardsInfo() { LOG(INFO) << "completing getAllShardsInfo() query"; - Ref proof1, proof2; - if (!make_mc_state_root_proof(proof1)) { + vm::MerkleProofBuilder mpb{mc_block_->root_cell()}; + block::gen::Block::Record blk; + block::gen::BlockExtra::Record extra; + block::gen::McBlockExtra::Record mc_extra; + if (!tlb::unpack_cell(mpb.root(), blk) || !tlb::unpack_cell(blk.extra, extra) || !extra.custom->have_refs() || + !tlb::unpack_cell(extra.custom->prefetch_ref(), mc_extra)) { + fatal_error("cannot unpack header of block "s + mc_block_->block_id().to_str()); return; } - vm::MerkleProofBuilder mpb{mc_state_->root_cell()}; - auto shards_dict = block::ShardConfig::extract_shard_hashes_dict(mpb.root()); - if (!shards_dict) { - fatal_error("cannot extract ShardHashes from last mc state"); - return; - } - if (!mpb.extract_proof_to(proof2)) { + vm::Dictionary shards_dict(std::move(mc_extra.shard_hashes), 32); + Ref proof; + if (!mpb.extract_proof_to(proof)) { fatal_error("cannot construct Merkle proof for all shards dictionary"); return; } - shards_dict = block::ShardConfig::extract_shard_hashes_dict(mc_state_->root_cell()); - vm::CellBuilder cb; - Ref cell; - if (!(std::move(shards_dict)->append_dict_to_bool(cb) && cb.finalize_to(cell))) { - fatal_error("cannot store ShardHashes from last mc state into a new cell"); + auto proof_boc = vm::std_boc_serialize(std::move(proof)); + if (proof_boc.is_error()) { + fatal_error(proof_boc.move_as_error()); return; } - auto proof = vm::std_boc_serialize_multi({std::move(proof1), std::move(proof2)}); - if (proof.is_error()) { - fatal_error(proof.move_as_error()); + vm::CellBuilder cb; + Ref cell; + if (!(shards_dict.append_dict_to_bool(cb) && cb.finalize_to(cell))) { + fatal_error("cannot store ShardHashes from last mc block into a new cell"); return; } auto data = vm::std_boc_serialize(std::move(cell)); @@ -1777,10 +2027,309 @@ void LiteQuery::continue_getAllShardsInfo() { } LOG(INFO) << "getAllShardInfo() query completed"; auto b = ton::create_serialize_tl_object( - ton::create_tl_lite_block_id(base_blk_id_), proof.move_as_ok(), data.move_as_ok()); + ton::create_tl_lite_block_id(base_blk_id_), proof_boc.move_as_ok(), data.move_as_ok()); finish_query(std::move(b)); } +void LiteQuery::perform_lookupBlockWithProof(BlockId blkid, BlockIdExt mc_blkid, int mode, LogicalTime lt, UnixTime utime) { + if (!((1 << (mode & 7)) & 0x16)) { + fatal_error("exactly one of mode.0, mode.1 and mode.2 bits must be set"); + return; + } + if (!mc_blkid.is_masterchain_ext()) { + fatal_error("masterchain block id must be specified"); + return; + } + if (!(mode & 1)) { + blkid.seqno = 0; + } + if (!(mode & 2)) { + lt = 0; + } + if (!(mode & 4)) { + utime = 0; + } + mode_ = mode; + base_blk_id_ = mc_blkid; + LOG(INFO) << "started a lookupBlockWithProof(" << blkid.to_str() << ", " << mc_blkid.to_str() << ", " << mode << ", " + << lt << ", " << utime << ") liteserver query"; + + ton::AccountIdPrefixFull pfx{blkid.workchain, blkid.shard}; + auto P = td::PromiseCreator::lambda( + [Self = actor_id(this), mc_blkid, manager = manager_, pfx](td::Result res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + return; + } + auto handle = res.move_as_ok(); + if (!handle->inited_masterchain_ref_block()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, td::Status::Error("block doesn't have masterchain ref")); + return; + } + if (handle->masterchain_ref_block() > mc_blkid.seqno()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, td::Status::Error("specified mc block is older than block's masterchain ref")); + return; + } + LOG(DEBUG) << "requesting data for block " << handle->id().to_str(); + td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle, + [Self, mc_ref_blkid = handle->masterchain_ref_block(), pfx](td::Result> res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::continue_lookupBlockWithProof_getHeaderProof, res.move_as_ok(), pfx, mc_ref_blkid); + } + }); + }); + + if (mode & 2) { + td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_lt_for_litequery, pfx, lt, std::move(P)); + } else if (mode & 4) { + td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_unix_time_for_litequery, pfx, utime, + std::move(P)); + } else { + td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_seqno_for_litequery, pfx, blkid.seqno, + std::move(P)); + } +} + +void LiteQuery::continue_lookupBlockWithProof_getHeaderProof(Ref block, AccountIdPrefixFull req_prefix, BlockSeqno masterchain_ref_seqno) { + blk_id_ = block->block_id(); + LOG(INFO) << "obtained data for getBlockHeader(" << blk_id_.to_str() << ", " << mode_ << ")"; + CHECK(block.not_null()); + auto block_root = block->root_cell(); + if (block_root.is_null()) { + fatal_error("block has no valid root cell"); + return; + } + + vm::MerkleProofBuilder mpb{block_root}; + std::vector prev; + BlockIdExt mc_blkid; + bool after_split; + td::Status S = block::unpack_block_prev_blk_try(mpb.root(), blk_id_, prev, mc_blkid, after_split); + if (S.is_error()) { + fatal_error(std::move(S)); + return; + } + auto proof_data = mpb.extract_proof_boc(); + if (proof_data.is_error()) { + fatal_error(proof_data.move_as_error()); + return; + } + lookup_header_proof_ = proof_data.move_as_ok(); + + bool include_prev = mode_ & 6; + if (include_prev) { + BlockIdExt prev_blkid; + for (auto& p : prev) { + if (ton::shard_contains(p.shard_full(), req_prefix)) { + prev_blkid = p; + } + } + CHECK(prev_blkid.is_valid()); + get_block_handle_checked(prev_blkid, [Self = actor_id(this), masterchain_ref_seqno, manager = manager_](td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure(manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(), + [Self, masterchain_ref_seqno](td::Result> res) mutable { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + return; + } + td::actor::send_closure(Self, &LiteQuery::continue_lookupBlockWithProof_gotPrevBlockData, res.move_as_ok(), masterchain_ref_seqno); + }); + }); + } else { + continue_lookupBlockWithProof_gotPrevBlockData(Ref(), masterchain_ref_seqno); + } +} + +void LiteQuery::continue_lookupBlockWithProof_gotPrevBlockData(Ref prev_block, BlockSeqno masterchain_ref_seqno) { + if (prev_block.not_null()) { + CHECK(prev_block.not_null()); + if (prev_block->root_cell().is_null()) { + fatal_error("block has no valid root cell"); + return; + } + vm::MerkleProofBuilder mpb{prev_block->root_cell()}; + block::gen::Block::Record blk; + block::gen::BlockInfo::Record info; + if (!(tlb::unpack_cell(mpb.root(), blk) && tlb::unpack_cell(blk.info, info))) { + fatal_error(td::Status::Error("cannot unpack prev block header")); + return; + } + auto proof_data = mpb.extract_proof_boc(); + if (proof_data.is_error()) { + fatal_error(proof_data.move_as_error()); + return; + } + lookup_prev_header_proof_ = proof_data.move_as_ok(); + } + + if (!blk_id_.is_masterchain()) { + ton::AccountIdPrefixFull pfx{ton::masterchainId, ton::shardIdAll}; + td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_seqno_from_db, pfx, masterchain_ref_seqno, + [manager = manager_, Self = actor_id(this)](td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure(manager, &ValidatorManager::get_block_data_from_db, R.move_as_ok(), + [Self](td::Result> res) mutable { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + return; + } + td::actor::send_closure(Self, &LiteQuery::continue_lookupBlockWithProof_buildProofLinks, res.move_as_ok(), std::vector>>()); + }); + }); + } else { + base_blk_id_alt_ = blk_id_; + td::actor::send_closure(actor_id(this), &LiteQuery::continue_lookupBlockWithProof_getClientMcBlockDataState, std::vector>>()); + } +} + +void LiteQuery::continue_lookupBlockWithProof_buildProofLinks(td::Ref cur_block, + std::vector>> result) { + BlockIdExt cur_id = cur_block->block_id(); + BlockIdExt prev_id; + vm::MerkleProofBuilder mpb{cur_block->root_cell()}; + if (cur_id.is_masterchain()) { + base_blk_id_alt_ = cur_id; + block::gen::Block::Record blk; + block::gen::BlockExtra::Record extra; + block::gen::McBlockExtra::Record mc_extra; + if (!tlb::unpack_cell(mpb.root(), blk) || !tlb::unpack_cell(blk.extra, extra) || !extra.custom->have_refs() || + !tlb::unpack_cell(extra.custom->prefetch_ref(), mc_extra)) { + fatal_error("cannot unpack header of block "s + cur_id.to_str()); + return; + } + block::ShardConfig shards(mc_extra.shard_hashes->prefetch_ref()); + ShardIdFull shard_id = blk_id_.shard_full(); + shard_id.shard = (shard_id.shard & ~(1 << (63 - shard_id.pfx_len()))) | 1; + Ref shard_hash = shards.get_shard_hash(shard_id, false); + if (shard_hash.is_null()) { + fatal_error("shard not found"); + return; + } + prev_id = shard_hash->top_block_id(); + } else { + std::vector prev; + BlockIdExt mc_blkid; + bool after_split; + td::Status S = block::unpack_block_prev_blk_try(mpb.root(), cur_id, prev, mc_blkid, after_split); + if (S.is_error()) { + fatal_error(std::move(S)); + return; + } + bool found = false; + for (const BlockIdExt& id : prev) { + if (shard_intersects(id.shard_full(), blk_id_.shard_full())) { + found = true; + prev_id = id; + break; + } + } + if (!found) { + fatal_error("failed to find block chain"); + return; + } + } + auto proof = mpb.extract_proof(); + if (proof.is_error()) { + fatal_error(proof.move_as_error_prefix("cannot serialize Merkle proof : ")); + return; + } + result.emplace_back(prev_id, proof.move_as_ok()); + + if (prev_id == blk_id_) { + CHECK(base_blk_id_alt_.is_masterchain()); + if (base_blk_id_alt_ != base_blk_id_) { + continue_lookupBlockWithProof_getClientMcBlockDataState(std::move(result)); + } else { + continue_lookupBlockWithProof_getMcBlockPrev(std::move(result)); + } + return; + } + if (result.size() == 8) { + // Chains of shardblocks between masterchain blocks can't be longer than 8 (see collator.cpp:991) + fatal_error("proof chain is too long"); + return; + } + + td::actor::send_closure_later( + manager_, &ValidatorManager::get_block_data_from_db_short, prev_id, + [Self = actor_id(this), result = std::move(result)](td::Result> R) mutable { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::continue_lookupBlockWithProof_buildProofLinks, R.move_as_ok(), + std::move(result)); + } + }); +} + +void LiteQuery::continue_lookupBlockWithProof_getClientMcBlockDataState(std::vector>> links) { + set_continuation([this, links = std::move(links)]() -> void { + continue_lookupBlockWithProof_getMcBlockPrev(std::move(links)); + }); + request_mc_block_data_state(base_blk_id_); +} + +void LiteQuery::continue_lookupBlockWithProof_getMcBlockPrev(std::vector>> links) { + td::BufferSlice mc_state_proof_buf, client_mc_blk_proof_buf; + + if (base_blk_id_alt_ != base_blk_id_) { + vm::MerkleProofBuilder mpb{mc_state_->root_cell()}; + auto prev_blocks_dict = block::get_prev_blocks_dict(mpb.root()); + if (!prev_blocks_dict) { + fatal_error(td::Status::Error("cannot extract prev_blocks from mc state")); + return; + } + if (!block::check_old_mc_block_id(*prev_blocks_dict, base_blk_id_alt_)) { + fatal_error(td::Status::Error("client mc blkid is not in prev_blocks")); + return; + } + auto client_mc_blk_proof = mpb.extract_proof_boc(); + if (client_mc_blk_proof.is_error()) { + fatal_error(client_mc_blk_proof.move_as_error()); + return; + } + client_mc_blk_proof_buf = client_mc_blk_proof.move_as_ok(); + + Ref mc_state_proof; + if (!make_mc_state_root_proof(mc_state_proof)) { + fatal_error(td::Status::Error("cannot create Merkle proof for mc state")); + return; + } + auto mc_state_proof_boc = vm::std_boc_serialize(std::move(mc_state_proof)); + if (mc_state_proof_boc.is_error()) { + fatal_error(mc_state_proof_boc.move_as_error()); + return; + } + mc_state_proof_buf = mc_state_proof_boc.move_as_ok(); + } + + std::vector> links_res; + for (auto& p : links) { + auto prev_block_proof = vm::std_boc_serialize(std::move(p.second)); + if (prev_block_proof.is_error()) { + fatal_error(prev_block_proof.move_as_error()); + return; + } + links_res.push_back( + create_tl_object(create_tl_lite_block_id(p.first), prev_block_proof.move_as_ok())); + } + + auto b = ton::create_serialize_tl_object(ton::create_tl_lite_block_id(blk_id_), + mode_, ton::create_tl_lite_block_id(base_blk_id_alt_), std::move(mc_state_proof_buf), std::move(client_mc_blk_proof_buf), + std::move(links_res), std::move(lookup_header_proof_), std::move(lookup_prev_header_proof_)); + finish_query(std::move(b)); +} + + void LiteQuery::perform_lookupBlock(BlockId blkid, int mode, LogicalTime lt, UnixTime utime) { if (!((1 << (mode & 7)) & 0x16)) { fatal_error("exactly one of mode.0, mode.1 and mode.2 bits must be set"); @@ -1800,10 +2349,6 @@ void LiteQuery::perform_lookupBlock(BlockId blkid, int mode, LogicalTime lt, Uni td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); } else { auto handle = res.move_as_ok(); - if (!handle->is_applied()) { - td::actor::send_closure(Self, &LiteQuery::abort_query, td::Status::Error("block is not applied")); - return; - } LOG(DEBUG) << "requesting data for block " << handle->id().to_str(); td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, handle, [Self, blkid = handle->id(), mode](td::Result> res) { @@ -1819,13 +2364,14 @@ void LiteQuery::perform_lookupBlock(BlockId blkid, int mode, LogicalTime lt, Uni ton::AccountIdPrefixFull pfx{blkid.workchain, blkid.shard}; if (mode & 2) { - td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_lt_from_db, pfx, lt, std::move(P)); + td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_lt_for_litequery, pfx, lt, + std::move(P)); } else if (mode & 4) { - td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_unix_time_from_db, pfx, utime, + td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_unix_time_for_litequery, pfx, utime, std::move(P)); } else { - td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_seqno_from_db, pfx, blkid.seqno, - std::move(P)); + td::actor::send_closure_later(manager_, &ValidatorManager::get_block_by_seqno_for_litequery, pfx, + blkid.seqno, std::move(P)); } } @@ -1839,6 +2385,45 @@ void LiteQuery::perform_listBlockTransactions(BlockIdExt blkid, int mode, int co request_block_data(blkid); } +static td::Result> get_in_msg_metadata( + const Ref& in_msg_descr_root, const Ref& trans_root) { + vm::AugmentedDictionary in_msg_descr{vm::load_cell_slice_ref(in_msg_descr_root), 256, block::tlb::aug_InMsgDescr}; + block::gen::Transaction::Record transaction; + if (!block::tlb::unpack_cell(trans_root, transaction)) { + return td::Status::Error("invalid Transaction in block"); + } + Ref msg = transaction.r1.in_msg->prefetch_ref(); + if (msg.is_null()) { + return nullptr; + } + td::Bits256 in_msg_hash = msg->get_hash().bits(); + Ref in_msg = in_msg_descr.lookup(in_msg_hash); + if (in_msg.is_null()) { + return td::Status::Error(PSTRING() << "no InMsg in InMsgDescr for message with hash " << in_msg_hash.to_hex()); + } + int tag = block::gen::t_InMsg.get_tag(*in_msg); + if (tag != block::gen::InMsg::msg_import_imm && tag != block::gen::InMsg::msg_import_fin && + tag != block::gen::InMsg::msg_import_deferred_fin) { + return nullptr; + } + Ref msg_env = in_msg->prefetch_ref(); + if (msg_env.is_null()) { + return td::Status::Error(PSTRING() << "no MsgEnvelope in InMsg for message with hash " << in_msg_hash.to_hex()); + } + block::tlb::MsgEnvelope::Record_std env; + if (!block::tlb::unpack_cell(std::move(msg_env), env)) { + return td::Status::Error(PSTRING() << "failed to unpack MsgEnvelope for message with hash " << in_msg_hash.to_hex()); + } + if (!env.metadata) { + return nullptr; + } + block::MsgMetadata& metadata = env.metadata.value(); + return create_tl_object( + 0, metadata.depth, + create_tl_object(metadata.initiator_wc, metadata.initiator_addr), + metadata.initiator_lt); +} + void LiteQuery::finish_listBlockTransactions(int mode, int req_count) { LOG(INFO) << "completing a listBlockTransactions(" << base_blk_id_.to_str() << ", " << mode << ", " << req_count << ", " << acc_addr_.to_hex() << ", " << trans_lt_ << ") liteserver query"; @@ -1858,6 +2443,8 @@ void LiteQuery::finish_listBlockTransactions(int mode, int req_count) { acc_addr_.set_ones(); trans_lt_ = ~0ULL; } + bool with_metadata = mode & 256; + mode &= ~256; std::vector> result; bool eof = false; ton::LogicalTime reverse = (mode & 64) ? ~0ULL : 0; @@ -1911,8 +2498,18 @@ void LiteQuery::finish_listBlockTransactions(int mode, int req_count) { trans_lt_ = reverse; break; } - result.push_back(create_tl_object(mode, cur_addr, cur_trans.to_long(), - tvalue->get_hash().bits())); + tl_object_ptr metadata; + if (with_metadata) { + auto r_metadata = get_in_msg_metadata(extra.in_msg_descr, tvalue); + if (r_metadata.is_error()) { + fatal_error(r_metadata.move_as_error()); + return; + } + metadata = r_metadata.move_as_ok(); + } + result.push_back(create_tl_object( + mode | (metadata ? 256 : 0), cur_addr, cur_trans.to_long(), tvalue->get_hash().bits(), + std::move(metadata))); ++count; } } @@ -1937,6 +2534,159 @@ void LiteQuery::finish_listBlockTransactions(int mode, int req_count) { finish_query(std::move(b)); } +void LiteQuery::perform_listBlockTransactionsExt(BlockIdExt blkid, int mode, int count, Bits256 account, LogicalTime lt) { + LOG(INFO) << "started a listBlockTransactionsExt(" << blkid.to_str() << ", " << mode << ", " << count << ", " + << account.to_hex() << ", " << lt << ") liteserver query"; + base_blk_id_ = blkid; + acc_addr_ = account; + trans_lt_ = lt; + set_continuation([this, mode, count]() -> void { finish_listBlockTransactionsExt(mode, count); }); + request_block_data(blkid); +} + +static td::Status process_all_in_msg_metadata(const Ref& in_msg_descr_root, + const std::vector>& trans_roots) { + vm::AugmentedDictionary in_msg_descr{vm::load_cell_slice_ref(in_msg_descr_root), 256, block::tlb::aug_InMsgDescr}; + for (const Ref& trans_root : trans_roots) { + block::gen::Transaction::Record transaction; + if (!block::tlb::unpack_cell(trans_root, transaction)) { + return td::Status::Error("invalid Transaction in block"); + } + Ref msg = transaction.r1.in_msg->prefetch_ref(); + if (msg.is_null()) { + continue; + } + td::Bits256 in_msg_hash = msg->get_hash().bits(); + Ref in_msg = in_msg_descr.lookup(in_msg_hash); + if (in_msg.is_null()) { + return td::Status::Error(PSTRING() << "no InMsg in InMsgDescr for message with hash " << in_msg_hash.to_hex()); + } + int tag = block::gen::t_InMsg.get_tag(*in_msg); + if (tag == block::gen::InMsg::msg_import_imm || tag == block::gen::InMsg::msg_import_fin || + tag == block::gen::InMsg::msg_import_deferred_fin) { + Ref msg_env = in_msg->prefetch_ref(); + if (msg_env.is_null()) { + return td::Status::Error(PSTRING() << "no MsgEnvelope in InMsg for message with hash " << in_msg_hash.to_hex()); + } + vm::load_cell_slice(msg_env); + } + } + return td::Status::OK(); +} + +void LiteQuery::finish_listBlockTransactionsExt(int mode, int req_count) { + LOG(INFO) << "completing a listBlockTransactionsExt(" << base_blk_id_.to_str() << ", " << mode << ", " << req_count + << ", " << acc_addr_.to_hex() << ", " << trans_lt_ << ") liteserver query"; + constexpr int max_answer_transactions = 256; + CHECK(block_.not_null()); + auto block_root = block_->root_cell(); + CHECK(block_root.not_null()); + RootHash rhash{block_root->get_hash().bits()}; + CHECK(rhash == base_blk_id_.root_hash); + vm::MerkleProofBuilder pb; + auto virt_root = block_root; + if (mode & 256) { + // with msg metadata in proof + mode |= 32; + } + if (mode & 32) { + // proof requested + virt_root = pb.init(std::move(virt_root)); + } + if ((mode & 192) == 64) { // reverse order, no starting point + acc_addr_.set_ones(); + trans_lt_ = ~0ULL; + } + std::vector> trans_roots; + bool eof = false; + ton::LogicalTime reverse = (mode & 64) ? ~0ULL : 0; + try { + block::gen::Block::Record blk; + block::gen::BlockExtra::Record extra; + if (!(tlb::unpack_cell(virt_root, blk) && tlb::unpack_cell(std::move(blk.extra), extra))) { + fatal_error("cannot find account transaction data in block "s + base_blk_id_.to_str()); + return; + } + vm::AugmentedDictionary acc_dict{vm::load_cell_slice_ref(extra.account_blocks), 256, + block::tlb::aug_ShardAccountBlocks}; + int count = 0; + bool allow_same = true; + td::Bits256 cur_addr = acc_addr_; + while (!eof && count < req_count && count < max_answer_transactions) { + Ref value; + try { + value = acc_dict.extract_value( + acc_dict.vm::DictionaryFixed::lookup_nearest_key(cur_addr.bits(), 256, !reverse, allow_same)); + } catch (vm::VmError err) { + fatal_error("error while traversing account block dictionary: "s + err.get_msg()); + return; + } + if (value.is_null()) { + eof = true; + break; + } + allow_same = false; + if (cur_addr != acc_addr_) { + trans_lt_ = reverse; + } + block::gen::AccountBlock::Record acc_blk; + if (!(tlb::csr_unpack(std::move(value), acc_blk) && acc_blk.account_addr == cur_addr)) { + fatal_error("invalid AccountBlock for account "s + cur_addr.to_hex()); + return; + } + vm::AugmentedDictionary trans_dict{vm::DictNonEmpty(), std::move(acc_blk.transactions), 64, + block::tlb::aug_AccountTransactions}; + td::BitArray<64> cur_trans{(long long)trans_lt_}; + while (count < req_count && count < max_answer_transactions) { + Ref tvalue; + try { + tvalue = trans_dict.extract_value_ref( + trans_dict.vm::DictionaryFixed::lookup_nearest_key(cur_trans.bits(), 64, !reverse)); + } catch (vm::VmError err) { + fatal_error("error while traversing transaction dictionary of an AccountBlock: "s + err.get_msg()); + return; + } + if (tvalue.is_null()) { + trans_lt_ = reverse; + break; + } + trans_roots.push_back(std::move(tvalue)); + ++count; + } + } + if (mode & 256) { + td::Status S = process_all_in_msg_metadata(extra.in_msg_descr, trans_roots); + if (S.is_error()) { + fatal_error(S.move_as_error()); + return; + } + } + } catch (vm::VmError err) { + fatal_error("error while parsing AccountBlocks of block "s + base_blk_id_.to_str() + " : " + err.get_msg()); + return; + } + td::BufferSlice proof_data; + if (mode & 32) { + // create proof + auto proof_boc = pb.extract_proof_boc(); + if (proof_boc.is_error()) { + fatal_error(proof_boc.move_as_error()); + return; + } + proof_data = proof_boc.move_as_ok(); + } + auto res = vm::std_boc_serialize_multi(std::move(trans_roots)); + if (res.is_error()) { + fatal_error(res.move_as_error()); + return; + } + + auto b = ton::create_serialize_tl_object( + ton::create_tl_lite_block_id(base_blk_id_), req_count, !eof, res.move_as_ok(), std::move(proof_data)); + LOG(INFO) << "listBlockTransactionsExt() query completed"; + finish_query(std::move(b)); +} + void LiteQuery::perform_getBlockProof(ton::BlockIdExt from, ton::BlockIdExt to, int mode) { if (!(mode & 1)) { to.invalidate_clear(); @@ -2471,7 +3221,7 @@ void LiteQuery::perform_getShardBlockProof(BlockIdExt blkid) { } AccountIdPrefixFull pfx{masterchainId, shardIdAll}; td::actor::send_closure_later( - manager, &ValidatorManager::get_block_by_seqno_from_db, pfx, handle->masterchain_ref_block(), + manager, &ValidatorManager::get_block_by_seqno_for_litequery, pfx, handle->masterchain_ref_block(), [Self, manager](td::Result R) { if (R.is_error()) { td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); @@ -2575,5 +3325,401 @@ void LiteQuery::continue_getShardBlockProof(Ref cur_block, }); } +void LiteQuery::perform_getOutMsgQueueSizes(td::optional shard) { + LOG(INFO) << "started a getOutMsgQueueSizes" << (shard ? shard.value().to_str() : "") << " liteserver query"; + td::actor::send_closure_later( + manager_, &ton::validator::ValidatorManager::get_last_liteserver_state_block, + [Self = actor_id(this), shard](td::Result, BlockIdExt>> res) { + if (res.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, res.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::continue_getOutMsgQueueSizes, shard, res.ok().first); + } + }); +} + +void LiteQuery::continue_getOutMsgQueueSizes(td::optional shard, Ref state) { + std::vector blocks; + if (!shard || shard_intersects(shard.value(), state->get_shard())) { + blocks.push_back(state->get_block_id()); + } + for (auto& x : state->get_shards()) { + if (!shard || shard_intersects(shard.value(), x->shard())) { + blocks.push_back(x->top_block_id()); + } + } + auto res = std::make_shared>>(blocks.size()); + td::MultiPromise mp; + auto ig = mp.init_guard(); + for (size_t i = 0; i < blocks.size(); ++i) { + td::actor::send_closure(manager_, &ValidatorManager::get_out_msg_queue_size, blocks[i], + [promise = ig.get_promise(), res, i, id = blocks[i]](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, value, std::move(R)); + res->at(i) = create_tl_object( + create_tl_lite_block_id(id), value); + promise.set_value(td::Unit()); + }); + } + ig.add_promise([Self = actor_id(this), res](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + return; + } + td::actor::send_closure(Self, &LiteQuery::finish_query, + create_serialize_tl_object( + std::move(*res), Collator::get_skip_externals_queue_size()), + false); + }); +} + +void LiteQuery::perform_getBlockOutMsgQueueSize(int mode, BlockIdExt blkid) { + LOG(INFO) << "started a getBlockOutMsgQueueSize(" << blkid.to_str() << ", " << mode << ") liteserver query"; + mode_ = mode; + if (!blkid.is_valid_full()) { + fatal_error("invalid BlockIdExt"); + return; + } + set_continuation([=]() -> void { finish_getBlockOutMsgQueueSize(); }); + request_block_data_state(blkid); +} + +void LiteQuery::finish_getBlockOutMsgQueueSize() { + LOG(INFO) << "completing getBlockOutNsgQueueSize() query"; + bool with_proof = mode_ & 1; + Ref state_root = state_->root_cell(); + vm::MerkleProofBuilder pb; + if (with_proof) { + pb = vm::MerkleProofBuilder{state_root}; + state_root = pb.root(); + } + block::gen::ShardStateUnsplit::Record sstate; + block::gen::OutMsgQueueInfo::Record out_msg_queue_info; + if (!tlb::unpack_cell(state_root, sstate) || !tlb::unpack_cell(sstate.out_msg_queue_info, out_msg_queue_info)) { + fatal_error("cannot unpack shard state"); + return; + } + vm::CellSlice& extra_slice = out_msg_queue_info.extra.write(); + if (extra_slice.fetch_long(1) == 0) { + fatal_error("no out_msg_queue_size in shard state"); + return; + } + block::gen::OutMsgQueueExtra::Record out_msg_queue_extra; + if (!tlb::unpack(extra_slice, out_msg_queue_extra)) { + fatal_error("cannot unpack OutMsgQueueExtra"); + return; + } + vm::CellSlice& size_slice = out_msg_queue_extra.out_queue_size.write(); + if (size_slice.fetch_long(1) == 0) { + fatal_error("no out_msg_queue_size in shard state"); + return; + } + td::uint64 size = size_slice.prefetch_ulong(48); + + td::BufferSlice proof; + if (with_proof) { + Ref proof1, proof2; + if (!make_state_root_proof(proof1)) { + return; + } + if (!pb.extract_proof_to(proof2)) { + fatal_error("unknown error creating Merkle proof"); + return; + } + auto r_proof = vm::std_boc_serialize_multi({std::move(proof1), std::move(proof2)}); + if (r_proof.is_error()) { + fatal_error(r_proof.move_as_error()); + return; + } + proof = r_proof.move_as_ok(); + } + LOG(INFO) << "getBlockOutMsgQueueSize(" << blk_id_.to_str() << ", " << mode_ << ") query completed"; + auto b = ton::create_serialize_tl_object( + mode_, ton::create_tl_lite_block_id(blk_id_), size, std::move(proof)); + finish_query(std::move(b)); +} + +void LiteQuery::perform_getDispatchQueueInfo(int mode, BlockIdExt blkid, StdSmcAddress after_addr, int max_accounts) { + LOG(INFO) << "started a getDispatchQueueInfo(" << blkid.to_str() << ", " << mode << ") liteserver query"; + mode_ = mode; + if (!blkid.is_valid_full()) { + fatal_error("invalid BlockIdExt"); + return; + } + if (max_accounts <= 0) { + fatal_error("invalid max_accounts"); + return; + } + set_continuation([=]() -> void { finish_getDispatchQueueInfo(after_addr, max_accounts); }); + request_block_data_state(blkid); +} + +void LiteQuery::finish_getDispatchQueueInfo(StdSmcAddress after_addr, int max_accounts) { + LOG(INFO) << "completing getDispatchQueueInfo() query"; + bool with_proof = mode_ & 1; + Ref state_root = state_->root_cell(); + vm::MerkleProofBuilder pb; + if (with_proof) { + pb = vm::MerkleProofBuilder{state_root}; + state_root = pb.root(); + } + + std::unique_ptr dispatch_queue; + + block::gen::ShardStateUnsplit::Record sstate; + block::gen::OutMsgQueueInfo::Record out_msg_queue_info; + block::gen::OutMsgQueueExtra::Record out_msg_queue_extra; + if (!tlb::unpack_cell(state_root, sstate) || !tlb::unpack_cell(sstate.out_msg_queue_info, out_msg_queue_info)) { + fatal_error("cannot unpack shard state"); + return; + } + vm::CellSlice& extra_slice = out_msg_queue_info.extra.write(); + if (extra_slice.fetch_long(1)) { + if (!tlb::unpack(extra_slice, out_msg_queue_extra)) { + fatal_error("cannot unpack OutMsgQueueExtra"); + return; + } + dispatch_queue = std::make_unique(out_msg_queue_extra.dispatch_queue, 256, + block::tlb::aug_DispatchQueue); + } else { + dispatch_queue = std::make_unique(256, block::tlb::aug_DispatchQueue); + } + + int remaining = std::min(max_accounts, 64); + bool complete = false; + std::vector> result; + bool allow_eq; + if (mode_ & 2) { + allow_eq = false; + } else { + allow_eq = true; + after_addr = td::Bits256::zero(); + } + while (true) { + auto value = dispatch_queue->extract_value(dispatch_queue->lookup_nearest_key(after_addr, true, allow_eq)); + allow_eq = false; + if (value.is_null()) { + complete = true; + break; + } + if (remaining == 0) { + break; + } + --remaining; + StdSmcAddress addr = after_addr; + vm::Dictionary dict{64}; + td::uint64 dict_size; + if (!block::unpack_account_dispatch_queue(value, dict, dict_size)) { + fatal_error(PSTRING() << "invalid account dispatch queue for account " << addr.to_hex()); + return; + } + CHECK(dict_size > 0); + td::BitArray<64> min_lt, max_lt; + dict.get_minmax_key(min_lt.bits(), 64, false, false); + dict.get_minmax_key(max_lt.bits(), 64, true, false); + result.push_back(create_tl_object(addr, dict_size, min_lt.to_ulong(), + max_lt.to_ulong())); + } + + td::BufferSlice proof; + if (with_proof) { + Ref proof1, proof2; + if (!make_state_root_proof(proof1)) { + return; + } + if (!pb.extract_proof_to(proof2)) { + fatal_error("unknown error creating Merkle proof"); + return; + } + auto r_proof = vm::std_boc_serialize_multi({std::move(proof1), std::move(proof2)}); + if (r_proof.is_error()) { + fatal_error(r_proof.move_as_error()); + return; + } + proof = r_proof.move_as_ok(); + } + LOG(INFO) << "getDispatchQueueInfo(" << blk_id_.to_str() << ", " << mode_ << ") query completed"; + auto b = ton::create_serialize_tl_object( + mode_, ton::create_tl_lite_block_id(blk_id_), std::move(result), complete, std::move(proof)); + finish_query(std::move(b)); +} + +void LiteQuery::perform_getDispatchQueueMessages(int mode, BlockIdExt blkid, StdSmcAddress addr, LogicalTime lt, + int max_messages) { + LOG(INFO) << "started a getDispatchQueueMessages(" << blkid.to_str() << ", " << mode << ") liteserver query"; + mode_ = mode; + if (!blkid.is_valid_full()) { + fatal_error("invalid BlockIdExt"); + return; + } + if (max_messages <= 0) { + fatal_error("invalid max_messages"); + return; + } + set_continuation([=]() -> void { finish_getDispatchQueueMessages(addr, lt, max_messages); }); + request_block_data_state(blkid); +} + +void LiteQuery::finish_getDispatchQueueMessages(StdSmcAddress addr, LogicalTime lt, int max_messages) { + LOG(INFO) << "completing getDispatchQueueMessages() query"; + bool with_proof = mode_ & lite_api::liteServer_getDispatchQueueMessages::WANT_PROOF_MASK; + bool one_account = mode_ & lite_api::liteServer_getDispatchQueueMessages::ONE_ACCOUNT_MASK; + bool with_messages_boc = mode_ & lite_api::liteServer_getDispatchQueueMessages::MESSAGES_BOC_MASK; + Ref state_root = state_->root_cell(); + vm::MerkleProofBuilder pb; + if (with_proof) { + pb = vm::MerkleProofBuilder{state_root}; + state_root = pb.root(); + } + + std::unique_ptr dispatch_queue; + + block::gen::ShardStateUnsplit::Record sstate; + block::gen::OutMsgQueueInfo::Record out_msg_queue_info; + block::gen::OutMsgQueueExtra::Record out_msg_queue_extra; + if (!tlb::unpack_cell(state_root, sstate) || !tlb::unpack_cell(sstate.out_msg_queue_info, out_msg_queue_info)) { + fatal_error("cannot unpack shard state"); + return; + } + vm::CellSlice& extra_slice = out_msg_queue_info.extra.write(); + if (extra_slice.fetch_long(1)) { + if (!tlb::unpack(extra_slice, out_msg_queue_extra)) { + fatal_error("cannot unpack OutMsgQueueExtra"); + return; + } + dispatch_queue = std::make_unique(out_msg_queue_extra.dispatch_queue, 256, + block::tlb::aug_DispatchQueue); + } else { + dispatch_queue = std::make_unique(256, block::tlb::aug_DispatchQueue); + } + + int remaining = std::min(max_messages, with_messages_boc ? 16 : 64); + bool complete = false; + std::vector> result; + std::vector> message_roots; + td::Bits256 orig_addr = addr; + bool first = true; + while (remaining > 0) { + auto value = dispatch_queue->extract_value(dispatch_queue->lookup_nearest_key(addr, true, first)); + if (value.is_null() || (one_account && addr != orig_addr)) { + complete = true; + break; + } + vm::Dictionary account_queue{64}; + td::uint64 dict_size; + if (!block::unpack_account_dispatch_queue(value, account_queue, dict_size)) { + fatal_error(PSTRING() << "invalid account dispatch queue for account " << addr.to_hex()); + return; + } + CHECK(dict_size > 0); + while (true) { + td::BitArray<64> lt_key; + lt_key.store_ulong(lt); + auto value2 = account_queue.lookup_nearest_key(lt_key, true, false); + if (value2.is_null()) { + break; + } + lt = lt_key.to_ulong(); + if (remaining == 0) { + break; + } + --remaining; + auto msg_env = value2->prefetch_ref(); + block::tlb::MsgEnvelope::Record_std env; + if (msg_env.is_null() || !tlb::unpack_cell(msg_env, env)) { + fatal_error(PSTRING() << "invalid message in dispatch queue for account " << addr.to_hex() << ", lt " << lt); + return; + } + message_roots.push_back(env.msg); + tl_object_ptr metadata_tl; + if (env.metadata) { + auto& metadata = env.metadata.value(); + metadata_tl = create_tl_object( + 0, metadata.depth, + create_tl_object(metadata.initiator_wc, metadata.initiator_addr), + metadata.initiator_lt); + } else { + metadata_tl = create_tl_object( + 0, -1, create_tl_object(workchainInvalid, td::Bits256::zero()), -1); + } + result.push_back(create_tl_object(addr, lt, env.msg->get_hash().bits(), + std::move(metadata_tl))); + } + first = false; + lt = 0; + } + + td::BufferSlice proof; + if (with_proof) { + Ref proof1, proof2; + if (!make_state_root_proof(proof1)) { + return; + } + if (!pb.extract_proof_to(proof2)) { + fatal_error("unknown error creating Merkle proof"); + return; + } + auto r_proof = vm::std_boc_serialize_multi({std::move(proof1), std::move(proof2)}); + if (r_proof.is_error()) { + fatal_error(r_proof.move_as_error()); + return; + } + proof = r_proof.move_as_ok(); + } + td::BufferSlice messages_boc; + if (with_messages_boc) { + auto r_messages_boc = vm::std_boc_serialize_multi(std::move(message_roots)); + if (r_messages_boc.is_error()) { + fatal_error(r_messages_boc.move_as_error()); + return; + } + messages_boc = std::move(messages_boc); + } + LOG(INFO) << "getDispatchQueueMessages(" << blk_id_.to_str() << ", " << mode_ << ") query completed"; + auto b = ton::create_serialize_tl_object( + mode_, ton::create_tl_lite_block_id(blk_id_), std::move(result), complete, std::move(proof), + std::move(messages_boc)); + finish_query(std::move(b)); +} + +void LiteQuery::perform_nonfinal_getCandidate(td::Bits256 source, BlockIdExt blkid, td::Bits256 collated_data_hash) { + LOG(INFO) << "started a nonfinal.getCandidate liteserver query"; + td::actor::send_closure_later( + manager_, &ValidatorManager::get_block_candidate_for_litequery, PublicKey{pubkeys::Ed25519{source}}, blkid, collated_data_hash, + [Self = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + } else { + BlockCandidate cand = R.move_as_ok(); + td::actor::send_closure_later( + Self, &LiteQuery::finish_query, + create_serialize_tl_object( + create_tl_object( + create_tl_lite_block_id(cand.id), cand.pubkey.as_bits256(), cand.collated_file_hash), + std::move(cand.data), std::move(cand.collated_data)), + false); + } + }); +} + +void LiteQuery::perform_nonfinal_getValidatorGroups(int mode, ShardIdFull shard) { + bool with_shard = mode & 1; + LOG(INFO) << "started a nonfinal.getValidatorGroups" << (with_shard ? shard.to_str() : "(all)") + << " liteserver query"; + td::optional maybe_shard; + if (with_shard) { + maybe_shard = shard; + } + td::actor::send_closure( + manager_, &ValidatorManager::get_validator_groups_info_for_litequery, maybe_shard, + [Self = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(Self, &LiteQuery::abort_query, R.move_as_error()); + } else { + td::actor::send_closure_later(Self, &LiteQuery::finish_query, serialize_tl_object(R.move_as_ok(), true), + false); + } + }); +} + } // namespace validator } // namespace ton diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index 47970aae..fc873533 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -27,7 +27,7 @@ #include "shard.hpp" #include "proof.hpp" #include "block/block-auto.h" - +#include "auto/tl/lite_api.h" namespace ton { @@ -37,11 +37,16 @@ using td::Ref; class LiteQuery : public td::actor::Actor { td::BufferSlice query_; td::actor::ActorId manager_; + td::actor::ActorId cache_; td::Timestamp timeout_; td::Promise promise_; td::Promise,UnixTime,LogicalTime,std::unique_ptr>> acc_state_promise_; + tl_object_ptr query_obj_; + bool use_cache_{false}; + td::Bits256 cache_key_; + int pending_{0}; int mode_{0}; WorkchainId acc_workchain_; @@ -57,13 +62,16 @@ class LiteQuery : public td::actor::Actor { td::BufferSlice buffer_; std::function continuation_; bool cont_set_{false}; - td::BufferSlice shard_proof_; + td::BufferSlice shard_proof_, proof_; std::vector> roots_; std::vector> aux_objs_; std::vector blk_ids_; std::unique_ptr chain_; Ref stack_; + td::BufferSlice lookup_header_proof_; + td::BufferSlice lookup_prev_header_proof_; + public: enum { default_timeout_msec = 4500, // 4.5 seconds @@ -75,11 +83,11 @@ class LiteQuery : public td::actor::Actor { ls_capabilities = 7 }; // version 1.1; +1 = build block proof chains, +2 = masterchainInfoExt, +4 = runSmcMethod LiteQuery(td::BufferSlice data, td::actor::ActorId manager, - td::Promise promise); + td::actor::ActorId cache, td::Promise promise); LiteQuery(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise); static void run_query(td::BufferSlice data, td::actor::ActorId manager, - td::Promise promise); + td::actor::ActorId cache, td::Promise promise); static void fetch_account_state(WorkchainId wc, StdSmcAddress acc_addr, td::actor::ActorId manager, td::Promise,UnixTime,LogicalTime,std::unique_ptr>> promise); @@ -89,10 +97,11 @@ class LiteQuery : public td::actor::Actor { bool fatal_error(std::string err_msg, int err_code = -400); bool fatal_error(int err_code, std::string err_msg = ""); void abort_query(td::Status reason); - void abort_query_ext(td::Status reason, std::string err_msg); - bool finish_query(td::BufferSlice result); + bool finish_query(td::BufferSlice result, bool skip_cache_update = false); void alarm() override; void start_up() override; + bool use_cache(); + void perform(); void perform_getTime(); void perform_getVersion(); void perform_getMasterchainInfo(int mode); @@ -117,6 +126,8 @@ class LiteQuery : public td::actor::Actor { UnixTime gen_utime, LogicalTime gen_lt); void perform_getLibraries(std::vector library_list); void continue_getLibraries(Ref mc_state, BlockIdExt blkid, std::vector library_list); + void perform_getLibrariesWithProof(BlockIdExt blkid, int mode, std::vector library_list); + void continue_getLibrariesWithProof(std::vector library_list, int mode); void perform_getOneTransaction(BlockIdExt blkid, WorkchainId workchain, StdSmcAddress addr, LogicalTime lt); void continue_getOneTransaction(); void perform_getTransactions(WorkchainId workchain, StdSmcAddress addr, LogicalTime lt, Bits256 hash, unsigned count); @@ -131,8 +142,16 @@ class LiteQuery : public td::actor::Actor { void perform_getConfigParams(BlockIdExt blkid, int mode, std::vector param_list = {}); void continue_getConfigParams(int mode, std::vector param_list); void perform_lookupBlock(BlockId blkid, int mode, LogicalTime lt, UnixTime utime); + void perform_lookupBlockWithProof(BlockId blkid, BlockIdExt client_mc_blkid, int mode, LogicalTime lt, UnixTime utime); + void continue_lookupBlockWithProof_getHeaderProof(Ref block, AccountIdPrefixFull req_prefix, BlockSeqno masterchain_ref_seqno); + void continue_lookupBlockWithProof_gotPrevBlockData(Ref prev_block, BlockSeqno masterchain_ref_seqno); + void continue_lookupBlockWithProof_buildProofLinks(td::Ref cur_block, std::vector>> result); + void continue_lookupBlockWithProof_getClientMcBlockDataState(std::vector>> links); + void continue_lookupBlockWithProof_getMcBlockPrev(std::vector>> links); void perform_listBlockTransactions(BlockIdExt blkid, int mode, int count, Bits256 account, LogicalTime lt); void finish_listBlockTransactions(int mode, int count); + void perform_listBlockTransactionsExt(BlockIdExt blkid, int mode, int count, Bits256 account, LogicalTime lt); + void finish_listBlockTransactionsExt(int mode, int count); void perform_getBlockProof(BlockIdExt from, BlockIdExt to, int mode); void continue_getBlockProof(BlockIdExt from, BlockIdExt to, int mode, BlockIdExt baseblk, Ref state); @@ -148,6 +167,18 @@ class LiteQuery : public td::actor::Actor { void perform_getShardBlockProof(BlockIdExt blkid); void continue_getShardBlockProof(Ref cur_block, std::vector> result); + void perform_getOutMsgQueueSizes(td::optional shard); + void continue_getOutMsgQueueSizes(td::optional shard, Ref state); + void perform_getBlockOutMsgQueueSize(int mode, BlockIdExt blkid); + void finish_getBlockOutMsgQueueSize(); + void perform_getDispatchQueueInfo(int mode, BlockIdExt blkid, StdSmcAddress after_addr, int max_accounts); + void finish_getDispatchQueueInfo(StdSmcAddress after_addr, int max_accounts); + void perform_getDispatchQueueMessages(int mode, BlockIdExt blkid, StdSmcAddress addr, LogicalTime lt, + int max_messages); + void finish_getDispatchQueueMessages(StdSmcAddress addr, LogicalTime lt, int max_messages); + + void perform_nonfinal_getCandidate(td::Bits256 source, BlockIdExt blkid, td::Bits256 collated_data_hash); + void perform_nonfinal_getValidatorGroups(int mode, ShardIdFull shard); void load_prevKeyBlock(ton::BlockIdExt blkid, td::Promise>>); void continue_loadPrevKeyBlock(ton::BlockIdExt blkid, td::Result, BlockIdExt>> res, diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp new file mode 100644 index 00000000..95ad4a41 --- /dev/null +++ b/validator/impl/out-msg-queue-proof.cpp @@ -0,0 +1,294 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "out-msg-queue-proof.hpp" +#include "interfaces/proof.h" +#include "shard.hpp" +#include "vm/cells/MerkleProof.h" +#include "common/delay.h" +#include "interfaces/validator-manager.h" +#include "block/block-parse.h" +#include "block/block-auto.h" +#include "output-queue-merger.h" + +namespace ton { + +namespace validator { + +static td::Status check_no_prunned(const Ref& cell) { + if (cell.is_null()) { + return td::Status::OK(); + } + TRY_RESULT(loaded_cell, cell->load_cell()); + if (loaded_cell.data_cell->get_level() > 0) { + return td::Status::Error("prunned branch"); + } + return td::Status::OK(); +} + +static td::Status check_no_prunned(const vm::CellSlice& cs) { + for (unsigned i = 0; i < cs.size_refs(); ++i) { + TRY_STATUS(check_no_prunned(cs.prefetch_ref(i))); + } + return td::Status::OK(); +} + +static td::Result> process_queue( + ShardIdFull dst_shard, std::vector> blocks, + block::ImportedMsgQueueLimits limits) { + td::uint64 estimated_proof_size = 0; + + td::HashSet visited; + std::function dfs_cs; + auto dfs = [&](const Ref& cell) { + if (cell.is_null() || !visited.insert(cell->get_hash()).second) { + return; + } + dfs_cs(vm::CellSlice(vm::NoVm(), cell)); + }; + dfs_cs = [&](const vm::CellSlice& cs) { + // Based on BlockLimitStatus::estimate_block_size + estimated_proof_size += 12 + (cs.size() + 7) / 8 + cs.size_refs() * 3; + for (unsigned i = 0; i < cs.size_refs(); i++) { + dfs(cs.prefetch_ref(i)); + } + }; + std::vector neighbors; + for (auto& b : blocks) { + TRY_STATUS_PREFIX(check_no_prunned(*b.second.proc_info), "invalid proc_info proof: ") + dfs_cs(*b.second.proc_info); + neighbors.emplace_back(b.first, b.second.out_queue->prefetch_ref()); + } + + block::OutputQueueMerger queue_merger{dst_shard, std::move(neighbors)}; + std::vector msg_count(blocks.size()); + td::int32 msg_count_total = 0; + bool limit_reached = false; + + while (!queue_merger.is_eof()) { + auto kv = queue_merger.extract_cur(); + queue_merger.next(); + block::EnqueuedMsgDescr enq; + auto msg = kv->msg; + if (!enq.unpack(msg.write())) { + return td::Status::Error("cannot unpack EnqueuedMsgDescr"); + } + if (limit_reached) { + break; + } + ++msg_count[kv->source]; + ++msg_count_total; + + dfs_cs(*kv->msg); + TRY_STATUS_PREFIX(check_no_prunned(*kv->msg), "invalid message proof: ") + if (estimated_proof_size >= limits.max_bytes || msg_count_total >= (long long)limits.max_msgs) { + limit_reached = true; + } + } + if (!limit_reached) { + std::fill(msg_count.begin(), msg_count.end(), -1); + } + return msg_count; +} + +td::Result> OutMsgQueueProof::build( + ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits) { + if (!dst_shard.is_valid_ext()) { + return td::Status::Error("invalid shard"); + } + if (blocks.empty()) { + return create_tl_object(td::BufferSlice{}, td::BufferSlice{}, + std::vector{}); + } + + std::vector> block_state_proofs; + for (auto& block : blocks) { + if (block.id.seqno() != 0) { + if (block.block_root.is_null()) { + return td::Status::Error("block is null"); + } + TRY_RESULT(proof, create_block_state_proof(block.block_root)); + block_state_proofs.push_back(std::move(proof)); + } + if (!block::ShardConfig::is_neighbor(dst_shard, block.id.shard_full())) { + return td::Status::Error("shards are not neighbors"); + } + } + TRY_RESULT(block_state_proof, vm::std_boc_serialize_multi(block_state_proofs)); + + vm::Dictionary states_dict_pure{32}; + for (size_t i = 0; i < blocks.size(); ++i) { + if (blocks[i].state_root.is_null()) { + return td::Status::Error("state is null"); + } + states_dict_pure.set_ref(td::BitArray<32>{(long long)i}, blocks[i].state_root); + } + + vm::MerkleProofBuilder mpb{states_dict_pure.get_root_cell()}; + vm::Dictionary states_dict{mpb.root(), 32}; + std::vector> data(blocks.size()); + for (size_t i = 0; i < blocks.size(); ++i) { + data[i].first = blocks[i].id; + TRY_RESULT(state, ShardStateQ::fetch(blocks[i].id, {}, states_dict.lookup_ref(td::BitArray<32>{(long long)i}))); + TRY_RESULT(outq_descr, state->message_queue()); + block::gen::OutMsgQueueInfo::Record qinfo; + if (!tlb::unpack_cell(outq_descr->root_cell(), data[i].second)) { + return td::Status::Error("invalid message queue"); + } + } + TRY_RESULT(msg_count, process_queue(dst_shard, std::move(data), limits)); + + TRY_RESULT(proof, mpb.extract_proof()); + vm::Dictionary states_dict_proof{vm::CellSlice{vm::NoVm(), proof}.prefetch_ref(), 32}; + std::vector> state_proofs; + for (size_t i = 0; i < blocks.size(); ++i) { + td::Ref proof_raw = states_dict_proof.lookup_ref(td::BitArray<32>{(long long)i}); + CHECK(proof_raw.not_null()); + state_proofs.push_back(vm::CellBuilder::create_merkle_proof(proof_raw)); + } + TRY_RESULT(queue_proof, vm::std_boc_serialize_multi(state_proofs)); + return create_tl_object(std::move(queue_proof), std::move(block_state_proof), + std::move(msg_count)); +} + +td::Result>> OutMsgQueueProof::fetch(ShardIdFull dst_shard, + std::vector blocks, + block::ImportedMsgQueueLimits limits, + const ton_api::tonNode_outMsgQueueProof& f) { + try { + std::vector> res; + TRY_RESULT(queue_proofs, vm::std_boc_deserialize_multi(f.queue_proofs_, (int)blocks.size())); + TRY_RESULT(block_state_proofs, vm::std_boc_deserialize_multi(f.block_state_proofs_, (int)blocks.size())); + if (queue_proofs.size() != blocks.size()) { + return td::Status::Error("invalid size of queue_proofs"); + } + if (f.msg_counts_.size() != blocks.size()) { + return td::Status::Error("invalid size of msg_counts"); + } + size_t j = 0; + std::vector> data(blocks.size()); + for (size_t i = 0; i < blocks.size(); ++i) { + td::Bits256 state_root_hash; + Ref block_state_proof = {}; + if (blocks[i].seqno() == 0) { + state_root_hash = blocks[i].root_hash; + } else { + if (j == block_state_proofs.size()) { + return td::Status::Error("invalid size of block_state_proofs"); + } + block_state_proof = block_state_proofs[j++]; + TRY_RESULT_ASSIGN(state_root_hash, unpack_block_state_proof(blocks[i], block_state_proof)); + } + auto state_root = vm::MerkleProof::virtualize(queue_proofs[i], 1); + if (state_root->get_hash().as_slice() != state_root_hash.as_slice()) { + return td::Status::Error("state root hash mismatch"); + } + res.emplace_back(true, blocks[i], state_root, block_state_proof, f.msg_counts_[i]); + + data[i].first = blocks[i]; + TRY_RESULT(state, ShardStateQ::fetch(blocks[i], {}, state_root)); + TRY_RESULT(outq_descr, state->message_queue()); + block::gen::OutMsgQueueInfo::Record qinfo; + if (!tlb::unpack_cell(outq_descr->root_cell(), data[i].second)) { + return td::Status::Error("invalid message queue"); + } + } + if (j != block_state_proofs.size()) { + return td::Status::Error("invalid size of block_state_proofs"); + } + TRY_RESULT(msg_count, process_queue(dst_shard, std::move(data), limits)); + if (msg_count != f.msg_counts_) { + return td::Status::Error("incorrect msg_count"); + } + return res; + } catch (vm::VmVirtError& err) { + return td::Status::Error(PSTRING() << "invalid proof: " << err.get_msg()); + } +} + +void BuildOutMsgQueueProof::abort_query(td::Status reason) { + if (promise_) { + FLOG(DEBUG) { + sb << "failed to build msg queue proof to " << dst_shard_.to_str() << " from"; + for (const auto& block : blocks_) { + sb << " " << block.id.id.to_str(); + } + sb << ": " << reason; + }; + promise_.set_error( + reason.move_as_error_prefix(PSTRING() << "failed to build msg queue proof to " << dst_shard_.to_str() << ": ")); + } + stop(); +} + +void BuildOutMsgQueueProof::start_up() { + for (size_t i = 0; i < blocks_.size(); ++i) { + BlockIdExt id = blocks_[i].id; + ++pending; + td::actor::send_closure(manager_, &ValidatorManagerInterface::get_shard_state_from_db_short, id, + [SelfId = actor_id(this), i](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, + R.move_as_error_prefix("failed to get shard state: ")); + } else { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_state_root, i, + R.move_as_ok()->root_cell()); + } + }); + if (id.seqno() != 0) { + ++pending; + td::actor::send_closure(manager_, &ValidatorManagerInterface::get_block_data_from_db_short, id, + [SelfId = actor_id(this), i](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, + R.move_as_error_prefix("failed to get block data: ")); + } else { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_block_root, i, + R.move_as_ok()->root_cell()); + } + }); + } + } + if (pending == 0) { + build_proof(); + } +} + +void BuildOutMsgQueueProof::got_state_root(size_t i, Ref root) { + blocks_[i].state_root = std::move(root); + if (--pending == 0) { + build_proof(); + } +} + +void BuildOutMsgQueueProof::got_block_root(size_t i, Ref root) { + blocks_[i].block_root = std::move(root); + if (--pending == 0) { + build_proof(); + } +} + +void BuildOutMsgQueueProof::build_proof() { + auto result = OutMsgQueueProof::build(dst_shard_, std::move(blocks_), limits_); + if (result.is_error()) { + LOG(ERROR) << "Failed to build msg queue proof: " << result.error(); + } + promise_.set_result(std::move(result)); + stop(); +} + +} // namespace validator +} // namespace ton diff --git a/validator/impl/out-msg-queue-proof.hpp b/validator/impl/out-msg-queue-proof.hpp new file mode 100644 index 00000000..e28561e2 --- /dev/null +++ b/validator/impl/out-msg-queue-proof.hpp @@ -0,0 +1,64 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "vm/cells.h" +#include "ton/ton-types.h" +#include "auto/tl/ton_api.h" +#include "interfaces/out-msg-queue-proof.h" +#include "td/actor/actor.h" +#include "interfaces/shard.h" +#include "validator.h" + +namespace ton { + +namespace validator { +using td::Ref; + +class ValidatorManager; +class ValidatorManagerInterface; + +class BuildOutMsgQueueProof : public td::actor::Actor { + public: + BuildOutMsgQueueProof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, + td::actor::ActorId manager, + td::Promise> promise) + : dst_shard_(dst_shard), limits_(limits), manager_(manager), promise_(std::move(promise)) { + blocks_.resize(blocks.size()); + for (size_t i = 0; i < blocks_.size(); ++i) { + blocks_[i].id = blocks[i]; + } + } + + void abort_query(td::Status reason); + void start_up() override; + void got_state_root(size_t i, Ref root); + void got_block_root(size_t i, Ref root); + void build_proof(); + + private: + ShardIdFull dst_shard_; + std::vector blocks_; + block::ImportedMsgQueueLimits limits_; + + td::actor::ActorId manager_; + td::Promise> promise_; + + size_t pending = 0; +}; + +} // namespace validator +} // namespace ton diff --git a/validator/impl/proof.cpp b/validator/impl/proof.cpp index 033a1ab1..d7222211 100644 --- a/validator/impl/proof.cpp +++ b/validator/impl/proof.cpp @@ -162,5 +162,40 @@ td::Result> ProofQ::get_signatures_root() const { return proof.signatures->prefetch_ref(); } +td::Result> create_block_state_proof(td::Ref root) { + if (root.is_null()) { + return td::Status::Error("root is null"); + } + vm::MerkleProofBuilder mpb{std::move(root)}; + block::gen::Block::Record block; + if (!tlb::unpack_cell(mpb.root(), block) || block.state_update->load_cell().is_error()) { + return td::Status::Error("invalid block"); + } + TRY_RESULT(proof, mpb.extract_proof()); + if (proof.is_null()) { + return td::Status::Error("failed to create proof"); + } + return proof; +} + +td::Result unpack_block_state_proof(BlockIdExt block_id, td::Ref proof) { + auto virt_root = vm::MerkleProof::virtualize(proof, 1); + if (virt_root.is_null()) { + return td::Status::Error("invalid Merkle proof"); + } + if (virt_root->get_hash().as_slice() != block_id.root_hash.as_slice()) { + return td::Status::Error("hash mismatch"); + } + block::gen::Block::Record block; + if (!tlb::unpack_cell(virt_root, block)) { + return td::Status::Error("invalid block"); + } + vm::CellSlice upd_cs{vm::NoVmSpec(), block.state_update}; + if (!(upd_cs.is_special() && upd_cs.prefetch_long(8) == 4 && upd_cs.size_ext() == 0x20228)) { + return td::Status::Error("invalid Merkle update"); + } + return upd_cs.prefetch_ref(1)->get_hash(0).bits(); +} + } // namespace validator } // namespace ton diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index a96f1a81..0ab216f7 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -44,6 +44,7 @@ ShardStateQ::ShardStateQ(const ShardStateQ& other) , root(other.root) , lt(other.lt) , utime(other.utime) + , global_id_(other.global_id_) , before_split_(other.before_split_) , fake_split_(other.fake_split_) , fake_merge_(other.fake_merge_) { @@ -121,6 +122,7 @@ td::Status ShardStateQ::init() { } lt = info.gen_lt; utime = info.gen_utime; + global_id_ = info.global_id; before_split_ = info.before_split; block::ShardId id{info.shard_id}; ton::BlockId hdr_id{ton::ShardIdFull(id), info.seq_no}; @@ -129,6 +131,15 @@ td::Status ShardStateQ::init() { " contains BlockId " + hdr_id.to_str() + " different from the one originally required"); } + if (info.r1.master_ref.write().fetch_long(1)) { + BlockIdExt mc_id; + if (!block::tlb::t_ExtBlkRef.unpack(info.r1.master_ref, mc_id, nullptr)) { + return td::Status::Error(-668, "cannot unpack master_ref in shardchain state of "s + blkid.to_str()); + } + master_ref = mc_id; + } else { + master_ref = {}; + } return td::Status::OK(); } @@ -279,6 +290,7 @@ td::Result, td::Ref>> ShardStateQ::spl } td::Result ShardStateQ::serialize() const { + TD_PERF_COUNTER(serialize_state); td::PerfWarningTimer perf_timer_{"serializestate", 0.1}; if (!data.is_null()) { return data.clone(); @@ -303,6 +315,7 @@ td::Result ShardStateQ::serialize() const { } td::Status ShardStateQ::serialize_to_file(td::FileFd& fd) const { + TD_PERF_COUNTER(serialize_state_to_file); td::PerfWarningTimer perf_timer_{"serializestate", 0.1}; if (!data.is_null()) { auto cur_data = data.clone(); @@ -364,7 +377,8 @@ td::Status MasterchainStateQ::mc_init() { td::Status MasterchainStateQ::mc_reinit() { auto res = block::ConfigInfo::extract_config( root_cell(), block::ConfigInfo::needStateRoot | block::ConfigInfo::needValidatorSet | - block::ConfigInfo::needShardHashes | block::ConfigInfo::needPrevBlocks); + block::ConfigInfo::needShardHashes | block::ConfigInfo::needPrevBlocks | + block::ConfigInfo::needWorkchainInfo); cur_validators_.reset(); next_validators_.reset(); if (res.is_error()) { @@ -510,15 +524,15 @@ bool MasterchainStateQ::check_old_mc_block_id(const ton::BlockIdExt& blkid, bool return config_ && config_->check_old_mc_block_id(blkid, strict); } -td::uint32 MasterchainStateQ::min_split_depth(WorkchainId workchain_id) const { +td::uint32 MasterchainStateQ::monitor_min_split_depth(WorkchainId workchain_id) const { if (!config_) { return 0; } auto wc_info = config_->get_workchain_info(workchain_id); - return wc_info.not_null() ? wc_info->actual_min_split : 0; + return wc_info.not_null() ? wc_info->monitor_min_split : 0; } -td::uint32 MasterchainStateQ::soft_min_split_depth(WorkchainId workchain_id) const { +td::uint32 MasterchainStateQ::min_split_depth(WorkchainId workchain_id) const { if (!config_) { return 0; } @@ -555,5 +569,9 @@ BlockIdExt MasterchainStateQ::prev_key_block_id(BlockSeqno seqno) const { return block_id; } +bool MasterchainStateQ::is_key_state() const { + return config_ ? config_->is_key_state() : false; +} + } // namespace validator } // namespace ton diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index 1c511ab5..fa36e1e6 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -38,9 +38,11 @@ class ShardStateQ : virtual public ShardState { Ref root; LogicalTime lt{0}; UnixTime utime{0}; + td::int32 global_id_{0}; bool before_split_{false}; bool fake_split_{false}; bool fake_merge_{false}; + td::optional master_ref; protected: friend class Ref; @@ -80,6 +82,12 @@ class ShardStateQ : virtual public ShardState { LogicalTime get_logical_time() const override { return lt; } + td::int32 get_global_id() const override { + return global_id_; + } + td::optional get_master_ref() const override { + return master_ref; + } td::Status validate_deep() const override; ShardStateQ* make_copy() const override; td::Result> message_queue() const override; @@ -116,8 +124,8 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { bool has_workchain(WorkchainId workchain) const { return config_ && config_->has_workchain(workchain); } + td::uint32 monitor_min_split_depth(WorkchainId workchain_id) const override; td::uint32 min_split_depth(WorkchainId workchain_id) const override; - td::uint32 soft_min_split_depth(WorkchainId workchain_id) const override; BlockSeqno min_ref_masterchain_seqno() const override; td::Status prepare() override; ZeroStateIdExt get_zerostate_id() const { @@ -133,6 +141,7 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { BlockIdExt last_key_block_id() const override; BlockIdExt next_key_block_id(BlockSeqno seqno) const override; BlockIdExt prev_key_block_id(BlockSeqno seqno) const override; + bool is_key_state() const override; MasterchainStateQ* make_copy() const override; static td::Result> fetch(const BlockIdExt& _id, td::BufferSlice _data, @@ -151,6 +160,9 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { return td::make_ref(config_); } } + block::WorkchainSet get_workchain_list() const override { + return config_ ? config_->get_workchain_list() : block::WorkchainSet(); + } private: ZeroStateIdExt zerostate_id_; diff --git a/validator/impl/signature-set.cpp b/validator/impl/signature-set.cpp index c7298216..0078a115 100644 --- a/validator/impl/signature-set.cpp +++ b/validator/impl/signature-set.cpp @@ -42,9 +42,6 @@ td::BufferSlice BlockSignatureSetQ::serialize() const { } Ref root; CHECK(serialize_to(root)); - //std::cerr << "serializing BlockSignatureSet: "; - //vm::CellSlice{vm::NoVm{}, root}.print_rec(std::cerr); - //std::cerr << std::endl; auto res = vm::std_boc_serialize(std::move(root)); LOG_CHECK(res.is_ok()) << res.move_as_error(); return res.move_as_ok(); diff --git a/validator/impl/top-shard-descr.cpp b/validator/impl/top-shard-descr.cpp index c6e00edf..9eadeef3 100644 --- a/validator/impl/top-shard-descr.cpp +++ b/validator/impl/top-shard-descr.cpp @@ -62,12 +62,10 @@ td::Status ShardTopBlockDescrQ::unpack_one_proof(BlockIdExt& cur_id, Ref prev, BlockCandidate candidate, Ref validator_set, +/** + * Constructs a ValidateQuery object. + * + * @param shard The shard of the block being validated. + * @param min_masterchain_block_id The minimum allowed masterchain block reference for the block. + * @param prev A vector of BlockIdExt representing the previous blocks. + * @param candidate The BlockCandidate to be validated. + * @param validator_set A reference to the ValidatorSet. + * @param manager The ActorId of the ValidatorManager. + * @param timeout The timeout for the validation. + * @param promise The Promise to return the ValidateCandidateResult to. + * @param is_fake A boolean indicating if the validation is fake (performed when creating a hardfork). + */ +ValidateQuery::ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, + BlockCandidate candidate, Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, bool is_fake) : shard_(shard) , id_(candidate.id) - , min_ts(min_ts) , min_mc_block_id(min_masterchain_block_id) , prev_blocks(std::move(prev)) , block_candidate(std::move(candidate)) @@ -68,21 +85,37 @@ ValidateQuery::ValidateQuery(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_ , perf_timer_("validateblock", 0.1, [manager](double duration) { send_closure(manager, &ValidatorManager::add_perf_timer_stat, "validateblock", duration); }) { - proc_hash_.zero(); } +/** + * Raises an error when timeout is reached. + */ void ValidateQuery::alarm() { abort_query(td::Status::Error(ErrorCode::timeout, "timeout")); } +/** + * Aborts the validation with the given error. + * + * @param error The error encountered. + */ void ValidateQuery::abort_query(td::Status error) { (void)fatal_error(std::move(error)); } +/** + * Rejects the validation and logs an error message. + * + * @param error The error message to be logged. + * @param reason The reason for rejecting the validation. + * + * @returns False indicating that the validation failed. + */ bool ValidateQuery::reject_query(std::string error, td::BufferSlice reason) { error = error_ctx() + error; LOG(ERROR) << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error; if (main_promise) { + record_stats(false); errorlog::ErrorLog::log(PSTRING() << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error << ": data=" << block_candidate.id.file_hash.to_hex() << " collated_data=" << block_candidate.collated_file_hash.to_hex()); @@ -94,15 +127,33 @@ bool ValidateQuery::reject_query(std::string error, td::BufferSlice reason) { return false; } +/** + * Rejects the validation and logs an error message. + * + * @param err_msg The error message to be displayed. + * @param error The error status. + * @param reason The reason for rejecting the query. + * + * @returns False indicating that the validation failed. + */ bool ValidateQuery::reject_query(std::string err_msg, td::Status error, td::BufferSlice reason) { error.ensure_error(); return reject_query(err_msg + " : " + error.to_string(), std::move(reason)); } +/** + * Rejects the validation and logs an error message. + * + * @param error The error message to be logged. + * @param reason The reason for rejecting the validation. + * + * @returns False indicating that the validation failed. + */ bool ValidateQuery::soft_reject_query(std::string error, td::BufferSlice reason) { error = error_ctx() + error; LOG(ERROR) << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error; if (main_promise) { + record_stats(false); errorlog::ErrorLog::log(PSTRING() << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error << ": data=" << block_candidate.id.file_hash.to_hex() << " collated_data=" << block_candidate.collated_file_hash.to_hex()); @@ -114,10 +165,18 @@ bool ValidateQuery::soft_reject_query(std::string error, td::BufferSlice reason) return false; } +/** + * Handles a fatal error during validation. + * + * @param error The error status. + * + * @returns False indicating that the validation failed. + */ bool ValidateQuery::fatal_error(td::Status error) { error.ensure_error(); LOG(ERROR) << "aborting validation of block candidate for " << shard_.to_str() << " : " << error.to_string(); if (main_promise) { + record_stats(false); auto c = error.code(); if (c <= -667 && c >= -670) { errorlog::ErrorLog::log(PSTRING() << "FATAL ERROR: aborting validation of block candidate for " << shard_.to_str() @@ -132,21 +191,51 @@ bool ValidateQuery::fatal_error(td::Status error) { return false; } +/** + * Handles a fatal error during validation. + * + * @param err_code Error code. + * @param error Error message. + * + * @returns False indicating that the validation failed. + */ bool ValidateQuery::fatal_error(int err_code, std::string err_msg) { return fatal_error(td::Status::Error(err_code, error_ctx() + err_msg)); } +/** + * Handles a fatal error during validation. + * + * @param err_code Error code. + * @param err_msg Error message. + * @param error Error status. + * + * @returns False indicating that the validation failed. + */ bool ValidateQuery::fatal_error(int err_code, std::string err_msg, td::Status error) { error.ensure_error(); return fatal_error(err_code, err_msg + " : " + error.to_string()); } +/** + * Handles a fatal error during validation. + * + * @param error Error message. + * @param err_code Error code. + * + * @returns False indicating that the validation failed. + */ bool ValidateQuery::fatal_error(std::string err_msg, int err_code) { return fatal_error(td::Status::Error(err_code, error_ctx() + err_msg)); } +/** + * Finishes the query and sends the result to the promise. + */ void ValidateQuery::finish_query() { if (main_promise) { + record_stats(true); + LOG(WARNING) << "validate query done"; main_promise.set_result(now_); } stop(); @@ -158,8 +247,14 @@ void ValidateQuery::finish_query() { * */ +/** + * Starts the validation process. + * + * This function performs various checks on the validation parameters and the block candidate. + * Then the function also sends requests to the ValidatorManager to fetch blocks and shard stated. + */ void ValidateQuery::start_up() { - LOG(INFO) << "validate query for " << block_candidate.id.to_str() << " started"; + LOG(WARNING) << "validate query for " << block_candidate.id.to_str() << " started"; alarm_timestamp() = timeout; rand_seed_.set_zero(); created_by_ = block_candidate.pubkey; @@ -251,16 +346,7 @@ void ValidateQuery::start_up() { // return; } } - // 2. learn latest masterchain state and block id - LOG(DEBUG) << "sending get_top_masterchain_state_block() to Manager"; - ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::get_top_masterchain_state_block, - [self = get_self()](td::Result, BlockIdExt>> res) { - LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; - td::actor::send_closure_later( - std::move(self), &ValidateQuery::after_get_latest_mc_state, std::move(res)); - }); - // 3. load state(s) corresponding to previous block(s) + // 2. load state(s) corresponding to previous block(s) prev_states.resize(prev_blocks.size()); for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { // 3.1. load state @@ -273,21 +359,13 @@ void ValidateQuery::start_up() { std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res)); }); } - // 4. unpack block candidate (while necessary data is being loaded) + // 3. unpack block candidate (while necessary data is being loaded) if (!unpack_block_candidate()) { reject_query("error unpacking block candidate"); return; } - // 5. request masterchain state referred to in the block + // 4. request masterchain handle and state referred to in the block if (!is_masterchain()) { - ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, mc_blkid_, priority(), timeout, - [self = get_self()](td::Result> res) { - LOG(DEBUG) << "got answer to wait_block_state() query for masterchain block"; - td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_mc_state, - std::move(res)); - }); - // 5.1. request corresponding block handle ++pending; td::actor::send_closure_later(manager, &ValidatorManager::get_block_handle, mc_blkid_, true, [self = get_self()](td::Result res) { @@ -307,7 +385,16 @@ void ValidateQuery::start_up() { CHECK(pending); } -// unpack block candidate, and check root hash and file hash +/** + * Unpacks and validates a block candidate. + * + * This function unpacks the block candidate data and performs various validation checks to ensure its integrity. + * It checks the file hash and root hash of the block candidate against the expected values. + * It then parses the block header and checks its validity. + * Finally, it deserializes the collated data and extracts the collated roots. + * + * @returns True if the block candidate was successfully unpacked, false otherwise. + */ bool ValidateQuery::unpack_block_candidate() { vm::BagOfCells boc1, boc2; // 1. deserialize block itself @@ -359,6 +446,11 @@ bool ValidateQuery::unpack_block_candidate() { return extract_collated_data(); } +/** + * Initializes the validation by parsing and checking the block header. + * + * @returns True if the initialization is successful, false otherwise. + */ bool ValidateQuery::init_parse() { CHECK(block_root_.not_null()); std::vector prev_blks; @@ -486,6 +578,14 @@ bool ValidateQuery::init_parse() { return true; } +/** + * Extracts collated data from a cell. + * + * @param croot The root cell containing the collated data. + * @param idx The index of the root. + * + * @returns True if the extraction is successful, false otherwise. + */ bool ValidateQuery::extract_collated_data_from(Ref croot, int idx) { bool is_special = false; auto cs = vm::load_cell_slice_special(croot, is_special); @@ -523,7 +623,11 @@ bool ValidateQuery::extract_collated_data_from(Ref croot, int idx) { return true; } -// processes further and sorts data in collated_roots_ +/** + * Extracts collated data from a list of collated roots. + * + * @returns True if the extraction is successful, False otherwise. + */ bool ValidateQuery::extract_collated_data() { int i = -1; for (auto croot : collated_roots_) { @@ -542,8 +646,26 @@ bool ValidateQuery::extract_collated_data() { return true; } +/** + * Send get_top_masterchain_state_block to manager, call after_get_latest_mc_state afterwards + */ +void ValidateQuery::request_latest_mc_state() { + ++pending; + td::actor::send_closure_later(manager, &ValidatorManager::get_top_masterchain_state_block, + [self = get_self()](td::Result, BlockIdExt>> res) { + LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; + td::actor::send_closure_later( + std::move(self), &ValidateQuery::after_get_latest_mc_state, std::move(res)); + }); +} + +/** + * Callback function called after retrieving the latest masterchain state. + * + * @param res The result of the retrieval of the latest masterchain state. + */ void ValidateQuery::after_get_latest_mc_state(td::Result, BlockIdExt>> res) { - LOG(DEBUG) << "in ValidateQuery::after_get_latest_mc_state()"; + LOG(WARNING) << "in ValidateQuery::after_get_latest_mc_state()"; --pending; if (res.is_error()) { fatal_error(res.move_as_error()); @@ -578,8 +700,14 @@ void ValidateQuery::after_get_latest_mc_state(td::Result> res) { - LOG(DEBUG) << "in ValidateQuery::after_get_mc_state() for " << mc_blkid_.to_str(); + CHECK(!is_masterchain()); + LOG(WARNING) << "in ValidateQuery::after_get_mc_state() for " << mc_blkid_.to_str(); --pending; if (res.is_error()) { fatal_error(res.move_as_error()); @@ -589,6 +717,7 @@ void ValidateQuery::after_get_mc_state(td::Result> res) { fatal_error("cannot process masterchain state for "s + mc_blkid_.to_str()); return; } + request_latest_mc_state(); if (!pending) { if (!try_validate()) { fatal_error("cannot validate new block"); @@ -596,23 +725,38 @@ void ValidateQuery::after_get_mc_state(td::Result> res) { } } +/** + * Callback function for handling the result of retrieving a masterchain block handle referenced in the block. + * + * @param res The result of retrieving the masterchain block handle. + */ void ValidateQuery::got_mc_handle(td::Result res) { LOG(DEBUG) << "in ValidateQuery::got_mc_handle() for " << mc_blkid_.to_str(); - --pending; if (res.is_error()) { fatal_error(res.move_as_error()); return; } - auto handle = res.move_as_ok(); - if (!handle->inited_proof() && mc_blkid_.seqno()) { - fatal_error(-666, "reference masterchain block "s + mc_blkid_.to_str() + " for block " + id_.to_str() + - " does not have a valid proof"); - return; - } + auto mc_handle = res.move_as_ok(); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state, mc_handle, priority(), timeout, + [self = get_self(), id = id_, mc_handle](td::Result> res) { + LOG(DEBUG) << "got answer to wait_block_state() query for masterchain block"; + if (res.is_ok() && mc_handle->id().seqno() > 0 && !mc_handle->inited_proof()) { + res = td::Status::Error(-666, "reference masterchain block "s + mc_handle->id().to_str() + " for block " + + id.to_str() + " does not have a valid proof"); + } + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_mc_state, std::move(res)); + }); } +/** + * Callback function called after retrieving the shard state for a previous block. + * + * @param idx The index of the previous block (0 or 1). + * @param res The result of the shard state retrieval. + */ void ValidateQuery::after_get_shard_state(int idx, td::Result> res) { - LOG(DEBUG) << "in ValidateQuery::after_get_shard_state(" << idx << ")"; + LOG(WARNING) << "in ValidateQuery::after_get_shard_state(" << idx << ")"; --pending; if (res.is_error()) { fatal_error(res.move_as_error()); @@ -636,6 +780,9 @@ void ValidateQuery::after_get_shard_state(int idx, td::Result> r return; } } + if (is_masterchain()) { + request_latest_mc_state(); + } if (!pending) { if (!try_validate()) { fatal_error("cannot validate new block"); @@ -643,6 +790,13 @@ void ValidateQuery::after_get_shard_state(int idx, td::Result> r } } +/** + * Processes the retreived masterchain state. + * + * @param mc_state The reference to the masterchain state. + * + * @returns True if the masterchain state is successfully processed, false otherwise. + */ bool ValidateQuery::process_mc_state(Ref mc_state) { if (mc_state.is_null()) { return fatal_error("could not obtain reference masterchain state "s + mc_blkid_.to_str()); @@ -664,6 +818,11 @@ bool ValidateQuery::process_mc_state(Ref mc_state) { return register_mc_state(mc_state_); } +/** + * Tries to unpack the masterchain state and perform necessary checks. + * + * @returns True if the unpacking and checks are successful, false otherwise. + */ bool ValidateQuery::try_unpack_mc_state() { LOG(DEBUG) << "unpacking reference masterchain state"; auto guard = error_ctx_add_guard("unpack last mc state"); @@ -679,7 +838,7 @@ bool ValidateQuery::try_unpack_mc_state() { mc_state_root_, block::ConfigInfo::needShardHashes | block::ConfigInfo::needLibraries | block::ConfigInfo::needValidatorSet | block::ConfigInfo::needWorkchainInfo | block::ConfigInfo::needStateExtraRoot | - block::ConfigInfo::needCapabilities | + block::ConfigInfo::needCapabilities | block::ConfigInfo::needPrevBlocks | (is_masterchain() ? block::ConfigInfo::needAccountsRoot | block::ConfigInfo::needSpecialSmc : 0)); if (res.is_error()) { return fatal_error(-666, "cannot extract configuration from reference masterchain state "s + mc_blkid_.to_str() + @@ -742,6 +901,9 @@ bool ValidateQuery::try_unpack_mc_state() { if (!is_masterchain() && !check_this_shard_mc_info()) { return fatal_error("masterchain configuration does not admit creating block "s + id_.to_str()); } + store_out_msg_queue_size_ = config_->has_capability(ton::capStoreOutMsgQueueSize); + msg_metadata_enabled_ = config_->has_capability(ton::capMsgMetadata); + deferring_messages_enabled_ = config_->has_capability(ton::capDeferMessages); } catch (vm::VmError& err) { return fatal_error(-666, err.get_msg()); } catch (vm::VmVirtError& err) { @@ -750,7 +912,12 @@ bool ValidateQuery::try_unpack_mc_state() { return true; } -// almost the same as in Collator +/** + * Fetches and validates configuration parameters from the masterchain configuration. + * Almost the same as in Collator. + * + * @returns True if all configuration parameters were successfully fetched and validated, false otherwise. + */ bool ValidateQuery::fetch_config_params() { old_mparams_ = config_->get_config_param(9); { @@ -782,11 +949,34 @@ bool ValidateQuery::fetch_config_params() { storage_phase_cfg_.delete_due_limit)) { return fatal_error("cannot unpack current gas prices and limits from masterchain configuration"); } + auto mc_gas_prices = config_->get_gas_limits_prices(true); + if (mc_gas_prices.is_error()) { + return fatal_error(mc_gas_prices.move_as_error_prefix("cannot unpack masterchain gas prices and limits: ")); + } + compute_phase_cfg_.mc_gas_prices = mc_gas_prices.move_as_ok(); + compute_phase_cfg_.special_gas_full = config_->get_global_version() >= 5; + storage_phase_cfg_.enable_due_payment = config_->get_global_version() >= 4; + storage_phase_cfg_.global_version = config_->get_global_version(); compute_phase_cfg_.block_rand_seed = rand_seed_; compute_phase_cfg_.libraries = std::make_unique(config_->get_libraries_root(), 256); compute_phase_cfg_.max_vm_data_depth = size_limits.max_vm_data_depth; compute_phase_cfg_.global_config = config_->get_root_cell(); + compute_phase_cfg_.global_version = config_->get_global_version(); + if (compute_phase_cfg_.global_version >= 4) { + auto prev_blocks_info = config_->get_prev_blocks_info(); + if (prev_blocks_info.is_error()) { + return fatal_error(prev_blocks_info.move_as_error_prefix( + "cannot fetch prev blocks info from masterchain configuration: ")); + } + compute_phase_cfg_.prev_blocks_info = prev_blocks_info.move_as_ok(); + } + if (compute_phase_cfg_.global_version >= 6) { + compute_phase_cfg_.unpacked_config_tuple = config_->get_unpacked_config_tuple(now_); + } compute_phase_cfg_.suspended_addresses = config_->get_suspended_addresses(now_); + compute_phase_cfg_.size_limits = size_limits; + compute_phase_cfg_.precompiled_contracts = config_->get_precompiled_contracts_config(); + compute_phase_cfg_.allow_external_unfreeze = compute_phase_cfg_.global_version >= 8; } { // compute action_phase_cfg @@ -808,6 +998,16 @@ bool ValidateQuery::fetch_config_params() { action_phase_cfg_.workchains = &config_->get_workchain_list(); action_phase_cfg_.bounce_msg_body = (config_->has_capability(ton::capBounceMsgBody) ? 256 : 0); action_phase_cfg_.size_limits = size_limits; + action_phase_cfg_.action_fine_enabled = config_->get_global_version() >= 4; + action_phase_cfg_.bounce_on_fail_enabled = config_->get_global_version() >= 4; + action_phase_cfg_.message_skip_enabled = config_->get_global_version() >= 8; + action_phase_cfg_.disable_custom_fess = config_->get_global_version() >= 8; + action_phase_cfg_.reserve_extra_enabled = config_->get_global_version() >= 9; + action_phase_cfg_.mc_blackhole_addr = config_->get_burning_config().blackhole_addr; + action_phase_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; + } + { + serialize_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; } { // fetch block_grams_created @@ -826,7 +1026,16 @@ bool ValidateQuery::fetch_config_params() { return true; } -// almost the same as in Collator +/** + * Checks the previous block against the block registered in the masterchain. + * Almost the same as in Collator. + * + * @param listed The BlockIdExt of the top block of this shard registered in the masterchain. + * @param prev The BlockIdExt of the previous block. + * @param chk_chain_len Flag indicating whether to check the chain length. + * + * @returns True if the previous block is valid, false otherwise. + */ bool ValidateQuery::check_prev_block(const BlockIdExt& listed, const BlockIdExt& prev, bool chk_chain_len) { if (listed.seqno() > prev.seqno()) { return reject_query(PSTRING() << "cannot generate a shardchain block after previous block " << prev.to_str() @@ -846,7 +1055,15 @@ bool ValidateQuery::check_prev_block(const BlockIdExt& listed, const BlockIdExt& return true; } -// almost the same as in Collator +/** + * Checks the previous block against the block registered in the masterchain. + * Almost the same as in Collator + * + * @param listed The BlockIdExt of the top block of this shard registered in the masterchain. + * @param prev The BlockIdExt of the previous block. + * + * @returns True if the previous block is equal to the one registered in the masterchain, false otherwise. + */ bool ValidateQuery::check_prev_block_exact(const BlockIdExt& listed, const BlockIdExt& prev) { if (listed != prev) { return reject_query(PSTRING() << "cannot generate shardchain block for shard " << shard_.to_str() @@ -857,8 +1074,12 @@ bool ValidateQuery::check_prev_block_exact(const BlockIdExt& listed, const Block return true; } -// almost the same as in Collator -// (main change: fatal_error -> reject_query) +/** + * Checks the validity of the shard configuration of the current shard. + * Almost the same as in Collator (main change: fatal_error -> reject_query). + * + * @returns True if the shard's configuration is valid, False otherwise. + */ bool ValidateQuery::check_this_shard_mc_info() { wc_info_ = config_->get_workchain_info(workchain()); if (wc_info_.is_null()) { @@ -1003,6 +1224,11 @@ bool ValidateQuery::check_this_shard_mc_info() { * */ +/** + * Computes the previous shard state. + * + * @returns True if the previous state is computed successfully, false otherwise. + */ bool ValidateQuery::compute_prev_state() { CHECK(prev_states.size() == 1u + after_merge_); // Extend validator timeout if previous block is too old @@ -1032,6 +1258,9 @@ bool ValidateQuery::compute_prev_state() { return true; } +/** + * Computes the next shard state using the previous state and the block's Merkle update. + */ bool ValidateQuery::compute_next_state() { LOG(DEBUG) << "computing next state"; auto res = vm::MerkleUpdate::validate(state_update_); @@ -1107,7 +1336,13 @@ bool ValidateQuery::compute_next_state() { return true; } -// similar to Collator::unpack_merge_last_state() +/** + * Unpacks and merges the states of two previous blocks. + * Used if the block is after_merge. + * Similar to Collator::unpack_merge_last_state() + * + * @returns True if the unpacking and merging was successful, false otherwise. + */ bool ValidateQuery::unpack_merge_prev_state() { LOG(DEBUG) << "unpack/merge previous states"; CHECK(prev_states.size() == 2); @@ -1135,7 +1370,13 @@ bool ValidateQuery::unpack_merge_prev_state() { return true; } -// similar to Collator::unpack_last_state() +/** + * Unpacks the state of the previous block. + * Used if the block is not after_merge. + * Similar to Collator::unpack_last_state() + * + * @returns True if the unpacking is successful, false otherwise. + */ bool ValidateQuery::unpack_prev_state() { LOG(DEBUG) << "unpacking previous state(s)"; CHECK(prev_state_root_.not_null()); @@ -1150,7 +1391,16 @@ bool ValidateQuery::unpack_prev_state() { return unpack_one_prev_state(ps_, prev_blocks.at(0), prev_state_root_) && (!after_split_ || split_prev_state(ps_)); } -// similar to Collator::unpack_one_last_state() +/** + * Unpacks the state of a previous block and performs necessary checks. + * Similar to Collator::unpack_one_last_state() + * + * @param ss The ShardState object to unpack the state into. + * @param blkid The BlockIdExt of the previous block. + * @param prev_state_root The root of the state. + * + * @returns True if the unpacking and checks are successful, false otherwise. + */ bool ValidateQuery::unpack_one_prev_state(block::ShardState& ss, BlockIdExt blkid, Ref prev_state_root) { auto res = ss.unpack_state_ext(blkid, std::move(prev_state_root), global_id_, mc_seqno_, after_split_, after_split_ | after_merge_, [this](ton::BlockSeqno mc_seqno) { @@ -1167,7 +1417,15 @@ bool ValidateQuery::unpack_one_prev_state(block::ShardState& ss, BlockIdExt blki return true; } -// similar to Collator::split_last_state() +/** + * Splits the state of previous block. + * Used if the block is after_split. + * Similar to Collator::split_last_state() + * + * @param ss The ShardState object representing the previous state. The result is stored here. + * + * @returns True if the split operation is successful, false otherwise. + */ bool ValidateQuery::split_prev_state(block::ShardState& ss) { LOG(INFO) << "Splitting previous state " << ss.id_.to_str() << " to subshard " << shard_.to_str(); CHECK(after_split_); @@ -1189,6 +1447,11 @@ bool ValidateQuery::split_prev_state(block::ShardState& ss) { return true; } +/** + * Unpacks the next state (obtained by applying the Merkle update) and performs checks. + * + * @returns True if the next state is successfully unpacked and passes all checks, false otherwise. + */ bool ValidateQuery::unpack_next_state() { LOG(DEBUG) << "unpacking new state"; CHECK(state_root_.not_null()); @@ -1218,7 +1481,12 @@ bool ValidateQuery::unpack_next_state() { return true; } -// almost the same as in Collator +/** + * Requests the message queues of neighboring shards. + * Almost the same as in Collator. + * + * @returns True if the request for neighbor message queues was successful, false otherwise. + */ bool ValidateQuery::request_neighbor_queues() { CHECK(new_shard_conf_); auto neighbor_list = new_shard_conf_->get_neighbor_shard_hash_ids(shard_); @@ -1248,9 +1516,14 @@ bool ValidateQuery::request_neighbor_queues() { return true; } -// almost the same as in Collator +/** + * Handles the result of obtaining the outbound queue for a neighbor. + * Almost the same as in Collator. + * + * @param i The index of the neighbor. + * @param res The obtained outbound queue. + */ void ValidateQuery::got_neighbor_out_queue(int i, td::Result> res) { - LOG(DEBUG) << "obtained outbound queue for neighbor #" << i; --pending; if (res.is_error()) { fatal_error(res.move_as_error()); @@ -1258,6 +1531,7 @@ void ValidateQuery::got_neighbor_out_queue(int i, td::Result> } Ref outq_descr = res.move_as_ok(); block::McShardDescr& descr = neighbors_.at(i); + LOG(WARNING) << "obtained outbound queue for neighbor #" << i << " : " << descr.shard().to_str(); if (outq_descr->get_block_id() != descr.blk_) { LOG(DEBUG) << "outq_descr->id = " << outq_descr->get_block_id().to_str() << " ; descr.id = " << descr.blk_.to_str(); fatal_error( @@ -1283,8 +1557,10 @@ void ValidateQuery::got_neighbor_out_queue(int i, td::Result> // unpack ProcessedUpto LOG(DEBUG) << "unpacking ProcessedUpto of neighbor " << descr.blk_.to_str(); if (verbosity >= 2) { - block::gen::t_ProcessedInfo.print(std::cerr, qinfo.proc_info); - qinfo.proc_info->print_rec(std::cerr); + FLOG(INFO) { + block::gen::t_ProcessedInfo.print(sb, qinfo.proc_info); + qinfo.proc_info->print_rec(sb); + }; } descr.processed_upto = block::MsgProcessedUptoCollection::unpack(descr.shard(), qinfo.proc_info); if (!descr.processed_upto) { @@ -1310,7 +1586,14 @@ void ValidateQuery::got_neighbor_out_queue(int i, td::Result> } } -// almost the same as in Collator +/** + * Registers a masterchain state. + * Almost the same as in Collator. + * + * @param other_mc_state The masterchain state to register. + * + * @returns True if the registration is successful, false otherwise. + */ bool ValidateQuery::register_mc_state(Ref other_mc_state) { if (other_mc_state.is_null() || mc_state_.is_null()) { return false; @@ -1336,7 +1619,15 @@ bool ValidateQuery::register_mc_state(Ref other_mc_state) { return true; } -// almost the same as in Collator +/** + * Requests the auxiliary masterchain state. + * Almost the same as in Collator + * + * @param seqno The seqno of the block. + * @param state A reference to the auxiliary masterchain state. + * + * @returns True if the auxiliary masterchain state is successfully requested, false otherwise. + */ bool ValidateQuery::request_aux_mc_state(BlockSeqno seqno, Ref& state) { if (mc_state_.is_null()) { return fatal_error(PSTRING() << "cannot find masterchain block with seqno " << seqno @@ -1370,7 +1661,14 @@ bool ValidateQuery::request_aux_mc_state(BlockSeqno seqno, Ref ValidateQuery::get_aux_mc_state(BlockSeqno seqno) const { auto it = aux_mc_states_.find(seqno); if (it != aux_mc_states_.end()) { @@ -1380,7 +1678,14 @@ Ref ValidateQuery::get_aux_mc_state(BlockSeqno seqno) const { } } -// almost the same as in Collator +/** + * Callback function called after retrieving the auxiliary shard state. + * Handles the retrieved shard state and performs necessary checks and registrations. + * Almost the same as in Collator. + * + * @param blkid The BlockIdExt of the shard state. + * @param res The result of retrieving the shard state. + */ void ValidateQuery::after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result> res) { LOG(DEBUG) << "in ValidateQuery::after_get_aux_shard_state(" << blkid.to_str() << ")"; --pending; @@ -1407,6 +1712,17 @@ void ValidateQuery::after_get_aux_shard_state(ton::BlockIdExt blkid, td::Result< } // similar to Collator::update_one_shard() +/** + * Checks one shard description in the masterchain shard configuration. + * Used in masterchain validation. + * + * @param info The shard information to be updated. + * @param sibling The sibling shard information. + * @param wc_info The workchain information. + * @param ccvc The Catchain validators configuration. + * + * @returns True if the validation wasa successful, false otherwise. + */ bool ValidateQuery::check_one_shard(const block::McShardHash& info, const block::McShardHash* sibling, const block::WorkchainInfo* wc_info, const block::CatchainValidatorsConfig& ccvc) { auto shard = info.shard(); @@ -1693,8 +2009,14 @@ bool ValidateQuery::check_one_shard(const block::McShardHash& info, const block: return true; } -// checks old_shard_conf_ -> new_shard_conf_ transition using top_shard_descr_dict_ from collated data -// similar to Collator::update_shard_config() +/** + * Checks the shard configuration in the masterchain. + * Used in masterchain collator. + * Checks old_shard_conf_ -> new_shard_conf_ transition using top_shard_descr_dict_ from collated data. + * Similar to Collator::update_shard_config() + * + * @returns True if the shard layout is valid, false otherwise. + */ bool ValidateQuery::check_shard_layout() { prev_now_ = config_->utime; if (prev_now_ > now_) { @@ -1748,7 +2070,14 @@ bool ValidateQuery::check_shard_layout() { return check_mc_validator_info(is_key_block_ || (now_ / ccvc.mc_cc_lifetime > prev_now_ / ccvc.mc_cc_lifetime)); } -// similar to Collator::register_shard_block_creators +/** + * Registers the shard block creators to block_create_count_ + * Similar to Collator::register_shard_block_creators + * + * @param creator_list A vector of Bits256 representing the shard block creators. + * + * @returns True if the registration was successful, False otherwise. + */ bool ValidateQuery::register_shard_block_creators(std::vector creator_list) { for (const auto& x : creator_list) { LOG(DEBUG) << "registering block creator " << x.to_hex(); @@ -1763,7 +2092,12 @@ bool ValidateQuery::register_shard_block_creators(std::vector creat return true; } -// similar to Collator::check_cur_validator_set() +/** + * Checks that the current validator set is entitled to create blocks in this shard and has a correct catchain seqno. + * Similar to Collator::check_cur_validator_set() + * + * @returns True if the current validator set is valid, false otherwise. + */ bool ValidateQuery::check_cur_validator_set() { CatchainSeqno cc_seqno = 0; auto nodes = config_->compute_validator_set_cc(shard_, now_, &cc_seqno); @@ -1786,8 +2120,14 @@ bool ValidateQuery::check_cur_validator_set() { return true; } -// parallel to 4. of Collator::create_mc_state_extra() -// checks validator_info in mc_state_extra +/** + * Checks validator_info in mc_state_extra. + * NB: could be run in parallel to 4. of Collator::create_mc_state_extra() + * + * @param update_mc_cc Flag indicating whether the masterchain catchain seqno should be updated. + * + * @returns True if the validator information is valid, false otherwise. + */ bool ValidateQuery::check_mc_validator_info(bool update_mc_cc) { block::gen::McStateExtra::Record old_state_extra; block::gen::ValidatorInfo::Record old_val_info; @@ -1829,6 +2169,11 @@ bool ValidateQuery::check_mc_validator_info(bool update_mc_cc) { return true; } +/** + * Checks if the Unix time and logical time of the block are valid. + * + * @returns True if the utime and logical time pass checks, False otherwise. + */ bool ValidateQuery::check_utime_lt() { if (start_lt_ <= ps_.lt_) { return reject_query(PSTRING() << "block has start_lt " << start_lt_ << " less than or equal to lt " << ps_.lt_ @@ -1869,14 +2214,66 @@ bool ValidateQuery::check_utime_lt() { return true; } +/** + * Reads the size of the outbound message queue from the previous state(s), or requests it if needed. + * + * @returns True if the request was successful, false otherwise. + */ +bool ValidateQuery::prepare_out_msg_queue_size() { + if (ps_.out_msg_queue_size_) { + // if after_split then out_msg_queue_size is always present, since it is calculated during split + old_out_msg_queue_size_ = ps_.out_msg_queue_size_.value(); + return true; + } + old_out_msg_queue_size_ = 0; + for (size_t i = 0; i < prev_blocks.size(); ++i) { + ++pending; + send_closure_later(manager, &ValidatorManager::get_out_msg_queue_size, prev_blocks[i], + [self = get_self(), i](td::Result res) { + td::actor::send_closure(std::move(self), &ValidateQuery::got_out_queue_size, i, + std::move(res)); + }); + } + return true; +} + +/** + * Handles the result of obtaining the size of the outbound message queue. + * + * If the block is after merge then the two sizes are added. + * + * @param i The index of the previous block (0 or 1). + * @param res The result object containing the size of the queue. + */ +void ValidateQuery::got_out_queue_size(size_t i, td::Result res) { + --pending; + if (res.is_error()) { + fatal_error( + res.move_as_error_prefix(PSTRING() << "failed to get message queue size from prev block #" << i << ": ")); + return; + } + td::uint64 size = res.move_as_ok(); + LOG(DEBUG) << "got outbound queue size from prev block #" << i << ": " << size; + old_out_msg_queue_size_ += size; + try_validate(); +} + /* * * METHODS CALLED FROM try_validate() stage 1 * */ -// almost the same as in Collator -// (but it can take into account the new state of the masterchain) +/** + * Adjusts one entry from the processed up to information using the masterchain state that is referenced in the entry. + * Almost the same as in Collator (but it can take into account the new state of the masterchain). + * + * @param proc The MsgProcessedUpto object. + * @param owner The shard that the MsgProcessesUpto information is taken from. + * @param allow_cur Allow using the new state of the msaterchain. + * + * @returns True if the processed up to information was successfully adjusted, false otherwise. + */ bool ValidateQuery::fix_one_processed_upto(block::MsgProcessedUpto& proc, ton::ShardIdFull owner, bool allow_cur) { if (proc.compute_shard_end_lt) { return true; @@ -1899,7 +2296,15 @@ bool ValidateQuery::fix_one_processed_upto(block::MsgProcessedUpto& proc, ton::S return (bool)proc.compute_shard_end_lt; } -// almost the same as in Collator +/** + * Adjusts the processed up to collection using the using the auxilliary masterchain states. + * Almost the same as in Collator. + * + * @param upto The MsgProcessedUptoCollection to be adjusted. + * @param allow_cur Allow using the new state of the msaterchain. + * + * @returns True if all entries were successfully adjusted, False otherwise. + */ bool ValidateQuery::fix_processed_upto(block::MsgProcessedUptoCollection& upto, bool allow_cur) { for (auto& entry : upto.list) { if (!fix_one_processed_upto(entry, upto.owner, allow_cur)) { @@ -1909,6 +2314,11 @@ bool ValidateQuery::fix_processed_upto(block::MsgProcessedUptoCollection& upto, return true; } +/** + * Adjusts the processed_upto values for all shard states, including neighbors. + * + * @returns True if all processed_upto values were successfully adjusted, false otherwise. + */ bool ValidateQuery::fix_all_processed_upto() { CHECK(ps_.processed_upto_); if (!fix_processed_upto(*ps_.processed_upto_)) { @@ -1929,7 +2339,13 @@ bool ValidateQuery::fix_all_processed_upto() { return true; } -// almost the same as in Collator +/** + * Adds trivials neighbor after merging two shards. + * Trivial neighbors are the two previous blocks. + * Almost the same as in Collator. + * + * @returns True if the operation is successful, false otherwise. + */ bool ValidateQuery::add_trivial_neighbor_after_merge() { LOG(DEBUG) << "in add_trivial_neighbor_after_merge()"; CHECK(prev_blocks.size() == 2); @@ -1964,7 +2380,13 @@ bool ValidateQuery::add_trivial_neighbor_after_merge() { return true; } -// almost the same as in Collator +/** + * Adds a trivial neighbor. + * A trivial neighbor is the previous block. + * Almost the same as in Collator. + * + * @returns True if the operation is successful, false otherwise. + */ bool ValidateQuery::add_trivial_neighbor() { LOG(DEBUG) << "in add_trivial_neighbor()"; if (after_merge_) { @@ -2100,6 +2522,11 @@ bool ValidateQuery::add_trivial_neighbor() { return true; } +/** + * Unpacks block data and performs validation checks. + * + * @returns True if the block data is successfully unpacked and passes all validation checks, false otherwise. + */ bool ValidateQuery::unpack_block_data() { LOG(DEBUG) << "unpacking block structures"; block::gen::Block::Record blk; @@ -2111,13 +2538,13 @@ bool ValidateQuery::unpack_block_data() { auto outmsg_cs = vm::load_cell_slice_ref(std::move(extra.out_msg_descr)); // run some hand-written checks from block::tlb:: // (automatic tests from block::gen:: have been already run for the entire block) - if (!block::tlb::t_InMsgDescr.validate_upto(1000000, *inmsg_cs)) { + if (!block::tlb::t_InMsgDescr.validate_upto(10000000, *inmsg_cs)) { return reject_query("InMsgDescr of the new block failed to pass handwritten validity tests"); } - if (!block::tlb::t_OutMsgDescr.validate_upto(1000000, *outmsg_cs)) { + if (!block::tlb::t_OutMsgDescr.validate_upto(10000000, *outmsg_cs)) { return reject_query("OutMsgDescr of the new block failed to pass handwritten validity tests"); } - if (!block::tlb::t_ShardAccountBlocks.validate_ref(1000000, extra.account_blocks)) { + if (!block::tlb::t_ShardAccountBlocks.validate_ref(10000000, extra.account_blocks)) { return reject_query("ShardAccountBlocks of the new block failed to pass handwritten validity tests"); } in_msg_dict_ = std::make_unique(std::move(inmsg_cs), 256, block::tlb::aug_InMsgDescr); @@ -2139,6 +2566,13 @@ bool ValidateQuery::unpack_block_data() { return unpack_precheck_value_flow(std::move(blk.value_flow)); } +/** + * Validates and unpacks the value flow of a new block. + * + * @param value_flow_root The root of the value flow to be unpacked and validated. + * + * @returns True if the value flow is valid and unpacked successfully, false otherwise. + */ bool ValidateQuery::unpack_precheck_value_flow(Ref value_flow_root) { vm::CellSlice cs{vm::NoVmOrd(), value_flow_root}; if (!(cs.is_valid() && value_flow_.fetch(cs) && cs.empty_ext())) { @@ -2161,6 +2595,11 @@ bool ValidateQuery::unpack_precheck_value_flow(Ref value_flow_root) { return reject_query("ValueFlow of block "s + id_.to_str() + " is invalid (non-zero recovered value in a non-masterchain block)"); } + if (!is_masterchain() && !value_flow_.burned.is_zero()) { + LOG(INFO) << "invalid value flow: " << os.str(); + return reject_query("ValueFlow of block "s + id_.to_str() + + " is invalid (non-zero burned value in a non-masterchain block)"); + } if (!value_flow_.recovered.is_zero() && recover_create_msg_.is_null()) { return reject_query("ValueFlow of block "s + id_.to_str() + " has a non-zero recovered fees value, but there is no recovery InMsg"); @@ -2223,7 +2662,6 @@ bool ValidateQuery::unpack_precheck_value_flow(Ref value_flow_root) { " but the sum over all accounts present in the new state is " + cc.to_str()); } auto msg_extra = in_msg_dict_->get_root_extra(); - // block::gen::t_ImportFees.print(std::cerr, msg_extra); if (!(block::tlb::t_Grams.as_integer_skip_to(msg_extra.write(), import_fees_) && cc.unpack(std::move(msg_extra)))) { return reject_query("cannot unpack ImportFees from the augmentation of the InMsgDescr dictionary"); } @@ -2243,20 +2681,22 @@ bool ValidateQuery::unpack_precheck_value_flow(Ref value_flow_root) { "cannot unpack CurrencyCollection with total transaction fees from the augmentation of the ShardAccountBlocks " "dictionary"); } - auto expected_fees = value_flow_.fees_imported + value_flow_.created + transaction_fees_ + import_fees_; - if (value_flow_.fees_collected != expected_fees) { - return reject_query(PSTRING() << "ValueFlow for " << id_.to_str() << " declares fees_collected=" - << value_flow_.fees_collected.to_str() << " but the total message import fees are " - << import_fees_ << ", the total transaction fees are " << transaction_fees_.to_str() - << ", creation fee for this block is " << value_flow_.created.to_str() - << " and the total imported fees from shards are " - << value_flow_.fees_imported.to_str() << " with a total of " - << expected_fees.to_str()); + if (is_masterchain()) { + auto x = config_->get_burning_config().calculate_burned_fees(transaction_fees_ + import_fees_); + fees_burned_ += x; + total_burned_ += x; } return true; } -// similar to Collator::compute_minted_amount() +/** + * Computes the amount of extra currencies to be minted. + * Similar to Collator::compute_minted_amount() + * + * @param to_mint A reference to the CurrencyCollection object to store the minted amount. + * + * @returns True if the computation is successful, false otherwise. + */ bool ValidateQuery::compute_minted_amount(block::CurrencyCollection& to_mint) { if (!is_masterchain()) { return to_mint.set_zero(); @@ -2308,6 +2748,15 @@ bool ValidateQuery::compute_minted_amount(block::CurrencyCollection& to_mint) { return true; } +/** + * Pre-validates the update of an account in a query. + * + * @param acc_id The 256-bit account address. + * @param old_value The old value of the account serialized as ShardAccount. Can be null. + * @param new_value The new value of the account serialized as ShardAccount. Can be null. + * + * @returns True if the accounts passes preliminary checks, false otherwise. + */ bool ValidateQuery::precheck_one_account_update(td::ConstBitPtr acc_id, Ref old_value, Ref new_value) { LOG(DEBUG) << "checking update of account " << acc_id.to_hex(256); @@ -2316,20 +2765,22 @@ bool ValidateQuery::precheck_one_account_update(td::ConstBitPtr acc_id, Reflookup(acc_id, 256); if (acc_blk_root.is_null()) { if (verbosity >= 3 * 0) { - std::cerr << "state of account " << workchain() << ":" << acc_id.to_hex(256) - << " in the old shardchain state:" << std::endl; - if (old_value.not_null()) { - block::gen::t_ShardAccount.print(std::cerr, *old_value); - } else { - std::cerr << "" << std::endl; - } - std::cerr << "state of account " << workchain() << ":" << acc_id.to_hex(256) - << " in the new shardchain state:" << std::endl; - if (new_value.not_null()) { - block::gen::t_ShardAccount.print(std::cerr, *new_value); - } else { - std::cerr << "" << std::endl; - } + FLOG(INFO) { + sb << "state of account " << workchain() << ":" << acc_id.to_hex(256) + << " in the old shardchain state:" << "\n"; + if (old_value.not_null()) { + block::gen::t_ShardAccount.print(sb, old_value); + } else { + sb << "" << "\n"; + } + sb << "state of account " << workchain() << ":" << acc_id.to_hex(256) + << " in the new shardchain state:" << "\n"; + if (new_value.not_null()) { + block::gen::t_ShardAccount.print(sb, new_value); + } else { + sb << "" << "\n"; + } + }; } return reject_query("the state of account "s + acc_id.to_hex(256) + " changed in the new state with respect to the old state, but the block contains no " @@ -2367,6 +2818,11 @@ bool ValidateQuery::precheck_one_account_update(td::ConstBitPtr acc_id, Ref trans_csr, ton::Bits256& prev_trans_hash, ton::LogicalTime& prev_trans_lt, unsigned& prev_trans_lt_len, @@ -2454,6 +2923,14 @@ bool ValidateQuery::precheck_one_transaction(td::ConstBitPtr acc_id, ton::Logica } // NB: could be run in parallel for different accounts +/** + * Pre-validates an AccountBlock and all transactions in it. + * + * @param acc_id The 256-bit account address. + * @param acc_blk_root The root of the AccountBlock. + * + * @returns True if the AccountBlock passes pre-checks, false otherwise. + */ bool ValidateQuery::precheck_one_account_block(td::ConstBitPtr acc_id, Ref acc_blk_root) { LOG(DEBUG) << "checking AccountBlock for " << acc_id.to_hex(256); if (!acc_id.equals(shard_pfx_.bits(), shard_pfx_len_)) { @@ -2461,8 +2938,6 @@ bool ValidateQuery::precheck_one_account_block(td::ConstBitPtr acc_id, Refprint_rec(std::cerr); - // block::gen::t_AccountBlock.print(std::cerr, acc_blk_root); block::gen::AccountBlock::Record acc_blk; block::gen::HASH_UPDATE::Record hash_upd; if (!(tlb::csr_unpack(acc_blk_root, acc_blk) && @@ -2535,6 +3010,11 @@ bool ValidateQuery::precheck_one_account_block(td::ConstBitPtr acc_id, Ref ValidateQuery::lookup_transaction(const ton::StdSmcAddress& addr, ton::LogicalTime lt) const { CHECK(account_blocks_dict_); block::gen::AccountBlock::Record ab_rec; @@ -2565,7 +3053,13 @@ Ref ValidateQuery::lookup_transaction(const ton::StdSmcAddress& addr, return trans_dict.lookup_ref(td::BitArray<64>{(long long)lt}); } -// checks that a ^Transaction refers to a transaction present in the ShardAccountBlocks +/** + * Checks that a Transaction cell refers to a transaction present in the ShardAccountBlocks. + * + * @param trans_ref The reference to the serialized transaction root. + * + * @returns True if the transaction reference is valid, False otherwise. + */ bool ValidateQuery::is_valid_transaction_ref(Ref trans_ref) const { ton::StdSmcAddress addr; ton::LogicalTime lt; @@ -2585,8 +3079,16 @@ bool ValidateQuery::is_valid_transaction_ref(Ref trans_ref) const { return true; } -// checks that any change in OutMsgQueue in the state is accompanied by an OutMsgDescr record in the block -// also checks that the keys are correct +/** + * Checks that any change in OutMsgQueue in the state is accompanied by an OutMsgDescr record in the block. + * Also checks that the keys are correct. + * + * @param out_msg_id The 32+64+256-bit ID of the outbound message. + * @param old_value The old value of the message queue entry. + * @param new_value The new value of the message queue entry. + * + * @returns True if the update is valid, false otherwise. + */ bool ValidateQuery::precheck_one_message_queue_update(td::ConstBitPtr out_msg_id, Ref old_value, Ref new_value) { LOG(DEBUG) << "checking update of enqueued outbound message " << out_msg_id.get_int(32) << ":" @@ -2601,6 +3103,7 @@ bool ValidateQuery::precheck_one_message_queue_update(td::ConstBitPtr out_msg_id return reject_query("new EnqueuedMsg with key "s + out_msg_id.to_hex(352) + " is invalid"); } if (new_value.not_null()) { + ++new_out_msg_queue_size_; if (!block::gen::t_EnqueuedMsg.validate_csr(new_value)) { return reject_query("new EnqueuedMsg with key "s + out_msg_id.to_hex(352) + " failed to pass automated validity checks"); @@ -2617,6 +3120,7 @@ bool ValidateQuery::precheck_one_message_queue_update(td::ConstBitPtr out_msg_id } } if (old_value.not_null()) { + --new_out_msg_queue_size_; if (!block::gen::t_EnqueuedMsg.validate_csr(old_value)) { return reject_query("old EnqueuedMsg with key "s + out_msg_id.to_hex(352) + " failed to pass automated validity checks"); @@ -2643,11 +3147,18 @@ bool ValidateQuery::precheck_one_message_queue_update(td::ConstBitPtr out_msg_id " has been changed in the OutMsgQueue, but the key did not change"); } auto q_msg_env = (old_value.not_null() ? old_value : new_value)->prefetch_ref(); - int tag = (int)out_msg_cs->prefetch_ulong(3); - // mode for msg_export_{ext,new,imm,tr,deq_imm,???,deq/deq_short,tr_req} - static const int tag_mode[8] = {0, 2, 0, 2, 1, 0, 1, 3}; - static const char* tag_str[8] = {"ext", "new", "imm", "tr", "deq_imm", "???", "deq", "tr_req"}; - if (tag < 0 || tag >= 8 || !(tag_mode[tag] & mode)) { + int tag = block::tlb::t_OutMsg.get_tag(*out_msg_cs); + if (tag == 12 || tag == 13) { + tag /= 2; + } else if (tag == 20) { + tag = 8; + } else if (tag == 21) { + tag = 9; + } + // mode for msg_export_{ext,new,imm,tr,deq_imm,???,deq/deq_short,tr_req,new_defer,deferred_tr} + static const int tag_mode[10] = {0, 2, 0, 2, 1, 0, 1, 3, 0, 2}; + static const char* tag_str[10] = {"ext", "new", "imm", "tr", "deq_imm", "???", "deq", "tr_req", "new_defer", "deferred_tr"}; + if (tag < 0 || tag >= 10 || !(tag_mode[tag] & mode)) { return reject_query(PSTRING() << "OutMsgDescr corresponding to " << m_str[mode] << "queued message with key " << out_msg_id.to_hex(352) << " has invalid tag " << tag << "(" << tag_str[tag & 7] << ")"); @@ -2752,11 +3263,17 @@ bool ValidateQuery::precheck_one_message_queue_update(td::ConstBitPtr out_msg_id return true; } +/** + * Performs a pre-check on the difference between the old and new outbound message queues. + * + * @returns True if the pre-check is successful, false otherwise. + */ bool ValidateQuery::precheck_message_queue_update() { LOG(INFO) << "pre-checking the difference between the old and the new outbound message queues"; try { CHECK(ps_.out_msg_queue_ && ns_.out_msg_queue_); CHECK(out_msg_dict_); + new_out_msg_queue_size_ = old_out_msg_queue_size_; if (!ps_.out_msg_queue_->scan_diff( *ns_.out_msg_queue_, [this](td::ConstBitPtr key, int key_len, Ref old_val_extra, @@ -2771,9 +3288,197 @@ bool ValidateQuery::precheck_message_queue_update() { return reject_query("invalid OutMsgQueue dictionary difference between the old and the new state: "s + err.get_msg()); } + LOG(INFO) << "outbound message queue size: " << old_out_msg_queue_size_ << " -> " << new_out_msg_queue_size_; + if (store_out_msg_queue_size_) { + if (!ns_.out_msg_queue_size_) { + return reject_query(PSTRING() << "outbound message queue size in the new state is not correct (expected: " + << new_out_msg_queue_size_ << ", found: none)"); + } + if (ns_.out_msg_queue_size_.value() != new_out_msg_queue_size_) { + return reject_query(PSTRING() << "outbound message queue size in the new state is not correct (expected: " + << new_out_msg_queue_size_ << ", found: " << ns_.out_msg_queue_size_.value() + << ")"); + } + } else { + if (ns_.out_msg_queue_size_) { + return reject_query("outbound message queue size in the new state is present, but shouldn't"); + } + } return true; } +/** + * Performs a check on the difference between the old and new dispatch queues for one account. + * + * @param addr The 256-bit address of the account. + * @param old_queue_csr The old value of the account dispatch queue. + * @param new_queue_csr The new value of the account dispatch queue. + * + * @returns True if the check is successful, false otherwise. + */ +bool ValidateQuery::check_account_dispatch_queue_update(td::Bits256 addr, Ref old_queue_csr, + Ref new_queue_csr) { + vm::Dictionary old_dict{64}; + td::uint64 old_dict_size = 0; + if (!block::unpack_account_dispatch_queue(old_queue_csr, old_dict, old_dict_size)) { + return reject_query(PSTRING() << "invalid AccountDispatchQueue for " << addr.to_hex() << " in the old state"); + } + vm::Dictionary new_dict{64}; + td::uint64 new_dict_size = 0; + if (!block::unpack_account_dispatch_queue(new_queue_csr, new_dict, new_dict_size)) { + return reject_query(PSTRING() << "invalid AccountDispatchQueue for " << addr.to_hex() << " in the new state"); + } + td::uint64 expected_dict_size = old_dict_size; + LogicalTime max_removed_lt = 0; + LogicalTime min_added_lt = (LogicalTime)-1; + bool res = old_dict.scan_diff( + new_dict, [&](td::ConstBitPtr key, int key_len, Ref old_val, Ref new_val) { + CHECK(key_len == 64); + CHECK(old_val.not_null() || new_val.not_null()); + if (old_val.not_null() && new_val.not_null()) { + return false; + } + td::uint64 lt = key.get_uint(64); + block::gen::EnqueuedMsg::Record rec; + if (old_val.not_null()) { + LOG(DEBUG) << "removed message from DispatchQueue: account=" << addr.to_hex() << ", lt=" << lt; + --expected_dict_size; + if (!block::tlb::csr_unpack(old_val, rec)) { + return reject_query(PSTRING() << "invalid EnqueuedMsg in AccountDispatchQueue for " << addr.to_hex()); + } + } else { + LOG(DEBUG) << "added message to DispatchQueue: account=" << addr.to_hex() << ", lt=" << lt; + ++expected_dict_size; + if (!block::tlb::csr_unpack(new_val, rec)) { + return reject_query(PSTRING() << "invalid EnqueuedMsg in AccountDispatchQueue for " << addr.to_hex()); + } + if (is_masterchain() && config_->is_special_smartcontract(addr)) { + return reject_query(PSTRING() << "cannot defer message from a special account -1:" << addr.to_hex()); + } + } + if (lt != rec.enqueued_lt) { + return reject_query(PSTRING() << "invalid EnqueuedMsg in AccountDispatchQueue for " << addr.to_hex() + << ": lt mismatch (" << lt << " != " << rec.enqueued_lt << ")"); + } + block::tlb::MsgEnvelope::Record_std env; + if (!block::gen::t_MsgEnvelope.validate_ref(rec.out_msg) || !block::tlb::unpack_cell(rec.out_msg, env)) { + return reject_query(PSTRING() << "invalid EnqueuedMsg in AccountDispatchQueue for " << addr.to_hex()); + } + if (env.emitted_lt) { + return reject_query(PSTRING() << "invalid EnqueuedMsg in AccountDispatchQueue for " << addr.to_hex() + << ", lt=" << lt << ": unexpected emitted_lt"); + } + unsigned long long created_lt; + vm::CellSlice msg_cs = vm::load_cell_slice(env.msg); + if (!block::tlb::t_Message.get_created_lt(msg_cs, created_lt)) { + return reject_query(PSTRING() << "invalid EnqueuedMsg in AccountDispatchQueue for " << addr.to_hex() + << ": cannot get created_lt"); + } + if (lt != created_lt) { + return reject_query(PSTRING() << "invalid EnqueuedMsg in AccountDispatchQueue for " << addr.to_hex() + << ": lt mismatch (" << lt << " != " << created_lt << ")"); + } + if (old_val.not_null()) { + removed_dispatch_queue_messages_[{addr, lt}] = rec.out_msg; + max_removed_lt = std::max(max_removed_lt, lt); + } else { + new_dispatch_queue_messages_[{addr, lt}] = rec.out_msg; + min_added_lt = std::min(min_added_lt, lt); + } + return true; + }); + if (!res) { + return reject_query(PSTRING() << "invalid AccountDispatchQueue diff for account " << addr.to_hex()); + } + if (expected_dict_size != new_dict_size) { + return reject_query(PSTRING() << "invalid count in AccountDispatchQuery for " << addr.to_hex() + << ": expected=" << expected_dict_size << ", found=" << new_dict_size); + } + if (!new_dict.is_empty()) { + td::BitArray<64> new_min_lt; + CHECK(new_dict.get_minmax_key(new_min_lt).not_null()); + if (new_min_lt.to_ulong() <= max_removed_lt) { + return reject_query(PSTRING() << "invalid AccountDispatchQuery update for " << addr.to_hex() + << ": max removed lt is " << max_removed_lt << ", but lt=" << new_min_lt.to_ulong() + << " is still in queue"); + } + } + if (!old_dict.is_empty()) { + td::BitArray<64> old_max_lt; + CHECK(old_dict.get_minmax_key(old_max_lt, true).not_null()); + if (old_max_lt.to_ulong() >= min_added_lt) { + return reject_query(PSTRING() << "invalid AccountDispatchQuery update for " << addr.to_hex() + << ": min added lt is " << min_added_lt << ", but lt=" << old_max_lt.to_ulong() + << " was present in the queue"); + } + if (max_removed_lt != old_max_lt.to_ulong()) { + // Some old messages are still in DispatchQueue, meaning that all new messages from this account must be deferred + account_expected_defer_all_messages_.insert(addr); + } + } + if (old_dict_size > 0 && max_removed_lt != 0) { + ++processed_account_dispatch_queues_; + } + return true; +} + +/** + * Pre-check the difference between the old and new dispatch queues and put the difference to + * new_dispatch_queue_messages_, old_dispatch_queue_messages_ + * + * @returns True if the pre-check and unpack is successful, false otherwise. + */ +bool ValidateQuery::unpack_dispatch_queue_update() { + LOG(INFO) << "checking the difference between the old and the new dispatch queues"; + try { + CHECK(ps_.dispatch_queue_ && ns_.dispatch_queue_); + CHECK(out_msg_dict_); + bool res = ps_.dispatch_queue_->scan_diff( + *ns_.dispatch_queue_, + [this](td::ConstBitPtr key, int key_len, Ref old_val_extra, Ref new_val_extra) { + CHECK(key_len == 256); + return check_account_dispatch_queue_update(key, ps_.dispatch_queue_->extract_value(std::move(old_val_extra)), + ns_.dispatch_queue_->extract_value(std::move(new_val_extra))); + }, + 3 /* check augmentation of changed nodes */); + if (!res) { + return reject_query("invalid DispatchQueue dictionary in the new state"); + } + + if (old_out_msg_queue_size_ <= compute_phase_cfg_.size_limits.defer_out_queue_size_limit) { + // Check that at least one message was taken from each AccountDispatchQueue + try { + have_unprocessed_account_dispatch_queue_ = false; + td::uint64 total_account_dispatch_queues = 0; + ps_.dispatch_queue_->check_for_each([&](Ref, td::ConstBitPtr, int n) -> bool { + ++total_account_dispatch_queues; + if (total_account_dispatch_queues > processed_account_dispatch_queues_) { + return false; + } + return true; + }); + have_unprocessed_account_dispatch_queue_ = + (total_account_dispatch_queues != processed_account_dispatch_queues_); + } catch (vm::VmVirtError&) { + // VmVirtError can happen if we have only a proof of ShardState + have_unprocessed_account_dispatch_queue_ = true; + } + } + } catch (vm::VmError& err) { + return reject_query("invalid DispatchQueue dictionary difference between the old and the new state: "s + + err.get_msg()); + } + return true; +} + +/** + * Updates the maximum processed logical time and hash value. + * + * @param lt The logical time to compare against the current maximum processed logical time. + * @param hash The hash value to compare against the current maximum processed hash value. + * + * @returns True if the update was successful, false otherwise. + */ bool ValidateQuery::update_max_processed_lt_hash(ton::LogicalTime lt, const ton::Bits256& hash) { if (proc_lt_ < lt || (proc_lt_ == lt && proc_hash_ < hash)) { proc_lt_ = lt; @@ -2782,6 +3487,14 @@ bool ValidateQuery::update_max_processed_lt_hash(ton::LogicalTime lt, const ton: return true; } +/** + * Updates the minimum enqueued logical time and hash values. + * + * @param lt The logical time to compare. + * @param hash The hash value to compare. + * + * @returns True if the update was successful, false otherwise. + */ bool ValidateQuery::update_min_enqueued_lt_hash(ton::LogicalTime lt, const ton::Bits256& hash) { if (lt < min_enq_lt_ || (lt == min_enq_lt_ && hash < min_enq_hash_)) { min_enq_lt_ = lt; @@ -2790,7 +3503,13 @@ bool ValidateQuery::update_min_enqueued_lt_hash(ton::LogicalTime lt, const ton:: return true; } -// check that the enveloped message (MsgEnvelope) was present in the output queue of a neighbor, and that it has not been processed before +/** + * Checks that the MsgEnvelope was present in the output queue of a neighbor, and that it has not been processed before. + * + * @param msg_env The message envelope of the imported message. + * + * @returns True if the imported internal message passes checks, false otherwise. + */ bool ValidateQuery::check_imported_message(Ref msg_env) { block::tlb::MsgEnvelope::Record_std env; block::gen::CommonMsgInfo::Record_int_msg_info info; @@ -2850,18 +3569,34 @@ bool ValidateQuery::check_imported_message(Ref msg_env) { " has previous address not belonging to any neighbor"); } +/** + * Checks if the given input message is a special message. + * A message is considered special if it recovers fees or mints extra currencies. + * + * @param in_msg The input message to be checked. + * + * @returns True if the input message is special, False otherwise. + */ bool ValidateQuery::is_special_in_msg(const vm::CellSlice& in_msg) const { return (recover_create_msg_.not_null() && vm::load_cell_slice(recover_create_msg_).contents_equal(in_msg)) || (mint_msg_.not_null() && vm::load_cell_slice(mint_msg_).contents_equal(in_msg)); } +/** + * Checks the validity of an inbound message listed in InMsgDescr. + * + * @param key The 256-bit key of the inbound message. + * @param in_msg The inbound message to be checked serialized using InMsg TLB-scheme. + * + * @returns True if the inbound message is valid, false otherwise. + */ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) { LOG(DEBUG) << "checking InMsg with key " << key.to_hex(256); CHECK(in_msg.not_null()); int tag = block::gen::t_InMsg.get_tag(*in_msg); CHECK(tag >= 0); // NB: the block has been already checked to be valid TL-B in try_validate() - ton::StdSmcAddress addr; - ton::WorkchainId wc; + ton::StdSmcAddress src_addr, dest_addr; + ton::WorkchainId src_wc, dest_wc; Ref src, dest; Ref transaction; Ref msg, msg_env, tr_msg_env; @@ -2874,6 +3609,7 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) block::gen::CommonMsgInfo::Record_int_msg_info info; ton::AccountIdPrefixFull src_prefix, dest_prefix, cur_prefix, next_prefix; td::RefInt256 fwd_fee, orig_fwd_fee; + bool from_dispatch_queue = false; // initial checks and unpack switch (tag) { case block::gen::InMsg::msg_import_ext: { @@ -2900,7 +3636,7 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) dest_prefix.to_str() + "... not in this shard"); } dest = std::move(info_ext.dest); - if (!block::tlb::t_MsgAddressInt.extract_std_address(dest, wc, addr)) { + if (!block::tlb::t_MsgAddressInt.extract_std_address(dest, dest_wc, dest_addr)) { return reject_query("cannot unpack destination address of inbound external message with hash "s + key.to_hex(256)); } @@ -2912,7 +3648,7 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) block::gen::InMsg::Record_msg_import_imm inp; unsigned long long created_lt = 0; CHECK(tlb::csr_unpack(in_msg, inp) && tlb::unpack_cell(inp.in_msg, env) && - block::tlb::t_MsgEnvelope.get_created_lt(vm::load_cell_slice(inp.in_msg), created_lt) && + block::tlb::t_MsgEnvelope.get_emitted_lt(vm::load_cell_slice(inp.in_msg), created_lt) && (fwd_fee = block::tlb::t_Grams.as_integer(std::move(inp.fwd_fee))).not_null()); transaction = std::move(inp.transaction); msg_env = std::move(inp.in_msg); @@ -2959,9 +3695,42 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) // msg_discard_fin$110 in_msg:^MsgEnvelope transaction_id:uint64 fwd_fee:Grams return reject_query("InMsg with key "s + key.to_hex(256) + " is a msg_discard_fin, but IHR messages are not enabled in this version"); + case block::gen::InMsg::msg_import_deferred_fin: { + from_dispatch_queue = true; + // msg_import_deferredfin$00100 in_msg:^MsgEnvelope transaction:^Transaction fwd_fee:Grams + // importing and processing an internal message from DispatchQueue with destination in this shard + block::gen::InMsg::Record_msg_import_deferred_fin inp; + CHECK(tlb::csr_unpack(in_msg, inp) && tlb::unpack_cell(inp.in_msg, env) && + (fwd_fee = block::tlb::t_Grams.as_integer(std::move(inp.fwd_fee))).not_null()); + transaction = std::move(inp.transaction); + msg_env = std::move(inp.in_msg); + msg = env.msg; + // ... + break; + } + case block::gen::InMsg::msg_import_deferred_tr: { + from_dispatch_queue = true; + // msg_import_deferred_tr$00101 in_msg:^MsgEnvelope out_msg:^MsgEnvelope + // importing and enqueueing internal message from DispatchQueue + block::gen::InMsg::Record_msg_import_deferred_tr inp; + CHECK(tlb::csr_unpack(in_msg, inp) && tlb::unpack_cell(inp.in_msg, env)); + fwd_fee = td::zero_refint(); + msg_env = std::move(inp.in_msg); + msg = env.msg; + tr_msg_env = std::move(inp.out_msg); + // ... + break; + } default: return reject_query(PSTRING() << "InMsg with key " << key.to_hex(256) << " has impossible tag " << tag); } + if (have_unprocessed_account_dispatch_queue_ && tag != block::gen::InMsg::msg_import_ext && + tag != block::gen::InMsg::msg_import_deferred_tr && tag != block::gen::InMsg::msg_import_deferred_fin) { + // Collator is requeired to take at least one message from each AccountDispatchQueue + // (unless the block is full or unless out_msg_queue_size is big) + // If some AccountDispatchQueue is unporcessed then it's not allowed to import other messages except for externals + return reject_query("required DispatchQueue processing is not done, but some other internal messages are imported"); + } // common checks for all (non-external) inbound messages CHECK(msg.not_null()); if (msg->get_hash().as_bitslice() != key) { @@ -3002,27 +3771,34 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) return reject_query("next hop address "s + next_prefix.to_str() + "... of inbound internal message with hash " + key.to_hex(256) + " does not belong to the current block's shard " + shard_.to_str()); } - // next hop may coincide with current address only if destination is already reached - if (next_prefix == cur_prefix && cur_prefix != dest_prefix) { + // next hop may coincide with current address only if destination is already reached (or it is deferred message) + if (!from_dispatch_queue && next_prefix == cur_prefix && cur_prefix != dest_prefix) { return reject_query( "next hop address "s + next_prefix.to_str() + "... of inbound internal message with hash " + key.to_hex(256) + " coincides with its current address, but this message has not reached its final destination " + dest_prefix.to_str() + "... yet"); } + if (from_dispatch_queue && next_prefix != cur_prefix) { + return reject_query( + "next hop address "s + next_prefix.to_str() + "... of deferred internal message with hash " + key.to_hex(256) + + " must coincide with its current prefix "s + cur_prefix.to_str() + "..."s); + } // if a message is processed by a transaction, it must have destination inside the current shard if (transaction.not_null() && !ton::shard_contains(shard_, dest_prefix)) { return reject_query("inbound internal message with hash "s + key.to_hex(256) + " has destination address " + dest_prefix.to_str() + "... not in this shard, but it is processed nonetheless"); } - // if a message is not processed by a transaction, its final destination must be outside this shard - if (transaction.is_null() && ton::shard_contains(shard_, dest_prefix)) { + // if a message is not processed by a transaction, its final destination must be outside this shard, + // or it is a deferred message (dispatch queue -> out msg queue) + if (tag != block::gen::InMsg::msg_import_deferred_tr && transaction.is_null() && + ton::shard_contains(shard_, dest_prefix)) { return reject_query("inbound internal message with hash "s + key.to_hex(256) + " has destination address " + dest_prefix.to_str() + "... in this shard, but it is not processed by a transaction"); } src = std::move(info.src); dest = std::move(info.dest); // unpack complete destination address if it is inside this shard - if (transaction.not_null() && !block::tlb::t_MsgAddressInt.extract_std_address(dest, wc, addr)) { + if (transaction.not_null() && !block::tlb::t_MsgAddressInt.extract_std_address(dest, dest_wc, dest_addr)) { return reject_query("cannot unpack destination address of inbound internal message with hash "s + key.to_hex(256)); } @@ -3034,6 +3810,44 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) td::dec_string(env.fwd_fee_remaining) + " larger than the original (total) forwarding fee " + td::dec_string(orig_fwd_fee)); } + // Unpacr src address + if (!block::tlb::t_MsgAddressInt.extract_std_address(src, src_wc, src_addr)) { + return reject_query("cannot unpack source address of inbound external message with hash "s + key.to_hex(256)); + } + } + + if (from_dispatch_queue) { + // Check that the message was removed from DispatchQueue + LogicalTime lt = info.created_lt; + auto it = removed_dispatch_queue_messages_.find({src_addr, lt}); + if (it == removed_dispatch_queue_messages_.end()) { + return reject_query(PSTRING() << "deferred InMsg with src_addr=" << src_addr.to_hex() << ", lt=" << lt + << " was not removed from the dispatch queue"); + } + // InMsg msg_import_deferred_* has emitted_lt in MessageEnv, but this emitted_lt is not present in DispatchQueue + Ref dispatched_msg_env = it->second; + td::Ref expected_msg_env; + if (!env.emitted_lt) { + return reject_query(PSTRING() << "no dispatch_lt in deferred InMsg with src_addr=" << src_addr.to_hex() + << ", lt=" << lt); + } + auto emitted_lt = env.emitted_lt.value(); + if (emitted_lt < start_lt_ || emitted_lt > end_lt_) { + return reject_query(PSTRING() << "dispatch_lt in deferred InMsg with src_addr=" << src_addr.to_hex() + << ", lt=" << lt << " is not between start and end of the block"); + } + auto env2 = env; + env2.emitted_lt = {}; + CHECK(block::tlb::pack_cell(expected_msg_env, env2)); + if (dispatched_msg_env->get_hash() != expected_msg_env->get_hash()) { + return reject_query(PSTRING() << "deferred InMsg with src_addr=" << src_addr.to_hex() << ", lt=" << lt + << " msg envelope hasg mismatch: " << dispatched_msg_env->get_hash().to_hex() + << " in DispatchQueue, " << expected_msg_env->get_hash().to_hex() << " expected"); + } + removed_dispatch_queue_messages_.erase(it); + if (tag == block::gen::InMsg::msg_import_deferred_fin) { + msg_emitted_lt_.emplace_back(src_addr, lt, env.emitted_lt.value()); + } } if (transaction.not_null()) { @@ -3050,10 +3864,12 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) ton::StdSmcAddress trans_addr; ton::LogicalTime trans_lt; CHECK(block::get_transaction_id(transaction, trans_addr, trans_lt)); - if (addr != trans_addr) { - block::gen::t_InMsg.print(std::cerr, *in_msg); + if (dest_addr != trans_addr) { + FLOG(INFO) { + block::gen::t_InMsg.print(sb, in_msg); + }; return reject_query(PSTRING() << "InMsg corresponding to inbound message with hash " << key.to_hex(256) - << " and destination address " << addr.to_hex() + << " and destination address " << dest_addr.to_hex() << " claims that the message is processed by transaction " << trans_lt << " of another account " << trans_addr.to_hex()); } @@ -3105,6 +3921,7 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) } case block::gen::InMsg::msg_import_fin: { // msg_import_fin$100 in_msg:^MsgEnvelope transaction:^Transaction fwd_fee:Grams + // msg_import_deferred_fin$00100 in_msg:^MsgEnvelope transaction:^Transaction fwd_fee:Grams // importing and processing an internal message with destination in this shard CHECK(transaction.not_null()); CHECK(shard_contains(shard_, next_prefix)); @@ -3137,22 +3954,39 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) // ... break; } + case block::gen::InMsg::msg_import_deferred_fin: { + // fwd_fee must be equal to the fwd_fee_remaining of this MsgEnvelope + if (*fwd_fee != *env.fwd_fee_remaining) { + return reject_query("msg_import_imm$011 InMsg with hash "s + key.to_hex(256) + + " is invalid because its collected fwd_fee=" + td::dec_string(fwd_fee) + + " is not equal to fwd_fee_remaining=" + td::dec_string(env.fwd_fee_remaining) + + " of this message (envelope)"); + } + // ... + break; + } + case block::gen::InMsg::msg_import_deferred_tr: case block::gen::InMsg::msg_import_tr: { // msg_import_tr$101 in_msg:^MsgEnvelope out_msg:^MsgEnvelope transit_fee:Grams + // msg_import_deferred_tr$00101 in_msg:^MsgEnvelope out_msg:^MsgEnvelope // importing and relaying a (transit) internal message with destination outside this shard - if (cur_prefix == dest_prefix) { + if (cur_prefix == dest_prefix && tag == block::gen::InMsg::msg_import_tr) { return reject_query("inbound internal message with hash "s + key.to_hex(256) + " is a msg_import_tr$101 (a transit message), but its current address " + cur_prefix.to_str() + " is already equal to its final destination"); } + if (cur_prefix != next_prefix && tag == block::gen::InMsg::msg_import_deferred_tr) { + return reject_query("internal message from DispatchQueue with hash "s + key.to_hex(256) + + " is a msg_import_deferred_tr$00101, but its current address " + + cur_prefix.to_str() + " is not equal to next address"); + } CHECK(transaction.is_null()); - CHECK(cur_prefix != next_prefix); auto out_msg_cs = out_msg_dict_->lookup(key, 256); if (out_msg_cs.is_null()) { return reject_query("inbound internal message with hash "s + key.to_hex(256) + " is a msg_import_tr$101 (transit message), but the corresponding OutMsg does not exist"); } - if (shard_contains(shard_, cur_prefix)) { + if (shard_contains(shard_, cur_prefix) && tag == block::gen::InMsg::msg_import_tr) { // we imported this message from our shard! // (very rare situation possible only after merge) tr_req = true; @@ -3165,7 +3999,7 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) } out_msg_env = std::move(out_msg.out_msg); reimport = std::move(out_msg.imported); - } else { + } else if (tag == block::gen::InMsg::msg_import_tr) { block::gen::OutMsg::Record_msg_export_tr out_msg; if (!tlb::csr_unpack_safe(out_msg_cs, out_msg)) { return reject_query( @@ -3179,6 +4013,16 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) if (!check_imported_message(msg_env)) { return false; } + } else { + block::gen::OutMsg::Record_msg_export_deferred_tr out_msg; + if (!tlb::csr_unpack_safe(out_msg_cs, out_msg)) { + return reject_query( + "inbound internal message with hash "s + key.to_hex(256) + + " is a msg_import_deferred_tr$00101 with current address " + cur_prefix.to_str() + + "... outside of our shard, but the corresponding OutMsg is not a valid msg_export_deferred_tr$10101"); + } + out_msg_env = std::move(out_msg.out_msg); + reimport = std::move(out_msg.imported); } // perform hypercube routing for this transit message auto route_info = block::perform_hypercube_routing(next_prefix, dest_prefix, shard_); @@ -3219,6 +4063,18 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) td::dec_string(env.fwd_fee_remaining) + " to " + td::dec_string(tr_env.fwd_fee_remaining) + " in transit"); } + if (tr_env.metadata != env.metadata) { + return reject_query( + PSTRING() << "InMsg for transit message with hash " << key.to_hex(256) << " contains invalid MsgMetadata: " + << (env.metadata ? env.metadata.value().to_str() : "") << " in in_msg, but " + << (tr_env.metadata ? tr_env.metadata.value().to_str() : "") << " in out_msg"); + } + if (tr_env.emitted_lt != env.emitted_lt) { + return reject_query( + PSTRING() << "InMsg for transit message with hash " << key.to_hex(256) << " contains invalid emitted_lt: " + << (env.emitted_lt ? td::to_string(env.emitted_lt.value()) : "") << " in in_msg, but " + << (tr_env.emitted_lt ? td::to_string(tr_env.emitted_lt.value()) : "") << " in out_msg"); + } if (tr_msg_env->get_hash() != out_msg_env->get_hash()) { return reject_query( "InMsg for transit message with hash "s + key.to_hex(256) + @@ -3226,7 +4082,8 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) (tr_req ? "requeued" : "usual") + "transit)"); } // check the amount of the transit fee - td::RefInt256 transit_fee = action_phase_cfg_.fwd_std.get_next_part(env.fwd_fee_remaining); + td::RefInt256 transit_fee = + from_dispatch_queue ? td::zero_refint() : action_phase_cfg_.fwd_std.get_next_part(env.fwd_fee_remaining); if (*transit_fee != *fwd_fee) { return reject_query("InMsg for transit message with hash "s + key.to_hex(256) + " declared collected transit fees to be " + td::dec_string(fwd_fee) + @@ -3252,7 +4109,8 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) " refers to a different reimport InMsg"); } // for transit messages, OutMsg refers to the newly-created outbound messages (not to the re-imported old outbound message) - if (tag != block::gen::InMsg::msg_import_tr && out_msg_env->get_hash() != msg_env->get_hash()) { + if (tag != block::gen::InMsg::msg_import_tr && tag != block::gen::InMsg::msg_import_deferred_tr && + out_msg_env->get_hash() != msg_env->get_hash()) { return reject_query( "InMsg with hash "s + key.to_hex(256) + " is a reimport record, but the corresponding OutMsg exports a MsgEnvelope with a different hash"); @@ -3261,6 +4119,11 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) return true; } +/** + * Checks the validity of the inbound messages listed in the InMsgDescr dictionary. + * + * @returns True if the inbound messages dictionary is valid, false otherwise. + */ bool ValidateQuery::check_in_msg_descr() { LOG(INFO) << "checking inbound messages listed in InMsgDescr"; try { @@ -3280,13 +4143,21 @@ bool ValidateQuery::check_in_msg_descr() { return true; } +/** + * Checks the validity of an outbound message listed in OutMsgDescr. + * + * @param key The 256-bit key of the outbound message. + * @param in_msg The outbound message to be checked serialized using OutMsg TLB-scheme. + * + * @returns True if the outbound message is valid, false otherwise. + */ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_msg) { LOG(DEBUG) << "checking OutMsg with key " << key.to_hex(256); CHECK(out_msg.not_null()); int tag = block::gen::t_OutMsg.get_tag(*out_msg); CHECK(tag >= 0); // NB: the block has been already checked to be valid TL-B in try_validate() - ton::StdSmcAddress addr; - ton::WorkchainId wc; + ton::StdSmcAddress src_addr; + ton::WorkchainId src_wc; Ref src, dest; Ref transaction; Ref msg, msg_env, tr_msg_env, reimport; @@ -3330,7 +4201,7 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms src_prefix.to_str() + "... not in this shard"); } src = std::move(info_ext.src); - if (!block::tlb::t_MsgAddressInt.extract_std_address(src, wc, addr)) { + if (!block::tlb::t_MsgAddressInt.extract_std_address(src, src_wc, src_addr)) { return reject_query("cannot unpack source address of outbound external message with hash "s + key.to_hex(256)); } break; @@ -3349,7 +4220,7 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms case block::gen::OutMsg::msg_export_new: { block::gen::OutMsg::Record_msg_export_new out; CHECK(tlb::csr_unpack(out_msg, out) && tlb::unpack_cell(out.out_msg, env) && - block::tlb::t_MsgEnvelope.get_created_lt(vm::load_cell_slice(out.out_msg), created_lt)); + block::tlb::t_MsgEnvelope.get_emitted_lt(vm::load_cell_slice(out.out_msg), created_lt)); transaction = std::move(out.transaction); msg_env = std::move(out.out_msg); msg = env.msg; @@ -3412,6 +4283,35 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms // ... break; } + case block::gen::OutMsg::msg_export_new_defer: { + block::gen::OutMsg::Record_msg_export_new_defer out; + CHECK(tlb::csr_unpack(out_msg, out) && tlb::unpack_cell(out.out_msg, env) && + block::tlb::t_MsgEnvelope.get_emitted_lt(vm::load_cell_slice(out.out_msg), created_lt)); + transaction = std::move(out.transaction); + msg_env = std::move(out.out_msg); + msg = env.msg; + // ... + break; + } + case block::gen::OutMsg::msg_export_deferred_tr: { + block::gen::OutMsg::Record_msg_export_deferred_tr out; + CHECK(tlb::csr_unpack(out_msg, out) && tlb::unpack_cell(out.out_msg, env)); + msg_env = std::move(out.out_msg); + msg = env.msg; + reimport = std::move(out.imported); + in_tag = block::gen::InMsg::msg_import_deferred_tr; + mode = 2; // added to OutMsgQueue + if (!env.emitted_lt) { + return reject_query(PSTRING() << "msg_export_deferred_tr for OutMsg with key " << key.to_hex(256) + << " does not have emitted_lt in MsgEnvelope"); + } + if (env.emitted_lt.value() < start_lt_ || env.emitted_lt.value() > end_lt_) { + return reject_query(PSTRING() << "emitted_lt for msg_export_deferred_tr with key " << key.to_hex(256) + << " is not between start and end lt of the block"); + } + // ... + break; + } default: return reject_query(PSTRING() << "OutMsg with key (message hash) " << key.to_hex(256) << " has an unknown tag " << tag); @@ -3446,30 +4346,36 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms return reject_query("destination of outbound internal message with hash "s + key.to_hex(256) + " is an invalid blockchain address"); } - cur_prefix = block::interpolate_addr(src_prefix, dest_prefix, env.cur_addr); - next_prefix = block::interpolate_addr(src_prefix, dest_prefix, env.next_addr); - if (!(cur_prefix.is_valid() && next_prefix.is_valid())) { - return reject_query("cannot compute current and next hop addresses of outbound internal message with hash "s + - key.to_hex(256)); - } - // check that next hop is nearer to the destination than the current address - if (count_matching_bits(dest_prefix, next_prefix) < count_matching_bits(dest_prefix, cur_prefix)) { - return reject_query("next hop address "s + next_prefix.to_str() + "... of outbound internal message with hash " + - key.to_hex(256) + " is further from its destination " + dest_prefix.to_str() + - "... than its current address " + cur_prefix.to_str() + "..."); - } - // current address must belong to this shard (otherwise we should never had exported this message) - if (!ton::shard_contains(shard_, cur_prefix)) { - return reject_query("current address "s + cur_prefix.to_str() + "... of outbound internal message with hash " + - key.to_hex(256) + " does not belong to the current block's shard " + shard_.to_str()); - } - // next hop may coincide with current address only if destination is already reached - if (next_prefix == cur_prefix && cur_prefix != dest_prefix) { - return reject_query( - "next hop address "s + next_prefix.to_str() + "... of outbound internal message with hash " + - key.to_hex(256) + - " coincides with its current address, but this message has not reached its final destination " + - dest_prefix.to_str() + "... yet"); + if (tag == block::gen::OutMsg::msg_export_new_defer) { + if (env.cur_addr != 0 || env.next_addr != 0) { + return reject_query("cur_addr and next_addr of the message in DispatchQueue must be zero"); + } + } else { + cur_prefix = block::interpolate_addr(src_prefix, dest_prefix, env.cur_addr); + next_prefix = block::interpolate_addr(src_prefix, dest_prefix, env.next_addr); + if (!(cur_prefix.is_valid() && next_prefix.is_valid())) { + return reject_query("cannot compute current and next hop addresses of outbound internal message with hash "s + + key.to_hex(256)); + } + // check that next hop is nearer to the destination than the current address + if (count_matching_bits(dest_prefix, next_prefix) < count_matching_bits(dest_prefix, cur_prefix)) { + return reject_query("next hop address "s + next_prefix.to_str() + "... of outbound internal message with hash " + + key.to_hex(256) + " is further from its destination " + dest_prefix.to_str() + + "... than its current address " + cur_prefix.to_str() + "..."); + } + // current address must belong to this shard (otherwise we should never had exported this message) + if (!ton::shard_contains(shard_, cur_prefix)) { + return reject_query("current address "s + cur_prefix.to_str() + "... of outbound internal message with hash " + + key.to_hex(256) + " does not belong to the current block's shard " + shard_.to_str()); + } + // next hop may coincide with current address only if destination is already reached + if (next_prefix == cur_prefix && cur_prefix != dest_prefix) { + return reject_query( + "next hop address "s + next_prefix.to_str() + "... of outbound internal message with hash " + + key.to_hex(256) + + " coincides with its current address, but this message has not reached its final destination " + + dest_prefix.to_str() + "... yet"); + } } // if a message is created by a transaction, it must have source inside the current shard if (transaction.not_null() && !ton::shard_contains(shard_, src_prefix)) { @@ -3480,7 +4386,7 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms src = std::move(info.src); dest = std::move(info.dest); // unpack complete source address if it is inside this shard - if (transaction.not_null() && !block::tlb::t_MsgAddressInt.extract_std_address(src, wc, addr)) { + if (!block::tlb::t_MsgAddressInt.extract_std_address(src, src_wc, src_addr)) { return reject_query("cannot unpack source address of outbound internal message with hash "s + key.to_hex(256) + " created in this shard"); } @@ -3508,10 +4414,12 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms ton::StdSmcAddress trans_addr; ton::LogicalTime trans_lt; CHECK(block::get_transaction_id(transaction, trans_addr, trans_lt)); - if (addr != trans_addr) { - block::gen::t_OutMsg.print(std::cerr, *out_msg); + if (src_addr != trans_addr) { + FLOG(INFO) { + block::gen::t_OutMsg.print(sb, out_msg); + }; return reject_query(PSTRING() << "OutMsg corresponding to outbound message with hash " << key.to_hex(256) - << " and source address " << addr.to_hex() + << " and source address " << src_addr.to_hex() << " claims that the message was created by transaction " << trans_lt << " of another account " << trans_addr.to_hex()); } @@ -3530,43 +4438,64 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms (q_key.bits() + 96).copy_from(key, 256); auto q_entry = ns_.out_msg_queue_->lookup(q_key); auto old_q_entry = ps_.out_msg_queue_->lookup(q_key); - if (old_q_entry.not_null() && q_entry.not_null()) { - return reject_query("OutMsg with key (message hash) "s + key.to_hex(256) + - " should have removed or added OutMsgQueue entry with key " + q_key.to_hex() + - ", but it is present both in the old and in the new output queues"); - } - if (old_q_entry.is_null() && q_entry.is_null() && mode) { - return reject_query("OutMsg with key (message hash) "s + key.to_hex(256) + - " should have removed or added OutMsgQueue entry with key " + q_key.to_hex() + - ", but it is absent both from the old and from the new output queues"); - } - if (!mode && (old_q_entry.not_null() || q_entry.not_null())) { - return reject_query("OutMsg with key (message hash) "s + key.to_hex(256) + - " is a msg_export_imm$010, so the OutMsgQueue entry with key " + q_key.to_hex() + - " should never be created, but it is present in either the old or the new output queue"); - } - // NB: if mode!=0, the OutMsgQueue entry has been changed, so we have already checked some conditions in precheck_one_message_queue_update() - if (mode & 2) { - if (q_entry.is_null()) { - return reject_query("OutMsg with key "s + key.to_hex(256) + - " was expected to create OutMsgQueue entry with key " + q_key.to_hex() + " but it did not"); + + if (tag == block::gen::OutMsg::msg_export_new_defer) { + // check the DispatchQueue update + if (old_q_entry.not_null() || q_entry.not_null()) { + return reject_query("OutMsg with key (message hash) "s + key.to_hex(256) + + " shouldn't exist in the old and the new message queues"); } - if (msg_env_hash != q_entry->prefetch_ref()->get_hash().bits()) { - return reject_query("OutMsg with key "s + key.to_hex(256) + " has created OutMsgQueue entry with key " + - q_key.to_hex() + " containing a different MsgEnvelope"); + auto it = new_dispatch_queue_messages_.find({src_addr, created_lt}); + if (it == new_dispatch_queue_messages_.end()) { + return reject_query(PSTRING() << "new deferred OutMsg with src_addr=" << src_addr.to_hex() + << ", lt=" << created_lt << " was not added to the dispatch queue"); } - // ... - } else if (mode & 1) { - if (old_q_entry.is_null()) { - return reject_query("OutMsg with key "s + key.to_hex(256) + - " was expected to remove OutMsgQueue entry with key " + q_key.to_hex() + - " but it did not exist in the old queue"); + Ref expected_msg_env = it->second; + if (expected_msg_env->get_hash() != msg_env->get_hash()) { + return reject_query(PSTRING() << "new deferred OutMsg with src_addr=" << src_addr.to_hex() << ", lt=" + << created_lt << " msg envelope hasg mismatch: " << msg_env->get_hash().to_hex() + << " in OutMsg, " << expected_msg_env->get_hash().to_hex() << " in DispatchQueue"); } - if (msg_env_hash != old_q_entry->prefetch_ref()->get_hash().bits()) { - return reject_query("OutMsg with key "s + key.to_hex(256) + " has dequeued OutMsgQueue entry with key " + - q_key.to_hex() + " containing a different MsgEnvelope"); + new_dispatch_queue_messages_.erase(it); + } else { + if (old_q_entry.not_null() && q_entry.not_null()) { + return reject_query("OutMsg with key (message hash) "s + key.to_hex(256) + + " should have removed or added OutMsgQueue entry with key " + q_key.to_hex() + + ", but it is present both in the old and in the new output queues"); + } + if (old_q_entry.is_null() && q_entry.is_null() && mode) { + return reject_query("OutMsg with key (message hash) "s + key.to_hex(256) + + " should have removed or added OutMsgQueue entry with key " + q_key.to_hex() + + ", but it is absent both from the old and from the new output queues"); + } + if (!mode && (old_q_entry.not_null() || q_entry.not_null())) { + return reject_query("OutMsg with key (message hash) "s + key.to_hex(256) + + " is a msg_export_imm$010, so the OutMsgQueue entry with key " + q_key.to_hex() + + " should never be created, but it is present in either the old or the new output queue"); + } + // NB: if mode!=0, the OutMsgQueue entry has been changed, so we have already checked some conditions in precheck_one_message_queue_update() + if (mode & 2) { + if (q_entry.is_null()) { + return reject_query("OutMsg with key "s + key.to_hex(256) + + " was expected to create OutMsgQueue entry with key " + q_key.to_hex() + " but it did not"); + } + if (msg_env_hash != q_entry->prefetch_ref()->get_hash().bits()) { + return reject_query("OutMsg with key "s + key.to_hex(256) + " has created OutMsgQueue entry with key " + + q_key.to_hex() + " containing a different MsgEnvelope"); + } + // ... + } else if (mode & 1) { + if (old_q_entry.is_null()) { + return reject_query("OutMsg with key "s + key.to_hex(256) + + " was expected to remove OutMsgQueue entry with key " + q_key.to_hex() + + " but it did not exist in the old queue"); + } + if (msg_env_hash != old_q_entry->prefetch_ref()->get_hash().bits()) { + return reject_query("OutMsg with key "s + key.to_hex(256) + " has dequeued OutMsgQueue entry with key " + + q_key.to_hex() + " containing a different MsgEnvelope"); + } + // ... } - // ... } // check reimport:^InMsg @@ -3594,8 +4523,8 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms int i_tag = block::gen::t_InMsg.get_tag(*in); if (i_tag < 0 || i_tag != in_tag) { return reject_query("OutMsg with key "s + key.to_hex(256) + - " refers to a (re)import InMsg, which is not one of msg_import_imm, msg_import_fin or " - "msg_import_tr as expected"); + " refers to a (re)import InMsg, which is not one of msg_import_imm, msg_import_fin, " + "msg_import_tr or msg_import_deferred_tr as expected"); } } @@ -3655,6 +4584,9 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms // ... break; } + case block::gen::OutMsg::msg_export_new_defer: { + break; + } case block::gen::OutMsg::msg_export_tr: { block::gen::InMsg::Record_msg_import_tr in; block::tlb::MsgEnvelope::Record_std in_env; @@ -3679,6 +4611,24 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms // ... break; } + case block::gen::OutMsg::msg_export_deferred_tr: { + block::gen::InMsg::Record_msg_import_deferred_tr in; + block::tlb::MsgEnvelope::Record_std in_env; + if (!(tlb::unpack_cell(reimport, in) && tlb::unpack_cell(in.in_msg, in_env))) { + return reject_query( + "cannot unpack msg_import_deferred_tr InMsg record corresponding to msg_export_deferred_tr OutMsg record with key "s + + key.to_hex(256)); + } + CHECK(in_env.msg->get_hash() == msg->get_hash()); + auto in_cur_prefix = block::interpolate_addr(src_prefix, dest_prefix, in_env.cur_addr); + if (!shard_contains(shard_, in_cur_prefix)) { + return reject_query( + "msg_export_deferred_tr OutMsg record with key "s + key.to_hex(256) + + " corresponds to msg_import_deferred_tr InMsg record with current imported message address " + + in_cur_prefix.to_str() + " NOT inside the current shard"); + } + break; + } case block::gen::OutMsg::msg_export_deq: case block::gen::OutMsg::msg_export_deq_short: { // check that the message has been indeed processed by a neighbor @@ -3794,9 +4744,32 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms return fatal_error(PSTRING() << "unknown OutMsg tag " << tag); } + if (tag == block::gen::OutMsg::msg_export_imm || tag == block::gen::OutMsg::msg_export_deq_imm || + tag == block::gen::OutMsg::msg_export_new || tag == block::gen::OutMsg::msg_export_deferred_tr) { + if (src_wc != workchain()) { + return true; + } + if (tag == block::gen::OutMsg::msg_export_imm && is_special_in_msg(vm::load_cell_slice(reimport))) { + return true; + } + unsigned long long created_lt; + auto cs = vm::load_cell_slice(env.msg); + if (!block::tlb::t_Message.get_created_lt(cs, created_lt)) { + return reject_query(PSTRING() << "cannot get created_lt for OutMsg with key " << key.to_hex(256) + << ", tag=" << tag); + } + auto emitted_lt = env.emitted_lt ? env.emitted_lt.value() : created_lt; + msg_emitted_lt_.emplace_back(src_addr, created_lt, emitted_lt); + } + return true; } +/** + * Checks the validity of the outbound messages listed in the OutMsgDescr dictionary. + * + * @returns True if the outbound messages dictionary is valid, false otherwise. + */ bool ValidateQuery::check_out_msg_descr() { LOG(INFO) << "checking outbound messages listed in OutMsgDescr"; try { @@ -3815,7 +4788,12 @@ bool ValidateQuery::check_out_msg_descr() { return true; } -// compare to Collator::update_processed_upto() +/** + * Checks if the processed up to information is valid and consistent. + * Compare to Collator::update_processed_upto() + * + * @returns True if the processed up to information is valid and consistent, false otherwise. + */ bool ValidateQuery::check_processed_upto() { LOG(INFO) << "checking ProcessedInfo"; CHECK(ps_.processed_upto_); @@ -3871,7 +4849,37 @@ bool ValidateQuery::check_processed_upto() { return true; } -// similar to Collator::process_inbound_message +/** + * Check that the difference between the old and new dispatch queues is reflected in OutMsgs and InMsgs + * + * @returns True if the check is successful, false otherwise. + */ +bool ValidateQuery::check_dispatch_queue_update() { + if (!new_dispatch_queue_messages_.empty()) { + auto it = new_dispatch_queue_messages_.begin(); + return reject_query(PSTRING() << "DispatchQueue has a new message with src_addr=" << it->first.first.to_hex() + << ", lt=" << it->first.second << ", but no correseponding OutMsg exists"); + } + if (!removed_dispatch_queue_messages_.empty()) { + auto it = removed_dispatch_queue_messages_.begin(); + return reject_query(PSTRING() << "message with src_addr=" << it->first.first.to_hex() << ", lt=" << it->first.second + << " was removed from DispatchQueue, but no correseponding InMsg exists"); + } + return true; +} + +/** + * Checks the validity of an outbound message in the neighbor's queue. + * Similar to Collator::process_inbound_message. + * + * @param enq_msg The enqueued message to validate. + * @param lt The logical time of the message. + * @param key The 32+64+256-bit key of the message. + * @param nb The neighbor's description. + * @param unprocessed A boolean flag that will be set to true if the message is unprocessed, false otherwise. + * + * @returns True if the message is valid, false otherwise. + */ bool ValidateQuery::check_neighbor_outbound_message(Ref enq_msg, ton::LogicalTime lt, td::ConstBitPtr key, const block::McShardDescr& nb, bool& unprocessed) { @@ -4010,6 +5018,11 @@ bool ValidateQuery::check_neighbor_outbound_message(Ref enq_msg, return true; } +/** + * Checks messages from the outbound queues of the neighbors. + * + * @returns True if the messages are valid, false otherwise. + */ bool ValidateQuery::check_in_queue() { block::OutputQueueMerger nb_out_msgs(shard_, neighbors_); while (!nb_out_msgs.is_eof()) { @@ -4018,32 +5031,37 @@ bool ValidateQuery::check_in_queue() { LOG(DEBUG) << "processing inbound message with (lt,hash)=(" << kv->lt << "," << kv->key.to_hex() << ") from neighbor #" << kv->source; if (verbosity > 3) { - std::cerr << "inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() << " msg="; - block::gen::t_EnqueuedMsg.print(std::cerr, *(kv->msg)); + FLOG(INFO) { + sb << "inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() << " msg="; + block::gen::t_EnqueuedMsg.print(sb, kv->msg); + }; } bool unprocessed = false; if (!check_neighbor_outbound_message(kv->msg, kv->lt, kv->key.cbits(), neighbors_.at(kv->source), unprocessed)) { if (verbosity > 1) { - std::cerr << "invalid neighbor outbound message: lt=" << kv->lt << " from=" << kv->source - << " key=" << kv->key.to_hex() << " msg="; - block::gen::t_EnqueuedMsg.print(std::cerr, *(kv->msg)); + FLOG(INFO) { + sb << "invalid neighbor outbound message: lt=" << kv->lt << " from=" << kv->source + << " key=" << kv->key.to_hex() << " msg="; + block::gen::t_EnqueuedMsg.print(sb, kv->msg); + }; } return reject_query("error processing outbound internal message "s + kv->key.to_hex() + " of neighbor " + neighbors_.at(kv->source).blk_.to_str()); } if (unprocessed) { - inbound_queues_empty_ = false; return true; } nb_out_msgs.next(); } - inbound_queues_empty_ = true; return true; } -// checks that all messages imported from our outbound queue into neighbor shards have been dequeued -// similar to Collator::out_msg_queue_cleanup() -// (but scans new outbound queue instead of the old) +/** + * Checks that all messages imported from our outbound queue into neighbor shards have been dequeued + * Similar to Collator::out_msg_queue_cleanup() (but scans the new outbound queue instead of the old). + * + * @returns True if the delivery status of all messages has been checked successfully, false otherwise. + */ bool ValidateQuery::check_delivered_dequeued() { LOG(INFO) << "scanning new outbound queue and checking delivery status of all messages"; bool ok = false; @@ -4090,26 +5108,42 @@ bool ValidateQuery::check_delivered_dequeued() { }) || ok; } -// similar to Collator::make_account_from() -std::unique_ptr ValidateQuery::make_account_from(td::ConstBitPtr addr, Ref account, - Ref extra) { +/** + * Creates a new Account object from the given address and serialized account data. + * Creates a new Account if not found. + * Similar to Collator::make_account_from() + * + * @param addr A pointer to the 256-bit address of the account. + * @param account A cell slice with an account serialized using ShardAccount TLB-scheme. + * + * @returns A unique pointer to the created Account object, or nullptr if the creation failed. + */ +std::unique_ptr ValidateQuery::make_account_from(td::ConstBitPtr addr, Ref account) { auto ptr = std::make_unique(workchain(), addr); if (account.is_null()) { if (!ptr->init_new(now_)) { return nullptr; } - } else if (!ptr->unpack(std::move(account), std::move(extra), now_, - is_masterchain() && config_->is_special_smartcontract(addr))) { + } else if (!ptr->unpack(std::move(account), now_, is_masterchain() && config_->is_special_smartcontract(addr))) { return nullptr; } ptr->block_lt = start_lt_; return ptr; } -// similar to Collator::make_account() +/** + * Retreives an Account object from the data in the shard state. + * Accounts are cached in the ValidatorQuery's map. + * Similar to Collator::make_account() + * + * @param addr The 256-bit address of the account. + * + * @returns Pointer to the account if found or created successfully. + * Returns nullptr if an error occured. + */ std::unique_ptr ValidateQuery::unpack_account(td::ConstBitPtr addr) { auto dict_entry = ps_.account_dict_->lookup_extra(addr, 256); - auto new_acc = make_account_from(addr, std::move(dict_entry.first), std::move(dict_entry.second)); + auto new_acc = make_account_from(addr, std::move(dict_entry.first)); if (!new_acc) { reject_query("cannot load state of account "s + addr.to_hex(256) + " from previous shardchain state"); return {}; @@ -4122,6 +5156,18 @@ std::unique_ptr ValidateQuery::unpack_account(td::ConstBitPtr ad return new_acc; } +/** + * Checks the validity of a single transaction for a given account. + * Performs transaction execution. + * + * @param account The account of the transaction. + * @param lt The logical time of the transaction. + * @param trans_root The root of the transaction. + * @param is_first Flag indicating if this is the first transaction of the account. + * @param is_last Flag indicating if this is the last transaction of the account. + * + * @returns True if the transaction is valid, false otherwise. + */ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalTime lt, Ref trans_root, bool is_first, bool is_last) { if (!check_timeout()) { @@ -4137,6 +5183,11 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT bool external{false}, ihr_delivered{false}, need_credit_phase{false}; // check input message block::CurrencyCollection money_imported(0), money_exported(0); + bool is_special_tx = false; // recover/mint transaction + auto td_cs = vm::load_cell_slice(trans.description); + int tag = block::gen::t_TransactionDescr.get_tag(td_cs); + CHECK(tag >= 0); // we have already validated the serialization of all Transactions + td::optional in_msg_metadata; if (in_msg_root.not_null()) { auto in_descr_cs = in_msg_dict_->lookup(in_msg_root->get_hash().as_bitslice()); if (in_descr_cs.is_null()) { @@ -4144,19 +5195,21 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT << " of transaction " << lt << " of account " << addr.to_hex() << " does not have a corresponding InMsg record"); } - auto tag = block::gen::t_InMsg.get_tag(*in_descr_cs); - if (tag != block::gen::InMsg::msg_import_ext && tag != block::gen::InMsg::msg_import_fin && - tag != block::gen::InMsg::msg_import_imm && tag != block::gen::InMsg::msg_import_ihr) { + auto in_msg_tag = block::gen::t_InMsg.get_tag(*in_descr_cs); + if (in_msg_tag != block::gen::InMsg::msg_import_ext && in_msg_tag != block::gen::InMsg::msg_import_fin && + in_msg_tag != block::gen::InMsg::msg_import_imm && in_msg_tag != block::gen::InMsg::msg_import_ihr && + in_msg_tag != block::gen::InMsg::msg_import_deferred_fin) { return reject_query(PSTRING() << "inbound message with hash " << in_msg_root->get_hash().to_hex() << " of transaction " << lt << " of account " << addr.to_hex() << " has an invalid InMsg record (not one of msg_import_ext, msg_import_fin, " - "msg_import_imm or msg_import_ihr)"); + "msg_import_imm, msg_import_ihr or msg_import_deferred_fin)"); } + is_special_tx = is_special_in_msg(*in_descr_cs); // once we know there is a InMsg with correct hash, we already know that it contains a message with this hash (by the verification of InMsg), so it is our message // have still to check its destination address and imported value // and that it refers to this transaction Ref dest; - if (tag == block::gen::InMsg::msg_import_ext) { + if (in_msg_tag == block::gen::InMsg::msg_import_ext) { block::gen::CommonMsgInfo::Record_ext_in_msg_info info; CHECK(tlb::unpack_cell_inexact(in_msg_root, info)); dest = std::move(info.dest); @@ -4169,12 +5222,26 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT << " processed inbound message created later at logical time " << info.created_lt); } - if (info.created_lt != start_lt_ || !is_special_in_msg(*in_descr_cs)) { - msg_proc_lt_.emplace_back(addr, lt, info.created_lt); + LogicalTime emitted_lt = info.created_lt; // See ValidateQuery::check_message_processing_order + if (in_msg_tag == block::gen::InMsg::msg_import_imm || in_msg_tag == block::gen::InMsg::msg_import_fin || + in_msg_tag == block::gen::InMsg::msg_import_deferred_fin) { + block::tlb::MsgEnvelope::Record_std msg_env; + if (!block::tlb::unpack_cell(in_descr_cs->prefetch_ref(), msg_env)) { + return reject_query(PSTRING() << "InMsg record for inbound message with hash " + << in_msg_root->get_hash().to_hex() << " of transaction " << lt + << " of account " << addr.to_hex() << " does not have a valid MsgEnvelope"); + } + in_msg_metadata = std::move(msg_env.metadata); + if (msg_env.emitted_lt) { + emitted_lt = msg_env.emitted_lt.value(); + } + } + if (info.created_lt != start_lt_ || !is_special_tx) { + msg_proc_lt_.emplace_back(addr, lt, emitted_lt); } dest = std::move(info.dest); CHECK(money_imported.validate_unpack(info.value)); - ihr_delivered = (tag == block::gen::InMsg::msg_import_ihr); + ihr_delivered = (in_msg_tag == block::gen::InMsg::msg_import_ihr); if (!ihr_delivered) { money_imported += block::tlb::t_Grams.as_integer(info.ihr_fee); } @@ -4196,6 +5263,15 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT } } // check output messages + td::optional new_msg_metadata; + if (msg_metadata_enabled_) { + if (external || is_special_tx || tag != block::gen::TransactionDescr::trans_ord) { + new_msg_metadata = block::MsgMetadata{0, account.workchain, account.addr, (LogicalTime)trans.lt}; + } else if (in_msg_metadata) { + new_msg_metadata = std::move(in_msg_metadata); + ++new_msg_metadata.value().depth; + } + } vm::Dictionary out_dict{trans.r1.out_msgs, 15}; for (int i = 0; i < trans.outmsg_cnt; i++) { auto out_msg_root = out_dict.lookup_ref(td::BitArray<15>{i}); @@ -4208,33 +5284,45 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT } auto tag = block::gen::t_OutMsg.get_tag(*out_descr_cs); if (tag != block::gen::OutMsg::msg_export_ext && tag != block::gen::OutMsg::msg_export_new && - tag != block::gen::OutMsg::msg_export_imm) { - return reject_query( - PSTRING() << "outbound message #" << i + 1 << " with hash " << out_msg_root->get_hash().to_hex() - << " of transaction " << lt << " of account " << addr.to_hex() - << " has an invalid OutMsg record (not one of msg_export_ext, msg_export_new or msg_export_imm)"); + tag != block::gen::OutMsg::msg_export_imm && tag != block::gen::OutMsg::msg_export_new_defer) { + return reject_query(PSTRING() << "outbound message #" << i + 1 << " with hash " + << out_msg_root->get_hash().to_hex() << " of transaction " << lt << " of account " + << addr.to_hex() + << " has an invalid OutMsg record (not one of msg_export_ext, msg_export_new, " + "msg_export_imm or msg_export_new_defer)"); } - // once we know there is an OutMsg with correct hash, we already know that it contains a message with this hash (by the verification of OutMsg), so it is our message + // once we know there is an OutMsg with correct hash, we already know that it contains a message with this hash + // (by the verification of OutMsg), so it is our message // have still to check its source address, lt and imported value // and that it refers to this transaction as its origin Ref src; + LogicalTime message_lt; if (tag == block::gen::OutMsg::msg_export_ext) { block::gen::CommonMsgInfo::Record_ext_out_msg_info info; CHECK(tlb::unpack_cell_inexact(out_msg_root, info)); src = std::move(info.src); + message_lt = info.created_lt; } else { block::gen::CommonMsgInfo::Record_int_msg_info info; CHECK(tlb::unpack_cell_inexact(out_msg_root, info)); src = std::move(info.src); - block::gen::MsgEnvelope::Record msg_env; + message_lt = info.created_lt; + block::tlb::MsgEnvelope::Record_std msg_env; CHECK(tlb::unpack_cell(out_descr_cs->prefetch_ref(), msg_env)); // unpack exported message value (from this transaction) block::CurrencyCollection msg_export_value; CHECK(msg_export_value.unpack(info.value)); msg_export_value += block::tlb::t_Grams.as_integer(info.ihr_fee); - msg_export_value += block::tlb::t_Grams.as_integer(msg_env.fwd_fee_remaining); + msg_export_value += msg_env.fwd_fee_remaining; CHECK(msg_export_value.is_valid()); money_exported += msg_export_value; + if (msg_env.metadata != new_msg_metadata) { + return reject_query(PSTRING() << "outbound message #" << i + 1 << " with hash " + << out_msg_root->get_hash().to_hex() << " of transaction " << lt << " of account " + << addr.to_hex() << " has invalid metadata in an OutMsg record: expected " + << (new_msg_metadata ? new_msg_metadata.value().to_str() : "") << ", found " + << (msg_env.metadata ? msg_env.metadata.value().to_str() : "")); + } } WorkchainId s_wc; StdSmcAddress ss_addr; // s_addr is some macros in Windows @@ -4251,13 +5339,32 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT << out_msg_root->get_hash().to_hex() << " of transaction " << lt << " of account " << addr.to_hex() << " refers to a different processing transaction"); } + if (tag != block::gen::OutMsg::msg_export_ext) { + bool is_deferred = tag == block::gen::OutMsg::msg_export_new_defer; + if (account_expected_defer_all_messages_.count(ss_addr) && !is_deferred) { + return reject_query( + PSTRING() << "outbound message #" << i + 1 << " on account " << workchain() << ":" << ss_addr.to_hex() + << " must be deferred because this account has earlier messages in DispatchQueue"); + } + if (is_deferred) { + LOG(INFO) << "message from account " << workchain() << ":" << ss_addr.to_hex() << " with lt " << message_lt + << " was deferred"; + if (!deferring_messages_enabled_ && !account_expected_defer_all_messages_.count(ss_addr)) { + return reject_query(PSTRING() << "outbound message #" << i + 1 << " on account " << workchain() << ":" + << ss_addr.to_hex() << " is deferred, but deferring messages is disabled"); + } + if (i == 0 && !account_expected_defer_all_messages_.count(ss_addr)) { + return reject_query(PSTRING() << "outbound message #1 on account " << workchain() << ":" << ss_addr.to_hex() + << " must not be deferred (the first message cannot be deferred unless some " + "prevoius messages are deferred)"); + } + account_expected_defer_all_messages_.insert(ss_addr); + } + } } CHECK(money_exported.is_valid()); // check general transaction data block::CurrencyCollection old_balance{account.get_balance()}; - auto td_cs = vm::load_cell_slice(trans.description); - int tag = block::gen::t_TransactionDescr.get_tag(td_cs); - CHECK(tag >= 0); // we have already validated the serialization of all Transactions if (tag == block::gen::TransactionDescr::trans_merge_prepare || tag == block::gen::TransactionDescr::trans_merge_install || tag == block::gen::TransactionDescr::trans_split_prepare || @@ -4359,17 +5466,10 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT << account.total_state->get_hash().to_hex()); } // some type-specific checks - int trans_type = block::Transaction::tr_none; + int trans_type = block::transaction::Transaction::tr_none; switch (tag) { case block::gen::TransactionDescr::trans_ord: { - if (!block_limit_status_->fits(block::ParamLimits::cl_medium)) { - return reject_query(PSTRING() << "cannod add ordinary transaction because hard block limits are exceeded: " - << "gas_used=" << block_limit_status_->gas_used - << "(limit=" << block_limits_->gas.hard() << "), " - << "lt_delta=" << block_limit_status_->cur_lt - block_limits_->start_lt - << "(limit=" << block_limits_->lt_delta.hard() << ")"); - } - trans_type = block::Transaction::tr_ord; + trans_type = block::transaction::Transaction::tr_ord; if (in_msg_root.is_null()) { return reject_query(PSTRING() << "ordinary transaction " << lt << " of account " << addr.to_hex() << " has no inbound message"); @@ -4378,7 +5478,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT break; } case block::gen::TransactionDescr::trans_storage: { - trans_type = block::Transaction::tr_storage; + trans_type = block::transaction::Transaction::tr_storage; if (in_msg_root.not_null()) { return reject_query(PSTRING() << "storage transaction " << lt << " of account " << addr.to_hex() << " has an inbound message"); @@ -4394,7 +5494,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT } case block::gen::TransactionDescr::trans_tick_tock: { bool is_tock = (td_cs.prefetch_ulong(4) & 1); - trans_type = is_tock ? block::Transaction::tr_tock : block::Transaction::tr_tick; + trans_type = is_tock ? block::transaction::Transaction::tr_tock : block::transaction::Transaction::tr_tick; if (in_msg_root.not_null()) { return reject_query(PSTRING() << (is_tock ? "tock" : "tick") << " transaction " << lt << " of account " << addr.to_hex() << " has an inbound message"); @@ -4402,7 +5502,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT break; } case block::gen::TransactionDescr::trans_merge_prepare: { - trans_type = block::Transaction::tr_merge_prepare; + trans_type = block::transaction::Transaction::tr_merge_prepare; if (in_msg_root.not_null()) { return reject_query(PSTRING() << "merge prepare transaction " << lt << " of account " << addr.to_hex() << " has an inbound message"); @@ -4417,7 +5517,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT break; } case block::gen::TransactionDescr::trans_merge_install: { - trans_type = block::Transaction::tr_merge_install; + trans_type = block::transaction::Transaction::tr_merge_install; if (in_msg_root.is_null()) { return reject_query(PSTRING() << "merge install transaction " << lt << " of account " << addr.to_hex() << " has no inbound message"); @@ -4429,7 +5529,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT break; } case block::gen::TransactionDescr::trans_split_prepare: { - trans_type = block::Transaction::tr_split_prepare; + trans_type = block::transaction::Transaction::tr_split_prepare; if (in_msg_root.not_null()) { return reject_query(PSTRING() << "split prepare transaction " << lt << " of account " << addr.to_hex() << " has an inbound message"); @@ -4444,7 +5544,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT break; } case block::gen::TransactionDescr::trans_split_install: { - trans_type = block::Transaction::tr_split_install; + trans_type = block::transaction::Transaction::tr_split_install; if (in_msg_root.is_null()) { return reject_query(PSTRING() << "split install transaction " << lt << " of account " << addr.to_hex() << " has no inbound message"); @@ -4459,8 +5559,8 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT // check transaction computation by re-doing it // similar to Collator::create_ordinary_transaction() and Collator::create_ticktock_transaction() // .... - std::unique_ptr trs = - std::make_unique(account, trans_type, lt, now_, in_msg_root); + std::unique_ptr trs = + std::make_unique(account, trans_type, lt, now_, in_msg_root); if (in_msg_root.not_null()) { if (!trs->unpack_input_msg(ihr_delivered, &action_phase_cfg_)) { // inbound external message was not accepted @@ -4506,19 +5606,41 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT return reject_query(PSTRING() << "cannot re-create action phase of transaction " << lt << " for smart contract " << addr.to_hex()); } - if (trs->bounce_enabled && (!trs->compute_phase->success || trs->action_phase->state_size_too_big) && + if (trs->bounce_enabled && + (!trs->compute_phase->success || trs->action_phase->state_exceeds_limits || trs->action_phase->bounce) && !trs->prepare_bounce_phase(action_phase_cfg_)) { return reject_query(PSTRING() << "cannot re-create bounce phase of transaction " << lt << " for smart contract " << addr.to_hex()); } - if (!trs->serialize()) { + if (!trs->serialize(serialize_cfg_)) { return reject_query(PSTRING() << "cannot re-create the serialization of transaction " << lt << " for smart contract " << addr.to_hex()); } - if (!trs->update_limits(*block_limit_status_, false)) { + if (!trs->update_limits(*block_limit_status_, /* with_gas = */ false, /* with_size = */ false)) { return fatal_error(PSTRING() << "cannot update block limit status to include transaction " << lt << " of account " << addr.to_hex()); } + + // Collator should stop if total gas usage exceeds limits, including transactions on special accounts, but without + // ticktocks and mint/recover. + // Here Validator checks a weaker condition + if (!is_special_tx && !trs->gas_limit_overridden && trans_type == block::transaction::Transaction::tr_ord) { + (account.is_special ? total_special_gas_used_ : total_gas_used_) += trs->gas_used(); + } + if (total_gas_used_ > block_limits_->gas.hard() + compute_phase_cfg_.gas_limit) { + return reject_query(PSTRING() << "gas block limits are exceeded: total_gas_used > gas_limit_hard + trx_gas_limit (" + << "total_gas_used=" << total_gas_used_ + << ", gas_limit_hard=" << block_limits_->gas.hard() + << ", trx_gas_limit=" << compute_phase_cfg_.gas_limit << ")"); + } + if (total_special_gas_used_ > block_limits_->gas.hard() + compute_phase_cfg_.special_gas_limit) { + return reject_query( + PSTRING() << "gas block limits are exceeded: total_special_gas_used > gas_limit_hard + special_gas_limit (" + << "total_special_gas_used=" << total_special_gas_used_ + << ", gas_limit_hard=" << block_limits_->gas.hard() + << ", special_gas_limit=" << compute_phase_cfg_.special_gas_limit << ")"); + } + auto trans_root2 = trs->commit(account); if (trans_root2.is_null()) { return reject_query(PSTRING() << "the re-created transaction " << lt << " for smart contract " << addr.to_hex() @@ -4527,10 +5649,12 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT // now compare the re-created transaction with the one we have if (trans_root2->get_hash() != trans_root->get_hash()) { if (verbosity >= 3 * 0) { - std::cerr << "original transaction " << lt << " of " << addr.to_hex() << ": "; - block::gen::t_Transaction.print_ref(std::cerr, trans_root); - std::cerr << "re-created transaction " << lt << " of " << addr.to_hex() << ": "; - block::gen::t_Transaction.print_ref(std::cerr, trans_root2); + FLOG(INFO) { + sb << "original transaction " << lt << " of " << addr.to_hex() << ": "; + block::gen::t_Transaction.print_ref(sb, trans_root); + sb << "re-created transaction " << lt << " of " << addr.to_hex() << ": "; + block::gen::t_Transaction.print_ref(sb, trans_root2); + }; } return reject_query(PSTRING() << "the transaction " << lt << " of " << addr.to_hex() << " has hash " << trans_root->get_hash().to_hex() @@ -4564,6 +5688,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT << "transaction " << lt << " of " << addr.to_hex() << " is invalid: it has produced a set of outbound messages different from that listed in the transaction"); } + total_burned_ += trs->blackhole_burned; // check new balance and value flow auto new_balance = account.get_balance(); block::CurrencyCollection total_fees; @@ -4571,17 +5696,27 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT return reject_query(PSTRING() << "transaction " << lt << " of " << addr.to_hex() << " has an invalid total_fees value"); } - if (old_balance + money_imported != new_balance + money_exported + total_fees) { - return reject_query(PSTRING() << "transaction " << lt << " of " << addr.to_hex() - << " violates the currency flow condition: old balance=" << old_balance.to_str() - << " + imported=" << money_imported.to_str() << " does not equal new balance=" - << new_balance.to_str() << " + exported=" << money_exported.to_str() - << " + total_fees=" << total_fees.to_str()); + if (old_balance + money_imported != new_balance + money_exported + total_fees + trs->blackhole_burned) { + return reject_query( + PSTRING() << "transaction " << lt << " of " << addr.to_hex() + << " violates the currency flow condition: old balance=" << old_balance.to_str() + << " + imported=" << money_imported.to_str() << " does not equal new balance=" << new_balance.to_str() + << " + exported=" << money_exported.to_str() << " + total_fees=" << total_fees.to_str() + << (trs->blackhole_burned.is_zero() ? "" + : PSTRING() << " burned=" << trs->blackhole_burned.to_str())); } return true; } -// NB: may be run in parallel for different accounts +/** + * Checks the validity of transactions for a given account block. + * NB: may be run in parallel for different accounts + * + * @param acc_addr The address of the account. + * @param acc_blk_root The root of the AccountBlock. + * + * @returns True if the account transactions are valid, false otherwise. + */ bool ValidateQuery::check_account_transactions(const StdSmcAddress& acc_addr, Ref acc_blk_root) { block::gen::AccountBlock::Record acc_blk; CHECK(tlb::csr_unpack(std::move(acc_blk_root), acc_blk) && acc_blk.account_addr == acc_addr); @@ -4613,6 +5748,11 @@ bool ValidateQuery::check_account_transactions(const StdSmcAddress& acc_addr, Re } } +/** + * Checks all transactions in the account blocks. + * + * @returns True if all transactions pass the check, False otherwise. + */ bool ValidateQuery::check_transactions() { LOG(INFO) << "checking all transactions"; return account_blocks_dict_->check_for_each_extra( @@ -4622,7 +5762,17 @@ bool ValidateQuery::check_transactions() { }); } -// similar to Collator::update_account_public_libraries() +/** + * Processes changes in libraries of an account. + * Used in masterchain validation. + * Similar to Collator::update_account_public_libraries() + * + * @param orig_libs The original libraries of the account. + * @param final_libs The final libraries of the account. + * @param addr The address of the account. + * + * @returns True if the update was successful, false otherwise. + */ bool ValidateQuery::scan_account_libraries(Ref orig_libs, Ref final_libs, const td::Bits256& addr) { vm::Dictionary dict1{std::move(orig_libs), 256}, dict2{std::move(final_libs), 256}; return dict1.scan_diff( @@ -4640,6 +5790,12 @@ bool ValidateQuery::scan_account_libraries(Ref orig_libs, Ref (t1.lt < t2.lt). + // New rule: + // If message was taken from dispatch queue, instead of created_lt use emitted_lt std::sort(msg_proc_lt_.begin(), msg_proc_lt_.end()); for (std::size_t i = 1; i < msg_proc_lt_.size(); i++) { auto &a = msg_proc_lt_[i - 1], &b = msg_proc_lt_[i]; @@ -4675,9 +5840,32 @@ bool ValidateQuery::check_message_processing_order() { << ") processes an earlier message created at logical time " << std::get<2>(b)); } } + + // Check that if messages m1 and m2 with the same source have m1.created_lt < m2.created_lt then + // m1.emitted_lt < m2.emitted_lt. + std::sort(msg_emitted_lt_.begin(), msg_emitted_lt_.end()); + for (std::size_t i = 1; i < msg_emitted_lt_.size(); i++) { + auto &a = msg_emitted_lt_[i - 1], &b = msg_emitted_lt_[i]; + if (std::get<0>(a) == std::get<0>(b) && std::get<2>(a) >= std::get<2>(b)) { + return reject_query(PSTRING() << "incorrect deferred message processing order for sender " + << std::get<0>(a).to_hex() << ": message with created_lt " << std::get<1>(a) + << " has emitted_lt" << std::get<2>(a) << ", but message with created_lt " + << std::get<1>(b) << " has emitted_lt" << std::get<2>(b)); + } + } return true; } +/** + * Checks if a special message is valid and exists in the incoming messages dictionary. + * A message is special if it recovers fees or mints extra currencies. + * + * @param in_msg_root The root of the message. + * @param amount The amount of currency recovered/minted. + * @param addr_cell The cell containing the destination address. + * + * @returns True if the special message is valid, false otherwise. + */ bool ValidateQuery::check_special_message(Ref in_msg_root, const block::CurrencyCollection& amount, Ref addr_cell) { if (in_msg_root.is_null()) { @@ -4769,37 +5957,54 @@ bool ValidateQuery::check_special_message(Ref in_msg_root, const block return true; } +/** + * Checks if all necessary special messages are valid and exist in the incoming messages dictionary. + * Used in masterchain validation. + * + * @returns True if special messages are valid, false otherwise. + */ bool ValidateQuery::check_special_messages() { return check_special_message(recover_create_msg_, value_flow_.recovered, config_->get_config_param(3, 1)) && check_special_message(mint_msg_, value_flow_.minted, config_->get_config_param(2, 0)); } +/** + * Checks if an update of LibDescr of as single library update is valid. + * Compares updates in LibDescr against updates of account states. + * Used in masterchain validation. + * + * @param key The 256-bit key of the library. + * @param old_value The old value of the LibDescr + * @param new_value The new value of the LibDescr. + * + * @returns True if the library update is valid, false otherwise. + */ bool ValidateQuery::check_one_library_update(td::ConstBitPtr key, Ref old_value, Ref new_value) { // shared_lib_descr$00 lib:^Cell publishers:(Hashmap 256 True) = LibDescr; std::unique_ptr old_publishers, new_publishers; if (new_value.not_null()) { - if (!block::gen::t_LibDescr.validate_csr(new_value)) { - return reject_query("LibDescr with key "s + key.to_hex(256) + - " in the libraries dictionary of the new state failed to pass automatic validity tests"); + block::gen::LibDescr::Record rec; + if (!block::gen::csr_unpack(std::move(new_value), rec)) { + return reject_query("failed to unpack LibDescr with key "s + key.to_hex(256) + + " in the libraries dictionary of the new state"); } - auto lib_ref = new_value->prefetch_ref(); - CHECK(lib_ref.not_null()); - if (lib_ref->get_hash().as_bitslice() != key) { + if (rec.lib->get_hash().as_bitslice() != key) { return reject_query("LibDescr with key "s + key.to_hex(256) + " in the libraries dictionary of the new state contains a library with different root hash " + - lib_ref->get_hash().to_hex()); + rec.lib->get_hash().to_hex()); } - CHECK(new_value.write().advance_ext(2, 1)); - new_publishers = std::make_unique(vm::DictNonEmpty(), std::move(new_value), 256); + new_publishers = std::make_unique(vm::DictNonEmpty(), std::move(rec.publishers), 256); } else { new_publishers = std::make_unique(256); } - if (old_value.not_null() && !block::gen::t_LibDescr.validate_csr(old_value)) { - return reject_query("LibDescr with key "s + key.to_hex(256) + - " in the libraries dictionary of the old state failed to pass automatic validity tests"); - CHECK(old_value.write().advance_ext(2, 1)); - old_publishers = std::make_unique(vm::DictNonEmpty(), std::move(old_value), 256); + if (old_value.not_null()) { + block::gen::LibDescr::Record rec; + if (!block::gen::csr_unpack(std::move(old_value), rec)) { + return reject_query("failed to unpack LibDescr with key "s + key.to_hex(256) + + " in the libraries dictionary of the old state"); + } + old_publishers = std::make_unique(vm::DictNonEmpty(), std::move(rec.publishers), 256); } else { old_publishers = std::make_unique(256); } @@ -4824,6 +6029,12 @@ bool ValidateQuery::check_one_library_update(td::ConstBitPtr key, Refscan_diff( @@ -4844,6 +6055,11 @@ bool ValidateQuery::check_shard_libraries() { return true; } +/** + * Checks the validity of the new shard state. + * + * @returns True if the new state is valid, false otherwise. + */ bool ValidateQuery::check_new_state() { LOG(INFO) << "checking header of the new shardchain state"; block::gen::ShardStateUnsplit::Record info; @@ -4938,6 +6154,15 @@ bool ValidateQuery::check_new_state() { return true; } +/** + * Checks if a masterchain configuration update is valid. + * Used in masterchain validation. + * + * @param old_conf_params The old configuration parameters. + * @param new_conf_params The new configuration parameters. + * + * @returns True if the update is valid, false otherwise. + */ bool ValidateQuery::check_config_update(Ref old_conf_params, Ref new_conf_params) { if (!block::gen::t_ConfigParams.validate_csr(10000, new_conf_params)) { return reject_query("new configuration failed to pass automated validity checks"); @@ -5042,6 +6267,16 @@ bool ValidateQuery::check_config_update(Ref old_conf_params, Ref< "reason (the suggested configuration appears to be valid)"); } +/** + * Checks if a single entry in the dictionary of previous masterchain blocks is valid and consistent. + * Used in masterchain validation. + * + * @param seqno The sequence number of the entry. + * @param old_val_extra The old value of the entry. + * @param new_val_extra The new value of the entry. + * + * @returns True if the update is valid and consistent, false otherwise. + */ bool ValidateQuery::check_one_prev_dict_update(ton::BlockSeqno seqno, Ref old_val_extra, Ref new_val_extra) { if (old_val_extra.not_null() && new_val_extra.is_null()) { @@ -5096,7 +6331,13 @@ bool ValidateQuery::check_one_prev_dict_update(ton::BlockSeqno seqno, Ref old_val, Ref new_val) { LOG(DEBUG) << "checking update of CreatorStats for "s + key.to_hex(256); @@ -5348,7 +6609,13 @@ bool ValidateQuery::check_one_block_creator_update(td::ConstBitPtr key, Refget_shard_hash(shard); @@ -5416,6 +6693,12 @@ bool ValidateQuery::check_one_shard_fee(ShardIdFull shard, const block::Currency return true; } +/** + * Checks the validity of the McBlockExtra in a masterchain block. + * Used in masterchain validation. + * + * @returns True if the data is valid, false otherwise. + */ bool ValidateQuery::check_mc_block_extra() { if (!is_masterchain()) { return true; @@ -5447,6 +6730,9 @@ bool ValidateQuery::check_mc_block_extra() { return reject_query("invalid fees_imported in value flow: declared "s + value_flow_.fees_imported.to_str() + ", correct value is " + fees_imported.to_str()); } + auto x = config_->get_burning_config().calculate_burned_fees(fees_imported - import_created_); + total_burned_ += x; + fees_burned_ += x; // ^[ prev_blk_signatures:(HashmapE 16 CryptoSignaturePair) if (prev_signatures_.not_null() && id_.seqno() == 1) { return reject_query("block contains non-empty signature set for the zero state of the masterchain"); @@ -5464,19 +6750,49 @@ bool ValidateQuery::check_mc_block_extra() { return true; } -/* - * - * MAIN VALIDATOR FUNCTION - * (invokes other methods in a suitable order) +/** + * Validates the value flow of a block. * + * @returns True if the value flow is valid, False otherwise. */ +bool ValidateQuery::postcheck_value_flow() { + auto expected_fees = + value_flow_.fees_imported + value_flow_.created + transaction_fees_ + import_fees_ - fees_burned_; + if (value_flow_.fees_collected != expected_fees) { + return reject_query(PSTRING() << "ValueFlow for " << id_.to_str() << " declares fees_collected=" + << value_flow_.fees_collected.to_str() << " but the total message import fees are " + << import_fees_ << ", the total transaction fees are " << transaction_fees_.to_str() + << ", creation fee for this block is " << value_flow_.created.to_str() + << ", the total imported fees from shards are " << value_flow_.fees_imported.to_str() + << " and the burned fees are " << fees_burned_.to_str() + << " with a total of " << expected_fees.to_str()); + } + if (total_burned_ != value_flow_.burned) { + return reject_query(PSTRING() << "invalid burned in value flow: " << id_.to_str() << " declared " + << value_flow_.burned.to_str() << ", correct value is " + << total_burned_.to_str()); + } + return true; +} +/** + * MAIN VALIDATOR FUNCTION (invokes other methods in a suitable order). + * + * @returns True if the validation is successful, False otherwise. + */ bool ValidateQuery::try_validate() { if (pending) { return true; } + work_timer_.resume(); + cpu_work_timer_.resume(); + SCOPE_EXIT { + work_timer_.pause(); + cpu_work_timer_.pause(); + }; try { if (!stage_) { + LOG(WARNING) << "try_validate stage 0"; if (!compute_prev_state()) { return fatal_error(-666, "cannot compute previous state"); } @@ -5501,13 +6817,17 @@ bool ValidateQuery::try_validate() { if (!check_utime_lt()) { return reject_query("creation utime/lt of the new block is invalid"); } + if (!prepare_out_msg_queue_size()) { + return reject_query("cannot request out msg queue size"); + } stage_ = 1; if (pending) { return true; } } + LOG(WARNING) << "try_validate stage 1"; LOG(INFO) << "running automated validity checks for block candidate " << id_.to_str(); - if (!block::gen::t_Block.validate_ref(1000000, block_root_)) { + if (!block::gen::t_Block.validate_ref(10000000, block_root_)) { return reject_query("block "s + id_.to_str() + " failed to pass automated validity checks"); } if (!fix_all_processed_upto()) { @@ -5528,21 +6848,28 @@ bool ValidateQuery::try_validate() { if (!precheck_message_queue_update()) { return reject_query("invalid OutMsgQueue update"); } + if (!unpack_dispatch_queue_update()) { + return reject_query("invalid DispatchQueue update"); + } if (!check_in_msg_descr()) { return reject_query("invalid InMsgDescr"); } if (!check_out_msg_descr()) { return reject_query("invalid OutMsgDescr"); } + if (!check_dispatch_queue_update()) { + return reject_query("invalid OutMsgDescr"); + } if (!check_processed_upto()) { return reject_query("invalid ProcessedInfo"); } if (!check_in_queue()) { return reject_query("cannot check inbound message queues"); } - if (!check_delivered_dequeued()) { + // Excessive check: validity of message in queue is checked elsewhere + /*if (!check_delivered_dequeued()) { return reject_query("cannot check delivery status of all outbound messages"); - } + }*/ if (!check_transactions()) { return reject_query("invalid collection of account transactions in ShardAccountBlocks"); } @@ -5564,6 +6891,9 @@ bool ValidateQuery::try_validate() { if (!check_mc_state_extra()) { return reject_query("new McStateExtra is invalid"); } + if (!postcheck_value_flow()) { + return reject_query("new ValueFlow is invalid"); + } } catch (vm::VmError& err) { return fatal_error(-666, err.get_msg()); } catch (vm::VmVirtError& err) { @@ -5572,6 +6902,11 @@ bool ValidateQuery::try_validate() { return save_candidate(); } +/** + * Saves the candidate to disk. + * + * @returns True. + */ bool ValidateQuery::save_candidate() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { @@ -5581,14 +6916,31 @@ bool ValidateQuery::save_candidate() { } }); - td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, id_, block_candidate.clone(), std::move(P)); + td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, id_, block_candidate.clone(), + validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), std::move(P)); return true; } +/** + * Callback function called after saving block candidate. + * Finishes validation. + */ void ValidateQuery::written_candidate() { finish_query(); } +/** + * Sends validation work time to manager. + */ +void ValidateQuery::record_stats(bool success) { + double work_time = work_timer_.elapsed(); + double cpu_work_time = cpu_work_timer_.elapsed(); + LOG(WARNING) << "validation took " << perf_timer_.elapsed() << "s"; + LOG(WARNING) << "Validate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; + td::actor::send_closure(manager, &ValidatorManager::record_validate_query_stats, block_candidate.id, work_time, + cpu_work_time, success); +} + } // namespace validator } // namespace ton diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 4ef02c86..60f0cc8a 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -28,6 +28,7 @@ #include #include #include +#include "common/global-version.h" namespace ton { @@ -108,14 +109,15 @@ inline ErrorCtxSet ErrorCtx::set_guard(std::vector str_list) { class ValidateQuery : public td::actor::Actor { static constexpr int supported_version() { - return 3; + return SUPPORTED_VERSION; } static constexpr long long supported_capabilities() { - return ton::capCreateStatsEnabled | ton::capBounceMsgBody | ton::capReportVersion | ton::capShortDequeue; + return ton::capCreateStatsEnabled | ton::capBounceMsgBody | ton::capReportVersion | ton::capShortDequeue | + ton::capStoreOutMsgQueueSize | ton::capMsgMetadata | ton::capDeferMessages; } public: - ValidateQuery(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, std::vector prev, + ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, BlockCandidate candidate, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, bool is_fake = false); @@ -125,7 +127,6 @@ class ValidateQuery : public td::actor::Actor { int pending{0}; const ShardIdFull shard_; const BlockIdExt id_; - UnixTime min_ts; BlockIdExt min_mc_block_id; std::vector prev_blocks; std::vector> prev_states; @@ -194,6 +195,7 @@ class ValidateQuery : public td::actor::Actor { ton::LogicalTime prev_key_block_lt_; std::unique_ptr block_limits_; std::unique_ptr block_limit_status_; + td::uint64 total_gas_used_{0}, total_special_gas_used_{0}; LogicalTime start_lt_, end_lt_; UnixTime prev_now_{~0u}, now_{~0u}; @@ -203,6 +205,7 @@ class ValidateQuery : public td::actor::Actor { block::StoragePhaseConfig storage_phase_cfg_{&storage_prices_}; block::ComputePhaseConfig compute_phase_cfg_; block::ActionPhaseConfig action_phase_cfg_; + block::SerializeConfig serialize_cfg_; td::RefInt256 masterchain_create_fee_, basechain_create_fee_; std::vector neighbors_; @@ -217,17 +220,29 @@ class ValidateQuery : public td::actor::Actor { std::unique_ptr in_msg_dict_, out_msg_dict_, account_blocks_dict_; block::ValueFlow value_flow_; - block::CurrencyCollection import_created_, transaction_fees_; + block::CurrencyCollection import_created_, transaction_fees_, total_burned_{0}, fees_burned_{0}; td::RefInt256 import_fees_; ton::LogicalTime proc_lt_{0}, claimed_proc_lt_{0}, min_enq_lt_{~0ULL}; - ton::Bits256 proc_hash_, claimed_proc_hash_, min_enq_hash_; - bool inbound_queues_empty_{false}; + ton::Bits256 proc_hash_ = ton::Bits256::zero(), claimed_proc_hash_, min_enq_hash_; std::vector> msg_proc_lt_; + std::vector> msg_emitted_lt_; std::vector> lib_publishers_, lib_publishers2_; + std::map, Ref> removed_dispatch_queue_messages_; + std::map, Ref> new_dispatch_queue_messages_; + std::set account_expected_defer_all_messages_; + td::uint64 old_out_msg_queue_size_ = 0, new_out_msg_queue_size_ = 0; + + bool msg_metadata_enabled_ = false; + bool deferring_messages_enabled_ = false; + bool store_out_msg_queue_size_ = false; + + td::uint64 processed_account_dispatch_queues_ = 0; + bool have_unprocessed_account_dispatch_queue_ = false; + td::PerfWarningTimer perf_timer_; static constexpr td::uint32 priority() { @@ -270,6 +285,7 @@ class ValidateQuery : public td::actor::Actor { return actor_id(this); } + void request_latest_mc_state(); void after_get_latest_mc_state(td::Result, BlockIdExt>> res); void after_get_mc_state(td::Result> res); void got_mc_handle(td::Result res); @@ -307,6 +323,8 @@ class ValidateQuery : public td::actor::Actor { bool check_cur_validator_set(); bool check_mc_validator_info(bool update_mc_cc); bool check_utime_lt(); + bool prepare_out_msg_queue_size(); + void got_out_queue_size(size_t i, td::Result res); bool fix_one_processed_upto(block::MsgProcessedUpto& proc, ton::ShardIdFull owner, bool allow_cur = false); bool fix_processed_upto(block::MsgProcessedUptoCollection& upto, bool allow_cur = false); @@ -328,6 +346,9 @@ class ValidateQuery : public td::actor::Actor { bool precheck_one_message_queue_update(td::ConstBitPtr out_msg_id, Ref old_value, Ref new_value); bool precheck_message_queue_update(); + bool check_account_dispatch_queue_update(td::Bits256 addr, Ref old_queue_csr, + Ref new_queue_csr); + bool unpack_dispatch_queue_update(); bool update_max_processed_lt_hash(ton::LogicalTime lt, const ton::Bits256& hash); bool update_min_enqueued_lt_hash(ton::LogicalTime lt, const ton::Bits256& hash); bool check_imported_message(Ref msg_env); @@ -336,13 +357,13 @@ class ValidateQuery : public td::actor::Actor { bool check_in_msg_descr(); bool check_out_msg(td::ConstBitPtr key, Ref out_msg); bool check_out_msg_descr(); + bool check_dispatch_queue_update(); bool check_processed_upto(); bool check_neighbor_outbound_message(Ref enq_msg, ton::LogicalTime lt, td::ConstBitPtr key, const block::McShardDescr& src_nb, bool& unprocessed); bool check_in_queue(); bool check_delivered_dequeued(); - std::unique_ptr make_account_from(td::ConstBitPtr addr, Ref account, - Ref extra); + std::unique_ptr make_account_from(td::ConstBitPtr addr, Ref account); std::unique_ptr unpack_account(td::ConstBitPtr addr); bool check_one_transaction(block::Account& account, LogicalTime lt, Ref trans_root, bool is_first, bool is_last); @@ -361,6 +382,7 @@ class ValidateQuery : public td::actor::Actor { bool check_one_prev_dict_update(ton::BlockSeqno seqno, Ref old_val_extra, Ref new_val_extra); bool check_mc_state_extra(); + bool postcheck_value_flow(); td::Status check_counter_update(const block::DiscountedCounter& oc, const block::DiscountedCounter& nc, unsigned expected_incr); bool check_one_block_creator_update(td::ConstBitPtr key, Ref old_val, Ref new_val); @@ -376,6 +398,10 @@ class ValidateQuery : public td::actor::Actor { } return true; } + + td::Timer work_timer_{true}; + td::ThreadCpuTimer cpu_work_timer_{true}; + void record_stats(bool success); }; } // namespace validator diff --git a/validator/impl/validator-set.cpp b/validator/impl/validator-set.cpp index 629337cf..d4b0d647 100644 --- a/validator/impl/validator-set.cpp +++ b/validator/impl/validator-set.cpp @@ -28,14 +28,14 @@ namespace ton { namespace validator { using td::Ref; -const ValidatorDescr *ValidatorSetQ::find_validator(const NodeIdShort &id) const { +const ValidatorDescr *ValidatorSetQ::get_validator(const NodeIdShort &id) const { auto it = std::lower_bound(ids_map_.begin(), ids_map_.end(), id, [](const auto &p, const auto &x) { return p.first < x; }); return it < ids_map_.end() && it->first == id ? &ids_[it->second] : nullptr; } bool ValidatorSetQ::is_validator(NodeIdShort id) const { - return find_validator(id); + return get_validator(id); } td::Result ValidatorSetQ::check_signatures(RootHash root_hash, FileHash file_hash, @@ -53,7 +53,7 @@ td::Result ValidatorSetQ::check_signatures(RootHash root_hash, } nodes.insert(sig.node); - auto vdescr = find_validator(sig.node); + auto vdescr = get_validator(sig.node); if (!vdescr) { return td::Status::Error(ErrorCode::protoviolation, "unknown node to sign"); } @@ -84,7 +84,7 @@ td::Result ValidatorSetQ::check_approve_signatures(RootHash roo } nodes.insert(sig.node); - auto vdescr = find_validator(sig.node); + auto vdescr = get_validator(sig.node); if (!vdescr) { return td::Status::Error(ErrorCode::protoviolation, "unknown node to sign"); } diff --git a/validator/impl/validator-set.hpp b/validator/impl/validator-set.hpp index 3141f36c..951ca4b7 100644 --- a/validator/impl/validator-set.hpp +++ b/validator/impl/validator-set.hpp @@ -32,6 +32,7 @@ namespace validator { class ValidatorSetQ : public ValidatorSet { public: + const ValidatorDescr* get_validator(const NodeIdShort& id) const override; bool is_validator(NodeIdShort id) const override; CatchainSeqno get_catchain_seqno() const override { return cc_seqno_; @@ -62,8 +63,6 @@ class ValidatorSetQ : public ValidatorSet { ValidatorWeight total_weight_; std::vector ids_; std::vector> ids_map_; - - const ValidatorDescr* find_validator(const NodeIdShort& id) const; }; class ValidatorSetCompute { diff --git a/validator/import-db-slice.cpp b/validator/import-db-slice.cpp index a93fb05b..06573d34 100644 --- a/validator/import-db-slice.cpp +++ b/validator/import-db-slice.cpp @@ -17,6 +17,7 @@ Copyright 2019-2020 Telegram Systems LLP */ #include "import-db-slice.hpp" + #include "validator/db/fileref.hpp" #include "td/utils/overloaded.h" #include "validator/fabric.h" @@ -26,35 +27,91 @@ #include "ton/ton-io.hpp" #include "downloaders/download-state.hpp" +#include + namespace ton { namespace validator { -ArchiveImporter::ArchiveImporter(std::string path, td::Ref state, BlockSeqno shard_client_seqno, +ArchiveImporter::ArchiveImporter(std::string db_root, td::Ref state, BlockSeqno shard_client_seqno, td::Ref opts, td::actor::ActorId manager, - td::Promise> promise) - : path_(std::move(path)) - , state_(std::move(state)) + std::vector to_import_files, + td::Promise> promise) + : db_root_(std::move(db_root)) + , last_masterchain_state_(std::move(state)) , shard_client_seqno_(shard_client_seqno) + , start_import_seqno_(shard_client_seqno + 1) , opts_(std::move(opts)) , manager_(manager) + , to_import_files_(std::move(to_import_files)) + , use_imported_files_(!to_import_files_.empty()) , promise_(std::move(promise)) { } void ArchiveImporter::start_up() { - auto R = Package::open(path_, false, false); - if (R.is_error()) { - abort_query(R.move_as_error()); + if (use_imported_files_) { + LOG(INFO) << "Importing archive for masterchain seqno #" << start_import_seqno_ << " from disk"; + for (const std::string& path : to_import_files_) { + LOG(INFO) << "Importing file from disk " << path; + td::Status S = process_package(path, true); + if (S.is_error()) { + LOG(INFO) << "Error processing package " << path << ": " << S; + } + } + files_to_cleanup_.clear(); + processed_mc_archive(); return; } - package_ = std::make_shared(R.move_as_ok()); + LOG(INFO) << "Importing archive for masterchain seqno #" << start_import_seqno_ << " from net"; + td::actor::send_closure(manager_, &ValidatorManager::send_download_archive_request, start_import_seqno_, + ShardIdFull{masterchainId}, db_root_ + "/tmp/", td::Timestamp::in(3600.0), + [SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporter::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveImporter::downloaded_mc_archive, R.move_as_ok()); + } + }); +} - bool fail = false; - package_->iterate([&](std::string filename, td::BufferSlice data, td::uint64 offset) -> bool { +void ArchiveImporter::downloaded_mc_archive(std::string path) { + td::Status S = process_package(path, true); + if (S.is_error()) { + abort_query(std::move(S)); + return; + } + processed_mc_archive(); +} + +void ArchiveImporter::processed_mc_archive() { + if (masterchain_blocks_.empty()) { + LOG(DEBUG) << "No masterhchain blocks in archive"; + last_masterchain_seqno_ = last_masterchain_state_->get_seqno(); + checked_all_masterchain_blocks(); + return; + } + + auto seqno = masterchain_blocks_.begin()->first; + LOG(DEBUG) << "First mc seqno in archive = " << seqno; + if (seqno > last_masterchain_state_->get_seqno() + 1) { + abort_query(td::Status::Error(ErrorCode::notready, "too big first masterchain seqno")); + return; + } + + check_masterchain_block(seqno); +} + +td::Status ArchiveImporter::process_package(std::string path, bool with_masterchain) { + LOG(DEBUG) << "Processing package " << path << " (with_masterchain=" << with_masterchain << ")"; + files_to_cleanup_.push_back(path); + TRY_RESULT(p, Package::open(path, false, false)); + auto package = std::make_shared(std::move(p)); + + td::Status S = td::Status::OK(); + package->iterate([&](std::string filename, td::BufferSlice, td::uint64 offset) -> bool { auto F = FileReference::create(filename); if (F.is_error()) { - abort_query(F.move_as_error()); - fail = true; + S = F.move_as_error(); return false; } auto f = F.move_as_ok(); @@ -79,33 +136,26 @@ void ArchiveImporter::start_up() { ignore = false; is_proof = false; }, - [&](const auto &p) { ignore = true; })); + [&](const auto &) { ignore = true; })); - if (!ignore) { - blocks_[b][is_proof ? 0 : 1] = offset; + if (!ignore && (with_masterchain || !b.is_masterchain())) { + if (is_proof) { + blocks_[b].proof_pkg = package; + blocks_[b].proof_offset = offset; + } else { + blocks_[b].data_pkg = package; + blocks_[b].data_offset = offset; + } if (b.is_masterchain()) { masterchain_blocks_[b.seqno()] = b; + last_masterchain_seqno_ = std::max(last_masterchain_seqno_, b.seqno()); + } else { + have_shard_blocks_ = true; } } return true; }); - - if (fail) { - return; - } - - if (masterchain_blocks_.size() == 0) { - abort_query(td::Status::Error(ErrorCode::notready, "archive does not contain any masterchain blocks")); - return; - } - - auto seqno = masterchain_blocks_.begin()->first; - if (seqno > state_->get_seqno() + 1) { - abort_query(td::Status::Error(ErrorCode::notready, "too big first masterchain seqno")); - return; - } - - check_masterchain_block(seqno); + return S; } void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { @@ -115,17 +165,17 @@ void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { abort_query(td::Status::Error(ErrorCode::notready, "no new blocks")); return; } - checked_all_masterchain_blocks(seqno - 1); + checked_all_masterchain_blocks(); return; } - while (seqno <= state_->get_block_id().seqno()) { - if (seqno < state_->get_block_id().seqno()) { - if (!state_->check_old_mc_block_id(it->second)) { + while (seqno <= last_masterchain_state_->get_block_id().seqno()) { + if (seqno < last_masterchain_state_->get_block_id().seqno()) { + if (!last_masterchain_state_->check_old_mc_block_id(it->second)) { abort_query(td::Status::Error(ErrorCode::protoviolation, "bad old masterchain block id")); return; } } else { - if (state_->get_block_id() != it->second) { + if (last_masterchain_state_->get_block_id() != it->second) { abort_query(td::Status::Error(ErrorCode::protoviolation, "bad old masterchain block id")); return; } @@ -133,18 +183,27 @@ void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { seqno++; it = masterchain_blocks_.find(seqno); if (it == masterchain_blocks_.end()) { - checked_all_masterchain_blocks(seqno - 1); + checked_all_masterchain_blocks(); return; } } - if (seqno != state_->get_block_id().seqno() + 1) { + LOG(DEBUG) << "Checking masterchain block #" << seqno; + if (seqno != last_masterchain_state_->get_block_id().seqno() + 1) { abort_query(td::Status::Error(ErrorCode::protoviolation, "hole in masterchain seqno")); return; } auto it2 = blocks_.find(it->second); CHECK(it2 != blocks_.end()); + if (!it2->second.proof_pkg) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block proof")); + return; + } + if (!it2->second.data_pkg) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block data")); + return; + } - auto R1 = package_->read(it2->second[0]); + auto R1 = it2->second.proof_pkg->read(it2->second.proof_offset); if (R1.is_error()) { abort_query(R1.move_as_error()); return; @@ -156,7 +215,7 @@ void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { return; } - auto R2 = package_->read(it2->second[1]); + auto R2 = it2->second.data_pkg->read(it2->second.data_offset); if (R2.is_error()) { abort_query(R2.move_as_error()); return; @@ -175,7 +234,7 @@ void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { auto proof = proofR.move_as_ok(); auto data = dataR.move_as_ok(); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id = state_->get_block_id(), + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id = last_masterchain_state_->get_block_id(), data](td::Result R) mutable { if (R.is_error()) { td::actor::send_closure(SelfId, &ArchiveImporter::abort_query, R.move_as_error()); @@ -191,11 +250,12 @@ void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { td::actor::send_closure(SelfId, &ArchiveImporter::checked_masterchain_proof, std::move(handle), std::move(data)); }); - run_check_proof_query(it->second, std::move(proof), manager_, td::Timestamp::in(2.0), std::move(P), state_, - opts_->is_hardfork(it->second)); + run_check_proof_query(it->second, std::move(proof), manager_, td::Timestamp::in(2.0), std::move(P), + last_masterchain_state_, opts_->is_hardfork(it->second)); } void ArchiveImporter::checked_masterchain_proof(BlockHandle handle, td::Ref data) { + LOG(DEBUG) << "Checked proof for masterchain block #" << handle->id().seqno(); CHECK(data.not_null()); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result R) { R.ensure(); @@ -205,6 +265,7 @@ void ArchiveImporter::checked_masterchain_proof(BlockHandle handle, td::Refid().seqno(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); td::actor::send_closure(SelfId, &ArchiveImporter::got_new_materchain_state, @@ -214,22 +275,87 @@ void ArchiveImporter::applied_masterchain_block(BlockHandle handle) { } void ArchiveImporter::got_new_materchain_state(td::Ref state) { - state_ = std::move(state); - check_masterchain_block(state_->get_block_id().seqno() + 1); + last_masterchain_state_ = std::move(state); + imported_any_ = true; + check_masterchain_block(last_masterchain_state_->get_block_id().seqno() + 1); } -void ArchiveImporter::checked_all_masterchain_blocks(BlockSeqno seqno) { - check_next_shard_client_seqno(shard_client_seqno_ + 1); +void ArchiveImporter::checked_all_masterchain_blocks() { + LOG(DEBUG) << "Done importing masterchain blocks. Last block seqno = " << last_masterchain_seqno_; + if (start_import_seqno_ > last_masterchain_state_->get_seqno()) { + abort_query(td::Status::Error("no new masterchain blocks were imported")); + return; + } + BlockIdExt block_id; + CHECK(last_masterchain_state_->get_old_mc_block_id(start_import_seqno_, block_id)); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db_short, block_id, + [SelfId = actor_id(this)](td::Result> R) { + R.ensure(); + td::Ref state{R.move_as_ok()}; + td::actor::send_closure(SelfId, &ArchiveImporter::download_shard_archives, + std::move(state)); + }); +} + +void ArchiveImporter::download_shard_archives(td::Ref start_state) { + start_state_ = start_state; + td::uint32 monitor_min_split = start_state->monitor_min_split_depth(basechainId); + LOG(DEBUG) << "Monitor min split = " << monitor_min_split; + // If monitor_min_split == 0, we use the old archive format (packages are not separated by shard) + // If masterchain package has shard blocks then it's old archive format, don't need to download shards + if (monitor_min_split > 0 && !have_shard_blocks_ && !use_imported_files_) { + for (td::uint64 i = 0; i < (1ULL << monitor_min_split); ++i) { + ShardIdFull shard_prefix{basechainId, (i * 2 + 1) << (64 - monitor_min_split - 1)}; + if (opts_->need_monitor(shard_prefix, start_state)) { + ++pending_shard_archives_; + LOG(DEBUG) << "Downloading shard archive #" << start_import_seqno_ << " " << shard_prefix.to_str(); + download_shard_archive(shard_prefix); + } + } + } else { + LOG(DEBUG) << "Skip downloading shard archives"; + } + if (pending_shard_archives_ == 0) { + check_next_shard_client_seqno(shard_client_seqno_ + 1); + } +} + +void ArchiveImporter::download_shard_archive(ShardIdFull shard_prefix) { + td::actor::send_closure( + manager_, &ValidatorManager::send_download_archive_request, start_import_seqno_, shard_prefix, db_root_ + "/tmp/", + td::Timestamp::in(3600.0), + [SelfId = actor_id(this), seqno = start_import_seqno_, shard_prefix](td::Result R) { + if (R.is_error()) { + LOG(WARNING) << "Failed to download archive slice #" << seqno << " for shard " << shard_prefix.to_str(); + delay_action( + [=]() { td::actor::send_closure(SelfId, &ArchiveImporter::download_shard_archive, shard_prefix); }, + td::Timestamp::in(2.0)); + } else { + LOG(DEBUG) << "Downloaded shard archive #" << seqno << " " << shard_prefix.to_str(); + td::actor::send_closure(SelfId, &ArchiveImporter::downloaded_shard_archive, R.move_as_ok()); + } + }); +} + +void ArchiveImporter::downloaded_shard_archive(std::string path) { + td::Status S = process_package(path, false); + if (S.is_error()) { + LOG(INFO) << "Error processing package: " << S; + } + --pending_shard_archives_; + if (pending_shard_archives_ == 0) { + check_next_shard_client_seqno(shard_client_seqno_ + 1); + } } void ArchiveImporter::check_next_shard_client_seqno(BlockSeqno seqno) { - if (seqno > state_->get_seqno()) { + if (seqno > last_masterchain_state_->get_seqno() || seqno > last_masterchain_seqno_) { finish_query(); - } else if (seqno == state_->get_seqno()) { - got_masterchain_state(state_); + } else if (seqno == last_masterchain_state_->get_seqno()) { + got_masterchain_state(last_masterchain_state_); } else { BlockIdExt b; - bool f = state_->get_old_mc_block_id(seqno, b); + bool f = last_masterchain_state_->get_old_mc_block_id(seqno, b); CHECK(f); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); @@ -241,33 +367,38 @@ void ArchiveImporter::check_next_shard_client_seqno(BlockSeqno seqno) { } void ArchiveImporter::got_masterchain_state(td::Ref state) { + if (state->get_seqno() != start_import_seqno_ && state->is_key_state()) { + finish_query(); + return; + } + LOG(DEBUG) << "Applying shard client seqno " << state->get_seqno(); auto s = state->get_shards(); - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), seqno = state->get_seqno()](td::Result R) { + td::MultiPromise mp; + auto ig = mp.init_guard(); + for (auto &shard : s) { + if (opts_->need_monitor(shard->shard(), state)) { + apply_shard_block(shard->top_block_id(), state->get_block_id(), ig.get_promise()); + } + } + ig.add_promise([SelfId = actor_id(this), seqno = state->get_seqno()](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &ArchiveImporter::abort_query, R.move_as_error()); } else { td::actor::send_closure(SelfId, &ArchiveImporter::checked_shard_client_seqno, seqno); } }); - - td::MultiPromise mp; - auto ig = mp.init_guard(); - ig.add_promise(std::move(P)); - - for (auto &shard : s) { - apply_shard_block(shard->top_block_id(), state->get_block_id(), ig.get_promise()); - } } void ArchiveImporter::checked_shard_client_seqno(BlockSeqno seqno) { CHECK(shard_client_seqno_ + 1 == seqno); shard_client_seqno_++; + imported_any_ = true; check_next_shard_client_seqno(seqno + 1); } void ArchiveImporter::apply_shard_block(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) { + LOG(DEBUG) << "Applying shard block " << block_id.id.to_str(); auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), masterchain_block_id, promise = std::move(promise)](td::Result R) mutable { R.ensure(); @@ -286,7 +417,7 @@ void ArchiveImporter::apply_shard_block_cont1(BlockHandle handle, BlockIdExt mas if (handle->id().seqno() == 0) { auto P = td::PromiseCreator::lambda( - [promise = std::move(promise)](td::Result> R) mutable { promise.set_value(td::Unit()); }); + [promise = std::move(promise)](td::Result>) mutable { promise.set_value(td::Unit()); }); td::actor::create_actor("downloadstate", handle->id(), masterchain_block_id, 2, manager_, td::Timestamp::in(3600), std::move(P)) .release(); @@ -294,12 +425,13 @@ void ArchiveImporter::apply_shard_block_cont1(BlockHandle handle, BlockIdExt mas } auto it = blocks_.find(handle->id()); - if (it == blocks_.end()) { - promise.set_error(td::Status::Error(ErrorCode::notready, PSTRING() << "no proof for shard block " << handle->id())); + if (it == blocks_.end() || !it->second.proof_pkg || !it->second.data_pkg) { + promise.set_error( + td::Status::Error(ErrorCode::notready, PSTRING() << "no data/proof for shard block " << handle->id())); return; } - TRY_RESULT_PROMISE(promise, data, package_->read(it->second[0])); - TRY_RESULT_PROMISE(promise, proof, create_proof_link(handle->id(), std::move(data.second))); + TRY_RESULT_PROMISE(promise, proof_data, it->second.proof_pkg->read(it->second.proof_offset)); + TRY_RESULT_PROMISE(promise, proof, create_proof_link(handle->id(), std::move(proof_data.second))); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle, masterchain_block_id, promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { @@ -345,8 +477,8 @@ void ArchiveImporter::apply_shard_block_cont2(BlockHandle handle, BlockIdExt mas void ArchiveImporter::apply_shard_block_cont3(BlockHandle handle, BlockIdExt masterchain_block_id, td::Promise promise) { auto it = blocks_.find(handle->id()); - CHECK(it != blocks_.end()); - TRY_RESULT_PROMISE(promise, data, package_->read(it->second[1])); + CHECK(it != blocks_.end() && it->second.data_pkg); + TRY_RESULT_PROMISE(promise, data, it->second.data_pkg->read(it->second.data_offset)); if (sha256_bits256(data.second.as_slice()) != handle->id().file_hash) { promise.set_error(td::Status::Error(ErrorCode::protoviolation, "bad block file hash")); return; @@ -367,6 +499,7 @@ void ArchiveImporter::check_shard_block_applied(BlockIdExt block_id, td::Promise if (!handle->is_applied()) { promise.set_error(td::Status::Error(ErrorCode::notready, "not applied")); } else { + LOG(DEBUG) << "Applied shard block " << handle->id().id.to_str(); promise.set_value(td::Unit()); } } @@ -375,13 +508,24 @@ void ArchiveImporter::check_shard_block_applied(BlockIdExt block_id, td::Promise } void ArchiveImporter::abort_query(td::Status error) { - LOG(INFO) << error; + if (!imported_any_) { + for (const std::string &f : files_to_cleanup_) { + td::unlink(f).ignore(); + } + promise_.set_error(std::move(error)); + return; + } + LOG(INFO) << "Archive import: " << error; finish_query(); } + void ArchiveImporter::finish_query() { + for (const std::string &f : files_to_cleanup_) { + td::unlink(f).ignore(); + } if (promise_) { - promise_.set_value( - std::vector{state_->get_seqno(), std::min(state_->get_seqno(), shard_client_seqno_)}); + promise_.set_value({last_masterchain_state_->get_seqno(), + std::min(last_masterchain_state_->get_seqno(), shard_client_seqno_)}); } stop(); } diff --git a/validator/import-db-slice.hpp b/validator/import-db-slice.hpp index 0993d4bb..04f22642 100644 --- a/validator/import-db-slice.hpp +++ b/validator/import-db-slice.hpp @@ -19,6 +19,7 @@ #pragma once #include "td/actor/actor.h" +#include "td/utils/port/path.h" #include "validator/interfaces/validator-manager.h" #include "validator/db/package.hpp" @@ -28,19 +29,27 @@ namespace validator { class ArchiveImporter : public td::actor::Actor { public: - ArchiveImporter(std::string path, td::Ref state, BlockSeqno shard_client_seqno, + ArchiveImporter(std::string db_root, td::Ref state, BlockSeqno shard_client_seqno, td::Ref opts, td::actor::ActorId manager, - td::Promise> promise); + std::vector to_import_files, td::Promise> promise); void start_up() override; void abort_query(td::Status error); void finish_query(); + void downloaded_mc_archive(std::string path); + td::Status process_package(std::string path, bool with_masterchain); + + void processed_mc_archive(); void check_masterchain_block(BlockSeqno seqno); void checked_masterchain_proof(BlockHandle handle, td::Ref data); void applied_masterchain_block(BlockHandle handle); void got_new_materchain_state(td::Ref state); - void checked_all_masterchain_blocks(BlockSeqno seqno); + + void checked_all_masterchain_blocks(); + void download_shard_archives(td::Ref start_state); + void download_shard_archive(ShardIdFull shard_prefix); + void downloaded_shard_archive(std::string path); void check_next_shard_client_seqno(BlockSeqno seqno); void checked_shard_client_seqno(BlockSeqno seqno); @@ -52,19 +61,36 @@ class ArchiveImporter : public td::actor::Actor { void check_shard_block_applied(BlockIdExt block_id, td::Promise promise); private: - std::string path_; - td::Ref state_; + std::string db_root_; + td::Ref last_masterchain_state_; BlockSeqno shard_client_seqno_; + BlockSeqno start_import_seqno_; td::Ref opts_; - std::shared_ptr package_; - td::actor::ActorId manager_; - td::Promise> promise_; + + std::vector to_import_files_; + bool use_imported_files_; + td::Promise> promise_; std::map masterchain_blocks_; - std::map> blocks_; + BlockSeqno last_masterchain_seqno_ = 0; + + struct BlockInfo { + std::shared_ptr data_pkg; + td::uint64 data_offset = 0; + std::shared_ptr proof_pkg; + td::uint64 proof_offset = 0; + }; + std::map blocks_; + + td::Ref start_state_; + size_t pending_shard_archives_ = 0; + + bool imported_any_ = false; + bool have_shard_blocks_ = false; + std::vector files_to_cleanup_; }; } // namespace validator diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index ba4d9dda..29ef715b 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -46,6 +46,7 @@ class Db : public td::actor::Actor { virtual void store_block_candidate(BlockCandidate candidate, td::Promise promise) = 0; virtual void get_block_candidate(ton::PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) = 0; + virtual void get_block_candidate_by_block_id(BlockIdExt id, td::Promise promise) = 0; virtual void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; @@ -66,6 +67,8 @@ class Db : public td::actor::Actor { virtual void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) = 0; virtual void get_zero_state_file(BlockIdExt block_id, td::Promise promise) = 0; virtual void check_zero_state_file_exists(BlockIdExt block_id, td::Promise promise) = 0; + virtual void get_previous_persistent_state_files( + BlockSeqno cur_mc_seqno, td::Promise>> promise) = 0; virtual void try_get_static_file(FileHash file_hash, td::Promise promise) = 0; @@ -114,12 +117,18 @@ class Db : public td::actor::Actor { virtual void check_key_block_proof_exists(BlockIdExt block_id, td::Promise promise) = 0; virtual void check_key_block_proof_link_exists(BlockIdExt block_id, td::Promise promise) = 0; - virtual void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) = 0; + virtual void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) = 0; virtual void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise) = 0; virtual void set_async_mode(bool mode, td::Promise promise) = 0; - virtual void run_gc(UnixTime ts, UnixTime archive_ttl) = 0; + virtual void run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) = 0; + + virtual void add_persistent_state_description(td::Ref desc, + td::Promise promise) = 0; + virtual void get_persistent_state_descriptions( + td::Promise>> promise) = 0; }; } // namespace validator diff --git a/validator/interfaces/liteserver.h b/validator/interfaces/liteserver.h index 3e803029..f2f78d41 100644 --- a/validator/interfaces/liteserver.h +++ b/validator/interfaces/liteserver.h @@ -19,16 +19,20 @@ #pragma once #include "td/actor/actor.h" +#include "td/utils/buffer.h" +#include "common/bitstring.h" -namespace ton { - -namespace validator { +namespace ton::validator { class LiteServerCache : public td::actor::Actor { public: - virtual ~LiteServerCache() = default; + ~LiteServerCache() override = default; + + virtual void lookup(td::Bits256 key, td::Promise promise) = 0; + virtual void update(td::Bits256 key, td::BufferSlice value) = 0; + + virtual void process_send_message(td::Bits256 key, td::Promise promise) = 0; + virtual void drop_send_message_from_cache(td::Bits256 key) = 0; }; -} // namespace validator - -} // namespace ton +} // namespace ton::validator \ No newline at end of file diff --git a/validator/interfaces/out-msg-queue-proof.h b/validator/interfaces/out-msg-queue-proof.h new file mode 100644 index 00000000..c0aa5610 --- /dev/null +++ b/validator/interfaces/out-msg-queue-proof.h @@ -0,0 +1,57 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "vm/cells.h" +#include "ton/ton-types.h" +#include "auto/tl/ton_api.h" +#include "block/block.h" + +namespace ton { + +namespace validator { +using td::Ref; + +struct OutMsgQueueProof : public td::CntObject { + OutMsgQueueProof(BlockIdExt block_id, Ref state_root, Ref block_state_proof, + td::int32 msg_count = -1) + : block_id_(block_id) + , state_root_(std::move(state_root)) + , block_state_proof_(std::move(block_state_proof)) + , msg_count_(msg_count) { + } + + BlockIdExt block_id_; + Ref state_root_; + Ref block_state_proof_; + td::int32 msg_count_; // -1 - no limit + + static td::Result>> fetch(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + const ton_api::tonNode_outMsgQueueProof &f); + + struct OneBlock { + BlockIdExt id; + Ref state_root; + Ref block_root; + }; + static td::Result> build(ShardIdFull dst_shard, + std::vector blocks, + block::ImportedMsgQueueLimits limits); +}; + +} // namespace validator +} // namespace ton diff --git a/validator/interfaces/proof.h b/validator/interfaces/proof.h index 99471a1f..6665ad08 100644 --- a/validator/interfaces/proof.h +++ b/validator/interfaces/proof.h @@ -48,6 +48,9 @@ class Proof : virtual public ProofLink { virtual td::Result> export_as_proof_link() const = 0; }; +td::Result> create_block_state_proof(td::Ref root); +td::Result unpack_block_state_proof(BlockIdExt block_id, td::Ref proof); + } // namespace validator } // namespace ton diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index 4a0e42fe..64aea9b6 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -39,11 +39,13 @@ class ShardState : public td::CntObject { virtual UnixTime get_unix_time() const = 0; virtual LogicalTime get_logical_time() const = 0; + virtual td::int32 get_global_id() const = 0; virtual ShardIdFull get_shard() const = 0; virtual BlockSeqno get_seqno() const = 0; virtual BlockIdExt get_block_id() const = 0; virtual RootHash root_hash() const = 0; virtual td::Ref root_cell() const = 0; + virtual td::optional get_master_ref() const = 0; virtual td::Status validate_deep() const = 0; @@ -69,18 +71,20 @@ class MasterchainState : virtual public ShardState { virtual std::vector> get_shards() const = 0; virtual td::Ref get_shard_from_config(ShardIdFull shard) const = 0; virtual bool workchain_is_active(WorkchainId workchain_id) const = 0; + virtual td::uint32 monitor_min_split_depth(WorkchainId workchain_id) const = 0; virtual td::uint32 min_split_depth(WorkchainId workchain_id) const = 0; - virtual td::uint32 soft_min_split_depth(WorkchainId workchain_id) const = 0; virtual BlockSeqno min_ref_masterchain_seqno() const = 0; virtual bool ancestor_is_valid(BlockIdExt id) const = 0; virtual ValidatorSessionConfig get_consensus_config() const = 0; virtual BlockIdExt last_key_block_id() const = 0; virtual BlockIdExt next_key_block_id(BlockSeqno seqno) const = 0; virtual BlockIdExt prev_key_block_id(BlockSeqno seqno) const = 0; + virtual bool is_key_state() const = 0; virtual bool get_old_mc_block_id(ton::BlockSeqno seqno, ton::BlockIdExt& blkid, ton::LogicalTime* end_lt = nullptr) const = 0; virtual bool check_old_mc_block_id(const ton::BlockIdExt& blkid, bool strict = false) const = 0; virtual td::Result> get_config_holder() const = 0; + virtual block::WorkchainSet get_workchain_list() const = 0; virtual td::Status prepare() { return td::Status::OK(); } diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index ae15ed4c..00fb77e1 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -29,6 +29,7 @@ #include "liteserver.h" #include "crypto/vm/db/DynamicBagOfCellsDb.h" #include "validator-session/validator-session-types.h" +#include "auto/tl/lite_api.h" namespace ton { @@ -51,10 +52,22 @@ struct AsyncSerializerState { UnixTime last_written_block_ts; }; +struct CollationStats { + td::uint32 bytes, gas, lt_delta; + int cat_bytes, cat_gas, cat_lt_delta; + std::string limits_log; + td::uint32 ext_msgs_total = 0; + td::uint32 ext_msgs_filtered = 0; + td::uint32 ext_msgs_accepted = 0; + td::uint32 ext_msgs_rejected = 0; +}; + using ValidateCandidateResult = td::Variant; class ValidatorManager : public ValidatorManagerInterface { public: + virtual void init_last_masterchain_state(td::Ref state) { + } virtual void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; virtual void get_cell_db_reader(td::Promise> promise) = 0; @@ -64,10 +77,6 @@ class ValidatorManager : public ValidatorManagerInterface { std::function write_data, td::Promise promise) = 0; virtual void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) = 0; - virtual void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) = 0; - virtual void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) = 0; virtual void set_block_data(BlockHandle handle, td::Ref data, td::Promise promise) = 0; virtual void wait_block_data(BlockHandle handle, td::uint32 priority, td::Timestamp, @@ -92,7 +101,10 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void wait_block_signatures_short(BlockIdExt id, td::Timestamp timeout, td::Promise> promise) = 0; - virtual void set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) = 0; + virtual void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) = 0; + virtual void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) = 0; virtual void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) = 0; @@ -103,7 +115,8 @@ class ValidatorManager : public ValidatorManagerInterface { td::Promise> promise) = 0; virtual void wait_block_message_queue_short(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) = 0; - virtual void get_external_messages(ShardIdFull shard, td::Promise>> promise) = 0; + virtual void get_external_messages(ShardIdFull shard, + td::Promise, int>>> promise) = 0; virtual void get_ihr_messages(ShardIdFull shard, td::Promise>> promise) = 0; virtual void get_shard_blocks(BlockIdExt masterchain_block_id, td::Promise>> promise) = 0; @@ -131,11 +144,16 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_external_message(td::Ref message) = 0; virtual void send_ihr_message(td::Ref message) = 0; virtual void send_top_shard_block_description(td::Ref desc) = 0; - virtual void send_block_broadcast(BlockBroadcast broadcast) = 0; + virtual void send_block_broadcast(BlockBroadcast broadcast, int mode) = 0; + virtual void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) = 0; + virtual void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + td::Promise>> promise) = 0; + virtual void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) = 0; virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_shard_client_state(bool from_db, td::Promise promise) = 0; - virtual void subscribe_to_shard(ShardIdFull shard) = 0; virtual void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) = 0; virtual void get_async_serializer_state(td::Promise promise) = 0; @@ -169,6 +187,34 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) = 0; virtual void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) = 0; + virtual void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) = 0; + virtual void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) = 0; + + virtual void get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) = 0; + virtual void get_block_data_for_litequery(BlockIdExt block_id, td::Promise> promise) = 0; + virtual void get_block_state_for_litequery(BlockIdExt block_id, td::Promise> promise) = 0; + virtual void get_block_by_lt_for_litequery(AccountIdPrefixFull account, LogicalTime lt, + td::Promise promise) = 0; + virtual void get_block_by_unix_time_for_litequery(AccountIdPrefixFull account, UnixTime ts, + td::Promise promise) = 0; + virtual void get_block_by_seqno_for_litequery(AccountIdPrefixFull account, BlockSeqno seqno, + td::Promise promise) = 0; + virtual void get_block_candidate_for_litequery(PublicKey source, BlockIdExt block_id, FileHash collated_data_hash, + td::Promise promise) = 0; + virtual void get_validator_groups_info_for_litequery( + td::optional shard, + td::Promise> promise) = 0; + + virtual void add_lite_query_stats(int lite_query_id, bool success) { + } + + virtual void record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, + td::optional stats) { + } + virtual void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, bool success) { + } + + virtual void add_persistent_state_description(td::Ref desc) = 0; static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { return ts / (1 << 17) != prev_ts / (1 << 17); diff --git a/validator/interfaces/validator-set.h b/validator/interfaces/validator-set.h index b71c0bfe..ad7fb9b5 100644 --- a/validator/interfaces/validator-set.h +++ b/validator/interfaces/validator-set.h @@ -30,6 +30,7 @@ namespace validator { class ValidatorSet : public td::CntObject { public: virtual ~ValidatorSet() = default; + virtual const ValidatorDescr* get_validator(const NodeIdShort& id) const = 0; virtual bool is_validator(NodeIdShort id) const = 0; virtual CatchainSeqno get_catchain_seqno() const = 0; virtual td::uint32 get_validator_set_hash() const = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 8818c86a..62fdc4b4 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -128,8 +128,8 @@ void ValidatorManagerImpl::sync_complete(td::Promise promise) { } Ed25519_PublicKey created_by{td::Bits256::zero()}; td::as(created_by.as_bits256().data() + 32 - 4) = ((unsigned)std::time(nullptr) >> 8); - run_collate_query(shard_id, 0, last_masterchain_block_id_, prev, created_by, val_set, actor_id(this), - td::Timestamp::in(10.0), std::move(P)); + run_collate_query(shard_id, last_masterchain_block_id_, prev, created_by, val_set, td::Ref{true}, + actor_id(this), td::Timestamp::in(10.0), std::move(P), td::CancellationToken{}, 0); } void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector prev, BlockIdExt last, @@ -152,7 +152,7 @@ void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector>> promise) { - promise.set_result(ext_messages_); +void ValidatorManagerImpl::get_external_messages( + ShardIdFull shard, td::Promise, int>>> promise) { + std::vector, int>> res; + for (const auto& x : ext_messages_) { + res.emplace_back(x, 0); + } + promise.set_result(std::move(res)); } void ValidatorManagerImpl::get_ihr_messages(ShardIdFull shard, td::Promise>> promise) { @@ -585,6 +589,11 @@ void ValidatorManagerImpl::get_block_candidate_from_db(PublicKey source, BlockId td::actor::send_closure(db_, &Db::get_block_candidate, source, id, collated_data_file_hash, std::move(promise)); } +void ValidatorManagerImpl::get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) { + td::actor::send_closure(db_, &Db::get_block_candidate_by_block_id, id, + promise.wrap([](BlockCandidate &&b) { return std::move(b.data); })); +} + void ValidatorManagerImpl::get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) { td::actor::send_closure(db_, &Db::get_block_proof, std::move(handle), std::move(promise)); } @@ -771,10 +780,16 @@ void ValidatorManagerImpl::set_next_block(BlockIdExt block_id, BlockIdExt next, get_block_handle(block_id, true, std::move(P)); } -void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) { +void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) { td::actor::send_closure(db_, &Db::store_block_candidate, std::move(candidate), std::move(promise)); } +void ValidatorManagerImpl::send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) { + callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data)); +} + void ValidatorManagerImpl::write_handle(BlockHandle handle, td::Promise promise) { td::actor::send_closure(db_, &Db::store_block_handle, std::move(handle), std::move(promise)); } @@ -901,7 +916,7 @@ void ValidatorManagerImpl::send_top_shard_block_description(td::Ref R) { R.ensure(); diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 2745e02e..cd06bf55 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -23,6 +23,7 @@ #include "validator-group.hpp" #include "manager-init.h" #include "manager-disk.h" +#include "queue-size-counter.hpp" #include #include @@ -115,6 +116,10 @@ class ValidatorManagerImpl : public ValidatorManager { td::int64 max_length, td::Promise promise) override { UNREACHABLE(); } + void get_previous_persistent_state_files( + BlockSeqno cur_mc_seqno, td::Promise>> promise) override { + UNREACHABLE(); + } void get_block_proof(BlockHandle handle, td::Promise promise) override; void get_block_proof_link(BlockHandle block_id, td::Promise promise) override { UNREACHABLE(); @@ -123,12 +128,14 @@ class ValidatorManagerImpl : public ValidatorManager { void get_key_block_proof_link(BlockIdExt block_id, td::Promise promise) override; //void get_block_description(BlockIdExt block_id, td::Promise promise) override; - void new_external_message(td::BufferSlice data) override; + void new_external_message(td::BufferSlice data, int priority) override; void check_external_message(td::BufferSlice data, td::Promise> promise) override { UNREACHABLE(); } void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; + void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override { + } void add_ext_server_id(adnl::AdnlNodeIdShort id) override { UNREACHABLE(); @@ -176,7 +183,10 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_block_signatures_short(BlockIdExt id, td::Timestamp timeout, td::Promise> promise) override; - void set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) override; + void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) override; + void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override; void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; @@ -187,7 +197,8 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void wait_block_message_queue_short(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; - void get_external_messages(ShardIdFull shard, td::Promise>> promise) override; + void get_external_messages(ShardIdFull shard, + td::Promise, int>>> promise) override; void get_ihr_messages(ShardIdFull shard, td::Promise>> promise) override; void get_shard_blocks(BlockIdExt masterchain_block_id, td::Promise>> promise) override; @@ -204,6 +215,7 @@ class ValidatorManagerImpl : public ValidatorManager { void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) override; void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) override; + void get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) override; void get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) override; void get_block_proof_from_db_short(BlockIdExt id, td::Promise> promise) override; void get_block_proof_link_from_db(ConstBlockHandle handle, td::Promise> promise) override; @@ -243,19 +255,28 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } void send_external_message(td::Ref message) override { - new_external_message(message->serialize()); + new_external_message(message->serialize(), 0); } void send_ihr_message(td::Ref message) override { new_ihr_message(message->serialize()); } void send_top_shard_block_description(td::Ref desc) override; - void send_block_broadcast(BlockBroadcast broadcast) override { + void send_block_broadcast(BlockBroadcast broadcast, int mode) override { + } + void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override { + } + void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + td::Promise>> promise) override { + UNREACHABLE(); + } + void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override { + UNREACHABLE(); } void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; - void subscribe_to_shard(ShardIdFull shard) override { - } void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override { UNREACHABLE(); @@ -267,11 +288,12 @@ class ValidatorManagerImpl : public ValidatorManager { void try_get_static_file(FileHash file_hash, td::Promise promise) override; void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) override { + td::Promise> promise) override { promise.set_error(td::Status::Error(ErrorCode::error, "download disabled")); } - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) override { + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) override { UNREACHABLE(); } void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, @@ -360,6 +382,10 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } + void prepare_actor_stats(td::Promise promise) override { + UNREACHABLE(); + } + void prepare_perf_timer_stats(td::Promise> promise) override { UNREACHABLE(); } @@ -376,6 +402,55 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override { UNREACHABLE(); } + void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override { + UNREACHABLE(); + } + void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override { + UNREACHABLE(); + } + void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { + if (queue_size_counter_.empty()) { + queue_size_counter_ = td::actor::create_actor("queuesizecounter", td::Ref{}, + opts_, actor_id(this)); + } + td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::get_queue_size, block_id, std::move(promise)); + } + void get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) override { + get_block_handle(block_id, false, promise.wrap([](BlockHandle &&handle) -> ConstBlockHandle { return handle; })); + } + void get_block_data_for_litequery(BlockIdExt block_id, td::Promise> promise) override { + get_block_data_from_db_short(block_id, std::move(promise)); + } + void get_block_state_for_litequery(BlockIdExt block_id, td::Promise> promise) override { + get_shard_state_from_db_short(block_id, std::move(promise)); + } + void get_block_by_lt_for_litequery(AccountIdPrefixFull account, LogicalTime lt, + td::Promise promise) override { + get_block_by_lt_from_db(account, lt, std::move(promise)); + } + void get_block_by_unix_time_for_litequery(AccountIdPrefixFull account, UnixTime ts, + td::Promise promise) override { + get_block_by_unix_time_from_db(account, ts, std::move(promise)); + } + void get_block_by_seqno_for_litequery(AccountIdPrefixFull account, BlockSeqno seqno, + td::Promise promise) override { + get_block_by_seqno_from_db(account, seqno, std::move(promise)); + } + void get_block_candidate_for_litequery(PublicKey source, BlockIdExt block_id, FileHash collated_data_hash, + td::Promise promise) override { + promise.set_result(td::Status::Error("not implemented")); + } + void get_validator_groups_info_for_litequery( + td::optional shard, + td::Promise> promise) override { + promise.set_result(td::Status::Error("not implemented")); + } + void add_persistent_state_description(td::Ref desc) override { + } + + void update_options(td::Ref opts) override { + opts_ = std::move(opts); + } private: PublicKeyHash local_id_; @@ -393,6 +468,7 @@ class ValidatorManagerImpl : public ValidatorManager { int pending_new_shard_block_descr_{0}; std::vector>>> waiting_new_shard_block_descr_; + td::actor::ActorOwn queue_size_counter_; void update_shards(); void update_shard_blocks(); diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp index 80a64d25..91c598aa 100644 --- a/validator/manager-hardfork.cpp +++ b/validator/manager-hardfork.cpp @@ -150,7 +150,7 @@ void ValidatorManagerImpl::get_key_block_proof_link(BlockIdExt block_id, td::Pro td::actor::send_closure(db_, &Db::get_key_block_proof_link, block_id, std::move(P)); } -void ValidatorManagerImpl::new_external_message(td::BufferSlice data) { +void ValidatorManagerImpl::new_external_message(td::BufferSlice data, int priority) { auto R = create_ext_message(std::move(data), block::SizeLimitsConfig::ExtMsgLimits()); if (R.is_ok()) { ext_messages_.emplace_back(R.move_as_ok()); @@ -357,9 +357,13 @@ void ValidatorManagerImpl::wait_block_message_queue_short(BlockIdExt block_id, t get_block_handle(block_id, true, std::move(P)); } -void ValidatorManagerImpl::get_external_messages(ShardIdFull shard, - td::Promise>> promise) { - promise.set_result(ext_messages_); +void ValidatorManagerImpl::get_external_messages( + ShardIdFull shard, td::Promise, int>>> promise) { + std::vector, int>> res; + for (const auto &x : ext_messages_) { + res.emplace_back(x, 0); + } + promise.set_result(std::move(res)); } void ValidatorManagerImpl::get_ihr_messages(ShardIdFull shard, td::Promise>> promise) { @@ -411,6 +415,11 @@ void ValidatorManagerImpl::get_block_candidate_from_db(PublicKey source, BlockId td::actor::send_closure(db_, &Db::get_block_candidate, source, id, collated_data_file_hash, std::move(promise)); } +void ValidatorManagerImpl::get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) { + td::actor::send_closure(db_, &Db::get_block_candidate_by_block_id, id, + promise.wrap([](BlockCandidate &&b) { return std::move(b.data); })); +} + void ValidatorManagerImpl::get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) { td::actor::send_closure(db_, &Db::get_block_proof, std::move(handle), std::move(promise)); } @@ -549,7 +558,7 @@ void ValidatorManagerImpl::register_block_handle(BlockHandle handle, td::Promise } void ValidatorManagerImpl::start_up() { - db_ = create_db_actor(actor_id(this), db_root_); + db_ = create_db_actor(actor_id(this), db_root_, opts_); } void ValidatorManagerImpl::try_get_static_file(FileHash file_hash, td::Promise promise) { diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 6a145191..0b8b9e73 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -20,9 +20,11 @@ #include "interfaces/validator-manager.h" #include "interfaces/db.h" +#include "ton/ton-types.h" #include "validator-group.hpp" #include "manager-init.h" #include "manager-hardfork.h" +#include "queue-size-counter.hpp" #include #include @@ -138,12 +140,16 @@ class ValidatorManagerImpl : public ValidatorManager { td::int64 max_length, td::Promise promise) override { UNREACHABLE(); } + void get_previous_persistent_state_files( + BlockSeqno cur_mc_seqno, td::Promise>> promise) override { + UNREACHABLE(); + } void get_block_proof(BlockHandle handle, td::Promise promise) override; void get_block_proof_link(BlockHandle block_id, td::Promise promise) override; void get_key_block_proof(BlockIdExt block_id, td::Promise promise) override; void get_key_block_proof_link(BlockIdExt block_id, td::Promise promise) override; - void new_external_message(td::BufferSlice data) override; + void new_external_message(td::BufferSlice data, int priority) override; void check_external_message(td::BufferSlice data, td::Promise> promise) override { UNREACHABLE(); } @@ -151,6 +157,9 @@ class ValidatorManagerImpl : public ValidatorManager { void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override { UNREACHABLE(); } + void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override { + UNREACHABLE(); + } void add_ext_server_id(adnl::AdnlNodeIdShort id) override { UNREACHABLE(); @@ -214,9 +223,14 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_block_signatures_short(BlockIdExt id, td::Timestamp timeout, td::Promise> promise) override; - void set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) override { + void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) override { promise.set_value(td::Unit()); } + void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) { + callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data)); + } void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; @@ -227,7 +241,8 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void wait_block_message_queue_short(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; - void get_external_messages(ShardIdFull shard, td::Promise>> promise) override; + void get_external_messages(ShardIdFull shard, + td::Promise, int>>> promise) override; void get_ihr_messages(ShardIdFull shard, td::Promise>> promise) override; void get_shard_blocks(BlockIdExt masterchain_block_id, td::Promise>> promise) override; @@ -248,6 +263,7 @@ class ValidatorManagerImpl : public ValidatorManager { void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) override; void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) override; + void get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) override; void get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) override; void get_block_proof_from_db_short(BlockIdExt id, td::Promise> promise) override; void get_block_proof_link_from_db(ConstBlockHandle handle, td::Promise> promise) override; @@ -307,7 +323,7 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } void send_external_message(td::Ref message) override { - new_external_message(message->serialize()); + new_external_message(message->serialize(), 0); } void send_ihr_message(td::Ref message) override { new_ihr_message(message->serialize()); @@ -315,7 +331,18 @@ class ValidatorManagerImpl : public ValidatorManager { void send_top_shard_block_description(td::Ref desc) override { UNREACHABLE(); } - void send_block_broadcast(BlockBroadcast broadcast) override { + void send_block_broadcast(BlockBroadcast broadcast, int mode) override { + } + void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override { + } + void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + td::Promise>> promise) override { + UNREACHABLE(); + } + void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override { + UNREACHABLE(); } void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override { @@ -324,8 +351,6 @@ class ValidatorManagerImpl : public ValidatorManager { void get_shard_client_state(bool from_db, td::Promise promise) override { UNREACHABLE(); } - void subscribe_to_shard(ShardIdFull shard) override { - } void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override { UNREACHABLE(); @@ -337,11 +362,12 @@ class ValidatorManagerImpl : public ValidatorManager { void try_get_static_file(FileHash file_hash, td::Promise promise) override; void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) override { + td::Promise> promise) override { promise.set_error(td::Status::Error(ErrorCode::error, "download disabled")); } - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) override { + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) override { UNREACHABLE(); } void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, @@ -421,6 +447,10 @@ class ValidatorManagerImpl : public ValidatorManager { UNREACHABLE(); } + void prepare_actor_stats(td::Promise promise) override { + UNREACHABLE(); + } + void prepare_perf_timer_stats(td::Promise> promise) override { UNREACHABLE(); } @@ -437,6 +467,54 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override { UNREACHABLE(); } + void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override { + UNREACHABLE(); + } + void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override { + UNREACHABLE(); + } + void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { + if (queue_size_counter_.empty()) { + queue_size_counter_ = td::actor::create_actor("queuesizecounter", td::Ref{}, + opts_, actor_id(this)); + } + td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::get_queue_size, block_id, std::move(promise)); + } + void get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) override { + get_block_handle(block_id, false, promise.wrap([](BlockHandle &&handle) -> ConstBlockHandle { return handle; })); + } + void get_block_data_for_litequery(BlockIdExt block_id, td::Promise> promise) override { + get_block_data_from_db_short(block_id, std::move(promise)); + } + void get_block_state_for_litequery(BlockIdExt block_id, td::Promise> promise) override { + get_shard_state_from_db_short(block_id, std::move(promise)); + } + void get_block_by_lt_for_litequery(AccountIdPrefixFull account, LogicalTime lt, + td::Promise promise) override { + get_block_by_lt_from_db(account, lt, std::move(promise)); + } + void get_block_by_unix_time_for_litequery(AccountIdPrefixFull account, UnixTime ts, + td::Promise promise) override { + get_block_by_unix_time_from_db(account, ts, std::move(promise)); + } + void get_block_by_seqno_for_litequery(AccountIdPrefixFull account, BlockSeqno seqno, + td::Promise promise) override { + get_block_by_seqno_from_db(account, seqno, std::move(promise)); + } + void get_block_candidate_for_litequery(PublicKey source, BlockIdExt block_id, FileHash collated_data_hash, + td::Promise promise) override { + promise.set_result(td::Status::Error("not implemented")); + } + void get_validator_groups_info_for_litequery( + td::optional shard, + td::Promise> promise) override { + promise.set_result(td::Status::Error("not implemented")); + } + void update_options(td::Ref opts) override { + opts_ = std::move(opts); + } + void add_persistent_state_description(td::Ref desc) override { + } private: td::Ref opts_; @@ -445,6 +523,7 @@ class ValidatorManagerImpl : public ValidatorManager { std::string db_root_; ShardIdFull shard_to_generate_; BlockIdExt block_to_generate_; + td::actor::ActorOwn queue_size_counter_; }; } // namespace validator diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index aa110380..6f304680 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -32,6 +32,8 @@ namespace ton { namespace validator { void ValidatorManagerMasterchainReiniter::start_up() { + status_ = ProcessStatus(manager_, "process.initial_sync"); + status_.set_status(PSTRING() << "starting, init block seqno " << block_id_.seqno()); LOG(INFO) << "init_block_id=" << block_id_; CHECK(block_id_.is_masterchain()); CHECK(block_id_.id.shard == shardIdAll); @@ -58,6 +60,7 @@ void ValidatorManagerMasterchainReiniter::got_masterchain_handle(BlockHandle han key_blocks_.push_back(handle_); if (opts_->initial_sync_disabled()) { + status_.set_status(PSTRING() << "downloading masterchain state " << handle_->id().seqno()); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_masterchain_state); @@ -181,6 +184,7 @@ void ValidatorManagerMasterchainReiniter::got_next_key_blocks(std::vector(key_blocks_.size()); key_blocks_.resize(key_blocks_.size() + vec.size(), nullptr); @@ -227,7 +231,7 @@ void ValidatorManagerMasterchainReiniter::choose_masterchain_state() { } if (!p || ValidatorManager::is_persistent_state(h->unix_time(), p->unix_time())) { auto ttl = ValidatorManager::persistent_state_ttl(h->unix_time()); - double time_to_download = 3600 * 3; + double time_to_download = 3600 * 8; if (ttl > td::Clocks::system() + time_to_download) { handle = h; break; @@ -247,6 +251,7 @@ void ValidatorManagerMasterchainReiniter::choose_masterchain_state() { } void ValidatorManagerMasterchainReiniter::download_masterchain_state() { + status_.set_status(PSTRING() << "downloading masterchain state " << block_id_.seqno()); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { LOG(WARNING) << "failed to download masterchain state: " << R.move_as_error(); @@ -268,11 +273,13 @@ void ValidatorManagerMasterchainReiniter::downloaded_masterchain_state(td::Refreceived_state()); CHECK(handle_->is_applied()); LOG(INFO) << "downloaded masterchain state"; + td::actor::send_closure(manager_, &ValidatorManager::init_last_masterchain_state, state_); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::downloaded_all_shards); }); client_ = td::actor::create_actor("shardclient", opts_, handle_, state_, manager_, std::move(P)); + status_.set_status(PSTRING() << "downloading all shard states, mc seqno " << block_id_.seqno()); } void ValidatorManagerMasterchainReiniter::downloaded_all_shards() { @@ -567,7 +574,7 @@ void ValidatorManagerMasterchainStarter::truncated() { truncate_shard_next(handle_->id(), ig.get_promise()); auto s = state_->get_shards(); for (auto &shard : s) { - if (opts_->need_monitor(shard->shard())) { + if (opts_->need_monitor(shard->shard(), state_)) { truncate_shard_next(shard->top_block_id(), ig.get_promise()); } } diff --git a/validator/manager-init.hpp b/validator/manager-init.hpp index 7dce4e47..901b826b 100644 --- a/validator/manager-init.hpp +++ b/validator/manager-init.hpp @@ -27,6 +27,8 @@ #include "manager-init.h" +#include + namespace ton { namespace validator { @@ -77,6 +79,8 @@ class ValidatorManagerMasterchainReiniter : public td::actor::Actor { td::uint32 pending_ = 0; td::actor::ActorOwn client_; + + ProcessStatus status_; }; class ValidatorManagerMasterchainStarter : public td::actor::Actor { diff --git a/validator/manager.cpp b/validator/manager.cpp index 5199eb01..b0ac5409 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -17,12 +17,12 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "manager.hpp" +#include "checksum.h" +#include "td/utils/buffer.h" #include "validator-group.hpp" -#include "adnl/utils.hpp" #include "downloaders/wait-block-state.hpp" #include "downloaders/wait-block-state-merge.hpp" #include "downloaders/wait-block-data.hpp" -#include "validator-group.hpp" #include "fabric.h" #include "manager.h" #include "validate-broadcast.hpp" @@ -42,6 +42,7 @@ #include "td/utils/JsonBuilder.h" #include "common/delay.h" +#include "td/utils/filesystem.h" #include "validator/stats-merger.h" @@ -199,7 +200,7 @@ void ValidatorManagerImpl::validate_block(ReceivedBlock block, td::Promise R) mutable { + [SelfId = actor_id(this), promise = std::move(promise), id = blkid](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); } else { @@ -214,24 +215,38 @@ void ValidatorManagerImpl::prevalidate_block(BlockBroadcast broadcast, td::Promi promise.set_error(td::Status::Error(ErrorCode::notready, "node not started")); return; } + if (!need_monitor(broadcast.block_id.shard_full())) { + promise.set_error(td::Status::Error("not monitoring shard")); + return; + } + promise = [SelfId = actor_id(this), promise = std::move(promise), block_id = broadcast.block_id, + cc_seqno = broadcast.catchain_seqno](td::Result R) mutable { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::validated_block_broadcast, block_id, cc_seqno); + } + promise.set_result(std::move(R)); + }; td::actor::create_actor("broadcast", std::move(broadcast), last_masterchain_block_handle_, last_masterchain_state_, last_known_key_block_handle_, actor_id(this), td::Timestamp::in(2.0), std::move(promise)) .release(); } +void ValidatorManagerImpl::validated_block_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno) { +} + void ValidatorManagerImpl::sync_complete(td::Promise promise) { started_ = true; VLOG(VALIDATOR_WARNING) << "completed sync. Validating " << validator_groups_.size() << " groups"; for (auto &v : validator_groups_) { - if (!v.second.empty()) { - td::actor::send_closure(v.second, &ValidatorGroup::create_session); + if (!v.second.actor.empty()) { + td::actor::send_closure(v.second.actor, &ValidatorGroup::create_session); } } for (auto &v : next_validator_groups_) { - if (!v.second.empty()) { - td::actor::send_closure(v.second, &ValidatorGroup::create_session); + if (!v.second.actor.empty()) { + td::actor::send_closure(v.second.actor, &ValidatorGroup::create_session); } } } @@ -306,6 +321,11 @@ void ValidatorManagerImpl::get_persistent_state_slice(BlockIdExt block_id, Block std::move(promise)); } +void ValidatorManagerImpl::get_previous_persistent_state_files( + BlockSeqno cur_mc_seqno, td::Promise>> promise) { + td::actor::send_closure(db_, &Db::get_previous_persistent_state_files, cur_mc_seqno, std::move(promise)); +} + void ValidatorManagerImpl::get_block_proof(BlockHandle handle, td::Promise promise) { auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result> R) mutable { if (R.is_error()) { @@ -368,7 +388,7 @@ void ValidatorManagerImpl::get_key_block_proof_link(BlockIdExt block_id, td::Pro td::actor::send_closure(db_, &Db::get_key_block_proof, block_id, std::move(P)); } -void ValidatorManagerImpl::new_external_message(td::BufferSlice data) { +void ValidatorManagerImpl::new_external_message(td::BufferSlice data, int priority) { if (!is_validator()) { return; } @@ -376,7 +396,7 @@ void ValidatorManagerImpl::new_external_message(td::BufferSlice data) { VLOG(VALIDATOR_NOTICE) << "dropping ext message: validator is not ready"; return; } - if (ext_messages_.size() > max_mempool_num()) { + if (ext_msgs_[priority].ext_messages_.size() > (size_t)max_mempool_num()) { return; } auto R = create_ext_message(std::move(data), last_masterchain_state_->get_ext_msg_limits()); @@ -384,30 +404,70 @@ void ValidatorManagerImpl::new_external_message(td::BufferSlice data) { VLOG(VALIDATOR_NOTICE) << "dropping bad ext message: " << R.move_as_error(); return; } - add_external_message(R.move_as_ok()); + add_external_message(R.move_as_ok(), priority); } -void ValidatorManagerImpl::add_external_message(td::Ref msg) { +void ValidatorManagerImpl::add_external_message(td::Ref msg, int priority) { + auto &msgs = ext_msgs_[priority]; auto message = std::make_unique>(msg); auto id = message->ext_id(); auto address = message->address(); unsigned long per_address_limit = 256; - if(ext_addr_messages_.count(address) < per_address_limit) { - if (ext_messages_hashes_.count(id.hash) == 0) { - ext_messages_.emplace(id, std::move(message)); - ext_messages_hashes_.emplace(id.hash, id); - ext_addr_messages_[address].emplace(id.hash, id); - } + auto it = msgs.ext_addr_messages_.find(address); + if (it != msgs.ext_addr_messages_.end() && it->second.size() >= per_address_limit) { + return; } + auto it2 = ext_messages_hashes_.find(id.hash); + if (it2 != ext_messages_hashes_.end()) { + int old_priority = it2->second.first; + if (old_priority >= priority) { + return; + } + ext_msgs_[old_priority].erase(id); + } + msgs.ext_messages_.emplace(id, std::move(message)); + msgs.ext_addr_messages_[address].emplace(id.hash, id); + ext_messages_hashes_[id.hash] = {priority, id}; } void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Promise> promise) { + if (!started_) { + promise.set_error(td::Status::Error(ErrorCode::notready, "node not synced")); + return; + } auto state = do_get_last_liteserver_state(); if (state.is_null()) { promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); return; } - run_check_external_message(std::move(data), state->get_ext_msg_limits(), actor_id(this), - std::move(promise)); + auto R = create_ext_message(std::move(data), state->get_ext_msg_limits()); + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix("failed to parse external message: ")); + return; + } + auto message = R.move_as_ok(); + WorkchainId wc = message->wc(); + StdSmcAddress addr = message->addr(); + if (checked_ext_msg_counter_.get_msg_count(wc, addr) >= max_ext_msg_per_addr()) { + promise.set_error( + td::Status::Error(PSTRING() << "too many external messages to address " << wc << ":" << addr.to_hex())); + return; + } + + promise = [self = this, wc, addr, promise = std::move(promise), + SelfId = actor_id(this)](td::Result> R) mutable { + td::actor::send_lambda(SelfId, [=, promise = std::move(promise), R = std::move(R)]() mutable { + ++(R.is_ok() ? self->total_check_ext_messages_ok_ : self->total_check_ext_messages_error_); + TRY_RESULT_PROMISE(promise, message, std::move(R)); + if (self->checked_ext_msg_counter_.inc_msg_count(wc, addr) > max_ext_msg_per_addr()) { + promise.set_error( + td::Status::Error(PSTRING() << "too many external messages to address " << wc << ":" << addr.to_hex())); + return; + } + promise.set_result(std::move(message)); + }); + }; + ++ls_stats_check_ext_messages_; + run_check_external_message(std::move(message), actor_id(this), std::move(promise)); } void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { @@ -428,7 +488,7 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { - if (!is_validator()) { + if (!is_validator() && !cached_block_candidates_.count(block_id)) { return; } if (!last_masterchain_block_handle_) { @@ -454,35 +514,75 @@ void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc actor_id(this), td::Timestamp::in(2.0), std::move(P)); } +void ValidatorManagerImpl::new_block_candidate(BlockIdExt block_id, td::BufferSlice data) { + if (!last_masterchain_block_handle_) { + VLOG(VALIDATOR_DEBUG) << "dropping top shard block broadcast: not inited"; + return; + } + if (!started_) { + return; + } + if (!need_monitor(block_id.shard_full())) { + VLOG(VALIDATOR_DEBUG) << "dropping block candidate broadcast: not monitoring shard"; + return; + } + add_cached_block_candidate(ReceivedBlock{block_id, std::move(data)}); +} + void ValidatorManagerImpl::add_shard_block_description(td::Ref desc) { - if (desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { - auto it = shard_blocks_.find(ShardTopBlockDescriptionId{desc->shard(), desc->catchain_seqno()}); - if (it != shard_blocks_.end() && desc->block_id().id.seqno <= it->second->block_id().id.seqno) { - VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block broadcast"; - return; + if (!desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { + return; + } + auto it = shard_blocks_.find(ShardTopBlockDescriptionId{desc->shard(), desc->catchain_seqno()}); + if (it != shard_blocks_.end() && desc->block_id().id.seqno <= it->second->block_id().id.seqno) { + VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block broadcast"; + return; + } + shard_blocks_[ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}] = desc; + VLOG(VALIDATOR_DEBUG) << "new shard block descr for " << desc->block_id(); + if (need_monitor(desc->block_id().shard_full())) { + auto P = td::PromiseCreator::lambda([](td::Result> R) { + if (R.is_error()) { + auto S = R.move_as_error(); + if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { + VLOG(VALIDATOR_NOTICE) << "failed to get shard state: " << S; + } else { + VLOG(VALIDATOR_DEBUG) << "failed to get shard state: " << S; + } + } + }); + wait_block_state_short(desc->block_id(), 0, td::Timestamp::in(60.0), std::move(P)); + } +} + +void ValidatorManagerImpl::add_cached_block_candidate(ReceivedBlock block) { + BlockIdExt id = block.id; + if (block.id.is_masterchain()) { + return; + } + if (cached_block_candidates_.emplace(id, std::move(block)).second) { + cached_block_candidates_lru_.push_back(id); + { + auto it = wait_block_data_.find(id); + if (it != wait_block_data_.end()) { + auto r_block = create_block(cached_block_candidates_[id].clone()); + if (r_block.is_ok()) { + td::actor::send_closure(it->second.actor_, &WaitBlockData::loaded_block_data, r_block.move_as_ok()); + } + } } - shard_blocks_[ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}] = desc; - VLOG(VALIDATOR_DEBUG) << "new shard block descr for " << desc->block_id(); - if (last_masterchain_block_handle_ && last_masterchain_seqno_ > 0 && - desc->generated_at() < last_masterchain_block_handle_->unix_time() + 60) { - delay_action( - [SelfId = actor_id(this), desc]() { - auto P = td::PromiseCreator::lambda([](td::Result> R) { - if (R.is_error()) { - auto S = R.move_as_error(); - if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { - VLOG(VALIDATOR_NOTICE) << "failed to get shard state: " << S; - } else { - VLOG(VALIDATOR_DEBUG) << "failed to get shard state: " << S; - } - } - }); - td::actor::send_closure(SelfId, &ValidatorManager::wait_block_state_short, desc->block_id(), 0, - td::Timestamp::in(60.0), std::move(P)); - }, - td::Timestamp::in(1.0)); + { + auto it = wait_state_.find(id); + if (it != wait_state_.end()) { + // Proof link is not ready at this point, but this will force WaitBlockState to redo send_get_proof_link_request + td::actor::send_closure(it->second.actor_, &WaitBlockState::after_get_proof_link); + } } } + if (cached_block_candidates_lru_.size() > max_cached_candidates()) { + CHECK(cached_block_candidates_.erase(cached_block_candidates_lru_.front())); + cached_block_candidates_lru_.pop_front(); + } } void ValidatorManagerImpl::add_ext_server_id(adnl::AdnlNodeIdShort id) { @@ -585,13 +685,24 @@ void ValidatorManagerImpl::run_ext_query(td::BufferSlice data, td::Promise> promise) { + if (last_masterchain_state_.not_null() && !opts_->need_monitor(handle->id().shard_full(), last_masterchain_state_)) { + return promise.set_error( + td::Status::Error(PSTRING() << "not monitoring shard " << handle->id().shard_full().to_str())); + } + auto it0 = block_state_cache_.find(handle->id()); + if (it0 != block_state_cache_.end()) { + it0->second.ttl_ = td::Timestamp::in(30.0); + promise.set_result(it0->second.state_); + return; + } auto it = wait_state_.find(handle->id()); if (it == wait_state_.end()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); }); auto id = td::actor::create_actor("waitstate", handle, priority, actor_id(this), - td::Timestamp::in(10.0), std::move(P)) + td::Timestamp::at(timeout.at() + 10.0), std::move(P), + get_block_persistent_state_to_download(handle->id())) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); @@ -624,7 +735,7 @@ void ValidatorManagerImpl::wait_block_data(BlockHandle handle, td::uint32 priori td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_data, handle, std::move(R)); }); auto id = td::actor::create_actor("waitdata", handle, priority, actor_id(this), - td::Timestamp::in(10.0), std::move(P)) + td::Timestamp::at(timeout.at() + 10.0), false, std::move(P)) .release(); wait_block_data_[handle->id()].actor_ = id; it = wait_block_data_.find(handle->id()); @@ -651,6 +762,10 @@ void ValidatorManagerImpl::wait_block_data_short(BlockIdExt block_id, td::uint32 void ValidatorManagerImpl::wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) { + if (last_masterchain_state_.not_null() && !opts_->need_monitor(left_id.shard_full(), last_masterchain_state_)) { + return promise.set_error( + td::Status::Error(PSTRING() << "not monitoring shard " << left_id.shard_full().to_str())); + } td::actor::create_actor("merge", left_id, right_id, priority, actor_id(this), timeout, std::move(promise)) .release(); @@ -776,27 +891,44 @@ void ValidatorManagerImpl::wait_block_message_queue_short(BlockIdExt block_id, t get_block_handle(block_id, true, std::move(P)); } -void ValidatorManagerImpl::get_external_messages(ShardIdFull shard, - td::Promise>> promise) { - std::vector> res; +void ValidatorManagerImpl::get_external_messages( + ShardIdFull shard, td::Promise, int>>> promise) { + td::Timer t; + size_t processed = 0, deleted = 0; + std::vector, int>> res; MessageId left{AccountIdPrefixFull{shard.workchain, shard.shard & (shard.shard - 1)}, Bits256::zero()}; - auto it = ext_messages_.lower_bound(left); - while (it != ext_messages_.end()) { - auto s = it->first; - if (!shard_contains(shard, s.dst)) { - break; + size_t total_msgs = 0; + td::Random::Fast rnd; + for (auto iter = ext_msgs_.rbegin(); iter != ext_msgs_.rend(); ++iter) { + std::vector, int>> cur_res; + int priority = iter->first; + auto &msgs = iter->second; + auto it = msgs.ext_messages_.lower_bound(left); + while (it != msgs.ext_messages_.end()) { + auto s = it->first; + if (!shard_contains(shard, s.dst)) { + break; + } + ++processed; + if (it->second->expired()) { + msgs.ext_addr_messages_[it->second->address()].erase(it->first.hash); + ext_messages_hashes_.erase(it->first.hash); + it = msgs.ext_messages_.erase(it); + ++deleted; + continue; + } + if (it->second->is_active()) { + cur_res.emplace_back(it->second->message(), priority); + } + it++; } - if (it->second->expired()) { - ext_addr_messages_[it->second->address()].erase(it->first.hash); - ext_messages_hashes_.erase(it->first.hash); - it = ext_messages_.erase(it); - continue; - } - if (it->second->is_active()) { - res.push_back(it->second->message()); - } - it++; + td::random_shuffle(td::as_mutable_span(cur_res), rnd); + res.insert(res.end(), cur_res.begin(), cur_res.end()); + total_msgs += msgs.ext_messages_.size(); } + LOG(WARNING) << "get_external_messages to shard " << shard.to_str() << " : time=" << t.elapsed() + << " result_size=" << res.size() << " processed=" << processed << " expired=" << deleted + << " total_size=" << total_msgs; promise.set_value(std::move(res)); } @@ -836,8 +968,9 @@ void ValidatorManagerImpl::complete_external_messages(std::vectorsecond]->address()].erase(it->first); - CHECK(ext_messages_.erase(it->second)); + int priority = it->second.first; + auto msg_id = it->second.second; + ext_msgs_[priority].erase(msg_id); ext_messages_hashes_.erase(it); } } @@ -845,12 +978,14 @@ void ValidatorManagerImpl::complete_external_messages(std::vectorsecond); - if ((ext_messages_.size() < soft_mempool_limit) && it2->second->can_postpone()) { + int priority = it->second.first; + auto msg_id = it->second.second; + auto &msgs = ext_msgs_[priority]; + auto it2 = msgs.ext_messages_.find(msg_id); + if ((msgs.ext_messages_.size() < soft_mempool_limit) && it2->second->can_postpone()) { it2->second->postpone(); } else { - ext_addr_messages_[it2->second->address()].erase(it2->first.hash); - ext_messages_.erase(it2); + msgs.erase(msg_id); ext_messages_hashes_.erase(it); } } @@ -922,6 +1057,16 @@ void ValidatorManagerImpl::get_block_candidate_from_db(PublicKey source, BlockId td::actor::send_closure(db_, &Db::get_block_candidate, source, id, collated_data_file_hash, std::move(promise)); } +void ValidatorManagerImpl::get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) { + auto it = cached_block_candidates_.find(id); + if (it != cached_block_candidates_.end()) { + promise.set_result(it->second.data.clone()); + return; + } + td::actor::send_closure(db_, &Db::get_block_candidate_by_block_id, id, + promise.wrap([](BlockCandidate &&b) { return std::move(b.data); })); +} + void ValidatorManagerImpl::get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) { td::actor::send_closure(db_, &Db::get_block_proof, std::move(handle), std::move(promise)); } @@ -988,6 +1133,9 @@ void ValidatorManagerImpl::get_block_by_seqno_from_db(AccountIdPrefixFull accoun } void ValidatorManagerImpl::finished_wait_state(BlockHandle handle, td::Result> R) { + if (R.is_ok()) { + block_state_cache_[handle->id()] = {R.ok(), td::Timestamp::in(30.0)}; + } auto it = wait_state_.find(handle->id()); if (it != wait_state_.end()) { if (R.is_error()) { @@ -1001,9 +1149,10 @@ void ValidatorManagerImpl::finished_wait_state(BlockHandle handle, td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); }); - auto id = td::actor::create_actor("waitstate", handle, X.second, actor_id(this), X.first, - std::move(P)) - .release(); + auto id = + td::actor::create_actor("waitstate", handle, X.second, actor_id(this), X.first, + std::move(P), get_block_persistent_state_to_download(handle->id())) + .release(); it->second.actor_ = id; return; } @@ -1032,8 +1181,9 @@ void ValidatorManagerImpl::finished_wait_data(BlockHandle handle, td::Result("waitdata", handle, X.second, actor_id(this), X.first, std::move(P)) - .release(); + td::actor::create_actor("waitdata", handle, X.second, actor_id(this), X.first, false, + std::move(P)) + .release(); it->second.actor_ = id; return; } @@ -1150,10 +1300,23 @@ void ValidatorManagerImpl::set_next_block(BlockIdExt block_id, BlockIdExt next, get_block_handle(block_id, true, std::move(P)); } -void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) { +void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) { + if (!candidates_buffer_.empty()) { + td::actor::send_closure(candidates_buffer_, &CandidatesBuffer::add_new_candidate, id, + PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}, candidate.collated_file_hash); + } + if (!id.is_masterchain()) { + add_cached_block_candidate(ReceivedBlock{id, candidate.data.clone()}); + } td::actor::send_closure(db_, &Db::store_block_candidate, std::move(candidate), std::move(promise)); } +void ValidatorManagerImpl::send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) { + callback_->send_block_candidate(id, cc_seqno, validator_set_hash, std::move(data)); +} + void ValidatorManagerImpl::write_handle(BlockHandle handle, td::Promise promise) { auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), handle, promise = std::move(promise)](td::Result R) mutable { @@ -1169,7 +1332,7 @@ void ValidatorManagerImpl::write_handle(BlockHandle handle, td::Promise promise) { bool received = handle->received(); bool inited_state = handle->received_state(); - bool inited_proof = handle->id().is_masterchain() ? handle->inited_proof() : handle->inited_proof(); + bool inited_proof = handle->id().is_masterchain() ? handle->inited_proof() : handle->inited_proof_link(); if (handle->need_flush()) { handle->flush(actor_id(this), handle, std::move(promise)); @@ -1187,6 +1350,19 @@ void ValidatorManagerImpl::written_handle(BlockHandle handle, td::Promisesecond.actor_, &WaitBlockState::force_read_from_db); } + } else { + if (handle->inited_proof_link()) { + auto it = wait_state_.find(handle->id()); + if (it != wait_state_.end()) { + td::actor::send_closure(it->second.actor_, &WaitBlockState::after_get_proof_link); + } + } + if (handle->id().is_masterchain() && handle->inited_proof()) { + auto it = wait_state_.find(handle->id()); + if (it != wait_state_.end()) { + td::actor::send_closure(it->second.actor_, &WaitBlockState::after_get_proof); + } + } } promise.set_value(td::Unit()); @@ -1347,7 +1523,18 @@ td::Ref ValidatorManagerImpl::do_get_last_liteserver_state() { if (last_masterchain_state_.is_null()) { return {}; } - if (last_liteserver_state_.is_null() || last_liteserver_state_->get_unix_time() < td::Clocks::system() - 30) { + if (last_liteserver_state_.is_null()) { + last_liteserver_state_ = last_masterchain_state_; + return last_liteserver_state_; + } + if (last_liteserver_state_->get_seqno() == last_masterchain_state_->get_seqno()) { + return last_liteserver_state_; + } + // If liteserver seqno (i.e. shard client) lags then use last masterchain state for liteserver + // Allowed lag depends on the block rate + double time_per_block = double(last_masterchain_state_->get_unix_time() - last_liteserver_state_->get_unix_time()) / + double(last_masterchain_state_->get_seqno() - last_liteserver_state_->get_seqno()); + if (td::Clocks::system() - double(last_liteserver_state_->get_unix_time()) > std::min(time_per_block * 8, 180.0)) { last_liteserver_state_ = last_masterchain_state_; } return last_liteserver_state_; @@ -1383,6 +1570,13 @@ void ValidatorManagerImpl::get_last_liteserver_state_block( void ValidatorManagerImpl::send_get_block_request(BlockIdExt id, td::uint32 priority, td::Promise promise) { + { + auto it = cached_block_candidates_.find(id); + if (it != cached_block_candidates_.end()) { + LOG(DEBUG) << "send_get_block_request: got result from candidates cache for " << id.to_str(); + return promise.set_value(it->second.clone()); + } + } callback_->download_block(id, priority, td::Timestamp::in(10.0), std::move(promise)); } @@ -1405,6 +1599,20 @@ void ValidatorManagerImpl::send_get_block_proof_request(BlockIdExt block_id, td: void ValidatorManagerImpl::send_get_block_proof_link_request(BlockIdExt block_id, td::uint32 priority, td::Promise promise) { + if (!block_id.is_masterchain()) { + auto it = cached_block_candidates_.find(block_id); + if (it != cached_block_candidates_.end()) { + // Proof link can be created from the cached block candidate + LOG(DEBUG) << "send_get_block_proof_link_request: creating proof link from cached caniddate for " + << block_id.to_str(); + TRY_RESULT_PROMISE_PREFIX(promise, block_root, vm::std_boc_deserialize(it->second.data), + "failed to create proof link: "); + TRY_RESULT_PROMISE_PREFIX(promise, proof_link, WaitBlockData::generate_proof_link(it->second.id, block_root), + "failed to create proof link: "); + promise.set_result(std::move(proof_link)); + return; + } + } callback_->download_block_proof_link(block_id, priority, td::Timestamp::in(10.0), std::move(promise)); } @@ -1415,7 +1623,7 @@ void ValidatorManagerImpl::send_get_next_key_blocks_request(BlockIdExt block_id, void ValidatorManagerImpl::send_external_message(td::Ref message) { callback_->send_ext_message(message->shard(), message->serialize()); - add_external_message(std::move(message)); + add_external_message(std::move(message), 0); } void ValidatorManagerImpl::send_ihr_message(td::Ref message) { @@ -1433,15 +1641,35 @@ void ValidatorManagerImpl::send_top_shard_block_description(td::Refblock_id().shard_full(), desc->catchain_seqno()}] = desc; callback_->send_shard_block_info(desc->block_id(), desc->catchain_seqno(), desc->serialize()); + add_shard_block_description(desc); } } -void ValidatorManagerImpl::send_block_broadcast(BlockBroadcast broadcast) { - callback_->send_broadcast(std::move(broadcast)); +void ValidatorManagerImpl::send_block_broadcast(BlockBroadcast broadcast, int mode) { + callback_->send_broadcast(std::move(broadcast), mode); +} + +void ValidatorManagerImpl::send_validator_telemetry(PublicKeyHash key, + tl_object_ptr telemetry) { + callback_->send_validator_telemetry(key, std::move(telemetry)); +} + +void ValidatorManagerImpl::send_get_out_msg_queue_proof_request( + ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, + td::Promise>> promise) { + callback_->download_out_msg_queue_proof(dst_shard, std::move(blocks), limits, td::Timestamp::in(10.0), + std::move(promise)); +} + +void ValidatorManagerImpl::send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, + std::string tmp_dir, td::Timestamp timeout, + td::Promise promise) { + callback_->download_archive(mc_seqno, shard_prefix, std::move(tmp_dir), timeout, std::move(promise)); } void ValidatorManagerImpl::start_up() { - db_ = create_db_actor(actor_id(this), db_root_); + db_ = create_db_actor(actor_id(this), db_root_, opts_); + actor_stats_ = td::actor::create_actor("actor_stats"); lite_server_cache_ = create_liteserver_cache_actor(actor_id(this), db_root_); token_manager_ = td::actor::create_actor("tokenmanager"); td::mkdir(db_root_ + "/tmp/").ensure(); @@ -1464,7 +1692,7 @@ void ValidatorManagerImpl::start_up() { auto S = td::WalkPath::run(to_import_dir, [&](td::CSlice cfname, td::WalkPath::Type t) -> void { auto fname = td::Slice(cfname); if (t == td::WalkPath::Type::NotDir) { - auto d = fname.rfind('/'); + auto d = fname.rfind(TD_DIR_SLASH); if (d != td::Slice::npos) { fname = fname.substr(d + 1); } @@ -1474,7 +1702,6 @@ void ValidatorManagerImpl::start_up() { if (fname.substr(fname.size() - 5) != ".pack") { return; } - fname = fname.substr(0, fname.size() - 5); if (fname.substr(0, 8) != "archive.") { return; } @@ -1483,13 +1710,18 @@ void ValidatorManagerImpl::start_up() { while (fname.size() > 1 && fname[0] == '0') { fname.remove_prefix(1); } + auto i = fname.find('.'); + if (i == td::Slice::npos) { + return; + } + fname = fname.substr(0, i); auto v = td::to_integer_safe(fname); if (v.is_error()) { return; } - auto pos = v.move_as_ok(); - LOG(INFO) << "found archive slice '" << cfname << "' for position " << pos; - to_import_[pos] = std::make_pair(cfname.str(), true); + auto seqno = v.move_as_ok(); + LOG(INFO) << "found archive slice '" << cfname << "' for seqno " << seqno; + to_import_[seqno].push_back(cfname.str()); } }); if (S.is_error()) { @@ -1502,6 +1734,14 @@ void ValidatorManagerImpl::start_up() { alarm_timestamp().relax(check_waiters_at_); } +void ValidatorManagerImpl::init_last_masterchain_state(td::Ref state) { + if (last_masterchain_state_.not_null()) { + return; + } + last_masterchain_state_ = std::move(state); + update_shard_overlays(); +} + void ValidatorManagerImpl::started(ValidatorManagerInitResult R) { CHECK(R.handle); CHECK(R.state.not_null()); @@ -1534,8 +1774,23 @@ void ValidatorManagerImpl::started(ValidatorManagerInitResult R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::read_gc_list, R.move_as_ok()); } }); - td::actor::send_closure(db_, &Db::get_destroyed_validator_sessions, std::move(P)); + + if (opts_->nonfinal_ls_queries_enabled()) { + candidates_buffer_ = td::actor::create_actor("candidates-buffer", actor_id(this)); + } + init_validator_telemetry(); + + auto Q = td::PromiseCreator::lambda( + [SelfId = actor_id(this)](td::Result>> R) { + if (R.is_error()) { + LOG(FATAL) << "db error: " << R.move_as_error(); + } else { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::got_persistent_state_descriptions, R.move_as_ok()); + } + }); + td::actor::send_closure(db_, &Db::get_persistent_state_descriptions, std::move(Q)); + update_shard_overlays(); } void ValidatorManagerImpl::read_gc_list(std::vector list) { @@ -1547,6 +1802,8 @@ void ValidatorManagerImpl::read_gc_list(std::vector list) { serializer_ = td::actor::create_actor("serializer", last_key_block_handle_->id(), opts_, actor_id(this)); + td::actor::send_closure(serializer_, &AsyncStateSerializer::update_last_known_key_block_ts, + last_key_block_handle_->unix_time()); if (last_masterchain_block_handle_->inited_next_left()) { auto b = last_masterchain_block_handle_->one_next(true); @@ -1636,61 +1893,35 @@ void ValidatorManagerImpl::download_next_archive() { } auto seqno = std::min(last_masterchain_seqno_, shard_client_handle_->id().seqno()); + std::vector to_import_files; auto it = to_import_.upper_bound(seqno + 1); if (it != to_import_.begin()) { - it--; - if (it->second.second) { - it->second.second = false; - downloaded_archive_slice(it->second.first, false); - return; - } + --it; + to_import_files = std::move(it->second); + it->second.clear(); } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { - LOG(INFO) << "failed to download archive slice: " << R.error(); + LOG(INFO) << "failed to download and import archive slice: " << R.error(); delay_action([SelfId]() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::download_next_archive); }, td::Timestamp::in(2.0)); } else { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::downloaded_archive_slice, R.move_as_ok(), true); + td::actor::send_closure(SelfId, &ValidatorManagerImpl::checked_archive_slice, R.ok().first, R.ok().second); } }); - callback_->download_archive(seqno + 1, db_root_ + "/tmp/", td::Timestamp::in(36000.0), std::move(P)); -} - -void ValidatorManagerImpl::downloaded_archive_slice(std::string name, bool is_tmp) { - LOG(INFO) << "downloaded archive slice: " << name; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), name, is_tmp](td::Result> R) { - if (is_tmp) { - td::unlink(name).ensure(); - } - if (R.is_error()) { - LOG(INFO) << "failed to check downloaded archive slice: " << R.error(); - delay_action([SelfId]() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::download_next_archive); }, - td::Timestamp::in(2.0)); - } else { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::checked_archive_slice, R.move_as_ok()); - } - }); - - auto seqno = std::min(last_masterchain_seqno_, shard_client_handle_->id().seqno()); - - td::actor::create_actor("archiveimport", name, last_masterchain_state_, seqno, opts_, actor_id(this), - std::move(P)) + td::actor::create_actor("archiveimport", db_root_, last_masterchain_state_, seqno, opts_, + actor_id(this), std::move(to_import_files), std::move(P)) .release(); } -void ValidatorManagerImpl::checked_archive_slice(std::vector seqno) { - CHECK(seqno.size() == 2); - LOG(INFO) << "checked downloaded archive slice: mc_top_seqno=" << seqno[0] << " shard_top_seqno_=" << seqno[1]; - CHECK(seqno[0] <= last_masterchain_seqno_); - CHECK(seqno[1] <= last_masterchain_seqno_); +void ValidatorManagerImpl::checked_archive_slice(BlockSeqno new_last_mc_seqno, BlockSeqno new_shard_client_seqno) { + LOG(INFO) << "checked downloaded archive slice: mc_top_seqno=" << new_last_mc_seqno + << " shard_top_seqno_=" << new_shard_client_seqno; + CHECK(new_last_mc_seqno <= last_masterchain_seqno_); + CHECK(new_shard_client_seqno <= last_masterchain_seqno_); - BlockIdExt b; - if (seqno[1] < last_masterchain_seqno_) { - CHECK(last_masterchain_state_->get_old_mc_block_id(seqno[1], b)); - } else { - b = last_masterchain_block_id_; - } + BlockIdExt shard_client_block_id; + CHECK(last_masterchain_state_->get_old_mc_block_id(new_shard_client_seqno, shard_client_block_id)); auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), db = db_.get(), client = shard_client_.get()](td::Result R) { @@ -1706,7 +1937,7 @@ void ValidatorManagerImpl::checked_archive_slice(std::vector seqno) }); td::actor::send_closure(db, &Db::get_block_state, std::move(handle), std::move(P)); }); - get_block_handle(b, true, std::move(P)); + get_block_handle(shard_client_block_id, true, std::move(P)); } void ValidatorManagerImpl::finish_prestart_sync() { @@ -1736,8 +1967,14 @@ void ValidatorManagerImpl::new_masterchain_block() { last_known_key_block_handle_ = last_key_block_handle_; callback_->new_key_block(last_key_block_handle_); } + if (!serializer_.empty()) { + td::actor::send_closure(serializer_, &AsyncStateSerializer::update_last_known_key_block_ts, + last_key_block_handle_->unix_time()); + } + init_validator_telemetry(); } + update_shard_overlays(); update_shards(); update_shard_blocks(); @@ -1751,6 +1988,26 @@ void ValidatorManagerImpl::new_masterchain_block() { } } +void ValidatorManagerImpl::update_shard_overlays() { + CHECK(last_masterchain_state_.not_null()); + std::set shards_to_monitor; + shards_to_monitor.insert(ShardIdFull{masterchainId}); + std::set workchains; + for (const auto& shard : last_masterchain_state_->get_shards()) { + workchains.insert(shard->shard().workchain); + if (opts_->need_monitor(shard->shard(),last_masterchain_state_)) { + shards_to_monitor.insert(shard->shard()); + } + } + for (const auto &[wc, desc] : last_masterchain_state_->get_workchain_list()) { + if (!workchains.count(wc) && desc->active && + opts_->need_monitor(ShardIdFull{wc, shardIdAll}, last_masterchain_state_)) { + shards_to_monitor.insert(ShardIdFull{wc, shardIdAll}); + } + } + callback_->on_new_masterchain_block(last_masterchain_state_, std::move(shards_to_monitor)); +} + void ValidatorManagerImpl::update_shards() { if ((last_masterchain_state_->rotated_all_shards() || last_masterchain_seqno_ == 0) && opts_->get_last_fork_masterchain_seqno() <= last_masterchain_seqno_) { @@ -1827,8 +2084,8 @@ void ValidatorManagerImpl::update_shards() { VLOG(VALIDATOR_DEBUG) << "total shards=" << new_shards.size() << " config shards=" << exp_vec.size(); - std::map> new_validator_groups_; - std::map> new_next_validator_groups_; + std::map new_validator_groups_; + std::map new_next_validator_groups_; bool force_recover = false; { @@ -1860,24 +2117,23 @@ void ValidatorManagerImpl::update_shards() { } else { auto it2 = next_validator_groups_.find(legacy_val_group_id); if (it2 != next_validator_groups_.end()) { - if (!it2->second.empty()) { - td::actor::send_closure(it2->second, &ValidatorGroup::start, prev, last_masterchain_block_id_, - last_masterchain_state_->get_unix_time()); + if (!it2->second.actor.empty()) { + td::actor::send_closure(it2->second.actor, &ValidatorGroup::start, prev, last_masterchain_block_id_); } new_validator_groups_.emplace(val_group_id, std::move(it2->second)); } else { - auto G = create_validator_group(val_group_id, shard, val_set, opts, started_); + auto G = create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_); if (!G.empty()) { - td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_, - last_masterchain_state_->get_unix_time()); + td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_); } - new_validator_groups_.emplace(val_group_id, std::move(G)); + new_validator_groups_.emplace(val_group_id, ValidatorGroupEntry{std::move(G), shard}); } } } } } + active_validator_groups_master_ = active_validator_groups_shard_ = 0; if (allow_validate_) { for (auto &desc : new_shards) { auto shard = desc.first; @@ -1894,6 +2150,7 @@ void ValidatorManagerImpl::update_shards() { auto validator_id = get_validator(shard, val_set); if (!validator_id.is_zero()) { + ++(shard.is_masterchain() ? active_validator_groups_master_ : active_validator_groups_shard_); auto val_group_id = get_validator_set_id(shard, val_set, opts_hash, key_seqno, opts); if (force_recover) { @@ -1916,18 +2173,16 @@ void ValidatorManagerImpl::update_shards() { } else { auto it2 = next_validator_groups_.find(val_group_id); if (it2 != next_validator_groups_.end()) { - if (!it2->second.empty()) { - td::actor::send_closure(it2->second, &ValidatorGroup::start, prev, last_masterchain_block_id_, - last_masterchain_state_->get_unix_time()); + if (!it2->second.actor.empty()) { + td::actor::send_closure(it2->second.actor, &ValidatorGroup::start, prev, last_masterchain_block_id_); } new_validator_groups_.emplace(val_group_id, std::move(it2->second)); } else { - auto G = create_validator_group(val_group_id, shard, val_set, opts, started_); + auto G = create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_); if (!G.empty()) { - td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_, - last_masterchain_state_->get_unix_time()); + td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_); } - new_validator_groups_.emplace(val_group_id, std::move(G)); + new_validator_groups_.emplace(val_group_id, ValidatorGroupEntry{std::move(G), shard}); } } } @@ -1947,23 +2202,24 @@ void ValidatorManagerImpl::update_shards() { //CHECK(!it->second.empty()); new_next_validator_groups_.emplace(val_group_id, std::move(it->second)); } else { - new_next_validator_groups_.emplace(val_group_id, - create_validator_group(val_group_id, shard, val_set, opts, started_)); + new_next_validator_groups_.emplace( + val_group_id, ValidatorGroupEntry{ + create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_), shard}); } } } std::vector> gc; for (auto &v : validator_groups_) { - if (!v.second.empty()) { + if (!v.second.actor.empty()) { gc_list_.push_back(v.first); - gc.push_back(v.second.release()); + gc.push_back(v.second.actor.release()); } } for (auto &v : next_validator_groups_) { - if (!v.second.empty()) { + if (!v.second.actor.empty()) { gc_list_.push_back(v.first); - gc.push_back(v.second.release()); + gc.push_back(v.second.actor.release()); } } @@ -1988,7 +2244,11 @@ void ValidatorManagerImpl::update_shards() { }); td::actor::send_closure(db_, &Db::update_destroyed_validator_sessions, gc_list_, std::move(P)); } -} // namespace validator + if (!serializer_.empty()) { + td::actor::send_closure(serializer_, &AsyncStateSerializer::auto_disable_serializer, + is_validator() && last_masterchain_state_->get_global_id() == -239); // mainnet only + } +} void ValidatorManagerImpl::written_destroyed_validator_sessions(std::vector> list) { for (auto &v : list) { @@ -2051,17 +2311,21 @@ ValidatorSessionId ValidatorManagerImpl::get_validator_set_id(ShardIdFull shard, } td::actor::ActorOwn ValidatorManagerImpl::create_validator_group( - ValidatorSessionId session_id, ShardIdFull shard, td::Ref validator_set, + ValidatorSessionId session_id, ShardIdFull shard, td::Ref validator_set, BlockSeqno key_seqno, validatorsession::ValidatorSessionOptions opts, bool init_session) { if (check_gc_list_.count(session_id) == 1) { return td::actor::ActorOwn{}; } else { + // Call get_external_messages to cleanup mempool for the shard + get_external_messages(shard, [](td::Result, int>>>) {}); + auto validator_id = get_validator(shard, validator_set); CHECK(!validator_id.is_zero()); auto G = td::actor::create_actor( - "validatorgroup", shard, validator_id, session_id, validator_set, opts, keyring_, adnl_, rldp_, overlays_, - db_root_, actor_id(this), init_session, - opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno())); + PSTRING() << "valgroup" << shard.to_str(), shard, validator_id, session_id, validator_set, key_seqno, opts, + keyring_, adnl_, rldp_, + overlays_, db_root_, actor_id(this), init_session, + opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_); return G; } } @@ -2235,7 +2499,7 @@ void ValidatorManagerImpl::allow_block_state_gc(BlockIdExt block_id, td::Promise return; } auto shards = gc_masterchain_state_->get_shards(); - for (auto shard : shards) { + for (const auto &shard : shards) { if (shard_intersects(shard->shard(), block_id.shard_full())) { promise.set_result(block_id.id.seqno < shard->top_block_id().id.seqno); return; @@ -2268,7 +2532,15 @@ void ValidatorManagerImpl::allow_block_info_gc(BlockIdExt block_id, td::Promise< void ValidatorManagerImpl::got_next_gc_masterchain_handle(BlockHandle handle) { CHECK(gc_advancing_); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result> R) { - R.ensure(); + if (R.is_error()) { + if (R.error().code() == ErrorCode::timeout) { + LOG(ERROR) << "Failed to get gc masterchain state, retrying: " << R.move_as_error(); + td::actor::send_closure(SelfId, &ValidatorManagerImpl::got_next_gc_masterchain_handle, std::move(handle)); + } else { + LOG(FATAL) << "Failed to get gc masterchain state: " << R.move_as_error(); + } + return; + } td::actor::send_closure(SelfId, &ValidatorManagerImpl::got_next_gc_masterchain_state, std::move(handle), td::Ref{R.move_as_ok()}); }); @@ -2299,8 +2571,11 @@ void ValidatorManagerImpl::update_shard_client_block_handle(BlockHandle handle, td::Promise promise) { shard_client_handle_ = std::move(handle); auto seqno = shard_client_handle_->id().seqno(); - if (last_liteserver_state_.is_null() || last_liteserver_state_->get_block_id().seqno() < seqno) { - last_liteserver_state_ = std::move(state); + if (state.not_null()) { + shard_client_shards_ = state->get_shards(); + if (last_liteserver_state_.is_null() || last_liteserver_state_->get_block_id().seqno() < seqno) { + last_liteserver_state_ = std::move(state); + } } shard_client_update(seqno); promise.set_value(td::Unit()); @@ -2333,21 +2608,21 @@ void ValidatorManagerImpl::state_serializer_update(BlockSeqno seqno) { void ValidatorManagerImpl::alarm() { try_advance_gc_masterchain_block(); alarm_timestamp() = td::Timestamp::in(1.0); - if (gc_masterchain_handle_) { - td::actor::send_closure(db_, &Db::run_gc, gc_masterchain_handle_->unix_time(), + if (shard_client_handle_ && gc_masterchain_handle_) { + td::actor::send_closure(db_, &Db::run_gc, shard_client_handle_->unix_time(), gc_masterchain_handle_->unix_time(), static_cast(opts_->archive_ttl())); } if (log_status_at_.is_in_past()) { if (last_masterchain_block_handle_) { - LOG(INFO) << "STATUS: last_masterchain_block_ago=" - << td::format::as_time(td::Clocks::system() - last_masterchain_block_handle_->unix_time()) - << " last_known_key_block_ago=" - << td::format::as_time(td::Clocks::system() - (last_known_key_block_handle_->inited_unix_time() - ? last_known_key_block_handle_->unix_time() - : 0)) - << " shard_client_ago=" - << td::format::as_time(td::Clocks::system() - - (shard_client_handle_ ? shard_client_handle_->unix_time() : 0)); + LOG(ERROR) << "STATUS: last_masterchain_block_ago=" + << td::format::as_time(td::Clocks::system() - last_masterchain_block_handle_->unix_time()) + << " last_known_key_block_ago=" + << td::format::as_time(td::Clocks::system() - (last_known_key_block_handle_->inited_unix_time() + ? last_known_key_block_handle_->unix_time() + : 0)) + << " shard_client_ago=" + << td::format::as_time(td::Clocks::system() - + (shard_client_handle_ ? shard_client_handle_->unix_time() : 0)); } log_status_at_ = td::Timestamp::in(60.0); } @@ -2373,6 +2648,31 @@ void ValidatorManagerImpl::alarm() { for (auto &w : shard_client_waiters_) { w.second.check_timers(); } + for (auto it = block_state_cache_.begin(); it != block_state_cache_.end();) { + bool del = it->second.ttl_.is_in_past(); + if (del) { + auto block_id = it->first; + if (block_id.is_masterchain()) { + if (block_id.seqno() == last_masterchain_seqno_) { + it->second.ttl_ = td::Timestamp::in(30.0); + del = false; + } + } else if (last_masterchain_state_.not_null()) { + auto shard = last_masterchain_state_->get_shard_from_config(block_id.shard_full()); + if (shard.not_null()) { + if (block_id.seqno() == shard->top_block_id().seqno()) { + it->second.ttl_ = td::Timestamp::in(30.0); + del = false; + } + } + } + } + if (del) { + it = block_state_cache_.erase(it); + } else { + ++it; + } + } } alarm_timestamp().relax(check_waiters_at_); if (check_shard_clients_.is_in_past()) { @@ -2390,6 +2690,39 @@ void ValidatorManagerImpl::alarm() { } } alarm_timestamp().relax(check_shard_clients_); + + if (log_ls_stats_at_.is_in_past()) { + if (!ls_stats_.empty() || ls_stats_check_ext_messages_ != 0) { + td::StringBuilder sb; + sb << "Liteserver stats (1 minute):"; + td::uint32 total = 0; + for (const auto &p : ls_stats_) { + sb << " " << lite_query_name_by_id(p.first) << ":" << p.second; + total += p.second; + } + if (total > 0) { + sb << " TOTAL:" << total; + } + if (ls_stats_check_ext_messages_ > 0) { + sb << " checkExtMessage:" << ls_stats_check_ext_messages_; + } + LOG(WARNING) << sb.as_cslice(); + } + ls_stats_.clear(); + ls_stats_check_ext_messages_ = 0; + log_ls_stats_at_ = td::Timestamp::in(60.0); + } + alarm_timestamp().relax(log_ls_stats_at_); + if (cleanup_mempool_at_.is_in_past()) { + if (is_validator()) { + get_external_messages(ShardIdFull{masterchainId, shardIdAll}, + [](td::Result, int>>>) {}); + get_external_messages(ShardIdFull{basechainId, shardIdAll}, + [](td::Result, int>>>) {}); + } + cleanup_mempool_at_ = td::Timestamp::in(250.0); + } + alarm_timestamp().relax(cleanup_mempool_at_); } void ValidatorManagerImpl::update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) { @@ -2397,17 +2730,13 @@ void ValidatorManagerImpl::update_shard_client_state(BlockIdExt masterchain_bloc } void ValidatorManagerImpl::get_shard_client_state(bool from_db, td::Promise promise) { - if (!shard_client_.empty() && !from_db) { - td::actor::send_closure(shard_client_, &ShardClient::get_processed_masterchain_block_id, std::move(promise)); + if (shard_client_handle_ && !from_db) { + promise.set_result(shard_client_handle_->id()); } else { td::actor::send_closure(db_, &Db::get_shard_client_state, std::move(promise)); } } -void ValidatorManagerImpl::subscribe_to_shard(ShardIdFull shard) { - callback_->add_shard(shard); -} - void ValidatorManagerImpl::update_async_serializer_state(AsyncSerializerState state, td::Promise promise) { td::actor::send_closure(db_, &Db::update_async_serializer_state, std::move(state), std::move(promise)); } @@ -2420,12 +2749,13 @@ void ValidatorManagerImpl::try_get_static_file(FileHash file_hash, td::Promise promise) { +void ValidatorManagerImpl::get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) { if (masterchain_seqno > last_masterchain_seqno_) { promise.set_error(td::Status::Error(ErrorCode::notready, "masterchain seqno too big")); return; } - td::actor::send_closure(db_, &Db::get_archive_id, masterchain_seqno, std::move(promise)); + td::actor::send_closure(db_, &Db::get_archive_id, masterchain_seqno, shard_prefix, std::move(promise)); } void ValidatorManagerImpl::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, @@ -2437,10 +2767,13 @@ bool ValidatorManagerImpl::is_validator() { return temp_keys_.size() > 0 || permanent_keys_.size() > 0; } +bool ValidatorManagerImpl::validating_masterchain() { + return !get_validator(ShardIdFull(masterchainId), + last_masterchain_state_->get_validator_set(ShardIdFull(masterchainId))) + .is_zero(); +} + PublicKeyHash ValidatorManagerImpl::get_validator(ShardIdFull shard, td::Ref val_set) { - if (!opts_->need_validate(shard, val_set->get_catchain_seqno())) { - return PublicKeyHash::zero(); - } for (auto &key : temp_keys_) { if (val_set->is_validator(key.bits256_value())) { return key; @@ -2495,6 +2828,10 @@ void ValidatorManagerImpl::send_peek_key_block_request() { send_get_next_key_blocks_request(last_known_key_block_handle_->id(), 1, std::move(P)); } +void ValidatorManagerImpl::prepare_actor_stats(td::Promise promise) { + send_closure(actor_stats_, &td::actor::ActorStats::prepare_stats, std::move(promise)); +} + void ValidatorManagerImpl::prepare_stats(td::Promise>> promise) { auto merger = StatsMerger::create(std::move(promise)); @@ -2508,9 +2845,12 @@ void ValidatorManagerImpl::prepare_stats(td::Promiseid().to_str()); vec.emplace_back("rotatemasterchainblock", last_rotate_block_id_.to_str()); //vec.emplace_back("shardclientmasterchainseqno", td::to_string(min_confirmed_masterchain_seqno_)); - vec.emplace_back("stateserializermasterchainseqno", td::to_string(state_serializer_masterchain_seqno_)); } + td::NamedThreadSafeCounter::get_default().for_each([&](auto key, auto value) { + vec.emplace_back("counter." + key, PSTRING() << value); + }); + if (!shard_client_.empty()) { auto P = td::PromiseCreator::lambda([promise = merger.make_promise("")](td::Result R) mutable { if (R.is_error()) { @@ -2524,9 +2864,48 @@ void ValidatorManagerImpl::prepare_stats(td::Promiseget_state_serializer_enabled(); + if (is_validator() && last_masterchain_state_->get_global_id() == -239) { + serializer_enabled = false; + } + vec.emplace_back("stateserializerenabled", serializer_enabled ? "true" : "false"); + merger.make_promise("").set_value(std::move(vec)); + if (!serializer_.empty()) { + td::actor::send_closure(serializer_, &AsyncStateSerializer::prepare_stats, merger.make_promise("")); + } + td::actor::send_closure(db_, &Db::prepare_stats, merger.make_promise("db.")); + for (auto &[_, p] : stats_providers_) { + p.second(merger.make_promise(p.first)); + } } void ValidatorManagerImpl::prepare_perf_timer_stats(td::Promise> promise) { @@ -2577,21 +2956,48 @@ void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, } std::vector> rounds; - for (const auto& round : stats.rounds) { + for (const auto &round : stats.rounds) { std::vector> producers; - for (const auto& producer : round.producers) { + for (const auto &producer : round.producers) { + BlockIdExt cur_block_id{block_id.id, producer.root_hash, producer.file_hash}; + auto it = recorded_block_stats_.find(cur_block_id); + tl_object_ptr collation_stats; + if (it != recorded_block_stats_.end() && it->second.collator_stats_) { + auto &stats = it->second.collator_stats_.value(); + collation_stats = create_tl_object( + stats.bytes, stats.gas, stats.lt_delta, stats.cat_bytes, stats.cat_gas, stats.cat_lt_delta, + stats.limits_log, stats.ext_msgs_total, stats.ext_msgs_filtered, stats.ext_msgs_accepted, + stats.ext_msgs_rejected); + } + std::string approvers, signers; + for (bool x : producer.approvers) { + approvers += (x ? '1' : '0'); + } + for (bool x : producer.signers) { + signers += (x ? '1' : '0'); + } producers.push_back(create_tl_object( - producer.id.bits256_value(), producer.block_status, producer.block_timestamp)); + producer.id.bits256_value(), producer.candidate_id, producer.block_status, producer.root_hash, + producer.file_hash, producer.comment, producer.block_timestamp, producer.is_accepted, producer.is_ours, + producer.got_submit_at, producer.collation_time, producer.collated_at, producer.collation_cached, + it == recorded_block_stats_.end() ? -1.0 : it->second.collator_work_time_, + it == recorded_block_stats_.end() ? -1.0 : it->second.collator_cpu_work_time_, std::move(collation_stats), + producer.validation_time, producer.validated_at, producer.validation_cached, + it == recorded_block_stats_.end() ? -1.0 : it->second.validator_work_time_, + it == recorded_block_stats_.end() ? -1.0 : it->second.validator_cpu_work_time_, producer.gen_utime, + producer.approved_weight, producer.approved_33pct_at, producer.approved_66pct_at, std::move(approvers), + producer.signed_weight, producer.signed_33pct_at, producer.signed_66pct_at, std::move(signers), + producer.serialize_time, producer.deserialize_time, producer.serialized_size)); } rounds.push_back(create_tl_object(round.timestamp, std::move(producers))); } auto obj = create_tl_object( - create_tl_block_id_simple(block_id.id), stats.timestamp, stats.self.bits256_value(), - stats.creator.bits256_value(), stats.total_validators, stats.total_weight, stats.signatures, + stats.success, create_tl_block_id(block_id), stats.timestamp, stats.self.bits256_value(), stats.session_id, + stats.cc_seqno, stats.creator.bits256_value(), stats.total_validators, stats.total_weight, stats.signatures, stats.signatures_weight, stats.approve_signatures, stats.approve_signatures_weight, stats.first_round, std::move(rounds)); - std::string s = td::json_encode(td::ToJson(*obj.get()), false); + auto s = td::json_encode(td::ToJson(*obj.get()), false); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); std::ofstream file; @@ -2599,7 +3005,389 @@ void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, file << s << "\n"; file.close(); - LOG(INFO) << "Writing validator session stats for " << block_id.id; + LOG(INFO) << "Writing validator session stats for " << block_id.id.to_str(); +} + +void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) { + std::string fname = opts_->get_session_logs_file(); + if (fname.empty()) { + return; + } + std::vector> nodes; + for (const auto &node : stats.nodes) { + nodes.push_back( + create_tl_object(node.id.bits256_value(), node.weight)); + } + auto obj = create_tl_object( + stats.session_id, stats.shard.workchain, stats.shard.shard, stats.cc_seqno, stats.last_key_block_seqno, + stats.timestamp, stats.self_idx, std::move(nodes)); + auto s = td::json_encode(td::ToJson(*obj.get()), false); + s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); + + std::ofstream file; + file.open(fname, std::ios_base::app); + file << s << "\n"; + file.close(); + + LOG(INFO) << "Writing new validator group stats for " << stats.session_id << " shard=" << stats.shard.to_str() + << " cc_seqno=" << stats.cc_seqno; +} + +void ValidatorManagerImpl::log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) { + std::string fname = opts_->get_session_logs_file(); + if (fname.empty()) { + return; + } + std::vector> nodes; + for (const auto &node : stats.nodes) { + nodes.push_back(create_tl_object(node.id.bits256_value(), + node.catchain_blocks)); + } + auto obj = create_tl_object(stats.session_id, stats.timestamp, + std::move(nodes)); + auto s = td::json_encode(td::ToJson(*obj.get()), false); + s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); + + std::ofstream file; + file.open(fname, std::ios_base::app); + file << s << "\n"; + file.close(); + + LOG(INFO) << "Writing end validator group stats for " << stats.session_id; +} + +void ValidatorManagerImpl::get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) { + get_block_handle(block_id, false, + [SelfId = actor_id(this), block_id, promise = std::move(promise), + allow_not_applied = opts_->nonfinal_ls_queries_enabled()](td::Result R) mutable { + if (R.is_ok() && (allow_not_applied || R.ok()->is_applied())) { + promise.set_value(R.move_as_ok()); + } else { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::process_block_handle_for_litequery_error, + block_id, std::move(R), std::move(promise)); + } + }); +} + +void ValidatorManagerImpl::get_block_data_for_litequery(BlockIdExt block_id, td::Promise> promise) { + if (candidates_buffer_.empty()) { + get_block_handle_for_litequery( + block_id, [manager = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, std::move(handle), + std::move(promise)); + }); + } else { + td::actor::send_closure( + candidates_buffer_, &CandidatesBuffer::get_block_data, block_id, + [manager = actor_id(this), promise = std::move(promise), block_id](td::Result> R) mutable { + if (R.is_ok()) { + promise.set_result(R.move_as_ok()); + return; + } + td::actor::send_closure(manager, &ValidatorManagerImpl::get_block_handle_for_litequery, block_id, + [manager, promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + td::actor::send_closure_later(manager, &ValidatorManager::get_block_data_from_db, + std::move(handle), std::move(promise)); + }); + }); + } +} + +void ValidatorManagerImpl::get_block_state_for_litequery(BlockIdExt block_id, + td::Promise> promise) { + if (candidates_buffer_.empty()) { + get_block_handle_for_litequery( + block_id, [manager = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db, std::move(handle), + std::move(promise)); + }); + } else { + td::actor::send_closure( + candidates_buffer_, &CandidatesBuffer::get_block_state, block_id, + [manager = actor_id(this), promise = std::move(promise), block_id](td::Result> R) mutable { + if (R.is_ok()) { + promise.set_result(R.move_as_ok()); + return; + } + td::actor::send_closure(manager, &ValidatorManagerImpl::get_block_handle_for_litequery, block_id, + [manager, promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, handle, std::move(R)); + td::actor::send_closure_later(manager, &ValidatorManager::get_shard_state_from_db, + std::move(handle), std::move(promise)); + }); + }); + } +} + +void ValidatorManagerImpl::get_block_by_lt_for_litequery(AccountIdPrefixFull account, LogicalTime lt, + td::Promise promise) { + get_block_by_lt_from_db( + account, lt, [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_ok() && R.ok()->is_applied()) { + promise.set_value(R.move_as_ok()); + } else { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::process_lookup_block_for_litequery_error, account, 0, + lt, std::move(R), std::move(promise)); + } + }); +} + +void ValidatorManagerImpl::get_block_by_unix_time_for_litequery(AccountIdPrefixFull account, UnixTime ts, + td::Promise promise) { + get_block_by_unix_time_from_db( + account, ts, [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_ok() && R.ok()->is_applied()) { + promise.set_value(R.move_as_ok()); + } else { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::process_lookup_block_for_litequery_error, account, 1, + ts, std::move(R), std::move(promise)); + } + }); +} + +void ValidatorManagerImpl::get_block_by_seqno_for_litequery(AccountIdPrefixFull account, BlockSeqno seqno, + td::Promise promise) { + get_block_by_seqno_from_db( + account, seqno, + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_ok() && R.ok()->is_applied()) { + promise.set_value(R.move_as_ok()); + } else { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::process_lookup_block_for_litequery_error, account, 2, + seqno, std::move(R), std::move(promise)); + } + }); +} + +void ValidatorManagerImpl::process_block_handle_for_litequery_error(BlockIdExt block_id, + td::Result r_handle, + td::Promise promise) { + td::Status err; + if (r_handle.is_error()) { + err = r_handle.move_as_error(); + } else { + auto handle = r_handle.move_as_ok(); + if (handle->is_applied()) { + promise.set_value(std::move(handle)); + return; + } + if (!handle->received() || !handle->received_state()) { + err = td::Status::Error(ErrorCode::notready, PSTRING() << "block " << block_id.id.to_str() << " is not in db"); + } else { + err = td::Status::Error(ErrorCode::notready, PSTRING() << "block " << block_id.id.to_str() << " is not applied"); + } + } + if (block_id.is_masterchain()) { + if (block_id.seqno() > last_masterchain_seqno_) { + err = err.move_as_error_suffix(PSTRING() << " (last known masterchain block: " << last_masterchain_seqno_ << ")"); + } + } else { + for (auto &shard : shard_client_shards_) { + if (shard_intersects(shard->shard(), block_id.shard_full())) { + if (block_id.seqno() > shard->top_block_id().seqno()) { + err = err.move_as_error_suffix( + PSTRING() << " (possibly out of sync: shard_client_seqno=" + << (shard_client_handle_ ? shard_client_handle_->id().seqno() : 0) << " ls_seqno=" + << (last_liteserver_state_.not_null() ? last_liteserver_state_->get_seqno() : 0) << ")"); + } + break; + } + } + } + promise.set_error(std::move(err)); +} + +void ValidatorManagerImpl::process_lookup_block_for_litequery_error(AccountIdPrefixFull account, int type, + td::uint64 value, + td::Result r_handle, + td::Promise promise) { + td::Status err; + if (r_handle.is_error()) { + err = r_handle.move_as_error(); + } else { + auto handle = r_handle.move_as_ok(); + if (handle->is_applied()) { + promise.set_value(std::move(handle)); + return; + } + if (!handle->received() || !handle->received_state()) { + err = td::Status::Error(ErrorCode::notready, PSTRING() << "block " << handle->id().to_str() << " is not in db"); + } else { + err = td::Status::Error(ErrorCode::notready, PSTRING() << "block " << handle->id().to_str() << " is not applied"); + } + } + if (account.is_masterchain()) { + if (value > (type == 0 + ? last_masterchain_state_->get_logical_time() + : (type == 1 ? last_masterchain_state_->get_unix_time() : last_masterchain_state_->get_seqno()))) { + err = err.move_as_error_suffix(PSTRING() << " (last known masterchain block: " << last_masterchain_seqno_ << ")"); + } + } else { + for (auto &shard : shard_client_shards_) { + if (shard_intersects(shard->shard(), account.as_leaf_shard())) { + if (value > (type == 0 ? shard->end_lt() + : (type == 1 ? (shard_client_handle_ ? shard_client_handle_->unix_time() : 0) + : shard->top_block_id().seqno()))) { + err = err.move_as_error_suffix( + PSTRING() << " (possibly out of sync: shard_client_seqno=" + << (shard_client_handle_ ? shard_client_handle_->id().seqno() : 0) << " ls_seqno=" + << (last_liteserver_state_.not_null() ? last_liteserver_state_->get_seqno() : 0) << ")"); + } + break; + } + } + } + static std::string names[3] = {"lt", "utime", "seqno"}; + err = err.move_as_error_prefix(PSTRING() << "cannot find block " << account.to_str() << " " << names[type] << "=" + << value << ": "); + promise.set_error(std::move(err)); +} + +void ValidatorManagerImpl::get_block_candidate_for_litequery(PublicKey source, BlockIdExt block_id, + FileHash collated_data_hash, + td::Promise promise) { + if (!opts_->nonfinal_ls_queries_enabled()) { + promise.set_error(td::Status::Error("query is not allowed")); + return; + } + get_block_candidate_from_db(source, block_id, collated_data_hash, std::move(promise)); +} + +void ValidatorManagerImpl::get_validator_groups_info_for_litequery( + td::optional shard, + td::Promise> promise) { + if (!opts_->nonfinal_ls_queries_enabled()) { + promise.set_error(td::Status::Error("query is not allowed")); + return; + } + class Actor : public td::actor::Actor { + public: + explicit Actor(std::vector> groups, + td::Promise> promise) + : groups_(std::move(groups)), promise_(std::move(promise)) { + } + + void start_up() override { + pending_ = groups_.size(); + if (pending_ == 0) { + promise_.set_result(std::move(result_)); + stop(); + return; + } + for (auto &x : groups_) { + td::actor::send_closure( + x, &ValidatorGroup::get_validator_group_info_for_litequery, + [SelfId = actor_id(this)](td::Result> R) { + td::actor::send_closure(SelfId, &Actor::on_result, R.is_ok() ? R.move_as_ok() : nullptr); + }); + } + } + + void on_result(tl_object_ptr r) { + if (r) { + result_->groups_.push_back(std::move(r)); + } + --pending_; + if (pending_ == 0) { + promise_.set_result(std::move(result_)); + stop(); + } + } + + private: + std::vector> groups_; + size_t pending_; + td::Promise> promise_; + tl_object_ptr result_ = + create_tl_object(); + }; + std::vector> groups; + for (auto &x : validator_groups_) { + if (x.second.actor.empty()) { + continue; + } + if (shard && shard.value() != x.second.shard) { + continue; + } + groups.push_back(x.second.actor.get()); + } + td::actor::create_actor("get-validator-groups-info", std::move(groups), std::move(promise)).release(); +} + +void ValidatorManagerImpl::update_options(td::Ref opts) { + if (!shard_client_.empty()) { + td::actor::send_closure(shard_client_, &ShardClient::update_options, opts); + } + if (!serializer_.empty()) { + td::actor::send_closure(serializer_, &AsyncStateSerializer::update_options, opts); + } + if (!queue_size_counter_.empty()) { + td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::update_options, opts); + } + for (auto &group : validator_groups_) { + td::actor::send_closure(group.second.actor, &ValidatorGroup::update_options, opts); + } + for (auto &group : next_validator_groups_) { + td::actor::send_closure(group.second.actor, &ValidatorGroup::update_options, opts); + } + opts_ = std::move(opts); +} + +void ValidatorManagerImpl::add_persistent_state_description(td::Ref desc) { + auto now = (UnixTime)td::Clocks::system(); + if (desc->end_time <= now) { + return; + } + td::actor::send_closure(db_, &Db::add_persistent_state_description, desc, [](td::Result) {}); + auto it = persistent_state_descriptions_.begin(); + while (it != persistent_state_descriptions_.end()) { + const auto &prev_desc = it->second; + if (prev_desc->end_time <= now) { + for (const BlockIdExt &block_id : prev_desc->shard_blocks) { + persistent_state_blocks_.erase(block_id); + } + it = persistent_state_descriptions_.erase(it); + } else { + ++it; + } + } + add_persistent_state_description_impl(std::move(desc)); +} + +void ValidatorManagerImpl::add_persistent_state_description_impl(td::Ref desc) { + if (!persistent_state_descriptions_.emplace(desc->masterchain_id.seqno(), desc).second) { + return; + } + LOG(DEBUG) << "Add persistent state description for mc block " << desc->masterchain_id.to_str() + << " start_time=" << desc->start_time << " end_time=" << desc->end_time; + for (const BlockIdExt &block_id : desc->shard_blocks) { + persistent_state_blocks_[block_id] = desc; + LOG(DEBUG) << "Persistent state description: shard block " << block_id.to_str(); + } +} + +void ValidatorManagerImpl::got_persistent_state_descriptions(std::vector> descs) { + for (auto &desc : descs) { + add_persistent_state_description_impl(std::move(desc)); + } +} + +td::Ref ValidatorManagerImpl::get_block_persistent_state_to_download(BlockIdExt block_id) { + if (block_id.is_masterchain()) { + return {}; + } + auto it = persistent_state_blocks_.find(block_id); + if (it == persistent_state_blocks_.end()) { + return {}; + } + if (it->second->masterchain_id.seqno() + 16 >= min_confirmed_masterchain_seqno_) { + // Do not download persistent states during ordinary shard client sync + return {}; + } + return it->second; } td::actor::ActorOwn ValidatorManagerFactory::create( @@ -2610,6 +3398,110 @@ td::actor::ActorOwn ValidatorManagerFactory::create( rldp, overlays); } +void ValidatorManagerImpl::record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, + td::optional stats) { + if (!stats) { + ++(block_id.is_masterchain() ? total_collated_blocks_master_error_ : total_collated_blocks_shard_error_); + return; + } + auto &record = new_block_stats_record(block_id); + record.collator_work_time_ = work_time; + record.collator_cpu_work_time_ = cpu_work_time; + record.collator_stats_ = std::move(stats.value()); + ++(block_id.is_masterchain() ? total_collated_blocks_master_ok_ : total_collated_blocks_shard_ok_); +} + +void ValidatorManagerImpl::record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, + bool success) { + auto &record = new_block_stats_record(block_id); + record.validator_work_time_ = work_time; + record.validator_cpu_work_time_ = cpu_work_time; + if (success) { + ++(block_id.is_masterchain() ? total_validated_blocks_master_ok_ : total_validated_blocks_shard_ok_); + } else { + ++(block_id.is_masterchain() ? total_validated_blocks_master_error_ : total_validated_blocks_shard_error_); + } +} + +ValidatorManagerImpl::RecordedBlockStats &ValidatorManagerImpl::new_block_stats_record(BlockIdExt block_id) { + if (!recorded_block_stats_.count(block_id)) { + recorded_block_stats_lru_.push(block_id); + if (recorded_block_stats_lru_.size() > 4096) { + recorded_block_stats_.erase(recorded_block_stats_lru_.front()); + recorded_block_stats_lru_.pop(); + } + } + return recorded_block_stats_[block_id]; +} + +void ValidatorManagerImpl::register_stats_provider( + td::uint64 idx, std::string prefix, + std::function>>)> callback) { + stats_providers_[idx] = {std::move(prefix), std::move(callback)}; +} + +void ValidatorManagerImpl::unregister_stats_provider(td::uint64 idx) { + stats_providers_.erase(idx); +} + +size_t ValidatorManagerImpl::CheckedExtMsgCounter::get_msg_count(WorkchainId wc, StdSmcAddress addr) { + before_query(); + auto it1 = counter_cur_.find({wc, addr}); + auto it2 = counter_prev_.find({wc, addr}); + return (it1 == counter_cur_.end() ? 0 : it1->second) + (it2 == counter_prev_.end() ? 0 : it2->second); +} +size_t ValidatorManagerImpl::CheckedExtMsgCounter::inc_msg_count(WorkchainId wc, StdSmcAddress addr) { + before_query(); + auto it2 = counter_prev_.find({wc, addr}); + return (it2 == counter_prev_.end() ? 0 : it2->second) + ++counter_cur_[{wc, addr}]; +} +void ValidatorManagerImpl::CheckedExtMsgCounter::before_query() { + while (cleanup_at_.is_in_past()) { + counter_prev_ = std::move(counter_cur_); + counter_cur_.clear(); + if (counter_prev_.empty()) { + cleanup_at_ = td::Timestamp::in(max_ext_msg_per_addr_time_window() / 2.0); + break; + } + cleanup_at_ += max_ext_msg_per_addr_time_window() / 2.0; + } +} + +void ValidatorManagerImpl::init_validator_telemetry() { + if (last_masterchain_state_.is_null()) { + return; + } + td::Ref validator_set = last_masterchain_state_->get_total_validator_set(0); + if (validator_set.is_null()) { + validator_telemetry_.clear(); + return; + } + std::set processed; + for (auto& key : temp_keys_) { + if (const ValidatorDescr* desc = validator_set->get_validator(key.bits256_value())) { + processed.insert(key); + adnl::AdnlNodeIdShort adnl_id; + if (desc->addr.is_zero()) { + adnl_id = adnl::AdnlNodeIdShort{ValidatorFullId{desc->key}.compute_short_id()}; + } else { + adnl_id = adnl::AdnlNodeIdShort{desc->addr}; + } + auto& telemetry = validator_telemetry_[key]; + if (telemetry.empty()) { + telemetry = td::actor::create_actor( + "telemetry", key, adnl_id, opts_->zero_block_id().file_hash, actor_id(this)); + } + } + } + for (auto it = validator_telemetry_.begin(); it != validator_telemetry_.end();) { + if (processed.contains(it->first)) { + ++it; + } else { + it = validator_telemetry_.erase(it); + } + } +} + } // namespace validator } // namespace ton diff --git a/validator/manager.hpp b/validator/manager.hpp index d133f83b..418deb35 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -18,20 +18,29 @@ */ #pragma once +#include "common/refcnt.hpp" #include "interfaces/validator-manager.h" #include "interfaces/db.h" +#include "td/actor/ActorStats.h" #include "td/actor/PromiseFuture.h" +#include "td/utils/SharedSlice.h" +#include "td/utils/buffer.h" #include "td/utils/port/Poll.h" +#include "td/utils/port/StdStreams.h" #include "validator-group.hpp" #include "shard-client.hpp" #include "manager-init.h" #include "state-serializer.hpp" #include "rldp/rldp.h" #include "token-manager.h" +#include "queue-size-counter.hpp" +#include "validator-telemetry.hpp" +#include "impl/candidates-buffer.hpp" #include #include #include +#include namespace ton { @@ -180,9 +189,21 @@ class ValidatorManagerImpl : public ValidatorManager { waiting_.resize(j); } }; + template + struct WaitListCaching : public WaitList { + bool done_ = false; + ResType result_; + td::Timestamp remove_at_; + }; std::map>> wait_state_; std::map>> wait_block_data_; + struct CachedBlockState { + td::Ref state_; + td::Timestamp ttl_; + }; + std::map block_state_cache_; + struct WaitBlockHandle { std::vector> waiting_; }; @@ -212,24 +233,52 @@ class ValidatorManagerImpl : public ValidatorManager { }; // DATA FOR COLLATOR std::map> shard_blocks_; - std::map, std::unique_ptr>> ext_messages_; - std::map, std::map>> ext_addr_messages_; - std::map> ext_messages_hashes_; + + std::map cached_block_candidates_; + std::list cached_block_candidates_lru_; + + struct ExtMessages { + std::map, std::unique_ptr>> ext_messages_; + std::map, std::map>> + ext_addr_messages_; + void erase(const MessageId& id) { + auto it = ext_messages_.find(id); + CHECK(it != ext_messages_.end()); + ext_addr_messages_[it->second->address()].erase(id.hash); + ext_messages_.erase(it); + } + }; + std::map ext_msgs_; // priority -> messages + std::map>> ext_messages_hashes_; // hash -> priority + td::Timestamp cleanup_mempool_at_; // IHR ? std::map, std::unique_ptr>> ihr_messages_; std::map> ihr_messages_hashes_; + struct CheckedExtMsgCounter { + std::map, size_t> counter_cur_, counter_prev_; + td::Timestamp cleanup_at_ = td::Timestamp::now(); + + size_t get_msg_count(WorkchainId wc, StdSmcAddress addr); + size_t inc_msg_count(WorkchainId wc, StdSmcAddress addr); + void before_query(); + } checked_ext_msg_counter_; + private: // VALIDATOR GROUPS ValidatorSessionId get_validator_set_id(ShardIdFull shard, td::Ref val_set, td::Bits256 opts_hash, BlockSeqno last_key_block_seqno, const validatorsession::ValidatorSessionOptions &opts); td::actor::ActorOwn create_validator_group(ValidatorSessionId session_id, ShardIdFull shard, - td::Ref validator_set, + td::Ref validator_set, BlockSeqno key_seqno, validatorsession::ValidatorSessionOptions opts, bool create_catchain); - std::map> validator_groups_; - std::map> next_validator_groups_; + struct ValidatorGroupEntry { + td::actor::ActorOwn actor; + ShardIdFull shard; + }; + std::map validator_groups_; + std::map next_validator_groups_; std::set check_gc_list_; std::vector gc_list_; @@ -245,6 +294,7 @@ class ValidatorManagerImpl : public ValidatorManager { BlockHandle last_key_block_handle_; BlockHandle last_known_key_block_handle_; BlockHandle shard_client_handle_; + std::vector> shard_client_shards_; td::Ref last_liteserver_state_; td::Ref do_get_last_liteserver_state(); @@ -261,6 +311,7 @@ class ValidatorManagerImpl : public ValidatorManager { std::vector perf_timer_stats; void new_masterchain_block(); + void update_shard_overlays(); void update_shards(); void update_shard_blocks(); void written_destroyed_validator_sessions(std::vector> groups); @@ -279,8 +330,7 @@ class ValidatorManagerImpl : public ValidatorManager { void applied_hardfork(); void prestart_sync(); void download_next_archive(); - void downloaded_archive_slice(std::string name, bool is_tmp); - void checked_archive_slice(std::vector seqno); + void checked_archive_slice(BlockSeqno new_last_mc_seqno, BlockSeqno new_shard_client_seqno); void finish_prestart_sync(); void completed_prestart_sync(); @@ -296,6 +346,7 @@ class ValidatorManagerImpl : public ValidatorManager { } void add_temp_key(PublicKeyHash key, td::Promise promise) override { temp_keys_.insert(key); + init_validator_telemetry(); promise.set_value(td::Unit()); } void del_permanent_key(PublicKeyHash key, td::Promise promise) override { @@ -304,6 +355,7 @@ class ValidatorManagerImpl : public ValidatorManager { } void del_temp_key(PublicKeyHash key, td::Promise promise) override { temp_keys_.erase(key); + init_validator_telemetry(); promise.set_value(td::Unit()); } @@ -315,6 +367,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise promise) override; void validate_block(ReceivedBlock block, td::Promise promise) override; void prevalidate_block(BlockBroadcast broadcast, td::Promise promise) override; + void validated_block_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno); //void create_validate_block(BlockId block, td::BufferSlice data, td::Promise promise) = 0; void sync_complete(td::Promise promise) override; @@ -330,18 +383,21 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise promise) override; void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, td::int64 max_length, td::Promise promise) override; + void get_previous_persistent_state_files( + BlockSeqno cur_mc_seqno, td::Promise>> promise) override; void get_block_proof(BlockHandle handle, td::Promise promise) override; void get_block_proof_link(BlockHandle block_id, td::Promise promise) override; void get_key_block_proof(BlockIdExt block_id, td::Promise promise) override; void get_key_block_proof_link(BlockIdExt block_id, td::Promise promise) override; //void get_block_description(BlockIdExt block_id, td::Promise promise) override; - void new_external_message(td::BufferSlice data) override; - void add_external_message(td::Ref message); + void new_external_message(td::BufferSlice data, int priority) override; + void add_external_message(td::Ref message, int priority); void check_external_message(td::BufferSlice data, td::Promise> promise) override; void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; + void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override; void add_ext_server_id(adnl::AdnlNodeIdShort id) override; void add_ext_server_port(td::uint16 port) override; @@ -386,7 +442,10 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_block_signatures_short(BlockIdExt id, td::Timestamp timeout, td::Promise> promise) override; - void set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) override; + void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) override; + void send_block_candidate_broadcast(BlockIdExt id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override; void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; @@ -397,7 +456,8 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override; void wait_block_message_queue_short(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; - void get_external_messages(ShardIdFull shard, td::Promise>> promise) override; + void get_external_messages(ShardIdFull shard, + td::Promise, int>>> promise) override; void get_ihr_messages(ShardIdFull shard, td::Promise>> promise) override; void get_shard_blocks(BlockIdExt masterchain_block_id, td::Promise>> promise) override; @@ -413,6 +473,7 @@ class ValidatorManagerImpl : public ValidatorManager { void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) override; void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) override; + void get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) override; void get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) override; void get_block_proof_from_db_short(BlockIdExt id, td::Promise> promise) override; void get_block_proof_link_from_db(ConstBlockHandle handle, td::Promise> promise) override; @@ -449,11 +510,16 @@ class ValidatorManagerImpl : public ValidatorManager { void send_external_message(td::Ref message) override; void send_ihr_message(td::Ref message) override; void send_top_shard_block_description(td::Ref desc) override; - void send_block_broadcast(BlockBroadcast broadcast) override; + void send_block_broadcast(BlockBroadcast broadcast, int mode) override; + void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override; + void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + td::Promise>> promise) override; + void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override; void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; - void subscribe_to_shard(ShardIdFull shard) override; void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override; void get_async_serializer_state(td::Promise promise) override; @@ -461,12 +527,12 @@ class ValidatorManagerImpl : public ValidatorManager { void try_get_static_file(FileHash file_hash, td::Promise promise) override; void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) override { - td::actor::send_closure(token_manager_, &TokenManager::get_download_token, download_size, priority, timeout, + td::Promise> promise) override { + td::actor::send_closure(token_manager_, &TokenManager::get_token, download_size, priority, timeout, std::move(promise)); } - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) override; + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::Promise promise) override; void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise) override; @@ -479,6 +545,7 @@ class ValidatorManagerImpl : public ValidatorManager { } void add_shard_block_description(td::Ref desc); + void add_cached_block_candidate(ReceivedBlock block); void register_block_handle(BlockHandle handle); @@ -486,10 +553,12 @@ class ValidatorManagerImpl : public ValidatorManager { void finished_wait_data(BlockHandle handle, td::Result> R); void start_up() override; + void init_last_masterchain_state(td::Ref state) override; void started(ValidatorManagerInitResult result); void read_gc_list(std::vector list); bool is_validator(); + bool validating_masterchain(); PublicKeyHash get_validator(ShardIdFull shard, td::Ref val_set); ValidatorManagerImpl(td::Ref opts, std::string db_root, @@ -533,6 +602,8 @@ class ValidatorManagerImpl : public ValidatorManager { void prepare_stats(td::Promise>> promise) override; + void prepare_actor_stats(td::Promise promise) override; + void prepare_perf_timer_stats(td::Promise> promise) override; void add_perf_timer_stat(std::string name, double duration) override; @@ -541,6 +612,53 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) override; void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override; + void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override; + void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override; + + void update_options(td::Ref opts) override; + + void add_persistent_state_description(td::Ref desc) override; + + void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { + if (queue_size_counter_.empty()) { + if (last_masterchain_state_.is_null()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); + return; + } + queue_size_counter_ = + td::actor::create_actor("queuesizecounter", last_masterchain_state_, opts_, actor_id(this)); + } + if (!opts_->need_monitor(block_id.shard_full(), last_masterchain_state_)) { + return promise.set_error( + td::Status::Error(PSTRING() << "not monitoring shard " << block_id.shard_full().to_str())); + } + td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::get_queue_size, block_id, std::move(promise)); + } + + void get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) override; + void get_block_data_for_litequery(BlockIdExt block_id, td::Promise> promise) override; + void get_block_state_for_litequery(BlockIdExt block_id, td::Promise> promise) override; + void get_block_by_lt_for_litequery(AccountIdPrefixFull account, LogicalTime lt, + td::Promise promise) override; + void get_block_by_unix_time_for_litequery(AccountIdPrefixFull account, UnixTime ts, + td::Promise promise) override; + void get_block_by_seqno_for_litequery(AccountIdPrefixFull account, BlockSeqno seqno, + td::Promise promise) override; + void process_block_handle_for_litequery_error(BlockIdExt block_id, td::Result r_handle, + td::Promise promise); + void process_lookup_block_for_litequery_error(AccountIdPrefixFull account, int type, td::uint64 value, + td::Result r_handle, + td::Promise promise); + void get_block_candidate_for_litequery(PublicKey source, BlockIdExt block_id, FileHash collated_data_hash, + td::Promise promise) override; + void get_validator_groups_info_for_litequery( + td::optional shard, + td::Promise> promise) override; + + void add_lite_query_stats(int lite_query_id, bool success) override { + ++ls_stats_[lite_query_id]; + ++(success ? total_ls_queries_ok_ : total_ls_queries_error_)[lite_query_id]; + } private: td::Timestamp resend_shard_blocks_at_; @@ -584,11 +702,12 @@ class ValidatorManagerImpl : public ValidatorManager { td::actor::ActorOwn serializer_; - std::map> to_import_; + std::map> to_import_; private: std::unique_ptr callback_; td::actor::ActorOwn db_; + td::actor::ActorOwn actor_stats_; bool started_ = false; bool allow_validate_ = false; @@ -603,9 +722,74 @@ class ValidatorManagerImpl : public ValidatorManager { double max_mempool_num() const { return opts_->max_mempool_num(); } + size_t max_cached_candidates() const { + return 128; + } + static double max_ext_msg_per_addr_time_window() { + return 10.0; + } + static size_t max_ext_msg_per_addr() { + return 3 * 10; + } + + void got_persistent_state_descriptions(std::vector> descs); + void add_persistent_state_description_impl(td::Ref desc); + td::Ref get_block_persistent_state_to_download(BlockIdExt block_id); private: + bool need_monitor(ShardIdFull shard) const { + return opts_->need_monitor(shard, last_masterchain_state_); + } + std::map> shard_client_waiters_; + td::actor::ActorOwn queue_size_counter_; + + td::Timestamp log_ls_stats_at_; + std::map ls_stats_; // lite_api ID -> count, 0 for unknown + td::uint32 ls_stats_check_ext_messages_{0}; + + UnixTime started_at_ = (UnixTime)td::Clocks::system(); + std::map total_ls_queries_ok_, total_ls_queries_error_; // lite_api ID -> count, 0 for unknown + td::uint64 total_check_ext_messages_ok_{0}, total_check_ext_messages_error_{0}; + td::uint64 total_collated_blocks_master_ok_{0}, total_collated_blocks_master_error_{0}; + td::uint64 total_validated_blocks_master_ok_{0}, total_validated_blocks_master_error_{0}; + td::uint64 total_collated_blocks_shard_ok_{0}, total_collated_blocks_shard_error_{0}; + td::uint64 total_validated_blocks_shard_ok_{0}, total_validated_blocks_shard_error_{0}; + + size_t active_validator_groups_master_{0}, active_validator_groups_shard_{0}; + + td::actor::ActorOwn candidates_buffer_; + + struct RecordedBlockStats { + double collator_work_time_ = -1.0; + double collator_cpu_work_time_ = -1.0; + td::optional collator_stats_; + double validator_work_time_ = -1.0; + double validator_cpu_work_time_ = -1.0; + }; + std::map recorded_block_stats_; + std::queue recorded_block_stats_lru_; + + void record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, + td::optional stats) override; + void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, bool success) override; + RecordedBlockStats &new_block_stats_record(BlockIdExt block_id); + + void register_stats_provider( + td::uint64 idx, std::string prefix, + std::function>>)> callback) override; + void unregister_stats_provider(td::uint64 idx) override; + + std::map> validator_telemetry_; + + void init_validator_telemetry(); + + std::map> persistent_state_descriptions_; + std::map> persistent_state_blocks_; + + std::map>>)>>> + stats_providers_; }; } // namespace validator diff --git a/validator/net/download-archive-slice.cpp b/validator/net/download-archive-slice.cpp index e3acb4ba..c2f8ecea 100644 --- a/validator/net/download-archive-slice.cpp +++ b/validator/net/download-archive-slice.cpp @@ -20,6 +20,8 @@ #include "td/utils/port/path.h" #include "td/utils/overloaded.h" +#include + namespace ton { namespace validator { @@ -27,12 +29,13 @@ namespace validator { namespace fullnode { DownloadArchiveSlice::DownloadArchiveSlice( - BlockSeqno masterchain_seqno, std::string tmp_dir, adnl::AdnlNodeIdShort local_id, + BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, td::Timestamp timeout, - td::actor::ActorId validator_manager, td::actor::ActorId rldp, + td::actor::ActorId validator_manager, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId adnl, td::actor::ActorId client, td::Promise promise) : masterchain_seqno_(masterchain_seqno) + , shard_prefix_(shard_prefix) , tmp_dir_(std::move(tmp_dir)) , local_id_(local_id) , overlay_id_(overlay_id) @@ -114,7 +117,13 @@ void DownloadArchiveSlice::got_node_to_download(adnl::AdnlNodeIdShort download_f } }); - auto q = create_serialize_tl_object(masterchain_seqno_); + td::BufferSlice q; + if (shard_prefix_.is_masterchain()) { + q = create_serialize_tl_object(masterchain_seqno_); + } else { + q = create_serialize_tl_object(masterchain_seqno_, + create_tl_shard_id(shard_prefix_)); + } if (client_.empty()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_query, download_from_, local_id_, overlay_id_, "get_archive_info", std::move(P), td::Timestamp::in(3.0), std::move(q)); @@ -144,6 +153,9 @@ void DownloadArchiveSlice::got_archive_info(td::BufferSlice data) { return; } + prev_logged_timer_ = td::Timer(); + LOG(INFO) << "downloading archive slice #" << masterchain_seqno_ << " " << shard_prefix_.to_str() << " from " + << download_from_; get_archive_slice(); } @@ -159,12 +171,12 @@ void DownloadArchiveSlice::get_archive_slice() { auto q = create_serialize_tl_object(archive_id_, offset_, slice_size()); if (client_.empty()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, overlay_id_, - "get_archive_slice", std::move(P), td::Timestamp::in(3.0), std::move(q), + "get_archive_slice", std::move(P), td::Timestamp::in(15.0), std::move(q), slice_size() + 1024, rldp_); } else { td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get_archive_slice", create_serialize_tl_object_suffix(std::move(q)), - td::Timestamp::in(1.0), std::move(P)); + td::Timestamp::in(15.0), std::move(P)); } } @@ -181,7 +193,18 @@ void DownloadArchiveSlice::got_archive_slice(td::BufferSlice data) { offset_ += data.size(); + double elapsed = prev_logged_timer_.elapsed(); + if (elapsed > 10.0) { + prev_logged_timer_ = td::Timer(); + LOG(INFO) << "downloading archive slice #" << masterchain_seqno_ << " " << shard_prefix_.to_str() + << ": total=" << offset_ << " (" + << td::format::as_size((td::uint64)(double(offset_ - prev_logged_sum_) / elapsed)) << "/s)"; + prev_logged_sum_ = offset_; + } + if (data.size() < slice_size()) { + LOG(INFO) << "finished downloading arcrive slice #" << masterchain_seqno_ << " " << shard_prefix_.to_str() + << ": total=" << offset_; finish_query(); } else { get_archive_slice(); diff --git a/validator/net/download-archive-slice.hpp b/validator/net/download-archive-slice.hpp index 6ef11d1a..42fd715f 100644 --- a/validator/net/download-archive-slice.hpp +++ b/validator/net/download-archive-slice.hpp @@ -21,7 +21,6 @@ #include "overlay/overlays.h" #include "ton/ton-types.h" #include "validator/validator.h" -#include "rldp/rldp.h" #include "adnl/adnl-ext-client.h" #include "td/utils/port/FileFd.h" @@ -33,12 +32,13 @@ namespace fullnode { class DownloadArchiveSlice : public td::actor::Actor { public: - DownloadArchiveSlice(BlockSeqno masterchain_seqno, std::string tmp_dir, adnl::AdnlNodeIdShort local_id, - overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, td::Timestamp timeout, + DownloadArchiveSlice(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, + adnl::AdnlNodeIdShort download_from, td::Timestamp timeout, td::actor::ActorId validator_manager, - td::actor::ActorId rldp, td::actor::ActorId overlays, - td::actor::ActorId adnl, td::actor::ActorId client, - td::Promise promise); + td::actor::ActorId rldp, + td::actor::ActorId overlays, td::actor::ActorId adnl, + td::actor::ActorId client, td::Promise promise); void abort_query(td::Status reason); void alarm() override; @@ -51,11 +51,12 @@ class DownloadArchiveSlice : public td::actor::Actor { void got_archive_slice(td::BufferSlice data); static constexpr td::uint32 slice_size() { - return 1 << 17; + return 1 << 21; } private: BlockSeqno masterchain_seqno_; + ShardIdFull shard_prefix_; std::string tmp_dir_; std::string tmp_name_; td::FileFd fd_; @@ -68,11 +69,14 @@ class DownloadArchiveSlice : public td::actor::Actor { td::Timestamp timeout_; td::actor::ActorId validator_manager_; - td::actor::ActorId rldp_; + td::actor::ActorId rldp_; td::actor::ActorId overlays_; td::actor::ActorId adnl_; td::actor::ActorId client_; td::Promise promise_; + + td::uint64 prev_logged_sum_ = 0; + td::Timer prev_logged_timer_; }; } // namespace fullnode diff --git a/validator/net/download-block-new.cpp b/validator/net/download-block-new.cpp index ef5ed7e5..37580cef 100644 --- a/validator/net/download-block-new.cpp +++ b/validator/net/download-block-new.cpp @@ -23,6 +23,7 @@ #include "td/utils/overloaded.h" #include "ton/ton-io.hpp" #include "validator/full-node.h" +#include "full-node-serializer.hpp" namespace ton { @@ -143,7 +144,7 @@ void DownloadBlockNew::got_block_handle(BlockHandle handle) { return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &DownloadBlockNew::abort_query, R.move_as_error_prefix("failed to get download token: ")); @@ -155,7 +156,7 @@ void DownloadBlockNew::got_block_handle(BlockHandle handle) { std::move(P)); } -void DownloadBlockNew::got_download_token(std::unique_ptr token) { +void DownloadBlockNew::got_download_token(std::unique_ptr token) { token_ = std::move(token); if (download_from_.is_zero() && client_.empty()) { @@ -201,12 +202,12 @@ void DownloadBlockNew::got_node_to_download(adnl::AdnlNodeIdShort node) { } if (client_.empty()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, overlay_id_, - "get_proof", std::move(P), td::Timestamp::in(3.0), std::move(q), + "get_block_full", std::move(P), td::Timestamp::in(15.0), std::move(q), FullNode::max_proof_size() + FullNode::max_block_size() + 128, rldp_); } else { - td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get_prepare", + td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get_block_full", create_serialize_tl_object_suffix(std::move(q)), - td::Timestamp::in(1.0), std::move(P)); + td::Timestamp::in(15.0), std::move(P)); } } @@ -219,52 +220,54 @@ void DownloadBlockNew::got_data(td::BufferSlice data) { } auto f = F.move_as_ok(); + if (f->get_id() == ton_api::tonNode_dataFullEmpty::ID) { + abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have this block")); + return; + } + BlockIdExt id; + td::BufferSlice proof, block_data; + bool is_link; + td::Status S = deserialize_block_full(*f, id, proof, block_data, is_link, overlay::Overlays::max_fec_broadcast_size()); + if (S.is_error()) { + abort_query(S.move_as_error_prefix("cannot deserialize block: ")); + return; + } - ton_api::downcast_call( - *f.get(), - td::overloaded( - [&](ton_api::tonNode_dataFullEmpty &x) { - abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have this block")); - }, - [&, self = this](ton_api::tonNode_dataFull &x) { - if (!allow_partial_proof_ && x.is_link_) { - abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have proof for this block")); - return; - } - auto id = create_block_id(x.id_); - if (block_id_.is_valid() && id != block_id_) { - abort_query(td::Status::Error(ErrorCode::notready, "received data for wrong block")); - return; - } - block_.id = id; - block_.data = std::move(x.block_); - if (td::sha256_bits256(block_.data.as_slice()) != id.file_hash) { - abort_query(td::Status::Error(ErrorCode::notready, "received data with bad hash")); - return; - } + if (!allow_partial_proof_ && is_link) { + abort_query(td::Status::Error(ErrorCode::notready, "node doesn't have proof for this block")); + return; + } + if (block_id_.is_valid() && id != block_id_) { + abort_query(td::Status::Error(ErrorCode::notready, "received data for wrong block")); + return; + } + block_.id = id; + block_.data = std::move(block_data); + if (td::sha256_bits256(block_.data.as_slice()) != id.file_hash) { + abort_query(td::Status::Error(ErrorCode::notready, "received data with bad hash")); + return; + } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(self)](td::Result R) { - if (R.is_error()) { - td::actor::send_closure(SelfId, &DownloadBlockNew::abort_query, - R.move_as_error_prefix("received bad proof: ")); - } else { - td::actor::send_closure(SelfId, &DownloadBlockNew::checked_block_proof); - } - }); - if (block_id_.is_valid()) { - if (x.is_link_) { - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_proof_link, - block_id_, std::move(x.proof_), std::move(P)); - } else { - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_proof, block_id_, - std::move(x.proof_), std::move(P)); - } - } else { - CHECK(!x.is_link_); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_is_next_proof, - prev_id_, id, std::move(x.proof_), std::move(P)); - } - })); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadBlockNew::abort_query, R.move_as_error_prefix("received bad proof: ")); + } else { + td::actor::send_closure(SelfId, &DownloadBlockNew::checked_block_proof); + } + }); + if (block_id_.is_valid()) { + if (is_link) { + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_proof_link, block_id_, + std::move(proof), std::move(P)); + } else { + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_proof, block_id_, + std::move(proof), std::move(P)); + } + } else { + CHECK(!is_link); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::validate_block_is_next_proof, prev_id_, id, + std::move(proof), std::move(P)); + } } void DownloadBlockNew::got_data_from_db(td::BufferSlice data) { diff --git a/validator/net/download-block-new.hpp b/validator/net/download-block-new.hpp index d2a0e136..ecd062ee 100644 --- a/validator/net/download-block-new.hpp +++ b/validator/net/download-block-new.hpp @@ -49,7 +49,7 @@ class DownloadBlockNew : public td::actor::Actor { void start_up() override; void got_block_handle(BlockHandle handle); - void got_download_token(std::unique_ptr token); + void got_download_token(std::unique_ptr token); void got_node_to_download(adnl::AdnlNodeIdShort node); void got_data(td::BufferSlice data); void got_data_from_db(td::BufferSlice data); @@ -79,7 +79,7 @@ class DownloadBlockNew : public td::actor::Actor { bool allow_partial_proof_ = false; - std::unique_ptr token_; + std::unique_ptr token_; }; } // namespace fullnode diff --git a/validator/net/download-block.cpp b/validator/net/download-block.cpp index 5e7c0be9..c60955ed 100644 --- a/validator/net/download-block.cpp +++ b/validator/net/download-block.cpp @@ -128,7 +128,7 @@ void DownloadBlock::got_block_handle(BlockHandle handle) { return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &DownloadBlock::abort_query, R.move_as_error_prefix("failed to get download token: ")); @@ -140,7 +140,7 @@ void DownloadBlock::got_block_handle(BlockHandle handle) { std::move(P)); } -void DownloadBlock::got_download_token(std::unique_ptr token) { +void DownloadBlock::got_download_token(std::unique_ptr token) { token_ = std::move(token); if (download_from_.is_zero() && !short_ && client_.empty()) { @@ -373,12 +373,12 @@ void DownloadBlock::got_block_data_description(td::BufferSlice data_description) auto q = create_serialize_tl_object(create_tl_block_id(block_id_)); if (client_.empty()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, - overlay_id_, "get_block", std::move(P), td::Timestamp::in(3.0), std::move(q), + overlay_id_, "get_block", std::move(P), td::Timestamp::in(15.0), std::move(q), FullNode::max_block_size(), rldp_); } else { td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get_block", create_serialize_tl_object_suffix(std::move(q)), - td::Timestamp::in(3.0), std::move(P)); + td::Timestamp::in(15.0), std::move(P)); } }, [&](ton_api::tonNode_notFound &val) { diff --git a/validator/net/download-block.hpp b/validator/net/download-block.hpp index b1847d58..2e2a715b 100644 --- a/validator/net/download-block.hpp +++ b/validator/net/download-block.hpp @@ -49,7 +49,7 @@ class DownloadBlock : public td::actor::Actor { void start_up() override; void got_block_handle(BlockHandle handle); - void got_download_token(std::unique_ptr token); + void got_download_token(std::unique_ptr token); void got_node_to_download(adnl::AdnlNodeIdShort node); void got_block_proof_description(td::BufferSlice proof_description); void got_block_proof(td::BufferSlice data); @@ -86,7 +86,7 @@ class DownloadBlock : public td::actor::Actor { bool allow_partial_proof_ = false; - std::unique_ptr token_; + std::unique_ptr token_; }; } // namespace fullnode diff --git a/validator/net/download-proof.cpp b/validator/net/download-proof.cpp index 2ff95b88..784ecac2 100644 --- a/validator/net/download-proof.cpp +++ b/validator/net/download-proof.cpp @@ -107,7 +107,7 @@ void DownloadProof::start_up() { } void DownloadProof::checked_db() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &DownloadProof::abort_query, R.move_as_error_prefix("failed to get download token: ")); @@ -119,7 +119,7 @@ void DownloadProof::checked_db() { std::move(P)); } -void DownloadProof::got_download_token(std::unique_ptr token) { +void DownloadProof::got_download_token(std::unique_ptr token) { token_ = std::move(token); if (download_from_.is_zero() && client_.empty()) { diff --git a/validator/net/download-proof.hpp b/validator/net/download-proof.hpp index 0739dcaf..9caf9c3a 100644 --- a/validator/net/download-proof.hpp +++ b/validator/net/download-proof.hpp @@ -45,7 +45,7 @@ class DownloadProof : public td::actor::Actor { void start_up() override; void checked_db(); - void got_download_token(std::unique_ptr token); + void got_download_token(std::unique_ptr token); void got_node_to_download(adnl::AdnlNodeIdShort node); void got_block_proof_description(td::BufferSlice proof_description); void got_block_proof(td::BufferSlice data); @@ -72,7 +72,7 @@ class DownloadProof : public td::actor::Actor { td::BufferSlice data_; - std::unique_ptr token_; + std::unique_ptr token_; }; } // namespace fullnode diff --git a/validator/net/download-state.cpp b/validator/net/download-state.cpp index fedfaae8..6735a2b5 100644 --- a/validator/net/download-state.cpp +++ b/validator/net/download-state.cpp @@ -32,9 +32,9 @@ DownloadState::DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_i overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, td::uint32 priority, td::Timestamp timeout, td::actor::ActorId validator_manager, - td::actor::ActorId rldp, td::actor::ActorId overlays, - td::actor::ActorId adnl, td::actor::ActorId client, - td::Promise promise) + td::actor::ActorId rldp, + td::actor::ActorId overlays, td::actor::ActorId adnl, + td::actor::ActorId client, td::Promise promise) : block_id_(block_id) , masterchain_block_id_(masterchain_block_id) , local_id_(local_id) @@ -52,12 +52,7 @@ DownloadState::DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_i void DownloadState::abort_query(td::Status reason) { if (promise_) { - if (reason.code() == ErrorCode::notready || reason.code() == ErrorCode::timeout) { - VLOG(FULL_NODE_DEBUG) << "failed to download state " << block_id_ << " from " << download_from_ << ": " << reason; - } else { - VLOG(FULL_NODE_NOTICE) << "failed to download state " << block_id_ << " from " << download_from_ << ": " - << reason; - } + LOG(WARNING) << "failed to download state " << block_id_.to_str() << " from " << download_from_ << ": " << reason; promise_.set_error(std::move(reason)); } stop(); @@ -75,8 +70,22 @@ void DownloadState::finish_query() { } void DownloadState::start_up() { + status_ = ProcessStatus(validator_manager_, "process.download_state_net"); alarm_timestamp() = timeout_; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id_, + masterchain_block_id_, + [SelfId = actor_id(this), block_id = block_id_](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &DownloadState::get_block_handle); + } else { + LOG(WARNING) << "got block state from disk: " << block_id.to_str(); + td::actor::send_closure(SelfId, &DownloadState::got_block_state, R.move_as_ok()); + } + }); +} + +void DownloadState::get_block_handle() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &DownloadState::abort_query, R.move_as_error()); @@ -115,7 +124,7 @@ void DownloadState::got_block_handle(BlockHandle handle) { void DownloadState::got_node_to_download(adnl::AdnlNodeIdShort node) { download_from_ = node; - LOG(INFO) << "downloading state " << block_id_ << " from " << download_from_; + LOG(WARNING) << "downloading state " << block_id_.to_str() << " from " << download_from_; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) mutable { if (R.is_error()) { @@ -182,6 +191,7 @@ void DownloadState::got_block_state_description(td::BufferSlice data) { td::Timestamp::in(3.0), std::move(P)); } })); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : 0 bytes, 0B/s"); } void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 requested_size) { @@ -190,14 +200,18 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques parts_.push_back(std::move(data)); double elapsed = prev_logged_timer_.elapsed(); - if (elapsed > 10.0) { + if (elapsed > 5.0) { prev_logged_timer_ = td::Timer(); - LOG(INFO) << "downloading state " << block_id_ << ": total=" << sum_ << - " (" << double(sum_ - prev_logged_sum_) / elapsed << " B/s)"; + auto speed = (td::uint64)((double)(sum_ - prev_logged_sum_) / elapsed); + LOG(WARNING) << "downloading state " << block_id_.to_str() << ": " << td::format::as_size(sum_) << " (" + << td::format::as_size(speed) << "/s)"; + status_.set_status(PSTRING() << block_id_.id.to_str() << " : " << sum_ << " bytes, " << td::format::as_size(speed) + << "/s"); prev_logged_sum_ = sum_; } if (last_part) { + status_.set_status(PSTRING() << block_id_.id.to_str() << " : " << sum_ << " bytes, finishing"); td::BufferSlice res{td::narrow_cast(sum_)}; auto S = res.as_slice(); for (auto &p : parts_) { @@ -210,7 +224,7 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques return; } - td::uint32 part_size = 1 << 18; + td::uint32 part_size = 1 << 21; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), part_size](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &DownloadState::abort_query, R.move_as_error()); @@ -223,18 +237,18 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques create_tl_block_id(block_id_), create_tl_block_id(masterchain_block_id_), sum_, part_size); if (client_.empty()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, overlay_id_, - "download state", std::move(P), td::Timestamp::in(10.0), std::move(query), + "download state", std::move(P), td::Timestamp::in(20.0), std::move(query), FullNode::max_state_size(), rldp_); } else { td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "download state", create_serialize_tl_object_suffix(std::move(query)), - td::Timestamp::in(10.0), std::move(P)); + td::Timestamp::in(20.0), std::move(P)); } } void DownloadState::got_block_state(td::BufferSlice data) { state_ = std::move(data); - LOG(INFO) << "finished downloading state " << block_id_ << ": total=" << sum_; + LOG(WARNING) << "finished downloading state " << block_id_.to_str() << ": " << td::format::as_size(state_.size()); finish_query(); } diff --git a/validator/net/download-state.hpp b/validator/net/download-state.hpp index a586f61f..470c5431 100644 --- a/validator/net/download-state.hpp +++ b/validator/net/download-state.hpp @@ -21,9 +21,10 @@ #include "overlay/overlays.h" #include "ton/ton-types.h" #include "validator/validator.h" -#include "rldp/rldp.h" #include "adnl/adnl-ext-client.h" +#include + namespace ton { namespace validator { @@ -35,7 +36,7 @@ class DownloadState : public td::actor::Actor { DownloadState(BlockIdExt block_id, BlockIdExt masterchain_block_id, adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, td::uint32 priority, td::Timestamp timeout, td::actor::ActorId validator_manager, - td::actor::ActorId rldp, td::actor::ActorId overlays, + td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId adnl, td::actor::ActorId client, td::Promise promise); @@ -44,6 +45,7 @@ class DownloadState : public td::actor::Actor { void finish_query(); void start_up() override; + void get_block_handle(); void got_block_handle(BlockHandle handle); void got_node_to_download(adnl::AdnlNodeIdShort node); void got_block_state_description(td::BufferSlice data_description); @@ -62,7 +64,7 @@ class DownloadState : public td::actor::Actor { td::Timestamp timeout_; td::actor::ActorId validator_manager_; - td::actor::ActorId rldp_; + td::actor::ActorId rldp_; td::actor::ActorId overlays_; td::actor::ActorId adnl_; td::actor::ActorId client_; @@ -75,6 +77,8 @@ class DownloadState : public td::actor::Actor { td::uint64 prev_logged_sum_ = 0; td::Timer prev_logged_timer_; + + ProcessStatus status_; }; } // namespace fullnode diff --git a/validator/net/get-next-key-blocks.cpp b/validator/net/get-next-key-blocks.cpp index 3354b005..2c12e495 100644 --- a/validator/net/get-next-key-blocks.cpp +++ b/validator/net/get-next-key-blocks.cpp @@ -84,7 +84,7 @@ void GetNextKeyBlocks::finish_query() { void GetNextKeyBlocks::start_up() { alarm_timestamp() = timeout_; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &GetNextKeyBlocks::abort_query, R.move_as_error_prefix("failed to get download token: ")); @@ -96,7 +96,7 @@ void GetNextKeyBlocks::start_up() { std::move(P)); } -void GetNextKeyBlocks::got_download_token(std::unique_ptr token) { +void GetNextKeyBlocks::got_download_token(std::unique_ptr token) { token_ = std::move(token); if (download_from_.is_zero() && client_.empty()) { diff --git a/validator/net/get-next-key-blocks.hpp b/validator/net/get-next-key-blocks.hpp index 074289e2..14c040bd 100644 --- a/validator/net/get-next-key-blocks.hpp +++ b/validator/net/get-next-key-blocks.hpp @@ -44,7 +44,7 @@ class GetNextKeyBlocks : public td::actor::Actor { void finish_query(); void start_up() override; - void got_download_token(std::unique_ptr token); + void got_download_token(std::unique_ptr token); void got_node_to_download(adnl::AdnlNodeIdShort node); void send_request(); void got_result(td::BufferSlice res); @@ -75,7 +75,7 @@ class GetNextKeyBlocks : public td::actor::Actor { std::vector pending_; std::vector res_; - std::unique_ptr token_; + std::unique_ptr token_; }; } // namespace fullnode diff --git a/validator/queue-size-counter.cpp b/validator/queue-size-counter.cpp new file mode 100644 index 00000000..4fe55ae3 --- /dev/null +++ b/validator/queue-size-counter.cpp @@ -0,0 +1,303 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "queue-size-counter.hpp" +#include "block/block-auto.h" +#include "block/block-parse.h" +#include "common/delay.h" +#include "td/actor/MultiPromise.h" +#include "td/utils/Random.h" + +namespace ton::validator { + +static td::Result calc_queue_size(const td::Ref &state) { + td::uint64 size = 0; + TRY_RESULT(outq_descr, state->message_queue()); + block::gen::OutMsgQueueInfo::Record qinfo; + if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { + return td::Status::Error("invalid message queue"); + } + vm::AugmentedDictionary queue{qinfo.out_queue->prefetch_ref(0), 352, block::tlb::aug_OutMsgQueue}; + bool ok = queue.check_for_each([&](td::Ref, td::ConstBitPtr, int) -> bool { + ++size; + return true; + }); + if (!ok) { + return td::Status::Error("invalid message queue dict"); + } + return size; +} + +static td::Result recalc_queue_size(const td::Ref &state, const td::Ref &prev_state, + td::uint64 prev_size) { + TRY_RESULT(outq_descr, state->message_queue()); + block::gen::OutMsgQueueInfo::Record qinfo; + if (!tlb::unpack_cell(outq_descr->root_cell(), qinfo)) { + return td::Status::Error("invalid message queue"); + } + vm::AugmentedDictionary queue{qinfo.out_queue->prefetch_ref(0), 352, block::tlb::aug_OutMsgQueue}; + + TRY_RESULT(prev_outq_descr, prev_state->message_queue()); + block::gen::OutMsgQueueInfo::Record prev_qinfo; + if (!tlb::unpack_cell(prev_outq_descr->root_cell(), prev_qinfo)) { + return td::Status::Error("invalid message queue"); + } + vm::AugmentedDictionary prev_queue{prev_qinfo.out_queue->prefetch_ref(0), 352, block::tlb::aug_OutMsgQueue}; + td::uint64 add = 0, rem = 0; + bool ok = prev_queue.scan_diff( + queue, [&](td::ConstBitPtr, int, td::Ref prev_val, td::Ref new_val) -> bool { + if (prev_val.not_null()) { + ++rem; + } + if (new_val.not_null()) { + ++add; + } + return true; + }); + if (!ok) { + return td::Status::Error("invalid message queue dict"); + } + if (prev_size + add < rem) { + return td::Status::Error("negative value"); + } + return prev_size + add - rem; +} + +void QueueSizeCounter::start_up() { + if (init_masterchain_state_.is_null()) { + // Used in manager-hardfork or manager-disk + simple_mode_ = true; + return; + } + current_seqno_ = init_masterchain_state_->get_seqno(); + process_top_shard_blocks_cont(init_masterchain_state_, true); + init_masterchain_state_ = {}; + alarm(); +} + +void QueueSizeCounter::get_queue_size(BlockIdExt block_id, td::Promise promise) { + get_queue_size_ex(block_id, simple_mode_ || is_block_too_old(block_id), std::move(promise)); +} + +void QueueSizeCounter::get_queue_size_ex(ton::BlockIdExt block_id, bool calc_whole, td::Promise promise) { + Entry &entry = results_[block_id]; + if (entry.done_) { + promise.set_result(entry.queue_size_); + return; + } + entry.promises_.push_back(std::move(promise)); + if (entry.started_) { + return; + } + entry.started_ = true; + entry.calc_whole_ = calc_whole; + td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, block_id, true, + [SelfId = actor_id(this), block_id, manager = manager_](td::Result R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &QueueSizeCounter::on_error, block_id, R.move_as_error()); + return; + } + BlockHandle handle = R.move_as_ok(); + td::actor::send_closure( + manager, &ValidatorManager::wait_block_state, handle, 0, td::Timestamp::in(10.0), + [SelfId, handle](td::Result> R) mutable { + if (R.is_error()) { + td::actor::send_closure(SelfId, &QueueSizeCounter::on_error, handle->id(), + R.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &QueueSizeCounter::get_queue_size_cont, + std::move(handle), R.move_as_ok()); + }); + }); +} + +void QueueSizeCounter::get_queue_size_cont(BlockHandle handle, td::Ref state) { + Entry &entry = results_[handle->id()]; + CHECK(entry.started_); + bool calc_whole = entry.calc_whole_ || handle->id().seqno() == 0; + if (!calc_whole) { + CHECK(handle->inited_prev()); + auto prev_blocks = handle->prev(); + bool after_split = prev_blocks.size() == 1 && handle->id().shard_full() != prev_blocks[0].shard_full(); + bool after_merge = prev_blocks.size() == 2; + calc_whole = after_split || after_merge; + } + if (calc_whole) { + auto r_size = calc_queue_size(state); + if (r_size.is_error()) { + on_error(handle->id(), r_size.move_as_error()); + return; + } + entry.done_ = true; + entry.queue_size_ = r_size.move_as_ok(); + for (auto &promise : entry.promises_) { + promise.set_result(entry.queue_size_); + } + entry.promises_.clear(); + return; + } + + auto prev_block_id = handle->one_prev(true); + get_queue_size(prev_block_id, [=, SelfId = actor_id(this), manager = manager_](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &QueueSizeCounter::on_error, state->get_block_id(), R.move_as_error()); + return; + } + td::uint64 prev_size = R.move_as_ok(); + td::actor::send_closure( + manager, &ValidatorManager::wait_block_state_short, prev_block_id, 0, td::Timestamp::in(10.0), + [=](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &QueueSizeCounter::on_error, state->get_block_id(), R.move_as_error()); + return; + } + td::actor::send_closure(SelfId, &QueueSizeCounter::get_queue_size_cont2, state, R.move_as_ok(), prev_size); + }); + }); +} + +void QueueSizeCounter::get_queue_size_cont2(td::Ref state, td::Ref prev_state, + td::uint64 prev_size) { + BlockIdExt block_id = state->get_block_id(); + Entry &entry = results_[block_id]; + CHECK(entry.started_); + auto r_size = recalc_queue_size(state, prev_state, prev_size); + if (r_size.is_error()) { + on_error(block_id, r_size.move_as_error()); + return; + } + entry.done_ = true; + entry.queue_size_ = r_size.move_as_ok(); + for (auto &promise : entry.promises_) { + promise.set_result(entry.queue_size_); + } + entry.promises_.clear(); +} + +void QueueSizeCounter::on_error(ton::BlockIdExt block_id, td::Status error) { + auto it = results_.find(block_id); + if (it == results_.end()) { + return; + } + Entry &entry = it->second; + CHECK(!entry.done_); + for (auto &promise : entry.promises_) { + promise.set_error(error.clone()); + } + results_.erase(it); +} + +void QueueSizeCounter::process_top_shard_blocks() { + LOG(DEBUG) << "QueueSizeCounter::process_top_shard_blocks seqno=" << current_seqno_; + td::actor::send_closure( + manager_, &ValidatorManager::get_block_by_seqno_from_db, AccountIdPrefixFull{masterchainId, 0}, current_seqno_, + [SelfId = actor_id(this), manager = manager_](td::Result R) { + if (R.is_error()) { + LOG(WARNING) << "Failed to get masterchain block id: " << R.move_as_error(); + delay_action([=]() { td::actor::send_closure(SelfId, &QueueSizeCounter::process_top_shard_blocks); }, + td::Timestamp::in(5.0)); + return; + } + td::actor::send_closure( + manager, &ValidatorManager::wait_block_state_short, R.ok()->id(), 0, td::Timestamp::in(10.0), + [=](td::Result> R) { + if (R.is_error()) { + LOG(WARNING) << "Failed to get masterchain state: " << R.move_as_error(); + delay_action([=]() { td::actor::send_closure(SelfId, &QueueSizeCounter::process_top_shard_blocks); }, + td::Timestamp::in(5.0)); + return; + } + td::actor::send_closure(SelfId, &QueueSizeCounter::process_top_shard_blocks_cont, + td::Ref(R.move_as_ok()), false); + }); + }); +} + +void QueueSizeCounter::process_top_shard_blocks_cont(td::Ref state, bool init) { + LOG(DEBUG) << "QueueSizeCounter::process_top_shard_blocks_cont seqno=" << current_seqno_ << " init=" << init; + td::MultiPromise mp; + auto ig = mp.init_guard(); + last_top_blocks_.clear(); + last_top_blocks_.push_back(state->get_block_id()); + for (auto &shard : state->get_shards()) { + if (opts_->need_monitor(shard->shard(), state)) { + last_top_blocks_.push_back(shard->top_block_id()); + } + } + for (const BlockIdExt &block_id : last_top_blocks_) { + get_queue_size_ex_retry(block_id, init, ig.get_promise()); + } + ig.add_promise([SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + return; + } + td::actor::send_closure(SelfId, &QueueSizeCounter::process_top_shard_blocks_finish); + }); + if (init) { + init_top_blocks_ = last_top_blocks_; + } +} + +void QueueSizeCounter::get_queue_size_ex_retry(BlockIdExt block_id, bool calc_whole, td::Promise promise) { + get_queue_size_ex(block_id, calc_whole, + [=, promise = std::move(promise), SelfId = actor_id(this)](td::Result R) mutable { + if (R.is_error()) { + LOG(WARNING) << "Failed to calculate queue size for block " << block_id.to_str() << ": " + << R.move_as_error(); + delay_action( + [=, promise = std::move(promise)]() mutable { + td::actor::send_closure(SelfId, &QueueSizeCounter::get_queue_size_ex_retry, block_id, + calc_whole, std::move(promise)); + }, + td::Timestamp::in(5.0)); + return; + } + promise.set_result(td::Unit()); + }); +} + +void QueueSizeCounter::process_top_shard_blocks_finish() { + ++current_seqno_; + wait_shard_client(); +} + +void QueueSizeCounter::wait_shard_client() { + LOG(DEBUG) << "QueueSizeCounter::wait_shard_client seqno=" << current_seqno_; + td::actor::send_closure( + manager_, &ValidatorManager::wait_shard_client_state, current_seqno_, td::Timestamp::in(60.0), + [SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + delay_action([=]() mutable { td::actor::send_closure(SelfId, &QueueSizeCounter::wait_shard_client); }, + td::Timestamp::in(5.0)); + return; + } + td::actor::send_closure(SelfId, &QueueSizeCounter::process_top_shard_blocks); + }); +} + +void QueueSizeCounter::alarm() { + for (auto it = results_.begin(); it != results_.end();) { + if (it->second.done_ && is_block_too_old(it->first)) { + it = results_.erase(it); + } else { + ++it; + } + } + alarm_timestamp() = td::Timestamp::in(td::Random::fast(20.0, 40.0)); +} + +} // namespace ton::validator \ No newline at end of file diff --git a/validator/queue-size-counter.hpp b/validator/queue-size-counter.hpp new file mode 100644 index 00000000..7e5f7627 --- /dev/null +++ b/validator/queue-size-counter.hpp @@ -0,0 +1,88 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "interfaces/validator-manager.h" + +namespace ton::validator { + +class QueueSizeCounter : public td::actor::Actor { + public: + QueueSizeCounter(td::Ref last_masterchain_state, td::Ref opts, + td::actor::ActorId manager) + : init_masterchain_state_(last_masterchain_state), opts_(std::move(opts)), manager_(std::move(manager)) { + } + + void start_up() override; + void get_queue_size(BlockIdExt block_id, td::Promise promise); + void alarm() override; + + void update_options(td::Ref opts) { + opts_ = std::move(opts); + } + + private: + td::Ref init_masterchain_state_; + td::Ref opts_; + td::actor::ActorId manager_; + bool simple_mode_ = false; + + BlockSeqno current_seqno_ = 0; + std::vector init_top_blocks_; + std::vector last_top_blocks_; + + struct Entry { + bool started_ = false; + bool done_ = false; + bool calc_whole_ = false; + td::uint64 queue_size_ = 0; + std::vector> promises_; + }; + std::map results_; + + void get_queue_size_ex(BlockIdExt block_id, bool calc_whole, td::Promise promise); + void get_queue_size_cont(BlockHandle handle, td::Ref state); + void get_queue_size_cont2(td::Ref state, td::Ref prev_state, td::uint64 prev_size); + void on_error(BlockIdExt block_id, td::Status error); + + void process_top_shard_blocks(); + void process_top_shard_blocks_cont(td::Ref state, bool init = false); + void get_queue_size_ex_retry(BlockIdExt block_id, bool calc_whole, td::Promise promise); + void process_top_shard_blocks_finish(); + void wait_shard_client(); + + bool is_block_too_old(const BlockIdExt& block_id) const { + for (const BlockIdExt& top_block : last_top_blocks_) { + if (shard_intersects(block_id.shard_full(), top_block.shard_full())) { + if (block_id.seqno() + 100 < top_block.seqno()) { + return true; + } + break; + } + } + for (const BlockIdExt& init_top_block : init_top_blocks_) { + if (shard_intersects(block_id.shard_full(), init_top_block.shard_full())) { + if (block_id.seqno() < init_top_block.seqno()) { + return true; + } + break; + } + } + return false; + } +}; + +} // namespace ton::validator diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index 8fad3612..ac86cf37 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -70,39 +70,34 @@ void ShardClient::got_init_handle_from_db(BlockHandle handle) { } void ShardClient::got_init_state_from_db(td::Ref state) { - masterchain_state_ = std::move(state); - build_shard_overlays(); - masterchain_state_.clear(); - saved_to_db(); } void ShardClient::start_up_init_mode() { - build_shard_overlays(); - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - R.ensure(); - td::actor::send_closure(SelfId, &ShardClient::applied_all_shards); - }); - - td::MultiPromise mp; - auto ig = mp.init_guard(); - ig.add_promise(std::move(P)); - - auto vec = masterchain_state_->get_shards(); - for (auto &shard : vec) { - if (opts_->need_monitor(shard->shard())) { - auto P = td::PromiseCreator::lambda([promise = ig.get_promise()](td::Result> R) mutable { - R.ensure(); - promise.set_value(td::Unit()); - }); - - td::actor::create_actor("downloadstate", shard->top_block_id(), - masterchain_block_handle_->id(), 2, manager_, - td::Timestamp::in(3600 * 3), std::move(P)) - .release(); + std::vector shards; + for (const auto& s : masterchain_state_->get_shards()) { + if (opts_->need_monitor(s->shard(), masterchain_state_)) { + shards.push_back(s->top_block_id()); } } + download_shard_states(masterchain_block_handle_->id(), std::move(shards), 0); +} + +void ShardClient::download_shard_states(BlockIdExt masterchain_block_id, std::vector shards, size_t idx) { + if (idx >= shards.size()) { + LOG(WARNING) << "downloaded all shard states"; + applied_all_shards(); + return; + } + BlockIdExt block_id = shards[idx]; + td::actor::create_actor( + "downloadstate", block_id, masterchain_block_handle_->id(), 2, manager_, td::Timestamp::in(3600 * 5), + [=, SelfId = actor_id(this), shards = std::move(shards)](td::Result> R) { + R.ensure(); + td::actor::send_closure(SelfId, &ShardClient::download_shard_states, masterchain_block_id, std::move(shards), + idx + 1); + }) + .release(); } void ShardClient::applied_all_shards() { @@ -166,7 +161,6 @@ void ShardClient::download_masterchain_state() { void ShardClient::got_masterchain_block_state(td::Ref state) { masterchain_state_ = std::move(state); - build_shard_overlays(); if (started_) { apply_all_shards(); } @@ -189,8 +183,10 @@ void ShardClient::apply_all_shards() { ig.add_promise(std::move(P)); auto vec = masterchain_state_->get_shards(); + std::set workchains; for (auto &shard : vec) { - if (opts_->need_monitor(shard->shard())) { + workchains.insert(shard->shard().workchain); + if (opts_->need_monitor(shard->shard(), masterchain_state_)) { auto Q = td::PromiseCreator::lambda([SelfId = actor_id(this), promise = ig.get_promise(), shard = shard->shard()](td::Result> R) mutable { if (R.is_error()) { @@ -200,7 +196,22 @@ void ShardClient::apply_all_shards() { } }); td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, shard->top_block_id(), - shard_client_priority(), td::Timestamp::in(600), std::move(Q)); + shard_client_priority(), td::Timestamp::in(1500), std::move(Q)); + } + } + for (const auto &[wc, desc] : masterchain_state_->get_workchain_list()) { + if (!workchains.count(wc) && desc->active && opts_->need_monitor(ShardIdFull{wc, shardIdAll}, masterchain_state_)) { + auto Q = td::PromiseCreator::lambda([SelfId = actor_id(this), promise = ig.get_promise(), + workchain = wc](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix(PSTRING() << "workchain " << workchain << ": ")); + } else { + td::actor::send_closure(SelfId, &ShardClient::downloaded_shard_state, R.move_as_ok(), std::move(promise)); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, + BlockIdExt{wc, shardIdAll, 0, desc->zerostate_root_hash, desc->zerostate_file_hash}, + shard_client_priority(), td::Timestamp::in(1500), std::move(Q)); } } } @@ -223,7 +234,6 @@ void ShardClient::new_masterchain_block_notification(BlockHandle handle, td::Ref masterchain_block_handle_ = std::move(handle); masterchain_state_ = std::move(state); waiting_ = false; - build_shard_overlays(); apply_all_shards(); } @@ -244,26 +254,6 @@ void ShardClient::get_processed_masterchain_block_id(td::Promise pro } } -void ShardClient::build_shard_overlays() { - auto v = masterchain_state_->get_shards(); - - for (auto &x : v) { - auto shard = x->shard(); - if (opts_->need_monitor(shard)) { - auto d = masterchain_state_->soft_min_split_depth(shard.workchain); - auto l = shard_prefix_length(shard.shard); - if (l > d) { - shard = shard_prefix(shard, d); - } - - if (created_overlays_.count(shard) == 0) { - created_overlays_.insert(shard); - td::actor::send_closure(manager_, &ValidatorManager::subscribe_to_shard, shard); - } - } - } -} - void ShardClient::force_update_shard_client(BlockHandle handle, td::Promise promise) { CHECK(!init_mode_); CHECK(!started_); @@ -294,10 +284,13 @@ void ShardClient::force_update_shard_client_ex(BlockHandle handle, td::Ref opts) { + opts_ = std::move(opts); +} + } // namespace validator } // namespace ton diff --git a/validator/shard-client.hpp b/validator/shard-client.hpp index 455b67e7..7c2c978c 100644 --- a/validator/shard-client.hpp +++ b/validator/shard-client.hpp @@ -42,8 +42,6 @@ class ShardClient : public td::actor::Actor { td::Promise promise_; - std::set created_overlays_; - public: ShardClient(td::Ref opts, BlockHandle masterchain_block_handle, td::Ref masterchain_state, td::actor::ActorId manager, @@ -64,23 +62,14 @@ class ShardClient : public td::actor::Actor { return 2; } - void build_shard_overlays(); - void start_up() override; void start_up_init_mode(); - void start_up_init_mode_finished(); + void download_shard_states(BlockIdExt masterchain_block_id, std::vector shards, size_t idx); void start(); void got_state_from_db(BlockIdExt masterchain_block_id); void got_init_handle_from_db(BlockHandle handle); void got_init_state_from_db(td::Ref state); - void im_download_shard_state(BlockIdExt block_id, td::Promise promise); - void im_downloaded_zero_state(BlockIdExt block_id, td::BufferSlice data, td::Promise promise); - void im_downloaded_proof_link(BlockIdExt block_id, td::BufferSlice data, td::Promise promise); - void im_checked_proof_link(BlockIdExt block_id, td::Promise promise); - void im_downloaded_shard_state(BlockIdExt block_id, td::Promise promise); - void im_got_shard_handle(BlockHandle handle, td::Promise promise); - void new_masterchain_block_id(BlockIdExt masterchain_block_id); void got_masterchain_block_handle(BlockHandle handle); void download_masterchain_state(); @@ -97,6 +86,8 @@ class ShardClient : public td::actor::Actor { void force_update_shard_client(BlockHandle handle, td::Promise promise); void force_update_shard_client_ex(BlockHandle handle, td::Ref state, td::Promise promise); + + void update_options(td::Ref opts); }; } // namespace validator diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index 1bb932c6..bc3d7b5e 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -18,15 +18,19 @@ */ #include "state-serializer.hpp" #include "td/utils/Random.h" -#include "adnl/utils.hpp" #include "ton/ton-io.hpp" #include "common/delay.h" +#include "td/utils/filesystem.h" +#include "td/utils/HashSet.h" namespace ton { namespace validator { void AsyncStateSerializer::start_up() { + if (!opts_->get_state_serializer_enabled()) { + LOG(ERROR) << "Persistent state serializer is disabled"; + } alarm_timestamp() = td::Timestamp::in(1.0 + td::Random::fast(0, 10) * 1.0); running_ = true; @@ -54,6 +58,12 @@ void AsyncStateSerializer::got_self_state(AsyncSerializerState state) { }); td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, last_block_id_, true, std::move(P)); } + + inited_block_id_ = true; + for (auto& promise : wait_init_block_id_) { + promise.set_value(td::Unit()); + } + wait_init_block_id_.clear(); } void AsyncStateSerializer::got_init_handle(BlockHandle handle) { @@ -81,6 +91,21 @@ void AsyncStateSerializer::alarm() { td::actor::send_closure(manager_, &ValidatorManager::get_top_masterchain_block, std::move(P)); } +void AsyncStateSerializer::request_previous_state_files() { + td::actor::send_closure( + manager_, &ValidatorManager::get_previous_persistent_state_files, masterchain_handle_->id().seqno(), + [SelfId = actor_id(this)](td::Result>> R) { + R.ensure(); + td::actor::send_closure(SelfId, &AsyncStateSerializer::got_previous_state_files, R.move_as_ok()); + }); +} + +void AsyncStateSerializer::got_previous_state_files(std::vector> files) { + previous_state_cache_ = std::make_shared(); + previous_state_cache_->state_files = std::move(files); + request_masterchain_state(); +} + void AsyncStateSerializer::request_masterchain_state() { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), manager = manager_](td::Result> R) { if (R.is_error()) { @@ -131,30 +156,57 @@ void AsyncStateSerializer::next_iteration() { CHECK(masterchain_handle_->id() == last_block_id_); if (attempt_ < max_attempt() && last_key_block_id_.id.seqno < last_block_id_.id.seqno && need_serialize(masterchain_handle_)) { - if (!have_masterchain_state_) { - LOG(INFO) << "started serializing persistent state for " << masterchain_handle_->id().id; - // block next attempts immediately, but send actual request later + if (!stored_persistent_state_description_) { + LOG(INFO) << "storing persistent state description for " << masterchain_handle_->id().id; running_ = true; - delay_action([SelfId = actor_id( - this)]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_masterchain_state); }, - td::Timestamp::in(td::Random::fast(0, 3600))); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &AsyncStateSerializer::fail_handler, + R.move_as_error_prefix("failed to get masterchain state: ")); + } else { + td::actor::send_closure(SelfId, &AsyncStateSerializer::store_persistent_state_description, + td::Ref(R.move_as_ok())); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, masterchain_handle_, std::move(P)); return; } - while (next_idx_ < shards_.size()) { - if (!need_monitor(shards_[next_idx_].shard_full())) { - next_idx_++; - } else { - // block next attempts immediately, but send actual request later - running_ = true; - delay_action( - [SelfId = actor_id(this), shard = shards_[next_idx_]]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_shard_state, shard); }, - td::Timestamp::in(td::Random::fast(0, 4 * 3600))); + if (!have_masterchain_state_ && !opts_->get_state_serializer_enabled()) { + LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str() + << ": serializer is disabled (by user)"; + } else if (!have_masterchain_state_ && auto_disabled_) { + LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str() + << ": serializer is disabled (automatically)"; + } else if (!have_masterchain_state_ && have_newer_persistent_state(masterchain_handle_->unix_time())) { + LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str() + << ": newer key block with ts=" << last_known_key_block_ts_ << " exists"; + } else { + if (!have_masterchain_state_) { + LOG(ERROR) << "started serializing persistent state for " << masterchain_handle_->id().id.to_str(); + // block next attempts immediately, but send actual request later + running_ = true; + double delay = td::Random::fast(0, 3600 * 6); + LOG(WARNING) << "serializer delay = " << delay << "s"; + delay_action( + [SelfId = actor_id(this)]() { + td::actor::send_closure(SelfId, &AsyncStateSerializer::request_previous_state_files); + }, + td::Timestamp::in(delay)); + current_status_ = PSTRING() << "delay before serializing seqno=" << masterchain_handle_->id().seqno() << " " + << (int)delay << "s"; + current_status_ts_ = td::Timestamp::now(); return; } + if (next_idx_ < shards_.size()) { + running_ = true; + request_shard_state(shards_[next_idx_]); + return; + } + LOG(ERROR) << "finished serializing persistent state for " << masterchain_handle_->id().id.to_str(); } - LOG(INFO) << "finished serializing persistent state for " << masterchain_handle_->id().id; last_key_block_ts_ = masterchain_handle_->unix_time(); last_key_block_id_ = masterchain_handle_->id(); + previous_state_cache_ = {}; } if (!saved_to_db_) { running_ = true; @@ -170,6 +222,7 @@ void AsyncStateSerializer::next_iteration() { if (masterchain_handle_->inited_next_left()) { last_block_id_ = masterchain_handle_->one_next(true); have_masterchain_state_ = false; + stored_persistent_state_description_ = false; masterchain_handle_ = nullptr; saved_to_db_ = false; shards_.clear(); @@ -184,6 +237,24 @@ void AsyncStateSerializer::got_top_masterchain_handle(BlockIdExt block_id) { } } +void AsyncStateSerializer::store_persistent_state_description(td::Ref state) { + stored_persistent_state_description_ = true; + attempt_ = 0; + running_ = false; + + PersistentStateDescription desc; + desc.masterchain_id = state->get_block_id(); + desc.start_time = state->get_unix_time(); + desc.end_time = ValidatorManager::persistent_state_ttl(desc.start_time); + for (const auto &v : state->get_shards()) { + desc.shard_blocks.push_back(v->top_block_id()); + } + td::actor::send_closure(manager_, &ValidatorManager::add_persistent_state_description, + td::Ref(true, std::move(desc))); + + next_iteration(); +} + void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) { CHECK(!masterchain_handle_); masterchain_handle_ = std::move(handle); @@ -192,32 +263,140 @@ void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) { next_iteration(); } +class CachedCellDbReader : public vm::CellDbReader { + public: + CachedCellDbReader(std::shared_ptr parent, + std::shared_ptr cache) + : parent_(std::move(parent)), cache_(std::move(cache)) { + } + td::Result> load_cell(td::Slice hash) override { + ++total_reqs_; + DCHECK(hash.size() == 32); + if (cache_) { + auto it = cache_->find(hash); + if (it != cache_->end()) { + ++cached_reqs_; + TRY_RESULT(loaded_cell, (*it)->load_cell()); + return loaded_cell.data_cell; + } + } + return parent_->load_cell(hash); + } + void print_stats() const { + LOG(WARNING) << "CachedCellDbReader stats : " << total_reqs_ << " reads, " << cached_reqs_ << " cached"; + } + private: + std::shared_ptr parent_; + std::shared_ptr cache_; + + td::uint64 total_reqs_ = 0; + td::uint64 cached_reqs_ = 0; +}; + +void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard) { + std::vector prev_shards; + for (const auto& [_, prev_shard] : state_files) { + if (shard_intersects(shard, prev_shard)) { + prev_shards.push_back(prev_shard); + } + } + if (prev_shards == cur_shards) { + return; + } + cur_shards = std::move(prev_shards); + cache = {}; + if (cur_shards.empty()) { + return; + } + td::Timer timer; + LOG(WARNING) << "Preloading previous persistent state for shard " << shard.to_str() << " (" + << cur_shards.size() << " files)"; + vm::CellHashSet cells; + std::function)> dfs = [&](td::Ref cell) { + if (!cells.insert(cell).second) { + return; + } + bool is_special; + vm::CellSlice cs = vm::load_cell_slice_special(cell, is_special); + for (unsigned i = 0; i < cs.size_refs(); ++i) { + dfs(cs.prefetch_ref(i)); + } + }; + for (const auto& [file, prev_shard] : state_files) { + if (!shard_intersects(shard, prev_shard)) { + continue; + } + auto r_data = td::read_file(file); + if (r_data.is_error()) { + LOG(INFO) << "Reading " << file << " : " << r_data.move_as_error(); + continue; + } + LOG(INFO) << "Reading " << file << " : " << td::format::as_size(r_data.ok().size()); + auto r_root = vm::std_boc_deserialize(r_data.move_as_ok()); + if (r_root.is_error()) { + LOG(WARNING) << "Deserialize error : " << r_root.move_as_error(); + continue; + } + r_data.clear(); + dfs(r_root.move_as_ok()); + } + LOG(WARNING) << "Preloaded previous state: " << cells.size() << " cells in " << timer.elapsed() << "s"; + cache = std::make_shared(std::move(cells)); +} + void AsyncStateSerializer::got_masterchain_state(td::Ref state, std::shared_ptr cell_db_reader) { - LOG(INFO) << "serializing masterchain state " << masterchain_handle_->id().id; + if (!opts_->get_state_serializer_enabled() || auto_disabled_) { + stored_masterchain_state(); + return; + } + LOG(ERROR) << "serializing masterchain state " << masterchain_handle_->id().id.to_str(); have_masterchain_state_ = true; CHECK(next_idx_ == 0); CHECK(shards_.size() == 0); auto vec = state->get_shards(); - for (auto& v : vec) { - shards_.push_back(v->top_block_id()); + for (auto &v : vec) { + if (opts_->need_monitor(v->shard(), state)) { + shards_.push_back(v->top_block_id()); + } } - auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader](td::FileFd& fd) { - return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31); + auto write_data = [shard = state->get_shard(), root = state->root_cell(), cell_db_reader, + previous_state_cache = previous_state_cache_, + fast_serializer_enabled = opts_->get_fast_state_serializer_enabled(), + cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable { + if (!cell_db_reader) { + return vm::std_boc_serialize_to_file(root, fd, 31, std::move(cancellation_token)); + } + if (fast_serializer_enabled) { + previous_state_cache->prepare_cache(shard); + } + auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache->cache); + auto res = vm::std_boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token)); + new_cell_db_reader->print_stats(); + return res; }; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - R.ensure(); + if (R.is_error() && R.error().code() == cancelled) { + LOG(ERROR) << "Persistent state serialization cancelled"; + } else { + R.ensure(); + } td::actor::send_closure(SelfId, &AsyncStateSerializer::stored_masterchain_state); }); td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, masterchain_handle_->id(), masterchain_handle_->id(), write_data, std::move(P)); + + current_status_ = PSTRING() << "serializing masterchain state " << state->get_block_id().id.to_str(); + current_status_ts_ = td::Timestamp::now(); } void AsyncStateSerializer::stored_masterchain_state() { - LOG(INFO) << "finished serializing masterchain state " << masterchain_handle_->id().id; + current_status_ = "pending"; + current_status_ts_ = {}; + LOG(ERROR) << "finished serializing masterchain state " << masterchain_handle_->id().id.to_str(); running_ = false; next_iteration(); } @@ -247,21 +426,46 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) { void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref state, std::shared_ptr cell_db_reader) { - LOG(INFO) << "serializing shard state " << handle->id().id; - auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader](td::FileFd& fd) { - return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31); + next_idx_++; + if (!opts_->get_state_serializer_enabled() || auto_disabled_) { + success_handler(); + return; + } + LOG(ERROR) << "serializing shard state " << handle->id().id.to_str(); + auto write_data = [shard = state->get_shard(), root = state->root_cell(), cell_db_reader, + previous_state_cache = previous_state_cache_, + fast_serializer_enabled = opts_->get_fast_state_serializer_enabled(), + cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable { + if (!cell_db_reader) { + return vm::std_boc_serialize_to_file(root, fd, 31, std::move(cancellation_token)); + } + if (fast_serializer_enabled) { + previous_state_cache->prepare_cache(shard); + } + auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache->cache); + auto res = vm::std_boc_serialize_to_file_large(new_cell_db_reader, root->get_hash(), fd, 31, std::move(cancellation_token)); + new_cell_db_reader->print_stats(); + return res; }; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result R) { - R.ensure(); - LOG(INFO) << "finished serializing shard state " << handle->id().id; + if (R.is_error() && R.error().code() == cancelled) { + LOG(ERROR) << "Persistent state serialization cancelled"; + } else { + R.ensure(); + LOG(ERROR) << "finished serializing shard state " << handle->id().id.to_str(); + } td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler); }); td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, handle->id(), masterchain_handle_->id(), write_data, std::move(P)); - next_idx_++; + current_status_ = PSTRING() << "serializing shard state " << next_idx_ << "/" << shards_.size() << " " + << state->get_block_id().id.to_str(); + current_status_ts_ = td::Timestamp::now(); } void AsyncStateSerializer::fail_handler(td::Status reason) { + current_status_ = PSTRING() << "pending, " << reason; + current_status_ts_ = {}; VLOG(VALIDATOR_NOTICE) << "failure: " << reason; attempt_++; delay_action( @@ -275,19 +479,59 @@ void AsyncStateSerializer::fail_handler_cont() { } void AsyncStateSerializer::success_handler() { + current_status_ = "pending"; + current_status_ts_ = {}; running_ = false; next_iteration(); } -bool AsyncStateSerializer::need_monitor(ShardIdFull shard) { - return opts_->need_monitor(shard); +void AsyncStateSerializer::update_options(td::Ref opts) { + opts_ = std::move(opts); + if (!opts_->get_state_serializer_enabled()) { + cancellation_token_source_.cancel(); + } +} + +void AsyncStateSerializer::auto_disable_serializer(bool disabled) { + auto_disabled_ = disabled; + if (auto_disabled_) { + cancellation_token_source_.cancel(); + } +} + +void AsyncStateSerializer::prepare_stats(td::Promise>> promise) { + if (!inited_block_id_) { + wait_init_block_id_.push_back( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_STATUS_PROMISE(promise, R.move_as_status()); + td::actor::send_closure(SelfId, &AsyncStateSerializer::prepare_stats, std::move(promise)); + }); + return; + } + std::vector> vec; + vec.emplace_back("stateserializermasterchainseqno", td::to_string(last_block_id_.seqno())); + td::StringBuilder sb; + sb << current_status_; + if (current_status_ts_) { + sb << " (started " << (int)(td::Timestamp::now() - current_status_ts_) << "s ago)"; + } + if (!opts_->get_state_serializer_enabled() || auto_disabled_) { + sb << " (disabled)"; + } + vec.emplace_back("stateserializerstatus", sb.as_cslice().str()); + promise.set_result(std::move(vec)); } bool AsyncStateSerializer::need_serialize(BlockHandle handle) { if (handle->id().id.seqno == 0 || !handle->is_key_block()) { return false; } - return ValidatorManager::is_persistent_state(handle->unix_time(), last_key_block_ts_); + return ValidatorManager::is_persistent_state(handle->unix_time(), last_key_block_ts_) && + ValidatorManager::persistent_state_ttl(handle->unix_time()) > (UnixTime)td::Clocks::system(); +} + +bool AsyncStateSerializer::have_newer_persistent_state(UnixTime cur_ts) { + return cur_ts / (1 << 17) < last_known_key_block_ts_ / (1 << 17); } } // namespace validator diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index ee2aace0..406ac350 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -36,16 +36,31 @@ class AsyncStateSerializer : public td::actor::Actor { UnixTime last_key_block_ts_ = 0; bool saved_to_db_ = true; + bool inited_block_id_ = false; + std::vector> wait_init_block_id_; + td::Ref opts_; + bool auto_disabled_ = false; + td::CancellationTokenSource cancellation_token_source_; + UnixTime last_known_key_block_ts_ = 0; td::actor::ActorId manager_; td::uint32 next_idx_ = 0; BlockHandle masterchain_handle_; + bool stored_persistent_state_description_ = false; bool have_masterchain_state_ = false; std::vector shards_; + struct PreviousStateCache { + std::vector> state_files; + std::shared_ptr cache; + std::vector cur_shards; + + void prepare_cache(ShardIdFull shard); + }; + std::shared_ptr previous_state_cache_; public: AsyncStateSerializer(BlockIdExt block_id, td::Ref opts, @@ -58,18 +73,21 @@ class AsyncStateSerializer : public td::actor::Actor { } bool need_serialize(BlockHandle handle); - bool need_monitor(ShardIdFull shard); + bool have_newer_persistent_state(UnixTime cur_ts); void alarm() override; void start_up() override; void got_self_state(AsyncSerializerState state); void got_init_handle(BlockHandle handle); + void request_previous_state_files(); + void got_previous_state_files(std::vector> files); void request_masterchain_state(); void request_shard_state(BlockIdExt shard); void next_iteration(); void got_top_masterchain_handle(BlockIdExt block_id); + void store_persistent_state_description(td::Ref state); void got_masterchain_handle(BlockHandle handle_); void got_masterchain_state(td::Ref state, std::shared_ptr cell_db_reader); void stored_masterchain_state(); @@ -80,6 +98,12 @@ class AsyncStateSerializer : public td::actor::Actor { promise.set_result(last_block_id_.id.seqno); } + void prepare_stats(td::Promise>> promise); + + void update_last_known_key_block_ts(UnixTime ts) { + last_known_key_block_ts_ = std::max(last_known_key_block_ts_, ts); + } + void saved_to_db() { saved_to_db_ = true; running_ = false; @@ -89,6 +113,12 @@ class AsyncStateSerializer : public td::actor::Actor { void fail_handler(td::Status reason); void fail_handler_cont(); void success_handler(); + + void update_options(td::Ref opts); + void auto_disable_serializer(bool disabled); + + std::string current_status_ = "pending"; + td::Timestamp current_status_ts_ = td::Timestamp::never(); }; } // namespace validator diff --git a/validator/stats-provider.h b/validator/stats-provider.h new file mode 100644 index 00000000..e0a7f565 --- /dev/null +++ b/validator/stats-provider.h @@ -0,0 +1,105 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "validator.h" +#include "common/AtomicRef.h" + +#include + +namespace ton { + +namespace validator { + +class StatsProvider { + public: + StatsProvider() = default; + StatsProvider(td::actor::ActorId manager, std::string prefix, + std::function>>)> callback) + : inited_(true), manager_(std::move(manager)) { + static std::atomic cur_idx{0}; + idx_ = cur_idx.fetch_add(1); + td::actor::send_closure(manager_, &ValidatorManagerInterface::register_stats_provider, idx_, std::move(prefix), + std::move(callback)); + } + StatsProvider(const StatsProvider&) = delete; + StatsProvider(StatsProvider&& other) noexcept + : inited_(other.inited_), idx_(other.idx_), manager_(std::move(other.manager_)) { + other.inited_ = false; + } + ~StatsProvider() { + if (inited_) { + td::actor::send_closure(manager_, &ValidatorManagerInterface::unregister_stats_provider, idx_); + } + } + + StatsProvider& operator=(const StatsProvider&) = delete; + StatsProvider& operator=(StatsProvider&& other) noexcept { + if (this != &other) { + inited_ = other.inited_; + idx_ = other.idx_; + manager_ = std::move(other.manager_); + other.inited_ = false; + } + return *this; + } + + bool inited() const { + return inited_; + } + + private: + bool inited_ = false; + td::uint64 idx_ = 0; + td::actor::ActorId manager_; +}; + +class ProcessStatus { + public: + ProcessStatus() = default; + ProcessStatus(td::actor::ActorId manager, std::string name) + : stats_provider_(std::move(manager), std::move(name), [value = value_](auto promise) { + auto status = value->load(); + if (status.is_null()) { + promise.set_error(td::Status::Error("empty")); + return; + } + std::vector> vec; + vec.emplace_back("", *status); + promise.set_value(std::move(vec)); + }) { + } + ProcessStatus(const ProcessStatus&) = delete; + ProcessStatus(ProcessStatus&& other) noexcept = default; + ProcessStatus& operator=(const ProcessStatus&) = delete; + ProcessStatus& operator=(ProcessStatus&& other) noexcept = default; + + void set_status(std::string s) { + if (!value_) { + return; + } + value_->store(td::Ref>(true, std::move(s))); + } + + private: + std::shared_ptr>> value_ = std::make_shared>>(); + StatsProvider stats_provider_; +}; + +} // namespace validator + +} // namespace ton diff --git a/validator/token-manager.cpp b/validator/token-manager.cpp index 2cca02ed..8242f921 100644 --- a/validator/token-manager.cpp +++ b/validator/token-manager.cpp @@ -22,23 +22,23 @@ namespace ton { namespace validator { -void TokenManager::get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { +void TokenManager::get_token(size_t size, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) { if (free_priority_tokens_ > 0 && priority > 0) { --free_priority_tokens_; - promise.set_value(gen_token(download_size, priority)); + promise.set_value(gen_token(size, priority)); return; } if (free_tokens_ > 0) { --free_tokens_; - promise.set_value(gen_token(download_size, priority)); + promise.set_value(gen_token(size, priority)); return; } - pending_.emplace(PendingPromiseKey{download_size, priority, seqno_++}, PendingPromise{timeout, std::move(promise)}); + pending_.emplace(PendingPromiseKey{size, priority, seqno_++}, PendingPromise{timeout, std::move(promise)}); } -void TokenManager::download_token_cleared(size_t download_size, td::uint32 priority) { +void TokenManager::token_cleared(size_t size, td::uint32 priority) { (priority ? free_priority_tokens_ : free_tokens_)++; if (free_priority_tokens_ > max_priority_tokens_) { free_priority_tokens_--; @@ -47,7 +47,7 @@ void TokenManager::download_token_cleared(size_t download_size, td::uint32 prior for (auto it = pending_.begin(); it != pending_.end();) { if (it->first.priority && (free_tokens_ || free_priority_tokens_)) { - it->second.promise.set_value(gen_token(download_size, priority)); + it->second.promise.set_value(gen_token(size, priority)); auto it2 = it++; pending_.erase(it2); if (free_priority_tokens_ > 0) { @@ -56,7 +56,7 @@ void TokenManager::download_token_cleared(size_t download_size, td::uint32 prior free_tokens_--; } } else if (!it->first.priority && free_tokens_) { - it->second.promise.set_value(gen_token(download_size, priority)); + it->second.promise.set_value(gen_token(size, priority)); auto it2 = it++; pending_.erase(it2); free_tokens_--; @@ -67,34 +67,33 @@ void TokenManager::download_token_cleared(size_t download_size, td::uint32 prior } void TokenManager::alarm() { - for (auto it = pending_.begin(); it != pending_.end(); it++) { + for (auto it = pending_.begin(); it != pending_.end();) { if (it->second.timeout.is_in_past()) { - it->second.promise.set_error(td::Status::Error(ErrorCode::timeout, "timeout in wait download token")); - auto it2 = it++; - pending_.erase(it2); + it->second.promise.set_error(td::Status::Error(ErrorCode::timeout, "timeout in wait token")); + it = pending_.erase(it); } else { it++; } } } -std::unique_ptr TokenManager::gen_token(size_t download_size, td::uint32 priority) { - class Token : public DownloadToken { +std::unique_ptr TokenManager::gen_token(size_t size, td::uint32 priority) { + class TokenImpl : public ActionToken { public: - Token(size_t download_size, td::uint32 priority, td::actor::ActorId manager) - : download_size_(download_size), priority_(priority), manager_(manager) { + TokenImpl(size_t size, td::uint32 priority, td::actor::ActorId manager) + : size_(size), priority_(priority), manager_(manager) { } - ~Token() override { - td::actor::send_closure(manager_, &TokenManager::download_token_cleared, download_size_, priority_); + ~TokenImpl() override { + td::actor::send_closure(manager_, &TokenManager::token_cleared, size_, priority_); } private: - size_t download_size_; + size_t size_; td::uint32 priority_; td::actor::ActorId manager_; }; - return std::make_unique(download_size, priority, actor_id(this)); + return std::make_unique(size, priority, actor_id(this)); } } // namespace validator diff --git a/validator/token-manager.h b/validator/token-manager.h index 0d75710f..0fd0126a 100644 --- a/validator/token-manager.h +++ b/validator/token-manager.h @@ -31,16 +31,19 @@ class TokenManager : public td::actor::Actor { public: TokenManager() { } + explicit TokenManager(td::uint32 max_tokens) + : free_tokens_(max_tokens), free_priority_tokens_(max_tokens), max_priority_tokens_(max_tokens) { + } void alarm() override; - void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise); - void download_token_cleared(size_t download_size, td::uint32 priority); + void get_token(size_t size, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise); + void token_cleared(size_t size, td::uint32 priority); private: - std::unique_ptr gen_token(size_t download_size, td::uint32 priority); + std::unique_ptr gen_token(size_t size, td::uint32 priority); struct PendingPromiseKey { - size_t download_size; + size_t size; td::uint32 priority; td::uint64 seqno; @@ -50,7 +53,7 @@ class TokenManager : public td::actor::Actor { }; struct PendingPromise { td::Timestamp timeout; - td::Promise> promise; + td::Promise> promise; }; td::uint64 seqno_ = 0; std::map pending_; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index b4c38e51..110ccd81 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -18,15 +18,24 @@ */ #include "validator-group.hpp" #include "fabric.h" +#include "full-node-master.hpp" #include "ton/ton-io.hpp" #include "td/utils/overloaded.h" #include "common/delay.h" +#include "ton/lite-tl.hpp" namespace ton { namespace validator { -void ValidatorGroup::generate_block_candidate(td::uint32 round_id, td::Promise promise) { +static bool need_send_candidate_broadcast(const validatorsession::BlockSourceInfo &source_info, bool is_masterchain) { + return source_info.first_block_round == source_info.round && source_info.source_priority == 0 && !is_masterchain; +} + +void ValidatorGroup::generate_block_candidate( + validatorsession::BlockSourceInfo source_info, + td::Promise promise) { + td::uint32 round_id = source_info.round; if (round_id > last_known_round_id_) { last_known_round_id_ = round_id; } @@ -36,23 +45,30 @@ void ValidatorGroup::generate_block_candidate(td::uint32 round_id, td::Promiseresult) { - promise.set_result(cached_collated_block_->result.value().clone()); + promise.set_value({cached_collated_block_->result.value().clone(), true}); } else { - cached_collated_block_->promises.push_back(std::move(promise)); + cached_collated_block_->promises.push_back(promise.wrap([](BlockCandidate &&res) { + return validatorsession::ValidatorSession::GeneratedCandidate{std::move(res), true}; + })); } return; } cached_collated_block_ = std::make_shared(); - cached_collated_block_->promises.push_back(std::move(promise)); + cached_collated_block_->promises.push_back(promise.wrap([](BlockCandidate &&res) { + return validatorsession::ValidatorSession::GeneratedCandidate{std::move(res), false}; + })); run_collate_query( - shard_, min_ts_, min_masterchain_block_id_, prev_block_ids_, - Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, validator_set_, manager_, td::Timestamp::in(10.0), - [SelfId = actor_id(this), cache = cached_collated_block_](td::Result R) { - td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_candidate, std::move(cache), std::move(R)); - }); + shard_, min_masterchain_block_id_, prev_block_ids_, Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, + validator_set_, opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), + [SelfId = actor_id(this), cache = cached_collated_block_, source_info](td::Result R) mutable { + td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_candidate, std::move(source_info), + std::move(cache), std::move(R)); + }, cancellation_token_source_.get_cancellation_token(), /* mode = */ 0); } -void ValidatorGroup::generated_block_candidate(std::shared_ptr cache, td::Result R) { +void ValidatorGroup::generated_block_candidate(validatorsession::BlockSourceInfo source_info, + std::shared_ptr cache, + td::Result R) { if (R.is_error()) { for (auto &p : cache->promises) { p.set_error(R.error().clone()); @@ -61,7 +77,12 @@ void ValidatorGroup::generated_block_candidate(std::shared_ptrresult = R.move_as_ok(); + auto candidate = R.move_as_ok(); + add_available_block_candidate(candidate.pubkey.as_bits256(), candidate.id, candidate.collated_file_hash); + if (need_send_candidate_broadcast(source_info, shard_.is_masterchain())) { + send_block_candidate_broadcast(candidate.id, candidate.data.clone()); + } + cache->result = std::move(candidate); for (auto &p : cache->promises) { p.set_value(cache->result.value().clone()); } @@ -69,8 +90,9 @@ void ValidatorGroup::generated_block_candidate(std::shared_ptrpromises.clear(); } -void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidate block, - td::Promise promise) { +void ValidatorGroup::validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block, + td::Promise> promise) { + td::uint32 round_id = source_info.round; if (round_id > last_known_round_id_) { last_known_round_id_ = round_id; } @@ -78,7 +100,19 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat promise.set_error(td::Status::Error(ErrorCode::notready, "too old")); return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), round_id, block = block.clone(), + + auto next_block_id = create_next_block_id(block.id.root_hash, block.id.file_hash); + block.id = next_block_id; + + CacheKey cache_key = block_to_cache_key(block); + auto it = approved_candidates_cache_.find(cache_key); + if (it != approved_candidates_cache_.end()) { + promise.set_value({it->second, true}); + return; + } + + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), source_info, block = block.clone(), manager = manager_, + validator_set = validator_set_, promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { auto S = R.move_as_error(); @@ -86,37 +120,52 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat LOG(ERROR) << "failed to validate candidate: " << S; } delay_action( - [SelfId, round_id, block = std::move(block), promise = std::move(promise)]() mutable { - td::actor::send_closure(SelfId, &ValidatorGroup::validate_block_candidate, round_id, std::move(block), - std::move(promise)); + [SelfId, source_info, block = std::move(block), promise = std::move(promise)]() mutable { + td::actor::send_closure(SelfId, &ValidatorGroup::validate_block_candidate, std::move(source_info), + std::move(block), std::move(promise)); }, td::Timestamp::in(0.1)); } else { auto v = R.move_as_ok(); - v.visit(td::overloaded([&](UnixTime ts) { promise.set_result(ts); }, - [&](CandidateReject reject) { - promise.set_error(td::Status::Error(ErrorCode::protoviolation, - PSTRING() << "bad candidate: " << reject.reason)); - })); + v.visit(td::overloaded( + [&](UnixTime ts) { + td::actor::send_closure(SelfId, &ValidatorGroup::update_approve_cache, block_to_cache_key(block), ts); + td::actor::send_closure(SelfId, &ValidatorGroup::add_available_block_candidate, block.pubkey.as_bits256(), + block.id, block.collated_file_hash); + if (need_send_candidate_broadcast(source_info, block.id.is_masterchain())) { + td::actor::send_closure(SelfId, &ValidatorGroup::send_block_candidate_broadcast, block.id, + block.data.clone()); + } + promise.set_value({ts, false}); + }, + [&](CandidateReject reject) { + promise.set_error( + td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad candidate: " << reject.reason)); + })); } }); if (!started_) { P.set_error(td::Status::Error(ErrorCode::notready, "validator group not started")); return; } - auto next_block_id = create_next_block_id(block.id.root_hash, block.id.file_hash); VLOG(VALIDATOR_DEBUG) << "validating block candidate " << next_block_id; block.id = next_block_id; - run_validate_query(shard_, min_ts_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_, - manager_, td::Timestamp::in(15.0), std::move(P)); + run_validate_query(shard_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_, manager_, + td::Timestamp::in(15.0), std::move(P)); } -void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash src, td::BufferSlice block_data, +void ValidatorGroup::update_approve_cache(CacheKey key, UnixTime value) { + approved_candidates_cache_[key] = value; +} + +void ValidatorGroup::accept_block_candidate(validatorsession::BlockSourceInfo source_info, td::BufferSlice block_data, RootHash root_hash, FileHash file_hash, std::vector signatures, std::vector approve_signatures, validatorsession::ValidatorSessionStats stats, td::Promise promise) { + stats.cc_seqno = validator_set_->get_catchain_seqno(); + td::uint32 round_id = source_info.round; if (round_id >= last_known_round_id_) { last_known_round_id_ = round_id + 1; } @@ -131,12 +180,27 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s return; } auto next_block_id = create_next_block_id(root_hash, file_hash); + LOG(WARNING) << "Accepted block " << next_block_id; td::actor::send_closure(manager_, &ValidatorManager::log_validator_session_stats, next_block_id, std::move(stats)); auto block = block_data.size() > 0 ? create_block(next_block_id, std::move(block_data)).move_as_ok() : td::Ref{}; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id = next_block_id, block, prev = prev_block_ids_, - sig_set, approve_sig_set, + // Creator of the block sends broadcast to public overlays + // Creator of the block sends broadcast to private block overlay unless candidate broadcast was sent + // Any node sends broadcast to custom overlays unless candidate broadcast was sent + int send_broadcast_mode = 0; + bool sent_candidate = sent_candidate_broadcasts_.contains(next_block_id); + if (source_info.source.compute_short_id() == local_id_) { + send_broadcast_mode |= fullnode::FullNode::broadcast_mode_public; + if (!sent_candidate) { + send_broadcast_mode |= fullnode::FullNode::broadcast_mode_private_block; + } + } + if (!sent_candidate) { + send_broadcast_mode |= fullnode::FullNode::broadcast_mode_custom; + } + + auto P = td::PromiseCreator::lambda([=, SelfId = actor_id(this), block_id = next_block_id, prev = prev_block_ids_, promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { if (R.error().code() == ErrorCode::cancelled) { @@ -145,35 +209,40 @@ void ValidatorGroup::accept_block_candidate(td::uint32 round_id, PublicKeyHash s } LOG_CHECK(R.error().code() == ErrorCode::timeout || R.error().code() == ErrorCode::notready) << R.move_as_error(); td::actor::send_closure(SelfId, &ValidatorGroup::retry_accept_block_query, block_id, std::move(block), - std::move(prev), std::move(sig_set), std::move(approve_sig_set), std::move(promise)); + std::move(prev), std::move(sig_set), std::move(approve_sig_set), send_broadcast_mode, + std::move(promise)); } else { promise.set_value(R.move_as_ok()); } }); run_accept_block_query(next_block_id, std::move(block), prev_block_ids_, validator_set_, std::move(sig_set), - std::move(approve_sig_set), src == local_id_, manager_, std::move(P)); + std::move(approve_sig_set), send_broadcast_mode, manager_, + std::move(P)); prev_block_ids_ = std::vector{next_block_id}; cached_collated_block_ = nullptr; + approved_candidates_cache_.clear(); + cancellation_token_source_.cancel(); } void ValidatorGroup::retry_accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, td::Ref sig_set, - td::Ref approve_sig_set, + td::Ref approve_sig_set, int send_broadcast_mode, td::Promise promise) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), block_id, block, prev, sig_set, approve_sig_set, - promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - LOG_CHECK(R.error().code() == ErrorCode::timeout) << R.move_as_error(); - td::actor::send_closure(SelfId, &ValidatorGroup::retry_accept_block_query, block_id, std::move(block), - std::move(prev), std::move(sig_set), std::move(approve_sig_set), std::move(promise)); - } else { - promise.set_value(R.move_as_ok()); - } - }); + auto P = td::PromiseCreator::lambda( + [=, SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + LOG_CHECK(R.error().code() == ErrorCode::timeout) << R.move_as_error(); + td::actor::send_closure(SelfId, &ValidatorGroup::retry_accept_block_query, block_id, std::move(block), + std::move(prev), std::move(sig_set), std::move(approve_sig_set), send_broadcast_mode, + std::move(promise)); + } else { + promise.set_value(R.move_as_ok()); + } + }); run_accept_block_query(block_id, std::move(block), prev, validator_set_, std::move(sig_set), - std::move(approve_sig_set), false, manager_, std::move(P)); + std::move(approve_sig_set), send_broadcast_mode, manager_, std::move(P)); } void ValidatorGroup::skip_round(td::uint32 round_id) { @@ -200,35 +269,51 @@ BlockIdExt ValidatorGroup::create_next_block_id(RootHash root_hash, FileHash fil return BlockIdExt{shard_.workchain, shard_.shard, seqno + 1, root_hash, file_hash}; } +BlockId ValidatorGroup::create_next_block_id_simple() const { + BlockSeqno seqno = 0; + for (auto &p : prev_block_ids_) { + if (seqno < p.id.seqno) { + seqno = p.id.seqno; + } + } + return BlockId{shard_.workchain, shard_.shard, seqno + 1}; +} + std::unique_ptr ValidatorGroup::make_validator_session_callback() { class Callback : public validatorsession::ValidatorSession::Callback { public: Callback(td::actor::ActorId id) : id_(id) { } - void on_candidate(td::uint32 round, PublicKey source, validatorsession::ValidatorSessionRootHash root_hash, - td::BufferSlice data, td::BufferSlice collated_data, + void on_candidate(validatorsession::BlockSourceInfo source_info, + validatorsession::ValidatorSessionRootHash root_hash, td::BufferSlice data, + td::BufferSlice collated_data, td::Promise promise) override { - auto P = td::PromiseCreator::lambda([id = id_, promise = std::move(promise)](td::Result R) mutable { - if (R.is_ok()) { - promise.set_value(validatorsession::ValidatorSession::CandidateDecision{R.move_as_ok()}); - } else { - auto S = R.move_as_error(); - promise.set_value( - validatorsession::ValidatorSession::CandidateDecision{S.message().c_str(), td::BufferSlice()}); - } - }); + auto P = + td::PromiseCreator::lambda([promise = std::move(promise)](td::Result> R) mutable { + if (R.is_ok()) { + validatorsession::ValidatorSession::CandidateDecision decision(R.ok().first); + decision.set_is_cached(R.ok().second); + promise.set_value(std::move(decision)); + } else { + auto S = R.move_as_error(); + promise.set_value( + validatorsession::ValidatorSession::CandidateDecision{S.message().c_str(), td::BufferSlice()}); + } + }); - BlockCandidate candidate{Ed25519_PublicKey{source.ed25519_value().raw()}, + BlockCandidate candidate{Ed25519_PublicKey{source_info.source.ed25519_value().raw()}, BlockIdExt{0, 0, 0, root_hash, sha256_bits256(data.as_slice())}, sha256_bits256(collated_data.as_slice()), data.clone(), collated_data.clone()}; - td::actor::send_closure(id_, &ValidatorGroup::validate_block_candidate, round, std::move(candidate), - std::move(P)); + td::actor::send_closure(id_, &ValidatorGroup::validate_block_candidate, std::move(source_info), + std::move(candidate), std::move(P)); } - void on_generate_slot(td::uint32 round, td::Promise promise) override { - td::actor::send_closure(id_, &ValidatorGroup::generate_block_candidate, round, std::move(promise)); + void on_generate_slot(validatorsession::BlockSourceInfo source_info, + td::Promise promise) override { + td::actor::send_closure(id_, &ValidatorGroup::generate_block_candidate, std::move(source_info), + std::move(promise)); } - void on_block_committed(td::uint32 round, PublicKey source, validatorsession::ValidatorSessionRootHash root_hash, + void on_block_committed(validatorsession::BlockSourceInfo source_info, validatorsession::ValidatorSessionRootHash root_hash, validatorsession::ValidatorSessionFileHash file_hash, td::BufferSlice data, std::vector> signatures, std::vector> approve_signatures, @@ -242,9 +327,9 @@ std::unique_ptr ValidatorGroup::ma approve_sigs.emplace_back(BlockSignature{sig.first.bits256_value(), std::move(sig.second)}); } auto P = td::PromiseCreator::lambda([](td::Result) {}); - td::actor::send_closure(id_, &ValidatorGroup::accept_block_candidate, round, source.compute_short_id(), - std::move(data), root_hash, file_hash, std::move(sigs), std::move(approve_sigs), - std::move(stats), std::move(P)); + td::actor::send_closure(id_, &ValidatorGroup::accept_block_candidate, std::move(source_info), std::move(data), + root_hash, file_hash, std::move(sigs), std::move(approve_sigs), std::move(stats), + std::move(P)); } void on_block_skipped(td::uint32 round) override { td::actor::send_closure(id_, &ValidatorGroup::skip_round, round); @@ -288,6 +373,7 @@ void ValidatorGroup::create_session() { } CHECK(found); + config_.catchain_opts.broadcast_speed_multiplier = opts_->get_catchain_broadcast_speed_multiplier(); if (!config_.new_catchain_ids) { session_ = validatorsession::ValidatorSession::create(session_id_, config_, local_id_, std::move(vec), make_validator_session_callback(), keyring_, adnl_, rldp_, @@ -300,16 +386,22 @@ void ValidatorGroup::create_session() { << ".", allow_unsafe_self_blocks_resync_); } + double catchain_delay = opts_->get_catchain_max_block_delay() ? opts_->get_catchain_max_block_delay().value() : 0.4; + double catchain_delay_slow = + std::max(catchain_delay, + opts_->get_catchain_max_block_delay_slow() ? opts_->get_catchain_max_block_delay_slow().value() : 1.0); + td::actor::send_closure(session_, &validatorsession::ValidatorSession::set_catchain_max_block_delay, catchain_delay, + catchain_delay_slow); if (started_) { td::actor::send_closure(session_, &validatorsession::ValidatorSession::start); } } -void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterchain_block_id, UnixTime min_ts) { +void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterchain_block_id) { prev_block_ids_ = prev; min_masterchain_block_id_ = min_masterchain_block_id; - min_ts_ = min_ts; cached_collated_block_ = nullptr; + approved_candidates_cache_.clear(); started_ = true; if (init_) { @@ -324,21 +416,106 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch auto block = p.block.size() > 0 ? create_block(next_block_id, std::move(p.block)).move_as_ok() : td::Ref{}; retry_accept_block_query(next_block_id, std::move(block), prev_block_ids_, std::move(p.sigs), - std::move(p.approve_sigs), std::move(p.promise)); + std::move(p.approve_sigs), 0, std::move(p.promise)); prev_block_ids_ = std::vector{next_block_id}; } postponed_accept_.clear(); + + validatorsession::NewValidatorGroupStats stats; + stats.session_id = session_id_; + stats.shard = shard_; + stats.cc_seqno = validator_set_->get_catchain_seqno(); + stats.last_key_block_seqno = last_key_block_seqno_; + stats.timestamp = td::Clocks::system(); + td::uint32 idx = 0; + for (const auto& node : validator_set_->export_vector()) { + PublicKeyHash id = ValidatorFullId{node.key}.compute_short_id(); + if (id == local_id_) { + stats.self_idx = idx; + } + stats.nodes.push_back(validatorsession::NewValidatorGroupStats::Node{id, node.weight}); + ++idx; + } + td::actor::send_closure(manager_, &ValidatorManager::log_new_validator_group_stats, std::move(stats)); } void ValidatorGroup::destroy() { if (!session_.empty()) { + td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_current_stats, + [manager = manager_, cc_seqno = validator_set_->get_catchain_seqno(), + block_id = create_next_block_id(RootHash::zero(), FileHash::zero())]( + td::Result R) { + if (R.is_error()) { + LOG(WARNING) << "Failed to get validator session stats: " << R.move_as_error(); + return; + } + auto stats = R.move_as_ok(); + if (stats.rounds.empty()) { + return; + } + stats.cc_seqno = cc_seqno; + td::actor::send_closure(manager, &ValidatorManager::log_validator_session_stats, block_id, + std::move(stats)); + }); + td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_end_stats, + [manager = manager_](td::Result R) { + if (R.is_error()) { + LOG(DEBUG) << "Failed to get validator session end stats: " << R.move_as_error(); + return; + } + auto stats = R.move_as_ok(); + td::actor::send_closure(manager, &ValidatorManager::log_end_validator_group_stats, + std::move(stats)); + }); auto ses = session_.release(); delay_action([ses]() mutable { td::actor::send_closure(ses, &validatorsession::ValidatorSession::destroy); }, td::Timestamp::in(10.0)); } + cancellation_token_source_.cancel(); stop(); } +void ValidatorGroup::get_validator_group_info_for_litequery( + td::Promise> promise) { + if (session_.empty()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "not started")); + return; + } + td::actor::send_closure( + session_, &validatorsession::ValidatorSession::get_validator_group_info_for_litequery, last_known_round_id_, + [SelfId = actor_id(this), promise = std::move(promise), round = last_known_round_id_]( + td::Result>> R) mutable { + TRY_RESULT_PROMISE(promise, result, std::move(R)); + td::actor::send_closure(SelfId, &ValidatorGroup::get_validator_group_info_for_litequery_cont, round, + std::move(result), std::move(promise)); + }); +} + +void ValidatorGroup::get_validator_group_info_for_litequery_cont( + td::uint32 expected_round, std::vector> candidates, + td::Promise> promise) { + if (expected_round != last_known_round_id_) { + candidates.clear(); + } + + BlockId next_block_id = create_next_block_id_simple(); + for (auto &candidate : candidates) { + BlockIdExt id{next_block_id, candidate->id_->block_id_->root_hash_, candidate->id_->block_id_->file_hash_}; + candidate->id_->block_id_ = create_tl_lite_block_id(id); + candidate->available_ = + available_block_candidates_.count({candidate->id_->creator_, id, candidate->id_->collated_data_hash_}); + } + + auto result = create_tl_object(); + result->next_block_id_ = create_tl_lite_block_id_simple(next_block_id); + for (const BlockIdExt& prev : prev_block_ids_) { + result->prev_.push_back(create_tl_lite_block_id(prev)); + } + result->cc_seqno_ = validator_set_->get_catchain_seqno(); + result->candidates_ = std::move(candidates); + promise.set_result(std::move(result)); +} + } // namespace validator } // namespace ton diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 6bba5887..db55614f 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -24,6 +24,8 @@ #include "rldp/rldp.h" +#include + namespace ton { namespace validator { @@ -32,21 +34,24 @@ class ValidatorManager; class ValidatorGroup : public td::actor::Actor { public: - void generate_block_candidate(td::uint32 round_id, td::Promise promise); - void validate_block_candidate(td::uint32 round_id, BlockCandidate block, td::Promise promise); - void accept_block_candidate(td::uint32 round_id, PublicKeyHash src, td::BufferSlice block, RootHash root_hash, + void generate_block_candidate(validatorsession::BlockSourceInfo source_info, + td::Promise promise); + void validate_block_candidate(validatorsession::BlockSourceInfo source_info, BlockCandidate block, + td::Promise> promise); + void accept_block_candidate(validatorsession::BlockSourceInfo source_info, td::BufferSlice block, RootHash root_hash, FileHash file_hash, std::vector signatures, std::vector approve_signatures, validatorsession::ValidatorSessionStats stats, td::Promise promise); void skip_round(td::uint32 round); void retry_accept_block_query(BlockIdExt block_id, td::Ref block, std::vector prev, td::Ref sigs, td::Ref approve_sigs, - td::Promise promise); + int send_broadcast_mode, td::Promise promise); void get_approved_candidate(PublicKey source, RootHash root_hash, FileHash file_hash, FileHash collated_data_file_hash, td::Promise promise); BlockIdExt create_next_block_id(RootHash root_hash, FileHash file_hash) const; + BlockId create_next_block_id_simple() const; - void start(std::vector prev, BlockIdExt min_masterchain_block_id, UnixTime min_ts); + void start(std::vector prev, BlockIdExt min_masterchain_block_id); void create_session(); void destroy(); void start_up() override { @@ -56,16 +61,25 @@ class ValidatorGroup : public td::actor::Actor { } } + void get_validator_group_info_for_litequery( + td::Promise> promise); + + void update_options(td::Ref opts) { + opts_ = std::move(opts); + } + ValidatorGroup(ShardIdFull shard, PublicKeyHash local_id, ValidatorSessionId session_id, - td::Ref validator_set, validatorsession::ValidatorSessionOptions config, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, - std::string db_root, td::actor::ActorId validator_manager, bool create_session, - bool allow_unsafe_self_blocks_resync) + td::Ref validator_set, BlockSeqno last_key_block_seqno, + validatorsession::ValidatorSessionOptions config, td::actor::ActorId keyring, + td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId overlays, std::string db_root, + td::actor::ActorId validator_manager, bool create_session, + bool allow_unsafe_self_blocks_resync, td::Ref opts) : shard_(shard) , local_id_(std::move(local_id)) , session_id_(session_id) , validator_set_(std::move(validator_set)) + , last_key_block_seqno_(last_key_block_seqno) , config_(std::move(config)) , keyring_(keyring) , adnl_(adnl) @@ -74,7 +88,8 @@ class ValidatorGroup : public td::actor::Actor { , db_root_(std::move(db_root)) , manager_(validator_manager) , init_(create_session) - , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) { + , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) + , opts_(std::move(opts)) { } private: @@ -99,9 +114,9 @@ class ValidatorGroup : public td::actor::Actor { std::vector prev_block_ids_; BlockIdExt min_masterchain_block_id_; - UnixTime min_ts_; td::Ref validator_set_; + BlockSeqno last_key_block_seqno_; validatorsession::ValidatorSessionOptions config_; td::actor::ActorId keyring_; @@ -115,6 +130,7 @@ class ValidatorGroup : public td::actor::Actor { bool init_ = false; bool started_ = false; bool allow_unsafe_self_blocks_resync_; + td::Ref opts_; td::uint32 last_known_round_id_ = 0; struct CachedCollatedBlock { @@ -122,8 +138,40 @@ class ValidatorGroup : public td::actor::Actor { std::vector> promises; }; std::shared_ptr cached_collated_block_; + td::CancellationTokenSource cancellation_token_source_; - void generated_block_candidate(std::shared_ptr cache, td::Result R); + void generated_block_candidate(validatorsession::BlockSourceInfo source_info, + std::shared_ptr cache, td::Result R); + + using CacheKey = std::tuple; + std::map approved_candidates_cache_; + + void update_approve_cache(CacheKey key, UnixTime value); + + static CacheKey block_to_cache_key(const BlockCandidate& block) { + return std::make_tuple(block.pubkey.as_bits256(), block.id, sha256_bits256(block.data), block.collated_file_hash); + } + + void get_validator_group_info_for_litequery_cont( + td::uint32 expected_round, + std::vector> candidates, + td::Promise> promise); + + std::set> available_block_candidates_; // source, id, collated hash + + void add_available_block_candidate(td::Bits256 source, BlockIdExt id, FileHash collated_data_hash) { + available_block_candidates_.emplace(source, id, collated_data_hash); + } + + std::set sent_candidate_broadcasts_; + + void send_block_candidate_broadcast(BlockIdExt id, td::BufferSlice data) { + if (sent_candidate_broadcasts_.insert(id).second) { + td::actor::send_closure(manager_, &ValidatorManager::send_block_candidate_broadcast, id, + validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), + std::move(data)); + } + } }; } // namespace validator diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index 93fe05e6..cb26fe44 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -26,7 +26,7 @@ namespace validator { td::Ref ValidatorManagerOptions::create( BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard, bool allow_blockchain_init, + std::function check_shard, bool allow_blockchain_init, double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) { return td::make_ref(zero_block_id, init_block_id, std::move(check_shard), diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index d23d8cc9..ace6b106 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -32,11 +32,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { BlockIdExt init_block_id() const override { return init_block_id_; } - bool need_monitor(ShardIdFull shard) const override { - return check_shard_(shard, 0, ShardCheckMode::m_monitor); - } - bool need_validate(ShardIdFull shard, CatchainSeqno cc_seqno) const override { - return check_shard_(shard, cc_seqno, ShardCheckMode::m_validate); + bool need_monitor(ShardIdFull shard, const td::Ref& state) const override { + td::uint32 min_split = state->monitor_min_split_depth(shard.workchain); + return check_shard_((td::uint32)shard.pfx_len() <= min_split ? shard : shard_prefix(shard, min_split)); } bool allow_blockchain_init() const override { return allow_blockchain_init_; @@ -114,6 +112,51 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { std::string get_session_logs_file() const override { return session_logs_file_; } + td::uint32 get_celldb_compress_depth() const override { + return celldb_compress_depth_; + } + size_t get_max_open_archive_files() const override { + return max_open_archive_files_; + } + double get_archive_preload_period() const override { + return archive_preload_period_; + } + bool get_disable_rocksdb_stats() const override { + return disable_rocksdb_stats_; + } + bool nonfinal_ls_queries_enabled() const override { + return nonfinal_ls_queries_enabled_; + } + td::optional get_celldb_cache_size() const override { + return celldb_cache_size_; + } + bool get_celldb_direct_io() const override { + return celldb_direct_io_; + } + bool get_celldb_preload_all() const override { + return celldb_preload_all_; + } + bool get_celldb_in_memory() const override { + return celldb_in_memory_; + } + td::optional get_catchain_max_block_delay() const override { + return catchain_max_block_delay_; + } + td::optional get_catchain_max_block_delay_slow() const override { + return catchain_max_block_delay_slow_; + } + bool get_state_serializer_enabled() const override { + return state_serializer_enabled_; + } + td::Ref get_collator_options() const override { + return collator_options_; + } + bool get_fast_state_serializer_enabled() const override { + return fast_state_serializer_enabled_; + } + double get_catchain_broadcast_speed_multiplier() const override { + return catchain_broadcast_speed_multipliers_; + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -121,7 +164,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_init_block_id(BlockIdExt block_id) override { init_block_id_ = block_id; } - void set_shard_check_function(std::function check_shard) override { + void set_shard_check_function(std::function check_shard) override { check_shard_ = std::move(check_shard); } void set_allow_blockchain_init(bool value) override { @@ -167,17 +210,60 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_session_logs_file(std::string f) override { session_logs_file_ = std::move(f); } + void set_celldb_compress_depth(td::uint32 value) override { + celldb_compress_depth_ = value; + } + void set_max_open_archive_files(size_t value) override { + max_open_archive_files_ = value; + } + void set_archive_preload_period(double value) override { + archive_preload_period_ = value; + } + void set_disable_rocksdb_stats(bool value) override { + disable_rocksdb_stats_ = value; + } + void set_nonfinal_ls_queries_enabled(bool value) override { + nonfinal_ls_queries_enabled_ = value; + } + void set_celldb_cache_size(td::uint64 value) override { + celldb_cache_size_ = value; + } + void set_celldb_direct_io(bool value) override { + celldb_direct_io_ = value; + } + void set_celldb_preload_all(bool value) override { + celldb_preload_all_ = value; + } + void set_celldb_in_memory(bool value) override { + celldb_in_memory_ = value; + } + void set_catchain_max_block_delay(double value) override { + catchain_max_block_delay_ = value; + } + void set_catchain_max_block_delay_slow(double value) override { + catchain_max_block_delay_slow_ = value; + } + void set_state_serializer_enabled(bool value) override { + state_serializer_enabled_ = value; + } + void set_collator_options(td::Ref value) override { + collator_options_ = std::move(value); + } + void set_fast_state_serializer_enabled(bool value) override { + fast_state_serializer_enabled_ = value; + } + void set_catchain_broadcast_speed_multiplier(double value) override { + catchain_broadcast_speed_multipliers_ = value; + } ValidatorManagerOptionsImpl *make_copy() const override { return new ValidatorManagerOptionsImpl(*this); } ValidatorManagerOptionsImpl(BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard, - bool allow_blockchain_init, double sync_blocks_before, - double block_ttl, double state_ttl, double max_mempool_num, - double archive_ttl, double key_proof_ttl, - bool initial_sync_disabled) + std::function check_shard, bool allow_blockchain_init, + double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, + double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) : zero_block_id_(zero_block_id) , init_block_id_(init_block_id) , check_shard_(std::move(check_shard)) @@ -194,7 +280,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { private: BlockIdExt zero_block_id_; BlockIdExt init_block_id_; - std::function check_shard_; + std::function check_shard_; bool allow_blockchain_init_; double sync_blocks_before_; double block_ttl_; @@ -209,6 +295,20 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { BlockSeqno truncate_{0}; BlockSeqno sync_upto_{0}; std::string session_logs_file_; + td::uint32 celldb_compress_depth_{0}; + size_t max_open_archive_files_ = 0; + double archive_preload_period_ = 0.0; + bool disable_rocksdb_stats_; + bool nonfinal_ls_queries_enabled_ = false; + td::optional celldb_cache_size_; + bool celldb_direct_io_ = false; + bool celldb_preload_all_ = false; + bool celldb_in_memory_ = false; + td::optional catchain_max_block_delay_, catchain_max_block_delay_slow_; + bool state_serializer_enabled_ = true; + td::Ref collator_options_{true}; + bool fast_state_serializer_enabled_ = false; + double catchain_broadcast_speed_multipliers_; }; } // namespace validator diff --git a/validator/validator-telemetry.cpp b/validator/validator-telemetry.cpp new file mode 100644 index 00000000..403dd6f9 --- /dev/null +++ b/validator/validator-telemetry.cpp @@ -0,0 +1,87 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#include "validator-telemetry.hpp" +#include "git.h" +#include "td/utils/Random.h" +#include "td/utils/port/uname.h" +#include "interfaces/validator-manager.h" + +namespace ton::validator { + +void ValidatorTelemetry::start_up() { + node_version_ = PSTRING() << "validator-engine, Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate(); + + os_version_ = td::get_operating_system_version().str(); + + auto r_total_mem_stat = td::get_total_mem_stat(); + if (r_total_mem_stat.is_error()) { + LOG(WARNING) << "Cannot get RAM size: " << r_total_mem_stat.move_as_error(); + } else { + ram_size_ = r_total_mem_stat.ok().total_ram; + } + + auto r_cpu_cores = td::get_cpu_cores(); + if (r_cpu_cores.is_error()) { + LOG(WARNING) << "Cannot get CPU info: " << r_cpu_cores.move_as_error(); + } else { + cpu_cores_ = r_cpu_cores.move_as_ok(); + } + + LOG(DEBUG) << "Initializing validator telemetry, key = " << key_ << ", adnl_id = " << local_id_; + alarm_timestamp().relax(send_telemetry_at_ = td::Timestamp::in(td::Random::fast(30.0, 60.0))); +} + +void ValidatorTelemetry::alarm() { + if (send_telemetry_at_.is_in_past()) { + send_telemetry_at_ = td::Timestamp::never(); + send_telemetry(); + } + alarm_timestamp().relax(send_telemetry_at_); +} + +void ValidatorTelemetry::send_telemetry() { + send_telemetry_at_ = td::Timestamp::in(PERIOD); + + auto telemetry = create_tl_object(); + telemetry->flags_ = 0; + telemetry->timestamp_ = td::Clocks::system(); + telemetry->adnl_id_ = local_id_.bits256_value(); + telemetry->node_version_ = node_version_; + telemetry->os_version_ = os_version_; + telemetry->node_started_at_ = adnl::Adnl::adnl_start_time(); + telemetry->ram_size_ = ram_size_; + telemetry->cpu_cores_ = cpu_cores_; + telemetry->node_threads_ = (td::int32)td::actor::SchedulerContext::get() + ->scheduler_group() + ->schedulers.at(td::actor::SchedulerContext::get()->get_scheduler_id().value()) + .cpu_threads_count; + + LOG(DEBUG) << "Sending validator telemetry for adnl id " << local_id_; + td::actor::send_closure(manager_, &ValidatorManager::send_validator_telemetry, key_, std::move(telemetry)); +} + +} // namespace ton::validator diff --git a/validator/validator-telemetry.hpp b/validator/validator-telemetry.hpp new file mode 100644 index 00000000..73908bdd --- /dev/null +++ b/validator/validator-telemetry.hpp @@ -0,0 +1,66 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#pragma once +#include "overlay.h" +#include "td/actor/actor.h" +#include "adnl/adnl.h" +#include "interfaces/shard.h" + +namespace ton::validator { +class ValidatorManager; + +class ValidatorTelemetry : public td::actor::Actor { +public: + ValidatorTelemetry(PublicKeyHash key, adnl::AdnlNodeIdShort local_id, td::Bits256 zero_state_file_hash, + td::actor::ActorId manager) + : key_(key) + , local_id_(local_id) + , zero_state_file_hash_(zero_state_file_hash) + , manager_(std::move(manager)) { + } + + void start_up() override; + void alarm() override; + +private: + PublicKeyHash key_; + adnl::AdnlNodeIdShort local_id_; + td::Bits256 zero_state_file_hash_; + td::actor::ActorId manager_; + + std::string node_version_; + std::string os_version_; + td::uint32 cpu_cores_ = 0; + td::uint64 ram_size_ = 0; + + td::Timestamp send_telemetry_at_ = td::Timestamp::never(); + + void send_telemetry(); + + static constexpr double PERIOD = 600.0; + static constexpr td::uint32 MAX_SIZE = 8192; +}; +} // namespace ton::validator \ No newline at end of file diff --git a/validator/validator.h b/validator/validator.h index 0687b160..5d6c0173 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -20,6 +20,7 @@ #include #include +#include #include "td/actor/actor.h" @@ -35,15 +36,16 @@ #include "interfaces/proof.h" #include "interfaces/shard.h" #include "catchain/catchain-types.h" +#include "interfaces/out-msg-queue-proof.h" #include "interfaces/external-message.h" namespace ton { namespace validator { -class DownloadToken { +class ActionToken { public: - virtual ~DownloadToken() = default; + virtual ~ActionToken() = default; }; struct PerfTimerStats { @@ -51,14 +53,33 @@ struct PerfTimerStats { std::deque> stats; // }; +struct CollatorOptions : public td::CntObject { + bool deferring_enabled = true; + + // Defer messages from account after Xth message in block (excluding first messages from transactions) + td::uint32 defer_messages_after = 10; + // Defer all messages if out msg queue size is greater than X (excluding first messages from transactions) + td::uint64 defer_out_queue_size_limit = 2048; + + // See Collator::process_dispatch_queue + td::uint32 dispatch_phase_2_max_total = 150; + td::uint32 dispatch_phase_3_max_total = 150; + td::uint32 dispatch_phase_2_max_per_initiator = 20; + td::optional dispatch_phase_3_max_per_initiator; // Default - depends on out msg queue size + + // Don't defer messages from these accounts + std::set> whitelist; + // Prioritize these accounts on each phase of process_dispatch_queue + std::set> prioritylist; +}; + struct ValidatorManagerOptions : public td::CntObject { public: enum class ShardCheckMode { m_monitor, m_validate }; virtual BlockIdExt zero_block_id() const = 0; virtual BlockIdExt init_block_id() const = 0; - virtual bool need_monitor(ShardIdFull shard) const = 0; - virtual bool need_validate(ShardIdFull shard, CatchainSeqno cc_seqno) const = 0; + virtual bool need_monitor(ShardIdFull shard, const td::Ref& state) const = 0; virtual bool allow_blockchain_init() const = 0; virtual double sync_blocks_before() const = 0; virtual double block_ttl() const = 0; @@ -81,11 +102,25 @@ struct ValidatorManagerOptions : public td::CntObject { virtual BlockSeqno get_truncate_seqno() const = 0; virtual BlockSeqno sync_upto() const = 0; virtual std::string get_session_logs_file() const = 0; + virtual td::uint32 get_celldb_compress_depth() const = 0; + virtual bool get_celldb_in_memory() const = 0; + virtual size_t get_max_open_archive_files() const = 0; + virtual double get_archive_preload_period() const = 0; + virtual bool get_disable_rocksdb_stats() const = 0; + virtual bool nonfinal_ls_queries_enabled() const = 0; + virtual td::optional get_celldb_cache_size() const = 0; + virtual bool get_celldb_direct_io() const = 0; + virtual bool get_celldb_preload_all() const = 0; + virtual td::optional get_catchain_max_block_delay() const = 0; + virtual td::optional get_catchain_max_block_delay_slow() const = 0; + virtual bool get_state_serializer_enabled() const = 0; + virtual td::Ref get_collator_options() const = 0; + virtual bool get_fast_state_serializer_enabled() const = 0; + virtual double get_catchain_broadcast_speed_multiplier() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; - virtual void set_shard_check_function( - std::function check_shard) = 0; + virtual void set_shard_check_function(std::function check_shard) = 0; virtual void set_allow_blockchain_init(bool value) = 0; virtual void set_sync_blocks_before(double value) = 0; virtual void set_block_ttl(double value) = 0; @@ -100,15 +135,29 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void truncate_db(BlockSeqno seqno) = 0; virtual void set_sync_upto(BlockSeqno seqno) = 0; virtual void set_session_logs_file(std::string f) = 0; + virtual void set_celldb_compress_depth(td::uint32 value) = 0; + virtual void set_max_open_archive_files(size_t value) = 0; + virtual void set_archive_preload_period(double value) = 0; + virtual void set_disable_rocksdb_stats(bool value) = 0; + virtual void set_nonfinal_ls_queries_enabled(bool value) = 0; + virtual void set_celldb_cache_size(td::uint64 value) = 0; + virtual void set_celldb_direct_io(bool value) = 0; + virtual void set_celldb_preload_all(bool value) = 0; + virtual void set_celldb_in_memory(bool value) = 0; + virtual void set_catchain_max_block_delay(double value) = 0; + virtual void set_catchain_max_block_delay_slow(double value) = 0; + virtual void set_state_serializer_enabled(bool value) = 0; + virtual void set_collator_options(td::Ref value) = 0; + virtual void set_fast_state_serializer_enabled(bool value) = 0; + virtual void set_catchain_broadcast_speed_multiplier(double value) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard = [](ShardIdFull, CatchainSeqno, - ShardCheckMode) { return true; }, - bool allow_blockchain_init = false, double sync_blocks_before = 86400, double block_ttl = 86400 * 7, - double state_ttl = 3600, double archive_ttl = 86400 * 365, double key_proof_ttl = 86400 * 3650, - double max_mempool_num = 999999, - bool initial_sync_disabled = false); + + std::function check_shard = [](ShardIdFull) { return true; }, + bool allow_blockchain_init = false, double sync_blocks_before = 3600, double block_ttl = 86400, + double state_ttl = 86400, double archive_ttl = 86400 * 7, double key_proof_ttl = 86400 * 3650, + double max_mempool_num = 999999, bool initial_sync_disabled = false); }; class ValidatorManagerInterface : public td::actor::Actor { @@ -118,13 +167,15 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual ~Callback() = default; virtual void initial_read_complete(BlockHandle top_masterchain_blocks) = 0; - virtual void add_shard(ShardIdFull shard) = 0; - virtual void del_shard(ShardIdFull shard) = 0; + virtual void on_new_masterchain_block(td::Ref state, + std::set shards_to_monitor) = 0; virtual void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; - virtual void send_broadcast(BlockBroadcast broadcast) = 0; + virtual void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) = 0; + virtual void send_broadcast(BlockBroadcast broadcast, int mode) = 0; virtual void download_block(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; virtual void download_zero_state(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, @@ -137,10 +188,14 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise promise) = 0; virtual void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) = 0; - virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) = 0; + virtual void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) = 0; + virtual void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise) = 0; virtual void new_key_block(BlockHandle handle) = 0; + virtual void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) = 0; }; virtual ~ValidatorManagerInterface() = default; @@ -178,6 +233,8 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise promise) = 0; virtual void get_persistent_state_slice(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::int64 offset, td::int64 max_length, td::Promise promise) = 0; + virtual void get_previous_persistent_state_files( + BlockSeqno cur_mc_seqno, td::Promise>> promise) = 0; virtual void get_block_proof(BlockHandle handle, td::Promise promise) = 0; virtual void get_block_proof_link(BlockHandle handle, td::Promise promise) = 0; virtual void get_block_handle(BlockIdExt block_id, bool force, td::Promise promise) = 0; @@ -188,21 +245,23 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_next_block(BlockIdExt block_id, td::Promise promise) = 0; virtual void write_handle(BlockHandle handle, td::Promise promise) = 0; - virtual void new_external_message(td::BufferSlice data) = 0; + virtual void new_external_message(td::BufferSlice data, int priority) = 0; virtual void check_external_message(td::BufferSlice data, td::Promise> promise) = 0; virtual void new_ihr_message(td::BufferSlice data) = 0; virtual void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; + virtual void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) = 0; virtual void add_ext_server_id(adnl::AdnlNodeIdShort id) = 0; virtual void add_ext_server_port(td::uint16 port) = 0; virtual void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) = 0; + td::Promise> promise) = 0; virtual void get_block_data_from_db(ConstBlockHandle handle, td::Promise> promise) = 0; virtual void get_block_data_from_db_short(BlockIdExt block_id, td::Promise> promise) = 0; virtual void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) = 0; + virtual void get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) = 0; virtual void get_shard_state_from_db(ConstBlockHandle handle, td::Promise> promise) = 0; virtual void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) = 0; virtual void get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) = 0; @@ -217,15 +276,32 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_block_by_seqno_from_db(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise promise) = 0; - virtual void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) = 0; + virtual void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) = 0; + virtual void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) = 0; + + virtual void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) = 0; virtual void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise) = 0; virtual void run_ext_query(td::BufferSlice data, td::Promise promise) = 0; virtual void prepare_stats(td::Promise>> promise) = 0; + virtual void prepare_actor_stats(td::Promise promise) = 0; virtual void prepare_perf_timer_stats(td::Promise> promise) = 0; virtual void add_perf_timer_stat(std::string name, double duration) = 0; + virtual void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) = 0; + + virtual void update_options(td::Ref opts) = 0; + + virtual void register_stats_provider( + td::uint64 idx, std::string prefix, + std::function>>)> callback) { + } + virtual void unregister_stats_provider(td::uint64 idx) { + } }; } // namespace validator