diff --git a/.github/script/amd64-20.04.Dockerfile b/.github/script/amd64-20.04.Dockerfile deleted file mode 100644 index 40d980e5..00000000 --- a/.github/script/amd64-20.04.Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM ubuntu:20.04 - -RUN apt update -RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata -RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git curl libreadline-dev ccache libmicrohttpd-dev ninja-build libsecp256k1-dev libsodium-dev pkg-config - -WORKDIR / - -ARG BRANCH -ARG REPO -RUN git clone --recurse-submodules https://github.com/$REPO ton && cd ton && git checkout $BRANCH && git submodule update - -WORKDIR /ton -RUN mkdir /ton/build -WORKDIR /ton/build -ENV CC clang -ENV CXX clang++ -ENV CCACHE_DISABLE 1 -RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. -RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client \ No newline at end of file diff --git a/.github/script/amd64-22.04.Dockerfile b/.github/script/amd64-22.04.Dockerfile deleted file mode 100644 index 44c9c40b..00000000 --- a/.github/script/amd64-22.04.Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM ubuntu:22.04 - -RUN apt update -RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata -RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git curl libreadline-dev ccache libmicrohttpd-dev ninja-build libsecp256k1-dev libsodium-dev pkg-config - -WORKDIR / - -ARG BRANCH -ARG REPO -RUN git clone --recurse-submodules https://github.com/$REPO ton && cd ton && git checkout $BRANCH && git submodule update - -WORKDIR /ton -RUN mkdir /ton/build -WORKDIR /ton/build -ENV CC clang -ENV CXX clang++ -ENV CCACHE_DISABLE 1 -RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. -RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client \ No newline at end of file diff --git a/.github/script/arm64-20.04.Dockerfile b/.github/script/arm64-20.04.Dockerfile deleted file mode 100644 index 1f57dc40..00000000 --- a/.github/script/arm64-20.04.Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM ubuntu:20.04 - -RUN apt update -RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata -RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git curl libreadline-dev ccache libmicrohttpd-dev ninja-build libsecp256k1-dev libsodium-dev pkg-config - -WORKDIR / - -ARG BRANCH -ARG REPO -RUN git clone --recurse-submodules https://github.com/$REPO ton && cd ton && git checkout $BRANCH && git submodule update - -WORKDIR /ton -RUN mkdir /ton/build -WORKDIR /ton/build -ENV CC clang -ENV CXX clang++ -ENV CCACHE_DISABLE 1 -RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= .. -RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id dht-server lite-client \ No newline at end of file diff --git a/.github/script/arm64-22.04.Dockerfile b/.github/script/arm64-22.04.Dockerfile deleted file mode 100644 index 2b595839..00000000 --- a/.github/script/arm64-22.04.Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM ubuntu:22.04 - -RUN apt update -RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata -RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git curl libreadline-dev ccache libmicrohttpd-dev ninja-build libsecp256k1-dev libsodium-dev pkg-config - -WORKDIR / - -ARG BRANCH -ARG REPO -RUN git clone --recurse-submodules https://github.com/$REPO ton && cd ton && git checkout $BRANCH && git submodule update - -WORKDIR /ton -RUN mkdir /ton/build -WORKDIR /ton/build -ENV CC clang -ENV CXX clang++ -ENV CCACHE_DISABLE 1 -RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= .. -RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id dht-server lite-client \ No newline at end of file diff --git a/.github/workflows/build-ton-linux-android-tonlib.yml b/.github/workflows/build-ton-linux-android-tonlib.yml index a1a6bc99..b1ef5281 100644 --- a/.github/workflows/build-ton-linux-android-tonlib.yml +++ b/.github/workflows/build-ton-linux-android-tonlib.yml @@ -21,6 +21,8 @@ jobs: - name: Build TON run: | + git submodule sync --recursive + git submodule update cp assembly/android/build-android-tonlib.sh . chmod +x build-android-tonlib.sh ./build-android-tonlib.sh -a diff --git a/.github/workflows/build-ton-linux-arm64-appimage.yml b/.github/workflows/build-ton-linux-arm64-appimage.yml new file mode 100644 index 00000000..d464d8a2 --- /dev/null +++ b/.github/workflows/build-ton-linux-arm64-appimage.yml @@ -0,0 +1,57 @@ +name: Ubuntu TON build (AppImages, arm64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + runs-on: ubuntu-22.04-arm + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Install system libraries + run: | + sudo apt update + sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev + sudo apt remove libgsl-dev + + - name: Install clang-16 + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 16 all + + - name: Build TON + run: | + git submodule sync --recursive + git submodule update + cp assembly/native/build-ubuntu-appimages.sh . + chmod +x build-ubuntu-appimages.sh + ./build-ubuntu-appimages.sh -a + + - name: Make AppImages + run: | + cp assembly/appimage/create-appimages.sh . + cp assembly/appimage/AppRun . + cp assembly/appimage/ton.png . + chmod +x create-appimages.sh + ./create-appimages.sh aarch64 + rm -rf artifacts + + + - name: Build TON libs + run: | + cp assembly/native/build-ubuntu-portable-libs.sh . + chmod +x build-ubuntu-portable-libs.sh + ./build-ubuntu-portable-libs.sh -a + cp ./artifacts/libtonlibjson.so appimages/artifacts/ + cp ./artifacts/libemulator.so appimages/artifacts/ + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-arm64-linux + path: appimages/artifacts diff --git a/.github/workflows/build-ton-linux-arm64-shared.yml b/.github/workflows/build-ton-linux-arm64-shared.yml new file mode 100644 index 00000000..6433df0b --- /dev/null +++ b/.github/workflows/build-ton-linux-arm64-shared.yml @@ -0,0 +1,43 @@ +name: Ubuntu TON build (shared, arm64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + strategy: + fail-fast: false + matrix: + os: [ubuntu-22.04-arm, ubuntu-24.04-arm] + runs-on: ${{ matrix.os }} + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Install system libraries + run: | + sudo apt-get update + sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev + + - if: matrix.os != 'ubuntu-24.04-arm' + name: Install llvm-16 + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 16 all + + - name: Build TON + run: | + git submodule sync --recursive + git submodule update + cp assembly/native/build-ubuntu-shared.sh . + chmod +x build-ubuntu-shared.sh + ./build-ubuntu-shared.sh -t -a + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-binaries-${{ matrix.os }} + path: artifacts diff --git a/.github/workflows/build-ton-linux-x86-64-appimage.yml b/.github/workflows/build-ton-linux-x86-64-appimage.yml new file mode 100644 index 00000000..4f78ece9 --- /dev/null +++ b/.github/workflows/build-ton-linux-x86-64-appimage.yml @@ -0,0 +1,63 @@ +name: Ubuntu TON build (AppImages, x86-64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + runs-on: ubuntu-20.04 + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Install system libraries + run: | + sudo apt update + sudo apt install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev + sudo apt remove libgsl-dev + + - name: Install gcc-11 g++-11 + run: | + sudo apt install -y manpages-dev software-properties-common + sudo add-apt-repository ppa:ubuntu-toolchain-r/test + sudo apt update && sudo apt install gcc-11 g++-11 + + - name: Install clang-16 + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 16 all + + - name: Build TON + run: | + git submodule sync --recursive + git submodule update + cp assembly/native/build-ubuntu-appimages.sh . + chmod +x build-ubuntu-appimages.sh + ./build-ubuntu-appimages.sh -a + + - name: Make AppImages + run: | + cp assembly/appimage/create-appimages.sh . + cp assembly/appimage/AppRun . + cp assembly/appimage/ton.png . + chmod +x create-appimages.sh + ./create-appimages.sh x86_64 + rm -rf artifacts + + + - name: Build TON libs + run: | + cp assembly/native/build-ubuntu-portable-libs.sh . + chmod +x build-ubuntu-portable-libs.sh + ./build-ubuntu-portable-libs.sh -a + cp ./artifacts/libtonlibjson.so appimages/artifacts/ + cp ./artifacts/libemulator.so appimages/artifacts/ + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-x86_64-linux + path: appimages/artifacts diff --git a/.github/workflows/build-ton-linux-x86-64-shared.yml b/.github/workflows/build-ton-linux-x86-64-shared.yml index e28ecc74..bf78f7df 100644 --- a/.github/workflows/build-ton-linux-x86-64-shared.yml +++ b/.github/workflows/build-ton-linux-x86-64-shared.yml @@ -35,6 +35,8 @@ jobs: - name: Build TON run: | + git submodule sync --recursive + git submodule update cp assembly/native/build-ubuntu-shared.sh . chmod +x build-ubuntu-shared.sh ./build-ubuntu-shared.sh -t -a diff --git a/.github/workflows/build-ton-macos-13-x86-64-portable.yml b/.github/workflows/build-ton-macos-13-x86-64-portable.yml new file mode 100644 index 00000000..5e50a468 --- /dev/null +++ b/.github/workflows/build-ton-macos-13-x86-64-portable.yml @@ -0,0 +1,27 @@ +name: MacOS-13 TON build (portable, x86-64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + runs-on: macos-13 + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Build TON + run: | + git submodule sync --recursive + git submodule update + cp assembly/native/build-macos-portable.sh . + chmod +x build-macos-portable.sh + ./build-macos-portable.sh -t -a + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-x86_64-macos + path: artifacts diff --git a/.github/workflows/build-ton-macos-14-arm64-portable.yml b/.github/workflows/build-ton-macos-14-arm64-portable.yml new file mode 100644 index 00000000..8eb3af70 --- /dev/null +++ b/.github/workflows/build-ton-macos-14-arm64-portable.yml @@ -0,0 +1,27 @@ +name: MacOS-14 TON build (portable, arm64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + runs-on: macos-14 + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Build TON + run: | + git submodule sync --recursive + git submodule update + cp assembly/native/build-macos-portable.sh . + chmod +x build-macos-portable.sh + ./build-macos-portable.sh -t -a + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-arm64-macos + path: artifacts diff --git a/.github/workflows/build-ton-macos-15-arm64-shared.yml b/.github/workflows/build-ton-macos-15-arm64-shared.yml index 8379a0b0..00d8f639 100644 --- a/.github/workflows/build-ton-macos-15-arm64-shared.yml +++ b/.github/workflows/build-ton-macos-15-arm64-shared.yml @@ -14,6 +14,8 @@ jobs: - name: Build TON run: | + git submodule sync --recursive + git submodule update cp assembly/native/build-macos-shared.sh . chmod +x build-macos-shared.sh ./build-macos-shared.sh -t -a diff --git a/.github/workflows/build-ton-macos-arm64-shared.yml b/.github/workflows/build-ton-macos-arm64-shared.yml index aadd23dd..7481f9ff 100644 --- a/.github/workflows/build-ton-macos-arm64-shared.yml +++ b/.github/workflows/build-ton-macos-arm64-shared.yml @@ -14,6 +14,8 @@ jobs: - name: Build TON run: | + git submodule sync --recursive + git submodule update cp assembly/native/build-macos-shared.sh . chmod +x build-macos-shared.sh ./build-macos-shared.sh -t -a diff --git a/.github/workflows/build-ton-macos-x86-64-shared.yml b/.github/workflows/build-ton-macos-x86-64-shared.yml index c9331e3b..6a69b2e3 100644 --- a/.github/workflows/build-ton-macos-x86-64-shared.yml +++ b/.github/workflows/build-ton-macos-x86-64-shared.yml @@ -4,7 +4,7 @@ on: [push,workflow_dispatch,workflow_call] jobs: build: - runs-on: macos-12 + runs-on: macos-13 steps: - name: Check out repository @@ -14,6 +14,8 @@ jobs: - name: Build TON run: | + git submodule sync --recursive + git submodule update cp assembly/native/build-macos-shared.sh . chmod +x build-macos-shared.sh ./build-macos-shared.sh -t -a @@ -21,5 +23,5 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@master with: - name: ton-binaries-macos-12 + name: ton-binaries-macos-13 path: artifacts diff --git a/.github/workflows/build-ton-wasm-emscripten.yml b/.github/workflows/build-ton-wasm-emscripten.yml index 534d312f..bac0cf98 100644 --- a/.github/workflows/build-ton-wasm-emscripten.yml +++ b/.github/workflows/build-ton-wasm-emscripten.yml @@ -15,16 +15,37 @@ jobs: - name: Install system libraries run: | sudo apt-get update - sudo apt-get install -y build-essential git openssl cmake ninja-build zlib1g-dev libssl-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev + sudo apt-get install -y build-essential git openssl cmake ninja-build zlib1g-dev libssl-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev - name: Build TON WASM artifacts run: | + git submodule sync --recursive + git submodule update cp assembly/wasm/fift-func-wasm-build-ubuntu.sh . chmod +x fift-func-wasm-build-ubuntu.sh ./fift-func-wasm-build-ubuntu.sh -a + - name: Prepare test + run: | + cp assembly/wasm/*.fc . + git clone https://github.com/ton-community/func-js.git + cd func-js + npm install + npm run build + npm link + + - name: Test TON WASM artifacts + run: | + base64 -w 0 artifacts/funcfiftlib.wasm > artifacts/funcfiftlib.wasm.js + printf "module.exports = { FuncFiftLibWasm: '" | cat - artifacts/funcfiftlib.wasm.js > temp.txt && mv temp.txt artifacts/funcfiftlib.wasm.js + echo "'}" >> artifacts/funcfiftlib.wasm.js + cp artifacts/funcfiftlib.wasm.js func-js/node_modules/@ton-community/func-js-bin/dist/funcfiftlib.wasm.js + cp artifacts/funcfiftlib.js func-js/node_modules/@ton-community/func-js-bin/dist/funcfiftlib.js + npx func-js stdlib.fc intrinsics.fc --fift ./output.f + + - name: Upload artifacts uses: actions/upload-artifact@master with: - name: ton-wasm-binaries + name: ton-wasm path: artifacts diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 263bd9a4..05a3db26 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -4,6 +4,9 @@ on: [workflow_dispatch] permissions: write-all +env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + jobs: create-release: runs-on: ubuntu-22.04 @@ -11,84 +14,112 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Download Linux x86-64 artifacts - uses: dawidd6/action-download-artifact@v2 + - name: Download Linux arm64 artifacts + uses: dawidd6/action-download-artifact@v6 with: - workflow: ton-x86-64-linux.yml + workflow: build-ton-linux-arm64-appimage.yml path: artifacts workflow_conclusion: success + branch: master + skip_unpack: true + + - name: Download and unzip Linux arm64 artifacts + uses: dawidd6/action-download-artifact@v6 + with: + workflow: build-ton-linux-arm64-appimage.yml + path: artifacts + workflow_conclusion: success + branch: master + skip_unpack: false + + - name: Download Linux x86-64 artifacts + uses: dawidd6/action-download-artifact@v6 + with: + workflow: build-ton-linux-x86-64-appimage.yml + path: artifacts + workflow_conclusion: success + branch: master skip_unpack: true - name: Download and unzip Linux x86-64 artifacts - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v6 with: - workflow: ton-x86-64-linux.yml + workflow: build-ton-linux-x86-64-appimage.yml path: artifacts workflow_conclusion: success + branch: master skip_unpack: false - name: Download Mac x86-64 artifacts - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v6 with: - workflow: ton-x86-64-macos.yml + workflow: build-ton-macos-13-x86-64-portable.yml path: artifacts workflow_conclusion: success + branch: master skip_unpack: true - name: Download Mac arm64 artifacts - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v6 with: - workflow: ton-arm64-macos.yml + workflow: build-ton-macos-14-arm64-portable.yml path: artifacts workflow_conclusion: success + branch: master skip_unpack: true - name: Download and unzip Mac x86-64 artifacts - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v6 with: - workflow: ton-x86-64-macos.yml + workflow: build-ton-macos-13-x86-64-portable.yml path: artifacts workflow_conclusion: success + branch: master skip_unpack: false - name: Download and unzip arm64 artifacts - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v6 with: - workflow: ton-arm64-macos.yml + workflow: build-ton-macos-14-arm64-portable.yml path: artifacts workflow_conclusion: success + branch: master skip_unpack: false - name: Download Windows artifacts - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v6 with: workflow: ton-x86-64-windows.yml path: artifacts workflow_conclusion: success + branch: master skip_unpack: true - name: Download and unzip Windows artifacts - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v6 with: workflow: ton-x86-64-windows.yml path: artifacts workflow_conclusion: success + branch: master skip_unpack: false - name: Download WASM artifacts - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v6 with: workflow: build-ton-wasm-emscripten.yml path: artifacts workflow_conclusion: success + branch: master skip_unpack: true - name: Download Android Tonlib artifacts - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v6 with: workflow: build-ton-linux-android-tonlib.yml path: artifacts workflow_conclusion: success + branch: master skip_unpack: true - name: Show all artifacts @@ -147,7 +178,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-win-binaries.zip + file: artifacts/ton-x86-64-windows.zip asset_name: ton-win-x86-64.zip tag: ${{ steps.tag.outputs.TAG }} @@ -155,7 +186,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-win-binaries/fift.exe + file: artifacts/ton-x86-64-windows/fift.exe asset_name: fift.exe tag: ${{ steps.tag.outputs.TAG }} @@ -163,23 +194,39 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-win-binaries/func.exe + file: artifacts/ton-x86-64-windows/func.exe asset_name: func.exe tag: ${{ steps.tag.outputs.TAG }} + - name: Upload Windows 2019 single artifact - tolk + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86-64-windows/tolk.exe + asset_name: tolk.exe + tag: ${{ steps.tag.outputs.TAG }} + - name: Upload Windows 2019 single artifact - lite-client uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-win-binaries/lite-client.exe + file: artifacts/ton-x86-64-windows/lite-client.exe asset_name: lite-client.exe tag: ${{ steps.tag.outputs.TAG }} + - name: Upload Windows 2019 single artifact - proxy-liteserver + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86-64-windows/proxy-liteserver.exe + asset_name: proxy-liteserver.exe + tag: ${{ steps.tag.outputs.TAG }} + - name: Upload Windows 2019 single artifact - rldp-http-proxy uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-win-binaries/rldp-http-proxy.exe + file: artifacts/ton-x86-64-windows/rldp-http-proxy.exe asset_name: rldp-http-proxy.exe tag: ${{ steps.tag.outputs.TAG }} @@ -187,7 +234,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-win-binaries/http-proxy.exe + file: artifacts/ton-x86-64-windows/http-proxy.exe asset_name: http-proxy.exe tag: ${{ steps.tag.outputs.TAG }} @@ -195,7 +242,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-win-binaries/storage-daemon-cli.exe + file: artifacts/ton-x86-64-windows/storage-daemon-cli.exe asset_name: storage-daemon-cli.exe tag: ${{ steps.tag.outputs.TAG }} @@ -203,7 +250,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-win-binaries/storage-daemon.exe + file: artifacts/ton-x86-64-windows/storage-daemon.exe asset_name: storage-daemon.exe tag: ${{ steps.tag.outputs.TAG }} @@ -211,7 +258,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-win-binaries/tonlibjson.dll + file: artifacts/ton-x86-64-windows/tonlibjson.dll asset_name: tonlibjson.dll tag: ${{ steps.tag.outputs.TAG }} @@ -219,7 +266,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-win-binaries/emulator.dll + file: artifacts/ton-x86-64-windows/emulator.dll asset_name: libemulator.dll tag: ${{ steps.tag.outputs.TAG }} @@ -227,7 +274,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-win-binaries/tonlib-cli.exe + file: artifacts/ton-x86-64-windows/tonlib-cli.exe asset_name: tonlib-cli.exe tag: ${{ steps.tag.outputs.TAG }} @@ -237,7 +284,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-x86_64-macos-binaries.zip + file: artifacts/ton-x86_64-macos.zip asset_name: ton-mac-x86-64.zip tag: ${{ steps.tag.outputs.TAG }} @@ -245,7 +292,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-x86_64-macos-binaries/fift + file: artifacts/ton-x86_64-macos/fift asset_name: fift-mac-x86-64 tag: ${{ steps.tag.outputs.TAG }} @@ -253,23 +300,39 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-x86_64-macos-binaries/func + file: artifacts/ton-x86_64-macos/func asset_name: func-mac-x86-64 tag: ${{ steps.tag.outputs.TAG }} + - name: Upload Mac x86-64 single artifact - tolk + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-macos/tolk + asset_name: tolk-mac-x86-64 + tag: ${{ steps.tag.outputs.TAG }} + - name: Upload Mac x86-64 single artifact - lite-client uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-x86_64-macos-binaries/lite-client + file: artifacts/ton-x86_64-macos/lite-client asset_name: lite-client-mac-x86-64 tag: ${{ steps.tag.outputs.TAG }} + - name: Upload Mac x86-64 single artifact - proxy-liteserver + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-macos/proxy-liteserver + asset_name: proxy-liteserver-mac-x86-64 + tag: ${{ steps.tag.outputs.TAG }} + - name: Upload Mac x86-64 single artifact - rldp-http-proxy uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-x86_64-macos-binaries/rldp-http-proxy + file: artifacts/ton-x86_64-macos/rldp-http-proxy asset_name: rldp-http-proxy-mac-x86-64 tag: ${{ steps.tag.outputs.TAG }} @@ -277,7 +340,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-x86_64-macos-binaries/http-proxy + file: artifacts/ton-x86_64-macos/http-proxy asset_name: http-proxy-mac-x86-64 tag: ${{ steps.tag.outputs.TAG }} @@ -285,7 +348,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-x86_64-macos-binaries/storage-daemon-cli + file: artifacts/ton-x86_64-macos/storage-daemon-cli asset_name: storage-daemon-cli-mac-x86-64 tag: ${{ steps.tag.outputs.TAG }} @@ -293,7 +356,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-x86_64-macos-binaries/storage-daemon + file: artifacts/ton-x86_64-macos/storage-daemon asset_name: storage-daemon-mac-x86-64 tag: ${{ steps.tag.outputs.TAG }} @@ -301,7 +364,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-x86_64-macos-binaries/libtonlibjson.dylib + file: artifacts/ton-x86_64-macos/libtonlibjson.dylib asset_name: tonlibjson-mac-x86-64.dylib tag: ${{ steps.tag.outputs.TAG }} @@ -309,7 +372,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-x86_64-macos-binaries/libemulator.dylib + file: artifacts/ton-x86_64-macos/libemulator.dylib asset_name: libemulator-mac-x86-64.dylib tag: ${{ steps.tag.outputs.TAG }} @@ -317,7 +380,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-x86_64-macos-binaries/tonlib-cli + file: artifacts/ton-x86_64-macos/tonlib-cli asset_name: tonlib-cli-mac-x86-64 tag: ${{ steps.tag.outputs.TAG }} @@ -328,7 +391,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-arm64-macos-binaries.zip + file: artifacts/ton-arm64-macos.zip asset_name: ton-mac-arm64.zip tag: ${{ steps.tag.outputs.TAG }} @@ -336,7 +399,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-arm64-macos-binaries/fift + file: artifacts/ton-arm64-macos/fift asset_name: fift-mac-arm64 tag: ${{ steps.tag.outputs.TAG }} @@ -344,23 +407,39 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-arm64-macos-binaries/func + file: artifacts/ton-arm64-macos/func asset_name: func-mac-arm64 tag: ${{ steps.tag.outputs.TAG }} + - name: Upload Mac arm64 single artifact - tolk + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos/tolk + asset_name: tolk-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + - name: Upload Mac arm64 single artifact - lite-client uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-arm64-macos-binaries/lite-client + file: artifacts/ton-arm64-macos/lite-client asset_name: lite-client-mac-arm64 tag: ${{ steps.tag.outputs.TAG }} + - name: Upload Mac arm64 single artifact - proxy-liteserver + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos/proxy-liteserver + asset_name: proxy-liteserver-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + - name: Upload Mac arm64 single artifact - rldp-http-proxy uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-arm64-macos-binaries/rldp-http-proxy + file: artifacts/ton-arm64-macos/rldp-http-proxy asset_name: rldp-http-proxy-mac-arm64 tag: ${{ steps.tag.outputs.TAG }} @@ -368,7 +447,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-arm64-macos-binaries/http-proxy + file: artifacts/ton-arm64-macos/http-proxy asset_name: http-proxy-mac-arm64 tag: ${{ steps.tag.outputs.TAG }} @@ -376,7 +455,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-arm64-macos-binaries/storage-daemon-cli + file: artifacts/ton-arm64-macos/storage-daemon-cli asset_name: storage-daemon-cli-mac-arm64 tag: ${{ steps.tag.outputs.TAG }} @@ -384,7 +463,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-arm64-macos-binaries/storage-daemon + file: artifacts/ton-arm64-macos/storage-daemon asset_name: storage-daemon-mac-arm64 tag: ${{ steps.tag.outputs.TAG }} @@ -392,7 +471,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-arm64-macos-binaries/libtonlibjson.dylib + file: artifacts/ton-arm64-macos/libtonlibjson.dylib asset_name: tonlibjson-mac-arm64.dylib tag: ${{ steps.tag.outputs.TAG }} @@ -400,7 +479,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-arm64-macos-binaries/libemulator.dylib + file: artifacts/ton-arm64-macos/libemulator.dylib asset_name: libemulator-mac-arm64.dylib tag: ${{ steps.tag.outputs.TAG }} @@ -408,7 +487,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-arm64-macos-binaries/tonlib-cli + file: artifacts/ton-arm64-macos/tonlib-cli asset_name: tonlib-cli-mac-arm64 tag: ${{ steps.tag.outputs.TAG }} @@ -418,15 +497,23 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-x86_64-linux-binaries.zip + file: artifacts/ton-x86_64-linux.zip asset_name: ton-linux-x86_64.zip tag: ${{ steps.tag.outputs.TAG }} + - name: Upload generic smartcont+lib artifact + run: | + mkdir smartcont_lib + cd smartcont_lib + cp -r ../artifacts/ton-x86_64-linux/{smartcont,lib} . + zip -r smartcont_lib.zip . + gh release upload ${{ steps.tag.outputs.TAG }} smartcont_lib.zip + - name: Upload Linux x86-64 single artifact - fift uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-x86_64-linux-binaries/fift + file: artifacts/ton-x86_64-linux/fift asset_name: fift-linux-x86_64 tag: ${{ steps.tag.outputs.TAG }} @@ -434,23 +521,39 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-x86_64-linux-binaries/func + file: artifacts/ton-x86_64-linux/func asset_name: func-linux-x86_64 tag: ${{ steps.tag.outputs.TAG }} + - name: Upload Linux x86-64 single artifact - tolk + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-linux/tolk + asset_name: tolk-linux-x86_64 + tag: ${{ steps.tag.outputs.TAG }} + - name: Upload Linux x86-64 single artifact - lite-client uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-x86_64-linux-binaries/lite-client + file: artifacts/ton-x86_64-linux/lite-client asset_name: lite-client-linux-x86_64 tag: ${{ steps.tag.outputs.TAG }} + - name: Upload Linux x86-64 single artifact - proxy-liteserver + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-linux/proxy-liteserver + asset_name: proxy-liteserver-linux-x86_64 + tag: ${{ steps.tag.outputs.TAG }} + - name: Upload Linux x86-64 single artifact - rldp-http-proxy uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-x86_64-linux-binaries/rldp-http-proxy + file: artifacts/ton-x86_64-linux/rldp-http-proxy asset_name: rldp-http-proxy-linux-x86_64 tag: ${{ steps.tag.outputs.TAG }} @@ -458,7 +561,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-x86_64-linux-binaries/http-proxy + file: artifacts/ton-x86_64-linux/http-proxy asset_name: http-proxy-linux-x86_64 tag: ${{ steps.tag.outputs.TAG }} @@ -466,7 +569,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-x86_64-linux-binaries/storage-daemon-cli + file: artifacts/ton-x86_64-linux/storage-daemon-cli asset_name: storage-daemon-cli-linux-x86_64 tag: ${{ steps.tag.outputs.TAG }} @@ -474,7 +577,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-x86_64-linux-binaries/storage-daemon + file: artifacts/ton-x86_64-linux/storage-daemon asset_name: storage-daemon-linux-x86_64 tag: ${{ steps.tag.outputs.TAG }} @@ -482,7 +585,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-x86_64-linux-binaries/libtonlibjson.so + file: artifacts/ton-x86_64-linux/libtonlibjson.so asset_name: tonlibjson-linux-x86_64.so tag: ${{ steps.tag.outputs.TAG }} @@ -490,7 +593,7 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-x86_64-linux-binaries/libemulator.so + file: artifacts/ton-x86_64-linux/libemulator.so asset_name: libemulator-linux-x86_64.so tag: ${{ steps.tag.outputs.TAG }} @@ -498,16 +601,124 @@ jobs: uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-x86_64-linux-binaries/tonlib-cli + file: artifacts/ton-x86_64-linux/tonlib-cli asset_name: tonlib-cli-linux-x86_64 tag: ${{ steps.tag.outputs.TAG }} + + # linux arm64 + + - name: Upload Linux arm64 artifacts + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux.zip + asset_name: ton-linux-arm64.zip + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - fift + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/fift + asset_name: fift-linux-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - func + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/func + asset_name: func-linux-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - tolk + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/tolk + asset_name: tolk-linux-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - lite-client + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/lite-client + asset_name: lite-client-linux-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - proxy-liteserver + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/proxy-liteserver + asset_name: proxy-liteserver-linux-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - rldp-http-proxy + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/rldp-http-proxy + asset_name: rldp-http-proxy-linux-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - http-proxy + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/http-proxy + asset_name: http-proxy-linux-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - storage-daemon-cli + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/storage-daemon-cli + asset_name: storage-daemon-cli-linux-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - storage-daemon + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/storage-daemon + asset_name: storage-daemon-linux-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - tonlibjson + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/libtonlibjson.so + asset_name: tonlibjson-linux-arm64.so + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - libemulator + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/libemulator.so + asset_name: libemulator-linux-arm64.so + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Linux arm64 single artifact - tonlib-cli + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/tonlib-cli + asset_name: tonlib-cli-linux-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload WASM artifacts uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: artifacts/ton-wasm-binaries.zip - asset_name: ton-wasm-binaries.zip + file: artifacts/ton-wasm.zip + asset_name: ton-wasm.zip tag: ${{ steps.tag.outputs.TAG }} - name: Upload Android Tonlib artifacts diff --git a/.github/workflows/create-tolk-release.yml b/.github/workflows/create-tolk-release.yml new file mode 100644 index 00000000..fb8438a1 --- /dev/null +++ b/.github/workflows/create-tolk-release.yml @@ -0,0 +1,154 @@ +name: Create tolk release + +on: + workflow_dispatch: + inputs: + tag: + description: 'tolk release and tag name' + required: true + +permissions: write-all + +jobs: + create-release: + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v3 + + - name: Download and unzip Linux arm64 artifacts + uses: dawidd6/action-download-artifact@v6 + with: + workflow: build-ton-linux-arm64-appimage.yml + path: artifacts + workflow_conclusion: success + branch: master + skip_unpack: false + + - name: Download and unzip Linux x86-64 artifacts + uses: dawidd6/action-download-artifact@v6 + with: + workflow: build-ton-linux-x86-64-appimage.yml + path: artifacts + workflow_conclusion: success + branch: master + skip_unpack: false + + - name: Download and unzip Mac x86-64 artifacts + uses: dawidd6/action-download-artifact@v6 + with: + workflow: build-ton-macos-13-x86-64-portable.yml + path: artifacts + workflow_conclusion: success + branch: master + skip_unpack: false + + - name: Download and unzip arm64 artifacts + uses: dawidd6/action-download-artifact@v6 + with: + workflow: build-ton-macos-14-arm64-portable.yml + path: artifacts + workflow_conclusion: success + branch: master + skip_unpack: false + + - name: Download and unzip Windows artifacts + uses: dawidd6/action-download-artifact@v6 + with: + workflow: ton-x86-64-windows.yml + path: artifacts + workflow_conclusion: success + branch: master + skip_unpack: false + + - name: Download WASM artifacts + uses: dawidd6/action-download-artifact@v6 + with: + workflow: build-ton-wasm-emscripten.yml + path: artifacts + workflow_conclusion: success + branch: master + skip_unpack: true + + - name: Show all artifacts + run: | + tree artifacts + + + # create release + - name: Get registration token + id: getRegToken + run: | + curl -X POST -H \"Accept: application/vnd.github+json\" -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' https://api.github.com/repos/ton-blockchain/ton/actions/runners/registration-token + + - name: Create release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ inputs.tag }} + release_name: ${{ inputs.tag }} + draft: false + prerelease: false + + # upload + + # win + + - name: Upload Windows 2019 single artifact - tolk + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86-64-windows/tolk.exe + asset_name: tolk.exe + tag: ${{ inputs.tag }} + + # mac x86-64 + + - name: Upload Mac x86-64 single artifact - tolk + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-macos/tolk + asset_name: tolk-mac-x86-64 + tag: ${{ inputs.tag }} + + # mac arm64 + + - name: Upload Mac arm64 single artifact - tolk + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos/tolk + asset_name: tolk-mac-arm64 + tag: ${{ inputs.tag }} + + # linux x86-64 + + - name: Upload Linux x86-64 single artifact - tolk + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-x86_64-linux/tolk + asset_name: tolk-linux-x86_64 + tag: ${{ inputs.tag }} + + # linux arm64 + + - name: Upload Linux arm64 single artifact - tolk + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-linux/tolk + asset_name: tolk-linux-arm64 + tag: ${{ inputs.tag }} + + - name: Upload WASM artifacts + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-wasm.zip + asset_name: ton-wasm.zip + tag: ${{ inputs.tag }} + diff --git a/.github/workflows/docker-ubuntu-branch-image.yml b/.github/workflows/docker-ubuntu-branch-image.yml index d749afa2..00aa5015 100644 --- a/.github/workflows/docker-ubuntu-branch-image.yml +++ b/.github/workflows/docker-ubuntu-branch-image.yml @@ -20,10 +20,12 @@ jobs: submodules: 'recursive' - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v3.5.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v3.10.0 + with: + driver-opts: image=moby/buildkit:v0.11.0 - name: Login to GitHub Container Registry uses: docker/login-action@v3 @@ -32,6 +34,17 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and export to Docker + uses: docker/build-push-action@v6 + with: + load: true + context: ./ + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test + + - name: Test + run: | + docker run --rm -e "TEST=1" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test + - name: Get tag as branch name id: tag run: | @@ -41,7 +54,7 @@ jobs: id: docker_build uses: docker/build-push-action@v6 with: - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 push: true context: ./ tags: | diff --git a/.github/workflows/docker-ubuntu-image.yml b/.github/workflows/docker-ubuntu-image.yml index 48c553ef..aa4eaeef 100644 --- a/.github/workflows/docker-ubuntu-image.yml +++ b/.github/workflows/docker-ubuntu-image.yml @@ -20,10 +20,10 @@ jobs: submodules: 'recursive' - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - + uses: docker/setup-qemu-action@v3.5.0 + - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v3.10.0 - name: Login to GitHub Container Registry uses: docker/login-action@v3 diff --git a/.github/workflows/ton-arm64-macos.yml b/.github/workflows/ton-arm64-macos.yml deleted file mode 100644 index 9e8302e8..00000000 --- a/.github/workflows/ton-arm64-macos.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: MacOS TON build (portable, arm64) - -on: [push,workflow_dispatch,workflow_call] - -jobs: - build: - runs-on: macos-14 - - steps: - - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - - uses: cachix/install-nix-action@v23 - with: - extra_nix_config: | - access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - - - name: Build TON - run: | - cp assembly/nix/build-macos-nix.sh . - chmod +x build-macos-nix.sh - ./build-macos-nix.sh -t - - - name: Simple binaries test - run: | - sudo mv /nix/store /nix/store2 - artifacts/validator-engine -V - artifacts/lite-client -V - artifacts/fift -V - artifacts/func -V - - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-arm64-macos-binaries - path: artifacts diff --git a/.github/workflows/ton-x86-64-linux.yml b/.github/workflows/ton-x86-64-linux.yml deleted file mode 100644 index abbe1cca..00000000 --- a/.github/workflows/ton-x86-64-linux.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Ubuntu TON build (portable, x86-64) - -on: [push,workflow_dispatch,workflow_call] - -jobs: - build: - runs-on: ubuntu-22.04 - - steps: - - run: | - sudo apt update - sudo apt install -y apt-utils - - - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - - uses: cachix/install-nix-action@v23 - with: - extra_nix_config: | - access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - - - name: Build TON - run: | - cp assembly/nix/build-linux-x86-64-nix.sh . - chmod +x build-linux-x86-64-nix.sh - ./build-linux-x86-64-nix.sh -t - - - name: Simple binaries test - run: | - sudo mv /nix/store /nix/store2 - artifacts/validator-engine -V - artifacts/lite-client -V - artifacts/fift -V - artifacts/func -V - - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-x86_64-linux-binaries - path: artifacts diff --git a/.github/workflows/ton-x86-64-macos.yml b/.github/workflows/ton-x86-64-macos.yml deleted file mode 100644 index 8c71f34a..00000000 --- a/.github/workflows/ton-x86-64-macos.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: MacOS TON build (portable, x86-64) - -on: [push,workflow_dispatch,workflow_call] - -jobs: - build: - runs-on: macos-12 - - steps: - - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - - uses: cachix/install-nix-action@v23 - with: - extra_nix_config: | - access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - - - name: Build TON - run: | - cp assembly/nix/build-macos-nix.sh . - chmod +x build-macos-nix.sh - ./build-macos-nix.sh -t - - - name: Simple binaries test - run: | - sudo mv /nix/store /nix/store2 - artifacts/validator-engine -V - artifacts/lite-client -V - artifacts/fift -V - artifacts/func -V - - - name: Upload artifacts - uses: actions/upload-artifact@master - with: - name: ton-x86_64-macos-binaries - path: artifacts diff --git a/.github/workflows/ton-x86-64-windows.yml b/.github/workflows/ton-x86-64-windows.yml index d5c9c243..baaad778 100644 --- a/.github/workflows/ton-x86-64-windows.yml +++ b/.github/workflows/ton-x86-64-windows.yml @@ -23,6 +23,8 @@ jobs: - name: Build TON run: | + git submodule sync --recursive + git submodule update copy assembly\native\build-windows-github-2019.bat . copy assembly\native\build-windows-2019.bat . build-windows-github-2019.bat Enterprise @@ -30,5 +32,5 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@master with: - name: ton-win-binaries + name: ton-x86-64-windows path: artifacts diff --git a/.gitignore b/.gitignore index 536918ab..e5bb366c 100644 --- a/.gitignore +++ b/.gitignore @@ -13,11 +13,12 @@ test/regression-tests.cache/ **/*build*/ .idea .vscode +.DS_Store +dev/ zlib/ libsodium/ libmicrohttpd-0.9.77-w32-bin/ readline-5.0-1-lib/ -secp256k1/ openssl-3.1.4/ libsodium-1.0.18-stable-msvc.zip libmicrohttpd-0.9.77-w32-bin.zip diff --git a/.gitmodules b/.gitmodules index f201ed73..325e3f4e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,7 @@ [submodule "third-party/blst"] path = third-party/blst url = https://github.com/supranational/blst.git +[submodule "third-party/secp256k1"] + path = third-party/secp256k1 + url = https://github.com/bitcoin-core/secp256k1 + branch = v0.3.2 diff --git a/CMake/BuildSECP256K1.cmake b/CMake/BuildSECP256K1.cmake new file mode 100644 index 00000000..f8b3c8ca --- /dev/null +++ b/CMake/BuildSECP256K1.cmake @@ -0,0 +1,55 @@ +if (NOT SECP256K1_LIBRARY) + + set(SECP256K1_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third-party/secp256k1) + set(SECP256K1_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/secp256k1) + set(SECP256K1_INCLUDE_DIR ${SECP256K1_BINARY_DIR}/include) + + file(MAKE_DIRECTORY ${SECP256K1_BINARY_DIR}) + file(MAKE_DIRECTORY "${SECP256K1_BINARY_DIR}/include") + + if (MSVC) + set(SECP256K1_BINARY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third-party/secp256k1) + set(SECP256K1_LIBRARY ${SECP256K1_SOURCE_DIR}/build/src/Release/libsecp256k1.lib) + set(SECP256K1_INCLUDE_DIR ${SECP256K1_BINARY_DIR}/include) + add_custom_command( + WORKING_DIRECTORY ${SECP256K1_SOURCE_DIR} + COMMAND cmake -E env CFLAGS="/WX" cmake -A x64 -B build -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DSECP256K1_ENABLE_MODULE_EXTRAKEYS=ON -DSECP256K1_BUILD_EXAMPLES=OFF -DBUILD_SHARED_LIBS=OFF + COMMAND cmake --build build --config Release + COMMENT "Build Secp256k1" + DEPENDS ${SECP256K1_SOURCE_DIR} + OUTPUT ${SECP256K1_LIBRARY} + ) + elseif (EMSCRIPTEN) + set(SECP256K1_BINARY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third-party/secp256k1) + set(SECP256K1_LIBRARY ${SECP256K1_BINARY_DIR}/.libs/libsecp256k1.a) + set(SECP256K1_INCLUDE_DIR ${SECP256K1_SOURCE_DIR}/include) + add_custom_command( + WORKING_DIRECTORY ${SECP256K1_SOURCE_DIR} + COMMAND ./autogen.sh + COMMAND emconfigure ./configure --enable-module-recovery --enable-module-extrakeys --disable-tests --disable-benchmark + COMMAND emmake make clean + COMMAND emmake make + COMMENT "Build Secp256k1 with emscripten" + DEPENDS ${SECP256K1_SOURCE_DIR} + OUTPUT ${SECP256K1_LIBRARY} + ) + else() + if (NOT NIX) + set(SECP256K1_LIBRARY ${SECP256K1_BINARY_DIR}/lib/libsecp256k1.a) + add_custom_command( + WORKING_DIRECTORY ${SECP256K1_SOURCE_DIR} + COMMAND ./autogen.sh + COMMAND ./configure -q --disable-option-checking --enable-module-recovery --enable-module-extrakeys --prefix ${SECP256K1_BINARY_DIR} --with-pic --disable-shared --enable-static --disable-tests --disable-benchmark + COMMAND make -j16 + COMMAND make install + COMMENT "Build secp256k1" + DEPENDS ${SECP256K1_SOURCE_DIR} + OUTPUT ${SECP256K1_LIBRARY} + ) + endif() + endif() +else() + message(STATUS "Use Secp256k1: ${SECP256K1_LIBRARY}") +endif() + +add_custom_target(secp256k1 DEPENDS ${SECP256K1_LIBRARY}) diff --git a/CMake/FindSecp256k1.cmake b/CMake/FindSecp256k1.cmake index 68a37c71..1f776796 100644 --- a/CMake/FindSecp256k1.cmake +++ b/CMake/FindSecp256k1.cmake @@ -1,9 +1,8 @@ -# - Try to find SECP256K1 +# - Try to find Secp256k1 # Once done this will define # -# SECP256K1_FOUND - system has SECP256K1 -# SECP256K1_INCLUDE_DIR - the SECP256K1 include directory -# SECP256K1_LIBRARY - Link these to use SECP256K1 +# SECP256K1_INCLUDE_DIR - the Secp256k1 include directory +# SECP256K1_LIBRARY - Link these to use Secp256k1 if (NOT SECP256K1_LIBRARY) find_path( diff --git a/CMakeLists.txt b/CMakeLists.txt index 573bc3a3..cea3fc7e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,6 +84,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED TRUE) set(CMAKE_CXX_EXTENSIONS FALSE) #BEGIN internal +option(BUILD_SHARED_LIBS "Use \"ON\" to build shared libraries instead of static where it's not specified (not recommended)" OFF) option(USE_EMSCRIPTEN "Use \"ON\" for config building wasm." OFF) option(TON_ONLY_TONLIB "Use \"ON\" to build only tonlib." OFF) if (USE_EMSCRIPTEN) @@ -183,6 +184,7 @@ message("Add ton") set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH}) include(BuildBLST) +include(BuildSECP256K1) # Configure CCache if available find_program(CCACHE_FOUND ccache) @@ -237,6 +239,11 @@ if (TON_USE_JEMALLOC) find_package(jemalloc REQUIRED) endif() +if (NIX) + find_package(Secp256k1 REQUIRED) +endif() + + set(MEMPROF "" CACHE STRING "Use one of \"ON\", \"FAST\" or \"SAFE\" to enable memory profiling. \ Works under macOS and Linux when compiled using glibc. \ In FAST mode stack is unwinded only using frame pointers, which may fail. \ @@ -413,6 +420,7 @@ add_subdirectory(adnl) add_subdirectory(crypto) add_subdirectory(lite-client) add_subdirectory(emulator) +add_subdirectory(tolk) #BEGIN tonlib add_subdirectory(tonlib) @@ -626,6 +634,30 @@ if (NOT NIX) endif() endif() +# Tolk tests +if (NOT NIX) + if (MSVC) + set(PYTHON_VER "python") + else() + set(PYTHON_VER "python3") + endif() + add_test( + NAME test-tolk + COMMAND ${PYTHON_VER} tolk-tester.py tests/ + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tolk-tester) + if (WIN32) + set_property(TEST test-tolk PROPERTY ENVIRONMENT + "TOLK_EXECUTABLE=${CMAKE_CURRENT_BINARY_DIR}/tolk/tolk.exe" + "FIFT_EXECUTABLE=${CMAKE_CURRENT_BINARY_DIR}/crypto/fift.exe" + "FIFTPATH=${CMAKE_CURRENT_SOURCE_DIR}/crypto/fift/lib/") + else() + set_property(TEST test-tolk PROPERTY ENVIRONMENT + "TOLK_EXECUTABLE=${CMAKE_CURRENT_BINARY_DIR}/tolk/tolk" + "FIFT_EXECUTABLE=${CMAKE_CURRENT_BINARY_DIR}/crypto/fift" + "FIFTPATH=${CMAKE_CURRENT_SOURCE_DIR}/crypto/fift/lib/") + endif() +endif() + #BEGIN internal if (NOT TON_ONLY_TONLIB) add_test(test-adnl test-adnl) diff --git a/Changelog.md b/Changelog.md index c8a7ea25..4dce39fc 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,44 @@ +## 2025.03 Update +1. New extracurrency behavior introduced, check [GlobalVersions.md](./doc/GlobalVersions.md#version-10) +2. Optmization of validation process, in particular CellStorageStat. +3. Flag for speeding up broadcasts in various overlays. +4. Fixes for static builds for emulator and tonlibjson +5. Improving getstats output: adds + * Liteserver queries count + * Collated/validated blocks count, number of active sessions + * Persistent state sizes + * Initial sync progress +6. Fixes in logging, TON Storage, external message checking, persistent state downloading, UB in tonlib + +Besides the work of the core team, this update is based on the efforts of @Sild from StonFi(UB in tonlib). + +## 2025.02 Update +1. Series of improvement/fixes for `Config8.version >= 9`, check [GlobalVersions.md](./doc/GlobalVersions.md) +2. Fix for better discovery of updated nodes' (validators') IPs: retry dht queries +3. Series of improvements for extra currency adoption: fixed c7 in rungetmethod, reserve modes +4. TVM: Fix processing continuation control data on deep jump +5. A few fixes of tl-b schemes: crc computation, incorrect tag for merkle proofs, advance_ext, NatWidth print +6. Emulator improvements: fix setting libraries, extracurrency support +7. Increase of gas limit for unlocking highload-v2 wallets locked in the beginning of 2024 +8. Validator console improvement: dashed names, better shard formats + + +Besides the work of the core team, this update is based on the efforts of @dbaranovstonfi from StonFi(libraries in emulator), @Rexagon (ret on deep jumps), @tvorogme from DTon (`advance_ext`), Nan from Zellic (`stk_und` and JNI) + +## 2024.12 Update + +1. FunC 0.4.6: Fix in try/catch handling, fixing pure flag for functions stored in variables +2. Merging parts of Accelerator: support of specific shard monitoring, archive/liteserver slice format, support for partial liteservers, proxy liteserver, on-demand neighbour queue loading +3. Fix of asynchronous cell loading +4. Various improvements: caching certificates checks, better block overloading detection, `_malloc` in emulator +5. Introduction of telemetry in overlays +6. Use non-null local-id for tonlib-LS interaction - mitigates MitM attack. +7. Adding `SECP256K1_XONLY_PUBKEY_TWEAK_ADD`, `SETCONTCTRMANY` instructions to TVM (activated by `Config8.version >= 9`) +8. Private keys export via validator-engine-console - required for better backups +9. Fix proof checking in tonlib, `hash` in `raw.Message` in tonlib_api + +Besides the work of the core team, this update is based on the efforts of OtterSec and LayerZero (FunC), tg:@throwunless (FunC), Aviv Frenkel and Dima Kogan from Fordefi (LS MitM), @hacker-volodya (Tonlib), OKX team (async cell loading), @krigga (emulator) + ## 2024.10 Update 1. Parallel write to celldb: substantial improvement of sync and GC speed, especially with slow disks. @@ -17,7 +58,7 @@ Besides the work of the core team, this update is based on the efforts of @krigg ## 2024.08 Update 1. Introduction of dispatch queues, message envelopes with transaction chain metadata, and explicitly stored msg_queue size, which will be activated by `Config8.version >= 8` and new `Config8.capabilities` bits: `capStoreOutMsgQueueSize`, `capMsgMetadata`, `capDeferMessages`. -2. A number of changes to transcation executor which will activated for `Config8.version >= 8`: +2. A number of changes to transaction executor which will activated for `Config8.version >= 8`: - Check mode on invalid `action_send_msg`. Ignore action if `IGNORE_ERROR` (+2) bit is set, bounce if `BOUNCE_ON_FAIL` (+16) bit is set. - Slightly change random seed generation to fix mix of `addr_rewrite` and `addr`. - Fill in `skipped_actions` for both invalid and valid messages with `IGNORE_ERROR` mode that can't be sent. @@ -89,7 +130,7 @@ Besides the work of the core team, this update is based on the efforts of @akifo * Fix error in proof generation for blocks after merge * Fix most of `block is not applied` issues related to sending too recent block in Proofs * LS now check external messages till `accept_message` (`set_gas`). -3. Improvements in DHT work and storage, CellDb, config.json ammendment, peer misbehavior detection, validator session stats collection, emulator. +3. Improvements in DHT work and storage, CellDb, config.json amendment, peer misbehavior detection, validator session stats collection, emulator. 4. Change in CTOS and XLOAD behavior activated through setting `version >= 5` in `ConfigParam 8;`: * Loading "nested libraries" (i.e. a library cell that points to another library cell) throws an exception. * Loading a library consumes gas for cell load only once (for the library cell), not twice (both for the library cell and the cell in the library). @@ -100,7 +141,7 @@ Besides the work of the Core team, this update is based on the efforts of @XaBbl ## 2023.12 Update 1. Optimized message queue handling, now queue cleaning speed doesn't depend on total queue size - * Cleaning delivered messages using lt augmentation instead of random search / consequtive walk + * Cleaning delivered messages using lt augmentation instead of random search / consecutive walk * Keeping root cell of queue message in memory until outdated (caching) 2. Changes to block collation/validation limits 3. Stop accepting new external message if message queue is overloaded @@ -192,7 +233,7 @@ Besides the work of the core team, this update is based on the efforts of @vtama Besides the work of the core team, this update is based on the efforts of @tvorogme (debug improvements), @AlexeyFSL (WASM builds) and third-party security auditors. ## 2022.08 Update -* Blockchain state serialization now works via separate db-handler which simplfies memory clearing after serialization +* Blockchain state serialization now works via separate db-handler which simplifies memory clearing after serialization * CellDB now works asynchronously which substantially increase database access throughput * Abseil-cpp and crc32 updated: solve issues with compilation on recent OS distributives * Fixed a series of UBs and issues for exotic endianness hosts diff --git a/Dockerfile b/Dockerfile index f4ea4375..f1b836bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,13 @@ FROM ubuntu:22.04 AS builder +ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool libjemalloc-dev lsb-release software-properties-common gnupg + rm /var/lib/dpkg/info/libc-bin.* && \ + apt-get clean && \ + apt-get update && \ + apt install libc-bin && \ + apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git \ + ninja-build libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool \ + libjemalloc-dev lsb-release software-properties-common gnupg RUN wget https://apt.llvm.org/llvm.sh && \ chmod +x llvm.sh && \ @@ -20,14 +27,18 @@ COPY ./ ./ RUN mkdir build && \ cd build && \ cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DTON_USE_JEMALLOC=ON .. && \ - ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console generate-random-id dht-server lite-client + ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console \ + generate-random-id dht-server lite-client tolk rldp-http-proxy dht-server proxy-liteserver create-state \ + blockchain-explorer emulator tonlibjson http-proxy adnl-proxy FROM ubuntu:22.04 +ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ - apt-get install -y wget curl libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev libjemalloc-dev htop net-tools netcat iptraf-ng jq tcpdump pv plzip && \ + apt-get install -y wget curl libatomic1 openssl libsodium-dev libmicrohttpd-dev liblz4-dev libjemalloc-dev htop \ + net-tools netcat iptraf-ng jq tcpdump pv plzip && \ rm -rf /var/lib/apt/lists/* -RUN mkdir -p /var/ton-work/db /var/ton-work/scripts /usr/share/ton/smartcont/ /usr/lib/fift/ +RUN mkdir -p /var/ton-work/db /var/ton-work/scripts /usr/share/ton/smartcont/auto /usr/lib/fift/ COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon /usr/local/bin/ COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon-cli /usr/local/bin/ @@ -35,9 +46,20 @@ COPY --from=builder /ton/build/lite-client/lite-client /usr/local/bin/ COPY --from=builder /ton/build/validator-engine/validator-engine /usr/local/bin/ COPY --from=builder /ton/build/validator-engine-console/validator-engine-console /usr/local/bin/ COPY --from=builder /ton/build/utils/generate-random-id /usr/local/bin/ +COPY --from=builder /ton/build/blockchain-explorer/blockchain-explorer /usr/local/bin/ +COPY --from=builder /ton/build/crypto/create-state /usr/local/bin/ +COPY --from=builder /ton/build/utils/proxy-liteserver /usr/local/bin/ +COPY --from=builder /ton/build/dht-server/dht-server /usr/local/bin/ +COPY --from=builder /ton/build/rldp-http-proxy/rldp-http-proxy /usr/local/bin/ +COPY --from=builder /ton/build/http/http-proxy /usr/local/bin/ +COPY --from=builder /ton/build/adnl/adnl-proxy /usr/local/bin/ +COPY --from=builder /ton/build/tonlib/libtonlibjson.so /usr/local/bin/ +COPY --from=builder /ton/build/emulator/libemulator.so /usr/local/bin/ +COPY --from=builder /ton/build/tolk/tolk /usr/local/bin/ COPY --from=builder /ton/build/crypto/fift /usr/local/bin/ COPY --from=builder /ton/build/crypto/func /usr/local/bin/ COPY --from=builder /ton/crypto/smartcont/* /usr/share/ton/smartcont/ +COPY --from=builder /ton/crypto/smartcont/auto/* /usr/share/ton/smartcont/auto/ COPY --from=builder /ton/crypto/fift/lib/* /usr/lib/fift/ WORKDIR /var/ton-work/db diff --git a/README.md b/README.md index 96516d44..897ba809 100644 --- a/README.md +++ b/README.md @@ -47,9 +47,9 @@ Main TON monorepo, which includes the code of the node/validator, lite-client, t __The Open Network (TON)__ is a fast, secure, scalable blockchain focused on handling _millions of transactions per second_ (TPS) with the goal of reaching hundreds of millions of blockchain users. - To learn more about different aspects of TON blockchain and its underlying ecosystem check [documentation](https://ton.org/docs) - To run node, validator or lite-server check [Participate section](https://ton.org/docs/participate/nodes/run-node) -- To develop decentralised apps check [Tutorials](https://ton.org/docs/develop/smart-contracts/), [FunC docs](https://ton.org/docs/develop/func/overview) and [DApp tutorials](https://ton.org/docs/develop/dapps/) +- To develop decentralised apps check [Tutorials](https://docs.ton.org/v3/guidelines/smart-contracts/guidelines), [FunC docs](https://ton.org/docs/develop/func/overview) and [DApp tutorials](https://docs.ton.org/v3/guidelines/dapps/overview) - To work on TON check [wallets](https://ton.app/wallets), [explorers](https://ton.app/explorers), [DEXes](https://ton.app/dex) and [utilities](https://ton.app/utilities) -- To interact with TON check [APIs](https://ton.org/docs/develop/dapps/apis/) +- To interact with TON check [APIs](https://docs.ton.org/v3/guidelines/dapps/apis-sdks/overview) ## Updates flow @@ -71,7 +71,7 @@ Usually, the response to your pull request will indicate which section it falls ## Build TON blockchain -### Ubuntu 20.4, 22.04 (x86-64, aarch64) +### Ubuntu 20.4, 22.04, 24.04 (x86-64, aarch64) Install additional system libraries ```bash sudo apt-get update @@ -141,18 +141,10 @@ Compile TON tonlib library ./build-android-tonlib.sh ``` -### Build TON portable binaries with Nix package manager -You need to install Nix first. -```bash - sh <(curl -L https://nixos.org/nix/install) --daemon -``` -Then compile TON with Nix by executing below command from the root folder: -```bash - cp -r assembly/nix/* . - export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz - nix-build linux-x86-64-static.nix -``` -More examples for other platforms can be found under `assembly/nix`. +### TON portable binaries + +Linux portable binaries are wrapped into AppImages, at the same time MacOS portable binaries are statically linked executables. +Linux and MacOS binaries are available for both x86-64 and arm64 architectures. ## Running tests diff --git a/adnl/adnl-ext-client.hpp b/adnl/adnl-ext-client.hpp index 13339725..1dd7d2ba 100644 --- a/adnl/adnl-ext-client.hpp +++ b/adnl/adnl-ext-client.hpp @@ -43,7 +43,10 @@ class AdnlOutboundConnection : public AdnlExtConnection { public: AdnlOutboundConnection(td::SocketFd fd, std::unique_ptr callback, AdnlNodeIdFull dst, td::actor::ActorId ext_client) - : AdnlExtConnection(std::move(fd), std::move(callback), true), dst_(std::move(dst)), ext_client_(ext_client) { + : AdnlExtConnection(std::move(fd), std::move(callback), true) + , dst_(std::move(dst)) + , local_id_(privkeys::Ed25519::random()) + , ext_client_(ext_client) { } AdnlOutboundConnection(td::SocketFd fd, std::unique_ptr callback, AdnlNodeIdFull dst, PrivateKey local_id, td::actor::ActorId ext_client) diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index 7f5c6039..ab460058 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -119,6 +119,7 @@ void AdnlPeerPairImpl::discover() { void AdnlPeerPairImpl::receive_packet_checked(AdnlPacket packet) { last_received_packet_ = td::Timestamp::now(); try_reinit_at_ = td::Timestamp::never(); + drop_addr_list_at_ = td::Timestamp::never(); request_reverse_ping_after_ = td::Timestamp::in(15.0); auto d = Adnl::adnl_start_time(); if (packet.dst_reinit_date() > d) { @@ -415,6 +416,9 @@ void AdnlPeerPairImpl::send_packet_continue(AdnlPacket packet, td::actor::ActorI if (!try_reinit_at_ && last_received_packet_ < td::Timestamp::in(-5.0)) { try_reinit_at_ = td::Timestamp::in(10.0); } + if (!drop_addr_list_at_ && last_received_packet_ < td::Timestamp::in(-60.0 * 9.0)) { + drop_addr_list_at_ = td::Timestamp::in(60.0); + } packet.run_basic_checks().ensure(); auto B = serialize_tl_object(packet.tl(), true); if (via_channel) { @@ -692,6 +696,16 @@ void AdnlPeerPairImpl::reinit(td::int32 date) { } td::Result, bool>> AdnlPeerPairImpl::get_conn() { + if (drop_addr_list_at_ && drop_addr_list_at_.is_in_past()) { + drop_addr_list_at_ = td::Timestamp::never(); + priority_addr_list_ = AdnlAddressList{}; + priority_conns_.clear(); + addr_list_ = AdnlAddressList{}; + conns_.clear(); + has_reverse_addr_ = false; + return td::Status::Error(ErrorCode::notready, "no active connections"); + } + if (!priority_addr_list_.empty() && priority_addr_list_.expire_at() < td::Clocks::system()) { priority_addr_list_ = AdnlAddressList{}; priority_conns_.clear(); diff --git a/adnl/adnl-peer.hpp b/adnl/adnl-peer.hpp index 7db2e2a1..243974ba 100644 --- a/adnl/adnl-peer.hpp +++ b/adnl/adnl-peer.hpp @@ -266,6 +266,7 @@ class AdnlPeerPairImpl : public AdnlPeerPair { td::Timestamp last_received_packet_ = td::Timestamp::never(); td::Timestamp try_reinit_at_ = td::Timestamp::never(); + td::Timestamp drop_addr_list_at_ = td::Timestamp::never(); bool has_reverse_addr_ = false; td::Timestamp request_reverse_ping_after_ = td::Timestamp::now(); diff --git a/assembly/android/build-android-tonlib.sh b/assembly/android/build-android-tonlib.sh index e470f602..29dbffc8 100644 --- a/assembly/android/build-android-tonlib.sh +++ b/assembly/android/build-android-tonlib.sh @@ -8,14 +8,19 @@ while getopts 'a' flag; do esac done +export CC=$(which clang-16) +export CXX=$(which clang++-16) +export CCACHE_DISABLE=1 + if [ ! -d android-ndk-r25b ]; then rm android-ndk-r25b-linux.zip + echo "Downloading https://dl.google.com/android/repository/android-ndk-r25b-linux.zip" wget -q https://dl.google.com/android/repository/android-ndk-r25b-linux.zip unzip -q android-ndk-r25b-linux.zip test $? -eq 0 || { echo "Can't unzip android-ndk-r25b-linux.zip"; exit 1; } - echo Android NDK extracted + echo "Android NDK extracted" else - echo Using extracted Android NDK + echo "Using extracted Android NDK" fi export JAVA_AWT_LIBRARY=NotNeeded diff --git a/assembly/appimage/AppRun b/assembly/appimage/AppRun new file mode 100644 index 00000000..c7f147b3 --- /dev/null +++ b/assembly/appimage/AppRun @@ -0,0 +1,3 @@ +#!/bin/sh +export LD_LIBRARY_PATH="${APPDIR}/usr/lib:${LD_LIBRARY_PATH}" +exec "$(dirname $0)"/usr/bin/app "$@" diff --git a/assembly/appimage/create-appimages.sh b/assembly/appimage/create-appimages.sh new file mode 100644 index 00000000..2a8cd0ec --- /dev/null +++ b/assembly/appimage/create-appimages.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +if [ ! -d "artifacts" ]; then + echo "No artifacts found." + exit 2 +fi +# x86_64 or aarch64 +ARCH=$1 + +rm -rf appimages + +mkdir -p appimages/artifacts + +wget -nc https://github.com/AppImage/appimagetool/releases/download/continuous/appimagetool-$ARCH.AppImage +chmod +x ./appimagetool-$ARCH.AppImage + +cd appimages +for file in ../artifacts/*; do + if [[ -f "$file" && "$file" != *.so ]]; then + appName=$(basename "$file") + echo $appName + # prepare AppDir + mkdir -p $appName.AppDir/usr/{bin,lib} + cp ../AppRun $appName.AppDir/AppRun + sed -i "s/app/$appName/g" $appName.AppDir/AppRun + chmod +x ./$appName.AppDir/AppRun + printf '[Desktop Entry]\nName='$appName'\nExec='$appName'\nIcon='$appName'\nType=Application\nCategories=Utility;\n' > $appName.AppDir/$appName.desktop + cp ../ton.png $appName.AppDir/$appName.png + cp $file $appName.AppDir/usr/bin/ + cp ../build/openssl_3/libcrypto.so.3 \ + /lib/$ARCH-linux-gnu/libatomic.so.1 \ + /lib/$ARCH-linux-gnu/libsodium.so.23 \ + /lib/$ARCH-linux-gnu/libz.so.1 \ + /lib/$ARCH-linux-gnu/liblz4.so.1 \ + /lib/$ARCH-linux-gnu/libmicrohttpd.so.12 \ + /lib/$ARCH-linux-gnu/libreadline.so.8 \ + /lib/$ARCH-linux-gnu/libstdc++.so.6 \ + $appName.AppDir/usr/lib/ + + chmod +x ./$appName.AppDir/usr/bin/$appName + # create AppImage + ./../appimagetool-$ARCH.AppImage -l $appName.AppDir + mv $appName-$ARCH.AppImage artifacts/$appName + fi +done + +ls -larth artifacts +cp -r ../artifacts/{smartcont,lib} artifacts/ +pwd +ls -larth artifacts diff --git a/assembly/appimage/ton.png b/assembly/appimage/ton.png new file mode 100644 index 00000000..2a25c863 Binary files /dev/null and b/assembly/appimage/ton.png differ diff --git a/assembly/cicd/jenkins/test-builds.groovy b/assembly/cicd/jenkins/test-builds.groovy deleted file mode 100644 index 0b5ab7a3..00000000 --- a/assembly/cicd/jenkins/test-builds.groovy +++ /dev/null @@ -1,237 +0,0 @@ -pipeline { - - agent none - stages { - stage('Run Builds') { - parallel { - stage('Ubuntu 20.04 x86-64 (shared)') { - agent { - label 'Ubuntu_x86-64' - } - steps { - timeout(time: 180, unit: 'MINUTES') { - sh ''' - cp assembly/native/build-ubuntu-shared.sh . - chmod +x build-ubuntu-shared.sh - ./build-ubuntu-shared.sh -a - ''' - sh ''' - cd artifacts - zip -9r ton-x86_64-linux-shared ./* - ''' - archiveArtifacts artifacts: 'artifacts/ton-x86_64-linux-shared.zip' - } - } - } - stage('Ubuntu 20.04 x86-64 (portable)') { - agent { - label 'Ubuntu_x86-64' - } - steps { - timeout(time: 180, unit: 'MINUTES') { - sh ''' - cp assembly/nix/build-linux-x86-64-nix.sh . - chmod +x build-linux-x86-64-nix.sh - ./build-linux-x86-64-nix.sh - ''' - sh ''' - cd artifacts - zip -9r ton-x86-64-linux-portable ./* - ''' - archiveArtifacts artifacts: 'artifacts/ton-x86-64-linux-portable.zip' - } - } - } - stage('Ubuntu 20.04 aarch64 (shared)') { - agent { - label 'Ubuntu_arm64' - } - steps { - timeout(time: 180, unit: 'MINUTES') { - sh ''' - cp assembly/native/build-ubuntu-shared.sh . - chmod +x build-ubuntu-shared.sh - ./build-ubuntu-shared.sh -a - ''' - sh ''' - cd artifacts - zip -9r ton-arm64-linux-shared ./* - ''' - archiveArtifacts artifacts: 'artifacts/ton-arm64-linux-shared.zip' - } - } - } - stage('Ubuntu 20.04 aarch64 (portable)') { - agent { - label 'Ubuntu_arm64' - } - steps { - timeout(time: 180, unit: 'MINUTES') { - sh ''' - cp assembly/nix/build-linux-arm64-nix.sh . - chmod +x build-linux-arm64-nix.sh - ./build-linux-arm64-nix.sh - ''' - sh ''' - cd artifacts - zip -9r ton-arm64-linux-portable ./* - ''' - archiveArtifacts artifacts: 'artifacts/ton-arm64-linux-portable.zip' - } - } - } - stage('macOS 12.7 x86-64 (shared)') { - agent { - label 'macOS_12.7_x86-64' - } - steps { - timeout(time: 180, unit: 'MINUTES') { - sh ''' - cp assembly/native/build-macos-shared.sh . - chmod +x build-macos-shared.sh - ./build-macos-shared.sh -a - ''' - sh ''' - cd artifacts - zip -9r ton-x86-64-macos-shared ./* - ''' - archiveArtifacts artifacts: 'artifacts/ton-x86-64-macos-shared.zip' - } - } - } - stage('macOS 12.7 x86-64 (portable)') { - agent { - label 'macOS_12.7_x86-64' - } - steps { - timeout(time: 180, unit: 'MINUTES') { - sh ''' - cp assembly/nix/build-macos-nix.sh . - chmod +x build-macos-nix.sh - ./build-macos-nix.sh - ''' - sh ''' - cd artifacts - zip -9r ton-x86-64-macos-portable ./* - ''' - archiveArtifacts artifacts: 'artifacts/ton-x86-64-macos-portable.zip' - } - } - } - stage('macOS 12.6 aarch64 (shared)') { - agent { - label 'macOS_12.6-arm64-m1' - } - steps { - timeout(time: 180, unit: 'MINUTES') { - sh ''' - cp assembly/native/build-macos-shared.sh . - chmod +x build-macos-shared.sh - ./build-macos-shared.sh -a - ''' - sh ''' - cd artifacts - zip -9r ton-arm64-macos-m1-shared ./* - ''' - archiveArtifacts artifacts: 'artifacts/ton-arm64-macos-m1-shared.zip' - } - } - } - stage('macOS 12.6 aarch64 (portable)') { - agent { - label 'macOS_12.6-arm64-m1' - } - steps { - timeout(time: 180, unit: 'MINUTES') { - sh ''' - cp assembly/nix/build-macos-nix.sh . - chmod +x build-macos-nix.sh - ./build-macos-nix.sh - ''' - sh ''' - cd artifacts - zip -9r ton-arm64-macos-portable ./* - ''' - archiveArtifacts artifacts: 'artifacts/ton-arm64-macos-portable.zip' - } - } - } - stage('macOS 13.2 aarch64 (shared)') { - agent { - label 'macOS_13.2-arm64-m2' - } - steps { - timeout(time: 180, unit: 'MINUTES') { - sh ''' - cp assembly/native/build-macos-shared.sh . - chmod +x build-macos-shared.sh - ./build-macos-shared.sh -a - ''' - sh ''' - cd artifacts - zip -9r ton-arm64-macos-m2-shared ./* - ''' - archiveArtifacts artifacts: 'artifacts/ton-arm64-macos-m2-shared.zip' - } - } - } - stage('Windows Server 2022 x86-64') { - agent { - label 'Windows_x86-64' - } - steps { - timeout(time: 180, unit: 'MINUTES') { - bat ''' - copy assembly\\native\\build-windows.bat . - build-windows.bat - ''' - bat ''' - cd artifacts - zip -9r ton-x86-64-windows ./* - ''' - archiveArtifacts artifacts: 'artifacts/ton-x86-64-windows.zip' - } - } - } - stage('Android Tonlib') { - agent { - label 'Ubuntu_x86-64' - } - steps { - timeout(time: 180, unit: 'MINUTES') { - sh ''' - cp assembly/android/build-android-tonlib.sh . - chmod +x build-android-tonlib.sh - ./build-android-tonlib.sh -a - ''' - sh ''' - cd artifacts/tonlib-android-jni - zip -9r ton-android-tonlib ./* - ''' - archiveArtifacts artifacts: 'artifacts/tonlib-android-jni/ton-android-tonlib.zip' - } - } - } - stage('WASM fift func emulator') { - agent { - label 'Ubuntu_x86-64' - } - steps { - timeout(time: 180, unit: 'MINUTES') { - sh ''' - cp assembly/wasm/fift-func-wasm-build-ubuntu.sh . - chmod +x fift-func-wasm-build-ubuntu.sh - ./fift-func-wasm-build-ubuntu.sh -a - ''' - sh ''' - cd artifacts - zip -9r ton-wasm-binaries ./* - ''' - archiveArtifacts artifacts: 'artifacts/ton-wasm-binaries.zip' - } - } - } - } - } - } -} diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index 0e1003b5..b785339e 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -52,21 +52,6 @@ else echo "Using compiled lz4" fi -if [ ! -d "secp256k1" ]; then -git clone https://github.com/bitcoin-core/secp256k1.git -cd secp256k1 -secp256k1Path=`pwd` -git checkout v0.3.2 -./autogen.sh -./configure --enable-module-recovery --enable-static --disable-tests --disable-benchmark --with-pic -make -j12 -test $? -eq 0 || { echo "Can't compile secp256k1"; exit 1; } -cd .. -else - secp256k1Path=$(pwd)/secp256k1 - echo "Using compiled secp256k1" -fi - if [ ! -d "libsodium" ]; then export LIBSODIUM_FULL_BUILD=1 git clone https://github.com/jedisct1/libsodium.git @@ -135,9 +120,6 @@ cmake -GNinja .. \ -DZLIB_FOUND=1 \ -DZLIB_INCLUDE_DIR=$zlibPath \ -DZLIB_LIBRARIES=$zlibPath/libz.a \ --DSECP256K1_FOUND=1 \ --DSECP256K1_INCLUDE_DIR=$secp256k1Path/include \ --DSECP256K1_LIBRARY=$secp256k1Path/.libs/libsecp256k1.a \ -DSODIUM_FOUND=1 \ -DSODIUM_INCLUDE_DIR=$sodiumPath/src/libsodium/include \ -DSODIUM_LIBRARY_RELEASE=$sodiumPath/src/libsodium/.libs/libsodium.a \ @@ -153,39 +135,21 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli blockchain-explorer \ - tonlib tonlibjson tonlib-cli validator-engine func fift \ + tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont \ test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ - test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state test-emulator + test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli blockchain-explorer \ - tonlib tonlibjson tonlib-cli validator-engine func fift \ + tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \ - http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator + http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi -strip -s storage/storage-daemon/storage-daemon -strip -s storage/storage-daemon/storage-daemon-cli -strip -s blockchain-explorer/blockchain-explorer -strip -s crypto/fift -strip -s crypto/func -strip -s crypto/create-state -strip -s crypto/tlbc -strip -s validator-engine-console/validator-engine-console -strip -s tonlib/tonlib-cli -strip -s http/http-proxy -strip -s rldp-http-proxy/rldp-http-proxy -strip -s dht-server/dht-server -strip -s lite-client/lite-client -strip -s validator-engine/validator-engine -strip -s utils/generate-random-id -strip -s utils/json2tlo -strip -s adnl/adnl-proxy - cd .. if [ "$with_artifacts" = true ]; then @@ -197,6 +161,7 @@ if [ "$with_artifacts" = true ]; then cp build/blockchain-explorer/blockchain-explorer artifacts/ cp build/crypto/fift artifacts/ cp build/crypto/func artifacts/ + cp build/tolk/tolk artifacts/ cp build/crypto/create-state artifacts/ cp build/crypto/tlbc artifacts/ cp build/validator-engine-console/validator-engine-console artifacts/ @@ -209,6 +174,7 @@ if [ "$with_artifacts" = true ]; then cp build/validator-engine/validator-engine artifacts/ cp build/utils/generate-random-id artifacts/ cp build/utils/json2tlo artifacts/ + cp build/utils/proxy-liteserver artifacts/ cp build/adnl/adnl-proxy artifacts/ cp build/emulator/libemulator.dylib artifacts/ rsync -r crypto/smartcont artifacts/ diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index 7fdcfb94..5c4a5fe0 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -36,21 +36,6 @@ else fi export CCACHE_DISABLE=1 -if [ ! -d "secp256k1" ]; then - git clone https://github.com/bitcoin-core/secp256k1.git - cd secp256k1 - secp256k1Path=`pwd` - git checkout v0.3.2 - ./autogen.sh - ./configure --enable-module-recovery --enable-static --disable-tests --disable-benchmark - make -j12 - test $? -eq 0 || { echo "Can't compile secp256k1"; exit 1; } - cd .. -else - secp256k1Path=$(pwd)/secp256k1 - echo "Using compiled secp256k1" -fi - if [ ! -d "lz4" ]; then git clone https://github.com/lz4/lz4 cd lz4 @@ -70,9 +55,6 @@ brew unlink openssl@3 && brew link --overwrite openssl@3 cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \ -DCMAKE_CXX_FLAGS="-stdlib=libc++" \ --DSECP256K1_FOUND=1 \ --DSECP256K1_INCLUDE_DIR=$secp256k1Path/include \ --DSECP256K1_LIBRARY=$secp256k1Path/.libs/libsecp256k1.a \ -DLZ4_FOUND=1 \ -DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a \ -DLZ4_INCLUDE_DIRS=$lz4Path/lib @@ -81,40 +63,21 @@ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then ninja storage-daemon storage-daemon-cli blockchain-explorer \ - tonlib tonlibjson tonlib-cli validator-engine func fift \ + tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \ http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator \ test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont \ test-net test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp \ - test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state test-emulator + test-rldp2 test-catchain test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else ninja storage-daemon storage-daemon-cli blockchain-explorer \ - tonlib tonlibjson tonlib-cli validator-engine func fift \ + tonlib tonlibjson tonlib-cli validator-engine func tolk fift \ lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server \ - http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator + http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi - -strip -s storage/storage-daemon/storage-daemon -strip -s storage/storage-daemon/storage-daemon-cli -strip -s blockchain-explorer/blockchain-explorer -strip -s crypto/fift -strip -s crypto/func -strip -s crypto/create-state -strip -s crypto/tlbc -strip -s validator-engine-console/validator-engine-console -strip -s tonlib/tonlib-cli -strip -s http/http-proxy -strip -s rldp-http-proxy/rldp-http-proxy -strip -s dht-server/dht-server -strip -s lite-client/lite-client -strip -s validator-engine/validator-engine -strip -s utils/generate-random-id -strip -s utils/json2tlo -strip -s adnl/adnl-proxy - cd .. if [ "$with_artifacts" = true ]; then @@ -126,6 +89,7 @@ if [ "$with_artifacts" = true ]; then cp build/blockchain-explorer/blockchain-explorer artifacts/ cp build/crypto/fift artifacts/ cp build/crypto/func artifacts/ + cp build/tolk/tolk artifacts/ cp build/crypto/create-state artifacts/ cp build/crypto/tlbc artifacts/ cp build/validator-engine-console/validator-engine-console artifacts/ @@ -138,6 +102,7 @@ if [ "$with_artifacts" = true ]; then cp build/validator-engine/validator-engine artifacts/ cp build/utils/generate-random-id artifacts/ cp build/utils/json2tlo artifacts/ + cp build/utils/proxy-liteserver artifacts/ cp build/adnl/adnl-proxy artifacts/ cp build/emulator/libemulator.dylib artifacts/ cp -R crypto/smartcont artifacts/ diff --git a/assembly/native/build-ubuntu-appimages.sh b/assembly/native/build-ubuntu-appimages.sh new file mode 100644 index 00000000..4e63234d --- /dev/null +++ b/assembly/native/build-ubuntu-appimages.sh @@ -0,0 +1,109 @@ +#/bin/bash + +with_tests=false +with_artifacts=false + + +while getopts 'ta' flag; do + case "${flag}" in + t) with_tests=true ;; + a) with_artifacts=true ;; + *) break + ;; + esac +done + +if [ ! -d "build" ]; then + mkdir build + cd build +else + cd build + rm -rf .ninja* CMakeCache.txt +fi + +export CC=$(which clang-16) +export CXX=$(which clang++-16) +export CCACHE_DISABLE=1 + +if [ ! -d "openssl_3" ]; then + git clone https://github.com/openssl/openssl openssl_3 + cd openssl_3 + opensslPath=`pwd` + git checkout openssl-3.1.4 + ./config + make build_libs -j12 + test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } + cd .. +else + opensslPath=$(pwd)/openssl_3 + echo "Using compiled openssl_3" +fi + +cmake -GNinja .. \ +-DCMAKE_BUILD_TYPE=Release \ +-DPORTABLE=1 \ +-DOPENSSL_ROOT_DIR=$opensslPath \ +-DOPENSSL_INCLUDE_DIR=$opensslPath/include \ +-DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.so + + +test $? -eq 0 || { echo "Can't configure ton"; exit 1; } + +if [ "$with_tests" = true ]; then +ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ + validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ + adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ + test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ + test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ + test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver + test $? -eq 0 || { echo "Can't compile ton"; exit 1; } +else +ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ + validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ + generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ + adnl-proxy create-state emulator proxy-liteserver + test $? -eq 0 || { echo "Can't compile ton"; exit 1; } +fi + +# simple binaries' test +./storage/storage-daemon/storage-daemon -V || exit 1 +./validator-engine/validator-engine -V || exit 1 +./lite-client/lite-client -V || exit 1 +./crypto/fift -V || exit 1 + +echo validator-engine +ldd ./validator-engine/validator-engine || exit 1 +ldd ./validator-engine-console/validator-engine-console || exit 1 +ldd ./crypto/fift || exit 1 +echo blockchain-explorer +ldd ./blockchain-explorer/blockchain-explorer || exit 1 +echo libtonlibjson.so +ldd ./tonlib/libtonlibjson.so.0.5 || exit 1 +echo libemulator.so +ldd ./emulator/libemulator.so || exit 1 + +cd .. + +if [ "$with_artifacts" = true ]; then + rm -rf artifacts + mkdir artifacts + mv build/tonlib/libtonlibjson.so.0.5 build/tonlib/libtonlibjson.so + cp build/storage/storage-daemon/storage-daemon build/storage/storage-daemon/storage-daemon-cli \ + build/crypto/fift build/crypto/tlbc build/crypto/func build/tolk/tolk build/crypto/create-state build/blockchain-explorer/blockchain-explorer \ + build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli build/utils/proxy-liteserver \ + build/tonlib/libtonlibjson.so build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy \ + build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine \ + build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.so \ + artifacts + test $? -eq 0 || { echo "Can't copy final binaries"; exit 1; } + cp -R crypto/smartcont artifacts + cp -R crypto/fift/lib artifacts + chmod -R +x artifacts/* +fi + +if [ "$with_tests" = true ]; then + cd build +# ctest --output-on-failure -E "test-catchain|test-actors|test-smartcont|test-adnl|test-validator-session-state|test-dht|test-rldp" + ctest --output-on-failure --timeout 1800 +fi diff --git a/assembly/native/build-ubuntu-portable-libs.sh b/assembly/native/build-ubuntu-portable-libs.sh new file mode 100644 index 00000000..2f0a1ba4 --- /dev/null +++ b/assembly/native/build-ubuntu-portable-libs.sh @@ -0,0 +1,132 @@ +#/bin/bash + +#sudo apt-get update +#sudo apt-get install -y build-essential git cmake ninja-build automake libtool texinfo autoconf libc++-dev libc++abi-dev + +with_artifacts=false + +while getopts 'ta' flag; do + case "${flag}" in + a) with_artifacts=true ;; + *) break + ;; + esac +done + +if [ ! -d "build" ]; then + mkdir build + cd build +else + cd build + rm -rf .ninja* CMakeCache.txt +fi + +export CC=$(which clang) +export CXX=$(which clang++) +export CCACHE_DISABLE=1 + +if [ ! -d "lz4" ]; then +git clone https://github.com/lz4/lz4.git +cd lz4 +lz4Path=`pwd` +git checkout v1.9.4 +CFLAGS="-fPIC" make -j12 +test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } +cd .. +# ./lib/liblz4.a +# ./lib +else + lz4Path=$(pwd)/lz4 + echo "Using compiled lz4" +fi + +if [ ! -d "libsodium" ]; then + export LIBSODIUM_FULL_BUILD=1 + git clone https://github.com/jedisct1/libsodium.git + cd libsodium + sodiumPath=`pwd` + git checkout 1.0.18 + ./autogen.sh + ./configure --with-pic --enable-static + make -j12 + test $? -eq 0 || { echo "Can't compile libsodium"; exit 1; } + cd .. +else + sodiumPath=$(pwd)/libsodium + echo "Using compiled libsodium" +fi + +if [ ! -d "openssl_3" ]; then + git clone https://github.com/openssl/openssl openssl_3 + cd openssl_3 + opensslPath=`pwd` + git checkout openssl-3.1.4 + ./config + make build_libs -j12 + test $? -eq 0 || { echo "Can't compile openssl_3"; exit 1; } + cd .. +else + opensslPath=$(pwd)/openssl_3 + echo "Using compiled openssl_3" +fi + +if [ ! -d "zlib" ]; then + git clone https://github.com/madler/zlib.git + cd zlib + zlibPath=`pwd` + ./configure --static + make -j12 + test $? -eq 0 || { echo "Can't compile zlib"; exit 1; } + cd .. +else + zlibPath=$(pwd)/zlib + echo "Using compiled zlib" +fi + +if [ ! -d "libmicrohttpd" ]; then + git clone https://git.gnunet.org/libmicrohttpd.git + cd libmicrohttpd + libmicrohttpdPath=`pwd` + ./autogen.sh + ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic + make -j12 + test $? -eq 0 || { echo "Can't compile libmicrohttpd"; exit 1; } + cd .. +else + libmicrohttpdPath=$(pwd)/libmicrohttpd + echo "Using compiled libmicrohttpd" +fi + +cmake -GNinja .. \ +-DPORTABLE=1 \ +-DCMAKE_BUILD_TYPE=Release \ +-DOPENSSL_FOUND=1 \ +-DOPENSSL_INCLUDE_DIR=$opensslPath/include \ +-DOPENSSL_CRYPTO_LIBRARY=$opensslPath/libcrypto.a \ +-DZLIB_FOUND=1 \ +-DZLIB_INCLUDE_DIR=$zlibPath \ +-DZLIB_LIBRARIES=$zlibPath/libz.a \ +-DSODIUM_FOUND=1 \ +-DSODIUM_INCLUDE_DIR=$sodiumPath/src/libsodium/include \ +-DSODIUM_LIBRARY_RELEASE=$sodiumPath/src/libsodium/.libs/libsodium.a \ +-DMHD_FOUND=1 \ +-DMHD_INCLUDE_DIR=$libmicrohttpdPath/src/include \ +-DMHD_LIBRARY=$libmicrohttpdPath/src/microhttpd/.libs/libmicrohttpd.a \ +-DLZ4_FOUND=1 \ +-DLZ4_INCLUDE_DIRS=$lz4Path/lib \ +-DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a + + + +test $? -eq 0 || { echo "Can't configure ton"; exit 1; } + +ninja tonlibjson emulator +test $? -eq 0 || { echo "Can't compile ton"; exit 1; } + +cd .. + +mkdir artifacts +mv build/tonlib/libtonlibjson.so.0.5 build/tonlib/libtonlibjson.so +cp build/tonlib/libtonlibjson.so \ + build/emulator/libemulator.so \ + artifacts diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh index 73ae5926..16e77ac8 100644 --- a/assembly/native/build-ubuntu-portable.sh +++ b/assembly/native/build-ubuntu-portable.sh @@ -1,7 +1,7 @@ #/bin/bash #sudo apt-get update -#sudo apt-get install -y build-essential git cmake ninja-build automake libtool texinfo autoconf +#sudo apt-get install -y build-essential git cmake ninja-build automake libtool texinfo autoconf libc++-dev libc++abi-dev with_tests=false with_artifacts=false @@ -24,8 +24,8 @@ else rm -rf .ninja* CMakeCache.txt fi -export CC=$(which clang-16) -export CXX=$(which clang++-16) +export CC=$(which clang) +export CXX=$(which clang++) export CCACHE_DISABLE=1 if [ ! -d "lz4" ]; then @@ -33,7 +33,7 @@ git clone https://github.com/lz4/lz4.git cd lz4 lz4Path=`pwd` git checkout v1.9.4 -make -j12 +CFLAGS="-fPIC" make -j12 test $? -eq 0 || { echo "Can't compile lz4"; exit 1; } cd .. # ./lib/liblz4.a @@ -43,23 +43,6 @@ else echo "Using compiled lz4" fi -if [ ! -d "secp256k1" ]; then -git clone https://github.com/bitcoin-core/secp256k1.git -cd secp256k1 -secp256k1Path=`pwd` -git checkout v0.3.2 -./autogen.sh -./configure --enable-module-recovery --enable-static --disable-tests --disable-benchmark --with-pic -make -j12 -test $? -eq 0 || { echo "Can't compile secp256k1"; exit 1; } -cd .. -# ./.libs/libsecp256k1.a -# ./include -else - secp256k1Path=$(pwd)/secp256k1 - echo "Using compiled secp256k1" -fi - if [ ! -d "libsodium" ]; then export LIBSODIUM_FULL_BUILD=1 git clone https://github.com/jedisct1/libsodium.git @@ -126,9 +109,6 @@ cmake -GNinja .. \ -DZLIB_FOUND=1 \ -DZLIB_INCLUDE_DIR=$zlibPath \ -DZLIB_LIBRARIES=$zlibPath/libz.a \ --DSECP256K1_FOUND=1 \ --DSECP256K1_INCLUDE_DIR=$secp256k1Path/include \ --DSECP256K1_LIBRARY=$secp256k1Path/.libs/libsecp256k1.a \ -DSODIUM_FOUND=1 \ -DSODIUM_INCLUDE_DIR=$sodiumPath/src/libsodium/include \ -DSODIUM_LIBRARY_RELEASE=$sodiumPath/src/libsodium/.libs/libsodium.a \ @@ -144,44 +124,22 @@ cmake -GNinja .. \ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then -ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli \ +ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ - test-fec test-tddb test-db test-validator-session-state test-emulator + test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else -ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli \ +ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ - adnl-proxy create-state emulator + adnl-proxy create-state emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi -strip -s storage/storage-daemon/storage-daemon \ - storage/storage-daemon/storage-daemon-cli \ - blockchain-explorer/blockchain-explorer \ - crypto/fift \ - crypto/tlbc \ - crypto/func \ - crypto/create-state \ - validator-engine-console/validator-engine-console \ - tonlib/tonlib-cli \ - tonlib/libtonlibjson.so.0.5 \ - http/http-proxy \ - rldp-http-proxy/rldp-http-proxy \ - dht-server/dht-server \ - lite-client/lite-client \ - validator-engine/validator-engine \ - utils/generate-random-id \ - utils/json2tlo \ - adnl/adnl-proxy \ - emulator/libemulator.* - -test $? -eq 0 || { echo "Can't strip final binaries"; exit 1; } - # simple binaries' test ./storage/storage-daemon/storage-daemon -V || exit 1 ./validator-engine/validator-engine -V || exit 1 @@ -195,8 +153,8 @@ if [ "$with_artifacts" = true ]; then mkdir artifacts mv build/tonlib/libtonlibjson.so.0.5 build/tonlib/libtonlibjson.so cp build/storage/storage-daemon/storage-daemon build/storage/storage-daemon/storage-daemon-cli \ - build/crypto/fift build/crypto/tlbc build/crypto/func build/crypto/create-state build/blockchain-explorer/blockchain-explorer \ - build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli \ + build/crypto/fift build/crypto/tlbc build/crypto/func build/tolk/tolk build/crypto/create-state build/blockchain-explorer/blockchain-explorer \ + build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli build/utils/proxy-liteserver \ build/tonlib/libtonlibjson.so build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy \ build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine \ build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.so \ diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh index 00b9aa9b..49cc8e1e 100644 --- a/assembly/native/build-ubuntu-shared.sh +++ b/assembly/native/build-ubuntu-shared.sh @@ -52,44 +52,22 @@ cmake -GNinja -DTON_USE_JEMALLOC=ON .. \ test $? -eq 0 || { echo "Can't configure ton"; exit 1; } if [ "$with_tests" = true ]; then -ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli \ +ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ adnl-proxy create-state emulator test-ed25519 test-ed25519-crypto test-bigint \ test-vm test-fift test-cells test-smartcont test-net test-tdactor test-tdutils \ test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain \ - test-fec test-tddb test-db test-validator-session-state test-emulator + test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } else -ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli \ +ninja storage-daemon storage-daemon-cli fift func tolk tonlib tonlibjson tonlib-cli \ validator-engine lite-client pow-miner validator-engine-console blockchain-explorer \ generate-random-id json2tlo dht-server http-proxy rldp-http-proxy \ - adnl-proxy create-state emulator + adnl-proxy create-state emulator proxy-liteserver test $? -eq 0 || { echo "Can't compile ton"; exit 1; } fi -strip -s storage/storage-daemon/storage-daemon \ - storage/storage-daemon/storage-daemon-cli \ - blockchain-explorer/blockchain-explorer \ - crypto/fift \ - crypto/tlbc \ - crypto/func \ - crypto/create-state \ - validator-engine-console/validator-engine-console \ - tonlib/tonlib-cli \ - tonlib/libtonlibjson.so.0.5 \ - http/http-proxy \ - rldp-http-proxy/rldp-http-proxy \ - dht-server/dht-server \ - lite-client/lite-client \ - validator-engine/validator-engine \ - utils/generate-random-id \ - utils/json2tlo \ - adnl/adnl-proxy \ - emulator/libemulator.* - -test $? -eq 0 || { echo "Can't strip final binaries"; exit 1; } - # simple binaries' test ./storage/storage-daemon/storage-daemon -V || exit 1 ./validator-engine/validator-engine -V || exit 1 @@ -105,8 +83,8 @@ if [ "$with_artifacts" = true ]; then mkdir artifacts mv build/tonlib/libtonlibjson.so.0.5 build/tonlib/libtonlibjson.so cp build/storage/storage-daemon/storage-daemon build/storage/storage-daemon/storage-daemon-cli \ - build/crypto/fift build/crypto/tlbc build/crypto/func build/crypto/create-state build/blockchain-explorer/blockchain-explorer \ - build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli \ + build/crypto/fift build/crypto/tlbc build/crypto/func build/tolk/tolk build/crypto/create-state build/blockchain-explorer/blockchain-explorer \ + build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli build/utils/proxy-liteserver \ build/tonlib/libtonlibjson.so build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy \ build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine \ build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.so \ diff --git a/assembly/native/build-windows-2019.bat b/assembly/native/build-windows-2019.bat index f728b88f..844c09fc 100644 --- a/assembly/native/build-windows-2019.bat +++ b/assembly/native/build-windows-2019.bat @@ -26,149 +26,129 @@ IF %errorlevel% NEQ 0 ( exit /b %errorlevel% ) -if not exist "zlib" ( -git clone https://github.com/madler/zlib.git -cd zlib -git checkout v1.3.1 -cd contrib\vstudio\vc14 -msbuild zlibstat.vcxproj /p:Configuration=ReleaseWithoutAsm /p:platform=x64 -p:PlatformToolset=v142 - +echo Installing nasm... +choco install -y nasm +where nasm +SET PATH=%PATH%;C:\Program Files\NASM IF %errorlevel% NEQ 0 ( - echo Can't install zlib + echo Can't install nasm exit /b %errorlevel% ) -cd ..\..\..\.. + +mkdir third_libs +cd third_libs + +set third_libs=%cd% +echo %third_libs% + +if not exist "zlib" ( + git clone https://github.com/madler/zlib.git + cd zlib + git checkout v1.3.1 + cd contrib\vstudio\vc14 + msbuild zlibstat.vcxproj /p:Configuration=ReleaseWithoutAsm /p:platform=x64 -p:PlatformToolset=v142 + cd ..\..\..\.. ) else ( -echo Using zlib... + echo Using zlib... ) if not exist "lz4" ( -git clone https://github.com/lz4/lz4.git -cd lz4 -git checkout v1.9.4 -cd build\VS2017\liblz4 -msbuild liblz4.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v142 - -IF %errorlevel% NEQ 0 ( - echo Can't install lz4 - exit /b %errorlevel% -) -cd ..\..\..\.. + git clone https://github.com/lz4/lz4.git + cd lz4 + git checkout v1.9.4 + cd build\VS2017\liblz4 + msbuild liblz4.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v142 + cd ..\..\..\.. ) else ( -echo Using lz4... + echo Using lz4... ) -if not exist "secp256k1" ( -git clone https://github.com/bitcoin-core/secp256k1.git -cd secp256k1 -git checkout v0.3.2 -cmake -G "Visual Studio 16 2019" -A x64 -S . -B build -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DBUILD_SHARED_LIBS=OFF -IF %errorlevel% NEQ 0 ( - echo Can't configure secp256k1 - exit /b %errorlevel% +if not exist "libsodium" ( + git clone https://github.com/jedisct1/libsodium + cd libsodium + git checkout 1.0.18-RELEASE + msbuild libsodium.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v142 + cd .. +) else ( + echo Using libsodium... ) -cmake --build build --config Release -IF %errorlevel% NEQ 0 ( - echo Can't install secp256k1 - exit /b %errorlevel% + +if not exist "openssl" ( + git clone https://github.com/openssl/openssl.git + cd openssl + git checkout openssl-3.1.4 + where perl + perl Configure VC-WIN64A + IF %errorlevel% NEQ 0 ( + echo Can't configure openssl + exit /b %errorlevel% + ) + nmake + cd .. +) else ( + echo Using openssl... ) + +if not exist "libmicrohttpd" ( + git clone https://github.com/Karlson2k/libmicrohttpd.git + cd libmicrohttpd + git checkout v1.0.1 + cd w32\VS2019 + msbuild libmicrohttpd.vcxproj /p:Configuration=Release-static /p:platform=x64 -p:PlatformToolset=v142 + IF %errorlevel% NEQ 0 ( + echo Can't compile libmicrohttpd + exit /b %errorlevel% + ) + cd ../../.. +) else ( + echo Using libmicrohttpd... +) + cd .. -) else ( -echo Using secp256k1... -) - - -curl --retry 5 --retry-delay 10 -Lo libsodium-1.0.18-stable-msvc.zip https://download.libsodium.org/libsodium/releases/libsodium-1.0.18-stable-msvc.zip -IF %errorlevel% NEQ 0 ( - echo Can't download libsodium - exit /b %errorlevel% -) -unzip libsodium-1.0.18-stable-msvc.zip -) else ( -echo Using libsodium... -) - -if not exist "openssl-3.1.4" ( -curl -Lo openssl-3.1.4.zip https://github.com/neodiX42/precompiled-openssl-win64/raw/main/openssl-3.1.4.zip -IF %errorlevel% NEQ 0 ( - echo Can't download OpenSSL - exit /b %errorlevel% -) -unzip -q openssl-3.1.4.zip -) else ( -echo Using openssl... -) - -if not exist "libmicrohttpd-0.9.77-w32-bin" ( -curl -Lo libmicrohttpd-0.9.77-w32-bin.zip https://github.com/neodiX42/precompiled-openssl-win64/raw/main/libmicrohttpd-0.9.77-w32-bin.zip -IF %errorlevel% NEQ 0 ( - echo Can't download libmicrohttpd - exit /b %errorlevel% -) -unzip -q libmicrohttpd-0.9.77-w32-bin.zip -) else ( -echo Using libmicrohttpd... -) - -if not exist "readline-5.0-1-lib" ( -curl -Lo readline-5.0-1-lib.zip https://github.com/neodiX42/precompiled-openssl-win64/raw/main/readline-5.0-1-lib.zip -IF %errorlevel% NEQ 0 ( - echo Can't download readline - exit /b %errorlevel% -) -unzip -q -d readline-5.0-1-lib readline-5.0-1-lib.zip -) else ( -echo Using readline... -) - - -set root=%cd% -echo %root% -set SODIUM_DIR=%root%\libsodium +echo Current dir %cd% mkdir build cd build cmake -GNinja -DCMAKE_BUILD_TYPE=Release ^ -DPORTABLE=1 ^ -DSODIUM_USE_STATIC_LIBS=1 ^ --DSECP256K1_FOUND=1 ^ --DSECP256K1_INCLUDE_DIR=%root%\secp256k1\include ^ --DSECP256K1_LIBRARY=%root%\secp256k1\build\src\Release\libsecp256k1.lib ^ +-DSODIUM_LIBRARY_RELEASE=%third_libs%\libsodium\Build\Release\x64\libsodium.lib ^ +-DSODIUM_LIBRARY_DEBUG=%third_libs%\libsodium\Build\Release\x64\libsodium.lib ^ +-DSODIUM_INCLUDE_DIR=%third_libs%\libsodium\src\libsodium\include ^ -DLZ4_FOUND=1 ^ --DLZ4_INCLUDE_DIRS=%root%\lz4\lib ^ --DLZ4_LIBRARIES=%root%\lz4\build\VS2017\liblz4\bin\x64_Release\liblz4_static.lib ^ +-DLZ4_INCLUDE_DIRS=%third_libs%\lz4\lib ^ +-DLZ4_LIBRARIES=%third_libs%\lz4\build\VS2017\liblz4\bin\x64_Release\liblz4_static.lib ^ -DMHD_FOUND=1 ^ --DMHD_LIBRARY=%root%\libmicrohttpd-0.9.77-w32-bin\x86_64\VS2019\Release-static\libmicrohttpd.lib ^ --DMHD_INCLUDE_DIR=%root%\libmicrohttpd-0.9.77-w32-bin\x86_64\VS2019\Release-static ^ +-DMHD_LIBRARY=%third_libs%\libmicrohttpd\w32\VS2019\Output\x64\libmicrohttpd.lib ^ +-DMHD_INCLUDE_DIR=%third_libs%\libmicrohttpd\src\include ^ -DZLIB_FOUND=1 ^ --DZLIB_INCLUDE_DIR=%root%\zlib ^ --DZLIB_LIBRARIES=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib ^ +-DZLIB_INCLUDE_DIR=%third_libs%\zlib ^ +-DZLIB_LIBRARIES=%third_libs%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib ^ -DOPENSSL_FOUND=1 ^ --DOPENSSL_INCLUDE_DIR=%root%\openssl-3.1.4\x64\include ^ --DOPENSSL_CRYPTO_LIBRARY=%root%\openssl-3.1.4\x64\lib\libcrypto_static.lib ^ --DREADLINE_INCLUDE_DIR=%root%\readline-5.0-1-lib\include ^ --DREADLINE_LIBRARY=%root%\readline-5.0-1-lib\lib\readline.lib ^ +-DOPENSSL_INCLUDE_DIR=%third_libs%\openssl\include ^ +-DOPENSSL_CRYPTO_LIBRARY=%third_libs%\openssl\libcrypto_static.lib ^ -DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj" .. + IF %errorlevel% NEQ 0 ( echo Can't configure TON exit /b %errorlevel% ) IF "%1"=="-t" ( -ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson ^ +ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net ^ test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ -test-fec test-tddb test-db test-validator-session-state test-emulator +test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver IF %errorlevel% NEQ 0 ( echo Can't compile TON exit /b %errorlevel% ) ) else ( -ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson ^ +ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^ -json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator +json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver IF %errorlevel% NEQ 0 ( echo Can't compile TON exit /b %errorlevel% @@ -191,31 +171,38 @@ REM ctest -C Release --output-on-failure -E "test-catchain|test-actors|test-val ) ) - -echo Creating artifacts... +echo Strip and copy artifacts cd .. +echo where strip +where strip mkdir artifacts mkdir artifacts\smartcont mkdir artifacts\lib for %%I in (build\storage\storage-daemon\storage-daemon.exe ^ -build\storage\storage-daemon\storage-daemon-cli.exe ^ -build\blockchain-explorer\blockchain-explorer.exe ^ -build\crypto\fift.exe ^ -build\crypto\tlbc.exe ^ -build\crypto\func.exe ^ -build\crypto\create-state.exe ^ -build\validator-engine-console\validator-engine-console.exe ^ -build\tonlib\tonlib-cli.exe ^ -build\tonlib\tonlibjson.dll ^ -build\http\http-proxy.exe ^ -build\rldp-http-proxy\rldp-http-proxy.exe ^ -build\dht-server\dht-server.exe ^ -build\lite-client\lite-client.exe ^ -build\validator-engine\validator-engine.exe ^ -build\utils\generate-random-id.exe ^ -build\utils\json2tlo.exe ^ -build\adnl\adnl-proxy.exe ^ -build\emulator\emulator.dll) do (strip -s %%I & copy %%I artifacts\) + build\storage\storage-daemon\storage-daemon-cli.exe ^ + build\blockchain-explorer\blockchain-explorer.exe ^ + build\crypto\fift.exe ^ + build\crypto\tlbc.exe ^ + build\crypto\func.exe ^ + build\tolk\tolk.exe ^ + build\crypto\create-state.exe ^ + build\validator-engine-console\validator-engine-console.exe ^ + build\tonlib\tonlib-cli.exe ^ + build\tonlib\tonlibjson.dll ^ + build\http\http-proxy.exe ^ + build\rldp-http-proxy\rldp-http-proxy.exe ^ + build\dht-server\dht-server.exe ^ + build\lite-client\lite-client.exe ^ + build\validator-engine\validator-engine.exe ^ + build\utils\generate-random-id.exe ^ + build\utils\json2tlo.exe ^ + build\utils\proxy-liteserver.exe ^ + build\adnl\adnl-proxy.exe ^ + build\emulator\emulator.dll) do ( + echo strip -s %%I & copy %%I artifacts\ + strip -s %%I & copy %%I artifacts\ +) + xcopy /e /k /h /i crypto\smartcont artifacts\smartcont xcopy /e /k /h /i crypto\fift\lib artifacts\lib diff --git a/assembly/native/build-windows.bat b/assembly/native/build-windows.bat index aa0fd69a..68f83c39 100644 --- a/assembly/native/build-windows.bat +++ b/assembly/native/build-windows.bat @@ -26,150 +26,129 @@ IF %errorlevel% NEQ 0 ( exit /b %errorlevel% ) -if not exist "zlib" ( -git clone https://github.com/madler/zlib.git -cd zlib -git checkout v1.3.1 -cd contrib\vstudio\vc14 -msbuild zlibstat.vcxproj /p:Configuration=ReleaseWithoutAsm /p:platform=x64 -p:PlatformToolset=v143 - +echo Installing nasm... +choco install -y nasm +where nasm +SET PATH=%PATH%;C:\Program Files\NASM IF %errorlevel% NEQ 0 ( - echo Can't install zlib + echo Can't install nasm exit /b %errorlevel% ) -cd ..\..\..\.. + +mkdir third_libs +cd third_libs + +set third_libs=%cd% +echo %third_libs% + +if not exist "zlib" ( + git clone https://github.com/madler/zlib.git + cd zlib + git checkout v1.3.1 + cd contrib\vstudio\vc14 + msbuild zlibstat.vcxproj /p:Configuration=ReleaseWithoutAsm /p:platform=x64 -p:PlatformToolset=v143 + cd ..\..\..\.. ) else ( -echo Using zlib... + echo Using zlib... ) if not exist "lz4" ( -git clone https://github.com/lz4/lz4.git -cd lz4 -git checkout v1.9.4 -cd build\VS2017\liblz4 -msbuild liblz4.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v143 -dir /s -IF %errorlevel% NEQ 0 ( - echo Can't install lz4 - exit /b %errorlevel% -) -cd ..\..\..\.. + git clone https://github.com/lz4/lz4.git + cd lz4 + git checkout v1.9.4 + cd build\VS2022\liblz4 + msbuild liblz4.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v143 + cd ..\..\..\.. ) else ( -echo Using lz4... + echo Using lz4... ) -if not exist "secp256k1" ( -git clone https://github.com/bitcoin-core/secp256k1.git -cd secp256k1 -git checkout v0.3.2 -cmake -G "Visual Studio 17 2022" -A x64 -S . -B build -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DBUILD_SHARED_LIBS=OFF -IF %errorlevel% NEQ 0 ( - echo Can't configure secp256k1 - exit /b %errorlevel% -) -cmake --build build --config Release -IF %errorlevel% NEQ 0 ( - echo Can't install secp256k1 - exit /b %errorlevel% -) -cd .. -) else ( -echo Using secp256k1... -) - - if not exist "libsodium" ( -curl -Lo libsodium-1.0.18-stable-msvc.zip https://download.libsodium.org/libsodium/releases/libsodium-1.0.18-stable-msvc.zip -IF %errorlevel% NEQ 0 ( - echo Can't download libsodium - exit /b %errorlevel% -) -unzip libsodium-1.0.18-stable-msvc.zip + git clone https://github.com/jedisct1/libsodium + cd libsodium + git checkout 1.0.18-RELEASE + msbuild libsodium.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v143 + cd .. ) else ( -echo Using libsodium... + echo Using libsodium... ) -if not exist "openssl-3.1.4" ( -curl -Lo openssl-3.1.4.zip https://github.com/neodiX42/precompiled-openssl-win64/raw/main/openssl-3.1.4.zip -IF %errorlevel% NEQ 0 ( - echo Can't download OpenSSL - exit /b %errorlevel% -) -unzip -q openssl-3.1.4.zip +if not exist "openssl" ( + git clone https://github.com/openssl/openssl.git + cd openssl + git checkout openssl-3.1.4 + where perl + perl Configure VC-WIN64A + IF %errorlevel% NEQ 0 ( + echo Can't configure openssl + exit /b %errorlevel% + ) + nmake + cd .. ) else ( -echo Using openssl... + echo Using openssl... ) -if not exist "libmicrohttpd-0.9.77-w32-bin" ( -curl -Lo libmicrohttpd-0.9.77-w32-bin.zip https://github.com/neodiX42/precompiled-openssl-win64/raw/main/libmicrohttpd-0.9.77-w32-bin.zip -IF %errorlevel% NEQ 0 ( - echo Can't download libmicrohttpd - exit /b %errorlevel% -) -unzip -q libmicrohttpd-0.9.77-w32-bin.zip +if not exist "libmicrohttpd" ( + git clone https://github.com/Karlson2k/libmicrohttpd.git + cd libmicrohttpd + git checkout v1.0.1 + cd w32\VS2022 + msbuild libmicrohttpd.vcxproj /p:Configuration=Release-static /p:platform=x64 -p:PlatformToolset=v143 + IF %errorlevel% NEQ 0 ( + echo Can't compile libmicrohttpd + exit /b %errorlevel% + ) + cd ../../.. ) else ( -echo Using libmicrohttpd... + echo Using libmicrohttpd... ) -if not exist "readline-5.0-1-lib" ( -curl -Lo readline-5.0-1-lib.zip https://github.com/neodiX42/precompiled-openssl-win64/raw/main/readline-5.0-1-lib.zip -IF %errorlevel% NEQ 0 ( - echo Can't download readline - exit /b %errorlevel% -) -unzip -q -d readline-5.0-1-lib readline-5.0-1-lib.zip -) else ( -echo Using readline... -) - - -set root=%cd% -echo %root% -set SODIUM_DIR=%root%\libsodium +cd .. +echo Current dir %cd% mkdir build cd build cmake -GNinja -DCMAKE_BUILD_TYPE=Release ^ -DPORTABLE=1 ^ -DSODIUM_USE_STATIC_LIBS=1 ^ --DSECP256K1_FOUND=1 ^ --DSECP256K1_INCLUDE_DIR=%root%\secp256k1\include ^ --DSECP256K1_LIBRARY=%root%\secp256k1\build\src\Release\libsecp256k1.lib ^ +-DSODIUM_LIBRARY_RELEASE=%third_libs%\libsodium\Build\Release\x64\libsodium.lib ^ +-DSODIUM_LIBRARY_DEBUG=%third_libs%\libsodium\Build\Release\x64\libsodium.lib ^ +-DSODIUM_INCLUDE_DIR=%third_libs%\libsodium\src\libsodium\include ^ -DLZ4_FOUND=1 ^ --DLZ4_INCLUDE_DIRS=%root%\lz4\lib ^ --DLZ4_LIBRARIES=%root%\lz4\build\VS2017\liblz4\bin\x64_Release\liblz4_static.lib ^ +-DLZ4_INCLUDE_DIRS=%third_libs%\lz4\lib ^ +-DLZ4_LIBRARIES=%third_libs%\lz4\build\VS2022\liblz4\bin\x64_Release\liblz4_static.lib ^ -DMHD_FOUND=1 ^ --DMHD_LIBRARY=%root%\libmicrohttpd-0.9.77-w32-bin\x86_64\VS2019\Release-static\libmicrohttpd.lib ^ --DMHD_INCLUDE_DIR=%root%\libmicrohttpd-0.9.77-w32-bin\x86_64\VS2019\Release-static ^ +-DMHD_LIBRARY=%third_libs%\libmicrohttpd\w32\VS2022\Output\x64\libmicrohttpd.lib ^ +-DMHD_INCLUDE_DIR=%third_libs%\libmicrohttpd\src\include ^ -DZLIB_FOUND=1 ^ --DZLIB_INCLUDE_DIR=%root%\zlib ^ --DZLIB_LIBRARIES=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib ^ +-DZLIB_INCLUDE_DIR=%third_libs%\zlib ^ +-DZLIB_LIBRARIES=%third_libs%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib ^ -DOPENSSL_FOUND=1 ^ --DOPENSSL_INCLUDE_DIR=%root%\openssl-3.1.4\x64\include ^ --DOPENSSL_CRYPTO_LIBRARY=%root%\openssl-3.1.4\x64\lib\libcrypto_static.lib ^ --DREADLINE_INCLUDE_DIR=%root%\readline-5.0-1-lib\include ^ --DREADLINE_LIBRARY=%root%\readline-5.0-1-lib\lib\readline.lib ^ +-DOPENSSL_INCLUDE_DIR=%third_libs%\openssl\include ^ +-DOPENSSL_CRYPTO_LIBRARY=%third_libs%\openssl\libcrypto_static.lib ^ -DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj" .. + IF %errorlevel% NEQ 0 ( echo Can't configure TON exit /b %errorlevel% ) IF "%1"=="-t" ( -ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson ^ +ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^ json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator ^ test-ed25519 test-ed25519-crypto test-bigint test-vm test-fift test-cells test-smartcont test-net ^ test-tdactor test-tdutils test-tonlib-offline test-adnl test-dht test-rldp test-rldp2 test-catchain ^ -test-fec test-tddb test-db test-validator-session-state test-emulator +test-fec test-tddb test-db test-validator-session-state test-emulator proxy-liteserver IF %errorlevel% NEQ 0 ( echo Can't compile TON exit /b %errorlevel% ) ) else ( -ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson ^ +ninja storage-daemon storage-daemon-cli blockchain-explorer fift func tolk tonlib tonlibjson ^ tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id ^ -json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator +json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork emulator proxy-liteserver IF %errorlevel% NEQ 0 ( echo Can't compile TON exit /b %errorlevel% @@ -192,31 +171,38 @@ REM ctest -C Release --output-on-failure -E "test-catchain|test-actors|test-val ) ) - -echo Creating artifacts... +echo Strip and copy artifacts cd .. +echo where strip +where strip mkdir artifacts mkdir artifacts\smartcont mkdir artifacts\lib for %%I in (build\storage\storage-daemon\storage-daemon.exe ^ -build\storage\storage-daemon\storage-daemon-cli.exe ^ -build\blockchain-explorer\blockchain-explorer.exe ^ -build\crypto\fift.exe ^ -build\crypto\tlbc.exe ^ -build\crypto\func.exe ^ -build\crypto\create-state.exe ^ -build\validator-engine-console\validator-engine-console.exe ^ -build\tonlib\tonlib-cli.exe ^ -build\tonlib\tonlibjson.dll ^ -build\http\http-proxy.exe ^ -build\rldp-http-proxy\rldp-http-proxy.exe ^ -build\dht-server\dht-server.exe ^ -build\lite-client\lite-client.exe ^ -build\validator-engine\validator-engine.exe ^ -build\utils\generate-random-id.exe ^ -build\utils\json2tlo.exe ^ -build\adnl\adnl-proxy.exe ^ -build\emulator\emulator.dll) do (strip -s %%I & copy %%I artifacts\) + build\storage\storage-daemon\storage-daemon-cli.exe ^ + build\blockchain-explorer\blockchain-explorer.exe ^ + build\crypto\fift.exe ^ + build\crypto\tlbc.exe ^ + build\crypto\func.exe ^ + build\tolk\tolk.exe ^ + build\crypto\create-state.exe ^ + build\validator-engine-console\validator-engine-console.exe ^ + build\tonlib\tonlib-cli.exe ^ + build\tonlib\tonlibjson.dll ^ + build\http\http-proxy.exe ^ + build\rldp-http-proxy\rldp-http-proxy.exe ^ + build\dht-server\dht-server.exe ^ + build\lite-client\lite-client.exe ^ + build\validator-engine\validator-engine.exe ^ + build\utils\generate-random-id.exe ^ + build\utils\json2tlo.exe ^ + build\utils\proxy-liteserver.exe ^ + build\adnl\adnl-proxy.exe ^ + build\emulator\emulator.dll) do ( + echo strip -s %%I & copy %%I artifacts\ + strip -s %%I & copy %%I artifacts\ +) + xcopy /e /k /h /i crypto\smartcont artifacts\smartcont xcopy /e /k /h /i crypto\fift\lib artifacts\lib diff --git a/assembly/nix/build-linux-arm64-nix.sh b/assembly/nix/build-linux-arm64-nix.sh index 2c7df521..6fc2fab2 100644 --- a/assembly/nix/build-linux-arm64-nix.sh +++ b/assembly/nix/build-linux-arm64-nix.sh @@ -36,22 +36,3 @@ cp ./result/lib/libemulator.so artifacts/ cp ./result/lib/fift/* artifacts/lib/ cp -r ./result/share/ton/smartcont artifacts/ chmod -R +x artifacts -cd artifacts -sudo strip -s storage-daemon \ - storage-daemon-cli \ - blockchain-explorer \ - fift \ - tlbc \ - func \ - create-state \ - validator-engine-console \ - tonlib-cli \ - http-proxy \ - rldp-http-proxy \ - dht-server \ - lite-client \ - validator-engine \ - generate-random-id \ - adnl-proxy \ - libemulator.so \ - libtonlibjson.so diff --git a/assembly/nix/build-linux-x86-64-nix.sh b/assembly/nix/build-linux-x86-64-nix.sh index ae478ec2..30ab79f7 100644 --- a/assembly/nix/build-linux-x86-64-nix.sh +++ b/assembly/nix/build-linux-x86-64-nix.sh @@ -36,22 +36,3 @@ cp ./result/lib/libemulator.so artifacts/ cp ./result/lib/fift/* artifacts/lib/ cp -r ./result/share/ton/smartcont artifacts/ chmod -R +x artifacts -cd artifacts -sudo strip -s storage-daemon \ - storage-daemon-cli \ - blockchain-explorer \ - fift \ - tlbc \ - func \ - create-state \ - validator-engine-console \ - tonlib-cli \ - http-proxy \ - rldp-http-proxy \ - dht-server \ - lite-client \ - validator-engine \ - generate-random-id \ - adnl-proxy \ - libemulator.so \ - libtonlibjson.so diff --git a/assembly/nix/build-macos-nix.sh b/assembly/nix/build-macos-nix.sh index c92eddb2..8a07bea2 100644 --- a/assembly/nix/build-macos-nix.sh +++ b/assembly/nix/build-macos-nix.sh @@ -36,22 +36,3 @@ cp ./result/lib/libemulator.dylib artifacts/ cp ./result/lib/fift/* artifacts/lib/ cp -r ./result/share/ton/smartcont artifacts/ chmod -R +x artifacts -cd artifacts -sudo strip -xSX storage-daemon \ - storage-daemon-cli \ - blockchain-explorer \ - fift \ - tlbc \ - func \ - create-state \ - validator-engine-console \ - tonlib-cli \ - http-proxy \ - rldp-http-proxy \ - dht-server \ - lite-client \ - validator-engine \ - generate-random-id \ - adnl-proxy \ - libemulator.dylib \ - libtonlibjson.dylib diff --git a/assembly/wasm/fift-func-wasm-build-ubuntu.sh b/assembly/wasm/fift-func-wasm-build-ubuntu.sh index 2d3507b2..a463c02a 100644 --- a/assembly/wasm/fift-func-wasm-build-ubuntu.sh +++ b/assembly/wasm/fift-func-wasm-build-ubuntu.sh @@ -2,7 +2,7 @@ # sudo apt update # sudo apt install -y build-essential git make cmake ninja-build clang libgflags-dev zlib1g-dev libssl-dev \ # libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev python3-pip \ -# nodejs libsecp256k1-dev libsodium-dev automake libtool +# nodejs libsodium-dev automake libtool libjemalloc-dev # wget https://apt.llvm.org/llvm.sh # chmod +x llvm.sh @@ -26,13 +26,14 @@ export CCACHE_DISABLE=1 echo `pwd` if [ "$scratch_new" = true ]; then - echo Compiling openssl zlib lz4 emsdk secp256k1 libsodium emsdk ton - rm -rf openssl zlib lz4 emsdk secp256k1 libsodium build + echo Compiling openssl zlib lz4 emsdk libsodium emsdk ton + rm -rf openssl zlib lz4 emsdk libsodium build openssl_em fi if [ ! -d "openssl" ]; then git clone https://github.com/openssl/openssl.git + cp -r openssl openssl_em cd openssl git checkout openssl-3.1.4 ./config @@ -47,21 +48,20 @@ fi if [ ! -d "build" ]; then mkdir build cd build - cmake -GNinja -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_STANDARD=17 \ - -DOPENSSL_FOUND=1 \ + cmake -GNinja -DTON_USE_JEMALLOC=ON .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DOPENSSL_ROOT_DIR=$OPENSSL_DIR \ -DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \ - -DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.so \ - -DTON_USE_ABSEIL=OFF .. + -DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.so test $? -eq 0 || { echo "Can't configure TON build"; exit 1; } ninja fift smc-envelope test $? -eq 0 || { echo "Can't compile fift "; exit 1; } - rm -rf * + rm -rf * .ninja* CMakeCache.txt cd .. else echo cleaning build... - rm -rf build/* + rm -rf build/* build/.ninja* build/CMakeCache.txt fi if [ ! -d "emsdk" ]; then @@ -82,9 +82,8 @@ export CCACHE_DISABLE=1 cd .. -if [ ! -f "openssl/openssl_em" ]; then - cd openssl - make clean +if [ ! -f "openssl_em/openssl_em" ]; then + cd openssl_em emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test no-tests no-fuzz-afl no-fuzz-libfuzzer sed -i 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile sed -i 's/-ldl//g' Makefile @@ -92,10 +91,12 @@ if [ ! -f "openssl/openssl_em" ]; then emmake make depend emmake make -j16 test $? -eq 0 || { echo "Can't compile OpenSSL with emmake "; exit 1; } + OPENSSL_DIR=`pwd` touch openssl_em cd .. else - echo Using compiled openssl with emscripten + OPENSSL_DIR=`pwd`/openssl_em + echo Using compiled with empscripten openssl at $OPENSSL_DIR fi if [ ! -d "zlib" ]; then @@ -125,21 +126,6 @@ else echo Using compiled lz4 with emscripten at $LZ4_DIR fi -if [ ! -d "secp256k1" ]; then - git clone https://github.com/bitcoin-core/secp256k1.git - cd secp256k1 - git checkout v0.3.2 - ./autogen.sh - SECP256K1_DIR=`pwd` - emconfigure ./configure --enable-module-recovery - emmake make -j16 - test $? -eq 0 || { echo "Can't compile secp256k1 with emmake "; exit 1; } - cd .. -else - SECP256K1_DIR=`pwd`/secp256k1 - echo Using compiled secp256k1 with emscripten at $SECP256K1_DIR -fi - if [ ! -d "libsodium" ]; then git clone https://github.com/jedisct1/libsodium cd libsodium @@ -168,9 +154,9 @@ emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAK -DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.a \ -DCMAKE_TOOLCHAIN_FILE=$EMSDK_DIR/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake \ -DCMAKE_CXX_FLAGS="-sUSE_ZLIB=1" \ --DSECP256K1_INCLUDE_DIR=$SECP256K1_DIR/include \ --DSECP256K1_LIBRARY=$SECP256K1_DIR/.libs/libsecp256k1.a \ +-DSODIUM_FOUND=1 \ -DSODIUM_INCLUDE_DIR=$SODIUM_DIR/src/libsodium/include \ +-DSODIUM_USE_STATIC_LIBS=1 \ -DSODIUM_LIBRARY_RELEASE=$SODIUM_DIR/src/libsodium/.libs/libsodium.a \ .. @@ -194,5 +180,3 @@ if [ "$with_artifacts" = true ]; then cp -R crypto/smartcont artifacts cp -R crypto/fift/lib artifacts fi - - diff --git a/assembly/wasm/intrinsics.fc b/assembly/wasm/intrinsics.fc new file mode 100644 index 00000000..14a4498d --- /dev/null +++ b/assembly/wasm/intrinsics.fc @@ -0,0 +1,61 @@ +#pragma allow-post-modification; +#pragma compute-asm-ltr; + +(slice, slice) __tact_load_address(slice cs) inline { + slice raw = cs~load_msg_addr(); + return (cs, raw); +} + +slice __gen_slice1 () asm """ + B{b5ee9c72410101010005000006abcdefe1e98884} B>boc boc boc boc boc tuple cons(X head, tuple tail) asm "CONS"; + +;;; Extracts the head and the tail of lisp-style list. +forall X -> (X, tuple) uncons(tuple list) asm "UNCONS"; + +;;; Extracts the tail and the head of lisp-style list. +forall X -> (tuple, X) list_next(tuple list) asm(-> 1 0) "UNCONS"; + +;;; Returns the head of lisp-style list. +forall X -> X car(tuple list) asm "CAR"; + +;;; Returns the tail of lisp-style list. +tuple cdr(tuple list) asm "CDR"; + +;;; Creates tuple with zero elements. +tuple empty_tuple() asm "NIL"; + +;;; Appends a value `x` to a `Tuple t = (x1, ..., xn)`, but only if the resulting `Tuple t' = (x1, ..., xn, x)` +;;; is of length at most 255. Otherwise throws a type check exception. +forall X -> tuple tpush(tuple t, X value) asm "TPUSH"; +forall X -> (tuple, ()) ~tpush(tuple t, X value) asm "TPUSH"; + +;;; Creates a tuple of length one with given argument as element. +forall X -> [X] single(X x) asm "SINGLE"; + +;;; Unpacks a tuple of length one +forall X -> X unsingle([X] t) asm "UNSINGLE"; + +;;; Creates a tuple of length two with given arguments as elements. +forall X, Y -> [X, Y] pair(X x, Y y) asm "PAIR"; + +;;; Unpacks a tuple of length two +forall X, Y -> (X, Y) unpair([X, Y] t) asm "UNPAIR"; + +;;; Creates a tuple of length three with given arguments as elements. +forall X, Y, Z -> [X, Y, Z] triple(X x, Y y, Z z) asm "TRIPLE"; + +;;; Unpacks a tuple of length three +forall X, Y, Z -> (X, Y, Z) untriple([X, Y, Z] t) asm "UNTRIPLE"; + +;;; Creates a tuple of length four with given arguments as elements. +forall X, Y, Z, W -> [X, Y, Z, W] tuple4(X x, Y y, Z z, W w) asm "4 TUPLE"; + +;;; Unpacks a tuple of length four +forall X, Y, Z, W -> (X, Y, Z, W) untuple4([X, Y, Z, W] t) asm "4 UNTUPLE"; + +;;; Returns the first element of a tuple (with unknown element types). +forall X -> X first(tuple t) asm "FIRST"; + +;;; Returns the second element of a tuple (with unknown element types). +forall X -> X second(tuple t) asm "SECOND"; + +;;; Returns the third element of a tuple (with unknown element types). +forall X -> X third(tuple t) asm "THIRD"; + +;;; Returns the fourth element of a tuple (with unknown element types). +forall X -> X fourth(tuple t) asm "3 INDEX"; + +;;; Returns the first element of a pair tuple. +forall X, Y -> X pair_first([X, Y] p) asm "FIRST"; + +;;; Returns the second element of a pair tuple. +forall X, Y -> Y pair_second([X, Y] p) asm "SECOND"; + +;;; Returns the first element of a triple tuple. +forall X, Y, Z -> X triple_first([X, Y, Z] p) asm "FIRST"; + +;;; Returns the second element of a triple tuple. +forall X, Y, Z -> Y triple_second([X, Y, Z] p) asm "SECOND"; + +;;; Returns the third element of a triple tuple. +forall X, Y, Z -> Z triple_third([X, Y, Z] p) asm "THIRD"; + + +;;; Push null element (casted to given type) +;;; By the TVM type `Null` FunC represents absence of a value of some atomic type. +;;; So `null` can actually have any atomic type. +forall X -> X null() asm "PUSHNULL"; + +;;; Moves a variable [x] to the top of the stack +forall X -> (X, ()) ~impure_touch(X x) impure asm "NOP"; + + + +;;; Returns the current Unix time as an Integer +int now() asm "NOW"; + +;;; Returns the internal address of the current smart contract as a Slice with a `MsgAddressInt`. +;;; If necessary, it can be parsed further using primitives such as [parse_std_addr]. +slice my_address() asm "MYADDR"; + +;;; Returns the balance of the smart contract as a tuple consisting of an int +;;; (balance in nanotoncoins) and a `cell` +;;; (a dictionary with 32-bit keys representing the balance of "extra currencies") +;;; at the start of Computation Phase. +;;; Note that RAW primitives such as [send_raw_message] do not update this field. +[int, cell] get_balance() asm "BALANCE"; + +;;; Returns the logical time of the current transaction. +int cur_lt() asm "LTIME"; + +;;; Returns the starting logical time of the current block. +int block_lt() asm "BLOCKLT"; + +;;; Computes the representation hash of a `cell` [c] and returns it as a 256-bit unsigned integer `x`. +;;; Useful for signing and checking signatures of arbitrary entities represented by a tree of cells. +int cell_hash(cell c) asm "HASHCU"; + +;;; Computes the hash of a `slice s` and returns it as a 256-bit unsigned integer `x`. +;;; The result is the same as if an ordinary cell containing only data and references from `s` had been created +;;; and its hash computed by [cell_hash]. +int slice_hash(slice s) asm "HASHSU"; + +;;; Computes sha256 of the data bits of `slice` [s]. If the bit length of `s` is not divisible by eight, +;;; throws a cell underflow exception. The hash value is returned as a 256-bit unsigned integer `x`. +int string_hash(slice s) asm "SHA256U"; + +{- + # Signature checks +-} + +;;; Checks the Ed25519-`signature` of a `hash` (a 256-bit unsigned integer, usually computed as the hash of some data) +;;; using [public_key] (also represented by a 256-bit unsigned integer). +;;; The signature must contain at least 512 data bits; only the first 512 bits are used. +;;; The result is `−1` if the signature is valid, `0` otherwise. +;;; Note that `CHKSIGNU` creates a 256-bit slice with the hash and calls `CHKSIGNS`. +;;; That is, if [hash] is computed as the hash of some data, these data are hashed twice, +;;; the second hashing occurring inside `CHKSIGNS`. +int check_signature(int hash, slice signature, int public_key) asm "CHKSIGNU"; + +;;; Checks whether [signature] is a valid Ed25519-signature of the data portion of `slice data` using `public_key`, +;;; similarly to [check_signature]. +;;; If the bit length of [data] is not divisible by eight, throws a cell underflow exception. +;;; The verification of Ed25519 signatures is the standard one, +;;; with sha256 used to reduce [data] to the 256-bit number that is actually signed. +int check_data_signature(slice data, slice signature, int public_key) asm "CHKSIGNS"; + +{--- + # Computation of boc size + The primitives below may be useful for computing storage fees of user-provided data. +-} + +;;; Returns `(x, y, z, -1)` or `(null, null, null, 0)`. +;;; Recursively computes the count of distinct cells `x`, data bits `y`, and cell references `z` +;;; in the DAG rooted at `cell` [c], effectively returning the total storage used by this DAG taking into account +;;; the identification of equal cells. +;;; The values of `x`, `y`, and `z` are computed by a depth-first traversal of this DAG, +;;; with a hash table of visited cell hashes used to prevent visits of already-visited cells. +;;; The total count of visited cells `x` cannot exceed non-negative [max_cells]; +;;; otherwise the computation is aborted before visiting the `(max_cells + 1)`-st cell and +;;; a zero flag is returned to indicate failure. If [c] is `null`, returns `x = y = z = 0`. +(int, int, int) compute_data_size(cell c, int max_cells) impure asm "CDATASIZE"; + +;;; Similar to [compute_data_size?], but accepting a `slice` [s] instead of a `cell`. +;;; The returned value of `x` does not take into account the cell that contains the `slice` [s] itself; +;;; however, the data bits and the cell references of [s] are accounted for in `y` and `z`. +(int, int, int) slice_compute_data_size(slice s, int max_cells) impure asm "SDATASIZE"; + +;;; A non-quiet version of [compute_data_size?] that throws a cell overflow exception (`8`) on failure. +(int, int, int, int) compute_data_size?(cell c, int max_cells) asm "CDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT"; + +;;; A non-quiet version of [slice_compute_data_size?] that throws a cell overflow exception (8) on failure. +(int, int, int, int) slice_compute_data_size?(cell c, int max_cells) asm "SDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT"; + +;;; Throws an exception with exit_code excno if cond is not 0 (commented since implemented in compilator) +;; () throw_if(int excno, int cond) impure asm "THROWARGIF"; + +{-- + # Debug primitives + Only works for local TVM execution with debug level verbosity +-} +;;; Dumps the stack (at most the top 255 values) and shows the total stack depth. +() dump_stack() impure asm "DUMPSTK"; + +{- + # Persistent storage save and load +-} + +;;; Returns the persistent contract storage cell. It can be parsed or modified with slice and builder primitives later. +cell get_data() asm "c4 PUSH"; + +;;; Sets `cell` [c] as persistent contract data. You can update persistent contract storage with this primitive. +() set_data(cell c) impure asm "c4 POP"; + +{- + # Continuation primitives +-} +;;; Usually `c3` has a continuation initialized by the whole code of the contract. It is used for function calls. +;;; The primitive returns the current value of `c3`. +cont get_c3() impure asm "c3 PUSH"; + +;;; Updates the current value of `c3`. Usually, it is used for updating smart contract code in run-time. +;;; Note that after execution of this primitive the current code +;;; (and the stack of recursive function calls) won't change, +;;; but any other function call will use a function from the new code. +() set_c3(cont c) impure asm "c3 POP"; + +;;; Transforms a `slice` [s] into a simple ordinary continuation `c`, with `c.code = s` and an empty stack and savelist. +cont bless(slice s) impure asm "BLESS"; + +{--- + # Gas related primitives +-} + +;;; Sets current gas limit `gl` to its maximal allowed value `gm`, and resets the gas credit `gc` to zero, +;;; decreasing the value of `gr` by `gc` in the process. +;;; In other words, the current smart contract agrees to buy some gas to finish the current transaction. +;;; This action is required to process external messages, which bring no value (hence no gas) with themselves. +;;; +;;; For more details check [accept_message effects](https://ton.org/docs/#/smart-contracts/accept). +() accept_message() impure asm "ACCEPT"; + +;;; Sets current gas limit `gl` to the minimum of limit and `gm`, and resets the gas credit `gc` to zero. +;;; If the gas consumed so far (including the present instruction) exceeds the resulting value of `gl`, +;;; an (unhandled) out of gas exception is thrown before setting new gas limits. +;;; Notice that [set_gas_limit] with an argument `limit ≥ 2^63 − 1` is equivalent to [accept_message]. +() set_gas_limit(int limit) impure asm "SETGASLIMIT"; + +;;; Commits the current state of registers `c4` (“persistent data”) and `c5` (“actions”) +;;; so that the current execution is considered “successful” with the saved values even if an exception +;;; in Computation Phase is thrown later. +() commit() impure asm "COMMIT"; + +;;; Not implemented +;;; Computes the amount of gas that can be bought for `amount` nanoTONs, +;;; and sets `gl` accordingly in the same way as [set_gas_limit]. +;;() buy_gas(int amount) impure asm "BUYGAS"; + +;;; Computes the minimum of two integers [x] and [y]. +int min(int x, int y) asm "MIN"; + +;;; Computes the maximum of two integers [x] and [y]. +int max(int x, int y) asm "MAX"; + +;;; Sorts two integers. +(int, int) minmax(int x, int y) asm "MINMAX"; + +;;; Computes the absolute value of an integer [x]. +int abs(int x) asm "ABS"; + +{- + # Slice primitives + + It is said that a primitive _loads_ some data, + if it returns the data and the remainder of the slice + (so it can also be used as [modifying method](https://ton.org/docs/#/func/statements?id=modifying-methods)). + + It is said that a primitive _preloads_ some data, if it returns only the data + (it can be used as [non-modifying method](https://ton.org/docs/#/func/statements?id=non-modifying-methods)). + + Unless otherwise stated, loading and preloading primitives read the data from a prefix of the slice. +-} + + +;;; Converts a `cell` [c] into a `slice`. Notice that [c] must be either an ordinary cell, +;;; or an exotic cell (see [TVM.pdf](https://ton-blockchain.github.io/docs/tvm.pdf), 3.1.2) +;;; which is automatically loaded to yield an ordinary cell `c'`, converted into a `slice` afterwards. +slice begin_parse(cell c) asm "CTOS"; + +;;; Checks if [s] is empty. If not, throws an exception. +() end_parse(slice s) impure asm "ENDS"; + +;;; Loads the first reference from the slice. +(slice, cell) load_ref(slice s) asm(-> 1 0) "LDREF"; + +;;; Preloads the first reference from the slice. +cell preload_ref(slice s) asm "PLDREF"; + +{- Functions below are commented because are implemented on compilator level for optimisation -} + +;;; Loads a signed [len]-bit integer from a slice [s]. +;; (slice, int) ~load_int(slice s, int len) asm(s len -> 1 0) "LDIX"; + +;;; Loads an unsigned [len]-bit integer from a slice [s]. +;; (slice, int) ~load_uint(slice s, int len) asm( -> 1 0) "LDUX"; + +;;; Preloads a signed [len]-bit integer from a slice [s]. +;; int preload_int(slice s, int len) asm "PLDIX"; + +;;; Preloads an unsigned [len]-bit integer from a slice [s]. +;; int preload_uint(slice s, int len) asm "PLDUX"; + +;;; Loads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate `slice s''`. +;; (slice, slice) load_bits(slice s, int len) asm(s len -> 1 0) "LDSLICEX"; + +;;; Preloads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate `slice s''`. +;; slice preload_bits(slice s, int len) asm "PLDSLICEX"; + +;;; Loads serialized amount of TonCoins (any unsigned integer up to `2^120 - 1`). +(slice, int) load_grams(slice s) asm(-> 1 0) "LDGRAMS"; +(slice, int) load_coins(slice s) asm(-> 1 0) "LDVARUINT16"; + +(slice, int) load_varint16(slice s) asm(-> 1 0) "LDVARINT16"; +(slice, int) load_varint32(slice s) asm(-> 1 0) "LDVARINT32"; +(slice, int) load_varuint16(slice s) asm(-> 1 0) "LDVARUINT16"; +(slice, int) load_varuint32(slice s) asm(-> 1 0) "LDVARUINT32"; + +;;; Returns all but the first `0 ≤ len ≤ 1023` bits of `slice` [s]. +slice skip_bits(slice s, int len) asm "SDSKIPFIRST"; +(slice, ()) ~skip_bits(slice s, int len) asm "SDSKIPFIRST"; + +;;; Returns the first `0 ≤ len ≤ 1023` bits of `slice` [s]. +slice first_bits(slice s, int len) asm "SDCUTFIRST"; + +;;; Returns all but the last `0 ≤ len ≤ 1023` bits of `slice` [s]. +slice skip_last_bits(slice s, int len) asm "SDSKIPLAST"; +(slice, ()) ~skip_last_bits(slice s, int len) asm "SDSKIPLAST"; + +;;; Returns the last `0 ≤ len ≤ 1023` bits of `slice` [s]. +slice slice_last(slice s, int len) asm "SDCUTLAST"; + +;;; Loads a dictionary `D` (HashMapE) from `slice` [s]. +;;; (returns `null` if `nothing` constructor is used). +(slice, cell) load_dict(slice s) asm(-> 1 0) "LDDICT"; + +;;; Preloads a dictionary `D` from `slice` [s]. +cell preload_dict(slice s) asm "PLDDICT"; + +;;; Loads a dictionary as [load_dict], but returns only the remainder of the slice. +slice skip_dict(slice s) asm "SKIPDICT"; +(slice, ()) ~skip_dict(slice s) asm "SKIPDICT"; + +;;; Loads (Maybe ^Cell) from `slice` [s]. +;;; In other words loads 1 bit and if it is true +;;; loads first ref and return it with slice remainder +;;; otherwise returns `null` and slice remainder +(slice, cell) load_maybe_ref(slice s) asm(-> 1 0) "LDOPTREF"; + +;;; Preloads (Maybe ^Cell) from `slice` [s]. +cell preload_maybe_ref(slice s) asm "PLDOPTREF"; + + +;;; Returns the depth of `cell` [c]. +;;; If [c] has no references, then return `0`; +;;; otherwise the returned value is one plus the maximum of depths of cells referred to from [c]. +;;; If [c] is a `null` instead of a cell, returns zero. +int cell_depth(cell c) asm "CDEPTH"; + + +{- + # Slice size primitives +-} + +;;; Returns the number of references in `slice` [s]. +int slice_refs(slice s) asm "SREFS"; + +;;; Returns the number of data bits in `slice` [s]. +int slice_bits(slice s) asm "SBITS"; + +;;; Returns both the number of data bits and the number of references in `slice` [s]. +(int, int) slice_bits_refs(slice s) asm "SBITREFS"; + +;;; Checks whether a `slice` [s] is empty (i.e., contains no bits of data and no cell references). +int slice_empty?(slice s) asm "SEMPTY"; + +;;; Checks whether `slice` [s] has no bits of data. +int slice_data_empty?(slice s) asm "SDEMPTY"; + +;;; Checks whether `slice` [s] has no references. +int slice_refs_empty?(slice s) asm "SREMPTY"; + +;;; Returns the depth of `slice` [s]. +;;; If [s] has no references, then returns `0`; +;;; otherwise the returned value is one plus the maximum of depths of cells referred to from [s]. +int slice_depth(slice s) asm "SDEPTH"; + +{- + # Builder size primitives +-} + +;;; Returns the number of cell references already stored in `builder` [b] +int builder_refs(builder b) asm "BREFS"; + +;;; Returns the number of data bits already stored in `builder` [b]. +int builder_bits(builder b) asm "BBITS"; + +;;; Returns the depth of `builder` [b]. +;;; If no cell references are stored in [b], then returns 0; +;;; otherwise the returned value is one plus the maximum of depths of cells referred to from [b]. +int builder_depth(builder b) asm "BDEPTH"; + +{- + # Builder primitives + It is said that a primitive _stores_ a value `x` into a builder `b` + if it returns a modified version of the builder `b'` with the value `x` stored at the end of it. + It can be used as [non-modifying method](https://ton.org/docs/#/func/statements?id=non-modifying-methods). + + All the primitives below first check whether there is enough space in the `builder`, + and only then check the range of the value being serialized. +-} + +;;; Creates a new empty `builder`. +builder begin_cell() asm "NEWC"; + +;;; Converts a `builder` into an ordinary `cell`. +cell end_cell(builder b) asm "ENDC"; + +;;; Stores a reference to `cell` [c] into `builder` [b]. +builder store_ref(builder b, cell c) asm(c b) "STREF"; + +;;; Stores an unsigned [len]-bit integer `x` into `b` for `0 ≤ len ≤ 256`. +;; builder store_uint(builder b, int x, int len) asm(x b len) "STUX"; + +;;; Stores a signed [len]-bit integer `x` into `b` for` 0 ≤ len ≤ 257`. +;; builder store_int(builder b, int x, int len) asm(x b len) "STIX"; + + +;;; Stores `slice` [s] into `builder` [b] +builder store_slice(builder b, slice s) asm "STSLICER"; + +;;; Stores (serializes) an integer [x] in the range `0..2^120 − 1` into `builder` [b]. +;;; The serialization of [x] consists of a 4-bit unsigned big-endian integer `l`, +;;; which is the smallest integer `l ≥ 0`, such that `x < 2^8l`, +;;; followed by an `8l`-bit unsigned big-endian representation of [x]. +;;; If [x] does not belong to the supported range, a range check exception is thrown. +;;; +;;; Store amounts of TonCoins to the builder as VarUInteger 16 +builder store_grams(builder b, int x) asm "STGRAMS"; +builder store_coins(builder b, int x) asm "STVARUINT16"; + +builder store_varint16(builder b, int x) asm "STVARINT16"; +builder store_varint32(builder b, int x) asm "STVARINT32"; +builder store_varuint16(builder b, int x) asm "STVARUINT16"; +builder store_varuint32(builder b, int x) asm "STVARUINT32"; + +;;; Stores dictionary `D` represented by `cell` [c] or `null` into `builder` [b]. +;;; In other words, stores a `1`-bit and a reference to [c] if [c] is not `null` and `0`-bit otherwise. +builder store_dict(builder b, cell c) asm(c b) "STDICT"; + +;;; Stores (Maybe ^Cell) to builder: +;;; if cell is null store 1 zero bit +;;; otherwise store 1 true bit and ref to cell +builder store_maybe_ref(builder b, cell c) asm(c b) "STOPTREF"; + + +{- + # Address manipulation primitives + The address manipulation primitives listed below serialize and deserialize values according to the following TL-B scheme: + ```TL-B + addr_none$00 = MsgAddressExt; + addr_extern$01 len:(## 8) external_address:(bits len) + = MsgAddressExt; + anycast_info$_ depth:(#<= 30) { depth >= 1 } + rewrite_pfx:(bits depth) = Anycast; + addr_std$10 anycast:(Maybe Anycast) + workchain_id:int8 address:bits256 = MsgAddressInt; + addr_var$11 anycast:(Maybe Anycast) addr_len:(## 9) + workchain_id:int32 address:(bits addr_len) = MsgAddressInt; + _ _:MsgAddressInt = MsgAddress; + _ _:MsgAddressExt = MsgAddress; + + int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool + src:MsgAddress dest:MsgAddressInt + value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams + created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; + ext_out_msg_info$11 src:MsgAddress dest:MsgAddressExt + created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; + ``` + A deserialized `MsgAddress` is represented by a tuple `t` as follows: + + - `addr_none` is represented by `t = (0)`, + i.e., a tuple containing exactly one integer equal to zero. + - `addr_extern` is represented by `t = (1, s)`, + where slice `s` contains the field `external_address`. In other words, ` + t` is a pair (a tuple consisting of two entries), containing an integer equal to one and slice `s`. + - `addr_std` is represented by `t = (2, u, x, s)`, + where `u` is either a `null` (if `anycast` is absent) or a slice `s'` containing `rewrite_pfx` (if anycast is present). + Next, integer `x` is the `workchain_id`, and slice `s` contains the address. + - `addr_var` is represented by `t = (3, u, x, s)`, + where `u`, `x`, and `s` have the same meaning as for `addr_std`. +-} + +;;; Loads from slice [s] the only prefix that is a valid `MsgAddress`, +;;; and returns both this prefix `s'` and the remainder `s''` of [s] as slices. +(slice, slice) load_msg_addr(slice s) asm(-> 1 0) "LDMSGADDR"; + +;;; Decomposes slice [s] containing a valid `MsgAddress` into a `tuple t` with separate fields of this `MsgAddress`. +;;; If [s] is not a valid `MsgAddress`, a cell deserialization exception is thrown. +tuple parse_addr(slice s) asm "PARSEMSGADDR"; + +;;; Parses slice [s] containing a valid `MsgAddressInt` (usually a `msg_addr_std`), +;;; applies rewriting from the anycast (if present) to the same-length prefix of the address, +;;; and returns both the workchain and the 256-bit address as integers. +;;; If the address is not 256-bit, or if [s] is not a valid serialization of `MsgAddressInt`, +;;; throws a cell deserialization exception. +(int, int) parse_std_addr(slice s) asm "REWRITESTDADDR"; + +;;; A variant of [parse_std_addr] that returns the (rewritten) address as a slice [s], +;;; even if it is not exactly 256 bit long (represented by a `msg_addr_var`). +(int, slice) parse_var_addr(slice s) asm "REWRITEVARADDR"; + +{- + # Dictionary primitives +-} + + +;;; Sets the value associated with [key_len]-bit key signed index in dictionary [dict] to [value] (cell), +;;; and returns the resulting dictionary. +cell idict_set_ref(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTISETREF"; +(cell, ()) ~idict_set_ref(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTISETREF"; + +;;; Sets the value associated with [key_len]-bit key unsigned index in dictionary [dict] to [value] (cell), +;;; and returns the resulting dictionary. +cell udict_set_ref(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTUSETREF"; +(cell, ()) ~udict_set_ref(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTUSETREF"; + +cell idict_get_ref(cell dict, int key_len, int index) asm(index dict key_len) "DICTIGETOPTREF"; +(cell, int) idict_get_ref?(cell dict, int key_len, int index) asm(index dict key_len) "DICTIGETREF" "NULLSWAPIFNOT"; +(cell, int) udict_get_ref?(cell dict, int key_len, int index) asm(index dict key_len) "DICTUGETREF" "NULLSWAPIFNOT"; +(cell, cell) idict_set_get_ref(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTISETGETOPTREF"; +(cell, cell) udict_set_get_ref(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTUSETGETOPTREF"; +(cell, int) idict_delete?(cell dict, int key_len, int index) asm(index dict key_len) "DICTIDEL"; +(cell, int) udict_delete?(cell dict, int key_len, int index) asm(index dict key_len) "DICTUDEL"; +(slice, int) idict_get?(cell dict, int key_len, int index) asm(index dict key_len) "DICTIGET" "NULLSWAPIFNOT"; +(slice, int) udict_get?(cell dict, int key_len, int index) asm(index dict key_len) "DICTUGET" "NULLSWAPIFNOT"; +(cell, slice, int) idict_delete_get?(cell dict, int key_len, int index) asm(index dict key_len) "DICTIDELGET" "NULLSWAPIFNOT"; +(cell, slice, int) udict_delete_get?(cell dict, int key_len, int index) asm(index dict key_len) "DICTUDELGET" "NULLSWAPIFNOT"; +(cell, (slice, int)) ~idict_delete_get?(cell dict, int key_len, int index) asm(index dict key_len) "DICTIDELGET" "NULLSWAPIFNOT"; +(cell, (slice, int)) ~udict_delete_get?(cell dict, int key_len, int index) asm(index dict key_len) "DICTUDELGET" "NULLSWAPIFNOT"; +(cell, cell, int) idict_delete_get_ref?(cell dict, int key_len, int index) asm(index dict key_len) "DICTIDELGETREF" "NULLSWAPIFNOT"; +(cell, cell, int) udict_delete_get_ref?(cell dict, int key_len, int index) asm(index dict key_len) "DICTUDELGETREF" "NULLSWAPIFNOT"; +(cell, (cell, int)) ~idict_delete_get_ref?(cell dict, int key_len, int index) asm(index dict key_len) "DICTIDELGETREF" "NULLSWAPIFNOT"; +(cell, (cell, int)) ~udict_delete_get_ref?(cell dict, int key_len, int index) asm(index dict key_len) "DICTUDELGETREF" "NULLSWAPIFNOT"; +cell udict_set(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTUSET"; +(cell, ()) ~udict_set(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTUSET"; +cell idict_set(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTISET"; +(cell, ()) ~idict_set(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTISET"; +cell dict_set(cell dict, int key_len, slice index, slice value) asm(value index dict key_len) "DICTSET"; +(cell, ()) ~dict_set(cell dict, int key_len, slice index, slice value) asm(value index dict key_len) "DICTSET"; +(cell, int) udict_add?(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTUADD"; +(cell, int) udict_replace?(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTUREPLACE"; +(cell, int) udict_replace_ref?(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTUREPLACEREF"; +(cell, slice, int) udict_replaceget?(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTUREPLACEGET" "NULLSWAPIFNOT"; +(cell, cell, int) udict_replaceget_ref?(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTUREPLACEGETREF" "NULLSWAPIFNOT"; +(cell, (slice, int)) ~udict_replaceget?(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTUREPLACEGET" "NULLSWAPIFNOT"; +(cell, (cell, int)) ~udict_replaceget_ref?(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTUREPLACEGETREF" "NULLSWAPIFNOT"; +(cell, int) idict_add?(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTIADD"; +(cell, int) idict_replace?(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTIREPLACE"; +(cell, int) idict_replace_ref?(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTIREPLACEREF"; +(cell, slice, int) idict_replaceget?(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTIREPLACEGET" "NULLSWAPIFNOT"; +(cell, cell, int) idict_replaceget_ref?(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTIREPLACEGETREF" "NULLSWAPIFNOT"; +(cell, (slice, int)) ~idict_replaceget?(cell dict, int key_len, int index, slice value) asm(value index dict key_len) "DICTIREPLACEGET" "NULLSWAPIFNOT"; +(cell, (cell, int)) ~idict_replaceget_ref?(cell dict, int key_len, int index, cell value) asm(value index dict key_len) "DICTIREPLACEGETREF" "NULLSWAPIFNOT"; +cell udict_set_builder(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTUSETB"; +(cell, ()) ~udict_set_builder(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTUSETB"; +cell idict_set_builder(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTISETB"; +(cell, ()) ~idict_set_builder(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTISETB"; +cell dict_set_builder(cell dict, int key_len, slice index, builder value) asm(value index dict key_len) "DICTSETB"; +(cell, ()) ~dict_set_builder(cell dict, int key_len, slice index, builder value) asm(value index dict key_len) "DICTSETB"; +(cell, int) dict_replace_builder?(cell dict, int key_len, slice index, builder value) asm(value index dict key_len) "DICTREPLACEB"; +(cell, builder, int) dict_replaceget_builder?(cell dict, int key_len, slice index, builder value) asm(value index dict key_len) "DICTREPLACEGETB" "NULLSWAPIFNOT"; +(cell, slice, int) dict_replaceget?(cell dict, int key_len, slice index, slice value) asm(value index dict key_len) "DICTREPLACEGET" "NULLSWAPIFNOT"; +(cell, (builder, int)) ~dict_replaceget_builder?(cell dict, int key_len, slice index, builder value) asm(value index dict key_len) "DICTREPLACEGETB" "NULLSWAPIFNOT"; +(cell, (slice, int)) ~dict_replaceget?(cell dict, int key_len, slice index, slice value) asm(value index dict key_len) "DICTREPLACEGET" "NULLSWAPIFNOT"; +(cell, int) udict_add_builder?(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTUADDB"; +(cell, int) udict_replace_builder?(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTUREPLACEB"; +(cell, builder, int) udict_replaceget_builder?(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTUREPLACEGETB" "NULLSWAPIFNOT"; +(cell, (builder, int)) ~udict_replaceget_builder?(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTUREPLACEGETB" "NULLSWAPIFNOT"; +(cell, int) idict_add_builder?(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTIADDB"; +(cell, int) idict_replace_builder?(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTIREPLACEB"; +(cell, builder, int) idict_replaceget_builder?(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTIREPLACEGETB" "NULLSWAPIFNOT"; +(cell, (builder, int)) ~idict_replaceget_builder?(cell dict, int key_len, int index, builder value) asm(value index dict key_len) "DICTIREPLACEGETB" "NULLSWAPIFNOT"; +(cell, int, slice, int) udict_delete_get_min(cell dict, int key_len) asm(-> 0 2 1 3) "DICTUREMMIN" "NULLSWAPIFNOT2"; +(cell, (int, slice, int)) ~udict::delete_get_min(cell dict, int key_len) asm(-> 0 2 1 3) "DICTUREMMIN" "NULLSWAPIFNOT2"; +(cell, int, slice, int) idict_delete_get_min(cell dict, int key_len) asm(-> 0 2 1 3) "DICTIREMMIN" "NULLSWAPIFNOT2"; +(cell, (int, slice, int)) ~idict::delete_get_min(cell dict, int key_len) asm(-> 0 2 1 3) "DICTIREMMIN" "NULLSWAPIFNOT2"; +(cell, slice, slice, int) dict_delete_get_min(cell dict, int key_len) asm(-> 0 2 1 3) "DICTREMMIN" "NULLSWAPIFNOT2"; +(cell, (slice, slice, int)) ~dict::delete_get_min(cell dict, int key_len) asm(-> 0 2 1 3) "DICTREMMIN" "NULLSWAPIFNOT2"; +(cell, int, slice, int) udict_delete_get_max(cell dict, int key_len) asm(-> 0 2 1 3) "DICTUREMMAX" "NULLSWAPIFNOT2"; +(cell, (int, slice, int)) ~udict::delete_get_max(cell dict, int key_len) asm(-> 0 2 1 3) "DICTUREMMAX" "NULLSWAPIFNOT2"; +(cell, int, slice, int) idict_delete_get_max(cell dict, int key_len) asm(-> 0 2 1 3) "DICTIREMMAX" "NULLSWAPIFNOT2"; +(cell, (int, slice, int)) ~idict::delete_get_max(cell dict, int key_len) asm(-> 0 2 1 3) "DICTIREMMAX" "NULLSWAPIFNOT2"; +(cell, slice, slice, int) dict_delete_get_max(cell dict, int key_len) asm(-> 0 2 1 3) "DICTREMMAX" "NULLSWAPIFNOT2"; +(cell, (slice, slice, int)) ~dict::delete_get_max(cell dict, int key_len) asm(-> 0 2 1 3) "DICTREMMAX" "NULLSWAPIFNOT2"; +(int, slice, int) udict_get_min?(cell dict, int key_len) asm (-> 1 0 2) "DICTUMIN" "NULLSWAPIFNOT2"; +(int, slice, int) udict_get_max?(cell dict, int key_len) asm (-> 1 0 2) "DICTUMAX" "NULLSWAPIFNOT2"; +(int, cell, int) udict_get_min_ref?(cell dict, int key_len) asm (-> 1 0 2) "DICTUMINREF" "NULLSWAPIFNOT2"; +(int, cell, int) udict_get_max_ref?(cell dict, int key_len) asm (-> 1 0 2) "DICTUMAXREF" "NULLSWAPIFNOT2"; +(int, slice, int) idict_get_min?(cell dict, int key_len) asm (-> 1 0 2) "DICTIMIN" "NULLSWAPIFNOT2"; +(int, slice, int) idict_get_max?(cell dict, int key_len) asm (-> 1 0 2) "DICTIMAX" "NULLSWAPIFNOT2"; +(int, cell, int) idict_get_min_ref?(cell dict, int key_len) asm (-> 1 0 2) "DICTIMINREF" "NULLSWAPIFNOT2"; +(int, cell, int) idict_get_max_ref?(cell dict, int key_len) asm (-> 1 0 2) "DICTIMAXREF" "NULLSWAPIFNOT2"; +(int, slice, int) udict_get_next?(cell dict, int key_len, int pivot) asm(pivot dict key_len -> 1 0 2) "DICTUGETNEXT" "NULLSWAPIFNOT2"; +(int, slice, int) udict_get_nexteq?(cell dict, int key_len, int pivot) asm(pivot dict key_len -> 1 0 2) "DICTUGETNEXTEQ" "NULLSWAPIFNOT2"; +(int, slice, int) udict_get_prev?(cell dict, int key_len, int pivot) asm(pivot dict key_len -> 1 0 2) "DICTUGETPREV" "NULLSWAPIFNOT2"; +(int, slice, int) udict_get_preveq?(cell dict, int key_len, int pivot) asm(pivot dict key_len -> 1 0 2) "DICTUGETPREVEQ" "NULLSWAPIFNOT2"; +(int, slice, int) idict_get_next?(cell dict, int key_len, int pivot) asm(pivot dict key_len -> 1 0 2) "DICTIGETNEXT" "NULLSWAPIFNOT2"; +(int, slice, int) idict_get_nexteq?(cell dict, int key_len, int pivot) asm(pivot dict key_len -> 1 0 2) "DICTIGETNEXTEQ" "NULLSWAPIFNOT2"; +(int, slice, int) idict_get_prev?(cell dict, int key_len, int pivot) asm(pivot dict key_len -> 1 0 2) "DICTIGETPREV" "NULLSWAPIFNOT2"; +(int, slice, int) idict_get_preveq?(cell dict, int key_len, int pivot) asm(pivot dict key_len -> 1 0 2) "DICTIGETPREVEQ" "NULLSWAPIFNOT2"; + +;;; Creates an empty dictionary, which is actually a null value. Equivalent to PUSHNULL +cell new_dict() asm "NEWDICT"; +;;; Checks whether a dictionary is empty. Equivalent to cell_null?. +int dict_empty?(cell c) asm "DICTEMPTY"; + + +{- Prefix dictionary primitives -} +(slice, slice, slice, int) pfxdict_get?(cell dict, int key_len, slice key) asm(key dict key_len) "PFXDICTGETQ" "NULLSWAPIFNOT2"; +(cell, int) pfxdict_set?(cell dict, int key_len, slice key, slice value) asm(value key dict key_len) "PFXDICTSET"; +(cell, int) pfxdict_delete?(cell dict, int key_len, slice key) asm(key dict key_len) "PFXDICTDEL"; + +;;; Returns the value of the global configuration parameter with integer index `i` as a `cell` or `null` value. +cell config_param(int x) asm "CONFIGOPTPARAM"; +;;; Checks whether c is a null. Note, that FunC also has polymorphic null? built-in. +int cell_null?(cell c) asm "ISNULL"; + +;;; Creates an output action which would reserve exactly amount nanotoncoins (if mode = 0), at most amount nanotoncoins (if mode = 2), or all but amount nanotoncoins (if mode = 1 or mode = 3), from the remaining balance of the account. It is roughly equivalent to creating an outbound message carrying amount nanotoncoins (or b − amount nanotoncoins, where b is the remaining balance) to oneself, so that the subsequent output actions would not be able to spend more money than the remainder. Bit +2 in mode means that the external action does not fail if the specified amount cannot be reserved; instead, all remaining balance is reserved. Bit +8 in mode means `amount <- -amount` before performing any further actions. Bit +4 in mode means that amount is increased by the original balance of the current account (before the compute phase), including all extra currencies, before performing any other checks and actions. Currently, amount must be a non-negative integer, and mode must be in the range 0..15. +() raw_reserve(int amount, int mode) impure asm "RAWRESERVE"; +;;; Similar to raw_reserve, but also accepts a dictionary extra_amount (represented by a cell or null) with extra currencies. In this way currencies other than TonCoin can be reserved. +() raw_reserve_extra(int amount, cell extra_amount, int mode) impure asm "RAWRESERVEX"; +;;; Sends a raw message contained in msg, which should contain a correctly serialized object Message X, with the only exception that the source address is allowed to have dummy value addr_none (to be automatically replaced with the current smart contract address), and ihr_fee, fwd_fee, created_lt and created_at fields can have arbitrary values (to be rewritten with correct values during the action phase of the current transaction). Integer parameter mode contains the flags. Currently mode = 0 is used for ordinary messages; mode = 128 is used for messages that are to carry all the remaining balance of the current smart contract (instead of the value originally indicated in the message); mode = 64 is used for messages that carry all the remaining value of the inbound message in addition to the value initially indicated in the new message (if bit 0 is not set, the gas fees are deducted from this amount); mode' = mode + 1 means that the sender wants to pay transfer fees separately; mode' = mode + 2 means that any errors arising while processing this message during the action phase should be ignored. Finally, mode' = mode + 32 means that the current account must be destroyed if its resulting balance is zero. This flag is usually employed together with +128. +() send_raw_message(cell msg, int mode) impure asm "SENDRAWMSG"; +;;; Creates an output action that would change this smart contract code to that given by cell new_code. Notice that this change will take effect only after the successful termination of the current run of the smart contract +() set_code(cell new_code) impure asm "SETCODE"; + +;;; Generates a new pseudo-random unsigned 256-bit integer x. The algorithm is as follows: if r is the old value of the random seed, considered as a 32-byte array (by constructing the big-endian representation of an unsigned 256-bit integer), then its sha512(r) is computed; the first 32 bytes of this hash are stored as the new value r' of the random seed, and the remaining 32 bytes are returned as the next random value x. +int random() impure asm "RANDU256"; +;;; Generates a new pseudo-random integer z in the range 0..range−1 (or range..−1, if range < 0). More precisely, an unsigned random value x is generated as in random; then z := x * range / 2^256 is computed. +int rand(int range) impure asm "RAND"; +;;; Returns the current random seed as an unsigned 256-bit Integer. +int get_seed() impure asm "RANDSEED"; +;;; Sets the random seed to unsigned 256-bit seed. +() set_seed(int x) impure asm "SETRAND"; +;;; Mixes unsigned 256-bit integer x into the random seed r by setting the random seed to sha256 of the concatenation of two 32-byte strings: the first with the big-endian representation of the old seed r, and the second with the big-endian representation of x. +() randomize(int x) impure asm "ADDRAND"; +;;; Equivalent to randomize(cur_lt());. +() randomize_lt() impure asm "LTIME" "ADDRAND"; + +;;; Checks whether the data parts of two slices coinside +int equal_slices_bits(slice a, slice b) asm "SDEQ"; +;;; Checks whether b is a null. Note, that FunC also has polymorphic null? built-in. +int builder_null?(builder b) asm "ISNULL"; +;;; Concatenates two builders +builder store_builder(builder to, builder from) asm "STBR"; + +;; CUSTOM: + +;; TVM UPGRADE 2023-07 https://docs.ton.org/learn/tvm-instructions/tvm-upgrade-2023-07 +;; In mainnet since 20 Dec 2023 https://t.me/tonblockchain/226 + +;;; Retrieves code of smart-contract from c7 + +cell my_code() asm "MYCODE"; diff --git a/blockchain-explorer/CMakeLists.txt b/blockchain-explorer/CMakeLists.txt index 36d4e506..86cb3e74 100644 --- a/blockchain-explorer/CMakeLists.txt +++ b/blockchain-explorer/CMakeLists.txt @@ -36,6 +36,7 @@ endif() target_include_directories(blockchain-explorer PUBLIC ${MHD_INCLUDE_DIR}) target_link_libraries(blockchain-explorer tdactor adnllite tl_lite_api tl-lite-utils ton_crypto ${MHD_LIBRARY}) +target_link_libraries(blockchain-explorer lite-client-common) install(TARGETS blockchain-explorer RUNTIME DESTINATION bin) diff --git a/blockchain-explorer/blockchain-explorer-query.cpp b/blockchain-explorer/blockchain-explorer-query.cpp index 1808a3c4..26a6787e 100644 --- a/blockchain-explorer/blockchain-explorer-query.cpp +++ b/blockchain-explorer/blockchain-explorer-query.cpp @@ -1432,7 +1432,7 @@ void HttpQueryStatus::finish_query() { for (td::uint32 i = 0; i < results_.ips.size(); i++) { A << ""; if (results_.ips[i].is_valid()) { - A << "" << results_.ips[i] << ""; + A << "" << results_.ips[i].get_ip_str() << ":" << results_.ips[i].get_port() << ""; } else { A << "hidden"; } diff --git a/blockchain-explorer/blockchain-explorer.cpp b/blockchain-explorer/blockchain-explorer.cpp index 3b5346b7..ca50d526 100644 --- a/blockchain-explorer/blockchain-explorer.cpp +++ b/blockchain-explorer/blockchain-explorer.cpp @@ -57,6 +57,7 @@ #include "auto/tl/lite_api.h" #include "ton/lite-tl.hpp" #include "tl-utils/lite-utils.hpp" +#include "lite-client/ext-client.h" #include @@ -127,7 +128,7 @@ class CoreActor : public CoreActorInterface { private: std::string global_config_ = "ton-global.config"; - std::vector> clients_; + td::actor::ActorOwn client_; td::uint32 http_port_ = 80; MHD_Daemon* daemon_ = nullptr; @@ -137,35 +138,29 @@ class CoreActor : public CoreActorInterface { bool hide_ips_ = false; - std::unique_ptr make_callback(td::uint32 idx) { - class Callback : public ton::adnl::AdnlExtClient::Callback { + td::unique_ptr make_callback() { + class Callback : public liteclient::ExtClient::Callback { public: - void on_ready() override { - td::actor::send_closure(id_, &CoreActor::conn_ready, idx_); - } - void on_stop_ready() override { - td::actor::send_closure(id_, &CoreActor::conn_closed, idx_); - } - Callback(td::actor::ActorId id, td::uint32 idx) : id_(std::move(id)), idx_(idx) { + Callback(td::actor::ActorId id) : id_(std::move(id)) { } private: td::actor::ActorId id_; - td::uint32 idx_; }; - return std::make_unique(actor_id(this), idx); + return td::make_unique(actor_id(this)); } std::shared_ptr new_result_; td::int32 attempt_ = 0; td::int32 waiting_ = 0; - std::vector ready_; + size_t n_servers_ = 0; void run_queries(); - void got_result(td::uint32 idx, td::int32 attempt, td::Result data); - void send_query(td::uint32 idx); + void got_servers_ready(td::int32 attempt, std::vector ready); + void send_ping(td::uint32 idx); + void got_ping_result(td::uint32 idx, td::int32 attempt, td::Result data); void add_result() { if (new_result_) { @@ -196,12 +191,6 @@ class CoreActor : public CoreActorInterface { static CoreActor* instance_; td::actor::ActorId self_id_; - void conn_ready(td::uint32 idx) { - ready_.at(idx) = true; - } - void conn_closed(td::uint32 idx) { - ready_.at(idx) = false; - } void set_global_config(std::string str) { global_config_ = str; } @@ -226,10 +215,7 @@ class CoreActor : public CoreActorInterface { hide_ips_ = value; } - void send_lite_query(td::uint32 idx, td::BufferSlice query, td::Promise promise); - void send_lite_query(td::BufferSlice data, td::Promise promise) override { - return send_lite_query(0, std::move(data), std::move(promise)); - } + void send_lite_query(td::BufferSlice query, td::Promise promise) override; void get_last_result(td::Promise> promise) override { } void get_results(td::uint32 max, td::Promise promise) override { @@ -449,33 +435,27 @@ class CoreActor : public CoreActorInterface { } void run() { + std::vector servers; if (remote_public_key_.empty()) { auto G = td::read_file(global_config_).move_as_ok(); auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); ton::ton_api::liteclient_config_global gc; ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); - - CHECK(gc.liteservers_.size() > 0); - td::uint32 size = static_cast(gc.liteservers_.size()); - ready_.resize(size, false); - - for (td::uint32 i = 0; i < size; i++) { - auto& cli = gc.liteservers_[i]; - td::IPAddress addr; - addr.init_host_port(td::IPAddress::ipv4_to_str(cli->ip_), cli->port_).ensure(); - addrs_.push_back(addr); - clients_.emplace_back(ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull::create(cli->id_).move_as_ok(), - addr, make_callback(i))); + auto r_servers = liteclient::LiteServerConfig::parse_global_config(gc); + r_servers.ensure(); + servers = r_servers.move_as_ok(); + for (const auto& serv : servers) { + addrs_.push_back(serv.addr); } } else { if (!remote_addr_.is_valid()) { LOG(FATAL) << "remote addr not set"; } - ready_.resize(1, false); addrs_.push_back(remote_addr_); - clients_.emplace_back(ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull{remote_public_key_}, - remote_addr_, make_callback(0))); + servers.push_back(liteclient::LiteServerConfig{ton::adnl::AdnlNodeIdFull{remote_public_key_}, remote_addr_}); } + n_servers_ = servers.size(); + client_ = liteclient::ExtClient::create(std::move(servers), make_callback(), true); daemon_ = MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, static_cast(http_port_), nullptr, nullptr, &process_http_request, nullptr, MHD_OPTION_NOTIFY_COMPLETED, request_completed, nullptr, MHD_OPTION_THREAD_POOL_SIZE, 16, MHD_OPTION_END); @@ -483,7 +463,46 @@ class CoreActor : public CoreActorInterface { } }; -void CoreActor::got_result(td::uint32 idx, td::int32 attempt, td::Result R) { +void CoreActor::run_queries() { + waiting_ = 0; + new_result_ = std::make_shared(n_servers_, td::Timestamp::at_unix(attempt_ * 60)); + td::actor::send_closure(client_, &liteclient::ExtClient::get_servers_status, + [SelfId = actor_id(this), attempt = attempt_](td::Result> R) { + R.ensure(); + td::actor::send_closure(SelfId, &CoreActor::got_servers_ready, attempt, R.move_as_ok()); + }); +} + +void CoreActor::got_servers_ready(td::int32 attempt, std::vector ready) { + if (attempt != attempt_) { + return; + } + CHECK(ready.size() == n_servers_); + for (td::uint32 i = 0; i < n_servers_; i++) { + if (ready[i]) { + send_ping(i); + } + } + CHECK(waiting_ >= 0); + if (waiting_ == 0) { + add_result(); + } +} + +void CoreActor::send_ping(td::uint32 idx) { + waiting_++; + auto query = ton::create_tl_object(); + auto q = ton::create_tl_object(serialize_tl_object(query, true)); + + auto P = + td::PromiseCreator::lambda([SelfId = actor_id(this), idx, attempt = attempt_](td::Result R) { + td::actor::send_closure(SelfId, &CoreActor::got_ping_result, idx, attempt, std::move(R)); + }); + td::actor::send_closure(client_, &liteclient::ExtClient::send_query_to_server, "query", serialize_tl_object(q, true), + idx, td::Timestamp::in(10.0), std::move(P)); +} + +void CoreActor::got_ping_result(td::uint32 idx, td::int32 attempt, td::Result R) { if (attempt != attempt_) { return; } @@ -524,39 +543,7 @@ void CoreActor::got_result(td::uint32 idx, td::int32 attempt, td::Result(); - auto q = ton::create_tl_object(serialize_tl_object(query, true)); - - auto P = - td::PromiseCreator::lambda([SelfId = actor_id(this), idx, attempt = attempt_](td::Result R) { - td::actor::send_closure(SelfId, &CoreActor::got_result, idx, attempt, std::move(R)); - }); - td::actor::send_closure(clients_[idx], &ton::adnl::AdnlExtClient::send_query, "query", serialize_tl_object(q, true), - td::Timestamp::in(10.0), std::move(P)); -} - -void CoreActor::run_queries() { - waiting_ = 0; - new_result_ = std::make_shared(ready_.size(), td::Timestamp::at_unix(attempt_ * 60)); - for (td::uint32 i = 0; i < ready_.size(); i++) { - send_query(i); - } - CHECK(waiting_ >= 0); - if (waiting_ == 0) { - add_result(); - } -} - -void CoreActor::send_lite_query(td::uint32 idx, td::BufferSlice query, td::Promise promise) { - if (!ready_[idx]) { - promise.set_error(td::Status::Error(ton::ErrorCode::notready, "ext conn not ready")); - return; - } +void CoreActor::send_lite_query(td::BufferSlice query, td::Promise promise) { auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); @@ -574,7 +561,7 @@ void CoreActor::send_lite_query(td::uint32 idx, td::BufferSlice query, td::Promi promise.set_value(std::move(B)); }); auto q = ton::create_tl_object(std::move(query)); - td::actor::send_closure(clients_[idx], &ton::adnl::AdnlExtClient::send_query, "query", serialize_tl_object(q, true), + td::actor::send_closure(client_, &liteclient::ExtClient::send_query, "query", serialize_tl_object(q, true), td::Timestamp::in(10.0), std::move(P)); } diff --git a/catchain/catchain-receiver.cpp b/catchain/catchain-receiver.cpp index edef9065..a6160383 100644 --- a/catchain/catchain-receiver.cpp +++ b/catchain/catchain-receiver.cpp @@ -526,10 +526,12 @@ void CatChainReceiverImpl::start_up() { for (td::uint32 i = 0; i < get_sources_cnt(); i++) { root_keys.emplace(get_source(i)->get_hash(), OVERLAY_MAX_ALLOWED_PACKET_SIZE); } - td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay, + overlay::OverlayOptions overlay_options; + overlay_options.broadcast_speed_multiplier_ = opts_.broadcast_speed_multiplier; + td::actor::send_closure(overlay_manager_, &overlay::Overlays::create_private_overlay_ex, get_source(local_idx_)->get_adnl_id(), overlay_full_id_.clone(), std::move(ids), make_callback(), overlay::OverlayPrivacyRules{0, 0, std::move(root_keys)}, - R"({ "type": "catchain" })"); + R"({ "type": "catchain" })", std::move(overlay_options)); CHECK(root_block_); diff --git a/common/global-version.h b/common/global-version.h index 533e5e8d..2308ce3e 100644 --- a/common/global-version.h +++ b/common/global-version.h @@ -19,6 +19,6 @@ namespace ton { // See doc/GlobalVersions.md -const int SUPPORTED_VERSION = 9; +constexpr int SUPPORTED_VERSION = 10; } diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index 501ce3b9..72bffae5 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -236,9 +236,8 @@ class HardforkCreator : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } - void add_shard(ton::ShardIdFull) override { - } - void del_shard(ton::ShardIdFull) override { + void on_new_masterchain_block(td::Ref state, + std::set shards_to_monitor) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } @@ -270,13 +269,19 @@ class HardforkCreator : public td::actor::Actor { void get_next_key_blocks(ton::BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) override { } - void download_archive(ton::BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - - td::Promise promise) override { + void download_archive(ton::BlockSeqno masterchain_seqno, ton::ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override { + } + void download_out_msg_queue_proof( + ton::ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, + td::Timestamp timeout, td::Promise>> promise) override { } void new_key_block(ton::validator::BlockHandle handle) override { } + void send_validator_telemetry(ton::PublicKeyHash key, + ton::tl_object_ptr telemetry) override { + } }; td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::install_callback, diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index c2a73714..06908338 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -318,20 +318,20 @@ endif() target_include_directories(ton_crypto SYSTEM PUBLIC $) add_dependencies(ton_crypto blst) +add_dependencies(ton_crypto_core secp256k1) + target_include_directories(ton_crypto PRIVATE ${BLST_INCLUDE_DIR}) target_link_libraries(ton_crypto PRIVATE ${BLST_LIB}) -if (NOT USE_EMSCRIPTEN) - find_package(Secp256k1 REQUIRED) -endif() +target_include_directories(ton_crypto_core PUBLIC $) if (MSVC) find_package(Sodium REQUIRED) target_compile_definitions(ton_crypto PUBLIC SODIUM_STATIC) - target_include_directories(ton_crypto_core PUBLIC $) + target_link_libraries(ton_crypto_core PUBLIC ${SECP256K1_LIBRARY}) target_link_libraries(ton_crypto PUBLIC ${SECP256K1_LIBRARY}) -elseif (ANDROID OR EMSCRIPTEN) - target_include_directories(ton_crypto_core PUBLIC $) +elseif (EMSCRIPTEN) + target_link_libraries(ton_crypto_core PUBLIC $) target_link_libraries(ton_crypto PUBLIC $) else() if (NOT SODIUM_FOUND) @@ -340,11 +340,10 @@ else() message(STATUS "Using Sodium ${SODIUM_LIBRARY_RELEASE}") endif() target_compile_definitions(ton_crypto PUBLIC SODIUM_STATIC) - target_include_directories(ton_crypto_core PUBLIC $) + target_link_libraries(ton_crypto_core PUBLIC ${SECP256K1_LIBRARY}) target_link_libraries(ton_crypto PUBLIC ${SECP256K1_LIBRARY}) endif() -target_link_libraries(ton_crypto_core PUBLIC ${SECP256K1_LIBRARY}) target_include_directories(ton_crypto_core PUBLIC $) target_link_libraries(ton_crypto PUBLIC ${SODIUM_LIBRARY_RELEASE}) diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp index 6b9ca8bf..e0782240 100644 --- a/crypto/block/block.cpp +++ b/crypto/block/block.cpp @@ -360,7 +360,6 @@ MsgProcessedUptoCollection::MsgProcessedUptoCollection(ton::ShardIdFull _owner, z.shard = key.get_uint(64); z.mc_seqno = (unsigned)((key + 64).get_uint(32)); z.last_inmsg_lt = value.write().fetch_ulong(64); - // std::cerr << "ProcessedUpto shard " << std::hex << z.shard << std::dec << std::endl; return value.write().fetch_bits_to(z.last_inmsg_hash) && z.shard && ton::shard_contains(owner.shard, z.shard); }); } @@ -660,6 +659,12 @@ bool EnqueuedMsgDescr::check_key(td::ConstBitPtr key) const { hash_ == key + 96; } +bool ImportedMsgQueueLimits::deserialize(vm::CellSlice& cs) { + return cs.fetch_ulong(8) == 0xd3 // imported_msg_queue_limits#d3 + && cs.fetch_uint_to(32, max_bytes) // max_bytes:# + && cs.fetch_uint_to(32, max_msgs); // max_msgs:# +} + bool ParamLimits::deserialize(vm::CellSlice& cs) { return cs.fetch_ulong(8) == 0xc3 // param_limits#c3 && cs.fetch_uint_to(32, limits_[0]) // underload:uint32 @@ -856,8 +861,10 @@ td::Status ShardState::unpack_out_msg_queue_info(Ref out_msg_queue_inf out_msg_queue_ = std::make_unique(std::move(qinfo.out_queue), 352, block::tlb::aug_OutMsgQueue); if (verbosity >= 3 * 1) { - LOG(DEBUG) << "unpacking ProcessedUpto of our previous block " << id_.to_str(); - block::gen::t_ProcessedInfo.print(std::cerr, qinfo.proc_info); + FLOG(DEBUG) { + sb << "unpacking ProcessedUpto of our previous block " << id_.to_str(); + block::gen::t_ProcessedInfo.print(sb, qinfo.proc_info); + }; } if (!block::gen::t_ProcessedInfo.validate_csr(1024, qinfo.proc_info)) { return td::Status::Error( @@ -1313,6 +1320,65 @@ CurrencyCollection CurrencyCollection::operator-(td::RefInt256 other_grams) cons } } +bool CurrencyCollection::clamp(const CurrencyCollection& other) { + if (!is_valid() || !other.is_valid()) { + return invalidate(); + } + grams = std::min(grams, other.grams); + vm::Dictionary dict1{extra, 32}, dict2(other.extra, 32); + bool ok = dict1.check_for_each([&](td::Ref cs1, td::ConstBitPtr key, int n) { + CHECK(n == 32); + td::Ref cs2 = dict2.lookup(key, 32); + td::RefInt256 val1 = tlb::t_VarUIntegerPos_32.as_integer(cs1); + if (val1.is_null()) { + return false; + } + td::RefInt256 val2 = cs2.is_null() ? td::zero_refint() : tlb::t_VarUIntegerPos_32.as_integer(cs2); + if (val2.is_null()) { + return false; + } + if (val1 > val2) { + if (val2->sgn() == 0) { + dict1.lookup_delete(key, 32); + } else { + dict1.set(key, 32, cs2); + } + } + return true; + }); + extra = dict1.get_root_cell(); + return ok || invalidate(); +} + +bool CurrencyCollection::check_extra_currency_limit(td::uint32 max_currencies) const { + td::uint32 count = 0; + return vm::Dictionary{extra, 32}.check_for_each([&](td::Ref, td::ConstBitPtr, int) { + ++count; + return count <= max_currencies; + }); +} + +bool CurrencyCollection::remove_zero_extra_currencies(Ref& root, td::uint32 max_currencies) { + td::uint32 count = 0; + vm::Dictionary dict{root, 32}; + int res = dict.filter([&](const vm::CellSlice& cs, td::ConstBitPtr, int) -> int { + ++count; + if (count > max_currencies) { + return -1; + } + td::RefInt256 val = tlb::t_VarUInteger_32.as_integer(cs); + if (val.is_null()) { + return -1; + } + return val->sgn() > 0; + }); + if (res < 0) { + return false; + } + root = dict.get_root_cell(); + return true; +} + bool CurrencyCollection::operator==(const CurrencyCollection& other) const { return is_valid() && other.is_valid() && !td::cmp(grams, other.grams) && (extra.not_null() == other.extra.not_null()) && diff --git a/crypto/block/block.h b/crypto/block/block.h index 0247d79c..685005b4 100644 --- a/crypto/block/block.h +++ b/crypto/block/block.h @@ -216,6 +216,16 @@ static inline std::ostream& operator<<(std::ostream& os, const MsgProcessedUptoC return proc_coll.print(os); } +struct ImportedMsgQueueLimits { + // Default values + td::uint32 max_bytes = 1 << 16; + td::uint32 max_msgs = 30; + bool deserialize(vm::CellSlice& cs); + ImportedMsgQueueLimits operator*(td::uint32 x) const { + return {max_bytes * x, max_msgs * x}; + } +}; + struct ParamLimits { enum { limits_cnt = 4 }; enum { cl_underload = 0, cl_normal = 1, cl_soft = 2, cl_medium = 3, cl_hard = 4 }; @@ -380,6 +390,9 @@ struct CurrencyCollection { CurrencyCollection operator-(const CurrencyCollection& other) const; CurrencyCollection operator-(CurrencyCollection&& other) const; CurrencyCollection operator-(td::RefInt256 other_grams) const; + bool clamp(const CurrencyCollection& other); + bool check_extra_currency_limit(td::uint32 max_currencies) const; + static bool remove_zero_extra_currencies(Ref& root, td::uint32 max_currencies); bool store(vm::CellBuilder& cb) const; bool store_or_zero(vm::CellBuilder& cb) const; bool fetch(vm::CellSlice& cs); diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index 6f975426..4a8bbc06 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -296,7 +296,7 @@ transaction$0111 account_addr:bits256 lt:uint64 total_fees:CurrencyCollection state_update:^(HASH_UPDATE Account) description:^TransactionDescr = Transaction; -!merkle_update#02 {X:Type} old_hash:bits256 new_hash:bits256 +!merkle_update#04 {X:Type} old_hash:bits256 new_hash:bits256 old_depth:uint16 new_depth:uint16 old:^X new:^X = MERKLE_UPDATE X; update_hashes#72 {X:Type} old_hash:bits256 new_hash:bits256 = HASH_UPDATE X; @@ -801,7 +801,7 @@ size_limits_config#01 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells max_ext_msg_size:uint32 max_ext_msg_depth:uint16 = SizeLimitsConfig; size_limits_config_v2#02 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16 max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_acc_state_bits:uint32 - max_acc_public_libraries:uint32 defer_out_queue_size_limit:uint32 = SizeLimitsConfig; + max_acc_public_libraries:uint32 defer_out_queue_size_limit:uint32 max_msg_extra_currencies:uint32 = SizeLimitsConfig; _ SizeLimitsConfig = ConfigParam 43; // key is [ wc:int32 addr:uint256 ] @@ -818,7 +818,7 @@ _ OracleBridgeParams = ConfigParam 72; // Binance Smart Chain bridge _ OracleBridgeParams = ConfigParam 73; // Polygon bridge // Note that chains in which bridge, minter and jetton-wallet operate are fixated -jetton_bridge_prices#_ bridge_burn_fee:Coins bridge_mint_fee:Coins +jetton_bridge_prices#_ bridge_burn_fee:Coins bridge_mint_fee:Coins wallet_min_tons_for_storage:Coins wallet_gas_consumption:Coins minter_min_tons_for_storage:Coins diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 56ee85ae..0f019b06 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -163,8 +163,11 @@ td::Status ConfigInfo::unpack() { } gen::McStateExtra::Record extra_info; if (!tlb::unpack_cell(state_extra_root_, extra_info)) { - vm::load_cell_slice(state_extra_root_).print_rec(std::cerr); - block::gen::t_McStateExtra.print_ref(std::cerr, state_extra_root_); + FLOG(WARNING) { + sb << "state extra information is invalid: "; + vm::load_cell_slice(state_extra_root_).print_rec(sb); + block::gen::t_McStateExtra.print_ref(sb, state_extra_root_); + }; return td::Status::Error("state extra information is invalid"); } gen::ValidatorInfo::Record validator_info; @@ -1067,7 +1070,6 @@ Ref ShardConfig::get_shard_hash(ton::ShardIdFull id, bool exact) co ton::ShardIdFull true_id; vm::CellSlice cs; if (get_shard_hash_raw(cs, id, true_id, exact)) { - // block::gen::t_ShardDescr.print(std::cerr, vm::CellSlice{cs}); return McShardHash::unpack(cs, true_id); } else { return {}; @@ -1637,8 +1639,10 @@ bool ShardConfig::set_shard_info(ton::ShardIdFull shard, Ref value) { if (!gen::t_BinTree_ShardDescr.validate_ref(1024, value)) { LOG(ERROR) << "attempting to store an invalid (BinTree ShardDescr) at shard configuration position " << shard.to_str(); - gen::t_BinTree_ShardDescr.print_ref(std::cerr, value); - vm::load_cell_slice(value).print_rec(std::cerr); + FLOG(WARNING) { + gen::t_BinTree_ShardDescr.print_ref(sb, value); + vm::load_cell_slice(value).print_rec(sb); + }; return false; } auto root = shard_hashes_dict_->lookup_ref(td::BitArray<32>{shard.workchain}); @@ -1956,6 +1960,7 @@ td::Result Config::do_get_size_limits_config(td::Ref ConfigInfo::lookup_library(td::ConstBitPtr root_hash) const { td::Result> ConfigInfo::get_prev_blocks_info() const { // [ wc:Integer shard:Integer seqno:Integer root_hash:Integer file_hash:Integer] = BlockId; // [ last_mc_blocks:[BlockId...] - // prev_key_block:BlockId ] : PrevBlocksInfo + // prev_key_block:BlockId + // last_mc_blocks_100[BlockId...] ] : PrevBlocksInfo auto block_id_to_tuple = [](const ton::BlockIdExt& block_id) -> vm::Ref { td::RefInt256 shard = td::make_refint(block_id.id.shard); if (shard->sgn() < 0) { @@ -2302,25 +2308,44 @@ td::Result> ConfigInfo::get_prev_blocks_info() const { td::make_refint(block_id.id.seqno), td::bits_to_refint(block_id.root_hash.bits(), 256), td::bits_to_refint(block_id.file_hash.bits(), 256)); }; - std::vector last_mc_blocks; + std::vector tuple; + std::vector last_mc_blocks; last_mc_blocks.push_back(block_id_to_tuple(block_id)); for (ton::BlockSeqno seqno = block_id.id.seqno; seqno > 0 && last_mc_blocks.size() < 16;) { --seqno; - ton::BlockIdExt block_id; - if (!get_old_mc_block_id(seqno, block_id)) { + ton::BlockIdExt id; + if (!get_old_mc_block_id(seqno, id)) { return td::Status::Error("cannot fetch old mc block"); } - last_mc_blocks.push_back(block_id_to_tuple(block_id)); + last_mc_blocks.push_back(block_id_to_tuple(id)); } + tuple.push_back(td::make_cnt_ref>(std::move(last_mc_blocks))); ton::BlockIdExt last_key_block; ton::LogicalTime last_key_block_lt; if (!get_last_key_block(last_key_block, last_key_block_lt)) { return td::Status::Error("cannot fetch last key block"); } - return vm::make_tuple_ref(td::make_cnt_ref>(std::move(last_mc_blocks)), - block_id_to_tuple(last_key_block)); + tuple.push_back(block_id_to_tuple(last_key_block)); + + if (get_global_version() >= 9) { + std::vector last_mc_blocks_100; + for (ton::BlockSeqno seqno = block_id.id.seqno / 100 * 100; last_mc_blocks_100.size() < 16;) { + ton::BlockIdExt id; + if (!get_old_mc_block_id(seqno, id)) { + return td::Status::Error("cannot fetch old mc block"); + } + last_mc_blocks_100.push_back(block_id_to_tuple(id)); + if (seqno < 100) { + break; + } + seqno -= 100; + } + tuple.push_back(td::make_cnt_ref>(std::move(last_mc_blocks_100))); + } + + return td::make_cnt_ref>(std::move(tuple)); } td::optional PrecompiledContractsConfig::get_contract( diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 7cb6613d..98e6a26d 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -397,6 +397,7 @@ struct SizeLimitsConfig { td::uint32 max_acc_state_bits = (1 << 16) * 1023; td::uint32 max_acc_public_libraries = 256; td::uint32 defer_out_queue_size_limit = 256; + td::uint32 max_msg_extra_currencies = 2; }; struct CatchainValidatorsConfig { diff --git a/crypto/block/output-queue-merger.cpp b/crypto/block/output-queue-merger.cpp index aa425f6b..7d258cfe 100644 --- a/crypto/block/output-queue-merger.cpp +++ b/crypto/block/output-queue-merger.cpp @@ -138,7 +138,6 @@ bool OutputQueueMerger::add_root(int src, Ref outmsg_root) { if (outmsg_root.is_null()) { return true; } - //block::gen::HashmapAug{352, block::gen::t_EnqueuedMsg, block::gen::t_uint64}.print_ref(std::cerr, outmsg_root); auto kv = std::make_unique(src, std::move(outmsg_root)); if (kv->replace_by_prefix(common_pfx.cbits(), common_pfx_len)) { heap.push_back(std::move(kv)); diff --git a/crypto/block/transaction.cpp b/crypto/block/transaction.cpp index a32bad52..34d23511 100644 --- a/crypto/block/transaction.cpp +++ b/crypto/block/transaction.cpp @@ -446,8 +446,10 @@ bool Account::unpack(Ref shard_account, ton::UnixTime now, bool s return false; } if (verbosity > 2) { - shard_account->print_rec(std::cerr, 2); - block::gen::t_ShardAccount.print(std::cerr, *shard_account); + FLOG(INFO) { + shard_account->print_rec(sb, 2); + block::gen::t_ShardAccount.print(sb, shard_account); + }; } block::gen::ShardAccount::Record acc_info; if (!(block::tlb::t_ShardAccount.validate_csr(shard_account) && tlb::unpack_exact(shard_account.write(), acc_info))) { @@ -737,9 +739,11 @@ bool Transaction::unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig* return false; } if (verbosity > 2) { - fprintf(stderr, "unpacking inbound message for a new transaction: "); - block::gen::t_Message_Any.print_ref(std::cerr, in_msg); - load_cell_slice(in_msg).print_rec(std::cerr); + FLOG(INFO) { + sb << "unpacking inbound message for a new transaction: "; + block::gen::t_Message_Any.print_ref(sb, in_msg); + load_cell_slice(in_msg).print_rec(sb); + }; } auto cs = vm::load_cell_slice(in_msg); int tag = block::gen::t_CommonMsgInfo.get_tag(cs); @@ -1145,31 +1149,66 @@ td::RefInt256 ComputePhaseConfig::compute_gas_price(td::uint64 gas_used) const { namespace transaction { /** - * Checks if it is required to increase gas_limit (from GasLimitsPrices config) to special_gas_limit * 2 - * from masterchain GasLimitsPrices config for the transaction. + * Checks if it is required to increase gas_limit (from GasLimitsPrices config) for the transaction * * In January 2024 a highload wallet of @wallet Telegram bot in mainnet was stuck because current gas limit (1M) is * not enough to clean up old queires, thus locking funds inside. * See comment in crypto/smartcont/highload-wallet-v2-code.fc for details on why this happened. * Account address: EQD_v9j1rlsuHHw2FIhcsCFFSD367ldfDdCKcsNmNpIRzUlu - * It was proposed to validators to increase gas limit for this account for a limited amount of time (until 2024-02-29). + * It was proposed to validators to increase gas limit for this account to 70M for a limited amount + * of time (until 2024-02-29). * It is activated by setting global version to 5 in ConfigParam 8. * This config change also activates new behavior for special accounts in masterchain. * + * In August 2024 it was decided to unlock other old highload wallets that got into the same situation. + * See https://t.me/tondev_news/129 + * It is activated by setting global version to 9. + * * @param cfg The compute phase configuration. * @param now The Unix time of the transaction. * @param account The account of the transaction. * - * @returns True if gas_limit override is required, false otherwise + * @returns Overridden gas limit or empty td::optional */ -static bool override_gas_limit(const ComputePhaseConfig& cfg, ton::UnixTime now, const Account& account) { - if (!cfg.special_gas_full) { - return false; +static td::optional override_gas_limit(const ComputePhaseConfig& cfg, ton::UnixTime now, + const Account& account) { + struct OverridenGasLimit { + td::uint64 new_limit; + int from_version; + ton::UnixTime until; + }; + static std::map, OverridenGasLimit> accounts = []() { + auto parse_addr = [](const char* s) -> std::pair { + auto r_addr = StdAddress::parse(td::Slice(s)); + r_addr.ensure(); + return {r_addr.ok().workchain, r_addr.ok().addr}; + }; + std::map, OverridenGasLimit> accounts; + + // Increase limit for EQD_v9j1rlsuHHw2FIhcsCFFSD367ldfDdCKcsNmNpIRzUlu until 2024-02-29 00:00:00 UTC + accounts[parse_addr("0:FFBFD8F5AE5B2E1C7C3614885CB02145483DFAEE575F0DD08A72C366369211CD")] = { + .new_limit = 70'000'000, .from_version = 5, .until = 1709164800}; + + // Increase limit for multiple accounts (https://t.me/tondev_news/129) until 2025-03-01 00:00:00 UTC + accounts[parse_addr("UQBeSl-dumOHieZ3DJkNKVkjeso7wZ0VpzR4LCbLGTQ8xr57")] = { + .new_limit = 70'000'000, .from_version = 9, .until = 1740787200}; + accounts[parse_addr("EQC3VcQ-43klww9UfimR58TBjBzk7GPupXQ3CNuthoNp-uTR")] = { + .new_limit = 70'000'000, .from_version = 9, .until = 1740787200}; + accounts[parse_addr("EQBhwBb8jvokGvfreHRRoeVxI237PrOJgyrsAhLA-4rBC_H5")] = { + .new_limit = 70'000'000, .from_version = 9, .until = 1740787200}; + accounts[parse_addr("EQCkoRp4OE-SFUoMEnYfL3vF43T3AzNfW8jyTC4yzk8cJqMS")] = { + .new_limit = 70'000'000, .from_version = 9, .until = 1740787200}; + accounts[parse_addr("UQBN5ICras79U8FYEm71ws34n-ZNIQ0LRNpckOUsIV3OebnC")] = { + .new_limit = 70'000'000, .from_version = 9, .until = 1740787200}; + accounts[parse_addr("EQBDanbCeUqI4_v-xrnAN0_I2wRvEIaLg1Qg2ZN5c6Zl1KOh")] = { + .new_limit = 225'000'000, .from_version = 9, .until = 1740787200}; + return accounts; + }(); + auto it = accounts.find({account.workchain, account.addr}); + if (it == accounts.end() || cfg.global_version < it->second.from_version || now >= it->second.until) { + return {}; } - ton::UnixTime until = 1709164800; // 2024-02-29 00:00:00 UTC - ton::WorkchainId wc = 0; - const char* addr_hex = "FFBFD8F5AE5B2E1C7C3614885CB02145483DFAEE575F0DD08A72C366369211CD"; - return now < until && account.workchain == wc && account.addr.to_hex() == addr_hex; + return it->second.new_limit; } /** @@ -1183,10 +1222,12 @@ static bool override_gas_limit(const ComputePhaseConfig& cfg, ton::UnixTime now, * @returns The amount of gas. */ td::uint64 Transaction::gas_bought_for(const ComputePhaseConfig& cfg, td::RefInt256 nanograms) { - if (override_gas_limit(cfg, now, account)) { + if (auto new_limit = override_gas_limit(cfg, now, account)) { gas_limit_overridden = true; // Same as ComputePhaseConfig::gas_bought for, but with other gas_limit and max_gas_threshold - auto gas_limit = cfg.mc_gas_prices.special_gas_limit * 2; + auto gas_limit = new_limit.value(); + LOG(INFO) << "overridding gas limit for account " << account.workchain << ":" << account.addr.to_hex() << " to " + << gas_limit; auto max_gas_threshold = compute_max_gas_threshold(cfg.gas_price256, gas_limit, cfg.flat_gas_limit, cfg.flat_gas_price); if (nanograms.is_null() || sgn(nanograms) < 0) { @@ -1336,7 +1377,8 @@ Ref Transaction::prepare_vm_c7(const ComputePhaseConfig& cfg) const { // See crypto/block/mc-config.cpp#2223 (get_prev_blocks_info) // [ wc:Integer shard:Integer seqno:Integer root_hash:Integer file_hash:Integer] = BlockId; // [ last_mc_blocks:[BlockId...] - // prev_key_block:BlockId ] : PrevBlocksInfo + // prev_key_block:BlockId + // last_mc_blocks_100:[BlockId...] ] : PrevBlocksInfo // The only context where PrevBlocksInfo (13 parameter of c7) is null is inside emulator // where it need to be set via transaction_emulator_set_prev_blocks_info (see emulator/emulator-extern.cpp) // Inside validator, collator and liteserver checking external message contexts @@ -1512,11 +1554,13 @@ bool Transaction::run_precompiled_contract(const ComputePhaseConfig& cfg, precom cp.actions = impl.get_c5(); int out_act_num = output_actions_count(cp.actions); if (verbosity > 2) { - std::cerr << "new smart contract data: "; - bool can_be_special = true; - load_cell_slice_special(cp.new_data, can_be_special).print_rec(std::cerr); - std::cerr << "output actions: "; - block::gen::OutList{out_act_num}.print_ref(std::cerr, cp.actions); + FLOG(INFO) { + sb << "new smart contract data: "; + bool can_be_special = true; + load_cell_slice_special(cp.new_data, can_be_special).print_rec(sb); + sb << "output actions: "; + block::gen::OutList{out_act_num}.print_ref(sb, cp.actions); + }; } } cp.mode = 0; @@ -1581,7 +1625,6 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { if (in_msg_state.not_null()) { LOG(DEBUG) << "HASH(in_msg_state) = " << in_msg_state->get_hash().bits().to_hex(256) << ", account_state_hash = " << account.state_hash.to_hex(); - // vm::load_cell_slice(in_msg_state).print_rec(std::cerr); } else { LOG(DEBUG) << "in_msg_state is null"; } @@ -1691,9 +1734,8 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { } } } - vm::VmState vm{new_code, std::move(stack), gas, 1, new_data, vm_log, compute_vm_libraries(cfg)}; + vm::VmState vm{new_code, cfg.global_version, std::move(stack), gas, 1, new_data, vm_log, compute_vm_libraries(cfg)}; vm.set_max_data_depth(cfg.max_vm_data_depth); - vm.set_global_version(cfg.global_version); vm.set_c7(prepare_vm_c7(cfg)); // tuple with SmartContractInfo vm.set_chksig_always_succeed(cfg.ignore_chksig); vm.set_stop_on_accept_message(cfg.stop_on_accept_message); @@ -1738,11 +1780,13 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) { cp.actions = vm.get_committed_state().c5; // c5 -> action list int out_act_num = output_actions_count(cp.actions); if (verbosity > 2) { - std::cerr << "new smart contract data: "; - bool can_be_special = true; - load_cell_slice_special(cp.new_data, can_be_special).print_rec(std::cerr); - std::cerr << "output actions: "; - block::gen::OutList{out_act_num}.print_ref(std::cerr, cp.actions); + FLOG(INFO) { + sb << "new smart contract data: "; + bool can_be_special = true; + load_cell_slice_special(cp.new_data, can_be_special).print_rec(sb); + sb << "output actions: "; + block::gen::OutList{out_act_num}.print_ref(sb, cp.actions); + }; } } cp.mode = 0; @@ -1956,9 +2000,9 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) { ap.remaining_balance += ap.reserved_balance; CHECK(ap.remaining_balance.is_valid()); if (ap.acc_delete_req) { - CHECK(ap.remaining_balance.is_zero()); + CHECK(cfg.extra_currency_v2 ? ap.remaining_balance.grams->sgn() == 0 : ap.remaining_balance.is_zero()); ap.acc_status_change = ActionPhase::acst_deleted; - acc_status = Account::acc_deleted; + acc_status = (ap.remaining_balance.is_zero() ? Account::acc_deleted : Account::acc_uninit); was_deleted = true; } ap.success = true; @@ -2428,6 +2472,20 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, LOG(DEBUG) << "invalid destination address in a proposed outbound message"; return check_skip_invalid(36); // invalid destination address } + if (cfg.extra_currency_v2) { + CurrencyCollection value; + if (!value.unpack(info.value)) { + LOG(DEBUG) << "invalid value:ExtraCurrencies in a proposed outbound message"; + return check_skip_invalid(37); // invalid value:CurrencyCollection + } + if (!CurrencyCollection::remove_zero_extra_currencies(value.extra, cfg.size_limits.max_msg_extra_currencies)) { + LOG(DEBUG) << "invalid value:ExtraCurrencies in a proposed outbound message: too many currencies (max " + << cfg.size_limits.max_msg_extra_currencies << ")"; + // Dict should be valid, since it was checked in t_OutListNode.validate_ref, so error here means limit exceeded + return check_skip_invalid(41); // invalid value:CurrencyCollection : too many extra currencies + } + info.value = value.pack(); + } // fetch message pricing info const MsgPrices& msg_prices = cfg.fetch_msg_prices(to_mc || account.is_masterchain()); @@ -2480,7 +2538,7 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, }; add_used_storage(msg.init, 3); // message init add_used_storage(msg.body, 3); // message body (the root cell itself is not counted) - if (!ext_msg) { + if (!ext_msg && !cfg.extra_currency_v2) { add_used_storage(info.value->prefetch_ref(), 0); } auto collect_fine = [&] { @@ -2551,11 +2609,19 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, if (act_rec.mode & 0x80) { // attach all remaining balance to this message - req = ap.remaining_balance; + if (cfg.extra_currency_v2) { + req.grams = ap.remaining_balance.grams; + } else { + req = ap.remaining_balance; + } act_rec.mode &= ~1; // pay fees from attached value } else if (act_rec.mode & 0x40) { // attach all remaining balance of the inbound message (in addition to the original value) - req += msg_balance_remaining; + if (cfg.extra_currency_v2) { + req.grams += msg_balance_remaining.grams; + } else { + req += msg_balance_remaining; + } if (!(act_rec.mode & 1)) { req -= ap.action_fine; if (compute_phase) { @@ -2595,6 +2661,11 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, return check_skip_invalid(37); // not enough grams } + if (cfg.extra_currency_v2 && !req.check_extra_currency_limit(cfg.size_limits.max_msg_extra_currencies)) { + LOG(DEBUG) << "too many extra currencies in the message : max " << cfg.size_limits.max_msg_extra_currencies; + return check_skip_invalid(41); // to many extra currencies + } + Ref new_extra; if (!block::sub_extra_currency(ap.remaining_balance.extra, req.extra, new_extra)) { @@ -2636,7 +2707,11 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, // clear msg_balance_remaining if it has been used if (act_rec.mode & 0xc0) { - msg_balance_remaining.set_zero(); + if (cfg.extra_currency_v2) { + msg_balance_remaining.grams = td::zero_refint(); + } else { + msg_balance_remaining.set_zero(); + } } // update balance @@ -2688,14 +2763,18 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, } if (!block::gen::t_Message_Any.validate_ref(new_msg)) { LOG(ERROR) << "generated outbound message is not a valid (Message Any) according to automated check"; - block::gen::t_Message_Any.print_ref(std::cerr, new_msg); - vm::load_cell_slice(new_msg).print_rec(std::cerr); + FLOG(INFO) { + block::gen::t_Message_Any.print_ref(sb, new_msg); + vm::load_cell_slice(new_msg).print_rec(sb); + }; collect_fine(); return -1; } if (verbosity > 2) { - std::cerr << "converted outbound message: "; - block::gen::t_Message_Any.print_ref(std::cerr, new_msg); + FLOG(INFO) { + sb << "converted outbound message: "; + block::gen::t_Message_Any.print_ref(sb, new_msg); + }; } ap.msgs_created++; @@ -2706,8 +2785,13 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap, ap.total_fwd_fees += fees_total; if ((act_rec.mode & 0xa0) == 0xa0) { - CHECK(ap.remaining_balance.is_zero()); - ap.acc_delete_req = ap.reserved_balance.is_zero(); + if (cfg.extra_currency_v2) { + CHECK(ap.remaining_balance.grams->sgn() == 0); + ap.acc_delete_req = ap.reserved_balance.grams->sgn() == 0; + } else { + CHECK(ap.remaining_balance.is_zero()); + ap.acc_delete_req = ap.reserved_balance.is_zero(); + } } ap.tot_msg_bits += sstat.bits + new_msg_bits; @@ -2760,22 +2844,25 @@ int Transaction::try_action_reserve_currency(vm::CellSlice& cs, ActionPhase& ap, LOG(DEBUG) << "cannot reserve a negative amount: " << reserve.to_str(); return -1; } - if (reserve.grams > ap.remaining_balance.grams) { - if (mode & 2) { - reserve.grams = ap.remaining_balance.grams; + if (mode & 2) { + if (cfg.reserve_extra_enabled) { + if (!reserve.clamp(ap.remaining_balance)) { + LOG(DEBUG) << "failed to clamp reserve amount" << mode; + return -1; + } } else { - LOG(DEBUG) << "cannot reserve " << reserve.grams << " nanograms : only " << ap.remaining_balance.grams - << " available"; - return 37; // not enough grams + reserve.grams = std::min(reserve.grams, ap.remaining_balance.grams); } } + if (reserve.grams > ap.remaining_balance.grams) { + LOG(DEBUG) << "cannot reserve " << reserve.grams << " nanograms : only " << ap.remaining_balance.grams + << " available"; + return 37; // not enough grams + } if (!block::sub_extra_currency(ap.remaining_balance.extra, reserve.extra, newc.extra)) { LOG(DEBUG) << "not enough extra currency to reserve: " << block::CurrencyCollection{0, reserve.extra}.to_str() << " required, only " << block::CurrencyCollection{0, ap.remaining_balance.extra}.to_str() << " available"; - if (mode & 2) { - // TODO: process (mode & 2) correctly by setting res_extra := inf (reserve.extra, ap.remaining_balance.extra) - } return 38; // not enough (extra) funds } newc.grams = ap.remaining_balance.grams - reserve.grams; @@ -2975,7 +3062,8 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { bp.fwd_fees -= bp.fwd_fees_collected; total_fees += td::make_refint(bp.fwd_fees_collected); // serialize outbound message - info.created_lt = end_lt++; + info.created_lt = start_lt + 1 + out_msgs.size(); + end_lt++; info.created_at = now; vm::CellBuilder cb; CHECK(cb.store_long_bool(5, 4) // int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool @@ -3005,8 +3093,10 @@ bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) { } CHECK(cb.finalize_to(bp.out_msg)); if (verbosity > 2) { - LOG(INFO) << "generated bounced message: "; - block::gen::t_Message_Any.print_ref(std::cerr, bp.out_msg); + FLOG(INFO) { + sb << "generated bounced message: "; + block::gen::t_Message_Any.print_ref(sb, bp.out_msg); + }; } out_msgs.push_back(bp.out_msg); bp.ok = true; @@ -3054,6 +3144,7 @@ bool Account::store_acc_status(vm::CellBuilder& cb, int acc_status) const { * Tries to update the storage statistics based on the old storage statistics and old account state without fully recomputing it. * * It succeeds if only root cell of AccountStorage is changed. + * old_cs and new_cell are AccountStorage without extra currencies (if global_version >= 10). * * @param old_stat The old storage statistics. * @param old_cs The old AccountStorage. @@ -3087,13 +3178,48 @@ static td::optional try_update_storage_stat(const vm::CellS return new_stat; } +/** + * Removes extra currencies dict from AccountStorage. + * + * This is used for computing account storage stats. + * + * @param storage_cs AccountStorage as CellSlice. + * + * @returns AccountStorage without extra currencies as Cell. + */ +static td::Ref storage_without_extra_currencies(td::Ref storage_cs) { + block::gen::AccountStorage::Record rec; + if (!block::gen::csr_unpack(storage_cs, rec)) { + LOG(ERROR) << "failed to unpack AccountStorage"; + return {}; + } + if (rec.balance->size_refs() > 0) { + block::gen::CurrencyCollection::Record balance; + if (!block::gen::csr_unpack(rec.balance, balance)) { + LOG(ERROR) << "failed to unpack AccountStorage"; + return {}; + } + balance.other = vm::CellBuilder{}.store_zeroes(1).as_cellslice_ref(); + if (!block::gen::csr_pack(rec.balance, balance)) { + LOG(ERROR) << "failed to pack AccountStorage"; + return {}; + } + } + td::Ref cell; + if (!block::gen::pack_cell(cell, rec)) { + LOG(ERROR) << "failed to pack AccountStorage"; + return {}; + } + return cell; +} + namespace transaction { /** * Computes the new state of the account. * * @returns True if the state computation is successful, false otherwise. */ -bool Transaction::compute_state() { +bool Transaction::compute_state(const SerializeConfig& cfg) { if (new_total_state.not_null()) { return true; } @@ -3127,11 +3253,13 @@ bool Transaction::compute_state() { auto frozen_state = cb2.finalize(); frozen_hash = frozen_state->get_hash().bits(); if (verbosity >= 3 * 1) { // !!!DEBUG!!! - std::cerr << "freezing state of smart contract: "; - block::gen::t_StateInit.print_ref(std::cerr, frozen_state); - CHECK(block::gen::t_StateInit.validate_ref(frozen_state)); - CHECK(block::tlb::t_StateInit.validate_ref(frozen_state)); - std::cerr << "with hash " << frozen_hash.to_hex() << std::endl; + FLOG(INFO) { + sb << "freezing state of smart contract: "; + block::gen::t_StateInit.print_ref(sb, frozen_state); + CHECK(block::gen::t_StateInit.validate_ref(frozen_state)); + CHECK(block::tlb::t_StateInit.validate_ref(frozen_state)); + sb << "with hash " << frozen_hash.to_hex(); + }; } } new_code.clear(); @@ -3163,13 +3291,27 @@ bool Transaction::compute_state() { new_inner_state.clear(); } vm::CellStorageStat& stats = new_storage_stat; - auto new_stats = try_update_storage_stat(account.storage_stat, account.storage, storage); + td::Ref old_storage_for_stat = account.storage; + td::Ref new_storage_for_stat = storage; + if (cfg.extra_currency_v2) { + new_storage_for_stat = storage_without_extra_currencies(new_storage); + if (new_storage_for_stat.is_null()) { + return false; + } + if (old_storage_for_stat.not_null()) { + old_storage_for_stat = vm::load_cell_slice_ref(storage_without_extra_currencies(old_storage_for_stat)); + if (old_storage_for_stat.is_null()) { + return false; + } + } + } + auto new_stats = try_update_storage_stat(account.storage_stat, old_storage_for_stat, storage); if (new_stats) { stats = new_stats.unwrap(); } else { TD_PERF_COUNTER(transaction_storage_stat_b); td::Timer timer; - stats.add_used_storage(Ref(storage)).ensure(); + stats.add_used_storage(new_storage_for_stat).ensure(); if (timer.elapsed() > 0.1) { LOG(INFO) << "Compute used storage took " << timer.elapsed() << "s"; } @@ -3189,8 +3331,10 @@ bool Transaction::compute_state() { CHECK(cb.append_data_cell_bool(std::move(storage))); new_total_state = cb.finalize(); if (verbosity > 2) { - std::cerr << "new account state: "; - block::gen::t_Account.print_ref(std::cerr, new_total_state); + FLOG(INFO) { + sb << "new account state: "; + block::gen::t_Account.print_ref(sb, new_total_state); + }; } CHECK(block::tlb::t_Account.validate_ref(new_total_state)); return true; @@ -3203,11 +3347,11 @@ bool Transaction::compute_state() { * * @returns True if the serialization is successful, False otherwise. */ -bool Transaction::serialize() { +bool Transaction::serialize(const SerializeConfig& cfg) { if (root.not_null()) { return true; } - if (!compute_state()) { + if (!compute_state(cfg)) { return false; } vm::Dictionary dict{15}; @@ -3282,22 +3426,28 @@ bool Transaction::serialize() { return false; } if (verbosity >= 3 * 1) { - std::cerr << "new transaction: "; - block::gen::t_Transaction.print_ref(std::cerr, root); - vm::load_cell_slice(root).print_rec(std::cerr); + FLOG(INFO) { + sb << "new transaction: "; + block::gen::t_Transaction.print_ref(sb, root); + vm::load_cell_slice(root).print_rec(sb); + }; } if (!block::gen::t_Transaction.validate_ref(4096, root)) { LOG(ERROR) << "newly-generated transaction failed to pass automated validation:"; - vm::load_cell_slice(root).print_rec(std::cerr); - block::gen::t_Transaction.print_ref(std::cerr, root); + FLOG(INFO) { + vm::load_cell_slice(root).print_rec(sb); + block::gen::t_Transaction.print_ref(sb, root); + }; root.clear(); return false; } if (!block::tlb::t_Transaction.validate_ref(4096, root)) { LOG(ERROR) << "newly-generated transaction failed to pass hand-written validation:"; - vm::load_cell_slice(root).print_rec(std::cerr); - block::gen::t_Transaction.print_ref(std::cerr, root); + FLOG(INFO) { + vm::load_cell_slice(root).print_rec(sb); + block::gen::t_Transaction.print_ref(sb, root); + }; root.clear(); return false; } @@ -3667,6 +3817,7 @@ bool Account::libraries_changed() const { * @param rand_seed Pointer to the random seed. Generates a new seed if the value is `td::Bits256::zero()`. * @param compute_phase_cfg Pointer to store the compute phase configuration. * @param action_phase_cfg Pointer to store the action phase configuration. + * @param serialize_cfg Pointer to store the serialize phase configuration. * @param masterchain_create_fee Pointer to store the masterchain create fee. * @param basechain_create_fee Pointer to store the basechain create fee. * @param wc The workchain ID. @@ -3675,15 +3826,15 @@ bool Account::libraries_changed() const { td::Status FetchConfigParams::fetch_config_params( const block::ConfigInfo& config, Ref* old_mparams, std::vector* storage_prices, StoragePhaseConfig* storage_phase_cfg, td::BitArray<256>* rand_seed, ComputePhaseConfig* compute_phase_cfg, - ActionPhaseConfig* action_phase_cfg, td::RefInt256* masterchain_create_fee, td::RefInt256* basechain_create_fee, - ton::WorkchainId wc, ton::UnixTime now) { + ActionPhaseConfig* action_phase_cfg, SerializeConfig* serialize_cfg, td::RefInt256* masterchain_create_fee, + td::RefInt256* basechain_create_fee, ton::WorkchainId wc, ton::UnixTime now) { auto prev_blocks_info = config.get_prev_blocks_info(); if (prev_blocks_info.is_error()) { return prev_blocks_info.move_as_error_prefix( td::Status::Error(-668, "cannot fetch prev blocks info from masterchain configuration: ")); } return fetch_config_params(config, prev_blocks_info.move_as_ok(), old_mparams, storage_prices, storage_phase_cfg, - rand_seed, compute_phase_cfg, action_phase_cfg, masterchain_create_fee, + rand_seed, compute_phase_cfg, action_phase_cfg, serialize_cfg, masterchain_create_fee, basechain_create_fee, wc, now); } @@ -3698,6 +3849,7 @@ td::Status FetchConfigParams::fetch_config_params( * @param rand_seed Pointer to the random seed. Generates a new seed if the value is `td::Bits256::zero()`. * @param compute_phase_cfg Pointer to store the compute phase configuration. * @param action_phase_cfg Pointer to store the action phase configuration. + * @param serialize_cfg Pointer to store the serialize phase configuration. * @param masterchain_create_fee Pointer to store the masterchain create fee. * @param basechain_create_fee Pointer to store the basechain create fee. * @param wc The workchain ID. @@ -3707,8 +3859,8 @@ td::Status FetchConfigParams::fetch_config_params( const block::Config& config, td::Ref prev_blocks_info, Ref* old_mparams, std::vector* storage_prices, StoragePhaseConfig* storage_phase_cfg, td::BitArray<256>* rand_seed, ComputePhaseConfig* compute_phase_cfg, ActionPhaseConfig* action_phase_cfg, - td::RefInt256* masterchain_create_fee, td::RefInt256* basechain_create_fee, ton::WorkchainId wc, - ton::UnixTime now) { + SerializeConfig* serialize_cfg, td::RefInt256* masterchain_create_fee, td::RefInt256* basechain_create_fee, + ton::WorkchainId wc, ton::UnixTime now) { *old_mparams = config.get_config_param(9); { auto res = config.get_storage_prices(); @@ -3778,7 +3930,12 @@ td::Status FetchConfigParams::fetch_config_params( action_phase_cfg->bounce_on_fail_enabled = config.get_global_version() >= 4; action_phase_cfg->message_skip_enabled = config.get_global_version() >= 8; action_phase_cfg->disable_custom_fess = config.get_global_version() >= 8; + action_phase_cfg->reserve_extra_enabled = config.get_global_version() >= 9; action_phase_cfg->mc_blackhole_addr = config.get_burning_config().blackhole_addr; + action_phase_cfg->extra_currency_v2 = config.get_global_version() >= 10; + } + { + serialize_cfg->extra_currency_v2 = config.get_global_version() >= 10; } { // fetch block_grams_created diff --git a/crypto/block/transaction.h b/crypto/block/transaction.h index 20d7cb29..8e612e6a 100644 --- a/crypto/block/transaction.h +++ b/crypto/block/transaction.h @@ -169,12 +169,18 @@ struct ActionPhaseConfig { bool bounce_on_fail_enabled{false}; bool message_skip_enabled{false}; bool disable_custom_fess{false}; + bool reserve_extra_enabled{false}; + bool extra_currency_v2{false}; td::optional mc_blackhole_addr; const MsgPrices& fetch_msg_prices(bool is_masterchain) const { return is_masterchain ? fwd_mc : fwd_std; } }; +struct SerializeConfig { + bool extra_currency_v2{false}; +}; + struct CreditPhase { td::RefInt256 due_fees_collected; block::CurrencyCollection credit; @@ -388,8 +394,8 @@ struct Transaction { bool prepare_action_phase(const ActionPhaseConfig& cfg); td::Status check_state_limits(const SizeLimitsConfig& size_limits, bool update_storage_stat = true); bool prepare_bounce_phase(const ActionPhaseConfig& cfg); - bool compute_state(); - bool serialize(); + bool compute_state(const SerializeConfig& cfg); + bool serialize(const SerializeConfig& cfg); td::uint64 gas_used() const { return compute_phase ? compute_phase->gas_used : 0; } @@ -427,14 +433,14 @@ struct FetchConfigParams { std::vector* storage_prices, StoragePhaseConfig* storage_phase_cfg, td::BitArray<256>* rand_seed, ComputePhaseConfig* compute_phase_cfg, ActionPhaseConfig* action_phase_cfg, - td::RefInt256* masterchain_create_fee, td::RefInt256* basechain_create_fee, - ton::WorkchainId wc, ton::UnixTime now); + SerializeConfig* serialize_cfg, td::RefInt256* masterchain_create_fee, + td::RefInt256* basechain_create_fee, ton::WorkchainId wc, ton::UnixTime now); static td::Status fetch_config_params(const block::Config& config, Ref prev_blocks_info, Ref* old_mparams, std::vector* storage_prices, StoragePhaseConfig* storage_phase_cfg, td::BitArray<256>* rand_seed, ComputePhaseConfig* compute_phase_cfg, ActionPhaseConfig* action_phase_cfg, - td::RefInt256* masterchain_create_fee, td::RefInt256* basechain_create_fee, - ton::WorkchainId wc, ton::UnixTime now); + SerializeConfig* serialize_cfg, td::RefInt256* masterchain_create_fee, + td::RefInt256* basechain_create_fee, ton::WorkchainId wc, ton::UnixTime now); }; } // namespace block diff --git a/crypto/common/bitstring.h b/crypto/common/bitstring.h index 25776478..12333522 100644 --- a/crypto/common/bitstring.h +++ b/crypto/common/bitstring.h @@ -554,11 +554,7 @@ class BitArray { set_same(0); } void set_zero_s() { - volatile uint8* p = data(); - auto x = m; - while (x--) { - *p++ = 0; - } + as_slice().fill_zero_secure(); } void set_ones() { set_same(1); diff --git a/crypto/ellcurve/secp256k1.cpp b/crypto/ellcurve/secp256k1.cpp index e890117a..b98eea7b 100644 --- a/crypto/ellcurve/secp256k1.cpp +++ b/crypto/ellcurve/secp256k1.cpp @@ -17,13 +17,22 @@ #include "secp256k1.h" #include "td/utils/check.h" +#include "td/utils/logging.h" + #include +#include #include -namespace td { +namespace td::secp256k1 { + +static const secp256k1_context* get_context() { + static secp256k1_context* ctx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY); + LOG_CHECK(ctx) << "Failed to create secp256k1_context"; + return ctx; +} bool ecrecover(const unsigned char* hash, const unsigned char* signature, unsigned char* public_key) { - static secp256k1_context* ctx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY); + const secp256k1_context* ctx = get_context(); secp256k1_ecdsa_recoverable_signature ecdsa_signature; if (signature[64] > 3 || !secp256k1_ecdsa_recoverable_signature_parse_compact(ctx, &ecdsa_signature, signature, signature[64])) { @@ -39,4 +48,22 @@ bool ecrecover(const unsigned char* hash, const unsigned char* signature, unsign return true; } +bool xonly_pubkey_tweak_add(const unsigned char* xonly_pubkey_bytes, const unsigned char* tweak, + unsigned char* output_pubkey_bytes) { + const secp256k1_context* ctx = get_context(); + + secp256k1_xonly_pubkey xonly_pubkey; + secp256k1_pubkey output_pubkey; + if (!secp256k1_xonly_pubkey_parse(ctx, &xonly_pubkey, xonly_pubkey_bytes)) { + return false; + } + if (!secp256k1_xonly_pubkey_tweak_add(ctx, &output_pubkey, &xonly_pubkey, tweak)) { + return false; + } + size_t len = 65; + secp256k1_ec_pubkey_serialize(ctx, output_pubkey_bytes, &len, &output_pubkey, SECP256K1_EC_UNCOMPRESSED); + CHECK(len == 65); + return true; } + +} // namespace td::secp256k1 diff --git a/crypto/ellcurve/secp256k1.h b/crypto/ellcurve/secp256k1.h index 80ab6a87..20f0b66b 100644 --- a/crypto/ellcurve/secp256k1.h +++ b/crypto/ellcurve/secp256k1.h @@ -16,8 +16,10 @@ */ #pragma once -namespace td { +namespace td::secp256k1 { bool ecrecover(const unsigned char* hash, const unsigned char* signature, unsigned char* public_key); +bool xonly_pubkey_tweak_add(const unsigned char* xonly_pubkey_bytes, const unsigned char* tweak, + unsigned char* output_pubkey_bytes); -} +} // namespace td::secp256k1 diff --git a/crypto/fift/lib/Asm.fif b/crypto/fift/lib/Asm.fif index 92ceab6d..976093f8 100644 --- a/crypto/fift/lib/Asm.fif +++ b/crypto/fift/lib/Asm.fif @@ -1015,6 +1015,10 @@ x{EDC} dup @Defop(c) SAVEBOTH @Defop(c) SAVEBOTHCTR x{EDE0} @Defop PUSHCTRX x{EDE1} @Defop POPCTRX x{EDE2} @Defop SETCONTCTRX +x{EDE3} @Defop(8u) SETCONTCTRMANY +x{EDE3} @Defop(8u) SETCONTMANY +x{EDE4} @Defop SETCONTCTRMANYX +x{EDE4} @Defop SETCONTMANYX x{EDF0} dup @Defop BOOLAND @Defop COMPOS x{EDF1} dup @Defop BOOLOR @Defop COMPOSALT x{EDF2} @Defop COMPOSBOTH @@ -1308,6 +1312,7 @@ x{F832} @Defop CONFIGPARAM x{F833} @Defop CONFIGOPTPARAM x{F83400} @Defop PREVMCBLOCKS x{F83401} @Defop PREVKEYBLOCK +x{F83402} @Defop PREVMCBLOCKS_100 x{F835} @Defop GLOBALID x{F836} @Defop GETGASFEE x{F837} @Defop GETSTORAGEFEE @@ -1354,6 +1359,7 @@ x{F90704} @Defop HASHEXTAR_KECCAK512 x{F910} @Defop CHKSIGNU x{F911} @Defop CHKSIGNS x{F912} @Defop ECRECOVER +x{F913} @Defop SECP256K1_XONLY_PUBKEY_TWEAK_ADD x{F914} @Defop P256_CHKSIGNU x{F915} @Defop P256_CHKSIGNS @@ -1589,6 +1595,9 @@ forget @proclist forget @proccnt { }END> b> } : }END>c { }END>c s +// This is the way how FunC assigns method_id for reserved functions. +// Note, that Tolk entrypoints have other names (`onInternalMessage`, etc.), +// but method_id is assigned not by Fift, but by Tolk code generation. 0 constant recv_internal -1 constant recv_external -2 constant run_ticktock diff --git a/crypto/fift/utils.cpp b/crypto/fift/utils.cpp index f37766a7..6057b2dc 100644 --- a/crypto/fift/utils.cpp +++ b/crypto/fift/utils.cpp @@ -114,7 +114,7 @@ class MemoryFileLoader : public fift::FileLoader { std::map> files_; }; -td::Result create_source_lookup(std::string main, bool need_preamble = true, bool need_asm = true, +td::Result create_source_lookup(std::string&& main, bool need_preamble = true, bool need_asm = true, bool need_ton_util = true, bool need_lisp = true, bool need_w3_code = true, bool need_fift_ext = true, bool need_disasm = true, std::string dir = "") { @@ -189,7 +189,7 @@ td::Result run_fift(fift::SourceLookup source_lookup, std::o } // namespace td::Result mem_run_fift(std::string source, std::vector args, std::string fift_dir) { std::stringstream ss; - TRY_RESULT(source_lookup, create_source_lookup(source, true, true, true, true, true, true, true, fift_dir)); + TRY_RESULT(source_lookup, create_source_lookup(std::move(source), true, true, true, true, true, true, true, fift_dir)); TRY_RESULT_ASSIGN(source_lookup, run_fift(std::move(source_lookup), &ss, true, std::move(args))); FiftOutput res; res.source_lookup = std::move(source_lookup); @@ -207,19 +207,43 @@ td::Result mem_run_fift(SourceLookup source_lookup, std::vector create_mem_source_lookup(std::string main, std::string fift_dir, bool need_preamble, bool need_asm, bool need_ton_util, bool need_lisp, bool need_w3_code) { - return create_source_lookup(main, need_preamble, need_asm, need_ton_util, need_lisp, need_w3_code, false, false, + return create_source_lookup(std::move(main), need_preamble, need_asm, need_ton_util, need_lisp, need_w3_code, false, false, fift_dir); } -td::Result> compile_asm(td::Slice asm_code, std::string fift_dir, bool is_raw) { +td::Result> compile_asm(td::Slice asm_code) { std::stringstream ss; - TRY_RESULT(source_lookup, - create_source_lookup(PSTRING() << "\"Asm.fif\" include\n " << (is_raw ? "<{" : "") << asm_code << "\n" - << (is_raw ? "}>c" : "") << " boc>B \"res\" B>file", - true, true, true, false, false, false, false, fift_dir)); + std::string sb; + sb.reserve(asm_code.size() + 100); + sb.append("\"Asm.fif\" include\n <{\n"); + sb.append(asm_code.data(), asm_code.size()); + sb.append("\n}>c boc>B \"res\" B>file"); + + TRY_RESULT(source_lookup, create_source_lookup(std::move(sb), true, true, true, false, false, false, false)); TRY_RESULT(res, run_fift(std::move(source_lookup), &ss)); TRY_RESULT(boc, res.read_file("res")); return vm::std_boc_deserialize(std::move(boc.data)); } +td::Result compile_asm_program(std::string&& program_code, const std::string& fift_dir) { + std::string main_fif; + main_fif.reserve(program_code.size() + 100); + main_fif.append(program_code.data(), program_code.size()); + main_fif.append(R"( dup hashB B>X $>B "hex" B>file)"); // write codeHashHex to a file + main_fif.append(R"( boc>B B>base64 $>B "boc" B>file)"); // write codeBoc64 to a file + + std::stringstream fift_output_stream; + TRY_RESULT(source_lookup, create_source_lookup(std::move(main_fif), true, true, false, false, false, false, false, fift_dir)); + TRY_RESULT(res, run_fift(std::move(source_lookup), &fift_output_stream)); + + TRY_RESULT(boc, res.read_file("boc")); + TRY_RESULT(hex, res.read_file("hex")); + + return CompiledProgramOutput{ + std::move(program_code), + std::move(boc.data), + std::move(hex.data), + }; +} + } // namespace fift diff --git a/crypto/fift/utils.h b/crypto/fift/utils.h index dd434fe0..fab92c54 100644 --- a/crypto/fift/utils.h +++ b/crypto/fift/utils.h @@ -26,11 +26,21 @@ struct FiftOutput { SourceLookup source_lookup; std::string output; }; + +// given a valid Fift code PROGRAM{ ... }END>c, compile_asm_program() returns this output +// now it's used primarily for wasm output (see tolk-js, for example) +struct CompiledProgramOutput { + std::string fiftCode; + std::string codeBoc64; + std::string codeHashHex; +}; + td::Result create_mem_source_lookup(std::string main, std::string fift_dir = "", bool need_preamble = true, bool need_asm = true, bool need_ton_util = true, bool need_lisp = true, bool need_w3_code = true); td::Result mem_run_fift(std::string source, std::vector args = {}, std::string fift_dir = ""); td::Result mem_run_fift(SourceLookup source_lookup, std::vector args); -td::Result> compile_asm(td::Slice asm_code, std::string fift_dir = "", bool is_raw = true); +td::Result> compile_asm(td::Slice asm_code); +td::Result compile_asm_program(std::string&& program_code, const std::string& fift_dir); } // namespace fift diff --git a/crypto/func/auto-tests/tests/try-func.fc b/crypto/func/auto-tests/tests/try-func.fc index 5678965e..05e7594c 100644 --- a/crypto/func/auto-tests/tests/try-func.fc +++ b/crypto/func/auto-tests/tests/try-func.fc @@ -1,4 +1,4 @@ -int foo(int x) method_id(11) { +int foo(int x) { try { if (x == 7) { throw(44); @@ -9,7 +9,7 @@ int foo(int x) method_id(11) { } } -int foo_inline(int x) inline method_id(12) { +int foo_inline(int x) inline { try { if (x == 7) { throw(44); @@ -20,7 +20,7 @@ int foo_inline(int x) inline method_id(12) { } } -int foo_inlineref(int x) inline_ref method_id(13) { +int foo_inlineref(int x) inline_ref { try { if (x == 7) { throw(44); @@ -31,17 +31,17 @@ int foo_inlineref(int x) inline_ref method_id(13) { } } -int test(int x, int y, int z) method_id(1) { +int test(int x, int y, int z) method_id(101) { y = foo(y); return x * 100 + y * 10 + z; } -int test_inline(int x, int y, int z) method_id(2) { +int test_inline(int x, int y, int z) method_id(102) { y = foo_inline(y); return x * 100 + y * 10 + z; } -int test_inlineref(int x, int y, int z) method_id(3) { +int test_inlineref(int x, int y, int z) method_id(103) { y = foo_inlineref(y); return x * 100 + y * 10 + z; } @@ -49,7 +49,7 @@ int test_inlineref(int x, int y, int z) method_id(3) { int foo_inline_big( int x1, int x2, int x3, int x4, int x5, int x6, int x7, int x8, int x9, int x10, int x11, int x12, int x13, int x14, int x15, int x16, int x17, int x18, int x19, int x20 -) inline method_id(14) { +) inline { try { if (x1 == 7) { throw(44); @@ -60,7 +60,7 @@ int foo_inline_big( } } -int test_inline_big(int x, int y, int z) method_id(4) { +int test_inline_big(int x, int y, int z) method_id(104) { y = foo_inline_big( y, y + 1, y + 2, y + 3, y + 4, y + 5, y + 6, y + 7, y + 8, y + 9, y + 10, y + 11, y + 12, y + 13, y + 14, y + 15, y + 16, y + 17, y + 18, y + 19); @@ -70,7 +70,7 @@ int test_inline_big(int x, int y, int z) method_id(4) { int foo_big( int x1, int x2, int x3, int x4, int x5, int x6, int x7, int x8, int x9, int x10, int x11, int x12, int x13, int x14, int x15, int x16, int x17, int x18, int x19, int x20 -) method_id(15) { +) { try { if (x1 == 7) { throw(44); @@ -81,29 +81,69 @@ int foo_big( } } -int test_big(int x, int y, int z) method_id(5) { +int test_big(int x, int y, int z) method_id(105) { y = foo_big( y, y + 1, y + 2, y + 3, y + 4, y + 5, y + 6, y + 7, y + 8, y + 9, y + 10, y + 11, y + 12, y + 13, y + 14, y + 15, y + 16, y + 17, y + 18, y + 19); return x * 1000000 + y * 1000 + z; } + +() some_throwing(int op) impure { + if (op == 1) { + return (); + } elseif (op == 2) { + return (); + } else { + throw(1); + } +} + +int test106() method_id(106) { + try { + some_throwing(1337); + return 1337; + } catch(_, code) { + return code; + } + return -1; +} + +global int g_reg; + +(int, int) test107() method_id(107) { + int l_reg = 10; + g_reg = 10; + try { + ;; note, that regardless of assignment, an exception RESTORES them to previous (to 10) + ;; it's very unexpected, but is considered to be a TVM feature, not a bug + g_reg = 999; + l_reg = 999; + some_throwing(999); + } catch(_, _) { + } + ;; returns (10,10) because of an exception, see a comment above + return (g_reg, l_reg); +} + () main() { } {- - method_id | in | out -TESTCASE | 1 | 1 2 3 | 123 -TESTCASE | 1 | 3 8 9 | 389 -TESTCASE | 1 | 3 7 9 | 329 -TESTCASE | 2 | 1 2 3 | 123 -TESTCASE | 2 | 3 8 9 | 389 -TESTCASE | 2 | 3 7 9 | 329 -TESTCASE | 3 | 1 2 3 | 123 -TESTCASE | 3 | 3 8 9 | 389 -TESTCASE | 3 | 3 7 9 | 329 -TESTCASE | 4 | 4 8 9 | 4350009 -TESTCASE | 4 | 4 7 9 | 4001009 -TESTCASE | 5 | 4 8 9 | 4350009 -TESTCASE | 5 | 4 7 9 | 4001009 + method_id | in | out +TESTCASE | 101 | 1 2 3 | 123 +TESTCASE | 101 | 3 8 9 | 389 +TESTCASE | 101 | 3 7 9 | 329 +TESTCASE | 102 | 1 2 3 | 123 +TESTCASE | 102 | 3 8 9 | 389 +TESTCASE | 102 | 3 7 9 | 329 +TESTCASE | 103 | 1 2 3 | 123 +TESTCASE | 103 | 3 8 9 | 389 +TESTCASE | 103 | 3 7 9 | 329 +TESTCASE | 104 | 4 8 9 | 4350009 +TESTCASE | 104 | 4 7 9 | 4001009 +TESTCASE | 105 | 4 8 9 | 4350009 +TESTCASE | 105 | 4 7 9 | 4001009 +TESTCASE | 106 | | 1 +TESTCASE | 107 | | 10 10 -} diff --git a/crypto/func/auto-tests/tests/var-apply.fc b/crypto/func/auto-tests/tests/var-apply.fc new file mode 100644 index 00000000..e6eb4f7e --- /dev/null +++ b/crypto/func/auto-tests/tests/var-apply.fc @@ -0,0 +1,132 @@ +tuple empty_tuple() asm "NIL"; +forall X -> (tuple, ()) ~tpush(tuple t, X value) asm "TPUSH"; +builder begin_cell() asm "NEWC"; +cell end_cell(builder b) asm "ENDC"; +slice begin_parse(cell c) asm "CTOS"; + + +_ getBeginCell() { + return begin_cell; +} + +_ getBeginParse() { + return begin_parse; +} + +(int, int) test101() method_id(101) { + var (_, f_end_cell) = (0, end_cell); + builder b = (getBeginCell())().store_int(1, 32); + b~store_int(2, 32); + var s = (getBeginParse())(f_end_cell(b)); + return (s~load_int(32), s~load_int(32)); +} + +() my_throw_always() inline { + throw(1000); +} + +_ get_raiser() inline { + return my_throw_always; +} + +int test102() method_id(102) { + try { + var raiser = get_raiser(); + raiser(); ;; `some_var()` is always impure, the compiler has no considerations about its runtime value + return 0; + } catch (_, code) { + return code; + } +} + +int sum(int a, int b) impure inline { + throw_unless(1000, a + b < 24); + return a + b; +} + +int mul(int a, int b) impure inline { + throw_unless(1001, a * b < 24); + return a * b; +} + +int sum_pure(int a, int b) inline { + throw_unless(1000, a + b < 24); + return a + b; +} + +int mul_pure(int a, int b) inline { + throw_unless(1001, a * b < 24); + return a * b; +} + +int demo_handler(int op, int query_id, int a, int b) { + if (op == 0xF2) { + var func = query_id % 2 == 0 ? sum : mul; + int result = func(a, b); + return 0; ;; result not used, we test that func is nevertheless called + } + if (op == 0xF3) { + var func = query_id % 2 == 0 ? sum_pure : mul_pure; + int result = func(a, b); + return 0; ;; the same for sum_pure, since `some_var()` is always impure + } + if (op == 0xF4) { + var func = query_id % 2 == 0 ? sum : mul; + int result = func(a, b); + return result; + } + return -1; +} + +tuple test103() method_id(103) { + tuple t = empty_tuple(); + try { + t~tpush(demo_handler(0xF2, 122, 100, 200)); + } catch(_, code) { + t~tpush(code); + } + try { + t~tpush(demo_handler(0xF4, 122, 100, 200)); + } catch(_, code) { + t~tpush(code); + } + try { + t~tpush(demo_handler(0xF3, 122, 10, 10)); + } catch(_, code) { + t~tpush(code); + } + try { + t~tpush(demo_handler(0xF3, 123, 10, 10)); + } catch(_, code) { + t~tpush(code); + } + return t; +} + +() always_throw2(int x) impure { + throw (239 + x); +} + +global int -> () global_f; + +int test104() method_id(104) { + try { + global_f = always_throw2; + global_f(1); + return 0; + } catch (_, code) { + return code; + } +} + +() main() { +} + + +{- + method_id | in | out +TESTCASE | 101 | | 1 2 +TESTCASE | 102 | | 1000 +TESTCASE | 103 | | [ 1000 1000 0 1001 ] +TESTCASE | 104 | | 240 +-} diff --git a/crypto/func/codegen.cpp b/crypto/func/codegen.cpp index de45c841..aa2972c2 100644 --- a/crypto/func/codegen.cpp +++ b/crypto/func/codegen.cpp @@ -830,6 +830,8 @@ bool Op::generate_code_step(Stack& stack) { catch_stack.push_new_var(left[1]); stack.rearrange_top(catch_vars, catch_last); stack.opt_show(); + stack.o << "c1 PUSH"; + stack.o << "c3 PUSH"; stack.o << "c4 PUSH"; stack.o << "c5 PUSH"; stack.o << "c7 PUSH"; @@ -846,6 +848,8 @@ bool Op::generate_code_step(Stack& stack) { stack.o << "c7 SETCONT"; stack.o << "c5 SETCONT"; stack.o << "c4 SETCONT"; + stack.o << "c3 SETCONT"; + stack.o << "c1 SETCONT"; for (size_t begin = catch_vars.size(), end = begin; end > 0; end = begin) { begin = end >= block_size ? end - block_size : 0; stack.o << std::to_string(end - begin) + " PUSHINT"; diff --git a/crypto/func/func.h b/crypto/func/func.h index e2476911..25711db8 100644 --- a/crypto/func/func.h +++ b/crypto/func/func.h @@ -45,7 +45,7 @@ extern std::string generated_from; constexpr int optimize_depth = 20; -const std::string func_version{"0.4.5"}; +const std::string func_version{"0.4.6"}; enum Keyword { _Eof = -1, diff --git a/crypto/func/gen-abscode.cpp b/crypto/func/gen-abscode.cpp index 9989d10c..7421dabb 100644 --- a/crypto/func/gen-abscode.cpp +++ b/crypto/func/gen-abscode.cpp @@ -380,7 +380,10 @@ std::vector Expr::pre_compile(CodeBlob& code, std::vectorflags = Expr::_IsRvalue | (fun->flags & Expr::_IsImpure); } else { res = new Expr{Expr::_VarApply, {fun, x}}; - res->flags = Expr::_IsRvalue; + res->flags = Expr::_IsRvalue | Expr::_IsImpure; // for `some_var()`, don't make any considerations about runtime value, it's impure } return res; } diff --git a/crypto/funcfiftlib/funcfiftlib.cpp b/crypto/funcfiftlib/funcfiftlib.cpp index 0bef9eac..403c075d 100644 --- a/crypto/funcfiftlib/funcfiftlib.cpp +++ b/crypto/funcfiftlib/funcfiftlib.cpp @@ -37,10 +37,10 @@ td::Result compile_internal(char *config_json) { TRY_RESULT(input_json, td::json_decode(td::MutableSlice(config_json))) - auto &obj = input_json.get_object(); + td::JsonObject& config = input_json.get_object(); - TRY_RESULT(opt_level, td::get_json_object_int_field(obj, "optLevel", false)); - TRY_RESULT(sources_obj, td::get_json_object_field(obj, "sources", td::JsonValue::Type::Array, false)); + TRY_RESULT(opt_level, td::get_json_object_int_field(config, "optLevel", false)); + TRY_RESULT(sources_obj, td::get_json_object_field(config, "sources", td::JsonValue::Type::Array, false)); auto &sources_arr = sources_obj.get_array(); @@ -52,29 +52,25 @@ td::Result compile_internal(char *config_json) { funC::opt_level = std::max(0, opt_level); funC::program_envelope = true; + funC::asm_preamble = true; funC::verbosity = 0; funC::indent = 1; std::ostringstream outs, errs; - auto compile_res = funC::func_proceed(sources, outs, errs); - - if (compile_res != 0) { - return td::Status::Error(std::string("Func compilation error: ") + errs.str()); + int funC_res = funC::func_proceed(sources, outs, errs); + if (funC_res != 0) { + return td::Status::Error("FunC compilation error: " + errs.str()); } - TRY_RESULT(code_cell, fift::compile_asm(outs.str(), "/fiftlib/", false)); - TRY_RESULT(boc, vm::std_boc_serialize(code_cell)); + TRY_RESULT(fift_res, fift::compile_asm_program(outs.str(), "/fiftlib/")); td::JsonBuilder result_json; - auto result_obj = result_json.enter_object(); - result_obj("status", "ok"); - result_obj("codeBoc", td::base64_encode(boc)); - result_obj("fiftCode", outs.str()); - result_obj("codeHashHex", code_cell->get_hash().to_hex()); - result_obj.leave(); - - outs.clear(); - errs.clear(); + auto obj = result_json.enter_object(); + obj("status", "ok"); + obj("fiftCode", std::move(fift_res.fiftCode)); + obj("codeBoc", std::move(fift_res.codeBoc64)); + obj("codeHashHex", std::move(fift_res.codeHashHex)); + obj.leave(); return result_json.string_builder().as_cslice().str(); } diff --git a/crypto/smartcont/tolk-stdlib/common.tolk b/crypto/smartcont/tolk-stdlib/common.tolk new file mode 100644 index 00000000..82757c22 --- /dev/null +++ b/crypto/smartcont/tolk-stdlib/common.tolk @@ -0,0 +1,756 @@ +// Standard library for Tolk (LGPL licence). +// It contains common functions that are available out of the box, the user doesn't have to import anything. +// More specific functions are required to be imported explicitly, like "@stdlib/tvm-dicts". +tolk 0.9 + +/** + Tuple manipulation primitives. + Elements of a tuple can be of arbitrary type. + Note that atomic type `tuple` can't be cast to composite tuple type (e.g. `[int, cell]`) and vise versa. + */ + +/// Creates a tuple with zero elements. +@pure +fun createEmptyTuple(): tuple + asm "NIL"; + +/// Appends a value to tuple, resulting in `Tuple t' = (x1, ..., xn, value)`. +/// If its size exceeds 255, throws a type check exception. +@pure +fun tuplePush(mutate self: tuple, value: T): void + asm "TPUSH"; + +/// Returns the first element of a non-empty tuple. +/// `t.0` is actually the same as `t.tupleFirst()` +@pure +fun tupleFirst(self: tuple): T + asm "FIRST"; + +/// Returns the [`index`]-th element of a tuple. +/// `t.i` is actually the same as `t.tupleAt(i)` +@pure +fun tupleAt(self: tuple, index: int): T + builtin; + +/// Sets the [`index`]-th element of a tuple to a specified value +/// (element with this index must already exist, a new element isn't created). +/// `t.i = value` is actually the same as `t.tupleSetAt(value, i)` +@pure +fun tupleSetAt(mutate self: tuple, value: T, index: int): void + builtin; + +/// Returns the size of a tuple (elements count in it). +@pure +fun tupleSize(self: tuple): int + asm "TLEN"; + +/// Returns the last element of a non-empty tuple. +@pure +fun tupleLast(self: tuple): T + asm "LAST"; + + +/** + Mathematical primitives. + */ + +/// Computes the minimum of two integers. +@pure +fun min(x: int, y: int): int + asm "MIN"; + +/// Computes the maximum of two integers. +@pure +fun max(x: int, y: int): int + asm "MAX"; + +/// Sorts two integers. +@pure +fun minMax(x: int, y: int): (int, int) + asm "MINMAX"; + +/// Computes the absolute value of an integer. +@pure +fun abs(x: int): int + asm "ABS"; + +/// Returns the sign of an integer: `-1` if x < 0, `0` if x == 0, `1` if x > 0. +@pure +fun sign(x: int): int + asm "SGN"; + +/// Computes the quotient and remainder of [x] / [y]. Example: divMod(112,3) = (37,1) +@pure +fun divMod(x: int, y: int): (int, int) + asm "DIVMOD"; + +/// Computes the remainder and quotient of [x] / [y]. Example: modDiv(112,3) = (1,37) +@pure +fun modDiv(x: int, y: int): (int, int) + asm(-> 1 0) "DIVMOD"; + +/// Computes multiple-then-divide: floor([x] * [y] / [z]). +/// The intermediate result is stored in a 513-bit integer to prevent precision loss. +@pure +fun mulDivFloor(x: int, y: int, z: int): int + builtin; + +/// Similar to `mulDivFloor`, but rounds the result: round([x] * [y] / [z]). +@pure +fun mulDivRound(x: int, y: int, z: int): int + builtin; + +/// Similar to `mulDivFloor`, but ceils the result: ceil([x] * [y] / [z]). +@pure +fun mulDivCeil(x: int, y: int, z: int): int + builtin; + +/// Computes the quotient and remainder of ([x] * [y] / [z]). Example: mulDivMod(112,3,10) = (33,6) +@pure +fun mulDivMod(x: int, y: int, z: int): (int, int) + builtin; + + +/** + Global getters of environment and contract state. + */ + +const MASTERCHAIN = -1; +const BASECHAIN = 0; + +/// Returns current Unix timestamp (in seconds). +@pure +fun now(): int + asm "NOW"; + +/// Returns the internal address of the current smart contract as a Slice with a `MsgAddressInt`. +/// If necessary, it can be parsed further using primitives such as [parseStandardAddress]. +@pure +fun getMyAddress(): slice + asm "MYADDR"; + +/// Returns the balance (in nanotoncoins) of the smart contract at the start of Computation Phase. +/// Note that RAW primitives such as [sendMessage] do not update this field. +@pure +fun getMyOriginalBalance(): int + asm "BALANCE" "FIRST"; + +/// Same as [getMyOriginalBalance], but returns a tuple: +/// `int` — balance in nanotoncoins; +/// `cell` — a dictionary with 32-bit keys representing the balance of "extra currencies". +@pure +fun getMyOriginalBalanceWithExtraCurrencies(): [int, cell?] + asm "BALANCE"; + +/// Returns the logical time of the current transaction. +@pure +fun getLogicalTime(): int + asm "LTIME"; + +/// Returns the starting logical time of the current block. +@pure +fun getCurrentBlockLogicalTime(): int + asm "BLOCKLT"; + +/// Returns the value of the global configuration parameter with integer index `i` as a `cell` or `null` value. +@pure +fun getBlockchainConfigParam(x: int): cell? + asm "CONFIGOPTPARAM"; + +/// Returns the persistent contract storage cell. It can be parsed or modified with slice and builder primitives later. +@pure +fun getContractData(): cell + asm "c4 PUSH"; + +/// Sets `cell` [c] as persistent contract data. You can update persistent contract storage with this primitive. +fun setContractData(c: cell): void + asm "c4 POP"; + +/// Retrieves code of smart-contract from c7 +@pure +fun getContractCode(): cell + asm "MYCODE"; + +/// Creates an output action that would change this smart contract code to that given by cell [newCode]. +/// Notice that this change will take effect only after the successful termination of the current run of the smart contract. +fun setContractCodePostponed(newCode: cell): void + asm "SETCODE"; + +/// Commits the current state of registers `c4` (“persistent data”) and `c5` (“actions”) +/// so that the current execution is considered “successful” with the saved values even if an exception +/// in Computation Phase is thrown later. +fun commitContractDataAndActions(): void + asm "COMMIT"; + + +/** + Signature checks, hashing, cryptography. + */ + +/// Computes the representation hash of a `cell` [c] and returns it as a 256-bit unsigned integer `x`. +/// Useful for signing and checking signatures of arbitrary entities represented by a tree of cells. +@pure +fun cellHash(c: cell): int + asm "HASHCU"; + +/// Computes the hash of a `slice s` and returns it as a 256-bit unsigned integer `x`. +/// The result is the same as if an ordinary cell containing only data and references from `s` had been created +/// and its hash computed by [cellHash]. +@pure +fun sliceHash(s: slice): int + asm "HASHSU"; + +/// Computes sha256 of the data bits of `slice` [s]. If the bit length of `s` is not divisible by eight, +/// throws a cell underflow exception. The hash value is returned as a 256-bit unsigned integer `x`. +@pure +fun stringHash(s: slice): int + asm "SHA256U"; + +/// Checks the Ed25519-`signature` of a `hash` (a 256-bit unsigned integer, usually computed as the hash of some data) +/// using [publicKey] (also represented by a 256-bit unsigned integer). +/// The signature must contain at least 512 data bits; only the first 512 bits are used. +/// The result is `−1` if the signature is valid, `0` otherwise. +/// Note that `CHKSIGNU` creates a 256-bit slice with the hash and calls `CHKSIGNS`. +/// That is, if [hash] is computed as the hash of some data, these data are hashed twice, +/// the second hashing occurring inside `CHKSIGNS`. +@pure +fun isSignatureValid(hash: int, signature: slice, publicKey: int): bool + asm "CHKSIGNU"; + +/// Checks whether [signature] is a valid Ed25519-signature of the data portion of `slice data` using `publicKey`, +/// similarly to [isSignatureValid]. +/// If the bit length of [data] is not divisible by eight, throws a cell underflow exception. +/// The verification of Ed25519 signatures is the standard one, +/// with sha256 used to reduce [data] to the 256-bit number that is actually signed. +@pure +fun isSliceSignatureValid(data: slice, signature: slice, publicKey: int): bool + asm "CHKSIGNS"; + +/// Generates a new pseudo-random unsigned 256-bit integer x. +fun random(): int + asm "RANDU256"; + +/// Generates a new pseudo-random integer z in the range 0..range−1 (or range..−1, if range < 0). +/// More precisely, an unsigned random value x is generated as in random; then z := x * range / 2^256 is computed. +fun randomRange(range: int): int + asm "RAND"; + +/// Returns the current random seed as an unsigned 256-bit integer. +@pure +fun randomGetSeed(): int + asm "RANDSEED"; + +/// Sets the random seed to unsigned 256-bit seed. +fun randomSetSeed(seed: int): void + asm "SETRAND"; + +/// Initializes (mixes) random seed with unsigned 256-bit integer x. +fun randomizeBy(x: int): void + asm "ADDRAND"; + +/// Initializes random seed using current time. Don't forget to call this before calling `random`! +fun randomizeByLogicalTime(): void + asm "LTIME" "ADDRAND"; + + +/** + Size computation primitives. + They may be useful for computing storage fees of user-provided data. + */ + +/// Returns `(x, y, z, -1)` or `(null, null, null, 0)`. +/// Recursively computes the count of distinct cells `x`, data bits `y`, and cell references `z` +/// in the DAG rooted at `cell` [c], effectively returning the total storage used by this DAG taking into account +/// the identification of equal cells. +/// The values of `x`, `y`, and `z` are computed by a depth-first traversal of this DAG, +/// with a hash table of visited cell hashes used to prevent visits of already-visited cells. +/// The total count of visited cells `x` cannot exceed non-negative [maxCells]; +/// otherwise the computation is aborted before visiting the `(maxCells + 1)`-st cell and +/// a zero flag is returned to indicate failure. If [c] is `null`, returns `x = y = z = 0`. +@pure +fun calculateCellSize(c: cell, maxCells: int): (int, int, int, bool) + asm "CDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT"; + +/// Similar to [calculateCellSize], but accepting a `slice` [s] instead of a `cell`. +/// The returned value of `x` does not take into account the cell that contains the `slice` [s] itself; +/// however, the data bits and the cell references of [s] are accounted for in `y` and `z`. +@pure +fun calculateSliceSize(s: slice, maxCells: int): (int, int, int, bool) + asm "SDATASIZEQ NULLSWAPIFNOT2 NULLSWAPIFNOT"; + +/// A non-quiet version of [calculateCellSize] that throws a cell overflow exception (`8`) on failure. +fun calculateCellSizeStrict(c: cell, maxCells: int): (int, int, int) + asm "CDATASIZE"; + +/// A non-quiet version of [calculateSliceSize] that throws a cell overflow exception (`8`) on failure. +fun calculateSliceSizeStrict(s: slice, maxCells: int): (int, int, int) + asm "SDATASIZE"; + +/// Returns the depth of `cell` [c]. +/// If [c] has no references, then return `0`; +/// otherwise the returned value is one plus the maximum of depths of cells referred to from [c]. +/// If [c] is a `null` instead of a cell, returns zero. +@pure +fun getCellDepth(c: cell?): int + asm "CDEPTH"; + +/// Returns the depth of `slice` [s]. +/// If [s] has no references, then returns `0`; +/// otherwise the returned value is one plus the maximum of depths of cells referred to from [s]. +@pure +fun getSliceDepth(s: slice): int + asm "SDEPTH"; + +/// Returns the depth of `builder` [b]. +/// If no cell references are stored in [b], then returns 0; +/// otherwise the returned value is one plus the maximum of depths of cells referred to from [b]. +@pure +fun getBuilderDepth(b: builder): int + asm "BDEPTH"; + + +/** + Debug primitives. + Only works for local TVM execution with debug level verbosity. + */ + +/// Dump a variable [x] to the debug log. +fun debugPrint(x: T): void + builtin; + +/// Dump a string [x] to the debug log. +fun debugPrintString(x: T): void + builtin; + +/// Dumps the stack (at most the top 255 values) and shows the total stack depth. +fun debugDumpStack(): void + builtin; + + +/** + Slice primitives: parsing cells. + When you _load_ some data, you mutate the slice (shifting an internal pointer on the stack). + When you _preload_ some data, you just get the result without mutating the slice. + */ + +/// Converts a `cell` [c] into a `slice`. Notice that [c] must be either an ordinary cell, +/// or an exotic cell (see [TVM.pdf](https://ton-blockchain.github.io/docs/tvm.pdf), 3.1.2) +/// which is automatically loaded to yield an ordinary cell `c'`, converted into a `slice` afterwards. +@pure +fun beginParse(c: cell): slice + asm "CTOS"; + +/// Checks if slice is empty. If not, throws an exception. +fun assertEndOfSlice(self: slice): void + asm "ENDS"; + +/// Loads the next reference from the slice. +@pure +fun loadRef(mutate self: slice): cell + asm( -> 1 0) "LDREF"; + +/// Preloads the next reference from the slice. +@pure +fun preloadRef(self: slice): cell + asm "PLDREF"; + +/// Loads a signed [len]-bit integer from a slice. +@pure +fun loadInt(mutate self: slice, len: int): int + builtin; + +/// Loads an unsigned [len]-bit integer from a slice. +@pure +fun loadUint(mutate self: slice, len: int): int + builtin; + +/// Loads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate slice `s''`. +@pure +fun loadBits(mutate self: slice, len: int): slice + builtin; + +/// Preloads a signed [len]-bit integer from a slice. +@pure +fun preloadInt(self: slice, len: int): int + builtin; + +/// Preloads an unsigned [len]-bit integer from a slice. +@pure +fun preloadUint(self: slice, len: int): int + builtin; + +/// Preloads the first `0 ≤ len ≤ 1023` bits from slice [s] into a separate slice. +@pure +fun preloadBits(self: slice, len: int): slice + builtin; + +/// Loads serialized amount of Toncoins (any unsigned integer up to `2^120 - 1`). +@pure +fun loadCoins(mutate self: slice): int + asm( -> 1 0) "LDGRAMS"; + +/// Loads bool (-1 or 0) from a slice +@pure +fun loadBool(mutate self: slice): bool + asm( -> 1 0) "1 LDI"; + +/// Shifts a slice pointer to [len] bits forward, mutating the slice. +@pure +fun skipBits(mutate self: slice, len: int): self + asm "SDSKIPFIRST"; + +/// Returns the first `0 ≤ len ≤ 1023` bits of a slice. +@pure +fun getFirstBits(self: slice, len: int): slice + asm "SDCUTFIRST"; + +/// Returns all but the last `0 ≤ len ≤ 1023` bits of a slice. +@pure +fun removeLastBits(mutate self: slice, len: int): self + asm "SDSKIPLAST"; + +/// Returns the last `0 ≤ len ≤ 1023` bits of a slice. +@pure +fun getLastBits(self: slice, len: int): slice + asm "SDCUTLAST"; + +/// Loads a dictionary (TL HashMapE structure, represented as TVM cell) from a slice. +/// Returns `null` if `nothing` constructor is used. +@pure +fun loadDict(mutate self: slice): cell? + asm( -> 1 0) "LDDICT"; + +/// Preloads a dictionary (cell) from a slice. +@pure +fun preloadDict(self: slice): cell? + asm "PLDDICT"; + +/// Loads a dictionary as [loadDict], but returns only the remainder of the slice. +@pure +fun skipDict(mutate self: slice): self + asm "SKIPDICT"; + +/// Loads (Maybe ^Cell) from a slice. +/// In other words, loads 1 bit: if it's true, loads the first ref, otherwise returns `null`. +@pure +fun loadMaybeRef(mutate self: slice): cell? + asm( -> 1 0) "LDOPTREF"; + +/// Preloads (Maybe ^Cell) from a slice. +@pure +fun preloadMaybeRef(self: slice): cell? + asm "PLDOPTREF"; + +/// Loads (Maybe ^Cell), but returns only the remainder of the slice. +@pure +fun skipMaybeRef(mutate self: slice): self + asm "SKIPOPTREF"; + +/** + Builder primitives: constructing cells. + When you _store_ some data, you mutate the builder (shifting an internal pointer on the stack). + All the primitives below first check whether there is enough space in the `builder`, + and only then check the range of the value being serialized. + */ + +/// Creates a new empty builder. +@pure +fun beginCell(): builder + asm "NEWC"; + +/// Converts a builder into an ordinary `cell`. +@pure +fun endCell(self: builder): cell + asm "ENDC"; + +/// Stores a reference to a cell into a builder. +@pure +fun storeRef(mutate self: builder, c: cell): self + asm(c self) "STREF"; + +/// Stores a signed [len]-bit integer into a builder (`0 ≤ len ≤ 257`). +@pure +fun storeInt(mutate self: builder, x: int, len: int): self + builtin; + +/// Stores an unsigned [len]-bit integer into a builder (`0 ≤ len ≤ 256`). +@pure +fun storeUint(mutate self: builder, x: int, len: int): self + builtin; + +/// Stores a slice into a builder. +@pure +fun storeSlice(mutate self: builder, s: slice): self + asm "STSLICER"; + +/// Stores amount of Toncoins into a builder. +@pure +fun storeCoins(mutate self: builder, x: int): self + asm "STGRAMS"; + +/// Stores bool (-1 or 0) into a builder. +/// Attention: true value is `-1`, not 1! If you pass `1` here, TVM will throw an exception. +@pure +fun storeBool(mutate self: builder, x: bool): self + asm(x self) "1 STI"; + +/// Stores dictionary (represented by TVM `cell` or `null`) into a builder. +/// In other words, stores a `1`-bit and a reference to [c] if [c] is not `null` and `0`-bit otherwise. +@pure +fun storeDict(mutate self: builder, c: cell?): self + asm(c self) "STDICT"; + +/// Stores (Maybe ^Cell) into a builder. +/// In other words, if cell is `null`, store '0' bit; otherwise, store '1' and a ref to [c]. +@pure +fun storeMaybeRef(mutate self: builder, c: cell?): self + asm(c self) "STOPTREF"; + +/// Concatenates two builders. +@pure +fun storeBuilder(mutate self: builder, from: builder): self + asm "STBR"; + +/// Stores a slice representing TL addr_none$00 (two `0` bits). +@pure +fun storeAddressNone(mutate self: builder): self + asm "b{00} STSLICECONST"; + + +/** + Slice size primitives. + */ + +/// Returns the number of references in a slice. +@pure +fun getRemainingRefsCount(self: slice): int + asm "SREFS"; + +/// Returns the number of data bits in a slice. +@pure +fun getRemainingBitsCount(self: slice): int + asm "SBITS"; + +/// Returns both the number of data bits and the number of references in a slice. +@pure +fun getRemainingBitsAndRefsCount(self: slice): (int, int) + asm "SBITREFS"; + +/// Checks whether a slice is empty (i.e., contains no bits of data and no cell references). +@pure +fun isEndOfSlice(self: slice): bool + asm "SEMPTY"; + +/// Checks whether a slice has no bits of data. +@pure +fun isEndOfSliceBits(self: slice): bool + asm "SDEMPTY"; + +/// Checks whether a slice has no references. +@pure +fun isEndOfSliceRefs(self: slice): bool + asm "SREMPTY"; + +/// Checks whether data parts of two slices coinside. +@pure +fun isSliceBitsEqual(self: slice, b: slice): bool + asm "SDEQ"; + +/// Returns the number of cell references already stored in a builder. +@pure +fun getBuilderRefsCount(self: builder): int + asm "BREFS"; + +/// Returns the number of data bits already stored in a builder. +@pure +fun getBuilderBitsCount(self: builder): int + asm "BBITS"; + + +/** + Address manipulation primitives. + The address manipulation primitives listed below serialize and deserialize values according to the following TL-B scheme: + ```TL-B + addr_none$00 = MsgAddressExt; + addr_extern$01 len:(## 8) external_address:(bits len) + = MsgAddressExt; + anycast_info$_ depth:(#<= 30) { depth >= 1 } + rewrite_pfx:(bits depth) = Anycast; + addr_std$10 anycast:(Maybe Anycast) + workchain_id:int8 address:bits256 = MsgAddressInt; + addr_var$11 anycast:(Maybe Anycast) addr_len:(## 9) + workchain_id:int32 address:(bits addr_len) = MsgAddressInt; + _ _:MsgAddressInt = MsgAddress; + _ _:MsgAddressExt = MsgAddress; + + int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool + src:MsgAddress dest:MsgAddressInt + value:CurrencyCollection ihr_fee:Grams fwd_fee:Grams + created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; + ext_out_msg_info$11 src:MsgAddress dest:MsgAddressExt + created_lt:uint64 created_at:uint32 = CommonMsgInfoRelaxed; + ``` + A deserialized `MsgAddress` is represented by a tuple `t` as follows: + + - `addr_none` is represented by `t = (0)`, + i.e., a tuple containing exactly one integer equal to zero. + - `addr_extern` is represented by `t = (1, s)`, + where slice `s` contains the field `external_address`. In other words, ` + t` is a pair (a tuple consisting of two entries), containing an integer equal to one and slice `s`. + - `addr_std` is represented by `t = (2, u, x, s)`, + where `u` is either a `null` (if `anycast` is absent) or a slice `s'` containing `rewrite_pfx` (if anycast is present). + Next, integer `x` is the `workchain_id`, and slice `s` contains the address. + - `addr_var` is represented by `t = (3, u, x, s)`, + where `u`, `x`, and `s` have the same meaning as for `addr_std`. + */ + +/// Loads from slice [s] the only prefix that is a valid `MsgAddress`, +/// and returns both this prefix `s'` and the remainder `s''` of [s] as slices. +@pure +fun loadAddress(mutate self: slice): slice + asm( -> 1 0) "LDMSGADDR"; + +/// Decomposes slice [s] containing a valid `MsgAddress` into a `tuple t` with separate fields of this `MsgAddress`. +/// If [s] is not a valid `MsgAddress`, a cell deserialization exception is thrown. +@pure +fun parseAddress(s: slice): tuple + asm "PARSEMSGADDR"; + +/// Parses slice [s] containing a valid `MsgAddressInt` (usually a `msg_addr_std`), +/// applies rewriting from the anycast (if present) to the same-length prefix of the address, +/// and returns both the workchain and the 256-bit address as integers. +/// If the address is not 256-bit, or if [s] is not a valid serialization of `MsgAddressInt`, +/// throws a cell deserialization exception. +@pure +fun parseStandardAddress(s: slice): (int, int) + asm "REWRITESTDADDR"; + +/// Creates a slice representing TL addr_none$00 (two `0` bits). +@pure +fun createAddressNone(): slice + asm "b{00} PUSHSLICE"; + +/// Returns if a slice pointer contains an empty address. +/// In other words, a slice starts with two `0` bits (TL addr_none$00). +@pure +fun addressIsNone(s: slice): bool + asm "2 PLDU" "0 EQINT"; + + +/** + Reserving Toncoins on balance and its flags. + */ + +/// mode = 0: Reserve exact amount of nanotoncoins +const RESERVE_MODE_EXACT_AMOUNT = 0; +/// +1: Actually reserves all but amount, meaning `currentContractBalance - amount` +const RESERVE_MODE_ALL_BUT_AMOUNT = 1; +/// +2: Actually set `min(amount, currentContractBalance)` (without this mode, if amount is greater, the action will fail) +const RESERVE_MODE_AT_MOST = 2; +/// +4: [amount] is increased by the _original_ balance of the current account (before the compute phase). +const RESERVE_MODE_INCREASE_BY_ORIGINAL_BALANCE = 4; +/// +8: Actually sets `amount = -amount` before performing any further actions. +const RESERVE_MODE_NEGATE_AMOUNT = 8; +/// +16: If this action fails, the transaction will be bounced. +const RESERVE_MODE_BOUNCE_ON_ACTION_FAIL = 16; + +/// Creates an output action which would reserve Toncoins on balance. +/// For [reserveMode] consider constants above. +fun reserveToncoinsOnBalance(nanoTonCoins: int, reserveMode: int): void + asm "RAWRESERVE"; + +/// Similar to [reserveToncoinsOnBalance], but also accepts a dictionary extraAmount (represented by a cell or null) +/// with extra currencies. In this way currencies other than Toncoin can be reserved. +fun reserveExtraCurrenciesOnBalance(nanoTonCoins: int, extraAmount: cell?, reserveMode: int): void + asm "RAWRESERVEX"; + + +/** + Messages sending and parsing primitives. + Working with messages is low-level right now, but still, every contract should do that. + + `Message` structure, its header and so on are specified in TL-B scheme, particularly: + int_msg_info$0 ihr_disabled:Bool bounce:Bool bounced:Bool ... = CommonMsgInfo; + */ + +/// 0b011000 tag - 0, ihr_disabled - 1, bounce - 1, bounced - 0, src = adr_none$00 +const BOUNCEABLE = 0x18; +/// 0b010000 tag - 0, ihr_disabled - 1, bounce - 0, bounced - 0, src = adr_none$00 +const NON_BOUNCEABLE = 0x10; + +/// Load msgFlags from incoming message body (4 bits). +@pure +fun loadMessageFlags(mutate self: slice): int + asm( -> 1 0) "4 LDU"; + +/// Having msgFlags (4 bits), check that a message is bounced. +/// Effectively, it's `msgFlags & 1` (the lowest bit present). +@pure +fun isMessageBounced(msgFlags: int): bool + asm "2 PUSHINT" "MODR"; + +/// Skip 0xFFFFFFFF prefix (when a message is bounced). +@pure +fun skipBouncedPrefix(mutate self: slice): self + asm "32 PUSHINT" "SDSKIPFIRST"; + +/// The guideline recommends to start the body of an internal message with uint32 `op` and uint64 `queryId`. +@pure +fun loadMessageOp(mutate self: slice): int + asm( -> 1 0) "32 LDU"; + +@pure +fun skipMessageOp(mutate self: slice): self + asm "32 PUSHINT" "SDSKIPFIRST"; + +@pure +fun storeMessageOp(mutate self: builder, op: int): self + asm(op self) "32 STU"; + +/// The guideline recommends that uint64 `queryId` should follow uint32 `op`. +@pure +fun loadMessageQueryId(mutate self: slice): int + asm( -> 1 0) "64 LDU"; + +@pure +fun skipMessageQueryId(mutate self: slice): self + asm "64 PUSHINT" "SDSKIPFIRST"; + +@pure +fun storeMessageQueryId(mutate self: builder, queryId: int): self + asm(queryId self) "64 STU"; + +/// SEND MODES - https://docs.ton.org/tvm.pdf page 137, SENDRAWMSG + +/// mode = 0 is used for ordinary messages; the gas fees are deducted from the senging amount; action phaes should NOT be ignored. +const SEND_MODE_REGULAR = 0; +/// +1 means that the sender wants to pay transfer fees separately. +const SEND_MODE_PAY_FEES_SEPARATELY = 1; +/// +2 means that any errors arising while processing this message during the action phase should be ignored. +const SEND_MODE_IGNORE_ERRORS = 2; +/// in the case of action fail - bounce transaction. No effect if SEND_MODE_IGNORE_ERRORS (+2) is used. TVM UPGRADE 2023-07. https://docs.ton.org/learn/tvm-instructions/tvm-upgrade-2023-07#sending-messages +const SEND_MODE_BOUNCE_ON_ACTION_FAIL = 16; +/// mode = 32 means that the current account must be destroyed if its resulting balance is zero. +const SEND_MODE_DESTROY = 32; +/// mode = 64 is used for messages that carry all the remaining value of the inbound message in addition to the value initially indicated in the new message. +const SEND_MODE_CARRY_ALL_REMAINING_MESSAGE_VALUE = 64; +/// mode = 128 is used for messages that are to carry all the remaining balance of the current smart contract (instead of the value originally indicated in the message). +const SEND_MODE_CARRY_ALL_BALANCE = 128; +/// do not create an action, only estimate fee. TVM UPGRADE 2023-07. https://docs.ton.org/learn/tvm-instructions/tvm-upgrade-2023-07#sending-messages +const SEND_MODE_ESTIMATE_FEE_ONLY = 1024; +/// Other modes affect the fee calculation as follows: +/// +64 substitutes the entire balance of the incoming message as an outcoming value (slightly inaccurate, gas expenses that cannot be estimated before the computation is completed are not taken into account). +/// +128 substitutes the value of the entire balance of the contract before the start of the computation phase (slightly inaccurate, since gas expenses that cannot be estimated before the completion of the computation phase are not taken into account). + +/// Sends a raw message — a correctly serialized TL object `Message X`. +/// For `mode`, see constants above (except SEND_MODE_ESTIMATE_FEE_ONLY). +/// This function is still available, but deprecated: consider using [sendMessage]. +@deprecated +fun sendRawMessage(msg: cell, mode: int): void + asm "SENDRAWMSG"; + +/// Creates an output action and returns a fee for creating a message. +/// Mode has the same effect as in the case of SENDRAWMSG. +/// For mode including SEND_MODE_ESTIMATE_FEE_ONLY it just returns estimated fee without sending a message. +fun sendMessage(msg: cell, mode: int): int + asm "SENDMSG"; diff --git a/crypto/smartcont/tolk-stdlib/gas-payments.tolk b/crypto/smartcont/tolk-stdlib/gas-payments.tolk new file mode 100644 index 00000000..9873ca94 --- /dev/null +++ b/crypto/smartcont/tolk-stdlib/gas-payments.tolk @@ -0,0 +1,69 @@ +// A part of standard library for Tolk +tolk 0.9 + +/** + Gas and payment related primitives. + */ + +/// Returns amount of gas (in gas units) consumed in current Computation Phase. +fun getGasConsumedAtTheMoment(): int + asm "GASCONSUMED"; + +/// This function is required to be called when you process an external message (from an outer world) +/// and "accept" it to blockchain. +/// Without calling this function, an external message would be discarded. +/// As an effect, the current smart contract agrees to buy some gas to finish the current transaction. +/// For more details, check [accept_message effects](https://ton.org/docs/#/smart-contracts/accept). +fun acceptExternalMessage(): void + asm "ACCEPT"; + +/// When processing an internal message, by default, the limit of gas consumption is determined by incoming message. +/// Functions [setGasLimit] and [setGasLimitToMaximum] allow you to change this behavior. +/// Sets current gas limit `gl` to its maximal allowed value `gm`, and resets the gas credit `gc` to zero, +/// decreasing the value of `gr` by `gc` in the process. +fun setGasLimitToMaximum(): void + asm "ACCEPT"; + +/// When processing an internal message, by default, the limit of gas consumption is determined by incoming message. +/// Functions [setGasLimit] and [setGasLimitToMaximum] allow you to change this behavior. +/// Sets current gas limit `gl` to the minimum of limit and `gm`, and resets the gas credit `gc` to zero. +/// If the gas consumed so far (including the present instruction) exceeds the resulting value of `gl`, +/// an (unhandled) out of gas exception is thrown before setting new gas limits. +fun setGasLimit(limit: int): void + asm "SETGASLIMIT"; + +/// Calculates fee (amount in nanotoncoins to be paid) for a transaction which consumed [gasUsed] gas units. +fun calculateGasFee(workchain: int, gasUsed: int): int + asm(gasUsed workchain) "GETGASFEE"; + +/// Same as [calculateGasFee], but without flat price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) +fun calculateGasFeeWithoutFlatPrice(workchain: int, gasUsed: int): int + asm(gasUsed workchain) "GETGASFEESIMPLE"; + +/// Calculates amount of nanotoncoins you should pay for storing a contract of provided size for [seconds]. +/// [bits] and [cells] represent contract state (code + data). +fun calculateStorageFee(workchain: int, seconds: int, bits: int, cells: int): int + asm(cells bits seconds workchain) "GETSTORAGEFEE"; + +/// Calculates amount of nanotoncoins you should pay to send a message of specified size. +fun calculateMessageFee(workchain: int, bits: int, cells: int): int + asm(cells bits workchain) "GETFORWARDFEE"; + +/// Same as [calculateMessageFee], but without lump price (you have supposed to read https://docs.ton.org/develop/howto/fees-low-level) +fun calculateMessageFeeWithoutLumpPrice(workchain: int, bits: int, cells: int): int + asm(cells bits workchain) "GETFORWARDFEESIMPLE"; + +/// Calculates fee that was paid by the sender of an incoming internal message. +fun calculateOriginalMessageFee(workchain: int, incomingFwdFee: int): int + asm(incomingFwdFee workchain) "GETORIGINALFWDFEE"; + +/// Returns the amount of nanotoncoins current contract debts for storage. ("due" and "debt" are synonyms) +/// If it has no debt, `0` is returned. +fun getMyStorageDuePayment(): int + asm "DUEPAYMENT"; + +/// Returns the amount of nanotoncoins charged for storage. +/// (during storage phase preceeding to current computation phase) +@pure +fun getMyStoragePaidPayment(): int + asm "STORAGEFEES"; diff --git a/crypto/smartcont/tolk-stdlib/lisp-lists.tolk b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk new file mode 100644 index 00000000..e63438b5 --- /dev/null +++ b/crypto/smartcont/tolk-stdlib/lisp-lists.tolk @@ -0,0 +1,39 @@ +// A part of standard library for Tolk +tolk 0.9 + +/** + Lisp-style lists are nested 2-elements tuples: `(1, (2, (3, null)))` represents list `[1, 2, 3]`. + Elements of a list can be of different types. + Empty list is conventionally represented as TVM `null` value. + */ + +@pure +fun createEmptyList(): tuple + asm "PUSHNULL"; + +/// Adds an element to the beginning of lisp-style list. +/// Note, that it does not mutate the list: instead, it returns a new one (it's a lisp pattern). +@pure +fun listPrepend(head: X, tail: tuple?): tuple + asm "CONS"; + +/// Extracts the head and the tail of lisp-style list. +@pure +fun listSplit(list: tuple): (X, tuple?) + asm "UNCONS"; + +/// Extracts the tail and the head of lisp-style list. +/// After extracting the last element, tuple is assigned to null. +@pure +fun listNext(mutate self: tuple?): X + asm( -> 1 0) "UNCONS"; + +/// Returns the head of lisp-style list. +@pure +fun listGetHead(list: tuple): X + asm "CAR"; + +/// Returns the tail of lisp-style list. +@pure +fun listGetTail(list: tuple): tuple? + asm "CDR"; diff --git a/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk new file mode 100644 index 00000000..ee205687 --- /dev/null +++ b/crypto/smartcont/tolk-stdlib/tvm-dicts.tolk @@ -0,0 +1,312 @@ +// A part of standard library for Tolk +tolk 0.9 + +/** + Dictionaries are represented as `cell` data type (cells can store anything, dicts in particular). + Currently, they have very low-level API very close to TVM internals. + Most of functions are duplicated for three common cases: + - iDict* - dicts with signed integer keys + - uDict* - dicts with unsigned integer keys + - sDict* - dicts with arbitrary slice keys + When accessing a dict element, you should not only provide a key, but provide keyLen, + since for optimization, key length is not stored in the dictionary itself. + Every dictionary object (`self` parameter) can be null. TVM NULL is essentially "empty dictionary". + */ + +/// Creates an empty dictionary, which is actually a null value. Equivalent to PUSHNULL +@pure +fun createEmptyDict(): cell? + asm "NEWDICT"; + +/// Checks whether a dictionary is empty. +@pure +fun dictIsEmpty(self: cell?): bool + asm "DICTEMPTY"; + + +@pure +fun iDictGet(self: cell?, keyLen: int, key: int): (slice?, bool) + asm(key self keyLen) "DICTIGET" "NULLSWAPIFNOT"; + +@pure +fun uDictGet(self: cell?, keyLen: int, key: int): (slice?, bool) + asm(key self keyLen) "DICTUGET" "NULLSWAPIFNOT"; + +@pure +fun sDictGet(self: cell?, keyLen: int, key: slice): (slice?, bool) + asm(key self keyLen) "DICTGET" "NULLSWAPIFNOT"; + + +@pure +fun iDictSet(mutate self: cell?, keyLen: int, key: int, value: slice): void + asm(value key self keyLen) "DICTISET"; + +@pure +fun uDictSet(mutate self: cell?, keyLen: int, key: int, value: slice): void + asm(value key self keyLen) "DICTUSET"; + +@pure +fun sDictSet(mutate self: cell?, keyLen: int, key: slice, value: slice): void + asm(value key self keyLen) "DICTSET"; + + +@pure +fun iDictSetRef(mutate self: cell?, keyLen: int, key: int, value: cell): void + asm(value key self keyLen) "DICTISETREF"; + +@pure +fun uDictSetRef(mutate self: cell?, keyLen: int, key: int, value: cell): void + asm(value key self keyLen) "DICTUSETREF"; + +@pure +fun sDictSetRef(mutate self: cell?, keyLen: int, key: slice, value: cell): void + asm(value key self keyLen) "DICTSETREF"; + + +@pure +fun iDictSetIfNotExists(mutate self: cell?, keyLen: int, key: int, value: slice): bool + asm(value key self keyLen) "DICTIADD"; + +@pure +fun uDictSetIfNotExists(mutate self: cell?, keyLen: int, key: int, value: slice): bool + asm(value key self keyLen) "DICTUADD"; + + +@pure +fun iDictSetIfExists(mutate self: cell?, keyLen: int, key: int, value: slice): bool + asm(value key self keyLen) "DICTIREPLACE"; + +@pure +fun uDictSetIfExists(mutate self: cell?, keyLen: int, key: int, value: slice): bool + asm(value key self keyLen) "DICTUREPLACE"; + + +@pure +fun iDictGetRef(self: cell?, keyLen: int, key: int): (cell?, bool) + asm(key self keyLen) "DICTIGETREF" "NULLSWAPIFNOT"; + +@pure +fun uDictGetRef(self: cell?, keyLen: int, key: int): (cell?, bool) + asm(key self keyLen) "DICTUGETREF" "NULLSWAPIFNOT"; + +@pure +fun sDictGetRef(self: cell?, keyLen: int, key: slice): (cell?, bool) + asm(key self keyLen) "DICTGETREF" "NULLSWAPIFNOT"; + + +@pure +fun iDictGetRefOrNull(self: cell?, keyLen: int, key: int): cell? + asm(key self keyLen) "DICTIGETOPTREF"; + +@pure +fun uDictGetRefOrNull(self: cell?, keyLen: int, key: int): cell? + asm(key self keyLen) "DICTUGETOPTREF"; + +@pure +fun sDictGetRefOrNull(self: cell?, keyLen: int, key: slice): cell? + asm(key self keyLen) "DICTGETOPTREF"; + + +@pure +fun iDictDelete(mutate self: cell?, keyLen: int, key: int): bool + asm(key self keyLen) "DICTIDEL"; + +@pure +fun uDictDelete(mutate self: cell?, keyLen: int, key: int): bool + asm(key self keyLen) "DICTUDEL"; + +@pure +fun sDictDelete(mutate self: cell?, keyLen: int, key: slice): bool + asm(key self keyLen) "DICTDEL"; + + +@pure +fun iDictSetAndGet(mutate self: cell?, keyLen: int, key: int, value: slice): (slice?, bool) + asm(value key self keyLen) "DICTISETGET" "NULLSWAPIFNOT"; + +@pure +fun uDictSetAndGet(mutate self: cell?, keyLen: int, key: int, value: slice): (slice?, bool) + asm(value key self keyLen) "DICTUSETGET" "NULLSWAPIFNOT"; + +@pure +fun sDictSetAndGet(mutate self: cell?, keyLen: int, key: slice, value: slice): (slice?, bool) + asm(value key self keyLen) "DICTSETGET" "NULLSWAPIFNOT"; + + +@pure +fun iDictSetAndGetRefOrNull(mutate self: cell?, keyLen: int, key: int, value: cell): cell? + asm(value key self keyLen) "DICTISETGETOPTREF"; + +@pure +fun uDictSetAndGetRefOrNull(mutate self: cell?, keyLen: int, key: int, value: cell): cell? + asm(value key self keyLen) "DICTUSETGETOPTREF"; + + +@pure +fun iDictDeleteAndGet(mutate self: cell?, keyLen: int, key: int): (slice?, bool) + asm(key self keyLen) "DICTIDELGET" "NULLSWAPIFNOT"; + +@pure +fun uDictDeleteAndGet(mutate self: cell?, keyLen: int, key: int): (slice?, bool) + asm(key self keyLen) "DICTUDELGET" "NULLSWAPIFNOT"; + +@pure +fun sDictDeleteAndGet(mutate self: cell?, keyLen: int, key: slice): (slice?, bool) + asm(key self keyLen) "DICTDELGET" "NULLSWAPIFNOT"; + + +@pure +fun iDictSetBuilder(mutate self: cell?, keyLen: int, key: int, value: builder): void + asm(value key self keyLen) "DICTISETB"; + +@pure +fun uDictSetBuilder(mutate self: cell?, keyLen: int, key: int, value: builder): void + asm(value key self keyLen) "DICTUSETB"; + +@pure +fun sDictSetBuilder(mutate self: cell?, keyLen: int, key: slice, value: builder): void + asm(value key self keyLen) "DICTSETB"; + + +@pure +fun iDictSetBuilderIfNotExists(mutate self: cell?, keyLen: int, key: int, value: builder): bool + asm(value key self keyLen) "DICTIADDB"; + +@pure +fun uDictSetBuilderIfNotExists(mutate self: cell?, keyLen: int, key: int, value: builder): bool + asm(value key self keyLen) "DICTUADDB"; + +@pure +fun iDictSetBuilderIfExists(mutate self: cell?, keyLen: int, key: int, value: builder): bool + asm(value key self keyLen) "DICTIREPLACEB"; + +@pure +fun uDictSetBuilderIfExists(mutate self: cell?, keyLen: int, key: int, value: builder): bool + asm(value key self keyLen) "DICTUREPLACEB"; + + +@pure +fun iDictDeleteFirstAndGet(mutate self: cell?, keyLen: int): (int?, slice?, bool) + asm(-> 0 2 1 3) "DICTIREMMIN" "NULLSWAPIFNOT2"; + +@pure +fun uDictDeleteFirstAndGet(mutate self: cell?, keyLen: int): (int?, slice?, bool) + asm(-> 0 2 1 3) "DICTUREMMIN" "NULLSWAPIFNOT2"; + +@pure +fun sDictDeleteFirstAndGet(mutate self: cell?, keyLen: int): (slice?, slice?, bool) + asm(-> 0 2 1 3) "DICTREMMIN" "NULLSWAPIFNOT2"; + + +@pure +fun iDictDeleteLastAndGet(mutate self: cell?, keyLen: int): (int?, slice?, bool) + asm(-> 0 2 1 3) "DICTIREMMAX" "NULLSWAPIFNOT2"; + +@pure +fun uDictDeleteLastAndGet(mutate self: cell?, keyLen: int): (int?, slice?, bool) + asm(-> 0 2 1 3) "DICTUREMMAX" "NULLSWAPIFNOT2"; + +@pure +fun sDictDeleteLastAndGet(mutate self: cell?, keyLen: int): (slice?, slice?, bool) + asm(-> 0 2 1 3) "DICTREMMAX" "NULLSWAPIFNOT2"; + + +@pure +fun iDictGetFirst(self: cell?, keyLen: int): (int?, slice?, bool) + asm (-> 1 0 2) "DICTIMIN" "NULLSWAPIFNOT2"; + +@pure +fun uDictGetFirst(self: cell?, keyLen: int): (int?, slice?, bool) + asm (-> 1 0 2) "DICTUMIN" "NULLSWAPIFNOT2"; + +@pure +fun sDictGetFirst(self: cell?, keyLen: int): (slice?, slice?, bool) + asm (-> 1 0 2) "DICTMIN" "NULLSWAPIFNOT2"; + +@pure +fun iDictGetFirstAsRef(self: cell?, keyLen: int): (int?, cell?, bool) + asm (-> 1 0 2) "DICTIMINREF" "NULLSWAPIFNOT2"; + +@pure +fun uDictGetFirstAsRef(self: cell?, keyLen: int): (int?, cell?, bool) + asm (-> 1 0 2) "DICTUMINREF" "NULLSWAPIFNOT2"; + +@pure +fun sDictGetFirstAsRef(self: cell?, keyLen: int): (slice?, cell?, bool) + asm (-> 1 0 2) "DICTMINREF" "NULLSWAPIFNOT2"; + + +@pure +fun iDictGetLast(self: cell?, keyLen: int): (int?, slice?, bool) + asm (-> 1 0 2) "DICTIMAX" "NULLSWAPIFNOT2"; + +@pure +fun uDictGetLast(self: cell?, keyLen: int): (int?, slice?, bool) + asm (-> 1 0 2) "DICTUMAX" "NULLSWAPIFNOT2"; + +@pure +fun sDictGetLast(self: cell?, keyLen: int): (slice?, slice?, bool) + asm (-> 1 0 2) "DICTMAX" "NULLSWAPIFNOT2"; + +@pure +fun iDictGetLastAsRef(self: cell?, keyLen: int): (int?, cell?, bool) + asm (-> 1 0 2) "DICTIMAXREF" "NULLSWAPIFNOT2"; + +@pure +fun uDictGetLastAsRef(self: cell?, keyLen: int): (int?, cell?, bool) + asm (-> 1 0 2) "DICTUMAXREF" "NULLSWAPIFNOT2"; + +@pure +fun sDictGetLastAsRef(self: cell?, keyLen: int): (slice?, cell?, bool) + asm (-> 1 0 2) "DICTMAXREF" "NULLSWAPIFNOT2"; + + +@pure +fun iDictGetNext(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXT" "NULLSWAPIFNOT2"; + +@pure +fun uDictGetNext(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXT" "NULLSWAPIFNOT2"; + +@pure +fun iDictGetNextOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTIGETNEXTEQ" "NULLSWAPIFNOT2"; + +@pure +fun uDictGetNextOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTUGETNEXTEQ" "NULLSWAPIFNOT2"; + + +@pure +fun iDictGetPrev(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTIGETPREV" "NULLSWAPIFNOT2"; + +@pure +fun uDictGetPrev(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTUGETPREV" "NULLSWAPIFNOT2"; + +@pure +fun iDictGetPrevOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTIGETPREVEQ" "NULLSWAPIFNOT2"; + +@pure +fun uDictGetPrevOrEqual(self: cell?, keyLen: int, pivot: int): (int?, slice?, bool) + asm(pivot self keyLen -> 1 0 2) "DICTUGETPREVEQ" "NULLSWAPIFNOT2"; + + +/** + Prefix dictionary primitives. + */ + +@pure +fun prefixDictGet(self: cell?, keyLen: int, key: slice): (slice, slice?, slice?, bool) + asm(key self keyLen) "PFXDICTGETQ" "NULLSWAPIFNOT2"; + +@pure +fun prefixDictSet(mutate self: cell?, keyLen: int, key: slice, value: slice): bool + asm(value key self keyLen) "PFXDICTSET"; + +@pure +fun prefixDictDelete(mutate self: cell?, keyLen: int, key: slice): bool + asm(key self keyLen) "PFXDICTDEL"; diff --git a/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk new file mode 100644 index 00000000..136eaa4a --- /dev/null +++ b/crypto/smartcont/tolk-stdlib/tvm-lowlevel.tolk @@ -0,0 +1,25 @@ +// A part of standard library for Tolk +tolk 0.9 + +/// Usually `c3` has a continuation initialized by the whole code of the contract. It is used for function calls. +/// The primitive returns the current value of `c3`. +@pure +fun getTvmRegisterC3(): continuation + asm "c3 PUSH"; + +/// Updates the current value of `c3`. Usually, it is used for updating smart contract code in run-time. +/// Note that after execution of this primitive the current code +/// (and the stack of recursive function calls) won't change, +/// but any other function call will use a function from the new code. +fun setTvmRegisterC3(c: continuation): void + asm "c3 POP"; + +/// Transforms a `slice` [s] into a simple ordinary continuation `c`, with `c.code = s` and an empty stack and savelist. +@pure +fun transformSliceToContinuation(s: slice): continuation + asm "BLESS"; + +/// Moves a variable or a value [x] to the top of the stack. +@pure +fun stackMoveToTop(mutate self: X): void + asm "NOP"; diff --git a/crypto/smc-envelope/SmartContract.cpp b/crypto/smc-envelope/SmartContract.cpp index 2578a951..4d860fba 100644 --- a/crypto/smc-envelope/SmartContract.cpp +++ b/crypto/smc-envelope/SmartContract.cpp @@ -149,16 +149,17 @@ td::Ref prepare_vm_c7(SmartContract::Args args, td::Ref cod } std::vector tuple = { - td::make_refint(0x076ef1ea), // [ magic:0x076ef1ea - td::make_refint(0), // actions:Integer - td::make_refint(0), // msgs_sent:Integer - td::make_refint(now), // unixtime:Integer - td::make_refint(0), //TODO: // block_lt:Integer - td::make_refint(0), //TODO: // trans_lt:Integer - std::move(rand_seed_int), // rand_seed:Integer - block::CurrencyCollection(args.balance).as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)] - vm::load_cell_slice_ref(address), // myself:MsgAddressInt - vm::StackEntry::maybe(config) // vm::StackEntry::maybe(td::Ref()) + td::make_refint(0x076ef1ea), // [ magic:0x076ef1ea + td::make_refint(0), // actions:Integer + td::make_refint(0), // msgs_sent:Integer + td::make_refint(now), // unixtime:Integer + td::make_refint(0), // block_lt:Integer (TODO) + td::make_refint(0), // trans_lt:Integer (TODO) + std::move(rand_seed_int), // rand_seed:Integer + block::CurrencyCollection(args.balance, args.extra_currencies) + .as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)] + vm::load_cell_slice_ref(address), // myself:MsgAddressInt + vm::StackEntry::maybe(config) // vm::StackEntry::maybe(td::Ref()) }; if (args.config && args.config.value()->get_global_version() >= 4) { tuple.push_back(vm::StackEntry::maybe(code)); // code:Cell @@ -222,14 +223,14 @@ SmartContract::Answer run_smartcont(SmartContract::State state, td::Refdump(os, 2); LOG(DEBUG) << "VM stack:\n" << os.str(); } - vm::VmState vm{state.code, std::move(stack), gas, 1, state.data, log}; + int global_version = config ? config->get_global_version() : 0; + vm::VmState vm{state.code, global_version, std::move(stack), gas, 1, state.data, log}; vm.set_c7(std::move(c7)); vm.set_chksig_always_succeed(ignore_chksig); if (!libraries.is_null()) { vm.register_library_collection(libraries); } if (config) { - vm.set_global_version(config->get_global_version()); auto r_limits = config->get_size_limits_config(); if (r_limits.is_ok()) { vm.set_max_data_depth(r_limits.ok().max_vm_data_depth); diff --git a/crypto/smc-envelope/SmartContract.h b/crypto/smc-envelope/SmartContract.h index 7fc93579..49edb969 100644 --- a/crypto/smc-envelope/SmartContract.h +++ b/crypto/smc-envelope/SmartContract.h @@ -64,6 +64,7 @@ class SmartContract : public td::CntObject { bool ignore_chksig{false}; td::uint64 amount{0}; td::uint64 balance{0}; + td::Ref extra_currencies; int vm_log_verbosity_level{0}; bool debug_enabled{false}; @@ -121,6 +122,10 @@ class SmartContract : public td::CntObject { this->balance = balance; return std::move(*this); } + Args&& set_extra_currencies(td::Ref extra_currencies) { + this->extra_currencies = std::move(extra_currencies); + return std::move(*this); + } Args&& set_address(block::StdAddress address) { this->address = address; return std::move(*this); diff --git a/crypto/test/fift.cpp b/crypto/test/fift.cpp index 3f370ee5..9a92e0b1 100644 --- a/crypto/test/fift.cpp +++ b/crypto/test/fift.cpp @@ -167,3 +167,7 @@ TEST(Fift, test_bls_ops) { TEST(Fift, test_levels) { run_fift("levels.fif"); } + +TEST(Fift, test_secp256k1) { + run_fift("secp256k1.fif"); +} diff --git a/crypto/test/fift/secp256k1.fif b/crypto/test/fift/secp256k1.fif new file mode 100644 index 00000000..9a0e5178 --- /dev/null +++ b/crypto/test/fift/secp256k1.fif @@ -0,0 +1,82 @@ +"Asm.fif" include +"FiftExt.fif" include + +{ + =: expected_result + =: tweak + =: pubkey + B{e80fe1639c9ca050e3af1b39c143c63e429cbceb15d940fbb5c5a1f4af57c5e9e80fe1639c9ca050e3af1b39c143c63e429cbceb15d940fbb5c5a1f4af57c5e9} + @' pubkey 256 u>B @' tweak 256 u>B B+ B+ Bhashu =: tweak2 + @' pubkey @' tweak2 + [[ <{ SECP256K1_XONLY_PUBKEY_TWEAK_ADD }>s ]] 0 runvmx + abort"exitcode != 0" -1 <> abort"xonly_pubkey_tweak_add failed" + drop + =: result + ."Pubkey = " @' pubkey x. cr + ."Tweak = " @' tweak x. cr + ."Tweak2 = " @' tweak2 x. cr + @' expected_result @' result <> { ."Expected = " @' expected_result x. cr ."Found = " @' result x. cr -1 abort"result mismatch" } if + ."Result = " @' result x. cr + 4 <> abort"first byte is expected to be 4" + cr +} : run-test + +0x4be5f2020ebe2f37b65fb1bcd8aecf2ab0a333427208a5e6dd3dc691f1fb6ae2 +0x7eeda5cc8b219363ffc29a77d170f923dbb939ea5164b71db862d4a192f2680f +0x5a001d32f9fc59c2965ef36d9ff154469a3ba87f00b95879f3b5198bb557494d +run-test + +0x3ad84c3844e66aa255a121ed46d0e69e7b98f31233941c5197917c97c7aeaec5 +0x3f339347d8b1dde7edade3dc59fc63d18cb21fc2326ffbd30f0711a02d25075a +0xc621ca417b8915474020c0c9471a13918b6a02fd20d48a0e526c896923457fcd +run-test + +0x57bb69f0d446ee2cf9f77a2bdca7a3a462a61d85997a1154f2321a2717951b02 +0x3197d6b03d78ebbe694d2b89945a21a5a671ca78393481d44739b7351767adef +0xcbf0da3bbb498fd575506060d04db426164cb9d1477f07481fc6e3a2f84b01ea +run-test + +0x4af537be1a1ae11770eef23e6087f83ce1019fbee7a5876837107f84929a1d19 +0xcf9bfedc7251c2f8e233ae1e2c8b7a6cbe25d96a46a38b79e45d5684d026b64c +0x19b1aa4551bf08363b2533c146c02fa61e26941336aaa16cdd4393a5440392ea +run-test + +0x4be5f2020ebe2f37b65fb1bcd8aecf2ab0a333427208a5e6dd3dc691f1fb6ae2 +0x1762cbfa935318fb1395b50c64f961baab3ecaed4afad3068ba2f7f3a9d15cec +0x38be4b9791c0cb4952b9fb944eb0fe9256ce0d48be7b92129caafdf2f521248a +run-test + +0x3ad84c3844e66aa255a121ed46d0e69e7b98f31233941c5197917c97c7aeaec5 +0x099ec2e6ee1c40f533a61abca2860733727204c2f31d078297194d5e93e148f7 +0x8e9d15851e9aeb652216378f6bc0b88fb66b491697c9b9588f37c11d0b0fbced +run-test + +0x57bb69f0d446ee2cf9f77a2bdca7a3a462a61d85997a1154f2321a2717951b02 +0x63494cee8217ca2a10076e2031534fdda39f132dbdf91606aa65021709cb3116 +0x5dd1aa62a438469f4934e1ab9ada9ba3945651a2641316ec91d0780ca71891f8 +run-test + +0x4af537be1a1ae11770eef23e6087f83ce1019fbee7a5876837107f84929a1d19 +0x36e26133dc62474040eae511ce89610dae6bb0359aa80738baf812ecfa03d2a1 +0xf9ab8bbbee0ad1e90492c952b362d43d56412debdd1224e9729dc79aff931943 +run-test + +0x4be5f2020ebe2f37b65fb1bcd8aecf2ab0a333427208a5e6dd3dc691f1fb6ae2 +0x94f8fb43dd6001bb8acd2c53d094f9c919766dad596498d3d582cebdb39c63da +0xf7b4c76d79925f62e7969cb0e4af0673808f646add78563f711dc33a2687dc1b +run-test + +0x3ad84c3844e66aa255a121ed46d0e69e7b98f31233941c5197917c97c7aeaec5 +0xa2f55c34c4eee5881584714bd5c7a63c397ef38ff1afedd04ada10fdeb541cef +0x73f9a15b76612ca4254e6c2758508589c0112eb724f42dbb4c65ff8b025d2103 +run-test + +0x57bb69f0d446ee2cf9f77a2bdca7a3a462a61d85997a1154f2321a2717951b02 +0x05130a68853e5e4aff579fa21ff10010410a3be47b94d908e203f69ec9dc7d00 +0x4ce6b5b1bd77b1666d33a0c9fd37b98952078bcc451d6de2d0bff65e8f2b46ed +run-test + +0x4af537be1a1ae11770eef23e6087f83ce1019fbee7a5876837107f84929a1d19 +0x9765369cb4467bebc8a468d44aa60f0154f04ee32fbcf1c8bdf646d4840163d1 +0x118953c642b8f25fea3519bcaab3ae6cea25402088e11a8efdc2e0bd222958ad +run-test \ No newline at end of file diff --git a/crypto/tl/tlbc-gen-cpp.cpp b/crypto/tl/tlbc-gen-cpp.cpp index 55b4a1c0..5730f169 100644 --- a/crypto/tl/tlbc-gen-cpp.cpp +++ b/crypto/tl/tlbc-gen-cpp.cpp @@ -2074,7 +2074,7 @@ void CppTypeCode::generate_skip_field(const Constructor& constr, const Field& fi output_cpp_expr(ss, expr, 100); ss << '.'; } - ss << "validate_skip_ref(ops, cs, weak)" << tail; + ss << "validate_skip_ref(ops, cs, " << (constr.is_special ? "true" : "weak") << ")" << tail; actions += Action{ss.str()}; } diff --git a/crypto/tl/tlbc.cpp b/crypto/tl/tlbc.cpp index 0050e161..d3a6edb5 100644 --- a/crypto/tl/tlbc.cpp +++ b/crypto/tl/tlbc.cpp @@ -1800,9 +1800,6 @@ void Constructor::show(std::ostream& os, int mode) const { } for (int i = 0; i < type_arity; i++) { os << ' '; - if (param_negated.at(i)) { - os << '~'; - } params.at(i)->show(os, this, 100, mode | 1); } if (!(mode & 2)) { diff --git a/crypto/tl/tlblib.cpp b/crypto/tl/tlblib.cpp index 0e0e5626..de5a483c 100644 --- a/crypto/tl/tlblib.cpp +++ b/crypto/tl/tlblib.cpp @@ -45,7 +45,7 @@ bool Bool::print_skip(PrettyPrinter& pp, vm::CellSlice& cs) const { } bool NatWidth::print_skip(PrettyPrinter& pp, vm::CellSlice& cs) const { - long long value = (long long)cs.fetch_ulong(32); + long long value = (long long)cs.fetch_ulong(n); return value >= 0 && pp.out_int(value); } @@ -133,7 +133,13 @@ bool TLB::validate_ref_internal(int* ops, Ref cell_ref, bool weak) con } bool is_special; auto cs = load_cell_slice_special(std::move(cell_ref), is_special); - return always_special() ? is_special : (is_special ? weak : (validate_skip(ops, cs) && cs.empty_ext())); + if (cs.special_type() == vm::Cell::SpecialType::PrunnedBranch && weak) { + return true; + } + if (always_special() != is_special) { + return false; + } + return validate_skip(ops, cs, weak) && cs.empty_ext(); } bool TLB::print_skip(PrettyPrinter& pp, vm::CellSlice& cs) const { @@ -190,6 +196,13 @@ bool TLB::print_ref(std::ostream& os, Ref cell_ref, int indent, int re return pp.fail_unless(print_ref(pp, std::move(cell_ref))); } +bool TLB::print_ref(td::StringBuilder& sb, Ref cell_ref, int indent, int rec_limit) const { + std::ostringstream ss; + auto result = print_ref(ss, std::move(cell_ref), indent, rec_limit); + sb << ss.str(); + return result; +} + std::string TLB::as_string_skip(vm::CellSlice& cs, int indent) const { std::ostringstream os; print_skip(os, cs, indent); diff --git a/crypto/tl/tlblib.hpp b/crypto/tl/tlblib.hpp index a6350ece..c10049a9 100644 --- a/crypto/tl/tlblib.hpp +++ b/crypto/tl/tlblib.hpp @@ -246,7 +246,14 @@ class TLB { bool print(std::ostream& os, Ref cs_ref, int indent = 0, int rec_limit = 0) const { return print(os, *cs_ref, indent, rec_limit); } + bool print(td::StringBuilder& sb, Ref cs_ref, int indent = 0, int rec_limit = 0) const { + std::ostringstream ss; + auto result = print(ss, *cs_ref, indent, rec_limit); + sb << ss.str(); + return result; + } bool print_ref(std::ostream& os, Ref cell_ref, int indent = 0, int rec_limit = 0) const; + bool print_ref(td::StringBuilder& sb, Ref cell_ref, int indent = 0, int rec_limit = 0) const; bool print_ref(int rec_limit, std::ostream& os, Ref cell_ref, int indent = 0) const { return print_ref(os, std::move(cell_ref), indent, rec_limit); } diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index 7ec8bdd1..72afb998 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -1153,8 +1153,12 @@ td::Result CellStorageStat::add_used_storage(Refsecond; } } - vm::CellSlice cs{vm::NoVm{}, std::move(cell)}; - return add_used_storage(std::move(cs), kill_dup, skip_count_root); + vm::CellSlice cs{vm::NoVm{}, cell}; + TRY_RESULT(res, add_used_storage(std::move(cs), kill_dup, skip_count_root)); + if (kill_dup) { + seen[cell->get_hash()] = res; + } + return res; } void NewCellStorageStat::add_cell(Ref cell) { diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index 8adf240f..17e7eb69 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -101,9 +101,9 @@ class NewCellStorageStat { private: const CellUsageTree* usage_tree_; - std::set seen_; + td::HashSet seen_; Stat stat_; - std::set proof_seen_; + td::HashSet proof_seen_; Stat proof_stat_; const NewCellStorageStat* parent_{nullptr}; @@ -117,7 +117,7 @@ struct CellStorageStat { struct CellInfo { td::uint32 max_merkle_depth = 0; }; - std::map seen; + td::HashMap seen; CellStorageStat() : cells(0), bits(0), public_cells(0) { } explicit CellStorageStat(unsigned long long limit_cells) @@ -173,7 +173,7 @@ class ProofStorageStat { enum CellStatus { c_none = 0, c_prunned = 1, c_loaded = 2 }; - std::map cells_; + td::HashMap cells_; td::uint64 proof_size_ = 0; }; diff --git a/crypto/vm/cells/CellSlice.cpp b/crypto/vm/cells/CellSlice.cpp index 466bcd8d..9cd3e931 100644 --- a/crypto/vm/cells/CellSlice.cpp +++ b/crypto/vm/cells/CellSlice.cpp @@ -264,7 +264,7 @@ bool CellSlice::advance_ext(unsigned bits, unsigned refs) { } bool CellSlice::advance_ext(unsigned bits_refs) { - return advance_ext(bits_refs >> 16, bits_refs & 0xffff); + return advance_ext(bits_refs & 0xffff, bits_refs >> 16); } // (PRIVATE) @@ -1026,6 +1026,13 @@ bool CellSlice::print_rec(std::ostream& os, int indent) const { return print_rec(os, &limit, indent); } +bool CellSlice::print_rec(td::StringBuilder& sb, int indent) const { + std::ostringstream ss; + auto result = print_rec(ss, indent); + sb << ss.str(); + return result; +} + bool CellSlice::print_rec(int limit, std::ostream& os, int indent) const { return print_rec(os, &limit, indent); } diff --git a/crypto/vm/cells/CellSlice.h b/crypto/vm/cells/CellSlice.h index 33fad741..ecce30f5 100644 --- a/crypto/vm/cells/CellSlice.h +++ b/crypto/vm/cells/CellSlice.h @@ -257,6 +257,7 @@ class CellSlice : public td::CntObject { void dump(std::ostream& os, int level = 0, bool endl = true) const; void dump_hex(std::ostream& os, int mode = 0, bool endl = false) const; bool print_rec(std::ostream& os, int indent = 0) const; + bool print_rec(td::StringBuilder& sb, int indent = 0) const; bool print_rec(std::ostream& os, int* limit, int indent = 0) const; bool print_rec(int limit, std::ostream& os, int indent = 0) const; void error() const { diff --git a/crypto/vm/contops.cpp b/crypto/vm/contops.cpp index 9610e4aa..1ccf53da 100644 --- a/crypto/vm/contops.cpp +++ b/crypto/vm/contops.cpp @@ -261,10 +261,10 @@ int exec_runvm_common(VmState* st, unsigned mode) { vm::GasLimits gas{gas_limit, gas_max}; VmStateInterface::Guard guard{nullptr}; // Don't consume gas for creating/loading cells during VM init - VmState new_state{std::move(code), std::move(new_stack), gas, (int)mode & 3, std::move(data), - VmLog{}, std::vector>{}, std::move(c7)}; + VmState new_state{ + std::move(code), st->get_global_version(), std::move(new_stack), gas, (int)mode & 3, std::move(data), + VmLog{}, std::vector>{}, std::move(c7)}; new_state.set_chksig_always_succeed(st->get_chksig_always_succeed()); - new_state.set_global_version(st->get_global_version()); st->run_child_vm(std::move(new_state), with_data, mode & 32, mode & 8, mode & 128, ret_vals); return 0; } @@ -923,6 +923,41 @@ int exec_setcont_ctr_var(VmState* st) { return 0; } +int exec_setcont_ctr_many(VmState* st, unsigned args) { + unsigned mask = args & 255; + VM_LOG(st) << "execute SETCONTCTRMANY " << mask; + if (mask & (1 << 6)) { + throw VmError{Excno::range_chk, "no control register c6"}; + } + Stack& stack = st->get_stack(); + auto cont = stack.pop_cont(); + for (int i = 0; i < 8; ++i) { + if (mask & (1 << i)) { + throw_typechk(force_cregs(cont)->define(i, st->get(i))); + } + } + st->get_stack().push_cont(std::move(cont)); + return 0; +} + +int exec_setcont_ctr_many_var(VmState* st) { + VM_LOG(st) << "execute SETCONTCTRMANYX"; + Stack& stack = st->get_stack(); + stack.check_underflow(2); + int mask = stack.pop_smallint_range(255); + if (mask & (1 << 6)) { + throw VmError{Excno::range_chk, "no control register c6"}; + } + auto cont = stack.pop_cont(); + for (int i = 0; i < 8; ++i) { + if (mask & (1 << i)) { + throw_typechk(force_cregs(cont)->define(i, st->get(i))); + } + } + st->get_stack().push_cont(std::move(cont)); + return 0; +} + int exec_compos(VmState* st, unsigned mask, const char* name) { Stack& stack = st->get_stack(); VM_LOG(st) << "execute " << name; @@ -1037,6 +1072,8 @@ void register_continuation_change_ops(OpcodeTable& cp0) { cp0.insert(OpcodeInstr::mksimple(0xede0, 16, "PUSHCTRX", exec_push_ctr_var)) .insert(OpcodeInstr::mksimple(0xede1, 16, "POPCTRX", exec_pop_ctr_var)) .insert(OpcodeInstr::mksimple(0xede2, 16, "SETCONTCTRX", exec_setcont_ctr_var)) + .insert(OpcodeInstr::mkfixed(0xede3, 16, 8, instr::dump_1c_l_add(1, "SETCONTCTRMANY "), exec_setcont_ctr_many)->require_version(9)) + .insert(OpcodeInstr::mksimple(0xede4, 16, "SETCONTCTRMANYX", exec_setcont_ctr_many_var)->require_version(9)) .insert(OpcodeInstr::mksimple(0xedf0, 16, "BOOLAND", std::bind(exec_compos, _1, 1, "BOOLAND"))) .insert(OpcodeInstr::mksimple(0xedf1, 16, "BOOLOR", std::bind(exec_compos, _1, 2, "BOOLOR"))) .insert(OpcodeInstr::mksimple(0xedf2, 16, "COMPOSBOTH", std::bind(exec_compos, _1, 3, "COMPOSBOTH"))) diff --git a/crypto/vm/db/DynamicBagOfCellsDb.cpp b/crypto/vm/db/DynamicBagOfCellsDb.cpp index b69cd8c0..09303758 100644 --- a/crypto/vm/db/DynamicBagOfCellsDb.cpp +++ b/crypto/vm/db/DynamicBagOfCellsDb.cpp @@ -100,8 +100,18 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat return get_cell_info_lazy(level_mask, hash, depth).cell; } td::Result> load_cell(td::Slice hash) override { - TRY_RESULT(loaded_cell, get_cell_info_force(hash).cell->load_cell()); - return std::move(loaded_cell.data_cell); + auto info = hash_table_.get_if_exists(hash); + if (info && info->sync_with_db) { + TRY_RESULT(loaded_cell, info->cell->load_cell()); + return std::move(loaded_cell.data_cell); + } + TRY_RESULT(res, loader_->load(hash, true, *this)); + if (res.status != CellLoader::LoadResult::Ok) { + return td::Status::Error("cell not found"); + } + Ref cell = res.cell(); + hash_table_.apply(hash, [&](CellInfo &info) { update_cell_info_loaded(info, hash, std::move(res)); }); + return cell; } td::Result> load_root(td::Slice hash) override { return load_cell(hash); @@ -111,14 +121,16 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat } void load_cell_async(td::Slice hash, std::shared_ptr executor, td::Promise> promise) override { + auto promise_ptr = std::make_shared>>(std::move(promise)); auto info = hash_table_.get_if_exists(hash); if (info && info->sync_with_db) { - TRY_RESULT_PROMISE(promise, loaded_cell, info->cell->load_cell()); - promise.set_result(loaded_cell.data_cell); + executor->execute_async([promise = std::move(promise_ptr), cell = info->cell]() mutable { + TRY_RESULT_PROMISE((*promise), loaded_cell, cell->load_cell()); + promise->set_result(loaded_cell.data_cell); + }); return; } SimpleExtCellCreator ext_cell_creator(cell_db_reader_); - auto promise_ptr = std::make_shared>>(std::move(promise)); executor->execute_async( [executor, loader = *loader_, hash = CellHash::from_slice(hash), db = this, ext_cell_creator = std::move(ext_cell_creator), promise = std::move(promise_ptr)]() mutable { @@ -143,9 +155,6 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat promise->set_result(std::move(cell)); }); } - CellInfo &get_cell_info_force(td::Slice hash) { - return hash_table_.apply(hash, [&](CellInfo &info) { update_cell_info_force(info, hash); }); - } CellInfo &get_cell_info_lazy(Cell::LevelMask level_mask, td::Slice hash, td::Slice depth) { return hash_table_.apply(hash.substr(hash.size() - Cell::hash_bytes), [&](CellInfo &info) { update_cell_info_lazy(info, level_mask, hash, depth); }); diff --git a/crypto/vm/db/StaticBagOfCellsDb.cpp b/crypto/vm/db/StaticBagOfCellsDb.cpp index c667f334..80dbfbf0 100644 --- a/crypto/vm/db/StaticBagOfCellsDb.cpp +++ b/crypto/vm/db/StaticBagOfCellsDb.cpp @@ -309,7 +309,9 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { return 0; } td::Slice offset_view; - CHECK(info_.offset_byte_size <= 8); + if (info_.offset_byte_size > 8) { + return td::Status::Error(PSTRING() << "bag-of-cell error: invalid offset_byte_size " << info_.offset_byte_size); + } char arr[8]; td::RwMutex::ReadLock guard; if (info_.has_index) { @@ -321,19 +323,25 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { offset_view = td::Slice(index_data_).substr((td::int64)idx * info_.offset_byte_size, info_.offset_byte_size); } - CHECK(offset_view.size() == (size_t)info_.offset_byte_size); + if (offset_view.size() != (size_t)info_.offset_byte_size) { + return td::Status::Error(PSTRING() << "bag-of-cell error: invalid offset view size" << offset_view.size()); + } return td::narrow_cast(info_.read_offset(offset_view.ubegin())); } td::Result load_root_idx(int root_i) { - CHECK(root_i >= 0 && root_i < info_.root_count); + if (root_i < 0 || root_i >= info_.root_count) { + return td::Status::Error(PSTRING() << "bag-of-cell error: invalid root index " << root_i); + } if (!info_.has_roots) { return 0; } char arr[8]; TRY_RESULT(idx_view, data_.view(td::MutableSlice(arr, info_.ref_byte_size), info_.roots_offset + (td::int64)root_i * info_.ref_byte_size)); - CHECK(idx_view.size() == (size_t)info_.ref_byte_size); + if (idx_view.size() != (size_t)info_.ref_byte_size) { + return td::Status::Error(PSTRING() << "bag-of-cell error: invalid idx_view size" << idx_view.size()); + } return info_.read_ref(idx_view.ubegin()); } @@ -343,8 +351,9 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { bool should_cache; }; td::Result get_cell_location(int idx) { - CHECK(idx >= 0); - CHECK(idx < info_.cell_count); + if (idx < 0 || idx >= info_.cell_count) { + return td::Status::Error(PSTRING() << "bag-of-cell error: invalid cell index " << idx); + } TRY_STATUS(preload_index(idx)); TRY_RESULT(from, load_idx_offset(idx - 1)); TRY_RESULT(till, load_idx_offset(idx)); @@ -357,10 +366,15 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { res.should_cache = res.end % 2 == 1; res.end /= 2; } - CHECK(std::numeric_limits::max() - res.begin >= info_.data_offset); - CHECK(std::numeric_limits::max() - res.end >= info_.data_offset); + if (std::numeric_limits::max() - res.begin < info_.data_offset || + std::numeric_limits::max() - res.end < info_.data_offset) { + return td::Status::Error(PSTRING() << "bag-of-cell error: invalid cell location (1) " << res.begin << ":" << res.end); + } res.begin += static_cast(info_.data_offset); res.end += static_cast(info_.data_offset); + if (res.begin > res.end) { + return td::Status::Error(PSTRING() << "bag-of-cell error: invalid cell location (2) " << res.begin << ":" << res.end); + } return res; } @@ -396,8 +410,6 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { if (info_.has_index) { return td::Status::OK(); } - - CHECK(idx < info_.cell_count); if (index_i_.load(std::memory_order_relaxed) > idx) { return td::Status::OK(); } @@ -407,12 +419,17 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { auto buf_slice = td::MutableSlice(buf.data(), buf.size()); for (; index_i_ <= idx; index_i_++) { auto offset = td::narrow_cast(info_.data_offset + index_offset_); - CHECK(data_.size() >= offset); + if (data_.size() < offset) { + return td::Status::Error(PSLICE() << "bag-of-cells error: invalid offset " << offset + << " (size=" << data_.size() << ")"); + } TRY_RESULT(cell, data_.view(buf_slice.copy().truncate(data_.size() - offset), offset)); CellSerializationInfo cell_info; TRY_STATUS(cell_info.init(cell, info_.ref_byte_size)); index_offset_ += cell_info.end_offset; - LOG_CHECK((unsigned)info_.offset_byte_size <= 8) << info_.offset_byte_size; + if ((unsigned)info_.offset_byte_size > 8) { + return td::Status::Error(PSTRING() << "bag-of-cell error: invalid offset_byte_size " << info_.offset_byte_size); + } td::uint8 tmp[8]; info_.write_offset(tmp, index_offset_); auto guard = index_data_rw_mutex_.lock_write(); @@ -488,7 +505,10 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { bool should_cache) { deserialize_cell_cnt_.add(1); Ref refs[4]; - CHECK(cell_info.refs_cnt <= 4); + if (cell_info.refs_cnt > 4) { + return td::Status::Error(PSLICE() << "invalid bag-of-cells cell #" << idx << " has " << cell_info.refs_cnt + << " refs"); + } auto* ref_ptr = cell_slice.ubegin() + cell_info.refs_offset; for (int k = 0; k < cell_info.refs_cnt; k++, ref_ptr += info_.ref_byte_size) { int ref_idx = td::narrow_cast(info_.read_ref(ref_ptr)); diff --git a/crypto/vm/dictops.cpp b/crypto/vm/dictops.cpp index d0ea8daa..02f26fdd 100644 --- a/crypto/vm/dictops.cpp +++ b/crypto/vm/dictops.cpp @@ -566,7 +566,7 @@ int exec_dict_getnear(VmState* st, unsigned args) { int exec_pfx_dict_set(VmState* st, Dictionary::SetMode mode, const char* name) { Stack& stack = st->get_stack(); VM_LOG(st) << "execute PFXDICT" << name; - stack.check_underflow(3); + stack.check_underflow(st->get_global_version() >= 9 ? 4 : 3); int n = stack.pop_smallint_range(PrefixDictionary::max_key_bits); PrefixDictionary dict{stack.pop_maybe_cell(), n}; auto key_slice = stack.pop_cellslice(); @@ -580,7 +580,7 @@ int exec_pfx_dict_set(VmState* st, Dictionary::SetMode mode, const char* name) { int exec_pfx_dict_delete(VmState* st) { Stack& stack = st->get_stack(); VM_LOG(st) << "execute PFXDICTDEL\n"; - stack.check_underflow(2); + stack.check_underflow(st->get_global_version() >= 9 ? 3 : 2); int n = stack.pop_smallint_range(PrefixDictionary::max_key_bits); PrefixDictionary dict{stack.pop_maybe_cell(), n}; auto key_slice = stack.pop_cellslice(); diff --git a/crypto/vm/tonops.cpp b/crypto/vm/tonops.cpp index 4b2d1734..aab1711f 100644 --- a/crypto/vm/tonops.cpp +++ b/crypto/vm/tonops.cpp @@ -279,6 +279,7 @@ int exec_get_global_id(VmState* st) { int exec_get_gas_fee(VmState* st) { VM_LOG(st) << "execute GETGASFEE"; Stack& stack = st->get_stack(); + stack.check_underflow(st->get_global_version() >= 9 ? 2 : 0); bool is_masterchain = stack.pop_bool(); td::uint64 gas = stack.pop_long_range(std::numeric_limits::max(), 0); block::GasLimitsPrices prices = util::get_gas_prices(get_unpacked_config_tuple(st), is_masterchain); @@ -289,6 +290,7 @@ int exec_get_gas_fee(VmState* st) { int exec_get_storage_fee(VmState* st) { VM_LOG(st) << "execute GETSTORAGEFEE"; Stack& stack = st->get_stack(); + stack.check_underflow(st->get_global_version() >= 9 ? 4 : 0); bool is_masterchain = stack.pop_bool(); td::int64 delta = stack.pop_long_range(std::numeric_limits::max(), 0); td::uint64 bits = stack.pop_long_range(std::numeric_limits::max(), 0); @@ -302,6 +304,7 @@ int exec_get_storage_fee(VmState* st) { int exec_get_forward_fee(VmState* st) { VM_LOG(st) << "execute GETFORWARDFEE"; Stack& stack = st->get_stack(); + stack.check_underflow(st->get_global_version() >= 9 ? 3 : 0); bool is_masterchain = stack.pop_bool(); td::uint64 bits = stack.pop_long_range(std::numeric_limits::max(), 0); td::uint64 cells = stack.pop_long_range(std::numeric_limits::max(), 0); @@ -320,6 +323,7 @@ int exec_get_precompiled_gas(VmState* st) { int exec_get_original_fwd_fee(VmState* st) { VM_LOG(st) << "execute GETORIGINALFWDFEE"; Stack& stack = st->get_stack(); + stack.check_underflow(st->get_global_version() >= 9 ? 2 : 0); bool is_masterchain = stack.pop_bool(); td::RefInt256 fwd_fee = stack.pop_int_finite(); if (fwd_fee->sgn() < 0) { @@ -333,6 +337,7 @@ int exec_get_original_fwd_fee(VmState* st) { int exec_get_gas_fee_simple(VmState* st) { VM_LOG(st) << "execute GETGASFEESIMPLE"; Stack& stack = st->get_stack(); + stack.check_underflow(st->get_global_version() >= 9 ? 2 : 0); bool is_masterchain = stack.pop_bool(); td::uint64 gas = stack.pop_long_range(std::numeric_limits::max(), 0); block::GasLimitsPrices prices = util::get_gas_prices(get_unpacked_config_tuple(st), is_masterchain); @@ -343,6 +348,7 @@ int exec_get_gas_fee_simple(VmState* st) { int exec_get_forward_fee_simple(VmState* st) { VM_LOG(st) << "execute GETFORWARDFEESIMPLE"; Stack& stack = st->get_stack(); + stack.check_underflow(st->get_global_version() >= 9 ? 3 : 0); bool is_masterchain = stack.pop_bool(); td::uint64 bits = stack.pop_long_range(std::numeric_limits::max(), 0); td::uint64 cells = stack.pop_long_range(std::numeric_limits::max(), 0); @@ -373,6 +379,7 @@ void register_ton_config_ops(OpcodeTable& cp0) { .insert(OpcodeInstr::mksimple(0xf833, 16, "CONFIGOPTPARAM", std::bind(exec_get_config_param, _1, true))) .insert(OpcodeInstr::mksimple(0xf83400, 24, "PREVMCBLOCKS", std::bind(exec_get_prev_blocks_info, _1, 0, "PREVMCBLOCKS"))->require_version(4)) .insert(OpcodeInstr::mksimple(0xf83401, 24, "PREVKEYBLOCK", std::bind(exec_get_prev_blocks_info, _1, 1, "PREVKEYBLOCK"))->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf83402, 24, "PREVMCBLOCKS_100", std::bind(exec_get_prev_blocks_info, _1, 2, "PREVMCBLOCKS_100"))->require_version(9)) .insert(OpcodeInstr::mksimple(0xf835, 16, "GLOBALID", exec_get_global_id)->require_version(4)) .insert(OpcodeInstr::mksimple(0xf836, 16, "GETGASFEE", exec_get_gas_fee)->require_version(6)) .insert(OpcodeInstr::mksimple(0xf837, 16, "GETSTORAGEFEE", exec_get_storage_fee)->require_version(6)) @@ -538,9 +545,10 @@ int exec_hash_ext(VmState* st, unsigned args) { VM_LOG(st) << "execute HASHEXT" << (append ? "A" : "") << (rev ? "R" : "") << " " << (hash_id == 255 ? -1 : hash_id); Stack& stack = st->get_stack(); if (hash_id == 255) { + stack.check_underflow(st->get_global_version() >= 9 ? 2 : 0); hash_id = stack.pop_smallint_range(254); } - int cnt = stack.pop_smallint_range(stack.depth() - 1); + int cnt = stack.pop_smallint_range(stack.depth() - 1 - (st->get_global_version() >= 9 ? (int)append : 0)); Hasher hasher{hash_id}; size_t total_bits = 0; long long gas_consumed = 0; @@ -661,7 +669,38 @@ int exec_ecrecover(VmState* st) { } st->consume_gas(VmState::ecrecover_gas_price); unsigned char public_key[65]; - if (td::ecrecover(hash_bytes, signature, public_key)) { + if (td::secp256k1::ecrecover(hash_bytes, signature, public_key)) { + td::uint8 h = public_key[0]; + td::RefInt256 x1{true}, x2{true}; + CHECK(x1.write().import_bytes(public_key + 1, 32, false)); + CHECK(x2.write().import_bytes(public_key + 33, 32, false)); + stack.push_smallint(h); + stack.push_int(std::move(x1)); + stack.push_int(std::move(x2)); + stack.push_bool(true); + } else { + stack.push_bool(false); + } + return 0; +} + +int exec_secp256k1_xonly_pubkey_tweak_add(VmState* st) { + VM_LOG(st) << "execute SECP256K1_XONLY_PUBKEY_TWEAK_ADD"; + Stack& stack = st->get_stack(); + stack.check_underflow(2); + auto tweak_int = stack.pop_int(); + auto key_int = stack.pop_int(); + + unsigned char key[32], tweak[32]; + if (!key_int->export_bytes(key, 32, false)) { + throw VmError{Excno::range_chk, "key must fit in an unsigned 256-bit integer"}; + } + if (!tweak_int->export_bytes(tweak, 32, false)) { + throw VmError{Excno::range_chk, "tweak must fit in an unsigned 256-bit integer"}; + } + st->consume_gas(VmState::secp256k1_xonly_pubkey_tweak_add_gas_price); + unsigned char public_key[65]; + if (td::secp256k1::xonly_pubkey_tweak_add(key, tweak, public_key)) { td::uint8 h = public_key[0]; td::RefInt256 x1{true}, x2{true}; CHECK(x1.write().import_bytes(public_key + 1, 32, false)); @@ -1214,6 +1253,7 @@ void register_ton_crypto_ops(OpcodeTable& cp0) { .insert(OpcodeInstr::mksimple(0xf910, 16, "CHKSIGNU", std::bind(exec_ed25519_check_signature, _1, false))) .insert(OpcodeInstr::mksimple(0xf911, 16, "CHKSIGNS", std::bind(exec_ed25519_check_signature, _1, true))) .insert(OpcodeInstr::mksimple(0xf912, 16, "ECRECOVER", exec_ecrecover)->require_version(4)) + .insert(OpcodeInstr::mksimple(0xf913, 16, "SECP256K1_XONLY_PUBKEY_TWEAK_ADD", exec_secp256k1_xonly_pubkey_tweak_add)->require_version(9)) .insert(OpcodeInstr::mksimple(0xf914, 16, "P256_CHKSIGNU", std::bind(exec_p256_chksign, _1, false))->require_version(4)) .insert(OpcodeInstr::mksimple(0xf915, 16, "P256_CHKSIGNS", std::bind(exec_p256_chksign, _1, true))->require_version(4)) @@ -1261,7 +1301,7 @@ void register_ton_crypto_ops(OpcodeTable& cp0) { } int exec_compute_data_size(VmState* st, int mode) { - VM_LOG(st) << (mode & 2 ? 'S' : 'C') << "DATASIZE" << (mode & 1 ? "Q" : ""); + VM_LOG(st) << "execute " << (mode & 2 ? 'S' : 'C') << "DATASIZE" << (mode & 1 ? "Q" : ""); Stack& stack = st->get_stack(); stack.check_underflow(2); auto bound = stack.pop_int(); @@ -1721,6 +1761,10 @@ int exec_send_message(VmState* st) { vm::VmStorageStat stat(max_cells); CellSlice cs = load_cell_slice(msg_cell); cs.skip_first(cs.size()); + if (st->get_global_version() >= 10 && have_extra_currencies) { + // Skip extra currency dict + cs.advance_refs(1); + } stat.add_storage(cs); if (!ext_msg) { @@ -1733,7 +1777,9 @@ int exec_send_message(VmState* st) { if (value.is_null()) { throw VmError{Excno::type_chk, "invalid param BALANCE"}; } - have_extra_currencies |= !tuple_index(balance, 1).as_cell().is_null(); + if (st->get_global_version() < 10) { + have_extra_currencies |= !tuple_index(balance, 1).as_cell().is_null(); + } } else if (mode & 64) { // value += value of incoming message Ref balance = get_param(st, 11).as_tuple(); if (balance.is_null()) { @@ -1744,7 +1790,9 @@ int exec_send_message(VmState* st) { throw VmError{Excno::type_chk, "invalid param INCOMINGVALUE"}; } value += balance_grams; - have_extra_currencies |= !tuple_index(balance, 1).as_cell().is_null(); + if (st->get_global_version() < 10) { + have_extra_currencies |= !tuple_index(balance, 1).as_cell().is_null(); + } } } diff --git a/crypto/vm/vm.cpp b/crypto/vm/vm.cpp index fb774f80..3c1118c6 100644 --- a/crypto/vm/vm.cpp +++ b/crypto/vm/vm.cpp @@ -22,6 +22,8 @@ #include "vm/log.h" #include "vm/vm.h" #include "cp0.h" +#include "memo.h" + #include namespace vm { @@ -31,33 +33,8 @@ VmState::VmState() : cp(-1), dispatch(&dummy_dispatch_table), quit0(true, 0), qu init_cregs(); } -VmState::VmState(Ref _code) - : code(std::move(_code)), cp(-1), dispatch(&dummy_dispatch_table), quit0(true, 0), quit1(true, 1) { - ensure_throw(init_cp(0)); - init_cregs(); -} - -VmState::VmState(Ref _code, Ref _stack, int flags, Ref _data, VmLog log, - std::vector> _libraries, Ref init_c7) - : code(std::move(_code)) - , stack(std::move(_stack)) - , cp(-1) - , dispatch(&dummy_dispatch_table) - , quit0(true, 0) - , quit1(true, 1) - , log(log) - , libraries(std::move(_libraries)) - , stack_trace((flags >> 2) & 1) { - ensure_throw(init_cp(0)); - set_c4(std::move(_data)); - if (init_c7.not_null()) { - set_c7(std::move(init_c7)); - } - init_cregs(flags & 1, flags & 2); -} - -VmState::VmState(Ref _code, Ref _stack, const GasLimits& gas, int flags, Ref _data, VmLog log, - std::vector> _libraries, Ref init_c7) +VmState::VmState(Ref _code, int global_version, Ref _stack, const GasLimits& gas, int flags, + Ref _data, VmLog log, std::vector> _libraries, Ref init_c7) : code(std::move(_code)) , stack(std::move(_stack)) , cp(-1) @@ -67,7 +44,8 @@ VmState::VmState(Ref _code, Ref _stack, const GasLimits& gas, , log(log) , gas(gas) , libraries(std::move(_libraries)) - , stack_trace((flags >> 2) & 1) { + , stack_trace((flags >> 2) & 1) + , global_version(global_version) { ensure_throw(init_cp(0)); set_c4(std::move(_data)); if (init_c7.not_null()) { @@ -102,12 +80,24 @@ void VmState::init_cregs(bool same_c3, bool push_0) { } } -Ref VmState::convert_code_cell(Ref code_cell) { +Ref VmState::convert_code_cell(Ref code_cell, int global_version, + const std::vector>& libraries) { if (code_cell.is_null()) { return {}; } - Ref csr{true, NoVmOrd(), code_cell}; - if (csr->is_valid()) { + Ref csr; + if (global_version >= 9) { + // Use DummyVmState instead of this to avoid consuming gas for cell loading + DummyVmState dummy{libraries, global_version}; + Guard guard(&dummy); + try { + csr = load_cell_slice_ref(code_cell); + } catch (VmError&) { // NOLINT(*-empty-catch) + } + } else { + csr = td::Ref{true, NoVmOrd(), code_cell}; + } + if (csr.not_null() && csr->is_valid()) { return csr; } return load_cell_slice_ref(CellBuilder{}.store_ref(std::move(code_cell)).finalize()); @@ -257,6 +247,11 @@ int VmState::jump(Ref cont) { // general jump to continuation cont int VmState::jump(Ref cont, int pass_args) { + cont = adjust_jump_cont(std::move(cont), pass_args); + return jump_to(std::move(cont)); +} + +Ref VmState::adjust_jump_cont(Ref cont, int pass_args) { const ControlData* cont_data = cont->get_cdata(); if (cont_data) { // first do the checks @@ -297,7 +292,7 @@ int VmState::jump(Ref cont, int pass_args) { consume_stack_gas(copy); } } - return jump_to(std::move(cont)); + return cont; } else { // have no continuation data, situation is somewhat simpler if (pass_args >= 0) { @@ -309,7 +304,7 @@ int VmState::jump(Ref cont, int pass_args) { consume_stack_gas(pass_args); } } - return jump_to(std::move(cont)); + return cont; } } @@ -577,6 +572,7 @@ int run_vm_code(Ref code, Ref& stack, int flags, Ref* da GasLimits* gas_limits, std::vector> libraries, Ref init_c7, Ref* actions_ptr, int global_version) { VmState vm{code, + global_version, std::move(stack), gas_limits ? *gas_limits : GasLimits{}, flags, @@ -584,7 +580,6 @@ int run_vm_code(Ref code, Ref& stack, int flags, Ref* da log, std::move(libraries), std::move(init_c7)}; - vm.set_global_version(global_version); int res = vm.run(); stack = vm.get_stack_ref(); if (vm.committed() && data_ptr) { diff --git a/crypto/vm/vm.h b/crypto/vm/vm.h index 7afa355c..a171ef27 100644 --- a/crypto/vm/vm.h +++ b/crypto/vm/vm.h @@ -127,6 +127,7 @@ class VmState final : public VmStateInterface { rist255_validate_gas_price = 200, ecrecover_gas_price = 1500, + secp256k1_xonly_pubkey_tweak_add_gas_price = 1250, chksgn_free_count = 10, chksgn_gas_price = 4000, p256_chksgn_gas_price = 3500, @@ -163,14 +164,12 @@ class VmState final : public VmStateInterface { bls_pairing_element_gas_price = 11800 }; VmState(); - VmState(Ref _code); - VmState(Ref _code, Ref _stack, int flags = 0, Ref _data = {}, VmLog log = {}, - std::vector> _libraries = {}, Ref init_c7 = {}); - VmState(Ref _code, Ref _stack, const GasLimits& _gas, int flags = 0, Ref _data = {}, + VmState(Ref _code, int global_version, Ref _stack, const GasLimits& _gas, int flags = 0, Ref _data = {}, VmLog log = {}, std::vector> _libraries = {}, Ref init_c7 = {}); - template - VmState(Ref code_cell, Args&&... args) - : VmState(convert_code_cell(std::move(code_cell)), std::forward(args)...) { + VmState(Ref _code, int global_version, Ref _stack, const GasLimits& _gas, int flags = 0, + Ref _data = {}, VmLog log = {}, std::vector> _libraries = {}, Ref init_c7 = {}) + : VmState(convert_code_cell(std::move(_code), global_version, _libraries), global_version, std::move(_stack), + _gas, flags, std::move(_data), std::move(log), _libraries, std::move(init_c7)) { } VmState(const VmState&) = delete; VmState(VmState&&) = default; @@ -344,13 +343,11 @@ class VmState final : public VmStateInterface { int get_global_version() const override { return global_version; } - void set_global_version(int version) { - global_version = version; - } int call(Ref cont); int call(Ref cont, int pass_args, int ret_args = -1); int jump(Ref cont); int jump(Ref cont, int pass_args); + Ref adjust_jump_cont(Ref cont, int pass_args); int ret(); int ret(int ret_args); int ret_alt(); @@ -378,10 +375,18 @@ class VmState final : public VmStateInterface { if (cnt > free_nested_cont_jump && global_version >= 9) { consume_gas(1); } + if (cont.not_null() && global_version >= 9) { + const ControlData* cont_data = cont->get_cdata(); + if (cont_data && (cont_data->stack.not_null() || cont_data->nargs >= 0)) { + // if cont has non-empty stack or expects fixed number of arguments, jump is not simple + cont = adjust_jump_cont(std::move(cont), -1); + } + } } return res; } - static Ref convert_code_cell(Ref code_cell); + static Ref convert_code_cell(Ref code_cell, int global_version, + const std::vector>& libraries); bool try_commit(); void force_commit(); diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index eb183cad..fa9fad13 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -54,7 +54,7 @@ Config::Config() { out_port = 3278; } -Config::Config(ton::ton_api::engine_validator_config &config) { +Config::Config(const ton::ton_api::engine_validator_config &config) { out_port = static_cast(config.out_port_); if (!out_port) { out_port = 3278; @@ -162,6 +162,7 @@ ton::tl_object_ptr Config::tl() const { control_vec.push_back(ton::create_tl_object(x.second.key.tl(), x.first, std::move(control_proc_vec))); } + std::vector> shard_vec; auto gc_vec = ton::create_tl_object(std::vector{}); for (auto &id : gc) { @@ -170,7 +171,7 @@ ton::tl_object_ptr Config::tl() const { return ton::create_tl_object( out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), - nullptr, nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); + nullptr, nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(shard_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, diff --git a/dht-server/dht-server.hpp b/dht-server/dht-server.hpp index 5b81875b..7c9e5619 100644 --- a/dht-server/dht-server.hpp +++ b/dht-server/dht-server.hpp @@ -94,7 +94,7 @@ struct Config { ton::tl_object_ptr tl() const; Config(); - Config(ton::ton_api::engine_validator_config &config); + Config(const ton::ton_api::engine_validator_config &config); }; class DhtServer : public td::actor::Actor { diff --git a/doc/GlobalVersions.md b/doc/GlobalVersions.md index 36d1ab36..77963e95 100644 --- a/doc/GlobalVersions.md +++ b/doc/GlobalVersions.md @@ -3,6 +3,7 @@ Global version is a parameter specified in `ConfigParam 8` ([block.tlb](https:// Various features are enabled depending on the global version. ## Version 4 +New features of version 4 are desctibed in detail in [the documentation](https://docs.ton.org/v3/documentation/tvm/changelog/tvm-upgrade-2023-07). ### New TVM instructions * `PREVMCBLOCKS`, `PREVKEYBLOCK` @@ -47,7 +48,7 @@ Version 5 enables higher gas limits for special contracts. Previously only ticktock transactions had this limit, while ordinary transactions had a default limit of `gas_limit` gas (1M). * Gas usage of special contracts is not taken into account when checking block limits. This allows keeping masterchain block limits low while having high gas limits for elector. -* Gas limit on `EQD_v9j1rlsuHHw2FIhcsCFFSD367ldfDdCKcsNmNpIRzUlu` is increased to `special_gas_limit * 2` until 2024-02-29. +* Gas limit on `EQD_v9j1rlsuHHw2FIhcsCFFSD367ldfDdCKcsNmNpIRzUlu` is increased to 70M (`special_gas_limit * 2`) until 2024-02-29. See [this post](https://t.me/tonstatus/88) for details. ### Loading libraries @@ -113,6 +114,45 @@ Operations for working with Merkle proofs, where cells can have non-zero level a ## Version 9 +### c7 tuple +c7 tuple parameter number **13** (previous blocks info tuple) now has the third element. It contains ids of the 16 last masterchain blocks with seqno divisible by 100. +Example: if the last masterchain block seqno is `19071` then the list contains block ids with seqnos `19000`, `18900`, ..., `17500`. + +### New TVM instructions +- `SECP256K1_XONLY_PUBKEY_TWEAK_ADD` (`key tweak - 0 or f x y -1`) - performs [`secp256k1_xonly_pubkey_tweak_add`](https://github.com/bitcoin-core/secp256k1/blob/master/include/secp256k1_extrakeys.h#L120). +`key` and `tweak` are 256-bit unsigned integers. 65-byte public key is returned as `uint8 f`, `uint256 x, y` (as in `ECRECOVER`). Gas cost: `1276`. +- `mask SETCONTCTRMANY` (`cont - cont'`) - takes continuation, performs the equivalent of `c[i] PUSHCTR SWAP c[i] SETCONTCNR` for each `i` that is set in `mask` (mask is in `0..255`). +- `SETCONTCTRMANYX` (`cont mask - cont'`) - same as `SETCONTCTRMANY`, but takes `mask` from stack. +- `PREVMCBLOCKS_100` returns the third element of the previous block info tuple (see above). + +### Other changes - Fix `RAWRESERVE` action with flag `4` (use original balance of the account) by explicitly setting `original_balance` to `balance - msg_balance_remaining`. - Previously it did not work if storage fee was greater than the original balance. -- Jumps to nested continuations of depth more than 8 consume 1 gas for eact subsequent continuation (this does not affect most of TVM code). \ No newline at end of file +- Jumps to nested continuations of depth more than 8 consume 1 gas for eact subsequent continuation (this does not affect most of TVM code). +- Support extra currencies in reserve action with `+2` mode. +- Fix exception code in some TVM instructions: now `stk_und` has priority over other error codes. + - `PFXDICTADD`, `PFXDICTSET`, `PFXDICTREPLACE`, `PFXDICTDEL`, `GETGASFEE`, `GETSTORAGEFEE`, `GETFORWARDFEE`, `GETORIGINALFWDFEE`, `GETGASFEESIMPLE`, `GETFORWARDFEESIMPLE`, `HASHEXT` +- Now setting the contract code to a library cell does not consume additional gas on execution of the code. +- Temporary increase gas limit for some accounts (see [this post](https://t.me/tondev_news/129) for details, `override_gas_limit` in `transaction.cpp` for the list of accounts). +- Fix recursive jump to continuations with non-null control data. + +## Version 10 + +### Extra currencies +- Internal messages cannot carry more than 2 different extra currencies. The limit can be changed in size limits config (`ConfigParam 43`). +- Amount of an extra currency in an output action "send message" can be zero. + - In action phase zero values are automatically deleted from the dictionary before sending. + - However, the size of the extra currency dictionary in the "send message" action should not be greater than 2 (or the value in size limits config). +- Extra currency dictionary is not counted in message size and does not affect message fees. +- Message mode `+64` (carry all remaining message balance) is now considered as "carry all remaining TONs from message balance". +- Message mode `+128` (carry all remaining account balance) is now considered as "carry all remaining TONs from account balance". +- Message mode `+32` (delete account if balance is zero) deletes account if it has zero TONs, regardless of extra currencies. + - Deleted accounts with extra currencies become `account_uninit`, extra currencies remain on the account. +- `SENDMSG` in TVM calculates message size and fees without extra currencies, uses new `+64` and `+128` mode behavior. + - `SENDMSG` does not check the number of extra currencies. +- Extra currency dictionary is not counted in the account size and does not affect storage fees. + - Accounts with already existing extra currencies will get their sizes recomputed without EC only after modifying `AccountState`. + +### TVM changes +- `SENDMSG` calculates messages size and fees without extra currencies, uses new +64 and +128 mode behavior. + - `SENDMSG` does not check the number of extra currencies. diff --git a/emulator/CMakeLists.txt b/emulator/CMakeLists.txt index 4b638300..663c8fd2 100644 --- a/emulator/CMakeLists.txt +++ b/emulator/CMakeLists.txt @@ -1,21 +1,12 @@ cmake_minimum_required(VERSION 3.5 FATAL_ERROR) -option(BUILD_SHARED_LIBS "Use \"OFF\" for a static build." ON) - -if (NOT OPENSSL_FOUND) - find_package(OpenSSL REQUIRED) -endif() +option(EMULATOR_STATIC "Build emulator as static library" OFF) set(EMULATOR_STATIC_SOURCE transaction-emulator.cpp tvm-emulator.hpp ) -set(EMULATOR_HEADERS - transaction-emulator.h - emulator-extern.h -) - set(EMULATOR_SOURCE emulator-extern.cpp ) @@ -29,14 +20,22 @@ include(GenerateExportHeader) add_library(emulator_static STATIC ${EMULATOR_STATIC_SOURCE}) target_link_libraries(emulator_static PUBLIC ton_crypto smc-envelope) -if (NOT USE_EMSCRIPTEN AND BUILD_SHARED_LIBS) - add_library(emulator SHARED ${EMULATOR_SOURCE} ${EMULATOR_HEADERS}) +if (EMULATOR_STATIC OR USE_EMSCRIPTEN) + add_library(emulator STATIC ${EMULATOR_SOURCE}) else() - add_library(emulator STATIC ${EMULATOR_SOURCE} ${EMULATOR_HEADERS}) + add_library(emulator SHARED ${EMULATOR_SOURCE}) +endif() + +if (PORTABLE AND NOT APPLE) + target_link_libraries(emulator PUBLIC emulator_static git -static-libgcc -static-libstdc++) +else() + target_link_libraries(emulator PUBLIC emulator_static git) endif() -target_link_libraries(emulator PUBLIC emulator_static git) generate_export_header(emulator EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/emulator_export.h) +if (EMULATOR_STATIC OR USE_EMSCRIPTEN) + target_compile_definitions(emulator PUBLIC EMULATOR_STATIC_DEFINE) +endif() target_include_directories(emulator PUBLIC $ $) @@ -47,8 +46,8 @@ endif() if (USE_EMSCRIPTEN) add_executable(emulator-emscripten ${EMULATOR_EMSCRIPTEN_SOURCE}) target_link_libraries(emulator-emscripten PUBLIC emulator) - target_link_options(emulator-emscripten PRIVATE -sEXPORTED_RUNTIME_METHODS=_malloc,free,UTF8ToString,stringToUTF8,allocate,ALLOC_NORMAL,lengthBytesUTF8) - target_link_options(emulator-emscripten PRIVATE -sEXPORTED_FUNCTIONS=_emulate,_free,_run_get_method,_create_emulator,_destroy_emulator,_emulate_with_emulator,_version) + target_link_options(emulator-emscripten PRIVATE -sEXPORTED_RUNTIME_METHODS=UTF8ToString,stringToUTF8,allocate,ALLOC_NORMAL,lengthBytesUTF8) + target_link_options(emulator-emscripten PRIVATE -sEXPORTED_FUNCTIONS=_emulate,_free,_malloc,_run_get_method,_create_emulator,_destroy_emulator,_emulate_with_emulator,_version) target_link_options(emulator-emscripten PRIVATE -sEXPORT_NAME=EmulatorModule) target_link_options(emulator-emscripten PRIVATE -sERROR_ON_UNDEFINED_SYMBOLS=0) target_link_options(emulator-emscripten PRIVATE -Oz) diff --git a/emulator/emulator-emscripten.cpp b/emulator/emulator-emscripten.cpp index 17639d28..efb14eff 100644 --- a/emulator/emulator-emscripten.cpp +++ b/emulator/emulator-emscripten.cpp @@ -65,6 +65,7 @@ struct GetMethodParams { std::string address; uint32_t unixtime; uint64_t balance; + std::string extra_currencies; std::string rand_seed_hex; int64_t gas_limit; int method_id; @@ -108,6 +109,32 @@ td::Result decode_get_method_params(const char* json) { TRY_RESULT(balance, td::to_integer_safe(balance_field.get_string())); params.balance = balance; + TRY_RESULT(ec_field, td::get_json_object_field(obj, "extra_currencies", td::JsonValue::Type::Object, true)); + if (ec_field.type() != td::JsonValue::Type::Null) { + if (ec_field.type() != td::JsonValue::Type::Object) { + return td::Status::Error("EC must be of type Object"); + } + td::StringBuilder ec_builder; + auto& ec_obj = ec_field.get_object(); + bool is_first = true; + for (auto &field_value : ec_obj) { + auto currency_id = field_value.first; + if (field_value.second.type() != td::JsonValue::Type::String) { + return td::Status::Error(PSLICE() << "EC amount must be of type String"); + } + auto amount = field_value.second.get_string(); + if (!is_first) { + ec_builder << " "; + is_first = false; + } + ec_builder << currency_id << "=" << amount; + } + if (ec_builder.is_error()) { + return td::Status::Error(PSLICE() << "Error building extra currencies string"); + } + params.extra_currencies = ec_builder.as_cslice().str(); + } + TRY_RESULT(rand_seed_str, td::get_json_object_string_field(obj, "rand_seed", false)); params.rand_seed_hex = rand_seed_str; @@ -228,8 +255,8 @@ const char *run_get_method(const char *params, const char* stack, const char* co if ((decoded_params.libs && !tvm_emulator_set_libraries(tvm, decoded_params.libs.value().c_str())) || !tvm_emulator_set_c7(tvm, decoded_params.address.c_str(), decoded_params.unixtime, decoded_params.balance, decoded_params.rand_seed_hex.c_str(), config) || - (decoded_params.prev_blocks_info && - !tvm_emulator_set_prev_blocks_info(tvm, decoded_params.prev_blocks_info.value().c_str())) || + (decoded_params.extra_currencies.size() > 0 && !tvm_emulator_set_extra_currencies(tvm, decoded_params.extra_currencies.c_str())) || + (decoded_params.prev_blocks_info && !tvm_emulator_set_prev_blocks_info(tvm, decoded_params.prev_blocks_info.value().c_str())) || (decoded_params.gas_limit > 0 && !tvm_emulator_set_gas_limit(tvm, decoded_params.gas_limit)) || !tvm_emulator_set_debug_enabled(tvm, decoded_params.debug_enabled)) { tvm_emulator_destroy(tvm); diff --git a/emulator/emulator-extern.cpp b/emulator/emulator-extern.cpp index 52c374ed..eb5ff9f9 100644 --- a/emulator/emulator-extern.cpp +++ b/emulator/emulator-extern.cpp @@ -496,6 +496,59 @@ bool tvm_emulator_set_c7(void *tvm_emulator, const char *address, uint32_t unixt return true; } +bool tvm_emulator_set_extra_currencies(void *tvm_emulator, const char *extra_currencies) { + auto emulator = static_cast(tvm_emulator); + vm::Dictionary dict{32}; + td::Slice extra_currencies_str{extra_currencies}; + while (true) { + auto next_space_pos = extra_currencies_str.find(' '); + auto currency_id_amount = next_space_pos == td::Slice::npos ? + extra_currencies_str.substr(0) : extra_currencies_str.substr(0, next_space_pos); + + if (!currency_id_amount.empty()) { + auto delim_pos = currency_id_amount.find('='); + if (delim_pos == td::Slice::npos) { + LOG(ERROR) << "Invalid extra currency format, missing '='"; + return false; + } + + auto currency_id_str = currency_id_amount.substr(0, delim_pos); + auto amount_str = currency_id_amount.substr(delim_pos + 1); + + auto currency_id = td::to_integer_safe(currency_id_str); + if (currency_id.is_error()) { + LOG(ERROR) << "Invalid extra currency id: " << currency_id_str; + return false; + } + auto amount = td::dec_string_to_int256(amount_str); + if (amount.is_null()) { + LOG(ERROR) << "Invalid extra currency amount: " << amount_str; + return false; + } + if (amount == 0) { + continue; + } + if (amount < 0) { + LOG(ERROR) << "Negative extra currency amount: " << amount_str; + return false; + } + + vm::CellBuilder cb; + block::tlb::t_VarUInteger_32.store_integer_value(cb, *amount); + if (!dict.set_builder(td::BitArray<32>(currency_id.ok()), cb, vm::DictionaryBase::SetMode::Add)) { + LOG(ERROR) << "Duplicate extra currency id"; + return false; + } + } + if (next_space_pos == td::Slice::npos) { + break; + } + extra_currencies_str.remove_prefix(next_space_pos + 1); + } + emulator->set_extra_currencies(std::move(dict).extract_root_cell()); + return true; +} + bool tvm_emulator_set_config_object(void* tvm_emulator, void* config) { auto emulator = static_cast(tvm_emulator); auto global_config = std::shared_ptr(static_cast(config), config_deleter); @@ -615,7 +668,7 @@ const char *tvm_emulator_emulate_run_method(uint32_t len, const char *params_boc emulator->set_vm_verbosity_level(0); emulator->set_gas_limit(gas_limit); emulator->set_c7_raw(c7->fetch(0).as_tuple()); - if (libs.is_empty()) { + if (!libs.is_empty()) { emulator->set_libraries(std::move(libs)); } auto result = emulator->run_get_method(int(method_id), stack); diff --git a/emulator/emulator-extern.h b/emulator/emulator-extern.h index e69a9cb0..14879e1e 100644 --- a/emulator/emulator-extern.h +++ b/emulator/emulator-extern.h @@ -182,6 +182,14 @@ EMULATOR_EXPORT bool tvm_emulator_set_libraries(void *tvm_emulator, const char * */ EMULATOR_EXPORT bool tvm_emulator_set_c7(void *tvm_emulator, const char *address, uint32_t unixtime, uint64_t balance, const char *rand_seed_hex, const char *config); +/** + * @brief Set extra currencies balance + * @param tvm_emulator Pointer to TVM emulator + * @param extra_currencies String with extra currencies balance in format "currency_id1=balance1 currency_id2=balance2 ..." + * @return true in case of success, false in case of error + */ +EMULATOR_EXPORT bool tvm_emulator_set_extra_currencies(void *tvm_emulator, const char *extra_currencies); + /** * @brief Set config for TVM emulator * @param tvm_emulator Pointer to TVM emulator diff --git a/emulator/emulator_export_list b/emulator/emulator_export_list index feb653e2..bd991cd7 100644 --- a/emulator/emulator_export_list +++ b/emulator/emulator_export_list @@ -17,6 +17,7 @@ _emulator_config_destroy _tvm_emulator_create _tvm_emulator_set_libraries _tvm_emulator_set_c7 +_tvm_emulator_set_extra_currencies _tvm_emulator_set_config_object _tvm_emulator_set_prev_blocks_info _tvm_emulator_set_gas_limit diff --git a/emulator/test/emulator-tests.cpp b/emulator/test/emulator-tests.cpp index 24394b49..ae273ddf 100644 --- a/emulator/test/emulator-tests.cpp +++ b/emulator/test/emulator-tests.cpp @@ -400,3 +400,58 @@ TEST(Emulator, tvm_emulator) { CHECK(stack_res->depth() == 1); CHECK(stack_res.write().pop_int()->to_long() == init_data.seqno); } + +TEST(Emulator, tvm_emulator_extra_currencies) { + void *tvm_emulator = tvm_emulator_create("te6cckEBBAEAHgABFP8A9KQT9LzyyAsBAgFiAgMABtBfBAAJofpP8E8XmGlj", "te6cckEBAQEAAgAAAEysuc0=", 1); + std::string addr = "0:" + std::string(64, 'F'); + tvm_emulator_set_c7(tvm_emulator, addr.c_str(), 1337, 1000, std::string(64, 'F').c_str(), nullptr); + CHECK(tvm_emulator_set_extra_currencies(tvm_emulator, "100=20000 200=1")); + unsigned method_crc = td::crc16("get_balance"); + unsigned method_id = (method_crc & 0xffff) | 0x10000; + + auto stack = td::make_ref(); + vm::CellBuilder stack_cb; + CHECK(stack->serialize(stack_cb)); + auto stack_cell = stack_cb.finalize(); + auto stack_boc = td::base64_encode(std_boc_serialize(stack_cell).move_as_ok()); + + std::string tvm_res = tvm_emulator_run_get_method(tvm_emulator, method_id, stack_boc.c_str()); + + auto result_json = td::json_decode(td::MutableSlice(tvm_res)); + auto result = result_json.move_as_ok(); + auto& result_obj = result.get_object(); + + auto success_field = td::get_json_object_field(result_obj, "success", td::JsonValue::Type::Boolean, false); + auto success = success_field.move_as_ok().get_boolean(); + CHECK(success); + + auto stack_field = td::get_json_object_field(result_obj, "stack", td::JsonValue::Type::String, false); + auto stack_val = stack_field.move_as_ok(); + auto& stack_obj = stack_val.get_string(); + auto stack_res_boc = td::base64_decode(stack_obj); + auto stack_res_cell = vm::std_boc_deserialize(stack_res_boc.move_as_ok()); + td::Ref stack_res; + auto stack_res_cs = vm::load_cell_slice(stack_res_cell.move_as_ok()); + CHECK(vm::Stack::deserialize_to(stack_res_cs, stack_res)); + CHECK(stack_res->depth() == 1); + auto tuple = stack_res.write().pop_tuple(); + CHECK(tuple->size() == 2); + + auto ton_balance = tuple->at(0).as_int(); + CHECK(ton_balance == 1000); + + auto cell = tuple->at(1).as_cell(); + auto dict = vm::Dictionary{cell, 32}; + auto it = dict.begin(); + std::map ec_balance; + while (!it.eof()) { + auto id = static_cast(td::BitArray<32>(it.cur_pos()).to_ulong()); + auto value_cs = it.cur_value(); + auto value = block::tlb::t_VarUInteger_32.as_integer(value_cs); + ec_balance[id] = value; + ++it; + } + CHECK(ec_balance.size() == 2); + CHECK(ec_balance[100] == 20000); + CHECK(ec_balance[200] == 1); +} diff --git a/emulator/transaction-emulator.cpp b/emulator/transaction-emulator.cpp index e87b2dfb..6267f9bd 100644 --- a/emulator/transaction-emulator.cpp +++ b/emulator/transaction-emulator.cpp @@ -16,6 +16,7 @@ td::Result> TransactionEmu block::StoragePhaseConfig storage_phase_cfg{&storage_prices}; block::ComputePhaseConfig compute_phase_cfg; block::ActionPhaseConfig action_phase_cfg; + block::SerializeConfig serialize_config; td::RefInt256 masterchain_create_fee, basechain_create_fee; if (!utime) { @@ -25,11 +26,9 @@ td::Result> TransactionEmu utime = (unsigned)std::time(nullptr); } - auto fetch_res = block::FetchConfigParams::fetch_config_params(*config_, prev_blocks_info_, &old_mparams, - &storage_prices, &storage_phase_cfg, - &rand_seed_, &compute_phase_cfg, - &action_phase_cfg, &masterchain_create_fee, - &basechain_create_fee, account.workchain, utime); + auto fetch_res = block::FetchConfigParams::fetch_config_params( + *config_, prev_blocks_info_, &old_mparams, &storage_prices, &storage_phase_cfg, &rand_seed_, &compute_phase_cfg, + &action_phase_cfg, &serialize_config, &masterchain_create_fee, &basechain_create_fee, account.workchain, utime); if(fetch_res.is_error()) { return fetch_res.move_as_error_prefix("cannot fetch config params "); } @@ -66,7 +65,7 @@ td::Result> TransactionEmu return std::make_unique(std::move(vm_log), vm_exit_code, elapsed); } - if (!trans->serialize()) { + if (!trans->serialize(serialize_config)) { return td::Status::Error(-669,"cannot serialize new transaction for smart contract "s + trans->account.addr.to_hex()); } diff --git a/emulator/tvm-emulator.hpp b/emulator/tvm-emulator.hpp index 413298c9..acc13627 100644 --- a/emulator/tvm-emulator.hpp +++ b/emulator/tvm-emulator.hpp @@ -33,6 +33,10 @@ public: } } + void set_extra_currencies(td::Ref extra_currencies) { + args_.set_extra_currencies(std::move(extra_currencies)); + } + void set_c7_raw(td::Ref c7) { args_.set_c7(std::move(c7)); } diff --git a/example/android/CMakeLists.txt b/example/android/CMakeLists.txt index b9651f61..0101ab64 100644 --- a/example/android/CMakeLists.txt +++ b/example/android/CMakeLists.txt @@ -3,7 +3,9 @@ # Sets the minimum version of CMake required to build the native library. -cmake_minimum_required(VERSION 3.4.1 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) + +project(TON_ANDROID VERSION 0.5 LANGUAGES C CXX) option(TONLIB_ENABLE_JNI "Enable JNI-compatible TonLib API" ON) diff --git a/example/android/build.sh b/example/android/build.sh index cd75f9d7..dc0618e9 100755 --- a/example/android/build.sh +++ b/example/android/build.sh @@ -48,7 +48,6 @@ mkdir -p build-$ARCH cd build-$ARCH cmake .. -GNinja \ --DPORTABLE=1 \ -DTON_ONLY_TONLIB=ON \ -DTON_ARCH="" \ -DANDROID_ABI=x86 \ @@ -58,12 +57,12 @@ cmake .. -GNinja \ -DCMAKE_BUILD_TYPE=Release \ -DANDROID_ABI=${ABI} \ -DOPENSSL_ROOT_DIR=${OPENSSL_DIR}/${ORIG_ARCH} \ --DSECP256K1_FOUND=1 \ -DSECP256K1_INCLUDE_DIR=${SECP256K1_INCLUDE_DIR} \ -DSECP256K1_LIBRARY=${SECP256K1_LIBRARY} \ -DLZ4_FOUND=1 \ -DLZ4_INCLUDE_DIRS=${LZ4_INCLUDE_DIR} \ -DLZ4_LIBRARIES=${LZ4_LIBRARY} \ +-DSODIUM_FOUND=1 \ -DSODIUM_INCLUDE_DIR=${SODIUM_INCLUDE_DIR} \ -DSODIUM_LIBRARY_RELEASE=${SODIUM_LIBRARY_RELEASE} \ -DSODIUM_USE_STATIC_LIBS=1 \ diff --git a/example/android/third_party/secp256k1/armv7/libsecp256k1.a b/example/android/third_party/secp256k1/armv7/libsecp256k1.a index 70464cba..f80ead61 100644 Binary files a/example/android/third_party/secp256k1/armv7/libsecp256k1.a and b/example/android/third_party/secp256k1/armv7/libsecp256k1.a differ diff --git a/example/android/third_party/secp256k1/armv7/libsecp256k1.so b/example/android/third_party/secp256k1/armv7/libsecp256k1.so deleted file mode 100644 index b5d939ab..00000000 Binary files a/example/android/third_party/secp256k1/armv7/libsecp256k1.so and /dev/null differ diff --git a/example/android/third_party/secp256k1/armv8/libsecp256k1.a b/example/android/third_party/secp256k1/armv8/libsecp256k1.a index 5b16c4ab..1b79e3af 100644 Binary files a/example/android/third_party/secp256k1/armv8/libsecp256k1.a and b/example/android/third_party/secp256k1/armv8/libsecp256k1.a differ diff --git a/example/android/third_party/secp256k1/armv8/libsecp256k1.so b/example/android/third_party/secp256k1/armv8/libsecp256k1.so deleted file mode 100644 index 6712bedf..00000000 Binary files a/example/android/third_party/secp256k1/armv8/libsecp256k1.so and /dev/null differ diff --git a/example/android/third_party/secp256k1/build.sh b/example/android/third_party/secp256k1/build.sh old mode 100644 new mode 100755 index d4251b44..68da6793 --- a/example/android/third_party/secp256k1/build.sh +++ b/example/android/third_party/secp256k1/build.sh @@ -5,28 +5,32 @@ export CC= export CXX= rm -rf secp256k1 -git clone https://github.com/libbitcoin/secp256k1.git +git clone https://github.com/bitcoin-core/secp256k1 cd secp256k1 +git checkout v0.3.2 ./autogen.sh +mkdir build +cd build -./configure --enable-module-recovery --enable-experimental --with-asm=arm --host=arm-linux-androideabi CC=armv7a-linux-androideabi21-clang CFLAGS="-mthumb -march=armv7-a" CCASFLAGS="-Wa,-mthumb -Wa,-march=armv7-a" +cmake .. -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake" -DANDROID_ABI=armeabi-v7a -DANDROID_PLATFORM=21 -DBUILD_SHARED_LIBS=OFF -DANDROID_TOOLCHAIN_NAME=arm-linux-androideabi make -cp .libs/libsecp256k1.a ../armv7/ -cp .libs/libsecp256k1.so ../armv7/ +cp lib/libsecp256k1.a ../../armv7/ +rm -rf * -./configure --enable-module-recovery --host=aarch64-linux-android CC=aarch64-linux-android21-clang CFLAGS="-mthumb -march=armv8-a" CCASFLAGS="-Wa,-mthumb -Wa,-march=armv8-a" +cmake .. -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake" -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=21 -DBUILD_SHARED_LIBS=OFF -DANDROID_TOOLCHAIN_NAME=aarch64-linux-android make -cp .libs/libsecp256k1.a ../armv8/ -cp .libs/libsecp256k1.so ../armv8/ +cp lib/libsecp256k1.a ../../armv8/ +rm -rf * -./configure --enable-module-recovery --host=x86_64-linux-android CC=x86_64-linux-android21-clang +cmake .. -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake" -DANDROID_ABI=x86_64 -DANDROID_PLATFORM=21 -DBUILD_SHARED_LIBS=OFF -DANDROID_TOOLCHAIN_NAME=x86_64 make -cp .libs/libsecp256k1.a ../x86-64/ -cp .libs/libsecp256k1.so ../x86-64/ +cp lib/libsecp256k1.a ../../x86-64/ +rm -rf * -./configure --enable-module-recovery --host=i686-linux-android CC=i686-linux-android21-clang +cmake .. -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake" -DANDROID_ABI= -DANDROID_PLATFORM=21 -DBUILD_SHARED_LIBS=OFF -DANDROID_TOOLCHAIN_NAME=x86- make -cp .libs/libsecp256k1.a ../i686/ -cp .libs/libsecp256k1.so ../i686/ +cp lib/libsecp256k1.a ../../i686/ +rm -rf * +rm -rf ../secp256k1 diff --git a/example/android/third_party/secp256k1/i686/libsecp256k1.a b/example/android/third_party/secp256k1/i686/libsecp256k1.a index 01ebe179..014b1fc7 100644 Binary files a/example/android/third_party/secp256k1/i686/libsecp256k1.a and b/example/android/third_party/secp256k1/i686/libsecp256k1.a differ diff --git a/example/android/third_party/secp256k1/i686/libsecp256k1.so b/example/android/third_party/secp256k1/i686/libsecp256k1.so deleted file mode 100644 index e245a724..00000000 Binary files a/example/android/third_party/secp256k1/i686/libsecp256k1.so and /dev/null differ diff --git a/example/android/third_party/secp256k1/include/secp256k1.h b/example/android/third_party/secp256k1/include/secp256k1.h index 36020e51..c6e9417f 100644 --- a/example/android/third_party/secp256k1/include/secp256k1.h +++ b/example/android/third_party/secp256k1/include/secp256k1.h @@ -7,14 +7,16 @@ extern "C" { #include -/* These rules specify the order of arguments in API calls: +/** Unless explicitly stated all pointer arguments must not be NULL. + * + * The following rules specify the order of arguments in API calls: * * 1. Context pointers go first, followed by output arguments, combined * output/input arguments, and finally input-only arguments. - * 2. Array lengths always immediately the follow the argument whose length + * 2. Array lengths always immediately follow the argument whose length * they describe, even if this violates rule 1. * 3. Within the OUT/OUTIN/IN groups, pointers to data that is typically generated - * later go first. This means: signatures, public nonces, private nonces, + * later go first. This means: signatures, public nonces, secret nonces, * messages, public keys, secret keys, tweaks. * 4. Arguments that are not data pointers go last, from more complex to less * complex: function pointers, algorithm names, messages, void pointers, @@ -22,15 +24,19 @@ extern "C" { * 5. Opaque data pointers follow the function pointer they are to be passed to. */ -/** Opaque data structure that holds context information (precomputed tables etc.). +/** Opaque data structure that holds context information * - * The purpose of context structures is to cache large precomputed data tables - * that are expensive to construct, and also to maintain the randomization data - * for blinding. + * The primary purpose of context objects is to store randomization data for + * enhanced protection against side-channel leakage. This protection is only + * effective if the context is randomized after its creation. See + * secp256k1_context_create for creation of contexts and + * secp256k1_context_randomize for randomization. * - * Do not create a new context object for each operation, as construction is - * far slower than all other API calls (~100 times slower than an ECDSA - * verification). + * A secondary purpose of context objects is to store pointers to callback + * functions that the library will call when certain error states arise. See + * secp256k1_context_set_error_callback as well as + * secp256k1_context_set_illegal_callback for details. Future library versions + * may use context objects for additional purposes. * * A constructed context can safely be used from multiple threads * simultaneously, but API calls that take a non-const pointer to a context @@ -43,32 +49,20 @@ extern "C" { */ typedef struct secp256k1_context_struct secp256k1_context; -/** Opaque data structure that holds rewriteable "scratch space" - * - * The purpose of this structure is to replace dynamic memory allocations, - * because we target architectures where this may not be available. It is - * essentially a resizable (within specified parameters) block of bytes, - * which is initially created either by memory allocation or TODO as a pointer - * into some fixed rewritable space. - * - * Unlike the context object, this cannot safely be shared between threads - * without additional synchronization logic. - */ -typedef struct secp256k1_scratch_space_struct secp256k1_scratch_space; - /** Opaque data structure that holds a parsed and valid public key. * * The exact representation of data inside is implementation defined and not * guaranteed to be portable between different platforms or versions. It is * however guaranteed to be 64 bytes in size, and can be safely copied/moved. - * If you need to convert to a format suitable for storage, transmission, or - * comparison, use secp256k1_ec_pubkey_serialize and secp256k1_ec_pubkey_parse. + * If you need to convert to a format suitable for storage or transmission, + * use secp256k1_ec_pubkey_serialize and secp256k1_ec_pubkey_parse. To + * compare keys, use secp256k1_ec_pubkey_cmp. */ -typedef struct { +typedef struct secp256k1_pubkey { unsigned char data[64]; } secp256k1_pubkey; -/** Opaque data structured that holds a parsed ECDSA signature. +/** Opaque data structure that holds a parsed ECDSA signature. * * The exact representation of data inside is implementation defined and not * guaranteed to be portable between different platforms or versions. It is @@ -77,7 +71,7 @@ typedef struct { * comparison, use the secp256k1_ecdsa_signature_serialize_* and * secp256k1_ecdsa_signature_parse_* functions. */ -typedef struct { +typedef struct secp256k1_ecdsa_signature { unsigned char data[64]; } secp256k1_ecdsa_signature; @@ -115,35 +109,62 @@ typedef int (*secp256k1_nonce_function)( # endif # endif -# if (!defined(__STDC_VERSION__) || (__STDC_VERSION__ < 199901L) ) -# if SECP256K1_GNUC_PREREQ(2,7) -# define SECP256K1_INLINE __inline__ -# elif (defined(_MSC_VER)) -# define SECP256K1_INLINE __inline -# else -# define SECP256K1_INLINE -# endif -# else -# define SECP256K1_INLINE inline -# endif +/* When this header is used at build-time the SECP256K1_BUILD define needs to be set + * to correctly setup export attributes and nullness checks. This is normally done + * by secp256k1.c but to guard against this header being included before secp256k1.c + * has had a chance to set the define (e.g. via test harnesses that just includes + * secp256k1.c) we set SECP256K1_NO_BUILD when this header is processed without the + * BUILD define so this condition can be caught. + */ +#ifndef SECP256K1_BUILD +# define SECP256K1_NO_BUILD +#endif -#ifndef SECP256K1_API -# if defined(_WIN32) -# ifdef SECP256K1_BUILD -# define SECP256K1_API __declspec(dllexport) +/* Symbol visibility. */ +#if defined(_WIN32) + /* GCC for Windows (e.g., MinGW) accepts the __declspec syntax + * for MSVC compatibility. A __declspec declaration implies (but is not + * exactly equivalent to) __attribute__ ((visibility("default"))), and so we + * actually want __declspec even on GCC, see "Microsoft Windows Function + * Attributes" in the GCC manual and the recommendations in + * https://gcc.gnu.org/wiki/Visibility. */ +# if defined(SECP256K1_BUILD) +# if defined(DLL_EXPORT) || defined(SECP256K1_DLL_EXPORT) + /* Building libsecp256k1 as a DLL. + * 1. If using Libtool, it defines DLL_EXPORT automatically. + * 2. In other cases, SECP256K1_DLL_EXPORT must be defined. */ +# define SECP256K1_API extern __declspec (dllexport) # else -# define SECP256K1_API + /* Building libsecp256k1 as a static library on Windows. + * No declspec is needed, and so we would want the non-Windows-specific + * logic below take care of this case. However, this may result in setting + * __attribute__ ((visibility("default"))), which is supposed to be a noop + * on Windows but may trigger warnings when compiling with -flto due to a + * bug in GCC, see + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=116478 . */ +# define SECP256K1_API extern # endif -# elif defined(__GNUC__) && defined(SECP256K1_BUILD) -# define SECP256K1_API __attribute__ ((visibility ("default"))) + /* The user must define SECP256K1_STATIC when consuming libsecp256k1 as a static + * library on Windows. */ +# elif !defined(SECP256K1_STATIC) + /* Consuming libsecp256k1 as a DLL. */ +# define SECP256K1_API extern __declspec (dllimport) +# endif +#endif +#ifndef SECP256K1_API +/* All cases not captured by the Windows-specific logic. */ +# if defined(__GNUC__) && (__GNUC__ >= 4) && defined(SECP256K1_BUILD) + /* Building libsecp256k1 using GCC or compatible. */ +# define SECP256K1_API extern __attribute__ ((visibility ("default"))) # else -# define SECP256K1_API + /* Fall back to standard C's extern. */ +# define SECP256K1_API extern # endif #endif -/**Warning attributes - * NONNULL is not used if SECP256K1_BUILD is set to avoid the compiler optimizing out - * some paranoid null checks. */ +/* Warning attributes + * NONNULL is not used if SECP256K1_BUILD is set to avoid the compiler optimizing out + * some paranoid null checks. */ # if defined(__GNUC__) && SECP256K1_GNUC_PREREQ(3, 4) # define SECP256K1_WARN_UNUSED_RESULT __attribute__ ((__warn_unused_result__)) # else @@ -155,20 +176,37 @@ typedef int (*secp256k1_nonce_function)( # define SECP256K1_ARG_NONNULL(_x) # endif -/** All flags' lower 8 bits indicate what they're for. Do not use directly. */ +/* Attribute for marking functions, types, and variables as deprecated */ +#if !defined(SECP256K1_BUILD) && defined(__has_attribute) +# if __has_attribute(__deprecated__) +# define SECP256K1_DEPRECATED(_msg) __attribute__ ((__deprecated__(_msg))) +# else +# define SECP256K1_DEPRECATED(_msg) +# endif +#else +# define SECP256K1_DEPRECATED(_msg) +#endif + +/* All flags' lower 8 bits indicate what they're for. Do not use directly. */ #define SECP256K1_FLAGS_TYPE_MASK ((1 << 8) - 1) #define SECP256K1_FLAGS_TYPE_CONTEXT (1 << 0) #define SECP256K1_FLAGS_TYPE_COMPRESSION (1 << 1) -/** The higher bits contain the actual data. Do not use directly. */ +/* The higher bits contain the actual data. Do not use directly. */ #define SECP256K1_FLAGS_BIT_CONTEXT_VERIFY (1 << 8) #define SECP256K1_FLAGS_BIT_CONTEXT_SIGN (1 << 9) +#define SECP256K1_FLAGS_BIT_CONTEXT_DECLASSIFY (1 << 10) #define SECP256K1_FLAGS_BIT_COMPRESSION (1 << 8) -/** Flags to pass to secp256k1_context_create, secp256k1_context_preallocated_size, and +/** Context flags to pass to secp256k1_context_create, secp256k1_context_preallocated_size, and * secp256k1_context_preallocated_create. */ +#define SECP256K1_CONTEXT_NONE (SECP256K1_FLAGS_TYPE_CONTEXT) + +/** Deprecated context flags. These flags are treated equivalent to SECP256K1_CONTEXT_NONE. */ #define SECP256K1_CONTEXT_VERIFY (SECP256K1_FLAGS_TYPE_CONTEXT | SECP256K1_FLAGS_BIT_CONTEXT_VERIFY) #define SECP256K1_CONTEXT_SIGN (SECP256K1_FLAGS_TYPE_CONTEXT | SECP256K1_FLAGS_BIT_CONTEXT_SIGN) -#define SECP256K1_CONTEXT_NONE (SECP256K1_FLAGS_TYPE_CONTEXT) + +/* Testing flag. Do not use. */ +#define SECP256K1_CONTEXT_DECLASSIFY (SECP256K1_FLAGS_TYPE_CONTEXT | SECP256K1_FLAGS_BIT_CONTEXT_DECLASSIFY) /** Flag to pass to secp256k1_ec_pubkey_serialize. */ #define SECP256K1_EC_COMPRESSED (SECP256K1_FLAGS_TYPE_COMPRESSION | SECP256K1_FLAGS_BIT_COMPRESSION) @@ -181,25 +219,68 @@ typedef int (*secp256k1_nonce_function)( #define SECP256K1_TAG_PUBKEY_HYBRID_EVEN 0x06 #define SECP256K1_TAG_PUBKEY_HYBRID_ODD 0x07 -/** A simple secp256k1 context object with no precomputed tables. These are useful for - * type serialization/parsing functions which require a context object to maintain - * API consistency, but currently do not require expensive precomputations or dynamic - * allocations. +/** A built-in constant secp256k1 context object with static storage duration, to be + * used in conjunction with secp256k1_selftest. + * + * This context object offers *only limited functionality* , i.e., it cannot be used + * for API functions that perform computations involving secret keys, e.g., signing + * and public key generation. If this restriction applies to a specific API function, + * it is mentioned in its documentation. See secp256k1_context_create if you need a + * full context object that supports all functionality offered by the library. + * + * It is highly recommended to call secp256k1_selftest before using this context. */ -SECP256K1_API extern const secp256k1_context *secp256k1_context_no_precomp; +SECP256K1_API const secp256k1_context *secp256k1_context_static; + +/** Deprecated alias for secp256k1_context_static. */ +SECP256K1_API const secp256k1_context *secp256k1_context_no_precomp +SECP256K1_DEPRECATED("Use secp256k1_context_static instead"); + +/** Perform basic self tests (to be used in conjunction with secp256k1_context_static) + * + * This function performs self tests that detect some serious usage errors and + * similar conditions, e.g., when the library is compiled for the wrong endianness. + * This is a last resort measure to be used in production. The performed tests are + * very rudimentary and are not intended as a replacement for running the test + * binaries. + * + * It is highly recommended to call this before using secp256k1_context_static. + * It is not necessary to call this function before using a context created with + * secp256k1_context_create (or secp256k1_context_preallocated_create), which will + * take care of performing the self tests. + * + * If the tests fail, this function will call the default error handler to abort the + * program (see secp256k1_context_set_error_callback). + */ +SECP256K1_API void secp256k1_selftest(void); + /** Create a secp256k1 context object (in dynamically allocated memory). * * This function uses malloc to allocate memory. It is guaranteed that malloc is * called at most once for every call of this function. If you need to avoid dynamic - * memory allocation entirely, see the functions in secp256k1_preallocated.h. + * memory allocation entirely, see secp256k1_context_static and the functions in + * secp256k1_preallocated.h. * - * Returns: a newly created context object. - * In: flags: which parts of the context to initialize. + * Returns: pointer to a newly created context object. + * In: flags: Always set to SECP256K1_CONTEXT_NONE (see below). * - * See also secp256k1_context_randomize. + * The only valid non-deprecated flag in recent library versions is + * SECP256K1_CONTEXT_NONE, which will create a context sufficient for all functionality + * offered by the library. All other (deprecated) flags will be treated as equivalent + * to the SECP256K1_CONTEXT_NONE flag. Though the flags parameter primarily exists for + * historical reasons, future versions of the library may introduce new flags. + * + * If the context is intended to be used for API functions that perform computations + * involving secret keys, e.g., signing and public key generation, then it is highly + * recommended to call secp256k1_context_randomize on the context before calling + * those API functions. This will provide enhanced protection against side-channel + * leakage, see secp256k1_context_randomize for details. + * + * Do not create a new context object for each operation, as construction and + * randomization can take non-negligible time. */ -SECP256K1_API secp256k1_context* secp256k1_context_create( +SECP256K1_API secp256k1_context *secp256k1_context_create( unsigned int flags ) SECP256K1_WARN_UNUSED_RESULT; @@ -209,11 +290,14 @@ SECP256K1_API secp256k1_context* secp256k1_context_create( * called at most once for every call of this function. If you need to avoid dynamic * memory allocation entirely, see the functions in secp256k1_preallocated.h. * - * Returns: a newly created context object. - * Args: ctx: an existing context to copy (cannot be NULL) + * Cloning secp256k1_context_static is not possible, and should not be emulated by + * the caller (e.g., using memcpy). Create a new context instead. + * + * Returns: pointer to a newly created context object. + * Args: ctx: pointer to a context to copy (not secp256k1_context_static). */ -SECP256K1_API secp256k1_context* secp256k1_context_clone( - const secp256k1_context* ctx +SECP256K1_API secp256k1_context *secp256k1_context_clone( + const secp256k1_context *ctx ) SECP256K1_ARG_NONNULL(1) SECP256K1_WARN_UNUSED_RESULT; /** Destroy a secp256k1 context object (created in dynamically allocated memory). @@ -226,12 +310,13 @@ SECP256K1_API secp256k1_context* secp256k1_context_clone( * behaviour is undefined. In that case, secp256k1_context_preallocated_destroy must * be used instead. * - * Args: ctx: an existing context to destroy, constructed using + * Args: ctx: pointer to a context to destroy, constructed using * secp256k1_context_create or secp256k1_context_clone + * (i.e., not secp256k1_context_static). */ SECP256K1_API void secp256k1_context_destroy( - secp256k1_context* ctx -); + secp256k1_context *ctx +) SECP256K1_ARG_NONNULL(1); /** Set a callback function to be called when an illegal argument is passed to * an API call. It will only trigger for violations that are mentioned @@ -248,36 +333,39 @@ SECP256K1_API void secp256k1_context_destroy( * undefined. * * When this function has not been called (or called with fn==NULL), then the - * default handler will be used. The library provides a default handler which + * default handler will be used. The library provides a default handler which * writes the message to stderr and calls abort. This default handler can be * replaced at link time if the preprocessor macro * USE_EXTERNAL_DEFAULT_CALLBACKS is defined, which is the case if the build * has been configured with --enable-external-default-callbacks. Then the * following two symbols must be provided to link against: - * - void secp256k1_default_illegal_callback_fn(const char* message, void* data); - * - void secp256k1_default_error_callback_fn(const char* message, void* data); + * - void secp256k1_default_illegal_callback_fn(const char *message, void *data); + * - void secp256k1_default_error_callback_fn(const char *message, void *data); * The library can call these default handlers even before a proper callback data * pointer could have been set using secp256k1_context_set_illegal_callback or * secp256k1_context_set_error_callback, e.g., when the creation of a context * fails. In this case, the corresponding default handler will be called with * the data pointer argument set to NULL. * - * Args: ctx: an existing context object (cannot be NULL) - * In: fun: a pointer to a function to call when an illegal argument is + * Args: ctx: pointer to a context object. + * In: fun: pointer to a function to call when an illegal argument is * passed to the API, taking a message and an opaque pointer. * (NULL restores the default handler.) - * data: the opaque pointer to pass to fun above. + * data: the opaque pointer to pass to fun above, must be NULL for the default handler. * * See also secp256k1_context_set_error_callback. */ SECP256K1_API void secp256k1_context_set_illegal_callback( - secp256k1_context* ctx, - void (*fun)(const char* message, void* data), - const void* data + secp256k1_context *ctx, + void (*fun)(const char *message, void *data), + const void *data ) SECP256K1_ARG_NONNULL(1); /** Set a callback function to be called when an internal consistency check - * fails. The default is crashing. + * fails. + * + * The default callback writes an error message to stderr and calls abort + * to abort the program. * * This can only trigger in case of a hardware failure, miscompilation, * memory corruption, serious bug in the library, or other error would can @@ -286,49 +374,26 @@ SECP256K1_API void secp256k1_context_set_illegal_callback( * for that). After this callback returns, anything may happen, including * crashing. * - * Args: ctx: an existing context object (cannot be NULL) - * In: fun: a pointer to a function to call when an internal error occurs, + * Args: ctx: pointer to a context object. + * In: fun: pointer to a function to call when an internal error occurs, * taking a message and an opaque pointer (NULL restores the * default handler, see secp256k1_context_set_illegal_callback * for details). - * data: the opaque pointer to pass to fun above. + * data: the opaque pointer to pass to fun above, must be NULL for the default handler. * * See also secp256k1_context_set_illegal_callback. */ SECP256K1_API void secp256k1_context_set_error_callback( - secp256k1_context* ctx, - void (*fun)(const char* message, void* data), - const void* data -) SECP256K1_ARG_NONNULL(1); - -/** Create a secp256k1 scratch space object. - * - * Returns: a newly created scratch space. - * Args: ctx: an existing context object (cannot be NULL) - * In: size: amount of memory to be available as scratch space. Some extra - * (<100 bytes) will be allocated for extra accounting. - */ -SECP256K1_API SECP256K1_WARN_UNUSED_RESULT secp256k1_scratch_space* secp256k1_scratch_space_create( - const secp256k1_context* ctx, - size_t size -) SECP256K1_ARG_NONNULL(1); - -/** Destroy a secp256k1 scratch space. - * - * The pointer may not be used afterwards. - * Args: ctx: a secp256k1 context object. - * scratch: space to destroy - */ -SECP256K1_API void secp256k1_scratch_space_destroy( - const secp256k1_context* ctx, - secp256k1_scratch_space* scratch + secp256k1_context *ctx, + void (*fun)(const char *message, void *data), + const void *data ) SECP256K1_ARG_NONNULL(1); /** Parse a variable-length public key into the pubkey object. * * Returns: 1 if the public key was fully valid. * 0 if the public key could not be parsed or is invalid. - * Args: ctx: a secp256k1 context object. + * Args: ctx: pointer to a context object. * Out: pubkey: pointer to a pubkey object. If 1 is returned, it is set to a * parsed version of input. If not, its value is undefined. * In: input: pointer to a serialized public key @@ -339,8 +404,8 @@ SECP256K1_API void secp256k1_scratch_space_destroy( * byte 0x06 or 0x07) format public keys. */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_parse( - const secp256k1_context* ctx, - secp256k1_pubkey* pubkey, + const secp256k1_context *ctx, + secp256k1_pubkey *pubkey, const unsigned char *input, size_t inputlen ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); @@ -348,65 +413,94 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_parse( /** Serialize a pubkey object into a serialized byte sequence. * * Returns: 1 always. - * Args: ctx: a secp256k1 context object. - * Out: output: a pointer to a 65-byte (if compressed==0) or 33-byte (if + * Args: ctx: pointer to a context object. + * Out: output: pointer to a 65-byte (if compressed==0) or 33-byte (if * compressed==1) byte array to place the serialized key * in. - * In/Out: outputlen: a pointer to an integer which is initially set to the + * In/Out: outputlen: pointer to an integer which is initially set to the * size of output, and is overwritten with the written * size. - * In: pubkey: a pointer to a secp256k1_pubkey containing an + * In: pubkey: pointer to a secp256k1_pubkey containing an * initialized public key. * flags: SECP256K1_EC_COMPRESSED if serialization should be in * compressed format, otherwise SECP256K1_EC_UNCOMPRESSED. */ SECP256K1_API int secp256k1_ec_pubkey_serialize( - const secp256k1_context* ctx, + const secp256k1_context *ctx, unsigned char *output, size_t *outputlen, - const secp256k1_pubkey* pubkey, + const secp256k1_pubkey *pubkey, unsigned int flags ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); +/** Compare two public keys using lexicographic (of compressed serialization) order + * + * Returns: <0 if the first public key is less than the second + * >0 if the first public key is greater than the second + * 0 if the two public keys are equal + * Args: ctx: pointer to a context object + * In: pubkey1: first public key to compare + * pubkey2: second public key to compare + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_cmp( + const secp256k1_context *ctx, + const secp256k1_pubkey *pubkey1, + const secp256k1_pubkey *pubkey2 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Sort public keys using lexicographic (of compressed serialization) order + * + * Returns: 0 if the arguments are invalid. 1 otherwise. + * + * Args: ctx: pointer to a context object + * In: pubkeys: array of pointers to pubkeys to sort + * n_pubkeys: number of elements in the pubkeys array + */ +SECP256K1_API int secp256k1_ec_pubkey_sort( + const secp256k1_context *ctx, + const secp256k1_pubkey **pubkeys, + size_t n_pubkeys +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); + /** Parse an ECDSA signature in compact (64 bytes) format. * * Returns: 1 when the signature could be parsed, 0 otherwise. - * Args: ctx: a secp256k1 context object - * Out: sig: a pointer to a signature object - * In: input64: a pointer to the 64-byte array to parse + * Args: ctx: pointer to a context object + * Out: sig: pointer to a signature object + * In: input64: pointer to the 64-byte array to parse * * The signature must consist of a 32-byte big endian R value, followed by a * 32-byte big endian S value. If R or S fall outside of [0..order-1], the * encoding is invalid. R and S with value 0 are allowed in the encoding. * * After the call, sig will always be initialized. If parsing failed or R or - * S are zero, the resulting sig value is guaranteed to fail validation for any - * message and public key. + * S are zero, the resulting sig value is guaranteed to fail verification for + * any message and public key. */ SECP256K1_API int secp256k1_ecdsa_signature_parse_compact( - const secp256k1_context* ctx, - secp256k1_ecdsa_signature* sig, + const secp256k1_context *ctx, + secp256k1_ecdsa_signature *sig, const unsigned char *input64 ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); /** Parse a DER ECDSA signature. * * Returns: 1 when the signature could be parsed, 0 otherwise. - * Args: ctx: a secp256k1 context object - * Out: sig: a pointer to a signature object - * In: input: a pointer to the signature to be parsed + * Args: ctx: pointer to a context object + * Out: sig: pointer to a signature object + * In: input: pointer to the signature to be parsed * inputlen: the length of the array pointed to be input * * This function will accept any valid DER encoded signature, even if the * encoded numbers are out of range. * * After the call, sig will always be initialized. If parsing failed or the - * encoded numbers are out of range, signature validation with it is + * encoded numbers are out of range, signature verification with it is * guaranteed to fail for every message and public key. */ SECP256K1_API int secp256k1_ecdsa_signature_parse_der( - const secp256k1_context* ctx, - secp256k1_ecdsa_signature* sig, + const secp256k1_context *ctx, + secp256k1_ecdsa_signature *sig, const unsigned char *input, size_t inputlen ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); @@ -414,71 +508,77 @@ SECP256K1_API int secp256k1_ecdsa_signature_parse_der( /** Serialize an ECDSA signature in DER format. * * Returns: 1 if enough space was available to serialize, 0 otherwise - * Args: ctx: a secp256k1 context object - * Out: output: a pointer to an array to store the DER serialization - * In/Out: outputlen: a pointer to a length integer. Initially, this integer + * Args: ctx: pointer to a context object + * Out: output: pointer to an array to store the DER serialization + * In/Out: outputlen: pointer to a length integer. Initially, this integer * should be set to the length of output. After the call * it will be set to the length of the serialization (even * if 0 was returned). - * In: sig: a pointer to an initialized signature object + * In: sig: pointer to an initialized signature object */ SECP256K1_API int secp256k1_ecdsa_signature_serialize_der( - const secp256k1_context* ctx, + const secp256k1_context *ctx, unsigned char *output, size_t *outputlen, - const secp256k1_ecdsa_signature* sig + const secp256k1_ecdsa_signature *sig ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); /** Serialize an ECDSA signature in compact (64 byte) format. * * Returns: 1 - * Args: ctx: a secp256k1 context object - * Out: output64: a pointer to a 64-byte array to store the compact serialization - * In: sig: a pointer to an initialized signature object + * Args: ctx: pointer to a context object + * Out: output64: pointer to a 64-byte array to store the compact serialization + * In: sig: pointer to an initialized signature object * * See secp256k1_ecdsa_signature_parse_compact for details about the encoding. */ SECP256K1_API int secp256k1_ecdsa_signature_serialize_compact( - const secp256k1_context* ctx, + const secp256k1_context *ctx, unsigned char *output64, - const secp256k1_ecdsa_signature* sig + const secp256k1_ecdsa_signature *sig ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); /** Verify an ECDSA signature. * * Returns: 1: correct signature * 0: incorrect or unparseable signature - * Args: ctx: a secp256k1 context object, initialized for verification. - * In: sig: the signature being verified (cannot be NULL) - * msg32: the 32-byte message hash being verified (cannot be NULL) - * pubkey: pointer to an initialized public key to verify with (cannot be NULL) + * Args: ctx: pointer to a context object + * In: sig: the signature being verified. + * msghash32: the 32-byte message hash being verified. + * The verifier must make sure to apply a cryptographic + * hash function to the message by itself and not accept an + * msghash32 value directly. Otherwise, it would be easy to + * create a "valid" signature without knowledge of the + * secret key. See also + * https://bitcoin.stackexchange.com/a/81116/35586 for more + * background on this topic. + * pubkey: pointer to an initialized public key to verify with. * * To avoid accepting malleable signatures, only ECDSA signatures in lower-S * form are accepted. * * If you need to accept ECDSA signatures from sources that do not obey this * rule, apply secp256k1_ecdsa_signature_normalize to the signature prior to - * validation, but be aware that doing so results in malleable signatures. + * verification, but be aware that doing so results in malleable signatures. * * For details, see the comments for that function. */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ecdsa_verify( - const secp256k1_context* ctx, + const secp256k1_context *ctx, const secp256k1_ecdsa_signature *sig, - const unsigned char *msg32, + const unsigned char *msghash32, const secp256k1_pubkey *pubkey ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); /** Convert a signature to a normalized lower-S form. * * Returns: 1 if sigin was not normalized, 0 if it already was. - * Args: ctx: a secp256k1 context object - * Out: sigout: a pointer to a signature to fill with the normalized form, + * Args: ctx: pointer to a context object + * Out: sigout: pointer to a signature to fill with the normalized form, * or copy if the input was already normalized. (can be NULL if * you're only interested in whether the input was already * normalized). - * In: sigin: a pointer to a signature to check/normalize (cannot be NULL, - * can be identical to sigout) + * In: sigin: pointer to a signature to check/normalize (can be identical to sigout) * * With ECDSA a third-party can forge a second distinct signature of the same * message, given a single initial signature, but without knowing the key. This @@ -512,7 +612,7 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ecdsa_verify( * secp256k1_ecdsa_signature_normalize must be called before verification. */ SECP256K1_API int secp256k1_ecdsa_signature_normalize( - const secp256k1_context* ctx, + const secp256k1_context *ctx, secp256k1_ecdsa_signature *sigout, const secp256k1_ecdsa_signature *sigin ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(3); @@ -521,185 +621,276 @@ SECP256K1_API int secp256k1_ecdsa_signature_normalize( * If a data pointer is passed, it is assumed to be a pointer to 32 bytes of * extra entropy. */ -SECP256K1_API extern const secp256k1_nonce_function secp256k1_nonce_function_rfc6979; +SECP256K1_API const secp256k1_nonce_function secp256k1_nonce_function_rfc6979; /** A default safe nonce generation function (currently equal to secp256k1_nonce_function_rfc6979). */ -SECP256K1_API extern const secp256k1_nonce_function secp256k1_nonce_function_default; +SECP256K1_API const secp256k1_nonce_function secp256k1_nonce_function_default; /** Create an ECDSA signature. * * Returns: 1: signature created - * 0: the nonce generation function failed, or the private key was invalid. - * Args: ctx: pointer to a context object, initialized for signing (cannot be NULL) - * Out: sig: pointer to an array where the signature will be placed (cannot be NULL) - * In: msg32: the 32-byte message hash being signed (cannot be NULL) - * seckey: pointer to a 32-byte secret key (cannot be NULL) - * noncefp:pointer to a nonce generation function. If NULL, secp256k1_nonce_function_default is used - * ndata: pointer to arbitrary data used by the nonce generation function (can be NULL) + * 0: the nonce generation function failed, or the secret key was invalid. + * Args: ctx: pointer to a context object (not secp256k1_context_static). + * Out: sig: pointer to an array where the signature will be placed. + * In: msghash32: the 32-byte message hash being signed. + * seckey: pointer to a 32-byte secret key. + * noncefp: pointer to a nonce generation function. If NULL, + * secp256k1_nonce_function_default is used. + * ndata: pointer to arbitrary data used by the nonce generation function + * (can be NULL). If it is non-NULL and + * secp256k1_nonce_function_default is used, then ndata must be a + * pointer to 32-bytes of additional data. * * The created signature is always in lower-S form. See * secp256k1_ecdsa_signature_normalize for more details. */ SECP256K1_API int secp256k1_ecdsa_sign( - const secp256k1_context* ctx, + const secp256k1_context *ctx, secp256k1_ecdsa_signature *sig, - const unsigned char *msg32, + const unsigned char *msghash32, const unsigned char *seckey, secp256k1_nonce_function noncefp, const void *ndata ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); -/** Verify an ECDSA secret key. +/** Verify an elliptic curve secret key. + * + * A secret key is valid if it is not 0 and less than the secp256k1 curve order + * when interpreted as an integer (most significant byte first). The + * probability of choosing a 32-byte string uniformly at random which is an + * invalid secret key is negligible. However, if it does happen it should + * be assumed that the randomness source is severely broken and there should + * be no retry. * * Returns: 1: secret key is valid * 0: secret key is invalid - * Args: ctx: pointer to a context object (cannot be NULL) - * In: seckey: pointer to a 32-byte secret key (cannot be NULL) + * Args: ctx: pointer to a context object. + * In: seckey: pointer to a 32-byte secret key. */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_seckey_verify( - const secp256k1_context* ctx, + const secp256k1_context *ctx, const unsigned char *seckey ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); /** Compute the public key for a secret key. * - * Returns: 1: secret was valid, public key stores - * 0: secret was invalid, try again - * Args: ctx: pointer to a context object, initialized for signing (cannot be NULL) - * Out: pubkey: pointer to the created public key (cannot be NULL) - * In: seckey: pointer to a 32-byte private key (cannot be NULL) + * Returns: 1: secret was valid, public key stores. + * 0: secret was invalid, try again. + * Args: ctx: pointer to a context object (not secp256k1_context_static). + * Out: pubkey: pointer to the created public key. + * In: seckey: pointer to a 32-byte secret key. */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_create( - const secp256k1_context* ctx, + const secp256k1_context *ctx, secp256k1_pubkey *pubkey, const unsigned char *seckey ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); -/** Negates a private key in place. +/** Negates a secret key in place. * - * Returns: 1 always - * Args: ctx: pointer to a context object - * In/Out: seckey: pointer to the 32-byte private key to be negated (cannot be NULL) + * Returns: 0 if the given secret key is invalid according to + * secp256k1_ec_seckey_verify. 1 otherwise + * Args: ctx: pointer to a context object + * In/Out: seckey: pointer to the 32-byte secret key to be negated. If the + * secret key is invalid according to + * secp256k1_ec_seckey_verify, this function returns 0 and + * seckey will be set to some unspecified value. */ -SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_negate( - const secp256k1_context* ctx, +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_seckey_negate( + const secp256k1_context *ctx, unsigned char *seckey ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); +/** Same as secp256k1_ec_seckey_negate, but DEPRECATED. Will be removed in + * future versions. */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_negate( + const secp256k1_context *ctx, + unsigned char *seckey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) + SECP256K1_DEPRECATED("Use secp256k1_ec_seckey_negate instead"); + /** Negates a public key in place. * * Returns: 1 always * Args: ctx: pointer to a context object - * In/Out: pubkey: pointer to the public key to be negated (cannot be NULL) + * In/Out: pubkey: pointer to the public key to be negated. */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_negate( - const secp256k1_context* ctx, + const secp256k1_context *ctx, secp256k1_pubkey *pubkey ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); -/** Tweak a private key by adding tweak to it. - * Returns: 0 if the tweak was out of range (chance of around 1 in 2^128 for - * uniformly random 32-byte arrays, or if the resulting private key - * would be invalid (only when the tweak is the complement of the - * private key). 1 otherwise. - * Args: ctx: pointer to a context object (cannot be NULL). - * In/Out: seckey: pointer to a 32-byte private key. - * In: tweak: pointer to a 32-byte tweak. +/** Tweak a secret key by adding tweak to it. + * + * Returns: 0 if the arguments are invalid or the resulting secret key would be + * invalid (only when the tweak is the negation of the secret key). 1 + * otherwise. + * Args: ctx: pointer to a context object. + * In/Out: seckey: pointer to a 32-byte secret key. If the secret key is + * invalid according to secp256k1_ec_seckey_verify, this + * function returns 0. seckey will be set to some unspecified + * value if this function returns 0. + * In: tweak32: pointer to a 32-byte tweak, which must be valid according to + * secp256k1_ec_seckey_verify or 32 zero bytes. For uniformly + * random 32-byte tweaks, the chance of being invalid is + * negligible (around 1 in 2^128). */ -SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_tweak_add( - const secp256k1_context* ctx, +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_seckey_tweak_add( + const secp256k1_context *ctx, unsigned char *seckey, - const unsigned char *tweak + const unsigned char *tweak32 ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); +/** Same as secp256k1_ec_seckey_tweak_add, but DEPRECATED. Will be removed in + * future versions. */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_tweak_add( + const secp256k1_context *ctx, + unsigned char *seckey, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) + SECP256K1_DEPRECATED("Use secp256k1_ec_seckey_tweak_add instead"); + /** Tweak a public key by adding tweak times the generator to it. - * Returns: 0 if the tweak was out of range (chance of around 1 in 2^128 for - * uniformly random 32-byte arrays, or if the resulting public key - * would be invalid (only when the tweak is the complement of the - * corresponding private key). 1 otherwise. - * Args: ctx: pointer to a context object initialized for validation - * (cannot be NULL). - * In/Out: pubkey: pointer to a public key object. - * In: tweak: pointer to a 32-byte tweak. + * + * Returns: 0 if the arguments are invalid or the resulting public key would be + * invalid (only when the tweak is the negation of the corresponding + * secret key). 1 otherwise. + * Args: ctx: pointer to a context object. + * In/Out: pubkey: pointer to a public key object. pubkey will be set to an + * invalid value if this function returns 0. + * In: tweak32: pointer to a 32-byte tweak, which must be valid according to + * secp256k1_ec_seckey_verify or 32 zero bytes. For uniformly + * random 32-byte tweaks, the chance of being invalid is + * negligible (around 1 in 2^128). */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_tweak_add( - const secp256k1_context* ctx, + const secp256k1_context *ctx, secp256k1_pubkey *pubkey, - const unsigned char *tweak + const unsigned char *tweak32 ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); -/** Tweak a private key by multiplying it by a tweak. - * Returns: 0 if the tweak was out of range (chance of around 1 in 2^128 for - * uniformly random 32-byte arrays, or equal to zero. 1 otherwise. - * Args: ctx: pointer to a context object (cannot be NULL). - * In/Out: seckey: pointer to a 32-byte private key. - * In: tweak: pointer to a 32-byte tweak. +/** Tweak a secret key by multiplying it by a tweak. + * + * Returns: 0 if the arguments are invalid. 1 otherwise. + * Args: ctx: pointer to a context object. + * In/Out: seckey: pointer to a 32-byte secret key. If the secret key is + * invalid according to secp256k1_ec_seckey_verify, this + * function returns 0. seckey will be set to some unspecified + * value if this function returns 0. + * In: tweak32: pointer to a 32-byte tweak. If the tweak is invalid according to + * secp256k1_ec_seckey_verify, this function returns 0. For + * uniformly random 32-byte arrays the chance of being invalid + * is negligible (around 1 in 2^128). */ -SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_tweak_mul( - const secp256k1_context* ctx, +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_seckey_tweak_mul( + const secp256k1_context *ctx, unsigned char *seckey, - const unsigned char *tweak + const unsigned char *tweak32 ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); +/** Same as secp256k1_ec_seckey_tweak_mul, but DEPRECATED. Will be removed in + * future versions. */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_tweak_mul( + const secp256k1_context *ctx, + unsigned char *seckey, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) + SECP256K1_DEPRECATED("Use secp256k1_ec_seckey_tweak_mul instead"); + /** Tweak a public key by multiplying it by a tweak value. - * Returns: 0 if the tweak was out of range (chance of around 1 in 2^128 for - * uniformly random 32-byte arrays, or equal to zero. 1 otherwise. - * Args: ctx: pointer to a context object initialized for validation - * (cannot be NULL). - * In/Out: pubkey: pointer to a public key object. - * In: tweak: pointer to a 32-byte tweak. + * + * Returns: 0 if the arguments are invalid. 1 otherwise. + * Args: ctx: pointer to a context object. + * In/Out: pubkey: pointer to a public key object. pubkey will be set to an + * invalid value if this function returns 0. + * In: tweak32: pointer to a 32-byte tweak. If the tweak is invalid according to + * secp256k1_ec_seckey_verify, this function returns 0. For + * uniformly random 32-byte arrays the chance of being invalid + * is negligible (around 1 in 2^128). */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_tweak_mul( - const secp256k1_context* ctx, + const secp256k1_context *ctx, secp256k1_pubkey *pubkey, - const unsigned char *tweak + const unsigned char *tweak32 ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); -/** Updates the context randomization to protect against side-channel leakage. - * Returns: 1: randomization successfully updated or nothing to randomize +/** Randomizes the context to provide enhanced protection against side-channel leakage. + * + * Returns: 1: randomization successful * 0: error - * Args: ctx: pointer to a context object (cannot be NULL) - * In: seed32: pointer to a 32-byte random seed (NULL resets to initial state) + * Args: ctx: pointer to a context object (not secp256k1_context_static). + * In: seed32: pointer to a 32-byte random seed (NULL resets to initial state). * - * While secp256k1 code is written to be constant-time no matter what secret - * values are, it's possible that a future compiler may output code which isn't, + * While secp256k1 code is written and tested to be constant-time no matter what + * secret values are, it is possible that a compiler may output code which is not, * and also that the CPU may not emit the same radio frequencies or draw the same - * amount power for all values. + * amount of power for all values. Randomization of the context shields against + * side-channel observations which aim to exploit secret-dependent behaviour in + * certain computations which involve secret keys. * - * This function provides a seed which is combined into the blinding value: that - * blinding value is added before each multiplication (and removed afterwards) so - * that it does not affect function results, but shields against attacks which - * rely on any input-dependent behaviour. + * It is highly recommended to call this function on contexts returned from + * secp256k1_context_create or secp256k1_context_clone (or from the corresponding + * functions in secp256k1_preallocated.h) before using these contexts to call API + * functions that perform computations involving secret keys, e.g., signing and + * public key generation. It is possible to call this function more than once on + * the same context, and doing so before every few computations involving secret + * keys is recommended as a defense-in-depth measure. Randomization of the static + * context secp256k1_context_static is not supported. * - * This function has currently an effect only on contexts initialized for signing - * because randomization is currently used only for signing. However, this is not - * guaranteed and may change in the future. It is safe to call this function on - * contexts not initialized for signing; then it will have no effect and return 1. - * - * You should call this after secp256k1_context_create or - * secp256k1_context_clone (and secp256k1_context_preallocated_create or - * secp256k1_context_clone, resp.), and you may call this repeatedly afterwards. + * Currently, the random seed is mainly used for blinding multiplications of a + * secret scalar with the elliptic curve base point. Multiplications of this + * kind are performed by exactly those API functions which are documented to + * require a context that is not secp256k1_context_static. As a rule of thumb, + * these are all functions which take a secret key (or a keypair) as an input. + * A notable exception to that rule is the ECDH module, which relies on a different + * kind of elliptic curve point multiplication and thus does not benefit from + * enhanced protection against side-channel leakage currently. */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_context_randomize( - secp256k1_context* ctx, + secp256k1_context *ctx, const unsigned char *seed32 ) SECP256K1_ARG_NONNULL(1); /** Add a number of public keys together. + * * Returns: 1: the sum of the public keys is valid. * 0: the sum of the public keys is not valid. - * Args: ctx: pointer to a context object - * Out: out: pointer to a public key object for placing the resulting public key - * (cannot be NULL) - * In: ins: pointer to array of pointers to public keys (cannot be NULL) - * n: the number of public keys to add together (must be at least 1) + * Args: ctx: pointer to a context object. + * Out: out: pointer to a public key object for placing the resulting public key. + * In: ins: pointer to array of pointers to public keys. + * n: the number of public keys to add together (must be at least 1). */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_combine( - const secp256k1_context* ctx, + const secp256k1_context *ctx, secp256k1_pubkey *out, - const secp256k1_pubkey * const * ins, + const secp256k1_pubkey * const *ins, size_t n -) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Compute a tagged hash as defined in BIP-340. + * + * This is useful for creating a message hash and achieving domain separation + * through an application-specific tag. This function returns + * SHA256(SHA256(tag)||SHA256(tag)||msg). Therefore, tagged hash + * implementations optimized for a specific tag can precompute the SHA256 state + * after hashing the tag hashes. + * + * Returns: 1 always. + * Args: ctx: pointer to a context object + * Out: hash32: pointer to a 32-byte array to store the resulting hash + * In: tag: pointer to an array containing the tag + * taglen: length of the tag array + * msg: pointer to an array containing the message + * msglen: length of the message array + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_tagged_sha256( + const secp256k1_context *ctx, + unsigned char *hash32, + const unsigned char *tag, + size_t taglen, + const unsigned char *msg, + size_t msglen +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(5); #ifdef __cplusplus } diff --git a/example/android/third_party/secp256k1/include/secp256k1_ecdh.h b/example/android/third_party/secp256k1/include/secp256k1_ecdh.h index df5fde23..4d9da346 100644 --- a/example/android/third_party/secp256k1/include/secp256k1_ecdh.h +++ b/example/android/third_party/secp256k1/include/secp256k1_ecdh.h @@ -7,43 +7,51 @@ extern "C" { #endif -/** A pointer to a function that applies hash function to a point +/** A pointer to a function that hashes an EC point to obtain an ECDH secret * - * Returns: 1 if a point was successfully hashed. 0 will cause ecdh to fail - * Out: output: pointer to an array to be filled by the function - * In: x: pointer to a 32-byte x coordinate - * y: pointer to a 32-byte y coordinate - * data: Arbitrary data pointer that is passed through + * Returns: 1 if the point was successfully hashed. + * 0 will cause secp256k1_ecdh to fail and return 0. + * Other return values are not allowed, and the behaviour of + * secp256k1_ecdh is undefined for other return values. + * Out: output: pointer to an array to be filled by the function + * In: x32: pointer to a 32-byte x coordinate + * y32: pointer to a 32-byte y coordinate + * data: arbitrary data pointer that is passed through */ typedef int (*secp256k1_ecdh_hash_function)( unsigned char *output, - const unsigned char *x, - const unsigned char *y, + const unsigned char *x32, + const unsigned char *y32, void *data ); -/** An implementation of SHA256 hash function that applies to compressed public key. */ -SECP256K1_API extern const secp256k1_ecdh_hash_function secp256k1_ecdh_hash_function_sha256; +/** An implementation of SHA256 hash function that applies to compressed public key. + * Populates the output parameter with 32 bytes. */ +SECP256K1_API const secp256k1_ecdh_hash_function secp256k1_ecdh_hash_function_sha256; -/** A default ecdh hash function (currently equal to secp256k1_ecdh_hash_function_sha256). */ -SECP256K1_API extern const secp256k1_ecdh_hash_function secp256k1_ecdh_hash_function_default; +/** A default ECDH hash function (currently equal to secp256k1_ecdh_hash_function_sha256). + * Populates the output parameter with 32 bytes. */ +SECP256K1_API const secp256k1_ecdh_hash_function secp256k1_ecdh_hash_function_default; /** Compute an EC Diffie-Hellman secret in constant time + * * Returns: 1: exponentiation was successful - * 0: scalar was invalid (zero or overflow) - * Args: ctx: pointer to a context object (cannot be NULL) - * Out: output: pointer to an array to be filled by the function - * In: pubkey: a pointer to a secp256k1_pubkey containing an - * initialized public key - * privkey: a 32-byte scalar with which to multiply the point - * hashfp: pointer to a hash function. If NULL, secp256k1_ecdh_hash_function_sha256 is used - * data: Arbitrary data pointer that is passed through + * 0: scalar was invalid (zero or overflow) or hashfp returned 0 + * Args: ctx: pointer to a context object. + * Out: output: pointer to an array to be filled by hashfp. + * In: pubkey: pointer to a secp256k1_pubkey containing an initialized public key. + * seckey: a 32-byte scalar with which to multiply the point. + * hashfp: pointer to a hash function. If NULL, + * secp256k1_ecdh_hash_function_sha256 is used + * (in which case, 32 bytes will be written to output). + * data: arbitrary data pointer that is passed through to hashfp + * (can be NULL for secp256k1_ecdh_hash_function_sha256). */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ecdh( - const secp256k1_context* ctx, + const secp256k1_context *ctx, unsigned char *output, const secp256k1_pubkey *pubkey, - const unsigned char *privkey, + const unsigned char *seckey, secp256k1_ecdh_hash_function hashfp, void *data ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); diff --git a/example/android/third_party/secp256k1/include/secp256k1_ellswift.h b/example/android/third_party/secp256k1/include/secp256k1_ellswift.h new file mode 100644 index 00000000..0d1293e9 --- /dev/null +++ b/example/android/third_party/secp256k1/include/secp256k1_ellswift.h @@ -0,0 +1,200 @@ +#ifndef SECP256K1_ELLSWIFT_H +#define SECP256K1_ELLSWIFT_H + +#include "secp256k1.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* This module provides an implementation of ElligatorSwift as well as a + * version of x-only ECDH using it (including compatibility with BIP324). + * + * ElligatorSwift is described in https://eprint.iacr.org/2022/759 by + * Chavez-Saab, Rodriguez-Henriquez, and Tibouchi. It permits encoding + * uniformly chosen public keys as 64-byte arrays which are indistinguishable + * from uniformly random arrays. + * + * Let f be the function from pairs of field elements to point X coordinates, + * defined as follows (all operations modulo p = 2^256 - 2^32 - 977) + * f(u,t): + * - Let C = 0xa2d2ba93507f1df233770c2a797962cc61f6d15da14ecd47d8d27ae1cd5f852, + * a square root of -3. + * - If u=0, set u=1 instead. + * - If t=0, set t=1 instead. + * - If u^3 + t^2 + 7 = 0, multiply t by 2. + * - Let X = (u^3 + 7 - t^2) / (2 * t) + * - Let Y = (X + t) / (C * u) + * - Return the first in [u + 4 * Y^2, (-X/Y - u) / 2, (X/Y - u) / 2] that is an + * X coordinate on the curve (at least one of them is, for any u and t). + * + * Then an ElligatorSwift encoding of x consists of the 32-byte big-endian + * encodings of field elements u and t concatenated, where f(u,t) = x. + * The encoding algorithm is described in the paper, and effectively picks a + * uniformly random pair (u,t) among those which encode x. + * + * If the Y coordinate is relevant, it is given the same parity as t. + * + * Changes w.r.t. the paper: + * - The u=0, t=0, and u^3+t^2+7=0 conditions result in decoding to the point + * at infinity in the paper. Here they are remapped to finite points. + * - The paper uses an additional encoding bit for the parity of y. Here the + * parity of t is used (negating t does not affect the decoded x coordinate, + * so this is possible). + * + * For mathematical background about the scheme, see the doc/ellswift.md file. + */ + +/** A pointer to a function used by secp256k1_ellswift_xdh to hash the shared X + * coordinate along with the encoded public keys to a uniform shared secret. + * + * Returns: 1 if a shared secret was successfully computed. + * 0 will cause secp256k1_ellswift_xdh to fail and return 0. + * Other return values are not allowed, and the behaviour of + * secp256k1_ellswift_xdh is undefined for other return values. + * Out: output: pointer to an array to be filled by the function + * In: x32: pointer to the 32-byte serialized X coordinate + * of the resulting shared point (will not be NULL) + * ell_a64: pointer to the 64-byte encoded public key of party A + * (will not be NULL) + * ell_b64: pointer to the 64-byte encoded public key of party B + * (will not be NULL) + * data: arbitrary data pointer that is passed through + */ +typedef int (*secp256k1_ellswift_xdh_hash_function)( + unsigned char *output, + const unsigned char *x32, + const unsigned char *ell_a64, + const unsigned char *ell_b64, + void *data +); + +/** An implementation of an secp256k1_ellswift_xdh_hash_function which uses + * SHA256(prefix64 || ell_a64 || ell_b64 || x32), where prefix64 is the 64-byte + * array pointed to by data. */ +SECP256K1_API const secp256k1_ellswift_xdh_hash_function secp256k1_ellswift_xdh_hash_function_prefix; + +/** An implementation of an secp256k1_ellswift_xdh_hash_function compatible with + * BIP324. It returns H_tag(ell_a64 || ell_b64 || x32), where H_tag is the + * BIP340 tagged hash function with tag "bip324_ellswift_xonly_ecdh". Equivalent + * to secp256k1_ellswift_xdh_hash_function_prefix with prefix64 set to + * SHA256("bip324_ellswift_xonly_ecdh")||SHA256("bip324_ellswift_xonly_ecdh"). + * The data argument is ignored. */ +SECP256K1_API const secp256k1_ellswift_xdh_hash_function secp256k1_ellswift_xdh_hash_function_bip324; + +/** Construct a 64-byte ElligatorSwift encoding of a given pubkey. + * + * Returns: 1 always. + * Args: ctx: pointer to a context object + * Out: ell64: pointer to a 64-byte array to be filled + * In: pubkey: pointer to a secp256k1_pubkey containing an + * initialized public key + * rnd32: pointer to 32 bytes of randomness + * + * It is recommended that rnd32 consists of 32 uniformly random bytes, not + * known to any adversary trying to detect whether public keys are being + * encoded, though 16 bytes of randomness (padded to an array of 32 bytes, + * e.g., with zeros) suffice to make the result indistinguishable from + * uniform. The randomness in rnd32 must not be a deterministic function of + * the pubkey (it can be derived from the private key, though). + * + * It is not guaranteed that the computed encoding is stable across versions + * of the library, even if all arguments to this function (including rnd32) + * are the same. + * + * This function runs in variable time. + */ +SECP256K1_API int secp256k1_ellswift_encode( + const secp256k1_context *ctx, + unsigned char *ell64, + const secp256k1_pubkey *pubkey, + const unsigned char *rnd32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +/** Decode a 64-bytes ElligatorSwift encoded public key. + * + * Returns: always 1 + * Args: ctx: pointer to a context object + * Out: pubkey: pointer to a secp256k1_pubkey that will be filled + * In: ell64: pointer to a 64-byte array to decode + * + * This function runs in variable time. + */ +SECP256K1_API int secp256k1_ellswift_decode( + const secp256k1_context *ctx, + secp256k1_pubkey *pubkey, + const unsigned char *ell64 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Compute an ElligatorSwift public key for a secret key. + * + * Returns: 1: secret was valid, public key was stored. + * 0: secret was invalid, try again. + * Args: ctx: pointer to a context object + * Out: ell64: pointer to a 64-byte array to receive the ElligatorSwift + * public key + * In: seckey32: pointer to a 32-byte secret key + * auxrnd32: (optional) pointer to 32 bytes of randomness + * + * Constant time in seckey and auxrnd32, but not in the resulting public key. + * + * It is recommended that auxrnd32 contains 32 uniformly random bytes, though + * it is optional (and does result in encodings that are indistinguishable from + * uniform even without any auxrnd32). It differs from the (mandatory) rnd32 + * argument to secp256k1_ellswift_encode in this regard. + * + * This function can be used instead of calling secp256k1_ec_pubkey_create + * followed by secp256k1_ellswift_encode. It is safer, as it uses the secret + * key as entropy for the encoding (supplemented with auxrnd32, if provided). + * + * Like secp256k1_ellswift_encode, this function does not guarantee that the + * computed encoding is stable across versions of the library, even if all + * arguments (including auxrnd32) are the same. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ellswift_create( + const secp256k1_context *ctx, + unsigned char *ell64, + const unsigned char *seckey32, + const unsigned char *auxrnd32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Given a private key, and ElligatorSwift public keys sent in both directions, + * compute a shared secret using x-only Elliptic Curve Diffie-Hellman (ECDH). + * + * Returns: 1: shared secret was successfully computed + * 0: secret was invalid or hashfp returned 0 + * Args: ctx: pointer to a context object. + * Out: output: pointer to an array to be filled by hashfp. + * In: ell_a64: pointer to the 64-byte encoded public key of party A + * (will not be NULL) + * ell_b64: pointer to the 64-byte encoded public key of party B + * (will not be NULL) + * seckey32: pointer to our 32-byte secret key + * party: boolean indicating which party we are: zero if we are + * party A, non-zero if we are party B. seckey32 must be + * the private key corresponding to that party's ell_?64. + * This correspondence is not checked. + * hashfp: pointer to a hash function. + * data: arbitrary data pointer passed through to hashfp. + * + * Constant time in seckey32. + * + * This function is more efficient than decoding the public keys, and performing + * ECDH on them. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ellswift_xdh( + const secp256k1_context *ctx, + unsigned char *output, + const unsigned char *ell_a64, + const unsigned char *ell_b64, + const unsigned char *seckey32, + int party, + secp256k1_ellswift_xdh_hash_function hashfp, + void *data +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(5) SECP256K1_ARG_NONNULL(7); + +#ifdef __cplusplus +} +#endif + +#endif /* SECP256K1_ELLSWIFT_H */ diff --git a/example/android/third_party/secp256k1/include/secp256k1_extrakeys.h b/example/android/third_party/secp256k1/include/secp256k1_extrakeys.h new file mode 100644 index 00000000..48c98693 --- /dev/null +++ b/example/android/third_party/secp256k1/include/secp256k1_extrakeys.h @@ -0,0 +1,250 @@ +#ifndef SECP256K1_EXTRAKEYS_H +#define SECP256K1_EXTRAKEYS_H + +#include "secp256k1.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** Opaque data structure that holds a parsed and valid "x-only" public key. + * An x-only pubkey encodes a point whose Y coordinate is even. It is + * serialized using only its X coordinate (32 bytes). See BIP-340 for more + * information about x-only pubkeys. + * + * The exact representation of data inside is implementation defined and not + * guaranteed to be portable between different platforms or versions. It is + * however guaranteed to be 64 bytes in size, and can be safely copied/moved. + * If you need to convert to a format suitable for storage, transmission, use + * use secp256k1_xonly_pubkey_serialize and secp256k1_xonly_pubkey_parse. To + * compare keys, use secp256k1_xonly_pubkey_cmp. + */ +typedef struct secp256k1_xonly_pubkey { + unsigned char data[64]; +} secp256k1_xonly_pubkey; + +/** Opaque data structure that holds a keypair consisting of a secret and a + * public key. + * + * The exact representation of data inside is implementation defined and not + * guaranteed to be portable between different platforms or versions. It is + * however guaranteed to be 96 bytes in size, and can be safely copied/moved. + */ +typedef struct secp256k1_keypair { + unsigned char data[96]; +} secp256k1_keypair; + +/** Parse a 32-byte sequence into a xonly_pubkey object. + * + * Returns: 1 if the public key was fully valid. + * 0 if the public key could not be parsed or is invalid. + * + * Args: ctx: pointer to a context object. + * Out: pubkey: pointer to a pubkey object. If 1 is returned, it is set to a + * parsed version of input. If not, it's set to an invalid value. + * In: input32: pointer to a serialized xonly_pubkey. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_xonly_pubkey_parse( + const secp256k1_context *ctx, + secp256k1_xonly_pubkey *pubkey, + const unsigned char *input32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Serialize an xonly_pubkey object into a 32-byte sequence. + * + * Returns: 1 always. + * + * Args: ctx: pointer to a context object. + * Out: output32: pointer to a 32-byte array to place the serialized key in. + * In: pubkey: pointer to a secp256k1_xonly_pubkey containing an initialized public key. + */ +SECP256K1_API int secp256k1_xonly_pubkey_serialize( + const secp256k1_context *ctx, + unsigned char *output32, + const secp256k1_xonly_pubkey *pubkey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Compare two x-only public keys using lexicographic order + * + * Returns: <0 if the first public key is less than the second + * >0 if the first public key is greater than the second + * 0 if the two public keys are equal + * Args: ctx: pointer to a context object. + * In: pubkey1: first public key to compare + * pubkey2: second public key to compare + */ +SECP256K1_API int secp256k1_xonly_pubkey_cmp( + const secp256k1_context *ctx, + const secp256k1_xonly_pubkey *pk1, + const secp256k1_xonly_pubkey *pk2 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Converts a secp256k1_pubkey into a secp256k1_xonly_pubkey. + * + * Returns: 1 always. + * + * Args: ctx: pointer to a context object. + * Out: xonly_pubkey: pointer to an x-only public key object for placing the converted public key. + * pk_parity: Ignored if NULL. Otherwise, pointer to an integer that + * will be set to 1 if the point encoded by xonly_pubkey is + * the negation of the pubkey and set to 0 otherwise. + * In: pubkey: pointer to a public key that is converted. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_xonly_pubkey_from_pubkey( + const secp256k1_context *ctx, + secp256k1_xonly_pubkey *xonly_pubkey, + int *pk_parity, + const secp256k1_pubkey *pubkey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(4); + +/** Tweak an x-only public key by adding the generator multiplied with tweak32 + * to it. + * + * Note that the resulting point can not in general be represented by an x-only + * pubkey because it may have an odd Y coordinate. Instead, the output_pubkey + * is a normal secp256k1_pubkey. + * + * Returns: 0 if the arguments are invalid or the resulting public key would be + * invalid (only when the tweak is the negation of the corresponding + * secret key). 1 otherwise. + * + * Args: ctx: pointer to a context object. + * Out: output_pubkey: pointer to a public key to store the result. Will be set + * to an invalid value if this function returns 0. + * In: internal_pubkey: pointer to an x-only pubkey to apply the tweak to. + * tweak32: pointer to a 32-byte tweak, which must be valid + * according to secp256k1_ec_seckey_verify or 32 zero + * bytes. For uniformly random 32-byte tweaks, the chance of + * being invalid is negligible (around 1 in 2^128). + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_xonly_pubkey_tweak_add( + const secp256k1_context *ctx, + secp256k1_pubkey *output_pubkey, + const secp256k1_xonly_pubkey *internal_pubkey, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +/** Checks that a tweaked pubkey is the result of calling + * secp256k1_xonly_pubkey_tweak_add with internal_pubkey and tweak32. + * + * The tweaked pubkey is represented by its 32-byte x-only serialization and + * its pk_parity, which can both be obtained by converting the result of + * tweak_add to a secp256k1_xonly_pubkey. + * + * Note that this alone does _not_ verify that the tweaked pubkey is a + * commitment. If the tweak is not chosen in a specific way, the tweaked pubkey + * can easily be the result of a different internal_pubkey and tweak. + * + * Returns: 0 if the arguments are invalid or the tweaked pubkey is not the + * result of tweaking the internal_pubkey with tweak32. 1 otherwise. + * Args: ctx: pointer to a context object. + * In: tweaked_pubkey32: pointer to a serialized xonly_pubkey. + * tweaked_pk_parity: the parity of the tweaked pubkey (whose serialization + * is passed in as tweaked_pubkey32). This must match the + * pk_parity value that is returned when calling + * secp256k1_xonly_pubkey with the tweaked pubkey, or + * this function will fail. + * internal_pubkey: pointer to an x-only public key object to apply the tweak to. + * tweak32: pointer to a 32-byte tweak. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_xonly_pubkey_tweak_add_check( + const secp256k1_context *ctx, + const unsigned char *tweaked_pubkey32, + int tweaked_pk_parity, + const secp256k1_xonly_pubkey *internal_pubkey, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(5); + +/** Compute the keypair for a valid secret key. + * + * See the documentation of `secp256k1_ec_seckey_verify` for more information + * about the validity of secret keys. + * + * Returns: 1: secret key is valid + * 0: secret key is invalid + * Args: ctx: pointer to a context object (not secp256k1_context_static). + * Out: keypair: pointer to the created keypair. + * In: seckey: pointer to a 32-byte secret key. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_keypair_create( + const secp256k1_context *ctx, + secp256k1_keypair *keypair, + const unsigned char *seckey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Get the secret key from a keypair. + * + * Returns: 1 always. + * Args: ctx: pointer to a context object. + * Out: seckey: pointer to a 32-byte buffer for the secret key. + * In: keypair: pointer to a keypair. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_keypair_sec( + const secp256k1_context *ctx, + unsigned char *seckey, + const secp256k1_keypair *keypair +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Get the public key from a keypair. + * + * Returns: 1 always. + * Args: ctx: pointer to a context object. + * Out: pubkey: pointer to a pubkey object, set to the keypair public key. + * In: keypair: pointer to a keypair. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_keypair_pub( + const secp256k1_context *ctx, + secp256k1_pubkey *pubkey, + const secp256k1_keypair *keypair +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Get the x-only public key from a keypair. + * + * This is the same as calling secp256k1_keypair_pub and then + * secp256k1_xonly_pubkey_from_pubkey. + * + * Returns: 1 always. + * Args: ctx: pointer to a context object. + * Out: pubkey: pointer to an xonly_pubkey object, set to the keypair + * public key after converting it to an xonly_pubkey. + * pk_parity: Ignored if NULL. Otherwise, pointer to an integer that will be set to the + * pk_parity argument of secp256k1_xonly_pubkey_from_pubkey. + * In: keypair: pointer to a keypair. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_keypair_xonly_pub( + const secp256k1_context *ctx, + secp256k1_xonly_pubkey *pubkey, + int *pk_parity, + const secp256k1_keypair *keypair +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(4); + +/** Tweak a keypair by adding tweak32 to the secret key and updating the public + * key accordingly. + * + * Calling this function and then secp256k1_keypair_pub results in the same + * public key as calling secp256k1_keypair_xonly_pub and then + * secp256k1_xonly_pubkey_tweak_add. + * + * Returns: 0 if the arguments are invalid or the resulting keypair would be + * invalid (only when the tweak is the negation of the keypair's + * secret key). 1 otherwise. + * + * Args: ctx: pointer to a context object. + * In/Out: keypair: pointer to a keypair to apply the tweak to. Will be set to + * an invalid value if this function returns 0. + * In: tweak32: pointer to a 32-byte tweak, which must be valid according to + * secp256k1_ec_seckey_verify or 32 zero bytes. For uniformly + * random 32-byte tweaks, the chance of being invalid is + * negligible (around 1 in 2^128). + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_keypair_xonly_tweak_add( + const secp256k1_context *ctx, + secp256k1_keypair *keypair, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +#ifdef __cplusplus +} +#endif + +#endif /* SECP256K1_EXTRAKEYS_H */ diff --git a/example/android/third_party/secp256k1/include/secp256k1_musig.h b/example/android/third_party/secp256k1/include/secp256k1_musig.h new file mode 100644 index 00000000..11b8f08c --- /dev/null +++ b/example/android/third_party/secp256k1/include/secp256k1_musig.h @@ -0,0 +1,588 @@ +#ifndef SECP256K1_MUSIG_H +#define SECP256K1_MUSIG_H + +#include "secp256k1_extrakeys.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +/** This module implements BIP 327 "MuSig2 for BIP340-compatible + * Multi-Signatures" + * (https://github.com/bitcoin/bips/blob/master/bip-0327.mediawiki) + * v1.0.0. You can find an example demonstrating the musig module in + * examples/musig.c. + * + * The module also supports BIP 341 ("Taproot") public key tweaking. + * + * It is recommended to read the documentation in this include file carefully. + * Further notes on API usage can be found in doc/musig.md + * + * Since the first version of MuSig is essentially replaced by MuSig2, we use + * MuSig, musig and MuSig2 synonymously unless noted otherwise. + */ + +/** Opaque data structures + * + * The exact representation of data inside the opaque data structures is + * implementation defined and not guaranteed to be portable between different + * platforms or versions. With the exception of `secp256k1_musig_secnonce`, the + * data structures can be safely copied/moved. If you need to convert to a + * format suitable for storage, transmission, or comparison, use the + * corresponding serialization and parsing functions. + */ + +/** Opaque data structure that caches information about public key aggregation. + * + * Guaranteed to be 197 bytes in size. No serialization and parsing functions + * (yet). + */ +typedef struct secp256k1_musig_keyagg_cache { + unsigned char data[197]; +} secp256k1_musig_keyagg_cache; + +/** Opaque data structure that holds a signer's _secret_ nonce. + * + * Guaranteed to be 132 bytes in size. + * + * WARNING: This structure MUST NOT be copied or read or written to directly. A + * signer who is online throughout the whole process and can keep this + * structure in memory can use the provided API functions for a safe standard + * workflow. + * + * Copying this data structure can result in nonce reuse which will leak the + * secret signing key. + */ +typedef struct secp256k1_musig_secnonce { + unsigned char data[132]; +} secp256k1_musig_secnonce; + +/** Opaque data structure that holds a signer's public nonce. + * + * Guaranteed to be 132 bytes in size. Serialized and parsed with + * `musig_pubnonce_serialize` and `musig_pubnonce_parse`. + */ +typedef struct secp256k1_musig_pubnonce { + unsigned char data[132]; +} secp256k1_musig_pubnonce; + +/** Opaque data structure that holds an aggregate public nonce. + * + * Guaranteed to be 132 bytes in size. Serialized and parsed with + * `musig_aggnonce_serialize` and `musig_aggnonce_parse`. + */ +typedef struct secp256k1_musig_aggnonce { + unsigned char data[132]; +} secp256k1_musig_aggnonce; + +/** Opaque data structure that holds a MuSig session. + * + * This structure is not required to be kept secret for the signing protocol to + * be secure. Guaranteed to be 133 bytes in size. No serialization and parsing + * functions (yet). + */ +typedef struct secp256k1_musig_session { + unsigned char data[133]; +} secp256k1_musig_session; + +/** Opaque data structure that holds a partial MuSig signature. + * + * Guaranteed to be 36 bytes in size. Serialized and parsed with + * `musig_partial_sig_serialize` and `musig_partial_sig_parse`. + */ +typedef struct secp256k1_musig_partial_sig { + unsigned char data[36]; +} secp256k1_musig_partial_sig; + +/** Parse a signer's public nonce. + * + * Returns: 1 when the nonce could be parsed, 0 otherwise. + * Args: ctx: pointer to a context object + * Out: nonce: pointer to a nonce object + * In: in66: pointer to the 66-byte nonce to be parsed + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_pubnonce_parse( + const secp256k1_context *ctx, + secp256k1_musig_pubnonce *nonce, + const unsigned char *in66 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Serialize a signer's public nonce + * + * Returns: 1 always + * Args: ctx: pointer to a context object + * Out: out66: pointer to a 66-byte array to store the serialized nonce + * In: nonce: pointer to the nonce + */ +SECP256K1_API int secp256k1_musig_pubnonce_serialize( + const secp256k1_context *ctx, + unsigned char *out66, + const secp256k1_musig_pubnonce *nonce +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Parse an aggregate public nonce. + * + * Returns: 1 when the nonce could be parsed, 0 otherwise. + * Args: ctx: pointer to a context object + * Out: nonce: pointer to a nonce object + * In: in66: pointer to the 66-byte nonce to be parsed + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_aggnonce_parse( + const secp256k1_context *ctx, + secp256k1_musig_aggnonce *nonce, + const unsigned char *in66 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Serialize an aggregate public nonce + * + * Returns: 1 always + * Args: ctx: pointer to a context object + * Out: out66: pointer to a 66-byte array to store the serialized nonce + * In: nonce: pointer to the nonce + */ +SECP256K1_API int secp256k1_musig_aggnonce_serialize( + const secp256k1_context *ctx, + unsigned char *out66, + const secp256k1_musig_aggnonce *nonce +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Parse a MuSig partial signature. + * + * Returns: 1 when the signature could be parsed, 0 otherwise. + * Args: ctx: pointer to a context object + * Out: sig: pointer to a signature object + * In: in32: pointer to the 32-byte signature to be parsed + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_partial_sig_parse( + const secp256k1_context *ctx, + secp256k1_musig_partial_sig *sig, + const unsigned char *in32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Serialize a MuSig partial signature + * + * Returns: 1 always + * Args: ctx: pointer to a context object + * Out: out32: pointer to a 32-byte array to store the serialized signature + * In: sig: pointer to the signature + */ +SECP256K1_API int secp256k1_musig_partial_sig_serialize( + const secp256k1_context *ctx, + unsigned char *out32, + const secp256k1_musig_partial_sig *sig +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Computes an aggregate public key and uses it to initialize a keyagg_cache + * + * Different orders of `pubkeys` result in different `agg_pk`s. + * + * Before aggregating, the pubkeys can be sorted with `secp256k1_ec_pubkey_sort` + * which ensures the same `agg_pk` result for the same multiset of pubkeys. + * This is useful to do before `pubkey_agg`, such that the order of pubkeys + * does not affect the aggregate public key. + * + * Returns: 0 if the arguments are invalid, 1 otherwise + * Args: ctx: pointer to a context object + * Out: agg_pk: the MuSig-aggregated x-only public key. If you do not need it, + * this arg can be NULL. + * keyagg_cache: if non-NULL, pointer to a musig_keyagg_cache struct that + * is required for signing (or observing the signing session + * and verifying partial signatures). + * In: pubkeys: input array of pointers to public keys to aggregate. The order + * is important; a different order will result in a different + * aggregate public key. + * n_pubkeys: length of pubkeys array. Must be greater than 0. + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_pubkey_agg( + const secp256k1_context *ctx, + secp256k1_xonly_pubkey *agg_pk, + secp256k1_musig_keyagg_cache *keyagg_cache, + const secp256k1_pubkey * const *pubkeys, + size_t n_pubkeys +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(4); + +/** Obtain the aggregate public key from a keyagg_cache. + * + * This is only useful if you need the non-xonly public key, in particular for + * plain (non-xonly) tweaking or batch-verifying multiple key aggregations + * (not implemented). + * + * Returns: 0 if the arguments are invalid, 1 otherwise + * Args: ctx: pointer to a context object + * Out: agg_pk: the MuSig-aggregated public key. + * In: keyagg_cache: pointer to a `musig_keyagg_cache` struct initialized by + * `musig_pubkey_agg` + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_pubkey_get( + const secp256k1_context *ctx, + secp256k1_pubkey *agg_pk, + const secp256k1_musig_keyagg_cache *keyagg_cache +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Apply plain "EC" tweaking to a public key in a given keyagg_cache by adding + * the generator multiplied with `tweak32` to it. This is useful for deriving + * child keys from an aggregate public key via BIP 32 where `tweak32` is set to + * a hash as defined in BIP 32. + * + * Callers are responsible for deriving `tweak32` in a way that does not reduce + * the security of MuSig (for example, by following BIP 32). + * + * The tweaking method is the same as `secp256k1_ec_pubkey_tweak_add`. So after + * the following pseudocode buf and buf2 have identical contents (absent + * earlier failures). + * + * secp256k1_musig_pubkey_agg(..., keyagg_cache, pubkeys, ...) + * secp256k1_musig_pubkey_get(..., agg_pk, keyagg_cache) + * secp256k1_musig_pubkey_ec_tweak_add(..., output_pk, tweak32, keyagg_cache) + * secp256k1_ec_pubkey_serialize(..., buf, ..., output_pk, ...) + * secp256k1_ec_pubkey_tweak_add(..., agg_pk, tweak32) + * secp256k1_ec_pubkey_serialize(..., buf2, ..., agg_pk, ...) + * + * This function is required if you want to _sign_ for a tweaked aggregate key. + * If you are only computing a public key but not intending to create a + * signature for it, use `secp256k1_ec_pubkey_tweak_add` instead. + * + * Returns: 0 if the arguments are invalid, 1 otherwise + * Args: ctx: pointer to a context object + * Out: output_pubkey: pointer to a public key to store the result. Will be set + * to an invalid value if this function returns 0. If you + * do not need it, this arg can be NULL. + * In/Out: keyagg_cache: pointer to a `musig_keyagg_cache` struct initialized by + * `musig_pubkey_agg` + * In: tweak32: pointer to a 32-byte tweak. The tweak is valid if it passes + * `secp256k1_ec_seckey_verify` and is not equal to the + * secret key corresponding to the public key represented + * by keyagg_cache or its negation. For uniformly random + * 32-byte arrays the chance of being invalid is + * negligible (around 1 in 2^128). + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_pubkey_ec_tweak_add( + const secp256k1_context *ctx, + secp256k1_pubkey *output_pubkey, + secp256k1_musig_keyagg_cache *keyagg_cache, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +/** Apply x-only tweaking to a public key in a given keyagg_cache by adding the + * generator multiplied with `tweak32` to it. This is useful for creating + * Taproot outputs where `tweak32` is set to a TapTweak hash as defined in BIP + * 341. + * + * Callers are responsible for deriving `tweak32` in a way that does not reduce + * the security of MuSig (for example, by following Taproot BIP 341). + * + * The tweaking method is the same as `secp256k1_xonly_pubkey_tweak_add`. So in + * the following pseudocode xonly_pubkey_tweak_add_check (absent earlier + * failures) returns 1. + * + * secp256k1_musig_pubkey_agg(..., agg_pk, keyagg_cache, pubkeys, ...) + * secp256k1_musig_pubkey_xonly_tweak_add(..., output_pk, keyagg_cache, tweak32) + * secp256k1_xonly_pubkey_serialize(..., buf, output_pk) + * secp256k1_xonly_pubkey_tweak_add_check(..., buf, ..., agg_pk, tweak32) + * + * This function is required if you want to _sign_ for a tweaked aggregate key. + * If you are only computing a public key but not intending to create a + * signature for it, use `secp256k1_xonly_pubkey_tweak_add` instead. + * + * Returns: 0 if the arguments are invalid, 1 otherwise + * Args: ctx: pointer to a context object + * Out: output_pubkey: pointer to a public key to store the result. Will be set + * to an invalid value if this function returns 0. If you + * do not need it, this arg can be NULL. + * In/Out: keyagg_cache: pointer to a `musig_keyagg_cache` struct initialized by + * `musig_pubkey_agg` + * In: tweak32: pointer to a 32-byte tweak. The tweak is valid if it passes + * `secp256k1_ec_seckey_verify` and is not equal to the + * secret key corresponding to the public key represented + * by keyagg_cache or its negation. For uniformly random + * 32-byte arrays the chance of being invalid is + * negligible (around 1 in 2^128). + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_pubkey_xonly_tweak_add( + const secp256k1_context *ctx, + secp256k1_pubkey *output_pubkey, + secp256k1_musig_keyagg_cache *keyagg_cache, + const unsigned char *tweak32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +/** Starts a signing session by generating a nonce + * + * This function outputs a secret nonce that will be required for signing and a + * corresponding public nonce that is intended to be sent to other signers. + * + * MuSig differs from regular Schnorr signing in that implementers _must_ take + * special care to not reuse a nonce. This can be ensured by following these rules: + * + * 1. Each call to this function must have a UNIQUE session_secrand32 that must + * NOT BE REUSED in subsequent calls to this function and must be KEPT + * SECRET (even from other signers). + * 2. If you already know the seckey, message or aggregate public key + * cache, they can be optionally provided to derive the nonce and increase + * misuse-resistance. The extra_input32 argument can be used to provide + * additional data that does not repeat in normal scenarios, such as the + * current time. + * 3. Avoid copying (or serializing) the secnonce. This reduces the possibility + * that it is used more than once for signing. + * + * If you don't have access to good randomness for session_secrand32, but you + * have access to a non-repeating counter, then see + * secp256k1_musig_nonce_gen_counter. + * + * Remember that nonce reuse will leak the secret key! + * Note that using the same seckey for multiple MuSig sessions is fine. + * + * Returns: 0 if the arguments are invalid and 1 otherwise + * Args: ctx: pointer to a context object (not secp256k1_context_static) + * Out: secnonce: pointer to a structure to store the secret nonce + * pubnonce: pointer to a structure to store the public nonce + * In/Out: + * session_secrand32: a 32-byte session_secrand32 as explained above. Must be unique to this + * call to secp256k1_musig_nonce_gen and must be uniformly + * random. If the function call is successful, the + * session_secrand32 buffer is invalidated to prevent reuse. + * In: + * seckey: the 32-byte secret key that will later be used for signing, if + * already known (can be NULL) + * pubkey: public key of the signer creating the nonce. The secnonce + * output of this function cannot be used to sign for any + * other public key. While the public key should correspond + * to the provided seckey, a mismatch will not cause the + * function to return 0. + * msg32: the 32-byte message that will later be signed, if already known + * (can be NULL) + * keyagg_cache: pointer to the keyagg_cache that was used to create the aggregate + * (and potentially tweaked) public key if already known + * (can be NULL) + * extra_input32: an optional 32-byte array that is input to the nonce + * derivation function (can be NULL) + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_nonce_gen( + const secp256k1_context *ctx, + secp256k1_musig_secnonce *secnonce, + secp256k1_musig_pubnonce *pubnonce, + unsigned char *session_secrand32, + const unsigned char *seckey, + const secp256k1_pubkey *pubkey, + const unsigned char *msg32, + const secp256k1_musig_keyagg_cache *keyagg_cache, + const unsigned char *extra_input32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(6); + + +/** Alternative way to generate a nonce and start a signing session + * + * This function outputs a secret nonce that will be required for signing and a + * corresponding public nonce that is intended to be sent to other signers. + * + * This function differs from `secp256k1_musig_nonce_gen` by accepting a + * non-repeating counter value instead of a secret random value. This requires + * that a secret key is provided to `secp256k1_musig_nonce_gen_counter` + * (through the keypair argument), as opposed to `secp256k1_musig_nonce_gen` + * where the seckey argument is optional. + * + * MuSig differs from regular Schnorr signing in that implementers _must_ take + * special care to not reuse a nonce. This can be ensured by following these rules: + * + * 1. The nonrepeating_cnt argument must be a counter value that never repeats, + * i.e., you must never call `secp256k1_musig_nonce_gen_counter` twice with + * the same keypair and nonrepeating_cnt value. For example, this implies + * that if the same keypair is used with `secp256k1_musig_nonce_gen_counter` + * on multiple devices, none of the devices should have the same counter + * value as any other device. + * 2. If the seckey, message or aggregate public key cache is already available + * at this stage, any of these can be optionally provided, in which case + * they will be used in the derivation of the nonce and increase + * misuse-resistance. The extra_input32 argument can be used to provide + * additional data that does not repeat in normal scenarios, such as the + * current time. + * 3. Avoid copying (or serializing) the secnonce. This reduces the possibility + * that it is used more than once for signing. + * + * Remember that nonce reuse will leak the secret key! + * Note that using the same keypair for multiple MuSig sessions is fine. + * + * Returns: 0 if the arguments are invalid and 1 otherwise + * Args: ctx: pointer to a context object (not secp256k1_context_static) + * Out: secnonce: pointer to a structure to store the secret nonce + * pubnonce: pointer to a structure to store the public nonce + * In: + * nonrepeating_cnt: the value of a counter as explained above. Must be + * unique to this call to secp256k1_musig_nonce_gen. + * keypair: keypair of the signer creating the nonce. The secnonce + * output of this function cannot be used to sign for any + * other keypair. + * msg32: the 32-byte message that will later be signed, if already known + * (can be NULL) + * keyagg_cache: pointer to the keyagg_cache that was used to create the aggregate + * (and potentially tweaked) public key if already known + * (can be NULL) + * extra_input32: an optional 32-byte array that is input to the nonce + * derivation function (can be NULL) + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_nonce_gen_counter( + const secp256k1_context *ctx, + secp256k1_musig_secnonce *secnonce, + secp256k1_musig_pubnonce *pubnonce, + uint64_t nonrepeating_cnt, + const secp256k1_keypair *keypair, + const unsigned char *msg32, + const secp256k1_musig_keyagg_cache *keyagg_cache, + const unsigned char *extra_input32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(5); + +/** Aggregates the nonces of all signers into a single nonce + * + * This can be done by an untrusted party to reduce the communication + * between signers. Instead of everyone sending nonces to everyone else, there + * can be one party receiving all nonces, aggregating the nonces with this + * function and then sending only the aggregate nonce back to the signers. + * + * If the aggregator does not compute the aggregate nonce correctly, the final + * signature will be invalid. + * + * Returns: 0 if the arguments are invalid, 1 otherwise + * Args: ctx: pointer to a context object + * Out: aggnonce: pointer to an aggregate public nonce object for + * musig_nonce_process + * In: pubnonces: array of pointers to public nonces sent by the + * signers + * n_pubnonces: number of elements in the pubnonces array. Must be + * greater than 0. + */ +SECP256K1_API int secp256k1_musig_nonce_agg( + const secp256k1_context *ctx, + secp256k1_musig_aggnonce *aggnonce, + const secp256k1_musig_pubnonce * const *pubnonces, + size_t n_pubnonces +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + +/** Takes the aggregate nonce and creates a session that is required for signing + * and verification of partial signatures. + * + * Returns: 0 if the arguments are invalid, 1 otherwise + * Args: ctx: pointer to a context object + * Out: session: pointer to a struct to store the session + * In: aggnonce: pointer to an aggregate public nonce object that is the + * output of musig_nonce_agg + * msg32: the 32-byte message to sign + * keyagg_cache: pointer to the keyagg_cache that was used to create the + * aggregate (and potentially tweaked) pubkey + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_nonce_process( + const secp256k1_context *ctx, + secp256k1_musig_session *session, + const secp256k1_musig_aggnonce *aggnonce, + const unsigned char *msg32, + const secp256k1_musig_keyagg_cache *keyagg_cache +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(5); + +/** Produces a partial signature + * + * This function overwrites the given secnonce with zeros and will abort if given a + * secnonce that is all zeros. This is a best effort attempt to protect against nonce + * reuse. However, this is of course easily defeated if the secnonce has been + * copied (or serialized). Remember that nonce reuse will leak the secret key! + * + * For signing to succeed, the secnonce provided to this function must have + * been generated for the provided keypair. This means that when signing for a + * keypair consisting of a seckey and pubkey, the secnonce must have been + * created by calling musig_nonce_gen with that pubkey. Otherwise, the + * illegal_callback is called. + * + * This function does not verify the output partial signature, deviating from + * the BIP 327 specification. It is recommended to verify the output partial + * signature with `secp256k1_musig_partial_sig_verify` to prevent random or + * adversarially provoked computation errors. + * + * Returns: 0 if the arguments are invalid or the provided secnonce has already + * been used for signing, 1 otherwise + * Args: ctx: pointer to a context object + * Out: partial_sig: pointer to struct to store the partial signature + * In/Out: secnonce: pointer to the secnonce struct created in + * musig_nonce_gen that has been never used in a + * partial_sign call before and has been created for the + * keypair + * In: keypair: pointer to keypair to sign the message with + * keyagg_cache: pointer to the keyagg_cache that was output when the + * aggregate public key for this session + * session: pointer to the session that was created with + * musig_nonce_process + */ +SECP256K1_API int secp256k1_musig_partial_sign( + const secp256k1_context *ctx, + secp256k1_musig_partial_sig *partial_sig, + secp256k1_musig_secnonce *secnonce, + const secp256k1_keypair *keypair, + const secp256k1_musig_keyagg_cache *keyagg_cache, + const secp256k1_musig_session *session +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(5) SECP256K1_ARG_NONNULL(6); + +/** Verifies an individual signer's partial signature + * + * The signature is verified for a specific signing session. In order to avoid + * accidentally verifying a signature from a different or non-existing signing + * session, you must ensure the following: + * 1. The `keyagg_cache` argument is identical to the one used to create the + * `session` with `musig_nonce_process`. + * 2. The `pubkey` argument must be identical to the one sent by the signer + * before aggregating it with `musig_pubkey_agg` to create the + * `keyagg_cache`. + * 3. The `pubnonce` argument must be identical to the one sent by the signer + * before aggregating it with `musig_nonce_agg` and using the result to + * create the `session` with `musig_nonce_process`. + * + * It is not required to call this function in regular MuSig sessions, because + * if any partial signature does not verify, the final signature will not + * verify either, so the problem will be caught. However, this function + * provides the ability to identify which specific partial signature fails + * verification. + * + * Returns: 0 if the arguments are invalid or the partial signature does not + * verify, 1 otherwise + * Args ctx: pointer to a context object + * In: partial_sig: pointer to partial signature to verify, sent by + * the signer associated with `pubnonce` and `pubkey` + * pubnonce: public nonce of the signer in the signing session + * pubkey: public key of the signer in the signing session + * keyagg_cache: pointer to the keyagg_cache that was output when the + * aggregate public key for this signing session + * session: pointer to the session that was created with + * `musig_nonce_process` + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_musig_partial_sig_verify( + const secp256k1_context *ctx, + const secp256k1_musig_partial_sig *partial_sig, + const secp256k1_musig_pubnonce *pubnonce, + const secp256k1_pubkey *pubkey, + const secp256k1_musig_keyagg_cache *keyagg_cache, + const secp256k1_musig_session *session +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4) SECP256K1_ARG_NONNULL(5) SECP256K1_ARG_NONNULL(6); + +/** Aggregates partial signatures + * + * Returns: 0 if the arguments are invalid, 1 otherwise (which does NOT mean + * the resulting signature verifies). + * Args: ctx: pointer to a context object + * Out: sig64: complete (but possibly invalid) Schnorr signature + * In: session: pointer to the session that was created with + * musig_nonce_process + * partial_sigs: array of pointers to partial signatures to aggregate + * n_sigs: number of elements in the partial_sigs array. Must be + * greater than 0. + */ +SECP256K1_API int secp256k1_musig_partial_sig_agg( + const secp256k1_context *ctx, + unsigned char *sig64, + const secp256k1_musig_session *session, + const secp256k1_musig_partial_sig * const *partial_sigs, + size_t n_sigs +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/example/android/third_party/secp256k1/include/secp256k1_preallocated.h b/example/android/third_party/secp256k1/include/secp256k1_preallocated.h index a9ae15d5..f2d95c24 100644 --- a/example/android/third_party/secp256k1/include/secp256k1_preallocated.h +++ b/example/android/third_party/secp256k1/include/secp256k1_preallocated.h @@ -52,17 +52,19 @@ SECP256K1_API size_t secp256k1_context_preallocated_size( * in the memory. In simpler words, the prealloc pointer (or any pointer derived * from it) should not be used during the lifetime of the context object. * - * Returns: a newly created context object. - * In: prealloc: a pointer to a rewritable contiguous block of memory of + * Returns: pointer to newly created context object. + * In: prealloc: pointer to a rewritable contiguous block of memory of * size at least secp256k1_context_preallocated_size(flags) - * bytes, as detailed above (cannot be NULL) + * bytes, as detailed above. * flags: which parts of the context to initialize. * + * See secp256k1_context_create (in secp256k1.h) for further details. + * * See also secp256k1_context_randomize (in secp256k1.h) * and secp256k1_context_preallocated_destroy. */ -SECP256K1_API secp256k1_context* secp256k1_context_preallocated_create( - void* prealloc, +SECP256K1_API secp256k1_context *secp256k1_context_preallocated_create( + void *prealloc, unsigned int flags ) SECP256K1_ARG_NONNULL(1) SECP256K1_WARN_UNUSED_RESULT; @@ -70,10 +72,10 @@ SECP256K1_API secp256k1_context* secp256k1_context_preallocated_create( * caller-provided memory. * * Returns: the required size of the caller-provided memory block. - * In: ctx: an existing context to copy (cannot be NULL) + * In: ctx: pointer to a context to copy. */ SECP256K1_API size_t secp256k1_context_preallocated_clone_size( - const secp256k1_context* ctx + const secp256k1_context *ctx ) SECP256K1_ARG_NONNULL(1) SECP256K1_WARN_UNUSED_RESULT; /** Copy a secp256k1 context object into caller-provided memory. @@ -86,15 +88,18 @@ SECP256K1_API size_t secp256k1_context_preallocated_clone_size( * the lifetime of this context object, see the description of * secp256k1_context_preallocated_create for details. * - * Returns: a newly created context object. - * Args: ctx: an existing context to copy (cannot be NULL) - * In: prealloc: a pointer to a rewritable contiguous block of memory of + * Cloning secp256k1_context_static is not possible, and should not be emulated by + * the caller (e.g., using memcpy). Create a new context instead. + * + * Returns: pointer to a newly created context object. + * Args: ctx: pointer to a context to copy (not secp256k1_context_static). + * In: prealloc: pointer to a rewritable contiguous block of memory of * size at least secp256k1_context_preallocated_size(flags) - * bytes, as detailed above (cannot be NULL) + * bytes, as detailed above. */ -SECP256K1_API secp256k1_context* secp256k1_context_preallocated_clone( - const secp256k1_context* ctx, - void* prealloc +SECP256K1_API secp256k1_context *secp256k1_context_preallocated_clone( + const secp256k1_context *ctx, + void *prealloc ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_WARN_UNUSED_RESULT; /** Destroy a secp256k1 context object that has been created in @@ -113,13 +118,14 @@ SECP256K1_API secp256k1_context* secp256k1_context_preallocated_clone( * preallocated pointer given to secp256k1_context_preallocated_create or * secp256k1_context_preallocated_clone. * - * Args: ctx: an existing context to destroy, constructed using + * Args: ctx: pointer to a context to destroy, constructed using * secp256k1_context_preallocated_create or - * secp256k1_context_preallocated_clone (cannot be NULL) + * secp256k1_context_preallocated_clone + * (i.e., not secp256k1_context_static). */ SECP256K1_API void secp256k1_context_preallocated_destroy( - secp256k1_context* ctx -); + secp256k1_context *ctx +) SECP256K1_ARG_NONNULL(1); #ifdef __cplusplus } diff --git a/example/android/third_party/secp256k1/include/secp256k1_recovery.h b/example/android/third_party/secp256k1/include/secp256k1_recovery.h index cf6c5ed7..93a2e4cc 100644 --- a/example/android/third_party/secp256k1/include/secp256k1_recovery.h +++ b/example/android/third_party/secp256k1/include/secp256k1_recovery.h @@ -7,7 +7,7 @@ extern "C" { #endif -/** Opaque data structured that holds a parsed ECDSA signature, +/** Opaque data structure that holds a parsed ECDSA signature, * supporting pubkey recovery. * * The exact representation of data inside is implementation defined and not @@ -21,21 +21,21 @@ extern "C" { * recoverability) will have identical representation, so they can be * memcmp'ed. */ -typedef struct { +typedef struct secp256k1_ecdsa_recoverable_signature { unsigned char data[65]; } secp256k1_ecdsa_recoverable_signature; /** Parse a compact ECDSA signature (64 bytes + recovery id). * * Returns: 1 when the signature could be parsed, 0 otherwise - * Args: ctx: a secp256k1 context object - * Out: sig: a pointer to a signature object - * In: input64: a pointer to a 64-byte compact signature + * Args: ctx: pointer to a context object + * Out: sig: pointer to a signature object + * In: input64: pointer to a 64-byte compact signature * recid: the recovery id (0, 1, 2 or 3) */ SECP256K1_API int secp256k1_ecdsa_recoverable_signature_parse_compact( - const secp256k1_context* ctx, - secp256k1_ecdsa_recoverable_signature* sig, + const secp256k1_context *ctx, + secp256k1_ecdsa_recoverable_signature *sig, const unsigned char *input64, int recid ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); @@ -43,45 +43,48 @@ SECP256K1_API int secp256k1_ecdsa_recoverable_signature_parse_compact( /** Convert a recoverable signature into a normal signature. * * Returns: 1 - * Out: sig: a pointer to a normal signature (cannot be NULL). - * In: sigin: a pointer to a recoverable signature (cannot be NULL). + * Args: ctx: pointer to a context object. + * Out: sig: pointer to a normal signature. + * In: sigin: pointer to a recoverable signature. */ SECP256K1_API int secp256k1_ecdsa_recoverable_signature_convert( - const secp256k1_context* ctx, - secp256k1_ecdsa_signature* sig, - const secp256k1_ecdsa_recoverable_signature* sigin + const secp256k1_context *ctx, + secp256k1_ecdsa_signature *sig, + const secp256k1_ecdsa_recoverable_signature *sigin ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); /** Serialize an ECDSA signature in compact format (64 bytes + recovery id). * * Returns: 1 - * Args: ctx: a secp256k1 context object - * Out: output64: a pointer to a 64-byte array of the compact signature (cannot be NULL) - * recid: a pointer to an integer to hold the recovery id (can be NULL). - * In: sig: a pointer to an initialized signature object (cannot be NULL) + * Args: ctx: pointer to a context object. + * Out: output64: pointer to a 64-byte array of the compact signature. + * recid: pointer to an integer to hold the recovery id. + * In: sig: pointer to an initialized signature object. */ SECP256K1_API int secp256k1_ecdsa_recoverable_signature_serialize_compact( - const secp256k1_context* ctx, + const secp256k1_context *ctx, unsigned char *output64, int *recid, - const secp256k1_ecdsa_recoverable_signature* sig + const secp256k1_ecdsa_recoverable_signature *sig ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); /** Create a recoverable ECDSA signature. * * Returns: 1: signature created - * 0: the nonce generation function failed, or the private key was invalid. - * Args: ctx: pointer to a context object, initialized for signing (cannot be NULL) - * Out: sig: pointer to an array where the signature will be placed (cannot be NULL) - * In: msg32: the 32-byte message hash being signed (cannot be NULL) - * seckey: pointer to a 32-byte secret key (cannot be NULL) - * noncefp:pointer to a nonce generation function. If NULL, secp256k1_nonce_function_default is used - * ndata: pointer to arbitrary data used by the nonce generation function (can be NULL) + * 0: the nonce generation function failed, or the secret key was invalid. + * Args: ctx: pointer to a context object (not secp256k1_context_static). + * Out: sig: pointer to an array where the signature will be placed. + * In: msghash32: the 32-byte message hash being signed. + * seckey: pointer to a 32-byte secret key. + * noncefp: pointer to a nonce generation function. If NULL, + * secp256k1_nonce_function_default is used. + * ndata: pointer to arbitrary data used by the nonce generation function + * (can be NULL for secp256k1_nonce_function_default). */ SECP256K1_API int secp256k1_ecdsa_sign_recoverable( - const secp256k1_context* ctx, + const secp256k1_context *ctx, secp256k1_ecdsa_recoverable_signature *sig, - const unsigned char *msg32, + const unsigned char *msghash32, const unsigned char *seckey, secp256k1_nonce_function noncefp, const void *ndata @@ -91,16 +94,16 @@ SECP256K1_API int secp256k1_ecdsa_sign_recoverable( * * Returns: 1: public key successfully recovered (which guarantees a correct signature). * 0: otherwise. - * Args: ctx: pointer to a context object, initialized for verification (cannot be NULL) - * Out: pubkey: pointer to the recovered public key (cannot be NULL) - * In: sig: pointer to initialized signature that supports pubkey recovery (cannot be NULL) - * msg32: the 32-byte message hash assumed to be signed (cannot be NULL) + * Args: ctx: pointer to a context object. + * Out: pubkey: pointer to the recovered public key. + * In: sig: pointer to initialized signature that supports pubkey recovery. + * msghash32: the 32-byte message hash assumed to be signed. */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ecdsa_recover( - const secp256k1_context* ctx, + const secp256k1_context *ctx, secp256k1_pubkey *pubkey, const secp256k1_ecdsa_recoverable_signature *sig, - const unsigned char *msg32 + const unsigned char *msghash32 ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); #ifdef __cplusplus diff --git a/example/android/third_party/secp256k1/include/secp256k1_schnorrsig.h b/example/android/third_party/secp256k1/include/secp256k1_schnorrsig.h new file mode 100644 index 00000000..013d4ee7 --- /dev/null +++ b/example/android/third_party/secp256k1/include/secp256k1_schnorrsig.h @@ -0,0 +1,190 @@ +#ifndef SECP256K1_SCHNORRSIG_H +#define SECP256K1_SCHNORRSIG_H + +#include "secp256k1.h" +#include "secp256k1_extrakeys.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** This module implements a variant of Schnorr signatures compliant with + * Bitcoin Improvement Proposal 340 "Schnorr Signatures for secp256k1" + * (https://github.com/bitcoin/bips/blob/master/bip-0340.mediawiki). + */ + +/** A pointer to a function to deterministically generate a nonce. + * + * Same as secp256k1_nonce function with the exception of accepting an + * additional pubkey argument and not requiring an attempt argument. The pubkey + * argument can protect signature schemes with key-prefixed challenge hash + * inputs against reusing the nonce when signing with the wrong precomputed + * pubkey. + * + * Returns: 1 if a nonce was successfully generated. 0 will cause signing to + * return an error. + * Out: nonce32: pointer to a 32-byte array to be filled by the function + * In: msg: the message being verified. Is NULL if and only if msglen + * is 0. + * msglen: the length of the message + * key32: pointer to a 32-byte secret key (will not be NULL) + * xonly_pk32: the 32-byte serialized xonly pubkey corresponding to key32 + * (will not be NULL) + * algo: pointer to an array describing the signature + * algorithm (will not be NULL) + * algolen: the length of the algo array + * data: arbitrary data pointer that is passed through + * + * Except for test cases, this function should compute some cryptographic hash of + * the message, the key, the pubkey, the algorithm description, and data. + */ +typedef int (*secp256k1_nonce_function_hardened)( + unsigned char *nonce32, + const unsigned char *msg, + size_t msglen, + const unsigned char *key32, + const unsigned char *xonly_pk32, + const unsigned char *algo, + size_t algolen, + void *data +); + +/** An implementation of the nonce generation function as defined in Bitcoin + * Improvement Proposal 340 "Schnorr Signatures for secp256k1" + * (https://github.com/bitcoin/bips/blob/master/bip-0340.mediawiki). + * + * If a data pointer is passed, it is assumed to be a pointer to 32 bytes of + * auxiliary random data as defined in BIP-340. If the data pointer is NULL, + * the nonce derivation procedure follows BIP-340 by setting the auxiliary + * random data to zero. The algo argument must be non-NULL, otherwise the + * function will fail and return 0. The hash will be tagged with algo. + * Therefore, to create BIP-340 compliant signatures, algo must be set to + * "BIP0340/nonce" and algolen to 13. + */ +SECP256K1_API const secp256k1_nonce_function_hardened secp256k1_nonce_function_bip340; + +/** Data structure that contains additional arguments for schnorrsig_sign_custom. + * + * A schnorrsig_extraparams structure object can be initialized correctly by + * setting it to SECP256K1_SCHNORRSIG_EXTRAPARAMS_INIT. + * + * Members: + * magic: set to SECP256K1_SCHNORRSIG_EXTRAPARAMS_MAGIC at initialization + * and has no other function than making sure the object is + * initialized. + * noncefp: pointer to a nonce generation function. If NULL, + * secp256k1_nonce_function_bip340 is used + * ndata: pointer to arbitrary data used by the nonce generation function + * (can be NULL). If it is non-NULL and + * secp256k1_nonce_function_bip340 is used, then ndata must be a + * pointer to 32-byte auxiliary randomness as per BIP-340. + */ +typedef struct secp256k1_schnorrsig_extraparams { + unsigned char magic[4]; + secp256k1_nonce_function_hardened noncefp; + void *ndata; +} secp256k1_schnorrsig_extraparams; + +#define SECP256K1_SCHNORRSIG_EXTRAPARAMS_MAGIC { 0xda, 0x6f, 0xb3, 0x8c } +#define SECP256K1_SCHNORRSIG_EXTRAPARAMS_INIT {\ + SECP256K1_SCHNORRSIG_EXTRAPARAMS_MAGIC,\ + NULL,\ + NULL\ +} + +/** Create a Schnorr signature. + * + * Does _not_ strictly follow BIP-340 because it does not verify the resulting + * signature. Instead, you can manually use secp256k1_schnorrsig_verify and + * abort if it fails. + * + * This function only signs 32-byte messages. If you have messages of a + * different size (or the same size but without a context-specific tag + * prefix), it is recommended to create a 32-byte message hash with + * secp256k1_tagged_sha256 and then sign the hash. Tagged hashing allows + * providing an context-specific tag for domain separation. This prevents + * signatures from being valid in multiple contexts by accident. + * + * Returns 1 on success, 0 on failure. + * Args: ctx: pointer to a context object (not secp256k1_context_static). + * Out: sig64: pointer to a 64-byte array to store the serialized signature. + * In: msg32: the 32-byte message being signed. + * keypair: pointer to an initialized keypair. + * aux_rand32: 32 bytes of fresh randomness. While recommended to provide + * this, it is only supplemental to security and can be NULL. A + * NULL argument is treated the same as an all-zero one. See + * BIP-340 "Default Signing" for a full explanation of this + * argument and for guidance if randomness is expensive. + */ +SECP256K1_API int secp256k1_schnorrsig_sign32( + const secp256k1_context *ctx, + unsigned char *sig64, + const unsigned char *msg32, + const secp256k1_keypair *keypair, + const unsigned char *aux_rand32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); + +/** Same as secp256k1_schnorrsig_sign32, but DEPRECATED. Will be removed in + * future versions. */ +SECP256K1_API int secp256k1_schnorrsig_sign( + const secp256k1_context *ctx, + unsigned char *sig64, + const unsigned char *msg32, + const secp256k1_keypair *keypair, + const unsigned char *aux_rand32 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4) + SECP256K1_DEPRECATED("Use secp256k1_schnorrsig_sign32 instead"); + +/** Create a Schnorr signature with a more flexible API. + * + * Same arguments as secp256k1_schnorrsig_sign except that it allows signing + * variable length messages and accepts a pointer to an extraparams object that + * allows customizing signing by passing additional arguments. + * + * Equivalent to secp256k1_schnorrsig_sign32(..., aux_rand32) if msglen is 32 + * and extraparams is initialized as follows: + * ``` + * secp256k1_schnorrsig_extraparams extraparams = SECP256K1_SCHNORRSIG_EXTRAPARAMS_INIT; + * extraparams.ndata = (unsigned char*)aux_rand32; + * ``` + * + * Returns 1 on success, 0 on failure. + * Args: ctx: pointer to a context object (not secp256k1_context_static). + * Out: sig64: pointer to a 64-byte array to store the serialized signature. + * In: msg: the message being signed. Can only be NULL if msglen is 0. + * msglen: length of the message. + * keypair: pointer to an initialized keypair. + * extraparams: pointer to an extraparams object (can be NULL). + */ +SECP256K1_API int secp256k1_schnorrsig_sign_custom( + const secp256k1_context *ctx, + unsigned char *sig64, + const unsigned char *msg, + size_t msglen, + const secp256k1_keypair *keypair, + secp256k1_schnorrsig_extraparams *extraparams +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(5); + +/** Verify a Schnorr signature. + * + * Returns: 1: correct signature + * 0: incorrect signature + * Args: ctx: pointer to a context object. + * In: sig64: pointer to the 64-byte signature to verify. + * msg: the message being verified. Can only be NULL if msglen is 0. + * msglen: length of the message + * pubkey: pointer to an x-only public key to verify with + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_schnorrsig_verify( + const secp256k1_context *ctx, + const unsigned char *sig64, + const unsigned char *msg, + size_t msglen, + const secp256k1_xonly_pubkey *pubkey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(5); + +#ifdef __cplusplus +} +#endif + +#endif /* SECP256K1_SCHNORRSIG_H */ diff --git a/example/android/third_party/secp256k1/libsecp256k1.a b/example/android/third_party/secp256k1/libsecp256k1.a deleted file mode 100644 index 0b3daaab..00000000 Binary files a/example/android/third_party/secp256k1/libsecp256k1.a and /dev/null differ diff --git a/example/android/third_party/secp256k1/libsecp256k1.so b/example/android/third_party/secp256k1/libsecp256k1.so deleted file mode 100644 index 784af261..00000000 Binary files a/example/android/third_party/secp256k1/libsecp256k1.so and /dev/null differ diff --git a/example/android/third_party/secp256k1/x86-64/libsecp256k1.a b/example/android/third_party/secp256k1/x86-64/libsecp256k1.a index 9f1a28d8..869dc5e6 100644 Binary files a/example/android/third_party/secp256k1/x86-64/libsecp256k1.a and b/example/android/third_party/secp256k1/x86-64/libsecp256k1.a differ diff --git a/example/android/third_party/secp256k1/x86-64/libsecp256k1.so b/example/android/third_party/secp256k1/x86-64/libsecp256k1.so deleted file mode 100644 index 077f47ac..00000000 Binary files a/example/android/third_party/secp256k1/x86-64/libsecp256k1.so and /dev/null differ diff --git a/keyring/keyring.cpp b/keyring/keyring.cpp index 0f45879d..a31d173e 100644 --- a/keyring/keyring.cpp +++ b/keyring/keyring.cpp @@ -28,7 +28,7 @@ namespace ton { namespace keyring { KeyringImpl::PrivateKeyDescr::PrivateKeyDescr(PrivateKey private_key, bool is_temp) - : public_key(private_key.compute_public_key()), is_temp(is_temp) { + : public_key(private_key.compute_public_key()), private_key(private_key), is_temp(is_temp) { auto D = private_key.create_decryptor_async(); D.ensure(); decryptor_sign = D.move_as_ok(); @@ -190,6 +190,16 @@ void KeyringImpl::decrypt_message(PublicKeyHash key_hash, td::BufferSlice data, } } +void KeyringImpl::export_all_private_keys(td::Promise> promise) { + std::vector keys; + for (auto& [_, descr] : map_) { + if (!descr->is_temp && descr->private_key.exportable()) { + keys.push_back(descr->private_key); + } + } + promise.set_value(std::move(keys)); +} + td::actor::ActorOwn Keyring::create(std::string db_root) { return td::actor::create_actor("keyring", db_root); } diff --git a/keyring/keyring.h b/keyring/keyring.h index 044d8d29..3b9064a7 100644 --- a/keyring/keyring.h +++ b/keyring/keyring.h @@ -44,6 +44,8 @@ class Keyring : public td::actor::Actor { virtual void decrypt_message(PublicKeyHash key_hash, td::BufferSlice data, td::Promise promise) = 0; + virtual void export_all_private_keys(td::Promise> promise) = 0; + static td::actor::ActorOwn create(std::string db_root); }; diff --git a/keyring/keyring.hpp b/keyring/keyring.hpp index ec658305..eca9073a 100644 --- a/keyring/keyring.hpp +++ b/keyring/keyring.hpp @@ -33,6 +33,7 @@ class KeyringImpl : public Keyring { td::actor::ActorOwn decryptor_sign; td::actor::ActorOwn decryptor_decrypt; PublicKey public_key; + PrivateKey private_key; bool is_temp; PrivateKeyDescr(PrivateKey private_key, bool is_temp); }; @@ -56,6 +57,8 @@ class KeyringImpl : public Keyring { void decrypt_message(PublicKeyHash key_hash, td::BufferSlice data, td::Promise promise) override; + void export_all_private_keys(td::Promise> promise) override; + KeyringImpl(std::string db_root) : db_root_(db_root) { } diff --git a/lite-client/CMakeLists.txt b/lite-client/CMakeLists.txt index c6988cf5..b28a14e9 100644 --- a/lite-client/CMakeLists.txt +++ b/lite-client/CMakeLists.txt @@ -1,9 +1,10 @@ cmake_minimum_required(VERSION 3.5 FATAL_ERROR) -add_library(lite-client-common STATIC lite-client-common.cpp lite-client-common.h) +add_library(lite-client-common STATIC lite-client-common.cpp lite-client-common.h ext-client.cpp ext-client.h + query-utils.hpp query-utils.cpp) target_link_libraries(lite-client-common PUBLIC tdactor adnllite tl_api tl_lite_api tl-lite-utils ton_crypto) -add_executable(lite-client lite-client.cpp lite-client.h) +add_executable(lite-client lite-client.cpp lite-client.h ext-client.h ext-client.cpp) target_link_libraries(lite-client tdutils tdactor adnllite tl_api tl_lite_api tl-lite-utils terminal lite-client-common git) install(TARGETS lite-client RUNTIME DESTINATION bin) diff --git a/lite-client/ext-client.cpp b/lite-client/ext-client.cpp new file mode 100644 index 00000000..a0e48e64 --- /dev/null +++ b/lite-client/ext-client.cpp @@ -0,0 +1,228 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "ext-client.h" +#include "td/utils/Random.h" +#include "ton/ton-shard.h" + +namespace liteclient { + +class ExtClientImpl : public ExtClient { + public: + ExtClientImpl(std::vector liteservers, td::unique_ptr callback, bool connect_to_all) + : callback_(std::move(callback)), connect_to_all_(connect_to_all) { + CHECK(!liteservers.empty()); + servers_.resize(liteservers.size()); + for (size_t i = 0; i < servers_.size(); ++i) { + servers_[i].config = std::move(liteservers[i]); + servers_[i].idx = i; + } + } + + void start_up() override { + LOG(INFO) << "Started ext client, " << servers_.size() << " liteservers"; + td::Random::Fast rnd; + td::random_shuffle(td::as_mutable_span(servers_), rnd); + server_indices_.resize(servers_.size()); + for (size_t i = 0; i < servers_.size(); ++i) { + server_indices_[servers_[i].idx] = i; + } + + if (connect_to_all_) { + for (size_t i = 0; i < servers_.size(); ++i) { + prepare_server(i, nullptr); + } + } + } + + void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, + td::Promise promise) override { + QueryInfo query_info = get_query_info(data); + TRY_RESULT_PROMISE(promise, server_idx, select_server(query_info)); + send_query_internal(std::move(name), std::move(data), std::move(query_info), server_idx, timeout, + std::move(promise)); + } + + void send_query_to_server(std::string name, td::BufferSlice data, size_t server_idx, td::Timestamp timeout, + td::Promise promise) override { + if (server_idx >= servers_.size()) { + promise.set_error(td::Status::Error(PSTRING() << "server idx " << server_idx << " is too big")); + return; + } + server_idx = server_indices_[server_idx]; + QueryInfo query_info = get_query_info(data); + prepare_server(server_idx, &query_info); + send_query_internal(std::move(name), std::move(data), std::move(query_info), server_idx, timeout, + std::move(promise)); + } + + void get_servers_status(td::Promise> promise) override { + std::vector status(servers_.size()); + for (const Server& s : servers_) { + status[s.idx] = s.alive; + } + promise.set_result(std::move(status)); + } + + void reset_servers() override { + LOG(INFO) << "Force resetting all liteservers"; + for (Server& server : servers_) { + server.alive = false; + server.timeout = {}; + server.ignore_until = {}; + server.client.reset(); + } + } + + private: + void send_query_internal(std::string name, td::BufferSlice data, QueryInfo query_info, size_t server_idx, + td::Timestamp timeout, td::Promise promise) { + auto& server = servers_[server_idx]; + CHECK(!server.client.empty()); + if (!connect_to_all_) { + alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); + } + td::Promise P = [SelfId = actor_id(this), server_idx, + promise = std::move(promise)](td::Result R) mutable { + if (R.is_error() && + (R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) { + td::actor::send_closure(SelfId, &ExtClientImpl::on_server_error, server_idx); + } + promise.set_result(std::move(R)); + }; + LOG(DEBUG) << "Sending query " << query_info.to_str() << " to server #" << server.idx << " (" + << server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")"; + send_closure(server.client, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout, + std::move(P)); + } + + td::Result select_server(const QueryInfo& query_info) { + for (size_t i = 0; i < servers_.size(); ++i) { + if (servers_[i].alive && servers_[i].config.accepts_query(query_info)) { + return i; + } + } + size_t server_idx = servers_.size(); + int cnt = 0; + int best_priority = -1; + for (size_t i = 0; i < servers_.size(); ++i) { + Server& server = servers_[i]; + if (!server.config.accepts_query(query_info)) { + continue; + } + int priority = 0; + priority += (server.ignore_until && !server.ignore_until.is_in_past() ? 0 : 10); + if (priority < best_priority) { + continue; + } + if (priority > best_priority) { + best_priority = priority; + cnt = 0; + } + if (td::Random::fast(0, cnt) == 0) { + server_idx = i; + } + ++cnt; + } + if (server_idx == servers_.size()) { + return td::Status::Error(PSTRING() << "no liteserver for query " << query_info.to_str()); + } + prepare_server(server_idx, &query_info); + return server_idx; + } + + void prepare_server(size_t server_idx, const QueryInfo* query_info) { + Server& server = servers_[server_idx]; + if (server.alive) { + return; + } + server.alive = true; + server.ignore_until = {}; + if (!connect_to_all_) { + alarm_timestamp().relax(server.timeout = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT)); + } + if (!server.client.empty()) { + return; + } + + class Callback : public ton::adnl::AdnlExtClient::Callback { + public: + explicit Callback(td::actor::ActorId parent, size_t idx) : parent_(std::move(parent)), idx_(idx) { + } + void on_ready() override { + } + void on_stop_ready() override { + td::actor::send_closure(parent_, &ExtClientImpl::on_server_error, idx_); + } + + private: + td::actor::ActorId parent_; + size_t idx_; + }; + LOG(INFO) << "Connecting to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() << ":" + << server.config.addr.get_port() << ") for query " << (query_info ? query_info->to_str() : "[none]"); + server.client = ton::adnl::AdnlExtClient::create(server.config.adnl_id, server.config.addr, + std::make_unique(actor_id(this), server_idx)); + } + + struct Server { + LiteServerConfig config; + size_t idx = 0; + td::actor::ActorOwn client; + bool alive = false; + td::Timestamp timeout = td::Timestamp::never(); + td::Timestamp ignore_until = td::Timestamp::never(); + }; + std::vector servers_; + std::vector server_indices_; + + td::unique_ptr callback_; + bool connect_to_all_ = false; + static constexpr double MAX_NO_QUERIES_TIMEOUT = 100.0; + static constexpr double BAD_SERVER_TIMEOUT = 30.0; + + void alarm() override { + if (connect_to_all_) { + return; + } + for (Server& server : servers_) { + if (server.timeout && server.timeout.is_in_past()) { + LOG(INFO) << "Closing connection to liteserver #" << server.idx << " (" << server.config.addr.get_ip_str() + << ":" << server.config.addr.get_port() << ")"; + server.client.reset(); + server.alive = false; + server.ignore_until = {}; + } + } + } + + void on_server_error(size_t idx) { + servers_[idx].alive = false; + servers_[idx].ignore_until = td::Timestamp::in(BAD_SERVER_TIMEOUT); + } +}; + +td::actor::ActorOwn ExtClient::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, + td::unique_ptr callback) { + return create({LiteServerConfig{dst, dst_addr}}, std::move(callback)); +} + +td::actor::ActorOwn ExtClient::create(std::vector liteservers, + td::unique_ptr callback, bool connect_to_all) { + return td::actor::create_actor("ExtClient", std::move(liteservers), std::move(callback), + connect_to_all); +} +} // namespace liteclient diff --git a/lite-client/ext-client.h b/lite-client/ext-client.h new file mode 100644 index 00000000..ef4523fd --- /dev/null +++ b/lite-client/ext-client.h @@ -0,0 +1,48 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "td/actor/actor.h" +#include "ton/ton-types.h" +#include "adnl/adnl-ext-client.h" +#include "query-utils.hpp" + +namespace liteclient { +class ExtClient : public td::actor::Actor { + public: + class Callback { + public: + virtual ~Callback() = default; + }; + + virtual void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, + td::Promise promise) = 0; + virtual void send_query_to_server(std::string name, td::BufferSlice data, size_t server_idx, td::Timestamp timeout, + td::Promise promise) { + promise.set_error(td::Status::Error("not supported")); + } + virtual void get_servers_status(td::Promise> promise) { + promise.set_error(td::Status::Error("not supported")); + } + virtual void reset_servers() { + } + + static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, + td::unique_ptr callback); + static td::actor::ActorOwn create(std::vector liteservers, + td::unique_ptr callback, bool connect_to_all = false); +}; +} // namespace liteclient diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 1a4201a7..1050e6d2 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -29,22 +29,16 @@ #include "lite-client-common.h" -#include "adnl/adnl-ext-client.h" #include "tl-utils/lite-utils.hpp" #include "auto/tl/ton_api_json.h" #include "auto/tl/lite_api.hpp" #include "td/utils/OptionParser.h" #include "td/utils/Time.h" #include "td/utils/filesystem.h" -#include "td/utils/format.h" #include "td/utils/Random.h" #include "td/utils/crypto.h" -#include "td/utils/overloaded.h" #include "td/utils/port/signals.h" -#include "td/utils/port/stacktrace.h" -#include "td/utils/port/StdStreams.h" #include "td/utils/port/FileFd.h" -#include "terminal/terminal.h" #include "ton/lite-tl.hpp" #include "block/block-db.h" #include "block/block.h" @@ -58,18 +52,14 @@ #include "vm/vm.h" #include "vm/cp0.h" #include "vm/memo.h" -#include "ton/ton-shard.h" -#include "openssl/rand.hpp" #include "crypto/vm/utils.h" #include "crypto/common/util.h" #include "common/checksum.h" #if TD_DARWIN || TD_LINUX #include -#include #endif #include -#include #include "git.h" using namespace std::literals::string_literals; @@ -77,24 +67,6 @@ using td::Ref; int verbosity; -std::unique_ptr TestNode::make_callback() { - class Callback : public ton::adnl::AdnlExtClient::Callback { - public: - void on_ready() override { - td::actor::send_closure(id_, &TestNode::conn_ready); - } - void on_stop_ready() override { - td::actor::send_closure(id_, &TestNode::conn_closed); - } - Callback(td::actor::ActorId id) : id_(std::move(id)) { - } - - private: - td::actor::ActorId id_; - }; - return std::make_unique(actor_id(this)); -} - void TestNode::run() { class Cb : public td::TerminalIO::Callback { public: @@ -110,19 +82,20 @@ void TestNode::run() { io_ = td::TerminalIO::create("> ", readline_enabled_, ex_mode_, std::make_unique(actor_id(this))); td::actor::send_closure(io_, &td::TerminalIO::set_log_interface); - if (remote_public_key_.empty()) { + std::vector servers; + if (!single_remote_public_key_.empty()) { // Use single provided liteserver + servers.push_back( + liteclient::LiteServerConfig{ton::adnl::AdnlNodeIdFull{single_remote_public_key_}, single_remote_addr_}); + td::TerminalIO::out() << "using liteserver " << single_remote_addr_ << "\n"; + } else { auto G = td::read_file(global_config_).move_as_ok(); auto gc_j = td::json_decode(G.as_slice()).move_as_ok(); ton::ton_api::liteclient_config_global gc; ton::ton_api::from_json(gc, gc_j.get_object()).ensure(); - CHECK(gc.liteservers_.size() > 0); - auto idx = liteserver_idx_ >= 0 ? liteserver_idx_ - : td::Random::fast(0, static_cast(gc.liteservers_.size() - 1)); - CHECK(idx >= 0 && static_cast(idx) <= gc.liteservers_.size()); - auto& cli = gc.liteservers_[idx]; - remote_addr_.init_host_port(td::IPAddress::ipv4_to_str(cli->ip_), cli->port_).ensure(); - remote_public_key_ = ton::PublicKey{cli->id_}; - td::TerminalIO::out() << "using liteserver " << idx << " with addr " << remote_addr_ << "\n"; + auto r_servers = liteclient::LiteServerConfig::parse_global_config(gc); + r_servers.ensure(); + servers = r_servers.move_as_ok(); + if (gc.validator_ && gc.validator_->zero_state_) { zstate_id_.workchain = gc.validator_->zero_state_->workchain_; if (zstate_id_.workchain != ton::workchainInvalid) { @@ -131,10 +104,19 @@ void TestNode::run() { td::TerminalIO::out() << "zerostate set to " << zstate_id_.to_str() << "\n"; } } - } - client_ = - ton::adnl::AdnlExtClient::create(ton::adnl::AdnlNodeIdFull{remote_public_key_}, remote_addr_, make_callback()); + if (single_liteserver_idx_ != -1) { // Use single liteserver from config + CHECK(single_liteserver_idx_ >= 0 && (size_t)single_liteserver_idx_ < servers.size()); + td::TerminalIO::out() << "using liteserver #" << single_liteserver_idx_ << " with addr " + << servers[single_liteserver_idx_].addr << "\n"; + servers = {servers[single_liteserver_idx_]}; + } + } + CHECK(!servers.empty()); + client_ = liteclient::ExtClient::create(std::move(servers), nullptr); + ready_ = true; + + run_init_queries(); } void TestNode::got_result(td::Result R, td::Promise promise) { @@ -191,8 +173,8 @@ bool TestNode::envelope_send_query(td::BufferSlice query, td::Promise(std::move(query)), true); - td::actor::send_closure(client_, &ton::adnl::AdnlExtClient::send_query, "query", std::move(b), - td::Timestamp::in(10.0), std::move(P)); + td::actor::send_closure(client_, &liteclient::ExtClient::send_query, "query", std::move(b), td::Timestamp::in(10.0), + std::move(P)); return true; } @@ -319,9 +301,10 @@ bool TestNode::get_server_time() { if (F.is_error()) { LOG(ERROR) << "cannot parse answer to liteServer.getTime"; } else { - server_time_ = F.move_as_ok()->now_; - server_time_got_at_ = now(); - LOG(INFO) << "server time is " << server_time_ << " (delta " << server_time_ - server_time_got_at_ << ")"; + mc_server_time_ = F.move_as_ok()->now_; + mc_server_time_got_at_ = now(); + LOG(INFO) << "server time is " << mc_server_time_ << " (delta " << mc_server_time_ - mc_server_time_got_at_ + << ")"; } } }); @@ -335,7 +318,7 @@ bool TestNode::get_server_version(int mode) { }; void TestNode::got_server_version(td::Result res, int mode) { - server_ok_ = false; + mc_server_ok_ = false; if (res.is_error()) { LOG(ERROR) << "cannot get server version and time (server too old?)"; } else { @@ -344,11 +327,11 @@ void TestNode::got_server_version(td::Result res, int mode) { LOG(ERROR) << "cannot parse answer to liteServer.getVersion"; } else { auto a = F.move_as_ok(); - set_server_version(a->version_, a->capabilities_); - set_server_time(a->now_); + set_mc_server_version(a->version_, a->capabilities_); + set_mc_server_time(a->now_); } } - if (!server_ok_) { + if (!mc_server_ok_) { LOG(ERROR) << "server version is too old (at least " << (min_ls_version >> 8) << "." << (min_ls_version & 0xff) << " with capabilities " << min_ls_capabilities << " required), some queries are unavailable"; } @@ -357,24 +340,24 @@ void TestNode::got_server_version(td::Result res, int mode) { } } -void TestNode::set_server_version(td::int32 version, td::int64 capabilities) { - if (server_version_ != version || server_capabilities_ != capabilities) { - server_version_ = version; - server_capabilities_ = capabilities; - LOG(WARNING) << "server version is " << (server_version_ >> 8) << "." << (server_version_ & 0xff) - << ", capabilities " << server_capabilities_; +void TestNode::set_mc_server_version(td::int32 version, td::int64 capabilities) { + if (mc_server_version_ != version || mc_server_capabilities_ != capabilities) { + mc_server_version_ = version; + mc_server_capabilities_ = capabilities; + LOG(WARNING) << "server version is " << (mc_server_version_ >> 8) << "." << (mc_server_version_ & 0xff) + << ", capabilities " << mc_server_capabilities_; } - server_ok_ = (server_version_ >= min_ls_version) && !(~server_capabilities_ & min_ls_capabilities); + mc_server_ok_ = (mc_server_version_ >= min_ls_version) && !(~mc_server_capabilities_ & min_ls_capabilities); } -void TestNode::set_server_time(int server_utime) { - server_time_ = server_utime; - server_time_got_at_ = now(); - LOG(INFO) << "server time is " << server_time_ << " (delta " << server_time_ - server_time_got_at_ << ")"; +void TestNode::set_mc_server_time(int server_utime) { + mc_server_time_ = server_utime; + mc_server_time_got_at_ = now(); + LOG(INFO) << "server time is " << mc_server_time_ << " (delta " << mc_server_time_ - mc_server_time_got_at_ << ")"; } bool TestNode::get_server_mc_block_id() { - int mode = (server_capabilities_ & 2) ? 0 : -1; + int mode = (mc_server_capabilities_ & 2) ? 0 : -1; if (mode < 0) { auto b = ton::serialize_tl_object(ton::create_tl_object(), true); return envelope_send_query(std::move(b), [Self = actor_id(this)](td::Result res) -> void { @@ -448,8 +431,8 @@ void TestNode::got_server_mc_block_id(ton::BlockIdExt blkid, ton::ZeroStateIdExt void TestNode::got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateIdExt zstateid, int mode, int version, long long capabilities, int last_utime, int server_now) { - set_server_version(version, capabilities); - set_server_time(server_now); + set_mc_server_version(version, capabilities); + set_mc_server_time(server_now); if (last_utime > server_now) { LOG(WARNING) << "server claims to have a masterchain block " << blkid.to_str() << " created at " << last_utime << " (" << last_utime - server_now << " seconds in the future)"; @@ -457,10 +440,10 @@ void TestNode::got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateI LOG(WARNING) << "server appears to be out of sync: its newest masterchain block is " << blkid.to_str() << " created at " << last_utime << " (" << server_now - last_utime << " seconds ago according to the server's clock)"; - } else if (last_utime < server_time_got_at_ - 60) { + } else if (last_utime < mc_server_time_got_at_ - 60) { LOG(WARNING) << "either the server is out of sync, or the local clock is set incorrectly: the newest masterchain " "block known to server is " - << blkid.to_str() << " created at " << last_utime << " (" << server_now - server_time_got_at_ + << blkid.to_str() << " created at " << last_utime << " (" << server_now - mc_server_time_got_at_ << " seconds ago according to the local clock)"; } got_server_mc_block_id(blkid, zstateid, last_utime); @@ -926,7 +909,7 @@ bool TestNode::show_help(std::string command) { "saveaccount[code|data] []\tSaves into specified file the most recent state " "(StateInit) or just the code or data of specified account; is in " "[:] format\n" - "runmethod[full] [] ...\tRuns GET method of account " + "runmethod[full] [] ...\tRuns GET method of account " " " "with specified parameters\n" "dnsresolve [] []\tResolves a domain starting from root dns smart contract\n" @@ -2244,7 +2227,7 @@ void TestNode::run_smc_method(int mode, ton::BlockIdExt ref_blk, ton::BlockIdExt // auto log = create_vm_log(ctx.error_stream ? &ostream_logger : nullptr); vm::GasLimits gas{gas_limit}; LOG(DEBUG) << "creating VM"; - vm::VmState vm{code, std::move(stack), gas, 1, data, vm::VmLog()}; + vm::VmState vm{code, ton::SUPPORTED_VERSION, std::move(stack), gas, 1, data, vm::VmLog()}; vm.set_c7(liteclient::prepare_vm_c7(info.gen_utime, info.gen_lt, td::make_ref(acc.addr->clone()), balance)); // tuple with SmartContractInfo // vm.incr_stack_trace(1); // enable stack dump after each step diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index 90a2fb8a..721d2b20 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -26,6 +26,7 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "ext-client.h" #include "adnl/adnl-ext-client.h" #include "tl-utils/tl-utils.hpp" #include "ton/ton-types.h" @@ -46,22 +47,24 @@ class TestNode : public td::actor::Actor { min_ls_version = 0x101, min_ls_capabilities = 1 }; // server version >= 1.1, capabilities at least +1 = build proof chains - td::actor::ActorOwn client_; + td::actor::ActorOwn client_; td::actor::ActorOwn io_; + bool ready_ = false; + + td::int32 single_liteserver_idx_ = -1; + td::IPAddress single_remote_addr_; + ton::PublicKey single_remote_public_key_; bool readline_enabled_ = true; - bool server_ok_ = false; - td::int32 liteserver_idx_ = -1; int print_limit_ = 1024; - bool ready_ = false; - bool inited_ = false; std::string db_root_; - int server_time_ = 0; - int server_time_got_at_ = 0; - int server_version_ = 0; - long long server_capabilities_ = 0; + int mc_server_time_ = 0; + int mc_server_time_got_at_ = 0; + int mc_server_version_ = 0; + long long mc_server_capabilities_ = 0; + bool mc_server_ok_ = false; ton::ZeroStateIdExt zstate_id_; ton::BlockIdExt mc_last_id_; @@ -76,9 +79,6 @@ class TestNode : public td::actor::Actor { const char *parse_ptr_, *parse_end_; td::Status error_; - td::IPAddress remote_addr_; - ton::PublicKey remote_public_key_; - std::vector known_blk_ids_; std::size_t shown_blk_ids_ = 0; @@ -89,8 +89,6 @@ class TestNode : public td::actor::Actor { std::map> cell_cache_; - std::unique_ptr make_callback(); - using creator_stats_func_t = std::function; @@ -183,8 +181,8 @@ class TestNode : public td::actor::Actor { void got_server_mc_block_id(ton::BlockIdExt blkid, ton::ZeroStateIdExt zstateid, int created_at); void got_server_mc_block_id_ext(ton::BlockIdExt blkid, ton::ZeroStateIdExt zstateid, int mode, int version, long long capabilities, int last_utime, int server_now); - void set_server_version(td::int32 version, td::int64 capabilities); - void set_server_time(int server_utime); + void set_mc_server_version(td::int32 version, td::int64 capabilities); + void set_mc_server_time(int server_utime); bool request_block(ton::BlockIdExt blkid); bool request_state(ton::BlockIdExt blkid); void got_mc_block(ton::BlockIdExt blkid, td::BufferSlice data); @@ -370,9 +368,6 @@ class TestNode : public td::actor::Actor { bool parse_shard_id(ton::ShardIdFull& shard); bool parse_block_id_ext(ton::BlockIdExt& blkid, bool allow_incomplete = false); bool parse_block_id_ext(std::string blk_id_string, ton::BlockIdExt& blkid, bool allow_incomplete = false) const; - bool parse_stack_value(td::Slice str, vm::StackEntry& value); - bool parse_stack_value(vm::StackEntry& value); - bool parse_stack_values(std::vector& values); bool register_blkid(const ton::BlockIdExt& blkid); bool show_new_blkids(bool all = false); bool complete_blkid(ton::BlockId partial_blkid, ton::BlockIdExt& complete_blkid) const; @@ -391,16 +386,6 @@ class TestNode : public td::actor::Actor { static const tlb::TypenameLookup& get_tlb_dict(); public: - void conn_ready() { - LOG(ERROR) << "conn ready"; - ready_ = true; - if (!inited_) { - run_init_queries(); - } - } - void conn_closed() { - ready_ = false; - } void set_global_config(std::string str) { global_config_ = str; } @@ -411,10 +396,10 @@ class TestNode : public td::actor::Actor { readline_enabled_ = value; } void set_liteserver_idx(td::int32 idx) { - liteserver_idx_ = idx; + single_liteserver_idx_ = idx; } void set_remote_addr(td::IPAddress addr) { - remote_addr_ = addr; + single_remote_addr_ = addr; } void set_public_key(td::BufferSlice file_name) { auto R = [&]() -> td::Result { @@ -425,7 +410,7 @@ class TestNode : public td::actor::Actor { if (R.is_error()) { LOG(FATAL) << "bad server public key: " << R.move_as_error(); } - remote_public_key_ = R.move_as_ok(); + single_remote_public_key_ = R.move_as_ok(); } void decode_public_key(td::BufferSlice b64_key) { auto R = [&]() -> td::Result { @@ -437,7 +422,7 @@ class TestNode : public td::actor::Actor { if (R.is_error()) { LOG(FATAL) << "bad b64 server public key: " << R.move_as_error(); } - remote_public_key_ = R.move_as_ok(); + single_remote_public_key_ = R.move_as_ok(); } void set_fail_timeout(td::Timestamp ts) { fail_timeout_ = ts; @@ -475,8 +460,7 @@ class TestNode : public td::actor::Actor { bool envelope_send_query(td::BufferSlice query, td::Promise promise); void parse_line(td::BufferSlice data); - TestNode() { - } + TestNode() = default; void run(); }; diff --git a/lite-client/query-utils.cpp b/lite-client/query-utils.cpp new file mode 100644 index 00000000..b46d46a5 --- /dev/null +++ b/lite-client/query-utils.cpp @@ -0,0 +1,400 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "query-utils.hpp" + +#include "block-parse.h" +#include "td/utils/overloaded.h" +#include "tl-utils/common-utils.hpp" + +#include "block/block-auto.h" +#include "auto/tl/lite_api.hpp" +#include "overlay/overlay-broadcast.hpp" +#include "tl-utils/lite-utils.hpp" +#include "ton/lite-tl.hpp" +#include "ton/ton-shard.h" + +#include + +namespace liteclient { + +using namespace ton; + +std::string QueryInfo::to_str() const { + td::StringBuilder sb; + sb << "[ " << lite_query_name_by_id(query_id) << " " << shard_id.to_str(); + switch (type) { + case t_simple: + break; + case t_seqno: + sb << " seqno=" << value; + break; + case t_utime: + sb << " utime=" << value; + break; + case t_lt: + sb << " lt=" << value; + break; + case t_mc_seqno: + sb << " mc_seqno=" << value; + break; + } + sb << " ]"; + return sb.as_cslice().str(); +} + +QueryInfo get_query_info(td::Slice data) { + auto F = fetch_tl_object(data, true); + if (F.is_ok()) { + data = F.ok()->data_; + } else { + fetch_tl_prefix(data, true).ignore(); + } + fetch_tl_prefix(data, true).ignore(); + auto Q = fetch_tl_object(data, true); + if (Q.is_error()) { + return {}; + } + return get_query_info(*Q.ok()); +} + +QueryInfo get_query_info(const lite_api::Function& f) { + QueryInfo info; + info.query_id = f.get_id(); + auto from_block_id = [&](const tl_object_ptr& id) { + BlockIdExt block_id = create_block_id(id); + info.shard_id = block_id.shard_full(); + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno(); + }; + downcast_call( + const_cast(f), + td::overloaded([&](const lite_api::liteServer_getTime& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getVersion& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getMasterchainInfo& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getMasterchainInfoExt& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getBlock& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getBlockHeader& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getState& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getAccountState& q) { + BlockIdExt block_id = create_block_id(q.id_); + AccountIdPrefixFull acc_id_prefix = extract_addr_prefix(q.account_->workchain_, q.account_->id_); + info.shard_id = acc_id_prefix.as_leaf_shard(); + // See LiteQuery::perform_getAccountState + if (block_id.id.workchain != masterchainId) { + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno(); + } else if (block_id.id.seqno != ~0U) { + info.type = QueryInfo::t_mc_seqno; + info.value = block_id.seqno(); + } else { + info.type = QueryInfo::t_simple; + } + }, + [&](const lite_api::liteServer_getAccountStatePrunned& q) { + BlockIdExt block_id = create_block_id(q.id_); + AccountIdPrefixFull acc_id_prefix = extract_addr_prefix(q.account_->workchain_, q.account_->id_); + info.shard_id = acc_id_prefix.as_leaf_shard(); + // See LiteQuery::perform_getAccountState + if (block_id.id.workchain != masterchainId) { + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno(); + } else if (block_id.id.seqno != ~0U) { + info.type = QueryInfo::t_mc_seqno; + info.value = block_id.seqno(); + } else { + info.type = QueryInfo::t_simple; + } + }, + [&](const lite_api::liteServer_getOneTransaction& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getTransactions& q) { + AccountIdPrefixFull acc_id_prefix = extract_addr_prefix(q.account_->workchain_, q.account_->id_); + info.shard_id = acc_id_prefix.as_leaf_shard(); + info.type = QueryInfo::t_lt; + info.value = q.lt_; + }, + [&](const lite_api::liteServer_sendMessage& q) { + info.type = QueryInfo::t_simple; + auto r_root = vm::std_boc_deserialize(q.body_); + if (r_root.is_error()) { + return; + } + block::gen::CommonMsgInfo::Record_ext_in_msg_info msg_info; + if (!tlb::unpack_cell_inexact(r_root.ok(), msg_info)) { + return; + } + auto dest_prefix = block::tlb::MsgAddressInt::get_prefix(msg_info.dest); + if (!dest_prefix.is_valid()) { + return; + } + info.shard_id = dest_prefix.as_leaf_shard(); + }, + [&](const lite_api::liteServer_getShardInfo& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getAllShardsInfo& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_lookupBlock& q) { + BlockId block_id = create_block_id_simple(q.id_); + info.shard_id = block_id.shard_full(); + // See LiteQuery::perform_lookupBlock + if (q.mode_ & 1) { + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno; + } else if (q.mode_ == 2) { + info.type = QueryInfo::t_lt; + info.value = q.lt_; + } else if (q.mode_ == 4) { + info.type = QueryInfo::t_utime; + info.value = q.utime_; + } + }, + [&](const lite_api::liteServer_lookupBlockWithProof& q) { + BlockId block_id = create_block_id_simple(q.id_); + info.shard_id = block_id.shard_full(); + // See LiteQuery::perform_lookupBlockWithProof + if (q.mode_ & 1) { + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno; + } else if (q.mode_ == 2) { + info.type = QueryInfo::t_lt; + info.value = q.lt_; + } else if (q.mode_ == 4) { + info.type = QueryInfo::t_utime; + info.value = q.utime_; + } + }, + [&](const lite_api::liteServer_listBlockTransactions& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_listBlockTransactionsExt& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getConfigParams& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getConfigAll& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getBlockProof& q) { + info.shard_id = ShardIdFull{masterchainId}; + BlockIdExt from = create_block_id(q.known_block_); + // See LiteQuery::perform_getBlockProof + if ((q.mode_ & 1) && (q.mode_ & 0x1000)) { + BlockIdExt to = create_block_id(q.target_block_); // target_block is non-null if (mode & 1) + info.type = QueryInfo::t_seqno; + info.value = std::max(from.seqno(), to.seqno()); + } else { + info.type = QueryInfo::t_simple; + } + }, + [&](const lite_api::liteServer_getValidatorStats& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_runSmcMethod& q) { + BlockIdExt block_id = create_block_id(q.id_); + AccountIdPrefixFull acc_id_prefix = extract_addr_prefix(q.account_->workchain_, q.account_->id_); + info.shard_id = acc_id_prefix.as_leaf_shard(); + // See LiteQuery::perform_getAccountState + if (block_id.id.workchain != masterchainId) { + info.type = QueryInfo::t_seqno; + info.value = block_id.seqno(); + } else if (block_id.id.seqno != ~0U) { + info.type = QueryInfo::t_mc_seqno; + info.value = block_id.seqno(); + } else { + info.type = QueryInfo::t_simple; + } + }, + [&](const lite_api::liteServer_getLibraries& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getLibrariesWithProof& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getShardBlockProof& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_nonfinal_getCandidate& q) { /* t_simple */ }, + [&](const lite_api::liteServer_nonfinal_getValidatorGroups& q) { /* t_simple */ }, + [&](const lite_api::liteServer_getOutMsgQueueSizes& q) { + // This query is expected to be removed, as it is not fully compatible with separated liteservers + /* t_simple */ + }, + [&](const lite_api::liteServer_getBlockOutMsgQueueSize& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getDispatchQueueInfo& q) { from_block_id(q.id_); }, + [&](const lite_api::liteServer_getDispatchQueueMessages& q) { from_block_id(q.id_); }, + [&](const auto&) { /* t_simple */ })); + if (info.shard_id.workchain == masterchainId) { + info.shard_id.shard = shardIdAll; + } + if (!info.shard_id.is_valid_ext()) { + info.shard_id = ShardIdFull{masterchainId}; + info.type = QueryInfo::t_simple; + info.value = 0; + } + return info; +} + +bool LiteServerConfig::accepts_query(const QueryInfo& query_info) const { + if (is_full) { + return true; + } + for (const Slice& s : slices) { + if (s.accepts_query(query_info)) { + return true; + } + } + return false; +} + +bool LiteServerConfig::Slice::accepts_query(const QueryInfo& query_info) const { + if (unlimited) { + for (const ShardInfo& shard : shards_from) { + if (shard_intersects(shard.shard_id, query_info.shard_id)) { + return true; + } + } + return false; + } + if (!shards_from.empty()) { + bool from_ok = false; + DCHECK(shards_from[0].shard_id.is_masterchain()); + for (const ShardInfo& shard : shards_from) { + if (shard_intersects(shard.shard_id, query_info.shard_id)) { + switch (query_info.type) { + case QueryInfo::t_simple: + from_ok = true; + break; + case QueryInfo::t_seqno: + from_ok = shard.seqno <= query_info.value; + break; + case QueryInfo::t_utime: + from_ok = shard.utime <= query_info.value; + break; + case QueryInfo::t_lt: + from_ok = shard.lt <= query_info.value; + break; + case QueryInfo::t_mc_seqno: + from_ok = shards_from[0].seqno <= query_info.value; + break; + } + if (from_ok) { + break; + } + } + } + if (!from_ok) { + return false; + } + } + if (!shards_to.empty()) { + bool to_ok = false; + DCHECK(shards_to[0].shard_id.is_masterchain()); + for (const ShardInfo& shard : shards_to) { + if (shard_intersects(shard.shard_id, query_info.shard_id)) { + switch (query_info.type) { + case QueryInfo::t_simple: + break; + case QueryInfo::t_seqno: + to_ok = shard.seqno >= query_info.value; + break; + case QueryInfo::t_utime: + to_ok = shard.utime >= query_info.value; + break; + case QueryInfo::t_lt: + to_ok = shard.lt >= query_info.value; + break; + case QueryInfo::t_mc_seqno: + to_ok = shards_from[0].seqno >= query_info.value; + break; + } + if (to_ok) { + break; + } + } + } + if (!to_ok) { + return false; + } + } + return true; +} + +td::Result> LiteServerConfig::parse_global_config( + const ton_api::liteclient_config_global& config) { + std::vector servers; + for (const auto& f : config.liteservers_) { + LiteServerConfig server; + TRY_STATUS(server.addr.init_host_port(td::IPAddress::ipv4_to_str(f->ip_), f->port_)); + server.adnl_id = adnl::AdnlNodeIdFull{PublicKey{f->id_}}; + server.is_full = true; + servers.push_back(std::move(server)); + } + for (const auto& f : config.liteservers_v2_) { + LiteServerConfig server; + TRY_STATUS(server.addr.init_host_port(td::IPAddress::ipv4_to_str(f->ip_), f->port_)); + server.adnl_id = adnl::AdnlNodeIdFull{PublicKey{f->id_}}; + server.is_full = false; + for (const auto& slice_obj : f->slices_) { + Slice slice; + td::Status S = td::Status::OK(); + downcast_call(*slice_obj, + td::overloaded( + [&](const ton_api::liteserver_descV2_sliceSimple& s) { + slice.unlimited = true; + slice.shards_from.push_back({ShardIdFull{masterchainId}, 0, 0, 0}); + for (const auto& shard_obj : s.shards_) { + ShardIdFull shard_id = create_shard_id(shard_obj); + if (!shard_id.is_valid_ext()) { + S = td::Status::Error(PSTRING() << "invalid shard id " << shard_id.to_str()); + break; + } + if (!shard_id.is_masterchain()) { + slice.shards_from.push_back({shard_id, 0, 0, 0}); + } + } + }, + [&](const ton_api::liteserver_descV2_sliceTimed& s) { + auto parse_shards = + [](const std::vector>& shard_objs, + std::vector& shards) -> td::Status { + if (shard_objs.empty()) { + return td::Status::OK(); + } + size_t i = 0; + int mc_idx = -1; + for (const auto& shard_obj : shard_objs) { + ShardIdFull shard_id = create_shard_id(shard_obj->shard_id_); + if (!shard_id.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard id " << shard_id.to_str()); + } + if (shard_id.is_masterchain()) { + shard_id = ShardIdFull{masterchainId}; + if (mc_idx != -1) { + return td::Status::Error("duplicate masterchain shard in sliceTimed"); + } + mc_idx = (int)i; + } + shards.push_back({shard_id, (BlockSeqno)shard_obj->seqno_, (UnixTime)shard_obj->utime_, + (LogicalTime)shard_obj->lt_}); + ++i; + } + if (mc_idx == -1) { + return td::Status::Error("no masterchain shard in sliceTimed"); + } + std::swap(shards[0], shards[mc_idx]); + return td::Status::OK(); + }; + S = parse_shards(s.shards_from_, slice.shards_from); + if (S.is_ok()) { + S = parse_shards(s.shards_to_, slice.shards_to); + } + if (S.is_ok() && slice.shards_from.empty() && slice.shards_to.empty()) { + S = td::Status::Error("shards_from and shards_to are both empty"); + } + })); + TRY_STATUS(std::move(S)); + server.slices.push_back(slice); + } + + servers.push_back(std::move(server)); + } + return servers; +} + +} // namespace liteclient \ No newline at end of file diff --git a/lite-client/query-utils.hpp b/lite-client/query-utils.hpp new file mode 100644 index 00000000..28500e26 --- /dev/null +++ b/lite-client/query-utils.hpp @@ -0,0 +1,89 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "ton/ton-types.h" +#include "auto/tl/lite_api.h" +#include "td/utils/port/IPAddress.h" +#include "adnl/adnl-node-id.hpp" + +namespace liteclient { + +struct QueryInfo { + enum Type { t_simple, t_seqno, t_utime, t_lt, t_mc_seqno }; + int query_id = 0; + ton::ShardIdFull shard_id{ton::masterchainId}; + Type type = t_simple; + td::uint64 value = 0; + /* Query types and examples: + * t_simple - query to the recent blocks in a shard, or general info. value = 0. + * getTime, getMasterchainInfo (shard_id = masterchain) + * sendMessage + * getAccountState, runSmcMethod - when no block is given + * t_seqno - query to block with seqno in a shard. value = seqno. + * lookupBlock by seqno + * getBlock, getBlockHeader + * getAccountState, runSmcMethod - when shard block is given + * t_utime - query to a block with given unixtime in a shard. value = utime. + * lookupBlock by utime + * t_lt - query to a block with given lt in a shard. value = lt. + * lookupBlock by lt + * getTransactions + * t_mc_seqno - query to a block in a shard, masterchain seqno is given. value = mc_seqno. + * getAccountState, runSmcMethod - when mc block is given + */ + + std::string to_str() const; +}; + +QueryInfo get_query_info(td::Slice data); +QueryInfo get_query_info(const ton::lite_api::Function& f); + +struct LiteServerConfig { + private: + struct ShardInfo { + ton::ShardIdFull shard_id; + ton::BlockSeqno seqno; + ton::UnixTime utime; + ton::LogicalTime lt; + }; + + struct Slice { + std::vector shards_from, shards_to; + bool unlimited = false; + + bool accepts_query(const QueryInfo& query_info) const; + }; + + bool is_full = false; + std::vector slices; + + public: + ton::adnl::AdnlNodeIdFull adnl_id; + td::IPAddress addr; + + LiteServerConfig() = default; + LiteServerConfig(ton::adnl::AdnlNodeIdFull adnl_id, td::IPAddress addr) + : is_full(true), adnl_id(adnl_id), addr(addr) { + } + + bool accepts_query(const QueryInfo& query_info) const; + + static td::Result> parse_global_config( + const ton::ton_api::liteclient_config_global& config); +}; + +} // namespace liteclient diff --git a/overlay/overlay-fec.cpp b/overlay/overlay-fec.cpp index b29fce22..817d3b7c 100644 --- a/overlay/overlay-fec.cpp +++ b/overlay/overlay-fec.cpp @@ -32,7 +32,7 @@ void OverlayOutboundFecBroadcast::alarm() { fec_type_.size(), flags_, std::move(X.data), X.id, fec_type_, date_); } - alarm_timestamp() = td::Timestamp::in(0.010); + alarm_timestamp() = td::Timestamp::in(delay_); if (seqno_ >= to_send_) { stop(); @@ -46,8 +46,9 @@ void OverlayOutboundFecBroadcast::start_up() { OverlayOutboundFecBroadcast::OverlayOutboundFecBroadcast(td::BufferSlice data, td::uint32 flags, td::actor::ActorId overlay, - PublicKeyHash local_id) + PublicKeyHash local_id, double speed_multiplier) : flags_(flags) { + delay_ /= speed_multiplier; CHECK(data.size() <= (1 << 27)); local_id_ = local_id; overlay_ = std::move(overlay); @@ -63,9 +64,10 @@ OverlayOutboundFecBroadcast::OverlayOutboundFecBroadcast(td::BufferSlice data, t } td::actor::ActorId OverlayOutboundFecBroadcast::create( - td::BufferSlice data, td::uint32 flags, td::actor::ActorId overlay, PublicKeyHash local_id) { - return td::actor::create_actor(td::actor::ActorOptions().with_name("bcast"), - std::move(data), flags, overlay, local_id) + td::BufferSlice data, td::uint32 flags, td::actor::ActorId overlay, PublicKeyHash local_id, + double speed_multiplier) { + return td::actor::create_actor( + td::actor::ActorOptions().with_name("bcast"), std::move(data), flags, overlay, local_id, speed_multiplier) .release(); } diff --git a/overlay/overlay-fec.hpp b/overlay/overlay-fec.hpp index a9cc3634..b72e830e 100644 --- a/overlay/overlay-fec.hpp +++ b/overlay/overlay-fec.hpp @@ -37,6 +37,7 @@ class OverlayOutboundFecBroadcast : public td::actor::Actor { PublicKeyHash local_id_; Overlay::BroadcastDataHash data_hash_; td::uint32 flags_ = 0; + double delay_ = 0.010; td::int32 date_; std::unique_ptr encoder_; td::actor::ActorId overlay_; @@ -45,9 +46,9 @@ class OverlayOutboundFecBroadcast : public td::actor::Actor { public: static td::actor::ActorId create(td::BufferSlice data, td::uint32 flags, td::actor::ActorId overlay, - PublicKeyHash local_id); + PublicKeyHash local_id, double speed_multiplier = 1.0); OverlayOutboundFecBroadcast(td::BufferSlice data, td::uint32 flags, td::actor::ActorId overlay, - PublicKeyHash local_id); + PublicKeyHash local_id, double speed_multiplier = 1.0); void alarm() override; void start_up() override; diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index f062cc33..f24c6cbc 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -68,6 +68,9 @@ void OverlayManager::register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdS } overlays_[local_id][overlay_id] = OverlayDescription{std::move(overlay), std::move(cert)}; + if (!with_db_) { + return; + } auto P = td::PromiseCreator::lambda([id = overlays_[local_id][overlay_id].overlay.get()](td::Result R) { R.ensure(); @@ -417,13 +420,19 @@ OverlayManager::OverlayManager(std::string db_root, td::actor::ActorId kv = - std::make_shared(td::RocksDb::open(PSTRING() << db_root_ << "/overlays").move_as_ok()); - db_ = DbType{std::move(kv)}; + if (!db_root_.empty()) { + with_db_ = true; + std::shared_ptr kv = + std::make_shared(td::RocksDb::open(PSTRING() << db_root_ << "/overlays").move_as_ok()); + db_ = DbType{std::move(kv)}; + } } void OverlayManager::save_to_db(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, std::vector nodes) { + if (!with_db_) { + return; + } std::vector> nodes_vec; for (auto &n : nodes) { nodes_vec.push_back(n.tl()); @@ -564,7 +573,7 @@ td::Result> Certificate::create(tl_object_ptr max_size_) { return BroadcastCheckResult::Forbidden; } @@ -575,16 +584,16 @@ BroadcastCheckResult Certificate::check(PublicKeyHash node, OverlayIdShort overl return BroadcastCheckResult::Forbidden; } - auto R1 = issued_by_.get().create_encryptor(); - if (R1.is_error()) { - return BroadcastCheckResult::Forbidden; - } - auto E = R1.move_as_ok(); - - auto B = to_sign(overlay_id, node); - - if (E->check_signature(B.as_slice(), signature_.as_slice()).is_error()) { - return BroadcastCheckResult::Forbidden; + if (!skip_check_signature) { + auto R1 = issued_by_.get().create_encryptor(); + if (R1.is_error()) { + return BroadcastCheckResult::Forbidden; + } + auto E = R1.move_as_ok(); + auto B = to_sign(overlay_id, node); + if (E->check_signature(B.as_slice(), signature_.as_slice()).is_error()) { + return BroadcastCheckResult::Forbidden; + } } return (flags_ & CertificateFlags::Trusted) ? BroadcastCheckResult::Allowed : BroadcastCheckResult::NeedCheck; diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index 12206e04..68b033a3 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -131,6 +131,7 @@ class OverlayManager : public Overlays { td::actor::ActorId dht_node_; using DbType = td::KeyValueAsync; + bool with_db_ = false; DbType db_; class AdnlCallback : public adnl::Adnl::Callback { diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index 00b508fd..7def4a2d 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -213,7 +213,7 @@ void OverlayImpl::add_peer(OverlayNode node) { peer_list_.peers_.insert(id, OverlayPeer(std::move(node))); del_some_peers(); auto X = peer_list_.peers_.get(id); - if (X != nullptr && peer_list_.neighbours_.size() < max_neighbours() && + if (X != nullptr && !X->is_neighbour() && peer_list_.neighbours_.size() < max_neighbours() && !(X->get_node()->flags() & OverlayMemberFlags::DoNotReceiveBroadcasts) && X->get_id() != local_id_) { peer_list_.neighbours_.push_back(X->get_id()); X->set_neighbour(true); @@ -440,7 +440,7 @@ void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) { VLOG(OVERLAY_INFO) << this << ": adding new neighbour " << X->get_id(); peer_list_.neighbours_.push_back(X->get_id()); X->set_neighbour(true); - } else { + } else if (X->is_alive()) { CHECK(nodes_to_change > 0); auto i = td::Random::fast(0, static_cast(peer_list_.neighbours_.size()) - 1); auto Y = peer_list_.peers_.get(peer_list_.neighbours_[i]); diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 93ae801c..30a40b1c 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -63,7 +63,7 @@ td::actor::ActorOwn Overlay::create_private( return td::actor::create_actor( overlay_actor_name(overlay_id), keyring, adnl, manager, dht_node, local_id, std::move(overlay_id), OverlayType::FixedMemberList, std::move(nodes), std::vector(), OverlayMemberCertificate{}, - std::move(callback), std::move(rules), std::move(scope)); + std::move(callback), std::move(rules), std::move(scope), std::move(opts)); } td::actor::ActorOwn Overlay::create_semiprivate( @@ -99,6 +99,7 @@ OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor overlay_id_ = id_full_.compute_short_id(); frequent_dht_lookup_ = opts_.frequent_dht_lookup_; peer_list_.local_member_flags_ = opts_.local_overlay_member_flags_; + opts_.broadcast_speed_multiplier_ = std::max(opts_.broadcast_speed_multiplier_, 1e-9); VLOG(OVERLAY_INFO) << this << ": creating"; @@ -347,7 +348,12 @@ void OverlayImpl::alarm() { update_db_at_ = td::Timestamp::in(60.0); } - update_neighbours(0); + if (update_neighbours_at_.is_in_past()) { + update_neighbours(2); + update_neighbours_at_ = td::Timestamp::in(td::Random::fast(30.0, 120.0)); + } else { + update_neighbours(0); + } alarm_timestamp() = td::Timestamp::in(1.0); } else { update_neighbours(0); @@ -485,7 +491,8 @@ void OverlayImpl::send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td VLOG(OVERLAY_WARNING) << "broadcast source certificate is invalid"; return; } - OverlayOutboundFecBroadcast::create(std::move(data), flags, actor_id(this), send_as); + OverlayOutboundFecBroadcast::create(std::move(data), flags, actor_id(this), send_as, + opts_.broadcast_speed_multiplier_); } void OverlayImpl::print(td::StringBuilder &sb) { @@ -503,37 +510,44 @@ td::Status OverlayImpl::check_date(td::uint32 date) { return td::Status::OK(); } -BroadcastCheckResult OverlayImpl::check_source_eligible(const PublicKeyHash &source, const Certificate *cert, +BroadcastCheckResult OverlayImpl::check_source_eligible(const PublicKeyHash& source, const Certificate* cert, td::uint32 size, bool is_fec) { if (size == 0) { return BroadcastCheckResult::Forbidden; } - auto r = rules_.check_rules(source, size, is_fec); if (!cert || r == BroadcastCheckResult::Allowed) { return r; } + td::Bits256 cert_hash = get_tl_object_sha_bits256(cert->tl()); + auto cached_cert = checked_certificates_cache_.find(source); + bool cached = cached_cert != checked_certificates_cache_.end() && cached_cert->second->cert_hash == cert_hash; - auto r2 = cert->check(source, overlay_id_, static_cast(td::Clocks::system()), size, is_fec); + auto r2 = cert->check(source, overlay_id_, static_cast(td::Clocks::system()), size, is_fec, + /* skip_check_signature = */ cached); + if (r2 != BroadcastCheckResult::Forbidden) { + if (cached_cert == checked_certificates_cache_.end()) { + cached_cert = checked_certificates_cache_.emplace( + source, std::make_unique(source, cert_hash)).first; + } else { + cached_cert->second->cert_hash = cert_hash; + cached_cert->second->remove(); + } + checked_certificates_cache_lru_.put(cached_cert->second.get()); + while (checked_certificates_cache_.size() > max_checked_certificates_cache_size_) { + auto to_remove = (CachedCertificate*)checked_certificates_cache_lru_.get(); + CHECK(to_remove); + to_remove->remove(); + checked_certificates_cache_.erase(to_remove->source); + } + } r2 = broadcast_check_result_min(r2, rules_.check_rules(cert->issuer_hash(), size, is_fec)); return broadcast_check_result_max(r, r2); } -BroadcastCheckResult OverlayImpl::check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size, +BroadcastCheckResult OverlayImpl::check_source_eligible(PublicKey source, const Certificate* cert, td::uint32 size, bool is_fec) { - if (size == 0) { - return BroadcastCheckResult::Forbidden; - } - auto short_id = source.compute_short_id(); - - auto r = rules_.check_rules(short_id, size, is_fec); - if (!cert || r == BroadcastCheckResult::Allowed) { - return r; - } - - auto r2 = cert->check(short_id, overlay_id_, static_cast(td::Clocks::system()), size, is_fec); - r2 = broadcast_check_result_min(r2, rules_.check_rules(cert->issuer_hash(), size, is_fec)); - return broadcast_check_result_max(r, r2); + return check_source_eligible(source.compute_short_id(), cert, size, is_fec); } td::Status OverlayImpl::check_delivered(BroadcastHash hash) { diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 6476779b..41a04dec 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -391,6 +391,7 @@ class OverlayImpl : public Overlay { td::Timestamp next_dht_store_query_ = td::Timestamp::in(1.0); td::Timestamp update_db_at_; td::Timestamp update_throughput_at_; + td::Timestamp update_neighbours_at_; td::Timestamp last_throughput_update_; std::unique_ptr callback_; @@ -466,6 +467,19 @@ class OverlayImpl : public Overlay { TrafficStats total_traffic_responses, total_traffic_responses_ctr; OverlayOptions opts_; + + struct CachedCertificate : td::ListNode { + CachedCertificate(PublicKeyHash source, td::Bits256 cert_hash) + : source(source) + , cert_hash(cert_hash) { + } + + PublicKeyHash source; + td::Bits256 cert_hash; + }; + std::map> checked_certificates_cache_; + td::ListNode checked_certificates_cache_lru_; + size_t max_checked_certificates_cache_size_ = 1000; }; } // namespace overlay diff --git a/overlay/overlays.h b/overlay/overlays.h index cc112bc4..5eb63b13 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -149,7 +149,7 @@ class Certificate { td::BufferSlice to_sign(OverlayIdShort overlay_id, PublicKeyHash issued_to) const; BroadcastCheckResult check(PublicKeyHash node, OverlayIdShort overlay_id, td::int32 unix_time, td::uint32 size, - bool is_fec) const; + bool is_fec, bool skip_check_signature = false) const; tl_object_ptr tl() const; const PublicKey &issuer() const; const PublicKeyHash issuer_hash() const; @@ -269,6 +269,7 @@ struct OverlayOptions { td::uint32 nodes_to_send_ = 4; td::uint32 propagate_broadcast_to_ = 5; td::uint32 default_permanent_members_flags_ = 0; + double broadcast_speed_multiplier_ = 1.0; }; class Overlays : public td::actor::Actor { diff --git a/recent_changelog.md b/recent_changelog.md index fb7433e5..820d2aa4 100644 --- a/recent_changelog.md +++ b/recent_changelog.md @@ -1,17 +1,13 @@ -## 2024.10 Update - -1. Parallel write to celldb: substantial improvement of sync and GC speed, especially with slow disks. -2. Decreased network traffic: only first block candidate is sent optimistically. -3. Improved channel creation and dht lookups, introduction of semi-private overlays -4. New LS dispatch queue related methods and improvement security -5. Fixing recursion in TVM continuations -6. Improved stats for actors, validator sessions, perf counters, overlays, adnl, rocksdb -7. Migration to C++20 -8. Improved block size estimates: account for depth in various structures -9. Fix bug with `<<` optimization in FunC -10. Minor changes of TVM which will be activated by `Config8.version >= 9` -11. Multiple minor improvements - -Besides the work of the core team, this update is based on the efforts of @krigga (emulator), Arayz @ TonBit (LS security, TVM recursion), @ret2happy (UB in BLST). - +## 2025.03 Update +1. New extracurrency behavior introduced, check [GlobalVersions.md](./doc/GlobalVersions.md#version-10) +2. Optmization of validation process, in particular CellStorageStat. +3. Flag for speeding up broadcasts in various overlays. +4. Fixes for static builds for emulator and tonlibjson +5. Improving getstats output: add + * Liteserver queries count + * Collated/validated blocks count, number of active sessions + * Persistent state sizes + * Initial sync progress +6. Fixes in logging, TON Storage, external message checking, persistent state downloading, UB in tonlib +Besides the work of the core team, this update is based on the efforts of @Sild from StonFi(UB in tonlib). diff --git a/storage/PeerActor.cpp b/storage/PeerActor.cpp index 48d45626..e140b4ce 100644 --- a/storage/PeerActor.cpp +++ b/storage/PeerActor.cpp @@ -251,7 +251,7 @@ void PeerActor::loop_update_init() { } s = s.substr(peer_init_offset_, UPDATE_INIT_BLOCK_SIZE); auto query = create_update_query(ton::create_tl_object( - td::BufferSlice(s), (int)peer_init_offset_, to_ton_api(node_state))); + td::BufferSlice(s), (int)peer_init_offset_ * 8, to_ton_api(node_state))); // take care about update_state_query initial state update_state_query_.state = node_state; @@ -502,11 +502,11 @@ void PeerActor::process_update_peer_parts(const tl_object_ptr(offset + i)); } diff --git a/tdutils/td/utils/SpinLock.h b/tdutils/td/utils/SpinLock.h index b5bb62db..f0856f0c 100644 --- a/tdutils/td/utils/SpinLock.h +++ b/tdutils/td/utils/SpinLock.h @@ -63,7 +63,10 @@ class SpinLock { } private: +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-pragma" std::atomic_flag flag_ = ATOMIC_FLAG_INIT; +#pragma clang diagnostic pop void unlock() { flag_.clear(std::memory_order_release); } diff --git a/tdutils/td/utils/StringBuilder.h b/tdutils/td/utils/StringBuilder.h index 99e9d517..685416fe 100644 --- a/tdutils/td/utils/StringBuilder.h +++ b/tdutils/td/utils/StringBuilder.h @@ -149,4 +149,19 @@ std::enable_if_t::value, string> to_string(const T &x) { return sb.as_cslice().str(); } +template +struct LambdaPrintHelper { + SB& sb; +}; +template +SB& operator<<(const LambdaPrintHelper& helper, F&& f) { + f(helper.sb); + return helper.sb; +} +struct LambdaPrint {}; + +inline LambdaPrintHelper operator<<(td::StringBuilder& sb, const LambdaPrint&) { + return LambdaPrintHelper{sb}; +} + } // namespace td diff --git a/tdutils/td/utils/Time.h b/tdutils/td/utils/Time.h index ece822d4..c7795ae4 100644 --- a/tdutils/td/utils/Time.h +++ b/tdutils/td/utils/Time.h @@ -128,6 +128,10 @@ inline Timestamp &operator+=(Timestamp &a, double b) { return a; } +inline double operator-(const Timestamp &a, const Timestamp &b) { + return a.at() - b.at(); +} + template void store(const Timestamp ×tamp, StorerT &storer) { storer.store_binary(timestamp.at() - Time::now() + Clocks::system()); diff --git a/tdutils/td/utils/logging.cpp b/tdutils/td/utils/logging.cpp index 345615f1..03d32ee2 100644 --- a/tdutils/td/utils/logging.cpp +++ b/tdutils/td/utils/logging.cpp @@ -176,7 +176,10 @@ void TsCerr::enterCritical() { void TsCerr::exitCritical() { lock_.clear(std::memory_order_release); } +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-pragma" TsCerr::Lock TsCerr::lock_ = ATOMIC_FLAG_INIT; +#pragma clang diagnostic pop class DefaultLog : public LogInterface { public: diff --git a/tdutils/td/utils/logging.h b/tdutils/td/utils/logging.h index d00fba15..bb28f6df 100644 --- a/tdutils/td/utils/logging.h +++ b/tdutils/td/utils/logging.h @@ -74,6 +74,7 @@ #define LOG(level) LOG_IMPL(level, level, true, ::td::Slice()) #define LOG_IF(level, condition) LOG_IMPL(level, level, condition, #condition) +#define FLOG(level) LOG_IMPL(level, level, true, ::td::Slice()) << td::LambdaPrint{} << [&](auto &sb) #define VLOG(level) LOG_IMPL(DEBUG, level, true, TD_DEFINE_STR(level)) #define VLOG_IF(level, condition) LOG_IMPL(DEBUG, level, condition, TD_DEFINE_STR(level) " " #condition) @@ -95,13 +96,13 @@ inline bool no_return_func() { #define DUMMY_LOG_CHECK(condition) LOG_IF(NEVER, !(condition)) #ifdef TD_DEBUG - #if TD_MSVC +#if TD_MSVC #define LOG_CHECK(condition) \ __analysis_assume(!!(condition)); \ LOG_IMPL(FATAL, FATAL, !(condition), #condition) - #else +#else #define LOG_CHECK(condition) LOG_IMPL(FATAL, FATAL, !(condition) && no_return_func(), #condition) - #endif +#endif #else #define LOG_CHECK DUMMY_LOG_CHECK #endif @@ -263,6 +264,9 @@ class Logger { sb_ << other; return *this; } + LambdaPrintHelper operator<<(const LambdaPrint &) { + return LambdaPrintHelper{sb_}; + } MutableCSlice as_cslice() { return sb_.as_cslice(); @@ -339,7 +343,10 @@ class TsLog : public LogInterface { private: LogInterface *log_ = nullptr; +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-pragma" std::atomic_flag lock_ = ATOMIC_FLAG_INIT; +#pragma clang diagnostic pop void enter_critical() { while (lock_.test_and_set(std::memory_order_acquire)) { // spin diff --git a/tdutils/td/utils/port/Stat.cpp b/tdutils/td/utils/port/Stat.cpp index 816d622e..73b00608 100644 --- a/tdutils/td/utils/port/Stat.cpp +++ b/tdutils/td/utils/port/Stat.cpp @@ -472,4 +472,45 @@ Result get_total_mem_stat() { #endif } +Result get_cpu_cores() { +#if TD_LINUX + uint32 result = 0; + TRY_RESULT(fd, FileFd::open("/proc/cpuinfo", FileFd::Read)); + SCOPE_EXIT { + fd.close(); + }; + std::string data; + char buf[10000]; + while (true) { + TRY_RESULT(size, fd.read(MutableSlice{buf, sizeof(buf) - 1})); + if (size == 0) { + break; + } + buf[size] = '\0'; + data += buf; + } + size_t i = 0; + while (i < data.size()) { + const char *line_begin = data.data() + i; + while (i < data.size() && data[i] != '\n') { + ++i; + } + auto line_end = data.data() + i; + ++i; + Slice line{line_begin, line_end}; + size_t j = 0; + while (j < line.size() && line[j] != ' ' && line[j] != '\t' && line[j] != ':') { + ++j; + } + Slice name = line.substr(0, j); + if (name == "processor") { + ++result; + } + } + return result; +#else + return Status::Error("Not supported"); +#endif +} + } // namespace td diff --git a/tdutils/td/utils/port/Stat.h b/tdutils/td/utils/port/Stat.h index ab97be0f..82e1832a 100644 --- a/tdutils/td/utils/port/Stat.h +++ b/tdutils/td/utils/port/Stat.h @@ -70,4 +70,6 @@ struct TotalMemStat { }; Result get_total_mem_stat() TD_WARN_UNUSED_RESULT; +Result get_cpu_cores() TD_WARN_UNUSED_RESULT; + } // namespace td diff --git a/tdutils/td/utils/port/detail/PollableFd.h b/tdutils/td/utils/port/detail/PollableFd.h index dceea4f3..6cb621e8 100644 --- a/tdutils/td/utils/port/detail/PollableFd.h +++ b/tdutils/td/utils/port/detail/PollableFd.h @@ -149,7 +149,10 @@ class PollableFdInfo : private ListNode { private: NativeFd fd_{}; +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-pragma" std::atomic_flag lock_ = ATOMIC_FLAG_INIT; +#pragma clang diagnostic pop PollFlagsSet flags_; #if TD_PORT_WINDOWS SpinLock observer_lock_; diff --git a/test/regression-tests.ans b/test/regression-tests.ans index bfea7089..14d5958b 100644 --- a/test/regression-tests.ans +++ b/test/regression-tests.ans @@ -21,6 +21,7 @@ Test_Fift_test_hmap_default c269246882039824bb5822e896c3e6e82ef8e1251b6b251f5af8 Test_Fift_test_levels_default 9fba4a7c98aec9000f42846d6e5fd820343ba61d68f9139dd16c88ccda757cf3 Test_Fift_test_namespaces_default e6419619c51332fb5e8bf22043ef415db686c47fe24f03061e5ad831014e7c6c Test_Fift_test_rist255_default f4d7558f200a656934f986145c19b1dedbe2ad029292a5a975576d6891e25fc4 +Test_Fift_test_secp256k1_default 3118450dace6af05fcdbd54a87d9446162ce11ac6ef6dfc57998cf113587d602 Test_Fift_test_sort2_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a Test_Fift_test_sort_default 9b57d47e6a10e7d1bbb565db35400debf2f963031f434742a702ec76555a5d3a Test_Fift_test_tvm_runvm_default ff3d2a4031b543c18d6b555f0a1f1a891c7825e6d1e2e9beb4bf13b37441450b diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 7c13870e..0fde1e68 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -323,9 +323,8 @@ class TestNode : public td::actor::Actor { td::actor::send_closure(id_, &ton::validator::ValidatorManager::sync_complete, td::PromiseCreator::lambda([](td::Unit) {})); } - void add_shard(ton::ShardIdFull) override { - } - void del_shard(ton::ShardIdFull) override { + void on_new_masterchain_block(td::Ref state, + std::set shards_to_monitor) override { } void send_ihr_message(ton::AccountIdPrefixFull dst, td::BufferSlice data) override { } @@ -371,13 +370,19 @@ class TestNode : public td::actor::Actor { void get_next_key_blocks(ton::BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) override { } - void download_archive(ton::BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - - td::Promise promise) override { + void download_archive(ton::BlockSeqno masterchain_seqno, ton::ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override { + } + void download_out_msg_queue_proof( + ton::ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, + td::Timestamp timeout, td::Promise>> promise) override { } void new_key_block(ton::validator::BlockHandle handle) override { } + void send_validator_telemetry(ton::PublicKeyHash key, + ton::tl_object_ptr telemetry) override { + } }; td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::install_callback, diff --git a/third-party/secp256k1 b/third-party/secp256k1 new file mode 160000 index 00000000..acf5c55a --- /dev/null +++ b/third-party/secp256k1 @@ -0,0 +1 @@ +Subproject commit acf5c55ae6a94e5ca847e07def40427547876101 diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 9ef47d95..cfc9f3a1 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -447,13 +447,19 @@ tonNode.dataFull id:tonNode.blockIdExt proof:bytes block:bytes is_link:Bool = to tonNode.dataFullCompressed id:tonNode.blockIdExt flags:# compressed:bytes is_link:Bool = tonNode.DataFull; tonNode.dataFullEmpty = tonNode.DataFull; -tonNode.capabilities version:int capabilities:long = tonNode.Capabilities; +tonNode.capabilities#f5bf60c0 version_major:int version_minor:int flags:# = tonNode.Capabilities; tonNode.success = tonNode.Success; tonNode.archiveNotFound = tonNode.ArchiveInfo; tonNode.archiveInfo id:long = tonNode.ArchiveInfo; +tonNode.importedMsgQueueLimits max_bytes:int max_msgs:int = ImportedMsgQueueLimits; +tonNode.outMsgQueueProof queue_proofs:bytes block_state_proofs:bytes msg_counts:(vector int) = tonNode.OutMsgQueueProof; +tonNode.outMsgQueueProofEmpty = tonNode.OutMsgQueueProof; + +tonNode.forgetPeer = tonNode.ForgetPeer; + ---functions--- tonNode.getNextBlockDescription prev_block:tonNode.blockIdExt = tonNode.BlockDescription; @@ -479,7 +485,10 @@ tonNode.downloadKeyBlockProof block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadBlockProofLink block:tonNode.blockIdExt = tonNode.Data; tonNode.downloadKeyBlockProofLink block:tonNode.blockIdExt = tonNode.Data; tonNode.getArchiveInfo masterchain_seqno:int = tonNode.ArchiveInfo; +tonNode.getShardArchiveInfo masterchain_seqno:int shard_prefix:tonNode.shardId = tonNode.ArchiveInfo; tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data; +tonNode.getOutMsgQueueProof dst_shard:tonNode.shardId blocks:(vector tonNode.blockIdExt) + limits:tonNode.importedMsgQueueLimits = tonNode.OutMsgQueueProof; tonNode.getCapabilities = tonNode.Capabilities; @@ -545,6 +554,9 @@ db.state.shardClient block:tonNode.blockIdExt = db.state.ShardClient; db.state.asyncSerializer block:tonNode.blockIdExt last:tonNode.blockIdExt last_ts:int = db.state.AsyncSerializer; db.state.hardforks blocks:(vector tonNode.blockIdExt) = db.state.Hardforks; db.state.dbVersion version:int = db.state.DbVersion; +db.state.persistentStateDescriptionShards shard_blocks:(vector tonNode.blockIdExt) = db.state.PersistentStateDescriptionShards; +db.state.persistentStateDescriptionHeader masterchain_id:tonNode.blockIdExt start_time:int end_time:int = db.state.PersistentStateDescriptionHeader; +db.state.persistentStateDescriptionsList list:(vector db.state.persistentStateDescriptionHeader) = db.state.PersistentStateDescriptionsList; db.state.key.destroyedSessions = db.state.Key; db.state.key.initBlockId = db.state.Key; @@ -553,6 +565,8 @@ db.state.key.shardClient = db.state.Key; db.state.key.asyncSerializer = db.state.Key; db.state.key.hardforks = db.state.Key; db.state.key.dbVersion = db.state.Key; +db.state.key.persistentStateDescriptionShards masterchain_seqno:int = db.state.Key; +db.state.key.persistentStateDescriptionsList = db.state.Key; db.lt.el.key workchain:int shard:long idx:int = db.lt.Key; db.lt.desc.key workchain:int shard:long = db.lt.Key; @@ -580,6 +594,10 @@ validator.group workchain:int shard:long catchain_seqno:int config_hash:int256 m validator.groupEx workchain:int shard:long vertical_seqno:int catchain_seqno:int config_hash:int256 members:(vector validator.groupMember) = validator.Group; validator.groupNew workchain:int shard:long vertical_seqno:int last_key_block_seqno:int catchain_seqno:int config_hash:int256 members:(vector validator.groupMember) = validator.Group; +validator.telemetry flags:# timestamp:double adnl_id:int256 + node_version:string os_version:string node_started_at:int + ram_size:long cpu_cores:int node_threads:int = validator.Telemetry; + ---functions--- @@ -604,8 +622,13 @@ dummyworkchain0.config.global zero_state_hash:int256 = dummyworkchain0.config.Gl validator.config.global zero_state:tonNode.blockIdExt init_block:tonNode.blockIdExt hardforks:(vector tonNode.blockIdExt) = validator.config.Global; config.global adnl:adnl.config.global dht:dht.config.Global validator:validator.config.global = config.Global; +liteserver.descV2.sliceSimple shards:(vector tonNode.shardId) = liteserver.descV2.Slice; +liteserver.descV2.shardInfo shard_id:tonNode.shardId seqno:int utime:int lt:long = liteserver.descV2.ShardInfo; +liteserver.descV2.sliceTimed shards_from:(vector liteserver.descV2.shardInfo) shards_to:(vector liteserver.descV2.shardInfo) = liteserver.descV2.Slice; + liteserver.desc id:PublicKey ip:int port:int = liteserver.Desc; -liteclient.config.global liteservers:(vector liteserver.desc) validator:validator.config.global = liteclient.config.Global; +liteserver.descV2 id:PublicKey ip:int port:int slices:(vector liteserver.descV2.Slice) = liteserver.DescV2; +liteclient.config.global liteservers:(vector liteserver.desc) liteservers_v2:(vector liteserver.descV2) validator:validator.config.global = liteclient.config.Global; engine.adnl id:int256 category:int = engine.Adnl; engine.addr ip:int port:int categories:(vector int) priority_categories:(vector int) = engine.Addr; @@ -632,10 +655,12 @@ engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector eng fullnodeconfig:engine.validator.fullNodeConfig extraconfig:engine.validator.extraConfig liteservers:(vector engine.liteServer) control:(vector engine.controlInterface) + shards_to_monitor:(vector tonNode.shardId) gc:engine.gc = engine.validator.Config; engine.validator.customOverlayNode adnl_id:int256 msg_sender:Bool msg_sender_priority:int block_sender:Bool = engine.validator.CustomOverlayNode; -engine.validator.customOverlay name:string nodes:(vector engine.validator.customOverlayNode) = engine.validator.CustomOverlay; +engine.validator.customOverlay name:string nodes:(vector engine.validator.customOverlayNode) sender_shards:(vector tonNode.shardId) + = engine.validator.CustomOverlay; engine.validator.customOverlaysConfig overlays:(vector engine.validator.customOverlay) = engine.validator.CustomOverlaysConfig; engine.validator.collatorOptions @@ -695,12 +720,19 @@ engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_i extra:string = engine.validator.OverlayStats; engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats; +engine.validator.shardOverlayStats.neighbour id:string verison_major:int version_minor:int flags:# + roundtrip:double unreliability:double = engine.validator.shardOverlayStats.Neighbour; +engine.validator.shardOverlayStats shard:string active:Bool + neighbours:(vector engine.validator.shardOverlayStats.neighbour) = engine.validator.ShardOverlayStats; + engine.validator.onePerfTimerStat time:int min:double avg:double max:double = engine.validator.OnePerfTimerStat; engine.validator.perfTimerStatsByName name:string stats:(vector engine.validator.OnePerfTimerStat) = engine.validator.PerfTimerStatsByName; engine.validator.perfTimerStats stats:(vector engine.validator.PerfTimerStatsByName) = engine.validator.PerfTimerStats; engine.validator.shardOutQueueSize size:long = engine.validator.ShardOutQueueSize; +engine.validator.exportedPrivateKeys encrypted_data:bytes = engine.validator.ExportedPrivateKeys; + ---functions--- @@ -731,6 +763,7 @@ engine.validator.delListeningPort ip:int port:int categories:(vector int) priori engine.validator.delProxy out_ip:int out_port:int categories:(vector int) priority_categories:(vector int) = engine.validator.Success; engine.validator.sign key_hash:int256 data:bytes = engine.validator.Signature; +engine.validator.exportAllPrivateKeys encryption_key:PublicKey = engine.validator.ExportedPrivateKeys; engine.validator.getStats = engine.validator.Stats; engine.validator.getConfig = engine.validator.JsonConfig; @@ -767,6 +800,9 @@ engine.validator.getCollatorOptionsJson = engine.validator.JsonConfig; engine.validator.getAdnlStats all:Bool = adnl.Stats; engine.validator.getActorTextStats = engine.validator.TextStats; +engine.validator.addShard shard:tonNode.shardId = engine.validator.Success; +engine.validator.delShard shard:tonNode.shardId = engine.validator.Success; + ---types--- storage.pong = storage.Pong; @@ -981,3 +1017,6 @@ storage.daemon.withdraw contract:string = storage.daemon.Success; storage.daemon.sendCoins address:string amount:string message:string = storage.daemon.Success; storage.daemon.closeStorageContract address:string = storage.daemon.Success; storage.daemon.removeStorageProvider = storage.daemon.Success; + +---types--- +proxyLiteserver.config port:int id:PublicKey = proxyLiteserver.Config; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index bc46d1b1..96ecb775 100644 Binary files a/tl/generate/scheme/ton_api.tlo and b/tl/generate/scheme/ton_api.tlo differ diff --git a/tl/generate/scheme/tonlib_api.tl b/tl/generate/scheme/tonlib_api.tl index 5110d6ec..31ca6fd4 100644 --- a/tl/generate/scheme/tonlib_api.tl +++ b/tl/generate/scheme/tonlib_api.tl @@ -53,7 +53,7 @@ ton.blockIdExt workchain:int32 shard:int64 seqno:int32 root_hash:bytes file_hash extraCurrency id:int32 amount:int64 = ExtraCurrency; raw.fullAccountState balance:int64 extra_currencies:vector code:bytes data:bytes last_transaction_id:internal.transactionId block_id:ton.blockIdExt frozen_hash:bytes sync_utime:int53 = raw.FullAccountState; -raw.message source:accountAddress destination:accountAddress value:int64 extra_currencies:vector fwd_fee:int64 ihr_fee:int64 created_lt:int64 body_hash:bytes msg_data:msg.Data = raw.Message; +raw.message hash:bytes source:accountAddress destination:accountAddress value:int64 extra_currencies:vector fwd_fee:int64 ihr_fee:int64 created_lt:int64 body_hash:bytes msg_data:msg.Data = raw.Message; raw.transaction address:accountAddress utime:int53 data:bytes transaction_id:internal.transactionId fee:int64 storage_fee:int64 other_fee:int64 in_msg:raw.message out_msgs:vector = raw.Transaction; raw.transactions transactions:vector previous_transaction_id:internal.transactionId = raw.Transactions; diff --git a/tl/generate/scheme/tonlib_api.tlo b/tl/generate/scheme/tonlib_api.tlo index 70c08459..10b9ed8d 100644 Binary files a/tl/generate/scheme/tonlib_api.tlo and b/tl/generate/scheme/tonlib_api.tlo differ diff --git a/tl/tl/tl_jni_object.cpp b/tl/tl/tl_jni_object.cpp index e7e69789..26a94d62 100644 --- a/tl/tl/tl_jni_object.cpp +++ b/tl/tl/tl_jni_object.cpp @@ -115,8 +115,9 @@ static size_t get_utf8_from_utf16_length(const jchar *p, jsize len) { for (jsize i = 0; i < len; i++) { unsigned int cur = p[i]; if ((cur & 0xF800) == 0xD800) { + ++i; if (i < len) { - unsigned int next = p[++i]; + unsigned int next = p[i]; if ((next & 0xFC00) == 0xDC00 && (cur & 0x400) == 0) { result += 4; continue; diff --git a/tolk-tester/tests/a10.tolk b/tolk-tester/tests/a10.tolk new file mode 100644 index 00000000..9d24f38d --- /dev/null +++ b/tolk-tester/tests/a10.tolk @@ -0,0 +1,159 @@ +import "@stdlib/tvm-lowlevel" + +fun pair_first(p: [X, Y]): X asm "FIRST"; + +fun one(dummy: tuple?) { + return 1; +} + +fun main(a: int, x: int) { + var y: int = 0; + var z: int = 0; + while ((y = x * x) > a) { + x -= 1; + z = one(null); + } + return (y, z); +} + +fun throwIfLt10(x: int): void { + if (x > 10) { + return; + } + throw 234; + return; +} + +@method_id(88) +fun test88(x: int) { + try { + var x: void = throwIfLt10(x); + return 0; + } catch(code) { + return code; + } +} + +@method_id(89) +fun test89(last: int): (int, int, int, int) { + var t: tuple = createEmptyTuple(); + t.tuplePush(1); + t.tuplePush(2); + t.tuplePush(3); + t.tuplePush(last); + return (t.tupleAt(0), t.tupleAt(t.tupleSize() - 1), t.tupleFirst(), t.tupleLast()); +} + +@pure fun get10() { return 10; } + +@method_id(91) +fun touchCodegen2() { + var f = get10(); + f.stackMoveToTop(); + return f; +} + +@method_id(92) +fun testDumpDontPolluteStack() { + var f = get10(); + f.debugPrint(); + debugPrint(10); + var s = "asdf"; + s.debugPrintString(); + debugDumpStack(); + debugPrintString("my"); + return (f, getRemainingBitsCount(s)); +} + +@method_id(93) +fun testStartBalanceCodegen1() { + var t = getMyOriginalBalanceWithExtraCurrencies(); + var first = t.pair_first(); + return first; +} + +@method_id(94) +fun testStartBalanceCodegen2() { + var first = getMyOriginalBalance(); + return first; +} + +global cur: [int, int, int]; +global next: [int, int, int]; + +@method_id(95) +fun test95() { + cur = [1, 2, 3]; + next = [2, 3, 4]; + (cur, next) = (next, [3, 4, 5]); + return (cur, next); +} + +/** + method_id | in | out +@testcase | 0 | 101 15 | 100 1 +@testcase | 0 | 101 14 | 100 1 +@testcase | 0 | 101 10 | 100 0 +@testcase | 0 | 100 10 | 100 0 +@testcase | 0 | 100 10 | 100 0 +@testcase | 88 | 5 | 234 +@testcase | 88 | 50 | 0 +@testcase | 89 | 4 | 1 4 1 4 +@testcase | 91 | | 10 +@testcase | 92 | | 10 32 +@testcase | 95 | | [ 2 3 4 ] [ 3 4 5 ] + +@fif_codegen +""" + touchCodegen2 PROC:<{ + // + get10 CALLDICT // f + }> +""" + +@fif_codegen +""" + testDumpDontPolluteStack PROC:<{ + ... + DUMPSTK + x{6d79} PUSHSLICE // f s '5 + STRDUMP DROP + SBITS // f '6 + }> +""" + +@fif_codegen +""" + testStartBalanceCodegen1 PROC:<{ + // + BALANCE // t + FIRST // first + }> +""" + +@fif_codegen +""" + testStartBalanceCodegen2 PROC:<{ + // + BALANCE + FIRST // first + }> +""" + +@fif_codegen +""" + test95 PROC:<{ + ... + next GETGLOB // g_next + 3 PUSHINT // g_next '14=3 + 4 PUSHINT // g_next '14=3 '15=4 + 5 PUSHINT // g_next '14=3 '15=4 '16=5 + TRIPLE // '10 '11 + SWAP + cur SETGLOB + next SETGLOB + cur GETGLOB // g_cur + next GETGLOB // g_cur g_next + }> +""" +*/ diff --git a/tolk-tester/tests/a6.tolk b/tolk-tester/tests/a6.tolk new file mode 100644 index 00000000..94494546 --- /dev/null +++ b/tolk-tester/tests/a6.tolk @@ -0,0 +1,83 @@ +fun f(a: int, b: int, c: int, d: int, e: int, f: int): (int, int) { + // solve a 2x2 linear equation + var D: int = a*d - b*c;;;; var Dx: int = e*d-b*f ;;;; var Dy: int = a * f - e * c; + __expect_type(D, "int"); + __expect_type(D*D, "int"); + __expect_type(calc_phi, "() -> int"); + return (Dx/D,Dy/D); +};;;; + +fun calc_phi(): int { + var n = 1; + repeat (70) { n*=10; }; + var p= 1; + var `q`=1; + _=`q`; + do { + (p,q)=(q,p+q); + } while (q <= n); //;; + return mulDivRound(p, n, q); +} + +fun calc_sqrt2(): int { + var n = 1; + repeat (70) { n *= 10; } + var p = 1; + var q = 1; + do { + var t = p + q; + (p, q) = (q, t + q); + } while (q <= n); + return mulDivRound(p, n, q); +} + +fun calc_root(m: int) { + var base: int=1; + repeat(70) { base *= 10; } + var (a, b, c) = (1,0,-m); + var (p1, q1, p2, q2) = (1, 0, 0, 1); + do { + var k: int=-1; + var (a1, b1, c1) = (0, 0, 0); + do { + k+=1; + (a1, b1, c1) = (a, b, c); + c+=b; + c += b += a; + } while (c <= 0); + (a, b, c) = (-c1, -b1, -a1); + (p1, q1) = (k * p1+q1, p1); + (p2, q2) = (k * p2+q2, p2); + } while (p1 <= base); + return (p1, q1, p2, q2); +} + +fun ataninv(base: int, q: int): int { // computes base*atan(1/q) + base=base~/q; + q*=-q; + var sum: int = 0; + var n: int = 1; + do { + sum += base~/n; + base = base~/q; + n += 2; + } while (base != 0); + return sum; +} + +fun calc_pi(): int { + var base: int = 64; + repeat (70) { base *= 10; } + return (ataninv(base << 2, 5) - ataninv(base, 239))~>>4; +} + +fun main(): int { + return calc_pi(); +} + +/** + method_id | in | out +@testcase | 0 | | 31415926535897932384626433832795028841971693993751058209749445923078164 + +@code_hash 84337043972311674339187056298873613816389434478842780265748859098303774481976 +*/ diff --git a/tolk-tester/tests/a6_1.tolk b/tolk-tester/tests/a6_1.tolk new file mode 100644 index 00000000..8079972b --- /dev/null +++ b/tolk-tester/tests/a6_1.tolk @@ -0,0 +1,22 @@ +fun main(a: int, b: int, c: int, d: int, e: int, f: int): (int, int) { + var D: int = a * d - b * c; + var Dx: int = e * d - b * f; + var Dy: int = a * f - e * c; + return (Dx / D, Dy / D); +} + +@method_id(101) +fun testDivMod(x: int, y: int) { + return (divMod(x, y), modDiv(x, y), mulDivMod(x, y, 10)); +} + +/** + method_id | in | out +@testcase | 0 | 1 1 1 -1 10 6 | 8 2 +@testcase | 0 | 817 -31 624 -241 132272 272276 | 132 -788 +@testcase | 0 | -886 562 498 -212 -36452 -68958 | -505 -861 +@testcase | 0 | 448 -433 -444 792 150012 -356232 | -218 -572 +@testcase | 0 | -40 -821 433 -734 -721629 -741724 | -206 889 +@testcase | 0 | -261 -98 -494 868 -166153 733738 | 263 995 +@testcase | 101 | 112 3 | 37 1 1 37 33 6 +*/ diff --git a/tolk-tester/tests/a6_5.tolk b/tolk-tester/tests/a6_5.tolk new file mode 100644 index 00000000..43fd59c5 --- /dev/null +++ b/tolk-tester/tests/a6_5.tolk @@ -0,0 +1,26 @@ +@deprecated +fun twice(f: int -> int, x: int) { + return f (f (x)); +} + +fun sqr(x: int) { + return x * x; +} + +fun main(x: int): int { + var f = sqr; + return twice(f, x) * f(x); +} + +@method_id(4) +fun pow6(x: int): int { + return twice(sqr, x) * sqr(x); +} + +/** + method_id | in | out +@testcase | 0 | 3 | 729 +@testcase | 0 | 10 | 1000000 +@testcase | 4 | 3 | 729 +@testcase | 4 | 10 | 1000000 +*/ diff --git a/tolk-tester/tests/a7.tolk b/tolk-tester/tests/a7.tolk new file mode 100644 index 00000000..1c0ae2eb --- /dev/null +++ b/tolk-tester/tests/a7.tolk @@ -0,0 +1,24 @@ +fun main() { } +@method_id(1) +fun steps(x: int): int { + var n = 0; + while (x > 1) { + n += 1; + if (x & 1) { + x = 3 * x + 1; + } else { + x >>= 1; + } + } + return n; +} + +/** + method_id | in | out +@testcase | 1 | 1 | 0 +@testcase | 1 | 2 | 1 +@testcase | 1 | 5 | 5 +@testcase | 1 | 19 | 20 +@testcase | 1 | 27 | 111 +@testcase | 1 | 100 | 25 +*/ diff --git a/tolk-tester/tests/allow_post_modification.tolk b/tolk-tester/tests/allow_post_modification.tolk new file mode 100644 index 00000000..e20e8218 --- /dev/null +++ b/tolk-tester/tests/allow_post_modification.tolk @@ -0,0 +1,151 @@ +fun unsafe_tuple(x: X): tuple + asm "NOP"; + +fun inc(x: int, y: int): (int, int) { + return (x + y, y * 10); +} +fun `~inc`(mutate self: int, y: int): int { + val (newX, newY) = inc(self, y); + self = newX; + return newY; +} + +fun eq(v: X): X { return v; } +fun eq2(v: (int, int)) { return v; } +fun mul2(mutate dest: int, v: int): int { dest = v*2; return dest; } +fun multens(mutate self: (int, int), v: (int, int)): (int, int) { var (f, s) = self; var (m1, m2) = v; self = (f*m1, s*m2); return self; } + +@method_id(11) +fun test_return(x: int): (int, int, int, int, int, int, int) { + return (x, x.`~inc`(x / 20), x, x = x * 2, x, x += 1, x); +} + +@method_id(12) +fun test_assign(x: int): (int, int, int, int, int, int, int) { + var (x1: int, x2: int, x3: int, x4: int, x5: int, x6: int, x7: int) = (x, x.`~inc`(x / 20), x, x=x*2, x, x+=1, x); + return (x1, x2, x3, x4, x5, x6, x7); +} + +@method_id(13) +fun test_tuple(x: int): tuple { + var t: tuple = unsafe_tuple([x, x.`~inc`(x / 20), x, x = x * 2, x, x += 1, x]); + return t; +} + +@method_id(14) +fun test_tuple_assign(x: int): (int, int, int, int, int, int, int) { + var [x1: int, x2: int, x3: int, x4: int, x5: int, x6: int, x7: int] = [x, x.`~inc`(x / 20), x, x = x * 2, x, x += 1, x]; + return (x1, x2, x3, x4, x5, x6, x7); +} + +fun foo1(x1: int, x2: int, x3: int, x4: int, x5: int, x6: int, x7: int): (int, int, int, int, int, int, int) { + return (x1, x2, x3, x4, x5, x6, x7); +} + +@method_id(15) +fun test_call_1(x: int): (int, int, int, int, int, int, int) { + return foo1(x, x.`~inc`(x / 20), x, x = x * 2, x, x += 1, x); +} + +fun foo2(x1: int, x2: int, x3456: (int, int, int, int), x7: int): (int, int, int, int, int, int, int) { + var (x3: int, x4: int, x5: int, x6: int) = x3456; + return (x1, x2, x3, x4, x5, x6, x7); +} + +@method_id(16) +fun test_call_2(x: int): (int, int, int, int, int, int, int) { + return foo2(x, x.`~inc`(x / 20), (x, x = x * 2, x, x += 1), x); +} + +fun asm_func(x1: int, x2: int, x3: int, x4: int, x5: int, x6: int, x7: int): (int, int, int, int, int, int, int) + asm (x4 x5 x6 x7 x1 x2 x3->0 1 2 3 4 5 6) "NOP"; + +@method_id(17) +fun test_call_asm_old(x: int): (int, int, int, int, int, int, int) { + return asm_func(x, x += 1, x, x, x.`~inc`(x / 20), x, x = x * 2); +} + +@method_id(18) +fun test_call_asm_new(x: int): (int, int, int, int, int, int, int) { + return asm_func(x, x.`~inc`(x / 20), x, x = x * 2, x, x += 1, x); +} + +global xx: int; +@method_id(19) +fun test_global(x: int) { + xx = x; + return (x, xx, xx.`~inc`(xx / 20), eq(xx += (x *= 0)), xx = xx * 2, xx, xx += 1, xx, x); +} + +@method_id(20) +fun test_if_else(x: int): (int, int, int, int, int) { + if (x > 10) { + return (x.`~inc`(8), x + 1, x = 1, x <<= 3, x); + } else { + xx = 9; + return (x, x.`~inc`(-4), x.`~inc`(-1), (x >= 1) as int, x = x + xx); + } +} + +@method_id(21) +fun test_assign_with_inner(x: int) { + var result = (x, x += 10, [x, x += 20, eq(x -= 50), x], eq2((x, x *= eq(x /= 2)))); + return result; +} + +@method_id(22) +fun test_assign_with_mutate(x: int) { + var (result, _) = ((x, mul2(mutate x, x += 5), x.`~inc`(mul2(mutate x, x)), x), 0); + return result; +} + +@method_id(23) +fun test_assign_tensor(x: (int, int)) { + var fs = (0, 0); + return (x, x = (20, 30), fs = x.multens((1, 2)), fs.multens(multens(mutate x, (-1, -1))), x, fs); +} + +global fs: (int, int); +@method_id(24) +fun test_assign_tensor_global(x: (int, int)) { + fs = (0, 0); + return (x, x = (20, 30), fs = x.multens((1, 2)), fs.multens(multens(mutate x, (-1, -1))), x, fs); +} + +fun main() { +} + +/** + method_id | in | out +@testcase | 11 | 100 | 100 50 105 210 210 211 211 +@testcase | 12 | 100 | 100 50 105 210 210 211 211 +@testcase | 13 | 100 | [ 100 50 105 210 210 211 211 ] +@testcase | 14 | 100 | 100 50 105 210 210 211 211 +@testcase | 15 | 100 | 100 50 105 210 210 211 211 +@testcase | 16 | 100 | 100 50 105 210 210 211 211 +@testcase | 17 | 100 | 101 50 106 212 100 101 101 +@testcase | 18 | 100 | 210 210 211 211 100 50 105 +@testcase | 19 | 100 | 100 100 50 105 210 210 211 211 0 +@testcase | 20 | 80 | 80 89 1 8 8 +@testcase | 20 | 9 | 9 -40 -10 -1 13 +@testcase | 21 | 100 | 100 110 [ 110 130 80 80 ] 80 3200 +@testcase | 22 | 100 | 100 210 4200 630 +@testcase | 23 | 1 1 | 1 1 20 30 20 60 -400 -3600 -20 -60 -400 -3600 +@testcase | 24 | 1 1 | 1 1 20 30 20 60 -400 -3600 -20 -60 -400 -3600 + +@fif_codegen +""" + ~inc PROC:<{ + // self y + inc CALLDICT // self newY + }> +""" + +@fif_codegen +""" + test_assign_tensor_global PROC:<{ + // x.0 x.1 +""" + +@code_hash 61280273714870328160131559159866470128402169974050439159015534193532598351244 +*/ diff --git a/tolk-tester/tests/asm_arg_order.tolk b/tolk-tester/tests/asm_arg_order.tolk new file mode 100644 index 00000000..b96e09ec --- /dev/null +++ b/tolk-tester/tests/asm_arg_order.tolk @@ -0,0 +1,139 @@ +@pure +fun empty_tuple2(): tuple +asm "NIL"; +@pure +fun tpush2(mutate self: tuple, x: X): void +asm "TPUSH"; + +@pure +fun asm_func_1(x: int, y: int, z: int): tuple +asm "3 TUPLE"; +@pure +fun asm_func_2(x: int, y: int, z: int): tuple +asm (z y x -> 0) "3 TUPLE"; +@pure +fun asm_func_3(x: int, y: int, z: int): tuple +asm (y z x -> 0) "3 TUPLE"; +@pure +fun asm_func_4(a: int, b: (int, (int, int)), c: int): tuple +asm (b a c -> 0) "5 TUPLE"; + +@pure +fun asm_func_modify(mutate self: tuple, b: int, c: int): void +asm (c b self) "SWAP TPUSH SWAP TPUSH"; + +global t: tuple; + +fun foo(x: int): int { + t.tpush2(x); + return x * 10; +} + +@method_id(11) +fun test_old_1(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = asm_func_1(foo(11), foo(22), foo(33)); + return (t, t2); +} + +@method_id(12) +fun test_old_2(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = asm_func_2(foo(11), foo(22), foo(33)); + return (t, t2); +} + +@method_id(13) +fun test_old_3(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = asm_func_3(foo(11), foo(22), foo(33)); + return (t, t2); +} + +@method_id(14) +fun test_old_4(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = empty_tuple2(); + // This actually computes left-to-right even without compute-asm-ltr + t2 = asm_func_4(foo(11), (foo(22), (foo(33), foo(44))), foo(55)); + return (t, t2); +} + +@method_id(15) +fun test_old_modify(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = empty_tuple2(); + t2.asm_func_modify(foo(22), foo(33)); + return (t, t2); +} + +@method_id(16) +fun test_old_dot(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = foo(11).asm_func_3(foo(22), foo(33)); + return (t, t2); +} + +@method_id(21) +fun test_new_1(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = asm_func_1(foo(11), foo(22), foo(33)); + return (t, t2); +} + +@method_id(22) +fun test_new_2(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = asm_func_2(foo(11), foo(22), foo(33)); + return (t, t2); +} + +@method_id(23) +fun test_new_3(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = asm_func_3(foo(11), foo(22), foo(33)); + return (t, t2); +} + +@method_id(24) +fun test_new_4(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = asm_func_4(foo(11), (foo(22), (foo(33), foo(44))), foo(55)); + return (t, t2); +} + +@method_id(25) +fun test_new_modify(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = empty_tuple2(); + t2.asm_func_modify(foo(22), foo(33)); + return (t, t2); +} + +@method_id(26) +fun test_new_dot(): (tuple, tuple) { + t = empty_tuple2(); + var t2: tuple = foo(11).asm_func_3(foo(22), foo(33)); + return (t, t2); +} + +fun main() { +} + +/** + method_id | in | out +@testcase | 11 | | [ 11 22 33 ] [ 110 220 330 ] +@testcase | 12 | | [ 11 22 33 ] [ 330 220 110 ] +@testcase | 13 | | [ 11 22 33 ] [ 220 330 110 ] +@testcase | 14 | | [ 11 22 33 44 55 ] [ 220 330 440 110 550 ] +@testcase | 15 | | [ 22 33 ] [ 220 330 ] +@testcase | 16 | | [ 11 22 33 ] [ 220 330 110 ] +@testcase | 21 | | [ 11 22 33 ] [ 110 220 330 ] +@testcase | 22 | | [ 11 22 33 ] [ 330 220 110 ] +@testcase | 23 | | [ 11 22 33 ] [ 220 330 110 ] +@testcase | 24 | | [ 11 22 33 44 55 ] [ 220 330 440 110 550 ] +@testcase | 25 | | [ 22 33 ] [ 220 330 ] +@testcase | 26 | | [ 11 22 33 ] [ 220 330 110 ] + +@code_hash 93068291567112337250118419287631047120002003622184251973082208096953112184588 +*/ diff --git a/tolk-tester/tests/assignment-tests.tolk b/tolk-tester/tests/assignment-tests.tolk new file mode 100644 index 00000000..bb647652 --- /dev/null +++ b/tolk-tester/tests/assignment-tests.tolk @@ -0,0 +1,250 @@ +fun extractFromTypedTuple(params: [int]) { + var [payload: int] = params; + return payload + 10; +} + +@method_id(101) +fun test101(x: int) { + var params = [x]; + return extractFromTypedTuple(params); +} + +fun autoInferIntNull(x: int) { + if (x > 10) { return null; } + return x; +} + +fun typesAsIdentifiers(builder: builder) { + var int = 1; + var cell = builder.endCell(); + var slice = cell.beginParse(); + { + var cell: cell = cell; + var tuple: tuple = createEmptyTuple(); + var bool: bool = tuple.tupleAt(0) > 0; + } + return int; +} + +global callOrder: tuple; + +fun getTensor_12() { + callOrder.tuplePush(100); + return (1, 2); +} +fun getTensor_1X(x: int) { + callOrder.tuplePush(101); + return (1, x); +} +fun getTuple_12() { + callOrder.tuplePush(110); + return [1, 2]; +} +fun getTuple_1X(x: int) { + callOrder.tuplePush(111); + return [1, x]; +} +fun getUntypedTuple_12() { + callOrder.tuplePush(120); + var t = createEmptyTuple(); t.tuplePush(1); t.tuplePush(2); + return t; +} +fun getUntypedTuple_1X(x: int) { + callOrder.tuplePush(121); + var t = createEmptyTuple(); t.tuplePush(1); t.tuplePush(x); + return t; +} +fun getIntValue5() { + callOrder.tuplePush(10); + return 5; +} +fun getIntValueX(x: int) { + callOrder.tuplePush(11); + return x; +} + +@method_id(102) +fun test102() { + callOrder = createEmptyTuple(); + var x = 0; + getTensor_12().0 = getIntValue5(); + getTensor_1X(5).1 = getIntValue5(); + getTensor_1X(x = 10).0 = getIntValueX(x); + return (callOrder, x); +} + +@method_id(103) +fun test103() { + callOrder = createEmptyTuple(); + var x = 0; + getTuple_12().0 = getIntValue5(); + getTuple_1X(5).1 = getIntValue5(); + getTuple_1X(x = 10).0 = getIntValueX(x); + return (callOrder, x); +} + +@method_id(104) +fun test104() { + callOrder = createEmptyTuple(); + var x = 0; + getUntypedTuple_12().0 = getIntValue5(); + getUntypedTuple_1X(5).1 = getIntValue5(); + getUntypedTuple_1X(x = 10).0 = getIntValueX(x); + return (callOrder, x); +} + +@method_id(105) +fun test105() { + callOrder = createEmptyTuple(); + getTensor_12().0 = getTensor_1X(getIntValue5()).1 = getIntValueX(getTensor_12().1); + return callOrder; +} + +@method_id(106) +fun test106() { + callOrder = createEmptyTuple(); + getTuple_12().0 = getTuple_1X(getIntValue5()).1 = getIntValueX(getTuple_12().1); + return callOrder; +} + +global t107: (int, int); + +@method_id(107) +fun test107() { + ((t107 = (1, 2)).0, (t107 = (3, 4)).1) = (5, 6); + return t107; +} + +global g108: int; +fun assertEq(a: int, b: int) { + assert(a == b, 10); + return b; +} + +@method_id(108) +fun test108() { + callOrder = createEmptyTuple(); + g108 = 0; + getTensor_1X(g108 = 8).1 = assertEq(g108, 8); + return (callOrder, g108); +} + +@method_id(109) +fun test109() { + callOrder = createEmptyTuple(); + var x = 0; + [getTuple_12().0, getTuple_1X(x = getIntValue5()).1, getTuple_1X(x += 10).0] = [getIntValue5(), getIntValue5(), getIntValueX(x)]; + return (callOrder, x); +} + +global g110: int; +global t110: (int, int); + +@method_id(110) +fun test110() { + callOrder = createEmptyTuple(); + var xy = [0, 0]; + [xy.0, getTuple_1X(g110 = 8).0] = [g110 += 5, getIntValueX(g110 += 10)]; + [xy.1, getTuple_1X((t110 = (8, 9)).0).1] = [t110.0 += 5, getIntValueX(t110.1 += 10)]; + return (xy, callOrder, g110, t110); +} + +@method_id(111) +fun test111() { + callOrder = createEmptyTuple(); + var z = -1; + var xy = [0, z = 0]; + var rhs = [getIntValueX(xy.1 += 10), xy.1, xy.0, z += 50]; + [xy.0, getTuple_1X(g110 = 8 + getIntValueX(xy.1)).0, xy.1, z] = rhs; + return (xy, g110, callOrder, z); +} + +@method_id(112) +fun test112() { + var xy = [1, 2]; + ((((xy))).0, ((xy.1))) = ((xy).1, ((xy.0))); + return xy; +} + +@method_id(113) +fun test113() { + var (a, t, z) = (1, [2,3], (-1,-1)); + (a, t, a, z, t.1, z.1) = (10, [a,12], 13, (a, t.1), 14, t.1); + return (a, t, z); +} + +global g114: int; +global t114: [int, int]; +global z114: (int, int); + +@method_id(114) +fun test114() { + g114 = 1; + t114 = [2, 3]; + (g114, t114, g114, z114, t114.1, z114.1) = (10, [g114,12], 13, (g114, t114.1), 14, t114.1); + return (g114, t114, z114); +} + +@method_id(115) +fun test115() { + callOrder = createEmptyTuple(); + var x = 0; + var y = 0; + [getTensor_1X(x = 5).0, y] = getTuple_1X(x = 9); + return (callOrder, x, y); +} + +@method_id(116) +fun test116() { + var (a,b,c,d) = (0,0,0,0); + var rhs = [1, 2, 3, 4]; + var rhs2 = ([a,b,c,d] = rhs); + __expect_type(rhs2, "[int, int, int, int]"); + return (a, b, c, d, rhs2); +} + + + +fun main(value: int) { + var (x: int?, y) = (autoInferIntNull(value), autoInferIntNull(value * 2)); + if (x == null && y == null) { return null; } + return x == null || y == null ? -1 : x! + y!; +} + +/** +@testcase | 0 | 3 | 9 +@testcase | 0 | 6 | -1 +@testcase | 0 | 11 | (null) +@testcase | 101 | 78 | 88 +@testcase | 102 | | [ 100 10 101 10 101 11 ] 10 +@testcase | 103 | | [ 110 10 111 10 111 11 ] 10 +@testcase | 104 | | [ 120 10 121 10 121 11 ] 10 +@testcase | 105 | | [ 100 10 101 100 11 ] +@testcase | 106 | | [ 110 10 111 110 11 ] +@testcase | 107 | | 3 4 +@testcase | 108 | | [ 101 ] 8 +@testcase | 109 | | [ 110 10 111 111 10 10 11 ] 15 +@testcase | 110 | | [ 13 13 ] [ 111 11 111 11 ] 23 13 19 +@testcase | 111 | | [ 10 0 ] 18 [ 11 11 111 ] 50 +@testcase | 112 | | [ 2 1 ] +@testcase | 113 | | 13 [ 1 14 ] 1 3 +@testcase | 114 | | 13 [ 1 14 ] 1 3 +@testcase | 115 | | [ 101 111 ] 9 9 +@testcase | 116 | | 1 2 3 4 [ 1 2 3 4 ] + + +@fif_codegen +""" + test116 PROC:<{ + // + 1 PUSHINT // '10=1 + 2 PUSHINT // '10=1 '11=2 + 3 PUSHINT // '10=1 '11=2 '12=3 + 4 PUSHINT // '10=1 '11=2 '12=3 '13=4 + 4 TUPLE // rhs + DUP // rhs rhs + 4 UNTUPLE // rhs2 a b c d + 4 ROLL // a b c d rhs2 + }> +""" +*/ diff --git a/tolk-tester/tests/bit-operators.tolk b/tolk-tester/tests/bit-operators.tolk new file mode 100644 index 00000000..b0106883 --- /dev/null +++ b/tolk-tester/tests/bit-operators.tolk @@ -0,0 +1,204 @@ +fun lshift(): bool { + return (1 << 0) == 1; +} + +fun rshift(): bool { + return (1 >> 0) == 1; +} + +fun lshift_var(i: int): bool { + return (1 << i) == 1; +} + +fun rshift_var(i: int): bool { + return (1 >> i) == 1; +} + +fun main(x: int): bool { + if (x == 0) { + return lshift(); + } else if (x == 1) { + return rshift(); + } else if (x == 2) { + return lshift_var(0); + } else if (x == 3) { + return rshift_var(0); + } else if (x == 4) { + return lshift_var(1); + } else { + return rshift_var(1); + } +} + +@method_id(11) +fun is_claimed(index: int): bool { + var claim_bit_index: int = index % 256; + var mask: int = 1 << claim_bit_index; + return (255 & mask) == mask; +} + +@method_id(12) +fun bit_not(i: int, b: bool): (int, bool, bool, bool, int, bool) { + var i2 = ~i; + var b2 = !b; + var (i3: int, b3: bool) = (i2, b2); + return (i3, b3, !i, !b, ~~~i, !!!b); +} + +@method_id(13) +fun boolWithBitwiseConst() { + var found = true; + return (found & false, found | true, found ^ true, found & found); +} + +global g14: int; +fun getBool() { return (g14 += 1) > 2; } + +@method_id(14) +fun boolWithBitwise(b: bool) { + g14 = 0; + return (b & getBool(), !b & getBool(), b | getBool(), !b | getBool(), b ^ getBool(), !b & getBool(), g14); +} + +@method_id(15) +fun boolWithBitwiseSet(b1: bool, b2: bool) { + b1 &= b2; + b2 |= true; + b1 |= b1 == false; + b2 ^= (b1 ^= b2); + return (b1, b2); +} + +@method_id(16) +fun testDoUntilCodegen(i: bool, n: int) { + var cnt = 0; + do { cnt += 1; } while (i); + do { cnt += 1; } while (!!i); + do { cnt += 1; } while (n); + return (cnt, !i, !n); +} + +@method_id(17) +fun testConstNegateCodegen() { + return (!0, !1, !true, !false, !!true, !!false); +} + +@method_id(18) +fun testBoolNegateOptimized(x: bool) { + return (x, !x, !!x, !!!x, !!!!true); +} + +fun eqX(x: bool) { return x; } + +@method_id(19) +fun testBoolCompareOptimized(x: bool) { + return (x == true, x != true, eqX(x) == false, eqX(x) != false, !!(x == !false)); +} + + + +/** + method_id | in | out +@testcase | 0 | 0 | -1 +@testcase | 0 | 1 | -1 +@testcase | 0 | 2 | -1 +@testcase | 0 | 3 | -1 +@testcase | 0 | 4 | 0 +@testcase | 0 | 5 | 0 +@testcase | 11 | 0 | -1 +@testcase | 11 | 1 | -1 +@testcase | 11 | 256 | -1 +@testcase | 11 | 8 | 0 +@testcase | 12 | 0 0 | -1 -1 -1 -1 -1 -1 +@testcase | 12 | -1 -1 | 0 0 0 0 0 0 +@testcase | 12 | 7 0 | -8 -1 0 -1 -8 -1 +@testcase | 14 | -1 | 0 0 -1 -1 0 0 6 +@testcase | 14 | 0 | 0 0 -1 -1 -1 -1 6 +@testcase | 15 | -1 -1 | 0 -1 +@testcase | 15 | -1 0 | 0 -1 +@testcase | 16 | 0 0 | 3 -1 -1 +@testcase | 17 | | -1 0 0 -1 -1 0 +@testcase | 18 | 0 | 0 -1 0 -1 -1 +@testcase | 18 | -1 | -1 0 -1 0 -1 +@testcase | 19 | 0 | 0 -1 -1 0 0 +@testcase | 19 | -1 | -1 0 0 -1 -1 + +@fif_codegen +""" + boolWithBitwiseConst PROC:<{ + // + 0 PUSHINT // '3 + -1 PUSHINT // '3 '5 + 0 PUSHINT // '3 '5 '7 + -1 PUSHINT // '3 '5 '7 '8 + }> +""" + +@fif_codegen +""" + testDoUntilCodegen PROC:<{ + // i n + 0 PUSHINT // i n cnt=0 + UNTIL:<{ + INC // i n cnt + s2 PUSH // i n cnt i + NOT // i n cnt '6 + }> // i n cnt + UNTIL:<{ + INC // i n cnt + s2 PUSH // i n cnt i + NOT // i n cnt '9 + }> // i n cnt + UNTIL:<{ + INC // i n cnt + OVER // i n cnt n + 0 EQINT // i n cnt '12 + }> // i n cnt + s0 s2 XCHG // cnt n i + NOT // cnt n '13 + SWAP // cnt '13 n + 0 EQINT // cnt '13 '14 + }> +""" + +@fif_codegen +""" + testConstNegateCodegen PROC:<{ + // + TRUE // '0 + FALSE // '0 '1 + FALSE // '0 '1 '2 + TRUE // '0 '1 '2 '3 + TRUE // '0 '1 '2 '3 '4 + FALSE // '0 '1 '2 '3 '4 '5 + }> +""" + +@fif_codegen +""" + testBoolNegateOptimized PROC:<{ + // x + DUP // x x + NOT // x '1 + OVER // x '1 x + NOT // x '1 '2 + s2 s(-1) PUXC + TRUE // x '1 x '2 '3 + }> +""" + +@fif_codegen +""" + testBoolCompareOptimized PROC:<{ + // x + DUP // x x + NOT // x '1 + OVER // x '1 x + eqX CALLDICT // x '1 '2 + NOT // x '1 '3 + s2 PUSH // x '1 '3 x + eqX CALLDICT // x '1 '3 '4 + s3 PUSH // x '1 '3 '4 x + }> +""" +*/ diff --git a/tolk-tester/tests/c2.tolk b/tolk-tester/tests/c2.tolk new file mode 100644 index 00000000..bcbc6c93 --- /dev/null +++ b/tolk-tester/tests/c2.tolk @@ -0,0 +1,28 @@ +global op: (int, int) -> int; + +fun check_assoc(a: int, b: int, c: int): bool { + return op(op(a, b), c) == op(a, op(b, c)); +} + +fun unnamed_args(_: int, _: slice, _: int) { + return true; +} + +fun main(x: int, y: int, z: int): bool? { + op = `_+_`; + if (0) { return null; } + return check_assoc(x, y, z); +} + +@method_id(101) +fun test101(x: int, z: int) { + return unnamed_args(x, "asdf", z); +} + +/** + method_id | in | out +@testcase | 0 | 2 3 9 | -1 +@testcase | 0 | 11 22 44 | -1 +@testcase | 0 | -1 -10 -20 | -1 +@testcase | 101 | 1 10 | -1 +*/ diff --git a/tolk-tester/tests/c2_1.tolk b/tolk-tester/tests/c2_1.tolk new file mode 100644 index 00000000..ef1e589a --- /dev/null +++ b/tolk-tester/tests/c2_1.tolk @@ -0,0 +1,14 @@ +fun check_assoc(op: (int, int) -> int, a: int, b: int, c: int) { + return op(op(a, b), c) == op(a, op(b, c)); +} + +fun main(x: int, y: int, z: int): bool { + return check_assoc(`_+_`, x, y, z); +} + +/** + method_id | in | out +@testcase | 0 | 2 3 9 | -1 +@testcase | 0 | 11 22 44 | -1 +@testcase | 0 | -1 -10 -20 | -1 +*/ diff --git a/tolk-tester/tests/cells-slices.tolk b/tolk-tester/tests/cells-slices.tolk new file mode 100644 index 00000000..772812eb --- /dev/null +++ b/tolk-tester/tests/cells-slices.tolk @@ -0,0 +1,232 @@ +fun store_u32(mutate self: builder, value: int): self { + return self.storeUint(value, 32); +} + +fun load_u32(mutate self: slice): int { + return self.loadUint(32); +} + +fun myLoadInt(mutate self: slice, len: int): int + asm(-> 1 0) "LDIX"; +fun myStoreInt(mutate self: builder, x: int, len: int): self + asm(x self len) "STIX"; + +@method_id(101) +fun test1(): [int,int,int,int,int] { + var b: builder = beginCell().storeUint(1, 32); + b = b.storeUint(2, 32); + b.storeUint(3, 32); + b = b.store_u32(4); + b.store_u32(5); + + var cs: slice = b.endCell().beginParse(); + var one: int = cs.loadUint(32); + var (two: int, three: int) = (cs.loadUint(32), cs.load_u32()); + var four: int = cs.load_u32(); + var five: int = cs.load_u32(); + + return [one,two,three,four,five]; +} + +@method_id(102) +fun test2(): [int,int,int] { + var b: builder = beginCell().myStoreInt(1, 32); + b = b.myStoreInt(2, 32); + // operator ! here and below is used just for testing purposes, it doesn't affect the result + b!.myStoreInt(3, 32); + + var cs: slice = b.endCell().beginParse(); + var one: int = cs.myLoadInt(32); + var (two: int, three: int) = (cs.myLoadInt(32), cs.myLoadInt(32)); + + return [one,two,three]; +} + +@method_id(103) +fun test3(ret: int): int { + val same: int = beginCell()!.storeUint(ret,32).endCell().beginParse().loadUint(32); + return same; +} + +@method_id(104) +fun test4(): [int,int] { + var b: builder = (beginCell() as builder).myStoreInt(1, 32); + b = b!.storeInt(2, 32)!.storeInt(3, 32); + + var cs: slice = b.endCell().beginParse(); + var (one, _, three) = (cs.getFirstBits(32).loadUint(32), cs.skipBits(64), cs.load_u32()); + + return [one,three]; +} + +@method_id(105) +fun test5(): [int,int] { + var cref: cell = endCell(beginCell().store_u32(105)); + var c: cell = beginCell().storeRef(cref).storeRef(cref).store_u32(1).endCell(); + + var cs: slice = beginParse(c); + var sto5x2: int = cs.loadRef().beginParse().load_u32() + cs.loadRef().beginParse().loadUint(32); + return [sto5x2, cs.load_u32()]; +} + +@method_id(106) +fun test6() { + return beginCell().storeUint(1, 32).storeUint(2, 32).storeUint(3, 32); +} + +@method_id(107) +fun test7() { + // since .store() methods now mutate, this piece of code works not as earlier (mutates uri_builder) + var uri_builder = beginCell(); + var uri_slice = uri_builder.storeSlice(".json").endCell().beginParse(); + var image_slice = uri_builder.storeSlice(".png").endCell().beginParse(); + return (uri_builder.getBuilderBitsCount(), uri_slice.getRemainingBitsCount(), image_slice.getRemainingBitsCount()); +} + +@method_id(108) +fun test8() { + var uri_builder = beginCell(); + var fresh = uri_builder; + var uri_slice = fresh.storeSlice(".json").endCell().beginParse(); + var fresh redef = uri_builder; + var image_slice = fresh.storeSlice(".png").endCell().beginParse(); + return (uri_builder.getBuilderBitsCount(), uri_slice.getRemainingBitsCount(), image_slice.getRemainingBitsCount()); +} + + +fun sumNumbersInSlice(mutate self: slice): int { + var result = 0; + while (!self.isEndOfSliceBits()) { + result += self.loadUint(32); + } + return result; +} + +@method_id(110) +fun test10() { + var ref = beginCell().storeInt(100, 32).endCell(); + var s: slice = beginCell().storeInt(1, 32).storeInt(2, 32).storeRef(ref).endCell().beginParse(); + var result = (getRemainingBitsCount(s), s.sumNumbersInSlice(), getRemainingBitsCount(s), isEndOfSlice(s), isEndOfSliceBits(s), isEndOfSliceRefs(s)); + var ref2: cell = s.loadRef(); + var s2: slice = ref2.beginParse(); + s.assertEndOfSlice(); + return (result, s2.loadInt(32), s2.isEndOfSlice()); +} + +@method_id(111) +fun test11() { + var s: slice = beginCell().storeInt(1, 32).storeInt(2, 32).storeInt(3, 32).storeInt(4, 32).storeInt(5, 32).storeInt(6, 32).storeInt(7, 32).endCell().beginParse(); + var size1 = getRemainingBitsCount(s); + s!.skipBits(32); + var s1: slice = s.getFirstBits(64); + var n1 = s1.loadInt(32); + var size2 = getRemainingBitsCount(s); + s.loadInt(32); + var size3 = getRemainingBitsCount(s); + s.removeLastBits(32); + var size4 = getRemainingBitsCount(s); + var n2 = s.loadInt(32); + var size5 = getRemainingBitsCount(s); + return (n1, n2, size1, size2, size3, size4, size5); +} + +@method_id(112) +fun test12() { + var (result1, result2) = (0, 0); + try { + beginCell().storeRef(beginCell().endCell()).endCell().beginParse().assertEndOfSlice(); + result1 = 100; + } catch (code) { + result1 = code; + } + try { + beginCell().endCell().beginParse().assertEndOfSlice(); + result2 = 100; + } catch (code) { + result2 = code; + } + return (result1, result2); +} + +@method_id(113) +fun test13() { + var ref2 = beginCell().storeInt(1, 32).endCell(); + var ref1 = beginCell().storeInt(1, 32).storeRef(ref2).endCell(); + var c = beginCell().storeInt(444, 32).storeRef(ref1).storeRef(ref1).storeRef(ref1).storeRef(ref2).storeInt(4, 32).endCell(); + var (n_cells1, n_bits1, n_refs1) = c.calculateCellSizeStrict(10); + var s = c.beginParse(); + s.loadRef(); + s.loadRef(); + var n = s.loadInt(32); + var (n_cells2, n_bits2, n_refs2) = s.calculateSliceSizeStrict(10); + return ([n_cells1, n_bits1, n_refs1], [n_cells2, n_bits2, n_refs2], n); +} + +@method_id(114) +fun test110(x: bool) { + var s = beginCell().storeBool(x == true).storeBool(false).storeBool(x).endCell().beginParse(); + return (s.loadBool(), s.loadBool(), s.loadBool()); +} + +@method_id(115) +fun test111() { + var s = beginCell().storeMessageOp(123).storeMessageQueryId(456) + .storeAddressNone().storeAddressNone() + .storeUint(0, 32) + .storeUint(123, 32).storeUint(456, 64).storeUint(789, 64) + .endCell().beginParse(); + var op1 = s.loadUint(32); + var q1 = s.loadUint(64); + if (s.addressIsNone()) { + s.skipBits(2); + } + if (s.loadBool() == false) { + assert(!s.loadBool()) throw 444; + s.skipBouncedPrefix(); + } + var op2 = s.loadMessageOp(); + var q2 = s.loadMessageQueryId(); + s.skipBits(64); + s.assertEndOfSlice(); + assert(isMessageBounced(0x001) && !isMessageBounced(0x002)) throw 444; + return (op1, q1, op2, q2); +} + +fun main(): int { + return 0; +} + +/** +@testcase | 101 | | [ 1 2 3 4 5 ] +@testcase | 102 | | [ 1 2 3 ] +@testcase | 103 | 103 | 103 +@testcase | 104 | | [ 1 3 ] +@testcase | 105 | | [ 210 1 ] +@testcase | 107 | | 72 40 72 +@testcase | 108 | | 0 40 32 +@testcase | 110 | | 64 3 0 0 -1 0 100 -1 +@testcase | 111 | | 2 3 224 192 160 128 96 +@testcase | 112 | | 9 100 +@testcase | 113 | | [ 3 128 5 ] [ 2 96 3 ] 444 +@testcase | 114 | -1 | -1 0 -1 +@testcase | 114 | 0 | 0 0 0 +@testcase | 115 | | 123 456 123 456 + +Note, that since 'compute-asm-ltr' became on be default, chaining methods codegen is not quite optimal. +@fif_codegen +""" + test6 PROC:<{ + // + NEWC // '0 + 1 PUSHINT // '0 '1=1 + SWAP // '1=1 '0 + 32 STU // '0 + 2 PUSHINT // '0 '4=2 + SWAP // '4=2 '0 + 32 STU // '0 + 3 PUSHINT // '0 '7=3 + SWAP // '7=3 '0 + 32 STU // '0 + }> +""" + */ diff --git a/tolk-tester/tests/co1.tolk b/tolk-tester/tests/co1.tolk new file mode 100644 index 00000000..f124e1de --- /dev/null +++ b/tolk-tester/tests/co1.tolk @@ -0,0 +1,72 @@ +const int1 = 1; +const int2 = 2; + +const int101: int = 101; +const int111: int = 111; + +const int1r = int1; + +const str1 = "const1"; +const str2 = "aabbcc"s; + +const str2r: slice = str2; + +const str1int = 0x636f6e737431; +const str2int = 0xAABBCC; + +const nibbles: int = 4; + +fun iget1(): int { return int1; } +fun iget2(): int { return int2; } +fun iget3(): int { return int1+int2; } + +fun iget1r(): int { return int1r; } + +fun sget1(): slice { return str1; } +fun sget2(): slice { return str2; } +fun sget2r(): slice { return str2r; } + +const int240: int = ((int1+int2)*10)<<3; + +fun iget240(): int { return int240; } + +@pure +fun newc(): builder +asm "NEWC"; +@pure +fun endcs(b: builder): slice +asm "ENDC" "CTOS"; +@pure +fun sdeq(s1: slice, s2: slice): int +asm "SDEQ"; +@pure +fun stslicer(b: builder, s: slice): builder +asm "STSLICER"; + +fun main() { + var i1: int = iget1(); + var i2: int = iget2(); + var i3: int = iget3(); + + assert(i1 == 1) throw int101; + assert(i2 == 2) throw 102; + assert(i3 == 3) throw 103; + + var s1: slice = sget1(); + var s2: slice = sget2(); + var s3: slice = newc().stslicer(str1).stslicer(str2r).endcs(); + + assert(sdeq(s1, newc().storeUint(str1int, 12 * nibbles).endcs())) throw int111; + assert(sdeq(s2, newc().storeUint(str2int, 6 * nibbles).endcs())) throw 112; + assert(sdeq(s3, newc().storeUint(0x636f6e737431AABBCC, 18 * nibbles).endcs())) throw 113; + + var i4: int = iget240(); + assert(i4 == 240) throw ((104)); + return 0; +} + +/** +@testcase | 0 | | 0 + +@code_hash 61273295789179921867241079778489100375537711211918844448475493726205774530743 +*/ diff --git a/tolk-tester/tests/code_after_ifelse.tolk b/tolk-tester/tests/code_after_ifelse.tolk new file mode 100644 index 00000000..6a16262f --- /dev/null +++ b/tolk-tester/tests/code_after_ifelse.tolk @@ -0,0 +1,41 @@ +fun elseif(cond: int) { + if (cond > 0) { + throw(cond); + } +} + +@inline +@method_id(101) +fun foo(x: int): int { + if (x==1) { + return 111; + } else { + x *= 2; + } + return x + 1; +} + +fun main(x: int): (int, int) { + return (foo(x), 222); +} + +@method_id(102) +fun test2(x: int) { + try { + if (x < 0) { return -1; } + elseif (x); + } catch(excNo) { + return excNo * 1000; + } + return 0; +} + +/** + method_id | in | out +@testcase | 0 | 1 | 111 222 +@testcase | 0 | 3 | 7 222 +@testcase | 101 | 1 | 111 +@testcase | 101 | 3 | 7 +@testcase | 102 | -5 | -1 +@testcase | 102 | 5 | 5000 +*/ diff --git a/tolk-tester/tests/codegen_check_demo.tolk b/tolk-tester/tests/codegen_check_demo.tolk new file mode 100644 index 00000000..5b46c093 --- /dev/null +++ b/tolk-tester/tests/codegen_check_demo.tolk @@ -0,0 +1,96 @@ +@method_id(101) +fun test1(): int { + var x: int = false as int; + if (x == true as int) { + x= 100500; + } + return x; +} + +fun main(s: int) { + var (z, t) = (17, s); + while (z > 0) { + t = s; + z -= 1; + } + return ~ t; +} + +/** + method_id | in | out +@testcase | 0 | 1 | -2 +@testcase | 0 | 5 | -6 +@testcase | 101 | | 0 + +Below, I just give examples of @fif_codegen tag: +* a pattern can be single-line (after the tag), or multi-line, surrounded with """ +* there may be multiple @fif_codegen, they all will be checked +* identation (spaces) is not checked intentionally +* "..." means any number of any lines +* lines not divided with "..." are expected to be consecutive in fif output +* //comments can be omitted, but if present, they are also expected to be equal +* there is also a tag @fif_codegen_avoid to check a pattern does not occur + +@fif_codegen +""" +main PROC:<{ + // s + 17 PUSHINT // s '3=17 + OVER // s z=17 t + WHILE:<{ + ... + }>DO<{ // s z t + ... + s1 s(-1) PUXC // s t z + ... + 2 1 BLKDROP2 + ... +}> +""" + +@fif_codegen +""" +main PROC:<{ + ... + WHILE:<{ + ... + }>DO<{ + ... + }> + }END>c +""" + +@fif_codegen +""" + OVER + 0 GTINT // s z t '5 +""" + +@fif_codegen +""" + "Asm.fif" include + ... + PROGRAM{ + ... + }END>c +""" + +@fif_codegen +""" +test1 PROC:<{ +// +FALSE +}> +""" + +@fif_codegen NOT // '8 +@fif_codegen main PROC:<{ + +@fif_codegen_avoid PROCINLINE +@fif_codegen_avoid END c +@fif_codegen_avoid +""" +multiline +can also be +""" +*/ diff --git a/tolk-tester/tests/comments.tolk b/tolk-tester/tests/comments.tolk new file mode 100644 index 00000000..cd287747 --- /dev/null +++ b/tolk-tester/tests/comments.tolk @@ -0,0 +1,31 @@ + +fun main(): int + +// inside a comment, /* doesn't start a new one +/* but if // is inside, a comment may end at this line*/ { + var cc = "a string may contain /* or // or /*, not parsed"; + // return 1; + return get10() + /* + traditional comment /* may not be nested + // line comment + // ends */1 + + 1; + /* moreover, different comment styles + may be used for opening and closing + */ +} + +/*** + first line + //two-lined*/ + +@method_id(10) +fun get10(): int { + return 10; +} + + +/** +@testcase | 0 | | 12 +@testcase | 10 | | 10 +*/ diff --git a/tolk-tester/tests/dicts-demo.tolk b/tolk-tester/tests/dicts-demo.tolk new file mode 100644 index 00000000..606318cb --- /dev/null +++ b/tolk-tester/tests/dicts-demo.tolk @@ -0,0 +1,105 @@ +import "@stdlib/tvm-dicts" + +fun addIntToIDict(mutate self: cell?, key: int, number: int): void { + return self.iDictSetBuilder(32, key, beginCell().storeInt(number, 32)); +} + +fun calculateDictLen(d: cell?) { + var len = 0; + var (k, v, f) = d.uDictGetFirst(32); + while (f) { + len += 1; + (k, v, f) = d.uDictGetNext(32, k!); + } + return len; +} + +fun loadTwoDigitNumberFromSlice(mutate self: slice): int { + var n1 = self.loadInt(8); + var n2 = self.loadInt(8); + return (n1 - 48) * 10 + (n2 - 48); +} + + +@method_id(101) +fun test101(getK1: int, getK2: int, getK3: int) { + var dict = createEmptyDict(); + dict.uDictSetBuilder(32, 1, beginCell().storeUint(1, 32)); + var (old1: slice?, found1) = dict.uDictSetAndGet(32, getK1, beginCell().storeUint(2, 32).endCell().beginParse()); + var (old2: slice?, found2) = dict.uDictSetAndGet(32, getK2, beginCell().storeUint(3, 32).endCell().beginParse()); + var (cur3: slice?, found3) = dict.uDictGet(32, getK3); + return ( + found1 ? old1!.loadUint(32) : -1, + found2 ? old2!.loadUint(32) : -1, + found3 ? cur3!.loadUint(32) : -1 + ); +} + +@method_id(102) +fun test102() { + var dict = createEmptyDict(); + dict.addIntToIDict(2, 102); + dict.addIntToIDict(1, 101); + dict.addIntToIDict(4, 104); + dict.addIntToIDict(3, 103); + var deleted = createEmptyTuple(); + var shouldBreak = false; + while (!shouldBreak) { + var (kDel, kVal, wasDel) = dict.iDictDeleteLastAndGet(32); + if (wasDel) { + deleted.tuplePush([kDel, kVal!.loadInt(32)]); + } else { + shouldBreak = true; + } + } + return deleted; +} + +@method_id(103) +fun test103() { + var dict = createEmptyDict(); + dict.uDictSetBuilderIfNotExists(32, 1,beginCell().storeInt(1, 32)); + dict.uDictSetBuilderIfNotExists(32, 1,beginCell().storeInt(1, 32)); + var len1 = calculateDictLen(dict); + dict.uDictSetBuilderIfExists(32, 2,beginCell().storeInt(1, 32)); + dict.uDictSetBuilderIfExists(32, 2,beginCell().storeInt(1, 32)); + var len2 = calculateDictLen(dict); + dict.uDictSetBuilder(32, 3,beginCell().storeInt(1, 32)); + dict.uDictSetBuilderIfExists(32, 3,beginCell().storeInt(1, 32)); + var len3 = calculateDictLen(dict); + var (delK1, _, _) = dict.uDictDeleteFirstAndGet(32); + var (delK2, _, _) = dict.uDictDeleteFirstAndGet(32); + var (delK3, _, _) = dict.uDictDeleteFirstAndGet(32); + return (len1, len2, len3, delK1, delK2, delK3); +} + +@method_id(104) +fun test104() { + var dict = createEmptyDict(); + dict.sDictSetBuilder(32, "7800", beginCell().storeUint(5 + 48, 8).storeUint(6 + 48, 8)); + dict.sDictSet(32, "key1", "12"); + var (old1, _) = dict.sDictSetAndGet(32, "key1", "34"); + var (old2, _) = dict.sDictDeleteAndGet(32, "key1"); + var (restK, restV, _) = dict.sDictGetFirst(32); + var (restK1, restV1, _) = dict.sDictDeleteLastAndGet(32); + assert (restK!.isSliceBitsEqual(restK1!)) throw 123; + assert (restV!.isSliceBitsEqual(restV1!)) throw 123; + return ( + old1!.loadTwoDigitNumberFromSlice(), + old2!.loadTwoDigitNumberFromSlice(), + restV!.loadTwoDigitNumberFromSlice(), + restK!.loadTwoDigitNumberFromSlice(), + restK!.loadTwoDigitNumberFromSlice() + ); +} + +fun main() {} + +/** +@testcase | 101 | 1 1 1 | 1 2 3 +@testcase | 101 | 1 2 1 | 1 -1 2 +@testcase | 101 | 1 2 3 | 1 -1 -1 +@testcase | 102 | | [ [ 4 104 ] [ 3 103 ] [ 2 102 ] [ 1 101 ] ] +@testcase | 103 | | 1 1 2 1 3 (null) +@testcase | 104 | | 12 34 56 78 0 + */ diff --git a/tolk-tester/tests/generics-1.tolk b/tolk-tester/tests/generics-1.tolk new file mode 100644 index 00000000..ca310927 --- /dev/null +++ b/tolk-tester/tests/generics-1.tolk @@ -0,0 +1,160 @@ +fun eq1(value: X): X { return value; } +fun eq2(value: X) { return value; } +fun eq3(value: X): X { var cp: [X] = [eq1(value)]; var ((([v: X]))) = cp; return v; } +fun eq4(value: X) { return eq1(value); } + +@method_id(101) +fun test101(x: int) { + var (a, b, c) = (x, (x,x), [x,x]); + return (eq1(a), eq1(b), eq1(c), eq2(a), eq2(b), eq2(c), eq3(a), eq4(b), eq3(createEmptyTuple())); +} + +fun getTwo(): X { return 2 as X; } + +fun takeInt(a: int) { return a; } + +@method_id(102) +fun test102(): (int, int, int, [int, int]) { + var a: int = getTwo(); + var _: int = getTwo(); + var b = getTwo() as int; + var c: int = 1 ? getTwo() : getTwo(); + var c redef = getTwo(); + var ab_tens = (0, (1, 2)); + ab_tens.0 = getTwo(); + ab_tens.1.1 = getTwo(); + var ab_tup = [0, [1, 2]]; + ab_tup.0 = getTwo(); + ab_tup.1.1 = getTwo(); + return (eq1(a), eq2(b), takeInt(getTwo()), [getTwo(), ab_tens.1.1]); +} + +@method_id(103) +fun test103(first: int): (int, int, int) { + var t = createEmptyTuple(); + var cs = beginCell().storeInt(100, 32).endCell().beginParse(); + t.tuplePush(first); + t.tuplePush(2); + t.tuplePush(cs); + cs = t.tupleAt(2); + cs = t.tupleAt(2) as slice; + return (t.tupleAt(0), cs.loadInt(32), t.tupleAt(2).loadInt(32)); +} + +fun manyEq(a: T1, b: T2, c: T3): [T1, T2, T3] { + return [a, b, c]; +} + +@method_id(104) +fun test104(f: int) { + var result = ( + manyEq(1 ? 1 : 1, f ? 0 : null, !f ? getTwo() as int : null), + manyEq(f ? null as int? : eq2(2), beginCell().storeBool(true).endCell().beginParse().loadBool(), eq4(f)) + ); + __expect_type(result, "([int, int?, int?], [int?, bool, int])"); + return result; +} + +fun calcSum(x: X, y: X) { return x! + y!; } + +@method_id(105) +fun test105() { + if (0) { calcSum(((0 as int?)), null); } + return (calcSum(1, 2)); +} + +fun calcYPlus1(value: Y) { return value + 1; } +fun calcLoad32(cs: slice) { return cs.loadInt(32); } +fun calcTensorPlus1(tens: (int, int)) { var (f, s) = tens; return (f + 1, s + 1); } +fun calcTensorMul2(tens: (int, int)) { var (f, s) = tens; return (f * 2, s * 2); } +fun cellToSlice(c: cell) { return c.beginParse(); } +fun abstractTransform(xToY: (X) -> Y, yToR: (((Y))) -> R, initialX: X): R { + var y = xToY(initialX); + return yToR(y); +} + +@method_id(106) +fun test106() { + var c = beginCell().storeInt(106, 32).endCell(); + __expect_type(calcYPlus1, "(int) -> int"); + return [ + abstractTransform(cellToSlice, calcLoad32, c), + abstractTransform(calcYPlus1, calcYPlus1, 0), + abstractTransform(calcTensorPlus1, calcTensorMul2, (2, 2)).0, + abstractTransform(calcTensorPlus1, calcTensorMul2, (2, 2)).1 + ]; +} + +fun callTupleFirst(t: X): Y { return t.tupleFirst(); } +fun callTuplePush(mutate self: T, v1: V, v2: V): self { self.tuplePush(v1); tuplePush(mutate self, v2); return self; } +fun getTupleLastInt(t: tuple) { return t.tupleLast(); } +fun getTupleSize(t: tuple) { return t.tupleSize(); } +fun callAnyFn(f: (TObj) -> TResult, arg: TObj) { return f(arg); } +fun callAnyFn2(f: TCallback, arg: tuple) { return f(arg); } + +global t107: tuple; + +@method_id(107) +fun test107() { + t107 = createEmptyTuple(); + callTuplePush(mutate t107, 1, 2); + t107.callTuplePush(3, 4).callTuplePush(5, 6); + var first: int = t107.callTupleFirst(); + return ( + callAnyFn(getTupleSize, t107), + callAnyFn2(getTupleSize, t107), + first, + callTupleFirst(t107) as int, + callAnyFn(getTupleLastInt, t107), + callAnyFn2(getTupleLastInt, t107) + ); +} + +global g108: int; + +fun inc108(by: int) { g108 += by; } +fun getInc108() { return inc108; } +fun returnResult(f: () -> RetT): RetT { return f(); } +fun applyAndReturn(f: () -> (ArgT) -> RetT, arg: ArgT): () -> ArgT -> RetT { + f()(arg); + return f; +} + +@method_id(108) +fun test108() { + g108 = 0; + getInc108()(1); + returnResult<(int) -> void>(getInc108)(2); + applyAndReturn(getInc108, 10)()(10); + returnResult(getInc108)(2); + applyAndReturn(getInc108, 10)()(10); + return g108; +} + +fun main(x: int): (int, [[int, int]]) { + try { if(x) { throw (1, x); } } + catch (excNo, arg) { return (arg as int, [[eq2(arg as int), getTwo()]]); } + return (0, [[x, 1]]); +} + +/** +@testcase | 0 | 1 | 1 [ [ 1 2 ] ] +@testcase | 101 | 0 | 0 0 0 [ 0 0 ] 0 0 0 [ 0 0 ] 0 0 0 [] +@testcase | 102 | | 2 2 2 [ 2 2 ] +@testcase | 103 | 0 | 0 100 100 +@testcase | 104 | 0 | [ 1 (null) 2 ] [ 2 -1 0 ] +@testcase | 105 | | 3 +@testcase | 106 | | [ 106 2 6 6 ] +@testcase | 107 | | 6 6 1 1 6 6 +@testcase | 108 | | 45 + +@fif_codegen DECLPROC eq1 +@fif_codegen DECLPROC eq1 +@fif_codegen DECLPROC eq1<(int,int)> +@fif_codegen DECLPROC eq1<[int,int]> +@fif_codegen DECLPROC getTwo + +@fif_codegen_avoid DECLPROC eq1 +@fif_codegen_avoid DECLPROC eq2 +@fif_codegen_avoid DECLPROC eq3 + */ diff --git a/tolk-tester/tests/if_stmt.tolk b/tolk-tester/tests/if_stmt.tolk new file mode 100644 index 00000000..0f78a516 --- /dev/null +++ b/tolk-tester/tests/if_stmt.tolk @@ -0,0 +1,66 @@ +@method_id(101) +fun test1(x: int): int { + if (x > 200) { + return 200; + } else if (x > 100) { + return 100; + } else if (!(x <= 50)) { + if (!(x > 90)) { + return x; + } else { + return 90; + } + } else { + return 0; + } +} + +@method_id(102) +fun test2(x: int) { + if (x == 20) { return 20; } + if (x != 50) { return 50; } + if (x == 0) { return 0; } + return -1; +} + +@method_id(103) +fun test3(x: int) { + if (!(x != 20)) { return 20; } + if (!(x == 50)) { return 50; } + if (!x) { return 0; } + return -1; +} + +fun main() { + +} + +/** +@testcase | 101 | 0 | 0 +@testcase | 101 | 1000 | 200 +@testcase | 101 | 150 | 100 +@testcase | 101 | -1 | 0 +@testcase | 101 | 87 | 87 +@testcase | 101 | 94 | 90 +@testcase | 102 | 20 | 20 +@testcase | 102 | 40 | 50 +@testcase | 102 | 50 | -1 +@testcase | 103 | 20 | 20 +@testcase | 103 | 40 | 50 +@testcase | 103 | 50 | -1 + +@fif_codegen +""" + test3 PROC:<{ + // x + DUP // x x + 20 NEQINT // x '2 + IFNOTJMP:<{ // x + DROP // + 20 PUSHINT // '3=20 + }> // x + DUP // x x + 50 EQINT // x '5 + IFNOTJMP:<{ // x +""" +*/ diff --git a/tolk-tester/tests/imports/invalid-no-import.tolk b/tolk-tester/tests/imports/invalid-no-import.tolk new file mode 100644 index 00000000..6c4ab6ce --- /dev/null +++ b/tolk-tester/tests/imports/invalid-no-import.tolk @@ -0,0 +1,4 @@ +fun demoOfInvalid(): (int) { + var f = someAdd; + return f(1, 2); +} diff --git a/tolk-tester/tests/imports/some-math.tolk b/tolk-tester/tests/imports/some-math.tolk new file mode 100644 index 00000000..dc0c9c9b --- /dev/null +++ b/tolk-tester/tests/imports/some-math.tolk @@ -0,0 +1,3 @@ +fun someAdd(a: int, b: int): int { + return a + b + 0; +} diff --git a/tolk-tester/tests/imports/use-dicts-err.tolk b/tolk-tester/tests/imports/use-dicts-err.tolk new file mode 100644 index 00000000..c5ba89d2 --- /dev/null +++ b/tolk-tester/tests/imports/use-dicts-err.tolk @@ -0,0 +1,21 @@ +fun prepareDict_3_30_4_40_5_x(valueAt5: int): cell { + var dict: cell = createEmptyDict(); + dict.idict_set_builder(32, 3, begin_cell().store_int(30, 32)); + dict.idict_set_builder(32, 4, begin_cell().store_int(40, 32)); + dict.idict_set_builder(32, 5, begin_cell().store_int(valueAt5, 32)); + return dict; +} + +fun lookupIdxByValue(idict32: cell, value: int): int { + var cur_key = -1; + do { + var (cur_key redef, cs: slice, found: int) = idict32.idictGetNext(32, cur_key); + // one-line condition (via &) doesn't work, since right side is calculated immediately + if (found) { + if (cs.loadInt(32) == value) { + return cur_key; + } + } + } while (found); + return -1; +} diff --git a/tolk-tester/tests/imports/use-dicts.tolk b/tolk-tester/tests/imports/use-dicts.tolk new file mode 100644 index 00000000..2daaf2b1 --- /dev/null +++ b/tolk-tester/tests/imports/use-dicts.tolk @@ -0,0 +1,23 @@ +import "@stdlib/tvm-dicts" + +fun prepareDict_3_30_4_40_5_x(valueAt5: int): cell? { + var dict: cell? = createEmptyDict(); + dict.iDictSetBuilder(32, 3, beginCell().storeInt(30, 32)); + dict.iDictSetBuilder(32, 4, beginCell().storeInt(40, 32)); + dict.iDictSetBuilder(32, 5, beginCell().storeInt(valueAt5, 32)); + return dict; +} + +fun lookupIdxByValue(idict32: cell?, value: int): int { + var cur_key: int? = -1; + do { + var (cur_key redef, cs: slice?, found: bool) = idict32.iDictGetNext(32, cur_key!); + // one-line condition (via &) doesn't work, since right side is calculated immediately + if (found) { + if (cs!.loadInt(32) == value) { + return cur_key!; + } + } + } while (found); + return -1; +} diff --git a/tolk-tester/tests/indexed-access.tolk b/tolk-tester/tests/indexed-access.tolk new file mode 100644 index 00000000..e2bd3dd9 --- /dev/null +++ b/tolk-tester/tests/indexed-access.tolk @@ -0,0 +1,358 @@ + +fun increment(mutate self: int) { + self += 1; +} + +fun increment2(mutate a: int, mutate b: int) { + a += 1; + b += 1; +} + +fun assign1020(mutate a: int, mutate b: int) { + a = 10; + b = 20; +} + +fun plus(mutate self: int, y: int): int { + val newVals = (self + y, y * 10); + self = newVals.0; + return newVals.1; +} + +fun eq(v: X): X { return v; } + +global gTup: [int]; +global gTens: (int, int); + +@method_id(100) +fun testCodegenSimple() { + var t1 = [1]; + t1.0 = 2; + debugPrintString(""); + var t2 = [[1]]; + t2.0.0 = 2; + debugPrintString(""); + gTup = [1]; + gTup.0 = 2; + debugPrintString(""); + gTens = (1,2); + gTens.1 = 4; + debugPrintString(""); + return (t1, t2, gTup, gTens); +} + +@method_id(101) +fun test101() { + var t = (1, (2, 3), [4, 5, [6, 7]], 8); + (t.0, t.1.0, t.2.0) = (2, 3, 5); + t.3.increment(); + t.2.1 += (t.1.1 += 1) - t.1.1 + 1; + increment2(mutate t.2.2.0, mutate t.2.2.1); + return t; +} + +global t102: (int, (int, int), [int, int, [int, int]], int); + +@method_id(102) +fun test102() { + t102 = (1, (2, 3), [4, 5, [6, 7]], 8); + (t102.0, t102.1.0, t102.2.0) = (2, 3, 5); + t102.3.increment(); + t102.2.1 += (t102.1.1 += 1) - t102.1.1 + 1; + increment2(mutate t102.2.2.0, mutate t102.2.2.1); + return t102; +} + +global t103: (int, int); + +@method_id(103) +fun test103() { + t103 = (5, 5); + assign1020(mutate t103.0, mutate t103.1); + var t = (5, 5); + assign1020(mutate t.0, mutate t.1); + return (t103, t); +} + +global t104: [[int, int]]; + +@method_id(104) +fun test104() { + var m = [[5, 5]]; + (m.0.0, m.0.1) = (10, 20); + t104 = [[5, 5]]; + (t104.0.0, t104.0.1) = (10, 20); + return (t104, m); +} + +@method_id(105) +fun test105(x: int, y: int): (tuple, int, (int?, int), int, int) { + var ab = (createEmptyTuple(), (x as int?, y), tupleSize); + ab.0.tuplePush(1); + tuplePush(mutate ab.0, 2); + ab.1.0 = null; + ab.1.1 += 10; + var cb = ab.2; + return (ab.0, ab.0.1, ab.1, cb(ab.0), ab.2(ab.0)); +} + +@method_id(106) +fun test106(x: int, y: int) { + var ab = [createEmptyTuple(), [x as int?, y], tupleSize]; + ab.0.tuplePush(1); + tuplePush(mutate ab.0, 2); + ab.1.0 = null; + ab.1.1 += 10; + var cb = ab.2; + return (ab.0, ab.1, cb(ab.0), ab.2(ab.0)); +} + +@method_id(107) +fun test107() { + var ab = createEmptyTuple(); + ab.tuplePush(1); + ab.tuplePush(beginCell().storeInt(1, 32)); + return (ab.0 as int, getBuilderBitsCount(ab.1)); +} + +global t108: [int, [int, [int]]]; + +@method_id(108) +fun test108(last: int) { + t108 = [1, [2, [last]]]; + t108.1.1.0.increment(); + var t = [1, [2, [last]]]; + t.1.1.0.increment(); + return (t108, t); +} + +@method_id(109) +fun test109(x: (int, int)): (int, int, int, int, int, int, int) { + return (x.1, x.1.plus(x.1 / 20), x.1, x.1 = x.1 * 2, x.1, x.1 += 1, x.1); +} + +@method_id(110) +fun test110(f: int, s: int) { + var x = [f, s]; + return (x, x.1, x.1.plus(x.1 / 20), x.1, x.1 = x.1 * 2, x.1, x.1 += 1, x.1, x); +} + +global xx: (int, int); + +@method_id(111) +fun test111(x: (int, int)) { + xx = x; + return (x, xx.1, xx.1.plus(xx.1 / 20), eq(xx.1 += (x.1 *= 0)), xx.1 = xx.1 * 2, xx.1, xx.1 += 1, xx.1, x); +} + +global yy: [int, int]; + +@method_id(112) +fun test112(f: int, s: int) { + yy = [f, s]; + return (yy, yy.1, yy.1.plus(yy.1 / 20), eq(yy.1 += (yy.1 *= 0)), yy.1 = yy.1 * 2, yy.1, yy.1 += 1, yy.1, yy); +} + +@pure +fun getConstTuple() { + return [1,2]; +} + +fun testCodegenNoPureIndexedAccess() { + (getConstTuple().1, getConstTuple().0) = (3, 4); + return 0; +} + +@method_id(113) +fun test113() { + var x = [[1, 2]]; + return (x, x.0, plus(mutate x.0.0, 10), x.0, x, x.0 = [10, 20], x); +} + +@method_id(114) +fun test114(f: int, s: int) { + var x = ((), (f, s), ()); + return (x, x.1, plus(mutate x.1.0, 10), x.1, x, x.1 = (10, 20), x); +} + +@method_id(115) +fun test115() { + var y = [[[[true]]]]; + return (y, ((((y).0).0).0).0 = !y.0.0.0.0, y.0); +} + +@method_id(116) +fun test116() { + var t = createEmptyTuple(); + t.tuplePush(1); + try { + return t.100500 as int; + } catch(excNo) { + return excNo; + } +} + +@method_id(117) +fun test117() { + var t = createEmptyTuple(); + t.tuplePush(1); + try { + return (t.0 as tuple).0 as int; + } catch(excNo) { + return excNo; + } +} + +@method_id(118) +fun testCodegenIndexPostfix1(x: (int, int)) { + var ab = (x.1, x.0); + return ab; +} + +@method_id(119) +fun testCodegenIndexPostfix2(x: (int, (int, int), int)) { + var y = x; + return (y.2, y.0, y.1.1); +} + +fun getT() { return (1, 2); } + +@method_id(120) +fun test120() { + return (getT().0 = 3, getT().0 = 4, [getT().0 = 5, getT().0 = 6]); +} + +@method_id(121) +fun test121(zero: int) { + var t = createEmptyTuple(); + t.tuplePush(-100); + t.tupleSetAt(0, zero); + (t.0 as int).increment(); + (((t.0) as int) as int).increment(); + increment(mutate t.0 as int); + return t; +} + +fun isFirstComponentGt0(t: (T1, T2)): bool { + return t.0 > 0; +} + +@method_id(122) +fun test122(x: (int, int)) { + return ( + isFirstComponentGt0(x), isFirstComponentGt0((2, beginCell())), isFirstComponentGt0((0, null)), + x.isFirstComponentGt0(), (2, beginCell()).isFirstComponentGt0(), (0, null).isFirstComponentGt0() + ); +} + +@method_id(123) +fun test123() { + var t = [[10, 20]] as [[int,int]]?; + ((t!).0).0 = ((t!).0).1 = 100; + return t; +} + +fun main(){} + + +/** +@testcase | 101 | | 2 3 4 [ 5 6 [ 7 8 ] ] 9 +@testcase | 102 | | 2 3 4 [ 5 6 [ 7 8 ] ] 9 +@testcase | 103 | | 10 20 10 20 +@testcase | 104 | | [ [ 10 20 ] ] [ [ 10 20 ] ] +@testcase | 105 | 5 6 | [ 1 2 ] 2 (null) 16 2 2 +@testcase | 106 | 5 6 | [ 1 2 ] [ (null) 16 ] 2 2 +@testcase | 107 | | 1 32 +@testcase | 108 | 3 | [ 1 [ 2 [ 4 ] ] ] [ 1 [ 2 [ 4 ] ] ] +@testcase | 109 | 0 100 | 100 50 105 210 210 211 211 +@testcase | 110 | 0 100 | [ 0 100 ] 100 50 105 210 210 211 211 [ 0 211 ] +@testcase | 111 | 0 100 | 0 100 100 50 105 210 210 211 211 0 0 +@testcase | 112 | 0 100 | [ 0 100 ] 100 50 105 210 210 211 211 [ 0 211 ] +@testcase | 113 | | [ [ 1 2 ] ] [ 1 2 ] 100 [ 11 2 ] [ [ 11 2 ] ] [ 10 20 ] [ [ 10 20 ] ] +@testcase | 114 | 1 2 | 1 2 1 2 100 11 2 11 2 10 20 10 20 +@testcase | 115 | | [ [ [ [ -1 ] ] ] ] 0 [ [ [ 0 ] ] ] +@testcase | 116 | | 5 +@testcase | 117 | | 7 +@testcase | 118 | 1 2 | 2 1 +@testcase | 119 | 1 2 3 4 | 4 1 3 +@testcase | 120 | | 3 4 [ 5 6 ] +@testcase | 121 | 0 | [ 3 ] +@testcase | 122 | 1 2 | -1 -1 0 -1 -1 0 +@testcase | 123 | | [ [ 100 100 ] ] + +@fif_codegen +""" + testCodegenSimple PROC:<{ + // + 1 PUSHINT // '2=1 + SINGLE // t1 + 2 PUSHINT // t1 '3=2 + 0 SETINDEX // t1 + x{} PUSHSLICE // t1 '6 + STRDUMP DROP + 1 PUSHINT // t1 '10=1 + SINGLE // t1 '9 + SINGLE // t1 t2 + 2 PUSHINT // t1 t2 '11=2 + OVER // t1 t2 '11=2 t2 + 0 INDEX // t1 t2 '11=2 '14 + SWAP // t1 t2 '14 '11=2 + 0 SETINDEX // t1 t2 '14 + 0 SETINDEX // t1 t2 + x{} PUSHSLICE // t1 t2 '17 + STRDUMP DROP + 1 PUSHINT // t1 t2 '20=1 + SINGLE // t1 t2 '18 + gTup SETGLOB + 2 PUSHINT // t1 t2 '21=2 + gTup GETGLOB // t1 t2 '21=2 g_gTup + SWAP // t1 t2 g_gTup '21=2 + 0 SETINDEX // t1 t2 g_gTup + gTup SETGLOB + x{} PUSHSLICE // t1 t2 '25 + STRDUMP DROP + 1 PUSHINT // t1 t2 '28=1 + 2 PUSHINT // t1 t2 '26=1 '27=2 + PAIR + gTens SETGLOB + 4 PUSHINT // t1 t2 g_gTens.1=4 + gTens GETGLOB + UNPAIR // t1 t2 g_gTens.1=4 g_gTens.0 g_gTens.1 + DROP // t1 t2 g_gTens.1=4 g_gTens.0 + SWAP // t1 t2 g_gTens.0 g_gTens.1=4 + PAIR + gTens SETGLOB + x{} PUSHSLICE // t1 t2 '36 + STRDUMP DROP + gTup GETGLOB // t1 t2 g_gTup + gTens GETGLOB + UNPAIR // t1 t2 g_gTup g_gTens.0 g_gTens.1 + }> +""" + +@fif_codegen +""" + testCodegenNoPureIndexedAccess PROC:<{ + // + 0 PUSHINT // '8=0 + }> +""" + +@fif_codegen +""" + testCodegenIndexPostfix1 PROC:<{ + // x.0 x.1 + // ab.1 ab.0 + SWAP // ab.0 ab.1 + }> +""" + +@fif_codegen +""" + testCodegenIndexPostfix2 PROC:<{ + // x.0 x.1.0 x.1.1 x.2 + s2 POP // y.0 y.2 y.1.1 + s1 s2 XCHG // y.2 y.0 y.1.1 + }> +""" + */ diff --git a/tolk-tester/tests/inference-tests.tolk b/tolk-tester/tests/inference-tests.tolk new file mode 100644 index 00000000..5020d0dd --- /dev/null +++ b/tolk-tester/tests/inference-tests.tolk @@ -0,0 +1,107 @@ +// the goal of this file is not only to @testcase results — +// but to check that this file compiles + +fun eq(value: X): X { return value; } + +fun test1(x: int, y: int) { + __expect_type(0, "int"); + __expect_type("0"c, "int"); + __expect_type(x, "int"); + __expect_type(x + y, "int"); + __expect_type(x * y, "int"); + __expect_type(x & y, "int"); + __expect_type(x << y, "int"); + __expect_type((((x))), "int"); + __expect_type(x = x, "int"); + __expect_type(x += x, "int"); + __expect_type(x &= x, "int"); + __expect_type(random() ? x : y, "int"); + __expect_type(eq(x), "int"); + __expect_type(eq(x), "int"); + __expect_type(eq(null), "int?"); + __expect_type(x as int, "int"); + __expect_type(+x, "int"); + __expect_type(~x, "int"); + __expect_type(x!, "int"); + __expect_type(x!!!, "int"); + { + var x: slice = beginCell().endCell().beginParse(); + __expect_type(x, "slice"); + __expect_type(beginCell(), "builder"); + __expect_type(beginCell().endCell(), "cell"); + } +} + +fun test2(x: int, y: bool) { + __expect_type(!x, "bool"); + __expect_type(x != x, "bool"); + __expect_type(x <= x, "bool"); + __expect_type(x <=> x, "bool"); + __expect_type(x <=> x, "bool"); + __expect_type(!random(), "bool"); + __expect_type(!!(x != null), "bool"); + __expect_type(x ? x != null : null == x, "bool"); + __expect_type(y & true, "bool"); + __expect_type(y ^= false, "bool"); + __expect_type(x && y, "bool"); + __expect_type(true && false && true, "bool"); + __expect_type(x || x, "bool"); + __expect_type(x || !x || (true & false), "bool"); +} + +fun test3() { + __expect_type(true as int, "int"); + __expect_type(!random() as int, "int"); +} + +fun test4(x: int) { + __expect_type((), "()"); + __expect_type((x, x), "(int, int)"); + __expect_type((x, (x, x), x), "(int, (int, int), int)"); +} + +fun test5(x: int) { + __expect_type([], "[]"); + __expect_type([x], "[int]"); + __expect_type([x, x >= 1], "[int, bool]"); + __expect_type([x, x >= 1, null as slice?], "[int, bool, slice?]"); + __expect_type((x, [x], [[x], x]), "(int, [int], [[int], int])"); + __expect_type(getMyOriginalBalanceWithExtraCurrencies(), "[int, cell?]"); +} + +fun test6() { + var t = createEmptyTuple(); + __expect_type(t, "tuple"); + t.tuplePush(1); + __expect_type(t, "tuple"); +} + +fun test7() { + __expect_type(test3(), "void"); + __expect_type(test3, "() -> void"); + var cb = test1; + __expect_type(cb, "(int, int) -> void"); + var t = createEmptyTuple(); + __expect_type(beginCell().endCell, "(builder) -> cell"); + // __expect_type(eq<(int, slice)>, "(int, slice) -> (int, slice)"); +} + +fun alwaysThrows(): never { throw 123; } +fun alwaysThrowsNotAnnotated() { throw 123; } +fun alwaysThrowsNotAnnotated2() { alwaysThrows(); } + +fun test9() { + __expect_type(alwaysThrows(), "never"); + __expect_type(alwaysThrows, "() -> never"); + __expect_type(alwaysThrowsNotAnnotated(), "void"); + __expect_type(alwaysThrowsNotAnnotated2(), "void"); +} + + +fun main() { + return 0; +} + +/** +@testcase | 0 | | 0 +*/ diff --git a/tolk-tester/tests/inline_big.tolk b/tolk-tester/tests/inline_big.tolk new file mode 100644 index 00000000..be014eb5 --- /dev/null +++ b/tolk-tester/tests/inline_big.tolk @@ -0,0 +1,62 @@ +@inline +fun foo(x: int): int { + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + x = x * 10 + 1; + return x; +} + +fun main(x: int): int { + return foo(x) * 10 + 5; +} +/** + method_id | in | out +@testcase | 0 | 9 | 9111111111111111111111111111111111111111111111111115 +*/ diff --git a/tolk-tester/tests/inline_if.tolk b/tolk-tester/tests/inline_if.tolk new file mode 100644 index 00000000..9f1fa8c1 --- /dev/null +++ b/tolk-tester/tests/inline_if.tolk @@ -0,0 +1,28 @@ +fun foo1(x: int): int { + if (x == 1) { + return 1; + } + return 2; +} +@inline +fun foo2(x: int): int { + if (x == 1) { + return 11; + } + return 22; +} +@inline_ref +fun foo3(x: int): int { + if (x == 1) { + return 111; + } + return 222; +} +fun main(x: int): (int, int, int) { + return (foo1(x)+1, foo2(x)+1, foo3(x)+1); +} +/** + method_id | in | out +@testcase | 0 | 1 | 2 12 112 +@testcase | 0 | 2 | 3 23 223 +*/ diff --git a/tolk-tester/tests/inline_loops.tolk b/tolk-tester/tests/inline_loops.tolk new file mode 100644 index 00000000..eba595a5 --- /dev/null +++ b/tolk-tester/tests/inline_loops.tolk @@ -0,0 +1,48 @@ +global g: int; + +@inline +fun foo_repeat() { + g = 1; + repeat(5) { + g *= 2; + } +} + +@inline +fun foo_until(): int { + g = 1; + var i: int = 0; + do { + g *= 2; + i += 1; + } while (i < 8); + return i; +} + +@inline +fun foo_while(): int { + g = 1; + var i: int = 0; + while (i < 10) { + g *= 2; + i += 1; + } + return i; +} + +fun main() { + foo_repeat(); + var x: int = g; + foo_until(); + var y: int = g; + foo_while(); + var z: int = g; + return (x, y, z); +} + +/** + method_id | in | out +@testcase | 0 | | 32 256 1024 + +@code_hash 102749806552989901976653997041637095139193406161777448419603700344770997608788 +*/ diff --git a/tolk-tester/tests/invalid-assign-1.tolk b/tolk-tester/tests/invalid-assign-1.tolk new file mode 100644 index 00000000..799176df --- /dev/null +++ b/tolk-tester/tests/invalid-assign-1.tolk @@ -0,0 +1,9 @@ +fun main() { + var t = createEmptyTuple(); + t.0 = (1, 2); +} + +/** +@compilation_should_fail +@stderr a tuple can not have `(int, int)` inside, because it occupies 2 stack slots in TVM, not 1 +*/ diff --git a/tolk-tester/tests/invalid-assign-2.tolk b/tolk-tester/tests/invalid-assign-2.tolk new file mode 100644 index 00000000..6a33e696 --- /dev/null +++ b/tolk-tester/tests/invalid-assign-2.tolk @@ -0,0 +1,8 @@ +fun main(cs: slice) { + var cb = cs.tupleSize; +} + +/** +@compilation_should_fail +@stderr referencing a method for `tuple` with object of type `slice` +*/ diff --git a/tolk-tester/tests/invalid-assign-3.tolk b/tolk-tester/tests/invalid-assign-3.tolk new file mode 100644 index 00000000..567ace33 --- /dev/null +++ b/tolk-tester/tests/invalid-assign-3.tolk @@ -0,0 +1,9 @@ +fun main() { + var t = createEmptyTuple(); + var xy = t.0 as (int, int); +} + +/** +@compilation_should_fail +@stderr a tuple can not have `(int, int)` inside, because it occupies 2 stack slots in TVM, not 1 +*/ diff --git a/tolk-tester/tests/invalid-bitwise-1.tolk b/tolk-tester/tests/invalid-bitwise-1.tolk new file mode 100644 index 00000000..f939d60d --- /dev/null +++ b/tolk-tester/tests/invalid-bitwise-1.tolk @@ -0,0 +1,9 @@ +fun main(flags: int): int { + return flags&0xFF!=0; +} + +/** +@compilation_should_fail +@stderr & has lower precedence than != +@stderr Use parenthesis +*/ diff --git a/tolk-tester/tests/invalid-bitwise-2.tolk b/tolk-tester/tests/invalid-bitwise-2.tolk new file mode 100644 index 00000000..e6fcd1e5 --- /dev/null +++ b/tolk-tester/tests/invalid-bitwise-2.tolk @@ -0,0 +1,8 @@ +fun justTrue(): int { return true; } + +const a = justTrue() | 1 < 9; + +/** +@compilation_should_fail +@stderr | has lower precedence than < +*/ diff --git a/tolk-tester/tests/invalid-bitwise-3.tolk b/tolk-tester/tests/invalid-bitwise-3.tolk new file mode 100644 index 00000000..ee43860b --- /dev/null +++ b/tolk-tester/tests/invalid-bitwise-3.tolk @@ -0,0 +1,8 @@ +fun justTrue(): int { return true; } + +const a = justTrue() | (1 < 9) | justTrue() != true; + +/** +@compilation_should_fail +@stderr | has lower precedence than != +*/ diff --git a/tolk-tester/tests/invalid-bitwise-4.tolk b/tolk-tester/tests/invalid-bitwise-4.tolk new file mode 100644 index 00000000..563ed535 --- /dev/null +++ b/tolk-tester/tests/invalid-bitwise-4.tolk @@ -0,0 +1,6 @@ +const a = (1) <=> (0) ^ 8; + +/** +@compilation_should_fail +@stderr ^ has lower precedence than <=> +*/ diff --git a/tolk-tester/tests/invalid-bitwise-5.tolk b/tolk-tester/tests/invalid-bitwise-5.tolk new file mode 100644 index 00000000..1030ed8d --- /dev/null +++ b/tolk-tester/tests/invalid-bitwise-5.tolk @@ -0,0 +1,11 @@ +const MAX_SLIPAGE = 100; + +fun main(jetton_amount: int, msg_value: int, slippage: int) { + if ((0 == jetton_amount) | (msg_value == 0) | true | false | slippage > MAX_SLIPAGE) { + } +} + +/** +@compilation_should_fail +@stderr | has lower precedence than > +*/ diff --git a/tolk-tester/tests/invalid-bitwise-6.tolk b/tolk-tester/tests/invalid-bitwise-6.tolk new file mode 100644 index 00000000..9c4dc67e --- /dev/null +++ b/tolk-tester/tests/invalid-bitwise-6.tolk @@ -0,0 +1,9 @@ +fun main() { + if ((1==1)|(2==2)&(3==3)) { + } +} + +/** +@compilation_should_fail +@stderr mixing | with & without parenthesis +*/ diff --git a/tolk-tester/tests/invalid-bitwise-7.tolk b/tolk-tester/tests/invalid-bitwise-7.tolk new file mode 100644 index 00000000..39fba401 --- /dev/null +++ b/tolk-tester/tests/invalid-bitwise-7.tolk @@ -0,0 +1,8 @@ +fun main() { + var c = x && y || x && y; +} + +/** +@compilation_should_fail +@stderr mixing && with || without parenthesis +*/ diff --git a/tolk-tester/tests/invalid-builtin-1.tolk b/tolk-tester/tests/invalid-builtin-1.tolk new file mode 100644 index 00000000..6a7f1ca7 --- /dev/null +++ b/tolk-tester/tests/invalid-builtin-1.tolk @@ -0,0 +1,10 @@ +fun moddiv2(x: int, y: int): (int, int) builtin; + +/** +@compilation_should_fail +@stderr +""" +`builtin` used for non-builtin function +fun moddiv2 +""" +*/ diff --git a/tolk-tester/tests/invalid-call-1.tolk b/tolk-tester/tests/invalid-call-1.tolk new file mode 100644 index 00000000..7435bb3c --- /dev/null +++ b/tolk-tester/tests/invalid-call-1.tolk @@ -0,0 +1,10 @@ +const asdf = 1; + +fun main(x: int) { + return x.asdf(); +} + +/** +@compilation_should_fail +@stderr non-existing method `asdf` of type `int` + */ diff --git a/tolk-tester/tests/invalid-call-10.tolk b/tolk-tester/tests/invalid-call-10.tolk new file mode 100644 index 00000000..4da85f4f --- /dev/null +++ b/tolk-tester/tests/invalid-call-10.tolk @@ -0,0 +1,11 @@ +fun takeInvalidTuple(t: [int, (int, builder), int]) { +} + +fun main() { + takeInvalidTuple([1, (2, beginCell()), 0]); +} + +/** +@compilation_should_fail +@stderr a tuple can not have `(int, builder)` inside, because it occupies 2 stack slots in TVM, not 1 + */ diff --git a/tolk-tester/tests/invalid-call-11.tolk b/tolk-tester/tests/invalid-call-11.tolk new file mode 100644 index 00000000..f631c546 --- /dev/null +++ b/tolk-tester/tests/invalid-call-11.tolk @@ -0,0 +1,11 @@ +fun main() { + var functions = (beginCell, beginCell); + var b = functions.1(); // ok + var c = functions.2(); // error +} + +/** +@compilation_should_fail +@stderr invalid tensor index, expected 0..1 +@stderr functions.2() + */ diff --git a/tolk-tester/tests/invalid-call-2.tolk b/tolk-tester/tests/invalid-call-2.tolk new file mode 100644 index 00000000..5a8c9fa5 --- /dev/null +++ b/tolk-tester/tests/invalid-call-2.tolk @@ -0,0 +1,14 @@ +fun add1(x: int) { + return x + 1; +} + +fun main() { + val adder_fn = add1; + var x = 10; + return adder_fn(mutate x); +} + +/** +@compilation_should_fail +@stderr `mutate` used for non-mutate argument + */ diff --git a/tolk-tester/tests/invalid-call-3.tolk b/tolk-tester/tests/invalid-call-3.tolk new file mode 100644 index 00000000..ac98df70 --- /dev/null +++ b/tolk-tester/tests/invalid-call-3.tolk @@ -0,0 +1,12 @@ +fun with2Params(x: int, y: int) { + +} + +fun main() { + return with2Params(1); +} + +/** +@compilation_should_fail +@stderr too few arguments in call to `with2Params`, expected 2, have 1 + */ diff --git a/tolk-tester/tests/invalid-call-4.tolk b/tolk-tester/tests/invalid-call-4.tolk new file mode 100644 index 00000000..c8f7dceb --- /dev/null +++ b/tolk-tester/tests/invalid-call-4.tolk @@ -0,0 +1,13 @@ +fun methodWith1Param(self: int, param: int) { + +} + +fun main() { + val x = 10; + x.methodWith1Param(2, "asdf"); +} + +/** +@compilation_should_fail +@stderr too many arguments in call to `methodWith1Param`, expected 1, have 2 + */ diff --git a/tolk-tester/tests/invalid-call-5.tolk b/tolk-tester/tests/invalid-call-5.tolk new file mode 100644 index 00000000..32905cd7 --- /dev/null +++ b/tolk-tester/tests/invalid-call-5.tolk @@ -0,0 +1,13 @@ +fun inc(x: int) { + return x + 1; +} + +fun main() { + return inc(_); +} + +/** +@compilation_should_fail +@stderr `_` can't be used as a value; it's a placeholder for a left side of assignment +@stderr inc(_) + */ diff --git a/tolk-tester/tests/invalid-call-6.tolk b/tolk-tester/tests/invalid-call-6.tolk new file mode 100644 index 00000000..cbf59806 --- /dev/null +++ b/tolk-tester/tests/invalid-call-6.tolk @@ -0,0 +1,12 @@ +fun nothing() { +} + +fun main() { + val x = 0; + return x.nothing(); +} + +/** +@compilation_should_fail +@stderr `nothing` has no parameters and can not be called as method + */ diff --git a/tolk-tester/tests/invalid-call-7.tolk b/tolk-tester/tests/invalid-call-7.tolk new file mode 100644 index 00000000..cf8c788c --- /dev/null +++ b/tolk-tester/tests/invalid-call-7.tolk @@ -0,0 +1,14 @@ +fun main() { + beginCell() + .storeAddressNone() + .storeUint(3, 32) + .storeUnexisting() + .storeInt(1, 32) + .endCell(); +} + +/** +@compilation_should_fail +@stderr non-existing method `storeUnexisting` of type `builder` +@stderr .storeUnexisting() + */ diff --git a/tolk-tester/tests/invalid-call-8.tolk b/tolk-tester/tests/invalid-call-8.tolk new file mode 100644 index 00000000..199aa681 --- /dev/null +++ b/tolk-tester/tests/invalid-call-8.tolk @@ -0,0 +1,10 @@ +fun get_incoming_value() { return 3; } + +fun main() { + var incoming_ton: int = get_incoming_value().3(); +} + +/** +@compilation_should_fail +@stderr type `int` is not indexable + */ diff --git a/tolk-tester/tests/invalid-call-9.tolk b/tolk-tester/tests/invalid-call-9.tolk new file mode 100644 index 00000000..87eb61e8 --- /dev/null +++ b/tolk-tester/tests/invalid-call-9.tolk @@ -0,0 +1,10 @@ +fun getOne() { return 1; } + +fun main() { + return getOne(); +} + +/** +@compilation_should_fail +@stderr calling a not generic function with generic T + */ diff --git a/tolk-tester/tests/invalid-catch-1.tolk b/tolk-tester/tests/invalid-catch-1.tolk new file mode 100644 index 00000000..54f6e182 --- /dev/null +++ b/tolk-tester/tests/invalid-catch-1.tolk @@ -0,0 +1,12 @@ +fun main() { + try { + + } catch(if, arg) {} + return 0; +} + +/** +@compilation_should_fail +@stderr expected identifier, got `if` +@stderr catch(if + */ diff --git a/tolk-tester/tests/invalid-catch-2.tolk b/tolk-tester/tests/invalid-catch-2.tolk new file mode 100644 index 00000000..a0276146 --- /dev/null +++ b/tolk-tester/tests/invalid-catch-2.tolk @@ -0,0 +1,9 @@ +fun main() { + try {} + catch(err, arg, more) {} +} + +/** +@compilation_should_fail +@stderr expected `)`, got `,` + */ diff --git a/tolk-tester/tests/invalid-cmt-nested.tolk b/tolk-tester/tests/invalid-cmt-nested.tolk new file mode 100644 index 00000000..807e7be8 --- /dev/null +++ b/tolk-tester/tests/invalid-cmt-nested.tolk @@ -0,0 +1,11 @@ +/* +in tolk we decided to drop nested comments support +/* +not nested + */ +*/ + +/** +@compilation_should_fail +@stderr error: expected fun or get, got `*` +*/ diff --git a/tolk-tester/tests/invalid-cmt-old.tolk b/tolk-tester/tests/invalid-cmt-old.tolk new file mode 100644 index 00000000..58927d3a --- /dev/null +++ b/tolk-tester/tests/invalid-cmt-old.tolk @@ -0,0 +1,8 @@ +fun main(): int { + ;; here is not a comment +} + +/** +@compilation_should_fail +@stderr error: expected `;`, got `is` + */ diff --git a/tolk-tester/tests/invalid-const-1.tolk b/tolk-tester/tests/invalid-const-1.tolk new file mode 100644 index 00000000..10e8303a --- /dev/null +++ b/tolk-tester/tests/invalid-const-1.tolk @@ -0,0 +1,8 @@ +fun main() { + return 9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999; +} + +/** +@compilation_should_fail +@stderr invalid integer constant + */ diff --git a/tolk-tester/tests/invalid-cyclic-1.tolk b/tolk-tester/tests/invalid-cyclic-1.tolk new file mode 100644 index 00000000..c46b1640 --- /dev/null +++ b/tolk-tester/tests/invalid-cyclic-1.tolk @@ -0,0 +1,8 @@ +const ONE = TWO - 1; +const TWO = ONE + 1; + +/** +@compilation_should_fail +@stderr const ONE +@stderr undefined symbol `TWO` + */ diff --git a/tolk-tester/tests/invalid-declaration-1.tolk b/tolk-tester/tests/invalid-declaration-1.tolk new file mode 100644 index 00000000..ea27e723 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-1.tolk @@ -0,0 +1,6 @@ +const a = 10, b = 20; + +/** +@compilation_should_fail +@stderr multiple declarations are not allowed + */ diff --git a/tolk-tester/tests/invalid-declaration-10.tolk b/tolk-tester/tests/invalid-declaration-10.tolk new file mode 100644 index 00000000..7ccb182d --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-10.tolk @@ -0,0 +1,8 @@ +get fun onInternalMessage() { + return 0; +} + +/** +@compilation_should_fail +@stderr invalid declaration of a reserved function + */ diff --git a/tolk-tester/tests/invalid-declaration-11.tolk b/tolk-tester/tests/invalid-declaration-11.tolk new file mode 100644 index 00000000..75ebb450 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-11.tolk @@ -0,0 +1,13 @@ +// this function is declared incorrectly, +// since it should return 2 values onto a stack (1 for returned slice, 1 for mutated int) +// but contains not 2 numbers in asm ret_order +fun loadAddress2(mutate self: int): slice + asm( -> 1 0 2) "LDMSGADDR"; + +fun main(){} + +/** +@compilation_should_fail +@stderr ret_order (after ->) expected to contain 2 numbers +@stderr asm( -> 1 0 2) + */ diff --git a/tolk-tester/tests/invalid-declaration-12.tolk b/tolk-tester/tests/invalid-declaration-12.tolk new file mode 100644 index 00000000..25ae9de6 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-12.tolk @@ -0,0 +1,16 @@ +fun proxy(x: int) { + return factorial(x); +} + +fun factorial(x: int) { + if (x <= 0) { + return 1; + } + return x * proxy(x-1); +} + +/** +@compilation_should_fail +@stderr could not infer return type of `factorial`, because it appears in a recursive call chain +@stderr fun factorial + */ diff --git a/tolk-tester/tests/invalid-declaration-13.tolk b/tolk-tester/tests/invalid-declaration-13.tolk new file mode 100644 index 00000000..758a4f21 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-13.tolk @@ -0,0 +1,7 @@ +const c: slice = 123 + 456; + +/** +@compilation_should_fail +@stderr expression type does not match declared type +@stderr const c + */ diff --git a/tolk-tester/tests/invalid-declaration-2.tolk b/tolk-tester/tests/invalid-declaration-2.tolk new file mode 100644 index 00000000..07ffb683 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-2.tolk @@ -0,0 +1,8 @@ +fun main(int): int { + +} + +/** +@compilation_should_fail +@stderr expected `: `, got `)` +*/ diff --git a/tolk-tester/tests/invalid-declaration-3.tolk b/tolk-tester/tests/invalid-declaration-3.tolk new file mode 100644 index 00000000..3edc09fd --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-3.tolk @@ -0,0 +1,8 @@ +int main() { + +} + +/** +@compilation_should_fail +@stderr expected fun or get, got `int` +*/ diff --git a/tolk-tester/tests/invalid-declaration-4.tolk b/tolk-tester/tests/invalid-declaration-4.tolk new file mode 100644 index 00000000..62fd6c56 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-4.tolk @@ -0,0 +1,8 @@ +fun main() { + int x = 0; +} + +/** +@compilation_should_fail +@stderr expected `;`, got `x` +*/ diff --git a/tolk-tester/tests/invalid-declaration-5.tolk b/tolk-tester/tests/invalid-declaration-5.tolk new file mode 100644 index 00000000..bf23d857 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-5.tolk @@ -0,0 +1,6 @@ +enum MyKind { } + +/** +@compilation_should_fail +@stderr `enum` is not supported yet +*/ diff --git a/tolk-tester/tests/invalid-declaration-6.tolk b/tolk-tester/tests/invalid-declaration-6.tolk new file mode 100644 index 00000000..42cb7b95 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-6.tolk @@ -0,0 +1,8 @@ +get seqno(self: int) { + return 0; +} + +/** +@compilation_should_fail +@stderr get methods can't have `mutate` and `self` params + */ diff --git a/tolk-tester/tests/invalid-declaration-7.tolk b/tolk-tester/tests/invalid-declaration-7.tolk new file mode 100644 index 00000000..8d188ea0 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-7.tolk @@ -0,0 +1,8 @@ +fun main() { + var a = 10, b = 20; +} + +/** +@compilation_should_fail +@stderr multiple declarations are not allowed + */ diff --git a/tolk-tester/tests/invalid-declaration-8.tolk b/tolk-tester/tests/invalid-declaration-8.tolk new file mode 100644 index 00000000..06cb9a98 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-8.tolk @@ -0,0 +1,8 @@ +fun someDemo() { + return 0; +} + +/** +@compilation_should_fail +@stderr the contract has no entrypoint + */ diff --git a/tolk-tester/tests/invalid-declaration-9.tolk b/tolk-tester/tests/invalid-declaration-9.tolk new file mode 100644 index 00000000..8cb71c73 --- /dev/null +++ b/tolk-tester/tests/invalid-declaration-9.tolk @@ -0,0 +1,9 @@ +fun recv_internal() { + return 0; +} + +/** +@compilation_should_fail +@stderr this is a reserved FunC/Fift identifier +@stderr you need `onInternalMessage` + */ diff --git a/tolk-tester/tests/invalid-generics-1.tolk b/tolk-tester/tests/invalid-generics-1.tolk new file mode 100644 index 00000000..0bbdeee6 --- /dev/null +++ b/tolk-tester/tests/invalid-generics-1.tolk @@ -0,0 +1,10 @@ +fun f(v: int, x: X) {} + +fun failCantDeduceWithoutArgument() { + return f(1); +} + +/** +@compilation_should_fail +@stderr too few arguments in call to `f`, expected 2, have 1 + */ diff --git a/tolk-tester/tests/invalid-generics-10.tolk b/tolk-tester/tests/invalid-generics-10.tolk new file mode 100644 index 00000000..c7f72bf4 --- /dev/null +++ b/tolk-tester/tests/invalid-generics-10.tolk @@ -0,0 +1,9 @@ +fun invalidReferencingGenericMethodWithoutGeneric() { + var t = createEmptyTuple(); + var cb = t.tupleLast; +} + +/** +@compilation_should_fail +@stderr can not use a generic function `tupleLast` as non-call + */ diff --git a/tolk-tester/tests/invalid-generics-11.tolk b/tolk-tester/tests/invalid-generics-11.tolk new file mode 100644 index 00000000..a399bc91 --- /dev/null +++ b/tolk-tester/tests/invalid-generics-11.tolk @@ -0,0 +1,11 @@ +global gVar: int; + +fun main() { + var x = gVar; + return x; +} + +/** +@compilation_should_fail +@stderr generic T not expected here + */ diff --git a/tolk-tester/tests/invalid-generics-12.tolk b/tolk-tester/tests/invalid-generics-12.tolk new file mode 100644 index 00000000..62a6f5da --- /dev/null +++ b/tolk-tester/tests/invalid-generics-12.tolk @@ -0,0 +1,15 @@ +fun getTwo(): X { return 2; } + +fun cantDeduceNonArgumentGeneric() { + var t1: [int] = [0]; + t1.0 = getTwo(); // ok + var t2 = createEmptyTuple(); + t2.tuplePush(0); + t2.0 = getTwo(); // error, can't decude X +} + +/** +@compilation_should_fail +@stderr can not deduce X for generic function `getTwo` +@stderr t2.0 = getTwo(); + */ diff --git a/tolk-tester/tests/invalid-generics-13.tolk b/tolk-tester/tests/invalid-generics-13.tolk new file mode 100644 index 00000000..d10e2174 --- /dev/null +++ b/tolk-tester/tests/invalid-generics-13.tolk @@ -0,0 +1,11 @@ +fun calcSum(x: X, y: X) { return x + y; } + +fun cantApplyPlusOnNullable() { + return calcSum(((0 as int?)), null); +} + +/** +@compilation_should_fail +@stderr in function `calcSum` +@stderr can not apply operator `+` to `int?` and `int?` + */ diff --git a/tolk-tester/tests/invalid-generics-14.tolk b/tolk-tester/tests/invalid-generics-14.tolk new file mode 100644 index 00000000..eb3adc92 --- /dev/null +++ b/tolk-tester/tests/invalid-generics-14.tolk @@ -0,0 +1,17 @@ +fun eq(v: X) {} + +fun cantDeduceWhenNotInferred() { + // at type inferring (before type checking) they are unknown + var (x, y) = 2; + + eq(x as int); // ok (since execution doesn't reach type checking) + eq(x); // ok (since execution doesn't reach type checking) + eq(x); +} + +/** +@compilation_should_fail +@stderr in function `cantDeduceWhenNotInferred` +@stderr can not deduce X for generic function `eq` +@stderr eq(x); + */ diff --git a/tolk-tester/tests/invalid-generics-2.tolk b/tolk-tester/tests/invalid-generics-2.tolk new file mode 100644 index 00000000..15594433 --- /dev/null +++ b/tolk-tester/tests/invalid-generics-2.tolk @@ -0,0 +1,10 @@ +fun f(v: int, x: T) {} + +fun failCantDeduceWithPlainNull() { + return f(0, null); +} + +/** +@compilation_should_fail +@stderr can not deduce T for generic function `f` + */ diff --git a/tolk-tester/tests/invalid-generics-3.tolk b/tolk-tester/tests/invalid-generics-3.tolk new file mode 100644 index 00000000..72b7df0e --- /dev/null +++ b/tolk-tester/tests/invalid-generics-3.tolk @@ -0,0 +1,11 @@ +fun f(x: T, y: T) {} + +fun failIncompatibleTypesForT() { + return f(32, ""); +} + +/** +@compilation_should_fail +@stderr T is both int and slice for generic function `f` +@stderr f(32 + */ diff --git a/tolk-tester/tests/invalid-generics-4.tolk b/tolk-tester/tests/invalid-generics-4.tolk new file mode 100644 index 00000000..07472ba3 --- /dev/null +++ b/tolk-tester/tests/invalid-generics-4.tolk @@ -0,0 +1,10 @@ +fun f(x: T): void asm "NOP"; + +fun failInstantiatingAsmFunctionWithNon1Slot() { + f((1, 2)); +} + +/** +@compilation_should_fail +@stderr can not call `f` with T=(int, int), because it occupies 2 stack slots in TVM, not 1 + */ diff --git a/tolk-tester/tests/invalid-generics-5.tolk b/tolk-tester/tests/invalid-generics-5.tolk new file mode 100644 index 00000000..4d4f2967 --- /dev/null +++ b/tolk-tester/tests/invalid-generics-5.tolk @@ -0,0 +1,10 @@ +fun f(x: T): void asm "NOP"; + +fun failUsingGenericFunctionPartially() { + var cb = f; +} + +/** +@compilation_should_fail +@stderr can not use a generic function `f` as non-call + */ diff --git a/tolk-tester/tests/invalid-generics-6.tolk b/tolk-tester/tests/invalid-generics-6.tolk new file mode 100644 index 00000000..73e6403f --- /dev/null +++ b/tolk-tester/tests/invalid-generics-6.tolk @@ -0,0 +1,10 @@ +fun eq(t: X) { return t; } + +fun failUsingGenericFunctionPartially() { + var cb = createEmptyTuple().eq().eq().tuplePush; +} + +/** +@compilation_should_fail +@stderr can not use a generic function `tuplePush` as non-call + */ diff --git a/tolk-tester/tests/invalid-generics-7.tolk b/tolk-tester/tests/invalid-generics-7.tolk new file mode 100644 index 00000000..076b7804 --- /dev/null +++ b/tolk-tester/tests/invalid-generics-7.tolk @@ -0,0 +1,17 @@ +fun failOnInstantiation(a: slice) { + var b: slice = foo(a); +} + +fun bar(value: X) : X { + return 1; +} +fun foo(value: X) : X { + return bar(value); +} + +/** +@compilation_should_fail +@stderr in function `bar` +@stderr can not convert type `int` to return type `slice` +@stderr return 1 + */ diff --git a/tolk-tester/tests/invalid-generics-8.tolk b/tolk-tester/tests/invalid-generics-8.tolk new file mode 100644 index 00000000..d2c24e53 --- /dev/null +++ b/tolk-tester/tests/invalid-generics-8.tolk @@ -0,0 +1,11 @@ +fun withT1T2(a: (T1, T2)) {} + +fun wrongTCountPassed() { + withT1T2((5, "")); +} + +/** +@compilation_should_fail +@stderr wrong count of generic T: expected 2, got 1 +@stderr + */ diff --git a/tolk-tester/tests/invalid-generics-9.tolk b/tolk-tester/tests/invalid-generics-9.tolk new file mode 100644 index 00000000..73fd6f87 --- /dev/null +++ b/tolk-tester/tests/invalid-generics-9.tolk @@ -0,0 +1,8 @@ +fun invalidProvidingGenericTsToNotGeneric() { + beginCell(); +} + +/** +@compilation_should_fail +@stderr calling a not generic function with generic T + */ diff --git a/tolk-tester/tests/invalid-get-method-1.tolk b/tolk-tester/tests/invalid-get-method-1.tolk new file mode 100644 index 00000000..263370d4 --- /dev/null +++ b/tolk-tester/tests/invalid-get-method-1.tolk @@ -0,0 +1,9 @@ +@method_id(123) +get fun hello(x: int, y: int): (int, int) { + return (x, y); +} + +/** +@compilation_should_fail +@stderr @method_id can be specified only for regular functions +*/ diff --git a/tolk-tester/tests/invalid-get-method-2.tolk b/tolk-tester/tests/invalid-get-method-2.tolk new file mode 100644 index 00000000..7c7a1413 --- /dev/null +++ b/tolk-tester/tests/invalid-get-method-2.tolk @@ -0,0 +1,17 @@ +@pure +get fun secret(): int { + return 0; +} +@pure +get fun balanced(): int { + return 1; +} + +fun main(): int { + return secret() + balanced(); +} + +/** +@compilation_should_fail +@stderr GET methods hash collision: `secret` and `balanced` produce the same hash +*/ diff --git a/tolk-tester/tests/invalid-import.tolk b/tolk-tester/tests/invalid-import.tolk new file mode 100644 index 00000000..416764b6 --- /dev/null +++ b/tolk-tester/tests/invalid-import.tolk @@ -0,0 +1,11 @@ +// line1 +/* */ import "unexisting.tolk"; +// line3 + +/** +@compilation_should_fail +On Linux/Mac, `realpath()` returns an error, and the error message is `cannot find file` +On Windows, it fails after, on reading, with a message "cannot open file" +@stderr invalid-import.tolk:2:7: error: Failed to import: cannot +@stderr import "unexisting.tolk"; + */ diff --git a/tolk-tester/tests/invalid-mutate-1.tolk b/tolk-tester/tests/invalid-mutate-1.tolk new file mode 100644 index 00000000..280d1e99 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-1.tolk @@ -0,0 +1,11 @@ +fun f(x: int) {} + +fun cantAssignToVal() { + val x = 10; + f(x += 1); +} + +/** +@compilation_should_fail +@stderr modifying immutable variable `x` + */ diff --git a/tolk-tester/tests/invalid-mutate-10.tolk b/tolk-tester/tests/invalid-mutate-10.tolk new file mode 100644 index 00000000..8cd37c51 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-10.tolk @@ -0,0 +1,16 @@ +fun increment(mutate x: int) { + x = x + 1; +} + +fun cantCallMutatingAsAMember() { + var x = 0; + x.increment(); + return x; +} + +/** +@compilation_should_fail +@stderr function `increment` mutates parameter `x` +@stderr consider calling `increment(mutate x)`, not `x.increment`() +@stderr alternatively, rename parameter to `self` to make it a method + */ diff --git a/tolk-tester/tests/invalid-mutate-11.tolk b/tolk-tester/tests/invalid-mutate-11.tolk new file mode 100644 index 00000000..dfc69851 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-11.tolk @@ -0,0 +1,8 @@ +fun load32(self: slice): int { + return self.loadUint(32); +} + +/** +@compilation_should_fail +@stderr modifying `self`, which is immutable by default + */ diff --git a/tolk-tester/tests/invalid-mutate-12.tolk b/tolk-tester/tests/invalid-mutate-12.tolk new file mode 100644 index 00000000..c8c8c68e --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-12.tolk @@ -0,0 +1,14 @@ +fun increment(mutate x: int) { + +} + +fun main() { + var x = 0; + var inc = increment; + inc(x); +} + +/** +@compilation_should_fail +@stderr saving `increment` into a variable is impossible, since it has `mutate` parameters and thus can only be called directly + */ diff --git a/tolk-tester/tests/invalid-mutate-13.tolk b/tolk-tester/tests/invalid-mutate-13.tolk new file mode 100644 index 00000000..ad861fd8 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-13.tolk @@ -0,0 +1,8 @@ +fun onInternalMessage(mutate in_msg_body: slice) { + +} + +/** +@compilation_should_fail +@stderr invalid declaration of a reserved function + */ diff --git a/tolk-tester/tests/invalid-mutate-14.tolk b/tolk-tester/tests/invalid-mutate-14.tolk new file mode 100644 index 00000000..2ba645d1 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-14.tolk @@ -0,0 +1,8 @@ +fun main(cs: slice) { + return loadInt(cs, 32); +} + +/** +@compilation_should_fail +@stderr `loadInt` is a mutating method; consider calling `cs.loadInt()`, not `loadInt(cs)` + */ diff --git a/tolk-tester/tests/invalid-mutate-15.tolk b/tolk-tester/tests/invalid-mutate-15.tolk new file mode 100644 index 00000000..f6874fb8 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-15.tolk @@ -0,0 +1,12 @@ +fun asdf(mutate cs: slice) {} + +fun main(cs: slice) { + cs.asdf(); +} + +/** +@compilation_should_fail +@stderr function `asdf` mutates parameter `cs` +@stderr consider calling `asdf(mutate cs)`, not `cs.asdf`() +@stderr alternatively, rename parameter to `self` to make it a method + */ diff --git a/tolk-tester/tests/invalid-mutate-16.tolk b/tolk-tester/tests/invalid-mutate-16.tolk new file mode 100644 index 00000000..9da6e253 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-16.tolk @@ -0,0 +1,9 @@ +fun cantCallMutatingFunctionWithAssignmentLValue() { + var t: tuple = createEmptyTuple(); + (t = createEmptyTuple()).tuplePush(1); +} + +/** +@compilation_should_fail +@stderr assignment can not be used as lvalue + */ diff --git a/tolk-tester/tests/invalid-mutate-17.tolk b/tolk-tester/tests/invalid-mutate-17.tolk new file mode 100644 index 00000000..9327f07d --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-17.tolk @@ -0,0 +1,13 @@ +@pure +fun tupleMut(mutate self: tuple): int + asm "TLEN"; + +fun main() { + var t = createEmptyTuple(); + return [[t.tupleMut]]; +} + +/** +@compilation_should_fail +@stderr saving `tupleMut` into a variable is impossible, since it has `mutate` parameters + */ diff --git a/tolk-tester/tests/invalid-mutate-18.tolk b/tolk-tester/tests/invalid-mutate-18.tolk new file mode 100644 index 00000000..bb8cde05 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-18.tolk @@ -0,0 +1,10 @@ +fun getNullableTuple(): tuple? { return createEmptyTuple(); } + +fun cantUseLValueUnwrappedNotNull() { + tuplePush(mutate getNullableTuple()!, 1); +} + +/** +@compilation_should_fail +@stderr function call can not be used as lvalue + */ diff --git a/tolk-tester/tests/invalid-mutate-19.tolk b/tolk-tester/tests/invalid-mutate-19.tolk new file mode 100644 index 00000000..bb8cde05 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-19.tolk @@ -0,0 +1,10 @@ +fun getNullableTuple(): tuple? { return createEmptyTuple(); } + +fun cantUseLValueUnwrappedNotNull() { + tuplePush(mutate getNullableTuple()!, 1); +} + +/** +@compilation_should_fail +@stderr function call can not be used as lvalue + */ diff --git a/tolk-tester/tests/invalid-mutate-2.tolk b/tolk-tester/tests/invalid-mutate-2.tolk new file mode 100644 index 00000000..71afe730 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-2.tolk @@ -0,0 +1,10 @@ +fun cantAssignToVal() { + val x = 10; + var y = 20; + [y, x] = [30, 40]; +} + +/** +@compilation_should_fail +@stderr modifying immutable variable `x` + */ diff --git a/tolk-tester/tests/invalid-mutate-20.tolk b/tolk-tester/tests/invalid-mutate-20.tolk new file mode 100644 index 00000000..f6eb2f9f --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-20.tolk @@ -0,0 +1,13 @@ +fun acceptMutateNullableTensor(mutate self: (int, int)?) { +} + +fun cantModifyTupleIndexWithTypeTransition() { + var t = [1, null]; + t.1.acceptMutateNullableTensor(); +} + +/** +@compilation_should_fail +@stderr can not call method for mutate `(int, int)?` with object of type `null` +@stderr because mutation is not type compatible + */ diff --git a/tolk-tester/tests/invalid-mutate-3.tolk b/tolk-tester/tests/invalid-mutate-3.tolk new file mode 100644 index 00000000..d556c9ed --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-3.tolk @@ -0,0 +1,11 @@ +const op_increase = 0x123; + +fun cantAssignToConst() { + var x = 10; + (x, op_increase) = (20, 30); +} + +/** +@compilation_should_fail +@stderr modifying immutable constant + */ diff --git a/tolk-tester/tests/invalid-mutate-4.tolk b/tolk-tester/tests/invalid-mutate-4.tolk new file mode 100644 index 00000000..5f2c111d --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-4.tolk @@ -0,0 +1,14 @@ + +fun inc(mutate x: int) { + x += 1; +} + +fun cantPassToMutatingFunction() { + val myVal = 10; + inc(mutate myVal); +} + +/** +@compilation_should_fail +@stderr modifying immutable variable `myVal` + */ diff --git a/tolk-tester/tests/invalid-mutate-5.tolk b/tolk-tester/tests/invalid-mutate-5.tolk new file mode 100644 index 00000000..2b282cf0 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-5.tolk @@ -0,0 +1,14 @@ +fun cantCallMutatingMethod(c: cell) { + val s: slice = c.beginParse(); + if (1) { + var s: slice = c.beginParse(); + s.loadRef(); // this is ok, 's' is another variable + } + val i = s.loadUint(32); +} + +/** +@compilation_should_fail +@stderr modifying immutable variable `s` +@stderr s.loadUint + */ diff --git a/tolk-tester/tests/invalid-mutate-6.tolk b/tolk-tester/tests/invalid-mutate-6.tolk new file mode 100644 index 00000000..749d9cab --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-6.tolk @@ -0,0 +1,16 @@ +const op_increase = 0x123; + +fun inc(mutate x: int): int { + x += 10; + return x + 1; +} + +fun cantCallMutatingFunctionWithImmutable() { + return inc(mutate op_increase); +} + +/** +@compilation_should_fail +@stderr modifying immutable constant +@stderr inc(mutate op_increase) + */ diff --git a/tolk-tester/tests/invalid-mutate-7.tolk b/tolk-tester/tests/invalid-mutate-7.tolk new file mode 100644 index 00000000..de3bce45 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-7.tolk @@ -0,0 +1,15 @@ +fun incBoth(mutate x: int, mutate y: int) { + x += 10; + y += 10; +} + +fun cantCallMutatingFunctionWithRvalue() { + var x = 10; + incBoth(mutate x, mutate 30); +} + +/** +@compilation_should_fail +@stderr literal can not be used as lvalue +@stderr incBoth(mutate x, mutate 30) + */ diff --git a/tolk-tester/tests/invalid-mutate-8.tolk b/tolk-tester/tests/invalid-mutate-8.tolk new file mode 100644 index 00000000..9b14e28f --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-8.tolk @@ -0,0 +1,10 @@ +fun cantRedefImmutable() { + val x = 10; + var (y: int, x redef) = (20, 30); + return (y, x); +} + +/** +@compilation_should_fail +@stderr `redef` for immutable variable + */ diff --git a/tolk-tester/tests/invalid-mutate-9.tolk b/tolk-tester/tests/invalid-mutate-9.tolk new file mode 100644 index 00000000..3489a288 --- /dev/null +++ b/tolk-tester/tests/invalid-mutate-9.tolk @@ -0,0 +1,9 @@ +fun increment(self: int) { + self = self + 1; +} + +/** +@compilation_should_fail +@stderr modifying `self`, which is immutable by default +@stderr probably, you want to declare `mutate self` + */ diff --git a/tolk-tester/tests/invalid-never-1.tolk b/tolk-tester/tests/invalid-never-1.tolk new file mode 100644 index 00000000..68c6c804 --- /dev/null +++ b/tolk-tester/tests/invalid-never-1.tolk @@ -0,0 +1,8 @@ +fun invalidNever(): never { + if (random()) { throw 123; } +} + +/** +@compilation_should_fail +@stderr a function returning `never` can not have a reachable endpoint + */ diff --git a/tolk-tester/tests/invalid-no-import-1.tolk b/tolk-tester/tests/invalid-no-import-1.tolk new file mode 100644 index 00000000..89f879a3 --- /dev/null +++ b/tolk-tester/tests/invalid-no-import-1.tolk @@ -0,0 +1,8 @@ +import "imports/some-math.tolk"; +import "imports/invalid-no-import.tolk"; + +/** +@compilation_should_fail +@stderr imports/invalid-no-import.tolk:2:13 +@stderr Using a non-imported symbol `someAdd` + */ diff --git a/tolk-tester/tests/invalid-no-import-2.tolk b/tolk-tester/tests/invalid-no-import-2.tolk new file mode 100644 index 00000000..d78346b9 --- /dev/null +++ b/tolk-tester/tests/invalid-no-import-2.tolk @@ -0,0 +1,9 @@ +import "@stdlib/tvm-dicts" +import "imports/use-dicts-err.tolk" + +/** +@compilation_should_fail +@stderr imports/use-dicts-err.tolk:2:22 +@stderr Using a non-imported symbol `createEmptyDict` +@stderr Forgot to import "@stdlib/tvm-dicts"? + */ diff --git a/tolk-tester/tests/invalid-nopar-1.tolk b/tolk-tester/tests/invalid-nopar-1.tolk new file mode 100644 index 00000000..a9a84865 --- /dev/null +++ b/tolk-tester/tests/invalid-nopar-1.tolk @@ -0,0 +1,12 @@ +fun eq(x: int): int { + return x; +} + +fun main(x: int): int { + return eq x; +} + +/** +@compilation_should_fail +@stderr expected `;`, got `x` + */ diff --git a/tolk-tester/tests/invalid-nopar-2.tolk b/tolk-tester/tests/invalid-nopar-2.tolk new file mode 100644 index 00000000..c7c13650 --- /dev/null +++ b/tolk-tester/tests/invalid-nopar-2.tolk @@ -0,0 +1,12 @@ + +fun main(x: int): int { + if x { + return 10; + } + return 0; +} + +/** +@compilation_should_fail +@stderr expected `(`, got `x` + */ diff --git a/tolk-tester/tests/invalid-nopar-3.tolk b/tolk-tester/tests/invalid-nopar-3.tolk new file mode 100644 index 00000000..8249ca28 --- /dev/null +++ b/tolk-tester/tests/invalid-nopar-3.tolk @@ -0,0 +1,12 @@ + +fun main(x: int): int { + if (x, 1) { + return 10; + } + return 0; +} + +/** +@compilation_should_fail +@stderr expected `)`, got `,` + */ diff --git a/tolk-tester/tests/invalid-nopar-4.tolk b/tolk-tester/tests/invalid-nopar-4.tolk new file mode 100644 index 00000000..033c483e --- /dev/null +++ b/tolk-tester/tests/invalid-nopar-4.tolk @@ -0,0 +1,8 @@ +fun load_u32(cs: slice): (slice, int) { + return cs.load_uint 32; +} + +/** +@compilation_should_fail +@stderr expected `;`, got `32` + */ diff --git a/tolk-tester/tests/invalid-pure-1.tolk b/tolk-tester/tests/invalid-pure-1.tolk new file mode 100644 index 00000000..4f0e9142 --- /dev/null +++ b/tolk-tester/tests/invalid-pure-1.tolk @@ -0,0 +1,20 @@ + +@pure +fun f_pure(): int { + return f_impure(); +} + +fun f_impure(): int { return 0; } + +fun main(): int { + return f_pure(); +} + +/** +@compilation_should_fail +@stderr +""" +an impure operation in a pure function +return f_impure(); +""" +*/ diff --git a/tolk-tester/tests/invalid-pure-2.tolk b/tolk-tester/tests/invalid-pure-2.tolk new file mode 100644 index 00000000..21320683 --- /dev/null +++ b/tolk-tester/tests/invalid-pure-2.tolk @@ -0,0 +1,23 @@ +global g: int; + +@pure +fun f_pure(): builder { + var b: builder = beginCell(); + g = g + 1; + return b; +} + +fun main(): int { + g = 0; + f_pure(); + return g; +} + +/** +@compilation_should_fail +@stderr +""" +an impure operation in a pure function +g = g + 1; +""" +*/ diff --git a/tolk-tester/tests/invalid-pure-3.tolk b/tolk-tester/tests/invalid-pure-3.tolk new file mode 100644 index 00000000..31d4f021 --- /dev/null +++ b/tolk-tester/tests/invalid-pure-3.tolk @@ -0,0 +1,24 @@ +@pure +fun validate_input(input: cell): (int, int) { + var (x, y, z, correct) = calculateCellSize(input, 10); + assert(correct) throw 102; + return (x, y); +} + +@pure +fun someF(): int { + var c: cell = beginCell().endCell(); + validate_input(c); + return 0; +} + +fun main() {} + +/** +@compilation_should_fail +@stderr +""" +an impure operation in a pure function +assert(correct) +""" +*/ diff --git a/tolk-tester/tests/invalid-redefinition-1.tolk b/tolk-tester/tests/invalid-redefinition-1.tolk new file mode 100644 index 00000000..5238a680 --- /dev/null +++ b/tolk-tester/tests/invalid-redefinition-1.tolk @@ -0,0 +1,7 @@ +global mulDivMod: int; + +/** +@compilation_should_fail +@stderr global mulDivMod: int; +@stderr redefinition of built-in symbol + */ diff --git a/tolk-tester/tests/invalid-redefinition-2.tolk b/tolk-tester/tests/invalid-redefinition-2.tolk new file mode 100644 index 00000000..3a300dc2 --- /dev/null +++ b/tolk-tester/tests/invalid-redefinition-2.tolk @@ -0,0 +1,12 @@ +global hello: int; + +fun hello(): int { + +} + +/** +@compilation_should_fail +@stderr fun hello() +@stderr redefinition of symbol, previous was at +@stderr invalid-redefinition-2.tolk:1:1 + */ diff --git a/tolk-tester/tests/invalid-redefinition-3.tolk b/tolk-tester/tests/invalid-redefinition-3.tolk new file mode 100644 index 00000000..04ed9383 --- /dev/null +++ b/tolk-tester/tests/invalid-redefinition-3.tolk @@ -0,0 +1,8 @@ +fun main(): int { + var demo_10: int = demo_10; +} + +/** +@compilation_should_fail +@stderr undefined symbol `demo_10` + */ diff --git a/tolk-tester/tests/invalid-redefinition-4.tolk b/tolk-tester/tests/invalid-redefinition-4.tolk new file mode 100644 index 00000000..993a869b --- /dev/null +++ b/tolk-tester/tests/invalid-redefinition-4.tolk @@ -0,0 +1,9 @@ +fun main(): int { + var (a: int, b: int) = (10, 20); + var (a, b: int) = (10, 20); +} + +/** +@compilation_should_fail +@stderr redeclaration of local variable `a` + */ diff --git a/tolk-tester/tests/invalid-redefinition-5.tolk b/tolk-tester/tests/invalid-redefinition-5.tolk new file mode 100644 index 00000000..4a8f5ea1 --- /dev/null +++ b/tolk-tester/tests/invalid-redefinition-5.tolk @@ -0,0 +1,9 @@ +fun main(x: int): int { + var (a: int, b: int) = (10, 20); + var (a redef, x: int) = (10, 20); +} + +/** +@compilation_should_fail +@stderr redeclaration of local variable `x` + */ diff --git a/tolk-tester/tests/invalid-redefinition-6.tolk b/tolk-tester/tests/invalid-redefinition-6.tolk new file mode 100644 index 00000000..e6b087c6 --- /dev/null +++ b/tolk-tester/tests/invalid-redefinition-6.tolk @@ -0,0 +1,10 @@ +const s1 = "asdf"; + +fun main() { + var s1 redef = "d"; +} + +/** +@compilation_should_fail +@stderr `redef` for unknown variable + */ diff --git a/tolk-tester/tests/invalid-self-1.tolk b/tolk-tester/tests/invalid-self-1.tolk new file mode 100644 index 00000000..40b54f16 --- /dev/null +++ b/tolk-tester/tests/invalid-self-1.tolk @@ -0,0 +1,8 @@ +fun cantReturnFromSelf(mutate self: int): self { + return 2; +} + +/** +@compilation_should_fail +@stderr invalid return from `self` function + */ diff --git a/tolk-tester/tests/invalid-self-2.tolk b/tolk-tester/tests/invalid-self-2.tolk new file mode 100644 index 00000000..c4aa758b --- /dev/null +++ b/tolk-tester/tests/invalid-self-2.tolk @@ -0,0 +1,8 @@ +fun cantUseSelfAsType(mutate x: int) { + var y: self = x; +} + +/** +@compilation_should_fail +@stderr `self` type can be used only as a return type of a function (enforcing it to be chainable) + */ diff --git a/tolk-tester/tests/invalid-self-3.tolk b/tolk-tester/tests/invalid-self-3.tolk new file mode 100644 index 00000000..330ac249 --- /dev/null +++ b/tolk-tester/tests/invalid-self-3.tolk @@ -0,0 +1,10 @@ +fun cantReturnSelf(mutate x: int): int { + x += 1; + return self; +} + +/** +@compilation_should_fail +@stderr using `self` in a non-member function (it does not accept the first `self` parameter) +@stderr return self + */ diff --git a/tolk-tester/tests/invalid-self-4.tolk b/tolk-tester/tests/invalid-self-4.tolk new file mode 100644 index 00000000..0be6b9e4 --- /dev/null +++ b/tolk-tester/tests/invalid-self-4.tolk @@ -0,0 +1,9 @@ +fun cantReturnNothingFromSelf(mutate self: int): self { + self = self + 1; +} + +/** +@compilation_should_fail +@stderr missing return +@stderr } + */ diff --git a/tolk-tester/tests/invalid-self-5.tolk b/tolk-tester/tests/invalid-self-5.tolk new file mode 100644 index 00000000..a007a93c --- /dev/null +++ b/tolk-tester/tests/invalid-self-5.tolk @@ -0,0 +1,15 @@ +fun increment(mutate self: int): self { + self = self + 1; + return self; +} + +fun cantReturnAnotherSelf(mutate self: int): self { + self = self + 1; + var x = 0; + return x.increment(); +} + +/** +@compilation_should_fail +@stderr invalid return from `self` function + */ diff --git a/tolk-tester/tests/invalid-self-6.tolk b/tolk-tester/tests/invalid-self-6.tolk new file mode 100644 index 00000000..588c70ab --- /dev/null +++ b/tolk-tester/tests/invalid-self-6.tolk @@ -0,0 +1,8 @@ +fun increment(x: int, self: int): int { + return x + self; +} + +/** +@compilation_should_fail +@stderr `self` can only be the first parameter + */ diff --git a/tolk-tester/tests/invalid-self-7.tolk b/tolk-tester/tests/invalid-self-7.tolk new file mode 100644 index 00000000..2fa2da49 --- /dev/null +++ b/tolk-tester/tests/invalid-self-7.tolk @@ -0,0 +1,8 @@ +fun increment(x: int): int { + return self + 1; +} + +/** +@compilation_should_fail +@stderr using `self` in a non-member function + */ diff --git a/tolk-tester/tests/invalid-shift-1.tolk b/tolk-tester/tests/invalid-shift-1.tolk new file mode 100644 index 00000000..5127ce05 --- /dev/null +++ b/tolk-tester/tests/invalid-shift-1.tolk @@ -0,0 +1,8 @@ +fun main(flags: int) { + return flags << 1 + 32; +} + +/** +@compilation_should_fail +@stderr << has lower precedence than + +*/ diff --git a/tolk-tester/tests/invalid-symbol-1.tolk b/tolk-tester/tests/invalid-symbol-1.tolk new file mode 100644 index 00000000..08a86f17 --- /dev/null +++ b/tolk-tester/tests/invalid-symbol-1.tolk @@ -0,0 +1,14 @@ +fun main(x: int): int { + if (x > 0) { + var y: int = 10; + } else { + var y: slice = "20"; + } + debugPrint(y); +} + +/** +@compilation_should_fail +@stderr debugPrint(y); +@stderr undefined symbol `y` + */ diff --git a/tolk-tester/tests/invalid-symbol-2.tolk b/tolk-tester/tests/invalid-symbol-2.tolk new file mode 100644 index 00000000..f55e15ce --- /dev/null +++ b/tolk-tester/tests/invalid-symbol-2.tolk @@ -0,0 +1,12 @@ +fun main(x: int): int { + try { + if (x > 10) { throw(44); } + } catch(code) {} + return code; +} + +/** +@compilation_should_fail +@stderr return code; +@stderr undefined symbol `code` + */ diff --git a/tolk-tester/tests/invalid-syntax-1.tolk b/tolk-tester/tests/invalid-syntax-1.tolk new file mode 100644 index 00000000..4ccc8f22 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax-1.tolk @@ -0,0 +1,15 @@ +fun main(x: int): int { + if (x > 0) { + return 1; + } + // 'elseif' doesn't exist anymore, it's treated as 'someFunction(arg)' + elseif(x < 0) { + return -1; + } + return x; +} + +/** +@compilation_should_fail +@stderr expected `;`, got `{` + */ diff --git a/tolk-tester/tests/invalid-syntax-2.tolk b/tolk-tester/tests/invalid-syntax-2.tolk new file mode 100644 index 00000000..1180dbbf --- /dev/null +++ b/tolk-tester/tests/invalid-syntax-2.tolk @@ -0,0 +1,13 @@ +fun main(x: int) { + while (x > 0) { + if (x == 10) { + break; + } + x = x -1; + } +} + +/** +@compilation_should_fail +@stderr break/continue from loops are not supported yet + */ diff --git a/tolk-tester/tests/invalid-syntax-3.tolk b/tolk-tester/tests/invalid-syntax-3.tolk new file mode 100644 index 00000000..259ea795 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax-3.tolk @@ -0,0 +1,8 @@ +fun main(x: int) { + return null(); +} + +/** +@compilation_should_fail +@stderr calling a non-function + */ diff --git a/tolk-tester/tests/invalid-syntax-4.tolk b/tolk-tester/tests/invalid-syntax-4.tolk new file mode 100644 index 00000000..044dd329 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax-4.tolk @@ -0,0 +1,8 @@ +fun main(x: int) { + assert(x > 0); +} + +/** +@compilation_should_fail +@stderr expected `throw excNo` after assert, got `;` + */ diff --git a/tolk-tester/tests/invalid-syntax-5.tolk b/tolk-tester/tests/invalid-syntax-5.tolk new file mode 100644 index 00000000..21774774 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax-5.tolk @@ -0,0 +1,8 @@ +fun main(s: auto) { + var (z, t) = ; + +/** +@compilation_should_fail +@stderr expected , got `;` +@stderr var (z, t) = ; +*/ diff --git a/tolk-tester/tests/invalid-syntax-6.tolk b/tolk-tester/tests/invalid-syntax-6.tolk new file mode 100644 index 00000000..12e02645 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax-6.tolk @@ -0,0 +1,9 @@ +fun main() { + var a = 1; + (a += 1) += 2; +} + +/** +@compilation_should_fail +@stderr assignment can not be used as lvalue +*/ diff --git a/tolk-tester/tests/invalid-syntax-7.tolk b/tolk-tester/tests/invalid-syntax-7.tolk new file mode 100644 index 00000000..9f63ac10 --- /dev/null +++ b/tolk-tester/tests/invalid-syntax-7.tolk @@ -0,0 +1,9 @@ +fun main() { + var x = 1; + x += (var y = 2); +} + +/** +@compilation_should_fail +@stderr expected , got `var` +*/ diff --git a/tolk-tester/tests/invalid-tolk-version.tolk b/tolk-tester/tests/invalid-tolk-version.tolk new file mode 100644 index 00000000..d66de9ff --- /dev/null +++ b/tolk-tester/tests/invalid-tolk-version.tolk @@ -0,0 +1,7 @@ +tolk asdf; + +/** +@compilation_should_fail +@stderr semver expected +@stderr tolk asdf; + */ diff --git a/tolk-tester/tests/invalid-typing-1.tolk b/tolk-tester/tests/invalid-typing-1.tolk new file mode 100644 index 00000000..0089bd62 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-1.tolk @@ -0,0 +1,10 @@ +fun main() { + var tri: [int, scli] = [10, null()]; + return; +} + +/** +@compilation_should_fail +@stderr .tolk:2 +@stderr unknown type name `scli` + */ diff --git a/tolk-tester/tests/invalid-typing-10.tolk b/tolk-tester/tests/invalid-typing-10.tolk new file mode 100644 index 00000000..8c1df4a2 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-10.tolk @@ -0,0 +1,8 @@ +fun failMathOnBoolean(c: cell) { + return (null == c) * 10; +} + +/** +@compilation_should_fail +@stderr can not apply operator `*` to `bool` and `int` + */ diff --git a/tolk-tester/tests/invalid-typing-11.tolk b/tolk-tester/tests/invalid-typing-11.tolk new file mode 100644 index 00000000..f6e89d08 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-11.tolk @@ -0,0 +1,12 @@ +fun failBitwiseNotOnBool() { + var eq = 1 == 0; + if (~eq) { + return 0; + } + return -1; +} + +/** +@compilation_should_fail +@stderr can not apply operator `~` to `bool` + */ diff --git a/tolk-tester/tests/invalid-typing-12.tolk b/tolk-tester/tests/invalid-typing-12.tolk new file mode 100644 index 00000000..3a5b1fe2 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-12.tolk @@ -0,0 +1,10 @@ +fun failAssignNullToTensor() { + var ab = (1, 2); + ab = null; + return ab; +} + +/** +@compilation_should_fail +@stderr can not assign `null` to variable of type `(int, int)` + */ diff --git a/tolk-tester/tests/invalid-typing-13.tolk b/tolk-tester/tests/invalid-typing-13.tolk new file mode 100644 index 00000000..e356d0f3 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-13.tolk @@ -0,0 +1,9 @@ +fun failAssignToInvalidTupleIndex() { + var ab = [1, 2]; + ab.100500 = 5; +} + +/** +@compilation_should_fail +@stderr invalid tuple index, expected 0..1 + */ diff --git a/tolk-tester/tests/invalid-typing-14.tolk b/tolk-tester/tests/invalid-typing-14.tolk new file mode 100644 index 00000000..657ab5f4 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-14.tolk @@ -0,0 +1,14 @@ + +fun autoGetIntOrNull() { + if (random()) { return 1; } + return null; +} + +fun testAutoInferredIntOrNull() { + var b: builder = autoGetIntOrNull() as builder; +} + +/** +@compilation_should_fail +@stderr type `int?` can not be cast to `builder` + */ diff --git a/tolk-tester/tests/invalid-typing-15.tolk b/tolk-tester/tests/invalid-typing-15.tolk new file mode 100644 index 00000000..fbcff8a2 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-15.tolk @@ -0,0 +1,13 @@ + +fun getNullable4(): int? { + return 4; +} + +fun testCantSumNullable() { + return 1 + getNullable4(); +} + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `int` and `int?` + */ diff --git a/tolk-tester/tests/invalid-typing-16.tolk b/tolk-tester/tests/invalid-typing-16.tolk new file mode 100644 index 00000000..1dca7822 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-16.tolk @@ -0,0 +1,13 @@ +@pure +fun myDictDeleteStrict(mutate self: cell, keyLen: int, key: int): bool + asm(key self keyLen) "DICTIDEL"; + + +fun testCantCallDictMethodsOnNullable(c: cell) { + c.beginParse().loadDict().myDictDeleteStrict(16, 1); +} + +/** +@compilation_should_fail +@stderr can not call method for `cell` with object of type `cell?` + */ diff --git a/tolk-tester/tests/invalid-typing-17.tolk b/tolk-tester/tests/invalid-typing-17.tolk new file mode 100644 index 00000000..b7302684 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-17.tolk @@ -0,0 +1,10 @@ + +fun testCantUseNullableAsCondition(x: int?) { + if (x) { return 1; } + return 0; +} + +/** +@compilation_should_fail +@stderr can not use `int?` as a boolean condition + */ diff --git a/tolk-tester/tests/invalid-typing-18.tolk b/tolk-tester/tests/invalid-typing-18.tolk new file mode 100644 index 00000000..cf985add --- /dev/null +++ b/tolk-tester/tests/invalid-typing-18.tolk @@ -0,0 +1,16 @@ +fun incrementOrSetNull(mutate x: int?) { + if (random()) { x! += 1; } + else { x = null; } +} + +fun cantCallMutateMethodNotNullable() { + var x = 1; + incrementOrSetNull(mutate x); + return x; +} + +/** +@compilation_should_fail +@stderr can not pass `int` to mutate `int?` +@stderr because mutation is not type compatible + */ diff --git a/tolk-tester/tests/invalid-typing-19.tolk b/tolk-tester/tests/invalid-typing-19.tolk new file mode 100644 index 00000000..58b6c1fc --- /dev/null +++ b/tolk-tester/tests/invalid-typing-19.tolk @@ -0,0 +1,12 @@ +fun getNullableInt(): int? { return 5; } + +fun testCantApplyNotNullForAlwaysNull() { + var x: int? = getNullableInt(); + if (x != null) { return 0; } + return x! + 1; +} + +/** +@compilation_should_fail +@stderr operator `!` used for always null expression + */ diff --git a/tolk-tester/tests/invalid-typing-2.tolk b/tolk-tester/tests/invalid-typing-2.tolk new file mode 100644 index 00000000..052596e4 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-2.tolk @@ -0,0 +1,9 @@ +fun main() { + var tri: (int, int) = (10, false); + return; +} + +/** +@compilation_should_fail +@stderr can not assign `(int, bool)` to variable of type `(int, int)` + */ diff --git a/tolk-tester/tests/invalid-typing-20.tolk b/tolk-tester/tests/invalid-typing-20.tolk new file mode 100644 index 00000000..457bc97a --- /dev/null +++ b/tolk-tester/tests/invalid-typing-20.tolk @@ -0,0 +1,15 @@ +fun getNullableInt(): int? { return 5; } + +fun testFlowContextAppliedInBinaryOperator() { + var x: int? = getNullableInt(); + var y: int? = getNullableInt(); + if ((y = null) < y) { + return -100; + } + return 0; +} + +/** +@compilation_should_fail +@stderr can not apply operator `<` to `null` and `null` + */ diff --git a/tolk-tester/tests/invalid-typing-21.tolk b/tolk-tester/tests/invalid-typing-21.tolk new file mode 100644 index 00000000..d2a815ee --- /dev/null +++ b/tolk-tester/tests/invalid-typing-21.tolk @@ -0,0 +1,14 @@ +fun getNullableInt(): int? { return 5; } + +fun testNeverTypeOccurs() { + var x: int? = getNullableInt(); + if (x == null && x != null) { + return x + 0; + } + return 0; +} + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `never` and `int` + */ diff --git a/tolk-tester/tests/invalid-typing-22.tolk b/tolk-tester/tests/invalid-typing-22.tolk new file mode 100644 index 00000000..f962f364 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-22.tolk @@ -0,0 +1,9 @@ +fun testLogicalAndNotConditionDoesntAffect(x: int?) { + var gt1 = x != null && x > 1; + return x + 0; +} + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `int?` and `int` + */ diff --git a/tolk-tester/tests/invalid-typing-23.tolk b/tolk-tester/tests/invalid-typing-23.tolk new file mode 100644 index 00000000..74feed52 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-23.tolk @@ -0,0 +1,15 @@ +fun getTensor(): (int?, int?) { return (5, null); } + +fun testSmartCastsForFieldsDropAfterAssign() { + var t = getTensor(); + if (t.0 != null && t.1 != null) { + t = getTensor(); + return t.0 + t.1; + } + return -1; +} + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `int?` and `int?` + */ diff --git a/tolk-tester/tests/invalid-typing-24.tolk b/tolk-tester/tests/invalid-typing-24.tolk new file mode 100644 index 00000000..75f61be9 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-24.tolk @@ -0,0 +1,16 @@ +fun getNullableInt(): int? { return 5; } + +fun getTensor(x: int?): (int?, int) { return (x, 0); } + +fun testSmartCastsDropAfterAssign() { + var x: int? = 0; + var y: int? = 0; + (getTensor(x = getNullableInt()).0, getTensor(y = getNullableInt()).0) = (x + y, x - y); + return x+y; +} + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `int?` and `int?` +@stderr x + y, x - y + */ diff --git a/tolk-tester/tests/invalid-typing-25.tolk b/tolk-tester/tests/invalid-typing-25.tolk new file mode 100644 index 00000000..1621bab1 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-25.tolk @@ -0,0 +1,14 @@ +fun takeNullableTensor(mutate ij: (int, int)?) { } + +fun testSmartCastsDropAfterMutate() { + var x: (int, int)? = (1, 2); + return x.0; // ok + takeNullableTensor(mutate x); + return x.1; // error +} + +/** +@compilation_should_fail +@stderr type `(int, int)?` is not indexable +@stderr return x.1 + */ diff --git a/tolk-tester/tests/invalid-typing-26.tolk b/tolk-tester/tests/invalid-typing-26.tolk new file mode 100644 index 00000000..bf5a1165 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-26.tolk @@ -0,0 +1,12 @@ +fun getNullableInt(): int? { return 5; } + +fun testAssertThrowIsConditional() { + var (x, y) = (getNullableInt(), getNullableInt()); + assert(x != null) throw(y = 10); + return x + y; +} + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `int` and `int?` + */ diff --git a/tolk-tester/tests/invalid-typing-27.tolk b/tolk-tester/tests/invalid-typing-27.tolk new file mode 100644 index 00000000..3861403b --- /dev/null +++ b/tolk-tester/tests/invalid-typing-27.tolk @@ -0,0 +1,18 @@ +fun assignNull2(mutate x: T1?, mutate y: T2?) { + if (false) { + x = null; + y = null; + } +} + +fun testSmartCastsDropAfterNullableGeneric() { + var (x: int?, y: int?) = (1, 2); + x * y; // ok + assignNull2(x, y); // treated like assignments to nullable + x << y; // error +} + +/** +@compilation_should_fail +@stderr can not apply operator `<<` to `int?` and `int?` + */ diff --git a/tolk-tester/tests/invalid-typing-28.tolk b/tolk-tester/tests/invalid-typing-28.tolk new file mode 100644 index 00000000..5d60ff22 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-28.tolk @@ -0,0 +1,15 @@ +fun getNullableInt(): int? { return 5; } + +fun testReassignInRedef() { + var t1: int? = getNullableInt(); + if (t1 != null) { + var (t1 redef, t2) = (getNullableInt(), 5); + return t1 + t2; + } + return -1; +} + +/** +@compilation_should_fail +@stderr can not apply operator `+` to `int?` and `int` + */ diff --git a/tolk-tester/tests/invalid-typing-29.tolk b/tolk-tester/tests/invalid-typing-29.tolk new file mode 100644 index 00000000..e8a4e5e2 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-29.tolk @@ -0,0 +1,14 @@ +fun getNullableInt(): int? { return 5; } + +fun testTryBodyDontSmartCast() { + var x = getNullableInt(); + try { + x = 5; + } catch {} + return x * 10; // x is not int here; for now, we have no exception edges, assuming it can be anywhere inside try +} + +/** +@compilation_should_fail +@stderr can not apply operator `*` to `int?` and `int` + */ diff --git a/tolk-tester/tests/invalid-typing-3.tolk b/tolk-tester/tests/invalid-typing-3.tolk new file mode 100644 index 00000000..ac019a42 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-3.tolk @@ -0,0 +1,19 @@ +fun incInt(mutate self: int): self { + self += 1; + return self; +} + +fun appendBuilder(mutate self: builder): self { + self.storeUint(1, 32); + return self; +} + +fun cantMixDifferentThis() { + var x = 0; + return x.incInt().appendBuilder().incInt(); +} + +/** +@compilation_should_fail +@stderr can not call method for `builder` with object of type `int` + */ diff --git a/tolk-tester/tests/invalid-typing-30.tolk b/tolk-tester/tests/invalid-typing-30.tolk new file mode 100644 index 00000000..53dfc5ca --- /dev/null +++ b/tolk-tester/tests/invalid-typing-30.tolk @@ -0,0 +1,15 @@ +fun getNullableInt(): int? { return 5; } + +fun testDoWhileCondition() { + var (x: int?, y: int?) = (10, 20); + do { + x = getNullableInt(); + y = getNullableInt(); + } while(x == null); + return x * y; // x is 100% int, but y is not +} + +/** +@compilation_should_fail +@stderr can not apply operator `*` to `int` and `int?` + */ diff --git a/tolk-tester/tests/invalid-typing-4.tolk b/tolk-tester/tests/invalid-typing-4.tolk new file mode 100644 index 00000000..1ee71290 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-4.tolk @@ -0,0 +1,12 @@ +fun incNotChained(mutate self: int) { + self = self + 1; +} + +fun cantCallNotChainedMethodsInAChain(x: int) { + return x.incNotChained().incNotChained(); +} + +/** +@compilation_should_fail +@stderr can not call method for `int` with object of type `void` + */ diff --git a/tolk-tester/tests/invalid-typing-44.tolk b/tolk-tester/tests/invalid-typing-44.tolk new file mode 100644 index 00000000..2ec5d0e8 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-44.tolk @@ -0,0 +1,9 @@ +fun cantAssignIntToTensor() { + var (x, y) = 2; + x + y; +} + +/** +@compilation_should_fail +@stderr can not assign `int` to a tensor + */ diff --git a/tolk-tester/tests/invalid-typing-45.tolk b/tolk-tester/tests/invalid-typing-45.tolk new file mode 100644 index 00000000..b357b637 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-45.tolk @@ -0,0 +1,9 @@ +fun cantAssignSizesMismatch() { + var [x, y] = [2, 3, 4]; + x + y; +} + +/** +@compilation_should_fail +@stderr can not assign `[int, int, int]`, sizes mismatch + */ diff --git a/tolk-tester/tests/invalid-typing-5.tolk b/tolk-tester/tests/invalid-typing-5.tolk new file mode 100644 index 00000000..9d8cd480 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-5.tolk @@ -0,0 +1,13 @@ +fun incNotChained(mutate self: int) { + self = self + 1; +} + +fun failWhenReturnANotChainedValue(x: int): int { + return x.incNotChained(); +} + +/** +@compilation_should_fail +@stderr x.incNotChained() +@stderr can not convert type `void` to return type `int` + */ diff --git a/tolk-tester/tests/invalid-typing-6.tolk b/tolk-tester/tests/invalid-typing-6.tolk new file mode 100644 index 00000000..f2e99c7d --- /dev/null +++ b/tolk-tester/tests/invalid-typing-6.tolk @@ -0,0 +1,8 @@ +fun failWhenTernaryConditionNotInt(cs: slice) { + return cs ? 1 : 0; +} + +/** +@compilation_should_fail +@stderr can not use `slice` as a boolean condition + */ diff --git a/tolk-tester/tests/invalid-typing-7.tolk b/tolk-tester/tests/invalid-typing-7.tolk new file mode 100644 index 00000000..c192a05b --- /dev/null +++ b/tolk-tester/tests/invalid-typing-7.tolk @@ -0,0 +1,9 @@ +fun failAssignPlainNullToVariable() { + var x = null; +} + +/** +@compilation_should_fail +@stderr can not infer type of `x`, it's always null +@stderr specify its type with `x: ` or use `null as ` + */ diff --git a/tolk-tester/tests/invalid-typing-8.tolk b/tolk-tester/tests/invalid-typing-8.tolk new file mode 100644 index 00000000..d696e132 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-8.tolk @@ -0,0 +1,8 @@ +fun failExplicitCastIncompatible(c: cell) { + return c as slice; +} + +/** +@compilation_should_fail +@stderr type `cell` can not be cast to `slice` + */ diff --git a/tolk-tester/tests/invalid-typing-9.tolk b/tolk-tester/tests/invalid-typing-9.tolk new file mode 100644 index 00000000..a0d5ee04 --- /dev/null +++ b/tolk-tester/tests/invalid-typing-9.tolk @@ -0,0 +1,13 @@ +fun getTupleLastGetter(): tuple -> X { + return tupleLast; +} + +fun failTypeMismatch() { + var t = createEmptyTuple(); + var c: cell = getTupleLastGetter()(t); +} + +/** +@compilation_should_fail +@stderr can not assign `int` to variable of type `cell` + */ diff --git a/tolk-tester/tests/logical-operators.tolk b/tolk-tester/tests/logical-operators.tolk new file mode 100644 index 00000000..700f2a3c --- /dev/null +++ b/tolk-tester/tests/logical-operators.tolk @@ -0,0 +1,346 @@ +import "imports/use-dicts.tolk" + +fun simpleAllConst() { + return (!0, !!0 & !false, !!!0, !1, !!1, !-1, !!-1, (!5 as int == 0) == !0, !0 == true); +} + +fun compileTimeEval1(x: int) { + // todo now compiler doesn't understand that bool can't be equal to number other than 0/-1 + // (but understands that it can't be positive) + // that's why for now, the last condition is evaluated at runtime + return (!x, !x as int > 10, (!x as int) < 10, !!x as int == 5, !x as int == -10); +} + +@method_id(101) +fun withIfNot(x: int, y: int) { + if (!x) { return 10; } + else if (!y) { return 20; } + return x+y; +} + +@method_id(102) +fun withAndOr(x: int, y: int, z: int) { + var return_at_end = -1; + if (!x & !y) { + if (!z & !y) { return 10; } + else if ((z != 0) | !!y) { return_at_end = 20; } + } else if (!!x & !!y & !z) { + if (!z & (x > 10)) { return_at_end = 30; } + if ((x != 11) & !z) { return 40; } + return_at_end = 50; + } else { + return_at_end = !x ? !y as int : (!z as int) | 1; + } + return return_at_end; +} + +@method_id(103) +fun someSum(upto: int) { + var x = 0; + var should_break = false; + while (!x & !should_break) { + if (upto < 10) { x = upto; should_break = true; } + else { upto = upto - 1; } + } + return x; +} + +@method_id(104) +fun testDict(last: int) { + // prepare dict: [3 => 30, 4 => 40, 5 => x] + var dict = prepareDict_3_30_4_40_5_x(!last ? 100 : last); + return (lookupIdxByValue(dict, 30), lookupIdxByValue(dict, last), lookupIdxByValue(dict, 100)); +} + +@method_id(105) +fun testNotNull(x: int?) { + return [x == null, null == x, !(x == null), null == null, (null != null) as int]; +} + +@method_id(106) +fun testAndConstCodegen() { + return ( + [1 && 0, 0 && 1, 0 && 0, 1 && 1], + [4 && 3 && 0, 5 && 0 && 7 && 8, (7 && 0) && -19], + [4 && 3 && -1, 5 && -100 && 7 && 8, (7 && (1 + 2)) && -19], + [true && false, true && true] + ); +} + +@method_id(107) +fun testOrConstCodegen() { + return ( + [1 || 0, 0 || 1, 0 || 0, 1 || 1], + [0 || 0 || 0, 0 || (0 || 0), ((0 || 0) || 0) || 0], + [4 || 3 || -1, 0 || -100 || 0 || 0, (0 || (1 + -1)) || -19], + [true || false, false || false] + ); +} + +global eqCallsCnt: int; + +fun eq(x: int) { return x; } +fun eqCnt(x: int) { eqCallsCnt += 1; return x; } +fun isGt0(x: int) { return x > 0; } + +fun alwaysThrows(): int { throw 444 ; return 444; } + +@method_id(108) +fun testAndSimpleCodegen(a: int, b: int) { + return a && b; +} + +@method_id(109) +fun testOrSimpleCodegen(a: int, b: int) { + return a > 0 || b > 0; +} + +@method_id(110) +fun testLogicalOps1(x: int) { + eqCallsCnt = 0; + return ( + isGt0(x) || !isGt0(x) || alwaysThrows(), + x && eqCnt(x) && eqCnt(x - 1) && eqCnt(x - 2), + (400 == eq(x)) && alwaysThrows(), + (500 == eq(x)) || eqCnt(x) || false, + (500 == eq(x)) || eqCnt(x) || true, + eqCallsCnt + ); +} + +@method_id(111) +fun testLogicalOps2(first: int) { + var s = beginCell().storeInt(1, 32).storeInt(2, 32).storeInt(3, 32).storeInt(4, 32).storeInt(5, 32).endCell().beginParse(); + var sum = 0; + if (first && s.loadUint(32)) { + (2 == s.loadUint(32)) && (sum += s.loadUint(32)); + (3 == s.loadUint(32)) && (sum += s.loadUint(32)); + (5 == s.preloadUint(32)) && (sum += s.loadUint(32)); + } else { + (10 == s.loadUint(32)) || (20 == s.loadUint(32)) || (3 == s.loadUint(32)) || (4 == s.loadUint(32)); + sum += s.loadUint(32); + } + return (s.getRemainingBitsCount(), sum); +} + +@method_id(112) +fun mixLogicalIntsAndBools(first: int, cond: bool) { + return ( + (first && cond) || (!first && cond), + ((first & -1) & cond as int) == ((first && true) && cond) as int, + 7 && cond, + first || cond || !cond || alwaysThrows(), + cond || first || !first || alwaysThrows() + ); +} + +@method_id(113) +fun testConvertIfToIfnot(x: bool) { + assert(!!(x == false), 100); + assert(!x, 100); + if (x == !!false) { + return 1; + } + if (!!(x != !false)) { + return 1; + } + assert(!!x, 100); + return -4; +} + +fun main() { + +} + +/** +@testcase | 101 | 0 0 | 10 +@testcase | 101 | 5 0 | 20 +@testcase | 101 | 5 8 | 13 +@testcase | 102 | 0 0 0 | 10 +@testcase | 102 | 0 0 5 | 20 +@testcase | 102 | 1 2 0 | 40 +@testcase | 102 | 11 2 0 | 50 +@testcase | 102 | 1 0 0 | -1 +@testcase | 102 | 0 1 0 | 0 +@testcase | 102 | 1 0 1 | 1 +@testcase | 103 | 15 | 9 +@testcase | 103 | 6 | 6 +@testcase | 103 | -1 | -1 +@testcase | 104 | 50 | 3 5 -1 +@testcase | 104 | 100 | 3 5 5 +@testcase | 104 | 0 | 3 -1 5 +@testcase | 105 | 0 | [ 0 0 -1 -1 0 ] +@testcase | 105 | null | [ -1 -1 0 -1 0 ] +@testcase | 106 | | [ 0 0 0 -1 ] [ 0 0 0 ] [ -1 -1 -1 ] [ 0 -1 ] +@testcase | 107 | | [ -1 -1 0 -1 ] [ 0 0 0 ] [ -1 -1 -1 ] [ -1 0 ] +@testcase | 108 | 1 2 | -1 +@testcase | 108 | 1 0 | 0 +@testcase | 109 | -5 -4 | 0 +@testcase | 109 | -5 4 | -1 +@testcase | 109 | 1 99 | -1 +@testcase | 110 | 0 | -1 0 0 0 -1 2 +@testcase | 110 | 1 | -1 0 0 -1 -1 4 +@testcase | 110 | 2 | -1 0 0 -1 -1 5 +@testcase | 110 | 500 | -1 -1 0 -1 -1 3 +@testcase | 111 | 0 | 32 4 +@testcase | 111 | -1 | 0 8 +@testcase | 112 | 5 0 | 0 -1 0 -1 -1 +@testcase | 112 | 0 -1 | -1 -1 -1 -1 -1 +@testcase | 113 | 0 | 1 + +@fif_codegen +""" + simpleAllConst PROC:<{ + // + TRUE + 0 PUSHINT + TRUE + FALSE + TRUE + FALSE + TRUE + TRUE + TRUE + }> +""" + +@fif_codegen +""" + compileTimeEval1 PROC:<{ + // x + DUP // x x + 0 EQINT // x '1 + FALSE // x '1 '4 + TRUE // x '1 '4 '7 + FALSE // x '1 '4 '7 '11 + s0 s4 XCHG // '11 '1 '4 '7 x + 0 EQINT // '11 '1 '4 '7 '12 + -10 EQINT // '11 '1 '4 '7 '14 + s3 s4 XCHG + s1 s3 s0 XCHG3 // '1 '4 '7 '11 '14 + }> +""" + +@fif_codegen +""" + withIfNot PROC:<{ + c2 SAVE + SAMEALTSAVE // x y + OVER // x y x + IFNOTJMP:<{ // x y + 2DROP // + 10 PUSHINT // '2=10 + }> // x y + DUP // x y y + IFNOTJMP:<{ // x y + 2DROP // + 20 PUSHINT // '3=20 + RETALT + }> // x y + ADD // '4 + }> +""" + +@fif_codegen +""" + testAndConstCodegen PROC:<{ + // + FALSE + 0 PUSHINT + DUP + TRUE + 4 TUPLE + FALSE + 0 PUSHINT + DUP + TRIPLE + TRUE + TRUE + TRUE + TRIPLE + FALSE + TRUE + PAIR + }> +""" + +@fif_codegen +""" + testOrConstCodegen PROC:<{ + // + -1 PUSHINT + TRUE + FALSE + s2 PUSH + 4 TUPLE + FALSE + FALSE + FALSE + TRIPLE + -1 PUSHINT + DUP + TRUE + TRIPLE + -1 PUSHINT + FALSE + PAIR + }> +""" + +Currently, && operator is implemented via ?: and is not optimal in primitive cases. +For example, `a && b` can be expressed without IFs. +These are moments of future optimizations. For now, it's more than enough. +@fif_codegen +""" + testAndSimpleCodegen PROC:<{ + // a b + SWAP // b a + IF:<{ // b + 0 NEQINT // '2 + }>ELSE<{ // b + DROP // + 0 PUSHINT // '2=0 + }> + }> +""" + +@fif_codegen +""" + testOrSimpleCodegen PROC:<{ + // a b + SWAP // b a + 0 GTINT // b '3 + IF:<{ // b + DROP // + -1 PUSHINT // '4=-1 + }>ELSE<{ // b + 0 GTINT // '7 + 0 NEQINT // '4 + }> + }> +""" + +@fif_codegen +""" + testConvertIfToIfnot PROC:<{ + // x + DUP // x x + 100 THROWIF + DUP // x x + 100 THROWIF + DUP // x x + IFNOTJMP:<{ // x + DROP // + 1 PUSHINT // '5=1 + }> // x + DUP // x x + IFNOTJMP:<{ // x + DROP // + 1 PUSHINT // '6=1 + }> // x + 100 THROWIFNOT + -4 PUSHINT // '9=-4 + }> +""" + + */ diff --git a/tolk-tester/tests/method_id.tolk b/tolk-tester/tests/method_id.tolk new file mode 100644 index 00000000..e7e70d24 --- /dev/null +++ b/tolk-tester/tests/method_id.tolk @@ -0,0 +1,17 @@ +@method_id(1) +fun foo1(): int { return 111; } +@method_id(3) +fun foo2(): int { return 222; } +@method_id(10) +fun foo3(): int { return 333; } +@method_id(11) +fun slice(slice: slice): slice { return slice; } +fun main(): int { return 999; } + +/** + method_id | in | out +@testcase | 1 | | 111 +@testcase | 3 | | 222 +@testcase | 10 | | 333 +@testcase | 0 | | 999 +*/ diff --git a/tolk-tester/tests/mutate-methods.tolk b/tolk-tester/tests/mutate-methods.tolk new file mode 100644 index 00000000..9ebf8b1d --- /dev/null +++ b/tolk-tester/tests/mutate-methods.tolk @@ -0,0 +1,344 @@ +fun incrementInPlace(mutate self: int, byValue: int): void { + self = self + byValue; +} + +fun incrementTwoInPlace(mutate self: int, mutate y: int, byValue: int): int { + self.incrementInPlace(byValue); + y += byValue; + return self + y; +} + +@method_id(101) +fun testIncrement1() { + var x = 50; + var y = 30; + incrementInPlace(mutate x, 10); + incrementInPlace(mutate x, 10); + incrementInPlace(mutate y, 10); + y.incrementInPlace(10); + incrementInPlace(mutate y, 10); + return (x, y); +} + +@method_id(102) +fun testIncrement2() { + var x = 50; + var y = 30; + val sum1 = incrementTwoInPlace(mutate x, mutate y, 10); + val sum2 = x.incrementTwoInPlace(mutate y, 10); + return (x, y, sum1, sum2); +} + + +fun load_next(mutate cs: slice): int { + return loadInt(mutate cs, 32); +} + +fun myLoadInt(mutate self: slice, len: int): int + asm(-> 1 0) "LDIX"; +fun myStoreInt(mutate self: builder, x: int, len: int): self + asm(x self len) "STIX"; + +@inline_ref +fun unpack_utils_info(mutate utils_info_sl: slice): (int, int) { + return ( + utils_info_sl.myLoadInt(32), + utils_info_sl.myLoadInt(32) + ); +} + +@method_id(103) +fun testSlices1() { + var b: builder = beginCell().storeInt(1, 32).myStoreInt(2, 32); + b.myStoreInt(3, 32); + var c: cell = b.myStoreInt(4, 32).storeInt(5, 32).endCell(); + var cs = c.beginParse(); + var first = cs.preloadInt(32); + unpack_utils_info(mutate cs); + return (first, cs.myLoadInt(32), cs.loadInt(32)); +} + +fun load_decimal_symbol(mutate self: slice): int { + // load decimal from bits using utf-8 table + var n: int = self.loadUint(8); + n = n - 48; + assert(n >= 0) throw 400; + assert(n <= 9) throw 400; + return n; +} + +@method_id(104) +fun testSlices2() { + var cs = "123"; + return (cs.load_decimal_symbol(), cs.load_decimal_symbol(), cs.load_decimal_symbol()); +} + +global v1: int; +global v2: int; +global v3: int; + +@method_id(105) +fun testGlobals() { + v1 = 0; + v2 = 0; + v3 = 100; + v3 += incrementTwoInPlace(mutate v1, mutate v2, 5); + return (v1, v2, v3); +} + +fun withNameShadowing(mutate x: int, pivot: int, extra: int) { + x += pivot; + if (pivot < 100) { + var x = 100 + extra; + if (pivot < 50) { + var x = 50 + extra; + return x + extra; + } else { + x += extra; + return x + extra; + } + } else { + x += extra; + return -100 + extra; + } +} + +@method_id(106) +fun testNameShadowing() { + var x = 0; + var sum = 0; + sum += withNameShadowing(mutate x, 100, 10); + sum += withNameShadowing(mutate x, 50, 10); + sum += withNameShadowing(mutate x, 0, 10); + return (x, sum); +} + +fun updateTwoItems(mutate self: (int, int), byValue: int) { + val (first, second) = self; + self = (first + byValue, second + byValue); +} + +global t107_1: int; +global t107_2: int; + +@method_id(107) +fun testMutableTensor() { + var t = (40, 50); + t.updateTwoItems(10); + updateTwoItems(mutate t, 10); + t107_1 = 1; + t107_2 = 2; + (t107_1, t107_2).updateTwoItems(10); + updateTwoItems(mutate (t107_1, t107_2), 10); + return (t, t107_1, t107_2); +} + +@pure +fun myStoreUint(mutate self: builder, x: int, len: int): self + asm(x self len) "STIX"; + +@pure +fun myStoreU32(mutate self: builder, x: int): self { + return self.storeUint(x, 32); +} + +fun getSumOfNumbersInCell(c: cell): int { + var sum = 0; + var s = c.beginParse(); + var n_numbers = s.getRemainingBitsCount() / 32; + repeat (n_numbers) { + sum += s.loadUint(32); + } + return sum; +} + +@method_id(110) +fun testStoreChaining() { + var b = ((beginCell()).storeUint(1, 32)).storeUint(2, 32).storeUint(3, 32); + b.storeUint(4, 32); + b.myStoreUint(5, 32).storeUint(6, 32); + storeUint(mutate b, 7, 32); + b = b.storeUint(8, 32); + b = b.storeUint(9, 32).storeUint(10, 32); + + return getBuilderBitsCount(b); +} + +@method_id(111) +fun testStoreChainingCustom() { + var b = beginCell().myStoreUint(1, 32).myStoreUint(2, 32).myStoreUint(3, 32); + b.myStoreUint(4, 32); + b.myStoreUint(5, 32).myStoreUint(6, 32); + myStoreUint(mutate b, 7, 32); + b = b.myStoreUint(8, 32); + b = b.myStoreUint(9, 32).myStoreUint(10, 32); + val sum1 = getSumOfNumbersInCell(b.endCell()); + + b = beginCell().myStoreU32(1).storeUint(2, 32).myStoreU32(3); + b.myStoreU32(4); + b.myStoreU32(5).myStoreU32(6); + myStoreU32(mutate b, 7); + b = b.myStoreU32(8); + b = b.storeUint(9, 32).myStoreU32(10); + val sum2 = getSumOfNumbersInCell(b.endCell()); + + return (sum1, sum2); +} + +fun myStoreU32_and_mutate_x(mutate self: builder, mutate x: int): void { + return myStoreUint(mutate self, x += 10, 32); +} + +@method_id(112) +fun testStoreAndMutateBoth() { + var x = 3; + var b: builder = beginCell().myStoreUint(1, 32); + b.myStoreU32_and_mutate_x(mutate x); + b.myStoreU32(3).myStoreU32_and_mutate_x(mutate x); + b.myStoreU32_and_mutate_x(mutate x); + + var cs: slice = b.endCell().beginParse(); + var (n1,n2,n3,n4,n5) = (cs.loadUint(32),((cs)).loadUint(32),cs.loadUint(32),cs.loadUint(32),cs.loadUint(32)); + assert(n5 == x) throw 100; + + return [n1,n2,n3,n4,n5]; +} + +global ccc: builder; + +@method_id(113) +fun testStoreChainingForGlobal() { + ccc = beginCell().storeUint(1, 32).myStoreUint(2, 32).myStoreU32(3); + ccc.storeUint(4, 32); + ccc.storeUint(5, 32).myStoreU32(6); + storeUint(mutate ccc, 7, 32); + ccc = ccc.myStoreU32(8); + ccc = ccc.storeUint(9, 32).myStoreUint(10, 32); + + return getBuilderBitsCount(ccc); +} + +fun alwaysThrows(): int { throw 123; return 123; } +fun loadIntFromCell(c: cell, len: int) { return c.beginParse().loadUint(len); } + +@method_id(114) +fun testLoadIntForTemporaryObject() { + val c0 = beginCell().storeUint(0, 32).endCell(); + val c4 = beginCell().storeUint(4, 32).endCell(); + return ( + beginCell().storeUint(1, 32).endCell().beginParse().loadUint(32), + beginCell().storeUint(2, 32).endCell().beginParse().loadUint(32), + c0.beginParse().loadUint(32) ? alwaysThrows() : loadIntFromCell(c4, 32) + ); +} + +@pure +fun myStoreUint_pure(mutate self: builder): void + asm "STIX"; + +fun myStoreUint_impure(mutate self: builder): void + asm "STIX"; + +fun testStoreUintPureUnusedResult() { + var b = beginCell(); + b.myStoreUint_pure(); + var s = b.endCell().beginParse(); + val ii = s.loadUint(32); + return 0; +} + +fun testStoreUintImpureUnusedResult() { + var b = beginCell(); + b.myStoreUint_impure(); + var s = b.endCell().beginParse(); + val ii = s.loadUint(32); + return 0; +} + +global counter: int; + +fun writeNext2(mutate self: builder): self { + return self.storeUint(counter += 1, 32).storeUint(counter += 1, 32); +} + +fun resetCounter(mutate self: builder): self { + counter = 0; + return self; +} + +@method_id(115) +fun testExplicitReturn() { + counter = 0; + return ( + beginCell().writeNext2().writeNext2().resetCounter().writeNext2().endCell().getSumOfNumbersInCell(), + counter + ); +} + + +fun main(){} + +/** +@testcase | 101 | | 70 60 +@testcase | 102 | | 70 50 100 120 +@testcase | 103 | | 1 3 4 +@testcase | 104 | | 1 2 3 +@testcase | 105 | | 5 5 110 +@testcase | 106 | | 160 110 +@testcase | 107 | | 60 70 21 22 +@testcase | 110 | | 320 +@testcase | 111 | | 55 55 +@testcase | 112 | | [ 1 13 3 23 33 ] +@testcase | 113 | | 320 +@testcase | 114 | | 1 2 4 +@testcase | 115 | | 13 2 + +@fif_codegen +""" + incrementInPlace PROC:<{ + // self byValue + ADD // self + }> +""" + +@fif_codegen +""" + testIncrement2 PROC:<{ + ... + incrementTwoInPlace CALLDICT // x y sum1 + -ROT + 10 PUSHINT // sum1 x y '11=10 + incrementTwoInPlace CALLDICT // sum1 x y sum2 + s1 s3 s0 XCHG3 // x y sum1 sum2 + }> +""" + +@fif_codegen +""" + load_next PROC:<{ + // cs + 32 LDI // '4 cs + SWAP // cs '4 + }> +""" + +@fif_codegen +""" + testStoreUintPureUnusedResult PROC:<{ + // + 0 PUSHINT // '11=0 + }> +""" + +@fif_codegen +""" + testStoreUintImpureUnusedResult PROC:<{ + // + NEWC // b + STIX // '2 + DROP // + 0 PUSHINT // '11=0 + }> +""" + + */ diff --git a/tolk-tester/tests/never-type-tests.tolk b/tolk-tester/tests/never-type-tests.tolk new file mode 100644 index 00000000..89447389 --- /dev/null +++ b/tolk-tester/tests/never-type-tests.tolk @@ -0,0 +1,28 @@ +fun takeInt(a: int) {} + +@method_id(101) +fun test1(x: int?) { + if (x == null && x != null) { + var y = x; + __expect_type(y, "never"); + __expect_type(y!, "never"); + // `never` type is assignable to anything, flow won't reach this point + var t: (int, int) = x; + t = y; + takeInt(x); + var cb: (int) -> int = x; + x as int?; + x as (int, int)?; + x as never; + return x; + } + return 123; +} + +fun main() { + __expect_type(test1, "(int?) -> int"); +} + +/** +@testcase | 101 | null | 123 + */ diff --git a/tolk-tester/tests/no-spaces.tolk b/tolk-tester/tests/no-spaces.tolk new file mode 100644 index 00000000..409bb342 --- /dev/null +++ b/tolk-tester/tests/no-spaces.tolk @@ -0,0 +1,115 @@ +const int10:int=10; + +fun just10(): int { return int10; } +fun eq(v: int): int { return`v`; } + +@method_id(101) fun `get_-1` (): int {return-1;} +@method_id(102) fun `get_--1` (): int {return--1;} +@method_id(103) fun `get_---1`(): int {return---1;} +@method_id(104) fun `get_+++1`(): int {return+++1;} +@method_id(105) fun `get_+-+1`(): int {return+-+1;} + +global `some()var`:int; + +@method_id(110) fun `some_math`(): int { + `some()var`=--6; + return 1*-2*-3*-4*just10()*-5+-`some()var`+--`some()var`---`some()var`; +} + +@method_id(111) fun `negative_nums`(a:int):int { + var m$0:int=1; + var m1:int=-(+0x1)*m$0; + return `a`*-1*-(1)*---(1)*+just10()+-`just10`()*m1*-m1+-eq(m1)----0x1; +} + +@method_id(112) fun `bitwise~ops`(flags:int):[bool,bool] { + return[ + (just10()-3==just10()-(4)--1)|((2==2)&(eq(eq(10)) -3==just10()--13)), + ((flags&0xFF)!=0) + ]; +} + +@method_id(113)fun`unary+bitwise-constant`():[int,int,int]{ + return [~-~~+-3, ~+3-~9, -(-~+-20-~ 10+3+~38&39)]; +} + +@method_id(114)fun`unary+bitwize-parametrized`(c3:int, c9:int, c20:int, c10:int, c38:int):[int,int,int]{ + return [~-~~+-c3, ~+c3-~`c9`, -(-~+-c20-~c10+c3+~c38&39)]; +} + +fun add3(a: int, b: int, c: int) { return a+b+c; } + +@method_id(115) fun unary_const_check(): [int,int] { + var fst1: int=-1; + var snd1: int=-1; + var trd1: int=+2; + var (fst2,snd2,trd2)=(-1,-1,+2); + return [add3(fst2,snd2,trd2),add3(fst1,snd1,trd1)]; +} + +fun `load:u32`(mutate self: slice): int { + return self.loadUint(32); +} + +@method_id(116) fun `call_~_via_backticks`():[int,int,int,int] { + var b:builder = beginCell().storeUint(1, 32).storeUint(2, 32).storeUint(3, 32).storeUint(4, 32); + var `cs`:slice = b.endCell().beginParse(); + val one:int=`cs`.`loadUint`(32); + val (two:int,three:int) = (`cs`.`loadUint`(32), cs.`load:u32`()); + val four:int = cs.`load:u32`(); + return [one,two,three,four]; +} + +fun`main`(){} + +/** + method_id | in | out +@testcase | 101 | | -1 +@testcase | 102 | | 1 +@testcase | 103 | | -1 +@testcase | 104 | | 1 +@testcase | 105 | | -1 +@testcase | 110 | | 1194 +@testcase | 111 | -1 | 22 +@testcase | 112 | 0 | [ -1 0 ] +@testcase | 113 | | [ -4 6 -4 ] +@testcase | 114 | 3 9 20 10 38 | [ -4 6 -4 ] +@testcase | 115 | | [ 0 0 ] +@testcase | 116 | | [ 1 2 3 4 ] + +@fif_codegen +""" + get_+-+1 PROC:<{ + // + -1 PUSHINT + }> +""" + +@fif_codegen +""" + unary+bitwise-constant PROC:<{ + // + -4 PUSHINT + 6 PUSHINT + -4 PUSHINT + TRIPLE + }> +""" + +@fif_codegen +""" + unary_const_check PROC:<{ + // + -1 PUSHINT // fst1=-1 + DUP // fst1=-1 snd1=-1 + 2 PUSHINT // fst1=-1 snd1=-1 trd1=2 + s1 s1 s0 PUSH3 // fst1=-1 snd1=-1 trd1=2 fst2=-1 snd2=-1 trd2=2 + add3 CALLDICT // fst1=-1 snd1=-1 trd1=2 '13 + 3 -ROLL // '13 fst1=-1 snd1=-1 trd1=2 + add3 CALLDICT // '13 '14 + PAIR // '12 + }> +""" + + */ + diff --git a/tolk-tester/tests/null-keyword.tolk b/tolk-tester/tests/null-keyword.tolk new file mode 100644 index 00000000..65890a92 --- /dev/null +++ b/tolk-tester/tests/null-keyword.tolk @@ -0,0 +1,136 @@ +import "@stdlib/lisp-lists" + +@method_id(101) +fun test1() { + var numbers: tuple? = createEmptyList(); + numbers = listPrepend(1, numbers); + numbers = listPrepend(2, numbers); + numbers = listPrepend(3, numbers); + numbers = listPrepend(4, numbers); + var (h: int, numbers redef) = listSplit(numbers!); + h += listGetHead(numbers!); + + _ = null; + (_, _) = (null, null); + var t = createEmptyTuple(); + do { + var num: int = numbers.listNext(); + t.tuplePush(num); + } while (numbers != null); + + return (h, numbers == null, t); +} + +@method_id(102) +fun test2(x: int?) { + if (null != x) { + var y: int? = null; + if (y != null) { return 10; } + if (10 < 20) { // always true at runtime (not at compile-time) + return y; + } + } + try { + return x! + 10; // will throw, since not a number + } catch { + return -1; + } + return 100; +} + +fun myIsNull(x: int?): int { + return x == null ? -1 : x!; +} + +@method_id(103) +fun test3(x: int) { + return myIsNull(x > 10 ? null : x); +} + +@method_id(104) +fun test4(): null { + var (_, (_, untyped: null)) = (3, (createEmptyTuple, null)); + if (true) { + return untyped; + } + return untyped; +} + +@method_id(107) +fun test7() { + var b = beginCell().storeMaybeRef(null) as builder?; + var s = b!.endCell().beginParse(); + var c = s.loadMaybeRef(); + return (null == c) as int * 10 + (b != null) as int; +} + +fun test8() { + __expect_type(null, "null"); + __expect_type([[null]], "[[null]]"); + __expect_type(null as tuple?, "tuple?"); + __expect_type(null as [int]?, "[int]?"); + __expect_type(((null)) as (int, int)?, "(int, int)?"); +} + +fun main() { + // the compiler optimizes this at compile-time + var i: int? = null; + if (i == null) { + return 1; + } + return 10; +} + +/** +@testcase | 101 | | 7 -1 [ 3 2 1 ] +@testcase | 102 | 5 | (null) +@testcase | 102 | null | -1 +@testcase | 103 | 5 | 5 +@testcase | 103 | 15 | -1 +@testcase | 104 | | (null) +@testcase | 107 | | -11 +@fif_codegen +""" + test1 PROC:<{ + // + PUSHNULL // numbers + 1 PUSHINT // numbers '2=1 + SWAP // '2=1 numbers + CONS // numbers + 2 PUSHINT // numbers '4=2 + SWAP // '4=2 numbers + CONS // numbers + 3 PUSHINT // numbers '6=3 + SWAP // '6=3 numbers + CONS // numbers + 4 PUSHINT // numbers '8=4 + SWAP // '8=4 numbers + CONS // numbers + UNCONS // h numbers + DUP // h numbers numbers + CAR // h numbers '13 +""" + +@fif_codegen +""" + main PROC:<{ + // + 1 PUSHINT // '3=1 + }> +""" + +@fif_codegen +""" + test7 PROC:<{ + ... + LDOPTREF // b '9 '8 + DROP // b c + ISNULL // b '11 + 10 MULCONST // b '13 + SWAP // '13 b + ISNULL // '13 '14 + NOT // '13 '14 + ADD // '15 + }> +""" +*/ diff --git a/tolk-tester/tests/nullable-tensors.tolk b/tolk-tester/tests/nullable-tensors.tolk new file mode 100644 index 00000000..d0720273 --- /dev/null +++ b/tolk-tester/tests/nullable-tensors.tolk @@ -0,0 +1,492 @@ +fun getNullableInt(): int? { return 5; } + +fun sumOfNullableTensorComponents(t: (int, int)?): int { + if (t == null) { return 0; } + return t!.0 + t!.1; +} + +fun isTensorNull(t: (int, int)?) { + return t == null; +} + +fun incrementNullableTensorComponents(mutate self: (int, int)?): self { + if (self != null) { + self!.0 += 1; + self!.1 += 1; + } + return self; +} + +fun incrementTensorComponents(mutate self: (int, int)): self { + self.0 += 1; + self.1 += 1; + return self; +} + +fun assignFirstComponent(mutate t: (int, int), first: int) { + t!.0 = first; +} + +fun assignFirstComponentNullable(mutate t: (int, int)?, first: int) { + if (t == null) { + t = (first, 0); + } else { + t!.0 = first; + } +} + +fun getNullableTensor(firstComponent: int?): (int, int)? { + return firstComponent == null ? null : (firstComponent!, 2); +} + +fun sumOfTensor(x: (int, int)) { + return x.0 + x.1; +} + +fun assignNullTo(mutate x: T?) { + x = null; +} + +fun getTensor12() { + return (1,2); +} + +@method_id(101) +fun test101(): (int, int)? { + return (1, 2); +} + +@method_id(102) +fun test102(): ((int, int)?, (int, int)?) { + var t = (1, 2); + return (t, null); +} + +@method_id(103) +fun test103(t: (int, int)) { + var t2: (int, int)? = t; + return (sumOfNullableTensorComponents(t), sumOfNullableTensorComponents(t2), sumOfNullableTensorComponents(null), t2); +} + +@method_id(104) +fun test104() { + var t1_1: (int, int)? = (1, 2); + var t1_2: (int, int)? = t1_1; + var t1_3: (int, int)? = t1_1!; + var t2_1: (int, int)? = getNullableTensor(null); + var t2_2 = t2_1; + return (t1_3, t2_2); +} + +@method_id(105) +fun test105() { + return (null as (int, slice, cell)?, (1, 2, 3) as (int, int, int)?); +} + +@method_id(106) +fun test106() { + var t: (int?, int?)? = (((((1, 2))) as (int, int))); + return t; +} + +@method_id(107) +fun test107() { + var ab = (1, 2); + var ab2: (int, int)? = ab; + return (isTensorNull(ab), isTensorNull(ab2), isTensorNull(null), ab.isTensorNull(), ab2.isTensorNull(), null.isTensorNull()); +} + +@method_id(108) +fun test108(x1: (int, int)) { + incrementTensorComponents(mutate x1); + x1.incrementTensorComponents(); + var x2: (int, int)? = x1; + __expect_type(x2, "(int, int)"); + x2.incrementNullableTensorComponents().incrementNullableTensorComponents(); + incrementNullableTensorComponents(mutate x2); + __expect_type(x2, "(int, int)?"); + var x3: (int, int)? = null; + __expect_type(x3, "null"); + x3.incrementNullableTensorComponents().incrementNullableTensorComponents(); + incrementNullableTensorComponents(mutate x3); + return (x1, x2, x3); +} + +fun isTensorNullGen(t: (T1, T2)?) { + return t == null; +} + +@method_id(109) +fun test109() { + var x1 = (1, 2); + var x2: (int, int)? = x1; + var x3: (int, int)? = x1.1 > 10 ? (1, 2) : null; + return ( + isTensorNullGen(x1), isTensorNullGen(x2), isTensorNullGen(null), + isTensorNullGen(x1), isTensorNullGen(x3), + x1.isTensorNullGen(), x2.isTensorNullGen(), x3.isTensorNullGen(), null.isTensorNullGen() + ); +} + +global g110_1: (int, int); +global g110_2: (int, int)?; + +@method_id(110) +fun test110() { + g110_1 = getNullableTensor(1)!; + incrementTensorComponents(mutate g110_1); + g110_1.incrementTensorComponents(); + g110_2 = g110_1; + g110_2.incrementNullableTensorComponents().incrementNullableTensorComponents(); + incrementNullableTensorComponents(mutate g110_2); + var tmp = g110_2; + g110_2 = null; + g110_2.incrementNullableTensorComponents(); + incrementNullableTensorComponents(mutate g110_2); + return (g110_1, g110_2, tmp); +} + +@method_id(111) +fun test111() { + var x = (1, 2); + assignFirstComponent(mutate x, 50); + var x2: (int, int)? = null; + var x3 = x2 as (int, int)?; + assignFirstComponentNullable(mutate x2, 30); + assignFirstComponentNullable(mutate x3, 70); + g110_1 = (1, 2); + g110_2 = null; + assignFirstComponent(mutate g110_1, 90); + assignFirstComponentNullable(mutate g110_2, 100); + return (x.0, x2!.0, x3!.0, g110_1.0, g110_2!.0); +} + +@method_id(112) +fun test112() { + var x: (int, int)? = (10, 20); + incrementTensorComponents(mutate x!); + x!.incrementTensorComponents(); + return x; +} + +@method_id(113) +fun test113() { + var t = [1, null]; // t.1 is always null + return isTensorNull(t.1); +} + +@method_id(114) +fun test114(): ((slice, (cell, [int, slice, tuple]))?, slice?, (int?, bool?)?) { + var t = [[null]]; + return (t.0.0, t.0.0, t.0.0); +} + +@method_id(115) +fun test115() { + var tt = getNullableTensor(null); + assignFirstComponentNullable(mutate tt, 5); + return ( + getNullableTensor(1)!.incrementTensorComponents(), + sumOfNullableTensorComponents(getNullableTensor(1).incrementNullableTensorComponents().incrementNullableTensorComponents()), + getNullableTensor(null).incrementNullableTensorComponents(), + tt, + sumOfNullableTensorComponents(getNullableTensor(null)) + ); +} + +@method_id(116) +fun test116(returnNull: bool) { + var t1: (int, int)? = returnNull ? null : getTensor12(); + var t2 = returnNull ? null as (int, int)? : getTensor12() as (int, int)?; + returnNull ? null : (1, 2); + return (t1, t2); +} + +@method_id(117) +fun test117() { + var (a, b: (int, int)?, c) = (1, null, 3); + return (b, a, c); +} + +fun autoInferNullableTensor(a: int?, b: int) { + if (a != null) { + return (a!, b); + } + return null; +} + +@method_id(118) +fun test118(a: int?) { + return autoInferNullableTensor(a, 10); +} + +@method_id(119) +fun test119() { + var x: (int, int)? = (1, 2); + x = null; + var tt: (int, (int, int)?) = (0, (1, 2)); + tt.1 = null; + var third: (int, (int, int)?, int) = (0, (1, 2), 3); + third.2 = 100; + return (x, tt.1, third.1, third.2); +} + +@method_id(120) +fun test120(setNull: bool) { + var x: (int, int)? = (1, 2); + if (setNull) { + assignNullTo(mutate x); + } + return x; +} + +@method_id(121) +fun test121() { + var t: [int?, [int?, int?]?] = [1, [2, 3]]; + t.1 = [3, 4]; + return t; +} + +@method_id(122) +fun test122(setNull: bool) { + var t: [int?, [int?, int?]?, int?, [int?, int?]?]? = [1, [2, 3], 4, null]; + if (setNull) { + assignNullTo(mutate t!.1); + } else { + var rhs = [3, 4]; + t!!.1 = rhs; + } + return t; +} + +@method_id(123) +fun test123() { + var t: (int?, (int?, int?)?) = (1, (2, 3)); + t.1 = (3, 4); + return t; +} + +@method_id(124) +fun test124(setNull: bool) { + var t: (int?, (int?, int?)?, int?, (int?, int?)?)? = (1, (2, 3), 4, null); + if (setNull) { + assignNullTo(mutate t!.1); + } else { + var rhs = (3, 4); + t!!.1 = rhs; + } + return t; +} + +global g125: int; +fun getT125(): (int, (int, int)?, (int?, int)?) { return (g125 += 1, null, null); } + +@method_id(125) +fun test125() { + g125 = 0; + getT125().1 = null; + getT125().2 = (1, 2); + (getT125()!! as (int, (int, int)?, (int?, int)?)).2 = null; + // test that nothing left on a stack + return g125; +} + +@method_id(126) +fun test126() { + var tt1: (int, null, int) = (1, null, 2); + var (a: int, b: (int, int)?, c: int) = tt1; + return (a, b, c); +} + +@method_id(127) +fun test127(choice: int) { + var tt1: (int, null, int) = (1, null, 2); + var tt2: (int, (int, int), int) = (1, (2, 3), 4); + var tt3: (int, (int, int)?, int) = (1, null, 5); + var abc: (int, (int, int)?, int) = choice == 1 ? tt1 : choice == 2 ? tt2 : tt3; + return abc; +} + +fun get128_1() { return (1, null, 2); } +fun get128_2() { return null; } +fun get128_3() { return (1, (2, 3), 4); } +fun takeT128(abc: (int, (int, int)?, int)?) { return abc; } + +@method_id(128) +fun test128(choice: int) { + if (choice == 1) { + return takeT128(get128_1())!; + } + if (choice == 2) { + return takeT128(get128_2()); + } + return takeT128(get128_3()); +} + +@method_id(129) +fun test129(setNull: bool) { + var t: (int?, int?) = (getNullableInt(), getNullableInt()); + var r1 = (t, t == null, t != null); + t = (setNull ? null : 1, setNull ? null : 2); + var r2 = (t, t == null, t != null); + return (r1, r2); +} + +@method_id(130) +fun test130(setNull: bool) { + var os: (int, (int, int)?) = (1, setNull ? null : (2, 3)); + return os; +} + +fun getEmptyNullableTensor(getNull: bool): ()? { + return getNull ? null : (); +} + +@method_id(131) +fun test131() { + var nonNullEmptyT = getEmptyNullableTensor(false); + var nullEmptyT = getEmptyNullableTensor(true); + var emptyT = nonNullEmptyT!; + __expect_type(emptyT, "()"); + var doubleNulls1 = (null, null) as (()?, ()?); + var doubleNulls2 = ((), ()) as (()?, ()?); + var doubleNulls3 = ((), ()) as (()?, ()?)?; + var stillEmpty = ((), ()); + return (nonNullEmptyT, 777, nullEmptyT, 777, emptyT, 777, nullEmptyT!, 777, doubleNulls1, doubleNulls2, 777, doubleNulls3, 777, stillEmpty); +} + +@method_id(132) +fun test132() { + var doubleNulls: (()?, ()?) = (getEmptyNullableTensor(true), getEmptyNullableTensor(false)); + var result = ((null as ()?) == null, (() as ()?) == null, doubleNulls.0 == null, doubleNulls.1 == null); + var aln1: int? = (doubleNulls.1 = null); + var aln2: null = (doubleNulls.1 = null); + return (result, 777, aln1, aln2, doubleNulls.1 == null, doubleNulls); +} + +@method_id(133) +fun test133() { + var x: (int, int)? = (10, 20); + return sumOfTensor(x) + x.0 + x.1; // smart casted +} + +@method_id(134) +fun test134(): (int, int)? { + var x: (int, int)? = (10, 20); + incrementTensorComponents(mutate x); // smart casted + return x; +} + + +fun getNormalNullableTensorWidth1(vLess100: int?): ([int?], ())? { + if (vLess100 != null && vLess100 >= 100) { + return null; + } + return ([vLess100], ()); // such a nullable tensor can store NULL in the same slot +} + +fun getTrickyNullableTensorWidth1(vLess100: int?): (int?, ())? { + if (vLess100 != null && vLess100 >= 100) { + return null; + } + return (vLess100, ()); // such a nullable tensor requires an extra stack slot for null presence +} + +fun getEvenTrickierNullableWidth1(vLess100: int?): ((), (int?, ()), ())? { + if (vLess100 != null && vLess100 >= 100) { + return null; + } + return ((), (vLess100, ()), ()); +} + +@method_id(135) +fun test135() { + var n1 = getNormalNullableTensorWidth1(10); // ([10], ()) + var n2 = getNormalNullableTensorWidth1(null); // ([null], ()) + var n3 = getNormalNullableTensorWidth1(100); // null + var t1 = getTrickyNullableTensorWidth1(10); // (10, ()) + var t2 = getTrickyNullableTensorWidth1(null); // (null, ()) + var t3 = getTrickyNullableTensorWidth1(100); // null + var e1 = getEvenTrickierNullableWidth1(10); // ((), (10, ()), ()) + var e2 = getEvenTrickierNullableWidth1(null); // ((), (null, (), ()) + var e3 = getEvenTrickierNullableWidth1(100); // null + return (n1, n2, n3, 777, t1, t2, t3, 777, e1, e2, e3, 777, + n1 == null, n2 == null, n3 == null, t1 == null, t2 == null, t3 == null, e1 == null, e2 == null, e3 == null, 777, + t1!.0 == null, t2!.0 == null, e1!.1.0 == null, e1!.1.1 == null, e2!.1.0 == null, e2!.1.1 == null); +} + + + +fun main(){} + +/** +@testcase | 101 | | 1 2 -1 +@testcase | 102 | | 1 2 -1 (null) (null) 0 +@testcase | 103 | 1 2 | 3 3 0 1 2 +@testcase | 104 | | 1 2 (null) (null) 0 +@testcase | 105 | | (null) (null) (null) 0 1 2 3 -1 +@testcase | 106 | | 1 2 +@testcase | 107 | | 0 0 -1 0 0 -1 +@testcase | 108 | 5 6 | 7 8 10 11 -1 (null) (null) 0 +@testcase | 109 | | 0 0 -1 0 -1 0 0 -1 -1 +@testcase | 110 | | 3 4 (null) (null) 0 6 7 -1 +@testcase | 111 | | 50 30 70 90 100 +@testcase | 112 | | 12 22 +@testcase | 113 | | -1 +@testcase | 114 | | (null) (null) (null) 0 (null) (null) (null) 0 +@testcase | 115 | | 2 3 7 (null) (null) 0 5 0 -1 0 +@testcase | 116 | -1 | (null) (null) 0 (null) (null) 0 +@testcase | 116 | 0 | 1 2 -1 1 2 -1 +@testcase | 117 | | (null) 1 3 +@testcase | 118 | 5 | 5 10 -1 +@testcase | 118 | null | (null) (null) 0 +@testcase | 119 | | (null) (null) 1 2 -1 100 +@testcase | 120 | -1 | (null) (null) 0 +@testcase | 120 | 0 | 1 2 -1 +@testcase | 121 | | [ 1 [ 3 4 ] ] +@testcase | 122 | 0 | [ 1 [ 3 4 ] 4 (null) ] +@testcase | 122 | -1 | [ 1 (null) 4 (null) ] +@testcase | 123 | | 1 3 4 -1 +@testcase | 124 | 0 | 1 3 4 -1 4 (null) (null) 0 +@testcase | 124 | -1 | 1 (null) (null) 0 4 (null) (null) 0 +@testcase | 125 | | 3 +@testcase | 126 | | 1 (null) 2 +@testcase | 127 | 1 | 1 (null) (null) 0 2 +@testcase | 127 | 2 | 1 2 3 -1 4 +@testcase | 127 | 3 | 1 (null) (null) 0 5 +@testcase | 128 | 1 | 1 (null) (null) 0 2 -1 +@testcase | 128 | 2 | (null) (null) (null) (null) (null) 0 +@testcase | 128 | 3 | 1 2 3 -1 4 -1 +@testcase | 129 | 0 | 5 5 0 -1 1 2 0 -1 +@testcase | 129 | -1 | 5 5 0 -1 (null) (null) 0 -1 +@testcase | 130 | 0 | 1 2 3 -1 +@testcase | 130 | -1 | 1 (null) (null) 0 +@testcase | 131 | | -1 777 0 777 777 777 0 0 -1 -1 777 -1 -1 -1 777 +@testcase | 132 | | -1 0 -1 0 777 (null) (null) -1 0 0 +@testcase | 133 | | 60 +@testcase | 134 | | 11 21 -1 +@testcase | 135 | | [ 10 ] [ (null) ] (null) 777 10 -1 (null) -1 (null) 0 777 10 -1 (null) -1 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0 + +@fif_codegen +""" + isTensorNull PROC:<{ + // t.0 t.1 t.NNFlag + 2 1 BLKDROP2 // t.NNFlag + 0 EQINT // '3 + }> +""" + +@fif_codegen +""" + test113 PROC:<{ + // + 1 PUSHINT // '2=1 + PUSHNULL // '2=1 '3 + PAIR // t + 1 INDEX // '5 + PUSHNULL // '5 '6 + 0 PUSHINT // '5 '6 '7=0 + isTensorNull CALLDICT // '8 + }> +""" +*/ diff --git a/tolk-tester/tests/nullable-types.tolk b/tolk-tester/tests/nullable-types.tolk new file mode 100644 index 00000000..24aa7f8a --- /dev/null +++ b/tolk-tester/tests/nullable-types.tolk @@ -0,0 +1,109 @@ + +fun getNullable4(): int? { return 4; } +fun getNullableIntNull(): int? asm "PUSHNULL"; + +fun eqInt(x: int) { return x; } +fun eq(x: T) { return x; } + +fun unwrap(x: T?): T { return x!; } +fun intOr0(x: int?): int { return null == x ? 0 : x!; } + +@method_id(101) +fun test101(x: int) { + var re = x == 0 ? null : 100; + return re == null ? re : 200 + getNullable4()!; +} + +@method_id(102) +fun test102(a: int) { + try { + throw (123, a > 10 ? null : a); + return 0; + } catch (excno, arg) { + var i = arg as int?; + return excno + (i != null ? i!!!!! : -100); + } +} + +@method_id(103) +fun test103(x: int?): (bool, bool, int) { + var x_gt_0 = x != null && eqInt(x!) > 0; + var x_lt_0 = x != null && eq(x)! < 0; + if (x == null) { + return (x_gt_0, x_lt_0, 0); + } + return (x_gt_0, x_lt_0, x!); +} + +@method_id(104) +fun test104(x: int?) { + var x2 = eq(x = 10); + var ab = (x2, getNullableIntNull()); + return (unwrap(ab.0) + (ab.1 == null ? -100 : ab.1!), ab.1); +} + +@method_id(105) +fun test105() { + var xy: (int?, int?) = (5, null); + var ab = [1 ? [xy.0, xy.1] : null]; + ab.0!.0 = intOr0(ab.0!.0); + ab.0!.1 = intOr0(ab.0!.1); + return ab.0!.0! + ab.0!.1!; +} + +global gTup106: tuple?; +global gInt106: int?; + +@method_id(106) +fun test106() { + gInt106 = 0; + gInt106! += 5; + var int106: int? = 0; + var gTup106 = createEmptyTuple(); + gTup106!.tuplePush(createEmptyTuple()); + (gTup106!.0 as tuple?)!.tuplePush(0 as int?); + tuplePush(mutate gTup106!, gInt106); + tuplePush(mutate gTup106!.0, int106! += 1); + return (gTup106 == null, null != gTup106, gTup106, gTup106!.0 as tuple?); +} + +@method_id(107) +fun test107() { + var b: builder? = beginCell(); + b!.storeInt(1, 32).storeInt(2, 32); + b = b!.storeInt(3, 32); + storeInt(mutate b!, 4, 32); + (b! as builder).storeInt(5, 32); + return b!.getBuilderBitsCount(); +} + +@method_id(108) +fun test108() { + var (a, b: cell?, c) = (1, beginCell().endCell(), 3); + if (10>3) { b = null; } + return a + (b == null ? 0 : b!.beginParse().loadInt(32)) + c; +} + +@method_id(109) +fun test109() { + var a = getNullable4(); + var b = getNullable4(); + return ([a, b] = [3, 4], a, b); +} + +fun main(x: int?, y: int?) { +} + +/** +@testcase | 101 | 0 | (null) +@testcase | 101 | -1 | 204 +@testcase | 102 | 5 | 128 +@testcase | 102 | 15 | 23 +@testcase | 103 | 10 | -1 0 10 +@testcase | 104 | 8 | -90 (null) +@testcase | 105 | | 5 +@testcase | 106 | | 0 -1 [ [ 0 1 ] 5 ] [ 0 1 ] +@testcase | 107 | | 160 +@testcase | 108 | | 4 +@testcase | 109 | | [ 3 4 ] 3 4 + */ diff --git a/tolk-tester/tests/op-priority.tolk b/tolk-tester/tests/op-priority.tolk new file mode 100644 index 00000000..e3c6bb6e --- /dev/null +++ b/tolk-tester/tests/op-priority.tolk @@ -0,0 +1,121 @@ +fun justTrue(): bool { return true; } + +fun unary_minus_1(a: int, b: int, c: int): int{return -(a+b) *c;} +fun unary_minus_2(a: int, b: int, c: int): int{return(-(a+b))*c;} +fun unary_minus_3(a: int, b: int, c: int): int{return-((a+b) *c);} + + +@method_id(101) +fun test1(x: int, y: int, z: int): bool { + return (x > 0) & (y > 0) & (z > 0); +} + +@method_id(102) +fun test2(x: int, y: int, z: int): bool { + return x > (0 & (y > 0) as int & (z > 0) as int); +} + +@method_id(103) +fun test3(x: int, y: int, z: int): bool { + if ((x < 0) | (y < 0)) { + return z < 0; + } + return (x > 0) & (y > 0); +} + +@method_id(104) +fun test4(x: int, y: int, mode: int): bool { + if (mode == 1) { + return (x == 10) | (y == 20); + } if (mode == 2) { + return (x == 10) | (y == 20); + } else { + return x == (10 | (y == 20) as int); + } +} + +@method_id(105) +fun test5(status: int): bool { + return justTrue() & (status == 1) & ((justTrue() as int & status) == 1); +} + +@method_id(106) +fun test6(a: int, b: int, c: int): bool { + return (unary_minus_1(a,b,c) == unary_minus_2(a,b,c)) & (unary_minus_1(a,b,c) == unary_minus_3(a,b,c)); +} + +@method_id(107) +fun test7(b: int): int { + var a = b == 3 ? 3 : b == 4 ? 4 : (b == 5) & true ? 5 : 100; + return a; +} + +@method_id(108) +fun test8(b: int): int { + var a = b == 3 ? 3 : b == 4 ? 4 : b = 5 ? 5 : 100; + return a; +} + +fun `_ 0, 3 & (3 > 0) as int, 3 & (`_<_`(3, 0)), + 3 & `_ + unary_minus_2 PROC:<{ + // a b c + -ROT // c a b + ADD // c '3 + NEGATE // c '4 + SWAP // '4 c + MUL // '5 + }> + unary_minus_3 PROC:<{ + // a b c + -ROT // c a b + ADD // c '3 + SWAP // '3 c + MUL // '4 + NEGATE // '5 + }> +""" + + */ diff --git a/tolk-tester/tests/parse-address.tolk b/tolk-tester/tests/parse-address.tolk new file mode 100644 index 00000000..385aa3b5 --- /dev/null +++ b/tolk-tester/tests/parse-address.tolk @@ -0,0 +1,113 @@ +const cc1 = "0:ca6e321c7cce9ecedf0a8ca2492ec8592494aa5fb5ce0387dff96ef6af982a3e"a; +const cc2 = "EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPrHF"a; + +fun verifyAddr(addr: slice, workchain: int, number: int) { + assert (addr.getRemainingBitsCount() == 3 + 8 + 256) throw 112; + addr.skipBits(3); + assert (addr.loadUint(8) == workchain) throw 111; + assert (addr.loadUint(256) == number) throw 111; +} + +fun main() { + verifyAddr("Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"a, 255, 23158417847463239084714197001737581570653996933128112807891516801582625927987); + verifyAddr("EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c"a, 0, 0); + verifyAddr("EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5"a, 0, 65607996509792174074532427555986248720836864382484024657400295821210434460432); + verifyAddr("UQCOgxbCOjOLH_cEuQdGgS23zBM5SrQQepMFedjK-oixYbis"a, 0, 64460038539088394980732229180523693489682583665805557562964506609821558550881); + verifyAddr("EQDa4VOnTYlLvDJ0gZjNYm5PXfSmmtL6Vs6A_CZEtXCNICq_"a, 0, 99002318936150612861744867526221033858534811876886359650897405270877291973920); + verifyAddr("Ef8BtXO9bcTMXjg9bgivKh4lhJmZWQPP6_rb9vfjlTP5FJtM"a, 255, 772910975127952880303441415761050161913031788763061162001556772893733681428); + verifyAddr("Ef89xh-uy860-mCcvS8zcAUs8bApmxLGygDLEKjUk5RL-311"a, 255, 27941138149036269893630478666581900122707382189183906805784676408403709676539); + verifyAddr("Ef_vA6yRfmt2P4UHnxlrQUZFcBnKux8mL2eMqBgpeMFPorr4"a, 255, 108109262375472472702582493362335418330829651067377177643099076957184687427490); + verifyAddr("Ef8o6AM9sUZ8rOqLFY8PYeaC3gbopZR1BMkE8fcD0r5NnmCi"a, 255, 18502444830824300068094395885436326119386947594392869497312068745716154912158); + verifyAddr("Ef_fvrd0hBoVJUxoi7wH173Zk8NPiyVvxh5IoYSjEYZbOhsu"a, 255, 101202732337223525952216789200341489000836292542250083765062769181728788863802); + verifyAddr("Ef9nzj6RBc4mQ6p3ng7mGJ7tp7MbzERhe7obkM9A0wnCCEcf"a, 255, 46952625717497919357580310066854892621799390294920450816077086267929711460872); + verifyAddr("Ef9rU-_AAnBkHB71TIC3QvUf5LcAsvj0B4IoYzAXLpEFd5CA"a, 255, 48545777798729612074233611768739897492467685225150339217043102685589809464695); + verifyAddr("Ef9LynHHKgBxY6-l-W_dWN-CtGT2_ji5rN3EzOI-p9zWEfq6"a, 255, 34281152017620085319078796986198022632548048219136747083019177301186013091345); + verifyAddr("Ef9hMd78gzSiVsK0zz0AHtEja8x1UoB_NDZMjn-l86NQK_2Y"a, 255, 43962460814164090767878334494257755557842170134382045184921495822637115592747); + verifyAddr("Ef80FNJ5NJO4-0QwlVAWckUZXdk-PfYDexDZ1-ju9SxhF0A6"a, 255, 23557057702048801338698514499604413540742716310574705490458593067566768087319); + verifyAddr("Ef_fdIbThooPs4_r2DE_Z6ZsWycJdHLnsuKAJHTcbaZaipez"a, 255, 101071650030310556115830521522496708686577365303530257137459798093298869361290); + verifyAddr("Ef_lva0qEiZhWrrZJl-IJxyCcTQmmTo71fIWyQ31HxJ8NurV"a, 255, 103914771557158282349484109182290824591675204108148026180964788916630125182006); + verifyAddr("Ef8sMGKypw006AeRYqimLjmY2Ufp-SHk8C0ZJBNgVBlzw_Nr"a, 255, 19987255184378161380023126214650814972824352533523055905552702178965809886147); + verifyAddr("EQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff+W72r5gqPrHF"a, 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr("EQCaSCHVak-jIc9ANutTAfHpZNM3YdGky7yaDzsTrg0WhFlm"a, 0, 69783625181781015447914682554083924798054947959007050695795761257887453484676); + verifyAddr("EQBS9U3AfD15fGmOtRMXQAxcPVBwNuItfLcDni9fkbTyyNX0"a, 0, 37523067738561024305547433298623118197038688994386001017161816416175242146504); + verifyAddr("EQBiMNL9qNWMAkJHuM0BFneYcuHL17kzS4pswpaEO-NGWrFG"a, 0, 44412924025649114419413541526870954696667907029239618728289150652715284776538); + verifyAddr("EQAUzE-Nef80O9dLZy91HfPiOb6EEQ8YqyWKyIU-KeaYLNUi"a, 0, 9407242825041766837311851458322335726136775042891143504070507665010681354284); + verifyAddr("EQD-nhrinjv0B4LTgr0dRHTHwH1MOsgGhKBXJZd7vESMZUf1"a, 0, 115166810931401616117484448645661180241548402534908005320733783571353775148133); + verifyAddr("EQAVD3Fni9I6j8XeSIl-wAGBEhqhame6OtAY0GScKT0D9X6f"a, 0, 9525855215156855607080079714361451576383963668563135377495902959388099150837); + verifyAddr("EQC6ACq3VANZjqfRBy7JMHkpLwqQ9qyYJsCIGx1mYbQgxaKw"a, 0, 84130484652351964071210477536969520113177637645401392541565606610268614566085); + verifyAddr("EQCIJLNFIko5CvpKn9oAkrDgLocDOoD4vwmHxNx_fsG_LkwW"a, 0, 61579391178099797614367237687950512448308156724136883899001108680249616482094); + verifyAddr("EQCe4AYIBce1pAk2qJJPSs1OzyZRlKjkfq8zuC8D7erv6DUP"a, 0, 71861245445432818728925844931259040612664802586395398157190478191760507596776); + verifyAddr("EQCtrtTXEAoSpoERmiqOnICe9LHxn2N89N4BH9qdHlrG-U0i"a, 0, 78559023162479717496981724991265882229440558807791659796411897368395464230649); + verifyAddr("EQBBlraAps0OZaB9Q8ePQn2wVAaL1G411A-dNppyWe3X3GIT"a, 0, 29666621803903557832193058147214384979915773445007872807927344851911086823388); + verifyAddr("EQBiASqUqaVizrozLRbszkWC2kETbkhpO2qniDVDPPg2_0W8"a, 0, 44328719889509369519441680467651025944540360433148852643949783408843779749631); + verifyAddr("EQBu2Q1EO8gIoNA1qoGWnHUudKfmqlKEDTQE-DxN-_4sdg14"a, 0, 50137910719490808065414827264266674858051167131188257457782826342827836714102); + verifyAddr("EQA5bvxWd5-q2vJUVqR9AlbEIfdFysLR0PXGgVlBf8x5hWuF"a, 0, 25977927117604457079092522008276392864656238504700352770597256138254994667909); + verifyAddr("EQBguMSHjFv5bfoOdshr3ruS9ymSZzhRKMovoNrxGxZXvmee"a, 0, 43748489720571123896506696370504498290006245978262404519821633796370658121662); + verifyAddr("EQAxL0oF1-zNgimPKthbDnYS4xj94rHtfNRN7_Pd1r2LNNv3"a, 0, 22246882279393590648219842750911786376759362211171398419754426796438233910068); + verifyAddr("EQANX1uRKGZfyPIwEaIXrR0ZOqadct5q10dvKxWIxx7SQqzW"a, 0, 6048549475100840191738010856156544571222758030966479209409932714701987172930); + verifyAddr("EQBitdFDoU5DWSjfKq7AsO29RIwAnBzzvcVVSn5ekQoB9Liv"a, 0, 44647902768175374073183447303109856895983123510911038181495138821771906122228); + verifyAddr("EQBgbux7VSjqJHP7ByRK1q4QuVZbpSCesNgvz5qad3lfXX_B"a, 0, 43618018778298854282398238948198420936771670943015013768514626213000552996701); + verifyAddr("EQDisBd8U7M3CEOZ8gcWCdetdmJi3AI31zIT5qBwOdmUbsxY"a, 0, 102533830955233207294921564956803510155400341370448300698800842506363763004526); + verifyAddr("EQAZpn_eynVlf7Ii2d6jP_p1URPrdF9F3S7DiudQyelkjzwE"a, 0, 11602000355550451044739442929923326898313570892134000961608306166632391730319); + verifyAddr("EQDE0HBgfkOiqHezLtExBGTvOs8eitthHQosBjW3BmDy1y2K"a, 0, 89021598108837008984355105304701054698583123510131754065320641619941010764503); + verifyAddr("EQDyT36zktBN9PVWvZ1joRxhIfEUgCPt4F2isa-enUA_d6CP"a, 0, 109600164736599393471831241268953938618560132398692391517933933264745646800759); + verifyAddr("EQDSMUGwt25IQd3_yHjI03F71G8Kp2GMaMEv2TiWoTKbsyRH"a, 0, 95072727086440754059372943502908629555499501854161516009430039520728770059187); + verifyAddr("EQAgK1EcrvEuL9sCtoj3cNhVNOuf3lo5GIPE2gn1fwZZYB3j"a, 0, 14550545393206146289454646242321274637527057595221202748348667645886114191712); + verifyAddr("EQCDKqL5w_6MD-Z7AOButu-uR-ZJTsgNU1fu464hn9grY81U"a, 0, 59328315557704100696483472039557119625141880163887490602190749720459366378339); + verifyAddr("EQB1aVMyFBhnlYXmQjsma0S63kvxKU7ccZKFNCFTwX7ASPv4"a, 0, 53106696421104300082516512931084483581353095629408473618166869610568148238408); + verifyAddr("EQBbjrXHoxDyh1ZYGBdBoQgLaScxW6pZR1hEhJC8BqF-5Kgq"a, 0, 41412616102566803060532874463898939692666425753852274254609049615175463829220); + verifyAddr("EQC-QeZ13QP0lszxNKt380fCWuaV94vwC/bfuqmrlg1/fJPA"a, 0, 86055876869280374285292827775555707420719385459150221433115419095878595346300); + verifyAddr("EQAiUwpF27vXCngqNhf_TQ5E_06ah0G4zuSrnfU7CLLaht5H"a, 0, 15525356059048115813946213102829493539706126913595626308144289257869196581510); + verifyAddr("EQBqiVjmhe2iVGmgOSDO1FGjSiz_AMtb1w7lLEiP4XIF_SFy"a, 0, 48187833566271418625754761625661652107159264793429628379411792200127405491709); + verifyAddr("EQDmwvaK2d_SbaPdpOM60effPWeKsksgDVwFPEyxuftM396K"a, 0, 104376425077737068747642645125299653296942252727305637929378053253273342397663); + verifyAddr("EQDWtPZZgF7wvIMUHZQojuD3utiuivsW7WslRJ33dgv-5yc8"a, 0, 97114682311034709685427168495629428400170984047839002197324103884924936519399); + verifyAddr("EQAA7z0JI0JKqbN-1uENKz9JrxIO5ZRY-ehMeg9fPncx50Ck"a, 0, 422697701361909095759185681783393186844038628935759044330165207027374567911); + verifyAddr("EQBVUHRoCq6coQYUwOAhGSoAmQ6Mpm7dFlDYon6HMgWV8Ftr"a, 0, 38588743302295548905191533977469452945717219128199196974980570837505276220912); + verifyAddr("EQCTdvDCf0bA5dOPI1-44tB2ZfNcMGiklzvg27TovgDEqM6E"a, 0, 66700138358140658950710678965721715920748906761125730971082529064117803730088); + verifyAddr("EQBDBKE5WGKIlnoi3OOzw7vkKKIX55eWjPvgxJWwek8AyL2J"a, 0, 30313140970524770883308749215942283658935592719811899513010665548955593408712); + verifyAddr("EQAvCSyLCo21GrqLAifdov4WkOxuGQCjMRxgF1cXSaNzLHZe"a, 0, 21274912932379789207153885262858116665851037273450532982121514600400844714796); + verifyAddr("EQCsLpDeHB2qpRbmsCb_0xmsYVNx1NgeYrvHGT1TDrHkDgL4"a, 0, 77880084760844670718511036557364450189323549135231036747480176919181282894862); + verifyAddr("EQCTQ8kPwyX92r48gCIL_pLN_RcQT9ghZygnmDTYkOkuW_j5"a, 0, 66609755171046741472463433430016629628609840960137226492652665879987546041947); + verifyAddr("EQCTrFRSHt-tfk7WxK9ZHQmqLcgxXxTK7wGfCEbqgY2W9Mcx"a, 0, 66794468397542182731534559853537484892417154018190804733043974345563210356468); + verifyAddr("EQCv28y49GdaLncoclv0ISdDlMUY_cxDPGNWFCPT8t4GuqUJ"a, 0, 79543100951881731989812212377176715376834409392116644269458867858071577560762); + verifyAddr("EQCVL-k6deDR56Z8pcb0Btg0lGfaivOGfdDCD1vvyRsyL9vS"a, 0, 67479265933941008511790471646564661743401752930295407567346938670637286896175); + verifyAddr("EQD6t2dXDjZxF1DqunKF-8dEWivJdliY_0FYiCXnthuqnDCa"a, 0, 113402258385556889021060606279033166272577193563727959698596277924908309916316); + verifyAddr("EQDE98XNzXiPq7VnbJJ2M4-Ht3tX_OWR0xUTTnDC8NObLmyU"a, 0, 89091094739778473356272490822716056624384395256388481953562551087642791090990); + verifyAddr("EQDfeRDE1TDhwt478CDR0Q7MDwqcTUhfjqyTT59mgoAaF6f7"a, 0, 101079669463449311486034260688909914923832300293253430457119371423825321269783); + verifyAddr("EQDijcEyUKa-QgCbeGlggQk1uBtt2ZRHyW4Y4gB4R6MN6RLW"a, 0, 102473162609487797404330889623966425536887610061087715571345738626121871855081); + verifyAddr("EQDOtFOt41skbjBkZF89oYXpoDECjlxIzD-ShWAOYyzuxqLA"a, 0, 93495056812773926196963707371665512785268729004579280701087533371037976424134); + verifyAddr("EQDuJKSFWU7AYqH6KLFfAbYvMuz346eWmJvG6_2NYE42_B4T"a, 0, 107715199938628393100813870735031557263256555616273999363057194279168054802172); + verifyAddr("EQDwGu4vFv1e3wn8min_iy7OPJXegOYTFQ5bZFZ5a5ZPiBpX"a, 0, 108602665568837301744601989570019709742180613578164394799026726718721456754568); + verifyAddr("EQC4G2ph6AS_mD_-cIv4aIYm1z5jAgCW_TTDEr72ygXOP2X-"a, 0, 83274003234732023403481554420859495155084746906198543572711543697320249249343); + verifyAddr("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"a, 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); + verifyAddr("EQDoIA20MF1qEcSPtROdCu5ukGx9dVjgMeJh1oQ4A4cf_Jif"a, 0, 104993214557977037193613824776415934089204193426692473563548548423424814817276); + verifyAddr("EQDpUkyAa6lZ12P3ZB2PL_rmWwI1I55BU4kxw_rssFL5dswA"a, 0, 105534303174146507629736518862713754948570412188900908177600861330298381728118); + verifyAddr("EQClLO4EnZ_rTyV1GVpWy53pLgWJRki5c4ZzuM_1O_ClBkO9"a, 0, 74711004027159342540251007601464500186374346239921204216319145006974068892934); + verifyAddr("EQDmkj65Ab_m0aZaW8IpKw4kYqIgITw_HRstYEkVQ6NIYCyW"a, 0, 104290347741656803921830951060768893809692975574470790497562993373950614128736); + verifyAddr("EQCqNTwAYUNhPFS0RgqZoTLGJcQQxbAJ7csUo4YO3_TONLab"a, 0, 76987241268612358571638783428744566580605181728938801022059780105627411729972); + verifyAddr("EQCL3DmCynaRK7-vsfeNmd4Jj-UxAIHPvA4qS2xwaL6UpLbF"a, 0, 63260589232981964910240894899061676480139492286430589202252472895352724165796); + verifyAddr("EQDbU1SVEjBE73oUqgAoM9gDcShUkM5EC2PgoCjuwVUKo-Ee"a, 0, 99203745911752606845646497420891218522647962685916739950275357890977532807843); + verifyAddr("EQD02VdcF4TDbCKLLhZJ39NQTu6aWq2LnLjp0oXqbNu_BANK"a, 0, 110748343802097970709980079967961144373090790244250392237586606542170934198020); + verifyAddr("EQBynBO23ywHy_CgarY9NK9FTz0yDsG82PtcbSTQgGoXwiuA"a, 0, 51839428943991432793039248316067731096592274748149794482308513726460953499586); + verifyAddr("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"a, 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr("EQAUTbQiM522Y_XJ_T98QPhPhTmb4nV--VSPiha8kC6kRfPO"a, 0, 9183547432069678364603018431103042146626948674383548774683927217595824907333); + verifyAddr("EQBlqsm144Dq6SjbPI4jjZvA1hqTIP3CvHovbIfW_t-SCALE"a, 0, 45985353862647206060987594732861817093328871106941773337270673759241903247880); + verifyAddr("UQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPuwA"a, 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr("kQDKbjIcfM6ezt8KjKJJLshZJJSqX7XOA4ff-W72r5gqPgpP"a, 0, 91561894446285001782438967260723928368560331318344957259023550817453781559870); + verifyAddr("kf-Dfdg-YQXaR2Q97gZJ4fGBtmV1DHOU1y1RPyyZZtRy_Ikh"a, 255, 59475331506450494976393625198911249698879029820580340449086829444312920781564); + verifyAddr("0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 0, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr("0:0000000000000000000000000000000000000000000000000000000000000000"a, 0, 0); + verifyAddr("0:FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFfffffffffffffffffffffffffffff"a, 0, 115792089237316195423570985008687907853269984665640564039457584007913129639935); + verifyAddr("0:zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"a, 0, 23158417847463239084714197001737581570653996933128112807891516801582625927987); + verifyAddr("0:0000000000000000000000000000000000000000000000000000000000000000"a, 0, 0); + verifyAddr("1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 1, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr("9:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 9, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr("99:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 99, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + verifyAddr("-1:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8"a, 255, 37304138005561100291416421295333982606153966175434134130332440738068913455320); + + return cc1.isSliceBitsEqual(cc2); +} + +/** +@testcase | 0 | | -1 + */ diff --git a/tolk-tester/tests/pure-functions.tolk b/tolk-tester/tests/pure-functions.tolk new file mode 100644 index 00000000..bb1a7d59 --- /dev/null +++ b/tolk-tester/tests/pure-functions.tolk @@ -0,0 +1,46 @@ + +@pure +fun f_pure1(): int { + return f_pure2(); +} + +@pure +fun f_pure2(): int { + return 2; +} + +@pure +fun get_contract_data(): (int, int) { + var c: cell = getContractData(); + var cs: slice = c.beginParse(); + cs.loadBits(32); + var value: int = cs.loadUint(16); + return (1, value); +} + +fun save_contract_data(value: int) { + var b: builder = beginCell().storeInt(1, 32).storeUint(value, 16); + setContractData(b.endCell()); +} + +@pure +@method_id(101) +fun test1(): int { + return f_pure1(); +} + +@method_id(102) +fun test2(value: int): int { + save_contract_data(value); + var (_, restored) = get_contract_data(); + return restored; +} + +fun main() { return; } + +/** + +@testcase | 101 | | 2 +@testcase | 102 | 44 | 44 + +*/ diff --git a/tolk-tester/tests/remove-unused-functions.tolk b/tolk-tester/tests/remove-unused-functions.tolk new file mode 100644 index 00000000..8e748ecf --- /dev/null +++ b/tolk-tester/tests/remove-unused-functions.tolk @@ -0,0 +1,48 @@ +fun unused1(): int { return 2; } +fun unused2(): int { return unused1(); } +fun unused3(x: int): int { return x * 2+unused2(); } + +fun used_from_noncall1(): int { return 10; } +fun used_as_noncall1(): int { return used_from_noncall1(); } + +const int20: int = 20; +fun used_from_noncall2(): int { return int20; } +fun used_as_noncall2(): int { return 0 * 0 + used_from_noncall2() + (0 << 0); } + +global unused_gv: int; +global used_gv: int; + +fun receiveGetter(): () -> int { return used_as_noncall2; } + +@pure +fun usedButOptimizedOut(x: int): int { return x + 2; } + +fun main(): (int, int, int) { + used_gv = 1; + used_gv = used_gv + 2; + var getter1 = used_as_noncall1; + var getter2 = receiveGetter(); + usedButOptimizedOut(used_gv); + return (used_gv, getter1(), getter2()); +} + +/** +@experimental_options remove-unused-functions + +@testcase | 0 | | 3 10 20 + +@fif_codegen DECLPROC used_as_noncall1 +@fif_codegen DECLGLOBVAR used_gv + +@fif_codegen_avoid DECLPROC unused1 +@fif_codegen_avoid DECLPROC unused2 +@fif_codegen_avoid DECLPROC unused3 +@fif_codegen_avoid DECLGLOBVAR unused_gv + +Note, that `usedButOptimizedOut()` (a pure function which result is unused) +is currently codegenerated, since it's formally reachable. +This is because optimizing code is a moment of codegen for now (later than marking unused symbols). + +@fif_codegen DECLPROC usedButOptimizedOut +@fif_codegen_avoid usedButOptimizedOut CALLDICT +*/ diff --git a/tolk-tester/tests/s1.tolk b/tolk-tester/tests/s1.tolk new file mode 100644 index 00000000..c7c4f694 --- /dev/null +++ b/tolk-tester/tests/s1.tolk @@ -0,0 +1,61 @@ +get ascii_slice(): slice { + return"string"; +} + +get raw_slice(): slice { + return "abcdef"s; +} + +get addr_slice(): slice { + return "Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF"a; +} + +get string_hex(): int { + return "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"u; +} + +get fun string_minihash(): int { // 'get' and 'get fun' both possible + return "transfer(slice, int)"h; +} + +get fun string_maxihash(): int { + return "transfer(slice, int)"H; +} + +get fun string_crc32(): int { + return "transfer(slice, int)"c; +} + +@pure +fun newc(): builder +asm "NEWC"; +fun endcs(b: builder): slice +asm "ENDC" "CTOS"; +@pure +fun sdeq(s1: slice, s2: slice): int +asm "SDEQ"; + +fun main() { + var s_ascii: slice = ascii_slice(); + var s_raw: slice = raw_slice(); + var s_addr: slice = addr_slice(); + var i_hex: int = string_hex(); + var i_mini: int = string_minihash(); + var i_maxi: int = string_maxihash(); + var i_crc: int = string_crc32(); + assert(sdeq(s_ascii, newc().storeUint(0x737472696E67, 12 * 4).endcs())) throw 101; + assert(sdeq(s_raw, newc().storeUint(0xABCDEF, 6 * 4).endcs())) throw 102; + assert(sdeq(s_addr, newc().storeUint(4, 3).storeInt(-1, 8) + .storeUint(0x3333333333333333333333333333333333333333333333333333333333333333, 256).endcs()), 103); + assert(i_hex == 0x4142434445464748494A4B4C4D4E4F505152535455565758595A303132333435) throw 104; + assert(i_mini == 0x7a62e8a8) throw 105; + assert(i_maxi == 0x7a62e8a8ebac41bd6de16c65e7be363bc2d2cbc6a0873778dead4795c13db979) throw 106; + assert(i_crc == 2235694568) throw 107; + return 0; +} + +/** +@testcase | 0 | | 0 + +@code_hash 13830542019509784148027107880226447201604257839069192762244575629978154217223 +*/ diff --git a/tolk-tester/tests/self-keyword.tolk b/tolk-tester/tests/self-keyword.tolk new file mode 100644 index 00000000..b0567696 --- /dev/null +++ b/tolk-tester/tests/self-keyword.tolk @@ -0,0 +1,253 @@ +fun incChained(mutate self: int): self { + self = self + 1; + return self; +} + +fun incChained2(mutate self: int): self { + return self.incChained(); +} + +fun incChained3(mutate self: int): self { + incChained(mutate self); + return self; +} + +fun incChained4(mutate self: int): self { + self.incChained(); + return self; +} + +@method_id(101) +fun testIncChainedCodegen(x: int) { + return x.incChained().incChained2().incChained3().incChained4(); +} + +@method_id(102) +fun testIncChained() { + var x: int = 10; + incChained(mutate x); + x.incChained(); + x.incChained2(); + x.incChained2().incChained(); + x = x.incChained(); + x = x.incChained2().incChained().incChained2(); + return x.incChained(); +} + +fun incChainedWithMiddleReturn(mutate self: int, maxValue: int): self { + if (self >= maxValue) { + return self; + } + self += 1; + return self; +} + +@method_id(103) +fun testIncChainedWithMiddleReturn(x: int) { + x.incChainedWithMiddleReturn(10).incChainedWithMiddleReturn(10); + x = x.incChainedWithMiddleReturn(10).incChainedWithMiddleReturn(10); + return x.incChainedWithMiddleReturn(10).incChainedWithMiddleReturn(999); +} + +fun incChainedMutatingBoth(mutate self: int, mutate y: int): self { + self += 1; + y += 1; + return self; +} + +global c104: int; + +@method_id(104) +fun testIncChainedMutatingBoth() { + var (x, y) = (0, 0); + c104 = 0; + x.incChainedMutatingBoth(mutate y).incChainedMutatingBoth(mutate y); + incChainedMutatingBoth(mutate x, mutate y); + x = x.incChainedMutatingBoth(mutate c104).incChainedMutatingBoth(mutate c104).incChainedMutatingBoth(mutate y); + return (x, y, c104); +} + +fun incTensorChained(mutate self: (int, int)): self { + val (f, s) = self; + self = (f + 1, s + 1); + return self; +} + +@method_id(105) +fun testIncTensorChained(f: int, s: int) { + var tens = (f, s); + tens.incTensorChained().incTensorChained(); + return tens.incTensorChained().incTensorChained(); +} + +fun incConditionalChainable(mutate self: int, mutate another: int, ifLessThan: int): self { + another += 1; + return self.incChained() < ifLessThan ? self.incChained().incChained() : self; +} + +@method_id(106) +fun testIncConditionalChainable(x: int) { + var y = 0; + x.incConditionalChainable(mutate y, 5).incConditionalChainable(mutate y, 5); + x = x.incConditionalChainable(mutate y, 5).incConditionalChainable(mutate y, 5); + return (x.incConditionalChainable(mutate y, 5), y); +} + +fun checkNotEq(self: int, throwIfEq: int): void { + if (self == throwIfEq) { + throw 100 + throwIfEq; + } +} + +@method_id(107) +fun testNotMutatingSelf(arg: int) { + try { + arg.checkNotEq(100); + arg.checkNotEq(101); + arg.checkNotEq(102); + return 0; + } catch (code) { + return code; + } +} + +global c108: int; + +fun checkNotEqChainable(self: int, throwIfEq: int): self { + c108 += 1; + if (self != throwIfEq) { + return self; + } + throw 100 + throwIfEq; + return self; +} + +@method_id(108) +fun testNotMutatingChainableSelf(arg: int) { + c108 = 0; + try { + arg.checkNotEqChainable(100).checkNotEqChainable(101).checkNotEqChainable(102); + arg = arg.checkNotEqChainable(100).checkNotEqChainable(101).checkNotEqChainable(102); + return (arg, c108); + } catch (code) { + return (code, c108); + } +} + +global onceFailed109: int; + +fun checkNotEqChainableMutateAnother(self: int, throwIfEq: int, mutate toInc: int): self { + if (onceFailed109) { return self; } + toInc += 1; + try { return self.checkNotEqChainable(throwIfEq); } + catch { onceFailed109 = 1; return self; } +} + +global c109: int; + +@method_id(109) +fun testNotMutatingChainableSelfMutateAnother(initial: int) { + val arg = initial; + var x = 0; + c108 = 0; + c109 = 0; + onceFailed109 = 0; + arg.checkNotEqChainableMutateAnother(100, mutate x) + .checkNotEqChainableMutateAnother(101, mutate c109) + .checkNotEqChainableMutateAnother(102, mutate x); + return (arg, c108, c109, x); +} + +fun pickG110(mutate self: int, mutate pushTo: tuple): self { + self += 10; + pushTo.tuplePush(c110); + return self; +} + +global tup110: tuple; +global c110: int; + +@method_id(110) +fun testMutateGlobalsLValue(init: int) { + c110 = init; + tup110 = createEmptyTuple(); + c110.incChained().incChained().pickG110(mutate tup110).incChained().pickG110(mutate tup110).incChained(); + return (c110, tup110); +} + +fun myTuplePush(mutate self: tuple, value: T): self { + self.tuplePush(value); + return self; +} + +fun myTupleAt(self: tuple, idx: int): T { + return self.tupleAt(idx); +} + +global tup111: tuple; + +@method_id(111) +fun testForallFunctionsWithSelf(): (int, int, tuple) { + var t = createEmptyTuple(); + tup111 = createEmptyTuple(); + t.myTuplePush(10); + tup111.myTuplePush(1).myTuplePush(2).myTuplePush(3); + return (t.myTupleAt(0), tup111.myTupleAt(tup111.tupleSize() - 1), tup111); +} + + + +fun main() { } + +/** +@testcase | 101 | 5 | 9 +@testcase | 102 | | 20 +@testcase | 103 | 1 | 7 +@testcase | 103 | 100 | 101 +@testcase | 103 | 8 | 11 +@testcase | 104 | | 6 4 2 +@testcase | 105 | 1 2 | 5 6 +@testcase | 106 | -20 | -5 5 +@testcase | 106 | -1 | 8 5 +@testcase | 106 | 7 | 12 5 +@testcase | 107 | 200 | 0 +@testcase | 107 | 102 | 202 +@testcase | 108 | 200 | 200 6 +@testcase | 108 | 101 | 201 0 +@testcase | 109 | 200 | 200 3 1 2 +@testcase | 109 | 100 | 100 0 0 1 +@testcase | 109 | 102 | 102 2 1 2 +@testcase | 110 | 0 | 24 [ 2 13 ] +@testcase | 111 | | 10 3 [ 1 2 3 ] + +@fif_codegen +""" + incChained PROC:<{ + // self + INC // self + }> + incChained2 PROC:<{ + // self + incChained CALLDICT // self + }> + incChained3 PROC:<{ + // self + incChained CALLDICT // self + }> + incChained4 PROC:<{ + // self + incChained CALLDICT // self + }> +""" + +@fif_codegen +""" + testIncChainedCodegen PROC:<{ + // x + incChained CALLDICT // x + incChained2 CALLDICT // x + incChained3 CALLDICT // x + incChained4 CALLDICT // x + }> +""" + */ diff --git a/tolk-tester/tests/smart-cast-tests.tolk b/tolk-tester/tests/smart-cast-tests.tolk new file mode 100644 index 00000000..4d71bb63 --- /dev/null +++ b/tolk-tester/tests/smart-cast-tests.tolk @@ -0,0 +1,678 @@ +// the goal of this file is not only to @testcase results — +// but to check that this file compiles + +fun getNullableInt(): int? { return 5; } +fun getNullableSlice(): slice? { return null; } +fun takeNullableInt(a: int?) {} +fun takeNullableSlice(a: slice?) {} +fun increment(mutate self: int) { self += 1; } +fun assignToInt(mutate self: int, value: int) { self = value; } +fun assignToNullableInt(mutate self: int?, value: int) { self = value; } +fun sameTensor(t: (int, int)) { return t; } +fun sameTensor2(t: (int?, (slice, slice, slice, builder)?)) { return t; } +fun eq(v: T) { return v; } +fun getTwo(): X { return 2 as X; } + +fun test1(): int { + var x = getNullableInt(); + var y = getNullableInt(); + if (x != null && y != null) { + __expect_type(x, "int"); + __expect_type(y, "int"); + return x + y; + } + return -1; +} + +fun test2() { + var (x, y) = (getNullableInt(), getNullableInt()); + if (x == null || y == null) { + return null; + } + __expect_type(x, "int"); + __expect_type(y, "int"); + return x + y; +} + +fun test3(): int { + var ([x, y]) = [getNullableInt(), getNullableInt()]; + if (x != null) { + if (((y)) != null) { + __expect_type(x, "int"); + __expect_type(y, "int"); + return x + y; + } + return x; + } + if (random() > -1) { + if (y == null) { return -1; } + else { return y; } + } + return 0; +} + +fun test4() { + var x = getNullableInt(); + if (x != null && x > 0) { + var x = getNullableInt(); + if ((x) != null && x + 10 < 0) { + var x = getNullableInt(); + return 10 > 3 && 10 < 10 && x != null && x + 8 > 10; + } + } + if (x != null && x < 1) { + return false; + } + if (x == null && x == null) { + __expect_type(x, "null"); + return true; + } + return x < x + 3; +} + +fun test5() { + var (a, (b, c)) = (getNullableInt(), (getNullableInt(), getNullableInt())); + if (a == null) { return -1; } + if (!(b != null)) { return -2; } + if (random() ? c == null && c == null : c == null) { return -3; } + return a + b + c; +} + +fun test6() { + var a: int? = 5; + __expect_type(a, "int"); + __expect_type(a != null ? a : null, "int"); + __expect_type(a == null ? "" : a, "int"); + takeNullableInt(a); + __expect_type(a, "int"); + if (random()) { + a = null; + } else { + if (random()) { a = null; } + else { a = null; } + } + __expect_type(a, "null"); + takeNullableSlice(a); // ok, `slice?` is `slice | null`, here a definitely null + var b: int? = true ? null : "sl"; + __expect_type(b, "null"); + takeNullableInt(b); + takeNullableSlice(b); // same reason + var c: int? = 10; + __expect_type(c, "int"); + takeNullableSlice(c = null); +} + +fun test7() { + var (a, b, c, d) = (getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt()); + if (a == null && true) { return -1; } + if (true && true && 1 && !0 && b == null) { return -2; } + if (true ? c == null && (((c))) == null && true : false) { return -3; } + if (!true ? random() > 0 : a != null && (d == null && b != null)) { return -4; } + return a + b + c + d; +} + +fun test8(x: int?, y: int?) { + var allGt1 = x != null && x > 1 && y != null && y > 1; + var xGtY = x != null && y != null && x > y; + var xLtEq0 = x == null || x < 0; + (x = 0) < random() || x > 10; + return x + 0; +} + +fun test9() { + var x = getNullableInt(); + var y = getNullableInt(); + if (x == null || y == null) { + return -1; + } + __expect_type(x, "int"); + __expect_type(y, "int"); + return x + y; +} + +fun test10(): int { + var (x, y) = (getNullableInt(), getNullableInt()); + if (x == null) { + if (y == null) { return -1; } + __expect_type(x, "null"); + __expect_type(y, "int"); + return y; + } + if (y == null) { + return x; + } + __expect_type(x, "int"); + __expect_type(y, "int"); + return x + y; +} + +fun test11() { + var [x, y] = [getNullableInt(), getNullableInt()]; + if (random()) { return x == null || y == null ? -1 : x + y; } + if (true && (x == null || y == null) && !!true) { return 0; } + return x + y; +} + +fun test12() { + var (x, y) = (getNullableInt(), getNullableInt()); + if (random() ? x == null || y == null : x == null || y == null) { return -1; } + __expect_type(x, "int"); + __expect_type(y, "int"); + return x + y; +} + +fun test13() { + var x: int? = getNullableInt(); + var y: int? = 10; + var z = getNullableInt(); + var w = getNullableInt(); + beginCell().storeInt(x!, 32).storeInt(x = getNullableInt()!, 32).storeInt(x, 32) + .storeInt(y, 32).storeInt(z = 10, 32).storeInt(x + y + z, 32) + .storeInt(w == null ? -1 : w, 32).storeInt(!(null == w) ? w : -1, 32); +} + +fun test14() { + var (x, y) = (getNullableInt(), getNullableInt()); + if (x == null) { + x = 0; + } + if (y == null) { + if (random()) { return 0; } + else { y = 0; } + } + return x + y; +} + +fun test20() { + var t = (getNullableInt(), getNullableInt()); + if (t.0 != null && t.1 != null) { + __expect_type(t.0, "int"); + __expect_type(t.1, "int"); + return t.0 + t.1; + } + t.0 = 10; + if (t.1 == null) { + t.1 = 20; + } + __expect_type(t.0, "int"); + __expect_type(t.1, "int"); + return t.0 + t.1; +} + +fun test21() { + var t = (getNullableInt(), (getNullableInt(), getNullableInt())); + if (t.0 != null && t.1.0 != null) { + if (t.1.1 != null) { return t.0 + t.1.0 + t.1.1; } + return t.0 + t.1.0; + } + if (t.0 != null) { + return t.0 + 0; + } + __expect_type(t.0, "null"); + __expect_type(t.1.0, "int?"); + return t.1.0 == null ? -1 : t.1.0 + 0; +} + +fun test22() { + var t = (getNullableInt(), (getNullableInt(), getNullableInt())); + if (t.0 == null || t.1.0 == null || t.1.1 == null) { + return -1; + } + return t.0 + t.1.0 + t.1.1; +} + +@method_id(123) +fun test23() { + var (x: int?, y: int?, z: int?) = (getNullableInt(), getNullableInt(), getNullableInt()); + ((x = 1, 0).0, (y = 2, 1).0) = (3, z = 4); + return x + y + z; +} + +@method_id(124) +fun test24(x: int?) { + if (x == null) { + __expect_type(x, "null"); + assignToNullableInt(mutate x, 10); + __expect_type(x, "int?"); + x.assignToNullableInt(x! + 5); + } else { + __expect_type(x, "int"); + increment(mutate x); + x.increment(); + __expect_type(x, "int"); + } + __expect_type(x, "int?"); + return x; +} + +fun test25() { + var x = (getNullableInt(), getNullableInt(), getNullableInt()); + x.0 = x.2 = random(); + return (x.0) + ((x.2)); +} + +fun test26() { + var x = [getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), + getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt()]; + if (~(x.0 = random())) { return; } + if ((x.1 = random()) < (x.2 = random())) { return; } + else if (!(x.2 <=> (x.3 = random()))) { return; } + x.5 = (x.4 = random()) ? (x.6 = random()) : (x.6 = random()); + if ((x.7 = random()) as int) { return; } + if (((((x.8 = random()) != null)))) { return; } + if ([x.1, (x.9 = random())!].1) { return; } + val result = x.0+x.1+x.2+x.3+x.4+x.5+x.6+x.7+x.8+x.9; +} + +fun test27() { + var (x, _) = ([getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), + getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt()], []); + +(x.0 = random()); + x.0 += [((x.1 = random()) < (x.2 = random() + x.1)) as int].0; + !(x.2 <=> (x.3 = random() + x.2)); + x.5 = (x.4 = random()) ? (x.6 = random()) : (x.6 = random()); + (x.7 = random()) as int; + (((((x.8 = random()) != null)))); + [x.1, (x.9 = random())!].1; + return x.0+x.1+x.2+x.3+x.4+x.5+x.6+x.7+x.8+x.9; +} + +fun test28() { + var x = (getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt()); + __expect_type((x.0 = random(), x.0 += (x.1 = random()) as int, !(x.1 <=> (x.2 = random() + x.0)) == null, (x.3 = random()) ? x.3 : (!x.3) as int), + "(int, int, bool, int)"); +} + +fun test29() { + var x = (getNullableInt(), getNullableInt(), getNullableInt(), getNullableInt()); + __expect_type([x.0 = random(), ((x.0 += (x.1 = random()) as int)), !(x.1 <=> (x.2 = random() + x.0)) == null, (x.3 = random()) ? x.3 : (!x.3) as int], + "[int, int, bool, int]"); +} + +@method_id(130) +fun test30(initial5: bool) { + var t: (int?, (int?, (int?, int?))) = initial5 + ? (getNullableInt(), (getNullableInt(), (getNullableInt(), getNullableInt()))) + : (null, (null, (null, null))); + if (t.0 == null || t.1.0 == null || t.1.1.0 == null || t.1.1.1 == null) { + if (t.1.0 == null || t.1.1.0 == null) { + if (t.1.1.0 == null) { + t.1.1.0 = 4; + } + __expect_type(t.1.1.0, "int"); + __expect_type(t.1.1.1, "int?"); + __expect_type(t.1.0, "int?"); + t.1.1.1 = 3; + t.1.0 = 2; + __expect_type(t.1.1.1, "int"); + __expect_type(t.1.0, "int"); + } + if (((((t.1.1.1)))) != null) {} + else { t.1.1.1 = 3; } + t.0 = 1; + } + return t.0 + t.1.0 + t.1.1.0 + t.1.1.1; +} + +fun test31() { + var t = (getNullableInt(), getNullableInt()); + t.0 == null ? (t.0, t.1) = (1, 2) : (t.1, t.0) = (4, 3); + return t.0 + t.1; +} + +@method_id(132) +fun test32() { + var t: (int?, (int?, int?)?, (int?, int?)) = (getNullableInt(), (getNullableInt(), getNullableInt()), (getNullableInt(), getNullableInt())); + if (t.0 == null) { return -1; } + t.1 != null && t.1.0 == null ? t.1 = (1, 2) : t.1 = (3, 4); + if (t.2.1 != null) { t.2.0 = 1; t.2.1 = 2; } + else { [t.2.0, t.2.1] = [3, 4]; } + return t.0 + t.1.0! + t.1.1! + t.2.0 + t.2.1; +} + +@method_id(133) +fun test33(): int { + var x = getNullableInt(); + repeat (eq(x = 5)) { + __expect_type(x, "int"); + increment(mutate x); + } + return x; +} + +fun test34() { + var (x, y) = (getNullableInt(), getNullableInt()); + if (random()) { throw (x = 1, y = 2); } + else { throw (x = 3, y = (1, getNullableInt()!).1); } + return x + y; +} + +fun test35() { + var (x, y, z, t) = (getNullableInt(), getNullableInt(), getNullableInt(), (getNullableInt(), getNullableInt())); + assert (x != null, 404); + assert (t.0 != null && true && !(t.1 == null) && !(z = 4)) throw (y = 404); + __expect_type(y, "int?"); + return x + t.0 + t.1 + z; +} + +fun test36() { + var x = getNullableInt(); + assert (x == null, x + 0); // check that x is int there + __expect_type(x, "null"); +} + +fun test37() { + var (x, code) = (getNullableInt()!, getNullableInt()); + try { + } catch(code) { + x = 20; + return x + code; // code is scoped + } + return code == null ? x : x + code; +} + +fun assignNull2(mutate x: T1?, mutate y: T2?) { + x = null; + y = null; +} + +fun test38() { + var (x: int?, y: int?) = (1, 2); + __expect_type(x, "int"); + __expect_type(y, "int"); + assignNull2(mutate x, mutate y); + __expect_type(x, "int?"); + __expect_type(y, "int?"); + if (x != null) { + if (y == null) { return -1; } + return x + y; + } + var t: (int?, slice?) = (null, null); + if (!false) { t.0 = 1; } + if (true) { t.1 = beginCell().endCell().beginParse(); } + __expect_type(t.0, "int"); + __expect_type(t.1, "slice"); + t.0 + t.1.loadInt(32); + assignNull2(mutate t.0, mutate t.1); + __expect_type(t.0, "int?"); + __expect_type(t.1, "slice?"); + t.0 != null && t.1 != null ? t.0 + loadInt(mutate t.1, 32) : -1; + return t.0 != null && t.1 != null ? t.0 + loadInt(mutate t.1, 32) : -1; +} + +@method_id(139) +fun test39() { + var x: (int?, int?)? = (4, null); + x.1 = 10; + x.1 += 1; + x!.1 += 1; + return (x!.0! + x.1); +} + +@method_id(140) +fun test40(second: int?) { + var x: (int?, int?)? = (4, second); + if (x.1 != null) { + val result = x.1 + x!.1 + x!!.1 + x.1! + x!!.1!!; + } + if (x!.1 != null) { + val result = x.1 + x!.1 + x!!.1 + x.1! + x!!.1!!; + } + if (!(x!!.1 != null)) { + return -1; + } + return x.1 + x!.1 + x!!.1 + x.1! + x!!.1!!; +} + +@method_id(141) +fun test41() { + var t: (int, int)? = null; + return sameTensor(t = (1, 2)); +} + +@method_id(142) +fun test42() { + var t: (int?, (int?, (int, int)?)?) = (getNullableInt(), (1, (2, 3))); + t.1 = (3,null); + __expect_type(t.1, "(int?, (int, int)?)"); + __expect_type(t, "(int?, (int?, (int, int)?)?)"); + return (t, t.1); +} + +@method_id(143) +fun test43() { + var t1: ((int, int), int?) = ((1, 2), 3); + var t2: ((int?, int?), (int?,int?)?) = ((null, null), (null, 5)); + t2.0 = t1.0 = (10, 11); + t2.1 = t1.1 = null; + return (t1, t2); +} + +@method_id(144) +fun test44() { + var t1: ((int, int), int?) = ((1, 2), 3); + var t2: ((int?, int?), (int?,int?)?) = ((null, null), (null, 5)); + t1.0 = t2.0 = (10, 11); + t1.1 = t2.1 = null; + __expect_type(t1, "((int, int), int?)"); + __expect_type(t2, "((int?, int?), (int?, int?)?)"); + return (t1, t2); +} + +@method_id(145) +fun test45() { + var t: (int?, (int?, (int, int)?)?) = (getNullableInt(), (1, (2, 3))); + var t2 = sameTensor2(t.1 = (3,null)); + return (t, t2, t.1); +} + +fun autoInfer46() { + var t1: int? = 3; + var t2: (int, int)? = (4, 5); + __expect_type(t1, "int"); + __expect_type(t2, "(int, int)"); + return (t1, t2); // proven to be not null, inferred (int, (int,int)) +} + +@method_id(146) +fun test46() { + var r46_1: (int, (int,int)) = autoInfer46(); + var r46_2: (int, (int,int)?) = autoInfer46(); + return (r46_1, r46_2); +} + +@method_id(147) +fun test47() { + var t1: int? = 3; + var t2: (int, int)? = (4, 5); + t1 = t2 = null; + __expect_type(t1, "null"); + __expect_type(t2, "null"); + var result = (t1, t2); // proven to be always null, inferred (null, null), 2 slots on a stack + return (result, 100, result.1, 100, t2 as (int, int)?); +} + +fun test48() { + var t1: int? = getNullableInt(); + if (t1 != null) { + var (t1 redef, t2) = (10, 5); + return t1 + t2; + var t2 redef = getNullableInt()!; + return t1 + t2; + } + return -1; +} + +fun test49(x: int?) { + while (x == null) { + x = getNullableInt(); + } + __expect_type(x, "int"); + return x + 1; +} + +fun test50() { + var (x: int?, y: int?) = (1, 2); + do { + x = getNullableInt(); + y = getNullableInt(); + } while (x == null || y == null); + return x + y; +} + +fun test51() { + while (true) { return; } + // test that no error "control reaches end of function" +} + +fun test52() { + do { } while (true); +} + +fun test53() { + var x1: int? = getNullableInt(); + var x2: int? = 5; + var x3: int? = 5; + var x10: int? = null; + var x11: int? = 5; + var x12: int? = 5; + while (x1 != null) { + __expect_type(x1, "int"); // because condition + __expect_type(x2, "int?"); // because re-assigned + __expect_type(x3, "int?"); // because re-assigned + __expect_type(x10, "null"); + __expect_type(x11, "int"); + x1 = getNullableInt(); + __expect_type(x1, "int?"); + assignToNullableInt(mutate x2, 5); + x3.assignToNullableInt(5); + x11 = 10; + assignToInt(mutate x12, 5); + } + __expect_type(x1, "null"); + __expect_type(x2, "int?"); + __expect_type(x3, "int?"); +} + +fun test54() { + var x1: int? = null; + var x2: int? = 5; + var x3: int? = 5; + var x10: int? = null; + var x11: int? = 5; + var x12: int? = 5; + do { + __expect_type(x1, "int?"); // because re-assigned + __expect_type(x2, "int?"); // because re-assigned + __expect_type(x3, "int?"); // because re-assigned + __expect_type(x10, "null"); + __expect_type(x11, "int"); + x1 = getNullableInt(); + __expect_type(x1, "int?"); + assignToNullableInt(mutate x2, 5); + if (random()) { x3.assignToNullableInt(5); } + x11 = 10; + assignToInt(mutate x12, 5); + } while (x1 != null); + __expect_type(x1, "null"); + __expect_type(x2, "int?"); + __expect_type(x3, "int?"); +} + +fun eq55(v: T) { return v; } + +fun test55() { + var x: int? = 4; + while (true) { + // currently, generic functions are instantiated at the type inferring step + // in case of loops, type inferring is re-enterable + // first iteration: x is int, eq instantiated + // second (final) iteration: x is int?, eq instantiated + // (checked via codegen) + eq55(x); + __expect_type(x, "int?"); // types are checked (unlike generics instantiated) after inferring + x = random() ? 1 : null; + } + __expect_type(x, "int?"); +} + +fun test56() { + var i: int? = null; + var (j: int?, k: int?) = (null, null); + __expect_type(i, "null"); + __expect_type(k, "null"); + i = getTwo(); + [j, ((k))] = [getTwo(), ((getTwo()))]; + __expect_type(i, "int?"); + __expect_type(j, "int?"); + __expect_type(k, "int?"); +} + +fun test57(mutate x: int?): int { + if (x == null) { x = 5; } + else { + if (x < 10) { x = 10; } + else { x = 20; } + } + if (x != null) { + return 123; + } + __expect_type(x, "int"); + // no "return" needed, because end of function is unreachable +} + +@method_id(158) +fun test58() { + var (x1, x2: int?) = (getNullableInt(), null); + return (test57(mutate x1), x1, test57(mutate x2), x2); +} + +fun test59() { + var (x1: int?, x2, x3) = (getNullableInt()!, getNullableInt(), 5); + if ((x2 = x3) != null) { + __expect_type(x2, "int"); + } + __expect_type(x2, "int"); + if ((x2 = getNullableInt()) != null) { + __expect_type(x2, "int"); + } + __expect_type(x2, "int?"); + if (((x1) = x2) == null) { + return; + } + __expect_type(x1, "int"); +} + + + +fun main(x: int?): int { + return x == null ? -1 : x; +} + +/** +@testcase | 0 | 1 | 1 +@testcase | 123 | | 7 +@testcase | 124 | 4 | 6 +@testcase | 124 | null | 15 +@testcase | 130 | -1 | 20 +@testcase | 130 | 0 | 10 +@testcase | 132 | | 15 +@testcase | 133 | | 10 +@testcase | 139 | | 16 +@testcase | 140 | 5 | 25 +@testcase | 141 | | 1 2 +@testcase | 142 | | 5 3 (null) (null) 0 -1 3 (null) (null) 0 +@testcase | 143 | | 10 11 (null) 10 11 (null) (null) 0 +@testcase | 144 | | 10 11 (null) 10 11 (null) (null) 0 +@testcase | 145 | | 5 3 (null) (null) 0 -1 3 (null) (null) (null) (null) 0 3 (null) (null) 0 +@testcase | 146 | | 3 4 5 3 4 5 -1 +@testcase | 147 | | (null) (null) 100 (null) 100 (null) (null) 0 +@testcase | 158 | | 123 10 123 5 + +@stderr warning: expression of type `int` is always not null, this condition is always true +@stderr warning: unreachable code +@stderr var t2 redef = getNullableInt()!; + +@fif_codegen eq55 PROC:<{ +@fif_codegen eq55 PROC:<{ +*/ diff --git a/tolk-tester/tests/special-fun-names.tolk b/tolk-tester/tests/special-fun-names.tolk new file mode 100644 index 00000000..8fae6d5d --- /dev/null +++ b/tolk-tester/tests/special-fun-names.tolk @@ -0,0 +1,24 @@ +fun onInternalMessage() { return 0; } +fun onExternalMessage() { return -1; } +fun onRunTickTock() { return -2; } +fun onSplitPrepare() { return -3; } +fun onSplitInstall() { return -4; } + +/** +@experimental_options remove-unused-functions + +@testcase | 0 | | 0 +@testcase | -1 | | -1 +@testcase | -2 | | -2 +@testcase | -3 | | -3 +@testcase | -4 | | -4 + +@fif_codegen +""" + 0 DECLMETHOD onInternalMessage + -1 DECLMETHOD onExternalMessage + -2 DECLMETHOD onRunTickTock + -3 DECLMETHOD onSplitPrepare + -4 DECLMETHOD onSplitInstall +""" + */ diff --git a/tolk-tester/tests/test-math.tolk b/tolk-tester/tests/test-math.tolk new file mode 100644 index 00000000..6b147c77 --- /dev/null +++ b/tolk-tester/tests/test-math.tolk @@ -0,0 +1,1288 @@ +// this is actually `mathlib.fc` transformed to Tolk + +import "@stdlib/tvm-lowlevel" + +/*--------------- MISSING OPERATIONS AND BUILT-INS ----------------*/ + +/// compute floor(log2(x))+1 +@pure +fun log2_floor_p1(x: int): int + asm "UBITSIZE"; + +@pure +fun mulrshiftr(x: int, y: int, s: int): int + asm "MULRSHIFTR"; + +@pure +fun mulrshiftr256(x: int, y: int): int + asm "256 MULRSHIFTR#"; + +@pure +fun mulrshift256mod(x: int, y: int): (int, int) + asm "256 MULRSHIFT#MOD"; + +@pure +fun mulrshiftr256mod(x: int, y: int): (int, int) + asm "256 MULRSHIFTR#MOD"; + +@pure +fun mulrshiftr255mod(x: int, y: int): (int, int) + asm "255 MULRSHIFTR#MOD"; + +@pure +fun mulrshiftr248mod(x: int, y: int): (int, int) + asm "248 MULRSHIFTR#MOD"; + +@pure +fun mulrshiftr5mod(x: int, y: int): (int, int) + asm "5 MULRSHIFTR#MOD"; + +@pure +fun mulrshiftr6mod(x: int, y: int): (int, int) + asm "6 MULRSHIFTR#MOD"; + +@pure +fun mulrshiftr7mod(x: int, y: int): (int, int) + asm "7 MULRSHIFTR#MOD"; + +@pure +fun lshift256divr(x: int, y: int): int + asm "256 LSHIFT#DIVR"; + +@pure +fun lshift256divmodr(x: int, y: int): (int, int) + asm "256 LSHIFT#DIVMODR"; + +@pure +fun lshift255divmodr(x: int, y: int): (int, int) + asm "255 LSHIFT#DIVMODR"; + +@pure +fun lshift2divmodr(x: int, y: int): (int, int) + asm "2 LSHIFT#DIVMODR"; + +@pure +fun lshift7divmodr(x: int, y: int): (int, int) + asm "7 LSHIFT#DIVMODR"; + +@pure +fun lshiftdivmodr(x: int, y: int, s: int): (int, int) + asm "LSHIFTDIVMODR"; + +@pure +fun rshiftr256mod(x: int): (int, int) + asm "256 RSHIFTR#MOD"; + +@pure +fun rshiftr248mod(x: int): (int, int) + asm "248 RSHIFTR#MOD"; + +@pure +fun rshiftr4mod(x: int): (int, int) + asm "4 RSHIFTR#MOD"; + +@pure +fun rshift3mod(x: int): (int, int) + asm "3 RSHIFT#MOD"; + +/// computes y - x (Tolk compiler does not try to use this by itself) +@pure +fun sub_rev(x: int, y: int): int + asm "SUBR"; + +@pure +fun nan(): int + asm "PUSHNAN"; + +@pure +fun is_nan(x: int): int + asm "ISNAN"; + +/*----------------------- SQUARE ROOTS ---------------------------*/ + +/// computes sqrt(a*b) exactly rounded to the nearest integer +/// for all 0 <= a, b <= 2^256-1 +/// may be used with b=1 or b=scale of fixed-point numbers +@pure +@inline_ref +fun geom_mean(a: int, b: int): int { + if (!min(a, b)) { + return 0; + } + var s: int = log2_floor_p1(a); // throws out of range error if a < 0 or b < 0 + var t: int = log2_floor_p1(b); + // NB: (a-b)/2+b == (a+b)/2, but without overflow for large a and b + var x: int = (s == t ? (a - b) / 2 + b : 1 << ((s + t) / 2)); + do { + // if always used with b=2^const, may be optimized to "const LSHIFTDIVC#" + // it is important to use `mulDivCeil` here, not `mulDivFloor` or `mulDivRound` + var q: int = (mulDivCeil(a, b, x) - x) / 2; + x += q; + } while (q); + return x; +} + +/// integer square root, computes round(sqrt(a)) for all a>=0. +/// note: `inline` is better than `inline_ref` for such simple functions +@pure +@inline +fun sqrt(a: int): int { + return geom_mean(a, 1); +} + +/// version for fixed248 = fixed-point numbers with scale 2^248 +/// fixed248 sqrt(fixed248 x) +@pure +@inline +fun fixed248_sqrt(x: int): int { + return geom_mean(x, 1 << 248); +} + +/// fixed255 sqrt(fixed255 x) +@pure +@inline +fun fixed255_sqrt(x: int): int { + return geom_mean(x, 1 << 255); +} + +/// fixed248 sqr(fixed248 x); +@pure +@inline +fun fixed248_sqr(x: int): int { + return mulDivRound(x, x, 1 << 248); +} + +/// fixed255 sqr(fixed255 x); +@pure +@inline +fun fixed255_sqr(x: int): int { + return mulDivRound(x, x, 1 << 255); +} + +const fixed248_One: int = (1 << 248); +const fixed255_One: int = (1 << 255); + +/*------------------- USEFUL CONSTANTS -------------------*/ + +/// store huge constants in inline_ref functions for reuse +/// (y,z) where y=round(log(2)*2^256), z=round((log(2)*2^256-y)*2^128) +/// then log(2) = y/2^256 + z/2^384 +@pure +@inline_ref +fun log2_xconst_f256(): (int, int) { + return (80260960185991308862233904206310070533990667611589946606122867505419956976172, -32272921378999278490133606779486332143); +} + +/// (y,z) where Pi = y/2^254 + z/2^382 +@pure +@inline_ref +fun Pi_xconst_f254(): (int, int) { + return (90942894222941581070058735694432465663348344332098107489693037779484723616546, 108051869516004014909778934258921521947); +} + +/// atan(1/16) as fixed260 +@pure +@inline_ref +fun Atan1_16_f260(): int { + return 115641670674223639132965820642403718536242645001775371762318060545014644837101; // true value is ...101.0089... +} + +/// atan(1/8) as fixed259 +@pure +@inline_ref +fun Atan1_8_f259(): int { + return 115194597005316551477397594802136977648153890007566736408151129975021336532841; // correction -0.1687... +} + +/// atan(1/32) as fixed261 +@pure +@inline_ref +fun Atan1_32_f261(): int { + return 115754418570128574501879331591757054405465733718902755858991306434399246026247; // correction 0.395... +} + +/// inline is better than inline_ref for such very small functions +@pure +@inline +fun log2_const_f256(): int { + var (c: int, _) = log2_xconst_f256(); + return c; +} + +@pure +@inline +fun fixed248_log2_const(): int { + return log2_const_f256() ~>> 8; +} + +@pure +@inline +fun Pi_const_f254(): int { + var (c, _) = Pi_xconst_f254(); + return c; +} + +@pure +@inline +fun fixed248_Pi_const(): int { + return Pi_const_f254() ~>> 6; +} + +/*-------------- HYPERBOLIC TANGENT AND EXPONENT ------------------*/ + +/// hyperbolic tangent of small x via n+2 terms of Lambert's continued fraction +/// n=17: good for |x| < log(2)/4 = 0.173 +/// fixed258 tanh_f258(fixed258 x, int n) +@pure +@inline_ref +fun tanh_f258(x: int, n: int): int { + var x2: int = mulDivRound(x, x, 1 << 255); // x^2 as fixed261 + var a: int = (2 * n + 5) << 250; // a=2n+5 as fixed250 + var c = a; + var Two: int = (1 << 251); // 2. as fixed250 + repeat (n) { + a = (c -= Two) + mulDivRound(x2, 1 << 239, a); // a := 2k+1+x^2/a as fixed250, k=n+1,n,...,2 + } + a = (3 << 254) + mulDivRound(x2, 1 << 243, a); // a := 3+x^2/a as fixed254 + // y = x/(1+a') = x - x*a'/(1+a') = x - x*x^2/(a+x^2) where a' = x^2/a + return x - (mulDivRound(x, x2, a + (x2 ~>> 7)) ~>> 7); +} + +/// fixed257 expm1_f257(fixed257 x) +/// computes exp(x)-1 for small x via 19 terms of Lambert's continued fraction for tanh(x/2) +/// good for |x| < log(2)/2 = 0.347 (n=17); consumes ~3500 gas +@pure +@inline_ref +fun expm1_f257(x: int): int { + // (almost) compute tanh(x/2) first; x/2 as fixed258 = x as fixed257 + var x2: int = mulDivRound(x, x, 1 << 255); // x^2 as fixed261 + var Two: int = (1 << 251); // 2. as fixed250 + var a: int = 39 << 250; // a=2n+5 as fixed250 + var c = a; + repeat (17) { + a = (c -= Two) + mulDivRound(x2, 1 << 239, a); // a := 2k+1+x^2/a as fixed250, k=n+1,n,...,2 + } + a = (3 << 254) + mulDivRound(x2, 1 << 243, a); // a := 3+x^2/a as fixed254 + // now tanh(x/2) = x/(1+a') where a'=x^2/a ; apply exp(x)-1=2*tanh(x/2)/(1-tanh(x/2)) + var t: int = (x ~>> 4) - a; // t:=x-a as fixed254 + return x - mulDivRound(x2, t / 2, a + mulrshiftr256(x, t) ~/ 4) ~/ 4; // x - x^2 * (x-a) / (a + x*(x-a)) +} + +/// expm1_f257() may be used to implement specific fixed-point exponentials +/// example: +/// fixed248 exp(fixed248 x) +@pure +@inline_ref +fun fixed248_exp(x: int): int { + var (l2c, l2d) = log2_xconst_f256(); + // divide x by log(2) and convert to fixed257 + // (int q, x) = muldivmodr(x, 256, l2c); // unfortunately, no such built-in + var (q: int, x redef) = lshiftdivmodr(x, l2c, 8); + x = 2 * x - mulDivRound(q, l2d, 1 << 127); + var y: int = expm1_f257(x); + // result is (1 + y) * (2^q) --> ((1 << 257) + y) >> (9 - q) + return (y ~>> (9 - q)) - (-1 << (248 + q)); + // note that (y ~>> (9 - q)) + (1 << (248 + q)) leads to overflow when q=8 +} + +/// compute 2^x in fixed248 +/// fixed248 exp2(fixed248 x) +@pure +@inline_ref +fun fixed248_exp2(x: int): int { + // (int q, x) = divmodr(x, 1 << 248); // no such built-in + var (q: int, x redef) = rshiftr248mod(x); + x = mulDivRound(x, log2_const_f256(), 1 << 247); + var y: int = expm1_f257(x); + return (y ~>> (9 - q)) - (-1 << (248 + q)); +} + +/*-------------------- TRIGONOMETRIC FUNCTIONS ----------------------*/ + +/// fixed260 tan(fixed260 x); +/// computes tan(x) for small |x|> 10)) ~>> 9); +} + +/// fixed260 tan(fixed260 x); +@pure +@inline_ref +fun tan_f260(x: int): int { + return tan_f260_inlined(x); +} + +/// fixed258 tan(fixed258 x); +/// computes tan(x) for small |x|> 6)) ~>> 5); +} + +/// fixed258 tan(fixed258 x); +@pure +@inline_ref +fun tan_f258(x: int): int { + return tan_f258_inlined(x); +} + +/// (fixed259, fixed263) sincosm1(fixed259 x) +/// computes (sin(x), 1-cos(x)) for small |x|<2*atan(1/16) +@pure +@inline +fun sincosm1_f259_inlined(x: int): (int, int) { + var t: int = tan_f260_inlined(x); // t=tan(x/2) as fixed260 + var tt: int = mulrshiftr256(t, t); // t^2 as fixed264 + var y: int = tt ~/ 512 + (1 << 255); // 1+t^2 as fixed255 + // 2*t/(1+t^2) as fixed259 and 2*t^2/(1+t^2) as fixed263 + // return (mulDivRound(t, 1 << 255, y), mulDivRound(tt, 1 << 255, y)); + return (t - mulDivRound(t / 2, tt, y) ~/ 256, tt - mulDivRound(tt / 2, tt, y) ~/ 256); +} + +@pure +@inline_ref +fun sincosm1_f259(x: int): (int, int) { + return sincosm1_f259_inlined(x); +} + +/// computes (sin(x+xe),-cos(x+xe)) for |x| <= Pi/4, xe very small +/// this function is very accurate, error less than 0.7 ulp (consumes ~ 5500 gas) +/// (fixed256, fixed256) sincosn(fixed256 x, fixed259 xe) +@pure +@inline_ref +fun sincosn_f256(x: int, xe: int): (int, int) { + // var (q, x1) = muldivmodr(x, 8, Atan1_8_f259()); // no muldivmodr() builtin + var (q, x1) = lshift2divmodr(abs(x), Atan1_8_f259()); // reduce mod theta where theta=2*atan(1/8) + var (si, co) = sincosm1_f259(x1 * 2 + xe); + var (a, b, c) = (-1, 0, 1); + repeat (q) { + // (a+b*I) *= (8+I)^2 = 63+16*I + (a, b, c) = (63 * a - 16 * b, 16 * a + 63 * b, 65 * c); + } + // now a/c = cos(q*theta), b/c = sin(q*theta) exactly(!) + // compute (a+b*I)*(1-co+si*I)/c + // (b, a) = (lshift256divr(b, c), lshift256divr(a, c)); + var (b redef, br: int) = lshift256divmodr(b, c); br = mulDivRound(br, 128, c); + var (a redef, ar: int) = lshift256divmodr(a, c); ar = mulDivRound(ar, 128, c); + return (sign(x) * (((mulrshiftr256(b, co) - br) ~/ 16 - mulrshiftr256(a, si)) ~/ 8 - b), + a - ((mulrshiftr256(a, co) - ar) ~/ 16 + mulrshiftr256(b, si)) ~/ 8); +} + +/// compute (sin(x),1-cos(x)) in fixed256 for |x| < 16*atan(1/16) = 0.9987 +/// (fixed256, fixed257) sincosm1_f256(fixed256 x); +/// slightly less accurate than sincosn_f256() (error up to 3/2^256), but faster (~ 4k gas) and shorter +@pure +@inline_ref +fun sincosm1_f256(x: int): (int, int) { + var (si, co) = sincosm1_f259_inlined(x); // compute (sin,1-cos)(x/8) in (fixed259,fixed263) + var r: int = 7; + repeat (r / 2) { + // 1-cos(2*x) = 2*sin(x)^2, sin(2*x) = 2*sin(x)*cos(x) + (co, si) = (mulrshiftr256(si, si), si - (mulrshiftr256(si, co) ~>> r)); + r -= 2; + } + return (si, co); +} + +/// compute (p, q) such that p/q = tan(x) for |x|<2*atan(1/2)=1899/2048=0.927 +/// (int, int) tan_aux(fixed256 x); +@pure +@inline_ref +fun tan_aux_f256(x: int): (int, int) { + var t: int = tan_f258_inlined(x); // t=tan(x/4) as fixed258 + // t:=2*t/(1-t^2)=2*(t-t^3/(t^2-1)) + var tt: int = mulrshiftr256(t, t); // t^2 as fixed260 + t = mulDivRound(t, tt, tt ~/ 16 + (-1 << 256)) ~/ 16 - t; // now t=-tan(x/2) as fixed259 + return (t, mulrshiftr256(t, t) ~/ 4 + (-1 << 256)); // return (2*t, t^2-1) as fixed256 +} + +/// sincosm1_f256() and sincosn_f256() may be used to implement trigonometric functions for different fixed-point types +/// example: +/// (fixed248, fixed248) sincos(fixed248 x); +@pure +@inline_ref +fun fixed248_sincos(x: int): (int, int) { + var (Pic, Pid) = Pi_xconst_f254(); + // (int q, x) = muldivmodr(x, 128, Pic); // no muldivmodr() builtin + var (q: int, x redef) = lshift7divmodr(x, Pic); // reduce mod Pi/2 + x = 2 * x - mulDivRound(q, Pid, 1 << 127); + var (si: int, co: int) = sincosm1_f256(x); // doesn't make sense to use more accurate sincosn_f256() + co = (1 << 248) - (co ~>> 9); + si = si ~>> 8; + repeat (q & 3) { + (si, co) = (co, -si); + } + return (si, co); +} + +/// fixed248 sin(fixed248 x); +/// inline is better than inline_ref for such simple functions +@pure +@inline +fun fixed248_sin(x: int): int { + var (si: int, _) = fixed248_sincos(x); + return si; +} + +/// fixed248 cos(fixed248 x); +@pure +@inline +fun fixed248_cos(x: int): int { + var (_, co: int) = fixed248_sincos(x); + return co; +} + +/// similarly, tan_aux_f256() may be used to implement tan() and cot() for specific fixed-point formats +/// fixed248 tan(fixed248 x); +/// not very accurate when |tan(x)| is very large (difficult to do better without floating-point numbers) +/// however, the relative accuracy is approximately 2^-247 in all cases, which is good enough for arguments given up to 2^-249 +@pure +@inline_ref +fun fixed248_tan(x: int): int { + var (Pic, Pid) = Pi_xconst_f254(); + // (int q, x) = muldivmodr(x, 128, Pic); // no muldivmodr() builtin + var (q: int, x redef) = lshift7divmodr(x, Pic); // reduce mod Pi/2 + x = 2 * x - mulDivRound(q, Pid, 1 << 127); + var (a, b) = tan_aux_f256(x); // now a/b = tan(x') + if (q & 1) { + (a, b) = (b, -a); + } + return mulDivRound(a, 1 << 248, b); // either -b/a or a/b as fixed248 +} + +/// fixed248 cot(fixed248 x); +@pure +@inline_ref +fun fixed248_cot(x: int): int { + var (Pic, Pid) = Pi_xconst_f254(); + var (q: int, x redef) = lshift7divmodr(x, Pic); // reduce mod Pi/2 + x = 2 * x - mulDivRound(q, Pid, 1 << 127); + var (b, a) = tan_aux_f256(x); // now b/a = tan(x') + if (q & 1) { + (a, b) = (b, -a); + } + return mulDivRound(a, 1 << 248, b); // either -b/a or a/b as fixed248 +} + +/*---------------- INVERSE HYPERBOLIC TANGENT AND LOGARITHMS ----------------*/ + +/// inverse hyperbolic tangent of small x, evaluated by means of n terms of the continued fraction +/// valid for |x| < 2^-2.5 ~ 0.18 if n=37 (slightly less accurate with n=36) +/// |x| < 1/8 if n=32; |x| < 2^-3.5 if n=28; |x| < 1/16 if n=25 +/// |x| < 2^-4.5 if n=23; |x| < 1/32 if n=21; |x| < 1/64 if n=18 +/// fixed258 atanh(fixed258 x); +@pure +@inline_ref +fun atanh_f258(x: int, n: int): int { + var x2: int = mulrshiftr256(x, x); // x^2 as fixed260 + var One: int = (1 << 254); + var a: int = One ~/ n + (1 << 255); // a := 2 + 1/n as fixed254 + repeat (n - 1) { + // a := 1 + (1 - x^2 / a)(1 + 1/n) as fixed254 + var t: int = One - mulDivRound(x2, 1 << 248, a); // t := 1 - x^2 / a + var n1: int = n - 1; + a = mulDivRound(t, n, n1) + One; + n = n1; + } + // x / (1 - x^2 / a) = x / (1 - d) = x + x * d / (1 - d) for d = x^2 / a + // int d = mulDivRound(x2, 1 << 255, a - (x2 ~>> 6)); // d/(1-d) = x^2/(a-x^2) as fixed261 + // return x + (mulrshiftr256(x, d) ~>> 5); + return x + mulDivRound(x, x2 / 2, a - x2 ~/ 64) ~/ 32; +} + +/// number of terms n should be chosen as for atanh_f258() +/// fixed261 atanh(fixed261 x); +@pure +@inline +fun atanh_f261_inlined(x: int, n: int): int { + var x2: int = mulrshiftr256(x, x); // x^2 as fixed266 + var One: int = (1 << 254); + var a: int = One ~/ n + (1 << 255); // a := 2 + 1/n as fixed254 + repeat (n - 1) { + // a := 1 + (1 - x^2 / a)(1 + 1/n) as fixed254 + var t: int = One - mulDivRound(x2, 1 << 242, a); // t := 1 - x^2 / a + var n1: int = n - 1; + a = mulDivRound(t, n, n1) + One; + n = n1; + } + // x / (1 - x^2 / a) = x / (1 - d) = x + x * d / (1 - d) for d = x^2 / a + // int d = mulDivRound(x2, 1 << 255, a - (x2 ~>> 12)); // d/(1-d) = x^2/(a-x^2) as fixed267 + // return x + (mulrshiftr256(x, d) ~>> 11); + return x + mulDivRound(x, x2, a - x2 ~/ 4096) ~/ 4096; +} + +/// fixed261 atanh(fixed261 x); +@pure +@inline_ref +fun atanh_f261(x: int, n: int): int { + return atanh_f261_inlined(x, n); +} + +/// returns (y, s) such that log(x) = y/2^257 + s*log(2) for positive integer x +/// (fixed257, int) log_aux(int x) +@pure +@inline_ref +fun log_aux_f257(x: int): (int, int) { + var s: int = log2_floor_p1(x); + x <<= 256 - s; + var t: int = -1 << 256; + if ((x >> 249) <= 90) { + t >>= 1; + s -= 1; + } + x += t; + var `2x`: int = 2 * x; + var y: int = lshift256divr(`2x`, (x >> 1) - t); + // y = `2x` - (mulrshiftr256(2x, y) ~>> 2); // this line could improve precision on very rare occasions + return (atanh_f258(y, 36), s); +} + +/// computes 33^m for small m +@pure +@inline +fun pow33(m: int): int { + var t: int = 1; + repeat (m) { + t *= 33; + } + return t; +} + +/// computes 33^m for small 0<=m<=22 +/// slightly faster than pow33() +@pure +@inline +fun pow33b(m: int): int { + var (mh: int, ml: int) = divMod(m, 5); + var t: int = 1; + repeat (ml) { + t *= 33; + } + repeat (mh) { + t *= 33 * 33 * 33 * 33 * 33; + } + return t; +} + +/// returns (s, q, y) such that log(x) = s*log(2) + q*log(33/32) + y/2^260 for positive integer x +/// (int, int, fixed260) log_auxx_f260(int x); +@pure +@inline_ref +fun log_auxx_f260(x: int): (int, int, int) { + var s: int = log2_floor_p1(x) - 1; + x <<= 255 - s; // rescale to 1 <= x < 2 as fixed255 + var t: int = 2873 << 244; // ~ (33/32)^11 ~ sqrt(2) as fixed255 + var x1: int = (x - t) >> 1; + var q: int = mulDivRound(x1, 65, x1 + t) + 11; // crude approximation to round(log(x)/log(33/32)) + // t = 1; repeat (q) { t *= 33; } // t:=33^q, 0<=q<=22 + t = pow33b(q); + t <<= (51 - q) * 5; // t:=(33/32)^q as fixed255, nearest power of 33/32 to x + x -= t; + var y: int = lshift256divr(x << 4, (x >> 1) + t); // y = (x-t)/(x+t) as fixed261 + y = atanh_f261(y, 18); // atanh((x-t)/(x+t)) as fixed261, or log(x/t) as fixed260 + return (s, q, y); +} + +/// returns (y, s) such that log(x) = y/2^256 + s*log(2) for positive integer x +/// this function is very precise (error less than 0.6 ulp) and consumes < 7k gas +/// may be used to implement specific fixed-point instances of log() and log2() +/// (fixed256, int) log_aux_f256(int x); +@pure +@inline_ref +fun log_aux_f256(x: int): (int, int) { + var (s, q, y) = log_auxx_f260(x); + var (yh, yl) = rshiftr4mod(y); // y ~/% 16 , but Tolk does not optimize this to RSHIFTR#MOD + // int Log33_32 = 3563114646320977386603103333812068872452913448227778071188132859183498739150; // log(33/32) as fixed256 + // int Log33_32_l = -3769; // log(33/32) = Log33_32 / 2^256 + Log33_32_l / 2^269 + yh += (yl * 512 + q * -3769) ~>> 13; // compensation, may be removed if slightly worse accuracy is acceptable + var Log33_32: int = 3563114646320977386603103333812068872452913448227778071188132859183498739150; // log(33/32) as fixed256 + return (yh + q * Log33_32, s); +} + +/// returns (y, s) such that log2(x) = y/2^256 + s for positive integer x +/// this function is very precise (error less than 0.6 ulp) and consumes < 7k gas +/// may be used to implement specific fixed-point instances of log() and log2() +/// (fixed256, int) log2_aux_f256(int x); +@pure +@inline_ref +fun log2_aux_f256(x: int): (int, int) { + var (s, q, y) = log_auxx_f260(x); + y = lshift256divr(y, log2_const_f256()) ~>> 4; // y/log(2) as fixed256 + var Log33_32: int = 5140487830366106860412008603913034462883915832139695448455767612111363481357; // log_2(33/32) as fixed256 + // Log33_32/2^256 happens to be a very precise approximation to log_2(33/32), no compensation required + return (y + q * Log33_32, s); +} + + +/// fixed248 log(fixed248 x) +@pure +@inline_ref +fun fixed248_log(x: int): int { + var (y, s) = log_aux_f256(x); + return mulDivRound(s - 248, log2_const_f256(), 1 << 8) + (y ~>> 8); + // return mulDivRound(s - 248, 80260960185991308862233904206310070533990667611589946606122867505419956976172, 1 << 8) + (y ~>> 8); +} + +/// fixed248 log2(fixed248 x) +@pure +@inline +fun fixed248_log2(x: int): int { + var (y, s) = log2_aux_f256(x); + return ((s - 248) << 248) + (y ~>> 8); +} + +/// computes x^y as exp(y*log(x)), x >= 0 +/// fixed248 pow(fixed248 x, fixed248 y); +@pure +@inline_ref +fun fixed248_pow(x: int, y: int): int { + if (!y) { + return 1 << 248; // x^0 = 1 + } + if (x <= 0) { + var bad: int = ((x | y) < 0) as int; + return 0 >> bad; // 0^y = 0 if x=0 and y>=0; "out of range" exception otherwise + } + var (l, s) = log2_aux_f256(x); + s -= 248; // log_2(x) = s+l, l is fixed256, 0<=l<1 + // compute (s+l)*y = q+ll + var (q1, r1) = mulrshiftr248mod(s, y); // muldivmodr(s, y, 1 << 248) + var (q2, r2) = mulrshift256mod(l, y); + r2 >>= 247; + var (q3, r3) = rshiftr248mod(q2); // divmodr(q2, 1 << 248); + var (q, ll) = rshiftr248mod(r1 + r3); + ll = 512 * ll + r2; + q += q1 + q3; + // now log_2(x^y) = y*log_2(x) = q + ll, ss integer, ll fixed257, -1/2<=ll<1/2 + var sq: int = q + 248; + if (sq <= 0) { + return -((sq == 0) as int); // underflow + } + y = expm1_f257(mulrshiftr256(ll, log2_const_f256())); + return (y ~>> (9 - q)) - (-1 << sq); +} + +/*-------------------- INVERSE TRIGONOMETRIC FUNCTIONS ------------------*/ + +/// number of terms n should be chosen as for atanh_f258() +/// fixed259 atan(fixed259 x); +@pure +@inline_ref +fun atan_f259(x: int, n: int): int { + var x2: int = mulrshiftr256(x, x); // x^2 as fixed262 + var One: int = (1 << 254); + var a: int = One ~/ n + (1 << 255); // a := 2 + 1/n as fixed254 + repeat (n - 1) { + // a := 1 + (1 + x^2 / a)(1 + 1/n) as fixed254 + var t: int = One + mulDivRound(x2, 1 << 246, a); // t := 1 + x^2 / a + var n1: int = n - 1; + a = mulDivRound(t, n, n1) + One; + n = n1; + } + // x / (1 + x^2 / a) = x / (1 + d) = x - x * d / (1 + d) = x - x * x^2/(a+x^2) for d = x^2 / a + return x - mulDivRound(x, x2, a + x2 ~/ 256) ~/ 256; +} + +/// number of terms n should be chosen as for atanh_f261() +/// fixed261 atan(fixed261 x); +@pure +@inline +fun atan_f261_inlined(x: int, n: int): int { + var x2: int = mulrshiftr256(x, x); // x^2 as fixed266 + var One: int = (1 << 254); + var a: int = One ~/ n + (1 << 255); // a := 2 + 1/n as fixed254 + repeat (n - 1) { + // a := 1 + (1 + x^2 / a)(1 + 1/n) as fixed254 + var t: int = One + mulDivRound(x2, 1 << 242, a); // t := 1 + x^2 / a + var n1: int = n - 1; + a = mulDivRound(t, n, n1) + One; + n = n1; + } + // x / (1 + x^2 / a) = x / (1 + d) = x - x * d / (1 + d) = x - x * x^2/(a+x^2) for d = x^2 / a + return x - mulDivRound(x, x2, a + x2 ~/ 4096) ~/ 4096; +} + +/// fixed261 atan(fixed261 x); +@pure +@inline_ref +fun atan_f261(x: int, n: int): int { + return atan_f261_inlined(x, n); +} + +/// computes (q,a,b) such that q is approximately atan(x)/atan(1/32) and a+b*I=(1+I/32)^q as fixed255 +/// then b/a=atan(q*atan(1/32)) exactly, and (a,b) is almost a unit vector pointing in the direction of (1,x) +/// must have |x|<1.1, x is fixed24 +/// (int, fixed255, fixed255) atan_aux_prereduce(fixed24 x); +@pure +@inline_ref +fun atan_aux_prereduce(x: int): (int, int, int) { + var xu: int = abs(x); + var tc: int = 7214596; // tan(13*theta) as fixed24 where theta=atan(1/32) + var t1: int = mulDivRound(xu - tc, 1 << 88, xu * tc + (1 << 48)); // tan(x') as fixed64 where x'=atan(x)-13*theta + // t1/(3+t1^2) * 3073/32 = x'/3 * 3072/32 = x' / (96/3072) = x' / theta + var q: int = mulDivRound(t1 * 3073, 1 << 59, t1 * t1 + (3 << 128)) + 13; // approximately round(atan(x)/theta), 0<=q<=25 + var (pa, pb) = (33226912, 5232641); // (32+I)^5 + var (qh, ql) = divMod(q, 5); + var (a, b) = (1 << (5 * (51 - q)), 0); // (1/32^q, 0) as fixed255 + repeat (ql) { + // a+b*I *= 32+I + b.stackMoveToTop(); + (a, b) = (sub_rev(b, 32 * a), a + 32 * b); // same as (32 * a - b, 32 * b + a), but more efficient + } + repeat (qh) { + // a+b*I *= (32+I)^5 = pa + pb*I + (a, b) = (a * pa - b * pb, a * pb + b * pa); + } + var xs: int = sign(x); + return (xs * q, a, xs * b); +} + +/// compute (q, z) such that atan(x)=q*atan(1/32)+z for -1 <= x < 1 +/// this function is reasonably accurate (error < 7 ulp with ulp = 2^-261), but it consumes >7k gas +/// this is sufficient for most purposes +/// (int, fixed261) atan_aux(fixed256 x) +@pure +@inline_ref +fun atan_aux_f256(x: int): (int, int) { + var (q, a, b) = atan_aux_prereduce(x ~>> 232); // convert x to fixed24 + // now b/a = tan(q*atan(1/32)) exactly, where q is near atan(x)/atan(1/32); so b/a is near x + // compute y = u/v = (a*x-b)/(a+b*x) as fixed261 ; then |y|<0.0167 = 1.07/64 and atan(x)=atan(y)+q*atan(1/32) + var (u, ul) = mulrshiftr256mod(a, x); + u = (ul ~>> 250) + ((u - b) << 6); // |u| < 1/32, convert fixed255 -> fixed261 + var v: int = a + mulrshiftr256(b, x); // v is scalar product of (a,b) and (1,x), it is approximately in [1..sqrt(2)] as fixed255 + var y: int = mulDivRound(u, 1 << 255, v); // y = u/v as fixed261 + var z: int = atan_f261_inlined(y, 18); // z = atan(x)-q*atan(1/32) + return (q, z); +} + +/// compute (q, z) such that atan(x)=q*atan(1/32)+z for -1 <= x < 1 +/// this function is very accurate (error < 2 ulp), but it consumes >7k gas +/// in most cases, faster function atan_aux_f256() should be used +/// (int, fixed261) atan_auxx(fixed256 x) +@pure +@inline_ref +fun atan_auxx_f256(x: int): (int, int) { + var (q, a, b) = atan_aux_prereduce(x ~>> 232); // convert x to fixed24 + // now b/a = tan(q*atan(1/32)) exactly, where q is near atan(x)/atan(1/32); so b/a is near x + // compute y = (a*x-b)/(a+b*x) as fixed261 ; then |y|<0.0167 = 1.07/64 and atan(x)=atan(y)+q*atan(1/32) + // use sort of double precision arithmetic for this + var (u, ul) = mulrshiftr256mod(a, x); + ul /= 2; + u -= b; // |u| < 1/32 as fixed255 + var (v, vl) = mulrshiftr256mod(b, x); + vl /= 2; + v += a; // v is scalar product of (a,b) and (1,x), it is approximately in [1..sqrt(2)] as fixed255 + // y = (u + ul*eps) / (v + vl*eps) = u/v + (ul - vl * u/v)/v * eps where eps=1/2^255 + var (y, r) = lshift255divmodr(u, v); // y = u/v as fixed255 + var yl: int = mulDivRound(ul + r, 1 << 255, v) - mulDivRound(vl, y, v); // y/2^255 + yl/2^510 represent u/v + y = (yl ~>> 249) + (y << 6); // convert y to fixed261 + var z: int = atan_f261_inlined(y, 18); // z = atan(x)-q*atan(1/32) + return (q, z); +} + +/// consumes ~ 8k gas +/// fixed255 atan(fixed255 x); +@pure +@inline_ref +fun atan_f255(x: int): int { + var s: int = (x ~>> 256); + x.stackMoveToTop(); + if (s) { + x = lshift256divr(-1 << 255, x); // x:=-1/x as fixed256 + } else { + x *= 2; // convert to fixed256 + } + var (q, z) = atan_aux_f256(x); + // now atan(x) = z + q*atan(1/32) + s*(Pi/2), z is fixed261 + var (Pi_h, Pi_l) = Pi_xconst_f254(); // Pi/2 as fixed255 + fixed383 + var (qh, ql) = mulrshiftr6mod(q, Atan1_32_f261()); + return qh + s * Pi_h + (z + ql + mulDivRound(s, Pi_l, 1 << 122)) ~/ 64; +} + +/// computes atan(x) for -1 <= x < 1 only +/// fixed256 atan_small(fixed256 x); +@pure +@inline_ref +fun atan_f256_small(x: int): int { + var (q, z) = atan_aux_f256(x); + // now atan(x) = z + q*atan(1/32), z is fixed261 + var (qh, ql) = mulrshiftr5mod(q, Atan1_32_f261()); + return qh + (z + ql) ~/ 32; +} + +/// fixed255 asin(fixed255 x); +@pure +@inline_ref +fun asin_f255(x: int): int { + var a: int = fixed255_One - fixed255_sqr(x); // a:=1-x^2 + if (!a) { + return sign(x) * Pi_const_f254(); // Pi/2 or -Pi/2 + } + var y: int = fixed255_sqrt(a); // sqrt(1-x^2) + var t: int = -lshift256divr(x, (-1 << 255) - y); // t = x/(1+sqrt(1-x^2)) avoiding overflow + return atan_f256_small(t); // asin(x)=2*atan(t) +} + +/// fixed254 acos(fixed255 x); +@pure +@inline_ref +fun acos_f255(x: int): int { + var Pi: int = Pi_const_f254(); + if (x == (-1 << 255)) { + return Pi; // acos(-1) = Pi + } + Pi /= 2; + var y: int = fixed255_sqrt(fixed255_One - fixed255_sqr(x)); // sqrt(1-x^2) + var t: int = lshift256divr(x, (-1 << 255) - y); // t = -x/(1+sqrt(1-x^2)) avoiding overflow + return Pi + atan_f256_small(t) ~/ 2; // acos(x)=Pi/2 + 2*atan(t) +} + +/// consumes ~ 10k gas +/// fixed248 asin(fixed248 x) +@pure +@inline +fun fixed248_asin(x: int): int { + return asin_f255(x << 7) ~>> 7; +} + +/// consumes ~ 10k gas +/// fixed248 acos(fixed248 x) +@pure +@inline +fun fixed248_acos(x: int): int { + return acos_f255(x << 7) ~>> 6; +} + +/// consumes ~ 7500 gas +/// fixed248 atan(fixed248 x); +@pure +@inline_ref +fun fixed248_atan(x: int): int { + var s: int = (x ~>> 249); + x.stackMoveToTop(); + if (s) { + s = sign(s); + x = lshift256divr(-1 << 248, x); // x:=-1/x as fixed256 + } else { + x <<= 8; // convert to fixed256 + } + var (q, z) = atan_aux_f256(x); + // now atan(x) = z + q*atan(1/32) + s*(Pi/2), z is fixed261 + return (z ~/ 64 + s * Pi_const_f254() + mulDivRound(q, Atan1_32_f261(), 64)) ~/ 128; // compute in fixed255, then convert +} + +/// fixed248 acot(fixed248 x); +@pure +@inline_ref +fun fixed248_acot(x: int): int { + var s: int = (x ~>> 249); + x.stackMoveToTop(); + if (s) { + x = lshift256divr(-1 << 248, x); // x:=-1/x as fixed256 + s = 0; + } else { + x <<= 8; // convert to fixed256 + s = sign(x); + } + var (q, z) = atan_aux_f256(x); + // now acot(x) = - z - q*atan(1/32) + s*(Pi/2), z is fixed261 + return (s * Pi_const_f254() - z ~/ 64 - mulDivRound(q, Atan1_32_f261(), 64)) ~/ 128; // compute in fixed255, then convert +} + +/*-------------------- PSEUDO-RANDOM NUMBERS ------------------*/ + +/// random number with standard normal distribution N(0,1) +/// generated by Kinderman--Monahan ratio method modified by J.Leva +/// spends ~ 2k..3k gas on average +/// fixed252 nrand(); +@inline_ref +fun nrand_f252(): int { + var (x, s, t, A, B, r0) = (nan(), 29483 << 236, -3167 << 239, 12845, 16693, 9043); + // 4/sqrt(e*Pi) = 1.369 loop iterations on average + do { + var (u, v) = (random() / 16 + 1, mulDivRound(random() - (1 << 255), 7027, 1 << 16)); // fixed252; 7027=ceil(sqrt(8/e)*2^12) + var va: int = abs(v); + var (u1, v1) = (u - s, va - t); // (u - 29483/2^16, abs(v) + 3167/2^13) as fixed252 + // Q := u1^2 + v1 * (A*v1 - B*u1) as fixed252 where A=12845/2^16, B=16693/2^16 + var Q: int = mulDivRound(u1, u1, 1 << 252) + mulDivRound(v1, mulDivRound(v1, A, 1 << 16) - mulDivRound(u1, B, 1 << 16), 1 << 252); + // must have 9043 / 2^15 < Q < 9125 / 2^15, otherwise accept if smaller, reject if larger + var Qd: int = (Q >> 237) - r0; + if ((Qd < 9125 - 9043) & (va / u < 16)) { + x = mulDivRound(v, 1 << 252, u); // x:=v/u as fixed252; reject immediately if |v/u| >= 16 + if (Qd >= 0) { + // immediately accept if Qd < 0 + // rarely taken branch - 0.012 times per call on average + // check condition v^2 < -4*u^2*log(u), or equivalent condition u < exp(-x^2/4) for x=v/u + var xx: int = mulrshiftr256(x, x) ~/ 4; // x^2/4 as fixed248 + var ex: int = fixed248_exp(-xx) * 16; // exp(-x^2/4) as fixed252 + if (u > ex) { + x = nan(); // condition false, reject + } + } + } + } while (!(~ is_nan(x))); + return x; +} + +/// generates a random number approximately distributed according to the standard normal distribution +/// much faster than nrand_f252(), should be suitable for most purposes when only several random numbers are needed +/// fixed252 nrand_fast(); +@inline_ref +fun nrand_fast_f252(): int { + var t: int = -3 << 253; // -6. as fixed252 + repeat (12) { + t += random() / 16; // add together 12 uniformly random numbers + } + return t; +} + +/// random number uniformly distributed in [0..1) +/// fixed248 random(); +@inline +fun fixed248_random(): int { + return random() >> 8; +} + +/// random number with standard normal distribution +/// fixed248 nrand(); +@inline +fun fixed248_nrand(): int { + return nrand_f252() ~>> 4; +} + +/// generates a random number approximately distributed according to the standard normal distribution +/// fixed248 nrand_fast(); +@inline +fun fixed248_nrand_fast(): int { + return nrand_fast_f252() ~>> 4; +} + +@pure +fun tset(mutate self: tuple, idx: int, value: X): void + asm(self value idx) "SETINDEXVAR"; + +// computes 1-acos(x)/Pi by a very simple, extremely slow (~70k gas) and imprecise method +// fixed256 acos_prepare_slow(fixed255 x); +@inline +fun acos_prepare_slow_f255(x: int): int { + x -= (x == 0) as int; + var t: int = 1; + repeat (255) { + t = t * sign(x) * 2 + 1; // decode Gray code (sign(x_0), sign(x_1), ...) + x = (-1 << 255) - mulDivRound(x, - x, 1 << 254); // iterate x := 2*x^2 - 1 = cos(2*acos(x)) + } + return abs(t); +} + +// extremely slow (~70k gas) and somewhat imprecise (very imprecise when x is small), for testing only +// fixed254 acos_slow(fixed255 x); +@inline_ref +fun acos_slow_f255(x: int): int { + var t: int = acos_prepare_slow_f255(x); + return - mulrshiftr256(t + (-1<<256), Pi_const_f254()); +} + +// fixed255 asin_slow(fixed255 x); +@inline_ref +fun asin_slow_f255(x: int): int { + var t: int = acos_prepare_slow_f255(abs(x)) % (1 << 255); + return mulDivRound(t, Pi_const_f254(), 1 << 255) * sign(x); +} + +@inline_ref +fun test_nrand(n: int): tuple { + var t: tuple = createEmptyTuple(); + repeat (255) { + t.tuplePush(0); + } + repeat (n) { + var x: int = fixed248_nrand(); + var bucket: int = (abs(x) >> 243); // 255 buckets starting from x=0, each 1/32 wide + var at_bucket: int = t.tupleAt(bucket); + t.tset(bucket, at_bucket + 1); + } + return t; +} + +@method_id(10000) +fun geom_mean_test(x: int, y: int): int { + return geom_mean(x, y); +} +@method_id(10001) +fun tan_f260_test(x: int): int { + return tan_f260(x); +} +@method_id(10002) +fun sincosm1_f259_test(x: int): (int, int) { + return sincosm1_f259(x); +} +@method_id(10003) +fun sincosn_f256_test(x: int, y: int): (int, int) { + return sincosn_f256(x, y); +} +@method_id(10004) +fun sincosm1_f256_test(x: int): (int, int) { + return sincosm1_f256(x); +} +@method_id(10005) +fun tan_aux_f256_test(x: int): (int, int) { + return tan_aux_f256(x); +} +@method_id(10006) +fun fixed248_tan_test(x: int): int { + return fixed248_tan(x); +} +/* + (int) atanh_alt_f258_test(x) method_id(10007) { + return atanh_alt_f258(x); + } +*/ +@method_id(10008) +fun atanh_f258_test(x:int, y:int): int { + return atanh_f258(x, y); +} +@method_id(10009) +fun atanh_f261_test(x:int, y:int): int { + return atanh_f261(x, y); +} + +@method_id(10010) +fun log2_aux_f256_test(x:int): (int, int) { + return log2_aux_f256(x); +} +@method_id(10011) +fun log_aux_f256_test(x:int): (int, int) { + return log_aux_f256(x); +} +@method_id(10012) +fun fixed248_pow_test(x:int, y:int): int { + return fixed248_pow(x, y); +} +@method_id(10013) +fun exp_log_div(x:int, y:int): int { + return fixed248_exp(fixed248_log(x << 248) ~/ y); +} +@method_id(10014) +fun fixed248_log_test(x:int): int { + return fixed248_log(x); +} +@method_id(10015) +fun log_aux_f257_test(x:int): (int,int) { + return log_aux_f257(x); +} +@method_id(10016) +fun fixed248_sincos_test(x:int): (int,int) { + return fixed248_sincos(x); +} +@method_id(10017) +fun fixed248_exp_test(x:int): int { + return fixed248_exp(x); +} +@method_id(10018) +fun fixed248_exp2_test(x:int): int { + return fixed248_exp2(x); +} +@method_id(10019) +fun expm1_f257_test(x:int): int { + return expm1_f257(x); +} +@method_id(10020) +fun atan_f255_test(x:int): int { + return atan_f255(x); +} +@method_id(10021) +fun atan_f259_test(x:int, n:int): int { + return atan_f259(x, n); +} +@method_id(10022) +fun atan_aux_f256_test(x:int): (int, int) { + return atan_aux_f256(x); +} +@method_id(10023) +fun asin_f255_test(x:int): int { + return asin_f255(x); +} +@method_id(10024) +fun asin_slow_f255_test(x:int): int { + return asin_slow_f255(x); +} +@method_id(10025) +fun acos_f255_test(x:int): int { + return acos_f255(x); +} +@method_id(10026) +fun acos_slow_f255_test(x:int): int { + return acos_slow_f255(x); +} +@method_id(10027) +fun fixed248_atan_test(x:int): int { + return fixed248_atan(x); +} +@method_id(10028) +fun fixed248_acot_test(x:int): int { + return fixed248_acot(x); +} + +fun main() { + var One: int = 1; + // repeat(76 / 4) { One *= 10000; } + var sqrt2: int = geom_mean(One, 2 * One); + var sqrt3: int = geom_mean(One, 3 * One); + // return geom_mean(-1 - (-1 << 256), -1 - (-1 << 256)); + // return geom_mean(-1 - (-1 << 256), -2 - (-1 << 256)); + // return geom_mean(-1 - (-1 << 256), 1 << 255); + // return (sqrt2, geom_mean(sqrt2, One)); // (sqrt(2), 2^(1/4)) + // return (sqrt3, geom_mean(sqrt3, One)); // (sqrt(3), 3^(1/4)) + // return geom_mean(3 << 254, 1 << 254); + // return geom_mean(3, 5); + // return tan_f260(115641670674223639132965820642403718536242645001775371762318060545014644837101 - 1); + // return tan_f260(15 << 252); // tan(15/256) * 2^260 + // return sincosm1_f259(1 << 255); // (sin,1-cos)(1/16) * 2^259 + // return sincosm1_f259(115641670674223639132965820642403718536242645001775371762318060545014644837101 - 1); + // return sincosm1_f256((1 << 255) - 1 + (1 << 255)); // (sin,1-cos)(1-2^(-256)) + // return sincosm1_f256(Pi_const_f254()); // (sin,1-cos)(Pi/4) + // return sincosn_f256(Pi_const_f254(), 0); // (sin,-cos)(Pi/4) + // return sincosn_f256((1 << 255) + 1, 0); // (sin,-cos)(1/2+1/2^256) + // return sincosn_f256(1 << 254, 0); + // return sincosn_f256(stackMoveToTop(15) << 252, 0); // (sin,-cos)(15/16) + // return sincosm1_f256(stackMoveToTop(15) << 252); // (sin,1-cos)(15/16) + // return sincosn_f256(60628596148627720713372490462954977108898896221398738326462025186323149077698, 0); // (sin,-cos)(Pi/6) + // return sincosm1_f256(60628596148627720713372490462954977108898896221398738326462025186323149077698); // (sin,1-cos)(Pi/6) + // return tan_aux_f256(1899 << 245); // (p,q) such that p/q=tan(1899/2048) + // return fixed248_tan(11 << 248); // tan(11) + // return atanh_alt_f258(1 << 252); // atanh(1/64) * 2^258 + // return atanh_f258(1 << 252, 18); // atanh(1/64) * 2^258 + // return atanh_f261(mulDivRound(64, 1 << 255, 55), 18); // atanh(1/55) * 2^261 + // return log2_aux_f256(1 << 255); + // return log2_aux_f256(-1 - (-1 << 256)); // log2(2-1/2^255))*2^256 ~ 2^256 - 1.43 + // return log_aux_f256(-1 - (-1 << 256)); + // return log_aux_f256(3); // log(3/2)*2^256 + // return fixed248_pow(3 << 248, 3 << 248); // 3^3 + // return fixed248_exp(fixed248_log(5 << 248) ~/ 7); // exp(log(5)/7) = 5^(1/7) + // return fixed248_log(Pi_const_f254() ~>> 6); // log(Pi) + // return atanh_alt_f258(1 << 255); // atanh(1/8) * 2^258 + // return atanh_f258(1 << 255, 37); // atanh(1/8) * 2^258 + // return atanh_f258(81877371507464127617551201542979628307507432471243237061821853600756754782485, 36); // atanh(sqrt(2)/8) * 2^258 + // return log_aux_f257(Pi_const_f254()); // log(Pi/4) + // return log_aux_f257(3 << 254); // log(3) + // return atanh_alt_f258(81877371507464127617551201542979628307507432471243237061821853600756754782485); // atanh(sqrt(2)/8) * 2^258 + // return fixed248_sincos(Pi_const_f254() ~/ (64 * 3)); // (sin,cos)(Pi/3) + // return fixed248_exp(3 << 248); // exp(3)*2^248 + // return fixed248_exp2((1 << 248) ~/ 5); // 2^(1/5)*2^248 + // return fixed248_pow(3 << 248, -3 << 247); // 3^(-1.5) + // return fixed248_pow(10 << 248, -70 << 248); // 10^(-70) + // return fixed248_pow(fixed248_Pi_const(), stackMoveToTop(3) << 248); // Pi^3 ~ 31.006, computed more precisely + // return fixed248_pow(fixed248_Pi_const(), fixed248_Pi_const()); // Pi^Pi, more precisely + // return fixed248_exp(fixed248_log(fixed248_Pi_const()) * 3); // Pi^3 ~ 31.006 + // return fixed248_exp(mulDivRound(fixed248_log(fixed248_Pi_const()), fixed248_Pi_const(), 1 << 248)); // Pi^Pi + // return fixed248_sin(fixed248_log(fixed248_exp(fixed248_Pi_const()))); // sin(log(e^Pi)) + // return expm1_f257(1 << 255); // (exp(1/4)-1)*2^256 + // return expm1_f257(-1 << 256); // (exp(-1/2)-1)*2^256 (argument out of range, will overflow) + // return expm1_f257(log2_const_f256()); // (exp(log(2)/2)-1)*2^256 + // return expm1_f257(- log2_const_f256()); // (exp(-log(2)/2)-1)*2^256 + // return tanh_f258(log2_const_f256(), 17); // tanh(log(2)/4)*2^258 + // return atan_f255(0xa0 << 247); + // return atan_f259(1 << 255, 26); // atan(1/16) + // return atan_f259(stackMoveToTop(2273) << 244, 26); // atan(2273/2^15) + // return atan_aux_f256(0xa0 << 248); + // return atan_aux_f256(-1 - (-1 << 256)); + // return atan_aux_f256(-1 << 256); + // return atan_aux_f256(1); // atan(1/2^256)*2^261 = 32 + //return fixed248_nrand(); + // return test_nrand(100000); + var One2: int = 1 << 255; + // return asin_f255(One); + // return asin_f255(-2 * One ~/ -3); + var arg: int = mulDivRound(12, One2, 17); // 12/17 + // return [ asin_slow_f255(arg), asin_f255(arg) ]; + // return [ acos_slow_f255(arg), acos_f255(arg) ]; + // return 4 * atan_f255(One ~/ 5) - atan_f255(One ~/ 239); // 4 * atan(1/5) - atan(1/239) = Pi/4 as fixed255 + var One3: int = 1 << 248; + // return fixed248_atan(One) ~/ 5); // atan(1/5) + // return fixed248_acot(One ~/ 239); // atan(1/5) +} + +/** + method_id | in | out +@testcase | 10000 | -1-(-1<<256) -1-(-1<<256) | 115792089237316195423570985008687907853269984665640564039457584007913129639935 +@testcase | 10000 | -1-(-1<<256) -2-(-1<<256) | 115792089237316195423570985008687907853269984665640564039457584007913129639934 +@testcase | 10000 | -1-(-1<<256) 1<<255 | 81877371507464127617551201542979628307507432471243237061821853600756754782485 +@testcase | 10000 | 1 2 | 1 +@testcase | 10000 | 1 3 | 2 +@testcase | 10000 | 3<<254 1<<254 | 50139445418395255283694704271811692336355250894665672355503583528635147053497 +@testcase | 10000 | 3 5 | 4 +@testcase | 10001 | 115641670674223639132965820642403718536242645001775371762318060545014644837101-1 | 115792089237316195423570985008687907853269984665640564039457584007913129639935 +@testcase | 10001 | 15<<252 | 108679485937549714997960660780289583146059954551846264494610741505469565211201 + +@testcase | 10002 | 1<<255 | 57858359242454268843682786479537198006144860419130642837770554273561536355094 28938600351875109040123440645416448095273333920390487381363947585666516031269 +@testcase | 10002 | 90942894222941581070058735694432465663348344332098107489693037779484723616546 | 90796875678616203090520439851979829600860326752181983760731669850687818036503 71369031536005973567205947792557760023823761636922618688720973932041901854510 +@testcase | 10002 | 115641670674223639132965820642403718536242645001775371762318060545014644837100 | 115341536360906404779899502576747487978354537254490211650198994186870666100480 115341536360906404779899502576747487978354537254490211650198994186870666100479 +@testcase | 10003 | 90942894222941581070058735694432465663348344332098107489693037779484723616546 0 | 81877371507464127617551201542979628307507432471243237061821853600756754782485 -81877371507464127617551201542979628307507432471243237061821853600756754782486 +@testcase | 10003 | (1<<255)+1 0 | 55513684748706254392157395574451324146997108788015526773113170656738693667657 -101617118319522600545601981648807607350213579319835970884288805016705398675944 +@testcase | 10003 | 1<<254 0 | 28647421327665059059430596260119789787021370826354543144805343654507971817712 -112192393597863122712065585177748900737784171216163716639418346853706594800924 +@testcase | 10003 | 15<<252 0 | 93337815620236900315136494926097782162348358704087992554326802765553037216157 -68526346066204767396483080633934170508153877799043171682610011603005473885083 +@testcase | 10004 | 15<<252 | 93337815620236900315136494926097782162348358704087992554326802765553037216158 94531486342222856054175808749507474690232213733194784713695144809815311509707 +@testcase | 10003 | 60628596148627720713372490462954977108898896221398738326462025186323149077698 0 | 57896044618658097711785492504343953926634992332820282019728792003956564819968 -100278890836790510567389408543623384672710501789331344711007167057270294106993 +@testcase | 10004 | 60628596148627720713372490462954977108898896221398738326462025186323149077698 | 57896044618658097711785492504343953926634992332820282019728792003956564819968 31026396801051369712363152930129046361118965752618438656900833901285671065886 +@testcase | 10005 | 1899<<245 | -115784979074977116522606932816046735344768048129666123117516779696532375620701 -86847621900007587791673148476644866514014227467564880140262768165345715058771 +@testcase | 10006 | 11<<248 | -102200470999497240398685962406597118965525125432278008915850368651878945159221 +@testcase | 10008 | 1<<252 18 | 7237594612640731814076778712183932891481921212865048737772958953246047977071 +@testcase! | 10009 | 64*(1<<255)//55 18 | 67377367986958444187782963285047188951340314639925508148698906136973510008513 +@testcase | 10010 | 1<<255 | 0 255 +@testcase | 10011 | -1-(-1<<256) | 80260960185991308862233904206310070533990667611589946606122867505419956976171 255 +@testcase | 10012 | 3<<248 3<<248 | 12212446911748192486079752325135052781399568695204278238536542063334587891712 +@testcase | 10013 | 5 7 | 569235245303856216139605450142923208167703167128528666640203654338408315932 +@testcase | 10014 | 1420982722233462204219667745225507275989817880189032929526453715304448806508 | 517776035526939558040896860590142614178014859368681705591403663865964112176 +@testcase | 10008 | 1<<255 37 | 58200445412255555045265806996802932280233368707362818578692888102488340124094 +@testcase | 10008 | 81877371507464127617551201542979628307507432471243237061821853600756754782485 36 | 82746618329032515754939514227666784789465120373484337368014239356561508382845 +@testcase | 10015 | 90942894222941581070058735694432465663348344332098107489693037779484723616546 | -55942510554172181731996424203087263676819062449594753161692794122306202470292 256 +@testcase | 10015 | 3<<254 | -66622616410625360568738677407433830899150908037353507097280251369610028875158 256 +@testcase | 10016 | 90942894222941581070058735694432465663348344332098107489693037779484723616546//(64*3) | 391714417331212931903864877123528846377775397614575565277371746317462086355 226156424291633194186662080095093570025917938800079226639565593765455331328 +@testcase | 10017 | 3<<248 | 9084946421051389814103830025729847734065792062362132089390904679466687950835 +@testcase | 10018 | (1<<248)//5 | 519571025111621076330285524602776985448579272766894385941850747946908706857 +@testcase | 10012 | 3<<248 -3<<247 | 87047648295825095978636639360784188083950088358794570061638165848324908079 +@testcase | 10012 | 10<<248 -70<<248 | 45231 +@testcase | 10012 | 1420982722233462204219667745225507275989817880189032929526453715304448806508 3<<248 | 14024537329227316173680050897643053638073167245065581681188087336877135047241 +@testcase | 10012 | 1420982722233462204219667745225507275989817880189032929526453715304448806508 1420982722233462204219667745225507275989817880189032929526453715304448806508 | 16492303277433924047657446877966346821161732581471802839855102123372676002295 +@testcase | 10019 | 1<<255 | 65775792789545756849501669218806308540691279864498696756901136302101823231959 +@testcase | 10019 | -1<<255 | -51226238931640701466578648374135745377468902266335737558089915608594425303282 + +@testcase | 10020 | 160<<247 | 32340690885082755723307749066376646841771751777398167772823878380310576779097 +@testcase | 10021 | 1<<255 26 | 57820835337111819566482910321201859268121322500887685881159030272507322418551 +@testcase | 10021 | 2273<<244 26 | 64153929153128256059565403901040178355488584937372975321150754259394300105908 +@testcase | 10022 | 160<<248 | 18 -13775317617017974742132028403521581424991093186766868001115299479309514610238 +@testcase | 10022 | -1-(-1<<256) | 25 16312150880916231694896252427912541090503675654570543195394548083530005073282 +@testcase | 10022 | -1<<256 | -25 -16312150880916231694896252427912541090503675654570543195394548083530005073298 +@testcase | 10022 | 1 | 0 32 + +@testcase | 10023 | 1<<255 | 90942894222941581070058735694432465663348344332098107489693037779484723616546 +@testcase | 10023 | (1-(1<<255))//-3 | 19675212872822715586637341573564384553677006914302429002469838095945333339604 +@testcase | 10023 | 12*(1<<255)//17 | 45371280744427205854111943101074857545572584208710061167826656461897302968384 +@testcase | 10024 | 12*(1<<255)//17 | 45371280744427205854111943101074857545572584208710061167826656461897302968387 +@testcase | 10025 | 12*(1<<255)//17 | 22785806739257187607973396296678804058887880061694023160933190658793710324081 +@testcase | 10026 | 12*(1<<255)//17 | 22785806739257187607973396296678804058887880061694023160933190658793710324080 + +@testcase | 10027 | (1<<248)//5 | 89284547973388213553327350968415123522888028497458323165947767504203347189 +@testcase | 10028 | (1<<248)//239 | 708598849781543798951441405045469962900811296151941404481049216461523216127 +*/ diff --git a/tolk-tester/tests/try-func.tolk b/tolk-tester/tests/try-func.tolk new file mode 100644 index 00000000..4ac86d96 --- /dev/null +++ b/tolk-tester/tests/try-func.tolk @@ -0,0 +1,323 @@ +fun foo(x: int): int { + try { + if (x == 7) { + throw 44; + } + return x; + } catch { + return 2; + } +} + +@inline +fun foo_inline(x: int): int { + try { + assert(!(x == 7)) throw 44; + return x; + } catch { + return 2; + } +} + +@inline_ref +fun foo_inlineref(x: int): int { + try { + if (x == 7) { throw (44, 2); } + return x; + } catch (_, arg) { + return arg as int; + } +} + +@method_id(101) +fun test(x: int, y: int, z: int): int { + y = foo(y); + return x * 100 + y * 10 + z; +} + +@method_id(102) +fun test_inline(x: int, y: int, z: int): int { + y = foo_inline(y); + return x * 100 + y * 10 + z; +} + +@method_id(103) +fun test_inlineref(x: int, y: int, z: int): int { + y = foo_inlineref(y); + return x * 100 + y * 10 + z; +} + +@inline +fun foo_inline_big( + x1: int, x2: int, x3: int, x4: int, x5: int, x6: int, x7: int, x8: int, x9: int, x10: int, + x11: int, x12: int, x13: int, x14: int, x15: int, x16: int, x17: int, x18: int, x19: int, x20: int +): int { + try { + if (x1 == 7) { + throw 44; + } + return x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + x10 + x11 + x12 + x13 + x14 + x15 + x16 + x17 + x18 + x19 + x20; + } catch { + return 1; + } +} + +@method_id(104) +fun test_inline_big(x: int, y: int, z: int): int { + y = foo_inline_big( + y, y + 1, y + 2, y + 3, y + 4, y + 5, y + 6, y + 7, y + 8, y + 9, + y + 10, y + 11, y + 12, y + 13, y + 14, y + 15, y + 16, y + 17, y + 18, y + 19); + return x * 1000000 + y * 1000 + z; +} + +fun foo_big( + x1: int, x2: int, x3: int, x4: int, x5: int, x6: int, x7: int, x8: int, x9: int, x10: int, + x11: int, x12: int, x13: int, x14: int, x15: int, x16: int, x17: int, x18: int, x19: int, x20: int +): int { + try { + if (x1 == 7) { + throw (44, 1); + } + return x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + x10 + x11 + x12 + x13 + x14 + x15 + x16 + x17 + x18 + x19 + x20; + } catch (code, arg) { + return arg as int; + } +} + +@method_id(105) +fun test_big(x: int, y: int, z: int): int { + y = foo_big( + y, y + 1, y + 2, y + 3, y + 4, y + 5, y + 6, y + 7, y + 8, y + 9, + y + 10, y + 11, y + 12, y + 13, y + 14, y + 15, y + 16, y + 17, y + 18, y + 19); + return x * 1000000 + y * 1000 + z; +} + +@method_id(106) +fun test_catch_into_same(x: int): int { + var code = x; + try { + assert(x <= 10, 44); + } catch(code) { + return code; + } + return code; +} + + +@method_id(107) +fun test_catch_into_same_2(x: int): int { + var code = x; + try { + if (x > 10) { + throw 44; + } + } catch(code) { + } + return code; +} + +global after046: int; + +// this bug existed in FunC and is fixed in v0.4.6 +fun bug_046_internal(op: int) { + if (op == 1) { + return; + } else if (op == 2) { + return; + } else { + throw 1; + } +} + +fun bug_046_called() { + after046 = 0; + try { + bug_046_internal(1337); + after046 = 1; // shouldn't be called + } catch(n) { + return; + } + return; +} + +@method_id(108) +fun bug_046_entrypoint() { + bug_046_called(); + return after046; +} + +global g_reg: int; + +@method_id(109) +fun test109(): (int, int) { + var l_reg = 10; + g_reg = 10; + try { + // note, that regardless of assignment, an exception RESTORES them to previous (to 10) + // it's very unexpected, but is considered to be a TVM feature, not a bug + g_reg = 999; + l_reg = 999; + bug_046_internal(999); // throws + } catch { + } + // returns (10,10) because of an exception, see a comment above + return (g_reg, l_reg); +} + +fun alwaysThrow123(): never { + throw 123; +} + +fun alwaysThrowX(x: int): never { + if (x > 10) { throw (x, beginCell()); } + else { throw (x, null); } +} + +fun anotherNever(throw123: bool): never { + if (throw123) { alwaysThrow123(); } + alwaysThrowX(456); +} + +fun testCodegen1(x: int) { + if (x > 10) { + throw 123; + anotherNever(true); // unreachable, will be dropped + } + else if (x < 10) { + throw x; + return -123; // unreachable, will be dropped + } + return 0; +} + +fun testCodegen2(x: int) { + if (x > 10) { + alwaysThrow123(); + anotherNever(true); // unreachable, will be dropped + } + else if (x < 10) { + anotherNever(false); + return -123; // unreachable, will be dropped + } + return 0; +} + +@method_id(110) +fun test110(b: bool) { + try { + if (b == true) { testCodegen1(100); } + testCodegen1(5); + return -1; + } catch (ex) { + return ex; + } +} + +@method_id(111) +fun test111(b: bool) { + try { + if (b == true) { testCodegen2(100); } + testCodegen2(5); + return -1; + } catch (ex) { + return ex; + } +} + +fun mySetCode(newCode: slice): void + asm "SETCODE"; + +fun testCodegen3(numberId: int, paramVal: cell) { + if (numberId == -1000) { + var cs = paramVal.beginParse(); + mySetCode(cs); + throw 0; + } + paramVal.beginParse(); +} + +fun main() { +} + +/** + method_id | in | out +@testcase | 101 | 1 2 3 | 123 +@testcase | 101 | 3 8 9 | 389 +@testcase | 101 | 3 7 9 | 329 +@testcase | 102 | 1 2 3 | 123 +@testcase | 102 | 3 8 9 | 389 +@testcase | 102 | 3 7 9 | 329 +@testcase | 103 | 1 2 3 | 123 +@testcase | 103 | 3 8 9 | 389 +@testcase | 103 | 3 7 9 | 329 +@testcase | 104 | 4 8 9 | 4350009 +@testcase | 104 | 4 7 9 | 4001009 +@testcase | 105 | 4 8 9 | 4350009 +@testcase | 105 | 4 7 9 | 4001009 +@testcase | 106 | 5 | 5 +@testcase | 106 | 20 | 44 +@testcase | 107 | 5 | 5 +@testcase | 107 | 20 | 20 +@testcase | 108 | | 0 +@testcase | 109 | | 10 10 +@testcase | 110 | -1 | 123 +@testcase | 110 | 0 | 5 +@testcase | 111 | -1 | 123 +@testcase | 111 | 0 | 456 + +@code_hash 57361460846265694653029920796509802052573595128418810728101968091567195330515 + +@fif_codegen +""" + testCodegen1 PROC:<{ + // x + DUP // x x + 10 GTINT // x '2 + IFJMP:<{ // x + 123 THROW + }> // x + DUP // x x + 10 LESSINT // x '6 + IFJMP:<{ // x + THROWANY + }> // x + DROP // + 0 PUSHINT // '8=0 + }> +""" + +@fif_codegen +""" + testCodegen2 PROC:<{ + // x + DUP // x x + 10 GTINT // x '2 + IFJMP:<{ // x + DROP // + alwaysThrow123 CALLDICT + }> // x + 10 LESSINT // '5 + IFJMP:<{ // + FALSE // '6 + anotherNever CALLDICT + }> // + 0 PUSHINT // '8=0 + }> +""" + +@fif_codegen +""" + testCodegen3 PROC:<{ + // numberId paramVal + SWAP + -1000 PUSHINT // paramVal numberId '2=-1000 + EQUAL // paramVal '3 + IFJMP:<{ // paramVal + CTOS // cs + SETCODE + 0 THROW + }> // paramVal + DROP // + }> +""" +*/ diff --git a/tolk-tester/tests/unbalanced_ret.tolk b/tolk-tester/tests/unbalanced_ret.tolk new file mode 100644 index 00000000..6cf42643 --- /dev/null +++ b/tolk-tester/tests/unbalanced_ret.tolk @@ -0,0 +1,17 @@ +fun main(x: int): (int, int) { + var y: int = 5; + if (x < 0) { + x *= 2; + y += 1; + if (x == -10) { + return (111, 0); + } + } + return (x + 1, y); +} +/** + method_id | in | out +@testcase | 0 | 10 | 11 5 +@testcase | 0 | -5 | 111 0 +@testcase | 0 | -4 | -7 6 +*/ diff --git a/tolk-tester/tests/unbalanced_ret_inline.tolk b/tolk-tester/tests/unbalanced_ret_inline.tolk new file mode 100644 index 00000000..4e24fbd8 --- /dev/null +++ b/tolk-tester/tests/unbalanced_ret_inline.tolk @@ -0,0 +1,19 @@ +@inline +fun foo(x: int): int { + if (x < 0) { + x *= 2; + if (x == -10) { + return 111; + } + } + return x + 1; +} +fun main(x: int): int { + return foo(x) * 10; +} +/** + method_id | in | out +@testcase | 0 | 10 | 110 +@testcase | 0 | -5 | 1110 +@testcase | 0 | -4 | -70 +*/ diff --git a/tolk-tester/tests/unbalanced_ret_loops.tolk b/tolk-tester/tests/unbalanced_ret_loops.tolk new file mode 100644 index 00000000..292b48da --- /dev/null +++ b/tolk-tester/tests/unbalanced_ret_loops.tolk @@ -0,0 +1,68 @@ +fun main() { } + +@method_id(1) +fun foo_repeat(x: int): int { + repeat(10) { + x += 10; + if (x >= 100) { + return x; + } + } + return -1; +} + +@method_id(2) +fun foo_while(x: int): int { + var i: int = 0; + while (i < 10) { + x += 10; + if (x >= 100) { + return x; + } + i += 1; + } + return -1; +} + +@method_id(3) +fun foo_until(x: int): int { + var i: int = 0; + do { + x += 10; + if (x >= 100) { + return x; + } + i += 1; + } while (i < 10); + return -1; +} + +@method_id(4) +fun test4(x: int): (int, bool) { + var s = 0; + var reached = false; + do { + x = x - 1; + s = s + 1; + if (x < 10) { + reached = true; + } + } while (!reached); + return (s, reached); +} + +/** + method_id | in | out +@testcase | 1 | 40 | 100 +@testcase | 1 | 33 | 103 +@testcase | 1 | -5 | -1 +@testcase | 2 | 40 | 100 +@testcase | 2 | 33 | 103 +@testcase | 2 | -5 | -1 +@testcase | 3 | 40 | 100 +@testcase | 3 | 33 | 103 +@testcase | 3 | -5 | -1 +@testcase | 4 | 18 | 9 -1 + +@code_hash 12359153928622198176298534554187062238616102949658930329300859312625793323482 +*/ diff --git a/tolk-tester/tests/unbalanced_ret_nested.tolk b/tolk-tester/tests/unbalanced_ret_nested.tolk new file mode 100644 index 00000000..05e60924 --- /dev/null +++ b/tolk-tester/tests/unbalanced_ret_nested.tolk @@ -0,0 +1,37 @@ +fun foo(y: int): int { + if (y < 0) { + y *= 2; + if (y == -10) { + return 111; + } + } + return y + 1; +} +fun bar(x: int, y: int): (int, int) { + if (x < 0) { + y = foo(y); + x *= 2; + if (x == -10) { + return (111, y); + } + } + return (x + 1, y); +} +fun main(x: int, y: int): (int, int) { + (x, y) = bar(x, y); + return (x, y * 10); +} +/** + method_id | in | out +@testcase | 0 | 3 3 | 4 30 +@testcase | 0 | 3 -5 | 4 -50 +@testcase | 0 | 3 -4 | 4 -40 +@testcase | 0 | -5 3 | 111 40 +@testcase | 0 | -5 -5 | 111 1110 +@testcase | 0 | -5 -4 | 111 -70 +@testcase | 0 | -4 3 | -7 40 +@testcase | 0 | -4 -5 | -7 1110 +@testcase | 0 | -4 -4 | -7 -70 + +@code_hash 68625253347714662162648433047986779710161195298061582217368558479961252943991 +*/ diff --git a/tolk-tester/tests/unreachable-1.tolk b/tolk-tester/tests/unreachable-1.tolk new file mode 100644 index 00000000..5b3cb1b0 --- /dev/null +++ b/tolk-tester/tests/unreachable-1.tolk @@ -0,0 +1,14 @@ +fun main(x: int) { + if (x) { + x = 10;;;;; + return x;;; + x = 20; + } + return -1; +} + +/** +@testcase | 0 | 1 | 10 +@stderr warning: unreachable code +@stderr x = 20; + */ diff --git a/tolk-tester/tests/unreachable-2.tolk b/tolk-tester/tests/unreachable-2.tolk new file mode 100644 index 00000000..aeadd8c6 --- /dev/null +++ b/tolk-tester/tests/unreachable-2.tolk @@ -0,0 +1,22 @@ +fun main(x: int) { + if (x) { + if (x > 10) { + return 1; // throw 1; + } else if (true) { + return -1; + } else { + return 2; // throw 2; + } + } else { + {{return 1;} + x = 30;} + } + assert(false, 10); +} + +/** +@testcase | 0 | 1 | -1 +@stderr warning: unreachable code +@stderr assert(false, 10) +@stderr x = 30 + */ diff --git a/tolk-tester/tests/unreachable-3.tolk b/tolk-tester/tests/unreachable-3.tolk new file mode 100644 index 00000000..fab21fd2 --- /dev/null +++ b/tolk-tester/tests/unreachable-3.tolk @@ -0,0 +1,22 @@ +fun main(x: int?) { + if (x != null && x == null) { + return 1 + 2; + } + if (x == null) { + return -1; + } + if (x != null) { + return -2; + } + return 3 + 4; +} + +/** +@testcase | 0 | 5 | -2 +@testcase | 0 | null | -1 + +@stderr warning: variable `x` of type `int` is always not null +@stderr if (x != null) +@stderr warning: unreachable code +@stderr return 3 + 4 + */ diff --git a/tolk-tester/tests/unreachable-4.tolk b/tolk-tester/tests/unreachable-4.tolk new file mode 100644 index 00000000..6b25b3d9 --- /dev/null +++ b/tolk-tester/tests/unreachable-4.tolk @@ -0,0 +1,24 @@ +fun alwaysThrows(): never { + throw 456; +} + +fun testUnreachable(x: int) { + if (x) { throw 123; } + else { alwaysThrows(); } + return 1; +} + +fun main() { + try { + testUnreachable(100); + throw 80; + } catch (excNo) { + return excNo; + } +} + +/** +@testcase | 0 | | 123 +@stderr warning: unreachable code +@stderr return 1; + */ diff --git a/tolk-tester/tests/use-before-declare.tolk b/tolk-tester/tests/use-before-declare.tolk new file mode 100644 index 00000000..2a0e0e7f --- /dev/null +++ b/tolk-tester/tests/use-before-declare.tolk @@ -0,0 +1,49 @@ +fun main(): int { + var c: cell = my_begin_cell().storeInt(demo_10, 32).my_end_cell(); + var cs: slice = my_begin_parse(c); + var ten: int = cs.loadInt(32); + return 1 + demo1(ten) + demo_var; +} + +@pure +fun my_begin_cell(): builder +asm "NEWC"; +@pure +fun my_end_cell(b: builder): cell +asm "ENDC"; +@pure +fun my_begin_parse(c: cell): slice +asm "CTOS"; + +fun demo1(v: int): int { + demo_var = 23; + return v; +} + +global demo_var: int; +const demo_10: int = 10; + +fun test1(): int { + var demo_var: int = demo_10; + var demo_slice: int = demo_20; + if (demo_var > 0) { + var demo_var: tuple? = null; + var demo_slice: tuple? = null; + } + return demo_var + demo_slice; +} + +global demo_slice: slice; +const demo_20: int = 20; + +/** +@testcase | 0 | | 34 + +@fif_codegen +""" + test1 PROC:<{ + // + 30 PUSHINT // '10 + }> +""" + */ diff --git a/tolk-tester/tests/var-apply.tolk b/tolk-tester/tests/var-apply.tolk new file mode 100644 index 00000000..d189430f --- /dev/null +++ b/tolk-tester/tests/var-apply.tolk @@ -0,0 +1,192 @@ +fun getBeginCell() { + return beginCell; +} + +fun getBeginParse() { + return beginParse; +} + +@method_id(101) +fun testVarApply1() { + var (_, f_end_cell) = (0, endCell); + var b: builder = (getBeginCell())().storeInt(1, 32); + b.storeInt(2, 32); + var s = (getBeginParse())(f_end_cell(b)); + return (s.loadInt(32), s.loadInt(32)); +} + +@inline +fun my_throw_always() { + throw 1000; +} + +@inline +fun get_raiser() { + return my_throw_always; +} + +@method_id(102) +fun testVarApplyWithoutSavingResult() { + try { + var raiser = get_raiser(); + raiser(); // `some_var()` is always impure, the compiler has no considerations about its runtime value + return 0; + } catch (code) { + return code; + } +} + +@inline +fun sum(a: int, b: int) { + assert(a + b < 24, 1000); + return a + b; +} + +@inline +fun mul(a: int, b: int) { + assert(a * b < 24, 1001); + return a * b; +} + +fun demo_handler(op: int, query_id: int, a: int, b: int): int { + if (op == 0xF2) { + val func = query_id % 2 == 0 ? sum : mul; + val result = func(a, b); + return 0; // result not used, we test that func is nevertheless called + } + if (op == 0xF4) { + val func = query_id % 2 == 0 ? sum : mul; + val result = func(a, b); + return result; + } + return -1; +} + +@method_id(103) +fun testVarApplyInTernary() { + var t: tuple = createEmptyTuple(); + try { + t.tuplePush(demo_handler(0xF2, 122, 100, 200)); + } catch(code) { + t.tuplePush(code); + } + try { + t.tuplePush(demo_handler(0xF4, 122, 100, 200)); + } catch(code) { + t.tuplePush(code); + } + try { + t.tuplePush(demo_handler(0xF2, 122, 10, 10)); + } catch(code) { + t.tuplePush(code); + } + try { + t.tuplePush(demo_handler(0xF2, 123, 10, 10)); + } catch(code) { + t.tuplePush(code); + } + return t; +} + +fun always_throw2(x: int) { + throw 239 + x; +} + +global global_f: int -> void; + +@method_id(104) +fun testGlobalVarApply() { + try { + global_f = always_throw2; + global_f(1); + return 0; + } catch (code) { + return code; + } +} + +@method_id(105) +fun testVarApply2() { + var creator = createEmptyTuple; + var t = creator(); + t.tuplePush(1); + var sizer = t.tupleSize; + return sizer(t); +} + +fun getTupleLastGetter(): (tuple) -> X { + return tupleLast; +} + +@method_id(106) +fun testVarApply3() { + var t = createEmptyTuple(); + t.tuplePush(1); + t.tuplePush([2]); + var getIntAt = t.tupleAt; + var getTupleFirstInt = createEmptyTuple().tupleFirst; + var getTupleLastTuple = getTupleLastGetter(); + return (getIntAt(t, 0), getTupleFirstInt(t), getTupleLastTuple(t), getTupleLastGetter()(t)); +} + +@method_id(107) +fun testIndexedAccessApply() { + var functions1 = (beginCell, endCell); + var functions2 = [beginParse]; + var b = functions1.0().storeInt(1, 16); + b.storeInt(1, 16); + return functions2.0(functions1.1(b)).loadInt(32); +} + +fun getNullable4(): int? { return 4; } +fun myBeginCell(): builder? asm "NEWC"; + +@method_id(108) +fun testCallingNotNull() { + var n4: () -> int? = getNullable4; + var creator: (() -> builder?)? = myBeginCell; + var end2: [int, (builder -> cell)?] = [0, endCell]; + var c: cell = end2.1!((creator!()!)!.storeInt(getNullable4()!, 32)); + return c.beginParse().loadInt(32); +} + +fun sumOfTensorIfNotNull(t: (int, int)?) { + if (t == null) { return 0; } + return t!.0 + t!.1; +} + +@method_id(109) +fun testTypeTransitionOfVarCall() { + var summer = sumOfTensorIfNotNull; + var hh1 = [1, null]; + var tt1 = (3, 4); + return (summer(null), summer((1,2)), summer(hh1.1), summer(tt1)); +} + +fun makeTensor(x1: int, x2: int, x3: int, x4: int, x5: int) { + return (x1, x2, x3, x4, x5); +} + +fun eq(x: T): T { return x; } + +@method_id(110) +fun testVarsModificationInsideVarCall(x: int) { + var cb = makeTensor; + return x > 3 ? cb(x, x += 5, eq(x *= x), x, eq(x)) : null; +} + +fun main() {} + +/** +@testcase | 101 | | 1 2 +@testcase | 102 | | 1000 +@testcase | 103 | | [ 1000 1000 0 1001 ] +@testcase | 104 | | 240 +@testcase | 105 | | 1 +@testcase | 106 | | 1 1 [ 2 ] [ 2 ] +@testcase | 107 | | 65537 +@testcase | 108 | | 4 +@testcase | 109 | | 0 3 0 7 +@testcase | 110 | 5 | 5 10 100 100 100 -1 +@testcase | 110 | 0 | (null) (null) (null) (null) (null) 0 + */ diff --git a/tolk-tester/tests/w1.tolk b/tolk-tester/tests/w1.tolk new file mode 100644 index 00000000..eb06bec6 --- /dev/null +++ b/tolk-tester/tests/w1.tolk @@ -0,0 +1,14 @@ +fun main(id: int): (int, int) { + if (id > 0) { + if (id > 10) { + return (2 * id, 3 * id); + } + } + return (5, 6); +} +/** + method_id | in | out +@testcase | 0 | 0 | 5 6 +@testcase | 0 | 4 | 5 6 +@testcase | 0 | 11 | 22 33 +*/ diff --git a/tolk-tester/tests/w2.tolk b/tolk-tester/tests/w2.tolk new file mode 100644 index 00000000..728b18d3 --- /dev/null +++ b/tolk-tester/tests/w2.tolk @@ -0,0 +1,34 @@ +@method_id(101) +fun test1(cs: slice) { + return cs.loadUint(8)+cs.loadUint(8)+cs.loadUint(8)+cs.loadUint(8); +} + +@method_id(102) +fun test2(cs: slice) { + var (x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, + x11, x12, x13, x14, x15, x16, x17, x18, x19) = f(cs); + return x0 + x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + + x10+ x11+ x12+ x13+ x14+ x15+ x16+ x17+ x18+ x19; +} + +fun main(cs: slice) { + return (cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8)); +} + +fun f(cs: slice) { + return (cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), + cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), + cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), + cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), + cs.loadUint(8), cs.loadUint(8), cs.loadUint(8), cs.loadUint(8)); +} + + +/** + method_id | in | out +@testcase | 102 | x{000102030405060708090a0b0c0d0e0f10111213} | 190 +@testcase | 101 | x{000102030405060708090a0b0c0d0e0f10111213} | 6 +@testcase | 0 | x{000102030405060708090a0b0c0d0e0f10111213} | 0 1 2 3 + +@code_hash 58474889199998908444151060994149070836199913191952040273624197630531731101157 +*/ diff --git a/tolk-tester/tests/w6.tolk b/tolk-tester/tests/w6.tolk new file mode 100644 index 00000000..489ffa8c --- /dev/null +++ b/tolk-tester/tests/w6.tolk @@ -0,0 +1,19 @@ +fun main(x: int): int { + var i: int = 0; + // int f = false; + do { + i = i + 1; + if (i > 5) { + return 1; + } + var f: bool = (i * i == 64); + } while (!f); + return -1; +} + +/** + method_id | in | out +@testcase | 0 | 0 | 1 + +@code_hash 36599880583276393028571473830850694081778552118303309411432666239740650614479 +*/ diff --git a/tolk-tester/tests/w7.tolk b/tolk-tester/tests/w7.tolk new file mode 100644 index 00000000..3d68c775 --- /dev/null +++ b/tolk-tester/tests/w7.tolk @@ -0,0 +1,26 @@ +@method_id(1) +fun test(y: int): int { + var x: int = 1; + if (y > 0) { + return 1; + } + return x > 0 ? -1 : 0; +} + +@method_id(2) +fun f(y: int): int { + if (y > 0) { + return 1; + } + return 2; +} + +fun main() { } + +/** + method_id | in | out +@testcase | 1 | 10 | 1 +@testcase | 1 | -5 | -1 +@testcase | 2 | 10 | 1 +@testcase | 2 | -5 | 2 +*/ diff --git a/tolk-tester/tests/w9.tolk b/tolk-tester/tests/w9.tolk new file mode 100644 index 00000000..b88dc736 --- /dev/null +++ b/tolk-tester/tests/w9.tolk @@ -0,0 +1,14 @@ +fun main(s: int) { + var (z, t) = (17, s); + while (z > 0) { + t = s; + z -= 1; + } + return ~ t; +} + +/** + method_id | in | out +@testcase | 0 | 1 | -2 +@testcase | 0 | 5 | -6 +*/ diff --git a/tolk-tester/tests/warnings-1.tolk b/tolk-tester/tests/warnings-1.tolk new file mode 100644 index 00000000..040057d1 --- /dev/null +++ b/tolk-tester/tests/warnings-1.tolk @@ -0,0 +1,28 @@ +fun getNullableInt(): int? { return null; } + +fun main() { + var c: int? = 6; + __expect_type(c, "int"); + if (c == null) {} + + var d: int? = c; + if (((d)) != null && tupleSize(createEmptyTuple())) {} + + var e: int? = getNullableInt(); + if (e != null) { + return true; + } + __expect_type(e, "null"); + null == e; + + return null != null; +} + +/** +@testcase | 0 | | 0 + +@stderr warning: variable `c` of type `int` is always not null, this condition is always false +@stderr warning: variable `d` of type `int` is always not null, this condition is always true +@stderr warning: variable `e` is always null, this condition is always true +@stderr warning: expression is always null, this condition is always false + */ diff --git a/tolk-tester/tests/warnings-2.tolk b/tolk-tester/tests/warnings-2.tolk new file mode 100644 index 00000000..57ecb21a --- /dev/null +++ b/tolk-tester/tests/warnings-2.tolk @@ -0,0 +1,26 @@ +fun main() { + var (a, b, c, d, e) = (1, beginCell(), beginCell().endCell().beginParse(), [1], true as bool?); + + var alwaysInt = a != null ? 1 : null; + __expect_type(alwaysInt, "int"); + + if (!(c == null)) { + if (10 < 3) { assert(b == null, 100); } + } + while (d == null || false) {} + + return e! != null; +} + +/** +@testcase | 0 | | -1 + +@stderr warning: variable `a` of type `int` is always not null, this condition is always true +@stderr warning: condition of ternary operator is always true +@stderr warning: variable `c` of type `slice` is always not null, this condition is always false +@stderr warning: condition of `if` is always true +@stderr warning: variable `b` of type `builder` is always not null, this condition is always false +@stderr warning: condition of `assert` is always false +@stderr warning: condition of `while` is always false +@stderr warning: expression of type `bool` is always not null, this condition is always true + */ diff --git a/tolk-tester/tolk-tester.js b/tolk-tester/tolk-tester.js new file mode 100644 index 00000000..c7e71021 --- /dev/null +++ b/tolk-tester/tolk-tester.js @@ -0,0 +1,533 @@ +// Usage: `node tolk-tester.js tests_dir` OR `node tolk-tester.js test_file.tolk` +// from current dir, providing some env (see getenv() calls). +// This is a JS version of tolk-tester.py to test Tolk compiled to WASM. +// Don't forget to keep it identical to Python version! + +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const child_process = require('child_process'); + +function print(...args) { + console.log(...args) +} + +/** @return {string} */ +function getenv(name, def = null) { + if (name in process.env) + return process.env[name] + if (def === null) { + print(`Environment variable ${name} is not set`) + process.exit(1) + } + return def +} + +const TOLKFIFTLIB_MODULE = getenv('TOLKFIFTLIB_MODULE') +const TOLKFIFTLIB_WASM = getenv('TOLKFIFTLIB_WASM') +const FIFT_EXECUTABLE = getenv('FIFT_EXECUTABLE') +const FIFT_LIBS_FOLDER = getenv('FIFTPATH') // this env is needed for fift to work properly +const STDLIB_FOLDER = __dirname + '/../crypto/smartcont/tolk-stdlib' +const TMP_DIR = os.tmpdir() + +class CmdLineOptions { + constructor(/**string[]*/ argv) { + if (argv.length !== 3) { + print("Usage: node tolk-tester.js tests_dir OR node tolk-tester.js test_file.tolk") + process.exit(1) + } + if (!fs.existsSync(argv[2])) { + print(`Input '${argv[2]}' doesn't exist`) + process.exit(1) + } + + if (fs.lstatSync(argv[2]).isDirectory()) { + this.tests_dir = argv[2] + this.test_file = null + } else { + this.tests_dir = path.dirname(argv[2]) + this.test_file = argv[2] + } + } + + /** @return {string[]} */ + find_tests() { + if (this.test_file) // an option to run (debug) a single test + return [this.test_file] + + let tests = fs.readdirSync(this.tests_dir).filter(f => f.endsWith('.tolk') || f.endsWith('.ton')) + tests.sort() + return tests.map(f => path.join(this.tests_dir, f)) + } +} + + +class ParseInputError extends Error { +} + +class TolkCompilationFailedError extends Error { + constructor(/**string*/ message, /**string*/ stderr) { + super(message); + this.stderr = stderr + } +} + +class TolkCompilationSucceededError extends Error { +} + +class FiftExecutionFailedError extends Error { + constructor(/**string*/ message, /**string*/ stderr) { + super(message); + this.stderr = stderr + } +} + +class CompareOutputError extends Error { + constructor(/**string*/ message, /**string*/ output) { + super(message); + this.output = output + } +} + +class CompareFifCodegenError extends Error { +} + +class CompareCodeHashError extends Error { +} + + +/* + * In positive tests, there are several testcases "input X should produce output Y". + */ +class TolkTestCaseInputOutput { + static reJustNumber = /^[-+]?\d+$/ + static reMathExpr = /^[0x123456789()+\-*/<>]*$/ + + constructor(/**string*/ method_id_str, /**string*/ input_str, /**string*/ output_str) { + let processed_inputs = [] + for (let in_arg of input_str.split(' ')) { + if (in_arg.length === 0) + continue + else if (in_arg.startsWith("x{") || TolkTestCaseInputOutput.reJustNumber.test(in_arg)) + processed_inputs.push(in_arg) + else if (TolkTestCaseInputOutput.reMathExpr.test(in_arg)) + // replace "3<<254" with "3n<<254n" (big number) before eval (in Python we don't need this) + processed_inputs.push(eval(in_arg.replace('//', '/').replace(/(\d)($|\D)/gmi, '$1n$2')).toString()) + else if (in_arg === "null") + processed_inputs.push("null") + else + throw new ParseInputError(`'${in_arg}' can't be evaluated`) + } + + this.method_id = +method_id_str + this.input = processed_inputs.join(' ') + this.expected_output = output_str + } + + check(/**string[]*/ stdout_lines, /**number*/ line_idx) { + if (stdout_lines[line_idx] !== this.expected_output) + throw new CompareOutputError(`error on case #${line_idx + 1} (${this.method_id} | ${this.input}): expected '${this.expected_output}', found '${stdout_lines[line_idx]}'`, stdout_lines.join("\n")) + } +} + +/* + * @stderr checks, when compilation fails, that stderr (compilation error) is expected. + * If it's multiline, all lines must be present in specified order. + */ +class TolkTestCaseStderr { + constructor(/**string[]*/ stderr_pattern, /**boolean*/ avoid) { + this.stderr_pattern = stderr_pattern + this.avoid = avoid + } + + check(/**string*/ stderr) { + const line_match = this.find_pattern_in_stderr(stderr.split(/\n/)) + if (line_match === -1 && !this.avoid) + throw new CompareOutputError("pattern not found in stderr:\n" + + this.stderr_pattern.map(x => " " + x).join("\n"), stderr) + else if (line_match !== -1 && this.avoid) + throw new CompareOutputError(`pattern found (line ${line_match + 1}), but not expected to be:\n` + + this.stderr_pattern.map(x => " " + x).join("\n"), stderr) + } + + find_pattern_in_stderr(/**string[]*/ stderr) { + for (let line_start = 0; line_start < stderr.length; ++line_start) + if (this.try_match_pattern(0, stderr, line_start)) + return line_start + return -1 + } + + try_match_pattern(/**number*/ pattern_offset, /**string[]*/ stderr, /**number*/ offset) { + if (pattern_offset >= this.stderr_pattern.length) + return true + if (offset >= stderr.length) + return false + + const line_pattern = this.stderr_pattern[pattern_offset] + const line_output = stderr[offset] + return line_output.includes(line_pattern) && this.try_match_pattern(pattern_offset + 1, stderr, offset + 1) + } +} + +/* + * @fif_codegen checks that contents of compiled.fif matches the expected pattern. + * @fif_codegen_avoid checks that is does not match the pattern. + * See comments in run_tests.py. + */ +class TolkTestCaseFifCodegen { + constructor(/**string[]*/ fif_pattern, /**boolean*/ avoid) { + /** @type {string[]} */ + this.fif_pattern = fif_pattern.map(s => s.trim()) + this.avoid = avoid + } + + check(/**string[]*/ fif_output) { + const line_match = this.find_pattern_in_fif_output(fif_output) + if (line_match === -1 && !this.avoid) + throw new CompareFifCodegenError("pattern not found:\n" + + this.fif_pattern.map(x => " " + x).join("\n")) + else if (line_match !== -1 && this.avoid) + throw new CompareFifCodegenError(`pattern found (line ${line_match + 1}), but not expected to be:\n` + + this.fif_pattern.map(x => " " + x).join("\n")) + } + + find_pattern_in_fif_output(/**string[]*/ fif_output) { + for (let line_start = 0; line_start < fif_output.length; ++line_start) + if (this.try_match_pattern(0, fif_output, line_start)) + return line_start + return -1 + } + + try_match_pattern(/**number*/ pattern_offset, /**string[]*/ fif_output, /**number*/ offset) { + if (pattern_offset >= this.fif_pattern.length) + return true + if (offset >= fif_output.length) + return false + const line_pattern = this.fif_pattern[pattern_offset] + const line_output = fif_output[offset] + + if (line_pattern !== "...") { + if (!TolkTestCaseFifCodegen.does_line_match(line_pattern, line_output)) + return false + return this.try_match_pattern(pattern_offset + 1, fif_output, offset + 1) + } + while (offset < fif_output.length) { + if (this.try_match_pattern(pattern_offset + 1, fif_output, offset)) + return true + offset = offset + 1 + } + return false + } + + static split_line_to_cmd_and_comment(/**string*/ trimmed_line) { + const pos = trimmed_line.indexOf("//") + if (pos === -1) + return [trimmed_line, null] + else + return [trimmed_line.substring(0, pos).trimEnd(), trimmed_line.substring(pos + 2).trimStart()] + } + + static does_line_match(/**string*/ line_pattern, /**string*/ line_output) { + const [cmd_pattern, comment_pattern] = TolkTestCaseFifCodegen.split_line_to_cmd_and_comment(line_pattern) + const [cmd_output, comment_output] = TolkTestCaseFifCodegen.split_line_to_cmd_and_comment(line_output.trim()) + return cmd_pattern === cmd_output && (comment_pattern === null || comment_pattern === comment_output) + } +} + +/* + * @code_hash checks that hash of compiled output.fif matches the provided value. + * It's used to "record" code boc hash and to check that it remains the same on compiler modifications. + * Being much less flexible than @fif_codegen, it nevertheless gives a guarantee of bytecode stability. + */ +class TolkTestCaseExpectedHash { + constructor(/**string*/ expected_hash) { + this.code_hash = expected_hash + } + + check(/**string*/ fif_code_hash) { + if (this.code_hash !== fif_code_hash) + throw new CompareCodeHashError(`expected ${this.code_hash}, actual ${fif_code_hash}`) + } +} + + +class TolkTestFile { + constructor(/**string*/ tolk_filename, /**string*/ artifacts_folder) { + this.line_idx = 0 + this.tolk_filename = tolk_filename + this.artifacts_folder = artifacts_folder + this.compilation_should_fail = false + /** @type {TolkTestCaseStderr[]} */ + this.stderr_includes = [] + /** @type {TolkTestCaseInputOutput[]} */ + this.input_output = [] + /** @type {TolkTestCaseFifCodegen[]} */ + this.fif_codegen = [] + /** @type {TolkTestCaseExpectedHash | null} */ + this.expected_hash = null + /** @type {string | null} */ + this.experimental_options = null + } + + parse_input_from_tolk_file() { + const lines = fs.readFileSync(this.tolk_filename, 'utf-8').split(/\r?\n/) + this.line_idx = 0 + + while (this.line_idx < lines.length) { + const line = lines[this.line_idx] + if (line.startsWith('@testcase')) { + let s = line.split("|").map(p => p.trim()) + if (s.length !== 4) + throw new ParseInputError(`incorrect format of @testcase: ${line}`) + this.input_output.push(new TolkTestCaseInputOutput(s[1], s[2], s[3])) + } else if (line.startsWith('@compilation_should_fail')) { + this.compilation_should_fail = true + } else if (line.startsWith('@stderr')) { + this.stderr_includes.push(new TolkTestCaseStderr(this.parse_string_value(lines), false)) + } else if (line.startsWith("@fif_codegen_avoid")) { + this.fif_codegen.push(new TolkTestCaseFifCodegen(this.parse_string_value(lines), true)) + } else if (line.startsWith("@fif_codegen")) { + this.fif_codegen.push(new TolkTestCaseFifCodegen(this.parse_string_value(lines), false)) + } else if (line.startsWith("@code_hash")) { + this.expected_hash = new TolkTestCaseExpectedHash(this.parse_string_value(lines, false)[0]) + } else if (line.startsWith("@experimental_options")) { + this.experimental_options = line.substring(22) + } + this.line_idx++ + } + + if (this.input_output.length === 0 && !this.compilation_should_fail) + throw new ParseInputError("no @testcase present") + if (this.input_output.length !== 0 && this.compilation_should_fail) + throw new ParseInputError("@testcase present, but compilation_should_fail") + } + + /** @return {string[]} */ + parse_string_value(/**string[]*/ lines, allow_multiline = true) { + // a tag must be followed by a space (single-line), e.g. '@stderr some text' + // or be a multi-line value, surrounded by """ + const line = lines[this.line_idx] + const pos_sp = line.indexOf(' ') + const is_multi_line = lines[this.line_idx + 1] === '"""' + const is_single_line = pos_sp !== -1 + if (!is_single_line && !is_multi_line) + throw new ParseInputError(`${line} value is empty (not followed by a string or a multiline """)`) + if (is_single_line && is_multi_line) + throw new ParseInputError(`${line.substring(0, pos_sp)} value is both single-line and followed by """`) + if (is_multi_line && !allow_multiline) + throw new ParseInputError(`${line} value should be single-line`); + + if (is_single_line) + return [line.substring(pos_sp + 1).trim()] + + this.line_idx += 2 + let s_multiline = [] + while (this.line_idx < lines.length && lines[this.line_idx] !== '"""') { + s_multiline.push(lines[this.line_idx]) + this.line_idx = this.line_idx + 1 + } + return s_multiline + } + + get_compiled_fif_filename() { + return this.artifacts_folder + "/compiled.fif" + } + + get_runner_fif_filename() { + return this.artifacts_folder + "/runner.fif" + } + + async run_and_check() { + const wasmModule = await compileWasm(TOLKFIFTLIB_MODULE, TOLKFIFTLIB_WASM) + let res = compileFile(wasmModule, this.tolk_filename, this.experimental_options) + let exit_code = res.status === 'ok' ? 0 : 1 + let stderr = res.message + let stdout = '' + + if (exit_code === 0 && this.compilation_should_fail) + throw new TolkCompilationSucceededError("compilation succeeded, but it should have failed") + + for (let should_include of this.stderr_includes) // @stderr is used to check errors and warnings + should_include.check(stderr) + + if (exit_code !== 0 && this.compilation_should_fail) + return + + if (exit_code !== 0 && !this.compilation_should_fail) + throw new TolkCompilationFailedError(`tolk exit_code = ${exit_code}`, stderr) + + fs.writeFileSync(this.get_compiled_fif_filename(), `"Asm.fif" include\n${res.fiftCode}`) + { + let runner = `"${this.get_compiled_fif_filename()}" include x.trim()).filter(s => s.length > 0) + let fif_code_hash = null + if (this.expected_hash !== null) { // then the last stdout line is a hash + fif_code_hash = stdout_lines[stdout_lines.length - 1] + stdout_lines = stdout_lines.slice(0, stdout_lines.length - 1) + } + + if (stdout_lines.length !== this.input_output.length) + throw new CompareOutputError(`unexpected number of fift output: ${stdout_lines.length} lines, but ${this.input_output.length} testcases`, stdout) + + for (let i = 0; i < stdout_lines.length; ++i) + this.input_output[i].check(stdout_lines, i) + + if (this.fif_codegen.length) { + const fif_output = fs.readFileSync(this.get_compiled_fif_filename(), 'utf-8').split(/\r?\n/) + for (let fif_codegen of this.fif_codegen) + fif_codegen.check(fif_output) + } + + if (this.expected_hash !== null) + this.expected_hash.check(fif_code_hash) + } +} + +async function run_all_tests(/**string[]*/ tests) { + for (let ti = 0; ti < tests.length; ++ti) { + let tolk_filename = tests[ti] + print(`Running test ${ti + 1}/${tests.length}: ${tolk_filename}`) + + let artifacts_folder = path.join(TMP_DIR, tolk_filename) + let testcase = new TolkTestFile(tolk_filename, artifacts_folder) + + try { + if (!fs.existsSync(artifacts_folder)) + fs.mkdirSync(artifacts_folder, {recursive: true}) + testcase.parse_input_from_tolk_file() + await testcase.run_and_check() + fs.rmSync(artifacts_folder, {recursive: true}) + + if (testcase.compilation_should_fail) + print(" OK, compilation failed as it should") + else + print(` OK, ${testcase.input_output.length} cases`) + } catch (e) { + if (e instanceof ParseInputError) { + print(` Error parsing input (cur line #${testcase.line_idx + 1}):`, e.message) + process.exit(2) + } else if (e instanceof TolkCompilationFailedError) { + print(" Error compiling tolk:", e.message) + print(" stderr:") + print(e.stderr.trimEnd()) + process.exit(2) + } else if (e instanceof FiftExecutionFailedError) { + print(" Error executing fift:", e.message) + print(" stderr:") + print(e.stderr.trimEnd()) + print(" compiled.fif at:", testcase.get_compiled_fif_filename()) + process.exit(2) + } else if (e instanceof CompareOutputError) { + print(" Mismatch in output:", e.message) + print(" Full output:") + print(e.output.trimEnd()) + print(" Was compiled to:", testcase.get_compiled_fif_filename()) + process.exit(2) + } else if (e instanceof CompareFifCodegenError) { + print(" Mismatch in fif codegen:", e.message) + print(" Was compiled to:", testcase.get_compiled_fif_filename()) + print(fs.readFileSync(testcase.get_compiled_fif_filename(), 'utf-8')) + process.exit(2) + } else if (e instanceof CompareCodeHashError) { + print(" Mismatch in code hash:", e.message) + print(" Was compiled to:", testcase.get_compiled_fif_filename()) + process.exit(2) + } + throw e + } + } +} + +const tests = new CmdLineOptions(process.argv).find_tests() +print(`Found ${tests.length} tests`) +run_all_tests(tests).then( + () => print(`Done, ${tests.length} tests`), + console.error +) + +// below are WASM helpers, which don't exist in Python version + +process.setMaxListeners(0); + +function copyToCString(mod, str) { + const len = mod.lengthBytesUTF8(str) + 1; + const ptr = mod._malloc(len); + mod.stringToUTF8(str, ptr, len); + return ptr; +} + +function copyToCStringPtr(mod, str, ptr) { + const allocated = copyToCString(mod, str); + mod.setValue(ptr, allocated, '*'); + return allocated; +} + +/** @return {string} */ +function copyFromCString(mod, ptr) { + return mod.UTF8ToString(ptr); +} + +/** @return {{status: string, message: string, fiftCode: string, codeBoc: string, codeHashHex: string}} */ +function compileFile(mod, filename, experimentalOptions) { + // see tolk-wasm.cpp: typedef void (*WasmFsReadCallback)(int, char const*, char**, char**) + const callbackPtr = mod.addFunction((kind, dataPtr, destContents, destError) => { + if (kind === 0) { // realpath + try { + let relative = copyFromCString(mod, dataPtr) + if (relative.startsWith('@stdlib/')) { + // import "@stdlib/filename" or import "@stdlib/filename.tolk" + relative = STDLIB_FOLDER + '/' + relative.substring(7) + if (!relative.endsWith('.tolk')) { + relative += '.tolk' + } + } + copyToCStringPtr(mod, fs.realpathSync(relative), destContents); + } catch (err) { + copyToCStringPtr(mod, 'cannot find file', destError); + } + } else if (kind === 1) { // read file + try { + const absolute = copyFromCString(mod, dataPtr) // already normalized (as returned above) + copyToCStringPtr(mod, fs.readFileSync(absolute).toString('utf-8'), destContents); + } catch (err) { + copyToCStringPtr(mod, err.message || err.toString(), destError); + } + } else { + copyToCStringPtr(mod, 'Unknown callback kind=' + kind, destError); + } + }, 'viiii'); + + const config = { + optimizationLevel: 2, + withStackComments: true, + experimentalOptions: experimentalOptions || undefined, + entrypointFileName: filename + }; + + const configPtr = copyToCString(mod, JSON.stringify(config)); + + const responsePtr = mod._tolk_compile(configPtr, callbackPtr); + + return JSON.parse(copyFromCString(mod, responsePtr)); +} + +async function compileWasm(tolkFiftLibJsFileName, tolkFiftLibWasmFileName) { + const wasmModule = require(tolkFiftLibJsFileName) + const wasmBinary = new Uint8Array(fs.readFileSync(tolkFiftLibWasmFileName)) + + return await wasmModule({ wasmBinary }) +} diff --git a/tolk-tester/tolk-tester.py b/tolk-tester/tolk-tester.py new file mode 100644 index 00000000..0b3c774c --- /dev/null +++ b/tolk-tester/tolk-tester.py @@ -0,0 +1,431 @@ +# Usage: `tolk-tester.py tests_dir` OR `tolk-tester.py test_file.tolk` +# from current dir, providing some env (see getenv() calls). +# Every .tolk file should provide /* testcase description in a comment */, consider tests/ folder. +# +# Tests for Tolk can be +# * positive (compiled to .fif, run with fift, compared output with the one expected) +# * negative (compilation fails, and it's expected; patterns in stderr can be specified) +# +# Note, that there is also tolk-tester.js to test Tolk compiled to WASM. +# Don't forget to keep it identical to Python version! + +import os +import os.path +import re +import shutil +import subprocess +import sys +import tempfile +from typing import List + + +def getenv(name, default=None): + if name in os.environ: + return os.environ[name] + if default is None: + print("Environment variable", name, "is not set", file=sys.stderr) + exit(1) + return default + + +TOLK_EXECUTABLE = getenv("TOLK_EXECUTABLE", "tolk") +FIFT_EXECUTABLE = getenv("FIFT_EXECUTABLE", "fift") +FIFT_LIBS_FOLDER = getenv("FIFTPATH") # this env is needed for fift to work properly +TMP_DIR = tempfile.mkdtemp() + + +class CmdLineOptions: + def __init__(self, argv: List[str]): + if len(argv) != 2: + print("Usage: tolk-tester.py tests_dir OR tolk-tester.py test_file.tolk", file=sys.stderr) + exit(1) + if not os.path.exists(argv[1]): + print("Input '%s' doesn't exist" % argv[1], file=sys.stderr) + exit(1) + + if os.path.isdir(argv[1]): + self.tests_dir = argv[1] + self.test_file = None + else: + self.tests_dir = os.path.dirname(argv[1]) + self.test_file = argv[1] + + def find_tests(self) -> List[str]: + if self.test_file is not None: # an option to run (debug) a single test + return [self.test_file] + + tests = [f for f in os.listdir(self.tests_dir) if f.endswith(".tolk") or f.endswith(".ton")] + tests.sort() + return [os.path.join(self.tests_dir, f) for f in tests] + + +class ParseInputError(Exception): + pass + + +class TolkCompilationFailedError(Exception): + def __init__(self, message: str, stderr: str): + super().__init__(message) + self.stderr = stderr + + +class TolkCompilationSucceededError(Exception): + pass + + +class FiftExecutionFailedError(Exception): + def __init__(self, message: str, stderr: str): + super().__init__(message) + self.stderr = stderr + + +class CompareOutputError(Exception): + def __init__(self, message: str, output: str): + super().__init__(message) + self.output = output + + +class CompareFifCodegenError(Exception): + pass + + +class CompareCodeHashError(Exception): + pass + + +class TolkTestCaseInputOutput: + """ + In positive tests, there are several testcases "input X should produce output Y". + They are written as a table: + @testcase | method_id | input (one or several) | output + """ + reJustNumber = re.compile(r"[-+]?\d+") + reMathExpr = re.compile(r"[0x123456789()+\-*/<>]+") + + def __init__(self, method_id_str: str, input_str: str, output_str: str): + processed_inputs = [] + for in_arg in input_str.split(" "): + if len(in_arg) == 0: + continue + elif in_arg.startswith("x{") or TolkTestCaseInputOutput.reJustNumber.fullmatch(in_arg): + processed_inputs.append(in_arg) + elif TolkTestCaseInputOutput.reMathExpr.fullmatch(in_arg): + processed_inputs.append(str(eval(in_arg))) + elif in_arg == "null": + processed_inputs.append("null") + else: + raise ParseInputError("'%s' can't be evaluated" % in_arg) + + self.method_id = int(method_id_str) + self.input = " ".join(processed_inputs) + self.expected_output = output_str + + def check(self, stdout_lines: List[str], line_idx: int): + if stdout_lines[line_idx] != self.expected_output: + raise CompareOutputError("error on case #%d (%d | %s): expected '%s', found '%s'" % (line_idx + 1, self.method_id, self.input, self.expected_output, stdout_lines[line_idx]), "\n".join(stdout_lines)) + + +class TolkTestCaseStderr: + """ + @stderr checks, when compilation fails, that stderr (compilation error) is expected. + If it's multiline, all lines must be present in specified order. + """ + + def __init__(self, stderr_pattern: List[str], avoid: bool): + self.stderr_pattern = stderr_pattern + self.avoid = avoid + + def check(self, stderr: str): + line_match = self.find_pattern_in_stderr(stderr.splitlines()) + if line_match == -1 and not self.avoid: + raise CompareOutputError("pattern not found in stderr:\n%s" % + "\n".join(map(lambda x: " " + x, self.stderr_pattern)), stderr) + elif line_match != -1 and self.avoid: + raise CompareOutputError("pattern found (line %d), but not expected to be:\n%s" % + (line_match + 1, "\n".join(map(lambda x: " " + x, self.stderr_pattern))), stderr) + + def find_pattern_in_stderr(self, stderr: List[str]) -> int: + for line_start in range(len(stderr)): + if self.try_match_pattern(0, stderr, line_start): + return line_start + return -1 + + def try_match_pattern(self, pattern_offset: int, stderr: List[str], offset: int) -> bool: + if pattern_offset >= len(self.stderr_pattern): + return True + if offset >= len(stderr): + return False + + line_pattern = self.stderr_pattern[pattern_offset] + line_output = stderr[offset] + return line_output.find(line_pattern) != -1 and self.try_match_pattern(pattern_offset + 1, stderr, offset + 1) + + +class TolkTestCaseFifCodegen: + """ + @fif_codegen checks that contents of compiled.fif matches the expected pattern. + @fif_codegen_avoid checks that is does not match the pattern. + The pattern is a multiline piece of fift code, optionally with "..." meaning "any lines here". + See tests/codegen_check_demo.tolk of how it looks. + A notable thing about indentations (spaces at line starts): + Taking them into account will complicate the code without reasonable profit, + that's why we just trim every string. + And one more word about //comments. Tolk inserts them into fift output. + If a line in the pattern contains a //comment, it's expected to be equal. + If a line does not, we just compare a command. + """ + + def __init__(self, fif_pattern: List[str], avoid: bool): + self.fif_pattern = [s.strip() for s in fif_pattern] + self.avoid = avoid + + def check(self, fif_output: List[str]): + line_match = self.find_pattern_in_fif_output(fif_output) + if line_match == -1 and not self.avoid: + raise CompareFifCodegenError("pattern not found:\n%s" % + "\n".join(map(lambda x: " " + x, self.fif_pattern))) + elif line_match != -1 and self.avoid: + raise CompareFifCodegenError("pattern found (line %d), but not expected to be:\n%s" % + (line_match + 1, "\n".join(map(lambda x: " " + x, self.fif_pattern)))) + + def find_pattern_in_fif_output(self, fif_output: List[str]) -> int: + for line_start in range(len(fif_output)): + if self.try_match_pattern(0, fif_output, line_start): + return line_start + return -1 + + def try_match_pattern(self, pattern_offset: int, fif_output: List[str], offset: int) -> bool: + if pattern_offset >= len(self.fif_pattern): + return True + if offset >= len(fif_output): + return False + line_pattern = self.fif_pattern[pattern_offset] + line_output = fif_output[offset] + + if line_pattern != "...": + if not TolkTestCaseFifCodegen.does_line_match(line_pattern, line_output): + return False + return self.try_match_pattern(pattern_offset + 1, fif_output, offset + 1) + while offset < len(fif_output): + if self.try_match_pattern(pattern_offset + 1, fif_output, offset): + return True + offset = offset + 1 + return False + + @staticmethod + def split_line_to_cmd_and_comment(trimmed_line: str) -> tuple: + pos = trimmed_line.find("//") + if pos == -1: + return trimmed_line, None + else: + return trimmed_line[:pos].rstrip(), trimmed_line[pos + 2:].lstrip() + + @staticmethod + def does_line_match(line_pattern: str, line_output: str) -> bool: + cmd_pattern, comment_pattern = TolkTestCaseFifCodegen.split_line_to_cmd_and_comment(line_pattern) + cmd_output, comment_output = TolkTestCaseFifCodegen.split_line_to_cmd_and_comment(line_output.strip()) + return cmd_pattern == cmd_output and (comment_pattern is None or comment_pattern == comment_output) + + +class TolkTestCaseExpectedHash: + """ + @code_hash checks that hash of compiled output.fif matches the provided value. + It's used to "record" code boc hash and to check that it remains the same on compiler modifications. + Being much less flexible than @fif_codegen, it nevertheless gives a guarantee of bytecode stability. + """ + + def __init__(self, expected_hash: str): + self.code_hash = expected_hash + + def check(self, fif_code_hash: str): + if self.code_hash != fif_code_hash: + raise CompareCodeHashError("expected %s, actual %s" % (self.code_hash, fif_code_hash)) + + +class TolkTestFile: + def __init__(self, tolk_filename: str, artifacts_folder: str): + self.line_idx = 0 + self.tolk_filename = tolk_filename + self.artifacts_folder = artifacts_folder + self.compilation_should_fail = False + self.stderr_includes: List[TolkTestCaseStderr] = [] + self.input_output: List[TolkTestCaseInputOutput] = [] + self.fif_codegen: List[TolkTestCaseFifCodegen] = [] + self.expected_hash: TolkTestCaseExpectedHash | None = None + self.experimental_options: str | None = None + + def parse_input_from_tolk_file(self): + with open(self.tolk_filename, "r") as fd: + lines = fd.read().splitlines() + self.line_idx = 0 + + while self.line_idx < len(lines): + line = lines[self.line_idx] + if line.startswith("@testcase"): + s = [x.strip() for x in line.split("|")] + if len(s) != 4: + raise ParseInputError("incorrect format of @testcase: %s" % line) + self.input_output.append(TolkTestCaseInputOutput(s[1], s[2], s[3])) + elif line.startswith("@compilation_should_fail"): + self.compilation_should_fail = True + elif line.startswith("@stderr"): + self.stderr_includes.append(TolkTestCaseStderr(self.parse_string_value(lines), False)) + elif line.startswith("@fif_codegen_avoid"): + self.fif_codegen.append(TolkTestCaseFifCodegen(self.parse_string_value(lines), True)) + elif line.startswith("@fif_codegen"): + self.fif_codegen.append(TolkTestCaseFifCodegen(self.parse_string_value(lines), False)) + elif line.startswith("@code_hash"): + self.expected_hash = TolkTestCaseExpectedHash(self.parse_string_value(lines, False)[0]) + elif line.startswith("@experimental_options"): + self.experimental_options = line[22:] + self.line_idx = self.line_idx + 1 + + if len(self.input_output) == 0 and not self.compilation_should_fail: + raise ParseInputError("no @testcase present") + if len(self.input_output) != 0 and self.compilation_should_fail: + raise ParseInputError("@testcase present, but compilation_should_fail") + + def parse_string_value(self, lines: List[str], allow_multiline = True) -> List[str]: + # a tag must be followed by a space (single-line), e.g. '@stderr some text' + # or be a multi-line value, surrounded by """ + line = lines[self.line_idx] + pos_sp = line.find(' ') + is_multi_line = lines[self.line_idx + 1] == '"""' + is_single_line = pos_sp != -1 + if not is_single_line and not is_multi_line: + raise ParseInputError('%s value is empty (not followed by a string or a multiline """)' % line) + if is_single_line and is_multi_line: + raise ParseInputError('%s value is both single-line and followed by """' % line[:pos_sp]) + if is_multi_line and not allow_multiline: + raise ParseInputError("%s value should be single-line" % line) + + if is_single_line: + return [line[pos_sp + 1:].strip()] + + self.line_idx += 2 + s_multiline = [] + while self.line_idx < len(lines) and lines[self.line_idx] != '"""': + s_multiline.append(lines[self.line_idx]) + self.line_idx = self.line_idx + 1 + return s_multiline + + def get_compiled_fif_filename(self): + return self.artifacts_folder + "/compiled.fif" + + def get_runner_fif_filename(self): + return self.artifacts_folder + "/runner.fif" + + def run_and_check(self): + cmd_args = [TOLK_EXECUTABLE, "-o", self.get_compiled_fif_filename()] + if self.experimental_options: + cmd_args = cmd_args + ["-x", self.experimental_options] + res = subprocess.run(cmd_args + [self.tolk_filename], capture_output=True, timeout=10) + exit_code = res.returncode + stderr = str(res.stderr, "utf-8") + stdout = str(res.stdout, "utf-8") + + if exit_code == 0 and self.compilation_should_fail: + raise TolkCompilationSucceededError("compilation succeeded, but it should have failed") + + for should_include in self.stderr_includes: # @stderr is used to check errors and warnings + should_include.check(stderr) + + if exit_code != 0 and self.compilation_should_fail: + return + + if exit_code != 0 and not self.compilation_should_fail: + raise TolkCompilationFailedError("tolk exit_code = %d" % exit_code, stderr) + + with open(self.get_runner_fif_filename(), "w") as fd: + fd.write("\"%s\" include ) +target_link_libraries(tolk PUBLIC git ton_crypto_core) +if (WINGETOPT_FOUND) + target_link_libraries_system(tolk wingetopt) +endif () +if (${TOLK_DEBUG}) # -DTOLK_DEBUG=1 in CMake options => #define TOLK_DEBUG (for development purposes) + message(STATUS "TOLK_DEBUG is ON") + target_compile_definitions(tolk PRIVATE TOLK_DEBUG=1) +endif() + +if (USE_EMSCRIPTEN) + add_executable(tolkfiftlib tolk-wasm.cpp ${TOLK_SOURCE}) + target_include_directories(tolkfiftlib PUBLIC $) + target_link_libraries(tolkfiftlib PUBLIC fift-lib git) + target_link_options(tolkfiftlib PRIVATE + -sEXPORTED_RUNTIME_METHODS=FS,ccall,cwrap,UTF8ToString,stringToUTF8,lengthBytesUTF8,addFunction,removeFunction,setValue + -sEXPORTED_FUNCTIONS=_tolk_compile,_version,_malloc,_free,_setThrew + -sEXPORT_NAME=CompilerModule + -sERROR_ON_UNDEFINED_SYMBOLS=0 + -sFILESYSTEM=1 -lnodefs.js + -Oz + -sIGNORE_MISSING_MAIN=1 + -sAUTO_NATIVE_LIBRARIES=0 + -sMODULARIZE=1 + -sTOTAL_MEMORY=33554432 + -sALLOW_MEMORY_GROWTH=1 + -sALLOW_TABLE_GROWTH=1 + --embed-file ${CMAKE_CURRENT_SOURCE_DIR}/../crypto/fift/lib@/fiftlib + -fexceptions + ) + target_compile_options(tolkfiftlib PRIVATE -fexceptions -fno-stack-protector) +endif () + +install(TARGETS tolk RUNTIME DESTINATION bin) diff --git a/tolk/abscode.cpp b/tolk/abscode.cpp new file mode 100644 index 00000000..fc160984 --- /dev/null +++ b/tolk/abscode.cpp @@ -0,0 +1,429 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" +#include "compiler-state.h" +#include "type-system.h" + +namespace tolk { + +/* + * + * ABSTRACT CODE + * + */ + +void TmpVar::show_as_stack_comment(std::ostream& os) const { + if (!name.empty()) { + os << name; + } else { + os << '\'' << ir_idx; + } +#ifdef TOLK_DEBUG + // uncomment for detailed stack output, like `'15(binary-op) '16(glob-var)` + // if (desc) os << desc; +#endif +} + +void TmpVar::show(std::ostream& os) const { + os << '\'' << ir_idx; // vars are printed out as `'1 '2` (in stack comments, debug info, etc.) + if (!name.empty()) { + os << '_' << name; + } +#ifdef TOLK_DEBUG + if (desc) { + os << ' ' << desc; // "origin" of implicitly created tmp var, like `'15 (binary-op) '16 (glob-var)` + } +#endif +} + +std::ostream& operator<<(std::ostream& os, const TmpVar& var) { + var.show(os); + return os; +} + +void VarDescr::show_value(std::ostream& os) const { + if (val & _Int) { + os << 'i'; + } + if (val & _Const) { + os << 'c'; + } + if (val & _Zero) { + os << '0'; + } + if (val & _NonZero) { + os << '!'; + } + if (val & _Pos) { + os << '>'; + } + if (val & _Neg) { + os << '<'; + } + if (val & _Even) { + os << 'E'; + } + if (val & _Odd) { + os << 'O'; + } + if (val & _Finite) { + os << 'f'; + } + if (val & _Nan) { + os << 'N'; + } + if (int_const.not_null()) { + os << '=' << int_const; + } +} + +void VarDescr::show(std::ostream& os, const char* name) const { + if (flags & _Last) { + os << '*'; + } + if (flags & _Unused) { + os << '?'; + } + if (name) { + os << name; + } + os << '\'' << idx; + show_value(os); +} + +void VarDescr::set_const(long long value) { + return set_const(td::make_refint(value)); +} + +void VarDescr::set_const(td::RefInt256 value) { + int_const = std::move(value); + if (!int_const->signed_fits_bits(257)) { + int_const.write().invalidate(); + } + val = _Const | _Int; + int s = sgn(int_const); + if (s < -1) { + val |= _Nan | _NonZero; + } else if (s < 0) { + val |= _NonZero | _Neg | _Finite; + } else if (s > 0) { + val |= _NonZero | _Pos | _Finite; + } else { + val |= _Zero | _Neg | _Pos | _Finite; + } + if (val & _Finite) { + val |= int_const->get_bit(0) ? _Odd : _Even; + } +} + +void VarDescr::set_const(std::string value) { + str_const = value; + val = _Const; +} + +void VarDescr::operator|=(const VarDescr& y) { + val &= y.val; + if (is_int_const() && y.is_int_const() && cmp(int_const, y.int_const) != 0) { + val &= ~_Const; + } + if (!(val & _Const)) { + int_const.clear(); + } +} + +void VarDescr::operator&=(const VarDescr& y) { + val |= y.val; + if (y.int_const.not_null() && int_const.is_null()) { + int_const = y.int_const; + } +} + +void VarDescr::set_value(const VarDescr& y) { + val = y.val; + int_const = y.int_const; +} + +void VarDescr::set_value(VarDescr&& y) { + val = y.val; + int_const = std::move(y.int_const); +} + +void VarDescr::clear_value() { + val = 0; + int_const.clear(); +} + +void VarDescrList::show(std::ostream& os) const { + if (unreachable) { + os << " "; + } + os << "["; + for (const auto& v : list) { + os << ' ' << v; + } + os << " ]\n"; +} + +void Op::show(std::ostream& os, const std::vector& vars, std::string pfx, int mode) const { + if (mode & 2) { + os << pfx << " ["; + for (const auto& v : var_info.list) { + os << ' '; + if (v.flags & VarDescr::_Last) { + os << '*'; + } + if (v.flags & VarDescr::_Unused) { + os << '?'; + } + os << vars[v.idx]; + if (mode & 4) { + os << ':'; + v.show_value(os); + } + } + os << " ]\n"; + } + std::string dis = disabled() ? " " : ""; + if (noreturn()) { + dis += " "; + } + if (impure()) { + dis += " "; + } + switch (cl) { + case _Undef: + os << pfx << dis << "???\n"; + break; + case _Nop: + os << pfx << dis << "NOP\n"; + break; + case _Call: + os << pfx << dis << "CALL: "; + show_var_list(os, left, vars); + os << " := " << (f_sym ? f_sym->name : "(null)") << " "; + if ((mode & 4) && args.size() == right.size()) { + show_var_list(os, args, vars); + } else { + show_var_list(os, right, vars); + } + os << std::endl; + break; + case _CallInd: + os << pfx << dis << "CALLIND: "; + show_var_list(os, left, vars); + os << " := EXEC "; + show_var_list(os, right, vars); + os << std::endl; + break; + case _Let: + os << pfx << dis << "LET "; + show_var_list(os, left, vars); + os << " := "; + show_var_list(os, right, vars); + os << std::endl; + break; + case _Tuple: + os << pfx << dis << "MKTUPLE "; + show_var_list(os, left, vars); + os << " := "; + show_var_list(os, right, vars); + os << std::endl; + break; + case _UnTuple: + os << pfx << dis << "UNTUPLE "; + show_var_list(os, left, vars); + os << " := "; + show_var_list(os, right, vars); + os << std::endl; + break; + case _IntConst: + os << pfx << dis << "CONST "; + show_var_list(os, left, vars); + os << " := " << int_const << std::endl; + break; + case _SliceConst: + os << pfx << dis << "SCONST "; + show_var_list(os, left, vars); + os << " := " << str_const << std::endl; + break; + case _Import: + os << pfx << dis << "IMPORT "; + show_var_list(os, left, vars); + os << std::endl; + break; + case _Return: + os << pfx << dis << "RETURN "; + show_var_list(os, left, vars); + os << std::endl; + break; + case _GlobVar: + os << pfx << dis << "GLOBVAR "; + show_var_list(os, left, vars); + os << " := " << (g_sym ? g_sym->name : "(null)") << std::endl; + break; + case _SetGlob: + os << pfx << dis << "SETGLOB "; + os << (g_sym ? g_sym->name : "(null)") << " := "; + show_var_list(os, right, vars); + os << std::endl; + break; + case _Repeat: + os << pfx << dis << "REPEAT "; + show_var_list(os, left, vars); + os << ' '; + show_block(os, block0.get(), vars, pfx, mode); + os << std::endl; + break; + case _If: + os << pfx << dis << "IF "; + show_var_list(os, left, vars); + os << ' '; + show_block(os, block0.get(), vars, pfx, mode); + os << " ELSE "; + show_block(os, block1.get(), vars, pfx, mode); + os << std::endl; + break; + case _While: + os << pfx << dis << "WHILE "; + show_var_list(os, left, vars); + os << ' '; + show_block(os, block0.get(), vars, pfx, mode); + os << " DO "; + show_block(os, block1.get(), vars, pfx, mode); + os << std::endl; + break; + case _Until: + os << pfx << dis << "UNTIL "; + show_var_list(os, left, vars); + os << ' '; + show_block(os, block0.get(), vars, pfx, mode); + os << std::endl; + break; + case _Again: + os << pfx << dis << "AGAIN "; + show_var_list(os, left, vars); + os << ' '; + show_block(os, block0.get(), vars, pfx, mode); + os << std::endl; + break; + default: + os << pfx << dis << " "; + show_var_list(os, left, vars); + os << " -- "; + show_var_list(os, right, vars); + os << std::endl; + break; + } +} + +void Op::show_var_list(std::ostream& os, const std::vector& idx_list, + const std::vector& vars) const { + if (!idx_list.size()) { + os << "()"; + } else if (idx_list.size() == 1) { + os << vars.at(idx_list[0]); + } else { + os << "(" << vars.at(idx_list[0]); + for (std::size_t i = 1; i < idx_list.size(); i++) { + os << ", " << vars.at(idx_list[i]); + } + os << ")"; + } +} + +void Op::show_var_list(std::ostream& os, const std::vector& list, const std::vector& vars) const { + auto n = list.size(); + if (!n) { + os << "()"; + } else { + os << "( "; + for (std::size_t i = 0; i < list.size(); i++) { + if (i) { + os << ", "; + } + if (list[i].is_unused()) { + os << '?'; + } + os << vars.at(list[i].idx) << ':'; + list[i].show_value(os); + } + os << " )"; + } +} + +void Op::show_block(std::ostream& os, const Op* block, const std::vector& vars, std::string pfx, int mode) { + os << "{" << std::endl; + std::string pfx2 = pfx + " "; + for (const Op& op : block) { + op.show(os, vars, pfx2, mode); + } + os << pfx << "}"; +} + +std::ostream& operator<<(std::ostream& os, const CodeBlob& code) { + code.print(os); + return os; +} + +// flags: +1 = show variable definition locations; +2 = show vars after each op; +4 = show var abstract value info after each op; +8 = show all variables at start +void CodeBlob::print(std::ostream& os, int flags) const { + os << "CODE BLOB: " << var_cnt << " variables, " << in_var_cnt << " input\n"; + if ((flags & 8) != 0) { + for (const auto& var : vars) { + var.show(os); + os << " : " << var.v_type << std::endl; + if (var.loc.is_defined() && (flags & 1) != 0) { + var.loc.show(os); + os << " defined here:\n"; + var.loc.show_context(os); + } + } + } + os << "------- BEGIN --------\n"; + for (const auto& op : ops) { + op.show(os, vars, "", flags); + } + os << "-------- END ---------\n\n"; +} + +std::vector CodeBlob::create_var(TypePtr var_type, SrcLocation loc, std::string name) { + std::vector ir_idx; + int stack_w = var_type->get_width_on_stack(); + ir_idx.reserve(stack_w); + if (const TypeDataTensor* t_tensor = var_type->try_as()) { + for (int i = 0; i < t_tensor->size(); ++i) { + std::string sub_name = name.empty() ? name : name + "." + std::to_string(i); + std::vector nested = create_var(t_tensor->items[i], loc, std::move(sub_name)); + ir_idx.insert(ir_idx.end(), nested.begin(), nested.end()); + } + } else if (const TypeDataNullable* t_nullable = var_type->try_as(); t_nullable && stack_w != 1) { + std::string null_flag_name = name.empty() ? name : name + ".NNFlag"; + ir_idx = create_var(t_nullable->inner, loc, std::move(name)); + ir_idx.emplace_back(create_var(TypeDataBool::create(), loc, std::move(null_flag_name))[0]); + } else if (var_type != TypeDataVoid::create() && var_type != TypeDataNever::create()) { +#ifdef TOLK_DEBUG + tolk_assert(stack_w == 1); +#endif + vars.emplace_back(var_cnt, var_type, std::move(name), loc); + ir_idx.emplace_back(var_cnt); + var_cnt++; + } + tolk_assert(static_cast(ir_idx.size()) == stack_w); + return ir_idx; +} + +} // namespace tolk diff --git a/tolk/analyzer.cpp b/tolk/analyzer.cpp new file mode 100644 index 00000000..c38b0bfa --- /dev/null +++ b/tolk/analyzer.cpp @@ -0,0 +1,902 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" +#include "compiler-state.h" +#include "type-system.h" + +namespace tolk { + +// functions returning "never" are assumed to interrupt flow +// for instance, variables after their call aren't considered used +// its main purpose is `throw` statement, it's a call to a built-in `__throw` function +static bool does_function_always_throw(FunctionPtr fun_ref) { + return fun_ref->declared_return_type == TypeDataNever::create(); +} + +/* + * + * ANALYZE AND PREPROCESS ABSTRACT CODE + * + */ + +bool CodeBlob::compute_used_code_vars() { + VarDescrList empty_var_info; + return compute_used_code_vars(ops, empty_var_info, true); +} + +bool CodeBlob::compute_used_code_vars(std::unique_ptr& ops_ptr, const VarDescrList& var_info, bool edit) const { + tolk_assert(ops_ptr); + if (!ops_ptr->next) { + tolk_assert(ops_ptr->cl == Op::_Nop); + return ops_ptr->set_var_info(var_info); + } + // here and below, bitwise | (not logical ||) are used to execute both left and right parts + return static_cast(compute_used_code_vars(ops_ptr->next, var_info, edit)) | + static_cast(ops_ptr->compute_used_vars(*this, edit)); +} + +bool operator==(const VarDescrList& x, const VarDescrList& y) { + if (x.size() != y.size()) { + return false; + } + for (std::size_t i = 0; i < x.size(); i++) { + if (x.list[i].idx != y.list[i].idx || x.list[i].flags != y.list[i].flags) { + return false; + } + } + return true; +} + +bool same_values(const VarDescr& x, const VarDescr& y) { + if (x.val != y.val || x.int_const.is_null() != y.int_const.is_null()) { + return false; + } + if (x.int_const.not_null() && cmp(x.int_const, y.int_const) != 0) { + return false; + } + return true; +} + +bool same_values(const VarDescrList& x, const VarDescrList& y) { + if (x.size() != y.size()) { + return false; + } + for (std::size_t i = 0; i < x.size(); i++) { + if (x.list[i].idx != y.list[i].idx || !same_values(x.list[i], y.list[i])) { + return false; + } + } + return true; +} + +bool Op::set_var_info(const VarDescrList& new_var_info) { + if (var_info == new_var_info) { + return false; + } + var_info = new_var_info; + return true; +} + +bool Op::set_var_info(VarDescrList&& new_var_info) { + if (var_info == new_var_info) { + return false; + } + var_info = std::move(new_var_info); + return true; +} + +bool Op::set_var_info_except(const VarDescrList& new_var_info, const std::vector& var_list) { + if (!var_list.size()) { + return set_var_info(new_var_info); + } + VarDescrList tmp_info{new_var_info}; + tmp_info -= var_list; + return set_var_info(tmp_info); +} + +bool Op::set_var_info_except(VarDescrList&& new_var_info, const std::vector& var_list) { + if (var_list.size()) { + new_var_info -= var_list; + } + return set_var_info(std::move(new_var_info)); +} +std::vector sort_unique_vars(const std::vector& var_list) { + std::vector vars{var_list}, unique_vars; + std::sort(vars.begin(), vars.end()); + vars.erase(std::unique(vars.begin(), vars.end()), vars.end()); + return vars; +} + +VarDescr* VarDescrList::operator[](var_idx_t idx) { + auto it = std::lower_bound(list.begin(), list.end(), idx); + return it != list.end() && it->idx == idx ? &*it : nullptr; +} + +const VarDescr* VarDescrList::operator[](var_idx_t idx) const { + auto it = std::lower_bound(list.begin(), list.end(), idx); + return it != list.end() && it->idx == idx ? &*it : nullptr; +} + +std::size_t VarDescrList::count(const std::vector idx_list) const { + std::size_t res = 0; + for (var_idx_t idx : idx_list) { + if (operator[](idx)) { + ++res; + } + } + return res; +} + +std::size_t VarDescrList::count_used(const std::vector idx_list) const { + std::size_t res = 0; + for (var_idx_t idx : idx_list) { + auto v = operator[](idx); + if (v && !v->is_unused()) { + ++res; + } + } + return res; +} + +VarDescrList& VarDescrList::operator-=(var_idx_t idx) { + auto it = std::lower_bound(list.begin(), list.end(), idx); + if (it != list.end() && it->idx == idx) { + list.erase(it); + } + return *this; +} + +VarDescrList& VarDescrList::operator-=(const std::vector& idx_list) { + for (var_idx_t idx : idx_list) { + *this -= idx; + } + return *this; +} + +VarDescrList& VarDescrList::add_var(var_idx_t idx, bool unused) { + auto it = std::lower_bound(list.begin(), list.end(), idx); + if (it == list.end() || it->idx != idx) { + list.emplace(it, idx, VarDescr::_Last | (unused ? VarDescr::_Unused : 0)); + } else if (it->is_unused() && !unused) { + it->clear_unused(); + } + return *this; +} + +VarDescrList& VarDescrList::add_vars(const std::vector& idx_list, bool unused) { + for (var_idx_t idx : idx_list) { + add_var(idx, unused); + } + return *this; +} + +VarDescr& VarDescrList::add(var_idx_t idx) { + auto it = std::lower_bound(list.begin(), list.end(), idx); + if (it == list.end() || it->idx != idx) { + it = list.emplace(it, idx); + } + return *it; +} + +VarDescr& VarDescrList::add_newval(var_idx_t idx) { + auto it = std::lower_bound(list.begin(), list.end(), idx); + if (it == list.end() || it->idx != idx) { + return *list.emplace(it, idx); + } else { + it->clear_value(); + return *it; + } +} + +VarDescrList& VarDescrList::clear_last() { + for (auto& var : list) { + if (var.flags & VarDescr::_Last) { + var.flags &= ~VarDescr::_Last; + } + } + return *this; +} + +VarDescrList VarDescrList::operator+(const VarDescrList& y) const { + VarDescrList res; + auto it1 = list.cbegin(); + auto it2 = y.list.cbegin(); + while (it1 != list.cend() && it2 != y.list.cend()) { + if (it1->idx < it2->idx) { + res.list.push_back(*it1++); + } else if (it1->idx > it2->idx) { + res.list.push_back(*it2++); + } else { + res.list.push_back(*it1++); + res.list.back() += *it2++; + } + } + while (it1 != list.cend()) { + res.list.push_back(*it1++); + } + while (it2 != y.list.cend()) { + res.list.push_back(*it2++); + } + return res; +} + +VarDescrList& VarDescrList::operator+=(const VarDescrList& y) { + return *this = *this + y; +} + +VarDescrList VarDescrList::operator|(const VarDescrList& y) const { + if (y.unreachable) { + return *this; + } + if (unreachable) { + return y; + } + VarDescrList res; + auto it1 = list.cbegin(); + auto it2 = y.list.cbegin(); + while (it1 != list.cend() && it2 != y.list.cend()) { + if (it1->idx < it2->idx) { + it1++; + } else if (it1->idx > it2->idx) { + it2++; + } else { + res.list.push_back(*it1++); + res.list.back() |= *it2++; + } + } + return res; +} + +VarDescrList& VarDescrList::operator|=(const VarDescrList& y) { + if (y.unreachable) { + return *this; + } else { + return *this = *this | y; + } +} + +VarDescrList& VarDescrList::import_values(const VarDescrList& values) { + if (values.unreachable) { + set_unreachable(); + } else + for (auto& vd : list) { + auto new_vd = values[vd.idx]; + if (new_vd) { + vd.set_value(*new_vd); + } else { + vd.clear_value(); + } + } + return *this; +} + +bool Op::std_compute_used_vars(bool disabled) { + // left = OP right + // var_info := (var_info - left) + right + VarDescrList new_var_info{next->var_info}; + new_var_info -= left; + new_var_info.clear_last(); + if (args.size() == right.size() && !disabled) { + for (const VarDescr& arg : args) { + new_var_info.add_var(arg.idx, arg.is_unused()); + } + } else { + new_var_info.add_vars(right, disabled); + } + return set_var_info(std::move(new_var_info)); +} + +bool Op::compute_used_vars(const CodeBlob& code, bool edit) { + tolk_assert(next); + const VarDescrList& next_var_info = next->var_info; + if (cl == _Nop) { + return set_var_info_except(next_var_info, left); + } + switch (cl) { + case _IntConst: + case _SliceConst: + case _GlobVar: + case _Call: + case _CallInd: + case _Tuple: + case _UnTuple: { + // left = EXEC right; + if (!next_var_info.count_used(left) && !impure()) { + // all variables in `left` are not needed + if (edit) { + set_disabled(); + } + return std_compute_used_vars(true); + } + if (cl == _Call && does_function_always_throw(f_sym)) { + VarDescrList new_var_info; // empty, not next->var_info + if (args.size() == right.size()) { + for (const VarDescr& arg : args) { + new_var_info.add_var(arg.idx, arg.is_unused()); + } + } else { + new_var_info.add_vars(right, false); + } + return set_var_info(std::move(new_var_info)); + } + return std_compute_used_vars(); + } + case _SetGlob: { + // GLOB = right + if (right.empty() && edit) { + set_disabled(); + } + return std_compute_used_vars(right.empty()); + } + case _Let: { + // left = right + std::size_t cnt = next_var_info.count_used(left); + tolk_assert(left.size() == right.size()); + auto l_it = left.cbegin(), r_it = right.cbegin(); + VarDescrList new_var_info{next_var_info}; + new_var_info -= left; + new_var_info.clear_last(); + std::vector new_left, new_right; + for (; l_it < left.cend(); ++l_it, ++r_it) { + if (std::find(l_it + 1, left.cend(), *l_it) == left.cend()) { + auto p = next_var_info[*l_it]; + new_var_info.add_var(*r_it, edit && (!p || p->is_unused())); + new_left.push_back(*l_it); + new_right.push_back(*r_it); + } + } + if (new_left.size() < left.size()) { + left = std::move(new_left); + right = std::move(new_right); + } + if (!cnt && edit) { + // all variables in `left` are not needed + set_disabled(); + } + return set_var_info(std::move(new_var_info)); + } + case _Return: { + // return left + if (var_info.count(left) == left.size()) { + return false; + } + std::vector unique_vars = sort_unique_vars(left); + var_info.list.clear(); + for (var_idx_t i : unique_vars) { + var_info.list.emplace_back(i, VarDescr::_Last); + } + return true; + } + case _Import: { + // import left + std::vector unique_vars = sort_unique_vars(left); + var_info.list.clear(); + for (var_idx_t i : unique_vars) { + var_info.list.emplace_back(i, next_var_info[i] ? 0 : VarDescr::_Last); + } + return true; + } + case _If: { + // if (left) then block0 else block1 + // VarDescrList nx_var_info = next_var_info; + // nx_var_info.clear_last(); + code.compute_used_code_vars(block0, next_var_info, edit); + VarDescrList merge_info; + if (block1) { + code.compute_used_code_vars(block1, next_var_info, edit); + merge_info = block0->var_info + block1->var_info; + } else { + merge_info = block0->var_info + next_var_info; + } + merge_info.clear_last(); + merge_info += left; + return set_var_info(std::move(merge_info)); + } + case _While: { + // while (block0 || left) block1; + // ... block0 left { block1 block0 left } next + VarDescrList new_var_info{next_var_info}; + bool changes = false; + do { + VarDescrList after_cond{new_var_info}; + after_cond += left; + code.compute_used_code_vars(block0, after_cond, changes); + code.compute_used_code_vars(block1, block0->var_info, changes); + std::size_t n = new_var_info.size(); + new_var_info += block1->var_info; + new_var_info.clear_last(); + if (changes) { + break; + } + changes = (new_var_info.size() == n); + } while (changes <= edit); + new_var_info += left; + code.compute_used_code_vars(block0, new_var_info, edit); + return set_var_info(block0->var_info); + } + case _Until: { + // until (block0 || left); + // .. { block0 left } block0 left next + VarDescrList after_cond_first{next_var_info}; + after_cond_first += left; + code.compute_used_code_vars(block0, after_cond_first, false); + VarDescrList new_var_info{block0->var_info}; + bool changes = false; + do { + VarDescrList after_cond{new_var_info}; + after_cond += next_var_info; + after_cond += left; + code.compute_used_code_vars(block0, after_cond, changes); + std::size_t n = new_var_info.size(); + new_var_info += block0->var_info; + new_var_info.clear_last(); + if (changes) { + break; + } + changes = (new_var_info.size() == n); + } while (changes <= edit); + return set_var_info(std::move(new_var_info) + next_var_info); + } + case _Repeat: { + // repeat (left) block0 + // left { block0 } next + VarDescrList new_var_info{next_var_info}; + bool changes = false; + do { + code.compute_used_code_vars(block0, new_var_info, changes); + std::size_t n = new_var_info.size(); + new_var_info += block0->var_info; + new_var_info.clear_last(); + if (changes) { + break; + } + changes = (new_var_info.size() == n); + } while (changes <= edit); + tolk_assert(left.size() == 1); + bool last = new_var_info.count_used(left) == 0; + new_var_info += left; + if (last) { + new_var_info[left[0]]->flags |= VarDescr::_Last; + } + return set_var_info(std::move(new_var_info)); + } + case _Again: { + // for(;;) block0 + // { block0 } + VarDescrList new_var_info; + bool changes = false; + do { + code.compute_used_code_vars(block0, new_var_info, changes); + std::size_t n = new_var_info.size(); + new_var_info += block0->var_info; + new_var_info.clear_last(); + if (changes) { + break; + } + changes = (new_var_info.size() == n); + } while (changes <= edit); + return set_var_info(std::move(new_var_info)); + } + case _TryCatch: { + code.compute_used_code_vars(block0, next_var_info, edit); + code.compute_used_code_vars(block1, next_var_info, edit); + VarDescrList merge_info = block0->var_info + block1->var_info + next_var_info; + merge_info -= left; + merge_info.clear_last(); + return set_var_info(std::move(merge_info)); + } + default: + std::cerr << "fatal: unknown operation in compute_used_vars()\n"; + throw ParseError{where, "unknown operation"}; + } +} + +bool prune_unreachable(std::unique_ptr& ops) { + if (!ops) { + return true; + } + Op& op = *ops; + if (op.cl == Op::_Nop) { + if (op.next) { + ops = std::move(op.next); + return prune_unreachable(ops); + } + return true; + } + bool reach; + switch (op.cl) { + case Op::_IntConst: + case Op::_SliceConst: + case Op::_GlobVar: + case Op::_SetGlob: + case Op::_CallInd: + case Op::_Tuple: + case Op::_UnTuple: + case Op::_Import: + case Op::_Let: + reach = true; + break; + case Op::_Return: + reach = false; + break; + case Op::_Call: + reach = !does_function_always_throw(op.f_sym); + break; + case Op::_If: { + // if left then block0 else block1; ... + VarDescr* c_var = op.var_info[op.left[0]]; + if (c_var && c_var->always_true()) { + op.block0->last().next = std::move(op.next); + ops = std::move(op.block0); + return prune_unreachable(ops); + } else if (c_var && c_var->always_false()) { + op.block1->last().next = std::move(op.next); + ops = std::move(op.block1); + return prune_unreachable(ops); + } else { + reach = static_cast(prune_unreachable(op.block0)) | static_cast(prune_unreachable(op.block1)); + } + break; + } + case Op::_While: { + // while (block0 || left) block1; + if (!prune_unreachable(op.block0)) { + // computation of block0 never returns + ops = std::move(op.block0); + return prune_unreachable(ops); + } + VarDescr* c_var = op.block0->last().var_info[op.left[0]]; + if (c_var && c_var->always_false()) { + // block1 never executed + op.block0->last().next = std::move(op.next); + ops = std::move(op.block0); + return prune_unreachable(ops); + } else if (c_var && c_var->always_true()) { + if (!prune_unreachable(op.block1)) { + // block1 never returns + op.block0->last().next = std::move(op.block1); + ops = std::move(op.block0); + return false; + } + // infinite loop + op.cl = Op::_Again; + op.block0->last().next = std::move(op.block1); + op.left.clear(); + reach = false; + } else { + if (!prune_unreachable(op.block1)) { + // block1 never returns, while equivalent to block0 ; if left then block1 else next + op.cl = Op::_If; + std::unique_ptr new_op = std::move(op.block0); + op.block0 = std::move(op.block1); + op.block1 = std::make_unique(op.next->where, Op::_Nop); + new_op->last().next = std::move(ops); + ops = std::move(new_op); + } + reach = true; // block1 may be never executed + } + break; + } + case Op::_Repeat: { + // repeat (left) block0 + VarDescr* c_var = op.var_info[op.left[0]]; + if (c_var && c_var->always_nonpos()) { + // loop never executed + ops = std::move(op.next); + return prune_unreachable(ops); + } + if (c_var && c_var->always_pos()) { + if (!prune_unreachable(op.block0)) { + // block0 executed at least once, and it never returns + // replace code with block0 + ops = std::move(op.block0); + return false; + } + } else { + prune_unreachable(op.block0); + } + reach = true; + break; + } + case Op::_Until: + case Op::_Again: { + // do block0 until left; ... + if (!prune_unreachable(op.block0)) { + // block0 never returns, replace loop by block0 + ops = std::move(op.block0); + return false; + } + reach = (op.cl != Op::_Again); + break; + } + case Op::_TryCatch: { + reach = static_cast(prune_unreachable(op.block0)) | static_cast(prune_unreachable(op.block1)); + break; + } + default: + std::cerr << "fatal: unknown operation \n"; + throw ParseError{op.where, "unknown operation in prune_unreachable()"}; + } + if (reach) { + return prune_unreachable(op.next); + } else { + while (op.next->next) { + op.next = std::move(op.next->next); + } + return false; + } +} + +void CodeBlob::prune_unreachable_code() { + if (prune_unreachable(ops)) { + throw ParseError{loc, "control reaches end of function"}; + } +} + +void CodeBlob::fwd_analyze() { + VarDescrList values; + tolk_assert(ops && ops->cl == Op::_Import); + for (var_idx_t i : ops->left) { + values += i; + if (vars[i].v_type == TypeDataInt::create()) { + values[i]->val |= VarDescr::_Int; + } + } + ops->fwd_analyze(values); +} + +void Op::prepare_args(VarDescrList values) { + if (args.size() != right.size()) { + args.clear(); + for (var_idx_t i : right) { + args.emplace_back(i); + } + } + for (std::size_t i = 0; i < right.size(); i++) { + const VarDescr* val = values[right[i]]; + if (val) { + args[i].set_value(*val); + // args[i].clear_unused(); + } else { + args[i].clear_value(); + } + args[i].clear_unused(); + } +} + +VarDescrList Op::fwd_analyze(VarDescrList values) { + var_info.import_values(values); + switch (cl) { + case _Nop: + case _Import: + break; + case _Return: + values.set_unreachable(); + break; + case _IntConst: { + values.add_newval(left[0]).set_const(int_const); + break; + } + case _SliceConst: { + values.add_newval(left[0]).set_const(str_const); + break; + } + case _Call: { + prepare_args(values); + if (!f_sym->is_code_function()) { + std::vector res; + res.reserve(left.size()); + for (var_idx_t i : left) { + res.emplace_back(i); + } + AsmOpList tmp; + if (f_sym->is_asm_function()) { + std::get(f_sym->body)->compile(tmp); // abstract interpretation of res := f (args) + } else { + std::get(f_sym->body)->compile(tmp, res, args, where); + } + int j = 0; + for (var_idx_t i : left) { + values.add_newval(i).set_value(res[j++]); + } + } else { + for (var_idx_t i : left) { + values.add_newval(i); + } + } + if (does_function_always_throw(f_sym)) { + values.set_unreachable(); + } + break; + } + case _Tuple: + case _UnTuple: + case _GlobVar: + case _CallInd: { + for (var_idx_t i : left) { + values.add_newval(i); + } + break; + } + case _SetGlob: + break; + case _Let: { + std::vector old_val; + tolk_assert(left.size() == right.size()); + for (std::size_t i = 0; i < right.size(); i++) { + const VarDescr* ov = values[right[i]]; + if (!ov && G.is_verbosity(5)) { + std::cerr << "FATAL: error in assignment at right component #" << i << " (no value for _" << right[i] << ")" + << std::endl; + for (auto x : left) { + std::cerr << '_' << x << " "; + } + std::cerr << "= "; + for (auto x : right) { + std::cerr << '_' << x << " "; + } + std::cerr << std::endl; + } + // tolk_assert(ov); + if (ov) { + old_val.push_back(*ov); + } else { + old_val.emplace_back(); + } + } + for (std::size_t i = 0; i < left.size(); i++) { + values.add_newval(left[i]).set_value(std::move(old_val[i])); + } + break; + } + case _If: { + VarDescrList val1 = block0->fwd_analyze(values); + VarDescrList val2 = block1 ? block1->fwd_analyze(std::move(values)) : std::move(values); + values = val1 | val2; + break; + } + case _Repeat: { + bool atl1 = (values[left[0]] && values[left[0]]->always_pos()); + VarDescrList next_values = block0->fwd_analyze(values); + while (true) { + VarDescrList new_values = values | next_values; + if (same_values(new_values, values)) { + break; + } + values = std::move(new_values); + next_values = block0->fwd_analyze(values); + } + if (atl1) { + values = std::move(next_values); + } + break; + } + case _While: { + auto values0 = values; + values = block0->fwd_analyze(values); + if (values[left[0]] && values[left[0]]->always_false()) { + // block1 never executed + block1->fwd_analyze(values); + break; + } + while (true) { + VarDescrList next_values = values | block0->fwd_analyze(values0 | block1->fwd_analyze(values)); + if (same_values(next_values, values)) { + break; + } + values = std::move(next_values); + } + break; + } + case _Until: + case _Again: { + while (true) { + VarDescrList next_values = values | block0->fwd_analyze(values); + if (same_values(next_values, values)) { + break; + } + values = std::move(next_values); + } + values = block0->fwd_analyze(values); + break; + } + case _TryCatch: { + VarDescrList val1 = block0->fwd_analyze(values); + VarDescrList val2 = block1->fwd_analyze(std::move(values)); + values = val1 | val2; + break; + } + default: + std::cerr << "fatal: unknown operation \n"; + throw ParseError{where, "unknown operation in fwd_analyze()"}; + } + if (next) { + return next->fwd_analyze(std::move(values)); + } else { + return values; + } +} + +void Op::set_disabled(bool flag) { + if (flag) { + flags |= _Disabled; + } else { + flags &= ~_Disabled; + } +} + + +bool Op::set_noreturn(bool flag) { + if (flag) { + flags |= _NoReturn; + } else { + flags &= ~_NoReturn; + } + return flag; +} + +void Op::set_impure_flag() { + flags |= _Impure; +} + +bool Op::mark_noreturn() { + switch (cl) { + case _Nop: + if (!next) { + return set_noreturn(false); + } + // fallthrough + case _Import: + case _IntConst: + case _SliceConst: + case _Let: + case _Tuple: + case _UnTuple: + case _SetGlob: + case _GlobVar: + case _CallInd: + return set_noreturn(next->mark_noreturn()); + case _Return: + return set_noreturn(); + case _Call: + return set_noreturn(next->mark_noreturn() || does_function_always_throw(f_sym)); + case _If: + case _TryCatch: + // note, that & | (not && ||) here and below is mandatory to invoke both left and right calls + return set_noreturn((static_cast(block0->mark_noreturn()) & static_cast(block1 && block1->mark_noreturn())) | static_cast(next->mark_noreturn())); + case _Again: + block0->mark_noreturn(); + return set_noreturn(); + case _Until: + return set_noreturn(static_cast(block0->mark_noreturn()) | static_cast(next->mark_noreturn())); + case _While: + block1->mark_noreturn(); + return set_noreturn(static_cast(block0->mark_noreturn()) | static_cast(next->mark_noreturn())); + case _Repeat: + block0->mark_noreturn(); + return set_noreturn(next->mark_noreturn()); + default: + std::cerr << "fatal: unknown operation \n"; + throw ParseError{where, "unknown operation in mark_noreturn()"}; + } +} + +void CodeBlob::mark_noreturn() { + ops->mark_noreturn(); +} + +} // namespace tolk diff --git a/tolk/asmops.cpp b/tolk/asmops.cpp new file mode 100644 index 00000000..2618ed26 --- /dev/null +++ b/tolk/asmops.cpp @@ -0,0 +1,364 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" +#include + +namespace tolk { + +/* + * + * ASM-OP LIST FUNCTIONS + * + */ + +int is_pos_pow2(td::RefInt256 x) { + if (sgn(x) > 0 && !sgn(x & (x - 1))) { + return x->bit_size(false) - 1; + } else { + return -1; + } +} + +int is_neg_pow2(td::RefInt256 x) { + return sgn(x) < 0 ? is_pos_pow2(-x) : 0; +} + +std::ostream& operator<<(std::ostream& os, AsmOp::SReg stack_reg) { + int i = stack_reg.idx; + if (i >= 0) { + if (i < 16) { + return os << 's' << i; + } else { + return os << i << " s()"; + } + } else if (i >= -2) { + return os << "s(" << i << ')'; + } else { + return os << i << " s()"; + } +} + +AsmOp AsmOp::Const(int arg, const std::string& push_op) { + std::ostringstream os; + os << arg << ' ' << push_op; + return AsmOp::Const(os.str()); +} + +AsmOp AsmOp::make_stk2(int a, int b, const char* str, int delta) { + std::ostringstream os; + os << SReg(a) << ' ' << SReg(b) << ' ' << str; + int c = std::max(a, b) + 1; + return AsmOp::Custom(os.str(), c, c + delta); +} + +AsmOp AsmOp::make_stk3(int a, int b, int c, const char* str, int delta) { + std::ostringstream os; + os << SReg(a) << ' ' << SReg(b) << ' ' << SReg(c) << ' ' << str; + int m = std::max(a, std::max(b, c)) + 1; + return AsmOp::Custom(os.str(), m, m + delta); +} + +AsmOp AsmOp::BlkSwap(int a, int b) { + std::ostringstream os; + if (a == 1 && b == 1) { + return AsmOp::Xchg(0, 1); + } else if (a == 1) { + if (b == 2) { + os << "ROT"; + } else { + os << b << " ROLL"; + } + } else if (b == 1) { + if (a == 2) { + os << "-ROT"; + } else { + os << a << " -ROLL"; + } + } else { + os << a << " " << b << " BLKSWAP"; + } + return AsmOp::Custom(os.str(), a + b, a + b); +} + +AsmOp AsmOp::BlkPush(int a, int b) { + std::ostringstream os; + if (a == 1) { + return AsmOp::Push(b); + } else if (a == 2 && b == 1) { + os << "2DUP"; + } else { + os << a << " " << b << " BLKPUSH"; + } + return AsmOp::Custom(os.str(), b + 1, a + b + 1); +} + +AsmOp AsmOp::BlkDrop(int a) { + std::ostringstream os; + if (a == 1) { + return AsmOp::Pop(); + } else if (a == 2) { + os << "2DROP"; + } else { + os << a << " BLKDROP"; + } + return AsmOp::Custom(os.str(), a, 0); +} + +AsmOp AsmOp::BlkDrop2(int a, int b) { + if (!b) { + return BlkDrop(a); + } + std::ostringstream os; + os << a << " " << b << " BLKDROP2"; + return AsmOp::Custom(os.str(), a + b, b); +} + +AsmOp AsmOp::BlkReverse(int a, int b) { + std::ostringstream os; + os << a << " " << b << " REVERSE"; + return AsmOp::Custom(os.str(), a + b, a + b); +} + +AsmOp AsmOp::Tuple(int a) { + switch (a) { + case 1: + return AsmOp::Custom("SINGLE", 1, 1); + case 2: + return AsmOp::Custom("PAIR", 2, 1); + case 3: + return AsmOp::Custom("TRIPLE", 3, 1); + } + std::ostringstream os; + os << a << " TUPLE"; + return AsmOp::Custom(os.str(), a, 1); +} + +AsmOp AsmOp::UnTuple(int a) { + switch (a) { + case 1: + return AsmOp::Custom("UNSINGLE", 1, 1); + case 2: + return AsmOp::Custom("UNPAIR", 1, 2); + case 3: + return AsmOp::Custom("UNTRIPLE", 1, 3); + } + std::ostringstream os; + os << a << " UNTUPLE"; + return AsmOp::Custom(os.str(), 1, a); +} + +AsmOp AsmOp::IntConst(const td::RefInt256& x) { + if (x->signed_fits_bits(8)) { + return AsmOp::Const(dec_string(x) + " PUSHINT"); + } + if (!x->is_valid()) { + return AsmOp::Const("PUSHNAN"); + } + int k = is_pos_pow2(x); + if (k >= 0) { + return AsmOp::Const(k, "PUSHPOW2"); + } + k = is_pos_pow2(x + 1); + if (k >= 0) { + return AsmOp::Const(k, "PUSHPOW2DEC"); + } + k = is_pos_pow2(-x); + if (k >= 0) { + return AsmOp::Const(k, "PUSHNEGPOW2"); + } + if (!x->mod_pow2_short(23)) { + return AsmOp::Const(dec_string(x) + " PUSHINTX"); + } + return AsmOp::Const(dec_string(x) + " PUSHINT"); +} + +AsmOp AsmOp::BoolConst(bool f) { + return AsmOp::Const(f ? "TRUE" : "FALSE"); +} + +AsmOp AsmOp::Parse(const std::string& custom_op) { + if (custom_op == "NOP") { + return AsmOp::Nop(); + } else if (custom_op == "SWAP") { + return AsmOp::Xchg(1); + } else if (custom_op == "DROP") { + return AsmOp::Pop(0); + } else if (custom_op == "NIP") { + return AsmOp::Pop(1); + } else if (custom_op == "DUP") { + return AsmOp::Push(0); + } else if (custom_op == "OVER") { + return AsmOp::Push(1); + } else { + return AsmOp::Custom(custom_op); + } +} + +AsmOp AsmOp::Parse(std::string custom_op, int args, int retv) { + auto res = Parse(custom_op); + if (res.is_custom()) { + res.a = args; + res.b = retv; + } + return res; +} + +void AsmOp::out(std::ostream& os) const { + if (!op.empty()) { + os << op; + return; + } + switch (t) { + case a_none: + break; + case a_xchg: + if (!a && !(b & -2)) { + os << (b ? "SWAP" : "NOP"); + break; + } + os << SReg(a) << ' ' << SReg(b) << " XCHG"; + break; + case a_push: + if (!(a & -2)) { + os << (a ? "OVER" : "DUP"); + break; + } + os << SReg(a) << " PUSH"; + break; + case a_pop: + if (!(a & -2)) { + os << (a ? "NIP" : "DROP"); + break; + } + os << SReg(a) << " POP"; + break; + default: + throw Fatal{"unknown assembler operation"}; + } +} + +void AsmOp::out_indent_nl(std::ostream& os, bool no_eol) const { + for (int i = 0; i < indent; i++) { + os << " "; + } + out(os); + if (!no_eol) { + os << std::endl; + } +} + +std::string AsmOp::to_string() const { + if (!op.empty()) { + return op; + } else { + std::ostringstream os; + out(os); + return os.str(); + } +} + +bool AsmOpList::append(const std::vector& ops) { + for (const auto& op : ops) { + if (!append(op)) { + return false; + } + } + return true; +} + +const_idx_t AsmOpList::register_const(Const new_const) { + if (new_const.is_null()) { + return not_const; + } + unsigned idx; + for (idx = 0; idx < constants_.size(); idx++) { + if (!td::cmp(new_const, constants_[idx])) { + return idx; + } + } + constants_.push_back(std::move(new_const)); + return (const_idx_t)idx; +} + +Const AsmOpList::get_const(const_idx_t idx) { + if ((unsigned)idx < constants_.size()) { + return constants_[idx]; + } else { + return {}; + } +} + +void AsmOpList::show_var_ext(std::ostream& os, std::pair idx_pair) const { + var_idx_t i = idx_pair.first; + const_idx_t j = idx_pair.second; + if (!var_names_ || (unsigned)i >= var_names_->size()) { + os << '\'' << i; + } else { + var_names_->at(i).show_as_stack_comment(os); + } + if ((unsigned)j < constants_.size() && constants_[j].not_null()) { + os << '=' << constants_[j]; + } +} + +void AsmOpList::out(std::ostream& os, int mode) const { + if (!(mode & 2)) { + for (const auto& op : list_) { + op.out_indent_nl(os); + } + } else { + std::size_t n = list_.size(); + for (std::size_t i = 0; i < n; i++) { + const auto& op = list_[i]; + if (!op.is_comment() && i + 1 < n && list_[i + 1].is_comment()) { + op.out_indent_nl(os, true); + os << '\t'; + do { + i++; + } while (i + 1 < n && list_[i + 1].is_comment()); + list_[i].out(os); + os << std::endl; + } else { + op.out_indent_nl(os, false); + } + } + } +} + +bool apply_op(StackTransform& trans, const AsmOp& op) { + if (!trans.is_valid()) { + return false; + } + switch (op.t) { + case AsmOp::a_none: + return true; + case AsmOp::a_xchg: + return trans.apply_xchg(op.a, op.b, true); + case AsmOp::a_push: + return trans.apply_push(op.a); + case AsmOp::a_pop: + return trans.apply_pop(op.a); + case AsmOp::a_const: + return !op.a && op.b == 1 && trans.apply_push_newconst(); + case AsmOp::a_custom: + return op.is_gconst() && trans.apply_push_newconst(); + default: + return false; + } +} + +} // namespace tolk diff --git a/tolk/ast-from-tokens.cpp b/tolk/ast-from-tokens.cpp new file mode 100644 index 00000000..fcaa1157 --- /dev/null +++ b/tolk/ast-from-tokens.cpp @@ -0,0 +1,1172 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "ast-from-tokens.h" +#include "ast.h" +#include "type-system.h" +#include "platform-utils.h" +#include "tolk-version.h" + +/* + * Here we construct AST for a tolk file. + * While constructing, no global state is modified. + * Historically, in FunC, there was no AST: while lexing, symbols were registered, types were inferred, and so on. + * There was no way to perform any more or less semantic analysis. + * Implementing AST gives a giant advance for future modifications and stability. + */ + +namespace tolk { + +// given a token, determine whether it's <, or >, or similar +static bool is_comparison_binary_op(TokenType tok) { + return tok == tok_lt || tok == tok_gt || tok == tok_leq || tok == tok_geq || tok == tok_eq || tok == tok_neq || tok == tok_spaceship; +} + +// same as above, but to detect bitwise operators: & | ^ +static bool is_bitwise_binary_op(TokenType tok) { + return tok == tok_bitwise_and || tok == tok_bitwise_or || tok == tok_bitwise_xor; +} + +// same as above, but to detect logical operators: && || +static bool is_logical_binary_op(TokenType tok) { + return tok == tok_logical_and || tok == tok_logical_or; +} + +// same as above, but to detect addition/subtraction +static bool is_add_or_sub_binary_op(TokenType tok) { + return tok == tok_plus || tok == tok_minus; +} + +// fire an error for a case "flags & 0xFF != 0" (equivalent to "flags & 1", probably unexpected) +// it would better be a warning, but we decided to make it a strict error +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_lower_precedence(SrcLocation loc, std::string_view op_lower, std::string_view op_higher) { + std::string name_lower = static_cast(op_lower); + std::string name_higher = static_cast(op_higher); + throw ParseError(loc, name_lower + " has lower precedence than " + name_higher + + ", probably this code won't work as you expected. " + "Use parenthesis: either (... " + name_lower + " ...) to evaluate it first, or (... " + name_higher + " ...) to suppress this error."); +} + +// fire an error for a case "arg1 & arg2 | arg3" +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_mix_and_or_no_parenthesis(SrcLocation loc, std::string_view op1, std::string_view op2) { + std::string name1 = static_cast(op1); + std::string name2 = static_cast(op2); + throw ParseError(loc, "mixing " + name1 + " with " + name2 + " without parenthesis may lead to accidental errors. " + "Use parenthesis to emphasize operator precedence."); +} + +// diagnose when bitwise operators are used in a probably wrong way due to tricky precedence +// example: "flags & 0xFF != 0" is equivalent to "flags & 1", most likely it's unexpected +// the only way to suppress this error for the programmer is to use parenthesis +// (how do we detect presence of parenthesis? simple: (0!=1) is ast_parenthesized_expr{ast_binary_operator}, +// that's why if rhs->type == ast_binary_operator, it's not surrounded by parenthesis) +static void diagnose_bitwise_precedence(SrcLocation loc, std::string_view operator_name, AnyExprV lhs, AnyExprV rhs) { + // handle "flags & 0xFF != 0" (rhs = "0xFF != 0") + if (rhs->type == ast_binary_operator && is_comparison_binary_op(rhs->as()->tok)) { + fire_error_lower_precedence(loc, operator_name, rhs->as()->operator_name); + } + + // handle "0 != flags & 0xFF" (lhs = "0 != flags") + if (lhs->type == ast_binary_operator && is_comparison_binary_op(lhs->as()->tok)) { + fire_error_lower_precedence(loc, operator_name, lhs->as()->operator_name); + } +} + +// similar to above, but detect potentially invalid usage of && and || +// since anyway, using parenthesis when both && and || occur in the same expression, +// && and || have equal operator precedence in Tolk +static void diagnose_and_or_precedence(SrcLocation loc, AnyExprV lhs, TokenType rhs_tok, std::string_view rhs_operator_name) { + if (auto lhs_op = lhs->try_as()) { + // handle "arg1 & arg2 | arg3" (lhs = "arg1 & arg2") + if (is_bitwise_binary_op(lhs_op->tok) && is_bitwise_binary_op(rhs_tok) && lhs_op->tok != rhs_tok) { + fire_error_mix_and_or_no_parenthesis(loc, lhs_op->operator_name, rhs_operator_name); + } + + // handle "arg1 && arg2 || arg3" (lhs = "arg1 && arg2") + if (is_logical_binary_op(lhs_op->tok) && is_logical_binary_op(rhs_tok) && lhs_op->tok != rhs_tok) { + fire_error_mix_and_or_no_parenthesis(loc, lhs_op->operator_name, rhs_operator_name); + } + } +} + +// diagnose "a << 8 + 1" (equivalent to "a << 9", probably unexpected) +static void diagnose_addition_in_bitshift(SrcLocation loc, std::string_view bitshift_operator_name, AnyExprV rhs) { + if (rhs->type == ast_binary_operator && is_add_or_sub_binary_op(rhs->as()->tok)) { + fire_error_lower_precedence(loc, bitshift_operator_name, rhs->as()->operator_name); + } +} + +// replace (a == null) and similar to ast_is_null_check(a) (special AST vertex) +static AnyExprV maybe_replace_eq_null_with_isNull_check(V v) { + bool has_null = v->get_lhs()->type == ast_null_keyword || v->get_rhs()->type == ast_null_keyword; + bool replace = has_null && (v->tok == tok_eq || v->tok == tok_neq); + if (!replace) { + return v; + } + + AnyExprV v_nullable = v->get_lhs()->type == ast_null_keyword ? v->get_rhs() : v->get_lhs(); + return createV(v->loc, v_nullable, v->tok == tok_neq); +} + + +/* + * + * PARSE SOURCE + * + */ + + +AnyExprV parse_expr(Lexer& lex); + +static AnyV parse_parameter(Lexer& lex, bool is_first) { + SrcLocation loc = lex.cur_location(); + + // optional keyword `mutate` meaning that a function will mutate a passed argument (like passed by reference) + bool declared_as_mutate = false; + if (lex.tok() == tok_mutate) { + lex.next(); + declared_as_mutate = true; + } + + // parameter name (or underscore for an unnamed parameter) + std::string_view param_name; + if (lex.tok() == tok_identifier) { + param_name = lex.cur_str(); + } else if (lex.tok() == tok_self) { + if (!is_first) { + lex.error("`self` can only be the first parameter"); + } + param_name = "self"; + } else if (lex.tok() != tok_underscore) { + lex.unexpected("parameter name"); + } + lex.next(); + + // parameter type after colon are mandatory + lex.expect(tok_colon, "`: `"); + TypePtr param_type = parse_type_from_tokens(lex); + + return createV(loc, param_name, param_type, declared_as_mutate); +} + +static AnyV parse_global_var_declaration(Lexer& lex, const std::vector>& annotations) { + if (!annotations.empty()) { + lex.error("@annotations are not applicable to global var declaration"); + } + SrcLocation loc = lex.cur_location(); + lex.expect(tok_global, "`global`"); + lex.check(tok_identifier, "global variable name"); + auto v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + lex.expect(tok_colon, "`:`"); + TypePtr declared_type = parse_type_from_tokens(lex); + if (lex.tok() == tok_comma) { + lex.error("multiple declarations are not allowed, split globals on separate lines"); + } + if (lex.tok() == tok_assign) { + lex.error("assigning to a global is not allowed at declaration"); + } + lex.expect(tok_semicolon, "`;`"); + return createV(loc, v_ident, declared_type); +} + +static AnyV parse_constant_declaration(Lexer& lex, const std::vector>& annotations) { + if (!annotations.empty()) { + lex.error("@annotations are not applicable to global var declaration"); + } + SrcLocation loc = lex.cur_location(); + lex.expect(tok_const, "`const`"); + lex.check(tok_identifier, "constant name"); + auto v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + TypePtr declared_type = nullptr; + if (lex.tok() == tok_colon) { + lex.next(); + declared_type = parse_type_from_tokens(lex); + } + lex.expect(tok_assign, "`=`"); + AnyExprV init_value = parse_expr(lex); + if (lex.tok() == tok_comma) { + lex.error("multiple declarations are not allowed, split constants on separate lines"); + } + lex.expect(tok_semicolon, "`;`"); + return createV(loc, v_ident, declared_type, init_value); +} + +// "parameters" are at function declaration: `fun f(param1: int, mutate param2: slice)` +static V parse_parameter_list(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + std::vector params; + lex.expect(tok_oppar, "parameter list"); + if (lex.tok() != tok_clpar) { + params.push_back(parse_parameter(lex, true)); + while (lex.tok() == tok_comma) { + lex.next(); + params.push_back(parse_parameter(lex, false)); + } + } + lex.expect(tok_clpar, "`)`"); + return createV(loc, std::move(params)); +} + +// "arguments" are at function call: `f(arg1, mutate arg2)` +static AnyExprV parse_argument(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + + // keyword `mutate` is necessary when a parameter is declared `mutate` (to make mutation obvious for the reader) + bool passed_as_mutate = false; + if (lex.tok() == tok_mutate) { + lex.next(); + passed_as_mutate = true; + } + + AnyExprV expr = parse_expr(lex); + return createV(loc, expr, passed_as_mutate); +} + +static V parse_argument_list(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + std::vector args; + lex.expect(tok_oppar, "`(`"); + if (lex.tok() != tok_clpar) { + args.push_back(parse_argument(lex)); + while (lex.tok() == tok_comma) { + lex.next(); + args.push_back(parse_argument(lex)); + } + } + lex.expect(tok_clpar, "`)`"); + return createV(loc, std::move(args)); +} + +static V parse_maybe_instantiationTs_after_identifier(Lexer& lex) { + lex.check(tok_lt, "`<`"); + Lexer::SavedPositionForLookahead backup = lex.save_parsing_position(); + try { + SrcLocation loc = lex.cur_location(); + lex.next(); + std::vector instantiationTs; + instantiationTs.push_back(createV(lex.cur_location(), parse_type_from_tokens(lex))); + while (lex.tok() == tok_comma) { + lex.next(); + instantiationTs.push_back(createV(lex.cur_location(), parse_type_from_tokens(lex))); + } + lex.expect(tok_gt, "`>`"); + return createV(loc, std::move(instantiationTs)); + } catch (const ParseError&) { + lex.restore_position(backup); + return nullptr; + } +} + +// parse (expr) / [expr] / identifier / number +static AnyExprV parse_expr100(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + switch (lex.tok()) { + case tok_oppar: { + lex.next(); + if (lex.tok() == tok_clpar) { + lex.next(); + return createV(loc, {}); + } + AnyExprV first = parse_expr(lex); + if (lex.tok() == tok_clpar) { + lex.next(); + return createV(loc, first); + } + std::vector items(1, first); + while (lex.tok() == tok_comma) { + lex.next(); + items.emplace_back(parse_expr(lex)); + } + lex.expect(tok_clpar, "`)`"); + return createV(loc, std::move(items)); + } + case tok_opbracket: { + lex.next(); + if (lex.tok() == tok_clbracket) { + lex.next(); + return createV(loc, {}); + } + std::vector items(1, parse_expr(lex)); + while (lex.tok() == tok_comma) { + lex.next(); + items.emplace_back(parse_expr(lex)); + } + lex.expect(tok_clbracket, "`]`"); + return createV(loc, std::move(items)); + } + case tok_int_const: { + std::string_view orig_str = lex.cur_str(); + td::RefInt256 intval = td::string_to_int256(static_cast(orig_str)); + if (intval.is_null() || !intval->signed_fits_bits(257)) { + lex.error("invalid integer constant"); + } + lex.next(); + return createV(loc, std::move(intval), orig_str); + } + case tok_string_const: { + std::string_view str_val = lex.cur_str(); + lex.next(); + char modifier = 0; + if (lex.tok() == tok_string_modifier) { + modifier = lex.cur_str()[0]; + lex.next(); + } + return createV(loc, str_val, modifier); + } + case tok_underscore: { + lex.next(); + return createV(loc); + } + case tok_true: { + lex.next(); + return createV(loc, true); + } + case tok_false: { + lex.next(); + return createV(loc, false); + } + case tok_null: { + lex.next(); + return createV(loc); + } + case tok_self: { + lex.next(); + auto v_ident = createV(loc, "self"); + return createV(loc, v_ident, nullptr); + } + case tok_identifier: { + auto v_ident = createV(loc, lex.cur_str()); + V v_instantiationTs = nullptr; + lex.next(); + if (lex.tok() == tok_lt) { + v_instantiationTs = parse_maybe_instantiationTs_after_identifier(lex); + } + return createV(loc, v_ident, v_instantiationTs); + } + default: + lex.unexpected(""); + } +} + +// parse E(...) and E! having parsed E already (left-to-right) +static AnyExprV parse_fun_call_postfix(Lexer& lex, AnyExprV lhs) { + while (true) { + if (lex.tok() == tok_oppar) { + lhs = createV(lhs->loc, lhs, parse_argument_list(lex)); + } else if (lex.tok() == tok_logical_not) { + lex.next(); + lhs = createV(lhs->loc, lhs); + } else { + break; + } + } + return lhs; +} + +// parse E(...) and E! (left-to-right) +static AnyExprV parse_expr90(Lexer& lex) { + AnyExprV res = parse_expr100(lex); + if (lex.tok() == tok_oppar || lex.tok() == tok_logical_not) { + res = parse_fun_call_postfix(lex, res); + } + return res; +} + +// parse E.field and E.method(...) and E.field! (left-to-right) +static AnyExprV parse_expr80(Lexer& lex) { + AnyExprV lhs = parse_expr90(lex); + while (lex.tok() == tok_dot) { + SrcLocation loc = lex.cur_location(); + lex.next(); + V v_ident = nullptr; + V v_instantiationTs = nullptr; + if (lex.tok() == tok_identifier) { // obj.field / obj.method + v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + if (lex.tok() == tok_lt) { + v_instantiationTs = parse_maybe_instantiationTs_after_identifier(lex); + } + } else if (lex.tok() == tok_int_const) { // obj.0 (indexed access) + v_ident = createV(lex.cur_location(), lex.cur_str()); + lex.next(); + } else { + lex.unexpected("method name"); + } + lhs = createV(loc, lhs, v_ident, v_instantiationTs); + if (lex.tok() == tok_oppar || lex.tok() == tok_logical_not) { + lhs = parse_fun_call_postfix(lex, lhs); + } + } + return lhs; +} + +// parse ! ~ - + E (unary) +static AnyExprV parse_expr75(Lexer& lex) { + TokenType t = lex.tok(); + if (t == tok_logical_not || t == tok_bitwise_not || t == tok_minus || t == tok_plus) { + SrcLocation loc = lex.cur_location(); + std::string_view operator_name = lex.cur_str(); + lex.next(); + AnyExprV rhs = parse_expr75(lex); + return createV(loc, operator_name, t, rhs); + } + return parse_expr80(lex); +} + +// parse E as +static AnyExprV parse_expr40(Lexer& lex) { + AnyExprV lhs = parse_expr75(lex); + if (lex.tok() == tok_as) { + SrcLocation loc = lex.cur_location(); + lex.next(); + TypePtr cast_to_type = parse_type_from_tokens(lex); + lhs = createV(loc, lhs, cast_to_type); + } + return lhs; +} + +// parse E * / % ^/ ~/ E (left-to-right) +static AnyExprV parse_expr30(Lexer& lex) { + AnyExprV lhs = parse_expr40(lex); + TokenType t = lex.tok(); + while (t == tok_mul || t == tok_div || t == tok_mod || t == tok_divC || t == tok_divR) { + SrcLocation loc = lex.cur_location(); + std::string_view operator_name = lex.cur_str(); + lex.next(); + AnyExprV rhs = parse_expr40(lex); + lhs = createV(loc, operator_name, t, lhs, rhs); + t = lex.tok(); + } + return lhs; +} + +// parse E + - E (left-to-right) +static AnyExprV parse_expr20(Lexer& lex) { + AnyExprV lhs = parse_expr30(lex); + TokenType t = lex.tok(); + while (t == tok_minus || t == tok_plus) { + SrcLocation loc = lex.cur_location(); + std::string_view operator_name = lex.cur_str(); + lex.next(); + AnyExprV rhs = parse_expr30(lex); + lhs = createV(loc, operator_name, t, lhs, rhs); + t = lex.tok(); + } + return lhs; +} + +// parse E << >> ~>> ^>> E (left-to-right) +static AnyExprV parse_expr17(Lexer& lex) { + AnyExprV lhs = parse_expr20(lex); + TokenType t = lex.tok(); + while (t == tok_lshift || t == tok_rshift || t == tok_rshiftC || t == tok_rshiftR) { + SrcLocation loc = lex.cur_location(); + std::string_view operator_name = lex.cur_str(); + lex.next(); + AnyExprV rhs = parse_expr20(lex); + diagnose_addition_in_bitshift(loc, operator_name, rhs); + lhs = createV(loc, operator_name, t, lhs, rhs); + t = lex.tok(); + } + return lhs; +} + +// parse E == < > <= >= != <=> E (left-to-right) +static AnyExprV parse_expr15(Lexer& lex) { + AnyExprV lhs = parse_expr17(lex); + TokenType t = lex.tok(); + if (t == tok_eq || t == tok_lt || t == tok_gt || t == tok_leq || t == tok_geq || t == tok_neq || t == tok_spaceship) { + SrcLocation loc = lex.cur_location(); + std::string_view operator_name = lex.cur_str(); + lex.next(); + AnyExprV rhs = parse_expr17(lex); + lhs = createV(loc, operator_name, t, lhs, rhs); + if (t == tok_eq || t == tok_neq) { + lhs = maybe_replace_eq_null_with_isNull_check(lhs->as()); + } + } + return lhs; +} + +// parse E & | ^ E (left-to-right) +static AnyExprV parse_expr14(Lexer& lex) { + AnyExprV lhs = parse_expr15(lex); + TokenType t = lex.tok(); + while (t == tok_bitwise_and || t == tok_bitwise_or || t == tok_bitwise_xor) { + SrcLocation loc = lex.cur_location(); + std::string_view operator_name = lex.cur_str(); + lex.next(); + AnyExprV rhs = parse_expr15(lex); + diagnose_bitwise_precedence(loc, operator_name, lhs, rhs); + diagnose_and_or_precedence(loc, lhs, t, operator_name); + lhs = createV(loc, operator_name, t, lhs, rhs); + t = lex.tok(); + } + return lhs; +} + +// parse E && || E (left-to-right) +static AnyExprV parse_expr13(Lexer& lex) { + AnyExprV lhs = parse_expr14(lex); + TokenType t = lex.tok(); + while (t == tok_logical_and || t == tok_logical_or) { + SrcLocation loc = lex.cur_location(); + std::string_view operator_name = lex.cur_str(); + lex.next(); + AnyExprV rhs = parse_expr14(lex); + diagnose_and_or_precedence(loc, lhs, t, operator_name); + lhs = createV(loc, operator_name, t, lhs, rhs); + t = lex.tok(); + } + return lhs; +} + +// parse E = += -= E and E ? E : E (right-to-left) +static AnyExprV parse_expr10(Lexer& lex) { + AnyExprV lhs = parse_expr13(lex); + TokenType t = lex.tok(); + if (t == tok_assign) { + SrcLocation loc = lex.cur_location(); + lex.next(); + AnyExprV rhs = parse_expr10(lex); + return createV(loc, lhs, rhs); + } + if (t == tok_set_plus || t == tok_set_minus || t == tok_set_mul || t == tok_set_div || + t == tok_set_mod || t == tok_set_lshift || t == tok_set_rshift || + t == tok_set_bitwise_and || t == tok_set_bitwise_or || t == tok_set_bitwise_xor) { + SrcLocation loc = lex.cur_location(); + std::string_view operator_name = lex.cur_str().substr(0, lex.cur_str().size() - 1); // "+" for += + lex.next(); + AnyExprV rhs = parse_expr10(lex); + return createV(loc, operator_name, t, lhs, rhs); + } + if (t == tok_question) { + SrcLocation loc = lex.cur_location(); + lex.next(); + AnyExprV when_true = parse_expr10(lex); + lex.expect(tok_colon, "`:`"); + AnyExprV when_false = parse_expr10(lex); + return createV(loc, lhs, when_true, when_false); + } + return lhs; +} + +AnyExprV parse_expr(Lexer& lex) { + return parse_expr10(lex); +} + +AnyV parse_statement(Lexer& lex); + +static AnyExprV parse_var_declaration_lhs(Lexer& lex, bool is_immutable) { + SrcLocation loc = lex.cur_location(); + if (lex.tok() == tok_oppar) { + lex.next(); + AnyExprV first = parse_var_declaration_lhs(lex, is_immutable); + if (lex.tok() == tok_clpar) { + lex.next(); + return first; + } + std::vector args(1, first); + while (lex.tok() == tok_comma) { + lex.next(); + args.push_back(parse_var_declaration_lhs(lex, is_immutable)); + } + lex.expect(tok_clpar, "`)`"); + return createV(loc, std::move(args)); + } + if (lex.tok() == tok_opbracket) { + lex.next(); + std::vector args(1, parse_var_declaration_lhs(lex, is_immutable)); + while (lex.tok() == tok_comma) { + lex.next(); + args.push_back(parse_var_declaration_lhs(lex, is_immutable)); + } + lex.expect(tok_clbracket, "`]`"); + return createV(loc, std::move(args)); + } + if (lex.tok() == tok_identifier) { + auto v_ident = createV(loc, lex.cur_str()); + TypePtr declared_type = nullptr; + bool marked_as_redef = false; + lex.next(); + if (lex.tok() == tok_colon) { + lex.next(); + declared_type = parse_type_from_tokens(lex); + } else if (lex.tok() == tok_redef) { + lex.next(); + marked_as_redef = true; + } + return createV(loc, v_ident, declared_type, is_immutable, marked_as_redef); + } + if (lex.tok() == tok_underscore) { + TypePtr declared_type = nullptr; + lex.next(); + if (lex.tok() == tok_colon) { + lex.next(); + declared_type = parse_type_from_tokens(lex); + } + return createV(loc, createV(loc, ""), declared_type, true, false); + } + lex.unexpected("variable name"); +} + +static AnyV parse_local_vars_declaration_assignment(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + bool is_immutable = lex.tok() == tok_val; + lex.next(); + + AnyExprV lhs = createV(loc, parse_var_declaration_lhs(lex, is_immutable)); + if (lex.tok() != tok_assign) { + lex.error("variables declaration must be followed by assignment: `var xxx = ...`"); + } + lex.next(); + AnyExprV rhs = parse_expr(lex); + + if (lex.tok() == tok_comma) { + lex.error("multiple declarations are not allowed, split variables on separate lines"); + } + lex.expect(tok_semicolon, "`;`"); + return createV(loc, lhs, rhs); +} + +static V parse_sequence(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_opbrace, "`{`"); + std::vector items; + while (lex.tok() != tok_clbrace) { + items.push_back(parse_statement(lex)); + } + SrcLocation loc_end = lex.cur_location(); + lex.expect(tok_clbrace, "`}`"); + return createV(loc, loc_end, items); +} + +static AnyV parse_return_statement(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_return, "`return`"); + AnyExprV child = lex.tok() == tok_semicolon // `return;` actually means "nothing" (inferred as void) + ? createV(lex.cur_location()) + : parse_expr(lex); + lex.expect(tok_semicolon, "`;`"); + return createV(loc, child); +} + +static AnyV parse_if_statement(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_if, "`if`"); + + lex.expect(tok_oppar, "`(`"); + AnyExprV cond = parse_expr(lex); + lex.expect(tok_clpar, "`)`"); + + V if_body = parse_sequence(lex); + V else_body = nullptr; + if (lex.tok() == tok_else) { // else if(e) { } or else { } + lex.next(); + if (lex.tok() == tok_if) { + AnyV v_inner_if = parse_if_statement(lex); + else_body = createV(v_inner_if->loc, lex.cur_location(), {v_inner_if}); + } else { + else_body = parse_sequence(lex); + } + } else { // no 'else', create empty block + else_body = createV(lex.cur_location(), lex.cur_location(), {}); + } + return createV(loc, false, cond, if_body, else_body); +} + +static AnyV parse_repeat_statement(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_repeat, "`repeat`"); + lex.expect(tok_oppar, "`(`"); + AnyExprV cond = parse_expr(lex); + lex.expect(tok_clpar, "`)`"); + V body = parse_sequence(lex); + return createV(loc, cond, body); +} + +static AnyV parse_while_statement(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_while, "`while`"); + lex.expect(tok_oppar, "`(`"); + AnyExprV cond = parse_expr(lex); + lex.expect(tok_clpar, "`)`"); + V body = parse_sequence(lex); + return createV(loc, cond, body); +} + +static AnyV parse_do_while_statement(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_do, "`do`"); + V body = parse_sequence(lex); + lex.expect(tok_while, "`while`"); + lex.expect(tok_oppar, "`(`"); + AnyExprV cond = parse_expr(lex); + lex.expect(tok_clpar, "`)`"); + lex.expect(tok_semicolon, "`;`"); + return createV(loc, body, cond); +} + +static AnyExprV parse_catch_variable(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + if (lex.tok() == tok_identifier) { + std::string_view var_name = lex.cur_str(); + lex.next(); + auto v_ident = createV(loc, var_name); + return createV(loc, v_ident, nullptr); + } + if (lex.tok() == tok_underscore) { + lex.next(); + auto v_ident = createV(loc, ""); + return createV(loc, v_ident, nullptr); + } + lex.unexpected("identifier"); +} + +static AnyExprV create_catch_underscore_variable(const Lexer& lex) { + auto v_ident = createV(lex.cur_location(), ""); + return createV(lex.cur_location(), v_ident, nullptr); +} + +static AnyV parse_throw_statement(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_throw, "`throw`"); + + AnyExprV thrown_code, thrown_arg; + if (lex.tok() == tok_oppar) { // throw (code) or throw (code, arg) + lex.next(); + thrown_code = parse_expr(lex); + if (lex.tok() == tok_comma) { + lex.next(); + thrown_arg = parse_expr(lex); + } else { + thrown_arg = createV(loc); + } + lex.expect(tok_clpar, "`)`"); + } else { // throw code + thrown_code = parse_expr(lex); + thrown_arg = createV(loc); + } + + lex.expect(tok_semicolon, "`;`"); + return createV(loc, thrown_code, thrown_arg); +} + +static AnyV parse_assert_statement(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_assert, "`assert`"); + + lex.expect(tok_oppar, "`(`"); + AnyExprV cond = parse_expr(lex); + AnyExprV thrown_code; + if (lex.tok() == tok_comma) { // assert(cond, code) + lex.next(); + thrown_code = parse_expr(lex); + lex.expect(tok_clpar, "`)`"); + } else { // assert(cond) throw code + lex.expect(tok_clpar, "`)`"); + lex.expect(tok_throw, "`throw excNo` after assert"); + thrown_code = parse_expr(lex); + } + + lex.expect(tok_semicolon, "`;`"); + return createV(loc, cond, thrown_code); +} + +static AnyV parse_try_catch_statement(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_try, "`try`"); + V try_body = parse_sequence(lex); + + std::vector catch_args; + lex.expect(tok_catch, "`catch`"); + SrcLocation catch_loc = lex.cur_location(); + if (lex.tok() == tok_oppar) { + lex.next(); + catch_args.push_back(parse_catch_variable(lex)); + if (lex.tok() == tok_comma) { // catch (excNo, arg) + lex.next(); + catch_args.push_back(parse_catch_variable(lex)); + } else { // catch (excNo) -> catch (excNo, _) + catch_args.push_back(create_catch_underscore_variable(lex)); + } + lex.expect(tok_clpar, "`)`"); + } else { // catch -> catch (_, _) + catch_args.push_back(create_catch_underscore_variable(lex)); + catch_args.push_back(create_catch_underscore_variable(lex)); + } + V catch_expr = createV(catch_loc, std::move(catch_args)); + + V catch_body = parse_sequence(lex); + return createV(loc, try_body, catch_expr, catch_body); +} + +AnyV parse_statement(Lexer& lex) { + switch (lex.tok()) { + case tok_var: // `var x = 0` is technically an expression, but can not appear in "any place", + case tok_val: // only as a separate declaration + return parse_local_vars_declaration_assignment(lex); + case tok_opbrace: + return parse_sequence(lex); + case tok_return: + return parse_return_statement(lex); + case tok_if: + return parse_if_statement(lex); + case tok_repeat: + return parse_repeat_statement(lex); + case tok_do: + return parse_do_while_statement(lex); + case tok_while: + return parse_while_statement(lex); + case tok_throw: + return parse_throw_statement(lex); + case tok_assert: + return parse_assert_statement(lex); + case tok_try: + return parse_try_catch_statement(lex); + case tok_semicolon: { + SrcLocation loc = lex.cur_location(); + lex.next(); + return createV(loc); + } + case tok_break: + case tok_continue: + lex.error("break/continue from loops are not supported yet"); + default: { + AnyExprV expr = parse_expr(lex); + lex.expect(tok_semicolon, "`;`"); + return expr; + } + } +} + +static AnyV parse_func_body(Lexer& lex) { + return parse_sequence(lex); +} + +static AnyV parse_asm_func_body(Lexer& lex, V param_list) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_asm, "`asm`"); + size_t n_params = param_list->size(); + if (n_params > 16) { + throw ParseError{loc, "assembler built-in function can have at most 16 arguments"}; + } + std::vector arg_order, ret_order; + if (lex.tok() == tok_oppar) { + lex.next(); + while (lex.tok() == tok_identifier || lex.tok() == tok_self) { + int arg_idx = param_list->lookup_idx(lex.cur_str()); + if (arg_idx == -1) { + lex.unexpected("parameter name"); + } + arg_order.push_back(arg_idx); + lex.next(); + } + if (lex.tok() == tok_arrow) { + lex.next(); + while (lex.tok() == tok_int_const) { + int ret_idx = std::atoi(static_cast(lex.cur_str()).c_str()); + ret_order.push_back(ret_idx); + lex.next(); + } + } + lex.expect(tok_clpar, "`)`"); + } + std::vector asm_commands; + lex.check(tok_string_const, "\"ASM COMMAND\""); + while (lex.tok() == tok_string_const) { + std::string_view asm_command = lex.cur_str(); + asm_commands.push_back(createV(lex.cur_location(), asm_command, 0)); + lex.next(); + } + lex.expect(tok_semicolon, "`;`"); + return createV(loc, std::move(arg_order), std::move(ret_order), std::move(asm_commands)); +} + +static AnyV parse_genericsT_list(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + std::vector genericsT_items; + lex.expect(tok_lt, "`<`"); + while (true) { + lex.check(tok_identifier, "T"); + std::string_view nameT = lex.cur_str(); + genericsT_items.emplace_back(createV(lex.cur_location(), nameT)); + lex.next(); + if (lex.tok() != tok_comma) { + break; + } + lex.next(); + } + lex.expect(tok_gt, "`>`"); + return createV{loc, std::move(genericsT_items)}; +} + +static V parse_annotation(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.check(tok_annotation_at, "`@`"); + std::string_view name = lex.cur_str(); + AnnotationKind kind = Vertex::parse_kind(name); + lex.next(); + + V v_arg = nullptr; + if (lex.tok() == tok_oppar) { + SrcLocation loc_args = lex.cur_location(); + lex.next(); + std::vector args; + args.push_back(parse_expr(lex)); + while (lex.tok() == tok_comma) { + lex.next(); + args.push_back(parse_expr(lex)); + } + lex.expect(tok_clpar, "`)`"); + v_arg = createV(loc_args, std::move(args)); + } + + switch (kind) { + case AnnotationKind::unknown: + throw ParseError(loc, "unknown annotation " + static_cast(name)); + case AnnotationKind::inline_simple: + case AnnotationKind::inline_ref: + case AnnotationKind::pure: + case AnnotationKind::deprecated: + if (v_arg) { + throw ParseError(v_arg->loc, "arguments aren't allowed for " + static_cast(name)); + } + v_arg = createV(loc, {}); + break; + case AnnotationKind::method_id: + if (!v_arg || v_arg->size() != 1 || v_arg->get_item(0)->type != ast_int_const) { + throw ParseError(loc, "expecting `(number)` after " + static_cast(name)); + } + break; + } + + return createV(loc, kind, v_arg); +} + +static AnyV parse_function_declaration(Lexer& lex, const std::vector>& annotations) { + SrcLocation loc = lex.cur_location(); + bool is_get_method = lex.tok() == tok_get; + lex.next(); + if (is_get_method && lex.tok() == tok_fun) { + lex.next(); // 'get f()' and 'get fun f()' both correct + } + + lex.check(tok_identifier, "function name identifier"); + + std::string_view f_name = lex.cur_str(); + bool is_entrypoint = + f_name == "main" || f_name == "onInternalMessage" || f_name == "onExternalMessage" || + f_name == "onRunTickTock" || f_name == "onSplitPrepare" || f_name == "onSplitInstall"; + bool is_FunC_entrypoint = + f_name == "recv_internal" || f_name == "recv_external" || + f_name == "run_ticktock" || f_name == "split_prepare" || f_name == "split_install"; + if (is_FunC_entrypoint) { + lex.error("this is a reserved FunC/Fift identifier; you need `onInternalMessage`"); + } + + auto v_ident = createV(lex.cur_location(), f_name); + lex.next(); + + V genericsT_list = nullptr; + if (lex.tok() == tok_lt) { // 'fun f' + genericsT_list = parse_genericsT_list(lex)->as(); + } + + V v_param_list = parse_parameter_list(lex)->as(); + bool accepts_self = !v_param_list->empty() && v_param_list->get_param(0)->param_name == "self"; + int n_mutate_params = v_param_list->get_mutate_params_count(); + + TypePtr ret_type = nullptr; + bool returns_self = false; + if (lex.tok() == tok_colon) { // : (if absent, it means "auto infer", not void) + lex.next(); + if (lex.tok() == tok_self) { + if (!accepts_self) { + lex.error("only a member function can return `self` (which accepts `self` first parameter)"); + } + lex.next(); + returns_self = true; + ret_type = TypeDataVoid::create(); + } else { + ret_type = parse_type_from_tokens(lex); + } + } + + if (is_entrypoint && (is_get_method || genericsT_list || n_mutate_params || accepts_self)) { + throw ParseError(loc, "invalid declaration of a reserved function"); + } + if (is_get_method && (genericsT_list || n_mutate_params || accepts_self)) { + throw ParseError(loc, "get methods can't have `mutate` and `self` params"); + } + + AnyV v_body = nullptr; + + if (lex.tok() == tok_builtin) { + v_body = createV(lex.cur_location()); + lex.next(); + lex.expect(tok_semicolon, "`;`"); + } else if (lex.tok() == tok_opbrace) { + v_body = parse_func_body(lex); + } else if (lex.tok() == tok_asm) { + if (!ret_type) { + lex.error("asm function must specify return type"); + } + v_body = parse_asm_func_body(lex, v_param_list); + } else { + lex.unexpected("{ function body }"); + } + + int flags = 0; + if (is_entrypoint) { + flags |= FunctionData::flagIsEntrypoint; + } + if (is_get_method) { + flags |= FunctionData::flagGetMethod; + } + if (accepts_self) { + flags |= FunctionData::flagAcceptsSelf; + } + if (returns_self) { + flags |= FunctionData::flagReturnsSelf; + } + + td::RefInt256 method_id; + for (auto v_annotation : annotations) { + switch (v_annotation->kind) { + case AnnotationKind::inline_simple: + flags |= FunctionData::flagInline; + break; + case AnnotationKind::inline_ref: + flags |= FunctionData::flagInlineRef; + break; + case AnnotationKind::pure: + flags |= FunctionData::flagMarkedAsPure; + break; + case AnnotationKind::method_id: { + if (is_get_method || genericsT_list || is_entrypoint || n_mutate_params || accepts_self) { + v_annotation->error("@method_id can be specified only for regular functions"); + } + auto v_int = v_annotation->get_arg()->get_item(0)->as(); + if (v_int->intval.is_null() || !v_int->intval->signed_fits_bits(32)) { + v_int->error("invalid integer constant"); + } + method_id = v_int->intval; + break; + } + case AnnotationKind::deprecated: + // no special handling + break; + + default: + v_annotation->error("this annotation is not applicable to functions"); + } + } + + return createV(loc, v_ident, v_param_list, v_body, ret_type, genericsT_list, std::move(method_id), flags); +} + +static AnyV parse_tolk_required_version(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.next_special(tok_semver, "semver"); // syntax: "tolk 0.6" + std::string semver = static_cast(lex.cur_str()); + lex.next(); + + // for simplicity, there is no syntax ">= version" and so on, just strict compare + if (TOLK_VERSION != semver && TOLK_VERSION != semver + ".0") { // 0.6 = 0.6.0 + loc.show_warning("the contract is written in Tolk v" + semver + ", but you use Tolk compiler v" + TOLK_VERSION + "; probably, it will lead to compilation errors or hash changes"); + } + + return createV(loc, semver); // semicolon is not necessary +} + +static AnyV parse_import_directive(Lexer& lex) { + SrcLocation loc = lex.cur_location(); + lex.expect(tok_import, "`import`"); + lex.check(tok_string_const, "source file name"); + std::string_view rel_filename = lex.cur_str(); + if (rel_filename.empty()) { + lex.error("imported file name is an empty string"); + } + auto v_str = createV(lex.cur_location(), rel_filename, 0); + lex.next(); + return createV(loc, v_str); // semicolon is not necessary +} + +// the main (exported) function +AnyV parse_src_file_to_ast(const SrcFile* file) { + std::vector toplevel_declarations; + std::vector> annotations; + Lexer lex(file); + + while (!lex.is_eof()) { + switch (lex.tok()) { + case tok_tolk: + if (!annotations.empty()) { + lex.unexpected("declaration after @annotations"); + } + toplevel_declarations.push_back(parse_tolk_required_version(lex)); + break; + case tok_import: + if (!annotations.empty()) { + lex.unexpected("declaration after @annotations"); + } + toplevel_declarations.push_back(parse_import_directive(lex)); + break; + case tok_semicolon: + if (!annotations.empty()) { + lex.unexpected("declaration after @annotations"); + } + lex.next(); // don't add ast_empty, no need + break; + + case tok_annotation_at: + annotations.push_back(parse_annotation(lex)); + break; + case tok_global: + toplevel_declarations.push_back(parse_global_var_declaration(lex, annotations)); + annotations.clear(); + break; + case tok_const: + toplevel_declarations.push_back(parse_constant_declaration(lex, annotations)); + annotations.clear(); + break; + case tok_fun: + case tok_get: + toplevel_declarations.push_back(parse_function_declaration(lex, annotations)); + annotations.clear(); + break; + + case tok_export: + case tok_struct: + case tok_enum: + case tok_operator: + case tok_infix: + lex.error("`" + static_cast(lex.cur_str()) +"` is not supported yet"); + + default: + lex.unexpected("fun or get"); + } + } + + return createV(file, std::move(toplevel_declarations)); +} + +} // namespace tolk diff --git a/tolk/ast-from-tokens.h b/tolk/ast-from-tokens.h new file mode 100644 index 00000000..39574f9c --- /dev/null +++ b/tolk/ast-from-tokens.h @@ -0,0 +1,25 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "fwd-declarations.h" + +namespace tolk { + +AnyV parse_src_file_to_ast(const SrcFile* file); + +} // namespace tolk diff --git a/tolk/ast-replacer.h b/tolk/ast-replacer.h new file mode 100644 index 00000000..5103cc92 --- /dev/null +++ b/tolk/ast-replacer.h @@ -0,0 +1,201 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "ast.h" +#include "platform-utils.h" + +/* + * A module of implementing traversing a vertex tree and replacing any vertex to another. + * For example, to replace "beginCell()" call to "begin_cell()" in a function body (in V) + * regardless of the place this call is performed, you need to iterate over all the function AST, + * to find ast_function_call(beginCell), create ast_function_call(begin_cell) instead and to replace + * a pointer inside its parent. + * Inheriting from ASTVisitor makes this task quite simple, without any boilerplate. + * + * If you need just to traverse a vertex tree without replacing vertices, + * consider another api: ast-visitor.h. + */ + +namespace tolk { + +class ASTReplacer { +protected: + GNU_ATTRIBUTE_ALWAYS_INLINE static AnyExprV replace_children(const ASTExprLeaf* v) { + return v; + } + + GNU_ATTRIBUTE_ALWAYS_INLINE AnyExprV replace_children(const ASTExprUnary* v) { + auto* v_mutable = const_cast(v); + v_mutable->child = replace(v_mutable->child); + return v_mutable; + } + + GNU_ATTRIBUTE_ALWAYS_INLINE AnyExprV replace_children(const ASTExprBinary* v) { + auto* v_mutable = const_cast(v); + v_mutable->lhs = replace(v->lhs); + v_mutable->rhs = replace(v->rhs); + return v_mutable; + } + + GNU_ATTRIBUTE_ALWAYS_INLINE AnyExprV replace_children(const ASTExprVararg* v) { + auto* v_mutable = const_cast(v); + for (AnyExprV& child : v_mutable->children) { + child = replace(child); + } + return v_mutable; + } + + GNU_ATTRIBUTE_ALWAYS_INLINE AnyV replace_children(const ASTStatementUnary* v) { + auto* v_mutable = const_cast(v); + v_mutable->child = replace(v_mutable->child); + return v_mutable; + } + + GNU_ATTRIBUTE_ALWAYS_INLINE AnyV replace_children(const ASTStatementVararg* v) { + auto* v_mutable = const_cast(v); + for (AnyV& child : v_mutable->children) { + child = replace(child); + } + return v_mutable; + } + +public: + virtual ~ASTReplacer() = default; + + virtual AnyV replace(AnyV v) = 0; + virtual AnyExprV replace(AnyExprV v) = 0; +}; + +class ASTReplacerInFunctionBody : public ASTReplacer { +protected: + using parent = ASTReplacerInFunctionBody; + + // expressions + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + virtual AnyExprV replace(V v) { return replace_children(v); } + // statements + virtual AnyV replace(V v) { return replace_children(v); } + virtual AnyV replace(V v) { return replace_children(v); } + virtual AnyV replace(V v) { return replace_children(v); } + virtual AnyV replace(V v) { return replace_children(v); } + virtual AnyV replace(V v) { return replace_children(v); } + virtual AnyV replace(V v) { return replace_children(v); } + virtual AnyV replace(V v) { return replace_children(v); } + virtual AnyV replace(V v) { return replace_children(v); } + virtual AnyV replace(V v) { return replace_children(v); } + virtual AnyV replace(V v) { return replace_children(v); } + + AnyExprV replace(AnyExprV v) final { + switch (v->type) { + case ast_empty_expression: return replace(v->as()); + case ast_parenthesized_expression: return replace(v->as()); + case ast_tensor: return replace(v->as()); + case ast_typed_tuple: return replace(v->as()); + case ast_reference: return replace(v->as()); + case ast_local_var_lhs: return replace(v->as()); + case ast_local_vars_declaration: return replace(v->as()); + case ast_int_const: return replace(v->as()); + case ast_string_const: return replace(v->as()); + case ast_bool_const: return replace(v->as()); + case ast_null_keyword: return replace(v->as()); + case ast_argument: return replace(v->as()); + case ast_argument_list: return replace(v->as()); + case ast_dot_access: return replace(v->as()); + case ast_function_call: return replace(v->as()); + case ast_underscore: return replace(v->as()); + case ast_assign: return replace(v->as()); + case ast_set_assign: return replace(v->as()); + case ast_unary_operator: return replace(v->as()); + case ast_binary_operator: return replace(v->as()); + case ast_ternary_operator: return replace(v->as()); + case ast_cast_as_operator: return replace(v->as()); + case ast_not_null_operator: return replace(v->as()); + case ast_is_null_check: return replace(v->as()); + default: + throw UnexpectedASTNodeType(v, "ASTReplacerInFunctionBody::replace"); + } + } + + AnyV replace(AnyV v) final { + switch (v->type) { + case ast_empty_statement: return replace(v->as()); + case ast_sequence: return replace(v->as()); + case ast_return_statement: return replace(v->as()); + case ast_if_statement: return replace(v->as()); + case ast_repeat_statement: return replace(v->as()); + case ast_while_statement: return replace(v->as()); + case ast_do_while_statement: return replace(v->as()); + case ast_throw_statement: return replace(v->as()); + case ast_assert_statement: return replace(v->as()); + case ast_try_catch_statement: return replace(v->as()); +#ifdef TOLK_DEBUG + case ast_asm_body: + throw UnexpectedASTNodeType(v, "ASTReplacer::replace"); +#endif + default: { + // be very careful, don't forget to handle all statements (not expressions) above! + AnyExprV as_expr = reinterpret_cast(v); + return replace(as_expr); + } + } + } + +public: + virtual bool should_visit_function(FunctionPtr fun_ref) = 0; + + void start_replacing_in_function(FunctionPtr fun_ref, V v_function) { + replace(v_function->get_body()); + } +}; + + +const std::vector& get_all_not_builtin_functions(); + +template +void replace_ast_of_all_functions() { + BodyReplacerT visitor; + for (FunctionPtr fun_ref : get_all_not_builtin_functions()) { + if (visitor.should_visit_function(fun_ref)) { + visitor.start_replacing_in_function(fun_ref, fun_ref->ast_root->as()); + } + } +} + +} // namespace tolk diff --git a/tolk/ast-replicator.h b/tolk/ast-replicator.h new file mode 100644 index 00000000..16bbbeb8 --- /dev/null +++ b/tolk/ast-replicator.h @@ -0,0 +1,263 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "ast.h" +#include "platform-utils.h" + +namespace tolk { + +class ASTReplicator { +protected: + virtual AnyV clone(AnyV v) = 0; + virtual AnyExprV clone(AnyExprV v) = 0; + virtual TypePtr clone(TypePtr) = 0; + +public: + virtual ~ASTReplicator() = default; +}; + +class ASTReplicatorFunction : public ASTReplicator { +protected: + using parent = ASTReplicatorFunction; + + std::vector clone(const std::vector& items) { + std::vector result; + result.reserve(items.size()); + for (AnyV item : items) { + result.push_back(clone(item)); + } + return result; + } + + std::vector clone(const std::vector& items) { + std::vector result; + result.reserve(items.size()); + for (AnyExprV item : items) { + result.push_back(clone(item)); + } + return result; + } + + // expressions + + virtual V clone(V v) { + return createV(v->loc); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_expr())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_items())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_items())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_identifier()), v->has_instantiationTs() ? clone(v->get_instantiationTs()) : nullptr); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_identifier()), clone(v->declared_type), v->is_immutable, v->marked_as_redef); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_expr())); + } + virtual V clone(V v) { + return createV(v->loc, v->intval, v->orig_str); + } + virtual V clone(V v) { + return createV(v->loc, v->str_val, v->modifier); + } + virtual V clone(V v) { + return createV(v->loc, v->bool_val); + } + virtual V clone(V v) { + return createV(v->loc); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_expr()), v->passed_as_mutate); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_arguments())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_obj()), clone(v->get_identifier()), v->has_instantiationTs() ? clone(v->get_instantiationTs()) : nullptr); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_callee()), clone(v->get_arg_list())); + } + virtual V clone(V v) { + return createV(v->loc); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_lhs()), clone(v->get_rhs())); + } + virtual V clone(V v) { + return createV(v->loc, v->operator_name, v->tok, clone(v->get_lhs()), clone(v->get_rhs())); + } + virtual V clone(V v) { + return createV(v->loc, v->operator_name, v->tok, clone(v->get_rhs())); + } + virtual V clone(V v) { + return createV(v->loc, v->operator_name, v->tok, clone(v->get_lhs()), clone(v->get_rhs())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_cond()), clone(v->get_when_true()), clone(v->get_when_false())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_expr()), clone(v->cast_to_type)); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_expr())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_expr()), v->is_negated); + } + + // statements + + virtual V clone(V v) { + return createV(v->loc); + } + virtual V clone(V v) { + return createV(v->loc, v->loc_end, clone(v->get_items())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_return_value())); + } + virtual V clone(V v) { + return createV(v->loc, v->is_ifnot, clone(v->get_cond()), clone(v->get_if_body()), clone(v->get_else_body())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_cond()), clone(v->get_body())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_cond()), clone(v->get_body())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_body()), clone(v->get_cond())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_thrown_code()), clone(v->get_thrown_arg())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_cond()), clone(v->get_thrown_code())); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_try_body()), clone(v->get_catch_expr()), clone(v->get_catch_body())); + } + virtual V clone(V v) { + return createV(v->loc, v->arg_order, v->ret_order, clone(v->get_asm_commands())); + } + + // other + + virtual V clone(V v) { + return createV(v->loc, v->name); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->substituted_type)); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_items())); + } + virtual V clone(V v) { + return createV(v->loc, v->param_name, clone(v->declared_type), v->declared_as_mutate); + } + virtual V clone(V v) { + return createV(v->loc, clone(v->get_params())); + } + + AnyExprV clone(AnyExprV v) final { + switch (v->type) { + case ast_empty_expression: return clone(v->as()); + case ast_parenthesized_expression: return clone(v->as()); + case ast_tensor: return clone(v->as()); + case ast_typed_tuple: return clone(v->as()); + case ast_reference: return clone(v->as()); + case ast_local_var_lhs: return clone(v->as()); + case ast_local_vars_declaration: return clone(v->as()); + case ast_int_const: return clone(v->as()); + case ast_string_const: return clone(v->as()); + case ast_bool_const: return clone(v->as()); + case ast_null_keyword: return clone(v->as()); + case ast_argument: return clone(v->as()); + case ast_argument_list: return clone(v->as()); + case ast_dot_access: return clone(v->as()); + case ast_function_call: return clone(v->as()); + case ast_underscore: return clone(v->as()); + case ast_assign: return clone(v->as()); + case ast_set_assign: return clone(v->as()); + case ast_unary_operator: return clone(v->as()); + case ast_binary_operator: return clone(v->as()); + case ast_ternary_operator: return clone(v->as()); + case ast_cast_as_operator: return clone(v->as()); + case ast_not_null_operator: return clone(v->as()); + case ast_is_null_check: return clone(v->as()); + default: + throw UnexpectedASTNodeType(v, "ASTReplicatorFunction::clone"); + } + } + + AnyV clone(AnyV v) final { + switch (v->type) { + case ast_empty_statement: return clone(v->as()); + case ast_sequence: return clone(v->as()); + case ast_return_statement: return clone(v->as()); + case ast_if_statement: return clone(v->as()); + case ast_repeat_statement: return clone(v->as()); + case ast_while_statement: return clone(v->as()); + case ast_do_while_statement: return clone(v->as()); + case ast_throw_statement: return clone(v->as()); + case ast_assert_statement: return clone(v->as()); + case ast_try_catch_statement: return clone(v->as()); + case ast_asm_body: return clone(v->as()); + // other AST nodes that can be children of ast nodes of function body + case ast_identifier: return clone(v->as()); + case ast_instantiationT_item: return clone(v->as()); + case ast_instantiationT_list: return clone(v->as()); + case ast_parameter: return clone(v->as()); + case ast_parameter_list: return clone(v->as()); + + default: { + // be very careful, don't forget to handle all statements/other (not expressions) above! + AnyExprV as_expr = reinterpret_cast(v); + return clone(as_expr); + } + } + } + + TypePtr clone(TypePtr t) override { + return t; + } + + public: + virtual V clone_function_body(V v_function) { + return createV( + v_function->loc, + clone(v_function->get_identifier()), + clone(v_function->get_param_list()), + clone(v_function->get_body()->as()), + clone(v_function->declared_return_type), + v_function->genericsT_list, + v_function->method_id, + v_function->flags + ); + } +}; + +} // namespace tolk diff --git a/tolk/ast-stringifier.h b/tolk/ast-stringifier.h new file mode 100644 index 00000000..a7f260de --- /dev/null +++ b/tolk/ast-stringifier.h @@ -0,0 +1,309 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#ifdef TOLK_DEBUG + +#include "ast.h" +#include "ast-visitor.h" +#include "type-system.h" +#include + +/* + * ASTStringifier is used to print out the whole vertex tree in a human-readable format. + * To stringify any vertex, call v->debug_print(), which uses this class. + */ + +namespace tolk { + +class ASTStringifier final : public ASTVisitor { + constexpr static std::pair name_pairs[] = { + {ast_identifier, "ast_identifier"}, + // expressions + {ast_empty_expression, "ast_empty_expression"}, + {ast_parenthesized_expression, "ast_parenthesized_expression"}, + {ast_tensor, "ast_tensor"}, + {ast_typed_tuple, "ast_typed_tuple"}, + {ast_reference, "ast_reference"}, + {ast_local_var_lhs, "ast_local_var_lhs"}, + {ast_local_vars_declaration, "ast_local_vars_declaration"}, + {ast_int_const, "ast_int_const"}, + {ast_string_const, "ast_string_const"}, + {ast_bool_const, "ast_bool_const"}, + {ast_null_keyword, "ast_null_keyword"}, + {ast_argument, "ast_argument"}, + {ast_argument_list, "ast_argument_list"}, + {ast_dot_access, "ast_dot_access"}, + {ast_function_call, "ast_function_call"}, + {ast_underscore, "ast_underscore"}, + {ast_assign, "ast_assign"}, + {ast_set_assign, "ast_set_assign"}, + {ast_unary_operator, "ast_unary_operator"}, + {ast_binary_operator, "ast_binary_operator"}, + {ast_ternary_operator, "ast_ternary_operator"}, + {ast_cast_as_operator, "ast_cast_as_operator"}, + {ast_not_null_operator, "ast_not_null_operator"}, + {ast_is_null_check, "ast_is_null_check"}, + // statements + {ast_empty_statement, "ast_empty_statement"}, + {ast_sequence, "ast_sequence"}, + {ast_return_statement, "ast_return_statement"}, + {ast_if_statement, "ast_if_statement"}, + {ast_repeat_statement, "ast_repeat_statement"}, + {ast_while_statement, "ast_while_statement"}, + {ast_do_while_statement, "ast_do_while_statement"}, + {ast_throw_statement, "ast_throw_statement"}, + {ast_assert_statement, "ast_assert_statement"}, + {ast_try_catch_statement, "ast_try_catch_statement"}, + {ast_asm_body, "ast_asm_body"}, + // other + {ast_genericsT_item, "ast_genericsT_item"}, + {ast_genericsT_list, "ast_genericsT_list"}, + {ast_instantiationT_item, "ast_instantiationT_item"}, + {ast_instantiationT_list, "ast_instantiationT_list"}, + {ast_parameter, "ast_parameter"}, + {ast_parameter_list, "ast_parameter_list"}, + {ast_annotation, "ast_annotation"}, + {ast_function_declaration, "ast_function_declaration"}, + {ast_global_var_declaration, "ast_global_var_declaration"}, + {ast_constant_declaration, "ast_constant_declaration"}, + {ast_tolk_required_version, "ast_tolk_required_version"}, + {ast_import_directive, "ast_import_directive"}, + {ast_tolk_file, "ast_tolk_file"}, + }; + + static_assert(std::size(name_pairs) == ast_tolk_file + 1, "name_pairs needs to be updated"); + + constexpr static std::pair annotation_kinds[] = { + {AnnotationKind::inline_simple, "@inline"}, + {AnnotationKind::inline_ref, "@inline_ref"}, + {AnnotationKind::method_id, "@method_id"}, + {AnnotationKind::pure, "@pure"}, + {AnnotationKind::deprecated, "@deprecated"}, + }; + + static_assert(std::size(annotation_kinds) == static_cast(AnnotationKind::unknown), "annotation_kinds needs to be updated"); + + template + constexpr static const char* ast_node_type_to_string() { + return name_pairs[node_type].second; + } + + int depth = 0; + std::string out; + bool colored = false; + + template + void handle_vertex(V v) { + out += std::string(depth * 2, ' '); + out += ast_node_type_to_string(); + if (std::string postfix = specific_str(v); !postfix.empty()) { + out += colored ? " \x1b[34m" : " // "; + out += postfix; + out += colored ? "\x1b[0m" : ""; + } + out += '\n'; + depth++; + visit_children(v); + depth--; + } + + static std::string specific_str(AnyV v) { + switch (v->type) { + case ast_identifier: + return static_cast(v->as()->name); + case ast_reference: { + std::string result(v->as()->get_name()); + if (v->as()->has_instantiationTs()) { + result += specific_str(v->as()->get_instantiationTs()); + } + return result; + } + case ast_int_const: + return static_cast(v->as()->orig_str); + case ast_string_const: + if (char modifier = v->as()->modifier) { + return "\"" + static_cast(v->as()->str_val) + "\"" + std::string(1, modifier); + } else { + return "\"" + static_cast(v->as()->str_val) + "\""; + } + case ast_bool_const: + return v->as()->bool_val ? "true" : "false"; + case ast_dot_access: { + std::string result = "." + static_cast(v->as()->get_field_name()); + if (v->as()->has_instantiationTs()) { + result += specific_str(v->as()->get_instantiationTs()); + } + return result; + } + case ast_function_call: { + std::string inner = specific_str(v->as()->get_callee()); + if (int n_args = v->as()->get_num_args()) { + return inner + "(..." + std::to_string(n_args) + ")"; + } + return inner + "()"; + } + case ast_global_var_declaration: + return static_cast(v->as()->get_identifier()->name); + case ast_constant_declaration: + return static_cast(v->as()->get_identifier()->name); + case ast_assign: + return "="; + case ast_set_assign: + return static_cast(v->as()->operator_name) + "="; + case ast_unary_operator: + return static_cast(v->as()->operator_name); + case ast_binary_operator: + return static_cast(v->as()->operator_name); + case ast_cast_as_operator: + return v->as()->cast_to_type->as_human_readable(); + case ast_sequence: + return "↓" + std::to_string(v->as()->get_items().size()); + case ast_instantiationT_item: + return v->as()->substituted_type->as_human_readable(); + case ast_if_statement: + return v->as()->is_ifnot ? "ifnot" : ""; + case ast_annotation: + return annotation_kinds[static_cast(v->as()->kind)].second; + case ast_parameter: { + std::ostringstream os; + os << v->as()->declared_type; + return static_cast(v->as()->param_name) + ": " + os.str(); + } + case ast_function_declaration: { + std::string param_names; + for (int i = 0; i < v->as()->get_num_params(); i++) { + if (!param_names.empty()) + param_names += ","; + param_names += v->as()->get_param(i)->param_name; + } + return "fun " + static_cast(v->as()->get_identifier()->name) + "(" + param_names + ")"; + } + case ast_local_var_lhs: { + std::ostringstream os; + os << (v->as()->inferred_type ? v->as()->inferred_type->as_human_readable() : v->as()->declared_type->as_human_readable()); + if (v->as()->get_name().empty()) { + return "_: " + os.str(); + } + return static_cast(v->as()->get_name()) + ":" + os.str(); + } + case ast_instantiationT_list: { + std::string result = "<"; + for (AnyV item : v->as()->get_items()) { + if (result.size() > 1) + result += ","; + result += item->as()->substituted_type->as_human_readable(); + } + return result + ">"; + } + case ast_tolk_required_version: + return static_cast(v->as()->semver); + case ast_import_directive: + return static_cast(v->as()->get_file_leaf()->str_val); + case ast_tolk_file: + return v->as()->file->rel_filename; + default: + return {}; + } + } + +public: + explicit ASTStringifier(bool colored) : colored(colored) { + } + + std::string to_string_with_children(AnyV v) { + out.clear(); + visit(v); + return std::move(out); + } + + static std::string to_string_without_children(AnyV v) { + std::string result = ast_node_type_to_string(v->type); + if (std::string postfix = specific_str(v); !postfix.empty()) { + result += ' '; + result += specific_str(v); + } + return result; + } + + static const char* ast_node_type_to_string(ASTNodeType node_type) { + return name_pairs[node_type].second; + } + + void visit(AnyV v) override { + switch (v->type) { + case ast_identifier: return handle_vertex(v->as()); + // expressions + case ast_empty_expression: return handle_vertex(v->as()); + case ast_parenthesized_expression: return handle_vertex(v->as()); + case ast_tensor: return handle_vertex(v->as()); + case ast_typed_tuple: return handle_vertex(v->as()); + case ast_reference: return handle_vertex(v->as()); + case ast_local_var_lhs: return handle_vertex(v->as()); + case ast_local_vars_declaration: return handle_vertex(v->as()); + case ast_int_const: return handle_vertex(v->as()); + case ast_string_const: return handle_vertex(v->as()); + case ast_bool_const: return handle_vertex(v->as()); + case ast_null_keyword: return handle_vertex(v->as()); + case ast_argument: return handle_vertex(v->as()); + case ast_argument_list: return handle_vertex(v->as()); + case ast_dot_access: return handle_vertex(v->as()); + case ast_function_call: return handle_vertex(v->as()); + case ast_underscore: return handle_vertex(v->as()); + case ast_assign: return handle_vertex(v->as()); + case ast_set_assign: return handle_vertex(v->as()); + case ast_unary_operator: return handle_vertex(v->as()); + case ast_binary_operator: return handle_vertex(v->as()); + case ast_ternary_operator: return handle_vertex(v->as()); + case ast_cast_as_operator: return handle_vertex(v->as()); + case ast_not_null_operator: return handle_vertex(v->as()); + case ast_is_null_check: return handle_vertex(v->as()); + // statements + case ast_empty_statement: return handle_vertex(v->as()); + case ast_sequence: return handle_vertex(v->as()); + case ast_return_statement: return handle_vertex(v->as()); + case ast_if_statement: return handle_vertex(v->as()); + case ast_repeat_statement: return handle_vertex(v->as()); + case ast_while_statement: return handle_vertex(v->as()); + case ast_do_while_statement: return handle_vertex(v->as()); + case ast_throw_statement: return handle_vertex(v->as()); + case ast_assert_statement: return handle_vertex(v->as()); + case ast_try_catch_statement: return handle_vertex(v->as()); + case ast_asm_body: return handle_vertex(v->as()); + // other + case ast_genericsT_item: return handle_vertex(v->as()); + case ast_genericsT_list: return handle_vertex(v->as()); + case ast_instantiationT_item: return handle_vertex(v->as()); + case ast_instantiationT_list: return handle_vertex(v->as()); + case ast_parameter: return handle_vertex(v->as()); + case ast_parameter_list: return handle_vertex(v->as()); + case ast_annotation: return handle_vertex(v->as()); + case ast_function_declaration: return handle_vertex(v->as()); + case ast_global_var_declaration: return handle_vertex(v->as()); + case ast_constant_declaration: return handle_vertex(v->as()); + case ast_tolk_required_version: return handle_vertex(v->as()); + case ast_import_directive: return handle_vertex(v->as()); + case ast_tolk_file: return handle_vertex(v->as()); + default: + throw UnexpectedASTNodeType(v, "ASTStringifier::visit"); + } + } +}; + +} // namespace tolk + +#endif // TOLK_DEBUG diff --git a/tolk/ast-visitor.h b/tolk/ast-visitor.h new file mode 100644 index 00000000..d697aa82 --- /dev/null +++ b/tolk/ast-visitor.h @@ -0,0 +1,194 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "ast.h" +#include "platform-utils.h" + +/* + * A module implementing base functionality of read-only traversing a vertex tree. + * Since a vertex in general doesn't store a vector of children, iterating is possible only for concrete node_type. + * E.g., for ast_if_statement, visit nodes cond, if-body and else-body. For ast_string_const, nothing. And so on. + * Visitors below are helpers to inherit from and handle specific vertex types. + * + * Note, that absence of "children" in ASTNodeBase is not a drawback. Instead, it encourages you to think + * about types and match the type system. + * + * The visitor is read-only, it does not modify visited nodes (except if you purposely call mutating methods). + * For example, if you want to replace "beginCell()" call with "begin_cell", a visitor isn't enough for you. + * To replace vertices, consider another API: ast-replacer.h. + */ + +namespace tolk { + +class ASTVisitor { +protected: + GNU_ATTRIBUTE_ALWAYS_INLINE static void visit_children(const ASTExprLeaf* v) { + static_cast(v); + } + + GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTExprUnary* v) { + visit(v->child); + } + + GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTExprBinary* v) { + visit(v->lhs); + visit(v->rhs); + } + + GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTExprVararg* v) { + for (AnyExprV child : v->children) { + visit(child); + } + } + + GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTStatementUnary* v) { + visit(v->child); + } + + GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTStatementVararg* v) { + for (AnyV child : v->children) { + visit(child); + } + } + + GNU_ATTRIBUTE_ALWAYS_INLINE static void visit_children(const ASTOtherLeaf* v) { + static_cast(v); + } + + GNU_ATTRIBUTE_ALWAYS_INLINE void visit_children(const ASTOtherVararg* v) { + for (AnyV child : v->children) { + visit(child); + } + } + + virtual void visit(AnyV v) = 0; + +public: + virtual ~ASTVisitor() = default; +}; + +class ASTVisitorFunctionBody : public ASTVisitor { +protected: + using parent = ASTVisitorFunctionBody; + + // expressions + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + // statements + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + virtual void visit(V v) { return visit_children(v); } + + void visit(AnyV v) final { + switch (v->type) { + // expressions + case ast_empty_expression: return visit(v->as()); + case ast_parenthesized_expression: return visit(v->as()); + case ast_tensor: return visit(v->as()); + case ast_typed_tuple: return visit(v->as()); + case ast_reference: return visit(v->as()); + case ast_local_var_lhs: return visit(v->as()); + case ast_local_vars_declaration: return visit(v->as()); + case ast_int_const: return visit(v->as()); + case ast_string_const: return visit(v->as()); + case ast_bool_const: return visit(v->as()); + case ast_null_keyword: return visit(v->as()); + case ast_argument: return visit(v->as()); + case ast_argument_list: return visit(v->as()); + case ast_dot_access: return visit(v->as()); + case ast_function_call: return visit(v->as()); + case ast_underscore: return visit(v->as()); + case ast_assign: return visit(v->as()); + case ast_set_assign: return visit(v->as()); + case ast_unary_operator: return visit(v->as()); + case ast_binary_operator: return visit(v->as()); + case ast_ternary_operator: return visit(v->as()); + case ast_cast_as_operator: return visit(v->as()); + case ast_not_null_operator: return visit(v->as()); + case ast_is_null_check: return visit(v->as()); + // statements + case ast_empty_statement: return visit(v->as()); + case ast_sequence: return visit(v->as()); + case ast_return_statement: return visit(v->as()); + case ast_if_statement: return visit(v->as()); + case ast_repeat_statement: return visit(v->as()); + case ast_while_statement: return visit(v->as()); + case ast_do_while_statement: return visit(v->as()); + case ast_throw_statement: return visit(v->as()); + case ast_assert_statement: return visit(v->as()); + case ast_try_catch_statement: return visit(v->as()); +#ifdef TOLK_DEBUG + case ast_asm_body: + throw UnexpectedASTNodeType(v, "ASTVisitor; forgot to filter out asm functions in should_visit_function()?"); +#endif + default: + throw UnexpectedASTNodeType(v, "ASTVisitorFunctionBody::visit"); + } + } + +public: + virtual bool should_visit_function(FunctionPtr fun_ref) = 0; + + virtual void start_visiting_function(FunctionPtr fun_ref, V v_function) { + visit(v_function->get_body()); + } +}; + + +const std::vector& get_all_not_builtin_functions(); + +template +void visit_ast_of_all_functions() { + BodyVisitorT visitor; + for (FunctionPtr fun_ref : get_all_not_builtin_functions()) { + if (visitor.should_visit_function(fun_ref)) { + visitor.start_visiting_function(fun_ref, fun_ref->ast_root->as()); + } + } +} + +} // namespace tolk diff --git a/tolk/ast.cpp b/tolk/ast.cpp new file mode 100644 index 00000000..26eaacd5 --- /dev/null +++ b/tolk/ast.cpp @@ -0,0 +1,209 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "ast.h" +#ifdef TOLK_DEBUG +#include "ast-stringifier.h" +#endif + +namespace tolk { + +static_assert(sizeof(ASTNodeBase) == 12); + +#ifdef TOLK_DEBUG + +std::string ASTNodeBase::to_debug_string(bool colored) const { + ASTStringifier s(colored); + return s.to_string_with_children(this); +} + +void ASTNodeBase::debug_print() const { + std::cerr << to_debug_string(true) << std::endl; +} + +#endif // TOLK_DEBUG + +UnexpectedASTNodeType::UnexpectedASTNodeType(AnyV v_unexpected, const char* place_where): v_unexpected(v_unexpected) { + message = "Unexpected ASTNodeType "; +#ifdef TOLK_DEBUG + message += ASTStringifier::ast_node_type_to_string(v_unexpected->type); + message += " "; +#endif + message += "in "; + message += place_where; +} + +void ASTNodeBase::error(const std::string& err_msg) const { + throw ParseError(loc, err_msg); +} + +AnnotationKind Vertex::parse_kind(std::string_view name) { + if (name == "@pure") { + return AnnotationKind::pure; + } + if (name == "@inline") { + return AnnotationKind::inline_simple; + } + if (name == "@inline_ref") { + return AnnotationKind::inline_ref; + } + if (name == "@method_id") { + return AnnotationKind::method_id; + } + if (name == "@deprecated") { + return AnnotationKind::deprecated; + } + return AnnotationKind::unknown; +} + +int Vertex::lookup_idx(std::string_view nameT) const { + for (size_t idx = 0; idx < children.size(); ++idx) { + if (children[idx] && children[idx]->as()->nameT == nameT) { + return static_cast(idx); + } + } + return -1; +} + +int Vertex::lookup_idx(std::string_view param_name) const { + for (size_t idx = 0; idx < children.size(); ++idx) { + if (children[idx] && children[idx]->as()->param_name == param_name) { + return static_cast(idx); + } + } + return -1; +} + +int Vertex::get_mutate_params_count() const { + int n = 0; + for (AnyV param : children) { + if (param->as()->declared_as_mutate) { + n++; + } + } + return n; +} + +// --------------------------------------------------------- +// "assign" methods +// +// From the user's point of view, all AST vertices are constant, fields are public, but can't be modified. +// The only way to modify a field is to call "mutate()" and then use these "assign_*" methods. +// Therefore, there is a guarantee, that all AST mutations are done via these methods, +// easily searched by usages, and there is no another way to modify any other field. + +void ASTNodeExpressionBase::assign_inferred_type(TypePtr type) { + this->inferred_type = type; +} + +void ASTNodeExpressionBase::assign_rvalue_true() { + this->is_rvalue = true; +} + +void ASTNodeExpressionBase::assign_lvalue_true() { + this->is_lvalue = true; +} + +void ASTNodeExpressionBase::assign_always_true_or_false(int flow_true_false_state) { + this->is_always_true = flow_true_false_state == 1; // see smart-casts-cfg.h + this->is_always_false = flow_true_false_state == 2; +} + +void Vertex::assign_sym(const Symbol* sym) { + this->sym = sym; +} + +void Vertex::assign_fun_ref(FunctionPtr fun_ref) { + this->fun_maybe = fun_ref; +} + +void Vertex::assign_resolved_type(TypePtr cast_to_type) { + this->cast_to_type = cast_to_type; +} + +void Vertex::assign_var_ref(GlobalVarPtr var_ref) { + this->var_ref = var_ref; +} + +void Vertex::assign_resolved_type(TypePtr declared_type) { + this->declared_type = declared_type; +} + +void Vertex::assign_const_ref(GlobalConstPtr const_ref) { + this->const_ref = const_ref; +} + +void Vertex::assign_resolved_type(TypePtr declared_type) { + this->declared_type = declared_type; +} + +void Vertex::assign_resolved_type(TypePtr substituted_type) { + this->substituted_type = substituted_type; +} + +void Vertex::assign_param_ref(LocalVarPtr param_ref) { + this->param_ref = param_ref; +} + +void Vertex::assign_resolved_type(TypePtr declared_type) { + this->declared_type = declared_type; +} + +void Vertex::assign_fun_ref(FunctionPtr fun_ref) { + this->fun_ref = fun_ref; +} + +void Vertex::assign_fun_ref(FunctionPtr fun_ref) { + this->fun_ref = fun_ref; +} + +void Vertex::assign_fun_ref(FunctionPtr fun_ref) { + this->fun_ref = fun_ref; +} + +void Vertex::assign_is_negated(bool is_negated) { + this->is_negated = is_negated; +} + +void Vertex::assign_first_unreachable(AnyV first_unreachable) { + this->first_unreachable = first_unreachable; +} + +void Vertex::assign_target(const DotTarget& target) { + this->target = target; +} + +void Vertex::assign_fun_ref(FunctionPtr fun_ref) { + this->fun_ref = fun_ref; +} + +void Vertex::assign_resolved_type(TypePtr declared_return_type) { + this->declared_return_type = declared_return_type; +} + +void Vertex::assign_var_ref(LocalVarPtr var_ref) { + this->var_ref = var_ref; +} + +void Vertex::assign_resolved_type(TypePtr declared_type) { + this->declared_type = declared_type; +} + +void Vertex::assign_src_file(const SrcFile* file) { + this->file = file; +} + +} // namespace tolk diff --git a/tolk/ast.h b/tolk/ast.h new file mode 100644 index 00000000..9b7c5d1a --- /dev/null +++ b/tolk/ast.h @@ -0,0 +1,1090 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include +#include "fwd-declarations.h" +#include "platform-utils.h" +#include "src-file.h" +#include "lexer.h" +#include "symtable.h" + +/* + * Here we introduce AST representation of Tolk source code. + * Historically, in FunC, there was no AST: while lexing, symbols were registered, types were inferred, and so on. + * There was no way to perform any more or less semantic analysis. + * In Tolk, I've implemented parsing .tolk files into AST at first, and then converting this AST + * into legacy representation (see pipe-ast-to-legacy.cpp). + * In the future, more and more code analysis will be moved out of legacy to AST-level. + * + * From the user's point of view, all AST vertices are constant. All API is based on constancy. + * Even though fields of vertex structs are public, they can't be modified, since vertices are accepted by const ref. + * Generally, there are three ways of accepting a vertex: + * * AnyV (= const ASTNodeBase*) + * the only you can do with this vertex is to see v->type (ASTNodeType) and to cast via v->as() + * * AnyExprV (= const ASTNodeExpressionBase*) + * in contains expression-specific properties (lvalue/rvalue, inferred type) + * * V (= const Vertex*) + * a specific type of vertex, you can use its fields and methods + * There is one way of creating a vertex: + * * createV(...constructor_args) (= new Vertex(...)) + * vertices are currently created on a heap, without any custom memory arena, just allocated and never deleted + * The only way to modify a field is to use "mutate()" method (drops constancy, the only point of mutation) + * and then to call "assign_*" method, like "assign_sym", "assign_src_file", etc. + * + * Having AnyV and knowing its node_type, a call + * v->as() + * will return a typed vertex. + * There is also a shorthand v->try_as() which returns V or nullptr if types don't match: + * if (auto v_int = v->try_as()) + * Note, that there casts are NOT DYNAMIC. ASTNode is not a virtual base, it has no vtable. + * So, as<...>() is just a compile-time casting, without any runtime overhead. + * + * Note, that ASTNodeBase doesn't store any vector of children. That's why there is no way to loop over + * a random (unknown) vertex. Only a concrete Vertex stores its children (if any). + * Hence, to iterate over a custom vertex (e.g., a function body), one should inherit some kind of ASTVisitor. + * Besides read-only visiting, there is a "visit and replace" pattern. + * See ast-visitor.h and ast-replacer.h. + */ + +namespace tolk { + +enum ASTNodeType { + ast_identifier, + // expressions + ast_empty_expression, + ast_parenthesized_expression, + ast_tensor, + ast_typed_tuple, + ast_reference, + ast_local_var_lhs, + ast_local_vars_declaration, + ast_int_const, + ast_string_const, + ast_bool_const, + ast_null_keyword, + ast_argument, + ast_argument_list, + ast_dot_access, + ast_function_call, + ast_underscore, + ast_assign, + ast_set_assign, + ast_unary_operator, + ast_binary_operator, + ast_ternary_operator, + ast_cast_as_operator, + ast_not_null_operator, + ast_is_null_check, + // statements + ast_empty_statement, + ast_sequence, + ast_return_statement, + ast_if_statement, + ast_repeat_statement, + ast_while_statement, + ast_do_while_statement, + ast_throw_statement, + ast_assert_statement, + ast_try_catch_statement, + ast_asm_body, + // other + ast_genericsT_item, + ast_genericsT_list, + ast_instantiationT_item, + ast_instantiationT_list, + ast_parameter, + ast_parameter_list, + ast_annotation, + ast_function_declaration, + ast_global_var_declaration, + ast_constant_declaration, + ast_tolk_required_version, + ast_import_directive, + ast_tolk_file, +}; + +enum class AnnotationKind { + inline_simple, + inline_ref, + method_id, + pure, + deprecated, + unknown, +}; + +template +struct Vertex; + +template +using V = const Vertex*; + +#define createV new Vertex + +struct UnexpectedASTNodeType final : std::exception { + AnyV v_unexpected; + std::string message; + + explicit UnexpectedASTNodeType(AnyV v_unexpected, const char* place_where); + + const char* what() const noexcept override { + return message.c_str(); + } +}; + +// --------------------------------------------------------- + +struct ASTNodeBase { + const ASTNodeType type; + const SrcLocation loc; + + ASTNodeBase(ASTNodeType type, SrcLocation loc) : type(type), loc(loc) {} + ASTNodeBase(const ASTNodeBase&) = delete; + + template + V as() const { +#ifdef TOLK_DEBUG + if (type != node_type) { + throw Fatal("v->as<...> to wrong node_type"); + } +#endif + return static_cast>(this); + } + + template + V try_as() const { + return type == node_type ? static_cast>(this) : nullptr; + } + +#ifdef TOLK_DEBUG + std::string to_debug_string() const { return to_debug_string(false); } + std::string to_debug_string(bool colored) const; + void debug_print() const; +#endif + + GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD + void error(const std::string& err_msg) const; +}; + +struct ASTNodeExpressionBase : ASTNodeBase { + friend class ASTDuplicatorFunction; + + TypePtr inferred_type = nullptr; + bool is_rvalue: 1 = false; + bool is_lvalue: 1 = false; + bool is_always_true: 1 = false; // inside `if`, `while`, ternary condition, `== null`, etc. + bool is_always_false: 1 = false; // (when expression is guaranteed to be always true or always false) + + ASTNodeExpressionBase* mutate() const { return const_cast(this); } + void assign_inferred_type(TypePtr type); + void assign_rvalue_true(); + void assign_lvalue_true(); + void assign_always_true_or_false(int flow_true_false_state); + + ASTNodeExpressionBase(ASTNodeType type, SrcLocation loc) : ASTNodeBase(type, loc) {} +}; + +struct ASTNodeStatementBase : ASTNodeBase { + ASTNodeStatementBase(ASTNodeType type, SrcLocation loc) : ASTNodeBase(type, loc) {} +}; + +struct ASTExprLeaf : ASTNodeExpressionBase { + friend class ASTVisitor; + friend class ASTReplacer; + +protected: + ASTExprLeaf(ASTNodeType type, SrcLocation loc) + : ASTNodeExpressionBase(type, loc) {} +}; + +struct ASTExprUnary : ASTNodeExpressionBase { + friend class ASTVisitor; + friend class ASTReplacer; + +protected: + AnyExprV child; + + ASTExprUnary(ASTNodeType type, SrcLocation loc, AnyExprV child) + : ASTNodeExpressionBase(type, loc), child(child) {} +}; + +struct ASTExprBinary : ASTNodeExpressionBase { + friend class ASTVisitor; + friend class ASTReplacer; + +protected: + AnyExprV lhs; + AnyExprV rhs; + + ASTExprBinary(ASTNodeType type, SrcLocation loc, AnyExprV lhs, AnyExprV rhs) + : ASTNodeExpressionBase(type, loc), lhs(lhs), rhs(rhs) {} +}; + +struct ASTExprVararg : ASTNodeExpressionBase { + friend class ASTVisitor; + friend class ASTReplacer; + +protected: + std::vector children; + + AnyExprV child(int i) const { return children.at(i); } + + ASTExprVararg(ASTNodeType type, SrcLocation loc, std::vector children) + : ASTNodeExpressionBase(type, loc), children(std::move(children)) {} + +public: + int size() const { return static_cast(children.size()); } + bool empty() const { return children.empty(); } +}; + +struct ASTStatementUnary : ASTNodeStatementBase { + friend class ASTVisitor; + friend class ASTReplacer; + +protected: + AnyV child; + + AnyExprV child_as_expr() const { return reinterpret_cast(child); } + + ASTStatementUnary(ASTNodeType type, SrcLocation loc, AnyV child) + : ASTNodeStatementBase(type, loc), child(child) {} +}; + +struct ASTStatementVararg : ASTNodeStatementBase { + friend class ASTVisitor; + friend class ASTReplacer; + +protected: + std::vector children; + + AnyExprV child_as_expr(int i) const { return reinterpret_cast(children.at(i)); } + + ASTStatementVararg(ASTNodeType type, SrcLocation loc, std::vector children) + : ASTNodeStatementBase(type, loc), children(std::move(children)) {} + +public: + int size() const { return static_cast(children.size()); } + bool empty() const { return children.empty(); } +}; + +struct ASTOtherLeaf : ASTNodeBase { + friend class ASTVisitor; + friend class ASTReplacer; + +protected: + ASTOtherLeaf(ASTNodeType type, SrcLocation loc) + : ASTNodeBase(type, loc) {} +}; + +struct ASTOtherVararg : ASTNodeBase { + friend class ASTVisitor; + friend class ASTReplacer; + +protected: + std::vector children; + + AnyExprV child_as_expr(int i) const { return reinterpret_cast(children.at(i)); } + + ASTOtherVararg(ASTNodeType type, SrcLocation loc, std::vector children) + : ASTNodeBase(type, loc), children(std::move(children)) {} + +public: + int size() const { return static_cast(children.size()); } + bool empty() const { return children.empty(); } +}; + + +template<> +// ast_identifier is "a name" in AST structure +// it's NOT a standalone expression, it's "implementation details" of other AST vertices +// example: `var x = 5` then "x" is identifier (inside local var declaration) +// example: `global g: int` then "g" is identifier +// example: `someF` is a reference, which contains identifier +// example: `someF` is a reference which contains identifier and generics instantiation +// example: `fun f()` then "f" is identifier, "" is a generics declaration +struct Vertex final : ASTOtherLeaf { + std::string_view name; // empty for underscore + + Vertex(SrcLocation loc, std::string_view name) + : ASTOtherLeaf(ast_identifier, loc) + , name(name) {} +}; + + +// +// --------------------------------------------------------- +// expressions +// + + +template<> +// ast_empty_expression is "nothing" in context of expression, it has "unknown" type +// example: `throw 123;` then "throw arg" is empty expression (opposed to `throw (123, arg)`) +struct Vertex final : ASTExprLeaf { + explicit Vertex(SrcLocation loc) + : ASTExprLeaf(ast_empty_expression, loc) {} +}; + + +template<> +// ast_parenthesized_expression is something surrounded embraced by (parenthesis) +// example: `(1)`, `((f()))` (two nested) +struct Vertex final : ASTExprUnary { + AnyExprV get_expr() const { return child; } + + Vertex(SrcLocation loc, AnyExprV expr) + : ASTExprUnary(ast_parenthesized_expression, loc, expr) {} +}; + +template<> +// ast_tensor is a set of expressions embraced by (parenthesis) +// in most languages, it's called "tuple", but in TVM, "tuple" is a TVM primitive, that's why "tensor" +// example: `(1, 2)`, `(1, (2, 3))` (nested), `()` (empty tensor) +// note, that `(1)` is not a tensor, it's a parenthesized expression +// a tensor of N elements occupies N slots on a stack (opposed to TVM tuple primitive, 1 slot) +struct Vertex final : ASTExprVararg { + const std::vector& get_items() const { return children; } + AnyExprV get_item(int i) const { return child(i); } + + Vertex(SrcLocation loc, std::vector items) + : ASTExprVararg(ast_tensor, loc, std::move(items)) {} +}; + +template<> +// ast_typed_tuple is a set of expressions in [square brackets] +// in TVM, it's a TVM tuple, that occupies 1 slot, but the compiler knows its "typed structure" +// example: `[1, x]`, `[[0]]` (nested) +// typed tuples can be assigned to N variables, like `[one, _, three] = [1,2,3]` +struct Vertex final : ASTExprVararg { + const std::vector& get_items() const { return children; } + AnyExprV get_item(int i) const { return child(i); } + + Vertex(SrcLocation loc, std::vector items) + : ASTExprVararg(ast_typed_tuple, loc, std::move(items)) {} +}; + +template<> +// ast_reference is "something that references a symbol" +// examples: `x` / `someF` / `someF` +// it's a leaf expression from traversing point of view, but actually, has children (not expressions) +// note, that both `someF()` and `someF()` are function calls, where a callee is just a reference +struct Vertex final : ASTExprLeaf { +private: + V identifier; // its name, `x` / `someF` + V instantiationTs; // not null if ``, otherwise nullptr + +public: + const Symbol* sym = nullptr; // filled on resolve or type inferring; points to local / global / function / constant + + auto get_identifier() const { return identifier; } + bool has_instantiationTs() const { return instantiationTs != nullptr; } + auto get_instantiationTs() const { return instantiationTs; } + std::string_view get_name() const { return identifier->name; } + + Vertex* mutate() const { return const_cast(this); } + void assign_sym(const Symbol* sym); + + Vertex(SrcLocation loc, V name_identifier, V instantiationTs) + : ASTExprLeaf(ast_reference, loc) + , identifier(name_identifier), instantiationTs(instantiationTs) {} +}; + +template<> +// ast_local_var_lhs is one variable inside `var` declaration +// example: `var x = 0;` then "x" is local var lhs +// example: `val (x: int, [y redef], _) = rhs` then "x" and "y" and "_" are +// it's a leaf from expression's point of view, though technically has an "identifier" child +struct Vertex final : ASTExprLeaf { +private: + V identifier; + +public: + LocalVarPtr var_ref = nullptr; // filled on resolve identifiers; for `redef` points to declared above; for underscore, name is empty + TypePtr declared_type; // not null for `var x: int = rhs`, otherwise nullptr + bool is_immutable; // declared via 'val', not 'var' + bool marked_as_redef; // var (existing_var redef, new_var: int) = ... + + V get_identifier() const { return identifier; } + std::string_view get_name() const { return identifier->name; } // empty for underscore + + Vertex* mutate() const { return const_cast(this); } + void assign_var_ref(LocalVarPtr var_ref); + void assign_resolved_type(TypePtr declared_type); + + Vertex(SrcLocation loc, V identifier, TypePtr declared_type, bool is_immutable, bool marked_as_redef) + : ASTExprLeaf(ast_local_var_lhs, loc) + , identifier(identifier), declared_type(declared_type), is_immutable(is_immutable), marked_as_redef(marked_as_redef) {} +}; + +template<> +// ast_local_vars_declaration is an expression declaring local variables on the left side of assignment +// examples: see above +// for `var (x, [y])` its expr is "tensor (local var, typed tuple (local var))" +// for assignment `var x = 5`, this node is `var x`, lhs of assignment +struct Vertex final : ASTExprUnary { + AnyExprV get_expr() const { return child; } // ast_local_var_lhs / ast_tensor / ast_typed_tuple + + Vertex(SrcLocation loc, AnyExprV expr) + : ASTExprUnary(ast_local_vars_declaration, loc, expr) {} +}; + +template<> +// ast_int_const is an integer literal +// examples: `0` / `0xFF` +// note, that `-1` is unary minus of `1` int const +struct Vertex final : ASTExprLeaf { + td::RefInt256 intval; // parsed value, 255 for "0xFF" + std::string_view orig_str; // original "0xFF"; empty for nodes generated by compiler (e.g. in constant folding) + + Vertex(SrcLocation loc, td::RefInt256 intval, std::string_view orig_str) + : ASTExprLeaf(ast_int_const, loc) + , intval(std::move(intval)) + , orig_str(orig_str) {} +}; + +template<> +// ast_string_const is a string literal in double quotes or """ when multiline +// examples: "asdf" / "Ef8zMz..."a / "to_calc_crc32_from"c +// an optional modifier specifies how a string is parsed (probably, like an integer) +// note, that TVM doesn't have strings, it has only slices, so "hello" has type slice +struct Vertex final : ASTExprLeaf { + std::string_view str_val; + char modifier; + + bool is_bitslice() const { + char m = modifier; + return m == 0 || m == 's' || m == 'a'; + } + bool is_intval() const { + char m = modifier; + return m == 'u' || m == 'h' || m == 'H' || m == 'c'; + } + + Vertex(SrcLocation loc, std::string_view str_val, char modifier) + : ASTExprLeaf(ast_string_const, loc) + , str_val(str_val), modifier(modifier) {} +}; + +template<> +// ast_bool_const is either `true` or `false` +struct Vertex final : ASTExprLeaf { + bool bool_val; + + Vertex(SrcLocation loc, bool bool_val) + : ASTExprLeaf(ast_bool_const, loc) + , bool_val(bool_val) {} +}; + +template<> +// ast_null_keyword is the `null` literal +// it should be handled with care; for instance, `null` takes special place in the type system +struct Vertex final : ASTExprLeaf { + explicit Vertex(SrcLocation loc) + : ASTExprLeaf(ast_null_keyword, loc) {} +}; + +template<> +// ast_argument is an element of an argument list of a function/method call +// example: `f(1, x)` has 2 arguments, `t.tupleFirst()` has no arguments (though `t` is passed as `self`) +// example: `f(mutate arg)` has 1 argument with `passed_as_mutate` flag +// (without `mutate` keyword, the entity "argument" could be replaced just by "any expression") +struct Vertex final : ASTExprUnary { + bool passed_as_mutate; + + AnyExprV get_expr() const { return child; } + + Vertex(SrcLocation loc, AnyExprV expr, bool passed_as_mutate) + : ASTExprUnary(ast_argument, loc, expr) + , passed_as_mutate(passed_as_mutate) {} +}; + +template<> +// ast_argument_list contains N arguments of a function/method call +struct Vertex final : ASTExprVararg { + const std::vector& get_arguments() const { return children; } + auto get_arg(int i) const { return child(i)->as(); } + + Vertex(SrcLocation loc, std::vector arguments) + : ASTExprVararg(ast_argument_list, loc, std::move(arguments)) {} +}; + +template<> +// ast_dot_access is "object before dot, identifier + optional after dot" +// examples: `tensorVar.0` / `obj.field` / `getObj().method` / `t.tupleFirst` +// from traversing point of view, it's an unary expression: only obj is expression, field name is not +// note, that `obj.method()` is a function call with "dot access `obj.method`" callee +struct Vertex final : ASTExprUnary { +private: + V identifier; // `0` / `field` / `method` + V instantiationTs; // not null if ``, otherwise nullptr + +public: + + typedef std::variant< + FunctionPtr, // for `t.tupleAt` target is `tupleAt` global function + int // for `t.0` target is "indexed access" 0 + > DotTarget; + DotTarget target = static_cast(nullptr); // filled at type inferring + + bool is_target_fun_ref() const { return std::holds_alternative(target); } + bool is_target_indexed_access() const { return std::holds_alternative(target); } + + AnyExprV get_obj() const { return child; } + auto get_identifier() const { return identifier; } + bool has_instantiationTs() const { return instantiationTs != nullptr; } + auto get_instantiationTs() const { return instantiationTs; } + std::string_view get_field_name() const { return identifier->name; } + + Vertex* mutate() const { return const_cast(this); } + void assign_target(const DotTarget& target); + + Vertex(SrcLocation loc, AnyExprV obj, V identifier, V instantiationTs) + : ASTExprUnary(ast_dot_access, loc, obj) + , identifier(identifier), instantiationTs(instantiationTs) {} +}; + +template<> +// ast_function_call is "calling some lhs with parenthesis", lhs is arbitrary expression (callee) +// example: `globalF()` then callee is reference +// example: `globalF()` then callee is reference (with instantiation Ts filled) +// example: `local_var()` then callee is reference (points to local var, filled at resolve identifiers) +// example: `getF()()` then callee is another func call (which type is TypeDataFunCallable) +// example: `obj.method()` then callee is dot access (resolved while type inferring) +struct Vertex final : ASTExprBinary { + FunctionPtr fun_maybe = nullptr; // filled while type inferring for `globalF()` / `obj.f()`; remains nullptr for `local_var()` / `getF()()` + + AnyExprV get_callee() const { return lhs; } + bool is_dot_call() const { return lhs->type == ast_dot_access; } + AnyExprV get_dot_obj() const { return lhs->as()->get_obj(); } + auto get_arg_list() const { return rhs->as(); } + int get_num_args() const { return rhs->as()->size(); } + auto get_arg(int i) const { return rhs->as()->get_arg(i); } + + Vertex* mutate() const { return const_cast(this); } + void assign_fun_ref(FunctionPtr fun_ref); + + Vertex(SrcLocation loc, AnyExprV lhs_f, V arguments) + : ASTExprBinary(ast_function_call, loc, lhs_f, arguments) {} +}; + +template<> +// ast_underscore represents `_` symbol used for left side of assignment +// example: `(cs, _) = cs.loadAndReturn()` +// though it's the only correct usage, using _ as rvalue like `var x = _;` is correct from AST point of view +// note, that for declaration `var _ = 1` underscore is a regular local var declared (with empty name) +// but for `_ = 1` (not declaration) it's underscore; it's because `var _:int` is also correct +struct Vertex final : ASTExprLeaf { + explicit Vertex(SrcLocation loc) + : ASTExprLeaf(ast_underscore, loc) {} +}; + +template<> +// ast_assign represents assignment "lhs = rhs" +// examples: `a = 4` / `var a = 4` / `(cs, b, mode) = rhs` / `f() = g()` +// note, that `a = 4` lhs is ast_reference, `var a = 4` lhs is ast_local_vars_declaration +struct Vertex final : ASTExprBinary { + AnyExprV get_lhs() const { return lhs; } + AnyExprV get_rhs() const { return rhs; } + + explicit Vertex(SrcLocation loc, AnyExprV lhs, AnyExprV rhs) + : ASTExprBinary(ast_assign, loc, lhs, rhs) {} +}; + +template<> +// ast_set_assign represents assignment-and-set operation "lhs = rhs" +// examples: `a += 4` / `b <<= c` +struct Vertex final : ASTExprBinary { + FunctionPtr fun_ref = nullptr; // filled at type inferring, points to `_+_` built-in for += + std::string_view operator_name; // without equal sign, "+" for operator += + TokenType tok; // tok_set_* + + AnyExprV get_lhs() const { return lhs; } + AnyExprV get_rhs() const { return rhs; } + + Vertex* mutate() const { return const_cast(this); } + void assign_fun_ref(FunctionPtr fun_ref); + + Vertex(SrcLocation loc, std::string_view operator_name, TokenType tok, AnyExprV lhs, AnyExprV rhs) + : ASTExprBinary(ast_set_assign, loc, lhs, rhs) + , operator_name(operator_name), tok(tok) {} +}; + +template<> +// ast_unary_operator is "some operator over one expression" +// examples: `-1` / `~found` +struct Vertex final : ASTExprUnary { + FunctionPtr fun_ref = nullptr; // filled at type inferring, points to some built-in function + std::string_view operator_name; + TokenType tok; + + AnyExprV get_rhs() const { return child; } + + Vertex* mutate() const { return const_cast(this); } + void assign_fun_ref(FunctionPtr fun_ref); + + Vertex(SrcLocation loc, std::string_view operator_name, TokenType tok, AnyExprV rhs) + : ASTExprUnary(ast_unary_operator, loc, rhs) + , operator_name(operator_name), tok(tok) {} +}; + +template<> +// ast_binary_operator is "some operator over two expressions" +// examples: `a + b` / `x & true` / `(a, b) << g()` +// note, that `a = b` is NOT a binary operator, it's ast_assign, also `a += b`, it's ast_set_assign +struct Vertex final : ASTExprBinary { + FunctionPtr fun_ref = nullptr; // filled at type inferring, points to some built-in function + std::string_view operator_name; + TokenType tok; + + AnyExprV get_lhs() const { return lhs; } + AnyExprV get_rhs() const { return rhs; } + + Vertex* mutate() const { return const_cast(this); } + void assign_fun_ref(FunctionPtr fun_ref); + + Vertex(SrcLocation loc, std::string_view operator_name, TokenType tok, AnyExprV lhs, AnyExprV rhs) + : ASTExprBinary(ast_binary_operator, loc, lhs, rhs) + , operator_name(operator_name), tok(tok) {} +}; + +template<> +// ast_ternary_operator is a traditional ternary construction +// example: `cond ? a : b` +struct Vertex final : ASTExprVararg { + AnyExprV get_cond() const { return child(0); } + AnyExprV get_when_true() const { return child(1); } + AnyExprV get_when_false() const { return child(2); } + + Vertex(SrcLocation loc, AnyExprV cond, AnyExprV when_true, AnyExprV when_false) + : ASTExprVararg(ast_ternary_operator, loc, {cond, when_true, when_false}) {} +}; + +template<> +// ast_cast_as_operator is explicit casting with "as" keyword +// examples: `arg as int` / `null as cell` / `t.tupleAt(2) as slice` +struct Vertex final : ASTExprUnary { + AnyExprV get_expr() const { return child; } + + TypePtr cast_to_type; + + Vertex* mutate() const { return const_cast(this); } + void assign_resolved_type(TypePtr cast_to_type); + + Vertex(SrcLocation loc, AnyExprV expr, TypePtr cast_to_type) + : ASTExprUnary(ast_cast_as_operator, loc, expr) + , cast_to_type(cast_to_type) {} +}; + +template<> +// ast_not_null_operator is non-null assertion: like TypeScript ! or Kotlin !! +// examples: `nullableInt!` / `getNullableBuilder()!` +struct Vertex final : ASTExprUnary { + AnyExprV get_expr() const { return child; } + + Vertex(SrcLocation loc, AnyExprV expr) + : ASTExprUnary(ast_not_null_operator, loc, expr) {} +}; + +template<> +// ast_is_null_check is an artificial vertex for "expr == null" / "expr != null" / same but null on the left +// it's created instead of a general binary expression to emphasize its purpose +struct Vertex final : ASTExprUnary { + bool is_negated; + + AnyExprV get_expr() const { return child; } + + Vertex* mutate() const { return const_cast(this); } + void assign_is_negated(bool is_negated); + + Vertex(SrcLocation loc, AnyExprV expr, bool is_negated) + : ASTExprUnary(ast_is_null_check, loc, expr) + , is_negated(is_negated) {} +}; + + +// +// --------------------------------------------------------- +// statements +// + + +template<> +// ast_empty_statement is very similar to "empty sequence" but has a special treatment +// example: `;` (just semicolon) +// example: body of `builtin` function is empty statement (not a zero sequence) +struct Vertex final : ASTStatementVararg { + explicit Vertex(SrcLocation loc) + : ASTStatementVararg(ast_empty_statement, loc, {}) {} +}; + +template<> +// ast_sequence is "some sequence of statements" +// example: function body is a sequence +// example: do while body is a sequence +struct Vertex final : ASTStatementVararg { + SrcLocation loc_end; + AnyV first_unreachable = nullptr; + + const std::vector& get_items() const { return children; } + AnyV get_item(int i) const { return children.at(i); } + + Vertex* mutate() const { return const_cast(this); } + void assign_first_unreachable(AnyV first_unreachable); + + Vertex(SrcLocation loc, SrcLocation loc_end, std::vector items) + : ASTStatementVararg(ast_sequence, loc, std::move(items)) + , loc_end(loc_end) {} +}; + +template<> +// ast_return_statement is "return something from a function" +// examples: `return a` / `return any_expr()()` / `return;` +// note, that for `return;` (without a value, meaning "void"), in AST, it's stored as empty expression +struct Vertex : ASTStatementUnary { + AnyExprV get_return_value() const { return child_as_expr(); } + bool has_return_value() const { return child->type != ast_empty_expression; } + + Vertex(SrcLocation loc, AnyExprV child) + : ASTStatementUnary(ast_return_statement, loc, child) {} +}; + +template<> +// ast_if_statement is a traditional if statement, probably followed by an else branch +// examples: `if (cond) { ... } else { ... }` / `if (cond) { ... }` +// when else branch is missing, it's stored as empty statement +// for "else if", it's just "if statement" inside a sequence of else branch +struct Vertex final : ASTStatementVararg { + bool is_ifnot; // if(!cond), to generate more optimal fift code + + AnyExprV get_cond() const { return child_as_expr(0); } + auto get_if_body() const { return children.at(1)->as(); } + auto get_else_body() const { return children.at(2)->as(); } // always exists (when else omitted, it's empty) + + Vertex(SrcLocation loc, bool is_ifnot, AnyExprV cond, V if_body, V else_body) + : ASTStatementVararg(ast_if_statement, loc, {cond, if_body, else_body}) + , is_ifnot(is_ifnot) {} +}; + +template<> +// ast_repeat_statement is "repeat something N times" +// example: `repeat (10) { ... }` +struct Vertex final : ASTStatementVararg { + AnyExprV get_cond() const { return child_as_expr(0); } + auto get_body() const { return children.at(1)->as(); } + + Vertex(SrcLocation loc, AnyExprV cond, V body) + : ASTStatementVararg(ast_repeat_statement, loc, {cond, body}) {} +}; + +template<> +// ast_while_statement is a standard "while" loop +// example: `while (x > 0) { ... }` +struct Vertex final : ASTStatementVararg { + AnyExprV get_cond() const { return child_as_expr(0); } + auto get_body() const { return children.at(1)->as(); } + + Vertex(SrcLocation loc, AnyExprV cond, V body) + : ASTStatementVararg(ast_while_statement, loc, {cond, body}) {} +}; + +template<> +// ast_do_while_statement is a standard "do while" loop +// example: `do { ... } while (x > 0);` +struct Vertex final : ASTStatementVararg { + auto get_body() const { return children.at(0)->as(); } + AnyExprV get_cond() const { return child_as_expr(1); } + + Vertex(SrcLocation loc, V body, AnyExprV cond) + : ASTStatementVararg(ast_do_while_statement, loc, {body, cond}) {} +}; + +template<> +// ast_throw_statement is throwing an exception, it accepts excNo and optional arg +// examples: `throw 10` / `throw (ERR_LOW_BALANCE)` / `throw (1001, incomingAddr)` +// when thrown arg is missing, it's stored as empty expression +struct Vertex final : ASTStatementVararg { + AnyExprV get_thrown_code() const { return child_as_expr(0); } + bool has_thrown_arg() const { return child_as_expr(1)->type != ast_empty_expression; } + AnyExprV get_thrown_arg() const { return child_as_expr(1); } + + Vertex(SrcLocation loc, AnyExprV thrown_code, AnyExprV thrown_arg) + : ASTStatementVararg(ast_throw_statement, loc, {thrown_code, thrown_arg}) {} +}; + +template<> +// ast_assert_statement is "assert that cond is true, otherwise throw an exception" +// examples: `assert (balance > 0, ERR_ZERO_BALANCE)` / `assert (balance > 0) throw (ERR_ZERO_BALANCE)` +struct Vertex final : ASTStatementVararg { + AnyExprV get_cond() const { return child_as_expr(0); } + AnyExprV get_thrown_code() const { return child_as_expr(1); } + + Vertex(SrcLocation loc, AnyExprV cond, AnyExprV thrown_code) + : ASTStatementVararg(ast_assert_statement, loc, {cond, thrown_code}) {} +}; + +template<> +// ast_try_catch_statement is a standard try catch (finally block doesn't exist) +// example: `try { ... } catch (excNo) { ... }` +// there are two formal "arguments" of catch: excNo and arg, but both can be omitted +// when omitted, they are stored as underscores, so len of a catch tensor is always 2 +struct Vertex final : ASTStatementVararg { + auto get_try_body() const { return children.at(0)->as(); } + auto get_catch_expr() const { return children.at(1)->as(); } // (excNo, arg), always len 2 + auto get_catch_body() const { return children.at(2)->as(); } + + Vertex(SrcLocation loc, V try_body, V catch_expr, V catch_body) + : ASTStatementVararg(ast_try_catch_statement, loc, {try_body, catch_expr, catch_body}) {} +}; + +template<> +// ast_asm_body is a body of `asm` function — a set of strings, and optionally stack order manipulations +// example: `fun skipMessageOp... asm "32 PUSHINT" "SDSKIPFIRST";` +// user can specify "arg order"; example: `fun store(self: builder, op: int) asm (op self)` then [1, 0] +// user can specify "ret order"; example: `fun modDiv... asm(-> 1 0) "DIVMOD";` then [1, 0] +struct Vertex final : ASTStatementVararg { + std::vector arg_order; + std::vector ret_order; + + const std::vector& get_asm_commands() const { return children; } // ast_string_const[] + + Vertex(SrcLocation loc, std::vector arg_order, std::vector ret_order, std::vector asm_commands) + : ASTStatementVararg(ast_asm_body, loc, std::move(asm_commands)) + , arg_order(std::move(arg_order)), ret_order(std::move(ret_order)) {} +}; + + +// +// --------------------------------------------------------- +// other +// + + +template<> +// ast_genericsT_item is generics T at declaration +// example: `fun f` has a list of 2 generic Ts +struct Vertex final : ASTOtherLeaf { + std::string_view nameT; + + Vertex(SrcLocation loc, std::string_view nameT) + : ASTOtherLeaf(ast_genericsT_item, loc) + , nameT(nameT) {} +}; + +template<> +// ast_genericsT_list is a container for generics T at declaration +// example: see above +struct Vertex final : ASTOtherVararg { + std::vector get_items() const { return children; } + auto get_item(int i) const { return children.at(i)->as(); } + + Vertex(SrcLocation loc, std::vector genericsT_items) + : ASTOtherVararg(ast_genericsT_list, loc, std::move(genericsT_items)) {} + + int lookup_idx(std::string_view nameT) const; +}; + + +template<> +// ast_instantiationT_item is manual substitution of generic T used in code, mostly for func calls +// examples: `g()` / `t.tupleFirst()` / `f<(int, slice), builder>()` +struct Vertex final : ASTOtherLeaf { + TypePtr substituted_type; + + Vertex* mutate() const { return const_cast(this); } + void assign_resolved_type(TypePtr substituted_type); + + Vertex(SrcLocation loc, TypePtr substituted_type) + : ASTOtherLeaf(ast_instantiationT_item, loc) + , substituted_type(substituted_type) {} +}; + +template<> +// ast_instantiationT_list is a container for generic T substitutions used in code +// examples: see above +struct Vertex final : ASTOtherVararg { + std::vector get_items() const { return children; } + auto get_item(int i) const { return children.at(i)->as(); } + + Vertex(SrcLocation loc, std::vector instantiationTs) + : ASTOtherVararg(ast_instantiationT_list, loc, std::move(instantiationTs)) {} +}; + +template<> +// ast_parameter is a parameter of a function in its declaration +// example: `fun f(a: int, mutate b: slice)` has 2 parameters +struct Vertex final : ASTOtherLeaf { + LocalVarPtr param_ref = nullptr; // filled on resolve identifiers + std::string_view param_name; + TypePtr declared_type; + bool declared_as_mutate; // declared as `mutate param_name` + + bool is_underscore() const { return param_name.empty(); } + + Vertex* mutate() const { return const_cast(this); } + void assign_param_ref(LocalVarPtr param_ref); + void assign_resolved_type(TypePtr declared_type); + + Vertex(SrcLocation loc, std::string_view param_name, TypePtr declared_type, bool declared_as_mutate) + : ASTOtherLeaf(ast_parameter, loc) + , param_name(param_name), declared_type(declared_type), declared_as_mutate(declared_as_mutate) {} +}; + +template<> +// ast_parameter_list is a container of parameters +// example: see above +struct Vertex final : ASTOtherVararg { + const std::vector& get_params() const { return children; } + auto get_param(int i) const { return children.at(i)->as(); } + + Vertex(SrcLocation loc, std::vector params) + : ASTOtherVararg(ast_parameter_list, loc, std::move(params)) {} + + int lookup_idx(std::string_view param_name) const; + int get_mutate_params_count() const; + bool has_mutate_params() const { return get_mutate_params_count() > 0; } +}; + +template<> +// ast_annotation is @annotation above a declaration +// example: `@pure fun ...` +struct Vertex final : ASTOtherVararg { + AnnotationKind kind; + + auto get_arg() const { return children.at(0)->as(); } + + static AnnotationKind parse_kind(std::string_view name); + + Vertex(SrcLocation loc, AnnotationKind kind, V arg_probably_empty) + : ASTOtherVararg(ast_annotation, loc, {arg_probably_empty}) + , kind(kind) {} +}; + +template<> +// ast_function_declaration is declaring a function/method +// methods are still global functions, just accepting "self" first parameter +// example: `fun f() { ... }` +// functions can be generic, `fun f(params) { ... }` +// their body is either sequence (regular code function), or `asm`, or `builtin` +struct Vertex final : ASTOtherVararg { + auto get_identifier() const { return children.at(0)->as(); } + int get_num_params() const { return children.at(1)->as()->size(); } + auto get_param_list() const { return children.at(1)->as(); } + auto get_param(int i) const { return children.at(1)->as()->get_param(i); } + AnyV get_body() const { return children.at(2); } // ast_sequence / ast_asm_body + + FunctionPtr fun_ref = nullptr; // filled after register + TypePtr declared_return_type; // filled at ast parsing; if unspecified (nullptr), means "auto infer" + V genericsT_list; // for non-generics it's nullptr + td::RefInt256 method_id; // specified via @method_id annotation + int flags; // from enum in FunctionData + + bool is_asm_function() const { return children.at(2)->type == ast_asm_body; } + bool is_code_function() const { return children.at(2)->type == ast_sequence; } + bool is_builtin_function() const { return children.at(2)->type == ast_empty_statement; } + + Vertex* mutate() const { return const_cast(this); } + void assign_fun_ref(FunctionPtr fun_ref); + void assign_resolved_type(TypePtr declared_return_type); + + Vertex(SrcLocation loc, V name_identifier, V parameters, AnyV body, TypePtr declared_return_type, V genericsT_list, td::RefInt256 method_id, int flags) + : ASTOtherVararg(ast_function_declaration, loc, {name_identifier, parameters, body}) + , declared_return_type(declared_return_type), genericsT_list(genericsT_list), method_id(std::move(method_id)), flags(flags) {} +}; + +template<> +// ast_global_var_declaration is declaring a global var, outside a function +// example: `global g: int;` +// note, that globals don't have default values, since there is no single "entrypoint" for a contract +struct Vertex final : ASTOtherVararg { + GlobalVarPtr var_ref = nullptr; // filled after register + TypePtr declared_type; // filled always, typing globals is mandatory + + auto get_identifier() const { return children.at(0)->as(); } + + Vertex* mutate() const { return const_cast(this); } + void assign_var_ref(GlobalVarPtr var_ref); + void assign_resolved_type(TypePtr declared_type); + + Vertex(SrcLocation loc, V name_identifier, TypePtr declared_type) + : ASTOtherVararg(ast_global_var_declaration, loc, {name_identifier}) + , declared_type(declared_type) {} +}; + +template<> +// ast_constant_declaration is declaring a global constant, outside a function +// example: `const op = 0x123;` +struct Vertex final : ASTOtherVararg { + GlobalConstPtr const_ref = nullptr; // filled after register + TypePtr declared_type; // not null for `const op: int = ...` + + auto get_identifier() const { return children.at(0)->as(); } + AnyExprV get_init_value() const { return child_as_expr(1); } + + Vertex* mutate() const { return const_cast(this); } + void assign_const_ref(GlobalConstPtr const_ref); + void assign_resolved_type(TypePtr declared_type); + + Vertex(SrcLocation loc, V name_identifier, TypePtr declared_type, AnyExprV init_value) + : ASTOtherVararg(ast_constant_declaration, loc, {name_identifier, init_value}) + , declared_type(declared_type) {} +}; + +template<> +// ast_tolk_required_version is a preamble fixating compiler's version at the top of the file +// example: `tolk 0.6` +// when compiler version mismatches, it means, that another compiler was earlier for that sources, a warning is emitted +struct Vertex final : ASTOtherLeaf { + std::string_view semver; + + Vertex(SrcLocation loc, std::string_view semver) + : ASTOtherLeaf(ast_tolk_required_version, loc) + , semver(semver) {} +}; + +template<> +// ast_import_directive is an import at the top of the file +// examples: `import "another.tolk"` / `import "@stdlib/tvm-dicts"` +struct Vertex final : ASTOtherVararg { + const SrcFile* file = nullptr; // assigned after imports have been resolved, just after parsing a file to ast + + auto get_file_leaf() const { return children.at(0)->as(); } + + std::string get_file_name() const { return static_cast(children.at(0)->as()->str_val); } + + Vertex* mutate() const { return const_cast(this); } + void assign_src_file(const SrcFile* file); + + Vertex(SrcLocation loc, V file_name) + : ASTOtherVararg(ast_import_directive, loc, {file_name}) {} +}; + +template<> +// ast_tolk_file represents a whole parsed input .tolk file +// with functions, constants, etc. +// particularly, it contains imports that lead to loading other files +// a whole program consists of multiple parsed files, each of them has a parsed ast tree (stdlib is also parsed) +struct Vertex final : ASTOtherVararg { + const SrcFile* const file; + + const std::vector& get_toplevel_declarations() const { return children; } + + Vertex(const SrcFile* file, std::vector toplevel_declarations) + : ASTOtherVararg(ast_tolk_file, SrcLocation(file), std::move(toplevel_declarations)) + , file(file) {} +}; + +} // namespace tolk diff --git a/tolk/builtins.cpp b/tolk/builtins.cpp new file mode 100644 index 00000000..cb89c984 --- /dev/null +++ b/tolk/builtins.cpp @@ -0,0 +1,1282 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" +#include "compiler-state.h" +#include "type-system.h" +#include "generics-helpers.h" + +namespace tolk { +using namespace std::literals::string_literals; + +// given func_type = `(slice, int) -> slice` and func flags, create SymLocalVarOrParameter +// currently (see at the bottom) parameters of built-in functions are unnamed: +// built-in functions are created using a resulting type +static std::vector define_builtin_parameters(const std::vector& params_types, int func_flags) { + // `loadInt()`, `storeInt()`: they accept `self` and mutate it; no other options available in built-ins for now + bool is_mutate_self = func_flags & FunctionData::flagHasMutateParams; + std::vector parameters; + parameters.reserve(params_types.size()); + + for (int i = 0; i < static_cast(params_types.size()); ++i) { + LocalVarData p_sym("", {}, params_types[i], (i == 0 && is_mutate_self) * LocalVarData::flagMutateParameter, i); + parameters.push_back(std::move(p_sym)); + } + + return parameters; +} + +static void define_builtin_func(const std::string& name, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags) { + auto* f_sym = new FunctionData(name, {}, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); + G.symtable.add_function(f_sym); +} + +static void define_builtin_func(const std::string& name, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const AsmOp& macro, int flags) { + auto* f_sym = new FunctionData(name, {}, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(make_simple_compile(macro)), nullptr); + G.symtable.add_function(f_sym); +} + +static void define_builtin_func(const std::string& name, const std::vector& params_types, TypePtr return_type, const GenericsDeclaration* genericTs, const simple_compile_func_t& func, int flags, + std::initializer_list arg_order, std::initializer_list ret_order) { + auto* f_sym = new FunctionData(name, {}, return_type, define_builtin_parameters(params_types, flags), flags, genericTs, nullptr, new FunctionBodyBuiltin(func), nullptr); + f_sym->arg_order = arg_order; + f_sym->ret_order = ret_order; + G.symtable.add_function(f_sym); +} + +void FunctionBodyBuiltin::compile(AsmOpList& dest, std::vector& out, std::vector& in, + SrcLocation where) const { + dest.append(simple_compile(out, in, where)); +} + +void FunctionBodyAsm::compile(AsmOpList& dest) const { + dest.append(ops); +} + + +/* + * + * DEFINE BUILT-IN FUNCTIONS + * + */ + +int emulate_negate(int a) { + int f = VarDescr::_Pos | VarDescr::_Neg; + if ((a & f) && (~a & f)) { + a ^= f; + } + return a; +} + +int emulate_add(int a, int b) { + if (b & VarDescr::_Zero) { + return a; + } else if (a & VarDescr::_Zero) { + return b; + } + int u = a & b, v = a | b; + int r = VarDescr::_Int; + int t = u & (VarDescr::_Pos | VarDescr::_Neg); + if (v & VarDescr::_Nan) { + return r | VarDescr::_Nan; + } + // non-quiet addition always returns finite results! + r |= t | VarDescr::_Finite; + if (t) { + r |= v & VarDescr::_NonZero; + } + r |= v & VarDescr::_Nan; + if (u & (VarDescr::_Odd | VarDescr::_Even)) { + r |= VarDescr::_Even; + } else if (!(~v & (VarDescr::_Odd | VarDescr::_Even))) { + r |= VarDescr::_Odd | VarDescr::_NonZero; + } + return r; +} + +int emulate_sub(int a, int b) { + return emulate_add(a, emulate_negate(b)); +} + +int emulate_mul(int a, int b) { + if ((b & VarDescr::ConstOne) == VarDescr::ConstOne) { + return a; + } else if ((a & VarDescr::ConstOne) == VarDescr::ConstOne) { + return b; + } + int u = a & b, v = a | b; + int r = VarDescr::_Int; + if (v & VarDescr::_Nan) { + return r | VarDescr::_Nan; + } + // non-quiet multiplication always yields finite results, if any + r |= VarDescr::_Finite; + if (v & VarDescr::_Zero) { + // non-quiet multiplication + // the result is zero, if any result at all + return VarDescr::ConstZero; + } + if (u & (VarDescr::_Pos | VarDescr::_Neg)) { + r |= VarDescr::_Pos; + } else if (!(~v & (VarDescr::_Pos | VarDescr::_Neg))) { + r |= VarDescr::_Neg; + } + r |= v & VarDescr::_Even; + r |= u & (VarDescr::_Odd | VarDescr::_NonZero); + return r; +} + +int emulate_bitwise_and(int a, int b) { + int both = a & b, any = a | b; + int r = VarDescr::_Int; + if (any & VarDescr::_Nan) { + return r | VarDescr::_Nan; + } + r |= VarDescr::_Finite; + if (any & VarDescr::_Zero) { + return VarDescr::ConstZero; + } + r |= both & (VarDescr::_Even | VarDescr::_Odd); + if (both & VarDescr::_Odd) { + r |= VarDescr::_NonZero; + } + return r; +} + +int emulate_bitwise_or(int a, int b) { + if (b & VarDescr::_Zero) { + return a; + } else if (a & VarDescr::_Zero) { + return b; + } + int both = a & b, any = a | b; + int r = VarDescr::_Int; + if (any & VarDescr::_Nan) { + return r | VarDescr::_Nan; + } + r |= VarDescr::_Finite; + r |= any & VarDescr::_NonZero; + r |= any & VarDescr::_Odd; + r |= both & VarDescr::_Even; + return r; +} + +int emulate_bitwise_xor(int a, int b) { + if (b & VarDescr::_Zero) { + return a; + } else if (a & VarDescr::_Zero) { + return b; + } + int both = a & b, any = a | b; + int r = VarDescr::_Int; + if (any & VarDescr::_Nan) { + return r | VarDescr::_Nan; + } + r |= VarDescr::_Finite; + r |= both & VarDescr::_Even; + if (both & VarDescr::_Odd) { + r |= VarDescr::_Even; + } + return r; +} + +int emulate_bitwise_not(int a) { + if ((a & VarDescr::ConstZero) == VarDescr::ConstZero) { + return VarDescr::ConstTrue; + } + if ((a & VarDescr::ConstTrue) == VarDescr::ConstTrue) { + return VarDescr::ConstZero; + } + int a2 = a; + int f = VarDescr::_Even | VarDescr::_Odd; + if ((a2 & f) && (~a2 & f)) { + a2 ^= f; + } + a2 &= ~(VarDescr::_Zero | VarDescr::_NonZero | VarDescr::_Pos | VarDescr::_Neg); + if ((a & VarDescr::_Neg) && (a & VarDescr::_NonZero)) { + a2 |= VarDescr::_Pos; + } + if (a & VarDescr::_Pos) { + a2 |= VarDescr::_Neg; + } + return a2; +} + +int emulate_lshift(int a, int b) { + if (((a | b) & VarDescr::_Nan) || !(~b & (VarDescr::_Neg | VarDescr::_NonZero))) { + return VarDescr::_Int | VarDescr::_Nan; + } + if (b & VarDescr::_Zero) { + return a; + } + int t = ((b & VarDescr::_NonZero) ? VarDescr::_Even : 0); + t |= b & VarDescr::_Finite; + return emulate_mul(a, VarDescr::_Int | VarDescr::_Pos | VarDescr::_NonZero | t); +} + +int emulate_div(int a, int b) { + if ((b & VarDescr::ConstOne) == VarDescr::ConstOne) { + return a; + } else if ((b & VarDescr::ConstOne) == VarDescr::ConstOne) { + return emulate_negate(a); + } + if (b & VarDescr::_Zero) { + return VarDescr::_Int | VarDescr::_Nan; + } + int u = a & b, v = a | b; + int r = VarDescr::_Int; + if (v & VarDescr::_Nan) { + return r | VarDescr::_Nan; + } + // non-quiet division always yields finite results, if any + r |= VarDescr::_Finite; + if (a & VarDescr::_Zero) { + // non-quiet division + // the result is zero, if any result at all + return VarDescr::ConstZero; + } + if (u & (VarDescr::_Pos | VarDescr::_Neg)) { + r |= VarDescr::_Pos; + } else if (!(~v & (VarDescr::_Pos | VarDescr::_Neg))) { + r |= VarDescr::_Neg; + } + return r; +} + +int emulate_rshift(int a, int b) { + if (((a | b) & VarDescr::_Nan) || !(~b & (VarDescr::_Neg | VarDescr::_NonZero))) { + return VarDescr::_Int | VarDescr::_Nan; + } + if (b & VarDescr::_Zero) { + return a; + } + int t = ((b & VarDescr::_NonZero) ? VarDescr::_Even : 0); + t |= b & VarDescr::_Finite; + return emulate_div(a, VarDescr::_Int | VarDescr::_Pos | VarDescr::_NonZero | t); +} + +int emulate_mod(int a, int b, int round_mode = -1) { + if ((b & VarDescr::ConstOne) == VarDescr::ConstOne) { + return VarDescr::ConstZero; + } + if (b & VarDescr::_Zero) { + return VarDescr::_Int | VarDescr::_Nan; + } + int r = VarDescr::_Int; + if ((a | b) & VarDescr::_Nan) { + return r | VarDescr::_Nan; + } + // non-quiet division always yields finite results, if any + r |= VarDescr::_Finite; + if (a & VarDescr::_Zero) { + // non-quiet division + // the result is zero, if any result at all + return VarDescr::ConstZero; + } + if (round_mode < 0) { + r |= b & (VarDescr::_Pos | VarDescr::_Neg); + } else if (round_mode > 0) { + r |= emulate_negate(b) & (VarDescr::_Pos | VarDescr::_Neg); + } + if (b & VarDescr::_Even) { + r |= a & (VarDescr::_Even | VarDescr::_Odd); + } + return r; +} + +bool VarDescr::always_less(const VarDescr& other) const { + if (is_int_const() && other.is_int_const()) { + return int_const < other.int_const; + } + return (always_nonpos() && other.always_pos()) || (always_neg() && other.always_nonneg()); +} + +bool VarDescr::always_leq(const VarDescr& other) const { + if (is_int_const() && other.is_int_const()) { + return int_const <= other.int_const; + } + return always_nonpos() && other.always_nonneg(); +} + +bool VarDescr::always_greater(const VarDescr& other) const { + return other.always_less(*this); +} + +bool VarDescr::always_geq(const VarDescr& other) const { + return other.always_leq(*this); +} + +bool VarDescr::always_equal(const VarDescr& other) const { + return is_int_const() && other.is_int_const() && *int_const == *other.int_const; +} + +bool VarDescr::always_neq(const VarDescr& other) const { + if (is_int_const() && other.is_int_const()) { + return *int_const != *other.int_const; + } + return always_greater(other) || always_less(other) || (always_even() && other.always_odd()) || + (always_odd() && other.always_even()); +} + +AsmOp exec_op(std::string op) { + return AsmOp::Custom(op); +} + +AsmOp exec_op(std::string op, int args, int retv = 1) { + return AsmOp::Custom(op, args, retv); +} + +AsmOp exec_arg_op(std::string op, long long arg) { + std::ostringstream os; + os << arg << ' ' << op; + return AsmOp::Custom(os.str()); +} + +AsmOp exec_arg_op(std::string op, long long arg, int args, int retv) { + std::ostringstream os; + os << arg << ' ' << op; + return AsmOp::Custom(os.str(), args, retv); +} + +AsmOp exec_arg_op(std::string op, td::RefInt256 arg) { + std::ostringstream os; + os << arg << ' ' << op; + return AsmOp::Custom(os.str()); +} + +AsmOp exec_arg_op(std::string op, td::RefInt256 arg, int args, int retv) { + std::ostringstream os; + os << arg << ' ' << op; + return AsmOp::Custom(os.str(), args, retv); +} + +AsmOp exec_arg2_op(std::string op, long long imm1, long long imm2, int args, int retv) { + std::ostringstream os; + os << imm1 << ' ' << imm2 << ' ' << op; + return AsmOp::Custom(os.str(), args, retv); +} + +AsmOp push_const(td::RefInt256 x) { + return AsmOp::IntConst(std::move(x)); +} + +AsmOp compile_add(std::vector& res, std::vector& args, SrcLocation where) { + tolk_assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (x.is_int_const() && y.is_int_const()) { + r.set_const(x.int_const + y.int_const); + if (!r.int_const->is_valid()) { + throw ParseError(where, "integer overflow"); + } + x.unused(); + y.unused(); + return push_const(r.int_const); + } + r.val = emulate_add(x.val, y.val); + if (y.is_int_const() && y.int_const->signed_fits_bits(8)) { + y.unused(); + if (y.always_zero()) { + return AsmOp::Nop(); + } + if (*y.int_const == 1) { + return exec_op("INC", 1); + } + if (*y.int_const == -1) { + return exec_op("DEC", 1); + } + return exec_arg_op("ADDCONST", y.int_const, 1); + } + if (x.is_int_const() && x.int_const->signed_fits_bits(8)) { + x.unused(); + if (x.always_zero()) { + return AsmOp::Nop(); + } + if (*x.int_const == 1) { + return exec_op("INC", 1); + } + if (*x.int_const == -1) { + return exec_op("DEC", 1); + } + return exec_arg_op("ADDCONST", x.int_const, 1); + } + return exec_op("ADD", 2); +} + +AsmOp compile_sub(std::vector& res, std::vector& args, SrcLocation where) { + tolk_assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (x.is_int_const() && y.is_int_const()) { + r.set_const(x.int_const - y.int_const); + if (!r.int_const->is_valid()) { + throw ParseError(where, "integer overflow"); + } + x.unused(); + y.unused(); + return push_const(r.int_const); + } + r.val = emulate_sub(x.val, y.val); + if (y.is_int_const() && (-y.int_const)->signed_fits_bits(8)) { + y.unused(); + if (y.always_zero()) { + return {}; + } + if (*y.int_const == 1) { + return exec_op("DEC", 1); + } + if (*y.int_const == -1) { + return exec_op("INC", 1); + } + return exec_arg_op("ADDCONST", -y.int_const, 1); + } + if (x.always_zero()) { + x.unused(); + return exec_op("NEGATE", 1); + } + return exec_op("SUB", 2); +} + +AsmOp compile_unary_minus(std::vector& res, std::vector& args, SrcLocation where) { + tolk_assert(res.size() == 1 && args.size() == 1); + VarDescr &r = res[0], &x = args[0]; + if (x.is_int_const()) { + r.set_const(-x.int_const); + if (!r.int_const->is_valid()) { + throw ParseError(where, "integer overflow"); + } + x.unused(); + return push_const(r.int_const); + } + r.val = emulate_negate(x.val); + return exec_op("NEGATE", 1); +} + +AsmOp compile_unary_plus(std::vector& res, std::vector& args, SrcLocation where) { + tolk_assert(res.size() == 1 && args.size() == 1); + VarDescr &r = res[0], &x = args[0]; + if (x.is_int_const()) { + r.set_const(x.int_const); + x.unused(); + return push_const(r.int_const); + } + r.val = x.val; + return AsmOp::Nop(); +} + +AsmOp compile_logical_not(std::vector& res, std::vector& args, SrcLocation where, bool for_int_arg) { + tolk_assert(res.size() == 1 && args.size() == 1); + VarDescr &r = res[0], &x = args[0]; + if (x.is_int_const()) { + r.set_const(x.int_const == 0 ? -1 : 0); + x.unused(); + return push_const(r.int_const); + } + r.val = VarDescr::ValBool; + // for integers, `!var` is `var != 0` + // for booleans, `!var` can be shortened to `~var` (works the same for 0/-1 but consumes less) + return for_int_arg ? exec_op("0 EQINT", 1) : exec_op("NOT", 1); +} + +AsmOp compile_bitwise_and(std::vector& res, std::vector& args, SrcLocation where) { + tolk_assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (x.is_int_const() && y.is_int_const()) { + r.set_const(x.int_const & y.int_const); + x.unused(); + y.unused(); + return push_const(r.int_const); + } + r.val = emulate_bitwise_and(x.val, y.val); + return exec_op("AND", 2); +} + +AsmOp compile_bitwise_or(std::vector& res, std::vector& args, SrcLocation where) { + tolk_assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (x.is_int_const() && y.is_int_const()) { + r.set_const(x.int_const | y.int_const); + x.unused(); + y.unused(); + return push_const(r.int_const); + } + r.val = emulate_bitwise_or(x.val, y.val); + return exec_op("OR", 2); +} + +AsmOp compile_bitwise_xor(std::vector& res, std::vector& args, SrcLocation where) { + tolk_assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (x.is_int_const() && y.is_int_const()) { + r.set_const(x.int_const ^ y.int_const); + x.unused(); + y.unused(); + return push_const(r.int_const); + } + r.val = emulate_bitwise_xor(x.val, y.val); + return exec_op("XOR", 2); +} + +AsmOp compile_bitwise_not(std::vector& res, std::vector& args, SrcLocation where) { + tolk_assert(res.size() == 1 && args.size() == 1); + VarDescr &r = res[0], &x = args[0]; + if (x.is_int_const()) { + r.set_const(~x.int_const); + x.unused(); + return push_const(r.int_const); + } + r.val = emulate_bitwise_not(x.val); + return exec_op("NOT", 1); +} + +AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation where) { + if (x.is_int_const() && y.is_int_const()) { + r.set_const(x.int_const * y.int_const); + if (!r.int_const->is_valid()) { + throw ParseError(where, "integer overflow"); + } + x.unused(); + y.unused(); + return push_const(r.int_const); + } + r.val = emulate_mul(x.val, y.val); + if (y.is_int_const()) { + int k = is_pos_pow2(y.int_const); + if (y.int_const->signed_fits_bits(8) && k < 0) { + y.unused(); + if (y.always_zero() && x.always_finite()) { + // dubious optimization: NaN * 0 = ? + r.set_const(y.int_const); + x.unused(); + return push_const(r.int_const); + } + if (*y.int_const == 1 && x.always_finite()) { + return AsmOp::Nop(); + } + if (*y.int_const == -1) { + return exec_op("NEGATE", 1); + } + return exec_arg_op("MULCONST", y.int_const, 1); + } + if (k > 0) { + y.unused(); + return exec_arg_op("LSHIFT#", k, 1); + } + if (k == 0) { + y.unused(); + return AsmOp::Nop(); + } + } + if (x.is_int_const()) { + int k = is_pos_pow2(x.int_const); + if (x.int_const->signed_fits_bits(8) && k < 0) { + x.unused(); + if (x.always_zero() && y.always_finite()) { + // dubious optimization: NaN * 0 = ? + r.set_const(x.int_const); + y.unused(); + return push_const(r.int_const); + } + if (*x.int_const == 1 && y.always_finite()) { + return AsmOp::Nop(); + } + if (*x.int_const == -1) { + return exec_op("NEGATE", 1); + } + return exec_arg_op("MULCONST", x.int_const, 1); + } + if (k > 0) { + x.unused(); + return exec_arg_op("LSHIFT#", k, 1); + } + if (k == 0) { + x.unused(); + return AsmOp::Nop(); + } + } + return exec_op("MUL", 2); +} + +AsmOp compile_mul(std::vector& res, std::vector& args, SrcLocation where) { + tolk_assert(res.size() == 1 && args.size() == 2); + return compile_mul_internal(res[0], args[0], args[1], where); +} + +AsmOp compile_lshift(std::vector& res, std::vector& args, SrcLocation where) { + tolk_assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (y.is_int_const()) { + auto yv = y.int_const->to_long(); + if (yv < 0 || yv > 256) { + throw ParseError(where, "lshift argument is out of range"); + } else if (x.is_int_const()) { + r.set_const(x.int_const << (int)yv); + if (!r.int_const->is_valid()) { + throw ParseError(where, "integer overflow"); + } + x.unused(); + y.unused(); + return push_const(r.int_const); + } + } + r.val = emulate_lshift(x.val, y.val); + if (y.is_int_const()) { + int k = (int)(y.int_const->to_long()); + if (!k /* && x.always_finite() */) { + // dubious optimization: what if x=NaN ? + y.unused(); + return AsmOp::Nop(); + } + y.unused(); + return exec_arg_op("LSHIFT#", k, 1); + } + if (x.is_int_const()) { + auto xv = x.int_const->to_long(); + if (xv == 1) { + x.unused(); + return exec_op("POW2", 1); + } + if (xv == -1) { + x.unused(); + return exec_op("-1 PUSHINT SWAP LSHIFT", 1); + } + } + return exec_op("LSHIFT", 2); +} + +AsmOp compile_rshift(std::vector& res, std::vector& args, SrcLocation where, + int round_mode) { + tolk_assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (y.is_int_const()) { + auto yv = y.int_const->to_long(); + if (yv < 0 || yv > 256) { + throw ParseError(where, "rshift argument is out of range"); + } else if (x.is_int_const()) { + r.set_const(td::rshift(x.int_const, (int)yv, round_mode)); + x.unused(); + y.unused(); + return push_const(r.int_const); + } + } + r.val = emulate_rshift(x.val, y.val); + std::string rshift = (round_mode < 0 ? "RSHIFT" : (round_mode ? "RSHIFTC" : "RSHIFTR")); + if (y.is_int_const()) { + int k = (int)(y.int_const->to_long()); + if (!k /* && x.always_finite() */) { + // dubious optimization: what if x=NaN ? + y.unused(); + return AsmOp::Nop(); + } + y.unused(); + return exec_arg_op(rshift + "#", k, 1); + } + return exec_op(rshift, 2); +} + +AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, SrcLocation where, int round_mode) { + if (x.is_int_const() && y.is_int_const()) { + r.set_const(div(x.int_const, y.int_const, round_mode)); + if (!r.int_const->is_valid()) { + throw ParseError(where, *y.int_const == 0 ? "division by zero" : "integer overflow"); + } + x.unused(); + y.unused(); + return push_const(r.int_const); + } + r.val = emulate_div(x.val, y.val); + if (y.is_int_const()) { + if (*y.int_const == 0) { + throw ParseError(where, "division by zero"); + } + if (*y.int_const == 1 && x.always_finite()) { + y.unused(); + return AsmOp::Nop(); + } + if (*y.int_const == -1) { + y.unused(); + return exec_op("NEGATE", 1); + } + int k = is_pos_pow2(y.int_const); + if (k > 0) { + y.unused(); + std::string op = "RSHIFT"; + if (round_mode >= 0) { + op += (round_mode > 0 ? 'C' : 'R'); + } + return exec_arg_op(op + '#', k, 1); + } + } + std::string op = "DIV"; + if (round_mode >= 0) { + op += (round_mode > 0 ? 'C' : 'R'); + } + return exec_op(op, 2); +} + +AsmOp compile_div(std::vector& res, std::vector& args, SrcLocation where, int round_mode) { + tolk_assert(res.size() == 1 && args.size() == 2); + return compile_div_internal(res[0], args[0], args[1], where, round_mode); +} + +AsmOp compile_mod(std::vector& res, std::vector& args, SrcLocation where, + int round_mode) { + tolk_assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (x.is_int_const() && y.is_int_const()) { + r.set_const(mod(x.int_const, y.int_const, round_mode)); + if (!r.int_const->is_valid()) { + throw ParseError(where, *y.int_const == 0 ? "division by zero" : "integer overflow"); + } + x.unused(); + y.unused(); + return push_const(r.int_const); + } + r.val = emulate_mod(x.val, y.val); + if (y.is_int_const()) { + if (*y.int_const == 0) { + throw ParseError(where, "division by zero"); + } + if ((*y.int_const == 1 || *y.int_const == -1) && x.always_finite()) { + x.unused(); + y.unused(); + r.set_const(td::zero_refint()); + return push_const(r.int_const); + } + int k = is_pos_pow2(y.int_const); + if (k > 0) { + y.unused(); + std::string op = "MODPOW2"; + if (round_mode >= 0) { + op += (round_mode > 0 ? 'C' : 'R'); + } + return exec_arg_op(op + '#', k, 1); + } + } + std::string op = "MOD"; + if (round_mode >= 0) { + op += (round_mode > 0 ? 'C' : 'R'); + } + return exec_op(op, 2); +} + +AsmOp compile_muldiv(std::vector& res, std::vector& args, SrcLocation where, + int round_mode) { + tolk_assert(res.size() == 1 && args.size() == 3); + VarDescr &r = res[0], &x = args[0], &y = args[1], &z = args[2]; + if (x.is_int_const() && y.is_int_const() && z.is_int_const()) { + r.set_const(muldiv(x.int_const, y.int_const, z.int_const, round_mode)); + if (!r.int_const->is_valid()) { + throw ParseError(where, *z.int_const == 0 ? "division by zero" : "integer overflow"); + } + x.unused(); + y.unused(); + z.unused(); + return push_const(r.int_const); + } + if (x.always_zero() || y.always_zero()) { + // dubious optimization for z=0... + x.unused(); + y.unused(); + z.unused(); + r.set_const(td::make_refint(0)); + return push_const(r.int_const); + } + char c = (round_mode < 0) ? 0 : (round_mode > 0 ? 'C' : 'R'); + r.val = emulate_div(emulate_mul(x.val, y.val), z.val); + if (z.is_int_const()) { + if (*z.int_const == 0) { + throw ParseError(where, "division by zero"); + } + if (*z.int_const == 1) { + z.unused(); + return compile_mul_internal(r, x, y, where); + } + } + if (y.is_int_const() && *y.int_const == 1) { + y.unused(); + return compile_div_internal(r, x, z, where, round_mode); + } + if (x.is_int_const() && *x.int_const == 1) { + x.unused(); + return compile_div_internal(r, y, z, where, round_mode); + } + if (z.is_int_const()) { + int k = is_pos_pow2(z.int_const); + if (k > 0) { + z.unused(); + std::string op = "MULRSHIFT"; + if (c) { + op += c; + } + return exec_arg_op(op + '#', k, 2); + } + } + if (y.is_int_const()) { + int k = is_pos_pow2(y.int_const); + if (k > 0) { + y.unused(); + std::string op = "LSHIFT#DIV"; + if (c) { + op += c; + } + return exec_arg_op(op, k, 2); + } + } + if (x.is_int_const()) { + int k = is_pos_pow2(x.int_const); + if (k > 0) { + x.unused(); + std::string op = "LSHIFT#DIV"; + if (c) { + op += c; + } + return exec_arg_op(op, k, 2); + } + } + std::string op = "MULDIV"; + if (c) { + op += c; + } + return exec_op(op, 3); +} + +int compute_compare(td::RefInt256 x, td::RefInt256 y, int mode) { + int s = td::cmp(x, y); + if (mode == 7) { + return s; + } else { + return -((mode >> (1 - s)) & 1); + } +} + +// return value: +// 4 -> constant 1 +// 2 -> constant 0 +// 1 -> constant -1 +// 3 -> 0 or -1 +int compute_compare(const VarDescr& x, const VarDescr& y, int mode) { + switch (mode) { + case 1: // > + return x.always_greater(y) ? 1 : (x.always_leq(y) ? 2 : 3); + case 2: // = + return x.always_equal(y) ? 1 : (x.always_neq(y) ? 2 : 3); + case 3: // >= + return x.always_geq(y) ? 1 : (x.always_less(y) ? 2 : 3); + case 4: // < + return x.always_less(y) ? 1 : (x.always_geq(y) ? 2 : 3); + case 5: // <> + return x.always_neq(y) ? 1 : (x.always_equal(y) ? 2 : 3); + case 6: // <= + return x.always_leq(y) ? 1 : (x.always_greater(y) ? 2 : 3); + case 7: // <=> + return x.always_less(y) + ? 1 + : (x.always_equal(y) + ? 2 + : (x.always_greater(y) + ? 4 + : (x.always_leq(y) ? 3 : (x.always_geq(y) ? 6 : (x.always_neq(y) ? 5 : 7))))); + default: + return 7; + } +} + +AsmOp compile_cmp_int(std::vector& res, std::vector& args, int mode) { + tolk_assert(mode >= 1 && mode <= 7); + tolk_assert(res.size() == 1 && args.size() == 2); + VarDescr &r = res[0], &x = args[0], &y = args[1]; + if (x.is_int_const() && y.is_int_const()) { + int v = compute_compare(x.int_const, y.int_const, mode); + r.set_const(v); + x.unused(); + y.unused(); + return mode == 7 ? push_const(r.int_const) : AsmOp::BoolConst(v != 0); + } + int v = compute_compare(x, y, mode); + // std::cerr << "compute_compare(" << x << ", " << y << ", " << mode << ") = " << v << std::endl; + tolk_assert(v); + if (!(v & (v - 1))) { + r.set_const(v - (v >> 2) - 2); + x.unused(); + y.unused(); + return mode == 7 ? push_const(r.int_const) : AsmOp::BoolConst(v & 1); + } + r.val = ~0; + if (v & 1) { + r.val &= VarDescr::ConstTrue; + } + if (v & 2) { + r.val &= VarDescr::ConstZero; + } + if (v & 4) { + r.val &= VarDescr::ConstOne; + } + // std::cerr << "result: " << r << std::endl; + static const char* cmp_int_names[] = {"", "GTINT", "EQINT", "GTINT", "LESSINT", "NEQINT", "LESSINT"}; + static const char* cmp_names[] = {"", "GREATER", "EQUAL", "GEQ", "LESS", "NEQ", "LEQ", "CMP"}; + static int cmp_int_delta[] = {0, 0, 0, -1, 0, 0, 1}; + if (mode != 7) { + if (y.is_int_const() && y.int_const >= -128 && y.int_const <= 127) { + y.unused(); + return exec_arg_op(cmp_int_names[mode], y.int_const + cmp_int_delta[mode], 1); + } + if (x.is_int_const() && x.int_const >= -128 && x.int_const <= 127) { + x.unused(); + mode = ((mode & 4) >> 2) | (mode & 2) | ((mode & 1) << 2); + return exec_arg_op(cmp_int_names[mode], x.int_const + cmp_int_delta[mode], 1); + } + } + return exec_op(cmp_names[mode], 2); +} + +AsmOp compile_throw(std::vector& res, std::vector& args, SrcLocation) { + tolk_assert(res.empty() && args.size() == 1); + VarDescr& x = args[0]; + if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { + x.unused(); + return exec_arg_op("THROW", x.int_const, 0, 0); + } else { + return exec_op("THROWANY", 1, 0); + } +} + +AsmOp compile_throw_if_unless(std::vector& res, std::vector& args, SrcLocation) { + tolk_assert(res.empty() && args.size() == 3); + VarDescr &x = args[0], &y = args[1], &z = args[2]; + if (!z.always_true() && !z.always_false()) { + throw Fatal("invalid usage of built-in symbol"); + } + bool mode = z.always_true(); + z.unused(); + std::string suff = (mode ? "IF" : "IFNOT"); + bool skip_cond = false; + if (y.always_true() || y.always_false()) { + y.unused(); + skip_cond = true; + if (y.always_true() != mode) { + x.unused(); + return AsmOp::Nop(); + } + } + if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { + x.unused(); + return skip_cond ? exec_arg_op("THROW", x.int_const, 0, 0) : exec_arg_op("THROW"s + suff, x.int_const, 1, 0); + } else { + return skip_cond ? exec_op("THROWANY", 1, 0) : exec_op("THROWANY"s + suff, 2, 0); + } +} + +AsmOp compile_throw_arg(std::vector& res, std::vector& args, SrcLocation) { + tolk_assert(res.empty() && args.size() == 2); + VarDescr &x = args[1]; + if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) { + x.unused(); + return exec_arg_op("THROWARG", x.int_const, 1, 0); + } else { + return exec_op("THROWARGANY", 2, 0); + } +} + +AsmOp compile_bool_const(std::vector& res, std::vector& args, bool val) { + tolk_assert(res.size() == 1 && args.empty()); + VarDescr& r = res[0]; + r.set_const(val ? -1 : 0); + return AsmOp::Const(val ? "TRUE" : "FALSE"); +} + +// fun loadInt (mutate s: slice, len: int): int asm(s len -> 1 0) "LDIX"; +// fun loadUint (mutate s: slice, len: int): int asm( -> 1 0) "LDUX"; +// fun preloadInt (s: slice, len: int): int asm "PLDIX"; +// fun preloadUint(s: slice, len: int): int asm "PLDUX"; +AsmOp compile_fetch_int(std::vector& res, std::vector& args, bool fetch, bool sgnd) { + tolk_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); + auto &y = args[1], &r = res.back(); + r.val = (sgnd ? VarDescr::FiniteInt : VarDescr::FiniteUInt); + int v = -1; + if (y.is_int_const() && y.int_const >= 0 && y.int_const <= 256) { + v = (int)y.int_const->to_long(); + if (!v) { + r.val = VarDescr::ConstZero; + } + if (v == 1) { + r.val = (sgnd ? VarDescr::ValBool : VarDescr::ValBit); + } + if (v > 0) { + y.unused(); + return exec_arg_op((fetch ? "LD"s : "PLD"s) + (sgnd ? 'I' : 'U'), v, 1, 1 + (unsigned)fetch); + } + } + return exec_op((fetch ? "LD"s : "PLD"s) + (sgnd ? "IX" : "UX"), 2, 1 + (unsigned)fetch); +} + +// fun storeInt (mutate self: builder, x: int, len: int): self asm(x b len) "STIX"; +// fun storeUint (mutate self: builder, x: int, len: int): self asm(x b len) "STUX"; +AsmOp compile_store_int(std::vector& res, std::vector& args, bool sgnd) { + tolk_assert(args.size() == 3 && res.size() == 1); + auto& z = args[2]; + if (z.is_int_const() && z.int_const > 0 && z.int_const <= 256) { + z.unused(); + return exec_arg_op("ST"s + (sgnd ? 'I' : 'U'), z.int_const, 2, 1); + } + return exec_op("ST"s + (sgnd ? "IX" : "UX"), 3, 1); +} + +// fun loadBits (mutate self: slice, len: int): self asm(s len -> 1 0) "LDSLICEX" +// fun preloadBits(self: slice, len: int): slice asm(s len -> 1 0) "PLDSLICEX" +AsmOp compile_fetch_slice(std::vector& res, std::vector& args, bool fetch) { + tolk_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch); + auto& y = args[1]; + int v = -1; + if (y.is_int_const() && y.int_const > 0 && y.int_const <= 256) { + v = (int)y.int_const->to_long(); + if (v > 0) { + y.unused(); + return exec_arg_op(fetch ? "LDSLICE" : "PLDSLICE", v, 1, 1 + (unsigned)fetch); + } + } + return exec_op(fetch ? "LDSLICEX" : "PLDSLICEX", 2, 1 + (unsigned)fetch); +} + +// fun tupleAt(t: tuple, index: int): X asm "INDEXVAR"; +AsmOp compile_tuple_at(std::vector& res, std::vector& args, SrcLocation) { + tolk_assert(args.size() == 2 && res.size() == 1); + auto& y = args[1]; + if (y.is_int_const() && y.int_const >= 0 && y.int_const < 16) { + y.unused(); + return exec_arg_op("INDEX", y.int_const, 1, 1); + } + return exec_op("INDEXVAR", 2, 1); +} + +// fun tupleSetAt(mutate self: tuple, value: X, index: int): void asm "SETINDEXVAR"; +AsmOp compile_tuple_set_at(std::vector& res, std::vector& args, SrcLocation) { + tolk_assert(args.size() == 3 && res.size() == 1); + auto& y = args[2]; + if (y.is_int_const() && y.int_const >= 0 && y.int_const < 16) { + y.unused(); + return exec_arg_op("SETINDEX", y.int_const, 1, 1); + } + return exec_op("SETINDEXVAR", 2, 1); +} + +// fun __isNull(X arg): bool +AsmOp compile_is_null(std::vector& res, std::vector& args, SrcLocation) { + tolk_assert(args.size() == 1 && res.size() == 1); + res[0].val = VarDescr::ValBool; + return exec_op("ISNULL", 1, 1); +} + + +void define_builtins() { + using namespace std::placeholders; + + TypePtr Unit = TypeDataVoid::create(); + TypePtr Int = TypeDataInt::create(); + TypePtr Bool = TypeDataBool::create(); + TypePtr Slice = TypeDataSlice::create(); + TypePtr Builder = TypeDataBuilder::create(); + TypePtr Tuple = TypeDataTuple::create(); + TypePtr Never = TypeDataNever::create(); + + std::vector itemsT; + itemsT.emplace_back("T"); + TypePtr typeT = TypeDataGenericT::create("T"); + const GenericsDeclaration* declGenericT = new GenericsDeclaration(std::move(itemsT)); + + std::vector ParamsInt1 = {Int}; + std::vector ParamsInt2 = {Int, Int}; + std::vector ParamsInt3 = {Int, Int, Int}; + std::vector ParamsSliceInt = {Slice, Int}; + + // builtin operators + // they are internally stored as functions, because at IR level, there is no difference + // between calling `userAdd(a,b)` and `_+_(a,b)` + // since they are registered in a global symtable, technically, they can even be referenced from Tolk code, + // though it's a "hidden feature" and won't work well for overloads (`==` for int and bool, for example) + + // unary operators + define_builtin_func("-_", ParamsInt1, Int, nullptr, + compile_unary_minus, + FunctionData::flagMarkedAsPure); + define_builtin_func("+_", ParamsInt1, Int, nullptr, + compile_unary_plus, + FunctionData::flagMarkedAsPure); + define_builtin_func("!_", ParamsInt1, Bool, nullptr, + std::bind(compile_logical_not, _1, _2, _3, true), + FunctionData::flagMarkedAsPure); + define_builtin_func("!b_", {Bool}, Bool, nullptr, // "overloaded" separate version for bool + std::bind(compile_logical_not, _1, _2, _3, false), + FunctionData::flagMarkedAsPure); + define_builtin_func("~_", ParamsInt1, Int, nullptr, + compile_bitwise_not, + FunctionData::flagMarkedAsPure); + + // binary operators + define_builtin_func("_+_", ParamsInt2, Int, nullptr, + compile_add, + FunctionData::flagMarkedAsPure); + define_builtin_func("_-_", ParamsInt2, Int, nullptr, + compile_sub, + FunctionData::flagMarkedAsPure); + define_builtin_func("_*_", ParamsInt2, Int, nullptr, + compile_mul, + FunctionData::flagMarkedAsPure); + define_builtin_func("_/_", ParamsInt2, Int, nullptr, + std::bind(compile_div, _1, _2, _3, -1), + FunctionData::flagMarkedAsPure); + define_builtin_func("_~/_", ParamsInt2, Int, nullptr, + std::bind(compile_div, _1, _2, _3, 0), + FunctionData::flagMarkedAsPure); + define_builtin_func("_^/_", ParamsInt2, Int, nullptr, + std::bind(compile_div, _1, _2, _3, 1), + FunctionData::flagMarkedAsPure); + define_builtin_func("_%_", ParamsInt2, Int, nullptr, + std::bind(compile_mod, _1, _2, _3, -1), + FunctionData::flagMarkedAsPure); + define_builtin_func("_<<_", ParamsInt2, Int, nullptr, + compile_lshift, + FunctionData::flagMarkedAsPure); + define_builtin_func("_>>_", ParamsInt2, Int, nullptr, + std::bind(compile_rshift, _1, _2, _3, -1), + FunctionData::flagMarkedAsPure); + define_builtin_func("_~>>_", ParamsInt2, Int, nullptr, + std::bind(compile_rshift, _1, _2, _3, 0), + FunctionData::flagMarkedAsPure); + define_builtin_func("_^>>_", ParamsInt2, Int, nullptr, + std::bind(compile_rshift, _1, _2, _3, 1), + FunctionData::flagMarkedAsPure); + define_builtin_func("_&_", ParamsInt2, Int, nullptr, // also works for bool + compile_bitwise_and, + FunctionData::flagMarkedAsPure); + define_builtin_func("_|_", ParamsInt2, Int, nullptr, // also works for bool + compile_bitwise_or, + FunctionData::flagMarkedAsPure); + define_builtin_func("_^_", ParamsInt2, Int, nullptr, // also works for bool + compile_bitwise_xor, + FunctionData::flagMarkedAsPure); + define_builtin_func("_==_", ParamsInt2, Int, nullptr, // also works for bool + std::bind(compile_cmp_int, _1, _2, 2), + FunctionData::flagMarkedAsPure); + define_builtin_func("_!=_", ParamsInt2, Int, nullptr, // also works for bool + std::bind(compile_cmp_int, _1, _2, 5), + FunctionData::flagMarkedAsPure); + define_builtin_func("_<_", ParamsInt2, Int, nullptr, + std::bind(compile_cmp_int, _1, _2, 4), + FunctionData::flagMarkedAsPure); + define_builtin_func("_>_", ParamsInt2, Int, nullptr, + std::bind(compile_cmp_int, _1, _2, 1), + FunctionData::flagMarkedAsPure); + define_builtin_func("_<=_", ParamsInt2, Int, nullptr, + std::bind(compile_cmp_int, _1, _2, 6), + FunctionData::flagMarkedAsPure); + define_builtin_func("_>=_", ParamsInt2, Int, nullptr, + std::bind(compile_cmp_int, _1, _2, 3), + FunctionData::flagMarkedAsPure); + define_builtin_func("_<=>_", ParamsInt2, Int, nullptr, + std::bind(compile_cmp_int, _1, _2, 7), + FunctionData::flagMarkedAsPure); + + // special function used for internal compilation of some lexical constructs + // for example, `throw 123;` is actually calling `__throw(123)` + define_builtin_func("__true", {}, Bool, nullptr, /* AsmOp::Const("TRUE") */ + std::bind(compile_bool_const, _1, _2, true), + FunctionData::flagMarkedAsPure); + define_builtin_func("__false", {}, Bool, nullptr, /* AsmOp::Const("FALSE") */ + std::bind(compile_bool_const, _1, _2, false), + FunctionData::flagMarkedAsPure); + define_builtin_func("__null", {}, typeT, declGenericT, + AsmOp::Const("PUSHNULL"), + FunctionData::flagMarkedAsPure); + define_builtin_func("__isNull", {typeT}, Bool, declGenericT, + compile_is_null, + FunctionData::flagMarkedAsPure); + define_builtin_func("__throw", ParamsInt1, Never, nullptr, + compile_throw, + 0); + define_builtin_func("__throw_arg", {typeT, Int}, Never, declGenericT, + compile_throw_arg, + 0); + define_builtin_func("__throw_if_unless", ParamsInt3, Unit, nullptr, + compile_throw_if_unless, + 0); + + // functions from stdlib marked as `builtin`, implemented at compiler level for optimizations + // (for example, `loadInt(1)` is `1 LDI`, but `loadInt(n)` for non-constant requires it be on a stack and `LDIX`) + define_builtin_func("mulDivFloor", ParamsInt3, Int, nullptr, + std::bind(compile_muldiv, _1, _2, _3, -1), + FunctionData::flagMarkedAsPure); + define_builtin_func("mulDivRound", ParamsInt3, Int, nullptr, + std::bind(compile_muldiv, _1, _2, _3, 0), + FunctionData::flagMarkedAsPure); + define_builtin_func("mulDivCeil", ParamsInt3, Int, nullptr, + std::bind(compile_muldiv, _1, _2, _3, 1), + FunctionData::flagMarkedAsPure); + define_builtin_func("mulDivMod", ParamsInt3, TypeDataTensor::create({Int, Int}), nullptr, + AsmOp::Custom("MULDIVMOD", 3, 2), + FunctionData::flagMarkedAsPure); + define_builtin_func("loadInt", ParamsSliceInt, Int, nullptr, + std::bind(compile_fetch_int, _1, _2, true, true), + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, + {}, {1, 0}); + define_builtin_func("loadUint", ParamsSliceInt, Int, nullptr, + std::bind(compile_fetch_int, _1, _2, true, false), + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, + {}, {1, 0}); + define_builtin_func("loadBits", ParamsSliceInt, Slice, nullptr, + std::bind(compile_fetch_slice, _1, _2, true), + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf, + {}, {1, 0}); + define_builtin_func("preloadInt", ParamsSliceInt, Int, nullptr, + std::bind(compile_fetch_int, _1, _2, false, true), + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); + define_builtin_func("preloadUint", ParamsSliceInt, Int, nullptr, + std::bind(compile_fetch_int, _1, _2, false, false), + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); + define_builtin_func("preloadBits", ParamsSliceInt, Slice, nullptr, + std::bind(compile_fetch_slice, _1, _2, false), + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); + define_builtin_func("storeInt", {Builder, Int, Int}, Unit, nullptr, + std::bind(compile_store_int, _1, _2, true), + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf, + {1, 0, 2}, {}); + define_builtin_func("storeUint", {Builder, Int, Int}, Unit, nullptr, + std::bind(compile_store_int, _1, _2, false), + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf | FunctionData::flagReturnsSelf, + {1, 0, 2}, {}); + define_builtin_func("tupleAt", {Tuple, Int}, typeT, declGenericT, + compile_tuple_at, + FunctionData::flagMarkedAsPure | FunctionData::flagAcceptsSelf); + define_builtin_func("tupleSetAt", {Tuple, typeT, Int}, Unit, declGenericT, + compile_tuple_set_at, + FunctionData::flagMarkedAsPure | FunctionData::flagHasMutateParams | FunctionData::flagAcceptsSelf); + define_builtin_func("debugPrint", {typeT}, Unit, declGenericT, + AsmOp::Custom("s0 DUMP DROP", 1, 1), + 0); + define_builtin_func("debugPrintString", {typeT}, Unit, declGenericT, + AsmOp::Custom("STRDUMP DROP", 1, 1), + 0); + define_builtin_func("debugDumpStack", {}, Unit, nullptr, + AsmOp::Custom("DUMPSTK", 0, 0), + 0); + + // functions not presented in stdlib at all + // used in tolk-tester to check/expose internal compiler state + // each of them is handled in a special way, search by its name + define_builtin_func("__expect_type", {TypeDataUnknown::create(), Slice}, Unit, nullptr, + AsmOp::Nop(), + FunctionData::flagMarkedAsPure); +} + +} // namespace tolk diff --git a/tolk/codegen.cpp b/tolk/codegen.cpp new file mode 100644 index 00000000..ac1cf639 --- /dev/null +++ b/tolk/codegen.cpp @@ -0,0 +1,922 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" +#include "compiler-state.h" +#include "type-system.h" + +namespace tolk { + +/* + * + * GENERATE TVM STACK CODE + * + */ + +StackLayout Stack::vars() const { + StackLayout res; + res.reserve(s.size()); + for (auto x : s) { + res.push_back(x.first); + } + return res; +} + +int Stack::find(var_idx_t var, int from) const { + for (int i = from; i < depth(); i++) { + if (at(i).first == var) { + return i; + } + } + return -1; +} + +// finds var in [from .. to) +int Stack::find(var_idx_t var, int from, int to) const { + for (int i = from; i < depth() && i < to; i++) { + if (at(i).first == var) { + return i; + } + } + return -1; +} + +// finds var outside [from .. to) +int Stack::find_outside(var_idx_t var, int from, int to) const { + from = std::max(from, 0); + if (from >= to) { + return find(var); + } else { + int t = find(var, 0, from); + return t >= 0 ? t : find(var, to); + } +} + +int Stack::find_const(const_idx_t cst, int from) const { + for (int i = from; i < depth(); i++) { + if (at(i).second == cst) { + return i; + } + } + return -1; +} + +void Stack::forget_const() { + for (auto& vc : s) { + if (vc.second != not_const) { + vc.second = not_const; + } + } +} + +void Stack::issue_pop(int i) { + validate(i); + if (output_enabled()) { + o << AsmOp::Pop(i); + } + at(i) = get(0); + s.pop_back(); + modified(); +} + +void Stack::issue_push(int i) { + validate(i); + if (output_enabled()) { + o << AsmOp::Push(i); + } + s.push_back(get(i)); + modified(); +} + +void Stack::issue_xchg(int i, int j) { + validate(i); + validate(j); + if (i != j && get(i) != get(j)) { + if (output_enabled()) { + o << AsmOp::Xchg(i, j); + } + std::swap(at(i), at(j)); + modified(); + } +} + +int Stack::drop_vars_except(const VarDescrList& var_info, int excl_var) { + int dropped = 0, changes; + do { + changes = 0; + int n = depth(); + for (int i = 0; i < n; i++) { + var_idx_t idx = at(i).first; + if (((!var_info[idx] || var_info[idx]->is_unused()) && idx != excl_var) || find(idx, 0, i - 1) >= 0) { + // unneeded + issue_pop(i); + changes = 1; + break; + } + } + dropped += changes; + } while (changes); + return dropped; +} + +void Stack::show() { + std::ostringstream os; + for (auto i : s) { + os << ' '; + o.show_var_ext(os, i); + } + o << AsmOp::Comment(os.str()); + mode |= _Shown; +} + +void Stack::forget_var(var_idx_t idx) { + for (auto& x : s) { + if (x.first == idx) { + x = std::make_pair(_Garbage, not_const); + modified(); + } + } +} + +void Stack::push_new_var(var_idx_t idx) { + forget_var(idx); + s.emplace_back(idx, not_const); + modified(); +} + +void Stack::push_new_const(var_idx_t idx, const_idx_t cidx) { + forget_var(idx); + s.emplace_back(idx, cidx); + modified(); +} + +void Stack::assign_var(var_idx_t new_idx, var_idx_t old_idx) { + int i = find(old_idx); + tolk_assert(i >= 0 && "variable not found in stack"); + if (new_idx != old_idx) { + at(i).first = new_idx; + modified(); + } +} + +void Stack::do_copy_var(var_idx_t new_idx, var_idx_t old_idx) { + int i = find(old_idx); + tolk_assert(i >= 0 && "variable not found in stack"); + if (find(old_idx, i + 1) < 0) { + issue_push(i); + tolk_assert(at(0).first == old_idx); + } + assign_var(new_idx, old_idx); +} + +void Stack::enforce_state(const StackLayout& req_stack) { + int k = (int)req_stack.size(); + for (int i = 0; i < k; i++) { + var_idx_t x = req_stack[i]; + if (i < depth() && s[i].first == x) { + continue; + } + while (depth() > 0 && std::find(req_stack.cbegin(), req_stack.cend(), get(0).first) == req_stack.cend()) { + // current TOS entry is unused in req_stack, drop it + issue_pop(0); + } + int j = find(x); + if (j >= depth() - i) { + issue_push(j); + j = 0; + } + issue_xchg(j, depth() - i - 1); + tolk_assert(s[i].first == x); + } + while (depth() > k) { + issue_pop(0); + } + tolk_assert(depth() == k); + for (int i = 0; i < k; i++) { + tolk_assert(s[i].first == req_stack[i]); + } +} + +void Stack::merge_const(const Stack& req_stack) { + tolk_assert(s.size() == req_stack.s.size()); + for (std::size_t i = 0; i < s.size(); i++) { + tolk_assert(s[i].first == req_stack.s[i].first); + if (s[i].second != req_stack.s[i].second) { + s[i].second = not_const; + } + } +} + +void Stack::merge_state(const Stack& req_stack) { + enforce_state(req_stack.vars()); + merge_const(req_stack); +} + +void Stack::rearrange_top(const StackLayout& top, std::vector last) { + while (last.size() < top.size()) { + last.push_back(false); + } + int k = (int)top.size(); + for (int i = 0; i < k; i++) { + for (int j = i + 1; j < k; j++) { + if (top[i] == top[j]) { + last[i] = false; + break; + } + } + } + int ss = 0; + for (int i = 0; i < k; i++) { + if (last[i]) { + ++ss; + } + } + for (int i = 0; i < k; i++) { + var_idx_t x = top[i]; + // find s(j) containing x with j not in [ss, ss+i) + int j = find_outside(x, ss, ss + i); + if (last[i]) { + // rearrange x to be at s(ss-1) + issue_xchg(--ss, j); + tolk_assert(get(ss).first == x); + } else { + // create a new copy of x + issue_push(j); + issue_xchg(0, ss); + tolk_assert(get(ss).first == x); + } + } + tolk_assert(!ss); +} + +void Stack::rearrange_top(var_idx_t top, bool last) { + int i = find(top); + if (last) { + issue_xchg(0, i); + } else { + issue_push(i); + } + tolk_assert(get(0).first == top); +} + +bool Op::generate_code_step(Stack& stack) { + stack.opt_show(); + + // detect `throw 123` (actually _IntConst 123 + _Call __throw) + // don't clear the stack, since dropping unused elements make no sense, an exception is thrown anyway + bool will_now_immediate_throw = (cl == _Call && f_sym->is_builtin_function() && f_sym->name == "__throw") + || (cl == _IntConst && next->cl == _Call && next->f_sym->is_builtin_function() && next->f_sym->name == "__throw"); + if (!will_now_immediate_throw) { + stack.drop_vars_except(var_info); + stack.opt_show(); + } + + bool inline_func = stack.mode & Stack::_InlineFunc; + switch (cl) { + case _Nop: + case _Import: + return true; + case _Return: { + stack.enforce_state(left); + if (stack.o.retalt_ && (stack.mode & Stack::_NeedRetAlt)) { + stack.o << "RETALT"; + stack.o.retalt_inserted_ = true; + } + stack.opt_show(); + return false; + } + case _IntConst: { + auto p = next->var_info[left[0]]; + if (!p || p->is_unused()) { + return true; + } + auto cidx = stack.o.register_const(int_const); + int i = stack.find_const(cidx); + if (i < 0) { + stack.o << push_const(int_const); + stack.push_new_const(left[0], cidx); + } else { + tolk_assert(stack.at(i).second == cidx); + stack.do_copy_var(left[0], stack[i]); + } + return true; + } + case _SliceConst: { + auto p = next->var_info[left[0]]; + if (!p || p->is_unused()) { + return true; + } + stack.o << AsmOp::Const("x{" + str_const + "} PUSHSLICE"); + stack.push_new_var(left[0]); + return true; + } + case _GlobVar: + if (g_sym) { + bool used = false; + for (auto i : left) { + auto p = next->var_info[i]; + if (p && !p->is_unused()) { + used = true; + } + } + if (!used || disabled()) { + return true; + } + stack.o << AsmOp::Custom(g_sym->name + " GETGLOB", 0, 1); + if (left.size() != 1) { + tolk_assert(left.size() <= 15); + stack.o << AsmOp::UnTuple((int)left.size()); + } + for (auto i : left) { + stack.push_new_var(i); + } + return true; + } else { + tolk_assert(left.size() == 1); + auto p = next->var_info[left[0]]; + if (!p || p->is_unused() || disabled()) { + return true; + } + stack.o << "CONT:<{"; + stack.o.indent(); + if (f_sym->is_asm_function() || f_sym->is_builtin_function()) { + // TODO: create and compile a true lambda instead of this (so that arg_order and ret_order would work correctly) + std::vector args0, res; + int w_arg = 0; + for (const LocalVarData& param : f_sym->parameters) { + w_arg += param.declared_type->get_width_on_stack(); + } + int w_ret = f_sym->inferred_return_type->get_width_on_stack(); + tolk_assert(w_ret >= 0 && w_arg >= 0); + for (int i = 0; i < w_ret; i++) { + res.emplace_back(0); + } + for (int i = 0; i < w_arg; i++) { + args0.emplace_back(0); + } + if (f_sym->is_asm_function()) { + std::get(f_sym->body)->compile(stack.o); // compile res := f (args0) + } else { + std::get(f_sym->body)->compile(stack.o, res, args0, where); // compile res := f (args0) + } + } else { + stack.o << AsmOp::Custom(f_sym->name + " CALLDICT", (int)right.size(), (int)left.size()); + } + stack.o.undent(); + stack.o << "}>"; + stack.push_new_var(left.at(0)); + return true; + } + case _Let: { + tolk_assert(left.size() == right.size()); + int i = 0; + std::vector active; + active.reserve(left.size()); + for (std::size_t k = 0; k < left.size(); k++) { + var_idx_t y = left[k]; // "y" = "x" + auto p = next->var_info[y]; + active.push_back(p && !p->is_unused()); + } + for (std::size_t k = 0; k < left.size(); k++) { + if (!active[k]) { + continue; + } + var_idx_t x = right[k]; // "y" = "x" + bool is_last = true; + for (std::size_t l = k + 1; l < right.size(); l++) { + if (right[l] == x && active[l]) { + is_last = false; + } + } + if (is_last) { + auto info = var_info[x]; + is_last = (info && info->is_last()); + } + if (is_last) { + stack.assign_var(--i, x); + } else { + stack.do_copy_var(--i, x); + } + } + i = 0; + for (std::size_t k = 0; k < left.size(); k++) { + if (active[k]) { + stack.assign_var(left[k], --i); + } + } + return true; + } + case _Tuple: + case _UnTuple: { + if (disabled()) { + return true; + } + std::vector last; + for (var_idx_t x : right) { + last.push_back(var_info[x] && var_info[x]->is_last()); + } + stack.rearrange_top(right, std::move(last)); + stack.opt_show(); + int k = (int)stack.depth() - (int)right.size(); + tolk_assert(k >= 0); + if (cl == _Tuple) { + stack.o << AsmOp::Tuple((int)right.size()); + tolk_assert(left.size() == 1); + } else { + stack.o << AsmOp::UnTuple((int)left.size()); + tolk_assert(right.size() == 1); + } + stack.s.resize(k); + for (int i = 0; i < (int)left.size(); i++) { + stack.push_new_var(left.at(i)); + } + return true; + } + case _Call: + case _CallInd: { + if (disabled()) { + return true; + } + // f_sym can be nullptr for Op::_CallInd (invoke a variable, not a function) + const std::vector* arg_order = f_sym ? f_sym->get_arg_order() : nullptr; + const std::vector* ret_order = f_sym ? f_sym->get_ret_order() : nullptr; + tolk_assert(!arg_order || arg_order->size() == right.size()); + tolk_assert(!ret_order || ret_order->size() == left.size()); + std::vector right1; + if (args.size()) { + tolk_assert(args.size() == right.size()); + for (int i = 0; i < (int)right.size(); i++) { + int j = arg_order ? arg_order->at(i) : i; + const VarDescr& arg = args.at(j); + if (!arg.is_unused()) { + tolk_assert(var_info[arg.idx] && !var_info[arg.idx]->is_unused()); + right1.push_back(arg.idx); + } + } + } else { + tolk_assert(!arg_order); + right1 = right; + } + std::vector last; + last.reserve(right1.size()); + for (var_idx_t x : right1) { + last.push_back(var_info[x] && var_info[x]->is_last()); + } + stack.rearrange_top(right1, std::move(last)); + stack.opt_show(); + int k = (int)stack.depth() - (int)right1.size(); + tolk_assert(k >= 0); + for (int i = 0; i < (int)right1.size(); i++) { + if (stack.s[k + i].first != right1[i]) { + std::cerr << stack.o; + } + tolk_assert(stack.s[k + i].first == right1[i]); + } + auto exec_callxargs = [&](int args, int ret) { + if (args <= 15 && ret <= 15) { + stack.o << exec_arg2_op("CALLXARGS", args, ret, args + 1, ret); + } else { + tolk_assert(args <= 254 && ret <= 254); + stack.o << AsmOp::Const(PSTRING() << args << " PUSHINT"); + stack.o << AsmOp::Const(PSTRING() << ret << " PUSHINT"); + stack.o << AsmOp::Custom("CALLXVARARGS", args + 3, ret); + } + }; + if (cl == _CallInd) { + exec_callxargs((int)right.size() - 1, (int)left.size()); + } else if (!f_sym->is_code_function()) { + std::vector res; + res.reserve(left.size()); + for (var_idx_t i : left) { + res.emplace_back(i); + } + if (f_sym->is_asm_function()) { + std::get(f_sym->body)->compile(stack.o); // compile res := f (args) + } else { + std::get(f_sym->body)->compile(stack.o, res, args, where); // compile res := f (args) + } + } else { + if (f_sym->is_inline() || f_sym->is_inline_ref()) { + stack.o << AsmOp::Custom(f_sym->name + " INLINECALLDICT", (int)right.size(), (int)left.size()); + } else if (f_sym->is_code_function() && std::get(f_sym->body)->code->require_callxargs) { + stack.o << AsmOp::Custom(f_sym->name + (" PREPAREDICT"), 0, 2); + exec_callxargs((int)right.size() + 1, (int)left.size()); + } else { + stack.o << AsmOp::Custom(f_sym->name + " CALLDICT", (int)right.size(), (int)left.size()); + } + } + stack.s.resize(k); + for (int i = 0; i < (int)left.size(); i++) { + int j = ret_order ? ret_order->at(i) : i; + stack.push_new_var(left.at(j)); + } + return !f_sym || f_sym->declared_return_type != TypeDataNever::create(); + } + case _SetGlob: { + tolk_assert(g_sym); + std::vector last; + for (var_idx_t x : right) { + last.push_back(var_info[x] && var_info[x]->is_last()); + } + stack.rearrange_top(right, std::move(last)); + stack.opt_show(); + int k = (int)stack.depth() - (int)right.size(); + tolk_assert(k >= 0); + for (int i = 0; i < (int)right.size(); i++) { + if (stack.s[k + i].first != right[i]) { + std::cerr << stack.o; + } + tolk_assert(stack.s[k + i].first == right[i]); + } + if (right.size() > 1) { + stack.o << AsmOp::Tuple((int)right.size()); + } + if (!right.empty()) { + stack.o << AsmOp::Custom(g_sym->name + " SETGLOB", 1, 0); + } + stack.s.resize(k); + return true; + } + case _If: { + if (block0->is_empty() && block1->is_empty()) { + return true; + } + if (!next->noreturn() && (block0->noreturn() != block1->noreturn())) { + stack.o.retalt_ = true; + } + var_idx_t x = left[0]; + stack.rearrange_top(x, var_info[x] && var_info[x]->is_last()); + tolk_assert(stack[0] == x); + stack.opt_show(); + stack.s.pop_back(); + stack.modified(); + if (inline_func && (block0->noreturn() || block1->noreturn())) { + bool is0 = block0->noreturn(); + Op* block_noreturn = is0 ? block0.get() : block1.get(); + Op* block_other = is0 ? block1.get() : block0.get(); + stack.mode &= ~Stack::_InlineFunc; + stack.o << (is0 ? "IF:<{" : "IFNOT:<{"); + stack.o.indent(); + Stack stack_copy{stack}; + block_noreturn->generate_code_all(stack_copy); + stack.o.undent(); + stack.o << "}>ELSE<{"; + stack.o.indent(); + block_other->generate_code_all(stack); + if (!block_other->noreturn()) { + next->generate_code_all(stack); + } + stack.o.undent(); + stack.o << "}>"; + return false; + } + if (block1->is_empty() || block0->is_empty()) { + bool is0 = block1->is_empty(); + Op* block = is0 ? block0.get() : block1.get(); + // if (left) block0; ... + // if (!left) block1; ... + if (block->noreturn()) { + stack.o << (is0 ? "IFJMP:<{" : "IFNOTJMP:<{"); + stack.o.indent(); + Stack stack_copy{stack}; + stack_copy.mode &= ~Stack::_InlineFunc; + stack_copy.mode |= next->noreturn() ? 0 : Stack::_NeedRetAlt; + block->generate_code_all(stack_copy); + stack.o.undent(); + stack.o << "}>"; + return true; + } + stack.o << (is0 ? "IF:<{" : "IFNOT:<{"); + stack.o.indent(); + Stack stack_copy{stack}, stack_target{stack}; + stack_target.disable_output(); + stack_target.drop_vars_except(next->var_info); + stack_copy.mode &= ~Stack::_InlineFunc; + block->generate_code_all(stack_copy); + stack_copy.drop_vars_except(var_info); + stack_copy.opt_show(); + if ((is0 && stack_copy == stack) || (!is0 && stack_copy.vars() == stack.vars())) { + stack.o.undent(); + stack.o << "}>"; + if (!is0) { + stack.merge_const(stack_copy); + } + return true; + } + // stack_copy.drop_vars_except(next->var_info); + stack_copy.enforce_state(stack_target.vars()); + stack_copy.opt_show(); + if (stack_copy.vars() == stack.vars()) { + stack.o.undent(); + stack.o << "}>"; + stack.merge_const(stack_copy); + return true; + } + stack.o.undent(); + stack.o << "}>ELSE<{"; + stack.o.indent(); + stack.merge_state(stack_copy); + stack.opt_show(); + stack.o.undent(); + stack.o << "}>"; + return true; + } + if (block0->noreturn() || block1->noreturn()) { + bool is0 = block0->noreturn(); + Op* block_noreturn = is0 ? block0.get() : block1.get(); + Op* block_other = is0 ? block1.get() : block0.get(); + stack.o << (is0 ? "IFJMP:<{" : "IFNOTJMP:<{"); + stack.o.indent(); + Stack stack_copy{stack}; + stack_copy.mode &= ~Stack::_InlineFunc; + stack_copy.mode |= (block_other->noreturn() || next->noreturn()) ? 0 : Stack::_NeedRetAlt; + block_noreturn->generate_code_all(stack_copy); + stack.o.undent(); + stack.o << "}>"; + block_other->generate_code_all(stack); + return !block_other->noreturn(); + } + stack.o << "IF:<{"; + stack.o.indent(); + Stack stack_copy{stack}; + stack_copy.mode &= ~Stack::_InlineFunc; + block0->generate_code_all(stack_copy); + stack_copy.drop_vars_except(next->var_info); + stack_copy.opt_show(); + stack.o.undent(); + stack.o << "}>ELSE<{"; + stack.o.indent(); + stack.mode &= ~Stack::_InlineFunc; + block1->generate_code_all(stack); + stack.merge_state(stack_copy); + stack.opt_show(); + stack.o.undent(); + stack.o << "}>"; + return true; + } + case _Repeat: { + var_idx_t x = left[0]; + //stack.drop_vars_except(block0->var_info, x); + stack.rearrange_top(x, var_info[x] && var_info[x]->is_last()); + tolk_assert(stack[0] == x); + stack.opt_show(); + stack.s.pop_back(); + stack.modified(); + if (block0->noreturn()) { + stack.o.retalt_ = true; + } + if (true || !next->is_empty()) { + stack.o << "REPEAT:<{"; + stack.o.indent(); + stack.forget_const(); + if (block0->noreturn()) { + Stack stack_copy{stack}; + StackLayout layout1 = stack.vars(); + stack_copy.mode &= ~Stack::_InlineFunc; + stack_copy.mode |= Stack::_NeedRetAlt; + block0->generate_code_all(stack_copy); + } else { + StackLayout layout1 = stack.vars(); + stack.mode &= ~Stack::_InlineFunc; + stack.mode |= Stack::_NeedRetAlt; + block0->generate_code_all(stack); + stack.enforce_state(std::move(layout1)); + stack.opt_show(); + } + stack.o.undent(); + stack.o << "}>"; + return true; + } else { + stack.o << "REPEATEND"; + stack.forget_const(); + StackLayout layout1 = stack.vars(); + block0->generate_code_all(stack); + stack.enforce_state(std::move(layout1)); + stack.opt_show(); + return false; + } + } + case _Again: { + stack.drop_vars_except(block0->var_info); + stack.opt_show(); + if (block0->noreturn()) { + stack.o.retalt_ = true; + } + if (!next->is_empty() || inline_func) { + stack.o << "AGAIN:<{"; + stack.o.indent(); + stack.forget_const(); + StackLayout layout1 = stack.vars(); + stack.mode &= ~Stack::_InlineFunc; + stack.mode |= Stack::_NeedRetAlt; + block0->generate_code_all(stack); + stack.enforce_state(std::move(layout1)); + stack.opt_show(); + stack.o.undent(); + stack.o << "}>"; + return true; + } else { + stack.o << "AGAINEND"; + stack.forget_const(); + StackLayout layout1 = stack.vars(); + block0->generate_code_all(stack); + stack.enforce_state(std::move(layout1)); + stack.opt_show(); + return false; + } + } + case _Until: { + // stack.drop_vars_except(block0->var_info); + // stack.opt_show(); + if (block0->noreturn()) { + stack.o.retalt_ = true; + } + if (true || !next->is_empty()) { + stack.o << "UNTIL:<{"; + stack.o.indent(); + stack.forget_const(); + auto layout1 = stack.vars(); + stack.mode &= ~Stack::_InlineFunc; + stack.mode |= Stack::_NeedRetAlt; + block0->generate_code_all(stack); + layout1.push_back(left[0]); + stack.enforce_state(std::move(layout1)); + stack.opt_show(); + stack.o.undent(); + stack.o << "}>"; + stack.s.pop_back(); + stack.modified(); + return true; + } else { + stack.o << "UNTILEND"; + stack.forget_const(); + StackLayout layout1 = stack.vars(); + block0->generate_code_all(stack); + layout1.push_back(left[0]); + stack.enforce_state(std::move(layout1)); + stack.opt_show(); + return false; + } + } + case _While: { + // while (block0 | left) block1; ...next + var_idx_t x = left[0]; + stack.drop_vars_except(block0->var_info); + stack.opt_show(); + StackLayout layout1 = stack.vars(); + bool next_empty = false && next->is_empty(); + if (block0->noreturn()) { + stack.o.retalt_ = true; + } + stack.o << "WHILE:<{"; + stack.o.indent(); + stack.forget_const(); + stack.mode &= ~Stack::_InlineFunc; + stack.mode |= Stack::_NeedRetAlt; + block0->generate_code_all(stack); + stack.rearrange_top(x, !next->var_info[x] && !block1->var_info[x]); + stack.opt_show(); + stack.s.pop_back(); + stack.modified(); + stack.o.undent(); + Stack stack_copy{stack}; + stack.o << (next_empty ? "}>DO:" : "}>DO<{"); + if (!next_empty) { + stack.o.indent(); + } + stack_copy.opt_show(); + block1->generate_code_all(stack_copy); + stack_copy.enforce_state(std::move(layout1)); + stack_copy.opt_show(); + if (!next_empty) { + stack.o.undent(); + stack.o << "}>"; + return true; + } else { + return false; + } + } + case _TryCatch: { + if (block0->is_empty() && block1->is_empty()) { + return true; + } + if (block0->noreturn() || block1->noreturn()) { + stack.o.retalt_ = true; + } + Stack catch_stack{stack.o}; + std::vector catch_vars; + std::vector catch_last; + for (const VarDescr& var : block1->var_info.list) { + if (stack.find(var.idx) >= 0) { + catch_vars.push_back(var.idx); + catch_last.push_back(!block0->var_info[var.idx]); + } + } + const size_t block_size = 255; + for (size_t begin = catch_vars.size(), end = begin; end > 0; end = begin) { + begin = end >= block_size ? end - block_size : 0; + for (size_t i = begin; i < end; ++i) { + catch_stack.push_new_var(catch_vars[i]); + } + } + catch_stack.push_new_var(left[0]); + catch_stack.push_new_var(left[1]); + stack.rearrange_top(catch_vars, catch_last); + stack.opt_show(); + stack.o << "c1 PUSH"; + stack.o << "c3 PUSH"; + stack.o << "c4 PUSH"; + stack.o << "c5 PUSH"; + stack.o << "c7 PUSH"; + stack.o << "<{"; + stack.o.indent(); + if (block1->noreturn()) { + catch_stack.mode |= Stack::_NeedRetAlt; + } + block1->generate_code_all(catch_stack); + catch_stack.drop_vars_except(next->var_info); + catch_stack.opt_show(); + stack.o.undent(); + stack.o << "}>CONT"; + stack.o << "c7 SETCONT"; + stack.o << "c5 SETCONT"; + stack.o << "c4 SETCONT"; + stack.o << "c3 SETCONT"; + stack.o << "c1 SETCONT"; + for (size_t begin = catch_vars.size(), end = begin; end > 0; end = begin) { + begin = end >= block_size ? end - block_size : 0; + stack.o << std::to_string(end - begin) + " PUSHINT"; + stack.o << "-1 PUSHINT"; + stack.o << "SETCONTVARARGS"; + } + stack.s.erase(stack.s.end() - catch_vars.size(), stack.s.end()); + stack.modified(); + stack.o << "<{"; + stack.o.indent(); + if (block0->noreturn()) { + stack.mode |= Stack::_NeedRetAlt; + } + block0->generate_code_all(stack); + if (block0->noreturn()) { + stack.s = std::move(catch_stack.s); + } else if (!block1->noreturn()) { + stack.merge_state(catch_stack); + } + stack.opt_show(); + stack.o.undent(); + stack.o << "}>CONT"; + stack.o << "c1 PUSH"; + stack.o << "COMPOSALT"; + stack.o << "SWAP"; + stack.o << "TRY"; + return true; + } + default: + std::cerr << "fatal: unknown operation \n"; + throw ParseError{where, "unknown operation in generate_code()"}; + } +} + +void Op::generate_code_all(Stack& stack) { + int saved_mode = stack.mode; + auto cont = generate_code_step(stack); + stack.mode = (stack.mode & ~Stack::_ModeSave) | (saved_mode & Stack::_ModeSave); + if (cont && next) { + next->generate_code_all(stack); + } +} + +void CodeBlob::generate_code(AsmOpList& out, int mode) { + Stack stack{out, mode}; + tolk_assert(ops && ops->cl == Op::_Import); + auto args = (int)ops->left.size(); + for (var_idx_t x : ops->left) { + stack.push_new_var(x); + } + ops->generate_code_all(stack); + stack.apply_wrappers(require_callxargs && (mode & Stack::_InlineAny) ? args : -1); +} + +void CodeBlob::generate_code(std::ostream& os, int mode, int indent) { + AsmOpList out_list(indent, &vars); + generate_code(out_list, mode); + if (G.settings.optimization_level >= 2) { + optimize_code(out_list); + } + out_list.out(os, mode); +} + +} // namespace tolk diff --git a/tolk/compiler-state.cpp b/tolk/compiler-state.cpp new file mode 100644 index 00000000..95a7e6a5 --- /dev/null +++ b/tolk/compiler-state.cpp @@ -0,0 +1,73 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "compiler-state.h" +#include +#include + +namespace tolk { + +CompilerState G; // the only mutable global variable in tolk internals + +void ExperimentalOption::mark_deprecated(const char* deprecated_from_v, const char* deprecated_reason) { + this->deprecated_from_v = deprecated_from_v; + this->deprecated_reason = deprecated_reason; +} + +std::string_view PersistentHeapAllocator::copy_string_to_persistent_memory(std::string_view str_in_tmp_memory) { + size_t len = str_in_tmp_memory.size(); + char* allocated = new char[len]; + memcpy(allocated, str_in_tmp_memory.data(), str_in_tmp_memory.size()); + auto new_chunk = std::make_unique(allocated, std::move(head)); + head = std::move(new_chunk); + return {head->allocated, len}; +} + +void PersistentHeapAllocator::clear() { + head = nullptr; +} + +void CompilerSettings::enable_experimental_option(std::string_view name) { + ExperimentalOption* to_enable = nullptr; + + if (name == remove_unused_functions.name) { + to_enable = &remove_unused_functions; + } + + if (to_enable == nullptr) { + std::cerr << "unknown experimental option: " << name << std::endl; + } else if (to_enable->deprecated_from_v) { + std::cerr << "experimental option " << name << " " + << "is deprecated since Tolk v" << to_enable->deprecated_from_v + << ": " << to_enable->deprecated_reason << std::endl; + } else { + to_enable->enabled = true; + } +} + +void CompilerSettings::parse_experimental_options_cmd_arg(const std::string& cmd_arg) { + std::istringstream stream(cmd_arg); + std::string token; + while (std::getline(stream, token, ',')) { + enable_experimental_option(token); + } +} + +const std::vector& get_all_not_builtin_functions() { + return G.all_functions; +} + +} // namespace tolk diff --git a/tolk/compiler-state.h b/tolk/compiler-state.h new file mode 100644 index 00000000..1d166a3a --- /dev/null +++ b/tolk/compiler-state.h @@ -0,0 +1,109 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "src-file.h" +#include "symtable.h" +#include "td/utils/Status.h" +#include +#include +#include + +namespace tolk { + +// with cmd option -x, the user can pass experimental options to use +class ExperimentalOption { + friend struct CompilerSettings; + + const std::string_view name; + bool enabled = false; + const char* deprecated_from_v = nullptr; // when an option becomes deprecated (after the next compiler release), + const char* deprecated_reason = nullptr; // but the user still passes it, we'll warn to stderr + +public: + explicit ExperimentalOption(std::string_view name) : name(name) {} + + void mark_deprecated(const char* deprecated_from_v, const char* deprecated_reason); + + explicit operator bool() const { return enabled; } +}; + +// CompilerSettings contains settings that can be passed via cmd line or (partially) wasm envelope. +// They are filled once at start and are immutable since the compilation started. +struct CompilerSettings { + enum class FsReadCallbackKind { Realpath, ReadFile }; + + using FsReadCallback = std::function(FsReadCallbackKind, const char*)>; + + int verbosity = 0; + int optimization_level = 2; + bool stack_layout_comments = true; + + std::string output_filename; + std::string boc_output_filename; + std::string stdlib_folder; // a path to tolk-stdlib/; files imported via @stdlib/xxx are there + + FsReadCallback read_callback; + + ExperimentalOption remove_unused_functions{"remove-unused-functions"}; + + void enable_experimental_option(std::string_view name); + void parse_experimental_options_cmd_arg(const std::string& cmd_arg); +}; + +// AST nodes contain std::string_view referencing to contents of .tolk files (kept in memory after reading). +// It's more than enough, except a situation when we create new AST nodes inside the compiler +// and want some "persistent place" for std::string_view to point to. +// This class copies strings to heap, so that they remain valid after closing scope. +class PersistentHeapAllocator { + struct ChunkInHeap { + const char* allocated; + std::unique_ptr next; + + ChunkInHeap(const char* allocated, std::unique_ptr&& next) + : allocated(allocated), next(std::move(next)) {} + }; + + std::unique_ptr head = nullptr; + +public: + std::string_view copy_string_to_persistent_memory(std::string_view str_in_tmp_memory); + void clear(); +}; + +// CompilerState contains a mutable state that is changed while the compilation is going on. +// It's a "global state" of all compilation. +// Historically, in FunC, this global state was spread along many global C++ variables. +// Now, no global C++ variables except `CompilerState G` are present. +struct CompilerState { + CompilerSettings settings; + + GlobalSymbolTable symtable; + PersistentHeapAllocator persistent_mem; + + std::vector all_functions; // all user-defined (not built-in) functions, with generic instantiations + std::vector all_get_methods; + std::vector all_global_vars; + std::vector all_constants; + AllRegisteredSrcFiles all_src_files; + + bool is_verbosity(int gt_eq) const { return settings.verbosity >= gt_eq; } +}; + +extern CompilerState G; + +} // namespace tolk diff --git a/tolk/constant-evaluator.cpp b/tolk/constant-evaluator.cpp new file mode 100644 index 00000000..4d11b922 --- /dev/null +++ b/tolk/constant-evaluator.cpp @@ -0,0 +1,317 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "constant-evaluator.h" +#include "ast.h" +#include "tolk.h" +#include "openssl/digest.hpp" +#include "crypto/common/util.h" +#include "td/utils/crypto.h" +#include "ton/ton-types.h" + +namespace tolk { + +// parse address like "EQCRDM9h4k3UJdOePPuyX40mCgA4vxge5Dc5vjBR8djbEKC5" +// based on unpack_std_smc_addr() from block.cpp +// (which is not included to avoid linking with ton_crypto) +static bool parse_friendly_address(const char packed[48], ton::WorkchainId& workchain, ton::StdSmcAddress& addr) { + unsigned char buffer[36]; + if (!td::buff_base64_decode(td::MutableSlice{buffer, 36}, td::Slice{packed, 48}, true)) { + return false; + } + td::uint16 crc = td::crc16(td::Slice{buffer, 34}); + if (buffer[34] != (crc >> 8) || buffer[35] != (crc & 0xff) || (buffer[0] & 0x3f) != 0x11) { + return false; + } + workchain = static_cast(buffer[1]); + std::memcpy(addr.data(), buffer + 2, 32); + return true; +} + +// parse address like "0:527964d55cfa6eb731f4bfc07e9d025098097ef8505519e853986279bd8400d8" +// based on StdAddress::parse_addr() from block.cpp +// (which is not included to avoid linking with ton_crypto) +static bool parse_raw_address(const std::string& acc_string, int& workchain, ton::StdSmcAddress& addr) { + size_t pos = acc_string.find(':'); + if (pos != std::string::npos) { + td::Result r_wc = td::to_integer_safe(acc_string.substr(0, pos)); + if (r_wc.is_error()) { + return false; + } + workchain = r_wc.move_as_ok(); + pos++; + } else { + pos = 0; + } + if (acc_string.size() != pos + 64) { + return false; + } + + for (int i = 0; i < 64; ++i) { // loop through each hex digit + char c = acc_string[pos + i]; + int x; + if (c >= '0' && c <= '9') { + x = c - '0'; + } else if (c >= 'a' && c <= 'z') { + x = c - 'a' + 10; + } else if (c >= 'A' && c <= 'Z') { + x = c - 'A' + 10; + } else { + return false; + } + + if ((i & 1) == 0) { + addr.data()[i >> 1] = static_cast((addr.data()[i >> 1] & 0x0F) | (x << 4)); + } else { + addr.data()[i >> 1] = static_cast((addr.data()[i >> 1] & 0xF0) | x); + } + } + return true; +} + + +static std::string parse_vertex_string_const_as_slice(V v) { + std::string str = static_cast(v->str_val); + switch (v->modifier) { + case 0: { + return td::hex_encode(str); + } + case 's': { + unsigned char buff[128]; + long bits = td::bitstring::parse_bitstring_hex_literal(buff, sizeof(buff), str.data(), str.data() + str.size()); + if (bits < 0) { + v->error("invalid hex bitstring constant '" + str + "'"); + } + return str; + } + case 'a': { // MsgAddress + ton::WorkchainId workchain; + ton::StdSmcAddress addr; + bool correct = (str.size() == 48 && parse_friendly_address(str.data(), workchain, addr)) || + (str.size() != 48 && parse_raw_address(str, workchain, addr)); + if (!correct) { + v->error("invalid standard address '" + str + "'"); + } + if (workchain < -128 || workchain >= 128) { + v->error("anycast addresses not supported"); + } + + unsigned char data[3 + 8 + 256]; // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; + td::bitstring::bits_store_long_top(data, 0, static_cast(4) << (64 - 3), 3); + td::bitstring::bits_store_long_top(data, 3, static_cast(workchain) << (64 - 8), 8); + td::bitstring::bits_memcpy(data, 3 + 8, addr.bits().ptr, 0, ton::StdSmcAddress::size()); + return td::BitSlice{data, sizeof(data)}.to_hex(); + } + default: + tolk_assert(false); + } +} + +static td::RefInt256 parse_vertex_string_const_as_int(V v) { + std::string str = static_cast(v->str_val); + switch (v->modifier) { + case 'u': { + td::RefInt256 intval = td::hex_string_to_int256(td::hex_encode(str)); + if (str.empty()) { + v->error("empty integer ascii-constant"); + } + if (intval.is_null()) { + v->error("too long integer ascii-constant"); + } + return intval; + } + case 'h': + case 'H': { + unsigned char hash[32]; + digest::hash_str(hash, str.data(), str.size()); + return td::bits_to_refint(hash, (v->modifier == 'h') ? 32 : 256, false); + } + case 'c': { + return td::make_refint(td::crc32(td::Slice{str})); + } + default: + tolk_assert(false); + } +} + + +struct ConstantEvaluator { + static bool is_overflow(const td::RefInt256& intval) { + return intval.is_null() || !intval->signed_fits_bits(257); + } + + static ConstantValue handle_unary_operator(V v, const ConstantValue& rhs) { + if (!rhs.is_int()) { + v->error("invalid operator, expecting integer"); + } + td::RefInt256 intval = std::get(rhs.value); + + switch (v->tok) { + case tok_minus: + intval = -intval; + break; + case tok_plus: + break; + case tok_bitwise_not: + intval = ~intval; + break; + case tok_logical_not: + intval = td::make_refint(intval == 0 ? -1 : 0); + break; + default: + v->error("not a constant expression"); + } + + if (is_overflow(intval)) { + v->error("integer overflow"); + } + return ConstantValue::from_int(std::move(intval)); + } + + static ConstantValue handle_binary_operator(V v, const ConstantValue& lhs, const ConstantValue& rhs) { + if (!lhs.is_int() || !rhs.is_int()) { + v->error("invalid operator, expecting integer"); + } + td::RefInt256 lhs_intval = std::get(lhs.value); + td::RefInt256 rhs_intval = std::get(rhs.value); + td::RefInt256 intval; + + switch (v->tok) { + case tok_minus: + intval = lhs_intval - rhs_intval; + break; + case tok_plus: + intval = lhs_intval + rhs_intval; + break; + case tok_mul: + intval = lhs_intval * rhs_intval; + break; + case tok_div: + intval = lhs_intval / rhs_intval; + break; + case tok_mod: + intval = lhs_intval % rhs_intval; + break; + case tok_lshift: + intval = lhs_intval << static_cast(rhs_intval->to_long()); + break; + case tok_rshift: + intval = lhs_intval >> static_cast(rhs_intval->to_long()); + break; + case tok_bitwise_and: + intval = lhs_intval & rhs_intval; + break; + case tok_bitwise_or: + intval = lhs_intval | rhs_intval; + break; + case tok_bitwise_xor: + intval = lhs_intval ^ rhs_intval; + break; + case tok_eq: + intval = td::make_refint(lhs_intval == rhs_intval ? -1 : 0); + break; + case tok_lt: + intval = td::make_refint(lhs_intval < rhs_intval ? -1 : 0); + break; + case tok_gt: + intval = td::make_refint(lhs_intval > rhs_intval ? -1 : 0); + break; + case tok_leq: + intval = td::make_refint(lhs_intval <= rhs_intval ? -1 : 0); + break; + case tok_geq: + intval = td::make_refint(lhs_intval >= rhs_intval ? -1 : 0); + break; + case tok_neq: + intval = td::make_refint(lhs_intval != rhs_intval ? -1 : 0); + break; + default: + v->error("unsupported binary operator in constant expression"); + } + + if (is_overflow(intval)) { + v->error("integer overflow"); + } + return ConstantValue::from_int(std::move(intval)); + } + + static ConstantValue handle_reference(V v) { + // todo better handle "appears, directly or indirectly, in its own initializer" + std::string_view name = v->get_name(); + const Symbol* sym = lookup_global_symbol(name); + if (!sym) { + v->error("undefined symbol `" + static_cast(name) + "`"); + } + GlobalConstPtr const_ref = sym->try_as(); + if (!const_ref) { + v->error("symbol `" + static_cast(name) + "` is not a constant"); + } + if (v->has_instantiationTs()) { // SOME_CONST + v->error("constant is not a generic"); + } + return {const_ref->value}; + } + + static ConstantValue visit(AnyExprV v) { + if (auto v_int = v->try_as()) { + return ConstantValue::from_int(v_int->intval); + } + if (auto v_bool = v->try_as()) { + return ConstantValue::from_int(v_bool->bool_val ? -1 : 0); + } + if (auto v_unop = v->try_as()) { + return handle_unary_operator(v_unop, visit(v_unop->get_rhs())); + } + if (auto v_binop = v->try_as()) { + return handle_binary_operator(v_binop, visit(v_binop->get_lhs()), visit(v_binop->get_rhs())); + } + if (auto v_ref = v->try_as()) { + return handle_reference(v_ref); + } + if (auto v_par = v->try_as()) { + return visit(v_par->get_expr()); + } + if (v->try_as()) { + return eval_const_init_value(v); + } + v->error("not a constant expression"); + } + + static ConstantValue eval_const_init_value(AnyExprV init_value) { + // it init_value is incorrect, an exception is thrown + return visit(init_value); + } +}; + +ConstantValue eval_const_init_value(AnyExprV init_value) { + // at first, handle most simple cases, not to launch heavy computation algorithm: just a number, just a string + // just `c = 1` or `c = 0xFF` + if (auto v_int = init_value->try_as()) { + return {v_int->intval}; + } + // just `c = "strval"`, probably with modifier (address, etc.) + if (auto v_string = init_value->try_as()) { + if (v_string->is_bitslice()) { + return {parse_vertex_string_const_as_slice(v_string)}; + } else { + return {parse_vertex_string_const_as_int(v_string)}; + } + } + // something more complex, like `c = anotherC` or `c = 1 << 8` + return ConstantEvaluator::eval_const_init_value(init_value); +} + +} // namespace tolk diff --git a/tolk/constant-evaluator.h b/tolk/constant-evaluator.h new file mode 100644 index 00000000..0f99867d --- /dev/null +++ b/tolk/constant-evaluator.h @@ -0,0 +1,45 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "fwd-declarations.h" +#include "crypto/common/refint.h" +#include + +namespace tolk { + +struct ConstantValue { + std::variant value; + + bool is_int() const { return std::holds_alternative(value); } + bool is_slice() const { return std::holds_alternative(value); } + + td::RefInt256 as_int() const { return std::get(value); } + const std::string& as_slice() const { return std::get(value); } + + static ConstantValue from_int(int value) { + return {td::make_refint(value)}; + } + + static ConstantValue from_int(td::RefInt256 value) { + return {std::move(value)}; + } +}; + +ConstantValue eval_const_init_value(AnyExprV init_value); + +} // namespace tolk diff --git a/tonlib/tonlib/ExtClientLazy.h b/tolk/fwd-declarations.h similarity index 52% rename from tonlib/tonlib/ExtClientLazy.h rename to tolk/fwd-declarations.h index dc4490b3..8d3b24a8 100644 --- a/tonlib/tonlib/ExtClientLazy.h +++ b/tolk/fwd-declarations.h @@ -13,29 +13,33 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP */ #pragma once -#include "td/actor/actor.h" -#include "adnl/adnl-ext-client.h" +namespace tolk { -namespace tonlib { -class ExtClientLazy : public ton::adnl::AdnlExtClient { - public: - class Callback { - public: - virtual ~Callback() { - } - }; +struct ASTNodeBase; +struct ASTNodeExpressionBase; +struct ASTNodeStatementBase; - virtual void force_change_liteserver() = 0; +using AnyV = const ASTNodeBase*; +using AnyExprV = const ASTNodeExpressionBase*; +using AnyStatementV = const ASTNodeStatementBase*; - static td::actor::ActorOwn create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, - td::unique_ptr callback); - static td::actor::ActorOwn create( - std::vector> servers, td::unique_ptr callback); -}; +struct Symbol; +struct LocalVarData; +struct FunctionData; +struct GlobalVarData; +struct GlobalConstData; -} // namespace tonlib +using LocalVarPtr = const LocalVarData*; +using FunctionPtr = const FunctionData*; +using GlobalVarPtr = const GlobalVarData*; +using GlobalConstPtr = const GlobalConstData*; + +class TypeData; +using TypePtr = const TypeData*; + +struct SrcFile; + +} // namespace tolk diff --git a/tolk/generics-helpers.cpp b/tolk/generics-helpers.cpp new file mode 100644 index 00000000..9dae3f00 --- /dev/null +++ b/tolk/generics-helpers.cpp @@ -0,0 +1,271 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "generics-helpers.h" +#include "tolk.h" +#include "ast.h" +#include "ast-replicator.h" +#include "type-system.h" +#include "compiler-state.h" +#include "pipeline.h" + +namespace tolk { + +// given orig = "(int, T)" and substitutions = [slice], return "(int, slice)" +static TypePtr replace_genericT_with_deduced(TypePtr orig, const GenericsDeclaration* genericTs, const std::vector& substitutionTs) { + if (!orig || !orig->has_genericT_inside()) { + return orig; + } + tolk_assert(genericTs->size() == substitutionTs.size()); + + return orig->replace_children_custom([genericTs, substitutionTs](TypePtr child) { + if (const TypeDataGenericT* asT = child->try_as()) { + int idx = genericTs->find_nameT(asT->nameT); + if (idx == -1) { + throw Fatal("can not replace generic " + asT->nameT); + } + if (substitutionTs[idx] == nullptr) { + throw GenericDeduceError("can not deduce " + asT->nameT); + } + return substitutionTs[idx]; + } + return child; + }); +} + +GenericSubstitutionsDeduceForCall::GenericSubstitutionsDeduceForCall(FunctionPtr fun_ref) + : fun_ref(fun_ref) { + substitutionTs.resize(fun_ref->genericTs->size()); // filled with nullptr (nothing deduced) +} + +void GenericSubstitutionsDeduceForCall::provide_deducedT(const std::string& nameT, TypePtr deduced) { + if (deduced == TypeDataNullLiteral::create() || deduced->has_unknown_inside()) { + return; // just 'null' doesn't give sensible info + } + + int idx = fun_ref->genericTs->find_nameT(nameT); + if (substitutionTs[idx] == nullptr) { + substitutionTs[idx] = deduced; + } else if (substitutionTs[idx] != deduced) { + throw GenericDeduceError(nameT + " is both " + substitutionTs[idx]->as_human_readable() + " and " + deduced->as_human_readable()); + } +} + +void GenericSubstitutionsDeduceForCall::provide_manually_specified(std::vector&& substitutionTs) { + this->substitutionTs = std::move(substitutionTs); + this->manually_specified = true; +} + +// purpose: having `f(value: T)` and call `f(5)`, deduce T = int +// generally, there may be many generic Ts for declaration, and many arguments +// for every argument, `consider_next_condition()` is called +// example: `f(a: int, b: T1, c: (T1, T2))` and call `f(6, 7, (8, cs))` +// - `a` does not affect, it doesn't depend on generic Ts +// - next condition: param_type = `T1`, arg_type = `int`, deduce T1 = int +// - next condition: param_type = `(T1, T2)`, arg_type = `(int, slice)`, deduce T1 = int, T2 = slice +// for call `f(6, cs, (8, cs))` T1 will be both `slice` and `int`, fired an error +void GenericSubstitutionsDeduceForCall::consider_next_condition(TypePtr param_type, TypePtr arg_type) { + if (const auto* asT = param_type->try_as()) { + // `(arg: T)` called as `f([1, 2])` => T is [int, int] + provide_deducedT(asT->nameT, arg_type); + } else if (const auto* p_nullable = param_type->try_as()) { + // `arg: T?` called as `f(nullableInt)` => T is int + if (const auto* a_nullable = arg_type->try_as()) { + consider_next_condition(p_nullable->inner, a_nullable->inner); + } + // `arg: T?` called as `f(int)` => T is int + else { + consider_next_condition(p_nullable->inner, arg_type); + } + } else if (const auto* p_tensor = param_type->try_as()) { + // `arg: (int, T)` called as `f((5, cs))` => T is slice + if (const auto* a_tensor = arg_type->try_as(); a_tensor && a_tensor->size() == p_tensor->size()) { + for (int i = 0; i < a_tensor->size(); ++i) { + consider_next_condition(p_tensor->items[i], a_tensor->items[i]); + } + } + } else if (const auto* p_tuple = param_type->try_as()) { + // `arg: [int, T]` called as `f([5, cs])` => T is slice + if (const auto* a_tuple = arg_type->try_as(); a_tuple && a_tuple->size() == p_tuple->size()) { + for (int i = 0; i < a_tuple->size(); ++i) { + consider_next_condition(p_tuple->items[i], a_tuple->items[i]); + } + } + } else if (const auto* p_callable = param_type->try_as()) { + // `arg: fun(TArg) -> TResult` called as `f(calcTupleLen)` => TArg is tuple, TResult is int + if (const auto* a_callable = arg_type->try_as(); a_callable && a_callable->params_size() == p_callable->params_size()) { + for (int i = 0; i < a_callable->params_size(); ++i) { + consider_next_condition(p_callable->params_types[i], a_callable->params_types[i]); + } + consider_next_condition(p_callable->return_type, a_callable->return_type); + } + } +} + +TypePtr GenericSubstitutionsDeduceForCall::replace_by_manually_specified(TypePtr param_type) const { + return replace_genericT_with_deduced(param_type, fun_ref->genericTs, substitutionTs); +} + +TypePtr GenericSubstitutionsDeduceForCall::auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type) { + try { + if (!manually_specified) { + consider_next_condition(param_type, arg_type); + } + return replace_genericT_with_deduced(param_type, fun_ref->genericTs, substitutionTs); + } catch (const GenericDeduceError& ex) { + throw ParseError(cur_f, loc, ex.message + " for generic function `" + fun_ref->as_human_readable() + "`; instantiate it manually with `" + fun_ref->name + "<...>()`"); + } +} + +int GenericSubstitutionsDeduceForCall::get_first_not_deduced_idx() const { + for (int i = 0; i < static_cast(substitutionTs.size()); ++i) { + if (substitutionTs[i] == nullptr) { + return i; + } + } + return -1; +} + +// clone the body of `f` replacing T everywhere with a substitution +// before: `fun f(v: T) { var cp: [T] = [v]; }` +// after: `fun f(v: int) { var cp: [int] = [v]; }` +// an instantiated function becomes a deep copy, all AST nodes are copied, no previous pointers left +class GenericFunctionReplicator final : public ASTReplicatorFunction { + const GenericsDeclaration* genericTs; + const std::vector& substitutionTs; + +protected: + using ASTReplicatorFunction::clone; + + TypePtr clone(TypePtr t) override { + return replace_genericT_with_deduced(t, genericTs, substitutionTs); + } + +public: + GenericFunctionReplicator(const GenericsDeclaration* genericTs, const std::vector& substitutionTs) + : genericTs(genericTs) + , substitutionTs(substitutionTs) { + } + + V clone_function_body(V v_function) override { + return createV( + v_function->loc, + clone(v_function->get_identifier()), + clone(v_function->get_param_list()), + clone(v_function->get_body()), + clone(v_function->declared_return_type), + nullptr, // a newly-created function is not generic + v_function->method_id, + v_function->flags + ); + } +}; + +std::string GenericsDeclaration::as_human_readable() const { + std::string result = "<"; + for (const GenericsItem& item : itemsT) { + if (result.size() > 1) { + result += ","; + } + result += item.nameT; + } + result += ">"; + return result; +} + +int GenericsDeclaration::find_nameT(std::string_view nameT) const { + for (int i = 0; i < static_cast(itemsT.size()); ++i) { + if (itemsT[i].nameT == nameT) { + return i; + } + } + return -1; +} + +// after creating a deep copy of `f` like `f`, its new and fresh body needs the previous pipeline to run +// for example, all local vars need to be registered as symbols, etc. +static void run_pipeline_for_instantiated_function(FunctionPtr inst_fun_ref) { + // these pipes are exactly the same as in tolk.cpp — all preceding (and including) type inferring + pipeline_resolve_identifiers_and_assign_symbols(inst_fun_ref); + pipeline_calculate_rvalue_lvalue(inst_fun_ref); + pipeline_infer_types_and_calls_and_fields(inst_fun_ref); +} + +std::string generate_instantiated_name(const std::string& orig_name, const std::vector& substitutions) { + // an instantiated function name will be "{orig_name}<{T1,T2,...}>" + std::string name = orig_name; + name += "<"; + for (TypePtr subs : substitutions) { + if (name.size() > orig_name.size() + 1) { + name += ","; + } + name += subs->as_human_readable(); + } + name.erase(std::remove(name.begin(), name.end(), ' '), name.end()); + name += ">"; + return name; +} + +FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, const std::string& inst_name, std::vector&& substitutionTs) { + tolk_assert(fun_ref->genericTs); + + // if `f` was earlier instantiated, return it + if (const auto* existing = lookup_global_symbol(inst_name)) { + FunctionPtr inst_ref = existing->try_as(); + tolk_assert(inst_ref); + return inst_ref; + } + + std::vector parameters; + parameters.reserve(fun_ref->get_num_params()); + for (const LocalVarData& orig_p : fun_ref->parameters) { + parameters.emplace_back(orig_p.name, orig_p.loc, replace_genericT_with_deduced(orig_p.declared_type, fun_ref->genericTs, substitutionTs), orig_p.flags, orig_p.param_idx); + } + TypePtr declared_return_type = replace_genericT_with_deduced(fun_ref->declared_return_type, fun_ref->genericTs, substitutionTs); + const GenericsInstantiation* instantiationTs = new GenericsInstantiation(loc, std::move(substitutionTs)); + + if (fun_ref->is_asm_function()) { + FunctionData* inst_ref = new FunctionData(inst_name, fun_ref->loc, declared_return_type, std::move(parameters), fun_ref->flags, nullptr, instantiationTs, new FunctionBodyAsm, fun_ref->ast_root); + inst_ref->arg_order = fun_ref->arg_order; + inst_ref->ret_order = fun_ref->ret_order; + G.symtable.add_function(inst_ref); + G.all_functions.push_back(inst_ref); + run_pipeline_for_instantiated_function(inst_ref); + return inst_ref; + } + + if (fun_ref->is_builtin_function()) { + FunctionData* inst_ref = new FunctionData(inst_name, fun_ref->loc, declared_return_type, std::move(parameters), fun_ref->flags, nullptr, instantiationTs, fun_ref->body, fun_ref->ast_root); + inst_ref->arg_order = fun_ref->arg_order; + inst_ref->ret_order = fun_ref->ret_order; + G.symtable.add_function(inst_ref); + return inst_ref; + } + + GenericFunctionReplicator replicator(fun_ref->genericTs, instantiationTs->substitutions); + V inst_root = replicator.clone_function_body(fun_ref->ast_root->as()); + + FunctionData* inst_ref = new FunctionData(inst_name, fun_ref->loc, declared_return_type, std::move(parameters), fun_ref->flags, nullptr, instantiationTs, new FunctionBodyCode, inst_root); + inst_ref->arg_order = fun_ref->arg_order; + inst_ref->ret_order = fun_ref->ret_order; + inst_root->mutate()->assign_fun_ref(inst_ref); + G.symtable.add_function(inst_ref); + G.all_functions.push_back(inst_ref); + run_pipeline_for_instantiated_function(inst_ref); + return inst_ref; +} + +} // namespace tolk diff --git a/tolk/generics-helpers.h b/tolk/generics-helpers.h new file mode 100644 index 00000000..5ed245af --- /dev/null +++ b/tolk/generics-helpers.h @@ -0,0 +1,102 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "src-file.h" +#include "fwd-declarations.h" +#include "td/utils/Status.h" +#include + +namespace tolk { + +// when a function is declared `f`, this "" is represented as this class +// (not at AST, but at symbol storage level) +struct GenericsDeclaration { + struct GenericsItem { + std::string_view nameT; + + explicit GenericsItem(std::string_view nameT) + : nameT(nameT) {} + }; + + explicit GenericsDeclaration(std::vector&& itemsT) + : itemsT(std::move(itemsT)) {} + + const std::vector itemsT; + + std::string as_human_readable() const; + + size_t size() const { return itemsT.size(); } + bool has_nameT(std::string_view nameT) const { return find_nameT(nameT) != -1; } + int find_nameT(std::string_view nameT) const; + std::string get_nameT(int idx) const { return static_cast(itemsT[idx].nameT); } +}; + +// when a function call is `f()`, this "" is represented as this class +struct GenericsInstantiation { + const std::vector substitutions; // for genericTs + const SrcLocation loc; // first instantiation location + + explicit GenericsInstantiation(SrcLocation loc, std::vector&& substitutions) + : substitutions(std::move(substitutions)) + , loc(loc) { + } +}; + +// this class helps to deduce Ts on the fly +// purpose: having `f(value: T)` and call `f(5)`, deduce T = int +// while analyzing a call, arguments are handled one by one, by `auto_deduce_from_argument()` +// this class also handles manually specified substitutions like `f(5)` +class GenericSubstitutionsDeduceForCall { + FunctionPtr fun_ref; + std::vector substitutionTs; + bool manually_specified = false; + + void provide_deducedT(const std::string& nameT, TypePtr deduced); + void consider_next_condition(TypePtr param_type, TypePtr arg_type); + +public: + explicit GenericSubstitutionsDeduceForCall(FunctionPtr fun_ref); + + bool is_manually_specified() const { + return manually_specified; + } + + void provide_manually_specified(std::vector&& substitutionTs); + TypePtr replace_by_manually_specified(TypePtr param_type) const; + TypePtr auto_deduce_from_argument(FunctionPtr cur_f, SrcLocation loc, TypePtr param_type, TypePtr arg_type); + int get_first_not_deduced_idx() const; + + std::vector&& flush() { + return std::move(substitutionTs); + } +}; + +struct GenericDeduceError final : std::exception { + std::string message; + explicit GenericDeduceError(std::string message) + : message(std::move(message)) { } + + const char* what() const noexcept override { + return message.c_str(); + } +}; + +std::string generate_instantiated_name(const std::string& orig_name, const std::vector& substitutions); +FunctionPtr instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, const std::string& inst_name, std::vector&& substitutionTs); + +} // namespace tolk diff --git a/tolk/lexer.cpp b/tolk/lexer.cpp new file mode 100644 index 00000000..06913a5f --- /dev/null +++ b/tolk/lexer.cpp @@ -0,0 +1,616 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "lexer.h" +#include +#include +#include + +namespace tolk { + +// By 'chunk' in lexer I mean a token or a list of tokens parsed simultaneously. +// E.g., when we meet "str", ChunkString is called, it emits tok_string. +// E.g., when we meet "str"x, ChunkString emits not only tok_string, but tok_string_modifier. +// E.g., when we meet //, ChunkInlineComment is called, it emits nothing (just skips a line). +// We store all valid chunks lexers in a prefix tree (LexingTrie), see below. +struct ChunkLexerBase { + ChunkLexerBase(const ChunkLexerBase&) = delete; + ChunkLexerBase &operator=(const ChunkLexerBase&) = delete; + ChunkLexerBase() = default; + + virtual bool parse(Lexer* lex) const = 0; + virtual ~ChunkLexerBase() = default; +}; + +template +static T* singleton() { + static T obj; + return &obj; +} + +// LexingTrie is a prefix tree storing all available Tolk language constructs. +// It's effectively a map of a prefix to ChunkLexerBase. +class LexingTrie { + LexingTrie** next{nullptr}; // either nullptr or [256] + ChunkLexerBase* val{nullptr}; // non-null for leafs + + GNU_ATTRIBUTE_ALWAYS_INLINE void ensure_next_allocated() { + if (next == nullptr) { + next = new LexingTrie*[256]; + std::memset(next, 0, 256 * sizeof(LexingTrie*)); + } + } + + GNU_ATTRIBUTE_ALWAYS_INLINE void ensure_symbol_allocated(uint8_t symbol) const { + if (next[symbol] == nullptr) { + next[symbol] = new LexingTrie; + } + } + +public: + // Maps a prefix onto a chunk lexer. + // E.g. " -> ChunkString + // E.g. """ -> ChunkMultilineString + void add_prefix(const char* s, ChunkLexerBase* val) { + LexingTrie* cur = this; + + for (; *s; ++s) { + uint8_t symbol = static_cast(*s); + cur->ensure_next_allocated(); + cur->ensure_symbol_allocated(symbol); + cur = cur->next[symbol]; + } + +#ifdef TOLK_DEBUG + assert(!cur->val); +#endif + cur->val = val; + } + + // Maps a pattern onto a chunk lexer. + // E.g. -[0-9] -> ChunkNegativeNumber + // Internally, it expands the pattern to all possible prefixes: -0, -1, etc. + // (for example, [0-9][a-z_$] gives 10*28=280 prefixes) + void add_pattern(const char* pattern, ChunkLexerBase* val) { + std::vector all_possible_trie{this}; + + for (const char* c = pattern; *c; ++c) { + std::string to_append; + if (*c == '[') { + c++; + while (*c != ']') { // assume that input is corrent, no out-of-string checks + if (*(c + 1) == '-') { + char l = *c, r = *(c + 2); + for (char symbol = l; symbol <= r; ++symbol) { + to_append += symbol; + } + c += 3; + } else { + to_append += *c; + c++; + } + } + } else { + to_append += *c; + } + + std::vector next_all_possible_trie; + next_all_possible_trie.reserve(all_possible_trie.size() * to_append.size()); + for (LexingTrie* cur : all_possible_trie) { + cur->ensure_next_allocated(); + for (uint8_t symbol : to_append) { + cur->ensure_symbol_allocated(symbol); + next_all_possible_trie.emplace_back(cur->next[symbol]); + } + } + all_possible_trie = std::move(next_all_possible_trie); + } + + for (LexingTrie* trie : all_possible_trie) { + trie->val = val; + } + } + + // Looks up a chunk lexer given a string (in practice, s points to cur position in the middle of the file). + // It returns the deepest case: pointing to ", it will return ChunkMultilineString if """, or ChunkString otherwize. + ChunkLexerBase* get_deepest(const char* s) const { + const LexingTrie* best = this; + + for (const LexingTrie* cur = this; cur && cur->next; ++s) { + cur = cur->next[static_cast(*s)]; // if s reaches \0, cur will just become nullptr, and loop will end + if (cur && cur->val) { + best = cur; + } + } + + return best->val; + } +}; + +// +// ---------------------------------------------------------------------- +// A list of valid parsed chunks. +// + +// An inline comment, starting from '//' +struct ChunkInlineComment final : ChunkLexerBase { + bool parse(Lexer* lex) const override { + lex->skip_line(); + return true; + } +}; + +// A multiline comment, starting from '/*' +// Note, that nested comments are not supported. +struct ChunkMultilineComment final : ChunkLexerBase { + bool parse(Lexer* lex) const override { + while (!lex->is_eof()) { + if (lex->char_at() == '*' && lex->char_at(1) == '/') { + lex->skip_chars(2); + return true; + } + lex->skip_chars(1); + } + return true; // it's okay if comment extends past end of file + } +}; + +// A string, starting from " +// Note, that there are no escape symbols inside: the purpose of strings in Tolk just doesn't need it. +// After a closing quote, a string modifier may be present, like "Ef8zMzMzMzMzMzMzMzMzMzM0vF"a. +// If present, it emits a separate tok_string_modifier. +struct ChunkString final : ChunkLexerBase { + bool parse(Lexer* lex) const override { + const char* str_begin = lex->c_str(); + lex->skip_chars(1); + while (!lex->is_eof() && lex->char_at() != '"' && lex->char_at() != '\n') { + lex->skip_chars(1); + } + if (lex->char_at() != '"') { + lex->error("string extends past end of line"); + } + + std::string_view str_val(str_begin + 1, lex->c_str() - str_begin - 1); + lex->skip_chars(1); + lex->add_token(tok_string_const, str_val); + + if (std::isalpha(lex->char_at())) { + std::string_view modifier_val(lex->c_str(), 1); + lex->skip_chars(1); + lex->add_token(tok_string_modifier, modifier_val); + } + + return true; + } +}; + +// A string starting from """ +// Used for multiline asm constructions. Can not have a postfix modifier. +struct ChunkMultilineString final : ChunkLexerBase { + bool parse(Lexer* lex) const override { + const char* str_begin = lex->c_str(); + lex->skip_chars(3); + while (!lex->is_eof()) { + if (lex->char_at() == '"' && lex->char_at(1) == '"' && lex->char_at(2) == '"') { + break; + } + lex->skip_chars(1); + } + if (lex->is_eof()) { + lex->error("string extends past end of file"); + } + + std::string_view str_val(str_begin + 3, lex->c_str() - str_begin - 3); + lex->skip_chars(3); + lex->add_token(tok_string_const, str_val); + return true; + } +}; + +// An annotation for a function (in the future, for vars also): +// @inline and others +struct ChunkAnnotation final : ChunkLexerBase { + bool parse(Lexer* lex) const override { + const char* str_begin = lex->c_str(); + lex->skip_chars(1); + while (std::isalnum(lex->char_at()) || lex->char_at() == '_') { + lex->skip_chars(1); + } + + std::string_view str_val(str_begin, lex->c_str() - str_begin); + lex->add_token(tok_annotation_at, str_val); + return true; + } +}; + +// A number, may be a hex one. +struct ChunkNumber final : ChunkLexerBase { + bool parse(Lexer* lex) const override { + const char* str_begin = lex->c_str(); + bool hex = false; + if (lex->char_at() == '0' && lex->char_at(1) == 'x') { + lex->skip_chars(2); + hex = true; + } + if (lex->is_eof()) { + return false; + } + while (!lex->is_eof()) { + char c = lex->char_at(); + if (c >= '0' && c <= '9') { + lex->skip_chars(1); + continue; + } + if (!hex) { + break; + } + c |= 0x20; + if (c < 'a' || c > 'f') { + break; + } + lex->skip_chars(1); + } + + std::string_view str_val(str_begin, lex->c_str() - str_begin); + lex->add_token(tok_int_const, str_val); + return true; + } +}; + +// Tokens like !=, &, etc. emit just a simple TokenType. +// Since they are stored in trie, "parsing" them is just skipping len chars. +struct ChunkSimpleToken final : ChunkLexerBase { + TokenType tp; + int len; + + ChunkSimpleToken(TokenType tp, int len) : tp(tp), len(len) {} + + bool parse(Lexer* lex) const override { + std::string_view str_val(lex->c_str(), len); + lex->add_token(tp, str_val); + lex->skip_chars(len); + return true; + } +}; + +// Spaces and other space-like symbols are just skipped. +struct ChunkSkipWhitespace final : ChunkLexerBase { + bool parse(Lexer* lex) const override { + lex->skip_chars(1); + lex->skip_spaces(); + return true; + } +}; + +// Here we handle corner cases of grammar that are requested on demand. +// E.g., for 'tolk >0.5.0', '0.5.0' should be parsed specially to emit tok_semver. +// See TolkLanguageGrammar::parse_next_chunk_special(). +struct ChunkSpecialParsing { + static bool parse_semver(Lexer* lex) { + const char* str_begin = lex->c_str(); + while (std::isdigit(lex->char_at()) || lex->char_at() == '.') { + lex->skip_chars(1); + } + + std::string_view str_val(str_begin, lex->c_str() - str_begin); + if (str_val.empty()) { + return false; + } + lex->add_token(tok_semver, str_val); + return true; + } +}; + +// Anything starting from a valid identifier beginning symbol is parsed as an identifier. +// But if a resulting string is a keyword, a corresponding token is emitted instead of tok_identifier. +struct ChunkIdentifierOrKeyword final : ChunkLexerBase { + // having parsed str up to the valid end, look up whether it's a valid keyword + // in the future, this could be a bit more effective than just comparing strings (e.g. gperf), + // but nevertheless, performance of the naive code below is reasonably good + static TokenType maybe_keyword(std::string_view str) { + switch (str.size()) { + case 1: + if (str == "_") return tok_underscore; + break; + case 2: + if (str == "do") return tok_do; + if (str == "if") return tok_if; + if (str == "as") return tok_as; + break; + case 3: + if (str == "var") return tok_var; + if (str == "fun") return tok_fun; + if (str == "asm") return tok_asm; + if (str == "get") return tok_get; + if (str == "try") return tok_try; + if (str == "val") return tok_val; + break; + case 4: + if (str == "else") return tok_else; + if (str == "true") return tok_true; + if (str == "null") return tok_null; + if (str == "self") return tok_self; + if (str == "tolk") return tok_tolk; + if (str == "type") return tok_type; + if (str == "enum") return tok_enum; + break; + case 5: + if (str == "const") return tok_const; + if (str == "false") return tok_false; + if (str == "redef") return tok_redef; + if (str == "while") return tok_while; + if (str == "break") return tok_break; + if (str == "throw") return tok_throw; + if (str == "catch") return tok_catch; + if (str == "infix") return tok_infix; + break; + case 6: + if (str == "return") return tok_return; + if (str == "assert") return tok_assert; + if (str == "import") return tok_import; + if (str == "global") return tok_global; + if (str == "mutate") return tok_mutate; + if (str == "repeat") return tok_repeat; + if (str == "struct") return tok_struct; + if (str == "export") return tok_export; + break; + case 7: + if (str == "builtin") return tok_builtin; + break; + case 8: + if (str == "continue") return tok_continue; + if (str == "operator") return tok_operator; + break; + default: + break; + } + return tok_empty; + } + + bool parse(Lexer* lex) const override { + const char* sym_begin = lex->c_str(); + lex->skip_chars(1); + while (!lex->is_eof()) { + char c = lex->char_at(); + bool allowed_in_identifier = std::isalnum(c) || c == '_' || c == '$'; + if (!allowed_in_identifier) { + break; + } + lex->skip_chars(1); + } + + std::string_view str_val(sym_begin, lex->c_str() - sym_begin); + if (TokenType kw_tok = maybe_keyword(str_val)) { + lex->add_token(kw_tok, str_val); + } else { + lex->add_token(tok_identifier, str_val); + } + return true; + } +}; + +// Like in Kotlin, `backticks` can be used to wrap identifiers (both in declarations/usage, both for vars/functions). +// E.g.: function `do`() { var `with spaces` = 1; } +// This could be useful to use reserved names as identifiers (in a probable codegen from TL, for example). +struct ChunkIdentifierInBackticks final : ChunkLexerBase { + bool parse(Lexer* lex) const override { + const char* str_begin = lex->c_str(); + lex->skip_chars(1); + while (!lex->is_eof() && lex->char_at() != '`' && lex->char_at() != '\n') { + if (std::isspace(lex->char_at())) { + lex->error("an identifier can't have a space in its name (even inside backticks)"); + } + lex->skip_chars(1); + } + if (lex->char_at() != '`') { + lex->error("unclosed backtick `"); + } + + std::string_view str_val(str_begin + 1, lex->c_str() - str_begin - 1); + lex->skip_chars(1); + lex->add_token(tok_identifier, str_val); + return true; + } +}; + +// +// ---------------------------------------------------------------------- +// Here we define a grammar of Tolk. +// All valid chunks prefixes are stored in trie. +// + +struct TolkLanguageGrammar { + static LexingTrie trie; + + static bool parse_next_chunk(Lexer* lex) { + const ChunkLexerBase* best = trie.get_deepest(lex->c_str()); + return best && best->parse(lex); + } + + static bool parse_next_chunk_special(Lexer* lex, TokenType parse_next_as) { + switch (parse_next_as) { + case tok_semver: + return ChunkSpecialParsing::parse_semver(lex); + default: + assert(false); + return false; + } + } + + static void register_token(const char* str, int len, TokenType tp) { + trie.add_prefix(str, new ChunkSimpleToken(tp, len)); + } + + static void init() { + trie.add_prefix("//", singleton()); + trie.add_prefix("/*", singleton()); + trie.add_prefix(R"(")", singleton()); + trie.add_prefix(R"(""")", singleton()); + trie.add_prefix("@", singleton()); + trie.add_prefix(" ", singleton()); + trie.add_prefix("\t", singleton()); + trie.add_prefix("\r", singleton()); + trie.add_prefix("\n", singleton()); + + trie.add_pattern("[0-9]", singleton()); + trie.add_pattern("[a-zA-Z_$]", singleton()); + trie.add_prefix("`", singleton()); + + register_token("+", 1, tok_plus); + register_token("-", 1, tok_minus); + register_token("*", 1, tok_mul); + register_token("/", 1, tok_div); + register_token("%", 1, tok_mod); + register_token("?", 1, tok_question); + register_token(":", 1, tok_colon); + register_token(",", 1, tok_comma); + register_token(";", 1, tok_semicolon); + register_token("(", 1, tok_oppar); + register_token(")", 1, tok_clpar); + register_token("[", 1, tok_opbracket); + register_token("]", 1, tok_clbracket); + register_token("{", 1, tok_opbrace); + register_token("}", 1, tok_clbrace); + register_token("=", 1, tok_assign); + register_token("<", 1, tok_lt); + register_token(">", 1, tok_gt); + register_token("!", 1, tok_logical_not); + register_token("&", 1, tok_bitwise_and); + register_token("|", 1, tok_bitwise_or); + register_token("^", 1, tok_bitwise_xor); + register_token("~", 1, tok_bitwise_not); + register_token(".", 1, tok_dot); + register_token("==", 2, tok_eq); + register_token("!=", 2, tok_neq); + register_token("<=", 2, tok_leq); + register_token(">=", 2, tok_geq); + register_token("<<", 2, tok_lshift); + register_token(">>", 2, tok_rshift); + register_token("&&", 2, tok_logical_and); + register_token("||", 2, tok_logical_or); + register_token("~/", 2, tok_divR); + register_token("^/", 2, tok_divC); + register_token("+=", 2, tok_set_plus); + register_token("-=", 2, tok_set_minus); + register_token("*=", 2, tok_set_mul); + register_token("/=", 2, tok_set_div); + register_token("%=", 2, tok_set_mod); + register_token("&=", 2, tok_set_bitwise_and); + register_token("|=", 2, tok_set_bitwise_or); + register_token("^=", 2, tok_set_bitwise_xor); + register_token("->", 2, tok_arrow); + register_token("<=>", 3, tok_spaceship); + register_token("~>>", 3, tok_rshiftR); + register_token("^>>", 3, tok_rshiftC); + register_token("<<=", 3, tok_set_lshift); + register_token(">>=", 3, tok_set_rshift); + } +}; + +LexingTrie TolkLanguageGrammar::trie; + +// +// ---------------------------------------------------------------------- +// The Lexer class is to be used outside (by parser, which constructs AST from tokens). +// It's streaming. It means, that `next()` parses a next token on demand +// (instead of parsing all file contents to vector and iterating over it). +// Parsing on demand uses effectively less memory. +// Note, that chunks, being parsed, call `add_token()`, and a chunk may add multiple tokens at once. +// That's why a small cirlular buffer for tokens is used. +// `last_token_idx` actually means a number of total tokens added. +// `cur_token_idx` is a number of returned by `next()`. +// It's assumed that an input file has already been loaded, its contents is present and won't be deleted +// (`start`, `cur` and `end`, as well as every Token str_val, points inside file->text). +// + +Lexer::Lexer(const SrcFile* file) + : file(file) + , p_start(file->text.data()) + , p_end(p_start + file->text.size()) + , p_next(p_start) + , location(file) { + next(); +} + +Lexer::Lexer(std::string_view text) + : file(nullptr) + , p_start(text.data()) + , p_end(p_start + text.size()) + , p_next(p_start) + , location() { + next(); +} + +void Lexer::next() { + while (cur_token_idx == last_token_idx && !is_eof()) { + update_location(); + if (!TolkLanguageGrammar::parse_next_chunk(this)) { + error("failed to parse"); + } + } + if (is_eof()) { + add_token(tok_eof, ""); + } + cur_token = tokens_circularbuf[++cur_token_idx & 7]; +} + +void Lexer::next_special(TokenType parse_next_as, const char* str_expected) { + assert(cur_token_idx == last_token_idx); + skip_spaces(); + update_location(); + if (!TolkLanguageGrammar::parse_next_chunk_special(this, parse_next_as)) { + error(std::string(str_expected) + " expected"); + } + cur_token = tokens_circularbuf[++cur_token_idx & 7]; +} + +Lexer::SavedPositionForLookahead Lexer::save_parsing_position() const { + return {p_next, cur_token_idx, cur_token}; +} + +void Lexer::restore_position(SavedPositionForLookahead saved) { + p_next = saved.p_next; + cur_token_idx = last_token_idx = saved.cur_token_idx; + cur_token = saved.cur_token; +} + +void Lexer::error(const std::string& err_msg) const { + throw ParseError(cur_location(), err_msg); +} + +void Lexer::unexpected(const char* str_expected) const { + throw ParseError(cur_location(), "expected " + std::string(str_expected) + ", got `" + std::string(cur_str()) + "`"); +} + +void lexer_init() { + TolkLanguageGrammar::init(); +} + +// todo #ifdef TOLK_PROFILING +// As told above, `next()` produces tokens on demand, while AST is being generated. +// Hence, it's difficult to measure Lexer performance separately. +// This function can be called just to tick Lexer performance, it just scans all input files. +// There is no sense to use it in production, but when refactoring and optimizing Lexer, it's useful. +void lexer_measure_performance(const AllRegisteredSrcFiles& files_to_just_parse) { + for (const SrcFile* file : files_to_just_parse) { + Lexer lex(file); + while (!lex.is_eof()) { + lex.next(); + } + } +} + +} // namespace tolk diff --git a/tolk/lexer.h b/tolk/lexer.h new file mode 100644 index 00000000..58bc3640 --- /dev/null +++ b/tolk/lexer.h @@ -0,0 +1,237 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "platform-utils.h" +#include "src-file.h" +#include + +namespace tolk { + +enum TokenType { + tok_empty, + + tok_fun, + tok_get, + tok_type, + tok_enum, + tok_struct, + tok_operator, + tok_infix, + + tok_global, + tok_const, + tok_var, + tok_val, + tok_redef, + tok_mutate, + tok_self, + + tok_annotation_at, + tok_colon, + tok_asm, + tok_builtin, + + tok_int_const, + tok_string_const, + tok_string_modifier, + tok_true, + tok_false, + tok_null, + + tok_identifier, + tok_dot, + + tok_plus, + tok_set_plus, + tok_minus, + tok_set_minus, + tok_mul, + tok_set_mul, + tok_div, + tok_set_div, + tok_mod, + tok_set_mod, + tok_lshift, + tok_set_lshift, + tok_rshift, + tok_set_rshift, + tok_rshiftR, + tok_rshiftC, + tok_bitwise_and, + tok_set_bitwise_and, + tok_bitwise_or, + tok_set_bitwise_or, + tok_bitwise_xor, + tok_set_bitwise_xor, + tok_bitwise_not, + + tok_question, + tok_comma, + tok_semicolon, + tok_oppar, + tok_clpar, + tok_opbracket, + tok_clbracket, + tok_opbrace, + tok_clbrace, + tok_assign, + tok_underscore, + tok_lt, + tok_gt, + tok_logical_not, + tok_logical_and, + tok_logical_or, + + tok_eq, + tok_neq, + tok_leq, + tok_geq, + tok_spaceship, + tok_divR, + tok_divC, + + tok_return, + tok_repeat, + tok_do, + tok_while, + tok_break, + tok_continue, + tok_try, + tok_catch, + tok_throw, + tok_assert, + tok_if, + tok_else, + + tok_arrow, + tok_as, + + tok_tolk, + tok_semver, + tok_import, + tok_export, + + tok_eof +}; + +// All tolk language is parsed into tokens. +// Lexer::next() returns a Token. +struct Token { + TokenType type = tok_empty; + std::string_view str_val; + + Token() = default; + Token(TokenType type, std::string_view str_val): type(type), str_val(str_val) {} +}; + +// Lexer::next() is a method to be used externally (while parsing tolk file to AST). +// It's streaming: `next()` parses a token on demand. +// For comments, see lexer.cpp, a comment above Lexer constructor. +class Lexer { + Token tokens_circularbuf[8]{}; + int last_token_idx = -1; + int cur_token_idx = -1; + Token cur_token; // = tokens_circularbuf[cur_token_idx & 7] + + const SrcFile* file; + const char *p_start, *p_end, *p_next; + SrcLocation location; + + void update_location() { + location.char_offset = static_cast(p_next - p_start); + } + +public: + + struct SavedPositionForLookahead { + const char* p_next = nullptr; + int cur_token_idx = 0; + Token cur_token; + }; + + explicit Lexer(const SrcFile* file); + explicit Lexer(std::string_view text); + Lexer(const Lexer&) = delete; + Lexer &operator=(const Lexer&) = delete; + + void add_token(TokenType type, std::string_view str) { + tokens_circularbuf[++last_token_idx & 7] = Token(type, str); + } + + void skip_spaces() { + while (std::isspace(*p_next)) { + ++p_next; + } + } + + void skip_line() { + while (p_next < p_end && *p_next != '\n' && *p_next != '\r') { + ++p_next; + } + while (*p_next == '\n' || *p_next == '\r') { + ++p_next; + } + } + + void skip_chars(int n) { + p_next += n; + } + + bool is_eof() const { + return p_next >= p_end; + } + + char char_at() const { return *p_next; } + char char_at(int shift) const { return *(p_next + shift); } + const char* c_str() const { return p_next; } + + TokenType tok() const { return cur_token.type; } + std::string_view cur_str() const { return cur_token.str_val; } + SrcLocation cur_location() const { return location; } + const SrcFile* cur_file() const { return file; } + + void next(); + void next_special(TokenType parse_next_as, const char* str_expected); + + SavedPositionForLookahead save_parsing_position() const; + void restore_position(SavedPositionForLookahead saved); + + void check(TokenType next_tok, const char* str_expected) const { + if (cur_token.type != next_tok) { + unexpected(str_expected); // unlikely path, not inlined + } + } + void expect(TokenType next_tok, const char* str_expected) { + if (cur_token.type != next_tok) { + unexpected(str_expected); + } + next(); + } + + GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD + void unexpected(const char* str_expected) const; + GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD + void error(const std::string& err_msg) const; +}; + +void lexer_init(); + +// todo #ifdef TOLK_PROFILING +void lexer_measure_performance(const AllRegisteredSrcFiles& files_to_just_parse); + +} // namespace tolk diff --git a/tolk/optimize.cpp b/tolk/optimize.cpp new file mode 100644 index 00000000..76d75638 --- /dev/null +++ b/tolk/optimize.cpp @@ -0,0 +1,652 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" + +namespace tolk { + +/* + * + * PEEPHOLE OPTIMIZER + * + */ + +void Optimizer::set_code(AsmOpConsList code) { + code_ = std::move(code); + unpack(); +} + +void Optimizer::unpack() { + int i = 0, j = 0; + for (AsmOpCons *p = code_.get(); p && i < optimize_depth; p = p->cdr.get(), ++j) { + if (p->car->is_very_custom()) { + break; + } + if (p->car->is_comment()) { + continue; + } + op_cons_[i] = p; + op_[i] = std::move(p->car); + offs_[i] = j; + ++i; + } + l_ = i; + indent_ = (i ? op_[0]->indent : 0); +} + +void Optimizer::pack() { + for (int i = 0; i < l_; i++) { + op_cons_[i]->car = std::move(op_[i]); + op_cons_[i] = nullptr; + } + l_ = 0; +} + +void Optimizer::apply() { + if (!p_ && !q_) { + return; + } + tolk_assert(p_ > 0 && p_ <= l_ && q_ >= 0 && q_ <= optimize_depth && l_ <= optimize_depth); + for (int i = p_; i < l_; i++) { + tolk_assert(op_[i]); + op_cons_[i]->car = std::move(op_[i]); + op_cons_[i] = nullptr; + } + for (int c = offs_[p_ - 1]; c >= 0; --c) { + code_ = std::move(code_->cdr); + } + for (int j = q_ - 1; j >= 0; j--) { + tolk_assert(oq_[j]); + oq_[j]->indent = indent_; + code_ = AsmOpCons::cons(std::move(oq_[j]), std::move(code_)); + } + l_ = 0; +} + +AsmOpConsList Optimizer::extract_code() { + pack(); + return std::move(code_); +} + +void Optimizer::show_head() const { + if (!debug_) { + return; + } + std::cerr << "optimizing"; + for (int i = 0; i < l_; i++) { + if (op_[i]) { + std::cerr << ' ' << *op_[i] << ' '; + } else { + std::cerr << " (null) "; + } + } + std::cerr << std::endl; +} + +void Optimizer::show_left() const { + if (!debug_) { + return; + } + std::cerr << "// *** rewriting"; + for (int i = 0; i < p_; i++) { + if (op_[i]) { + std::cerr << ' ' << *op_[i] << ' '; + } else { + std::cerr << " (null) "; + } + } +} + +void Optimizer::show_right() const { + if (!debug_) { + return; + } + std::cerr << "->"; + for (int i = 0; i < q_; i++) { + if (oq_[i]) { + std::cerr << ' ' << *oq_[i] << ' '; + } else { + std::cerr << " (null) "; + } + } + std::cerr << std::endl; +} + +bool Optimizer::say(std::string str) const { + if (debug_) { + std::cerr << str << std::endl; + } + return true; +} + +bool Optimizer::find_const_op(int* op_idx, int cst) { + for (int i = 0; i < l2_; i++) { + if (op_[i]->is_gconst() && tr_[i].get(0) == cst) { + *op_idx = i; + return true; + } + } + return false; +} + +bool Optimizer::is_push_const(int* i, int* c) const { + return pb_ >= 3 && pb_ <= l2_ && tr_[pb_ - 1].is_push_const(i, c); +} + +// PUSHCONST c ; PUSH s(i+1) ; SWAP -> PUSH s(i) ; PUSHCONST c +bool Optimizer::rewrite_push_const(int i, int c) { + p_ = pb_; + q_ = 2; + int idx = -1; + if (!(p_ >= 2 && find_const_op(&idx, c) && idx < p_)) { + return false; + } + show_left(); + oq_[1] = std::move(op_[idx]); + oq_[0] = std::move(op_[!idx]); + *oq_[0] = AsmOp::Push(i); + show_right(); + return true; +} + +bool Optimizer::is_const_rot(int* c) const { + return pb_ >= 3 && pb_ <= l2_ && tr_[pb_ - 1].is_const_rot(c); +} + +bool Optimizer::rewrite_const_rot(int c) { + p_ = pb_; + q_ = 2; + int idx = -1; + if (!(p_ >= 2 && find_const_op(&idx, c) && idx < p_)) { + return false; + } + show_left(); + oq_[0] = std::move(op_[idx]); + oq_[1] = std::move(op_[!idx]); + *oq_[1] = AsmOp::Custom("ROT", 3, 3); + show_right(); + return true; +} + +bool Optimizer::is_const_pop(int* c, int* i) const { + return pb_ >= 3 && pb_ <= l2_ && tr_[pb_ - 1].is_const_pop(c, i); +} + +bool Optimizer::rewrite_const_pop(int c, int i) { + p_ = pb_; + q_ = 2; + int idx = -1; + if (!(p_ >= 2 && find_const_op(&idx, c) && idx < p_)) { + return false; + } + show_left(); + oq_[0] = std::move(op_[idx]); + oq_[1] = std::move(op_[!idx]); + *oq_[1] = AsmOp::Pop(i); + show_right(); + return true; +} + +bool Optimizer::is_const_push_xchgs() { + if (!(pb_ >= 2 && pb_ <= l2_ && op_[0]->is_gconst())) { + return false; + } + StackTransform t; + int pos = 0, i; + for (i = 1; i < pb_; i++) { + int a, b; + if (op_[i]->is_xchg(&a, &b)) { + if (pos == a) { + pos = b; + } else if (pos == b) { + pos = a; + } else { + t.apply_xchg(a - (a > pos), b - (b > pos)); + } + } else if (op_[i]->is_push(&a)) { + if (pos == a) { + return false; + } + t.apply_push(a - (a > pos)); + ++pos; + } else { + return false; + } + } + if (pos) { + return false; + } + t.apply_push_newconst(); + if (t <= tr_[i - 1]) { + p_ = i; + return true; + } else { + return false; + } +} + +bool Optimizer::rewrite_const_push_xchgs() { + if (!p_) { + return false; + } + show_left(); + auto c_op = std::move(op_[0]); + tolk_assert(c_op->is_gconst()); + StackTransform t; + q_ = 0; + int pos = 0; + for (int i = 1; i < p_; i++) { + int a, b; + if (op_[i]->is_xchg(&a, &b)) { + if (a == pos) { + pos = b; + } else if (b == pos) { + pos = a; + } else { + oq_[q_] = std::move(op_[i]); + if (a > pos) { + oq_[q_]->a = a - 1; + } + if (b > pos) { + oq_[q_]->b = b - 1; + } + tolk_assert(apply_op(t, *oq_[q_])); + ++q_; + } + } else { + tolk_assert(op_[i]->is_push(&a)); + tolk_assert(a != pos); + oq_[q_] = std::move(op_[i]); + if (a > pos) { + oq_[q_]->a = a - 1; + } + tolk_assert(apply_op(t, *oq_[q_])); + ++q_; + ++pos; + } + } + tolk_assert(!pos); + t.apply_push_newconst(); + tolk_assert(t <= tr_[p_ - 1]); + oq_[q_++] = std::move(c_op); + show_right(); + return true; +} + +bool Optimizer::rewrite(int p, AsmOp&& new_op) { + tolk_assert(p > 0 && p <= l_); + p_ = p; + q_ = 1; + show_left(); + oq_[0] = std::move(op_[0]); + *oq_[0] = new_op; + show_right(); + return true; +} + +bool Optimizer::rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2) { + tolk_assert(p > 1 && p <= l_); + p_ = p; + q_ = 2; + show_left(); + oq_[0] = std::move(op_[0]); + *oq_[0] = new_op1; + oq_[1] = std::move(op_[1]); + *oq_[1] = new_op2; + show_right(); + return true; +} + +bool Optimizer::rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2, AsmOp&& new_op3) { + tolk_assert(p > 2 && p <= l_); + p_ = p; + q_ = 3; + show_left(); + oq_[0] = std::move(op_[0]); + *oq_[0] = new_op1; + oq_[1] = std::move(op_[1]); + *oq_[1] = new_op2; + oq_[2] = std::move(op_[2]); + *oq_[2] = new_op3; + show_right(); + return true; +} + +bool Optimizer::rewrite_nop() { + tolk_assert(p_ > 0 && p_ <= l_); + q_ = 0; + show_left(); + show_right(); + return true; +} + +bool Optimizer::is_pred(const std::function& pred, int min_p) { + min_p = std::max(min_p, pb_); + for (int p = l2_; p >= min_p; p--) { + if (pred(tr_[p - 1])) { + p_ = p; + return true; + } + } + return false; +} + +bool Optimizer::is_same_as(const StackTransform& trans, int min_p) { + return is_pred([&trans](const auto& t) { return t >= trans; }, min_p); +} + +// s1 s3 XCHG ; s0 s2 XCHG -> 2SWAP +bool Optimizer::is_2swap() { + static const StackTransform t_2swap{2, 3, 0, 1, 4}; + return is_same_as(t_2swap); +} + +// s3 PUSH ; s3 PUSH -> 2OVER +bool Optimizer::is_2over() { + static const StackTransform t_2over{2, 3, 0}; + return is_same_as(t_2over); +} + +bool Optimizer::is_2dup() { + static const StackTransform t_2dup{0, 1, 0}; + return is_same_as(t_2dup); +} + +bool Optimizer::is_tuck() { + static const StackTransform t_tuck{0, 1, 0, 2}; + return is_same_as(t_tuck); +} + +bool Optimizer::is_2drop() { + static const StackTransform t_2drop{2}; + return is_same_as(t_2drop); +} + +bool Optimizer::is_rot() { + return is_pred([](const auto& t) { return t.is_rot(); }); +} + +bool Optimizer::is_rotrev() { + return is_pred([](const auto& t) { return t.is_rotrev(); }); +} + +bool Optimizer::is_nop() { + return is_pred([](const auto& t) { return t.is_id(); }, 1); +} + +bool Optimizer::is_xchg(int* i, int* j) { + return is_pred([i, j](const auto& t) { return t.is_xchg(i, j) && ((*i < 16 && *j < 16) || (!*i && *j < 256)); }); +} + +bool Optimizer::is_xchg_xchg(int* i, int* j, int* k, int* l) { + return is_pred([i, j, k, l](const auto& t) { + return t.is_xchg_xchg(i, j, k, l) && (*i < 2 && *j < (*i ? 16 : 256) && *k < 2 && *l < (*k ? 16 : 256)); + }) && + (!(p_ == 2 && op_[0]->is_xchg(*i, *j) && op_[1]->is_xchg(*k, *l))); +} + +bool Optimizer::is_push(int* i) { + return is_pred([i](const auto& t) { return t.is_push(i) && *i < 256; }); +} + +bool Optimizer::is_pop(int* i) { + return is_pred([i](const auto& t) { return t.is_pop(i) && *i < 256; }); +} + +bool Optimizer::is_pop_pop(int* i, int* j) { + return is_pred([i, j](const auto& t) { return t.is_pop_pop(i, j) && *i < 256 && *j < 256; }, 3); +} + +bool Optimizer::is_push_rot(int* i) { + return is_pred([i](const auto& t) { return t.is_push_rot(i) && *i < 16; }, 3); +} + +bool Optimizer::is_push_rotrev(int* i) { + return is_pred([i](const auto& t) { return t.is_push_rotrev(i) && *i < 16; }, 3); +} + +bool Optimizer::is_push_xchg(int* i, int* j, int* k) { + return is_pred([i, j, k](const auto& t) { return t.is_push_xchg(i, j, k) && *i < 16 && *j < 16 && *k < 16; }) && + !(p_ == 2 && op_[0]->is_push() && op_[1]->is_xchg()); +} + +bool Optimizer::is_xchg2(int* i, int* j) { + return is_pred([i, j](const auto& t) { return t.is_xchg2(i, j) && *i < 16 && *j < 16; }); +} + +bool Optimizer::is_xcpu(int* i, int* j) { + return is_pred([i, j](const auto& t) { return t.is_xcpu(i, j) && *i < 16 && *j < 16; }); +} + +bool Optimizer::is_puxc(int* i, int* j) { + return is_pred([i, j](const auto& t) { return t.is_puxc(i, j) && *i < 16 && *j < 15; }); +} + +bool Optimizer::is_push2(int* i, int* j) { + return is_pred([i, j](const auto& t) { return t.is_push2(i, j) && *i < 16 && *j < 16; }); +} + +bool Optimizer::is_xchg3(int* i, int* j, int* k) { + return is_pred([i, j, k](const auto& t) { return t.is_xchg3(i, j, k) && *i < 16 && *j < 16 && *k < 16; }); +} + +bool Optimizer::is_xc2pu(int* i, int* j, int* k) { + return is_pred([i, j, k](const auto& t) { return t.is_xc2pu(i, j, k) && *i < 16 && *j < 16 && *k < 16; }); +} + +bool Optimizer::is_xcpuxc(int* i, int* j, int* k) { + return is_pred([i, j, k](const auto& t) { return t.is_xcpuxc(i, j, k) && *i < 16 && *j < 16 && *k < 15; }); +} + +bool Optimizer::is_xcpu2(int* i, int* j, int* k) { + return is_pred([i, j, k](const auto& t) { return t.is_xcpu2(i, j, k) && *i < 16 && *j < 16 && *k < 16; }); +} + +bool Optimizer::is_puxc2(int* i, int* j, int* k) { + return is_pred( + [i, j, k](const auto& t) { return t.is_puxc2(i, j, k) && *i < 16 && *j < 15 && *k < 15 && *j + *k != -1; }); +} + +bool Optimizer::is_puxcpu(int* i, int* j, int* k) { + return is_pred([i, j, k](const auto& t) { return t.is_puxcpu(i, j, k) && *i < 16 && *j < 15 && *k < 15; }); +} + +bool Optimizer::is_pu2xc(int* i, int* j, int* k) { + return is_pred([i, j, k](const auto& t) { return t.is_pu2xc(i, j, k) && *i < 16 && *j < 15 && *k < 14; }); +} + +bool Optimizer::is_push3(int* i, int* j, int* k) { + return is_pred([i, j, k](const auto& t) { return t.is_push3(i, j, k) && *i < 16 && *j < 16 && *k < 16; }); +} + +bool Optimizer::is_blkswap(int* i, int* j) { + return is_pred([i, j](const auto& t) { return t.is_blkswap(i, j) && *i > 0 && *j > 0 && *i <= 16 && *j <= 16; }); +} + +bool Optimizer::is_blkpush(int* i, int* j) { + return is_pred([i, j](const auto& t) { return t.is_blkpush(i, j) && *i > 0 && *i < 16 && *j < 16; }); +} + +bool Optimizer::is_blkdrop(int* i) { + return is_pred([i](const auto& t) { return t.is_blkdrop(i) && *i > 0 && *i < 16; }); +} + +bool Optimizer::is_blkdrop2(int* i, int* j) { + return is_pred([i, j](const auto& t) { return t.is_blkdrop2(i, j) && *i > 0 && *i < 16 && *j > 0 && *j < 16; }); +} + +bool Optimizer::is_reverse(int* i, int* j) { + return is_pred([i, j](const auto& t) { return t.is_reverse(i, j) && *i >= 2 && *i <= 17 && *j < 16; }); +} + +bool Optimizer::is_nip_seq(int* i, int* j) { + return is_pred([i, j](const auto& t) { return t.is_nip_seq(i, j) && *i >= 3 && *i <= 15; }); +} + +bool Optimizer::is_pop_blkdrop(int* i, int* k) { + return is_pred([i, k](const auto& t) { return t.is_pop_blkdrop(i, k) && *i >= *k && *k >= 2 && *k <= 15; }, 3); +} + +bool Optimizer::is_2pop_blkdrop(int* i, int* j, int* k) { + return is_pred( + [i, j, k](const auto& t) { return t.is_2pop_blkdrop(i, j, k) && *i >= *k && *j >= *k && *k >= 2 && *k <= 15; }, + 3); +} + +bool Optimizer::compute_stack_transforms() { + StackTransform trans; + for (int i = 0; i < l_; i++) { + if (!apply_op(trans, *op_[i])) { + l2_ = i; + return true; + } + tr_[i] = trans; + } + l2_ = l_; + return true; +} + +bool Optimizer::show_stack_transforms() const { + show_head(); + // slow version + /* + StackTransform trans2; + std::cerr << "id = " << trans2 << std::endl; + for (int i = 0; i < l_; i++) { + StackTransform op; + if (!apply_op(op, *op_[i])) { + std::cerr << "* (" << *op_[i] << " = invalid)\n"; + break; + } + trans2 *= op; + std::cerr << "* " << *op_[i] << " = " << op << " -> " << trans2 << std::endl; + } + */ + // fast version + StackTransform trans; + for (int i = 0; i < l_; i++) { + std::cerr << trans << std::endl << *op_[i] << " -> "; + if (!apply_op(trans, *op_[i])) { + std::cerr << " " << std::endl; + return true; + } + } + std::cerr << trans << std::endl; + return true; +} + +bool Optimizer::find_at_least(int pb) { + p_ = q_ = 0; + pb_ = pb; + // show_stack_transforms(); + int i, j, k, l, c; + return (is_push_const(&i, &c) && rewrite_push_const(i, c)) || (is_nop() && rewrite_nop()) || + (!(mode_ & 1) && is_const_rot(&c) && rewrite_const_rot(c)) || + (is_const_push_xchgs() && rewrite_const_push_xchgs()) || (is_const_pop(&c, &i) && rewrite_const_pop(c, i)) || + (is_xchg(&i, &j) && rewrite(AsmOp::Xchg(i, j))) || (is_push(&i) && rewrite(AsmOp::Push(i))) || + (is_pop(&i) && rewrite(AsmOp::Pop(i))) || (is_pop_pop(&i, &j) && rewrite(AsmOp::Pop(i), AsmOp::Pop(j))) || + (is_xchg_xchg(&i, &j, &k, &l) && rewrite(AsmOp::Xchg(i, j), AsmOp::Xchg(k, l))) || + (!(mode_ & 1) && + ((is_rot() && rewrite(AsmOp::Custom("ROT", 3, 3))) || (is_rotrev() && rewrite(AsmOp::Custom("-ROT", 3, 3))) || + (is_2dup() && rewrite(AsmOp::Custom("2DUP", 2, 4))) || + (is_2swap() && rewrite(AsmOp::Custom("2SWAP", 2, 4))) || + (is_2over() && rewrite(AsmOp::Custom("2OVER", 2, 4))) || + (is_tuck() && rewrite(AsmOp::Custom("TUCK", 2, 3))) || + (is_2drop() && rewrite(AsmOp::Custom("2DROP", 2, 0))) || (is_xchg2(&i, &j) && rewrite(AsmOp::Xchg2(i, j))) || + (is_xcpu(&i, &j) && rewrite(AsmOp::XcPu(i, j))) || (is_puxc(&i, &j) && rewrite(AsmOp::PuXc(i, j))) || + (is_push2(&i, &j) && rewrite(AsmOp::Push2(i, j))) || (is_blkswap(&i, &j) && rewrite(AsmOp::BlkSwap(i, j))) || + (is_blkpush(&i, &j) && rewrite(AsmOp::BlkPush(i, j))) || (is_blkdrop(&i) && rewrite(AsmOp::BlkDrop(i))) || + (is_push_rot(&i) && rewrite(AsmOp::Push(i), AsmOp::Custom("ROT"))) || + (is_push_rotrev(&i) && rewrite(AsmOp::Push(i), AsmOp::Custom("-ROT"))) || + (is_push_xchg(&i, &j, &k) && rewrite(AsmOp::Push(i), AsmOp::Xchg(j, k))) || + (is_reverse(&i, &j) && rewrite(AsmOp::BlkReverse(i, j))) || + (is_blkdrop2(&i, &j) && rewrite(AsmOp::BlkDrop2(i, j))) || + (is_nip_seq(&i, &j) && rewrite(AsmOp::Xchg(i, j), AsmOp::BlkDrop(i))) || + (is_pop_blkdrop(&i, &k) && rewrite(AsmOp::Pop(i), AsmOp::BlkDrop(k))) || + (is_2pop_blkdrop(&i, &j, &k) && (k >= 3 && k <= 13 && i != j + 1 && i <= 15 && j <= 14 + ? rewrite(AsmOp::Xchg2(j + 1, i), AsmOp::BlkDrop(k + 2)) + : rewrite(AsmOp::Pop(i), AsmOp::Pop(j), AsmOp::BlkDrop(k)))) || + (is_xchg3(&i, &j, &k) && rewrite(AsmOp::Xchg3(i, j, k))) || + (is_xc2pu(&i, &j, &k) && rewrite(AsmOp::Xc2Pu(i, j, k))) || + (is_xcpuxc(&i, &j, &k) && rewrite(AsmOp::XcPuXc(i, j, k))) || + (is_xcpu2(&i, &j, &k) && rewrite(AsmOp::XcPu2(i, j, k))) || + (is_puxc2(&i, &j, &k) && rewrite(AsmOp::PuXc2(i, j, k))) || + (is_puxcpu(&i, &j, &k) && rewrite(AsmOp::PuXcPu(i, j, k))) || + (is_pu2xc(&i, &j, &k) && rewrite(AsmOp::Pu2Xc(i, j, k))) || + (is_push3(&i, &j, &k) && rewrite(AsmOp::Push3(i, j, k))))); +} + +bool Optimizer::find() { + if (!compute_stack_transforms()) { + return false; + } + for (int pb = l_; pb > 0; --pb) { + if (find_at_least(pb)) { + return true; + } + } + return false; +} + +bool Optimizer::optimize() { + bool f = false; + while (find()) { + f = true; + apply(); + unpack(); + } + return f; +} + +AsmOpConsList optimize_code_head(AsmOpConsList op_list, int mode) { + Optimizer opt(std::move(op_list), false, mode); + opt.optimize(); + return opt.extract_code(); +} + +AsmOpConsList optimize_code(AsmOpConsList op_list, int mode) { + std::vector> v; + while (op_list) { + if (!op_list->car->is_comment()) { + op_list = optimize_code_head(std::move(op_list), mode); + } + if (op_list) { + v.push_back(std::move(op_list->car)); + op_list = std::move(op_list->cdr); + } + } + for (auto it = v.rbegin(); it < v.rend(); ++it) { + op_list = AsmOpCons::cons(std::move(*it), std::move(op_list)); + } + return std::move(op_list); +} + +void optimize_code(AsmOpList& ops) { + AsmOpConsList op_list; + for (auto it = ops.list_.rbegin(); it < ops.list_.rend(); ++it) { + op_list = AsmOpCons::cons(std::make_unique(std::move(*it)), std::move(op_list)); + } + for (int mode : {1, 1, 1, 1, 0, 0, 0, 0}) { + op_list = optimize_code(std::move(op_list), mode); + } + ops.list_.clear(); + while (op_list) { + ops.list_.push_back(std::move(*(op_list->car))); + op_list = std::move(op_list->cdr); + } +} + +} // namespace tolk diff --git a/tolk/pipe-ast-to-legacy.cpp b/tolk/pipe-ast-to-legacy.cpp new file mode 100644 index 00000000..1561aa40 --- /dev/null +++ b/tolk/pipe-ast-to-legacy.cpp @@ -0,0 +1,1437 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" +#include "src-file.h" +#include "ast.h" +#include "ast-visitor.h" +#include "type-system.h" +#include "common/refint.h" +#include "constant-evaluator.h" +#include + +/* + * This pipe is the last one operating AST: it transforms AST to IR. + * IR is described as "Op" struct. So, here AST is transformed to Ops, and then all the rest "legacy" + * kernel (initially forked from FunC) comes into play. + * Up to this point, all types have been inferred, all validity checks have been passed, etc. + * All properties in AST nodes are assigned and can be safely used (fun_ref, etc.). + * So, if execution reaches this pass, the input is (almost) correct, and code generation should succeed. + * (previously, there was a check for one variable modified twice like `(t.0, t.0) = rhs`, but after changing + * execution order of assignment to "first lhs, then lhs", it was removed for several reasons) +* + * A noticeable property for IR generation is "target_type" used to extend/shrink stack. + * Example: `var a: (int,int)? = null`. This `null` has inferred_type "null literal", but target_type "nullable tensor", + * and when it's assigned, it's "expanded" from 1 stack slot to 3 (int + int + null flag). + * Example: `fun analyze(t: (int,int)?)` and a call `analyze((1,2))`. `(1,2)` is `(int,int)` (2 stack slots), + * and when passed to target (3 slots, one for null flag), this null flag is implicitly added (zero value). + * Example: `nullableInt!`; for `nullableInt` inferred_type is `int?`, and target_type is `int` + * (this doesn't lead to stack reorganization, but in case `nullableTensor!` does) + * (inferred_type of `nullableInt!` is `int`, and its target_type depends on its usage). + * The same mechanism will work for union types in the future. + */ + +namespace tolk { + +class LValContext; +std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr target_type = nullptr, LValContext* lval_ctx = nullptr); +std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, CodeBlob& code, LValContext* lval_ctx); +void process_any_statement(AnyV v, CodeBlob& code); + + +// The goal of VarsModificationWatcher is to detect such cases: `return (x, x += y, x)`. +// Without any changes, ops will be { _Call $2 = +($0_x, $1_y); _Return $0_x, $2, $0_x } - incorrect +// Correct will be to introduce tmp var: { _Let $3 = $0_x; _Call $2 = ...; _Return $3, $2, $0_x } +// This "introducing" is done when compiling tensors, whereas this class allows to watch vars for modification. +class VarsModificationWatcher { + struct WatchedVar { + var_idx_t ir_idx; + std::function on_modification_callback; + + WatchedVar(var_idx_t ir_idx, std::function on_modification_callback) + : ir_idx(ir_idx), on_modification_callback(std::move(on_modification_callback)) {} + }; + + std::vector all_callbacks; + +public: + + bool empty() const { return all_callbacks.empty(); } + + void push_callback(var_idx_t ir_idx, std::function callback) { + all_callbacks.emplace_back(ir_idx, std::move(callback)); + } + + void pop_callback(var_idx_t ir_idx) { + for (auto it = all_callbacks.rbegin(); it != all_callbacks.rend(); ++it) { + if (it->ir_idx == ir_idx) { + all_callbacks.erase((it + 1).base()); + return; + } + } + tolk_assert(false); + } + + void trigger_callbacks(const std::vector& left_lval_indices, SrcLocation loc) const { + for (const WatchedVar& w : all_callbacks) { + for (var_idx_t changed_var : left_lval_indices) { + if (w.ir_idx == changed_var) { + w.on_modification_callback(loc, w.ir_idx); + } + } + } + } +}; + +static VarsModificationWatcher vars_modification_watcher; + + +// Main goal of LValContext is to handle non-primitive lvalues. At IR level, a usual local variable +// exists, but on its change, something non-trivial should happen. +// Example: `globalVar = 9` actually does `Const $5 = 9` + `Let $6 = $5` + `SetGlob "globVar" = $6` +// Example: `tupleVar.0 = 9` actually does `Const $5 = 9` + `Let $6 = $5` + `Const $7 = 0` + `Call tupleSetAt($4, $6, $7)` +// Of course, mixing globals with tuples should also be supported. +// To achieve this, treat tupleObj inside "tupleObj.i" like "rvalue inside lvalue". +// For instance, `globalTuple.0 = 9` reads global (like rvalue), assigns 9 to tmp var, modifies tuple, writes global. +// Note, that tensors (not tuples) `tensorVar.0 = 9` do not emit anything special (unless global). +class LValContext { + // every global variable used as lvalue is registered here + // example: `globalInt = 9`, implicit var is created `$tmp = 9`, and `SetGlob "globalInt" $tmp` is done after + struct ModifiedGlobal { + GlobalVarPtr glob_ref; + std::vector lval_ir_idx; // typically 1, generally get_width_on_stack() of global var (tensors) + + // for 1-slot globals int/cell/slice, assigning to them is just SETGLOB + // same for tensors, if they are fully rewritten in an expression: `gTensor = (5,6)` + void apply_fully_rewrite(CodeBlob& code, SrcLocation loc) const { + Op& op = code.emplace_back(loc, Op::_SetGlob, std::vector{}, lval_ir_idx, glob_ref); + op.set_impure_flag(); + } + + // for N-slot globals tensor/struct/union, assigning to their parts, like `gTensor.1 = 6` + // we need to read gTensor as a whole (0-th and 1-th component), rewrite 1-th component, and SETGLOB a whole back + void apply_partially_rewrite(CodeBlob& code, SrcLocation loc, std::vector&& was_modified_by_let) const { + LValContext local_lval; + local_lval.enter_rval_inside_lval(); + std::vector local_ir_idx = pre_compile_symbol(loc, glob_ref, code, &local_lval); + for (size_t i = 0; i < local_ir_idx.size(); ++i) { + if (was_modified_by_let[i]) { + code.emplace_back(loc, Op::_Let, std::vector{local_ir_idx[i]}, std::vector{lval_ir_idx[i]}); + } + } + + Op& op = code.emplace_back(loc, Op::_SetGlob, std::vector{}, local_ir_idx, glob_ref); + op.set_impure_flag(); + } + }; + + // every tensor index, when a tensor is a global, is registered here (same for structs and fields) + // example: `global v: (int, int); v.1 = 5`, implicit var is created `$tmp = 5`, and when it's modified, + // we need to partially update w; essentially, apply_partially_rewrite() above will be called + struct ModifiedFieldOfGlobal { + AnyExprV tensor_obj; + int index_at; + std::vector lval_ir_idx; + + void apply(CodeBlob& code, SrcLocation loc) const { + LValContext local_lval; + local_lval.enter_rval_inside_lval(); + std::vector obj_ir_idx = pre_compile_expr(tensor_obj, code, nullptr, &local_lval); + const TypeDataTensor* t_tensor = tensor_obj->inferred_type->try_as(); + tolk_assert(t_tensor); + int stack_width = t_tensor->items[index_at]->get_width_on_stack(); + int stack_offset = 0; + for (int i = 0; i < index_at; ++i) { + stack_offset += t_tensor->items[i]->get_width_on_stack(); + } + std::vector field_ir_idx = {obj_ir_idx.begin() + stack_offset, obj_ir_idx.begin() + stack_offset + stack_width}; + tolk_assert(field_ir_idx.size() == lval_ir_idx.size()); + + vars_modification_watcher.trigger_callbacks(field_ir_idx, loc); + code.emplace_back(loc, Op::_Let, field_ir_idx, lval_ir_idx); + local_lval.after_let(std::move(field_ir_idx), code, loc); + } + }; + + // every tuple index used as lvalue is registered here + // example: `t.0 = 9`, implicit var is created `$tmp = 9`, as well as `$tmp_idx = 0` and `tupleSetAt()` is done after + // for `t.0.0` if t is `[[int, ...]]`, `tupleAt()` for it is done since it's rvalue, and `tupleSetAt()` is done 2 times + struct ModifiedTupleIndex { + AnyExprV tuple_obj; + int index_at; + std::vector lval_ir_idx; + + void apply(CodeBlob& code, SrcLocation loc) const { + LValContext local_lval; + local_lval.enter_rval_inside_lval(); + std::vector tuple_ir_idx = pre_compile_expr(tuple_obj, code, nullptr, &local_lval); + std::vector index_ir_idx = code.create_tmp_var(TypeDataInt::create(), loc, "(tuple-idx)"); + code.emplace_back(loc, Op::_IntConst, index_ir_idx, td::make_refint(index_at)); + + vars_modification_watcher.trigger_callbacks(tuple_ir_idx, loc); + FunctionPtr builtin_sym = lookup_global_symbol("tupleSetAt")->try_as(); + code.emplace_back(loc, Op::_Call, std::vector{tuple_ir_idx}, std::vector{tuple_ir_idx[0], lval_ir_idx[0], index_ir_idx[0]}, builtin_sym); + local_lval.after_let(std::move(tuple_ir_idx), code, loc); + } + }; + + int level_rval_inside_lval = 0; + std::vector> modifications; + + static bool vector_contains(const std::vector& ir_vars, var_idx_t ir_idx) { + for (var_idx_t var_in_vector : ir_vars) { + if (var_in_vector == ir_idx) { + return true; + } + } + return false; + } + +public: + void enter_rval_inside_lval() { level_rval_inside_lval++; } + void exit_rval_inside_lval() { level_rval_inside_lval--; } + bool is_rval_inside_lval() const { return level_rval_inside_lval > 0; } + + void capture_global_modification(GlobalVarPtr glob_ref, std::vector lval_ir_idx) { + modifications.emplace_back(ModifiedGlobal{glob_ref, std::move(lval_ir_idx)}); + } + + void capture_field_of_global_modification(AnyExprV tensor_obj, int index_at, std::vector lval_ir_idx) { + modifications.emplace_back(ModifiedFieldOfGlobal{tensor_obj, index_at, std::move(lval_ir_idx)}); + } + + void capture_tuple_index_modification(AnyExprV tuple_obj, int index_at, std::vector lval_ir_idx) { + modifications.emplace_back(ModifiedTupleIndex{tuple_obj, index_at, std::move(lval_ir_idx)}); + } + + void after_let(std::vector&& let_left_vars, CodeBlob& code, SrcLocation loc) const { + for (const auto& modification : modifications) { + if (const auto* m_glob = std::get_if(&modification)) { + int n_modified_by_let = 0; + std::vector was_modified_by_let; + was_modified_by_let.resize(m_glob->lval_ir_idx.size()); + for (size_t i = 0; i < m_glob->lval_ir_idx.size(); ++i) { + if (vector_contains(let_left_vars, m_glob->lval_ir_idx[i])) { + was_modified_by_let[i] = true; + n_modified_by_let++; + } + } + if (n_modified_by_let == static_cast(m_glob->lval_ir_idx.size())) { + m_glob->apply_fully_rewrite(code, loc); + } else if (n_modified_by_let > 0) { + m_glob->apply_partially_rewrite(code, loc, std::move(was_modified_by_let)); + } + } else if (const auto* m_tup = std::get_if(&modification)) { + bool was_tuple_index_modified = false; + for (var_idx_t field_ir_idx : m_tup->lval_ir_idx) { + was_tuple_index_modified |= vector_contains(let_left_vars, field_ir_idx); + } + if (was_tuple_index_modified) { + m_tup->apply(code, loc); + } + } else if (const auto* m_tens = std::get_if(&modification)) { + bool was_tensor_index_modified = false; + for (var_idx_t field_ir_idx : m_tens->lval_ir_idx) { + was_tensor_index_modified |= vector_contains(let_left_vars, field_ir_idx); + } + if (was_tensor_index_modified) { + m_tens->apply(code, loc); + } + } + } + } +}; + +// given `{some_expr}!`, return some_expr +static AnyExprV unwrap_not_null_operator(AnyExprV v) { + while (auto v_notnull = v->try_as()) { + v = v_notnull->get_expr(); + } + return v; +} + +// given `{some_expr}.{i}`, check it for pattern `some_var.0` / `some_var.0.1` / etc. +// return some_var if satisfies (it may be a local or a global var, a tensor or a tuple) +// return nullptr otherwise: `f().0` / `(v = rhs).0` / `some_var.method().0` / etc. +static V calc_sink_leftmost_obj(V v) { + AnyExprV leftmost_obj = unwrap_not_null_operator(v->get_obj()); + while (auto v_dot = leftmost_obj->try_as()) { + if (!v_dot->is_target_indexed_access()) { + break; + } + leftmost_obj = unwrap_not_null_operator(v_dot->get_obj()); + } + return leftmost_obj->type == ast_reference ? leftmost_obj->as() : nullptr; +} + + +static std::vector> pre_compile_tensor_inner(CodeBlob& code, const std::vector& args, + const TypeDataTensor* tensor_target_type, LValContext* lval_ctx) { + const int n = static_cast(args.size()); + if (n == 0) { // just `()` + return {}; + } + tolk_assert(!tensor_target_type || tensor_target_type->size() == n); + if (n == 1) { // just `(x)`: even if x is modified (e.g. `f(x=x+2)`), there are no next arguments + TypePtr child_target_type = tensor_target_type ? tensor_target_type->items[0] : nullptr; + return {pre_compile_expr(args[0], code, child_target_type, lval_ctx)}; + } + + // the purpose is to handle such cases: `return (x, x += y, x)` + // without this, ops will be { _Call $2 = +($0_x, $1_y); _Return $0_x, $2, $0_x } - invalid + // with this, ops will be { _Let $3 = $0_x; _Call $2 = ...; _Return $3, $2, $0_x } - valid, tmp var for x + // how it works: for every arg, after transforming to ops, start tracking ir_idx inside it + // on modification attempt, create Op::_Let to a tmp var and replace old ir_idx with tmp_idx in result + struct WatchingVarList { + std::vector watched_vars; + std::vector> res_lists; + + explicit WatchingVarList(int n_args) { + res_lists.reserve(n_args); + } + + bool is_watched(var_idx_t ir_idx) const { + return std::find(watched_vars.begin(), watched_vars.end(), ir_idx) != watched_vars.end(); + } + + void add_and_watch_modifications(std::vector&& vars_of_ith_arg, CodeBlob& code) { + for (var_idx_t ir_idx : vars_of_ith_arg) { + if (!code.vars[ir_idx].name.empty() && !is_watched(ir_idx)) { + watched_vars.emplace_back(ir_idx); + vars_modification_watcher.push_callback(ir_idx, [this, &code](SrcLocation loc, var_idx_t ir_idx) { + on_var_modified(ir_idx, loc, code); + }); + } + } + res_lists.emplace_back(std::move(vars_of_ith_arg)); + } + + void on_var_modified(var_idx_t ir_idx, SrcLocation loc, CodeBlob& code) { + tolk_assert(is_watched(ir_idx)); + std::vector tmp_idx_arr = code.create_tmp_var(code.vars[ir_idx].v_type, loc, "(pre-modified)"); + tolk_assert(tmp_idx_arr.size() == 1); + var_idx_t tmp_idx = tmp_idx_arr[0]; + code.emplace_back(loc, Op::_Let, std::vector{tmp_idx}, std::vector{ir_idx}); + for (std::vector& prev_vars : res_lists) { + std::replace(prev_vars.begin(), prev_vars.end(), ir_idx, tmp_idx); + } + } + + std::vector> clear_and_stop_watching() { + for (var_idx_t ir_idx : watched_vars) { + vars_modification_watcher.pop_callback(ir_idx); + } + watched_vars.clear(); + return std::move(res_lists); + } + }; + + WatchingVarList watched_vars(n); + for (int arg_idx = 0; arg_idx < n; ++arg_idx) { + TypePtr child_target_type = tensor_target_type ? tensor_target_type->items[arg_idx] : nullptr; + std::vector vars_of_ith_arg = pre_compile_expr(args[arg_idx], code, child_target_type, lval_ctx); + watched_vars.add_and_watch_modifications(std::move(vars_of_ith_arg), code); + } + return watched_vars.clear_and_stop_watching(); +} + +static std::vector pre_compile_tensor(CodeBlob& code, const std::vector& args, + LValContext* lval_ctx = nullptr) { + std::vector types_list; + types_list.reserve(args.size()); + for (AnyExprV item : args) { + types_list.push_back(item->inferred_type); + } + const TypeDataTensor* tensor_target_type = TypeDataTensor::create(std::move(types_list))->try_as(); + std::vector> res_lists = pre_compile_tensor_inner(code, args, tensor_target_type, lval_ctx); + std::vector res; + for (const std::vector& list : res_lists) { + res.insert(res.end(), list.cbegin(), list.cend()); + } + return res; +} + +static std::vector pre_compile_let(CodeBlob& code, AnyExprV lhs, AnyExprV rhs, SrcLocation loc) { + // [lhs] = [rhs]; since type checking is ok, it's the same as "lhs = rhs" + if (lhs->type == ast_typed_tuple && rhs->type == ast_typed_tuple) { + // note: there are no type transitions (adding nullability flag, etc.), since only 1-slot elements allowed in tuples + LValContext local_lval; + std::vector left = pre_compile_tensor(code, lhs->as()->get_items(), &local_lval); + vars_modification_watcher.trigger_callbacks(left, loc); + std::vector rvect = pre_compile_tensor(code, rhs->as()->get_items()); + code.emplace_back(loc, Op::_Let, left, rvect); + local_lval.after_let(std::move(left), code, loc); + std::vector right = code.create_tmp_var(TypeDataTuple::create(), loc, "(tuple)"); + code.emplace_back(lhs->loc, Op::_Tuple, right, std::move(rvect)); + return right; + } + // [lhs] = rhs; it's un-tuple to N left vars + if (lhs->type == ast_typed_tuple) { + LValContext local_lval; + std::vector left = pre_compile_tensor(code, lhs->as()->get_items(), &local_lval); + vars_modification_watcher.trigger_callbacks(left, loc); + std::vector right = pre_compile_expr(rhs, code, nullptr); + const TypeDataTypedTuple* inferred_tuple = rhs->inferred_type->try_as(); + std::vector types_list = inferred_tuple->items; + std::vector rvect = code.create_tmp_var(TypeDataTensor::create(std::move(types_list)), rhs->loc, "(unpack-tuple)"); + code.emplace_back(lhs->loc, Op::_UnTuple, rvect, std::move(right)); + code.emplace_back(loc, Op::_Let, left, rvect); + local_lval.after_let(std::move(left), code, loc); + return right; + } + // small optimization: `var x = rhs` or `local_var = rhs` (90% cases), LValContext not needed actually + if (lhs->type == ast_local_var_lhs || (lhs->type == ast_reference && lhs->as()->sym->try_as())) { + std::vector left = pre_compile_expr(lhs, code, nullptr); // effectively, local_var->ir_idx + vars_modification_watcher.trigger_callbacks(left, loc); + std::vector right = pre_compile_expr(rhs, code, lhs->inferred_type); + code.emplace_back(loc, Op::_Let, std::move(left), right); + return right; + } + // lhs = rhs + LValContext local_lval; + std::vector left = pre_compile_expr(lhs, code, nullptr, &local_lval); + vars_modification_watcher.trigger_callbacks(left, loc); + std::vector right = pre_compile_expr(rhs, code, lhs->inferred_type); + code.emplace_back(loc, Op::_Let, left, right); + local_lval.after_let(std::move(left), code, loc); + return right; +} + +static std::vector gen_op_call(CodeBlob& code, TypePtr ret_type, SrcLocation loc, + std::vector&& args_vars, FunctionPtr fun_ref, const char* debug_desc) { + std::vector rvect = code.create_tmp_var(ret_type, loc, debug_desc); + Op& op = code.emplace_back(loc, Op::_Call, rvect, std::move(args_vars), fun_ref); + if (!fun_ref->is_marked_as_pure()) { + op.set_impure_flag(); + } + return rvect; +} + +// "Transition to target (runtime) type" is the following process. +// Imagine `fun analyze(t: (int,int)?)` and a call `analyze((1,2))`. +// `(1,2)` (inferred_type) is 2 stack slots, but `t` (target_type) is 3 (one for null-flag). +// So, this null flag should be implicitly added (non-zero, since a variable is not null). +// Another example: `var t: (int, int)? = null`. +// `null` (inferred_type) is 1 stack slots, but target_type is 3, we should add 2 nulls. +// Another example: `var t1 = (1, null); var t2: (int, (int,int)?) = t1;`. +// Then t1's rvect is 2 vars (1 and null), but t1's `null` should be converted to 3 stack slots (resulting in 4 total). +// The same mechanism will work for union types in the future. +// Here rvect is a list of IR vars for inferred_type, probably patched due to target_type. +GNU_ATTRIBUTE_NOINLINE +static std::vector transition_expr_to_runtime_type_impl(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc) { + // pass `T` to `T` + // could occur for passing tensor `(..., T, ...)` to `(..., T, ...)` while traversing tensor's components + if (target_type == original_type) { + return rvect; + } + + int target_w = target_type->get_width_on_stack(); + const TypeDataNullable* t_nullable = target_type->try_as(); + const TypeDataNullable* o_nullable = original_type->try_as(); + + // handle `never` + // it may occur due to smart cast and in unreachable branches + // we can't do anything reasonable here, but (hopefully) execution will never reach this point, and stack won't be polluted + if (original_type == TypeDataNever::create()) { + std::vector dummy_rvect; + dummy_rvect.reserve(target_w); + for (int i = 0; i < target_w; ++i) { + dummy_rvect.push_back(code.create_tmp_var(TypeDataUnknown::create(), loc, "(never)")[0]); + } + return dummy_rvect; + } + if (target_type == TypeDataNever::create()) { + return {}; + } + + // pass `null` to `T?` + // for primitives like `int?`, no changes in rvect, null occupies the same TVM slot + // for tensors like `(int,int)?`, `null` is represented as N nulls + 1 null flag, insert N nulls + if (t_nullable && original_type == TypeDataNullLiteral::create()) { + tolk_assert(rvect.size() == 1); + if (target_w == 1 && !t_nullable->is_primitive_nullable()) { // `null` to `()?` + rvect = code.create_tmp_var(TypeDataInt::create(), loc, "(NNFlag)"); + code.emplace_back(loc, Op::_IntConst, rvect, td::make_refint(0)); + } + if (target_w > 1) { + FunctionPtr builtin_sym = lookup_global_symbol("__null")->try_as(); + rvect.reserve(target_w + 1); + for (int i = 1; i < target_w - 1; ++i) { + std::vector ith_null = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); + code.emplace_back(loc, Op::_Call, ith_null, std::vector{}, builtin_sym); + rvect.push_back(ith_null[0]); + } + std::vector null_flag_ir = code.create_tmp_var(TypeDataInt::create(), loc, "(NNFlag)"); + var_idx_t null_flag_ir_idx = null_flag_ir[0]; + code.emplace_back(loc, Op::_IntConst, std::move(null_flag_ir), td::make_refint(0)); + rvect.push_back(null_flag_ir_idx); + } + return rvect; + } + // pass `T` to `T?` + // for primitives like `int?`, no changes in rvect: `int` and `int?` occupy the same TVM slot (null is represented as NULL TVM value) + // for passing `(int, int)` to `(int, int)?` / `(int, null)` to `(int, (int,int)?)?`, add a null flag equals to 0 + if (t_nullable && !o_nullable) { + if (!t_nullable->is_primitive_nullable()) { + rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type, t_nullable->inner, loc); + tolk_assert(target_w == static_cast(rvect.size() + 1)); + std::vector null_flag_ir = code.create_tmp_var(TypeDataInt::create(), loc, "(NNFlag)"); + var_idx_t null_flag_ir_idx = null_flag_ir[0]; + code.emplace_back(loc, Op::_IntConst, std::move(null_flag_ir), td::make_refint(-1)); + rvect.push_back(null_flag_ir_idx); + } + return rvect; + } + // pass `T1?` to `T2?` + // for example, `int8?` to `int16?` + // transition inner types, leaving nullable flag unchanged for tensors + if (t_nullable && o_nullable) { + if (target_w > 1) { + var_idx_t null_flag_ir_idx = rvect.back(); + rvect.pop_back(); + rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, o_nullable->inner, t_nullable->inner, loc); + rvect.push_back(null_flag_ir_idx); + } + return rvect; + } + // pass `T?` to `null` + // it may occur due to smart cast, when a `T?` variable is guaranteed to be always null + // (for instance, always-null `(int,int)?` will be represented as 1 TVM NULL value, not 3) + if (target_type == TypeDataNullLiteral::create() && original_type->can_rhs_be_assigned(target_type)) { + tolk_assert(o_nullable || original_type == TypeDataUnknown::create()); + if (o_nullable && !o_nullable->is_primitive_nullable()) { + FunctionPtr builtin_sym = lookup_global_symbol("__null")->try_as(); + rvect = code.create_tmp_var(TypeDataNullLiteral::create(), loc, "(null-literal)"); + code.emplace_back(loc, Op::_Call, rvect, std::vector{}, builtin_sym); + } + return rvect; + } + // pass `T?` to `T` (or, more generally, `T1?` to `T2`) + // it may occur due to operator `!` or smart cast + // for primitives like `int?`, no changes in rvect + // for passing `(int, int)?` to `(int, int)`, drop the null flag from the tail + // for complex scenarios like passing `(int, (int,int)?)?` to `(int, null)`, recurse the call + // (it may occur on `someF(t = (3,null))` when `(3,null)` at first targeted to lhs, but actually its result is rhs) + if (!t_nullable && o_nullable) { + if (!o_nullable->is_primitive_nullable()) { + rvect.pop_back(); + rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type->try_as()->inner, target_type, loc); + } + return rvect; + } + // pass `bool` to `int` + // in code, it's done via `as` operator, like `boolVar as int` + // no changes in rvect, boolVar is guaranteed to be -1 or 0 at TVM level + if (target_type == TypeDataInt::create() && original_type == TypeDataBool::create()) { + return rvect; + } + // pass something to `unknown` + // probably, it comes from `_ = rhs`, type of `_` is unknown, it's target_type of rhs + // no changes in rvect + if (target_type == TypeDataUnknown::create()) { + return rvect; + } + // pass `unknown` to something + // probably, it comes from `arg` in exception, it's inferred as `unknown` and could be cast to any value + if (original_type == TypeDataUnknown::create()) { + tolk_assert(rvect.size() == 1); + return rvect; + } + // pass tensor to tensor, e.g. `(1, null)` to `(int, slice?)` / `(1, null)` to `(int, (int,int)?)` + // every element of rhs tensor should be transitioned + if (target_type->try_as() && original_type->try_as()) { + const TypeDataTensor* target_tensor = target_type->try_as(); + const TypeDataTensor* inferred_tensor = original_type->try_as(); + tolk_assert(target_tensor->size() == inferred_tensor->size()); + tolk_assert(inferred_tensor->get_width_on_stack() == static_cast(rvect.size())); + std::vector result_rvect; + result_rvect.reserve(target_w); + int stack_offset = 0; + for (int i = 0; i < inferred_tensor->size(); ++i) { + int ith_w = inferred_tensor->items[i]->get_width_on_stack(); + std::vector rvect_i{rvect.begin() + stack_offset, rvect.begin() + stack_offset + ith_w}; + std::vector result_i = transition_expr_to_runtime_type_impl(std::move(rvect_i), code, inferred_tensor->items[i], target_tensor->items[i], loc); + result_rvect.insert(result_rvect.end(), result_i.begin(), result_i.end()); + stack_offset += ith_w; + } + return result_rvect; + } + // pass tuple to tuple, e.g. `[1, null]` to `[int, int?]` / `[1, null]` to `[int, [int?,int?]?]` + // to changes to rvect, since tuples contain only 1-slot elements + if (target_type->try_as() && original_type->try_as()) { + tolk_assert(target_type->get_width_on_stack() == original_type->get_width_on_stack()); + return rvect; + } + + throw Fatal("unhandled transition_expr_to_runtime_type_impl() combination"); +} + +// invoke the function above only if potentially needed to +// (if an expression is targeted to another type) +#ifndef TOLK_DEBUG +GNU_ATTRIBUTE_ALWAYS_INLINE +#endif +static std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr target_type, AnyExprV v) { + if (target_type != nullptr && target_type != v->inferred_type) { + rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, v->inferred_type, target_type, v->loc); + } + return rvect; +} + +// the second overload of the same function, invoke impl only when original and target differ +#ifndef TOLK_DEBUG +GNU_ATTRIBUTE_ALWAYS_INLINE +#endif +static std::vector transition_to_target_type(std::vector&& rvect, CodeBlob& code, TypePtr original_type, TypePtr target_type, SrcLocation loc) { + if (target_type != original_type) { + rvect = transition_expr_to_runtime_type_impl(std::move(rvect), code, original_type, target_type, loc); + } + return rvect; +} + + +std::vector pre_compile_symbol(SrcLocation loc, const Symbol* sym, CodeBlob& code, LValContext* lval_ctx) { + if (GlobalVarPtr glob_ref = sym->try_as()) { + // handle `globalVar = rhs` / `mutate globalVar` + if (lval_ctx && !lval_ctx->is_rval_inside_lval()) { + std::vector lval_ir_idx = code.create_tmp_var(glob_ref->declared_type, loc, "(lval-glob)"); + lval_ctx->capture_global_modification(glob_ref, lval_ir_idx); + return lval_ir_idx; + } + // `globalVar` is used for reading, just create local IR var to represent its value, Op GlobVar will fill it + // note, that global tensors are stored as a tuple an unpacked to N vars on read, N determined by declared_type + std::vector local_ir_idx = code.create_var(glob_ref->declared_type, loc, "g_" + glob_ref->name); + code.emplace_back(loc, Op::_GlobVar, local_ir_idx, std::vector{}, glob_ref); + if (lval_ctx) { // `globalVar.0 = rhs`, globalVar is rval inside lval + lval_ctx->capture_global_modification(glob_ref, local_ir_idx); + } + return local_ir_idx; + } + if (GlobalConstPtr const_ref = sym->try_as()) { + if (const_ref->is_int_const()) { + std::vector rvect = code.create_tmp_var(TypeDataInt::create(), loc, "(glob-const)"); + code.emplace_back(loc, Op::_IntConst, rvect, const_ref->as_int_const()); + return rvect; + } else { + std::vector rvect = code.create_tmp_var(TypeDataSlice::create(), loc, "(glob-const)"); + code.emplace_back(loc, Op::_SliceConst, rvect, const_ref->as_slice_const()); + return rvect; + } + } + if (FunctionPtr fun_ref = sym->try_as()) { + std::vector rvect = code.create_tmp_var(fun_ref->inferred_full_type, loc, "(glob-var-fun)"); + code.emplace_back(loc, Op::_GlobVar, rvect, std::vector{}, fun_ref); + return rvect; + } + if (LocalVarPtr var_ref = sym->try_as()) { +#ifdef TOLK_DEBUG + tolk_assert(static_cast(var_ref->ir_idx.size()) == var_ref->declared_type->get_width_on_stack()); +#endif + return var_ref->ir_idx; + } + throw Fatal("pre_compile_symbol"); +} + +static std::vector process_reference(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + std::vector rvect = pre_compile_symbol(v->loc, v->sym, code, lval_ctx); + + // a local variable might be smart cast at this point, for example we're in `if (v != null)` + // it means that we must drop the null flag (if it's a tensor), or maybe perform other stack transformations + // (from original var_ref->ir_idx to fit smart cast) + if (LocalVarPtr var_ref = v->sym->try_as()) { + // note, inside `if (v != null)` when `v` is used for writing, v->inferred_type is an original (declared_type) + // (smart casts apply only for rvalue, not for lvalue, we don't check it here, it's a property of inferring) + rvect = transition_to_target_type(std::move(rvect), code, var_ref->declared_type, v->inferred_type, v->loc); + } + + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_assignment(V v, CodeBlob& code, TypePtr target_type) { + AnyExprV lhs = v->get_lhs(); + AnyExprV rhs = v->get_rhs(); + + if (auto lhs_decl = lhs->try_as()) { + std::vector rvect = pre_compile_let(code, lhs_decl->get_expr(), rhs, v->loc); + return transition_to_target_type(std::move(rvect), code, target_type, v); + } else { + std::vector rvect = pre_compile_let(code, lhs, rhs, v->loc); + // now rvect contains rhs IR vars constructed to fit lhs (for correct assignment, lhs type was target_type for rhs) + // but the type of `lhs = rhs` is RHS (see type inferring), so rvect now should fit rhs->inferred_type (= v->inferred_type) + // example: `t1 = t2 = null`, we're at `t2 = null`, earlier declared t1: `int?`, t2: `(int,int)?` + // currently "null" matches t2 (3 null slots), but type of this assignment is "plain null" (1 slot) assigned later to t1 + rvect = transition_to_target_type(std::move(rvect), code, lhs->inferred_type, v->inferred_type, v->loc); + return transition_to_target_type(std::move(rvect), code, target_type, v); + } +} + +static std::vector process_set_assign(V v, CodeBlob& code, TypePtr target_type) { + // for "a += b", emulate "a = a + b" + // seems not beautiful, but it works; probably, this transformation should be done at AST level in advance + std::string_view calc_operator = v->operator_name; // "+" for operator += + auto v_apply = createV(v->loc, calc_operator, static_cast(v->tok - 1), v->get_lhs(), v->get_rhs()); + v_apply->assign_inferred_type(v->inferred_type); + v_apply->assign_fun_ref(v->fun_ref); + + std::vector rvect = pre_compile_let(code, v->get_lhs(), v_apply, v->loc); + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_binary_operator(V v, CodeBlob& code, TypePtr target_type) { + TokenType t = v->tok; + + if (v->fun_ref) { // almost all operators, fun_ref was assigned at type inferring + std::vector args_vars = pre_compile_tensor(code, {v->get_lhs(), v->get_rhs()}); + std::vector rvect = gen_op_call(code, v->inferred_type, v->loc, std::move(args_vars), v->fun_ref, "(binary-op)"); + return transition_to_target_type(std::move(rvect), code, target_type, v); + } + if (t == tok_logical_and || t == tok_logical_or) { + // do the following transformations: + // a && b -> a ? (b != 0) : 0 + // a || b -> a ? 1 : (b != 0) + AnyExprV v_0 = createV(v->loc, td::make_refint(0), "0"); + v_0->mutate()->assign_inferred_type(TypeDataInt::create()); + AnyExprV v_1 = createV(v->loc, td::make_refint(-1), "-1"); + v_1->mutate()->assign_inferred_type(TypeDataInt::create()); + auto v_b_ne_0 = createV(v->loc, "!=", tok_neq, v->get_rhs(), v_0); + v_b_ne_0->mutate()->assign_inferred_type(TypeDataInt::create()); + v_b_ne_0->mutate()->assign_fun_ref(lookup_global_symbol("_!=_")->try_as()); + std::vector cond = pre_compile_expr(v->get_lhs(), code, nullptr); + tolk_assert(cond.size() == 1); + std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(ternary)"); + Op& if_op = code.emplace_back(v->loc, Op::_If, cond); + code.push_set_cur(if_op.block0); + code.emplace_back(v->loc, Op::_Let, rvect, pre_compile_expr(t == tok_logical_and ? v_b_ne_0 : v_1, code, nullptr)); + code.close_pop_cur(v->loc); + code.push_set_cur(if_op.block1); + code.emplace_back(v->loc, Op::_Let, rvect, pre_compile_expr(t == tok_logical_and ? v_0 : v_b_ne_0, code, nullptr)); + code.close_pop_cur(v->loc); + return transition_to_target_type(std::move(rvect), code, target_type, v); + } + + throw UnexpectedASTNodeType(v, "process_binary_operator"); +} + +static std::vector process_unary_operator(V v, CodeBlob& code, TypePtr target_type) { + std::vector rhs_vars = pre_compile_expr(v->get_rhs(), code, nullptr); + std::vector rvect = gen_op_call(code, v->inferred_type, v->loc, std::move(rhs_vars), v->fun_ref, "(unary-op)"); + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_ternary_operator(V v, CodeBlob& code, TypePtr target_type) { + std::vector cond = pre_compile_expr(v->get_cond(), code, nullptr); + tolk_assert(cond.size() == 1); + std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(cond)"); + + if (v->get_cond()->is_always_true) { + code.emplace_back(v->get_when_true()->loc, Op::_Let, rvect, pre_compile_expr(v->get_when_true(), code, v->inferred_type)); + } else if (v->get_cond()->is_always_false) { + code.emplace_back(v->get_when_false()->loc, Op::_Let, rvect, pre_compile_expr(v->get_when_false(), code, v->inferred_type)); + } else { + Op& if_op = code.emplace_back(v->loc, Op::_If, cond); + code.push_set_cur(if_op.block0); + code.emplace_back(v->get_when_true()->loc, Op::_Let, rvect, pre_compile_expr(v->get_when_true(), code, v->inferred_type)); + code.close_pop_cur(v->get_when_true()->loc); + code.push_set_cur(if_op.block1); + code.emplace_back(v->get_when_false()->loc, Op::_Let, rvect, pre_compile_expr(v->get_when_false(), code, v->inferred_type)); + code.close_pop_cur(v->get_when_false()->loc); + } + + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_cast_as_operator(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + TypePtr child_target_type = v->cast_to_type; + std::vector rvect = pre_compile_expr(v->get_expr(), code, child_target_type, lval_ctx); + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_not_null_operator(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + TypePtr child_target_type = v->get_expr()->inferred_type; + if (const auto* as_nullable = child_target_type->try_as()) { + child_target_type = as_nullable->inner; + } + std::vector rvect = pre_compile_expr(v->get_expr(), code, child_target_type, lval_ctx); + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_is_null_check(V v, CodeBlob& code, TypePtr target_type) { + std::vector expr_ir_idx = pre_compile_expr(v->get_expr(), code, nullptr); + std::vector isnull_ir_idx = code.create_tmp_var(TypeDataBool::create(), v->loc, "(is-null)"); + TypePtr expr_type = v->get_expr()->inferred_type; + + if (const TypeDataNullable* t_nullable = expr_type->try_as()) { + if (!t_nullable->is_primitive_nullable()) { + std::vector zero_ir_idx = code.create_tmp_var(TypeDataInt::create(), v->loc, "(zero)"); + code.emplace_back(v->loc, Op::_IntConst, zero_ir_idx, td::make_refint(0)); + FunctionPtr eq_sym = lookup_global_symbol("_==_")->try_as(); + code.emplace_back(v->loc, Op::_Call, isnull_ir_idx, std::vector{expr_ir_idx.back(), zero_ir_idx[0]}, eq_sym); + } else { + FunctionPtr builtin_sym = lookup_global_symbol("__isNull")->try_as(); + code.emplace_back(v->loc, Op::_Call, isnull_ir_idx, expr_ir_idx, builtin_sym); + } + } else { + bool always_null = expr_type == TypeDataNullLiteral::create(); + code.emplace_back(v->loc, Op::_IntConst, isnull_ir_idx, td::make_refint(always_null ? -1 : 0)); + } + + if (v->is_negated) { + FunctionPtr not_sym = lookup_global_symbol("!b_")->try_as(); + code.emplace_back(v->loc, Op::_Call, isnull_ir_idx, std::vector{isnull_ir_idx}, not_sym); + } + return transition_to_target_type(std::move(isnull_ir_idx), code, target_type, v); +} + +static std::vector process_dot_access(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + // it's NOT a method call `t.tupleSize()` (since such cases are handled by process_function_call) + // it's `t.0`, `getUser().id`, and `t.tupleSize` (as a reference, not as a call) + if (!v->is_target_fun_ref()) { + TypePtr obj_type = v->get_obj()->inferred_type; + int index_at = std::get(v->target); + // `tensorVar.0` + if (const auto* t_tensor = obj_type->try_as()) { + // handle `tensorVar.0 = rhs` if tensors is a global, special case, then the global will be read on demand + if (lval_ctx && !lval_ctx->is_rval_inside_lval()) { + if (auto sink = calc_sink_leftmost_obj(v); sink && sink->sym->try_as()) { + std::vector lval_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(lval-global-tensor)"); + lval_ctx->capture_field_of_global_modification(v->get_obj(), index_at, lval_ir_idx); + return lval_ir_idx; + } + } + // since a tensor of N elems are N vars on a stack actually, calculate offset + std::vector lhs_vars = pre_compile_expr(v->get_obj(), code, nullptr, lval_ctx); + int stack_width = t_tensor->items[index_at]->get_width_on_stack(); + int stack_offset = 0; + for (int i = 0; i < index_at; ++i) { + stack_offset += t_tensor->items[i]->get_width_on_stack(); + } + std::vector rvect{lhs_vars.begin() + stack_offset, lhs_vars.begin() + stack_offset + stack_width}; + // a tensor index might be smart cast at this point, for example we're in `if (t.1 != null)` + // it means that we must drop the null flag (if `t.1` is a tensor), or maybe perform other stack transformations + // (from original rvect = (vars of t.1) to fit smart cast) + rvect = transition_to_target_type(std::move(rvect), code, t_tensor->items[index_at], v->inferred_type, v->loc); + return transition_to_target_type(std::move(rvect), code, target_type, v); + } + // `tupleVar.0` + if (obj_type->try_as() || obj_type->try_as()) { + // handle `tupleVar.0 = rhs`, "0 SETINDEX" will be called when this was is modified + if (lval_ctx && !lval_ctx->is_rval_inside_lval() && calc_sink_leftmost_obj(v)) { + std::vector lval_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(lval-tuple-field)"); + lval_ctx->capture_tuple_index_modification(v->get_obj(), index_at, lval_ir_idx); + return lval_ir_idx; + } + // `tupleVar.0` as rvalue: the same as "tupleAt(tupleVar, 0)" written in terms of IR vars + std::vector tuple_ir_idx = pre_compile_expr(v->get_obj(), code); + std::vector index_ir_idx = code.create_tmp_var(TypeDataInt::create(), v->get_identifier()->loc, "(tuple-idx)"); + code.emplace_back(v->loc, Op::_IntConst, index_ir_idx, td::make_refint(index_at)); + std::vector field_ir_idx = code.create_tmp_var(v->inferred_type, v->loc, "(tuple-field)"); + tolk_assert(tuple_ir_idx.size() == 1 && field_ir_idx.size() == 1); // tuples contain only 1-slot values + FunctionPtr builtin_sym = lookup_global_symbol("tupleAt")->try_as(); + code.emplace_back(v->loc, Op::_Call, field_ir_idx, std::vector{tuple_ir_idx[0], index_ir_idx[0]}, builtin_sym); + if (lval_ctx && calc_sink_leftmost_obj(v)) { // `tupleVar.0.1 = rhs`, then `tupleVar.0` is rval inside lval + lval_ctx->capture_tuple_index_modification(v->get_obj(), index_at, field_ir_idx); + } + // like tensor index, `tupleVar.1` also might be smart cast, for example we're in `if (tupleVar.1 != null)` + // but since tuple's elements are only 1-slot width (no tensors and unions), no stack transformations required + return transition_to_target_type(std::move(field_ir_idx), code, target_type, v); + } + tolk_assert(false); + } + + // okay, v->target refs a function, like `obj.method`, filled at type inferring + // (currently, nothing except a global function can be referenced, no object-scope methods exist) + FunctionPtr fun_ref = std::get(v->target); + tolk_assert(fun_ref); + std::vector rvect = pre_compile_symbol(v->loc, fun_ref, code, lval_ctx); + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_function_call(V v, CodeBlob& code, TypePtr target_type) { + // v is `globalF(args)` / `globalF(args)` / `obj.method(args)` / `local_var(args)` / `getF()(args)` + FunctionPtr fun_ref = v->fun_maybe; + if (!fun_ref) { + // it's `local_var(args)`, treat args like a tensor: + // 1) when variables are modified like `local_var(x, x += 2, x)`, regular mechanism of watching automatically works + // 2) when `null` is passed to `(int, int)?`, or any other type transitions, it automatically works + std::vector args; + args.reserve(v->get_num_args()); + for (int i = 0; i < v->get_num_args(); ++i) { + args.push_back(v->get_arg(i)->get_expr()); + } + std::vector params_types = v->get_callee()->inferred_type->try_as()->params_types; + const TypeDataTensor* tensor_tt = TypeDataTensor::create(std::move(params_types))->try_as(); + std::vector> vars_per_arg = pre_compile_tensor_inner(code, args, tensor_tt, nullptr); + std::vector args_vars; + for (const std::vector& list : vars_per_arg) { + args_vars.insert(args_vars.end(), list.cbegin(), list.cend()); + } + std::vector tfunc = pre_compile_expr(v->get_callee(), code, nullptr); + tolk_assert(tfunc.size() == 1); + args_vars.push_back(tfunc[0]); + std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(call-ind)"); + Op& op = code.emplace_back(v->loc, Op::_CallInd, rvect, std::move(args_vars)); + op.set_impure_flag(); + return transition_to_target_type(std::move(rvect), code, target_type, v); + } + + int delta_self = v->is_dot_call(); + AnyExprV obj_leftmost = nullptr; + std::vector args; + args.reserve(delta_self + v->get_num_args()); + if (delta_self) { + args.push_back(v->get_dot_obj()); + obj_leftmost = v->get_dot_obj(); + while (obj_leftmost->type == ast_function_call && obj_leftmost->as()->is_dot_call() && obj_leftmost->as()->fun_maybe && obj_leftmost->as()->fun_maybe->does_return_self()) { + obj_leftmost = obj_leftmost->as()->get_dot_obj(); + } + } + for (int i = 0; i < v->get_num_args(); ++i) { + args.push_back(v->get_arg(i)->get_expr()); + } + // the purpose of tensor_tt ("tensor target type") is to transition `null` to `(int, int)?` and so on + // the purpose of calling `pre_compile_tensor_inner` is to have 0-th IR vars to handle return self + std::vector params_types = fun_ref->inferred_full_type->try_as()->params_types; + const TypeDataTensor* tensor_tt = TypeDataTensor::create(std::move(params_types))->try_as(); + std::vector> vars_per_arg = pre_compile_tensor_inner(code, args, tensor_tt, nullptr); + + TypePtr op_call_type = v->inferred_type; + TypePtr real_ret_type = v->inferred_type; + if (delta_self && fun_ref->does_return_self()) { + real_ret_type = TypeDataVoid::create(); + if (!fun_ref->parameters[0].is_mutate_parameter()) { + op_call_type = TypeDataVoid::create(); + } + } + if (fun_ref->has_mutate_params()) { + std::vector types_list; + for (int i = 0; i < delta_self + v->get_num_args(); ++i) { + if (fun_ref->parameters[i].is_mutate_parameter()) { + types_list.push_back(fun_ref->parameters[i].declared_type); + } + } + types_list.push_back(real_ret_type); + op_call_type = TypeDataTensor::create(std::move(types_list)); + } + + std::vector args_vars; + for (const std::vector& list : vars_per_arg) { + args_vars.insert(args_vars.end(), list.cbegin(), list.cend()); + } + std::vector rvect_apply = gen_op_call(code, op_call_type, v->loc, std::move(args_vars), fun_ref, "(fun-call)"); + + if (fun_ref->has_mutate_params()) { + LValContext local_lval; + std::vector left; + for (int i = 0; i < delta_self + v->get_num_args(); ++i) { + if (fun_ref->parameters[i].is_mutate_parameter()) { + AnyExprV arg_i = obj_leftmost && i == 0 ? obj_leftmost : args[i]; + tolk_assert(arg_i->is_lvalue || i == 0); + if (arg_i->is_lvalue) { + std::vector ith_var_idx = pre_compile_expr(arg_i, code, nullptr, &local_lval); + left.insert(left.end(), ith_var_idx.begin(), ith_var_idx.end()); + } else { + left.insert(left.end(), vars_per_arg[0].begin(), vars_per_arg[0].end()); + } + } + } + std::vector rvect = code.create_tmp_var(real_ret_type, v->loc, "(fun-call)"); + left.insert(left.end(), rvect.begin(), rvect.end()); + vars_modification_watcher.trigger_callbacks(left, v->loc); + code.emplace_back(v->loc, Op::_Let, left, rvect_apply); + local_lval.after_let(std::move(left), code, v->loc); + rvect_apply = rvect; + } + + if (obj_leftmost && fun_ref->does_return_self()) { + if (obj_leftmost->is_lvalue) { // to handle if obj is global var, potentially re-assigned inside a chain + rvect_apply = pre_compile_expr(obj_leftmost, code, nullptr); + } else { // temporary object, not lvalue, pre_compile_expr + rvect_apply = vars_per_arg[0]; + } + } + + return transition_to_target_type(std::move(rvect_apply), code, target_type, v); +} + +static std::vector process_tensor(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + // tensor is compiled "as is", for example `(1, null)` occupies 2 slots + // and if assigned/passed to something other, like `(int, (int,int)?)`, a whole tensor is transitioned, it works + std::vector rvect = pre_compile_tensor(code, v->get_items(), lval_ctx); + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_typed_tuple(V v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + if (lval_ctx) { // todo some time, make "var (a, [b,c]) = (1, [2,3])" work + v->error("[...] can not be used as lvalue here"); + } + std::vector left = code.create_tmp_var(v->inferred_type, v->loc, "(pack-tuple)"); + std::vector right = pre_compile_tensor(code, v->get_items(), lval_ctx); + code.emplace_back(v->loc, Op::_Tuple, left, std::move(right)); + return transition_to_target_type(std::move(left), code, target_type, v); +} + +static std::vector process_int_const(V v, CodeBlob& code, TypePtr target_type) { + std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(int-const)"); + code.emplace_back(v->loc, Op::_IntConst, rvect, v->intval); + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_string_const(V v, CodeBlob& code, TypePtr target_type) { + ConstantValue value = eval_const_init_value(v); + std::vector rvect = code.create_tmp_var(v->inferred_type, v->loc, "(str-const)"); + if (value.is_int()) { + code.emplace_back(v->loc, Op::_IntConst, rvect, value.as_int()); + } else { + code.emplace_back(v->loc, Op::_SliceConst, rvect, value.as_slice()); + } + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_bool_const(V v, CodeBlob& code, TypePtr target_type) { + FunctionPtr builtin_sym = lookup_global_symbol(v->bool_val ? "__true" : "__false")->try_as(); + std::vector rvect = gen_op_call(code, v->inferred_type, v->loc, {}, builtin_sym, "(bool-const)"); + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_null_keyword(V v, CodeBlob& code, TypePtr target_type) { + FunctionPtr builtin_sym = lookup_global_symbol("__null")->try_as(); + std::vector rvect = gen_op_call(code, v->inferred_type, v->loc, {}, builtin_sym, "(null-literal)"); + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_local_var(V v, CodeBlob& code, TypePtr target_type) { + if (v->marked_as_redef) { + std::vector rvect = pre_compile_symbol(v->loc, v->var_ref, code, nullptr); + return transition_to_target_type(std::move(rvect), code, target_type, v); + } + + tolk_assert(v->var_ref->ir_idx.empty()); + v->var_ref->mutate()->assign_ir_idx(code.create_var(v->inferred_type, v->loc, v->var_ref->name)); + std::vector rvect = v->var_ref->ir_idx; + return transition_to_target_type(std::move(rvect), code, target_type, v); +} + +static std::vector process_local_vars_declaration(V, CodeBlob&) { + // it can not appear as a standalone expression + // `var ... = rhs` is handled by ast_assign + tolk_assert(false); +} + +static std::vector process_underscore(V v, CodeBlob& code) { + // when _ is used as left side of assignment, like `(cs, _) = cs.loadAndReturn()` + return code.create_tmp_var(v->inferred_type, v->loc, "(underscore)"); +} + +std::vector pre_compile_expr(AnyExprV v, CodeBlob& code, TypePtr target_type, LValContext* lval_ctx) { + switch (v->type) { + case ast_reference: + return process_reference(v->as(), code, target_type, lval_ctx); + case ast_assign: + return process_assignment(v->as(), code, target_type); + case ast_set_assign: + return process_set_assign(v->as(), code, target_type); + case ast_binary_operator: + return process_binary_operator(v->as(), code, target_type); + case ast_unary_operator: + return process_unary_operator(v->as(), code, target_type); + case ast_ternary_operator: + return process_ternary_operator(v->as(), code, target_type); + case ast_cast_as_operator: + return process_cast_as_operator(v->as(), code, target_type, lval_ctx); + case ast_not_null_operator: + return process_not_null_operator(v->as(), code, target_type, lval_ctx); + case ast_is_null_check: + return process_is_null_check(v->as(), code, target_type); + case ast_dot_access: + return process_dot_access(v->as(), code, target_type, lval_ctx); + case ast_function_call: + return process_function_call(v->as(), code, target_type); + case ast_parenthesized_expression: + return pre_compile_expr(v->as()->get_expr(), code, target_type, lval_ctx); + case ast_tensor: + return process_tensor(v->as(), code, target_type, lval_ctx); + case ast_typed_tuple: + return process_typed_tuple(v->as(), code, target_type, lval_ctx); + case ast_int_const: + return process_int_const(v->as(), code, target_type); + case ast_string_const: + return process_string_const(v->as(), code, target_type); + case ast_bool_const: + return process_bool_const(v->as(), code, target_type); + case ast_null_keyword: + return process_null_keyword(v->as(), code, target_type); + case ast_local_var_lhs: + return process_local_var(v->as(), code, target_type); + case ast_local_vars_declaration: + return process_local_vars_declaration(v->as(), code); + case ast_underscore: + return process_underscore(v->as(), code); + default: + throw UnexpectedASTNodeType(v, "pre_compile_expr"); + } +} + + +static void process_sequence(V v, CodeBlob& code) { + for (AnyV item : v->get_items()) { + process_any_statement(item, code); + } +} + +static void process_assert_statement(V v, CodeBlob& code) { + std::vector args(3); + if (auto v_not = v->get_cond()->try_as(); v_not && v_not->tok == tok_logical_not) { + args[0] = v->get_thrown_code(); + args[1] = v->get_cond()->as()->get_rhs(); + args[2] = createV(v->loc, true); + args[2]->mutate()->assign_inferred_type(TypeDataInt::create()); + } else { + args[0] = v->get_thrown_code(); + args[1] = v->get_cond(); + args[2] = createV(v->loc, false); + args[2]->mutate()->assign_inferred_type(TypeDataInt::create()); + } + + FunctionPtr builtin_sym = lookup_global_symbol("__throw_if_unless")->try_as(); + std::vector args_vars = pre_compile_tensor(code, args); + gen_op_call(code, TypeDataVoid::create(), v->loc, std::move(args_vars), builtin_sym, "(throw-call)"); +} + +static void process_catch_variable(AnyExprV v_catch_var, CodeBlob& code) { + if (auto v_ref = v_catch_var->try_as(); v_ref && v_ref->sym) { // not underscore + LocalVarPtr var_ref = v_ref->sym->try_as(); + tolk_assert(var_ref->ir_idx.empty()); + var_ref->mutate()->assign_ir_idx(code.create_var(v_catch_var->inferred_type, v_catch_var->loc, var_ref->name)); + } +} + +static void process_try_catch_statement(V v, CodeBlob& code) { + code.require_callxargs = true; + Op& try_catch_op = code.emplace_back(v->loc, Op::_TryCatch); + code.push_set_cur(try_catch_op.block0); + process_any_statement(v->get_try_body(), code); + code.close_pop_cur(v->get_try_body()->loc_end); + code.push_set_cur(try_catch_op.block1); + + // transform catch (excNo, arg) into TVM-catch (arg, excNo), where arg is untyped and thus almost useless now + const std::vector& catch_vars = v->get_catch_expr()->get_items(); + tolk_assert(catch_vars.size() == 2); + process_catch_variable(catch_vars[0], code); + process_catch_variable(catch_vars[1], code); + try_catch_op.left = pre_compile_tensor(code, {catch_vars[1], catch_vars[0]}); + process_any_statement(v->get_catch_body(), code); + code.close_pop_cur(v->get_catch_body()->loc_end); +} + +static void process_repeat_statement(V v, CodeBlob& code) { + std::vector tmp_vars = pre_compile_expr(v->get_cond(), code, nullptr); + Op& repeat_op = code.emplace_back(v->loc, Op::_Repeat, tmp_vars); + code.push_set_cur(repeat_op.block0); + process_any_statement(v->get_body(), code); + code.close_pop_cur(v->get_body()->loc_end); +} + +static void process_if_statement(V v, CodeBlob& code) { + std::vector cond = pre_compile_expr(v->get_cond(), code, nullptr); + tolk_assert(cond.size() == 1); + + if (v->get_cond()->is_always_true) { + process_any_statement(v->get_if_body(), code); // v->is_ifnot does not matter here + return; + } + if (v->get_cond()->is_always_false) { + process_any_statement(v->get_else_body(), code); + return; + } + + Op& if_op = code.emplace_back(v->loc, Op::_If, std::move(cond)); + code.push_set_cur(if_op.block0); + process_any_statement(v->get_if_body(), code); + code.close_pop_cur(v->get_if_body()->loc_end); + code.push_set_cur(if_op.block1); + process_any_statement(v->get_else_body(), code); + code.close_pop_cur(v->get_else_body()->loc_end); + if (v->is_ifnot) { + std::swap(if_op.block0, if_op.block1); + } +} + +static void process_do_while_statement(V v, CodeBlob& code) { + Op& until_op = code.emplace_back(v->loc, Op::_Until); + code.push_set_cur(until_op.block0); + process_any_statement(v->get_body(), code); + + // in TVM, there is only "do until", but in Tolk, we want "do while" + // here we negate condition to pass it forward to legacy to Op::_Until + // also, handle common situations as a hardcoded "optimization": replace (a<0) with (a>=0) and so on + // todo these hardcoded conditions should be removed from this place in the future + AnyExprV cond = v->get_cond(); + AnyExprV until_cond; + if (auto v_not = cond->try_as(); v_not && v_not->tok == tok_logical_not) { + until_cond = v_not->get_rhs(); + } else if (auto v_eq = cond->try_as(); v_eq && v_eq->tok == tok_eq) { + until_cond = createV(cond->loc, "!=", tok_neq, v_eq->get_lhs(), v_eq->get_rhs()); + } else if (auto v_neq = cond->try_as(); v_neq && v_neq->tok == tok_neq) { + until_cond = createV(cond->loc, "==", tok_eq, v_neq->get_lhs(), v_neq->get_rhs()); + } else if (auto v_leq = cond->try_as(); v_leq && v_leq->tok == tok_leq) { + until_cond = createV(cond->loc, ">", tok_gt, v_leq->get_lhs(), v_leq->get_rhs()); + } else if (auto v_lt = cond->try_as(); v_lt && v_lt->tok == tok_lt) { + until_cond = createV(cond->loc, ">=", tok_geq, v_lt->get_lhs(), v_lt->get_rhs()); + } else if (auto v_geq = cond->try_as(); v_geq && v_geq->tok == tok_geq) { + until_cond = createV(cond->loc, "<", tok_lt, v_geq->get_lhs(), v_geq->get_rhs()); + } else if (auto v_gt = cond->try_as(); v_gt && v_gt->tok == tok_gt) { + until_cond = createV(cond->loc, "<=", tok_geq, v_gt->get_lhs(), v_gt->get_rhs()); + } else if (cond->inferred_type == TypeDataBool::create()) { + until_cond = createV(cond->loc, "!b", tok_logical_not, cond); + } else { + until_cond = createV(cond->loc, "!", tok_logical_not, cond); + } + until_cond->mutate()->assign_inferred_type(TypeDataInt::create()); + if (auto v_bin = until_cond->try_as(); v_bin && !v_bin->fun_ref) { + v_bin->mutate()->assign_fun_ref(lookup_global_symbol("_" + static_cast(v_bin->operator_name) + "_")->try_as()); + } else if (auto v_un = until_cond->try_as(); v_un && !v_un->fun_ref) { + v_un->mutate()->assign_fun_ref(lookup_global_symbol(static_cast(v_un->operator_name) + "_")->try_as()); + } + + until_op.left = pre_compile_expr(until_cond, code, nullptr); + tolk_assert(until_op.left.size() == 1); + code.close_pop_cur(v->get_body()->loc_end); +} + +static void process_while_statement(V v, CodeBlob& code) { + Op& while_op = code.emplace_back(v->loc, Op::_While); + code.push_set_cur(while_op.block0); + while_op.left = pre_compile_expr(v->get_cond(), code, nullptr); + tolk_assert(while_op.left.size() == 1); + code.close_pop_cur(v->get_body()->loc); + code.push_set_cur(while_op.block1); + process_any_statement(v->get_body(), code); + code.close_pop_cur(v->get_body()->loc_end); +} + +static void process_throw_statement(V v, CodeBlob& code) { + if (v->has_thrown_arg()) { + FunctionPtr builtin_sym = lookup_global_symbol("__throw_arg")->try_as(); + std::vector args_vars = pre_compile_tensor(code, {v->get_thrown_arg(), v->get_thrown_code()}); + gen_op_call(code, TypeDataVoid::create(), v->loc, std::move(args_vars), builtin_sym, "(throw-call)"); + } else { + FunctionPtr builtin_sym = lookup_global_symbol("__throw")->try_as(); + std::vector args_vars = pre_compile_tensor(code, {v->get_thrown_code()}); + gen_op_call(code, TypeDataVoid::create(), v->loc, std::move(args_vars), builtin_sym, "(throw-call)"); + } +} + +static void process_return_statement(V v, CodeBlob& code) { + std::vector return_vars; + if (v->has_return_value()) { + TypePtr child_target_type = code.fun_ref->inferred_return_type; + if (code.fun_ref->does_return_self()) { + child_target_type = code.fun_ref->parameters[0].declared_type; + } + return_vars = pre_compile_expr(v->get_return_value(), code, child_target_type); + } + if (code.fun_ref->does_return_self()) { + return_vars = {}; + } + if (code.fun_ref->has_mutate_params()) { + std::vector mutated_vars; + for (const LocalVarData& p_sym: code.fun_ref->parameters) { + if (p_sym.is_mutate_parameter()) { + mutated_vars.insert(mutated_vars.end(), p_sym.ir_idx.begin(), p_sym.ir_idx.end()); + } + } + return_vars.insert(return_vars.begin(), mutated_vars.begin(), mutated_vars.end()); + } + code.emplace_back(v->loc, Op::_Return, std::move(return_vars)); +} + +// append "return" (void) to the end of the function +// if it's not reachable, it will be dropped +// (IR cfg reachability may differ from FlowContext in case of "never" types, so there may be situations, +// when IR will consider this "return" reachable and leave it, but actually execution will never reach it) +static void append_implicit_return_statement(SrcLocation loc_end, CodeBlob& code) { + std::vector mutated_vars; + if (code.fun_ref->has_mutate_params()) { + for (const LocalVarData& p_sym: code.fun_ref->parameters) { + if (p_sym.is_mutate_parameter()) { + mutated_vars.insert(mutated_vars.end(), p_sym.ir_idx.begin(), p_sym.ir_idx.end()); + } + } + } + code.emplace_back(loc_end, Op::_Return, std::move(mutated_vars)); +} + + +void process_any_statement(AnyV v, CodeBlob& code) { + switch (v->type) { + case ast_sequence: + return process_sequence(v->as(), code); + case ast_return_statement: + return process_return_statement(v->as(), code); + case ast_repeat_statement: + return process_repeat_statement(v->as(), code); + case ast_if_statement: + return process_if_statement(v->as(), code); + case ast_do_while_statement: + return process_do_while_statement(v->as(), code); + case ast_while_statement: + return process_while_statement(v->as(), code); + case ast_throw_statement: + return process_throw_statement(v->as(), code); + case ast_assert_statement: + return process_assert_statement(v->as(), code); + case ast_try_catch_statement: + return process_try_catch_statement(v->as(), code); + case ast_empty_statement: + return; + default: + pre_compile_expr(reinterpret_cast(v), code, nullptr); + } +} + +static void convert_function_body_to_CodeBlob(FunctionPtr fun_ref, FunctionBodyCode* code_body) { + auto v_body = fun_ref->ast_root->as()->get_body()->as(); + CodeBlob* blob = new CodeBlob{fun_ref->name, fun_ref->loc, fun_ref}; + + std::vector rvect_import; + int total_arg_width = 0; + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + total_arg_width += fun_ref->parameters[i].declared_type->get_width_on_stack(); + } + rvect_import.reserve(total_arg_width); + + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + const LocalVarData& param_i = fun_ref->parameters[i]; + std::vector ir_idx = blob->create_var(param_i.declared_type, param_i.loc, param_i.name); + rvect_import.insert(rvect_import.end(), ir_idx.begin(), ir_idx.end()); + param_i.mutate()->assign_ir_idx(std::move(ir_idx)); + } + blob->emplace_back(fun_ref->loc, Op::_Import, rvect_import); + blob->in_var_cnt = blob->var_cnt; + tolk_assert(blob->var_cnt == total_arg_width); + + for (AnyV item : v_body->get_items()) { + process_any_statement(item, *blob); + } + append_implicit_return_statement(v_body->loc_end, *blob); + + blob->close_blk(v_body->loc_end); + code_body->set_code(blob); + tolk_assert(vars_modification_watcher.empty()); +} + +static void convert_asm_body_to_AsmOp(FunctionPtr fun_ref, FunctionBodyAsm* asm_body) { + int cnt = fun_ref->get_num_params(); + int width = fun_ref->inferred_return_type->get_width_on_stack(); + std::vector asm_ops; + for (AnyV v_child : fun_ref->ast_root->as()->get_body()->as()->get_asm_commands()) { + std::string_view ops = v_child->as()->str_val; // \n\n... + std::string op; + for (char c : ops) { + if (c == '\n' || c == '\r') { + if (!op.empty()) { + asm_ops.push_back(AsmOp::Parse(op, cnt, width)); + if (asm_ops.back().is_custom()) { + cnt = width; + } + op.clear(); + } + } else { + op.push_back(c); + } + } + if (!op.empty()) { + asm_ops.push_back(AsmOp::Parse(op, cnt, width)); + if (asm_ops.back().is_custom()) { + cnt = width; + } + } + } + + asm_body->set_code(std::move(asm_ops)); +} + +class UpdateArgRetOrderConsideringStackWidth final { +public: + static bool should_visit_function(FunctionPtr fun_ref) { + return !fun_ref->is_generic_function() && (!fun_ref->ret_order.empty() || !fun_ref->arg_order.empty()); + } + + static void start_visiting_function(FunctionPtr fun_ref, V v_function) { + int total_arg_mutate_width = 0; + bool has_arg_width_not_1 = false; + for (const LocalVarData& param : fun_ref->parameters) { + int arg_width = param.declared_type->get_width_on_stack(); + has_arg_width_not_1 |= arg_width != 1; + total_arg_mutate_width += param.is_mutate_parameter() * arg_width; + } + + // example: `fun f(a: int, b: (int, (int, int)), c: int)` with `asm (b a c)` + // current arg_order is [1 0 2] + // needs to be converted to [1 2 3 0 4] because b width is 3 + if (has_arg_width_not_1) { + int total_arg_width = 0; + std::vector cum_arg_width; + cum_arg_width.reserve(1 + fun_ref->get_num_params()); + cum_arg_width.push_back(0); + for (const LocalVarData& param : fun_ref->parameters) { + cum_arg_width.push_back(total_arg_width += param.declared_type->get_width_on_stack()); + } + std::vector arg_order; + for (int i = 0; i < fun_ref->get_num_params(); ++i) { + int j = fun_ref->arg_order[i]; + int c1 = cum_arg_width[j], c2 = cum_arg_width[j + 1]; + while (c1 < c2) { + arg_order.push_back(c1++); + } + } + fun_ref->mutate()->assign_arg_order(std::move(arg_order)); + } + + // example: `fun f(mutate self: slice): slice` with `asm(-> 1 0)` + // ret_order is a shuffled range 0...N + // validate N: a function should return value and mutated arguments onto a stack + if (!fun_ref->ret_order.empty()) { + size_t expected_width = fun_ref->inferred_return_type->get_width_on_stack() + total_arg_mutate_width; + if (expected_width != fun_ref->ret_order.size()) { + v_function->get_body()->error("ret_order (after ->) expected to contain " + std::to_string(expected_width) + " numbers"); + } + } + } +}; + +class ConvertASTToLegacyOpVisitor final { +public: + static bool should_visit_function(FunctionPtr fun_ref) { + return !fun_ref->is_generic_function(); + } + + static void start_visiting_function(FunctionPtr fun_ref, V) { + tolk_assert(fun_ref->is_type_inferring_done()); + if (fun_ref->is_code_function()) { + convert_function_body_to_CodeBlob(fun_ref, std::get(fun_ref->body)); + } else if (fun_ref->is_asm_function()) { + convert_asm_body_to_AsmOp(fun_ref, std::get(fun_ref->body)); + } + } +}; + +void pipeline_convert_ast_to_legacy_Expr_Op() { + visit_ast_of_all_functions(); + visit_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipe-calc-rvalue-lvalue.cpp b/tolk/pipe-calc-rvalue-lvalue.cpp new file mode 100644 index 00000000..1f374bc8 --- /dev/null +++ b/tolk/pipe-calc-rvalue-lvalue.cpp @@ -0,0 +1,229 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-visitor.h" + +/* + * This pipe assigns lvalue/rvalue flags for AST expressions. + * It happens after identifiers have been resolved, but before type inferring (before methods binding). + * + * Example: `a = b`, `a` is lvalue, `b` is rvalue. + * Example: `a + b`, both are rvalue. + * + * Note, that this pass only assigns, not checks. So, for `f() = 4`, expr `f()` is lvalue. + * Checking (firing this as incorrect later) is performed after type inferring, see pipe-check-rvalue-lvalue. + */ + +namespace tolk { + +enum class MarkingState { + None, + LValue, + RValue, + LValueAndRValue +}; + +class CalculateRvalueLvalueVisitor final : public ASTVisitorFunctionBody { + MarkingState cur_state = MarkingState::None; + + MarkingState enter_state(MarkingState activated) { + MarkingState saved = cur_state; + cur_state = activated; + return saved; + } + + void restore_state(MarkingState saved) { + cur_state = saved; + } + + void mark_vertex_cur_or_rvalue(AnyExprV v) const { + if (cur_state == MarkingState::LValue || cur_state == MarkingState::LValueAndRValue) { + v->mutate()->assign_lvalue_true(); + } + if (cur_state == MarkingState::RValue || cur_state == MarkingState::LValueAndRValue || cur_state == MarkingState::None) { + v->mutate()->assign_rvalue_true(); + } + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + parent::visit(v); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + parent::visit(v); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + parent::visit(v); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + MarkingState saved = enter_state(v->passed_as_mutate ? MarkingState::LValueAndRValue : MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + parent::visit(v); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v->get_obj()); + restore_state(saved); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + // underscore is a placeholder to ignore left side of assignment: `(a, _) = get2params()` + // so, if current state is "lvalue", `_` will be marked as lvalue, and ok + // but if used incorrectly, like `f(_)` or just `_;`, it will be marked rvalue + // and will fire an error later, in pipe lvalue/rvalue check + mark_vertex_cur_or_rvalue(v); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + MarkingState saved = enter_state(MarkingState::LValue); + parent::visit(v->get_lhs()); + enter_state(MarkingState::RValue); + parent::visit(v->get_rhs()); + restore_state(saved); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + MarkingState saved = enter_state(MarkingState::LValueAndRValue); + parent::visit(v->get_lhs()); + enter_state(MarkingState::RValue); + parent::visit(v->get_rhs()); + restore_state(saved); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); + restore_state(saved); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v); // both cond, when_true and when_false are rvalue, `(cond ? a : b) = 5` prohibited + restore_state(saved); + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + parent::visit(v->get_expr()); // leave lvalue state unchanged, for `mutate (t.0 as int)` both `t.0 as int` and `t.0` are lvalue + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + parent::visit(v->get_expr()); // leave lvalue state unchanged, for `mutate x!` both `x!` and `x` are lvalue + } + + void visit(V v) override { + mark_vertex_cur_or_rvalue(v); + MarkingState saved = enter_state(MarkingState::RValue); + parent::visit(v->get_expr()); + restore_state(saved); + } + + void visit(V v) override { + tolk_assert(cur_state == MarkingState::LValue); + mark_vertex_cur_or_rvalue(v); + parent::visit(v); + } + + void visit(V v) override { + tolk_assert(cur_state == MarkingState::LValue); + mark_vertex_cur_or_rvalue(v); + parent::visit(v); + } + + void visit(V v) override { + parent::visit(v->get_try_body()); + MarkingState saved = enter_state(MarkingState::LValue); + parent::visit(v->get_catch_expr()); + restore_state(saved); + parent::visit(v->get_catch_body()); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } +}; + +void pipeline_calculate_rvalue_lvalue() { + visit_ast_of_all_functions(); +} + +void pipeline_calculate_rvalue_lvalue(FunctionPtr fun_ref) { + CalculateRvalueLvalueVisitor visitor; + if (visitor.should_visit_function(fun_ref)) { + visitor.start_visiting_function(fun_ref, fun_ref->ast_root->as()); + } +} + +} // namespace tolk diff --git a/tolk/pipe-check-inferred-types.cpp b/tolk/pipe-check-inferred-types.cpp new file mode 100644 index 00000000..bae67c5f --- /dev/null +++ b/tolk/pipe-check-inferred-types.cpp @@ -0,0 +1,586 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-visitor.h" +#include "type-system.h" + +namespace tolk { + +GNU_ATTRIBUTE_NOINLINE +static std::string to_string(TypePtr type) { + return "`" + type->as_human_readable() + "`"; +} + +GNU_ATTRIBUTE_NOINLINE +static std::string to_string(AnyExprV v_with_type) { + return "`" + v_with_type->inferred_type->as_human_readable() + "`"; +} + +GNU_ATTRIBUTE_NOINLINE +static std::string expression_as_string(AnyExprV v) { + if (auto v_ref = v->try_as()) { + if (v_ref->sym->try_as() || v_ref->sym->try_as()) { + return "variable `" + static_cast(v_ref->get_identifier()->name) + "`"; + } + } + if (auto v_par = v->try_as()) { + return expression_as_string(v_par->get_expr()); + } + return "expression"; +} + +// fire a general "type mismatch" error, just a wrapper over `throw` +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire(FunctionPtr cur_f, SrcLocation loc, const std::string& message) { + throw ParseError(cur_f, loc, message); +} + +// fire an error on `!cell` / `+slice` +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_cannot_apply_operator(FunctionPtr cur_f, SrcLocation loc, std::string_view operator_name, AnyExprV unary_expr) { + std::string op = static_cast(operator_name); + fire(cur_f, loc, "can not apply operator `" + op + "` to " + to_string(unary_expr->inferred_type)); +} + +// fire an error on `int + cell` / `slice & int` +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_cannot_apply_operator(FunctionPtr cur_f, SrcLocation loc, std::string_view operator_name, AnyExprV lhs, AnyExprV rhs) { + std::string op = static_cast(operator_name); + fire(cur_f, loc, "can not apply operator `" + op + "` to " + to_string(lhs->inferred_type) + " and " + to_string(rhs->inferred_type)); +} + +GNU_ATTRIBUTE_NOINLINE +static void warning_condition_always_true_or_false(FunctionPtr cur_f, SrcLocation loc, AnyExprV cond, const char* operator_name) { + loc.show_warning("condition of " + static_cast(operator_name) + " is always " + (cond->is_always_true ? "true" : "false")); +} + +// given `f(x: int)` and a call `f(expr)`, check that expr_type is assignable to `int` +static void check_function_argument_passed(FunctionPtr cur_f, TypePtr param_type, AnyExprV ith_arg, bool is_obj_of_dot_call) { + if (!param_type->can_rhs_be_assigned(ith_arg->inferred_type)) { + if (is_obj_of_dot_call) { + fire(cur_f, ith_arg->loc, "can not call method for " + to_string(param_type) + " with object of type " + to_string(ith_arg)); + } else { + fire(cur_f, ith_arg->loc, "can not pass " + to_string(ith_arg) + " to " + to_string(param_type)); + } + } +} + +// given `f(x: mutate int?)` and a call `f(expr)`, check that `int?` is assignable to expr_type +// (for instance, can't call `f(mutate intVal)`, since f can potentially assign null to it) +static void check_function_argument_mutate_back(FunctionPtr cur_f, TypePtr param_type, AnyExprV ith_arg, bool is_obj_of_dot_call) { + if (!ith_arg->inferred_type->can_rhs_be_assigned(param_type)) { + if (is_obj_of_dot_call) { + fire(cur_f, ith_arg->loc,"can not call method for mutate " + to_string(param_type) + " with object of type " + to_string(ith_arg) + ", because mutation is not type compatible"); + } else { + fire(cur_f, ith_arg->loc,"can not pass " + to_string(ith_arg) + " to mutate " + to_string(param_type) + ", because mutation is not type compatible"); + } + } +} + +// fire an error on `var n = null` +// technically it's correct, type of `n` is TypeDataNullLiteral, but it's not what the user wanted +// so, it's better to see an error on assignment, that later, on `n` usage and types mismatch +// (most common is situation above, but generally, `var (x,n) = xn` where xn is a tensor with 2-nd always-null, can be) +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_assign_always_null_to_variable(FunctionPtr cur_f, SrcLocation loc, LocalVarPtr assigned_var, bool is_assigned_null_literal) { + std::string var_name = assigned_var->name; + fire(cur_f, loc, "can not infer type of `" + var_name + "`, it's always null; specify its type with `" + var_name + ": `" + (is_assigned_null_literal ? " or use `null as `" : "")); +} + +// fire an error on `untypedTupleVar.0` when inferred as (int,int), or `[int, (int,int)]`, or other non-1 width in a tuple +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_cannot_put_non1_stack_width_arg_to_tuple(FunctionPtr cur_f, SrcLocation loc, TypePtr inferred_type) { + fire(cur_f, loc, "a tuple can not have " + to_string(inferred_type) + " inside, because it occupies " + std::to_string(inferred_type->get_width_on_stack()) + " stack slots in TVM, not 1"); +} + +// handle __expect_type(expr, "type") call +// this is used in compiler tests +GNU_ATTRIBUTE_NOINLINE GNU_ATTRIBUTE_COLD +static void handle_possible_compiler_internal_call(FunctionPtr cur_f, V v) { + FunctionPtr fun_ref = v->fun_maybe; + tolk_assert(fun_ref && fun_ref->is_builtin_function()); + + if (fun_ref->name == "__expect_type") { + tolk_assert(v->get_num_args() == 2); + TypePtr expected_type = parse_type_from_string(v->get_arg(1)->get_expr()->as()->str_val); + TypePtr expr_type = v->get_arg(0)->inferred_type; + if (expected_type != expr_type) { + fire(cur_f, v->loc, "__expect_type failed: expected " + to_string(expected_type) + ", got " + to_string(expr_type)); + } + } +} + +static bool expect_integer(AnyExprV v_inferred) { + return v_inferred->inferred_type == TypeDataInt::create(); +} + +static bool expect_boolean(AnyExprV v_inferred) { + return v_inferred->inferred_type == TypeDataBool::create(); +} + + +class CheckInferredTypesVisitor final : public ASTVisitorFunctionBody { + FunctionPtr cur_f = nullptr; // may be nullptr if checking `const a = ...` init_value + +protected: + void visit(V v) override { + AnyExprV lhs = v->get_lhs(); + AnyExprV rhs = v->get_rhs(); + parent::visit(lhs); + parent::visit(rhs); + + // all operators (+=, etc.) can work for integers (if both sides are integers) + bool types_ok = expect_integer(lhs) && expect_integer(rhs); + // bitwise operators &= |= ^= are "overloaded" for booleans also (if both sides are booleans) + if (!types_ok && (v->tok == tok_set_bitwise_and || v->tok == tok_set_bitwise_or || v->tok == tok_set_bitwise_xor)) { + types_ok = expect_boolean(lhs) && expect_boolean(rhs); + } + // using += for other types (e.g. `tensorVar += tensorVar`) is not allowed + if (!types_ok) { + fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); + } + } + + void visit(V v) override { + AnyExprV rhs = v->get_rhs(); + parent::visit(rhs); + + switch (v->tok) { + case tok_logical_not: + if (!expect_integer(rhs) && !expect_boolean(rhs)) { + fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, rhs); + } + break; + default: + if (!expect_integer(rhs)) { + fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, rhs); + } + } + } + + void visit(V v) override { + AnyExprV lhs = v->get_lhs(); + AnyExprV rhs = v->get_rhs(); + parent::visit(lhs); + parent::visit(rhs); + + switch (v->tok) { + // == != can compare both integers and booleans, (int == bool) is NOT allowed + // note, that `int?` and `int?` can't be compared, since Fift `EQUAL` works with integers only + // (if to allow `int?` in the future, `==` must be expressed in a complicated Fift code considering TVM NULL) + case tok_eq: + case tok_neq: { + bool both_int = expect_integer(lhs) && expect_integer(rhs); + bool both_bool = expect_boolean(lhs) && expect_boolean(rhs); + if (!both_int && !both_bool) { + if (lhs->inferred_type == rhs->inferred_type) { // compare slice with slice, int? with int? + fire(cur_f, v->loc, "type " + to_string(lhs) + " can not be compared with `== !=`"); + } else { + fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); + } + } + break; + } + // < > can compare only strict integers + case tok_lt: + case tok_gt: + case tok_leq: + case tok_geq: + case tok_spaceship: + if (!expect_integer(lhs) || !expect_integer(rhs)) { + fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); + } + break; + // & | ^ are "overloaded" both for integers and booleans, (int & bool) is NOT allowed + case tok_bitwise_and: + case tok_bitwise_or: + case tok_bitwise_xor: { + bool both_int = expect_integer(lhs) && expect_integer(rhs); + bool both_bool = expect_boolean(lhs) && expect_boolean(rhs); + if (!both_int && !both_bool) { + fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); + } + break; + } + // && || can work with integers and booleans, (int && bool) is allowed, (int16 && int32) also + case tok_logical_and: + case tok_logical_or: { + bool lhs_ok = expect_integer(lhs) || expect_boolean(lhs); + bool rhs_ok = expect_integer(rhs) || expect_boolean(rhs); + if (!lhs_ok || !rhs_ok) { + fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); + } + break; + } + // others are mathematical: + * ... + default: + if (!expect_integer(lhs) || !expect_integer(rhs)) { + fire_error_cannot_apply_operator(cur_f, v->loc, v->operator_name, lhs, rhs); + } + } + } + + void visit(V v) override { + parent::visit(v->get_expr()); + + if (!v->get_expr()->inferred_type->can_be_casted_with_as_operator(v->cast_to_type)) { + fire(cur_f, v->loc, "type " + to_string(v->get_expr()) + " can not be cast to " + to_string(v->cast_to_type)); + } + } + + void visit(V v) override { + parent::visit(v->get_expr()); + + if (v->get_expr()->inferred_type == TypeDataNullLiteral::create()) { + // operator `!` used for always-null (proven by smart casts, for example), it's an error + fire(cur_f, v->loc, "operator `!` used for always null expression"); + } + // if operator `!` used for non-nullable, probably a warning should be printed + } + + void visit(V v) override { + parent::visit(v->get_expr()); + + if ((v->is_always_true && !v->is_negated) || (v->is_always_false && v->is_negated)) { + v->loc.show_warning(expression_as_string(v->get_expr()) + " is always null, this condition is always " + (v->is_always_true ? "true" : "false")); + } + if ((v->is_always_false && !v->is_negated) || (v->is_always_true && v->is_negated)) { + v->loc.show_warning(expression_as_string(v->get_expr()) + " of type " + to_string(v->get_expr()) + " is always not null, this condition is always " + (v->is_always_true ? "true" : "false")); + } + } + + void visit(V v) override { + parent::visit(v); + + for (int i = 0; i < v->size(); ++i) { + AnyExprV item = v->get_item(i); + if (item->inferred_type->get_width_on_stack() != 1) { + fire_error_cannot_put_non1_stack_width_arg_to_tuple(cur_f, v->get_item(i)->loc, item->inferred_type); + } + } + } + + void visit(V v) override { + parent::visit(v); + + TypePtr obj_type = v->get_obj()->inferred_type; + if (v->is_target_indexed_access()) { + if (obj_type->try_as() && v->inferred_type->get_width_on_stack() != 1) { + fire_error_cannot_put_non1_stack_width_arg_to_tuple(cur_f, v->loc, v->inferred_type); + } + } + } + + void visit(V v) override { + parent::visit(v); // check against type mismatch inside nested arguments + + FunctionPtr fun_ref = v->fun_maybe; + if (!fun_ref) { + // `local_var(args)` and similar + const TypeDataFunCallable* f_callable = v->get_callee()->inferred_type->try_as(); + tolk_assert(f_callable && f_callable->params_size() == v->get_num_args()); + for (int i = 0; i < v->get_num_args(); ++i) { + auto arg_i = v->get_arg(i)->get_expr(); + TypePtr param_type = f_callable->params_types[i]; + if (!param_type->can_rhs_be_assigned(arg_i->inferred_type)) { + fire(cur_f, arg_i->loc, "can not pass " + to_string(arg_i) + " to " + to_string(param_type)); + } + } + return; + } + + // so, we have a call `f(args)` or `obj.f(args)`, f is a global function (fun_ref) (code / asm / builtin) + int delta_self = 0; + AnyExprV dot_obj = nullptr; + if (auto v_dot = v->get_callee()->try_as()) { + delta_self = 1; + dot_obj = v_dot->get_obj(); + } + + if (dot_obj) { + const LocalVarData& param_0 = fun_ref->parameters[0]; + TypePtr param_type = param_0.declared_type; + check_function_argument_passed(cur_f, param_type, dot_obj, true); + if (param_0.is_mutate_parameter()) { + check_function_argument_mutate_back(cur_f, param_type, dot_obj, true); + } + } + for (int i = 0; i < v->get_num_args(); ++i) { + const LocalVarData& param_i = fun_ref->parameters[delta_self + i]; + AnyExprV arg_i = v->get_arg(i)->get_expr(); + TypePtr param_type = param_i.declared_type; + check_function_argument_passed(cur_f, param_type, arg_i, false); + if (param_i.is_mutate_parameter()) { + check_function_argument_mutate_back(cur_f, param_type, arg_i, false); + } + } + + if (fun_ref->is_builtin_function() && fun_ref->name[0] == '_') { + handle_possible_compiler_internal_call(cur_f, v); + } + } + + void visit(V v) override { + parent::visit(v->get_lhs()); + parent::visit(v->get_rhs()); + + process_assignment_lhs(v->get_lhs(), v->get_rhs()->inferred_type, v->get_rhs()); + } + + // handle (and dig recursively) into `var lhs = rhs` + // examples: `var z = 5`, `var (x, [y]) = (2, [3])`, `var (x, [y]) = xy` + // while recursing, keep track of rhs if lhs and rhs have common shape (5 for z, 2 for x, [3] for [y], 3 for y) + // (so that on type mismatch, point to corresponding rhs, example: `var (x, y:slice) = (1, 2)` point to 2 + void process_assignment_lhs(AnyExprV lhs, TypePtr rhs_type, AnyExprV corresponding_maybe_rhs) { + AnyExprV err_loc = corresponding_maybe_rhs ? corresponding_maybe_rhs : lhs; + + // `var ... = rhs` - dig into left part + if (auto lhs_decl = lhs->try_as()) { + process_assignment_lhs(lhs_decl->get_expr(), rhs_type, corresponding_maybe_rhs); + return; + } + + // inside `var v: int = rhs` / `var _ = rhs` / `var v redef = rhs` (lhs is "v" / "_" / "v") + if (auto lhs_var = lhs->try_as()) { + TypePtr declared_type = lhs_var->declared_type; // `var v: int = rhs` (otherwise, nullptr) + if (lhs_var->marked_as_redef) { + tolk_assert(lhs_var->var_ref && lhs_var->var_ref->declared_type); + declared_type = lhs_var->var_ref->declared_type; + } + if (declared_type) { + if (!declared_type->can_rhs_be_assigned(rhs_type)) { + fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to variable of type " + to_string(declared_type)); + } + } else { + if (rhs_type == TypeDataNullLiteral::create()) { + fire_error_assign_always_null_to_variable(cur_f, err_loc->loc, lhs_var->var_ref->try_as(), corresponding_maybe_rhs && corresponding_maybe_rhs->type == ast_null_keyword); + } + } + return; + } + + // `(v1, v2) = rhs` / `var (v1, v2) = rhs` (rhs may be `(1,2)` or `tensorVar` or `someF()`, doesn't matter) + // dig recursively into v1 and v2 with corresponding rhs i-th item of a tensor + if (auto lhs_tensor = lhs->try_as()) { + const TypeDataTensor* rhs_type_tensor = rhs_type->try_as(); + if (!rhs_type_tensor) { + fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to a tensor"); + } + if (lhs_tensor->size() != rhs_type_tensor->size()) { + fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + ", sizes mismatch"); + } + V rhs_tensor_maybe = corresponding_maybe_rhs ? corresponding_maybe_rhs->try_as() : nullptr; + for (int i = 0; i < lhs_tensor->size(); ++i) { + process_assignment_lhs(lhs_tensor->get_item(i), rhs_type_tensor->items[i], rhs_tensor_maybe ? rhs_tensor_maybe->get_item(i) : nullptr); + } + return; + } + + // `[v1, v2] = rhs` / `var [v1, v2] = rhs` (rhs may be `[1,2]` or `tupleVar` or `someF()`, doesn't matter) + // dig recursively into v1 and v2 with corresponding rhs i-th item of a tuple + if (auto lhs_tuple = lhs->try_as()) { + const TypeDataTypedTuple* rhs_type_tuple = rhs_type->try_as(); + if (!rhs_type_tuple) { + fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to a tuple"); + } + if (lhs_tuple->size() != rhs_type_tuple->size()) { + fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + ", sizes mismatch"); + } + V rhs_tuple_maybe = corresponding_maybe_rhs ? corresponding_maybe_rhs->try_as() : nullptr; + for (int i = 0; i < lhs_tuple->size(); ++i) { + process_assignment_lhs(lhs_tuple->get_item(i), rhs_type_tuple->items[i], rhs_tuple_maybe ? rhs_tuple_maybe->get_item(i) : nullptr); + } + return; + } + + // check `untypedTuple.0 = rhs_tensor` and other non-1 width elements + if (auto lhs_dot = lhs->try_as()) { + if (lhs_dot->is_target_indexed_access() && lhs_dot->get_obj()->inferred_type == TypeDataTuple::create()) { + if (rhs_type->get_width_on_stack() != 1) { + fire_error_cannot_put_non1_stack_width_arg_to_tuple(cur_f, err_loc->loc, rhs_type); + } + } + } + + // here is `v = rhs` (just assignment, not `var v = rhs`) / `a.0 = rhs` / `getObj(z=f()).0 = rhs` etc. + // types were already inferred, so just check their compatibility + // for strange lhs like `f() = rhs` type checking will pass, but will fail lvalue check later + if (!lhs->inferred_type->can_rhs_be_assigned(rhs_type)) { + if (lhs->try_as()) { + fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to variable of type " + to_string(lhs)); + } else { + fire(cur_f, err_loc->loc, "can not assign " + to_string(rhs_type) + " to " + to_string(lhs)); + } + } + } + + void visit(V v) override { + parent::visit(v->get_return_value()); + + if (cur_f->does_return_self()) { + if (!is_expr_valid_as_return_self(v->get_return_value())) { + fire(cur_f, v->loc, "invalid return from `self` function"); + } + return; + } + + TypePtr expr_type = v->get_return_value()->inferred_type; + if (!cur_f->inferred_return_type->can_rhs_be_assigned(expr_type)) { + fire(cur_f, v->get_return_value()->loc, "can not convert type " + to_string(expr_type) + " to return type " + to_string(cur_f->inferred_return_type)); + } + } + + static bool is_expr_valid_as_return_self(AnyExprV return_expr) { + // `return self` + if (return_expr->type == ast_reference && return_expr->as()->get_name() == "self") { + return true; + } + // `return self.someMethod()` + if (auto v_call = return_expr->try_as(); v_call && v_call->is_dot_call()) { + return v_call->fun_maybe && v_call->fun_maybe->does_return_self() && is_expr_valid_as_return_self(v_call->get_dot_obj()); + } + // `return cond ? ... : ...` + if (auto v_ternary = return_expr->try_as()) { + return is_expr_valid_as_return_self(v_ternary->get_when_true()) && is_expr_valid_as_return_self(v_ternary->get_when_false()); + } + return false; + } + + void visit(V v) override { + parent::visit(v); + + AnyExprV cond = v->get_cond(); + if (!expect_integer(cond) && !expect_boolean(cond)) { + fire(cur_f, cond->loc, "can not use " + to_string(cond) + " as a boolean condition"); + } + + if (cond->is_always_true || cond->is_always_false) { + warning_condition_always_true_or_false(cur_f, v->loc, cond, "ternary operator"); + } + } + + void visit(V v) override { + parent::visit(v); + + AnyExprV cond = v->get_cond(); + if (!expect_integer(cond) && !expect_boolean(cond)) { + fire(cur_f, cond->loc, "can not use " + to_string(cond) + " as a boolean condition"); + } + + if (cond->is_always_true || cond->is_always_false) { + warning_condition_always_true_or_false(cur_f, v->loc, cond, "`if`"); + } + } + + void visit(V v) override { + parent::visit(v); + + AnyExprV cond = v->get_cond(); + if (!expect_integer(cond)) { + fire(cur_f, cond->loc, "condition of `repeat` must be an integer, got " + to_string(cond)); + } + } + + void visit(V v) override { + parent::visit(v); + + AnyExprV cond = v->get_cond(); + if (!expect_integer(cond) && !expect_boolean(cond)) { + fire(cur_f, cond->loc, "can not use " + to_string(cond) + " as a boolean condition"); + } + + if (cond->is_always_true || cond->is_always_false) { + warning_condition_always_true_or_false(cur_f, v->loc, cond, "`while`"); + } + } + + void visit(V v) override { + parent::visit(v); + + AnyExprV cond = v->get_cond(); + if (!expect_integer(cond) && !expect_boolean(cond)) { + fire(cur_f, cond->loc, "can not use " + to_string(cond) + " as a boolean condition"); + } + + if (cond->is_always_true || cond->is_always_false) { + warning_condition_always_true_or_false(cur_f, v->loc, cond, "`do while`"); + } + } + + void visit(V v) override { + parent::visit(v); + + if (!expect_integer(v->get_thrown_code())) { + fire(cur_f, v->get_thrown_code()->loc, "excNo of `throw` must be an integer, got " + to_string(v->get_thrown_code())); + } + if (v->has_thrown_arg() && v->get_thrown_arg()->inferred_type->get_width_on_stack() != 1) { + fire(cur_f, v->get_thrown_arg()->loc, "can not throw " + to_string(v->get_thrown_arg()) + ", exception arg must occupy exactly 1 stack slot"); + } + } + + void visit(V v) override { + parent::visit(v); + + AnyExprV cond = v->get_cond(); + if (!expect_integer(cond) && !expect_boolean(cond)) { + fire(cur_f, cond->loc, "can not use " + to_string(cond) + " as a boolean condition"); + } + if (!expect_integer(v->get_thrown_code())) { + fire(cur_f, v->get_thrown_code()->loc, "thrown excNo of `assert` must be an integer, got " + to_string(v->get_thrown_code())); + } + + if (cond->is_always_true || cond->is_always_false) { + warning_condition_always_true_or_false(cur_f, v->loc, cond, "`assert`"); + } + } + + void visit(V v) override { + parent::visit(v); + + if (v->first_unreachable) { + // it's essential to print "unreachable code" warning AFTER type checking + // (printing it while inferring might be a false positive if types are incorrect, due to smart casts for example) + // a more correct approach would be to access cfg here somehow, but since cfg is now available only while inferring, + // a special v->first_unreachable was set specifically for this warning (again, which is correct if types match) + v->first_unreachable->loc.show_warning("unreachable code"); + } + } + + public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) override { + cur_f = fun_ref; + parent::visit(v_function->get_body()); + cur_f = nullptr; + + if (fun_ref->is_implicit_return() && fun_ref->declared_return_type) { + if (!fun_ref->declared_return_type->can_rhs_be_assigned(TypeDataVoid::create()) || fun_ref->does_return_self()) { + fire(fun_ref, v_function->get_body()->as()->loc_end, "missing return"); + } + } + } +}; + +void pipeline_check_inferred_types() { + visit_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipe-check-pure-impure.cpp b/tolk/pipe-check-pure-impure.cpp new file mode 100644 index 00000000..366ff160 --- /dev/null +++ b/tolk/pipe-check-pure-impure.cpp @@ -0,0 +1,93 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-visitor.h" +#include "platform-utils.h" + +/* + * This pipe checks for impure operations inside pure functions. + * It happens after type inferring (after methods binding) since it operates fun_ref of calls. + */ + +namespace tolk { + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_impure_operation_inside_pure_function(AnyV v) { + v->error("an impure operation in a pure function"); +} + +class CheckImpureOperationsInPureFunctionVisitor final : public ASTVisitorFunctionBody { + static void fire_if_global_var(AnyExprV v) { + if (auto v_ident = v->try_as()) { + if (v_ident->sym->try_as()) { + fire_error_impure_operation_inside_pure_function(v); + } + } + } + + void visit(V v) override { + fire_if_global_var(v->get_lhs()); + parent::visit(v); + } + + void visit(V v) override { + fire_if_global_var(v->get_lhs()); + parent::visit(v); + } + + void visit(V v) override { + // v is `globalF(args)` / `globalF(args)` / `obj.method(args)` / `local_var(args)` / `getF()(args)` + if (!v->fun_maybe) { + // `local_var(args)` is always impure, no considerations about what's there at runtime + fire_error_impure_operation_inside_pure_function(v); + } + + if (!v->fun_maybe->is_marked_as_pure()) { + fire_error_impure_operation_inside_pure_function(v); + } + + parent::visit(v); + } + + void visit(V v) override { + if (v->passed_as_mutate) { + fire_if_global_var(v->get_expr()); + } + + parent::visit(v); + } + + void visit(V v) override { + fire_error_impure_operation_inside_pure_function(v); + } + + void visit(V v) override { + fire_error_impure_operation_inside_pure_function(v); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function() && fun_ref->is_marked_as_pure(); + } +}; + +void pipeline_check_pure_impure_operations() { + visit_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipe-check-rvalue-lvalue.cpp b/tolk/pipe-check-rvalue-lvalue.cpp new file mode 100644 index 00000000..3ec47a16 --- /dev/null +++ b/tolk/pipe-check-rvalue-lvalue.cpp @@ -0,0 +1,210 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-visitor.h" +#include "platform-utils.h" + +/* + * This pipe checks lvalue/rvalue for validity. + * It happens after type inferring (after methods binding) and after lvalue/rvalue are refined based on fun_ref. + * + * Example: `f() = 4`, `f()` was earlier marked as lvalue, it's incorrect. + * Example: `f(mutate 5)`, `5` was marked also, it's incorrect. + */ + +namespace tolk { + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_cannot_be_used_as_lvalue(AnyV v, const std::string& details) { + // example: `f() = 32` + // example: `loadUint(c.beginParse(), 32)` (since `loadUint()` mutates the first argument) + v->error(details + " can not be used as lvalue"); +} + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_modifying_immutable_variable(AnyExprV v, LocalVarPtr var_ref) { + if (var_ref->param_idx == 0 && var_ref->name == "self") { + v->error("modifying `self`, which is immutable by default; probably, you want to declare `mutate self`"); + } else { + v->error("modifying immutable variable `" + var_ref->name + "`"); + } +} + +// validate a function used as rvalue, like `var cb = f` +// it's not a generic function (ensured earlier at type inferring) and has some more restrictions +static void validate_function_used_as_noncall(AnyExprV v, FunctionPtr fun_ref) { + if (!fun_ref->arg_order.empty() || !fun_ref->ret_order.empty()) { + v->error("saving `" + fun_ref->name + "` into a variable will most likely lead to invalid usage, since it changes the order of variables on the stack"); + } + if (fun_ref->has_mutate_params()) { + v->error("saving `" + fun_ref->name + "` into a variable is impossible, since it has `mutate` parameters and thus can only be called directly"); + } +} + +class CheckRValueLvalueVisitor final : public ASTVisitorFunctionBody { + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(v, "assignment"); + } + parent::visit(v); + } + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(v, "assignment"); + } + parent::visit(v); + } + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(v, "operator " + static_cast(v->operator_name)); + } + parent::visit(v); + } + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(v, "operator " + static_cast(v->operator_name)); + } + parent::visit(v); + } + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(v, "operator ?:"); + } + parent::visit(v); + } + + void visit(V v) override { + // if `x as int` is lvalue, then `x` is also lvalue, so check that `x` is ok + parent::visit(v->get_expr()); + } + + void visit(V v) override { + // if `x!` is lvalue, then `x` is also lvalue, so check that `x` is ok + parent::visit(v->get_expr()); + } + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(v, v->is_negated ? "operator !=" : "operator =="); + } + parent::visit(v->get_expr()); + } + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(v, "literal"); + } + } + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(v, "literal"); + } + } + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(v, "literal"); + } + } + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(v, "literal"); + } + } + + void visit(V v) override { + // a reference to a method used as rvalue, like `var v = t.tupleAt` + if (v->is_rvalue && v->is_target_fun_ref()) { + validate_function_used_as_noncall(v, std::get(v->target)); + } + } + + void visit(V v) override { + if (v->is_lvalue) { + fire_error_cannot_be_used_as_lvalue(v, "function call"); + } + if (!v->fun_maybe) { + parent::visit(v->get_callee()); + } + // for `f()` don't visit ast_reference `f`, to detect `f` usage as non-call, like `var cb = f` + // same for `obj.method()`, don't visit ast_reference method, visit only obj + if (v->is_dot_call()) { + parent::visit(v->get_dot_obj()); + } + + for (int i = 0; i < v->get_num_args(); ++i) { + parent::visit(v->get_arg(i)); + } + } + + void visit(V v) override { + if (v->marked_as_redef) { + tolk_assert(v->var_ref); + if (v->var_ref->is_immutable()) { + v->error("`redef` for immutable variable"); + } + } + } + + void visit(V v) override { + if (v->is_lvalue) { + tolk_assert(v->sym); + if (LocalVarPtr var_ref = v->sym->try_as(); var_ref && var_ref->is_immutable()) { + fire_error_modifying_immutable_variable(v, var_ref); + } else if (v->sym->try_as()) { + v->error("modifying immutable constant"); + } else if (v->sym->try_as()) { + v->error("function can't be used as lvalue"); + } + } + + // a reference to a function used as rvalue, like `var v = someFunction` + if (FunctionPtr fun_ref = v->sym->try_as(); fun_ref && v->is_rvalue) { + validate_function_used_as_noncall(v, fun_ref); + } + } + + void visit(V v) override { + if (v->is_rvalue) { + v->error("`_` can't be used as a value; it's a placeholder for a left side of assignment"); + } + } + + void visit(V v) override { + parent::visit(v->get_try_body()); + // skip catch(_,excNo), there are always vars due to grammar, lvalue/rvalue aren't set to them + parent::visit(v->get_catch_body()); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } +}; + +void pipeline_check_rvalue_lvalue() { + visit_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipe-constant-folding.cpp b/tolk/pipe-constant-folding.cpp new file mode 100644 index 00000000..9c27029b --- /dev/null +++ b/tolk/pipe-constant-folding.cpp @@ -0,0 +1,112 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-replacer.h" +#include "type-system.h" + +/* + * This pipe is supposed to do constant folding, like replacing `2 + 3` with `5`. + * It happens after type inferring and validity checks, one of the last ones. + * + * Currently, it just replaces `-1` (ast_unary_operator ast_int_const) with a number -1 + * and `!true` with false. + * Also, all parenthesized `((expr))` are replaced with `expr`, it's a constant transformation. + * (not to handle parenthesized in optimization passes, like `((x)) == true`) + * More rich constant folding should be done some day, but even without this, IR optimizations + * (operating low-level stack variables) pretty manage to do all related optimizations. + * Constant folding in the future, done at AST level, just would slightly reduce amount of work for optimizer. + */ + +namespace tolk { + +class ConstantFoldingReplacer final : public ASTReplacerInFunctionBody { + static V create_int_const(SrcLocation loc, td::RefInt256&& intval) { + auto v_int = createV(loc, std::move(intval), {}); + v_int->assign_inferred_type(TypeDataInt::create()); + v_int->assign_rvalue_true(); + return v_int; + } + + static V create_bool_const(SrcLocation loc, bool bool_val) { + auto v_bool = createV(loc, bool_val); + v_bool->assign_inferred_type(TypeDataBool::create()); + v_bool->assign_rvalue_true(); + return v_bool; + } + + AnyExprV replace(V v) override { + AnyExprV inner = parent::replace(v->get_expr()); + if (v->is_lvalue) { + inner->mutate()->assign_lvalue_true(); + } + return inner; + } + + AnyExprV replace(V v) override { + parent::replace(v); + + TokenType t = v->tok; + // convert "-1" (tok_minus tok_int_const) to a const -1 + if (t == tok_minus && v->get_rhs()->type == ast_int_const) { + td::RefInt256 intval = v->get_rhs()->as()->intval; + tolk_assert(!intval.is_null()); + intval = -intval; + if (intval.is_null() || !intval->signed_fits_bits(257)) { + v->error("integer overflow"); + } + return create_int_const(v->loc, std::move(intval)); + } + // same for "+1" + if (t == tok_plus && v->get_rhs()->type == ast_int_const) { + return v->get_rhs(); + } + + // `!true` / `!false` + if (t == tok_logical_not && v->get_rhs()->type == ast_bool_const) { + return create_bool_const(v->loc, !v->get_rhs()->as()->bool_val); + } + // `!0` + if (t == tok_logical_not && v->get_rhs()->type == ast_int_const) { + return create_bool_const(v->loc, v->get_rhs()->as()->intval == 0); + } + + return v; + } + + AnyExprV replace(V v) override { + parent::replace(v); + + // `null == null` / `null != null` + if (v->get_expr()->type == ast_null_keyword) { + return create_bool_const(v->loc, !v->is_negated); + } + + return v; + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } +}; + +void pipeline_constant_folding() { + replace_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipe-discover-parse-sources.cpp b/tolk/pipe-discover-parse-sources.cpp new file mode 100644 index 00000000..d31348ba --- /dev/null +++ b/tolk/pipe-discover-parse-sources.cpp @@ -0,0 +1,70 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-from-tokens.h" +#include "compiler-state.h" + +/* + * This is the starting point of compilation pipeline. + * It parses Tolk files to AST, analyzes `import` statements and loads/parses imported files. + * + * When it finishes, all files have been parsed to AST, and no more files will later be added. + * If a parsing error happens (invalid syntax), an exception is thrown immediately from ast-from-tokens.cpp. + */ + +namespace tolk { + +void pipeline_discover_and_parse_sources(const std::string& stdlib_filename, const std::string& entrypoint_filename) { + G.all_src_files.locate_and_register_source_file(stdlib_filename, {}); + G.all_src_files.locate_and_register_source_file(entrypoint_filename, {}); + + while (SrcFile* file = G.all_src_files.get_next_unparsed_file()) { + tolk_assert(!file->ast); + + file->ast = parse_src_file_to_ast(file); + // if (!file->is_stdlib_file()) file->ast->debug_print(); + + for (AnyV v_toplevel : file->ast->as()->get_toplevel_declarations()) { + if (auto v_import = v_toplevel->try_as()) { + std::string imported_str = v_import->get_file_name(); + size_t cur_slash_pos = file->rel_filename.rfind('/'); + std::string rel_filename = cur_slash_pos == std::string::npos || imported_str[0] == '@' + ? std::move(imported_str) + : file->rel_filename.substr(0, cur_slash_pos + 1) + imported_str; + + const SrcFile* imported = G.all_src_files.locate_and_register_source_file(rel_filename, v_import->loc); + file->imports.push_back(SrcFile::ImportDirective{imported}); + v_import->mutate()->assign_src_file(imported); + } + } + } + + // todo #ifdef TOLK_PROFILING + lexer_measure_performance(G.all_src_files); +} + +} // namespace tolk diff --git a/tolk/pipe-find-unused-symbols.cpp b/tolk/pipe-find-unused-symbols.cpp new file mode 100644 index 00000000..2b7e5557 --- /dev/null +++ b/tolk/pipe-find-unused-symbols.cpp @@ -0,0 +1,76 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#include "tolk.h" +#include "compiler-state.h" + +/* + * This pipe finds unused symbols (global functions and variables) to strip them off codegen. + * It happens after converting AST to Op, so it does not traverse AST. + * In the future, when control flow graph is introduced, this should be done at AST level. + */ + +namespace tolk { + +static void mark_function_used_dfs(const std::unique_ptr& op); + +static void mark_function_used(FunctionPtr fun_ref) { + if (!fun_ref->is_code_function() || fun_ref->is_really_used()) { // already handled + return; + } + + fun_ref->mutate()->assign_is_really_used(); + mark_function_used_dfs(std::get(fun_ref->body)->code->ops); +} + +static void mark_global_var_used(GlobalVarPtr glob_ref) { + glob_ref->mutate()->assign_is_really_used(); +} + +static void mark_function_used_dfs(const std::unique_ptr& op) { + if (!op) { + return; + } + + if (op->f_sym) { // for Op::_Call + mark_function_used(op->f_sym); + } + if (op->g_sym) { // for Op::_GlobVar + mark_global_var_used(op->g_sym); + } + mark_function_used_dfs(op->next); + mark_function_used_dfs(op->block0); + mark_function_used_dfs(op->block1); +} + +void pipeline_find_unused_symbols() { + for (FunctionPtr fun_ref : G.all_functions) { + if (fun_ref->is_method_id_not_empty()) { // get methods, main and other entrypoints, regular functions with @method_id + mark_function_used(fun_ref); + } + } +} + +} // namespace tolk diff --git a/tolk/pipe-generate-fif-output.cpp b/tolk/pipe-generate-fif-output.cpp new file mode 100644 index 00000000..57f481f0 --- /dev/null +++ b/tolk/pipe-generate-fif-output.cpp @@ -0,0 +1,170 @@ +/* + This file is part of TON Blockchain source code-> + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#include "tolk.h" +#include "src-file.h" +#include "ast.h" +#include "compiler-state.h" + +namespace tolk { + +void FunctionBodyCode::set_code(CodeBlob* code) { + this->code = code; +} + +void FunctionBodyAsm::set_code(std::vector&& code) { + this->ops = std::move(code); +} + + +static void generate_output_func(FunctionPtr fun_ref) { + tolk_assert(fun_ref->is_code_function()); + if (G.is_verbosity(2)) { + std::cerr << "\n\n=========================\nfunction " << fun_ref->name << " : " << fun_ref->inferred_return_type << std::endl; + } + + CodeBlob* code = std::get(fun_ref->body)->code; + if (G.is_verbosity(3)) { + code->print(std::cerr, 9); + } + code->prune_unreachable_code(); + if (G.is_verbosity(5)) { + std::cerr << "after prune_unreachable: \n"; + code->print(std::cerr, 0); + } + for (int i = 0; i < 8; i++) { + code->compute_used_code_vars(); + if (G.is_verbosity(4)) { + std::cerr << "after compute_used_vars: \n"; + code->print(std::cerr, 6); + } + code->fwd_analyze(); + if (G.is_verbosity(5)) { + std::cerr << "after fwd_analyze: \n"; + code->print(std::cerr, 6); + } + code->prune_unreachable_code(); + if (G.is_verbosity(5)) { + std::cerr << "after prune_unreachable: \n"; + code->print(std::cerr, 6); + } + } + code->mark_noreturn(); + if (G.is_verbosity(3)) { + code->print(std::cerr, 15); + } + if (G.is_verbosity(2)) { + std::cerr << "\n---------- resulting code for " << fun_ref->name << " -------------\n"; + } + const char* modifier = ""; + if (fun_ref->is_inline()) { + modifier = "INLINE"; + } else if (fun_ref->is_inline_ref()) { + modifier = "REF"; + } + std::cout << std::string(2, ' ') << fun_ref->name << " PROC" << modifier << ":<{\n"; + int mode = 0; + if (G.settings.stack_layout_comments) { + mode |= Stack::_StkCmt | Stack::_CptStkCmt; + } + if (fun_ref->is_inline() && code->ops->noreturn()) { + mode |= Stack::_InlineFunc; + } + if (fun_ref->is_inline() || fun_ref->is_inline_ref()) { + mode |= Stack::_InlineAny; + } + code->generate_code(std::cout, mode, 2); + std::cout << std::string(2, ' ') << "}>\n"; + if (G.is_verbosity(2)) { + std::cerr << "--------------\n"; + } +} + +void pipeline_generate_fif_output_to_std_cout() { + std::cout << "\"Asm.fif\" include\n"; + std::cout << "// automatically generated from "; + bool need_comma = false; + for (const SrcFile* file : G.all_src_files) { + if (!file->is_stdlib_file()) { + if (need_comma) { + std::cout << ", "; + } + std::cout << file->rel_filename; + need_comma = true; + } + } + std::cout << std::endl; + std::cout << "PROGRAM{\n"; + + bool has_main_procedure = false; + for (FunctionPtr fun_ref : G.all_functions) { + if (!fun_ref->does_need_codegen()) { + if (G.is_verbosity(2) && fun_ref->is_code_function()) { + std::cerr << fun_ref->name << ": code not generated, function does not need codegen\n"; + } + continue; + } + + if (fun_ref->is_entrypoint() && (fun_ref->name == "main" || fun_ref->name == "onInternalMessage")) { + has_main_procedure = true; + } + + std::cout << std::string(2, ' '); + if (fun_ref->is_method_id_not_empty()) { + std::cout << fun_ref->method_id << " DECLMETHOD " << fun_ref->name << "\n"; + } else { + std::cout << "DECLPROC " << fun_ref->name << "\n"; + } + } + + if (!has_main_procedure) { + throw Fatal("the contract has no entrypoint; forgot `fun onInternalMessage(...)`?"); + } + + for (GlobalVarPtr var_ref : G.all_global_vars) { + if (!var_ref->is_really_used() && G.settings.remove_unused_functions) { + if (G.is_verbosity(2)) { + std::cerr << var_ref->name << ": variable not generated, it's unused\n"; + } + continue; + } + + std::cout << std::string(2, ' ') << "DECLGLOBVAR " << var_ref->name << "\n"; + } + + for (FunctionPtr fun_ref : G.all_functions) { + if (!fun_ref->does_need_codegen()) { + continue; + } + generate_output_func(fun_ref); + } + + std::cout << "}END>c\n"; + if (!G.settings.boc_output_filename.empty()) { + std::cout << "boc>B \"" << G.settings.boc_output_filename << "\" B>file\n"; + } +} + +} // namespace tolk diff --git a/tolk/pipe-infer-types-and-calls.cpp b/tolk/pipe-infer-types-and-calls.cpp new file mode 100644 index 00000000..5fb12059 --- /dev/null +++ b/tolk/pipe-infer-types-and-calls.cpp @@ -0,0 +1,1301 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" +#include "src-file.h" +#include "ast.h" +#include "ast-visitor.h" +#include "generics-helpers.h" +#include "type-system.h" +#include "smart-casts-cfg.h" + +/* + * This is a complicated and crucial part of the pipeline. It simultaneously does the following: + * * infers types of all expressions; example: `2 + 3` both are TypeDataInt, result is also + * * AND binds function/method calls (assigns fun_ref); example: `globalF()`, fun_ref is assigned to `globalF` (unless generic) + * * AND instantiates generic functions; example: `t.tuplePush(2)` creates `tuplePush` and assigns fun_ref to dot field + * * AND infers return type of functions if it's omitted (`fun f() { ... }` means "auto infer", not "void") + * * AND builds data flow graph, mostly used for smart casts (right at the time of inferring) + * Note, that type checking (errors about types mismatch) is a later compilation step, due to loops. + * + * It's important to do all these parts simultaneously, they can't be split or separated. + * For example, we can't bind `f(2)` earlier, because if `f` is a generic `f`, we should instantiate it, + * and in order to do it, we need to know argument types. + * For example, we can't bind `c.cellHash()` earlier, because in order to bind it, we need to know object type. + * For example, we can't infer `var y = x` without smart casts, because if x's type is refined, it affects y. + * And vice versa, to infer type of expression in the middle, we need to have inferred all expressions preceding it, + * which may also include generics, etc. + * + * About generics. They are more like "C++ templates". If `f` and `f` called from somewhere, + * there will be TWO new functions, inserted into symtable, and both will be code generated to Fift. + * Body of a generic function is NOT analyzed. Hence, `fun f(v: T) { v.method(); }` we don't know + * whether `v.method()` is a valid call until instantiate it with `f` for example. + * Same for `v + 2`, we don't know whether + operator can be applied until instantiation. + * In other words, we have a closed type system, not open. + * That's why generic functions' bodies aren't traversed here (and in most following pipes). + * Instead, when an instantiated function is created, it follows all the preceding pipeline (registering symbols, etc.), + * and type inferring is done inside instantiated functions (which can recursively instantiate another, etc.). + * + * A noticeable part of inferring is "hints". + * Example: `var a: User = { id: 3, name: "" }`. To infer type of `{...}` we need to know it's `User`. This hint is taken from lhs. + * Example: `fun tupleAt(t: tuple, idx: int):T`, just `t.tupleGet(2)` can't be deduced (T left unspecified), + * but for assignment with left-defined type, or a call to `fInt(t.tupleGet(2))` hint "int" helps deduce T. + * + * Control flow is represented NOT as a "graph with edges". Instead, it's a "structured DFS" for the AST: + * 1) at every point of inferring, we have "current flow facts" (FlowContext) + * 2) when we see an `if (...)`, we create two derived contexts (by cloning current) + * 3) after `if`, finalize them at the end and unify + * 4) if we detect unreachable code, we mark that path's context as "unreachable" + * In other words, we get the effect of a CFG but in a more direct approach. That's enough for AST-level data-flow. + * FlowContext contains "data-flow facts that are definitely known". + * // current facts: x is int?, t is (int, int) + * if (x != null && t.0 > 0) + * // current facts: x is int, t is (int, int), t.0 is positive + * else + * // current facts: x is null, t is (int, int), t.0 is not positive + * When branches rejoin, facts are merged back (int+null = int? and so on, here they would be equal to before if). + * See smart-casts-cfg.cpp for detailed comments. + * + * About loops and partial re-entering. Consider the following: + * var x: int? = 5; + * // <- here x is `int` (smart cast) + * while (true) { + * // <- but here x is `int?` (not `int`) due to assignment in a loop + * if (...) { x = getNullableInt(); } + * } + * When building control flow, loops are inferred twice. In the above, at first iteration, x will be `int`, + * but at the second, x will be `int?` (after merged with loop end). + * That's why type checking is done later, not to make false errors on the first iteration. + * Note, that it would also be better to postpone generics "materialization" also: here only to infer type arguments, + * but to instantiate and re-assign fun_ref later. But it complicates the architecture significantly. + * For now, generics may encounter problems within loops on first iteration, though it's unlikely to face this + * in practice. (example: in the loop above, `genericFn(x)` will at first instantiate and then ) + * + * Unlike other pipes, inferring can dig recursively on demand. + * Example: + * fun getInt() { return 1; } + * fun main() { var i = getInt(); } + * If `main` is handled the first, it should know the return type if `getInt`. It's not declared, so we need + * to launch type inferring for `getInt` and then proceed back to `main`. + * When a generic function is instantiated, type inferring inside it is also run. + */ + +namespace tolk { + +static void infer_and_save_return_type_of_function(FunctionPtr fun_ref); + +static TypePtr get_or_infer_return_type(FunctionPtr fun_ref) { + if (!fun_ref->inferred_return_type) { + infer_and_save_return_type_of_function(fun_ref); + } + return fun_ref->inferred_return_type; +} + +GNU_ATTRIBUTE_NOINLINE +static std::string to_string(TypePtr type) { + return "`" + type->as_human_readable() + "`"; +} + +GNU_ATTRIBUTE_NOINLINE +static std::string to_string(AnyExprV v_with_type) { + return "`" + v_with_type->inferred_type->as_human_readable() + "`"; +} + +GNU_ATTRIBUTE_NOINLINE +static std::string to_string(FunctionPtr fun_ref) { + return "`" + fun_ref->as_human_readable() + "`"; +} + +// fire a general error, just a wrapper over `throw` +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire(FunctionPtr cur_f, SrcLocation loc, const std::string& message) { + throw ParseError(cur_f, loc, message); +} + +// fire an error when `fun f(...) asm ...` is called with T=(int,int) or other non-1 width on stack +// asm functions generally can't handle it, they expect T to be a TVM primitive +// (in FunC, `forall` type just couldn't be unified with non-primitives; in Tolk, generic T is expectedly inferred) +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_calling_asm_function_with_non1_stack_width_arg(FunctionPtr cur_f, SrcLocation loc, FunctionPtr fun_ref, const std::vector& substitutions, int arg_idx) { + fire(cur_f, loc, "can not call `" + fun_ref->as_human_readable() + "` with " + fun_ref->genericTs->get_nameT(arg_idx) + "=" + substitutions[arg_idx]->as_human_readable() + ", because it occupies " + std::to_string(substitutions[arg_idx]->get_width_on_stack()) + " stack slots in TVM, not 1"); +} + +// fire an error on `untypedTupleVar.0` when used without a hint +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_cannot_deduce_untyped_tuple_access(FunctionPtr cur_f, SrcLocation loc, int index) { + std::string idx_access = "." + std::to_string(index); + fire(cur_f, loc, "can not deduce type of `" + idx_access + "`; either assign it to variable like `var c: int = " + idx_access + "` or cast the result like `" + idx_access + " as int`"); +} + + +/* + * This class handles all types of AST vertices and traverses them, filling all AnyExprV::inferred_type. + * Note, that it isn't derived from ASTVisitor, it has manual `switch` over all existing vertex types. + * There are two reasons for this: + * 1) when a new AST node type is introduced, I want it to fail here, not to be left un-inferred with UB at next steps + * 2) easy to maintain a hint (see comments at the top of the file) + */ +class InferTypesAndCallsAndFieldsVisitor final { + FunctionPtr cur_f = nullptr; + std::vector return_statements; + + GNU_ATTRIBUTE_ALWAYS_INLINE + static void assign_inferred_type(AnyExprV dst, AnyExprV src) { +#ifdef TOLK_DEBUG + tolk_assert(src->inferred_type != nullptr && !src->inferred_type->has_unresolved_inside() && !src->inferred_type->has_genericT_inside()); +#endif + dst->mutate()->assign_inferred_type(src->inferred_type); + } + + GNU_ATTRIBUTE_ALWAYS_INLINE + static void assign_inferred_type(AnyExprV dst, TypePtr inferred_type) { +#ifdef TOLK_DEBUG + tolk_assert(inferred_type != nullptr && !inferred_type->has_unresolved_inside() && !inferred_type->has_genericT_inside()); +#endif + dst->mutate()->assign_inferred_type(inferred_type); + } + + static void assign_inferred_type(LocalVarPtr local_var_or_param, TypePtr inferred_type) { +#ifdef TOLK_DEBUG + tolk_assert(inferred_type != nullptr && !inferred_type->has_unresolved_inside() && !inferred_type->has_genericT_inside()); +#endif + local_var_or_param->mutate()->assign_inferred_type(inferred_type); + } + + static void assign_inferred_type(FunctionPtr fun_ref, TypePtr inferred_return_type, TypePtr inferred_full_type) { +#ifdef TOLK_DEBUG + tolk_assert(inferred_return_type != nullptr && !inferred_return_type->has_unresolved_inside() && !inferred_return_type->has_genericT_inside()); +#endif + fun_ref->mutate()->assign_inferred_type(inferred_return_type, inferred_full_type); + } + + // traverse children in any statement + FlowContext process_any_statement(AnyV v, FlowContext&& flow) { + switch (v->type) { + case ast_sequence: + return process_sequence(v->as(), std::move(flow)); + case ast_return_statement: + return process_return_statement(v->as(), std::move(flow)); + case ast_if_statement: + return process_if_statement(v->as(), std::move(flow)); + case ast_repeat_statement: + return process_repeat_statement(v->as(), std::move(flow)); + case ast_while_statement: + return process_while_statement(v->as(), std::move(flow)); + case ast_do_while_statement: + return process_do_while_statement(v->as(), std::move(flow)); + case ast_throw_statement: + return process_throw_statement(v->as(), std::move(flow)); + case ast_assert_statement: + return process_assert_statement(v->as(), std::move(flow)); + case ast_try_catch_statement: + return process_try_catch_statement(v->as(), std::move(flow)); + case ast_empty_statement: + return flow; + default: + return process_expression_statement(reinterpret_cast(v), std::move(flow)); + } + } + + // assigns inferred_type for any expression (by calling assign_inferred_type) + // returns ExprFlow: out_facts that are "definitely known" after evaluating the whole expression + // if used_as_condition, true_facts/false_facts are also calculated (don't calculate them always for optimization) + ExprFlow infer_any_expr(AnyExprV v, FlowContext&& flow, bool used_as_condition, TypePtr hint = nullptr) { + switch (v->type) { + case ast_int_const: + return infer_int_const(v->as(), std::move(flow), used_as_condition); + case ast_string_const: + return infer_string_const(v->as(), std::move(flow), used_as_condition); + case ast_bool_const: + return infer_bool_const(v->as(), std::move(flow), used_as_condition); + case ast_local_vars_declaration: + return infer_local_vars_declaration(v->as(), std::move(flow), used_as_condition); + case ast_local_var_lhs: + return infer_local_var_lhs(v->as(), std::move(flow), used_as_condition); + case ast_assign: + return infer_assignment(v->as(), std::move(flow), used_as_condition); + case ast_set_assign: + return infer_set_assign(v->as(), std::move(flow), used_as_condition); + case ast_unary_operator: + return infer_unary_operator(v->as(), std::move(flow), used_as_condition); + case ast_binary_operator: + return infer_binary_operator(v->as(), std::move(flow), used_as_condition); + case ast_ternary_operator: + return infer_ternary_operator(v->as(), std::move(flow), used_as_condition, hint); + case ast_cast_as_operator: + return infer_cast_as_operator(v->as(), std::move(flow), used_as_condition); + case ast_not_null_operator: + return infer_not_null_operator(v->as(), std::move(flow), used_as_condition); + case ast_is_null_check: + return infer_is_null_check(v->as(), std::move(flow), used_as_condition); + case ast_parenthesized_expression: + return infer_parenthesized(v->as(), std::move(flow), used_as_condition, hint); + case ast_reference: + return infer_reference(v->as(), std::move(flow), used_as_condition); + case ast_dot_access: + return infer_dot_access(v->as(), std::move(flow), used_as_condition, hint); + case ast_function_call: + return infer_function_call(v->as(), std::move(flow), used_as_condition, hint); + case ast_tensor: + return infer_tensor(v->as(), std::move(flow), used_as_condition, hint); + case ast_typed_tuple: + return infer_typed_tuple(v->as(), std::move(flow), used_as_condition, hint); + case ast_null_keyword: + return infer_null_keyword(v->as(), std::move(flow), used_as_condition); + case ast_underscore: + return infer_underscore(v->as(), std::move(flow), used_as_condition, hint); + case ast_empty_expression: + return infer_empty_expression(v->as(), std::move(flow), used_as_condition); + default: + throw UnexpectedASTNodeType(v, "infer_any_expr"); + } + } + + static ExprFlow infer_int_const(V v, FlowContext&& flow, bool used_as_condition) { + assign_inferred_type(v, TypeDataInt::create()); + + ExprFlow after_v(std::move(flow), used_as_condition); + if (used_as_condition) { // `if (0)` always false + if (v->intval == 0) { + after_v.true_flow.mark_unreachable(UnreachableKind::CantHappen); + } else { + after_v.false_flow.mark_unreachable(UnreachableKind::CantHappen); + } + } + return after_v; + } + + static ExprFlow infer_string_const(V v, FlowContext&& flow, bool used_as_condition) { + if (v->is_bitslice()) { + assign_inferred_type(v, TypeDataSlice::create()); + } else { + assign_inferred_type(v, TypeDataInt::create()); + } + + return ExprFlow(std::move(flow), used_as_condition); + } + + static ExprFlow infer_bool_const(V v, FlowContext&& flow, bool used_as_condition) { + assign_inferred_type(v, TypeDataBool::create()); + + ExprFlow after_v(std::move(flow), used_as_condition); + if (used_as_condition) { // `if (false)` always false + if (v->bool_val == false) { + after_v.true_flow.mark_unreachable(UnreachableKind::CantHappen); + } else { + after_v.false_flow.mark_unreachable(UnreachableKind::CantHappen); + } + } + return after_v; + } + + ExprFlow infer_local_vars_declaration(V v, FlowContext&& flow, bool used_as_condition) { + flow = infer_any_expr(v->get_expr(), std::move(flow), used_as_condition).out_flow; + assign_inferred_type(v, v->get_expr()); + return ExprFlow(std::move(flow), used_as_condition); + } + + static ExprFlow infer_local_var_lhs(V v, FlowContext&& flow, bool used_as_condition) { + // `var v = rhs`, inferring is called for `v` + // at the moment of inferring left side of assignment, we don't know type of rhs (since lhs is executed first) + // so, mark `v` as unknown + // later, v's inferred_type will be reassigned; see process_assignment_lhs_after_infer_rhs() + if (v->marked_as_redef) { + assign_inferred_type(v, v->var_ref->declared_type); + } else { + assign_inferred_type(v, v->declared_type ? v->declared_type : TypeDataUnknown::create()); + } + return ExprFlow(std::move(flow), used_as_condition); + } + + ExprFlow infer_assignment(V v, FlowContext&& flow, bool used_as_condition) { + // v is assignment: `x = 5` / `var x = 5` / `var x: slice = 5` / `(cs,_) = f()` / `val (a,[b],_) = (a,t,0)` + // execution flow is: lhs first, rhs second (at IR generation, also lhs is evaluated first, unlike FunC) + // after inferring lhs, use it for hint when inferring rhs + // example: `var i: int = t.tupleAt(0)` is ok (hint=int, T=int), but `var i = t.tupleAt(0)` not, since `tupleAt(t,i): T` + AnyExprV lhs = v->get_lhs(); + AnyExprV rhs = v->get_rhs(); + flow = infer_left_side_of_assignment(lhs, std::move(flow)); + flow = infer_any_expr(rhs, std::move(flow), false, lhs->inferred_type).out_flow; + process_assignment_lhs_after_infer_rhs(lhs, rhs->inferred_type, flow); + assign_inferred_type(v, rhs); // note, that the resulting type is rhs, not lhs + + return ExprFlow(std::move(flow), used_as_condition); + } + + // for `v = rhs` (NOT `var v = lhs`), variable `v` may be smart cast at this point + // the purpose of this function is to drop smart casts from expressions used as left side of assignments + // another example: `x.0 = rhs`, smart cast is dropped for `x.0` (not for `x`) + // the goal of dropping smart casts is to have lhs->inferred_type as actually declared, used as hint to infer rhs + FlowContext infer_left_side_of_assignment(AnyExprV lhs, FlowContext&& flow) { + if (auto lhs_tensor = lhs->try_as()) { + std::vector types_list; + types_list.reserve(lhs_tensor->size()); + for (int i = 0; i < lhs_tensor->size(); ++i) { + flow = infer_left_side_of_assignment(lhs_tensor->get_item(i), std::move(flow)); + types_list.push_back(lhs_tensor->get_item(i)->inferred_type); + } + assign_inferred_type(lhs, TypeDataTensor::create(std::move(types_list))); + + } else if (auto lhs_tuple = lhs->try_as()) { + std::vector types_list; + types_list.reserve(lhs_tuple->size()); + for (int i = 0; i < lhs_tuple->size(); ++i) { + flow = infer_left_side_of_assignment(lhs_tuple->get_item(i), std::move(flow)); + types_list.push_back(lhs_tuple->get_item(i)->inferred_type); + } + assign_inferred_type(lhs, TypeDataTypedTuple::create(std::move(types_list))); + + } else if (auto lhs_par = lhs->try_as()) { + flow = infer_left_side_of_assignment(lhs_par->get_expr(), std::move(flow)); + assign_inferred_type(lhs, lhs_par->get_expr()->inferred_type); + + } else { + flow = infer_any_expr(lhs, std::move(flow), false).out_flow; + if (extract_sink_expression_from_vertex(lhs)) { + TypePtr lhs_declared_type = calc_declared_type_before_smart_cast(lhs); + assign_inferred_type(lhs, lhs_declared_type); + } + } + + return flow; + } + + // handle (and dig recursively) into `var lhs = rhs` + // at this point, both lhs and rhs are already inferred, but lhs newly-declared vars are unknown (unless have declared_type) + // examples: `var z = 5`, `var (x, [y]) = (2, [3])`, `var (x, [y]) = xy` + // the purpose is to update inferred_type of lhs vars (z, x, y) + // and to re-assign types of tensors/tuples inside: `var (x,[y]) = ...` was `(unknown,[unknown])`, becomes `(int,[int])` + // while recursing, keep track of rhs if lhs and rhs have common shape (5 for z, 2 for x, [3] for [y], 3 for y) + // (so that on type mismatch, point to corresponding rhs, example: `var (x, y:slice) = (1, 2)` point to 2 + static void process_assignment_lhs_after_infer_rhs(AnyExprV lhs, TypePtr rhs_type, FlowContext& out_flow) { + tolk_assert(lhs->inferred_type != nullptr); + + // `var ... = rhs` - dig into left part + if (auto lhs_decl = lhs->try_as()) { + process_assignment_lhs_after_infer_rhs(lhs_decl->get_expr(), rhs_type, out_flow); + return; + } + + // inside `var v: int = rhs` / `var _ = rhs` / `var v redef = rhs` (lhs is "v" / "_" / "v") + if (auto lhs_var = lhs->try_as()) { + TypePtr declared_type = lhs_var->marked_as_redef ? lhs_var->var_ref->declared_type : lhs_var->declared_type; + if (lhs_var->inferred_type == TypeDataUnknown::create()) { + assign_inferred_type(lhs_var, rhs_type); + assign_inferred_type(lhs_var->var_ref, rhs_type); + } + TypePtr smart_casted_type = declared_type ? calc_smart_cast_type_on_assignment(declared_type, rhs_type) : rhs_type; + out_flow.register_known_type(SinkExpression(lhs_var->var_ref), smart_casted_type); + return; + } + + // `(v1, v2) = rhs` / `var (v1, v2) = rhs` (rhs may be `(1,2)` or `tensorVar` or `someF()`, doesn't matter) + // dig recursively into v1 and v2 with corresponding rhs i-th item of a tensor + if (auto lhs_tensor = lhs->try_as()) { + const TypeDataTensor* rhs_type_tensor = rhs_type->try_as(); + std::vector types_list; + types_list.reserve(lhs_tensor->size()); + for (int i = 0; i < lhs_tensor->size(); ++i) { + TypePtr ith_rhs_type = rhs_type_tensor && i < rhs_type_tensor->size() ? rhs_type_tensor->items[i] : TypeDataUnknown::create(); + process_assignment_lhs_after_infer_rhs(lhs_tensor->get_item(i), ith_rhs_type, out_flow); + types_list.push_back(lhs_tensor->get_item(i)->inferred_type); + } + assign_inferred_type(lhs, TypeDataTensor::create(std::move(types_list))); + return; + } + + // `[v1, v2] = rhs` / `var [v1, v2] = rhs` (rhs may be `[1,2]` or `tupleVar` or `someF()`, doesn't matter) + // dig recursively into v1 and v2 with corresponding rhs i-th item of a tuple + if (auto lhs_tuple = lhs->try_as()) { + const TypeDataTypedTuple* rhs_type_tuple = rhs_type->try_as(); + std::vector types_list; + types_list.reserve(lhs_tuple->size()); + for (int i = 0; i < lhs_tuple->size(); ++i) { + TypePtr ith_rhs_type = rhs_type_tuple && i < rhs_type_tuple->size() ? rhs_type_tuple->items[i] : TypeDataUnknown::create(); + process_assignment_lhs_after_infer_rhs(lhs_tuple->get_item(i), ith_rhs_type, out_flow); + types_list.push_back(lhs_tuple->get_item(i)->inferred_type); + } + assign_inferred_type(lhs, TypeDataTypedTuple::create(std::move(types_list))); + return; + } + + // `(v) = (rhs)`, just surrounded by parenthesis + if (auto lhs_par = lhs->try_as()) { + process_assignment_lhs_after_infer_rhs(lhs_par->get_expr(), rhs_type, out_flow); + assign_inferred_type(lhs, lhs_par->get_expr()); + return; + } + + // here is `v = rhs` (just assignment, not `var v = rhs`) / `a.0 = rhs` / `getObj(z=f()).0 = rhs` etc. + // for instance, `tensorVar.0 = rhs` / `obj.field = rhs` has already checked index correctness while inferring lhs + // for strange lhs like `f() = rhs` type inferring (and later checking) will pass, but will fail lvalue check later + if (SinkExpression s_expr = extract_sink_expression_from_vertex(lhs)) { + TypePtr lhs_declared_type = calc_declared_type_before_smart_cast(lhs); + TypePtr smart_casted_type = calc_smart_cast_type_on_assignment(lhs_declared_type, rhs_type); + out_flow.register_known_type(s_expr, smart_casted_type); + assign_inferred_type(lhs, lhs_declared_type); + } + } + + ExprFlow infer_set_assign(V v, FlowContext&& flow, bool used_as_condition) { + AnyExprV lhs = v->get_lhs(); + AnyExprV rhs = v->get_rhs(); + ExprFlow after_lhs = infer_any_expr(lhs, std::move(flow), false); + FlowContext rhs_flow = std::move(after_lhs.out_flow); + ExprFlow after_rhs = infer_any_expr(rhs, std::move(rhs_flow), false, lhs->inferred_type); + + // almost all operators implementation is hardcoded by built-in functions `_+_` and similar + std::string_view builtin_func = v->operator_name; // "+" for operator += + + assign_inferred_type(v, lhs); + if (!builtin_func.empty()) { + FunctionPtr builtin_sym = lookup_global_symbol("_" + static_cast(builtin_func) + "_")->try_as(); + v->mutate()->assign_fun_ref(builtin_sym); + } + + return ExprFlow(std::move(after_rhs.out_flow), used_as_condition); + } + + ExprFlow infer_unary_operator(V v, FlowContext&& flow, bool used_as_condition) { + AnyExprV rhs = v->get_rhs(); + ExprFlow after_rhs = infer_any_expr(rhs, std::move(flow), used_as_condition); + + // all operators implementation is hardcoded by built-in functions `~_` and similar + std::string_view builtin_func = v->operator_name; + + switch (v->tok) { + case tok_minus: + case tok_plus: + case tok_bitwise_not: + assign_inferred_type(v, TypeDataInt::create()); + break; + case tok_logical_not: + if (rhs->inferred_type == TypeDataBool::create()) { + builtin_func = "!b"; // "overloaded" for bool + } + assign_inferred_type(v, TypeDataBool::create()); + std::swap(after_rhs.false_flow, after_rhs.true_flow); + break; + default: + tolk_assert(false); + } + + FunctionPtr builtin_sym = lookup_global_symbol(static_cast(builtin_func) + "_")->try_as(); + v->mutate()->assign_fun_ref(builtin_sym); + + return after_rhs; + } + + ExprFlow infer_binary_operator(V v, FlowContext&& flow, bool used_as_condition) { + AnyExprV lhs = v->get_lhs(); + AnyExprV rhs = v->get_rhs(); + + // almost all operators implementation is hardcoded by built-in functions `_+_` and similar + std::string_view builtin_func = v->operator_name; + + switch (v->tok) { + // comparison operators, returning bool + case tok_eq: + case tok_neq: + case tok_lt: + case tok_gt: + case tok_leq: + case tok_geq: + case tok_spaceship: + flow = infer_any_expr(lhs, std::move(flow), false).out_flow; + flow = infer_any_expr(rhs, std::move(flow), false).out_flow; + assign_inferred_type(v, TypeDataBool::create()); + break; + // & | ^ are "overloaded" both for integers and booleans + case tok_bitwise_and: + case tok_bitwise_or: + case tok_bitwise_xor: + flow = infer_any_expr(lhs, std::move(flow), false).out_flow; + flow = infer_any_expr(rhs, std::move(flow), false).out_flow; + if (lhs->inferred_type == TypeDataBool::create() && rhs->inferred_type == TypeDataBool::create()) { + assign_inferred_type(v, TypeDataBool::create()); + } else { + assign_inferred_type(v, TypeDataInt::create()); + } + assign_inferred_type(v, rhs); // (int & int) is int, (bool & bool) is bool + break; + // && || result in booleans, but building flow facts is tricky due to short-circuit + case tok_logical_and: { + ExprFlow after_lhs = infer_any_expr(lhs, std::move(flow), true); + ExprFlow after_rhs = infer_any_expr(rhs, std::move(after_lhs.true_flow), true); + assign_inferred_type(v, TypeDataBool::create()); + if (!used_as_condition) { + FlowContext out_flow = FlowContext::merge_flow(std::move(after_lhs.false_flow), std::move(after_rhs.out_flow)); + return ExprFlow(std::move(out_flow), false); + } + FlowContext out_flow = FlowContext::merge_flow(std::move(after_lhs.out_flow), std::move(after_rhs.out_flow)); + FlowContext true_flow = std::move(after_rhs.true_flow); + FlowContext false_flow = FlowContext::merge_flow(std::move(after_lhs.false_flow), std::move(after_rhs.false_flow)); + return ExprFlow(std::move(out_flow), std::move(true_flow), std::move(false_flow)); + } + case tok_logical_or: { + ExprFlow after_lhs = infer_any_expr(lhs, std::move(flow), true); + ExprFlow after_rhs = infer_any_expr(rhs, std::move(after_lhs.false_flow), true); + assign_inferred_type(v, TypeDataBool::create()); + if (!used_as_condition) { + FlowContext out_flow = FlowContext::merge_flow(std::move(after_lhs.true_flow), std::move(after_rhs.out_flow)); + return ExprFlow(std::move(after_rhs.out_flow), false); + } + FlowContext out_flow = FlowContext::merge_flow(std::move(after_lhs.out_flow), std::move(after_rhs.out_flow)); + FlowContext true_flow = FlowContext::merge_flow(std::move(after_lhs.true_flow), std::move(after_rhs.true_flow)); + FlowContext false_flow = std::move(after_rhs.false_flow); + return ExprFlow(std::move(out_flow), std::move(true_flow), std::move(false_flow)); + } + // others are mathematical: + * ... + default: + flow = infer_any_expr(lhs, std::move(flow), false).out_flow; + flow = infer_any_expr(rhs, std::move(flow), false).out_flow; + assign_inferred_type(v, TypeDataInt::create()); + } + + if (!builtin_func.empty()) { + FunctionPtr builtin_sym = lookup_global_symbol("_" + static_cast(builtin_func) + "_")->try_as(); + v->mutate()->assign_fun_ref(builtin_sym); + } + + return ExprFlow(std::move(flow), used_as_condition); + } + + ExprFlow infer_ternary_operator(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + ExprFlow after_cond = infer_any_expr(v->get_cond(), std::move(flow), true); + v->get_cond()->mutate()->assign_always_true_or_false(after_cond.get_always_true_false_state()); + + ExprFlow after_true = infer_any_expr(v->get_when_true(), std::move(after_cond.true_flow), used_as_condition, hint); + ExprFlow after_false = infer_any_expr(v->get_when_false(), std::move(after_cond.false_flow), used_as_condition, hint); + + if (v->get_cond()->is_always_true) { + assign_inferred_type(v, v->get_when_true()); + return after_true; + } + if (v->get_cond()->is_always_false) { + assign_inferred_type(v, v->get_when_false()); + return after_false; + } + + TypeInferringUnifyStrategy tern_type; + tern_type.unify_with(v->get_when_true()->inferred_type); + if (!tern_type.unify_with(v->get_when_false()->inferred_type)) { + fire(cur_f, v->loc, "types of ternary branches are incompatible: " + to_string(v->get_when_true()) + " and " + to_string(v->get_when_false())); + } + assign_inferred_type(v, tern_type.get_result()); + + FlowContext out_flow = FlowContext::merge_flow(std::move(after_true.out_flow), std::move(after_false.out_flow)); + return ExprFlow(std::move(out_flow), std::move(after_true.true_flow), std::move(after_false.false_flow)); + } + + ExprFlow infer_cast_as_operator(V v, FlowContext&& flow, bool used_as_condition) { + // for `expr as `, use this type for hint, so that `t.tupleAt(0) as int` is ok + ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false, v->cast_to_type); + assign_inferred_type(v, v->cast_to_type); + + if (!used_as_condition) { + return after_expr; + } + return ExprFlow(std::move(after_expr.out_flow), true); + } + + ExprFlow infer_is_null_check(V v, FlowContext&& flow, bool used_as_condition) { + ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false); + assign_inferred_type(v, TypeDataBool::create()); + + TypePtr expr_type = v->get_expr()->inferred_type; + TypePtr non_null_type = calculate_type_subtract_null(expr_type); + if (expr_type == TypeDataNullLiteral::create()) { // `expr == null` is always true + v->mutate()->assign_always_true_or_false(v->is_negated ? 2 : 1); + } else if (non_null_type == TypeDataNever::create()) { // `expr == null` is always false + v->mutate()->assign_always_true_or_false(v->is_negated ? 1 : 2); + } else { + v->mutate()->assign_always_true_or_false(0); + } + + if (!used_as_condition) { + return after_expr; + } + + FlowContext true_flow = after_expr.out_flow.clone(); + FlowContext false_flow = after_expr.out_flow.clone(); + if (SinkExpression s_expr = extract_sink_expression_from_vertex(v->get_expr())) { + if (v->is_always_true) { + false_flow.mark_unreachable(UnreachableKind::CantHappen); + false_flow.register_known_type(s_expr, TypeDataNever::create()); + } else if (v->is_always_false) { + true_flow.mark_unreachable(UnreachableKind::CantHappen); + true_flow.register_known_type(s_expr, TypeDataNever::create()); + } else if (!v->is_negated) { + true_flow.register_known_type(s_expr, TypeDataNullLiteral::create()); + false_flow.register_known_type(s_expr, non_null_type); + } else { + true_flow.register_known_type(s_expr, non_null_type); + false_flow.register_known_type(s_expr, TypeDataNullLiteral::create()); + } + } + return ExprFlow(std::move(after_expr.out_flow), std::move(true_flow), std::move(false_flow)); + } + + ExprFlow infer_not_null_operator(V v, FlowContext&& flow, bool used_as_condition) { + ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), false); + + if (const auto* as_nullable = v->get_expr()->inferred_type->try_as()) { + assign_inferred_type(v, as_nullable->inner); + } else { + assign_inferred_type(v, v->get_expr()); + } + + if (!used_as_condition) { + return after_expr; + } + return ExprFlow(std::move(after_expr.out_flow), true); + } + + ExprFlow infer_parenthesized(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + ExprFlow after_expr = infer_any_expr(v->get_expr(), std::move(flow), used_as_condition, hint); + assign_inferred_type(v, v->get_expr()); + return after_expr; + } + + ExprFlow infer_reference(V v, FlowContext&& flow, bool used_as_condition) { + if (LocalVarPtr var_ref = v->sym->try_as()) { + TypePtr declared_or_smart_casted = flow.smart_cast_if_exists(SinkExpression(var_ref)); + tolk_assert(declared_or_smart_casted != nullptr); // all local vars are presented in flow + assign_inferred_type(v, declared_or_smart_casted); + + } else if (GlobalConstPtr const_ref = v->sym->try_as()) { + assign_inferred_type(v, const_ref->is_int_const() ? TypeDataInt::create() : TypeDataSlice::create()); + + } else if (GlobalVarPtr glob_ref = v->sym->try_as()) { + // there are no smart casts for globals, it's a way of preventing reading one global multiple times, it costs gas + assign_inferred_type(v, glob_ref->declared_type); + + } else if (FunctionPtr fun_ref = v->sym->try_as()) { + // it's `globalF` / `globalF` - references to functions used as non-call + V v_instantiationTs = v->get_instantiationTs(); + + if (fun_ref->is_generic_function() && !v_instantiationTs) { + // `genericFn` is invalid as non-call, can't be used without + fire(cur_f, v->loc, "can not use a generic function " + to_string(fun_ref) + " as non-call"); + + } else if (fun_ref->is_generic_function()) { + // `genericFn` is valid, it's a reference to instantiation + std::vector substitutions = collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref, v_instantiationTs); + fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutions)); + v->mutate()->assign_sym(fun_ref); + + } else if (v_instantiationTs != nullptr && !fun_ref->is_instantiation_of_generic_function()) { + // non-generic function referenced like `return beginCell;` + fire(cur_f, v_instantiationTs->loc, "not generic function used with generic T"); + } + + fun_ref->mutate()->assign_is_used_as_noncall(); + get_or_infer_return_type(fun_ref); + assign_inferred_type(v, fun_ref->inferred_full_type); + return ExprFlow(std::move(flow), used_as_condition); + + } else { + tolk_assert(false); + } + + // for non-functions: `local_var` and similar not allowed + if (UNLIKELY(v->has_instantiationTs())) { + fire(cur_f, v->get_instantiationTs()->loc, "generic T not expected here"); + } + return ExprFlow(std::move(flow), used_as_condition); + } + + // given `genericF` / `t.tupleFirst` (the user manually specified instantiation Ts), + // validate and collect them + // returns: [int, slice] / [cell] + std::vector collect_fun_generic_substitutions_from_manually_specified(SrcLocation loc, FunctionPtr fun_ref, V instantiationT_list) const { + if (fun_ref->genericTs->size() != instantiationT_list->get_items().size()) { + fire(cur_f, loc, "wrong count of generic T: expected " + std::to_string(fun_ref->genericTs->size()) + ", got " + std::to_string(instantiationT_list->size())); + } + + std::vector substitutions; + substitutions.reserve(instantiationT_list->size()); + for (int i = 0; i < instantiationT_list->size(); ++i) { + substitutions.push_back(instantiationT_list->get_item(i)->substituted_type); + } + + return substitutions; + } + + // when generic Ts have been collected from user-specified or deduced from arguments, + // instantiate a generic function + // example: was `t.tuplePush(2)`, deduced , instantiate `tuplePush` + // example: was `t.tuplePush(2)`, read , instantiate `tuplePush` (will later fail type check) + // example: was `var cb = t.tupleFirst;` (used as reference, as non-call), instantiate `tupleFirst` + // returns fun_ref to instantiated function + FunctionPtr check_and_instantiate_generic_function(SrcLocation loc, FunctionPtr fun_ref, std::vector&& substitutionTs) const { + // T for asm function must be a TVM primitive (width 1), otherwise, asm would act incorrectly + if (fun_ref->is_asm_function() || fun_ref->is_builtin_function()) { + for (int i = 0; i < static_cast(substitutionTs.size()); ++i) { + if (substitutionTs[i]->get_width_on_stack() != 1) { + fire_error_calling_asm_function_with_non1_stack_width_arg(cur_f, loc, fun_ref, substitutionTs, i); + } + } + } + + std::string inst_name = generate_instantiated_name(fun_ref->name, substitutionTs); + // make deep clone of `f` with substitutionTs + // (if `f` was already instantiated, it will be immediately returned from a symbol table) + return instantiate_generic_function(loc, fun_ref, inst_name, std::move(substitutionTs)); + } + + ExprFlow infer_dot_access(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + // it's NOT a method call `t.tupleSize()` (since such cases are handled by infer_function_call) + // it's `t.0`, `getUser().id`, and `t.tupleSize` (as a reference, not as a call) + flow = infer_any_expr(v->get_obj(), std::move(flow), false).out_flow; + + TypePtr obj_type = v->get_obj()->inferred_type; + // our goal is to fill v->target knowing type of obj + V v_ident = v->get_identifier(); // field/method name vertex + V v_instantiationTs = v->get_instantiationTs(); + std::string_view field_name = v_ident->name; + + // it can be indexed access (`tensorVar.0`, `tupleVar.1`) or a method (`t.tupleSize`) + // at first, check for indexed access + if (field_name[0] >= '0' && field_name[0] <= '9') { + int index_at = std::stoi(std::string(field_name)); + if (const auto* t_tensor = obj_type->try_as()) { + if (index_at >= t_tensor->size()) { + fire(cur_f, v_ident->loc, "invalid tensor index, expected 0.." + std::to_string(t_tensor->items.size() - 1)); + } + v->mutate()->assign_target(index_at); + TypePtr inferred_type = t_tensor->items[index_at]; + if (SinkExpression s_expr = extract_sink_expression_from_vertex(v)) { + if (TypePtr smart_casted = flow.smart_cast_if_exists(s_expr)) { + inferred_type = smart_casted; + } + } + assign_inferred_type(v, inferred_type); + return ExprFlow(std::move(flow), used_as_condition); + } + if (const auto* t_tuple = obj_type->try_as()) { + if (index_at >= t_tuple->size()) { + fire(cur_f, v_ident->loc, "invalid tuple index, expected 0.." + std::to_string(t_tuple->items.size() - 1)); + } + v->mutate()->assign_target(index_at); + TypePtr inferred_type = t_tuple->items[index_at]; + if (SinkExpression s_expr = extract_sink_expression_from_vertex(v)) { + if (TypePtr smart_casted = flow.smart_cast_if_exists(s_expr)) { + inferred_type = smart_casted; + } + } + assign_inferred_type(v, inferred_type); + return ExprFlow(std::move(flow), used_as_condition); + } + if (obj_type->try_as()) { + TypePtr item_type = nullptr; + if (v->is_lvalue && !hint) { // left side of assignment + item_type = TypeDataUnknown::create(); + } else { + if (hint == nullptr) { + fire_error_cannot_deduce_untyped_tuple_access(cur_f, v->loc, index_at); + } + item_type = hint; + } + v->mutate()->assign_target(index_at); + assign_inferred_type(v, item_type); + return ExprFlow(std::move(flow), used_as_condition); + } + fire(cur_f, v_ident->loc, "type " + to_string(obj_type) + " is not indexable"); + } + + // for now, Tolk doesn't have fields and object-scoped methods; `t.tupleSize` is a global function `tupleSize` + const Symbol* sym = lookup_global_symbol(field_name); + FunctionPtr fun_ref = sym ? sym->try_as() : nullptr; + if (!fun_ref) { + fire(cur_f, v_ident->loc, "non-existing field `" + static_cast(field_name) + "` of type " + to_string(obj_type)); + } + + // `t.tupleSize` is ok, `cs.tupleSize` not + if (!fun_ref->parameters[0].declared_type->can_rhs_be_assigned(obj_type)) { + fire(cur_f, v_ident->loc, "referencing a method for " + to_string(fun_ref->parameters[0].declared_type) + " with object of type " + to_string(obj_type)); + } + + if (fun_ref->is_generic_function() && !v_instantiationTs) { + // `genericFn` and `t.tupleAt` are invalid as non-call, they can't be used without + fire(cur_f, v->loc, "can not use a generic function " + to_string(fun_ref) + " as non-call"); + + } else if (fun_ref->is_generic_function()) { + // `t.tupleAt` is valid, it's a reference to instantiation + std::vector substitutions = collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref, v_instantiationTs); + fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, std::move(substitutions)); + + } else if (UNLIKELY(v_instantiationTs != nullptr)) { + // non-generic method referenced like `var cb = c.cellHash;` + fire(cur_f, v_instantiationTs->loc, "not generic function used with generic T"); + } + + fun_ref->mutate()->assign_is_used_as_noncall(); + v->mutate()->assign_target(fun_ref); + get_or_infer_return_type(fun_ref); + assign_inferred_type(v, fun_ref->inferred_full_type); // type of `t.tupleSize` is TypeDataFunCallable + return ExprFlow(std::move(flow), used_as_condition); + } + + ExprFlow infer_function_call(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + AnyExprV callee = v->get_callee(); + + // v is `globalF(args)` / `globalF(args)` / `obj.method(args)` / `local_var(args)` / `getF()(args)` + int delta_self = 0; + AnyExprV dot_obj = nullptr; + FunctionPtr fun_ref = nullptr; + V v_instantiationTs = nullptr; + + if (auto v_ref = callee->try_as()) { + // `globalF()` / `globalF()` / `local_var()` / `SOME_CONST()` + fun_ref = v_ref->sym->try_as(); // not null for `globalF` + v_instantiationTs = v_ref->get_instantiationTs(); // present for `globalF()` + + } else if (auto v_dot = callee->try_as()) { + // `obj.someMethod()` / `obj.someMethod()` / `getF().someMethod()` / `obj.SOME_CONST()` + // note, that dot_obj->target is not filled yet, since callee was not inferred yet + delta_self = 1; + dot_obj = v_dot->get_obj(); + v_instantiationTs = v_dot->get_instantiationTs(); // present for `obj.someMethod()` + flow = infer_any_expr(dot_obj, std::move(flow), false).out_flow; + + // it can be indexed access (`tensorVar.0()`, `tupleVar.1()`) or a method (`t.tupleSize()`) + std::string_view field_name = v_dot->get_field_name(); + if (field_name[0] >= '0' && field_name[0] <= '9') { + // indexed access `ab.2()`, then treat `ab.2` just like an expression, fun_ref remains nullptr + // infer_dot_access() will be called for a callee, it will check index correctness + } else { + // for now, Tolk doesn't have fields and object-scoped methods; `t.tupleSize` is a global function `tupleSize` + const Symbol* sym = lookup_global_symbol(field_name); + fun_ref = sym ? sym->try_as() : nullptr; + if (!fun_ref) { + fire(cur_f, v_dot->get_identifier()->loc, "non-existing method `" + static_cast(field_name) + "` of type " + to_string(dot_obj)); + } + } + + } else { + // `getF()()` / `5()` + // fun_ref remains nullptr + } + + // handle `local_var()` / `getF()()` / `5()` / `SOME_CONST()` / `obj.method()()()` / `tensorVar.0()` + if (!fun_ref) { + // treat callee like a usual expression + flow = infer_any_expr(callee, std::move(flow), false).out_flow; + // it must have "callable" inferred type + const TypeDataFunCallable* f_callable = callee->inferred_type->try_as(); + if (!f_callable) { // `5()` / `SOME_CONST()` / `null()` + fire(cur_f, v->loc, "calling a non-function " + to_string(callee->inferred_type)); + } + // check arguments count (their types will be checked in a later pipe) + if (v->get_num_args() != static_cast(f_callable->params_types.size())) { + fire(cur_f, v->loc, "expected " + std::to_string(f_callable->params_types.size()) + " arguments, got " + std::to_string(v->get_arg_list()->size())); + } + for (int i = 0; i < v->get_num_args(); ++i) { + auto arg_i = v->get_arg(i)->get_expr(); + flow = infer_any_expr(arg_i, std::move(flow), false, f_callable->params_types[i]).out_flow; + assign_inferred_type(v->get_arg(i), arg_i); + } + v->mutate()->assign_fun_ref(nullptr); // no fun_ref to a global function + assign_inferred_type(v, f_callable->return_type); + return ExprFlow(std::move(flow), used_as_condition); + } + + // so, we have a call `f(args)` or `obj.f(args)`, f is a global function (fun_ref) (code / asm / builtin) + // we're going to iterate over passed arguments, and (if generic) infer substitutionTs + // at first, check arguments count (Tolk doesn't have optional parameters, so just compare counts) + int n_arguments = v->get_num_args() + delta_self; + int n_parameters = fun_ref->get_num_params(); + if (!n_parameters && dot_obj) { + fire(cur_f, v->loc, "`" + fun_ref->name + "` has no parameters and can not be called as method"); + } + if (n_parameters < n_arguments) { + fire(cur_f, v->loc, "too many arguments in call to `" + fun_ref->name + "`, expected " + std::to_string(n_parameters - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); + } + if (n_arguments < n_parameters) { + fire(cur_f, v->loc, "too few arguments in call to `" + fun_ref->name + "`, expected " + std::to_string(n_parameters - delta_self) + ", have " + std::to_string(n_arguments - delta_self)); + } + + // now, for every passed argument, we need to infer its type + // for regular functions, it's obvious + // but for generic functions, we need to infer type arguments (substitutionTs) on the fly + // (unless Ts are specified by a user like `f(args)` / `t.tupleAt()`, take them) + GenericSubstitutionsDeduceForCall* deducingTs = fun_ref->is_generic_function() ? new GenericSubstitutionsDeduceForCall(fun_ref) : nullptr; + if (deducingTs && v_instantiationTs) { + deducingTs->provide_manually_specified(collect_fun_generic_substitutions_from_manually_specified(v->loc, fun_ref, v_instantiationTs)); + } + + // loop over every argument, for `obj.method()` obj is the first one + // if genericT deducing has a conflict, ParseError is thrown + // note, that deducing Ts one by one is important to manage control flow (mutate params work like assignments) + // a corner case, e.g. `f(v1:T?, v2:T?)` and `f(null,2)` will fail on first argument, won't try the second one + if (dot_obj) { + const LocalVarData& param_0 = fun_ref->parameters[0]; + TypePtr param_type = param_0.declared_type; + if (param_type->has_genericT_inside()) { + param_type = deducingTs->auto_deduce_from_argument(cur_f, dot_obj->loc, param_type, dot_obj->inferred_type); + } + if (param_0.is_mutate_parameter() && dot_obj->inferred_type != param_type) { + if (SinkExpression s_expr = extract_sink_expression_from_vertex(dot_obj)) { + assign_inferred_type(dot_obj, calc_declared_type_before_smart_cast(dot_obj)); + flow.register_known_type(s_expr, param_type); + } + } + } + for (int i = 0; i < v->get_num_args(); ++i) { + const LocalVarData& param_i = fun_ref->parameters[delta_self + i]; + AnyExprV arg_i = v->get_arg(i)->get_expr(); + TypePtr param_type = param_i.declared_type; + if (param_type->has_genericT_inside() && deducingTs->is_manually_specified()) { // `f(a)` + param_type = deducingTs->replace_by_manually_specified(param_type); + } + if (param_type->has_genericT_inside()) { // `f(a)` where f is generic: use `a` to infer param type + // then arg_i is inferred without any hint + flow = infer_any_expr(arg_i, std::move(flow), false).out_flow; + param_type = deducingTs->auto_deduce_from_argument(cur_f, arg_i->loc, param_type, arg_i->inferred_type); + } else { + // param_type is hint, helps infer arg_i + flow = infer_any_expr(arg_i, std::move(flow), false, param_type).out_flow; + } + assign_inferred_type(v->get_arg(i), arg_i); // arg itself is an expression + if (param_i.is_mutate_parameter() && arg_i->inferred_type != param_type) { + if (SinkExpression s_expr = extract_sink_expression_from_vertex(arg_i)) { + assign_inferred_type(arg_i, calc_declared_type_before_smart_cast(arg_i)); + flow.register_known_type(s_expr, param_type); + } + } + } + + // if it's a generic function `f`, we need to instantiate it, like `f` + // same for generic methods `t.tupleAt`, need to achieve `t.tupleAt` + + if (fun_ref->is_generic_function()) { + // if `f(args)` was called, Ts were inferred; check that all of them are known + int idx = deducingTs->get_first_not_deduced_idx(); + if (idx != -1 && hint && fun_ref->declared_return_type->has_genericT_inside()) { + // example: `t.tupleFirst()`, T doesn't depend on arguments, but is determined by return type + // if used like `var x: int = t.tupleFirst()` / `t.tupleFirst() as int` / etc., use hint + deducingTs->auto_deduce_from_argument(cur_f, v->loc, fun_ref->declared_return_type, hint); + idx = deducingTs->get_first_not_deduced_idx(); + } + if (idx != -1) { + fire(cur_f, v->loc, "can not deduce " + fun_ref->genericTs->get_nameT(idx)); + } + fun_ref = check_and_instantiate_generic_function(v->loc, fun_ref, deducingTs->flush()); + delete deducingTs; + + } else if (UNLIKELY(v_instantiationTs != nullptr)) { + // non-generic function/method called with type arguments, like `c.cellHash()` / `beginCell()` + fire(cur_f, v_instantiationTs->loc, "calling a not generic function with generic T"); + } + + v->mutate()->assign_fun_ref(fun_ref); + // since for `t.tupleAt()`, infer_dot_access() not called for callee = "t.tupleAt", assign its target here + if (v->is_dot_call()) { + v->get_callee()->as()->mutate()->assign_target(fun_ref); + } + // get return type either from user-specified declaration or infer here on demand traversing its body + get_or_infer_return_type(fun_ref); + TypePtr inferred_type = dot_obj && fun_ref->does_return_self() ? dot_obj->inferred_type : fun_ref->inferred_return_type; + assign_inferred_type(v, inferred_type); + assign_inferred_type(callee, fun_ref->inferred_full_type); + if (inferred_type == TypeDataNever::create()) { + flow.mark_unreachable(UnreachableKind::CallNeverReturnFunction); + } + // note, that mutate params don't affect typing, they are handled when converting to IR + return ExprFlow(std::move(flow), used_as_condition); + } + + ExprFlow infer_tensor(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + const TypeDataTensor* tensor_hint = hint ? hint->try_as() : nullptr; + std::vector types_list; + types_list.reserve(v->get_items().size()); + for (int i = 0; i < v->size(); ++i) { + AnyExprV item = v->get_item(i); + flow = infer_any_expr(item, std::move(flow), false, tensor_hint && i < tensor_hint->size() ? tensor_hint->items[i] : nullptr).out_flow; + types_list.emplace_back(item->inferred_type); + } + assign_inferred_type(v, TypeDataTensor::create(std::move(types_list))); + return ExprFlow(std::move(flow), used_as_condition); + } + + ExprFlow infer_typed_tuple(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + const TypeDataTypedTuple* tuple_hint = hint ? hint->try_as() : nullptr; + std::vector types_list; + types_list.reserve(v->get_items().size()); + for (int i = 0; i < v->size(); ++i) { + AnyExprV item = v->get_item(i); + flow = infer_any_expr(item, std::move(flow), false, tuple_hint && i < tuple_hint->size() ? tuple_hint->items[i] : nullptr).out_flow; + types_list.emplace_back(item->inferred_type); + } + assign_inferred_type(v, TypeDataTypedTuple::create(std::move(types_list))); + return ExprFlow(std::move(flow), used_as_condition); + } + + static ExprFlow infer_null_keyword(V v, FlowContext&& flow, bool used_as_condition) { + assign_inferred_type(v, TypeDataNullLiteral::create()); + + return ExprFlow(std::move(flow), used_as_condition); + } + + static ExprFlow infer_underscore(V v, FlowContext&& flow, bool used_as_condition, TypePtr hint) { + // if execution is here, underscore is either used as lhs of assignment, or incorrectly, like `f(_)` + // more precise is to always set unknown here, but for incorrect usages, instead of an error + // "can not pass unknown to X" would better be an error it can't be used as a value, at later steps + assign_inferred_type(v, hint ? hint : TypeDataUnknown::create()); + return ExprFlow(std::move(flow), used_as_condition); + } + + static ExprFlow infer_empty_expression(V v, FlowContext&& flow, bool used_as_condition) { + assign_inferred_type(v, TypeDataUnknown::create()); + return ExprFlow(std::move(flow), used_as_condition); + } + + FlowContext process_sequence(V v, FlowContext&& flow) { + // we'll print a warning if after some statement, control flow became unreachable + // (but don't print a warning if it's already unreachable, for example we're inside always-false if) + bool initially_unreachable = flow.is_unreachable(); + for (AnyV item : v->get_items()) { + if (flow.is_unreachable() && !initially_unreachable && !v->first_unreachable && item->type != ast_empty_statement) { + v->mutate()->assign_first_unreachable(item); // a warning will be printed later, after type checking + } + flow = process_any_statement(item, std::move(flow)); + } + return flow; + } + + FlowContext process_return_statement(V v, FlowContext&& flow) { + if (v->has_return_value()) { + flow = infer_any_expr(v->get_return_value(), std::move(flow), false, cur_f->declared_return_type).out_flow; + } else { + assign_inferred_type(v->get_return_value(), TypeDataVoid::create()); + } + flow.mark_unreachable(UnreachableKind::ReturnStatement); + + if (!cur_f->declared_return_type) { + return_statements.push_back(v->get_return_value()); // for future unification + } + return flow; + } + + FlowContext process_if_statement(V v, FlowContext&& flow) { + ExprFlow after_cond = infer_any_expr(v->get_cond(), std::move(flow), true); + v->get_cond()->mutate()->assign_always_true_or_false(after_cond.get_always_true_false_state()); + + FlowContext true_flow = process_any_statement(v->get_if_body(), std::move(after_cond.true_flow)); + FlowContext false_flow = process_any_statement(v->get_else_body(), std::move(after_cond.false_flow)); + + return FlowContext::merge_flow(std::move(true_flow), std::move(false_flow)); + } + + FlowContext process_repeat_statement(V v, FlowContext&& flow) { + ExprFlow after_cond = infer_any_expr(v->get_cond(), std::move(flow), false); + + return process_any_statement(v->get_body(), std::move(after_cond.out_flow)); + } + + FlowContext process_while_statement(V v, FlowContext&& flow) { + // loops are inferred twice, to merge body outcome with the state before the loop + // (a more correct approach would be not "twice", but "find a fixed point when state stop changing") + // also remember, we don't have a `break` statement, that's why when loop exits, condition became false + FlowContext loop_entry_facts = flow.clone(); + ExprFlow after_cond = infer_any_expr(v->get_cond(), std::move(flow), true); + FlowContext body_out = process_any_statement(v->get_body(), std::move(after_cond.true_flow)); + // second time, to refine all types + flow = FlowContext::merge_flow(std::move(loop_entry_facts), std::move(body_out)); + ExprFlow after_cond2 = infer_any_expr(v->get_cond(), std::move(flow), true); + v->get_cond()->mutate()->assign_always_true_or_false(after_cond2.get_always_true_false_state()); + + process_any_statement(v->get_body(), std::move(after_cond2.true_flow)); + + return std::move(after_cond2.false_flow); + } + + FlowContext process_do_while_statement(V v, FlowContext&& flow) { + // do while is also handled twice; read comments above + FlowContext loop_entry_facts = flow.clone(); + flow = process_any_statement(v->get_body(), std::move(flow)); + ExprFlow after_cond = infer_any_expr(v->get_cond(), std::move(flow), true); + // second time + flow = FlowContext::merge_flow(std::move(loop_entry_facts), std::move(after_cond.true_flow)); + flow = process_any_statement(v->get_body(), std::move(flow)); + ExprFlow after_cond2 = infer_any_expr(v->get_cond(), std::move(flow), true); + v->get_cond()->mutate()->assign_always_true_or_false(after_cond2.get_always_true_false_state()); + + return std::move(after_cond2.false_flow); + } + + FlowContext process_throw_statement(V v, FlowContext&& flow) { + flow = infer_any_expr(v->get_thrown_code(), std::move(flow), false).out_flow; + flow = infer_any_expr(v->get_thrown_arg(), std::move(flow), false).out_flow; + flow.mark_unreachable(UnreachableKind::ThrowStatement); + return flow; + } + + FlowContext process_assert_statement(V v, FlowContext&& flow) { + ExprFlow after_cond = infer_any_expr(v->get_cond(), std::move(flow), true); + v->get_cond()->mutate()->assign_always_true_or_false(after_cond.get_always_true_false_state()); + + ExprFlow after_throw = infer_any_expr(v->get_thrown_code(), std::move(after_cond.false_flow), false); + return std::move(after_cond.true_flow); + } + + static FlowContext process_catch_variable(AnyExprV catch_var, TypePtr catch_var_type, FlowContext&& flow) { + if (auto v_ref = catch_var->try_as(); v_ref && v_ref->sym) { // not underscore + LocalVarPtr var_ref = v_ref->sym->try_as(); + assign_inferred_type(var_ref, catch_var_type); + flow.register_known_type(SinkExpression(var_ref), catch_var_type); + } + assign_inferred_type(catch_var, catch_var_type); + return flow; + } + + FlowContext process_try_catch_statement(V v, FlowContext&& flow) { + FlowContext before_try = flow.clone(); + FlowContext try_end = process_any_statement(v->get_try_body(), std::move(flow)); + + // `catch` has exactly 2 variables: excNo and arg (when missing, they are implicit underscores) + // `arg` is a curious thing, it can be any TVM primitive, so assign unknown to it + // hence, using `fInt(arg)` (int from parameter is a target type) or `arg as slice` works well + // it's not truly correct, because `arg as (int,int)` also compiles, but can never happen, but let it be user responsibility + FlowContext catch_flow = std::move(before_try); + tolk_assert(v->get_catch_expr()->size() == 2); + std::vector types_list = {TypeDataInt::create(), TypeDataUnknown::create()}; + catch_flow = process_catch_variable(v->get_catch_expr()->get_item(0), types_list[0], std::move(catch_flow)); + catch_flow = process_catch_variable(v->get_catch_expr()->get_item(1), types_list[1], std::move(catch_flow)); + assign_inferred_type(v->get_catch_expr(), TypeDataTensor::create(std::move(types_list))); + + FlowContext catch_end = process_any_statement(v->get_catch_body(), std::move(catch_flow)); + return FlowContext::merge_flow(std::move(try_end), std::move(catch_end)); + } + + FlowContext process_expression_statement(AnyExprV v, FlowContext&& flow) { + ExprFlow after_v = infer_any_expr(v, std::move(flow), false); + return std::move(after_v.out_flow); + } + +public: + static void assign_fun_full_type(FunctionPtr fun_ref, TypePtr inferred_return_type) { + // calculate function full type `(params) -> ret_type` + std::vector params_types; + params_types.reserve(fun_ref->get_num_params()); + for (const LocalVarData& param : fun_ref->parameters) { + params_types.push_back(param.declared_type); + } + assign_inferred_type(fun_ref, inferred_return_type, TypeDataFunCallable::create(std::move(params_types), inferred_return_type)); + } + + void start_visiting_function(FunctionPtr fun_ref, V v_function) { + TypePtr inferred_return_type = fun_ref->declared_return_type; + if (fun_ref->is_code_function()) { + FlowContext body_start; + for (const LocalVarData& param : fun_ref->parameters) { + body_start.register_known_type(SinkExpression(¶m), param.declared_type); + } + + cur_f = fun_ref; + FlowContext body_end = process_any_statement(v_function->get_body(), std::move(body_start)); + cur_f = nullptr; + + if (!body_end.is_unreachable()) { + fun_ref->mutate()->assign_is_implicit_return(); + if (fun_ref->declared_return_type == TypeDataNever::create()) { // `never` can only be declared, it can't be inferred + fire(fun_ref, v_function->get_body()->as()->loc_end, "a function returning `never` can not have a reachable endpoint"); + } + } + + if (!fun_ref->declared_return_type) { + TypeInferringUnifyStrategy return_unifier; + if (fun_ref->does_return_self()) { + return_unifier.unify_with(fun_ref->parameters[0].declared_type); + } + for (AnyExprV return_value : return_statements) { + if (!return_unifier.unify_with(return_value->inferred_type)) { + fire(cur_f, return_value->loc, "can not unify type " + to_string(return_value) + " with previous return type " + to_string(return_unifier.get_result())); + } + } + if (!body_end.is_unreachable()) { + if (!return_unifier.unify_with_implicit_return_void()) { + fire(cur_f, v_function->get_body()->as()->loc_end, "missing return"); + } + } + inferred_return_type = return_unifier.get_result(); + if (inferred_return_type == nullptr && body_end.is_unreachable()) { + inferred_return_type = TypeDataVoid::create(); + } + } + } else { + // asm functions should be strictly typed, this was checked earlier + tolk_assert(fun_ref->declared_return_type); + } + + assign_fun_full_type(fun_ref, inferred_return_type); + fun_ref->mutate()->assign_is_type_inferring_done(); + } +}; + +class LaunchInferTypesAndMethodsOnce final { +public: + static bool should_visit_function(FunctionPtr fun_ref) { + // since inferring can be requested on demand, prevent second execution from a regular pipeline launcher + return !fun_ref->is_type_inferring_done() && !fun_ref->is_generic_function(); + } + + static void start_visiting_function(FunctionPtr fun_ref, V v_function) { + InferTypesAndCallsAndFieldsVisitor visitor; + visitor.start_visiting_function(fun_ref, v_function); + } +}; + +// infer return type "on demand" +// example: `fun f() { return g(); } fun g() { ... }` +// when analyzing `f()`, we need to infer what fun_ref=g returns +// (if `g` is generic, it was already instantiated, so fun_ref=g is here) +static void infer_and_save_return_type_of_function(FunctionPtr fun_ref) { + static std::vector called_stack; + + tolk_assert(!fun_ref->is_generic_function() && !fun_ref->is_type_inferring_done()); + // if `g` has return type declared, like `fun g(): int { ... }`, don't traverse its body + if (fun_ref->declared_return_type) { + InferTypesAndCallsAndFieldsVisitor::assign_fun_full_type(fun_ref, fun_ref->declared_return_type); + return; + } + + // prevent recursion of untyped functions, like `fun f() { return g(); } fun g() { return f(); }` + bool contains = std::find(called_stack.begin(), called_stack.end(), fun_ref) != called_stack.end(); + if (contains) { + fire(fun_ref, fun_ref->loc, "could not infer return type of " + to_string(fun_ref) + ", because it appears in a recursive call chain; specify `: ` manually"); + } + + // dig into g's body; it's safe, since the compiler is single-threaded + // on finish, fun_ref->inferred_return_type is filled, and won't be called anymore + called_stack.push_back(fun_ref); + InferTypesAndCallsAndFieldsVisitor visitor; + visitor.start_visiting_function(fun_ref, fun_ref->ast_root->as()); + called_stack.pop_back(); +} + +void pipeline_infer_types_and_calls_and_fields() { + visit_ast_of_all_functions(); +} + +void pipeline_infer_types_and_calls_and_fields(FunctionPtr fun_ref) { + InferTypesAndCallsAndFieldsVisitor visitor; + visitor.start_visiting_function(fun_ref, fun_ref->ast_root->as()); +} + +} // namespace tolk diff --git a/tolk/pipe-optimize-boolean-expr.cpp b/tolk/pipe-optimize-boolean-expr.cpp new file mode 100644 index 00000000..c4c5d1dc --- /dev/null +++ b/tolk/pipe-optimize-boolean-expr.cpp @@ -0,0 +1,139 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-replacer.h" +#include "type-system.h" + +/* + * This pipe does some optimizations related to booleans. + * It happens after type inferring, when we know types of all expressions. + * + * Example: `boolVar == true` -> `boolVar`. + * Example: `!!boolVar` -> `boolVar`. + * + * todo some day, replace && || with & | when it's safe (currently, && always produces IFs in Fift) + * It's tricky to implement whether replacing is safe. + * For example, safe: `a > 0 && a < 10` / `a != 3 && a != 5` + * For example, unsafe: `cached && calc()` / `a > 0 && log(a)` / `b != 0 && a / b > 1` / `i >= 0 && arr[idx]` / `f != null && close(f)` + */ + +namespace tolk { + +struct OptimizerBooleanExpressionsReplacer final : ASTReplacerInFunctionBody { + static V create_int_const(SrcLocation loc, td::RefInt256&& intval) { + auto v_int = createV(loc, std::move(intval), {}); + v_int->assign_inferred_type(TypeDataInt::create()); + v_int->assign_rvalue_true(); + return v_int; + } + + static V create_bool_const(SrcLocation loc, bool bool_val) { + auto v_bool = createV(loc, bool_val); + v_bool->assign_inferred_type(TypeDataInt::create()); + v_bool->assign_rvalue_true(); + return v_bool; + } + + static V create_logical_not_for_bool(SrcLocation loc, AnyExprV rhs) { + auto v_not = createV(loc, "!", tok_logical_not, rhs); + v_not->assign_inferred_type(TypeDataBool::create()); + v_not->assign_rvalue_true(); + v_not->assign_fun_ref(lookup_global_symbol("!b_")->try_as()); + return v_not; + } + +protected: + + AnyExprV replace(V v) override { + parent::replace(v); + + if (v->tok == tok_logical_not) { + if (auto inner_not = v->get_rhs()->try_as(); inner_not && inner_not->tok == tok_logical_not) { + AnyExprV cond_not_not = inner_not->get_rhs(); + // `!!boolVar` => `boolVar` + if (cond_not_not->inferred_type == TypeDataBool::create()) { + return cond_not_not; + } + // `!!intVar` => `intVar != 0` + if (cond_not_not->inferred_type == TypeDataInt::create()) { + auto v_zero = create_int_const(v->loc, td::make_refint(0)); + auto v_neq = createV(v->loc, "!=", tok_neq, cond_not_not, v_zero); + v_neq->mutate()->assign_rvalue_true(); + v_neq->mutate()->assign_inferred_type(TypeDataBool::create()); + v_neq->mutate()->assign_fun_ref(lookup_global_symbol("_!=_")->try_as()); + return v_neq; + } + } + if (auto inner_bool = v->get_rhs()->try_as()) { + // `!true` / `!false` + return create_bool_const(v->loc, !inner_bool->bool_val); + } + } + + return v; + } + + AnyExprV replace(V v) override { + parent::replace(v); + + if (v->tok == tok_eq || v->tok == tok_neq) { + AnyExprV lhs = v->get_lhs(); + AnyExprV rhs = v->get_rhs(); + if (lhs->inferred_type == TypeDataBool::create() && rhs->type == ast_bool_const) { + // `boolVar == true` / `boolVar != false` + if (rhs->as()->bool_val ^ (v->tok == tok_neq)) { + return lhs; + } + // `boolVar != true` / `boolVar == false` + return create_logical_not_for_bool(v->loc, lhs); + } + } + + return v; + } + + AnyV replace(V v) override { + parent::replace(v); + + // `if (!x)` -> ifnot(x) + while (auto v_cond_unary = v->get_cond()->try_as()) { + if (v_cond_unary->tok != tok_logical_not) { + break; + } + v = createV(v->loc, !v->is_ifnot, v_cond_unary->get_rhs(), v->get_if_body(), v->get_else_body()); + } + // `if (x != null)` -> ifnot(x == null) + if (auto v_cond_isnull = v->get_cond()->try_as(); v_cond_isnull && v_cond_isnull->is_negated) { + v_cond_isnull->mutate()->assign_is_negated(!v_cond_isnull->is_negated); + v = createV(v->loc, !v->is_ifnot, v_cond_isnull, v->get_if_body(), v->get_else_body()); + } + + return v; + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } +}; + +void pipeline_optimize_boolean_expressions() { + replace_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipe-refine-lvalue-for-mutate.cpp b/tolk/pipe-refine-lvalue-for-mutate.cpp new file mode 100644 index 00000000..a8b4f1ae --- /dev/null +++ b/tolk/pipe-refine-lvalue-for-mutate.cpp @@ -0,0 +1,128 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "ast.h" +#include "ast-visitor.h" + +/* + * This pipe refines rvalue/lvalue and checks `mutate` arguments validity. + * It happens after type inferring (after methods binding), because it uses fun_ref of calls. + * + * Example: `a.increment().increment()`, the first `a.increment()` becomes lvalue (assume that increment mutates self). + * Example: `increment(a)` is invalid, should be `increment(mutate a)`. + * + * Note, that explicitly specifying `mutate` for arguments, like `increment(mutate a)` is on purpose. + * If we wished `increment(a)` to be valid (to work and mutate `a`, like passing by ref), it would also be done here, + * refining `a` to be lvalue. But to avoid unexpected mutations, `mutate` keyword for an argument is required. + * So, for mutated arguments, instead of setting lvalue, we check its presence. + */ + +namespace tolk { + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_invalid_mutate_arg_passed(AnyExprV v, FunctionPtr fun_ref, const LocalVarData& p_sym, bool called_as_method, bool arg_passed_as_mutate, AnyV arg_expr) { + std::string arg_str(arg_expr->type == ast_reference ? arg_expr->as()->get_name() : "obj"); + + // case: `loadInt(cs, 32)`; suggest: `cs.loadInt(32)` + if (p_sym.is_mutate_parameter() && !arg_passed_as_mutate && !called_as_method && p_sym.param_idx == 0 && fun_ref->does_accept_self()) { + v->error("`" + fun_ref->name + "` is a mutating method; consider calling `" + arg_str + "." + fun_ref->name + "()`, not `" + fun_ref->name + "(" + arg_str + ")`"); + } + // case: `cs.mutating_function()`; suggest: `mutating_function(mutate cs)` or make it a method + if (p_sym.is_mutate_parameter() && called_as_method && p_sym.param_idx == 0 && !fun_ref->does_accept_self()) { + v->error("function `" + fun_ref->name + "` mutates parameter `" + p_sym.name + "`; consider calling `" + fun_ref->name + "(mutate " + arg_str + ")`, not `" + arg_str + "." + fun_ref->name + "`(); alternatively, rename parameter to `self` to make it a method"); + } + // case: `mutating_function(arg)`; suggest: `mutate arg` + if (p_sym.is_mutate_parameter() && !arg_passed_as_mutate) { + v->error("function `" + fun_ref->name + "` mutates parameter `" + p_sym.name + "`; you need to specify `mutate` when passing an argument, like `mutate " + arg_str + "`"); + } + // case: `usual_function(mutate arg)` + if (!p_sym.is_mutate_parameter() && arg_passed_as_mutate) { + v->error("incorrect `mutate`, since `" + fun_ref->name + "` does not mutate this parameter"); + } + throw Fatal("unreachable"); +} + + +class RefineLvalueForMutateArgumentsVisitor final : public ASTVisitorFunctionBody { + void visit(V v) override { + // v is `globalF(args)` / `globalF(args)` / `obj.method(args)` / `local_var(args)` / `getF()(args)` + FunctionPtr fun_ref = v->fun_maybe; + if (!fun_ref) { + parent::visit(v); + for (int i = 0; i < v->get_num_args(); ++i) { + auto v_arg = v->get_arg(i); + if (v_arg->passed_as_mutate) { + v_arg->error("`mutate` used for non-mutate argument"); + } + } + return; + } + + int delta_self = v->is_dot_call(); + tolk_assert(fun_ref->get_num_params() == delta_self + v->get_num_args()); + + if (v->is_dot_call()) { + if (fun_ref->does_mutate_self()) { + // for `b.storeInt()`, `b` should become lvalue, since `storeInt` is a method mutating self + // but: `beginCell().storeInt()`, then `beginCell()` is not lvalue + // (it will be extracted as tmp var when transforming AST to IR) + AnyExprV leftmost_obj = v->get_dot_obj(); + while (true) { + if (auto as_par = leftmost_obj->try_as()) { + leftmost_obj = as_par->get_expr(); + } else if (auto as_cast = leftmost_obj->try_as()) { + leftmost_obj = as_cast->get_expr(); + } else if (auto as_nn = leftmost_obj->try_as()) { + leftmost_obj = as_nn->get_expr(); + } else { + break; + } + } + bool will_be_extracted_as_tmp_var = leftmost_obj->type == ast_function_call; + if (!will_be_extracted_as_tmp_var) { + leftmost_obj->mutate()->assign_lvalue_true(); + v->get_dot_obj()->mutate()->assign_lvalue_true(); + } + } + + if (!fun_ref->does_accept_self() && fun_ref->parameters[0].is_mutate_parameter()) { + fire_error_invalid_mutate_arg_passed(v, fun_ref, fun_ref->parameters[0], true, false, v->get_dot_obj()); + } + } + + for (int i = 0; i < v->get_num_args(); ++i) { + const LocalVarData& p_sym = fun_ref->parameters[delta_self + i]; + auto arg_i = v->get_arg(i); + if (p_sym.is_mutate_parameter() != arg_i->passed_as_mutate) { + fire_error_invalid_mutate_arg_passed(arg_i, fun_ref, p_sym, false, arg_i->passed_as_mutate, arg_i->get_expr()); + } + parent::visit(arg_i); + } + parent::visit(v->get_callee()); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + return fun_ref->is_code_function() && !fun_ref->is_generic_function(); + } +}; + +void pipeline_refine_lvalue_for_mutate_arguments() { + visit_ast_of_all_functions(); +} + +} // namespace tolk diff --git a/tolk/pipe-register-symbols.cpp b/tolk/pipe-register-symbols.cpp new file mode 100644 index 00000000..45246d6b --- /dev/null +++ b/tolk/pipe-register-symbols.cpp @@ -0,0 +1,262 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . +*/ +#include "tolk.h" +#include "platform-utils.h" +#include "src-file.h" +#include "ast.h" +#include "compiler-state.h" +#include "constant-evaluator.h" +#include "generics-helpers.h" +#include "td/utils/crypto.h" +#include "type-system.h" +#include + +/* + * This pipe registers global symbols: functions, constants, global vars, etc. + * It happens just after all files have been parsed to AST. + * + * "Registering" means adding symbols to a global symbol table. + * After this pass, any global symbol can be looked up. + * Note, that local variables are not analyzed here, it's a later step. + * Before digging into locals, we need a global symtable to be filled, exactly done here. + */ + +namespace tolk { + +static int calculate_method_id_for_entrypoint(std::string_view func_name) { + if (func_name == "main" || func_name == "onInternalMessage") { + return 0; + } + if (func_name == "onExternalMessage") { + return -1; + } + if (func_name == "onRunTickTock") { + return -2; + } + if (func_name == "onSplitPrepare") { + return -3; + } + if (func_name == "onSplitInstall") { + return -4; + } + tolk_assert(false); +} + +static int calculate_method_id_by_func_name(std::string_view func_name) { + unsigned int crc = td::crc16(static_cast(func_name)); + return static_cast(crc & 0xffff) | 0x10000; +} + +static void validate_arg_ret_order_of_asm_function(V v_body, int n_params, TypePtr ret_type) { + if (!ret_type) { + v_body->error("asm function must declare return type (before asm instructions)"); + } + if (n_params > 16) { + v_body->error("asm function can have at most 16 parameters"); + } + + // asm(param1 ... paramN), param names were previously mapped into indices + if (!v_body->arg_order.empty()) { + if (static_cast(v_body->arg_order.size()) != n_params) { + v_body->error("arg_order of asm function must specify all parameters"); + } + std::vector visited(v_body->arg_order.size(), false); + for (int j : v_body->arg_order) { + if (visited[j]) { + v_body->error("arg_order of asm function contains duplicates"); + } + visited[j] = true; + } + } + + // asm(-> 0 2 1 3), check for a shuffled range 0...N + // correctness of N (actual return width onto a stack) will be checked after type inferring and generics instantiation + if (!v_body->ret_order.empty()) { + std::vector visited(v_body->ret_order.size(), false); + for (int j : v_body->ret_order) { + if (j < 0 || j >= static_cast(v_body->ret_order.size()) || visited[j]) { + v_body->error("ret_order contains invalid integer, not in range 0 .. N"); + } + visited[j] = true; + } + } +} + +static const GenericsDeclaration* construct_genericTs(V v_list) { + std::vector itemsT; + itemsT.reserve(v_list->size()); + + for (int i = 0; i < v_list->size(); ++i) { + auto v_item = v_list->get_item(i); + auto it_existing = std::find_if(itemsT.begin(), itemsT.end(), [v_item](const GenericsDeclaration::GenericsItem& prev) { + return prev.nameT == v_item->nameT; + }); + if (it_existing != itemsT.end()) { + v_item->error("duplicate generic parameter `" + static_cast(v_item->nameT) + "`"); + } + itemsT.emplace_back(v_item->nameT); + } + + return new GenericsDeclaration(std::move(itemsT)); +} + +static void register_constant(V v) { + ConstantValue init_value = eval_const_init_value(v->get_init_value()); + GlobalConstData* c_sym = new GlobalConstData(static_cast(v->get_identifier()->name), v->loc, v->declared_type, std::move(init_value)); + + if (v->declared_type) { + bool ok = (c_sym->is_int_const() && (v->declared_type == TypeDataInt::create())) + || (c_sym->is_slice_const() && (v->declared_type == TypeDataSlice::create())); + if (!ok) { + v->error("expression type does not match declared type"); + } + } + + G.symtable.add_global_const(c_sym); + G.all_constants.push_back(c_sym); + v->mutate()->assign_const_ref(c_sym); +} + +static void register_global_var(V v) { + GlobalVarData* g_sym = new GlobalVarData(static_cast(v->get_identifier()->name), v->loc, v->declared_type); + + G.symtable.add_global_var(g_sym); + G.all_global_vars.push_back(g_sym); + v->mutate()->assign_var_ref(g_sym); +} + +static LocalVarData register_parameter(V v, int idx) { + if (v->is_underscore()) { + return {"", v->loc, v->declared_type, 0, idx}; + } + + int flags = 0; + if (v->declared_as_mutate) { + flags |= LocalVarData::flagMutateParameter; + } + if (!v->declared_as_mutate && idx == 0 && v->param_name == "self") { + flags |= LocalVarData::flagImmutable; + } + return LocalVarData(static_cast(v->param_name), v->loc, v->declared_type, flags, idx); +} + +static void register_function(V v) { + std::string_view func_name = v->get_identifier()->name; + + // calculate TypeData of a function + std::vector arg_types; + std::vector parameters; + int n_params = v->get_num_params(); + int n_mutate_params = 0; + arg_types.reserve(n_params); + parameters.reserve(n_params); + for (int i = 0; i < n_params; ++i) { + auto v_param = v->get_param(i); + arg_types.emplace_back(v_param->declared_type); + parameters.emplace_back(register_parameter(v_param, i)); + n_mutate_params += static_cast(v_param->declared_as_mutate); + } + + const GenericsDeclaration* genericTs = nullptr; + if (v->genericsT_list) { + genericTs = construct_genericTs(v->genericsT_list); + } + if (v->is_builtin_function()) { + const Symbol* sym = lookup_global_symbol(func_name); + FunctionPtr fun_ref = sym ? sym->try_as() : nullptr; + if (!fun_ref || !fun_ref->is_builtin_function()) { + v->error("`builtin` used for non-builtin function"); + } + v->mutate()->assign_fun_ref(fun_ref); + return; + } + + if (G.is_verbosity(1) && v->is_code_function()) { + std::cerr << "fun " << func_name << " : " << v->declared_return_type << std::endl; + } + + FunctionBody f_body = v->get_body()->type == ast_sequence ? static_cast(new FunctionBodyCode) : static_cast(new FunctionBodyAsm); + FunctionData* f_sym = new FunctionData(static_cast(func_name), v->loc, v->declared_return_type, std::move(parameters), 0, genericTs, nullptr, f_body, v); + + if (const auto* v_asm = v->get_body()->try_as()) { + validate_arg_ret_order_of_asm_function(v_asm, v->get_num_params(), v->declared_return_type); + f_sym->arg_order = v_asm->arg_order; + f_sym->ret_order = v_asm->ret_order; + } + + if (v->method_id.not_null()) { + f_sym->method_id = static_cast(v->method_id->to_long()); + } else if (v->flags & FunctionData::flagGetMethod) { + f_sym->method_id = calculate_method_id_by_func_name(func_name); + for (FunctionPtr other : G.all_get_methods) { + if (other->method_id == f_sym->method_id) { + v->error(PSTRING() << "GET methods hash collision: `" << other->name << "` and `" << f_sym->name << "` produce the same hash. Consider renaming one of these functions."); + } + } + } else if (v->flags & FunctionData::flagIsEntrypoint) { + f_sym->method_id = calculate_method_id_for_entrypoint(func_name); + } + f_sym->flags |= v->flags; + if (n_mutate_params) { + f_sym->flags |= FunctionData::flagHasMutateParams; + } + + G.symtable.add_function(f_sym); + G.all_functions.push_back(f_sym); + if (f_sym->is_get_method()) { + G.all_get_methods.push_back(f_sym); + } + v->mutate()->assign_fun_ref(f_sym); +} + +static void iterate_through_file_symbols(const SrcFile* file) { + static std::unordered_set seen; + if (!seen.insert(file).second) { + return; + } + tolk_assert(file && file->ast); + + for (AnyV v : file->ast->as()->get_toplevel_declarations()) { + switch (v->type) { + case ast_import_directive: + // on `import "another-file.tolk"`, register symbols from that file at first + // (for instance, it can calculate constants, which are used in init_val of constants in current file below import) + iterate_through_file_symbols(v->as()->file); + break; + + case ast_constant_declaration: + register_constant(v->as()); + break; + case ast_global_var_declaration: + register_global_var(v->as()); + break; + case ast_function_declaration: + register_function(v->as()); + break; + default: + break; + } + } +} + +void pipeline_register_global_symbols() { + for (const SrcFile* file : G.all_src_files) { + iterate_through_file_symbols(file); + } +} + +} // namespace tolk diff --git a/tolk/pipe-resolve-identifiers.cpp b/tolk/pipe-resolve-identifiers.cpp new file mode 100644 index 00000000..5a735885 --- /dev/null +++ b/tolk/pipe-resolve-identifiers.cpp @@ -0,0 +1,353 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" +#include "platform-utils.h" +#include "compiler-state.h" +#include "src-file.h" +#include "generics-helpers.h" +#include "ast.h" +#include "ast-visitor.h" +#include "type-system.h" +#include + +/* + * This pipe resolves identifiers (local variables and types) in all functions bodies. + * It happens before type inferring, but after all global symbols are registered. + * It means, that for any symbol `x` we can look up whether it's a global name or not. + * + * About resolving variables. + * Example: `var x = 10; x = 20;` both `x` point to one LocalVarData. + * Example: `x = 20` undefined symbol `x` is also here (unless it's a global) + * Variables scoping and redeclaration are also here. + * Note, that `x` is stored as `ast_reference (ast_identifier "x")`. More formally, "references" are resolved. + * "Reference" in AST, besides the identifier, stores optional generics instantiation. `x` is grammar-valid. + * + * About resolving types. At the moment of parsing, `int`, `cell` and other predefined are parsed as TypeDataInt, etc. + * All the others are stored as TypeDataUnresolved, to be resolved here, after global symtable is filled. + * Example: `var x: T = 0` unresolved "T" is replaced by TypeDataGenericT inside `f`. + * Example: `f()` unresolved "MyAlias" is replaced by TypeDataAlias inside the reference. + * Example: `fun f(): KKK` unresolved "KKK" fires an error "unknown type name". + * When structures and type aliases are implemented, their resolving will also be done here. + * See finalize_type_data(). + * + * Note, that functions/methods binding is NOT here. + * In other words, for ast_function_call `beginCell()` and `t.tupleAt(0)`, their fun_ref is NOT filled here. + * Functions/methods binding is done later, simultaneously with type inferring and generics instantiation. + * For instance, to call a generic function `t.tuplePush(1)`, we need types of `t` and `1` to be inferred, + * as well as `tuplePush` to be instantiated, and fun_ref to point at that exact instantiations. + * + * As a result of this step, + * * every V::sym is filled, pointing either to a local var/parameter, or to a global symbol + * (exceptional for function calls and methods, their references are bound later) + * * all TypeData in all symbols is ready for analyzing, TypeDataUnresolved won't occur later in pipeline + */ + +namespace tolk { + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_undefined_symbol(FunctionPtr cur_f, V v) { + if (v->name == "self") { + throw ParseError(cur_f, v->loc, "using `self` in a non-member function (it does not accept the first `self` parameter)"); + } else { + throw ParseError(cur_f, v->loc, "undefined symbol `" + static_cast(v->name) + "`"); + } +} + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_unknown_type_name(FunctionPtr cur_f, SrcLocation loc, const std::string &text) { + throw ParseError(cur_f, loc, "unknown type name `" + text + "`"); +} + +static void check_import_exists_when_using_sym(FunctionPtr cur_f, AnyV v_usage, const Symbol* used_sym) { + SrcLocation sym_loc = used_sym->loc; + if (!v_usage->loc.is_symbol_from_same_or_builtin_file(sym_loc)) { + const SrcFile* declared_in = sym_loc.get_src_file(); + bool has_import = false; + for (const SrcFile::ImportDirective& import : v_usage->loc.get_src_file()->imports) { + if (import.imported_file == declared_in) { + has_import = true; + } + } + if (!has_import) { + throw ParseError(cur_f, v_usage->loc, "Using a non-imported symbol `" + used_sym->name + "`. Forgot to import \"" + declared_in->rel_filename + "\"?"); + } + } +} + +struct NameAndScopeResolver { + std::vector> scopes; + + static uint64_t key_hash(std::string_view name_key) { + return std::hash{}(name_key); + } + + void open_scope([[maybe_unused]] SrcLocation loc) { + // std::cerr << "open_scope " << scopes.size() + 1 << " at " << loc << std::endl; + scopes.emplace_back(); + } + + void close_scope([[maybe_unused]] SrcLocation loc) { + // std::cerr << "close_scope " << scopes.size() << " at " << loc << std::endl; + if (UNLIKELY(scopes.empty())) { + throw Fatal{"cannot close the outer scope"}; + } + scopes.pop_back(); + } + + const Symbol* lookup_symbol(std::string_view name) const { + uint64_t key = key_hash(name); + for (auto it = scopes.rbegin(); it != scopes.rend(); ++it) { // NOLINT(*-loop-convert) + const auto& scope = *it; + if (auto it_sym = scope.find(key); it_sym != scope.end()) { + return it_sym->second; + } + } + return G.symtable.lookup(name); + } + + void add_local_var(LocalVarPtr v_sym) { + if (UNLIKELY(scopes.empty())) { + throw Fatal("unexpected scope_level = 0"); + } + if (v_sym->name.empty()) { // underscore + return; + } + + uint64_t key = key_hash(v_sym->name); + const auto& [_, inserted] = scopes.rbegin()->emplace(key, v_sym); + if (UNLIKELY(!inserted)) { + throw ParseError(v_sym->loc, "redeclaration of local variable `" + v_sym->name + "`"); + } + } +}; + +struct TypeDataResolver { + GNU_ATTRIBUTE_NOINLINE + static TypePtr resolve_identifiers_in_type_data(FunctionPtr cur_f, TypePtr type_data, const GenericsDeclaration* genericTs) { + return type_data->replace_children_custom([cur_f, genericTs](TypePtr child) { + if (const TypeDataUnresolved* un = child->try_as()) { + if (genericTs && genericTs->has_nameT(un->text)) { + std::string nameT = un->text; + return TypeDataGenericT::create(std::move(nameT)); + } + if (un->text == "auto") { + throw ParseError(cur_f, un->loc, "`auto` type does not exist; just omit a type for local variable (will be inferred from assignment); parameters should always be typed"); + } + if (un->text == "self") { + throw ParseError(cur_f, un->loc, "`self` type can be used only as a return type of a function (enforcing it to be chainable)"); + } + fire_error_unknown_type_name(cur_f, un->loc, un->text); + } + return child; + }); + } +}; + +static TypePtr finalize_type_data(FunctionPtr cur_f, TypePtr type_data, const GenericsDeclaration* genericTs) { + if (!type_data || !type_data->has_unresolved_inside()) { + return type_data; + } + return TypeDataResolver::resolve_identifiers_in_type_data(cur_f, type_data, genericTs); +} + + +class AssignSymInsideFunctionVisitor final : public ASTVisitorFunctionBody { + // more correctly this field shouldn't be static, but currently there is no need to make it a part of state + static NameAndScopeResolver current_scope; + static FunctionPtr cur_f; + static const GenericsDeclaration* current_genericTs; + + static LocalVarPtr create_local_var_sym(std::string_view name, SrcLocation loc, TypePtr declared_type, bool immutable) { + LocalVarData* v_sym = new LocalVarData(static_cast(name), loc, declared_type, immutable * LocalVarData::flagImmutable, -1); + current_scope.add_local_var(v_sym); + return v_sym; + } + + static void process_catch_variable(AnyExprV catch_var) { + if (auto v_ref = catch_var->try_as()) { + LocalVarPtr var_ref = create_local_var_sym(v_ref->get_name(), catch_var->loc, nullptr, true); + v_ref->mutate()->assign_sym(var_ref); + } + } + +protected: + void visit(V v) override { + if (v->marked_as_redef) { + const Symbol* sym = current_scope.lookup_symbol(v->get_name()); + if (sym == nullptr) { + throw ParseError(cur_f, v->loc, "`redef` for unknown variable"); + } + LocalVarPtr var_ref = sym->try_as(); + if (!var_ref) { + throw ParseError(cur_f, v->loc, "`redef` for unknown variable"); + } + v->mutate()->assign_var_ref(var_ref); + } else { + TypePtr declared_type = finalize_type_data(cur_f, v->declared_type, current_genericTs); + LocalVarPtr var_ref = create_local_var_sym(v->get_name(), v->loc, declared_type, v->is_immutable); + v->mutate()->assign_resolved_type(declared_type); + v->mutate()->assign_var_ref(var_ref); + } + } + + void visit(V v) override { + parent::visit(v->get_rhs()); // in this order, so that `var x = x` is invalid, "x" on the right unknown + parent::visit(v->get_lhs()); + } + + void visit(V v) override { + const Symbol* sym = current_scope.lookup_symbol(v->get_name()); + if (!sym) { + fire_error_undefined_symbol(cur_f, v->get_identifier()); + } + v->mutate()->assign_sym(sym); + + // for global functions, global vars and constants, `import` must exist + if (!sym->try_as()) { + check_import_exists_when_using_sym(cur_f, v, sym); + } + + // for `f` / `f`, resolve "MyAlias" and "T" + // (for function call `f()`, this v (ast_reference `f`) is callee) + if (auto v_instantiationTs = v->get_instantiationTs()) { + for (int i = 0; i < v_instantiationTs->size(); ++i) { + TypePtr substituted_type = finalize_type_data(cur_f, v_instantiationTs->get_item(i)->substituted_type, current_genericTs); + v_instantiationTs->get_item(i)->mutate()->assign_resolved_type(substituted_type); + } + } + } + + void visit(V v) override { + // for `t.tupleAt` / `obj.method`, resolve "MyAlias" and "T" + // (for function call `t.tupleAt()`, this v (ast_dot_access `t.tupleAt`) is callee) + if (auto v_instantiationTs = v->get_instantiationTs()) { + for (int i = 0; i < v_instantiationTs->size(); ++i) { + TypePtr substituted_type = finalize_type_data(cur_f, v_instantiationTs->get_item(i)->substituted_type, current_genericTs); + v_instantiationTs->get_item(i)->mutate()->assign_resolved_type(substituted_type); + } + } + parent::visit(v->get_obj()); + } + + void visit(V v) override { + TypePtr cast_to_type = finalize_type_data(cur_f, v->cast_to_type, current_genericTs); + v->mutate()->assign_resolved_type(cast_to_type); + parent::visit(v->get_expr()); + } + + void visit(V v) override { + if (v->empty()) { + return; + } + current_scope.open_scope(v->loc); + parent::visit(v); + current_scope.close_scope(v->loc_end); + } + + void visit(V v) override { + current_scope.open_scope(v->loc); + parent::visit(v->get_body()); + parent::visit(v->get_cond()); // in 'while' condition it's ok to use variables declared inside do + current_scope.close_scope(v->get_body()->loc_end); + } + + void visit(V v) override { + visit(v->get_try_body()); + current_scope.open_scope(v->get_catch_body()->loc); + const std::vector& catch_items = v->get_catch_expr()->get_items(); + tolk_assert(catch_items.size() == 2); + process_catch_variable(catch_items[1]); + process_catch_variable(catch_items[0]); + parent::visit(v->get_catch_body()); + current_scope.close_scope(v->get_catch_body()->loc_end); + } + +public: + bool should_visit_function(FunctionPtr fun_ref) override { + // this pipe is done just after parsing + // visit both asm and code functions, resolve identifiers in parameter/return types everywhere + // for generic functions, unresolved "T" will be replaced by TypeDataGenericT + return true; + } + + void start_visiting_function(FunctionPtr fun_ref, V v) override { + cur_f = fun_ref; + current_genericTs = fun_ref->genericTs; + + for (int i = 0; i < v->get_num_params(); ++i) { + const LocalVarData& param_var = fun_ref->parameters[i]; + TypePtr declared_type = finalize_type_data(cur_f, param_var.declared_type, fun_ref->genericTs); + v->get_param(i)->mutate()->assign_param_ref(¶m_var); + v->get_param(i)->mutate()->assign_resolved_type(declared_type); + param_var.mutate()->assign_resolved_type(declared_type); + } + TypePtr return_type = finalize_type_data(cur_f, fun_ref->declared_return_type, fun_ref->genericTs); + v->mutate()->assign_resolved_type(return_type); + fun_ref->mutate()->assign_resolved_type(return_type); + + if (fun_ref->is_code_function()) { + auto v_seq = v->get_body()->as(); + current_scope.open_scope(v->loc); + for (int i = 0; i < v->get_num_params(); ++i) { + current_scope.add_local_var(&fun_ref->parameters[i]); + } + parent::visit(v_seq); + current_scope.close_scope(v_seq->loc_end); + tolk_assert(current_scope.scopes.empty()); + } + + current_genericTs = nullptr; + cur_f = nullptr; + } +}; + +NameAndScopeResolver AssignSymInsideFunctionVisitor::current_scope; +FunctionPtr AssignSymInsideFunctionVisitor::cur_f = nullptr; +const GenericsDeclaration* AssignSymInsideFunctionVisitor::current_genericTs = nullptr; + +void pipeline_resolve_identifiers_and_assign_symbols() { + AssignSymInsideFunctionVisitor visitor; + for (const SrcFile* file : G.all_src_files) { + for (AnyV v : file->ast->as()->get_toplevel_declarations()) { + if (auto v_func = v->try_as()) { + tolk_assert(v_func->fun_ref); + visitor.start_visiting_function(v_func->fun_ref, v_func); + + } else if (auto v_global = v->try_as()) { + TypePtr declared_type = finalize_type_data(nullptr, v_global->var_ref->declared_type, nullptr); + v_global->mutate()->assign_resolved_type(declared_type); + v_global->var_ref->mutate()->assign_resolved_type(declared_type); + + } else if (auto v_const = v->try_as()) { + if (v_const->declared_type) { + TypePtr declared_type = finalize_type_data(nullptr, v_const->const_ref->declared_type, nullptr); + v_const->mutate()->assign_resolved_type(declared_type); + v_const->const_ref->mutate()->assign_resolved_type(declared_type); + } + } + } + } +} + +void pipeline_resolve_identifiers_and_assign_symbols(FunctionPtr fun_ref) { + AssignSymInsideFunctionVisitor visitor; + if (visitor.should_visit_function(fun_ref)) { + visitor.start_visiting_function(fun_ref, fun_ref->ast_root->as()); + } +} + +} // namespace tolk diff --git a/tolk/pipeline.h b/tolk/pipeline.h new file mode 100644 index 00000000..0a71d751 --- /dev/null +++ b/tolk/pipeline.h @@ -0,0 +1,58 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#pragma once + +#include "fwd-declarations.h" +#include + +namespace tolk { + +void pipeline_discover_and_parse_sources(const std::string& stdlib_filename, const std::string& entrypoint_filename); + +void pipeline_register_global_symbols(); +void pipeline_resolve_identifiers_and_assign_symbols(); +void pipeline_calculate_rvalue_lvalue(); +void pipeline_infer_types_and_calls_and_fields(); +void pipeline_check_inferred_types(); +void pipeline_refine_lvalue_for_mutate_arguments(); +void pipeline_check_rvalue_lvalue(); +void pipeline_check_pure_impure_operations(); +void pipeline_constant_folding(); +void pipeline_optimize_boolean_expressions(); +void pipeline_convert_ast_to_legacy_Expr_Op(); + +void pipeline_find_unused_symbols(); +void pipeline_generate_fif_output_to_std_cout(); + +// these pipes also can be called per-function individually +// they are called for instantiated generics functions, when `f` is deeply cloned as `f` +void pipeline_resolve_identifiers_and_assign_symbols(FunctionPtr); +void pipeline_calculate_rvalue_lvalue(FunctionPtr); +void pipeline_detect_unreachable_statements(FunctionPtr); +void pipeline_infer_types_and_calls_and_fields(FunctionPtr); + + +} // namespace tolk diff --git a/tolk/platform-utils.h b/tolk/platform-utils.h new file mode 100644 index 00000000..5ab01220 --- /dev/null +++ b/tolk/platform-utils.h @@ -0,0 +1,48 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#pragma once + +#if __GNUC__ +#define GNU_ATTRIBUTE_COLD [[gnu::cold]] +#define GNU_ATTRIBUTE_FLATTEN [[gnu::flatten]] +#define GNU_ATTRIBUTE_NORETURN [[gnu::noreturn]] +#define GNU_ATTRIBUTE_NOINLINE [[gnu::noinline]] +#define GNU_ATTRIBUTE_ALWAYS_INLINE [[gnu::always_inline]] +#else +#define GNU_ATTRIBUTE_COLD +#define GNU_ATTRIBUTE_FLATTEN +#define GNU_ATTRIBUTE_NORETURN [[noreturn]] +#define GNU_ATTRIBUTE_NOINLINE [[noinline]] +#define GNU_ATTRIBUTE_ALWAYS_INLINE +#endif + +#if defined(__GNUC__) +#define LIKELY(x) __builtin_expect(x, true) +#define UNLIKELY(x) __builtin_expect(x, false) +#else +#define LIKELY(x) (x) +#define UNLIKELY(x) (x) +#endif diff --git a/tolk/smart-casts-cfg.cpp b/tolk/smart-casts-cfg.cpp new file mode 100644 index 00000000..7b86f519 --- /dev/null +++ b/tolk/smart-casts-cfg.cpp @@ -0,0 +1,472 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "smart-casts-cfg.h" +#include "ast.h" +#include "tolk.h" + +/* + * This file represents internals of AST-level control flow and data flow analysis. + * Data flow is mostly used for smart casts and is calculated AT THE TIME of type inferring. + * Not before, not after, but simultaneously with type inferring, because any local variable can be smart cast, + * which affects other expressions/variables types, generics instantiation, return auto-infer, etc. + * Though it's a part of type inferring, it's extracted as a separate file to keep inferring a bit clearer. + * + * Control flow is represented NOT as a "graph with edges". Instead, it's a "structured DFS" for the AST: + * 1) at every point of inferring, we have "current flow facts" (FlowContext) + * 2) when we see an `if (...)`, we create two derived contexts (by cloning current) + * 3) after `if`, finalize them at the end and unify + * 4) if we detect unreachable code, we mark that path's context as "unreachable" + * In other words, we get the effect of a CFG but in a more direct approach. That's enough for AST-level data-flow. + * + * FlowContext contains "data-flow facts that are definitely known": variables types (original or refined), + * sign state (definitely positive, definitely zero, etc.), boolean state (definitely true, definitely false). + * Each local variable is contained there, and possibly sub-fields of tensors/objects if definitely known: + * // current facts: x is int?, t is (int, int) + * if (x != null && t.0 > 0) + * // current facts: x is int, t is (int, int), t.0 is positive + * else + * // current facts: x is null, t is (int, int), t.0 is not positive + * When branches rejoin, facts are merged back (int+null = int? and so on, here they would be equal to before if). + * Another example: + * // current facts: x is int? + * if (x == null) { + * // current facts: x is null + * x = 1; + * // current facts: x is int + * } // else branch is empty, its facts are: x is int + * // current facts (after rejoin): x is int + * + * Every expression analysis result (performed along with type inferring) returns ExprFlow: + * 1) out_flow: facts after evaluating the whole expression, no matter how it evaluates (true or false) + * 2) true_flow: the environment if expression is definitely true + * 3) false_flow: the environment if expression is definitely false + * + * Note, that globals are NOT analyzed (smart casts work for locals only). The explanation is simple: + * don't encourage to use a global twice, it costs gas, better assign it to a local. + * See SinkExpression. + * + * An important highlight about internal structure of tensors / tuples / objects and `t.1` sink expressions. + * When a tensor/object is assigned, its fields are NOT tracked individually. + * For better understanding, I'll give some examples in TypeScript (having the same behavior): + * interface User { id: number | string, ... } + * var u: User = { id: 123, ... } + * u.id // it's number|string, not number + * u = { id: 'asdf', ... } + * u.id // it's number|string, not string + * if (typeof u.id === 'string') { + * // here `u.id` is string (smart cast) + * } + * u.id = 123; + * u.id // now it's number (smart cast) (until `u.id` or `u` are reassigned) + * // but `u` still has type `{ id: number | string, ... }`, not `{ id: number, ... }`; only `u.id` is refined + * The same example, but with nullable tensor in Tolk: + * var t: (int?, ...) = (123, ...) + * t.0 // it's int?, not int + * t = (null, ...) + * t.0 // it's int?, not null + * if (t.0 == null) { + * // here `t.0` is null (smart cast) + * } + * t.0 = 123; + * t.0 // now it's int (smart cast) (until `t.0` or `t` are reassigned) + * // but `t` still has type `(int?, ...)`, not `(int, ...)`; only `t.0` is refined + * + * In the future, not only smart casts, but other data-flow analysis can be implemented. + * 1) detect signs: `if (x > 0) { ... if (x < 0)` to warn always false + * 2) detect always true/false: `if (x) { return; } ... if (!x)` to warn always true + * These potential improvements are SignState and BoolState. Now they are NOT IMPLEMENTED, though declared. + * Their purpose is to show, that data flow is not only about smart casts, but eventually for other facts also. + * (though it's not obvious whether they should be analyzed at AST level or at IR level, like constants now) + */ + +namespace tolk { + +std::string SinkExpression::to_string() const { + std::string result = var_ref->name; + uint64_t cur_path = index_path; + while (cur_path != 0) { + result += "."; + result += std::to_string((cur_path & 0xFF) - 1); + cur_path >>= 8; + } + return result; +} + +static std::string to_string(SignState s) { + static const char* txt[6 + 1] = {"sign=unknown", ">0", "<0", "=0", ">=0", "<=0", "sign=never"}; + return txt[static_cast(s)]; +} + +static std::string to_string(BoolState s) { + static const char* txt[4 + 1] = {"unknown", "always_true", "always_false", "bool=never"}; + return txt[static_cast(s)]; +} + +// from `expr!` get `expr` +static AnyExprV unwrap_not_null_operator(AnyExprV expr) { + while (auto v_not_null = expr->try_as()) { + expr = v_not_null->get_expr(); + } + return expr; +} + +// "type lca" for a and b is T, so that both are assignable to T +// it's used +// 1) for auto-infer return type of the function if not specified +// example: `fun f(x: int?) { ... return 1; ... return x; }`; lca(`int`,`int?`) = `int?` +// 2) for auto-infer type of ternary and `match` expressions +// example: `cond ? beginCell() : null`; lca(`builder`,`null`) = `builder?` +// 3) when two data flows rejoin +// example: `if (tensorVar != null) ... else ...` rejoin `(int,int)` and `null` into `(int,int)?` +// when lca can't be calculated (example: `(int,int)` and `(int,int,int)`), nullptr is returned +static TypePtr calculate_type_lca(TypePtr a, TypePtr b) { + if (a == b) { + return a; + } + if (a == TypeDataNever::create()) { + return b; + } + if (b == TypeDataNever::create()) { + return a; + } + + if (a->can_rhs_be_assigned(b)) { + return a; + } + if (b->can_rhs_be_assigned(a)) { + return b; + } + + if (a == TypeDataUnknown::create() || b == TypeDataUnknown::create()) { + return TypeDataUnknown::create(); + } + + if (a == TypeDataNullLiteral::create()) { + return TypeDataNullable::create(b); + } + if (b == TypeDataNullLiteral::create()) { + return TypeDataNullable::create(a); + } + + const auto* tensor1 = a->try_as(); + const auto* tensor2 = b->try_as(); + if (tensor1 && tensor2 && tensor1->size() == tensor2->size()) { + std::vector types_lca; + types_lca.reserve(tensor1->size()); + for (int i = 0; i < tensor1->size(); ++i) { + TypePtr next = calculate_type_lca(tensor1->items[i], tensor2->items[i]); + if (next == nullptr) { + return nullptr; + } + types_lca.push_back(next); + } + return TypeDataTensor::create(std::move(types_lca)); + } + + const auto* tuple1 = a->try_as(); + const auto* tuple2 = b->try_as(); + if (tuple1 && tuple2 && tuple1->size() == tuple2->size()) { + std::vector types_lca; + types_lca.reserve(tuple1->size()); + for (int i = 0; i < tuple1->size(); ++i) { + TypePtr next = calculate_type_lca(tuple1->items[i], tuple2->items[i]); + if (next == nullptr) { + return nullptr; + } + types_lca.push_back(next); + } + return TypeDataTypedTuple::create(std::move(types_lca)); + } + + return nullptr; +} + +// merge (unify) of two sign states: what sign do we definitely have +// it's used on data flow rejoin +// example: `if (x > 0) ... else ...`; lca(Positive, NonPositive) = Unknown +SignState calculate_sign_lca(SignState a, SignState b) { + using s = SignState; + // a transformation lookup table, using the following rules: + // 1) if one is Unknown, the result is Unknown ("no definite constraints") + // 2) if one is Never (can't happen), the result is the other + // example: x is known > 0 already, given code `if (x > 0) {} else {}` merges Positive (always true) and Never + // 3) handle all other combinations carefully + static constexpr SignState transformations[7][7] = { + // b= Unknown | Positive | Negative | Zero | NonNegative | NonPositive | Never | + /* a=Unknown */ {s::Unknown, s::Unknown, s::Unknown, s::Unknown, s::Unknown, s::Unknown, s::Unknown }, + /* a=Positive */ {s::Unknown, s::Positive, s::Unknown, s::NonNegative, s::NonNegative, s::Unknown, s::Positive }, + /* a=Negative */ {s::Unknown, s::Unknown, s::Negative, s::NonPositive, s::Unknown, s::NonPositive, s::Negative }, + /* a=Zero */ {s::Unknown, s::NonNegative, s::NonPositive, s::Zero, s::NonNegative, s::NonPositive, s::Zero }, + /* a=NonNegative */ {s::Unknown, s::NonNegative, s::Unknown, s::NonNegative, s::NonNegative, s::Unknown, s::NonNegative}, + /* a=NonPositive */ {s::Unknown, s::Unknown, s::NonPositive, s::NonPositive, s::Unknown, s::NonPositive, s::NonPositive}, + /* a=Never */ {s::Unknown, s::Positive, s::Negative, s::Zero, s::NonNegative, s::NonPositive, s::Never } + }; + + return transformations[static_cast(a)][static_cast(b)]; +} + +// merge (unify) two bool state: what state do we definitely have +// it's used on data flow rejoin +// example: `if (x) ... else ...`; lca(AlwaysTrue, AlwaysFalse) = Unknown +BoolState calculate_bool_lca(BoolState a, BoolState b) { + using s = BoolState; + static constexpr BoolState transformations[4][4] = { + // b= Unknown | AlwaysTrue | AlwaysFalse | Never | + /* a=Unknown */ {s::Unknown, s::Unknown, s::Unknown, s::Unknown }, + /* a=AlwaysTrue */ {s::Unknown, s::AlwaysTrue, s::Unknown, s::AlwaysTrue }, + /* a=AlwaysFalse */ {s::Unknown, s::Unknown, s::AlwaysFalse, s::AlwaysFalse}, + /* a=Never */ {s::Unknown, s::AlwaysTrue, s::AlwaysFalse, s::Never } + }; + + return transformations[static_cast(a)][static_cast(b)]; +} + +// see comments above TypeInferringUnifyStrategy +// this function calculates lca or currently stored result and next +bool TypeInferringUnifyStrategy::unify_with(TypePtr next) { + if (unified_result == nullptr) { + unified_result = next; + return true; + } + if (unified_result == next) { + return true; + } + + TypePtr combined = calculate_type_lca(unified_result, next); + if (!combined) { + return false; + } + + unified_result = combined; + return true; +} + +bool TypeInferringUnifyStrategy::unify_with_implicit_return_void() { + if (unified_result == nullptr) { + unified_result = TypeDataVoid::create(); + return true; + } + + return unified_result == TypeDataVoid::create(); +} + +// invalidate knowledge about sub-fields of a variable or its field +// example: `tensorVar = 2`, invalidate facts about `tensorVar`, `tensorVar.0`, `tensorVar.1.2`, and all others +// example: `user.id = rhs`, invalidate facts about `user.id` (sign, etc.) and `user.id.*` if exist +void FlowContext::invalidate_all_subfields(LocalVarPtr var_ref, uint64_t parent_path, uint64_t parent_mask) { + for (auto it = known_facts.begin(); it != known_facts.end();) { + bool is_self_or_field = it->first.var_ref == var_ref && (it->first.index_path & parent_mask) == parent_path; + if (is_self_or_field) { + it = known_facts.erase(it); + } else { + ++it; + } + } +} + +// update current type of `local_var` / `tensorVar.0` / `obj.field` +// example: `local_var = rhs` +// example: `f(mutate obj.field)` +// example: `if (t.0 != null)`, in true_flow `t.0` assigned to "not-null of current", in false_flow to null +void FlowContext::register_known_type(SinkExpression s_expr, TypePtr assigned_type) { + // having index_path = (some bytes filled in the end), + // calc index_mask: replace every filled byte with 0xFF + // example: `t.0.1`, index_path = (1<<8) + 2, index_mask = 0xFFFF + uint64_t index_path = s_expr.index_path; + uint64_t index_mask = 0; + while (index_path > 0) { + index_mask = index_mask << 8 | 0xFF; + index_path >>= 8; + } + invalidate_all_subfields(s_expr.var_ref, s_expr.index_path, index_mask); + + // if just `int` assigned, we have no considerations about its sign + // so, even if something existed by the key s_expr, drop all knowledge + known_facts[s_expr] = FactsAboutExpr(assigned_type, SignState::Unknown, BoolState::Unknown); +} + +// mark control flow unreachable / interrupted +void FlowContext::mark_unreachable(UnreachableKind reason) { + unreachable = true; + // currently we don't save why control flow became unreachable (it's not obvious how, there may be consequent reasons), + // but it helps debugging and reading outer code + static_cast(reason); +} + + +// "merge" two data-flow contexts occurs on control flow rejoins (if/else branches merging, for example) +// it's generating a new context that describes "knowledge that definitely outcomes from these two" +// example: in one branch x is `int`, in x is `null`, result is `int?` unless any of them is unreachable +FlowContext FlowContext::merge_flow(FlowContext&& c1, FlowContext&& c2) { + if (!c1.unreachable && c2.unreachable) { + return merge_flow(std::move(c2), std::move(c1)); + } + + std::map unified; + + if (c1.unreachable && !c2.unreachable) { + // `if (...) return; else ...;` — copy facts about common variables only from else (c2) + for (const auto& [s_expr, i2] : c2.known_facts) { + auto it1 = c1.known_facts.find(s_expr); + bool need_add = it1 != c1.known_facts.end() || s_expr.index_path != 0; + if (need_add) { + unified.emplace(s_expr, i2); + } + } + + } else { + // either both reachable, or both not — merge types and restrictions of common variables and fields + for (const auto& [s_expr, i1] : c1.known_facts) { + if (auto it2 = c2.known_facts.find(s_expr); it2 != c2.known_facts.end()) { + const FactsAboutExpr& i2 = it2->second; + unified.emplace(s_expr, i1 == i2 ? i1 : FactsAboutExpr( + calculate_type_lca(i1.expr_type, i2.expr_type), + calculate_sign_lca(i1.sign_state, i2.sign_state), + calculate_bool_lca(i1.bool_state, i2.bool_state) + )); + } + } + } + + return FlowContext(std::move(unified), c1.unreachable && c2.unreachable); +} + +// return `T`, so that `T?` = type +// what for: `if (x != null)`, to smart cast x inside if +TypePtr calculate_type_subtract_null(TypePtr type) { + if (const auto* as_nullable = type->try_as()) { + return as_nullable->inner; + } + // union types will be handled here + return TypeDataNever::create(); +} + +// given any expression vertex, extract SinkExpression is possible +// example: `x.0` is { var_ref: x, index_path: 1 } +// example: `x.1` is { var_ref: x, index_path: 2 } +// example: `x!.1` is the same +// example: `x.1.2` is { var_ref: x, index_path: 2<<8 + 3 } +// example: `x!.1!.2` is the same +// not SinkExpressions: `globalVar` / `f()` / `obj.method().1` +SinkExpression extract_sink_expression_from_vertex(AnyExprV v) { + if (auto as_ref = v->try_as()) { + if (LocalVarPtr var_ref = as_ref->sym->try_as()) { + return SinkExpression(var_ref); + } + } + + if (auto as_dot = v->try_as(); as_dot && as_dot->is_target_indexed_access()) { + V cur_dot = as_dot; + uint64_t index_path = 0; + while (cur_dot->is_target_indexed_access()) { + int index_at = std::get(cur_dot->target); + index_path = (index_path << 8) + index_at + 1; + if (auto parent_dot = unwrap_not_null_operator(cur_dot->get_obj())->try_as()) { + cur_dot = parent_dot; + } else { + break; + } + } + if (auto as_ref = unwrap_not_null_operator(cur_dot->get_obj())->try_as()) { + if (LocalVarPtr var_ref = as_ref->sym->try_as()) { + return SinkExpression(var_ref, index_path); + } + } + } + + if (auto as_par = v->try_as()) { + return extract_sink_expression_from_vertex(as_par->get_expr()); + } + + if (auto as_assign = v->try_as()) { + return extract_sink_expression_from_vertex(as_assign->get_lhs()); + } + + return {}; +} + +// given `lhs = rhs`, calculate "original" type of `lhs` +// example: `var x: int? = ...; if (x != null) { x (here) = null; }` +// "(here)" x is `int` (smart cast), but originally declared as `int?` +// example: `if (x is (int,int)?) { x!.0 = rhs }`, here `x!.0` is `int` +TypePtr calc_declared_type_before_smart_cast(AnyExprV v) { + if (auto as_ref = v->try_as()) { + if (LocalVarPtr var_ref = as_ref->sym->try_as()) { + return var_ref->declared_type; + } + } + + if (auto as_dot = v->try_as(); as_dot && as_dot->is_target_indexed_access()) { + TypePtr obj_type = as_dot->get_obj()->inferred_type; // v already inferred; hence, index_at is correct + int index_at = std::get(as_dot->target); + if (const auto* t_tensor = obj_type->try_as()) { + return t_tensor->items[index_at]; + } + if (const auto* t_tuple = obj_type->try_as()) { + return t_tuple->items[index_at]; + } + } + + return v->inferred_type; +} + +// given `lhs = rhs` (and `var x = rhs`), calculate probable smart cast for lhs +// it's NOT directly type of rhs! see comment at the top of the file about internal structure of tensors/tuples. +// obvious example: `var x: int? = 5`, it's `int` (most cases are like this) +// obvious example: `var x: (int,int)? = null`, it's `null` (`x == null` is always true, `x` can be passed to any `T?`) +// not obvious example: `var x: (int?, int?)? = (3,null)`, result is `(int?,int?)`, whereas type of rhs is `(int,null)` +TypePtr calc_smart_cast_type_on_assignment(TypePtr lhs_declared_type, TypePtr rhs_inferred_type) { + // assign `T` to `T?` (or at least "assignable-to-T" to "T?") + // smart cast to `T` + if (const auto* lhs_nullable = lhs_declared_type->try_as()) { + if (lhs_nullable->inner->can_rhs_be_assigned(rhs_inferred_type)) { + return lhs_nullable->inner; + } + } + + // assign `null` to `T?` + // smart cast to `null` + if (lhs_declared_type->try_as() && rhs_inferred_type == TypeDataNullLiteral::create()) { + return TypeDataNullLiteral::create(); + } + + // no smart cast, type is the same as declared + // example: `var x: (int?,slice?) = (1, null)`, it's `(int?,slice?)`, not `(int,null)` + return lhs_declared_type; +} + + +std::ostream& operator<<(std::ostream& os, const FlowContext& flow) { + os << "(" << flow.known_facts.size() << " facts) " << (flow.unreachable ? "(unreachable) " : ""); + for (const auto& [s_expr, facts] : flow.known_facts) { + os << ", " << s_expr.to_string() << ": " << facts; + } + return os; +} + +std::ostream& operator<<(std::ostream& os, const FactsAboutExpr& facts) { + os << facts.expr_type; + if (facts.sign_state != SignState::Unknown) { + os << " " << to_string(facts.sign_state); + } + if (facts.bool_state != BoolState::Unknown) { + os << " " << to_string(facts.bool_state); + } + return os; +} + +} // namespace tolk diff --git a/tolk/smart-casts-cfg.h b/tolk/smart-casts-cfg.h new file mode 100644 index 00000000..b97c8864 --- /dev/null +++ b/tolk/smart-casts-cfg.h @@ -0,0 +1,208 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "fwd-declarations.h" +#include "type-system.h" +#include +#include + +namespace tolk { + +/* + * TypeInferringUnifyStrategy unifies types from various branches to a common result (lca). + * It's used to auto infer function return type based on return statements, like in TypeScript. + * Example: `fun f() { ... return 1; ... return null; }` inferred as `int?`. + * + * Besides function returns, it's also used for ternary `return cond ? 1 : null` and `match` expression. + * If types can't be unified (a function returns int and cell, for example), `unify()` returns false, handled outside. + * BTW, don't confuse this way of inferring with Hindley-Milner, they have nothing in common. + */ +class TypeInferringUnifyStrategy { + TypePtr unified_result = nullptr; + +public: + bool unify_with(TypePtr next); + bool unify_with_implicit_return_void(); + + TypePtr get_result() const { return unified_result; } +}; + +/* + * SinkExpression is an expression that can be smart cast like `if (x != null)` (x is int inside) + * or analyzed by data flow is some other way like `if (x > 0) ... else ...` (x <= 0 inside else). + * In other words, it "absorbs" data flow facts. + * Examples: `localVar`, `localTensor.1`, `localTuple.1.2.3`, `localObj.field` + * These are NOT sink expressions: `globalVar`, `f()`, `f().1` + * Note, that globals are NOT sink: don't encourage to use a global twice, it costs gas, better assign it to a local. + */ +struct SinkExpression { + LocalVarPtr const var_ref; // smart casts and data flow applies only to locals + const uint64_t index_path; // 0 for just `v`; for `v.N` it's (N+1), for `v.N.M` it's (N+1) + (M+1)<<8, etc. + + SinkExpression() + : var_ref(nullptr), index_path(0) {} + explicit SinkExpression(LocalVarPtr var_ref) + : var_ref(var_ref), index_path(0) {} + explicit SinkExpression(LocalVarPtr var_ref, uint64_t index_path) + : var_ref(var_ref), index_path(index_path) {} + + SinkExpression(const SinkExpression&) = default; + SinkExpression& operator=(const SinkExpression&) = delete; + + bool operator==(const SinkExpression& rhs) const { return var_ref == rhs.var_ref && index_path == rhs.index_path; } + bool operator<(const SinkExpression& rhs) const { return var_ref == rhs.var_ref ? index_path < rhs.index_path : var_ref < rhs.var_ref; } + explicit operator bool() const { return var_ref != nullptr; } + + std::string to_string() const; +}; + +// UnreachableKind is a reason of why control flow is unreachable or interrupted +// example: `return;` interrupts control flow +// example: `if (true) ... else ...` inside "else" flow is unreachable because it can't happen +enum class UnreachableKind { + Unknown, // no definite info or not unreachable + CantHappen, + ThrowStatement, + ReturnStatement, + CallNeverReturnFunction, +}; + +// SignState is "definitely positive", etc. +// example: inside `if (x > 0)`, x is Positive, in `else` it's NonPositive (if x is local, until reassigned) +enum class SignState { + Unknown, // no definite info + Positive, + Negative, + Zero, + NonNegative, + NonPositive, + Never // can't happen, like "never" type +}; + +// BoolState is "definitely true" or "definitely false" +// example: inside `if (x)`, x is AlwaysTrue, in `else` it's AlwaysFalse +enum class BoolState { + Unknown, // no definite info + AlwaysTrue, + AlwaysFalse, + Never // can't happen, like "never" type +}; + +// FactsAboutExpr represents "everything known about SinkExpression at a given execution point" +// example: after `var x = getNullableInt()`, x is `int?`, sign/bool is Unknown +// example: after `x = 2;`, x is `int`, sign is Positive, bool is AlwaysTrue +// example: inside `if (x != null && x > 0)`, x is `int`, sign is Positive (in else, no definite knowledge) +// remember, that indices/fields are also expressions, `t.1 = 2` or `u.id = 2` also store such facts +// WARNING! Detecting data-flow facts about sign state and bool state is NOT IMPLEMENTED +// (e.g. `if (x > 0)` / `if (!t.1)` is NOT analysed, therefore not updated, always Unknown now) +// it's a potential improvement for the future, for example `if (x > 0) { ... if (x < 0)` to warn always false +// their purpose for now is to show, that data flow is not only about smart casts, but eventually for other facts also +struct FactsAboutExpr { + TypePtr expr_type; // originally declared type or smart cast (Unknown if no info) + SignState sign_state; // definitely positive, etc. (Unknown if no info) + BoolState bool_state; // definitely true/false (Unknown if no info) + + FactsAboutExpr() + : expr_type(nullptr), sign_state(SignState::Unknown), bool_state(BoolState::Unknown) {} + FactsAboutExpr(TypePtr smart_cast_type, SignState sign_state, BoolState bool_state) + : expr_type(smart_cast_type), sign_state(sign_state), bool_state(bool_state) {} + + bool operator==(const FactsAboutExpr& rhs) const = default; +}; + +// FlowContext represents "everything known about control flow at a given execution point" +// while traversing AST, each statement node gets "in" FlowContext (prior knowledge) +// and returns "output" FlowContext (representing a state AFTER execution of a statement) +// on branching, like if/else, input context is cloned, two contexts for each branch calculated, and merged to a result +class FlowContext { + // std::map, not std::unordered_map, because LLDB visualises it better, for debugging + std::map known_facts; // all local vars plus (optionally) indices/fields of tensors/tuples/objects + bool unreachable = false; // if execution can't reach this point (after `return`, for example) + + FlowContext(std::map&& known_facts, bool unreachable) + : known_facts(std::move(known_facts)), unreachable(unreachable) {} + + void invalidate_all_subfields(LocalVarPtr var_ref, uint64_t parent_path, uint64_t parent_mask); + + friend std::ostream& operator<<(std::ostream& os, const FlowContext& flow); + +public: + FlowContext() = default; + FlowContext(FlowContext&&) noexcept = default; + FlowContext(const FlowContext&) = delete; + FlowContext& operator=(FlowContext&&) = default; + FlowContext& operator=(const FlowContext&) = delete; + + FlowContext clone() const { + std::map copy = known_facts; + return FlowContext(std::move(copy), unreachable); + } + + bool is_unreachable() const { return unreachable; } + + TypePtr smart_cast_if_exists(SinkExpression s_expr) const { + auto it = known_facts.find(s_expr); + return it == known_facts.end() ? nullptr : it->second.expr_type; + } + + void register_known_type(SinkExpression s_expr, TypePtr assigned_type); + void mark_unreachable(UnreachableKind reason); + + static FlowContext merge_flow(FlowContext&& c1, FlowContext&& c2); +}; + +struct ExprFlow { + FlowContext out_flow; + + // only calculated inside `if`, left of `&&`, etc. — there this expression is immediate condition, empty otherwise + FlowContext true_flow; + FlowContext false_flow; + + ExprFlow(FlowContext&& out_flow, FlowContext&& true_flow, FlowContext&& false_flow) + : out_flow(std::move(out_flow)) + , true_flow(std::move(true_flow)) + , false_flow(std::move(false_flow)) {} + ExprFlow(FlowContext&& out_flow, const bool clone_flow_for_condition) + : out_flow(std::move(out_flow)) { + if (clone_flow_for_condition) { + true_flow = this->out_flow.clone(); + false_flow = this->out_flow.clone(); + } + } + + ExprFlow(ExprFlow&&) noexcept = default; + ExprFlow(const ExprFlow&) = delete; + ExprFlow& operator=(ExprFlow&&) = delete; + ExprFlow& operator=(const ExprFlow&) = delete; + + int get_always_true_false_state() const { + if (true_flow.is_unreachable() != false_flow.is_unreachable()) { + return false_flow.is_unreachable() ? 1 : 2; // 1 is "always true" + } + return 0; + } +}; + +std::ostream& operator<<(std::ostream& os, const FactsAboutExpr& facts); +std::ostream& operator<<(std::ostream& os, const FlowContext& flow); +TypePtr calculate_type_subtract_null(TypePtr type); +SinkExpression extract_sink_expression_from_vertex(AnyExprV v); +TypePtr calc_declared_type_before_smart_cast(AnyExprV v); +TypePtr calc_smart_cast_type_on_assignment(TypePtr lhs_declared_type, TypePtr rhs_inferred_type); + +} // namespace tolk diff --git a/tolk/src-file.cpp b/tolk/src-file.cpp new file mode 100644 index 00000000..1286c1f9 --- /dev/null +++ b/tolk/src-file.cpp @@ -0,0 +1,205 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "src-file.h" +#include "compiler-state.h" +#include +#include +#include + +namespace tolk { + +static_assert(sizeof(SrcLocation) == 8); + +const SrcFile* AllRegisteredSrcFiles::find_file(int file_id) const { + for (const SrcFile* file : all_src_files) { + if (file->file_id == file_id) { + return file; + } + } + return nullptr; +} + +const SrcFile* AllRegisteredSrcFiles::find_file(const std::string& abs_filename) const { + for (const SrcFile* file : all_src_files) { + if (file->abs_filename == abs_filename) { + return file; + } + } + return nullptr; +} + +const SrcFile* AllRegisteredSrcFiles::locate_and_register_source_file(const std::string& rel_filename, SrcLocation included_from) { + td::Result path = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::Realpath, rel_filename.c_str()); + if (path.is_error()) { + if (included_from.is_defined()) { + throw ParseError(included_from, "Failed to import: " + path.move_as_error().message().str()); + } + throw Fatal("Failed to locate " + rel_filename + ": " + path.move_as_error().message().str()); + } + + std::string abs_filename = path.move_as_ok(); + if (const SrcFile* file = find_file(abs_filename)) { + return file; + } + + td::Result text = G.settings.read_callback(CompilerSettings::FsReadCallbackKind::ReadFile, abs_filename.c_str()); + if (text.is_error()) { + if (included_from.is_defined()) { + throw ParseError(included_from, "Failed to import: " + text.move_as_error().message().str()); + } + throw Fatal("Failed to read " + rel_filename + ": " + text.move_as_error().message().str()); + } + + SrcFile* created = new SrcFile(++last_registered_file_id, rel_filename, std::move(abs_filename), text.move_as_ok()); + if (G.is_verbosity(1)) { + std::cerr << "register file_id " << created->file_id << " " << created->abs_filename << std::endl; + } + all_src_files.push_back(created); + return created; +} + +SrcFile* AllRegisteredSrcFiles::get_next_unparsed_file() { + if (last_parsed_file_id >= last_registered_file_id) { + return nullptr; + } + return const_cast(all_src_files[++last_parsed_file_id]); +} + +bool SrcFile::is_stdlib_file() const { + std::string_view rel(rel_filename); + return rel.size() > 10 && rel.substr(0, 8) == "@stdlib/"; // common.tolk, tvm-dicts.tolk, etc +} + +bool SrcFile::is_offset_valid(int offset) const { + return offset >= 0 && offset < static_cast(text.size()); +} + +SrcFile::SrcPosition SrcFile::convert_offset(int offset) const { + if (!is_offset_valid(offset)) { + return SrcPosition{offset, -1, -1, "invalid offset"}; + } + + int line_idx = 0; + int char_idx = 0; + int line_offset = 0; + for (int i = 0; i < offset; ++i) { + char c = text[i]; + if (c == '\n') { + line_idx++; + char_idx = 0; + line_offset = i + 1; + } else { + char_idx++; + } + } + + size_t line_len = text.size() - line_offset; + for (int i = line_offset; i < static_cast(text.size()); ++i) { + if (text[i] == '\n') { + line_len = i - line_offset; + break; + } + } + + std::string_view line_str(text.data() + line_offset, line_len); + return SrcPosition{offset, line_idx + 1, char_idx + 1, line_str}; +} + + +std::ostream& operator<<(std::ostream& os, const SrcFile* src_file) { + return os << (src_file ? src_file->rel_filename : "unknown-location"); +} + +std::ostream& operator<<(std::ostream& os, const Fatal& fatal) { + return os << fatal.what(); +} + +const SrcFile* SrcLocation::get_src_file() const { + return G.all_src_files.find_file(file_id); +} + +void SrcLocation::show(std::ostream& os) const { + const SrcFile* src_file = get_src_file(); + os << src_file; + if (src_file && src_file->is_offset_valid(char_offset)) { + SrcFile::SrcPosition pos = src_file->convert_offset(char_offset); + os << ':' << pos.line_no << ':' << pos.char_no; + } +} + +void SrcLocation::show_context(std::ostream& os) const { + const SrcFile* src_file = get_src_file(); + if (!src_file || !src_file->is_offset_valid(char_offset)) { + return; + } + SrcFile::SrcPosition pos = src_file->convert_offset(char_offset); + os << std::right << std::setw(4) << pos.line_no << " | "; + os << pos.line_str << "\n"; + + os << " " << " | "; + for (int i = 1; i < pos.char_no; ++i) { + os << ' '; + } + os << '^' << "\n"; +} + +std::string SrcLocation::to_string() const { + std::ostringstream os; + show(os); + return os.str(); +} + +std::ostream& operator<<(std::ostream& os, SrcLocation loc) { + loc.show(os); + return os; +} + +void SrcLocation::show_general_error(std::ostream& os, const std::string& message, const std::string& err_type) const { + show(os); + if (!err_type.empty()) { + os << ": " << err_type; + } + os << ": " << message << std::endl; + show_context(os); +} + +void SrcLocation::show_note(const std::string& err_msg) const { + show_general_error(std::cerr, err_msg, "note"); +} + +void SrcLocation::show_warning(const std::string& err_msg) const { + show_general_error(std::cerr, err_msg, "warning"); +} + +void SrcLocation::show_error(const std::string& err_msg) const { + show_general_error(std::cerr, err_msg, "error"); +} + +std::ostream& operator<<(std::ostream& os, const ParseError& error) { + error.show(os); + return os; +} + +void ParseError::show(std::ostream& os) const { + os << loc << ": error: " << message << std::endl; + if (current_function) { + os << " // in function `" << current_function->as_human_readable() << "`" << std::endl; + } + loc.show_context(os); +} + +} // namespace tolk diff --git a/tolk/src-file.h b/tolk/src-file.h new file mode 100644 index 00000000..b0f9cba3 --- /dev/null +++ b/tolk/src-file.h @@ -0,0 +1,144 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include +#include +#include "fwd-declarations.h" + +namespace tolk { + +struct SrcFile { + struct SrcPosition { + int offset; + int line_no; + int char_no; + std::string_view line_str; + }; + + struct ImportDirective { + const SrcFile* imported_file; + }; + + int file_id; // an incremental counter through all parsed files + std::string rel_filename; // relative to cwd + std::string abs_filename; // absolute from root + std::string text; // file contents loaded into memory, every Token::str_val points inside it + AnyV ast = nullptr; // when a file has been parsed, its ast_tolk_file is kept here + std::vector imports; // to check strictness (can't use a symbol without importing its file) + + SrcFile(int file_id, std::string rel_filename, std::string abs_filename, std::string&& text) + : file_id(file_id) + , rel_filename(std::move(rel_filename)) + , abs_filename(std::move(abs_filename)) + , text(std::move(text)) { } + + SrcFile(const SrcFile& other) = delete; + SrcFile &operator=(const SrcFile&) = delete; + + bool is_stdlib_file() const; + + bool is_offset_valid(int offset) const; + SrcPosition convert_offset(int offset) const; +}; + + +// SrcLocation points to a location (line, column) in some loaded .tolk source SrcFile. +// Note, that instead of storing src_file, line_no, etc., only 2 ints are stored. +// The purpose is: sizeof(SrcLocation) == 8, so it's just passed/stored without pointers/refs, just like int64_t. +// When decoding SrcLocation into human-readable format, it's converted to SrcFile::SrcPosition via offset. +class SrcLocation { + friend class Lexer; + + int file_id = -1; // = SrcFile::file_id (note, that get_src_file() does linear search) + int char_offset = -1; // offset from SrcFile::text + +public: + + SrcLocation() = default; + explicit SrcLocation(const SrcFile* src_file) : file_id(src_file->file_id) { + } + + bool is_defined() const { return file_id != -1; } + bool is_stdlib() const { return file_id == 0; } + const SrcFile* get_src_file() const; + + // similar to `this->get_src_file() == symbol->get_src_file() || symbol->get_src_file()->is_stdlib()` + // (but effectively, avoiding linear search) + bool is_symbol_from_same_or_builtin_file(SrcLocation symbol_loc) const { + return file_id == symbol_loc.file_id || symbol_loc.file_id < 1; + } + + void show(std::ostream& os) const; + void show_context(std::ostream& os) const; + std::string to_string() const; + + void show_general_error(std::ostream& os, const std::string& message, const std::string& err_type) const; + void show_note(const std::string& err_msg) const; + void show_warning(const std::string& err_msg) const; + void show_error(const std::string& err_msg) const; +}; + +std::ostream& operator<<(std::ostream& os, SrcLocation loc); + +class AllRegisteredSrcFiles { + std::vector all_src_files; + int last_registered_file_id = -1; + int last_parsed_file_id = -1; + +public: + const SrcFile* find_file(int file_id) const; + const SrcFile* find_file(const std::string& abs_filename) const; + + const SrcFile* locate_and_register_source_file(const std::string& rel_filename, SrcLocation included_from); + SrcFile* get_next_unparsed_file(); + + auto begin() const { return all_src_files.begin(); } + auto end() const { return all_src_files.end(); } +}; + +struct Fatal final : std::exception { + std::string message; + + explicit Fatal(std::string _msg) : message(std::move(_msg)) { + } + const char* what() const noexcept override { + return message.c_str(); + } +}; + +std::ostream& operator<<(std::ostream& os, const Fatal& fatal); + +struct ParseError : std::exception { + FunctionPtr current_function; + SrcLocation loc; + std::string message; + + ParseError(SrcLocation loc, std::string message) + : current_function(nullptr), loc(loc), message(std::move(message)) {} + ParseError(FunctionPtr current_function, SrcLocation loc, std::string message) + : current_function(current_function), loc(loc), message(std::move(message)) {} + + const char* what() const noexcept override { + return message.c_str(); + } + void show(std::ostream& os) const; +}; + +std::ostream& operator<<(std::ostream& os, const ParseError& error); + +} // namespace tolk diff --git a/tolk/stack-transform.cpp b/tolk/stack-transform.cpp new file mode 100644 index 00000000..fe5735e5 --- /dev/null +++ b/tolk/stack-transform.cpp @@ -0,0 +1,1054 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "tolk.h" + +namespace tolk { + +/* + * + * GENERIC STACK TRANSFORMATIONS + * + */ + +StackTransform::StackTransform(std::initializer_list list) { + *this = list; +} + +StackTransform &StackTransform::operator=(std::initializer_list list) { + if (list.size() > 255) { + invalidate(); + return *this; + } + set_id(); + if (!list.size()) { + return *this; + } + int m = (int)list.size(); + d = list.begin()[m - 1] - (m - 1); + if (d >= 128 || d < -128) { + invalidate(); + return *this; + } + for (int i = 0; i < m - 1; i++) { + int x = d + i; + int y = list.begin()[i]; + if (y != x) { + if (x != (short)x || y != (short)y || n == max_n) { + invalidate(); + return *this; + } + dp = std::max(dp, std::max(x, y) + 1); + A[n++] = std::make_pair((short)x, (short)y); + } + } + return *this; +} + +bool StackTransform::assign(const StackTransform &other) { + if (!other.is_valid() || (unsigned)other.n > max_n) { + return invalidate(); + } + d = other.d; + n = other.n; + dp = other.dp; + c = other.c; + invalid = false; + for (int i = 0; i < n; i++) { + A[i] = other.A[i]; + } + return true; +} + +int StackTransform::get(int x) const { + if (!is_valid()) { + return -1; + } + if (x <= c_start) { + return x - c; + } + x += d; + int i; + for (i = 0; i < n && A[i].first < x; i++) { + } + if (i < n && A[i].first == x) { + return A[i].second; + } else { + return x; + } +} + +bool StackTransform::set(int x, int y, bool relaxed) { + if (!is_valid()) { + return false; + } + if (x < 0) { + return (relaxed && y == x + d) || invalidate(); + } + if (!relaxed) { + touch(x); + } + x += d; + int i; + for (i = 0; i < n && A[i].first < x; i++) { + } + if (i < n && A[i].first == x) { + if (x != y) { + if (y != (short)y) { + return invalidate(); + } + A[i].second = (short)y; + } else { + --n; + for (; i < n; i++) { + A[i] = A[i + 1]; + } + } + } else { + if (x != y) { + if (x != (short)x || y != (short)y || n == max_n) { + return invalidate(); + } + for (int j = n++; j > i; j--) { + A[j] = A[j - 1]; + } + A[i].first = (short)x; + A[i].second = (short)y; + touch(x - d); + touch(y); + } + } + return true; +} + +// f(x') = x' + d for all x' >= x ? +bool StackTransform::is_trivial_after(int x) const { + return is_valid() && (!n || A[n - 1].first < x + d); +} + +// card f^{-1}(y) +int StackTransform::preimage_count(int y) const { + if (!is_valid()) { + return -1; + } + int count = (y >= d); + for (const auto &pair : A) { + if (pair.second == y) { + ++count; + } else if (pair.first == y) { + --count; + } + } + return count; +} + +// f^{-1}(y) +std::vector StackTransform::preimage(int y) const { + if (!is_valid()) { + return {}; + } + std::vector res; + bool f = (y >= d); + for (const auto &pair : A) { + if (pair.first > y && f) { + res.push_back(y - d); + f = false; + } + if (pair.first == y) { + f = false; + } else if (pair.second == y) { + res.push_back(pair.first - d); + } + } + return res; +} + +// is f:N->N bijective ? +bool StackTransform::is_permutation() const { + if (!is_valid() || d) { + return false; + } + tolk_assert(n <= max_n); + std::array X, Y; + for (int i = 0; i < n; i++) { + X[i] = A[i].first; + Y[i] = A[i].second; + if (Y[i] < 0) { + return false; + } + } + std::sort(Y.begin(), Y.begin() + n); + for (int i = 0; i < n; i++) { + if (X[i] != Y[i]) { + return false; + } + } + return true; +} + +bool StackTransform::remove_negative() { + int s = 0; + while (s < n && A[s].first < d) { + ++s; + } + if (s) { + n -= s; + for (int i = 0; i < n; i++) { + A[i] = A[i + s]; + } + } + return true; +} + +int StackTransform::try_load(int &i, int offs) const { + return i < n ? A[i++].first + offs : inf_x; +} + +bool StackTransform::try_store(int x, int y) { + if (x == y || x < d) { + return true; + } + if (n == max_n || x != (short)x || y != (short)y) { + return invalidate(); + } + A[n].first = (short)x; + A[n++].second = (short)y; + return true; +} + +// c := a * b +bool StackTransform::compose(const StackTransform &a, const StackTransform &b, StackTransform &c) { + if (!a.is_valid() || !b.is_valid()) { + return c.invalidate(); + } + c.d = a.d + b.d; + c.n = 0; + c.dp = std::max(a.dp, b.dp + a.d); + c.c = a.c + b.c; + c.invalid = false; + int i = 0, j = 0; + int x1 = a.try_load(i); + int x2 = b.try_load(j, a.d); + while (true) { + if (x1 < x2) { + int y = a.A[i - 1].second; + if (!c.try_store(x1, y)) { + return false; + } + x1 = a.try_load(i); + } else if (x2 < inf_x) { + if (x1 == x2) { + x1 = a.try_load(i); + } + int y = b.A[j - 1].second; + if (!c.try_store(x2, a(y))) { + return false; + } + x2 = b.try_load(j, a.d); + } else { + return true; + } + } +} + +// this = this * other +bool StackTransform::apply(const StackTransform &other) { + StackTransform res; + if (!compose(*this, other, res)) { + return invalidate(); + } + return assign(res); +} + +// this = other * this +bool StackTransform::preapply(const StackTransform &other) { + StackTransform res; + if (!compose(other, *this, res)) { + return invalidate(); + } + return assign(res); +} + +StackTransform StackTransform::operator*(const StackTransform &b) const & { + StackTransform res; + compose(*this, b, res); + return res; +} + +// this = this * other +StackTransform &StackTransform::operator*=(const StackTransform &other) { + StackTransform res; + (compose(*this, other, res) && assign(res)) || invalidate(); + return *this; +} + +bool StackTransform::apply_xchg(int i, int j, bool relaxed) { + if (!is_valid() || i < 0 || j < 0) { + return invalidate(); + } + if (i == j) { + return relaxed || touch(i); + } + int u = touch_get(i), v = touch_get(j); + return set(i, v) && set(j, u); +} + +bool StackTransform::apply_push(int i) { + if (!is_valid() || i < 0) { + return invalidate(); + } + int u = touch_get(i); + return shift(-1) && set(0, u); +} + +bool StackTransform::apply_push_newconst() { + if (!is_valid()) { + return false; + } + return shift(-1) && set(0, c_start - c++); +} + +bool StackTransform::apply_pop(int i) { + if (!is_valid() || i < 0) { + return invalidate(); + } + if (!i) { + return touch(0) && shift(1); + } else { + return set(i, get(0)) && shift(1); + } +} + +bool StackTransform::apply_blkpop(int k) { + if (!is_valid() || k < 0) { + return invalidate(); + } + return !k || (touch(k - 1) && shift(k)); +} + +bool StackTransform::equal(const StackTransform &other, bool relaxed) const { + if (!is_valid() || !other.is_valid()) { + return false; + } + if (!(n == other.n && d == other.d)) { + return false; + } + for (int i = 0; i < n; i++) { + if (A[i] != other.A[i]) { + return false; + } + } + return relaxed || dp == other.dp; +} + +StackTransform StackTransform::Xchg(int i, int j, bool relaxed) { + StackTransform t; + t.apply_xchg(i, j, relaxed); + return t; +} + +StackTransform StackTransform::Push(int i) { + StackTransform t; + t.apply_push(i); + return t; +} + +StackTransform StackTransform::Pop(int i) { + StackTransform t; + t.apply_pop(i); + return t; +} + +bool StackTransform::is_xchg(int i, int j) const { + if (i == j) { + return is_id(); + } + return is_valid() && !d && n == 2 && i >= 0 && j >= 0 && get(i) == j && get(j) == i; +} + +bool StackTransform::is_xchg(int *i, int *j) const { + if (!is_valid() || d || n > 2 || !dp) { + return false; + } + if (!n) { + *i = *j = 0; + return true; + } + if (n != 2) { + return false; + } + int a = A[0].first, b = A[1].first; + if (A[0].second != b || A[1].second != a) { + return false; + } + *i = std::min(a, b); + *j = std::max(a, b); + return true; +} + +bool StackTransform::is_xchg_xchg(int i, int j, int k, int l) const { + if (is_valid() && !d && n <= 4 && (i | j | k | l) >= 0) { + StackTransform t; + return t.apply_xchg(i, j) && t.apply_xchg(k, l) && t <= *this; + } else { + return false; + } +} + +bool StackTransform::is_xchg_xchg(int *i, int *j, int *k, int *l) const { + if (!is_valid() || d || n > 4 || !dp || !is_permutation()) { + return false; + } + if (!n) { + *i = *j = *k = *l = 0; + return true; + } + if (n <= 2) { + *k = *l = 0; + return is_xchg(i, j); + } + if (n == 3) { + // rotation: a -> b -> c -> a + int a = A[0].first; + int b = A[0].second; + int s = (b == A[2].first ? 2 : 1); + int c = A[s].second; + if (b != A[s].first || c != A[3 - s].first || a != A[3 - s].second) { + return false; + } + // implement as XCHG s(a),s(c) ; XCHG s(a),s(b) + *i = *k = a; + *j = c; + *l = b; + return is_xchg_xchg(*i, *j, *k, *l); + } + *i = A[0].first; + *j = A[0].second; + if (get(*j) != *i) { + return false; + } + for (int s = 1; s < 4; s++) { + if (A[s].first != *j) { + *k = A[s].first; + *l = A[s].second; + return get(*l) == *k && is_xchg_xchg(*i, *j, *k, *l); + } + } + return false; +} + +bool StackTransform::is_push(int i) const { + return is_valid() && d == -1 && n == 1 && A[0].first == -1 && A[0].second == i; +} + +bool StackTransform::is_push(int *i) const { + if (is_valid() && d == -1 && n == 1 && A[0].first == -1 && A[0].second >= 0) { + *i = A[0].second; + return true; + } else { + return false; + } +} + +// 1 2 3 4 .. = pop0 +// 0 2 3 4 .. = pop1 +// 1 0 3 4 .. = pop2 +// 1 2 0 4 .. = pop3 +// POP s(i) : 1 2 ... i-1 0 i+1 ... ; d=1, n=1, {(i,0)} +bool StackTransform::is_pop(int i) const { + if (!is_valid() || d != 1 || n > 1 || i < 0) { + return false; + } + if (!i) { + return !n; + } + return n == 1 && A[0].first == i && !A[0].second; +} + +bool StackTransform::is_pop(int *i) const { + if (!is_valid() || d != 1 || n > 1) { + return false; + } + if (!n) { + *i = 0; + return true; + } + if (n == 1 && !A[0].second) { + *i = A[0].first; + return true; + } + return false; +} + +// POP s(i) ; POP s(j) : 2 ... i-1 0 i+1 ... j 1 j+2 ... ; d=2, n=2, {(i,0),(j+1,1)} if i <> j+1 +bool StackTransform::is_pop_pop(int i, int j) const { + if (is_valid() && d == 2 && n <= 2 && i >= 0 && j >= 0) { + StackTransform t; + return t.apply_pop(i) && t.apply_pop(j) && t <= *this; + } else { + return false; + } +} + +bool StackTransform::is_pop_pop(int *i, int *j) const { + if (!is_valid() || d != 2 || n > 2) { + return false; + } + if (!n) { + *i = *j = 0; // 2DROP + } else if (n == 2) { + *i = A[0].first - A[0].second; + *j = A[1].first - A[1].second; + if (A[0].second > A[1].second) { + std::swap(*i, *j); + } + } else if (!A[0].second) { + *i = A[0].first; + *j = 0; + } else { + *i = 0; + *j = A[0].first - 1; + } + return is_pop_pop(*i, *j); +} + +const StackTransform StackTransform::rot{2, 0, 1, 3}; +const StackTransform StackTransform::rot_rev{1, 2, 0, 3}; + +bool StackTransform::is_rot() const { + return equal(rot, true); +} + +bool StackTransform::is_rotrev() const { + return equal(rot_rev, true); +} + +// PUSH i ; ROT == 1 i 0 2 3 +bool StackTransform::is_push_rot(int i) const { + return is_valid() && d == -1 && i >= 0 && is_trivial_after(3) && get(0) == 1 && get(1) == i && get(2) == 0; +} + +bool StackTransform::is_push_rot(int *i) const { + return is_valid() && (*i = get(1)) >= 0 && is_push_rot(*i); +} + +// PUSH i ; -ROT == 0 1 i 2 3 +bool StackTransform::is_push_rotrev(int i) const { + return is_valid() && d == -1 && i >= 0 && is_trivial_after(3) && get(0) == 0 && get(1) == 1 && get(2) == i; +} + +bool StackTransform::is_push_rotrev(int *i) const { + return is_valid() && (*i = get(2)) >= 0 && is_push_rotrev(*i); +} + +// PUSH s(i) ; XCHG s(j),s(k) --> i 0 1 .. i .. +// PUSH s(i) ; XCHG s(0),s(k) --> k-1 0 1 .. k-2 i k .. +bool StackTransform::is_push_xchg(int i, int j, int k) const { + StackTransform t; + return is_valid() && d == -1 && n <= 3 && t.apply_push(i) && t.apply_xchg(j, k) && t <= *this; +} + +bool StackTransform::is_push_xchg(int *i, int *j, int *k) const { + if (!(is_valid() && d == -1 && n <= 3 && n > 0)) { + return false; + } + int s = get(0); + if (s < 0) { + return false; + } + *i = s; + *j = 0; + if (n == 1) { + *k = 0; + } else if (n == 2) { + *k = s + 1; + *i = get(s + 1); + } else { + *j = A[1].first + 1; + *k = A[2].first + 1; + } + return is_push_xchg(*i, *j, *k); +} + +// XCHG s1,s(i) ; XCHG s0,s(j) +bool StackTransform::is_xchg2(int i, int j) const { + StackTransform t; + return is_valid() && !d && t.apply_xchg(1, i) && t.apply_xchg(0, j) && t <= *this; +} + +bool StackTransform::is_xchg2(int *i, int *j) const { + if (!is_valid() || d || n > 4 || n == 1 || dp < 2) { + return false; + } + *i = get(1); + *j = get(0); + if (!n) { + return true; + } + if (*i < 0 || *j < 0) { + return false; + } + if (n == 2 && !*i) { + *j = *i; // XCHG s0,s1 = XCHG2 s0,s0 + } else if (n == 3 && *i) { + // XCHG2 s(i),s(i) = XCHG s1,s(i) ; XCHG s0,s(i) : 0->1, 1->i + *j = *i; + } // XCHG2 s0,s(i) = XCHG s0,s1 ; XCHG s0,s(i) : 0->i, 1->0 + return is_xchg2(*i, *j); +} + +// XCHG s0,s(i) ; PUSH s(j) = PUSH s(j') ; XCHG s1,s(i+1) +// j'=j if j!=0, j!=i +// j'=0 if j=i +// j'=i if j=0 +bool StackTransform::is_xcpu(int i, int j) const { + StackTransform t; + return is_valid() && d == -1 && t.apply_xchg(0, i) && t.apply_push(j) && t <= *this; +} + +bool StackTransform::is_xcpu(int *i, int *j) const { + if (!is_valid() || d != -1 || n > 3 || dp < 1) { + return false; + } + *i = get(1); + *j = get(0); + if (!*j) { + *j = *i; + } else if (*j == *i) { + *j = 0; + } + return is_xcpu(*i, *j); +} + +// PUSH s(i) ; XCHG s0, s1 ; XCHG s0, s(j+1) +bool StackTransform::is_puxc(int i, int j) const { + StackTransform t; + return is_valid() && d == -1 && t.apply_push(i) && t.apply_xchg(0, 1) && t.apply_xchg(0, j + 1) && t <= *this; +} + +// j > 0 : 0 -> j, 1 -> i +// j = 0 : 0 -> i, 1 -> 0 ( PUSH s(i) ) +// j = -1 : 0 -> 0, 1 -> i ( PUSH s(i) ; XCHG s0, s1 ) +bool StackTransform::is_puxc(int *i, int *j) const { + if (!is_valid() || d != -1 || n > 3) { + return false; + } + *i = get(1); + *j = get(0); + if (!*i && is_push(*j)) { + std::swap(*i, *j); + return is_puxc(*i, *j); + } + if (!*j) { + --*j; + } + return is_puxc(*i, *j); +} + +// PUSH s(i) ; PUSH s(j+1) +bool StackTransform::is_push2(int i, int j) const { + StackTransform t; + return is_valid() && d == -2 && t.apply_push(i) && t.apply_push(j + 1) && t <= *this; +} + +bool StackTransform::is_push2(int *i, int *j) const { + if (!is_valid() || d != -2 || n > 2) { + return false; + } + *i = get(1); + *j = get(0); + return is_push2(*i, *j); +} + +// XCHG s2,s(i) ; XCHG s1,s(j) ; XCHG s0,s(k) +bool StackTransform::is_xchg3(int *i, int *j, int *k) const { + if (!is_valid() || d || dp < 3 || !is_permutation()) { + return false; + } + for (int s = 2; s >= 0; s--) { + *i = get(s); + StackTransform t = Xchg(2, *i) * *this; + if (t.is_xchg2(j, k)) { + return true; + } + } + return false; +} + +// XCHG s1,s(i) ; XCHG s0,s(j) ; PUSH s(k) +bool StackTransform::is_xc2pu(int *i, int *j, int *k) const { + if (!is_valid() || d != -1 || dp < 2) { + return false; + } + for (int s = 2; s >= 1; s--) { + *i = get(s); + StackTransform t = Xchg(1, *i) * *this; + if (t.is_xcpu(j, k)) { + return true; + } + } + return false; +} + +// XCHG s1,s(i) ; PUSH s(j) ; XCHG s0,s1 ; XCHG s0,s(k+1) +bool StackTransform::is_xcpuxc(int *i, int *j, int *k) const { + if (!is_valid() || d != -1 || dp < 2) { + return false; + } + for (int s = 2; s >= 0; s--) { + *i = get(s); + StackTransform t = Xchg(1, *i) * *this; + if (t.is_puxc(j, k)) { + return true; + } + } + return false; +} + +// XCHG s0,s(i) ; PUSH s(j) ; PUSH s(k+1) +bool StackTransform::is_xcpu2(int *i, int *j, int *k) const { + if (!is_valid() || d != -2 || dp < 1) { + return false; + } + *i = get(2); + StackTransform t = Xchg(0, *i) * *this; + return t.is_push2(j, k); +} + +// PUSH s(i) ; XCHG s0,s2 ; XCHG s1,s(j+1) ; XCHG s0,s(k+1) +// 0 -> i or 1 -> i or 2 -> i ; i has two preimages +// 0 -> k if k >= 2, k != j +// 1 -> j=k if j = k >= 2 +// 1 -> j if j >= 2, k != 0 +// 0 -> j if j >= 2, k = 0 +// => i in {f(0), f(1), f(2)} ; j in {-1, 0, 1, f(0), f(1)} ; k in {-1, 0, 1, f(0), f(1)} +bool StackTransform::is_puxc2(int *i, int *j, int *k) const { + if (!is_valid() || d != -1 || dp < 2) { + return false; + } + for (int s = 2; s >= 0; s--) { + *i = get(s); + if (preimage_count(*i) != 2) { + continue; + } + for (int u = -1; u <= 3; u++) { + *j = (u >= 2 ? get(u - 2) : u); + for (int v = -1; v <= 3; v++) { + *k = (v >= 2 ? get(v - 2) : v); + if (is_puxc2(*i, *j, *k)) { + return true; + } + } + } + } + return false; +} + +// PUSH s(i) ; XCHG s0,s2 ; XCHG s1,s(j+1) ; XCHG s0,s(k+1) +bool StackTransform::is_puxc2(int i, int j, int k) const { + StackTransform t; + return is_valid() && d == -1 && dp >= 2 // basic checks + && t.apply_push(i) && t.apply_xchg(0, 2) // PUSH s(i) ; XCHG s0,s2 + && t.apply_xchg(1, j + 1) // XCHG s1,s(j+1) + && t.apply_xchg(0, k + 1) && t <= *this; // XCHG s0,s(k+2) +} + +// PUSH s(i) ; XCHG s0,s1 ; XCHG s0,s(j+1) ; PUSH s(k+1) +bool StackTransform::is_puxcpu(int *i, int *j, int *k) const { + if (!is_valid() || d != -2 || dp < 1) { + return false; + } + StackTransform t = *this; + if (t.apply_pop() && t.is_puxc(i, j)) { + int y = get(0); + auto v = t.preimage(y); + if (!v.empty()) { + *k = v[0] - 1; + t.apply_push(*k + 1); + return t <= *this; + } + } + return false; +} + +// PUSH s(i) ; XCHG s0,s1 ; PUSH s(j+1) ; XCHG s0,s1 ; XCHG s0,s(k+2) +// 2 -> i; 1 -> j (if j >= 1, k != -1), 1 -> i (if j = 0, k != -1), 1 -> 0 (if j = -1, k != -1) +// 0 -> k (if k >= 1), 0 -> i (if k = 0), 0 -> j (if k = -1, j >= 1) +bool StackTransform::is_pu2xc(int *i, int *j, int *k) const { + if (!is_valid() || d != -2 || dp < 1) { + return false; + } + *i = get(2); + for (int v = -2; v <= 1; v++) { + *k = (v <= 0 ? v : get(0)); // one of -2, -1, 0, get(0) + for (int u = -1; u <= 1; u++) { + *j = (u <= 0 ? u : get(v != -1)); // one of -1, 0, get(0), get(1) + if (is_pu2xc(*i, *j, *k)) { + return true; + } + } + } + return false; +} + +bool StackTransform::is_pu2xc(int i, int j, int k) const { + StackTransform t; + return is_valid() && d == -2 && dp >= 1 // basic checks + && t.apply_push(i) && t.apply_xchg(0, 1) // PUSH s(i) ; XCHG s0,s1 + && t.apply_push(j + 1) && t.apply_xchg(0, 1) // PUSH s(j+1) ; XCHG s0,s1 + && t.apply_xchg(0, k + 2) && t <= *this; // XCHG s0,s(k+2) +} + +// PUSH s(i) ; PUSH s(j+1) ; PUSH s(k+2) +bool StackTransform::is_push3(int i, int j, int k) const { + StackTransform t; + return is_valid() && d == -3 && t.apply_push(i) && t.apply_push(j + 1) && t.apply_push(k + 2) && t <= *this; +} + +bool StackTransform::is_push3(int *i, int *j, int *k) const { + if (!is_valid() || d != -3 || n > 3) { + return false; + } + *i = get(2); + *j = get(1); + *k = get(0); + return is_push3(*i, *j, *k); +} + +bool StackTransform::is_blkswap(int *i, int *j) const { + if (!is_valid() || d || !is_permutation()) { + return false; + } + *j = get(0); + if (*j <= 0) { + return false; + } + auto v = preimage(0); + if (v.size() != 1) { + return false; + } + *i = v[0]; + return *i > 0 && is_blkswap(*i, *j); +} + +bool StackTransform::is_blkswap(int i, int j) const { + if (!is_valid() || d || i <= 0 || j <= 0 || dp < i + j || !is_trivial_after(i + j)) { + return false; + } + for (int s = 0; s < i; s++) { + if (get(s) != s + j) { + return false; + } + } + for (int s = 0; s < j; s++) { + if (get(s + i) != s) { + return false; + } + } + return true; +} + +// equivalent to i times DROP +bool StackTransform::is_blkdrop(int *i) const { + if (is_valid() && d > 0 && !n) { + *i = d; + return true; + } + return false; +} + +// 0 1 .. j-1 j+i j+i+1 ... +bool StackTransform::is_blkdrop2(int i, int j) const { + if (!is_valid() || d != i || i <= 0 || j < 0 || dp < i + j || n != j || !is_trivial_after(j)) { + return false; + } + for (int s = 0; s < j; s++) { + if (get(s) != s) { + return false; + } + } + return true; +} + +bool StackTransform::is_blkdrop2(int *i, int *j) const { + if (is_valid() && is_blkdrop2(d, n)) { + *i = d; + *j = n; + return true; + } + return false; +} + +// equivalent to i times PUSH s(j) +bool StackTransform::is_blkpush(int *i, int *j) const { + if (!is_valid() || d >= 0) { + return false; + } + *i = -d; + *j = get(*i - 1); + return is_blkpush(*i, *j); +} + +bool StackTransform::is_blkpush(int i, int j) const { + if (!is_valid() || d >= 0 || d != -i || j < 0 || dp < i + j || !is_trivial_after(i)) { + return false; + } + StackTransform t; + for (int s = 0; s < i; s++) { + if (!t.apply_push(j)) { + return false; + } + } + return t <= *this; +} + +bool StackTransform::is_reverse(int *i, int *j) const { + if (!is_valid() || d || !is_permutation() || n < 2) { + return false; + } + *j = A[0].first; + *i = A[n - 1].first - A[0].first + 1; + return is_reverse(*i, *j); +} + +bool StackTransform::is_reverse(int i, int j) const { + if (!is_valid() || d || !is_trivial_after(i + j) || n < 2 || A[0].first != j || A[n - 1].first != j + i - 1) { + return false; + } + for (int s = 0; s < i; s++) { + if (get(j + s) != j + i - 1 - s) { + return false; + } + } + return true; +} + +// 0 i+1 i+2 ... == i*NIP +// j i+1 i+2 ... == XCHG s(i),s(j) ; BLKDROP i +bool StackTransform::is_nip_seq(int i, int j) const { + return is_valid() && d == i && i > j && j >= 0 && n == 1 && A[0].first == i && A[0].second == j; +} + +bool StackTransform::is_nip_seq(int *i) const { + *i = d; + return is_nip_seq(*i); +} + +bool StackTransform::is_nip_seq(int *i, int *j) const { + if (is_valid() && n > 0) { + *i = d; + *j = A[0].second; + return is_nip_seq(*i, *j); + } else { + return false; + } +} + +// POP s(i); BLKDROP k (usually for i >= k >= 0) +bool StackTransform::is_pop_blkdrop(int i, int k) const { + StackTransform t; + return is_valid() && d == k + 1 && t.apply_pop(i) && t.apply_blkpop(k) && t <= *this; +} + +// POP s(i); BLKDROP k == XCHG s0,s(i); BLKDROP k+1 for i >= k >= 0 +// k+1 k+2 .. i-1 0 i+1 .. +bool StackTransform::is_pop_blkdrop(int *i, int *k) const { + if (is_valid() && n == 1 && d > 0 && !A[0].second) { + *k = d - 1; + *i = A[0].first; + return is_pop_blkdrop(*i, *k); + } else { + return false; + } +} + +// POP s(i); POP s(j); BLKDROP k (usually for i<>j >= k >= 0) +bool StackTransform::is_2pop_blkdrop(int i, int j, int k) const { + StackTransform t; + return is_valid() && d == k + 2 && t.apply_pop(i) && t.apply_pop(j) && t.apply_blkpop(k) && t <= *this; +} + +// POP s(i); POP s(j); BLKDROP k == XCHG s0,s(i); XCHG s1,s(j+1); BLKDROP k+2 (usually for i<>j >= k >= 2) +// k+2 k+3 .. i-1 0 i+1 ... j 1 j+2 ... +bool StackTransform::is_2pop_blkdrop(int *i, int *j, int *k) const { + if (is_valid() && n == 2 && d >= 2 && A[0].second + A[1].second == 1) { + *k = d - 2; + int t = (A[0].second > 0); + *i = A[t].first; + *j = A[1 - t].first - 1; + return is_2pop_blkdrop(*i, *j, *k); + } else { + return false; + } +} + +// PUSHCONST c ; ROT == 1 -1000 0 2 3 +bool StackTransform::is_const_rot(int c) const { + return is_valid() && d == -1 && is_trivial_after(3) && get(0) == 1 && c <= c_start && get(1) == c && get(2) == 0; +} + +bool StackTransform::is_const_rot(int *c) const { + return is_valid() && (*c = get(1)) <= c_start && is_const_rot(*c); +} + +// PUSHCONST c ; POP s(i) == 0 1 .. i-1 -1000 i+1 ... +bool StackTransform::is_const_pop(int c, int i) const { + return is_valid() && !d && n == 1 && i > 0 && c <= c_start && get(i - 1) == c; +} + +bool StackTransform::is_const_pop(int *c, int *i) const { + if (is_valid() && !d && n == 1 && A[0].second <= c_start) { + *i = A[0].first + 1; + *c = A[0].second; + return is_const_pop(*c, *i); + } else { + return false; + } +} + +// PUSH i ; PUSHCONST c == c i 0 1 2 ... +bool StackTransform::is_push_const(int i, int c) const { + return is_valid() && d == -2 && c <= c_start && i >= 0 && is_trivial_after(2) && get(0) == c && get(1) == i; +} + +bool StackTransform::is_push_const(int *i, int *c) const { + return is_valid() && d == -2 && n == 2 && is_push_const(*i = get(1), *c = get(0)); +} + +void StackTransform::show(std::ostream &os, int mode) const { + if (!is_valid()) { + os << ""; + return; + } + int mi = 0, ma = 0; + if (n > 0 && A[0].first < d) { + mi = A[0].first - d; + } + if (n > 0) { + ma = std::max(ma, A[n - 1].first - d + 1); + } + ma = std::max(ma + 1, dp - d); + os << '{'; + if (dp == d) { + os << '|'; + } + for (int i = mi; i < ma; i++) { + os << get(i) << (i == -1 ? '?' : (i == dp - d - 1 ? '|' : ' ')); + } + os << get(ma) << "..}"; +} + +} // namespace tolk diff --git a/tolk/symtable.cpp b/tolk/symtable.cpp new file mode 100644 index 00000000..51dc3440 --- /dev/null +++ b/tolk/symtable.cpp @@ -0,0 +1,148 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "symtable.h" +#include "compiler-state.h" +#include "platform-utils.h" +#include "generics-helpers.h" + +namespace tolk { + +std::string FunctionData::as_human_readable() const { + if (!genericTs) { + return name; // if it's generic instantiation like `f`, its name is "f", not "f" + } + return name + genericTs->as_human_readable(); +} + +bool FunctionData::does_need_codegen() const { + // when a function is declared, but not referenced from code in any way, don't generate its body + if (!is_really_used() && G.settings.remove_unused_functions) { + return false; + } + // functions with asm body don't need code generation + // (even if used as non-call: `var a = beginCell;` inserts TVM continuation inline) + if (is_asm_function() || is_builtin_function()) { + return false; + } + // when a function is referenced like `var a = some_fn;` (or in some other non-call way), its continuation should exist + if (is_used_as_noncall()) { + return true; + } + // generic functions also don't need code generation, only generic instantiations do + if (is_generic_function()) { + return false; + } + // currently, there is no inlining, all functions are codegenerated + // (but actually, unused ones are later removed by Fift) + // in the future, we may want to implement a true AST inlining for "simple" functions + return true; +} + +void FunctionData::assign_resolved_type(TypePtr declared_return_type) { + this->declared_return_type = declared_return_type; +} + +void FunctionData::assign_inferred_type(TypePtr inferred_return_type, TypePtr inferred_full_type) { + this->inferred_return_type = inferred_return_type; + this->inferred_full_type = inferred_full_type; +} + +void FunctionData::assign_is_used_as_noncall() { + this->flags |= flagUsedAsNonCall; +} + +void FunctionData::assign_is_implicit_return() { + this->flags |= flagImplicitReturn; +} + +void FunctionData::assign_is_type_inferring_done() { + this->flags |= flagTypeInferringDone; +} + +void FunctionData::assign_is_really_used() { + this->flags |= flagReallyUsed; +} + +void FunctionData::assign_arg_order(std::vector&& arg_order) { + this->arg_order = std::move(arg_order); +} + +void GlobalVarData::assign_resolved_type(TypePtr declared_type) { + this->declared_type = declared_type; +} + +void GlobalVarData::assign_is_really_used() { + this->flags |= flagReallyUsed; +} + +void GlobalConstData::assign_resolved_type(TypePtr declared_type) { + this->declared_type = declared_type; +} + +void LocalVarData::assign_ir_idx(std::vector&& ir_idx) { + this->ir_idx = std::move(ir_idx); +} + +void LocalVarData::assign_resolved_type(TypePtr declared_type) { + this->declared_type = declared_type; +} + +void LocalVarData::assign_inferred_type(TypePtr inferred_type) { + this->declared_type = inferred_type; +} + +GNU_ATTRIBUTE_NORETURN GNU_ATTRIBUTE_COLD +static void fire_error_redefinition_of_symbol(SrcLocation loc, const Symbol* previous) { + SrcLocation prev_loc = previous->loc; + if (prev_loc.is_stdlib()) { + throw ParseError(loc, "redefinition of a symbol from stdlib"); + } + if (prev_loc.is_defined()) { + throw ParseError(loc, "redefinition of symbol, previous was at: " + prev_loc.to_string()); + } + throw ParseError(loc, "redefinition of built-in symbol"); +} + +void GlobalSymbolTable::add_function(FunctionPtr f_sym) { + auto key = key_hash(f_sym->name); + auto [it, inserted] = entries.emplace(key, f_sym); + if (!inserted) { + fire_error_redefinition_of_symbol(f_sym->loc, it->second); + } +} + +void GlobalSymbolTable::add_global_var(GlobalVarPtr g_sym) { + auto key = key_hash(g_sym->name); + auto [it, inserted] = entries.emplace(key, g_sym); + if (!inserted) { + fire_error_redefinition_of_symbol(g_sym->loc, it->second); + } +} + +void GlobalSymbolTable::add_global_const(GlobalConstPtr c_sym) { + auto key = key_hash(c_sym->name); + auto [it, inserted] = entries.emplace(key, c_sym); + if (!inserted) { + fire_error_redefinition_of_symbol(c_sym->loc, it->second); + } +} + +const Symbol* lookup_global_symbol(std::string_view name) { + return G.symtable.lookup(name); +} + +} // namespace tolk diff --git a/tolk/symtable.h b/tolk/symtable.h new file mode 100644 index 00000000..9419afce --- /dev/null +++ b/tolk/symtable.h @@ -0,0 +1,239 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "src-file.h" +#include "fwd-declarations.h" +#include "constant-evaluator.h" +#include "crypto/common/refint.h" +#include +#include +#include + +namespace tolk { + +struct Symbol { + std::string name; + SrcLocation loc; + + Symbol(std::string name, SrcLocation loc) + : name(std::move(name)) + , loc(loc) { + } + + virtual ~Symbol() = default; + + template + ConstTPtr try_as() const { +#ifdef TOLK_DEBUG + assert(this != nullptr); +#endif + return dynamic_cast(this); + } +}; + +struct LocalVarData final : Symbol { + enum { + flagMutateParameter = 1, // parameter was declared with `mutate` keyword + flagImmutable = 2, // variable was declared via `val` (not `var`) + }; + + TypePtr declared_type; // either at declaration `var x:int`, or if omitted, from assigned value `var x=2` + int flags; + int param_idx; // 0...N for function parameters, -1 for local vars + std::vector ir_idx; + + LocalVarData(std::string name, SrcLocation loc, TypePtr declared_type, int flags, int param_idx) + : Symbol(std::move(name), loc) + , declared_type(declared_type) + , flags(flags) + , param_idx(param_idx) { + } + + bool is_parameter() const { return param_idx >= 0; } + + bool is_immutable() const { return flags & flagImmutable; } + bool is_mutate_parameter() const { return flags & flagMutateParameter; } + + LocalVarData* mutate() const { return const_cast(this); } + void assign_ir_idx(std::vector&& ir_idx); + void assign_resolved_type(TypePtr declared_type); + void assign_inferred_type(TypePtr inferred_type); +}; + +struct FunctionBodyCode; +struct FunctionBodyAsm; +struct FunctionBodyBuiltin; +struct GenericsDeclaration; +struct GenericsInstantiation; + +typedef std::variant< + FunctionBodyCode*, + FunctionBodyAsm*, + FunctionBodyBuiltin* +> FunctionBody; + +struct FunctionData final : Symbol { + static constexpr int EMPTY_METHOD_ID = -10; + + enum { + flagInline = 1, // marked `@inline` + flagInlineRef = 2, // marked `@inline_ref` + flagTypeInferringDone = 4, // type inferring step of function's body (all AST nodes assigning v->inferred_type) is done + flagUsedAsNonCall = 8, // used not only as `f()`, but as a 1-st class function (assigned to var, pushed to tuple, etc.) + flagMarkedAsPure = 16, // declared as `pure`, can't call impure and access globals, unused invocations are optimized out + flagImplicitReturn = 32, // control flow reaches end of function, so it needs implicit return at the end + flagGetMethod = 64, // was declared via `get func(): T`, method_id is auto-assigned + flagIsEntrypoint = 128, // it's `main` / `onExternalMessage` / etc. + flagHasMutateParams = 256, // has parameters declared as `mutate` + flagAcceptsSelf = 512, // is a member function (has `self` first parameter) + flagReturnsSelf = 1024, // return type is `self` (returns the mutated 1st argument), calls can be chainable + flagReallyUsed = 2048, // calculated via dfs from used functions; declared but unused functions are not codegenerated + }; + + int method_id = EMPTY_METHOD_ID; + int flags; + + std::vector parameters; + std::vector arg_order, ret_order; + TypePtr declared_return_type; // may be nullptr, meaning "auto infer" + TypePtr inferred_return_type = nullptr; // assigned on type inferring + TypePtr inferred_full_type = nullptr; // assigned on type inferring, it's TypeDataFunCallable(params -> return) + + const GenericsDeclaration* genericTs; + const GenericsInstantiation* instantiationTs; + FunctionBody body; + AnyV ast_root; // V for user-defined (not builtin) + + FunctionData(std::string name, SrcLocation loc, TypePtr declared_return_type, std::vector parameters, int initial_flags, const GenericsDeclaration* genericTs, const GenericsInstantiation* instantiationTs, FunctionBody body, AnyV ast_root) + : Symbol(std::move(name), loc) + , flags(initial_flags) + , parameters(std::move(parameters)) + , declared_return_type(declared_return_type) + , genericTs(genericTs) + , instantiationTs(instantiationTs) + , body(body) + , ast_root(ast_root) { + } + + std::string as_human_readable() const; + + const std::vector* get_arg_order() const { + return arg_order.empty() ? nullptr : &arg_order; + } + const std::vector* get_ret_order() const { + return ret_order.empty() ? nullptr : &ret_order; + } + + int get_num_params() const { return static_cast(parameters.size()); } + const LocalVarData& get_param(int idx) const { return parameters[idx]; } + + bool is_code_function() const { return std::holds_alternative(body); } + bool is_asm_function() const { return std::holds_alternative(body); } + bool is_builtin_function() const { return ast_root == nullptr; } + + bool is_generic_function() const { return genericTs != nullptr; } + bool is_instantiation_of_generic_function() const { return instantiationTs != nullptr; } + + bool is_inline() const { return flags & flagInline; } + bool is_inline_ref() const { return flags & flagInlineRef; } + bool is_type_inferring_done() const { return flags & flagTypeInferringDone; } + bool is_used_as_noncall() const { return flags & flagUsedAsNonCall; } + bool is_marked_as_pure() const { return flags & flagMarkedAsPure; } + bool is_implicit_return() const { return flags & flagImplicitReturn; } + bool is_get_method() const { return flags & flagGetMethod; } + bool is_method_id_not_empty() const { return method_id != EMPTY_METHOD_ID; } + bool is_entrypoint() const { return flags & flagIsEntrypoint; } + bool has_mutate_params() const { return flags & flagHasMutateParams; } + bool does_accept_self() const { return flags & flagAcceptsSelf; } + bool does_return_self() const { return flags & flagReturnsSelf; } + bool does_mutate_self() const { return (flags & flagAcceptsSelf) && parameters[0].is_mutate_parameter(); } + bool is_really_used() const { return flags & flagReallyUsed; } + + bool does_need_codegen() const; + + FunctionData* mutate() const { return const_cast(this); } + void assign_resolved_type(TypePtr declared_return_type); + void assign_inferred_type(TypePtr inferred_return_type, TypePtr inferred_full_type); + void assign_is_used_as_noncall(); + void assign_is_implicit_return(); + void assign_is_type_inferring_done(); + void assign_is_really_used(); + void assign_arg_order(std::vector&& arg_order); +}; + +struct GlobalVarData final : Symbol { + enum { + flagReallyUsed = 1, // calculated via dfs from used functions; unused globals are not codegenerated + }; + + TypePtr declared_type; // always exists, declaring globals without type is prohibited + int flags = 0; + + GlobalVarData(std::string name, SrcLocation loc, TypePtr declared_type) + : Symbol(std::move(name), loc) + , declared_type(declared_type) { + } + + bool is_really_used() const { return flags & flagReallyUsed; } + + GlobalVarData* mutate() const { return const_cast(this); } + void assign_resolved_type(TypePtr declared_type); + void assign_is_really_used(); +}; + +struct GlobalConstData final : Symbol { + ConstantValue value; + TypePtr declared_type; // may be nullptr + + GlobalConstData(std::string name, SrcLocation loc, TypePtr declared_type, ConstantValue&& value) + : Symbol(std::move(name), loc) + , value(std::move(value)) + , declared_type(declared_type) { + } + + bool is_int_const() const { return value.is_int(); } + bool is_slice_const() const { return value.is_slice(); } + + td::RefInt256 as_int_const() const { return value.as_int(); } + const std::string& as_slice_const() const { return value.as_slice(); } + + GlobalConstData* mutate() const { return const_cast(this); } + void assign_resolved_type(TypePtr declared_type); +}; + +class GlobalSymbolTable { + std::unordered_map entries; + + static uint64_t key_hash(std::string_view name_key) { + return std::hash{}(name_key); + } + +public: + void add_function(FunctionPtr f_sym); + void add_global_var(GlobalVarPtr g_sym); + void add_global_const(GlobalConstPtr c_sym); + + const Symbol* lookup(std::string_view name) const { + const auto it = entries.find(key_hash(name)); + return it == entries.end() ? nullptr : it->second; + } +}; + +const Symbol* lookup_global_symbol(std::string_view name); + +} // namespace tolk diff --git a/tolk/tolk-main.cpp b/tolk/tolk-main.cpp new file mode 100644 index 00000000..7f939670 --- /dev/null +++ b/tolk/tolk-main.cpp @@ -0,0 +1,285 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#include "tolk.h" +#include "tolk-version.h" +#include "compiler-state.h" +#include "td/utils/port/path.h" +#include +#include +#include +#ifdef TD_DARWIN +#include +#elif TD_WINDOWS +#include +#else // linux +#include +#endif +#include "git.h" + +using namespace tolk; + +void usage(const char* progname) { + std::cerr + << "usage: " << progname << " [options] \n" + "\tGenerates Fift TVM assembler code from a .tolk file\n" + "-o\tWrites generated code into specified .fif file instead of stdout\n" + "-b\tGenerate Fift instructions to save TVM bytecode into .boc file\n" + "-O\tSets optimization level (2 by default)\n" + "-x\tEnables experimental options, comma-separated\n" + "-S\tDon't include stack layout comments into Fift output\n" + "-e\tIncreases verbosity level (extra output into stderr)\n" + "-v\tOutput version of Tolk and exit\n"; + std::exit(2); +} + +static bool stdlib_folder_exists(const char* stdlib_folder) { + struct stat f_stat; + int res = stat(stdlib_folder, &f_stat); + return res == 0 && (f_stat.st_mode & S_IFMT) == S_IFDIR; +} + +// getting current executable path is a complicated and not cross-platform task +// for instance, we can't just use argv[0] or even filesystem::canonical +// https://stackoverflow.com/questions/1023306/finding-current-executables-path-without-proc-self-exe/1024937 +static bool get_current_executable_filename(std::string& out) { +#ifdef TD_DARWIN + char name_buf[1024]; + unsigned int size = 1024; + if (0 == _NSGetExecutablePath(name_buf, &size)) { // may contain ../, so normalize it + char *exe_path = realpath(name_buf, nullptr); + if (exe_path != nullptr) { + out = exe_path; + return true; + } + } +#elif TD_WINDOWS + char exe_path[1024]; + if (GetModuleFileNameA(nullptr, exe_path, 1024)) { + out = exe_path; + std::replace(out.begin(), out.end(), '\\', '/'); // modern Windows correctly deals with / separator + return true; + } +#else // linux + char exe_path[1024]; + ssize_t res = readlink("/proc/self/exe", exe_path, 1024 - 1); + if (res >= 0) { + exe_path[res] = 0; + out = exe_path; + return true; + } +#endif + return false; +} + +// simple join "/some/folder/" (guaranteed to end with /) and "../relative/path" +static std::string join_path(std::string dir, const char* relative) { + while (relative[0] == '.' && relative[1] == '.' && relative[2] == '/') { + size_t slash_pos = dir.find_last_of('/', dir.size() - 2); // last symbol is slash, find before it + if (slash_pos != std::string::npos) { + dir = dir.substr(0, slash_pos + 1); + } + relative += 3; + } + + return dir + relative; +} + +static std::string auto_discover_stdlib_folder() { + // if the user launches tolk compiler from a package installed (e.g. /usr/bin/tolk), + // locate stdlib in /usr/share/ton/smartcont (this folder exists on package installation) + // (note, that paths are not absolute, they are relative to the launched binary) + // consider https://github.com/ton-blockchain/packages for actual paths + std::string executable_filename; + if (!get_current_executable_filename(executable_filename)) { + return {}; + } + + // extract dirname to concatenate with relative paths (separator / is ok even for windows) + size_t slash_pos = executable_filename.find_last_of('/'); + std::string executable_dir = executable_filename.substr(0, slash_pos + 1); + +#ifdef TD_DARWIN + std::string def_location = join_path(executable_dir, "../share/ton/ton/smartcont/tolk-stdlib"); +#elif TD_WINDOWS + std::string def_location = join_path(executable_dir, "smartcont/tolk-stdlib"); +#else // linux + std::string def_location = join_path(executable_dir, "../share/ton/smartcont/tolk-stdlib"); +#endif + + if (stdlib_folder_exists(def_location.c_str())) { + return def_location; + } + + // so, the binary is not from a system package + // maybe it's just built from sources? e.g. ~/ton/cmake-build-debug/tolk/tolk + // then, check the ~/ton/crypto/smartcont folder + std::string near_when_built_from_sources = join_path(executable_dir, "../../crypto/smartcont/tolk-stdlib"); + if (stdlib_folder_exists(near_when_built_from_sources.c_str())) { + return near_when_built_from_sources; + } + + // no idea of where to find stdlib; let's show an error for the user, he should provide env var above + return {}; +} + +td::Result fs_read_callback(CompilerSettings::FsReadCallbackKind kind, const char* query) { + switch (kind) { + case CompilerSettings::FsReadCallbackKind::Realpath: { + td::Result res_realpath; + if (query[0] == '@' && strlen(query) > 8 && !strncmp(query, "@stdlib/", 8)) { + // import "@stdlib/filename" or import "@stdlib/filename.tolk" + std::string path = G.settings.stdlib_folder + static_cast(query + 7); + if (strncmp(path.c_str() + path.size() - 5, ".tolk", 5) != 0) { + path += ".tolk"; + } + res_realpath = td::realpath(td::CSlice(path.c_str())); + } else { + // import "relative/to/cwd/path.tolk" + res_realpath = td::realpath(td::CSlice(query)); + } + + if (res_realpath.is_error()) { + // note, that for non-existing files, `realpath()` on Linux/Mac returns an error, + // whereas on Windows, it returns okay, but fails after, on reading, with a message "cannot open file" + return td::Status::Error(std::string{"cannot find file "} + query); + } + return res_realpath; + } + case CompilerSettings::FsReadCallbackKind::ReadFile: { + struct stat f_stat; + int res = stat(query, &f_stat); // query here is already resolved realpath + if (res != 0 || (f_stat.st_mode & S_IFMT) != S_IFREG) { + return td::Status::Error(std::string{"cannot open file "} + query); + } + + size_t file_size = static_cast(f_stat.st_size); + std::string str; + str.resize(file_size); + FILE* f = fopen(query, "rb"); + fread(str.data(), file_size, 1, f); + fclose(f); + return std::move(str); + } + default: { + return td::Status::Error("unknown query kind"); + } + } +} + +class StdCoutRedirectToFile { + std::unique_ptr output_file; + std::streambuf* backup_sbuf = nullptr; + +public: + explicit StdCoutRedirectToFile(const std::string& output_filename) { + if (!output_filename.empty()) { + output_file = std::make_unique(output_filename, std::fstream::trunc | std::fstream::out); + if (output_file->is_open()) { + backup_sbuf = std::cout.rdbuf(output_file->rdbuf()); + } + } + } + + ~StdCoutRedirectToFile() { + if (backup_sbuf) { + std::cout.rdbuf(backup_sbuf); + } + } + + bool is_failed() const { return output_file && !output_file->is_open(); } +}; + +int main(int argc, char* const argv[]) { + int i; + while ((i = getopt(argc, argv, "o:b:O:x:Sevh")) != -1) { + switch (i) { + case 'o': + G.settings.output_filename = optarg; + break; + case 'b': + G.settings.boc_output_filename = optarg; + break; + case 'O': + G.settings.optimization_level = std::max(0, atoi(optarg)); + break; + case 'x': + G.settings.parse_experimental_options_cmd_arg(optarg); + break; + case 'S': + G.settings.stack_layout_comments = false; + break; + case 'e': + G.settings.verbosity++; + break; + case 'v': + std::cout << "Tolk compiler v" << TOLK_VERSION << std::endl; + std::cout << "Build commit: " << GitMetadata::CommitSHA1() << std::endl; + std::cout << "Build date: " << GitMetadata::CommitDate() << std::endl; + std::exit(0); + case 'h': + default: + usage(argv[0]); + } + } + + StdCoutRedirectToFile redirect_cout(G.settings.output_filename); + if (redirect_cout.is_failed()) { + std::cerr << "Failed to create output file " << G.settings.output_filename << std::endl; + return 2; + } + + // locate tolk-stdlib/ based on env or default system paths + if (const char* env_var = getenv("TOLK_STDLIB")) { + std::string stdlib_filename = static_cast(env_var) + "/common.tolk"; + td::Result res = td::realpath(td::CSlice(stdlib_filename.c_str())); + if (res.is_error()) { + std::cerr << "Environment variable TOLK_STDLIB is invalid: " << res.move_as_error().message().c_str() << std::endl; + return 2; + } + G.settings.stdlib_folder = env_var; + } else { + G.settings.stdlib_folder = auto_discover_stdlib_folder(); + } + if (G.settings.stdlib_folder.empty()) { + std::cerr << "Failed to discover Tolk stdlib.\n" + "Probably, you have a non-standard Tolk installation.\n" + "Please, provide env variable TOLK_STDLIB referencing to tolk-stdlib/ folder.\n"; + return 2; + } + if (G.is_verbosity(2)) { + std::cerr << "stdlib folder: " << G.settings.stdlib_folder << std::endl; + } + + if (optind != argc - 1) { + std::cerr << "invalid usage: should specify exactly one input file.tolk" << std::endl; + return 2; + } + + G.settings.read_callback = fs_read_callback; + + int exit_code = tolk_proceed(argv[optind]); + return exit_code; +} diff --git a/tolk/tolk-version.h b/tolk/tolk-version.h new file mode 100644 index 00000000..bbea63ff --- /dev/null +++ b/tolk/tolk-version.h @@ -0,0 +1,23 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +namespace tolk { + +constexpr const char* TOLK_VERSION = "0.9.0"; + +} // namespace tolk diff --git a/tolk/tolk-wasm.cpp b/tolk/tolk-wasm.cpp new file mode 100644 index 00000000..e74589ce --- /dev/null +++ b/tolk/tolk-wasm.cpp @@ -0,0 +1,127 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#include "tolk.h" +#include "tolk-version.h" +#include "compiler-state.h" +#include "git.h" +#include "td/utils/JsonBuilder.h" +#include "fift/utils.h" +#include "td/utils/Status.h" +#include + +using namespace tolk; + +static td::Result compile_internal(char *config_json) { + TRY_RESULT(input_json, td::json_decode(td::MutableSlice(config_json))) + td::JsonObject& config = input_json.get_object(); + + TRY_RESULT(opt_level, td::get_json_object_int_field(config, "optimizationLevel", true, 2)); + TRY_RESULT(stack_comments, td::get_json_object_bool_field(config, "withStackComments", true, false)); + TRY_RESULT(entrypoint_filename, td::get_json_object_string_field(config, "entrypointFileName", false)); + TRY_RESULT(experimental_options, td::get_json_object_string_field(config, "experimentalOptions", true)); + + G.settings.verbosity = 0; + G.settings.optimization_level = std::max(0, opt_level); + G.settings.stack_layout_comments = stack_comments; + if (!experimental_options.empty()) { + G.settings.parse_experimental_options_cmd_arg(experimental_options.c_str()); + } + + std::ostringstream outs, errs; + std::cout.rdbuf(outs.rdbuf()); + std::cerr.rdbuf(errs.rdbuf()); + int exit_code = tolk_proceed(entrypoint_filename); + if (exit_code != 0) { + return td::Status::Error("Tolk compilation error: " + errs.str()); + } + + TRY_RESULT(fift_res, fift::compile_asm_program(outs.str(), "/fiftlib/")); + + td::JsonBuilder result_json; + auto obj = result_json.enter_object(); + obj("status", "ok"); + obj("fiftCode", fift_res.fiftCode); + obj("codeBoc64", fift_res.codeBoc64); + obj("codeHashHex", fift_res.codeHashHex); + obj("stderr", errs.str().c_str()); + obj.leave(); + + return result_json.string_builder().as_cslice().str(); +} + +/// Callback used to retrieve file contents from a "not file system". See tolk-js for implementation. +/// The callback must fill either destContents or destError. +/// The implementor must use malloc() for them and use free() after tolk_compile returns. +typedef void (*WasmFsReadCallback)(int kind, char const* data, char** destContents, char** destError); + +static CompilerSettings::FsReadCallback wrap_wasm_read_callback(WasmFsReadCallback _readCallback) { + return [_readCallback](CompilerSettings::FsReadCallbackKind kind, char const* data) -> td::Result { + char* destContents = nullptr; + char* destError = nullptr; + if (_readCallback) { + _readCallback(static_cast(kind), data, &destContents, &destError); + } + if (destContents) { + return destContents; + } + if (destError) { + return td::Status::Error(std::string(destError)); + } + return td::Status::Error("Invalid callback from wasm"); + }; +} + +extern "C" { + +const char* version() { + td::JsonBuilder version_json = td::JsonBuilder(); + auto obj = version_json.enter_object(); + obj("tolkVersion", TOLK_VERSION); + obj("tolkFiftLibCommitHash", GitMetadata::CommitSHA1()); + obj("tolkFiftLibCommitDate", GitMetadata::CommitDate()); + obj.leave(); + return strdup(version_json.string_builder().as_cslice().c_str()); +} + +const char *tolk_compile(char *config_json, WasmFsReadCallback callback) { + G.settings.read_callback = wrap_wasm_read_callback(callback); + + td::Result res = compile_internal(config_json); + + if (res.is_error()) { + td::JsonBuilder error_res = td::JsonBuilder(); + auto obj = error_res.enter_object(); + obj("status", "error"); + obj("message", res.move_as_error().message().str()); + obj.leave(); + return strdup(error_res.string_builder().as_cslice().c_str()); + } + + std::string res_string = res.move_as_ok(); + return strdup(res_string.c_str()); +} + +} // extern "C" diff --git a/tolk/tolk.cpp b/tolk/tolk.cpp new file mode 100644 index 00000000..71d1969d --- /dev/null +++ b/tolk/tolk.cpp @@ -0,0 +1,87 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#include "tolk.h" +#include "pipeline.h" +#include "compiler-state.h" +#include "lexer.h" +#include "ast.h" +#include "type-system.h" + +namespace tolk { + + +void on_assertion_failed(const char *description, const char *file_name, int line_number) { + std::string message = static_cast("Assertion failed at ") + file_name + ":" + std::to_string(line_number) + ": " + description; +#ifdef TOLK_DEBUG +#ifdef __arm64__ + // when developing, it's handy when the debugger stops on assertion failure (stacktraces and watches are available) + std::cerr << message << std::endl; + __builtin_debugtrap(); +#endif +#endif + throw Fatal(std::move(message)); +} + +int tolk_proceed(const std::string &entrypoint_filename) { + type_system_init(); + define_builtins(); + lexer_init(); + + // on any error, an exception is thrown, and the message is printed out below + // (currently, only a single error can be printed) + try { + pipeline_discover_and_parse_sources("@stdlib/common.tolk", entrypoint_filename); + + pipeline_register_global_symbols(); + pipeline_resolve_identifiers_and_assign_symbols(); + pipeline_calculate_rvalue_lvalue(); + pipeline_infer_types_and_calls_and_fields(); + pipeline_check_inferred_types(); + pipeline_refine_lvalue_for_mutate_arguments(); + pipeline_check_rvalue_lvalue(); + pipeline_check_pure_impure_operations(); + pipeline_constant_folding(); + pipeline_optimize_boolean_expressions(); + pipeline_convert_ast_to_legacy_Expr_Op(); + + pipeline_find_unused_symbols(); + pipeline_generate_fif_output_to_std_cout(); + + return 0; + } catch (Fatal& fatal) { + std::cerr << "fatal: " << fatal << std::endl; + return 2; + } catch (ParseError& error) { + std::cerr << error << std::endl; + return 2; + } catch (UnexpectedASTNodeType& error) { + std::cerr << "fatal: " << error.what() << std::endl; + std::cerr << "It's a compiler bug, please report to developers" << std::endl; + return 2; + } +} + +} // namespace tolk diff --git a/tolk/tolk.h b/tolk/tolk.h new file mode 100644 index 00000000..3f00d0d4 --- /dev/null +++ b/tolk/tolk.h @@ -0,0 +1,1166 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "platform-utils.h" +#include "src-file.h" +#include "symtable.h" +#include "crypto/common/refint.h" +#include "td/utils/Status.h" +#include +#include +#include +#include +#include + +#define tolk_assert(expr) if(UNLIKELY(!(expr))) on_assertion_failed(#expr, __FILE__, __LINE__); + +namespace tolk { + +GNU_ATTRIBUTE_COLD GNU_ATTRIBUTE_NORETURN +void on_assertion_failed(const char *description, const char *file_name, int line_number); + +/* + * + * ABSTRACT CODE + * + */ + +typedef int var_idx_t; +typedef int const_idx_t; + +struct TmpVar { + var_idx_t ir_idx; // every var in IR represents 1 stack slot + TypePtr v_type; // get_width_on_stack() is 1 + std::string name; // "x" for vars originated from user sources; "x.0" for tensor components; empty for implicitly created tmp vars + SrcLocation loc; // location of var declaration in sources or where a tmp var was originated +#ifdef TOLK_DEBUG + const char* desc = nullptr; // "origin" of tmp var, for debug output like `'15 (binary-op) '16 (glob-var)` +#endif + + TmpVar(var_idx_t ir_idx, TypePtr v_type, std::string name, SrcLocation loc) + : ir_idx(ir_idx) + , v_type(v_type) + , name(std::move(name)) + , loc(loc) { + } + + void show_as_stack_comment(std::ostream& os) const; + void show(std::ostream& os) const; +}; + +struct VarDescr { + var_idx_t idx; + enum { _Last = 1, _Unused = 2 }; + int flags; + enum { + _Const = 16, + _Int = 32, + _Zero = 64, + _NonZero = 128, + _Pos = 256, + _Neg = 512, + _Finite = 4096, + _Nan = 8192, + _Even = 16384, + _Odd = 32768, + }; + static constexpr int ConstZero = _Const | _Int | _Zero | _Pos | _Neg | _Finite | _Even; + static constexpr int ConstOne = _Const | _Int | _NonZero | _Pos | _Finite | _Odd; + static constexpr int ConstTrue = _Const | _Int | _NonZero | _Neg | _Finite | _Odd; + static constexpr int ValBit = _Int | _Pos | _Finite; + static constexpr int ValBool = _Int | _Neg | _Finite; + static constexpr int FiniteInt = _Int | _Finite; + static constexpr int FiniteUInt = _Int | _Finite | _Pos; + int val; + td::RefInt256 int_const; + std::string str_const; + + explicit VarDescr(var_idx_t _idx = -1, int _flags = 0, int _val = 0) : idx(_idx), flags(_flags), val(_val) { + } + bool operator<(var_idx_t other_idx) const { + return idx < other_idx; + } + bool is_unused() const { + return flags & _Unused; + } + bool is_last() const { + return flags & _Last; + } + bool always_true() const { + return val & _NonZero; + } + bool always_false() const { + return val & _Zero; + } + bool always_nonzero() const { + return val & _NonZero; + } + bool always_zero() const { + return val & _Zero; + } + bool always_even() const { + return val & _Even; + } + bool always_odd() const { + return val & _Odd; + } + bool is_int_const() const { + return (val & (_Int | _Const)) == (_Int | _Const) && int_const.not_null(); + } + bool always_nonpos() const { + return val & _Neg; + } + bool always_nonneg() const { + return val & _Pos; + } + bool always_pos() const { + return (val & (_Pos | _NonZero)) == (_Pos | _NonZero); + } + bool always_neg() const { + return (val & (_Neg | _NonZero)) == (_Neg | _NonZero); + } + bool always_finite() const { + return val & _Finite; + } + bool always_less(const VarDescr& other) const; + bool always_leq(const VarDescr& other) const; + bool always_greater(const VarDescr& other) const; + bool always_geq(const VarDescr& other) const; + bool always_equal(const VarDescr& other) const; + bool always_neq(const VarDescr& other) const; + void unused() { + flags |= _Unused; + } + void clear_unused() { + flags &= ~_Unused; + } + void set_const(long long value); + void set_const(td::RefInt256 value); + void set_const(std::string value); + void operator+=(const VarDescr& y) { + flags &= y.flags; + } + void operator|=(const VarDescr& y); + void operator&=(const VarDescr& y); + void set_value(const VarDescr& y); + void set_value(VarDescr&& y); + void set_value(const VarDescr* y) { + if (y) { + set_value(*y); + } + } + void clear_value(); + void show_value(std::ostream& os) const; + void show(std::ostream& os, const char* var_name = nullptr) const; +}; + +inline std::ostream& operator<<(std::ostream& os, const VarDescr& vd) { + vd.show(os); + return os; +} + +struct VarDescrList { + std::vector list; + bool unreachable{false}; + VarDescrList() : list() { + } + VarDescrList(const std::vector& _list) : list(_list) { + } + VarDescrList(std::vector&& _list) : list(std::move(_list)) { + } + std::size_t size() const { + return list.size(); + } + VarDescr* operator[](var_idx_t idx); + const VarDescr* operator[](var_idx_t idx) const; + VarDescrList operator+(const VarDescrList& y) const; + VarDescrList& operator+=(const VarDescrList& y); + VarDescrList& clear_last(); + VarDescrList& operator+=(var_idx_t idx) { + return add_var(idx); + } + VarDescrList& operator+=(const std::vector& idx_list) { + return add_vars(idx_list); + } + VarDescrList& add_var(var_idx_t idx, bool unused = false); + VarDescrList& add_vars(const std::vector& idx_list, bool unused = false); + VarDescrList& operator-=(const std::vector& idx_list); + VarDescrList& operator-=(var_idx_t idx); + std::size_t count(const std::vector idx_list) const; + std::size_t count_used(const std::vector idx_list) const; + VarDescr& add(var_idx_t idx); + VarDescr& add_newval(var_idx_t idx); + VarDescrList& import_values(const VarDescrList& values); + VarDescrList operator|(const VarDescrList& y) const; + VarDescrList& operator|=(const VarDescrList& values); + void show(std::ostream& os) const; + void set_unreachable() { + list.clear(); + unreachable = true; + } +}; + +inline std::ostream& operator<<(std::ostream& os, const VarDescrList& values) { + values.show(os); + return os; +} + +struct CodeBlob; + +template +class ListIterator { + T* ptr; + + public: + ListIterator() : ptr(nullptr) { + } + explicit ListIterator(T* _ptr) : ptr(_ptr) { + } + ListIterator& operator++() { + ptr = ptr->next.get(); + return *this; + } + ListIterator operator++(int) { + T* z = ptr; + ptr = ptr->next.get(); + return ListIterator{z}; + } + T& operator*() const { + return *ptr; + } + T* operator->() const { + return ptr; + } + bool operator==(const ListIterator& y) const { + return ptr == y.ptr; + } + bool operator!=(const ListIterator& y) const { + return ptr != y.ptr; + } +}; + +struct Stack; + +struct Op { + enum OpKind { + _Undef, + _Nop, + _Call, + _CallInd, + _Let, + _IntConst, + _GlobVar, + _SetGlob, + _Import, + _Return, + _Tuple, + _UnTuple, + _If, + _While, + _Until, + _Repeat, + _Again, + _TryCatch, + _SliceConst, + }; + OpKind cl; + enum { _Disabled = 1, _NoReturn = 4, _Impure = 24 }; + int flags; + std::unique_ptr next; + FunctionPtr f_sym = nullptr; + GlobalVarPtr g_sym = nullptr; + SrcLocation where; + VarDescrList var_info; + std::vector args; + std::vector left, right; + std::unique_ptr block0, block1; + td::RefInt256 int_const; + std::string str_const; + Op(SrcLocation _where = {}, OpKind _cl = _Undef) : cl(_cl), flags(0), f_sym(nullptr), where(_where) { + } + Op(SrcLocation _where, OpKind _cl, const std::vector& _left) + : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(_left) { + } + Op(SrcLocation _where, OpKind _cl, std::vector&& _left) + : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(std::move(_left)) { + } + Op(SrcLocation _where, OpKind _cl, const std::vector& _left, td::RefInt256 _const) + : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(_left), int_const(_const) { + } + Op(SrcLocation _where, OpKind _cl, const std::vector& _left, std::string _const) + : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(_left), str_const(_const) { + } + Op(SrcLocation _where, OpKind _cl, const std::vector& _left, const std::vector& _right) + : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(_left), right(_right) { + } + Op(SrcLocation _where, OpKind _cl, std::vector&& _left, std::vector&& _right) + : cl(_cl), flags(0), f_sym(nullptr), where(_where), left(std::move(_left)), right(std::move(_right)) { + } + Op(SrcLocation _where, OpKind _cl, const std::vector& _left, const std::vector& _right, + FunctionPtr _fun) + : cl(_cl), flags(0), f_sym(_fun), where(_where), left(_left), right(_right) { + } + Op(SrcLocation _where, OpKind _cl, std::vector&& _left, std::vector&& _right, + FunctionPtr _fun) + : cl(_cl), flags(0), f_sym(_fun), where(_where), left(std::move(_left)), right(std::move(_right)) { + } + Op(SrcLocation _where, OpKind _cl, const std::vector& _left, const std::vector& _right, + GlobalVarPtr _gvar) + : cl(_cl), flags(0), g_sym(_gvar), where(_where), left(_left), right(_right) { + } + Op(SrcLocation _where, OpKind _cl, std::vector&& _left, std::vector&& _right, + GlobalVarPtr _gvar) + : cl(_cl), flags(0), g_sym(_gvar), where(_where), left(std::move(_left)), right(std::move(_right)) { + } + + bool disabled() const { return flags & _Disabled; } + void set_disabled() { flags |= _Disabled; } + void set_disabled(bool flag); + + bool noreturn() const { return flags & _NoReturn; } + bool set_noreturn() { flags |= _NoReturn; return true; } + bool set_noreturn(bool flag); + + bool impure() const { return flags & _Impure; } + void set_impure_flag(); + + void show(std::ostream& os, const std::vector& vars, std::string pfx = "", int mode = 0) const; + void show_var_list(std::ostream& os, const std::vector& idx_list, const std::vector& vars) const; + void show_var_list(std::ostream& os, const std::vector& list, const std::vector& vars) const; + static void show_block(std::ostream& os, const Op* block, const std::vector& vars, std::string pfx = "", + int mode = 0); + bool compute_used_vars(const CodeBlob& code, bool edit); + bool std_compute_used_vars(bool disabled = false); + bool set_var_info(const VarDescrList& new_var_info); + bool set_var_info(VarDescrList&& new_var_info); + bool set_var_info_except(const VarDescrList& new_var_info, const std::vector& var_list); + bool set_var_info_except(VarDescrList&& new_var_info, const std::vector& var_list); + void prepare_args(VarDescrList values); + VarDescrList fwd_analyze(VarDescrList values); + bool mark_noreturn(); + bool is_empty() const { + return cl == _Nop && !next; + } + bool generate_code_step(Stack& stack); + void generate_code_all(Stack& stack); + Op& last() { + return next ? next->last() : *this; + } + const Op& last() const { + return next ? next->last() : *this; + } +}; + +inline ListIterator begin(const std::unique_ptr& op_list) { + return ListIterator{op_list.get()}; +} + +inline ListIterator end(const std::unique_ptr& op_list) { + return ListIterator{}; +} + +inline ListIterator begin(const Op* op_list) { + return ListIterator{op_list}; +} + +inline ListIterator end(const Op* op_list) { + return ListIterator{}; +} + +struct AsmOpList; + +struct FunctionBodyCode { + CodeBlob* code = nullptr; + void set_code(CodeBlob* code); +}; + +/* + * + * GENERATE CODE + * + */ + +typedef std::vector StackLayout; +typedef std::pair var_const_idx_t; +typedef std::vector StackLayoutExt; +constexpr const_idx_t not_const = -1; +using Const = td::RefInt256; + +struct AsmOp { + enum Type { a_none, a_xchg, a_push, a_pop, a_const, a_custom, a_magic }; + Type t{a_none}; + int indent{0}; + int a, b; + bool gconst{false}; + std::string op; + struct SReg { + int idx; + SReg(int _idx) : idx(_idx) { + } + }; + AsmOp() = default; + AsmOp(Type _t) : t(_t) { + } + AsmOp(Type _t, std::string _op) : t(_t), op(std::move(_op)) { + } + AsmOp(Type _t, int _a) : t(_t), a(_a) { + } + AsmOp(Type _t, int _a, std::string _op) : t(_t), a(_a), op(std::move(_op)) { + } + AsmOp(Type _t, int _a, int _b) : t(_t), a(_a), b(_b) { + } + AsmOp(Type _t, int _a, int _b, std::string _op) : t(_t), a(_a), b(_b), op(std::move(_op)) { + compute_gconst(); + } + void out(std::ostream& os) const; + void out_indent_nl(std::ostream& os, bool no_nl = false) const; + std::string to_string() const; + void compute_gconst() { + gconst = (is_custom() && (op == "PUSHNULL" || op == "NEWC" || op == "NEWB" || op == "TRUE" || op == "FALSE" || op == "NOW")); + } + bool is_nop() const { + return t == a_none && op.empty(); + } + bool is_comment() const { + return t == a_none && !op.empty(); + } + bool is_custom() const { + return t == a_custom; + } + bool is_very_custom() const { + return is_custom() && a >= 255; + } + bool is_push() const { + return t == a_push; + } + bool is_push(int x) const { + return is_push() && a == x; + } + bool is_push(int* x) const { + *x = a; + return is_push(); + } + bool is_pop() const { + return t == a_pop; + } + bool is_pop(int x) const { + return is_pop() && a == x; + } + bool is_xchg() const { + return t == a_xchg; + } + bool is_xchg(int x, int y) const { + return is_xchg() && b == y && a == x; + } + bool is_xchg(int* x, int* y) const { + *x = a; + *y = b; + return is_xchg(); + } + bool is_xchg_short() const { + return is_xchg() && (a <= 1 || b <= 1); + } + bool is_swap() const { + return is_xchg(0, 1); + } + bool is_const() const { + return t == a_const && !a && b == 1; + } + bool is_gconst() const { + return !a && b == 1 && (t == a_const || gconst); + } + static AsmOp Nop() { + return AsmOp(a_none); + } + static AsmOp Xchg(int a, int b = 0) { + return a == b ? AsmOp(a_none) : (a < b ? AsmOp(a_xchg, a, b) : AsmOp(a_xchg, b, a)); + } + static AsmOp Push(int a) { + return AsmOp(a_push, a); + } + static AsmOp Pop(int a = 0) { + return AsmOp(a_pop, a); + } + static AsmOp Xchg2(int a, int b) { + return make_stk2(a, b, "XCHG2", 0); + } + static AsmOp XcPu(int a, int b) { + return make_stk2(a, b, "XCPU", 1); + } + static AsmOp PuXc(int a, int b) { + return make_stk2(a, b, "PUXC", 1); + } + static AsmOp Push2(int a, int b) { + return make_stk2(a, b, "PUSH2", 2); + } + static AsmOp Xchg3(int a, int b, int c) { + return make_stk3(a, b, c, "XCHG3", 0); + } + static AsmOp Xc2Pu(int a, int b, int c) { + return make_stk3(a, b, c, "XC2PU", 1); + } + static AsmOp XcPuXc(int a, int b, int c) { + return make_stk3(a, b, c, "XCPUXC", 1); + } + static AsmOp XcPu2(int a, int b, int c) { + return make_stk3(a, b, c, "XCPU2", 3); + } + static AsmOp PuXc2(int a, int b, int c) { + return make_stk3(a, b, c, "PUXC2", 3); + } + static AsmOp PuXcPu(int a, int b, int c) { + return make_stk3(a, b, c, "PUXCPU", 3); + } + static AsmOp Pu2Xc(int a, int b, int c) { + return make_stk3(a, b, c, "PU2XC", 3); + } + static AsmOp Push3(int a, int b, int c) { + return make_stk3(a, b, c, "PUSH3", 3); + } + static AsmOp BlkSwap(int a, int b); + static AsmOp BlkPush(int a, int b); + static AsmOp BlkDrop(int a); + static AsmOp BlkDrop2(int a, int b); + static AsmOp BlkReverse(int a, int b); + static AsmOp make_stk2(int a, int b, const char* str, int delta); + static AsmOp make_stk3(int a, int b, int c, const char* str, int delta); + static AsmOp IntConst(const td::RefInt256& x); + static AsmOp BoolConst(bool f); + static AsmOp Const(std::string push_op) { + return AsmOp(a_const, 0, 1, std::move(push_op)); + } + static AsmOp Const(int arg, const std::string& push_op); + static AsmOp Comment(const std::string& comment) { + return AsmOp(a_none, std::string{"// "} + comment); + } + static AsmOp Custom(const std::string& custom_op) { + return AsmOp(a_custom, 255, 255, custom_op); + } + static AsmOp Parse(const std::string& custom_op); + static AsmOp Custom(const std::string& custom_op, int args, int retv = 1) { + return AsmOp(a_custom, args, retv, custom_op); + } + static AsmOp Parse(std::string custom_op, int args, int retv = 1); + static AsmOp Tuple(int a); + static AsmOp UnTuple(int a); +}; + +inline std::ostream& operator<<(std::ostream& os, const AsmOp& op) { + op.out(os); + return os; +} + +std::ostream& operator<<(std::ostream& os, AsmOp::SReg stack_reg); +std::ostream& operator<<(std::ostream& os, TypePtr type_data); + +struct AsmOpList { + std::vector list_; + int indent_{0}; + const std::vector* var_names_{nullptr}; + std::vector constants_; + bool retalt_{false}; + bool retalt_inserted_{false}; + void out(std::ostream& os, int mode = 0) const; + AsmOpList(int indent = 0, const std::vector* var_names = nullptr) : indent_(indent), var_names_(var_names) { + } + template + AsmOpList& add(Args&&... args) { + append(AsmOp(std::forward(args)...)); + adjust_last(); + return *this; + } + bool append(const AsmOp& op) { + list_.push_back(op); + adjust_last(); + return true; + } + bool append(const std::vector& ops); + bool append(std::initializer_list ops) { + return append(std::vector(std::move(ops))); + } + AsmOpList& operator<<(const AsmOp& op) { + return add(op); + } + AsmOpList& operator<<(AsmOp&& op) { + return add(std::move(op)); + } + AsmOpList& operator<<(std::string str) { + return add(AsmOp::Type::a_custom, 255, 255, str); + } + const_idx_t register_const(Const new_const); + Const get_const(const_idx_t idx); + void show_var_ext(std::ostream& os, std::pair idx_pair) const; + void adjust_last() { + if (list_.back().is_nop()) { + list_.pop_back(); + } else { + list_.back().indent = indent_; + } + } + void indent() { + ++indent_; + } + void undent() { + --indent_; + } + void set_indent(int new_indent) { + indent_ = new_indent; + } + void insert(size_t pos, std::string str) { + insert(pos, AsmOp(AsmOp::a_custom, 255, 255, str)); + } + void insert(size_t pos, const AsmOp& op) { + auto ip = list_.begin() + pos; + ip = list_.insert(ip, op); + ip->indent = (ip == list_.begin()) ? indent_ : (ip - 1)->indent; + } + void indent_all() { + for (auto &op : list_) { + ++op.indent; + } + } +}; + +inline std::ostream& operator<<(std::ostream& os, const AsmOpList& op_list) { + op_list.out(os); + return os; +} + +struct AsmOpCons { + std::unique_ptr car; + std::unique_ptr cdr; + AsmOpCons(std::unique_ptr head, std::unique_ptr tail) : car(std::move(head)), cdr(std::move(tail)) { + } + static std::unique_ptr cons(std::unique_ptr head, std::unique_ptr tail) { + return std::make_unique(std::move(head), std::move(tail)); + } +}; + +using AsmOpConsList = std::unique_ptr; + +int is_pos_pow2(td::RefInt256 x); +int is_neg_pow2(td::RefInt256 x); + +/* + * + * STACK TRANSFORMS + * + */ + +/* +A stack transform is a map f:N={0,1,...} -> N, such that f(x) = x + d_f for almost all x:N and for a fixed d_f:N. +They form a monoid under composition: (fg)(x)=f(g(x)). +They act on stacks S on the right: Sf=S', such that S'[n]=S[f(n)]. + +A stack transform f is determined by d_f and the finite set A of all pairs (x,y), such that x>=d_f, f(x-d_f) = y and y<>x. They are listed in increasing order by x. +*/ +struct StackTransform { + enum { max_n = 16, inf_x = 0x7fffffff, c_start = -1000 }; + int d{0}, n{0}, dp{0}, c{0}; + bool invalid{false}; + std::array, max_n> A; + StackTransform() = default; + // list of f(0),f(1),...,f(s); assumes next values are f(s)+1,f(s)+2,... + StackTransform(std::initializer_list list); + StackTransform& operator=(std::initializer_list list); + bool assign(const StackTransform& other); + static StackTransform id() { + return {}; + } + bool invalidate() { + invalid = true; + return false; + } + bool is_valid() const { + return !invalid; + } + bool set_id() { + d = n = dp = c = 0; + invalid = false; + return true; + } + bool shift(int offs) { // post-composes with x -> x + offs + d += offs; + return offs <= 0 || remove_negative(); + } + bool remove_negative(); + bool touch(int i) { + dp = std::max(dp, i + d + 1); + return true; + } + bool is_permutation() const; // is f:N->N bijective ? + bool is_trivial_after(int x) const; // f(x') = x' + d for all x' >= x + int preimage_count(int y) const; // card f^{-1}(y) + std::vector preimage(int y) const; + bool apply_xchg(int i, int j, bool relaxed = false); + bool apply_push(int i); + bool apply_pop(int i = 0); + bool apply_push_newconst(); + bool apply_blkpop(int k); + bool apply(const StackTransform& other); // this = this * other + bool preapply(const StackTransform& other); // this = other * this + // c := a * b + static bool compose(const StackTransform& a, const StackTransform& b, StackTransform& c); + StackTransform& operator*=(const StackTransform& other); + StackTransform operator*(const StackTransform& b) const &; + bool equal(const StackTransform& other, bool relaxed = false) const; + bool almost_equal(const StackTransform& other) const { + return equal(other, true); + } + bool operator==(const StackTransform& other) const { + return dp == other.dp && almost_equal(other); + } + bool operator<=(const StackTransform& other) const { + return dp <= other.dp && almost_equal(other); + } + bool operator>=(const StackTransform& other) const { + return dp >= other.dp && almost_equal(other); + } + int get(int i) const; + int touch_get(int i, bool relaxed = false) { + if (!relaxed) { + touch(i); + } + return get(i); + } + bool set(int i, int v, bool relaxed = false); + int operator()(int i) const { + return get(i); + } + class Pos { + StackTransform& t_; + int p_; + + public: + Pos(StackTransform& t, int p) : t_(t), p_(p) { + } + Pos& operator=(const Pos& other) = delete; + operator int() const { + return t_.get(p_); + } + const Pos& operator=(int v) const { + t_.set(p_, v); + return *this; + } + }; + Pos operator[](int i) { + return Pos(*this, i); + } + static const StackTransform rot; + static const StackTransform rot_rev; + bool is_id() const { + return is_valid() && !d && !n; + } + bool is_xchg(int i, int j) const; + bool is_xchg(int* i, int* j) const; + bool is_xchg_xchg(int i, int j, int k, int l) const; + bool is_xchg_xchg(int* i, int* j, int* k, int* l) const; + bool is_push(int i) const; + bool is_push(int* i) const; + bool is_pop(int i) const; + bool is_pop(int* i) const; + bool is_pop_pop(int i, int j) const; + bool is_pop_pop(int* i, int* j) const; + bool is_rot() const; + bool is_rotrev() const; + bool is_push_rot(int i) const; + bool is_push_rot(int* i) const; + bool is_push_rotrev(int i) const; + bool is_push_rotrev(int* i) const; + bool is_push_xchg(int i, int j, int k) const; + bool is_push_xchg(int* i, int* j, int* k) const; + bool is_xchg2(int i, int j) const; + bool is_xchg2(int* i, int* j) const; + bool is_xcpu(int i, int j) const; + bool is_xcpu(int* i, int* j) const; + bool is_puxc(int i, int j) const; + bool is_puxc(int* i, int* j) const; + bool is_push2(int i, int j) const; + bool is_push2(int* i, int* j) const; + bool is_xchg3(int* i, int* j, int* k) const; + bool is_xc2pu(int* i, int* j, int* k) const; + bool is_xcpuxc(int* i, int* j, int* k) const; + bool is_xcpu2(int* i, int* j, int* k) const; + bool is_puxc2(int i, int j, int k) const; + bool is_puxc2(int* i, int* j, int* k) const; + bool is_puxcpu(int* i, int* j, int* k) const; + bool is_pu2xc(int i, int j, int k) const; + bool is_pu2xc(int* i, int* j, int* k) const; + bool is_push3(int i, int j, int k) const; + bool is_push3(int* i, int* j, int* k) const; + bool is_blkswap(int i, int j) const; + bool is_blkswap(int* i, int* j) const; + bool is_blkpush(int i, int j) const; + bool is_blkpush(int* i, int* j) const; + bool is_blkdrop(int* i) const; + bool is_blkdrop2(int i, int j) const; + bool is_blkdrop2(int* i, int* j) const; + bool is_reverse(int i, int j) const; + bool is_reverse(int* i, int* j) const; + bool is_nip_seq(int i, int j = 0) const; + bool is_nip_seq(int* i) const; + bool is_nip_seq(int* i, int* j) const; + bool is_pop_blkdrop(int i, int k) const; + bool is_pop_blkdrop(int* i, int* k) const; + bool is_2pop_blkdrop(int i, int j, int k) const; + bool is_2pop_blkdrop(int* i, int* j, int* k) const; + bool is_const_rot(int c) const; + bool is_const_rot(int* c) const; + bool is_const_pop(int c, int i) const; + bool is_const_pop(int* c, int* i) const; + bool is_push_const(int i, int c) const; + bool is_push_const(int* i, int* c) const; + + void show(std::ostream& os, int mode = 0) const; + + static StackTransform Xchg(int i, int j, bool relaxed = false); + static StackTransform Push(int i); + static StackTransform Pop(int i); + + private: + int try_load(int& i, int offs = 0) const; // returns A[i++].first + offs or inf_x + bool try_store(int x, int y); // appends (x,y) to A +}; + +inline std::ostream& operator<<(std::ostream& os, const StackTransform& trans) { + trans.show(os); + return os; +} + +bool apply_op(StackTransform& trans, const AsmOp& op); + +/* + * + * STACK OPERATION OPTIMIZER + * + */ + +struct Optimizer { + static constexpr int optimize_depth = 20; + AsmOpConsList code_; + int l_{0}, l2_{0}, p_, pb_, q_, indent_; + bool debug_{false}; + std::unique_ptr op_[optimize_depth], oq_[optimize_depth]; + AsmOpCons* op_cons_[optimize_depth]; + int offs_[optimize_depth]; + StackTransform tr_[optimize_depth]; + int mode_{0}; + Optimizer() { + } + Optimizer(bool debug, int mode = 0) : debug_(debug), mode_(mode) { + } + Optimizer(AsmOpConsList code, bool debug = false, int mode = 0) : Optimizer(debug, mode) { + set_code(std::move(code)); + } + void set_code(AsmOpConsList code_); + void unpack(); + void pack(); + void apply(); + bool find_at_least(int pb); + bool find(); + bool optimize(); + bool compute_stack_transforms(); + bool say(std::string str) const; + bool show_stack_transforms() const; + void show_head() const; + void show_left() const; + void show_right() const; + bool find_const_op(int* op_idx, int cst); + bool is_push_const(int* i, int* c) const; + bool rewrite_push_const(int i, int c); + bool is_const_push_xchgs(); + bool rewrite_const_push_xchgs(); + bool is_const_rot(int* c) const; + bool rewrite_const_rot(int c); + bool is_const_pop(int* c, int* i) const; + bool rewrite_const_pop(int c, int i); + bool rewrite(int p, AsmOp&& new_op); + bool rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2); + bool rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2, AsmOp&& new_op3); + bool rewrite(AsmOp&& new_op) { + return rewrite(p_, std::move(new_op)); + } + bool rewrite(AsmOp&& new_op1, AsmOp&& new_op2) { + return rewrite(p_, std::move(new_op1), std::move(new_op2)); + } + bool rewrite(AsmOp&& new_op1, AsmOp&& new_op2, AsmOp&& new_op3) { + return rewrite(p_, std::move(new_op1), std::move(new_op2), std::move(new_op3)); + } + bool rewrite_nop(); + bool is_pred(const std::function& pred, int min_p = 2); + bool is_same_as(const StackTransform& trans, int min_p = 2); + bool is_rot(); + bool is_rotrev(); + bool is_tuck(); + bool is_2dup(); + bool is_2drop(); + bool is_2swap(); + bool is_2over(); + bool is_xchg(int* i, int* j); + bool is_xchg_xchg(int* i, int* j, int* k, int* l); + bool is_push(int* i); + bool is_pop(int* i); + bool is_pop_pop(int* i, int* j); + bool is_nop(); + bool is_push_rot(int* i); + bool is_push_rotrev(int* i); + bool is_push_xchg(int* i, int* j, int* k); + bool is_xchg2(int* i, int* j); + bool is_xcpu(int* i, int* j); + bool is_puxc(int* i, int* j); + bool is_push2(int* i, int* j); + bool is_xchg3(int* i, int* j, int* k); + bool is_xc2pu(int* i, int* j, int* k); + bool is_xcpuxc(int* i, int* j, int* k); + bool is_xcpu2(int* i, int* j, int* k); + bool is_puxc2(int* i, int* j, int* k); + bool is_puxcpu(int* i, int* j, int* k); + bool is_pu2xc(int* i, int* j, int* k); + bool is_push3(int* i, int* j, int* k); + bool is_blkswap(int* i, int* j); + bool is_blkpush(int* i, int* j); + bool is_blkdrop(int* i); + bool is_blkdrop2(int* i, int* j); + bool is_reverse(int* i, int* j); + bool is_nip_seq(int* i, int* j); + bool is_pop_blkdrop(int* i, int* k); + bool is_2pop_blkdrop(int* i, int* j, int* k); + AsmOpConsList extract_code(); +}; + +AsmOpConsList optimize_code_head(AsmOpConsList op_list, int mode = 0); +AsmOpConsList optimize_code(AsmOpConsList op_list, int mode); +void optimize_code(AsmOpList& ops); + +struct Stack { + StackLayoutExt s; + AsmOpList& o; + enum { + _StkCmt = 1, _CptStkCmt = 2, _DisableOut = 128, _Shown = 256, + _InlineFunc = 512, _NeedRetAlt = 1024, _InlineAny = 2048, + _ModeSave = _InlineFunc | _NeedRetAlt | _InlineAny, + _Garbage = -0x10000 + }; + int mode; + Stack(AsmOpList& _o, int _mode = 0) : o(_o), mode(_mode) { + } + Stack(AsmOpList& _o, const StackLayoutExt& _s, int _mode = 0) : s(_s), o(_o), mode(_mode) { + } + Stack(AsmOpList& _o, StackLayoutExt&& _s, int _mode = 0) : s(std::move(_s)), o(_o), mode(_mode) { + } + int depth() const { + return (int)s.size(); + } + var_idx_t operator[](int i) const { + validate(i); + return s[depth() - i - 1].first; + } + var_const_idx_t& at(int i) { + validate(i); + return s[depth() - i - 1]; + } + var_const_idx_t at(int i) const { + validate(i); + return s[depth() - i - 1]; + } + var_const_idx_t get(int i) const { + return at(i); + } + bool output_disabled() const { + return mode & _DisableOut; + } + bool output_enabled() const { + return !output_disabled(); + } + void disable_output() { + mode |= _DisableOut; + } + StackLayout vars() const; + int find(var_idx_t var, int from = 0) const; + int find(var_idx_t var, int from, int to) const; + int find_const(const_idx_t cst, int from = 0) const; + int find_outside(var_idx_t var, int from, int to) const; + void forget_const(); + void validate(int i) const { + if (i > 255) { + throw Fatal{"Too deep stack"}; + } + tolk_assert(i >= 0 && i < depth() && "invalid stack reference"); + } + void modified() { + mode &= ~_Shown; + } + void issue_pop(int i); + void issue_push(int i); + void issue_xchg(int i, int j); + int drop_vars_except(const VarDescrList& var_info, int excl_var = 0x80000000); + void forget_var(var_idx_t idx); + void push_new_var(var_idx_t idx); + void push_new_const(var_idx_t idx, const_idx_t cidx); + void assign_var(var_idx_t new_idx, var_idx_t old_idx); + void do_copy_var(var_idx_t new_idx, var_idx_t old_idx); + void enforce_state(const StackLayout& req_stack); + void rearrange_top(const StackLayout& top, std::vector last); + void rearrange_top(var_idx_t top, bool last); + void merge_const(const Stack& req_stack); + void merge_state(const Stack& req_stack); + void show(); + void opt_show() { + if ((mode & (_StkCmt | _Shown)) == _StkCmt) { + show(); + } + } + bool operator==(const Stack& y) const & { + return s == y.s; + } + void apply_wrappers(int callxargs_count) { + bool is_inline = mode & _InlineFunc; + if (o.retalt_inserted_) { + o.insert(0, "SAMEALTSAVE"); + o.insert(0, "c2 SAVE"); + } + if (callxargs_count != -1 || (is_inline && o.retalt_)) { + o.indent_all(); + o.insert(0, "CONT:<{"); + o << "}>"; + if (callxargs_count != -1) { + if (callxargs_count <= 15) { + o << AsmOp::Custom(PSTRING() << callxargs_count << " -1 CALLXARGS"); + } else { + tolk_assert(callxargs_count <= 254); + o << AsmOp::Custom(PSTRING() << callxargs_count << " PUSHINT -1 PUSHINT CALLXVARARGS"); + } + } else { + o << "EXECUTE"; + } + } + } +}; + +/* + * + * SPECIFIC SYMBOL VALUES, + * BUILT-IN FUNCTIONS AND OPERATIONS + * + */ + +typedef std::function&, std::vector&, SrcLocation)> simple_compile_func_t; + +inline simple_compile_func_t make_simple_compile(AsmOp op) { + return [op](std::vector& out, std::vector& in, SrcLocation) -> AsmOp { return op; }; +} + +struct FunctionBodyBuiltin { + simple_compile_func_t simple_compile; + + explicit FunctionBodyBuiltin(simple_compile_func_t compile) + : simple_compile(std::move(compile)) {} + + void compile(AsmOpList& dest, std::vector& out, std::vector& in, SrcLocation where) const; +}; + +struct FunctionBodyAsm { + std::vector ops; + + void set_code(std::vector&& code); + void compile(AsmOpList& dest) const; +}; + +struct CodeBlob { + int var_cnt, in_var_cnt; + FunctionPtr fun_ref; + std::string name; + SrcLocation loc; + std::vector vars; + std::unique_ptr ops; + std::unique_ptr* cur_ops; +#ifdef TOLK_DEBUG + std::vector _vector_of_ops; // to see it in debugger instead of nested pointers +#endif + std::stack*> cur_ops_stack; + bool require_callxargs = false; + CodeBlob(std::string name, SrcLocation loc, FunctionPtr fun_ref) + : var_cnt(0), in_var_cnt(0), fun_ref(fun_ref), name(std::move(name)), loc(loc), cur_ops(&ops) { + } + template + Op& emplace_back(Args&&... args) { + Op& res = *(*cur_ops = std::make_unique(args...)); + cur_ops = &(res.next); +#ifdef TOLK_DEBUG + _vector_of_ops.push_back(&res); +#endif + return res; + } + std::vector create_var(TypePtr var_type, SrcLocation loc, std::string name); + std::vector create_tmp_var(TypePtr var_type, SrcLocation loc, const char* desc) { + std::vector ir_idx = create_var(var_type, loc, {}); +#ifdef TOLK_DEBUG + for (var_idx_t v : ir_idx) { + vars[v].desc = desc; + } +#endif + return ir_idx; + } + bool compute_used_code_vars(); + bool compute_used_code_vars(std::unique_ptr& ops, const VarDescrList& var_info, bool edit) const; + void print(std::ostream& os, int flags = 0) const; + void push_set_cur(std::unique_ptr& new_cur_ops) { + cur_ops_stack.push(cur_ops); + cur_ops = &new_cur_ops; + } + void close_blk(SrcLocation location) { + *cur_ops = std::make_unique(location, Op::_Nop); + } + void pop_cur() { + cur_ops = cur_ops_stack.top(); + cur_ops_stack.pop(); + } + void close_pop_cur(SrcLocation location) { + close_blk(location); + pop_cur(); + } + void prune_unreachable_code(); + void fwd_analyze(); + void mark_noreturn(); + void generate_code(AsmOpList& out_list, int mode = 0); + void generate_code(std::ostream& os, int mode = 0, int indent = 0); +}; + +// defined in builtins.cpp +AsmOp exec_arg_op(std::string op, long long arg); +AsmOp exec_arg_op(std::string op, long long arg, int args, int retv = 1); +AsmOp exec_arg_op(std::string op, td::RefInt256 arg); +AsmOp exec_arg_op(std::string op, td::RefInt256 arg, int args, int retv = 1); +AsmOp exec_arg2_op(std::string op, long long imm1, long long imm2, int args, int retv = 1); +AsmOp push_const(td::RefInt256 x); + +void define_builtins(); + + + +/* + * + * OUTPUT CODE GENERATOR + * + */ + +int tolk_proceed(const std::string &entrypoint_filename); + +} // namespace tolk + + diff --git a/tolk/type-system.cpp b/tolk/type-system.cpp new file mode 100644 index 00000000..d73625c2 --- /dev/null +++ b/tolk/type-system.cpp @@ -0,0 +1,756 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "type-system.h" +#include "lexer.h" +#include "platform-utils.h" +#include "compiler-state.h" +#include + +namespace tolk { + +/* + * This class stores a big hashtable [hash => TypeData] + * Every non-trivial TypeData*::create() method at first looks here, and allocates an object only if not found. + * That's why all allocated TypeData objects are unique, storing unique type_id. + */ +class TypeDataTypeIdCalculation { + uint64_t cur_hash; + int children_flags_mask = 0; + + static std::unordered_map all_unique_occurred_types; + +public: + explicit TypeDataTypeIdCalculation(uint64_t initial_arbitrary_unique_number) + : cur_hash(initial_arbitrary_unique_number) {} + + void feed_hash(uint64_t val) { + cur_hash = cur_hash * 56235515617499ULL + val; + } + + void feed_string(const std::string& s) { + feed_hash(std::hash{}(s)); + } + + void feed_child(TypePtr inner) { + feed_hash(inner->type_id); + children_flags_mask |= inner->flags; + } + + uint64_t type_id() const { + return cur_hash; + } + + int children_flags() const { + return children_flags_mask; + } + + GNU_ATTRIBUTE_FLATTEN + TypePtr get_existing() const { + auto it = all_unique_occurred_types.find(cur_hash); + return it != all_unique_occurred_types.end() ? it->second : nullptr; + } + + GNU_ATTRIBUTE_NOINLINE + TypePtr register_unique(TypePtr newly_created) const { +#ifdef TOLK_DEBUG + assert(newly_created->type_id == cur_hash); +#endif + all_unique_occurred_types[cur_hash] = newly_created; + return newly_created; + } +}; + +std::unordered_map TypeDataTypeIdCalculation::all_unique_occurred_types; +TypePtr TypeDataInt::singleton; +TypePtr TypeDataBool::singleton; +TypePtr TypeDataCell::singleton; +TypePtr TypeDataSlice::singleton; +TypePtr TypeDataBuilder::singleton; +TypePtr TypeDataTuple::singleton; +TypePtr TypeDataContinuation::singleton; +TypePtr TypeDataNullLiteral::singleton; +TypePtr TypeDataUnknown::singleton; +TypePtr TypeDataNever::singleton; +TypePtr TypeDataVoid::singleton; + +void type_system_init() { + TypeDataInt::singleton = new TypeDataInt; + TypeDataBool::singleton = new TypeDataBool; + TypeDataCell::singleton = new TypeDataCell; + TypeDataSlice::singleton = new TypeDataSlice; + TypeDataBuilder::singleton = new TypeDataBuilder; + TypeDataTuple::singleton = new TypeDataTuple; + TypeDataContinuation::singleton = new TypeDataContinuation; + TypeDataNullLiteral::singleton = new TypeDataNullLiteral; + TypeDataUnknown::singleton = new TypeDataUnknown; + TypeDataNever::singleton = new TypeDataNever; + TypeDataVoid::singleton = new TypeDataVoid; +} + + +// -------------------------------------------- +// create() +// +// all constructors of TypeData classes are private, only TypeData*::create() is allowed +// each non-trivial create() method calculates hash (type_id) +// and creates an object only if it isn't found in a global hashtable +// + +TypePtr TypeDataNullable::create(TypePtr inner) { + TypeDataTypeIdCalculation hash(1774084920039440885ULL); + hash.feed_child(inner); + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + // most types (int?, slice?, etc.), when nullable, still occupy 1 stack slot (holding TVM NULL at runtime) + // but for example for `(int, int)` we need an extra stack slot "null flag" + int width_on_stack = inner->can_hold_tvm_null_instead() ? 1 : inner->get_width_on_stack() + 1; + return hash.register_unique(new TypeDataNullable(hash.type_id(), hash.children_flags(), width_on_stack, inner)); +} + +TypePtr TypeDataFunCallable::create(std::vector&& params_types, TypePtr return_type) { + TypeDataTypeIdCalculation hash(3184039965511020991ULL); + for (TypePtr param : params_types) { + hash.feed_child(param); + hash.feed_hash(767721); + } + hash.feed_child(return_type); + hash.feed_hash(767722); + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + return hash.register_unique(new TypeDataFunCallable(hash.type_id(), hash.children_flags(), std::move(params_types), return_type)); +} + +TypePtr TypeDataGenericT::create(std::string&& nameT) { + TypeDataTypeIdCalculation hash(9145033724911680012ULL); + hash.feed_string(nameT); + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + return hash.register_unique(new TypeDataGenericT(hash.type_id(), std::move(nameT))); +} + +TypePtr TypeDataTensor::create(std::vector&& items) { + TypeDataTypeIdCalculation hash(3159238551239480381ULL); + for (TypePtr item : items) { + hash.feed_child(item); + hash.feed_hash(819613); + } + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + int width_on_stack = 0; + for (TypePtr item : items) { + width_on_stack += item->get_width_on_stack(); + } + return hash.register_unique(new TypeDataTensor(hash.type_id(), hash.children_flags(), width_on_stack, std::move(items))); +} + +TypePtr TypeDataTypedTuple::create(std::vector&& items) { + TypeDataTypeIdCalculation hash(9189266157349499320ULL); + for (TypePtr item : items) { + hash.feed_child(item); + hash.feed_hash(735911); + } + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + return hash.register_unique(new TypeDataTypedTuple(hash.type_id(), hash.children_flags(), std::move(items))); +} + +TypePtr TypeDataUnresolved::create(std::string&& text, SrcLocation loc) { + TypeDataTypeIdCalculation hash(3680147223540048162ULL); + hash.feed_string(text); + // hash.feed_hash(*reinterpret_cast(&loc)); + + if (TypePtr existing = hash.get_existing()) { + return existing; + } + return hash.register_unique(new TypeDataUnresolved(hash.type_id(), std::move(text), loc)); +} + + +// -------------------------------------------- +// as_human_readable() +// +// is used only for error messages and debugging, therefore no optimizations for simplicity +// only non-trivial implementations are here; trivial are defined in .h file +// + +std::string TypeDataNullable::as_human_readable() const { + std::string nested = inner->as_human_readable(); + bool embrace = inner->try_as(); + return embrace ? "(" + nested + ")?" : nested + "?"; +} + +std::string TypeDataFunCallable::as_human_readable() const { + std::string result = "("; + for (TypePtr param : params_types) { + if (result.size() > 1) { + result += ", "; + } + result += param->as_human_readable(); + } + result += ") -> "; + result += return_type->as_human_readable(); + return result; +} + +std::string TypeDataTensor::as_human_readable() const { + std::string result = "("; + for (TypePtr item : items) { + if (result.size() > 1) { + result += ", "; + } + result += item->as_human_readable(); + } + result += ')'; + return result; +} + +std::string TypeDataTypedTuple::as_human_readable() const { + std::string result = "["; + for (TypePtr item : items) { + if (result.size() > 1) { + result += ", "; + } + result += item->as_human_readable(); + } + result += ']'; + return result; +} + + +// -------------------------------------------- +// traverse() +// +// invokes a callback for TypeData itself and all its children +// only non-trivial implementations are here; by default (no children), `callback(this)` is executed +// + +void TypeDataNullable::traverse(const TraverserCallbackT& callback) const { + callback(this); + inner->traverse(callback); +} + +void TypeDataFunCallable::traverse(const TraverserCallbackT& callback) const { + callback(this); + for (TypePtr param : params_types) { + param->traverse(callback); + } + return_type->traverse(callback); +} + +void TypeDataTensor::traverse(const TraverserCallbackT& callback) const { + callback(this); + for (TypePtr item : items) { + item->traverse(callback); + } +} + +void TypeDataTypedTuple::traverse(const TraverserCallbackT& callback) const { + callback(this); + for (TypePtr item : items) { + item->traverse(callback); + } +} + + +// -------------------------------------------- +// replace_children_custom() +// +// returns new TypeData with children replaced by a custom callback +// used to replace generic T on generics expansion — to convert `f` to `f` +// only non-trivial implementations are here; by default (no children), `return callback(this)` is executed +// + +TypePtr TypeDataNullable::replace_children_custom(const ReplacerCallbackT& callback) const { + return callback(create(inner->replace_children_custom(callback))); +} + +TypePtr TypeDataFunCallable::replace_children_custom(const ReplacerCallbackT& callback) const { + std::vector mapped; + mapped.reserve(params_types.size()); + for (TypePtr param : params_types) { + mapped.push_back(param->replace_children_custom(callback)); + } + return callback(create(std::move(mapped), return_type->replace_children_custom(callback))); +} + +TypePtr TypeDataTensor::replace_children_custom(const ReplacerCallbackT& callback) const { + std::vector mapped; + mapped.reserve(items.size()); + for (TypePtr item : items) { + mapped.push_back(item->replace_children_custom(callback)); + } + return callback(create(std::move(mapped))); +} + +TypePtr TypeDataTypedTuple::replace_children_custom(const ReplacerCallbackT& callback) const { + std::vector mapped; + mapped.reserve(items.size()); + for (TypePtr item : items) { + mapped.push_back(item->replace_children_custom(callback)); + } + return callback(create(std::move(mapped))); +} + + +// -------------------------------------------- +// can_rhs_be_assigned() +// +// on `var lhs: = rhs`, having inferred rhs_type, check that it can be assigned without any casts +// the same goes for passing arguments, returning values, etc. — where the "receiver" (lhs) checks "applier" (rhs) +// + +bool TypeDataInt::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataBool::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataCell::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataSlice::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataBuilder::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataTuple::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataContinuation::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataNullLiteral::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataNullable::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + if (rhs == TypeDataNullLiteral::create()) { + return true; + } + if (const TypeDataNullable* rhs_nullable = rhs->try_as()) { + return inner->can_rhs_be_assigned(rhs_nullable->inner); + } + if (inner->can_rhs_be_assigned(rhs)) { + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataFunCallable::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataGenericT::can_rhs_be_assigned(TypePtr rhs) const { + assert(false); + return false; +} + +bool TypeDataTensor::can_rhs_be_assigned(TypePtr rhs) const { + if (const auto* as_tensor = rhs->try_as(); as_tensor && as_tensor->size() == size()) { + for (int i = 0; i < size(); ++i) { + if (!items[i]->can_rhs_be_assigned(as_tensor->items[i])) { + return false; + } + } + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataTypedTuple::can_rhs_be_assigned(TypePtr rhs) const { + if (const auto* as_tuple = rhs->try_as(); as_tuple && as_tuple->size() == size()) { + for (int i = 0; i < size(); ++i) { + if (!items[i]->can_rhs_be_assigned(as_tuple->items[i])) { + return false; + } + } + return true; + } + return rhs == TypeDataNever::create(); +} + +bool TypeDataUnknown::can_rhs_be_assigned(TypePtr rhs) const { + return true; +} + +bool TypeDataUnresolved::can_rhs_be_assigned(TypePtr rhs) const { + assert(false); + return false; +} + +bool TypeDataNever::can_rhs_be_assigned(TypePtr rhs) const { + return true; +} + +bool TypeDataVoid::can_rhs_be_assigned(TypePtr rhs) const { + if (rhs == this) { + return true; + } + return rhs == TypeDataNever::create(); +} + + +// -------------------------------------------- +// can_be_casted_with_as_operator() +// +// on `expr as `, check whether casting is applicable +// note, that it's not auto-casts `var lhs: = rhs`, it's an expression `rhs as ` +// + +bool TypeDataInt::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_nullable = cast_to->try_as()) { // `int` as `int?` + return can_be_casted_with_as_operator(to_nullable->inner); + } + return cast_to == this; +} + +bool TypeDataBool::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_nullable = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_nullable->inner); + } + return cast_to == this || cast_to == TypeDataInt::create(); +} + +bool TypeDataCell::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_nullable = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_nullable->inner); + } + return cast_to == this; +} + +bool TypeDataSlice::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_nullable = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_nullable->inner); + } + return cast_to == this; +} + +bool TypeDataBuilder::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_nullable = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_nullable->inner); + } + return cast_to == this; +} + +bool TypeDataTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_nullable = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_nullable->inner); + } + return cast_to == this; +} + +bool TypeDataContinuation::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_nullable = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_nullable->inner); + } + return cast_to == this; +} + +bool TypeDataNullLiteral::can_be_casted_with_as_operator(TypePtr cast_to) const { + return cast_to == this || cast_to->try_as(); +} + +bool TypeDataNullable::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_nullable = cast_to->try_as()) { + return inner->can_be_casted_with_as_operator(to_nullable->inner); + } + return false; +} + +bool TypeDataFunCallable::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_nullable = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_nullable->inner); + } + return this == cast_to; +} + +bool TypeDataGenericT::can_be_casted_with_as_operator(TypePtr cast_to) const { + return true; +} + +bool TypeDataTensor::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_tensor = cast_to->try_as(); to_tensor && to_tensor->size() == size()) { + for (int i = 0; i < size(); ++i) { + if (!items[i]->can_be_casted_with_as_operator(to_tensor->items[i])) { + return false; + } + } + return true; + } + if (const auto* to_nullable = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_nullable->inner); + } + return false; +} + +bool TypeDataTypedTuple::can_be_casted_with_as_operator(TypePtr cast_to) const { + if (const auto* to_tuple = cast_to->try_as(); to_tuple && to_tuple->size() == size()) { + for (int i = 0; i < size(); ++i) { + if (!items[i]->can_be_casted_with_as_operator(to_tuple->items[i])) { + return false; + } + } + return true; + } + if (const auto* to_nullable = cast_to->try_as()) { + return can_be_casted_with_as_operator(to_nullable->inner); + } + return false; +} + +bool TypeDataUnknown::can_be_casted_with_as_operator(TypePtr cast_to) const { + // 'unknown' can be cast to any TVM value + return cast_to->get_width_on_stack() == 1; +} + +bool TypeDataUnresolved::can_be_casted_with_as_operator(TypePtr cast_to) const { + return false; +} + +bool TypeDataNever::can_be_casted_with_as_operator(TypePtr cast_to) const { + return true; +} + +bool TypeDataVoid::can_be_casted_with_as_operator(TypePtr cast_to) const { + return cast_to == this; +} + + +// -------------------------------------------- +// can_hold_tvm_null_instead() +// +// assigning `null` to a primitive variable like `int?` / `cell?` can store TVM NULL inside the same slot +// (that's why the default implementation is just "return true", and most of types occupy 1 slot) +// but for complex variables, like `(int, int)?`, "null presence" is kept in a separate slot (UTag for union types) +// though still, tricky situations like `(int, ())?` can still "embed" TVM NULL in parallel with original value +// + +bool TypeDataNullable::can_hold_tvm_null_instead() const { + if (get_width_on_stack() != 1) { // `(int, int)?` / `()?` can not hold null instead + return false; // only `int?` / `cell?` / `StructWith1IntField?` can + } // and some tricky situations like `(int, ())?`, but not `(int?, ())?` + return !inner->can_hold_tvm_null_instead(); +} + +bool TypeDataTensor::can_hold_tvm_null_instead() const { + if (get_width_on_stack() != 1) { // `(int, int)` / `()` can not hold null instead, since null is 1 slot + return false; // only `((), int)` and similar can: + } // one item is width 1 (and not nullable), others are 0 + for (TypePtr item : items) { + if (item->get_width_on_stack() == 1 && !item->can_hold_tvm_null_instead()) { + return false; + } + } + return true; +} + +bool TypeDataNever::can_hold_tvm_null_instead() const { + return false; +} + +bool TypeDataVoid::can_hold_tvm_null_instead() const { + return false; +} + + +// -------------------------------------------- +// parsing type from tokens +// +// here we implement parsing types (mostly after colon) to TypeData +// example: `var v: int` is TypeDataInt +// example: `var v: (builder?, [cell])` is TypeDataTensor(TypeDataNullable(TypeDataBuilder), TypeDataTypedTuple(TypeDataCell)) +// example: `fun f(): ()` is TypeDataTensor() (an empty one) +// +// note, that unrecognized type names (MyEnum, MyStruct, T) are parsed as TypeDataUnresolved, +// and later, when all files are parsed and all symbols registered, such identifiers are resolved +// example: `fun f(v: T)` at first v is TypeDataUnresolved("T"), later becomes TypeDataGenericT +// see finalize_type_data() +// +// note, that `self` does not name a type, it can appear only as a return value of a function (parsed specially) +// when `self` appears as a type, it's parsed as TypeDataUnresolved, and later an error is emitted +// + +static TypePtr parse_type_expression(Lexer& lex); + +std::vector parse_nested_type_list(Lexer& lex, TokenType tok_op, const char* s_op, TokenType tok_cl, const char* s_cl) { + lex.expect(tok_op, s_op); + std::vector sub_types; + while (true) { + if (lex.tok() == tok_cl) { // empty lists allowed + lex.next(); + break; + } + + sub_types.emplace_back(parse_type_expression(lex)); + if (lex.tok() == tok_comma) { + lex.next(); + } else if (lex.tok() != tok_cl) { + lex.unexpected(s_cl); + } + } + return sub_types; +} + +std::vector parse_nested_type_list_in_parenthesis(Lexer& lex) { + return parse_nested_type_list(lex, tok_oppar, "`(`", tok_clpar, "`)` or `,`"); +} + +static TypePtr parse_simple_type(Lexer& lex) { + switch (lex.tok()) { + case tok_self: + case tok_identifier: { + SrcLocation loc = lex.cur_location(); + std::string_view str = lex.cur_str(); + lex.next(); + switch (str.size()) { + case 3: + if (str == "int") return TypeDataInt::create(); + break; + case 4: + if (str == "cell") return TypeDataCell::create(); + if (str == "void") return TypeDataVoid::create(); + if (str == "bool") return TypeDataBool::create(); + break; + case 5: + if (str == "slice") return TypeDataSlice::create(); + if (str == "tuple") return TypeDataTuple::create(); + if (str == "never") return TypeDataNever::create(); + break; + case 7: + if (str == "builder") return TypeDataBuilder::create(); + break; + case 12: + if (str == "continuation") return TypeDataContinuation::create(); + break; + default: + break; + } + return TypeDataUnresolved::create(std::string(str), loc); + } + case tok_null: + lex.next(); + return TypeDataNullLiteral::create(); + case tok_oppar: { + std::vector items = parse_nested_type_list_in_parenthesis(lex); + if (items.size() == 1) { + return items.front(); + } + return TypeDataTensor::create(std::move(items)); + } + case tok_opbracket: { + std::vector items = parse_nested_type_list(lex, tok_opbracket, "`[`", tok_clbracket, "`]` or `,`"); + return TypeDataTypedTuple::create(std::move(items)); + } + default: + lex.unexpected(""); + } +} + +static TypePtr parse_type_nullable(Lexer& lex) { + TypePtr result = parse_simple_type(lex); + + if (lex.tok() == tok_question) { + lex.next(); + result = TypeDataNullable::create(result); + } + + return result; +} + +static TypePtr parse_type_expression(Lexer& lex) { + TypePtr result = parse_type_nullable(lex); + + if (lex.tok() == tok_arrow) { // `int -> int`, `(cell, slice) -> void` + lex.next(); + TypePtr return_type = parse_type_expression(lex); + std::vector params_types = {result}; + if (const auto* as_tensor = result->try_as()) { + params_types = as_tensor->items; + } + return TypeDataFunCallable::create(std::move(params_types), return_type); + } + + if (lex.tok() != tok_bitwise_or) { + return result; + } + + lex.error("union types are not supported yet"); +} + +TypePtr parse_type_from_tokens(Lexer& lex) { + return parse_type_expression(lex); +} + +// for internal usage only +TypePtr parse_type_from_string(std::string_view text) { + Lexer lex(text); + return parse_type_expression(lex); +} + +std::ostream& operator<<(std::ostream& os, TypePtr type_data) { + return os << (type_data ? type_data->as_human_readable() : "(nullptr-type)"); +} + +} // namespace tolk diff --git a/tolk/type-system.h b/tolk/type-system.h new file mode 100644 index 00000000..4b671e30 --- /dev/null +++ b/tolk/type-system.h @@ -0,0 +1,463 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "src-file.h" +#include +#include +#include + +namespace tolk { + +/* + * TypeData is both a user-given and an inferred type representation. + * `int`, `cell`, `T`, `(int, [tuple])` are instances of TypeData. + * Every unique TypeData is created only once, so for example TypeDataTensor::create(int, int) + * returns one and the same pointer always. This "uniqueness" is called type_id, calculated before creation. + * + * In Tolk code, types after colon `var v: (int, T)` are parsed to TypeData. + * See parse_type_from_tokens(). + * So, AST nodes which can have declared types (local/global variables and others) store a pointer to TypeData. + * + * Type inferring also creates TypeData for inferred expressions. All AST expression nodes have inferred_type. + * For example, `1 + 2`, both operands are TypeDataInt, its result is also TypeDataInt. + * Type checking also uses TypeData. For example, `var i: slice = 1 + 2`, at first rhs (TypeDataInt) is inferred, + * then lhs (TypeDataSlice from declaration) is checked whether rhs can be assigned. + * See can_rhs_be_assigned(). + * + * Note, that while initial parsing Tolk files to AST, known types (`int`, `cell`, etc.) are created as-is, + * but user-defined types (`T`, `MyStruct`, `MyAlias`) are saved as TypeDataUnresolved. + * After all symbols have been registered, resolving identifiers step is executed, where particularly + * all TypeDataUnresolved instances are converted to a resolved type. At inferring, no unresolved remain. + * For instance, `fun f(v: T)`, at first "T" of `v` is unresolved, and then converted to TypeDataGenericT. + */ +class TypeData { + // all unique types have unique type_id; it's used both for allocating memory once and for tagged unions + const uint64_t type_id; + // bits of flag_mask, to store often-used properties and return them without tree traversing + const int flags; + // how many slots on a stack this type occupies (calculated on creation), e.g. `int`=1, `(int,int)`=2, `(int,int)?`=3 + const int width_on_stack; + + friend class TypeDataTypeIdCalculation; + +protected: + enum flag_mask { + flag_contains_unknown_inside = 1 << 1, + flag_contains_genericT_inside = 1 << 2, + flag_contains_unresolved_inside = 1 << 3, + }; + + explicit TypeData(uint64_t type_id, int flags_with_children, int width_on_stack) + : type_id(type_id) + , flags(flags_with_children) + , width_on_stack(width_on_stack) { + } + +public: + virtual ~TypeData() = default; + + template + const Derived* try_as() const { + return dynamic_cast(this); + } + + uint64_t get_type_id() const { return type_id; } + int get_width_on_stack() const { return width_on_stack; } + + bool has_unknown_inside() const { return flags & flag_contains_unknown_inside; } + bool has_genericT_inside() const { return flags & flag_contains_genericT_inside; } + bool has_unresolved_inside() const { return flags & flag_contains_unresolved_inside; } + + using TraverserCallbackT = std::function; + using ReplacerCallbackT = std::function; + + virtual std::string as_human_readable() const = 0; + virtual bool can_rhs_be_assigned(TypePtr rhs) const = 0; + virtual bool can_be_casted_with_as_operator(TypePtr cast_to) const = 0; + + virtual bool can_hold_tvm_null_instead() const { + return true; + } + + virtual void traverse(const TraverserCallbackT& callback) const { + callback(this); + } + + virtual TypePtr replace_children_custom(const ReplacerCallbackT& callback) const { + return callback(this); + } +}; + +/* + * `int` is TypeDataInt, representation of TVM int. + */ +class TypeDataInt final : public TypeData { + TypeDataInt() : TypeData(1ULL, 0, 1) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "int"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * `bool` is TypeDataBool. TVM has no bool, only integers. Under the hood, -1 is true, 0 is false. + * From the type system point of view, int and bool are different, not-autocastable types. + */ +class TypeDataBool final : public TypeData { + TypeDataBool() : TypeData(2ULL, 0, 1) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "bool"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * `cell` is TypeDataCell, representation of TVM cell. + */ +class TypeDataCell final : public TypeData { + TypeDataCell() : TypeData(3ULL, 0, 1) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "cell"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * `slice` is TypeDataSlice, representation of TVM slice. + */ +class TypeDataSlice final : public TypeData { + TypeDataSlice() : TypeData(4ULL, 0, 1) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "slice"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * `builder` is TypeDataBuilder, representation of TVM builder. + */ +class TypeDataBuilder final : public TypeData { + TypeDataBuilder() : TypeData(5ULL, 0, 1) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "builder"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * `tuple` is TypeDataTuple, representation of TVM tuple. + * Note, that it's UNTYPED tuple. It occupies 1 stack slot in TVM. Its elements are any TVM values at runtime, + * so getting its element results in TypeDataUnknown (which must be assigned/cast explicitly). + */ +class TypeDataTuple final : public TypeData { + TypeDataTuple() : TypeData(6ULL, 0, 1) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "tuple"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * `continuation` is TypeDataContinuation, representation of TVM continuation. + * It's like "untyped callable", not compatible with other types. + */ +class TypeDataContinuation final : public TypeData { + TypeDataContinuation() : TypeData(7ULL, 0, 1) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "continuation"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * `null` has TypeDataNullLiteral type. + * It can be assigned only to nullable types (`int?`, etc.), to ensure null safety. + * Note, that `var i = null`, though valid (i would be constant null), fires an "always-null" compilation error + * (it's much better for user to see an error here than when he passes this variable somewhere). + */ +class TypeDataNullLiteral final : public TypeData { + TypeDataNullLiteral() : TypeData(8ULL, 0, 1) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "null"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * `T?` is "nullable T". + * It can be converted to T either with ! (non-null assertion operator) or with smart casts. + */ +class TypeDataNullable final : public TypeData { + TypeDataNullable(uint64_t type_id, int children_flags, int width_on_stack, TypePtr inner) + : TypeData(type_id, children_flags, width_on_stack) + , inner(inner) {} + +public: + const TypePtr inner; + + static TypePtr create(TypePtr inner); + + bool is_primitive_nullable() const { return get_width_on_stack() == 1 && inner->get_width_on_stack() == 1; } + + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + void traverse(const TraverserCallbackT& callback) const override; + TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; + bool can_hold_tvm_null_instead() const override; +}; + +/* + * `fun(int, int) -> void` is TypeDataFunCallable, think of is as a typed continuation. + * A type of function `fun f(x: int) { return x; }` is actually `fun(int) -> int`. + * So, when assigning it to a variable `var cb = f`, this variable also has this type. + */ +class TypeDataFunCallable final : public TypeData { + TypeDataFunCallable(uint64_t type_id, int children_flags, std::vector&& params_types, TypePtr return_type) + : TypeData(type_id, children_flags, 1) + , params_types(std::move(params_types)) + , return_type(return_type) {} + +public: + const std::vector params_types; + const TypePtr return_type; + + static TypePtr create(std::vector&& params_types, TypePtr return_type); + + int params_size() const { return static_cast(params_types.size()); } + + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + void traverse(const TraverserCallbackT& callback) const override; + TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; +}; + +/* + * `T` inside generic functions is TypeDataGenericT. + * Example: `fun f(a: X, b: Y): [X, Y]` (here X and Y are). + * On instantiation like `f(1,"")`, a new function `f` is created with type `fun(int,slice)->[int,slice]`. + */ +class TypeDataGenericT final : public TypeData { + TypeDataGenericT(uint64_t type_id, std::string&& nameT) + : TypeData(type_id, flag_contains_genericT_inside, -999999) // width undefined until instantiated + , nameT(std::move(nameT)) {} + +public: + const std::string nameT; + + static TypePtr create(std::string&& nameT); + + std::string as_human_readable() const override { return nameT; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * `(int, slice)` is TypeDataTensor of 2 elements. Tensor of N elements occupies N stack slots. + * Of course, there may be nested tensors, like `(int, (int, slice), cell)`. + * Arguments, variables, globals, return values, etc. can be tensors. + * A tensor can be empty. + */ +class TypeDataTensor final : public TypeData { + TypeDataTensor(uint64_t type_id, int children_flags, int width_on_stack, std::vector&& items) + : TypeData(type_id, children_flags, width_on_stack) + , items(std::move(items)) {} + +public: + const std::vector items; + + static TypePtr create(std::vector&& items); + + int size() const { return static_cast(items.size()); } + + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + void traverse(const TraverserCallbackT& callback) const override; + TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; + bool can_hold_tvm_null_instead() const override; +}; + +/* + * `[int, slice]` is TypeDataTypedTuple, a TVM 'tuple' under the hood, contained in 1 stack slot. + * Unlike TypeDataTuple (untyped tuples), it has a predefined inner structure and can be assigned as + * `var [i, cs] = [0, ""]` (where a and b become two separate variables on a stack, int and slice). + */ +class TypeDataTypedTuple final : public TypeData { + TypeDataTypedTuple(uint64_t type_id, int children_flags, std::vector&& items) + : TypeData(type_id, children_flags, 1) + , items(std::move(items)) {} + +public: + const std::vector items; + + static TypePtr create(std::vector&& items); + + int size() const { return static_cast(items.size()); } + + std::string as_human_readable() const override; + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + void traverse(const TraverserCallbackT& callback) const override; + TypePtr replace_children_custom(const ReplacerCallbackT& callback) const override; +}; + +/* + * `unknown` is a special type, which can appear in corner cases. + * The type of exception argument (which can hold any TVM value at runtime) is unknown. + * The type of `_` used as rvalue is unknown. + * The only thing available to do with unknown is to cast it: `catch (excNo, arg) { var i = arg as int; }` + */ +class TypeDataUnknown final : public TypeData { + TypeDataUnknown() : TypeData(20ULL, flag_contains_unknown_inside, 1) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "unknown"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * "Unresolved" is not actually a type — it's an intermediate state between parsing and resolving. + * At parsing to AST, unrecognized type names (MyEnum, MyStruct, T) are parsed as TypeDataUnresolved, + * and after all source files parsed and global symbols registered, they are replaced by actual ones. + * Example: `fun f(v: T)` at first v is TypeDataUnresolved("T"), later becomes TypeDataGenericT. + */ +class TypeDataUnresolved final : public TypeData { + TypeDataUnresolved(uint64_t type_id, std::string&& text, SrcLocation loc) + : TypeData(type_id, flag_contains_unresolved_inside, -999999) + , text(std::move(text)) + , loc(loc) {} + +public: + const std::string text; + const SrcLocation loc; + + static TypePtr create(std::string&& text, SrcLocation loc); + + std::string as_human_readable() const override { return text + "*"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; +}; + +/* + * `never` is a special type meaning "no value can be hold". + * Is may appear due to smart casts, for example `if (x == null && x != null)` makes x "never". + * Functions returning "never" assume to never exit, calling them interrupts control flow. + * Such variables can not be cast to any other types, all their usage will trigger type mismatch errors. + */ +class TypeDataNever final : public TypeData { + TypeDataNever() : TypeData(19ULL, 0, 0) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "never"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + bool can_hold_tvm_null_instead() const override; +}; + +/* + * `void` is TypeDataVoid. + * From the type system point of view, `void` functions return nothing. + * Empty tensor is not compatible with void, although at IR level they are similar, 0 stack slots. + */ +class TypeDataVoid final : public TypeData { + TypeDataVoid() : TypeData(10ULL, 0, 0) {} + + static TypePtr singleton; + friend void type_system_init(); + +public: + static TypePtr create() { return singleton; } + + std::string as_human_readable() const override { return "void"; } + bool can_rhs_be_assigned(TypePtr rhs) const override; + bool can_be_casted_with_as_operator(TypePtr cast_to) const override; + bool can_hold_tvm_null_instead() const override; +}; + + +// -------------------------------------------- + + +class Lexer; +TypePtr parse_type_from_tokens(Lexer& lex); +TypePtr parse_type_from_string(std::string_view text); + +void type_system_init(); + +} // namespace tolk diff --git a/ton/ton-types.h b/ton/ton-types.h index efdb795d..c7aff644 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -120,6 +120,26 @@ struct ShardIdFull { char buffer[64]; return std::string{buffer, (unsigned)snprintf(buffer, 63, "(%d,%016llx)", workchain, (unsigned long long)shard)}; } + static td::Result parse(td::Slice s) { + // Formats: (0,2000000000000000) (0:2000000000000000) 0,2000000000000000 0:2000000000000000 + if (s.empty()) { + return td::Status::Error("empty string"); + } + if (s[0] == '(' && s.back() == ')') { + s = s.substr(1, s.size() - 2); + } + auto sep = s.find(':'); + if (sep == td::Slice::npos) { + sep = s.find(','); + } + if (sep == td::Slice::npos || s.size() - sep - 1 != 16) { + return td::Status::Error(PSTRING() << "invalid shard " << s); + } + ShardIdFull shard; + TRY_RESULT_ASSIGN(shard.workchain, td::to_integer_safe(s.substr(0, sep))); + TRY_RESULT_ASSIGN(shard.shard, td::hex_to_integer_safe(s.substr(sep + 1))); + return shard; + } }; struct AccountIdPrefixFull { @@ -408,6 +428,9 @@ struct Ed25519_PublicKey { bool operator==(const Ed25519_PublicKey& other) const { return _pubkey == other._pubkey; } + bool operator!=(const Ed25519_PublicKey& other) const { + return _pubkey != other._pubkey; + } bool clear() { _pubkey.set_zero(); return true; @@ -470,6 +493,7 @@ struct CatChainOptions { td::uint64 max_block_height_coeff = 0; bool debug_disable_db = false; + double broadcast_speed_multiplier = 1.0; }; struct ValidatorSessionConfig { @@ -490,4 +514,14 @@ struct ValidatorSessionConfig { static const td::uint32 BLOCK_HASH_COVERS_DATA_FROM_VERSION = 2; }; +struct PersistentStateDescription : public td::CntObject { + BlockIdExt masterchain_id; + std::vector shard_blocks; + UnixTime start_time, end_time; + + virtual CntObject* make_copy() const { + return new PersistentStateDescription(*this); + } +}; + } // namespace ton diff --git a/tonlib/CMakeLists.txt b/tonlib/CMakeLists.txt index 67d18e01..3dbd628d 100644 --- a/tonlib/CMakeLists.txt +++ b/tonlib/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.5 FATAL_ERROR) -option(BUILD_SHARED_LIBS "Use \"OFF\" for a static build." ON) +option(TONLIBJSON_STATIC "Build tonlibjson as static library" OFF) if (NOT OPENSSL_FOUND) find_package(OpenSSL REQUIRED) @@ -10,7 +10,6 @@ set(TONLIB_SOURCE tonlib/Client.cpp tonlib/Config.cpp tonlib/ExtClient.cpp - tonlib/ExtClientLazy.cpp tonlib/ExtClientOutbound.cpp tonlib/KeyStorage.cpp tonlib/KeyValue.cpp @@ -25,7 +24,6 @@ set(TONLIB_SOURCE tonlib/Client.h tonlib/Config.h tonlib/ExtClient.h - tonlib/ExtClientLazy.h tonlib/ExtClientOutbound.h tonlib/KeyStorage.h tonlib/KeyValue.h @@ -94,15 +92,20 @@ set(TONLIB_JSON_HEADERS tonlib/tonlib_client_json.h) set(TONLIB_JSON_SOURCE tonlib/tonlib_client_json.cpp) include(GenerateExportHeader) -if (NOT USE_EMSCRIPTEN AND BUILD_SHARED_LIBS) - add_library(tonlibjson SHARED ${TONLIB_JSON_SOURCE} ${TONLIB_JSON_HEADERS}) +if (TONLIBJSON_STATIC OR USE_EMSCRIPTEN) + add_library(tonlibjson STATIC ${TONLIB_JSON_SOURCE}) else() - add_library(tonlibjson STATIC ${TONLIB_JSON_SOURCE} ${TONLIB_JSON_HEADERS}) + add_library(tonlibjson SHARED ${TONLIB_JSON_SOURCE}) +endif() + +if (PORTABLE AND NOT APPLE) + target_link_libraries(tonlibjson PRIVATE tonlibjson_private -static-libgcc -static-libstdc++) +else() + target_link_libraries(tonlibjson PRIVATE tonlibjson_private) endif() -target_link_libraries(tonlibjson PRIVATE tonlibjson_private) generate_export_header(tonlibjson EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/tonlib/tonlibjson_export.h) -if (!BUILD_SHARED_LIBS) +if (TONLIBJSON_STATIC OR USE_EMSCRIPTEN) target_compile_definitions(tonlibjson PUBLIC TONLIBJSON_STATIC_DEFINE) endif() target_include_directories(tonlibjson PUBLIC @@ -156,7 +159,7 @@ endif() install(FILES ${TONLIB_JSON_HEADERS} ${CMAKE_CURRENT_BINARY_DIR}/tonlib/tonlibjson_export.h DESTINATION include/tonlib/) -if (NOT USE_EMSCRIPTEN AND BUILD_SHARED_LIBS) +if (NOT USE_EMSCRIPTEN AND NOT TONLIBJSON_STATIC) install(EXPORT Tonlib FILE TonlibTargets.cmake NAMESPACE Tonlib:: diff --git a/tonlib/test/offline.cpp b/tonlib/test/offline.cpp index b7423853..47d5d6a2 100644 --- a/tonlib/test/offline.cpp +++ b/tonlib/test/offline.cpp @@ -659,11 +659,12 @@ TEST(Tonlib, ConfigCache) { ], "validator": { "@type": "validator.config.global", - "zero_state": { + "init_block": { "workchain": -1, "shard": -9223372036854775808, "seqno": 0, "file_hash": "eh9yveSz1qMdJ7mOsO+I+H77jkLr9NpAuEkoJuseXBo=" + "root_hash": "ZXSXxDHhTALFxReyTZRd8E4Ya3ySOmpOWAS4rBX9XBY=", } } })abc"; diff --git a/tonlib/tonlib/Config.cpp b/tonlib/tonlib/Config.cpp index 3cd9d50e..063b34bb 100644 --- a/tonlib/tonlib/Config.cpp +++ b/tonlib/tonlib/Config.cpp @@ -19,6 +19,8 @@ #include "Config.h" #include "adnl/adnl-node-id.hpp" #include "td/utils/JsonBuilder.h" +#include "auto/tl/ton_api_json.h" +#include "ton/ton-tl.hpp" namespace tonlib { td::Result parse_block_id_ext(td::JsonObject &obj) { @@ -63,75 +65,26 @@ td::Result parse_block_id_ext(td::JsonObject &obj) { td::Result Config::parse(std::string str) { TRY_RESULT(json, td::json_decode(str)); if (json.type() != td::JsonValue::Type::Object) { - return td::Status::Error("Invalid config (1)"); + return td::Status::Error("Invalid config: json is not an object"); } - //TRY_RESULT(main_type, td::get_json_object_string_field(json.get_object(), "@type", false)); - //if (main_type != "config.global") { - //return td::Status::Error("Invalid config (3)"); - //} - TRY_RESULT(lite_clients_obj, - td::get_json_object_field(json.get_object(), "liteservers", td::JsonValue::Type::Array, false)); - auto &lite_clients = lite_clients_obj.get_array(); - Config res; - for (auto &value : lite_clients) { - if (value.type() != td::JsonValue::Type::Object) { - return td::Status::Error("Invalid config (2)"); - } - auto &object = value.get_object(); - //TRY_RESULT(value_type, td::get_json_object_string_field(object, "@type", false)); - //if (value_type != "liteclient.config.global") { - //return td::Status::Error("Invalid config (4)"); - //} + ton::ton_api::liteclient_config_global conf; + TRY_STATUS(ton::ton_api::from_json(conf, json.get_object())); + TRY_RESULT_ASSIGN(res.lite_servers, liteclient::LiteServerConfig::parse_global_config(conf)); - TRY_RESULT(ip, td::get_json_object_long_field(object, "ip", false)); - TRY_RESULT(port, td::get_json_object_int_field(object, "port", false)); - Config::LiteClient client; - TRY_STATUS(client.address.init_host_port(td::IPAddress::ipv4_to_str(static_cast(ip)), port)); - - TRY_RESULT(id_obj, td::get_json_object_field(object, "id", td::JsonValue::Type::Object, false)); - auto &id = id_obj.get_object(); - TRY_RESULT(id_type, td::get_json_object_string_field(id, "@type", false)); - if (id_type != "pub.ed25519") { - return td::Status::Error("Invalid config (5)"); - } - TRY_RESULT(key_base64, td::get_json_object_string_field(id, "key", false)); - TRY_RESULT(key, td::base64_decode(key_base64)); - if (key.size() != 32) { - return td::Status::Error("Invalid config (6)"); - } - - client.adnl_id = ton::adnl::AdnlNodeIdFull(ton::pubkeys::Ed25519(td::Bits256(td::Slice(key).ubegin()))); - res.lite_clients.push_back(std::move(client)); + if (!conf.validator_) { + return td::Status::Error("Invalid config: no 'validator' section"); + } + if (!conf.validator_->zero_state_) { + return td::Status::Error("Invalid config: no zerostate"); + } + res.zero_state_id = ton::create_block_id(conf.validator_->zero_state_); + if (conf.validator_->init_block_) { + res.init_block_id = ton::create_block_id(conf.validator_->init_block_); } - TRY_RESULT(validator_obj, - td::get_json_object_field(json.get_object(), "validator", td::JsonValue::Type::Object, false)); - auto &validator = validator_obj.get_object(); - TRY_RESULT(validator_type, td::get_json_object_string_field(validator, "@type", false)); - if (validator_type != "validator.config.global") { - return td::Status::Error("Invalid config (7)"); - } - TRY_RESULT(zero_state_obj, td::get_json_object_field(validator, "zero_state", td::JsonValue::Type::Object, false)); - TRY_RESULT(zero_state_id, parse_block_id_ext(zero_state_obj.get_object())); - res.zero_state_id = zero_state_id; - auto r_init_block_obj = td::get_json_object_field(validator, "init_block", td::JsonValue::Type::Object, false); - if (r_init_block_obj.is_ok()) { - TRY_RESULT(init_block_id, parse_block_id_ext(r_init_block_obj.move_as_ok().get_object())); - res.init_block_id = init_block_id; - } - - auto r_hardforks = td::get_json_object_field(validator, "hardforks", td::JsonValue::Type::Array, false); - if (r_hardforks.is_ok()) { - auto hardforks_obj = r_hardforks.move_as_ok(); - auto &hardforks = hardforks_obj.get_array(); - for (auto &fork : hardforks) { - if (fork.type() != td::JsonValue::Type::Object) { - return td::Status::Error("Invalid config (8)"); - } - TRY_RESULT(fork_block, parse_block_id_ext(fork.get_object())); - res.hardforks.push_back(std::move(fork_block)); - } + for (auto &fork : conf.validator_->hardforks_) { + res.hardforks.push_back(ton::create_block_id(fork)); } for (auto hardfork : res.hardforks) { diff --git a/tonlib/tonlib/Config.h b/tonlib/tonlib/Config.h index 3902c341..28f23881 100644 --- a/tonlib/tonlib/Config.h +++ b/tonlib/tonlib/Config.h @@ -20,17 +20,14 @@ #include "adnl/adnl-node-id.hpp" #include "td/utils/port/IPAddress.h" #include "ton/ton-types.h" +#include "lite-client/ext-client.h" namespace tonlib { struct Config { - struct LiteClient { - ton::adnl::AdnlNodeIdFull adnl_id; - td::IPAddress address; - }; ton::BlockIdExt zero_state_id; ton::BlockIdExt init_block_id; std::vector hardforks; - std::vector lite_clients; + std::vector lite_servers; std::string name; static td::Result parse(std::string str); }; diff --git a/tonlib/tonlib/ExtClient.cpp b/tonlib/tonlib/ExtClient.cpp index 30a29b59..b66ca25c 100644 --- a/tonlib/tonlib/ExtClient.cpp +++ b/tonlib/tonlib/ExtClient.cpp @@ -65,7 +65,7 @@ void ExtClient::send_raw_query(td::BufferSlice query, td::Promise adnl_ext_client_; + td::actor::ActorId adnl_ext_client_; td::actor::ActorId last_block_actor_; td::actor::ActorId last_config_actor_; }; @@ -97,7 +97,7 @@ class ExtClient { void force_change_liteserver() { if (!client_.adnl_ext_client_.empty()) { - td::actor::send_closure(client_.adnl_ext_client_, &ExtClientLazy::force_change_liteserver); + td::actor::send_closure(client_.adnl_ext_client_, &liteclient::ExtClient::reset_servers); } } diff --git a/tonlib/tonlib/ExtClientLazy.cpp b/tonlib/tonlib/ExtClientLazy.cpp deleted file mode 100644 index 335a0ff9..00000000 --- a/tonlib/tonlib/ExtClientLazy.cpp +++ /dev/null @@ -1,152 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#include "ExtClientLazy.h" -#include "TonlibError.h" -#include "td/utils/Random.h" -namespace tonlib { - -class ExtClientLazyImp : public ExtClientLazy { - public: - ExtClientLazyImp(std::vector> servers, - td::unique_ptr callback) - : servers_(std::move(servers)), callback_(std::move(callback)) { - CHECK(!servers_.empty()); - } - - void start_up() override { - td::Random::Fast rnd; - td::random_shuffle(td::as_mutable_span(servers_), rnd); - } - - void check_ready(td::Promise promise) override { - before_query(); - if (client_.empty()) { - return promise.set_error(TonlibError::Cancelled()); - } - send_closure(client_, &ton::adnl::AdnlExtClient::check_ready, std::move(promise)); - } - - void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, - td::Promise promise) override { - before_query(); - if (client_.empty()) { - return promise.set_error(TonlibError::Cancelled()); - } - td::Promise P = [SelfId = actor_id(this), idx = cur_server_idx_, - promise = std::move(promise)](td::Result R) mutable { - if (R.is_error() && - (R.error().code() == ton::ErrorCode::timeout || R.error().code() == ton::ErrorCode::cancelled)) { - td::actor::send_closure(SelfId, &ExtClientLazyImp::set_server_bad, idx, true); - } - promise.set_result(std::move(R)); - }; - send_closure(client_, &ton::adnl::AdnlExtClient::send_query, std::move(name), std::move(data), timeout, - std::move(P)); - } - - void force_change_liteserver() override { - if (servers_.size() == 1) { - return; - } - cur_server_bad_ = cur_server_bad_force_ = true; - } - - private: - void before_query() { - if (is_closing_) { - return; - } - alarm_timestamp() = td::Timestamp::in(MAX_NO_QUERIES_TIMEOUT); - if (cur_server_bad_) { - ++cur_server_idx_; - } else if (!client_.empty()) { - return; - } - class Callback : public ton::adnl::AdnlExtClient::Callback { - public: - explicit Callback(td::actor::ActorShared parent, size_t idx) - : parent_(std::move(parent)), idx_(idx) { - } - void on_ready() override { - td::actor::send_closure(parent_, &ExtClientLazyImp::set_server_bad, idx_, false); - } - void on_stop_ready() override { - td::actor::send_closure(parent_, &ExtClientLazyImp::set_server_bad, idx_, true); - } - - private: - td::actor::ActorShared parent_; - size_t idx_; - }; - ref_cnt_++; - cur_server_bad_ = false; - cur_server_bad_force_ = false; - const auto& s = servers_[cur_server_idx_ % servers_.size()]; - LOG(INFO) << "Connecting to liteserver " << s.second; - client_ = ton::adnl::AdnlExtClient::create( - s.first, s.second, std::make_unique(td::actor::actor_shared(this), cur_server_idx_)); - } - - std::vector> servers_; - size_t cur_server_idx_ = 0; - bool cur_server_bad_ = false; - bool cur_server_bad_force_ = false; - - td::actor::ActorOwn client_; - td::unique_ptr callback_; - static constexpr double MAX_NO_QUERIES_TIMEOUT = 100; - - bool is_closing_{false}; - td::uint32 ref_cnt_{1}; - - void set_server_bad(size_t idx, bool bad) { - if (idx == cur_server_idx_ && servers_.size() > 1 && !cur_server_bad_force_) { - cur_server_bad_ = bad; - } - } - void alarm() override { - client_.reset(); - } - void hangup_shared() override { - ref_cnt_--; - try_stop(); - } - void hangup() override { - is_closing_ = true; - ref_cnt_--; - client_.reset(); - try_stop(); - } - void try_stop() { - if (is_closing_ && ref_cnt_ == 0) { - stop(); - } - } -}; - -td::actor::ActorOwn ExtClientLazy::create(ton::adnl::AdnlNodeIdFull dst, td::IPAddress dst_addr, - td::unique_ptr callback) { - return create({std::make_pair(dst, dst_addr)}, std::move(callback)); -} - -td::actor::ActorOwn ExtClientLazy::create( - std::vector> servers, td::unique_ptr callback) { - return td::actor::create_actor("ExtClientLazy", std::move(servers), std::move(callback)); -} -} // namespace tonlib diff --git a/tonlib/tonlib/ExtClientOutbound.cpp b/tonlib/tonlib/ExtClientOutbound.cpp index 025ba848..e5fac8b4 100644 --- a/tonlib/tonlib/ExtClientOutbound.cpp +++ b/tonlib/tonlib/ExtClientOutbound.cpp @@ -20,15 +20,12 @@ #include "ExtClientOutbound.h" #include "TonlibError.h" #include + namespace tonlib { -class ExtClientOutboundImp : public ExtClientOutbound { +class ExtClientOutboundImpl : public ExtClientOutbound { public: - ExtClientOutboundImp(td::unique_ptr callback) : callback_(std::move(callback)) { - } - - void check_ready(td::Promise promise) override { - promise.set_error(td::Status::Error("Not supported")); + ExtClientOutboundImpl(td::unique_ptr callback) : callback_(std::move(callback)) { } void send_query(std::string name, td::BufferSlice data, td::Timestamp timeout, @@ -38,9 +35,6 @@ class ExtClientOutboundImp : public ExtClientOutbound { callback_->request(query_id, data.as_slice().str()); } - void force_change_liteserver() override { - } - void on_query_result(td::int64 id, td::Result r_data, td::Promise promise) override { auto it = queries_.find(id); if (it == queries_.end()) { @@ -66,6 +60,6 @@ class ExtClientOutboundImp : public ExtClientOutbound { }; td::actor::ActorOwn ExtClientOutbound::create(td::unique_ptr callback) { - return td::actor::create_actor("ExtClientOutbound", std::move(callback)); + return td::actor::create_actor("ExtClientOutbound", std::move(callback)); } } // namespace tonlib diff --git a/tonlib/tonlib/ExtClientOutbound.h b/tonlib/tonlib/ExtClientOutbound.h index 87b73b9e..bf52c8c2 100644 --- a/tonlib/tonlib/ExtClientOutbound.h +++ b/tonlib/tonlib/ExtClientOutbound.h @@ -18,11 +18,10 @@ */ #pragma once #include "td/actor/actor.h" - -#include "ExtClientLazy.h" +#include "lite-client/ext-client.h" namespace tonlib { -class ExtClientOutbound : public ExtClientLazy { +class ExtClientOutbound : public liteclient::ExtClient { public: class Callback { public: diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 7b223839..d73e715c 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -18,7 +18,6 @@ */ #include "TonlibClient.h" -#include "tonlib/ExtClientLazy.h" #include "tonlib/ExtClientOutbound.h" #include "tonlib/LastBlock.h" #include "tonlib/LastConfig.h" @@ -1051,15 +1050,17 @@ class Query { } vm::GasLimits gas_limits = compute_gas_limits(td::make_refint(raw_.source->get_balance()), gas_limits_prices); - auto res = smc.write().send_external_message(raw_.message_body, ton::SmartContract::Args() - .set_limits(gas_limits) - .set_balance(raw_.source->get_balance()) - .set_now(raw_.source->get_sync_time()) - .set_ignore_chksig(ignore_chksig) - .set_address(raw_.source->get_address()) - .set_config(cfg) - .set_prev_blocks_info(state.prev_blocks_info) - .set_libraries(libraries)); + auto res = smc.write().send_external_message(raw_.message_body, + ton::SmartContract::Args() + .set_limits(gas_limits) + .set_balance(raw_.source->get_balance()) + .set_extra_currencies(raw_.source->get_extra_currencies()) + .set_now(raw_.source->get_sync_time()) + .set_ignore_chksig(ignore_chksig) + .set_address(raw_.source->get_address()) + .set_config(cfg) + .set_prev_blocks_info(state.prev_blocks_info) + .set_libraries(libraries)); td::int64 fwd_fee = 0; if (res.success) { LOG(DEBUG) << "output actions:\n" @@ -2184,21 +2185,8 @@ void TonlibClient::init_ext_client() { ext_client_outbound_ = client.get(); raw_client_ = std::move(client); } else { - std::vector> servers; - for (const auto& s : config_.lite_clients) { - servers.emplace_back(s.adnl_id, s.address); - } - class Callback : public ExtClientLazy::Callback { - public: - explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { - } - - private: - td::actor::ActorShared<> parent_; - }; ext_client_outbound_ = {}; - ref_cnt_++; - raw_client_ = ExtClientLazy::create(std::move(servers), td::make_unique(td::actor::actor_shared())); + raw_client_ = liteclient::ExtClient::create(config_.lite_servers, nullptr); } } @@ -2860,7 +2848,7 @@ td::Result TonlibClient::validate_config(tonlib_api::o TRY_RESULT_PREFIX(new_config, Config::parse(std::move(config->config_)), TonlibError::InvalidConfig("can't parse config")); - if (new_config.lite_clients.empty() && !config->use_callbacks_for_network_) { + if (new_config.lite_servers.empty() && !config->use_callbacks_for_network_) { return TonlibError::InvalidConfig("no lite clients"); } td::optional o_master_config; @@ -3046,6 +3034,7 @@ struct ToRawTransactions { } auto body_cell = vm::CellBuilder().append_cellslice(*body).finalize(); auto body_hash = body_cell->get_hash().as_slice().str(); + auto msg_hash = cell->get_hash().as_slice().str(); td::Ref init_state_cell; auto& init_state_cs = message.init.write(); @@ -3115,6 +3104,7 @@ struct ToRawTransactions { auto created_lt = static_cast(msg_info.created_lt); return tonlib_api::make_object( + msg_hash, tonlib_api::make_object(src), tonlib_api::make_object(std::move(dest)), balance, std::move(extra_currencies), fwd_fee, ihr_fee, created_lt, std::move(body_hash), @@ -3127,6 +3117,7 @@ struct ToRawTransactions { } TRY_RESULT(dest, to_std_address(msg_info.dest)); return tonlib_api::make_object( + msg_hash, tonlib_api::make_object(), tonlib_api::make_object(std::move(dest)), 0, std::vector>{}, 0, 0, 0, std::move(body_hash), @@ -3140,6 +3131,7 @@ struct ToRawTransactions { TRY_RESULT(src, to_std_address(msg_info.src)); auto created_lt = static_cast(msg_info.created_lt); return tonlib_api::make_object( + msg_hash, tonlib_api::make_object(src), tonlib_api::make_object(), 0, std::vector>{}, 0, 0, created_lt, std::move(body_hash), @@ -4609,7 +4601,7 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_getLibraries& request, if (query_context_.block_id) { get_libraries(query_context_.block_id.value(), request.library_list_, std::move(promise)); } else { - client_.with_last_block([this, promise = std::move(promise), library_list = request.library_list_](td::Result r_last_block) mutable { + client_.with_last_block([this, promise = std::move(promise), library_list = request.library_list_](td::Result r_last_block) mutable { if (r_last_block.is_error()) { promise.set_error(r_last_block.move_as_error_prefix(TonlibError::Internal("get last block failed "))); } else { @@ -4627,6 +4619,8 @@ void TonlibClient::get_libraries(ton::BlockIdExt blkid, std::vector std::vector> result_entries; result_entries.reserve(library_list.size()); std::vector not_cached_hashes; + not_cached_hashes.reserve(library_list.size()); + for (auto& library_hash : library_list) { if (libraries.key_exists(library_hash)) { auto library_content = vm::std_boc_serialize(libraries.lookup_ref(library_hash)).move_as_ok().as_slice().str(); @@ -4641,9 +4635,10 @@ void TonlibClient::get_libraries(ton::BlockIdExt blkid, std::vector return; } - client_.send_query(ton::lite_api::liteServer_getLibrariesWithProof(ton::create_tl_lite_block_id(blkid), 1, std::move(not_cached_hashes)), + auto missed_lib_ids = not_cached_hashes; + client_.send_query(ton::lite_api::liteServer_getLibrariesWithProof(ton::create_tl_lite_block_id(blkid), 1, std::move(missed_lib_ids)), promise.wrap([self=this, blkid, result_entries = std::move(result_entries), not_cached_hashes] - (td::Result> r_libraries) mutable + (td::Result> r_libraries) mutable -> td::Result> { if (r_libraries.is_error()) { LOG(WARNING) << "cannot obtain found libraries: " << r_libraries.move_as_error().to_string(); @@ -4670,7 +4665,7 @@ void TonlibClient::get_libraries(ton::BlockIdExt blkid, std::vector auto csr = libraries_dict.lookup(hash.bits(), 256); if (csr.is_null()) { LOG(WARNING) << "library " << hash.to_hex() << " not found in config"; - if (std::any_of(libraries->result_.begin(), libraries->result_.end(), + if (std::any_of(libraries->result_.begin(), libraries->result_.end(), [&hash](const auto& lib) { return lib->hash_.bits().equals(hash.cbits(), 256); })) { return TonlibError::Internal("library is included in response but it's not found in proof"); } @@ -4681,7 +4676,7 @@ void TonlibClient::get_libraries(ton::BlockIdExt blkid, std::vector return TonlibError::Internal("cannot unpack LibDescr record"); } - auto lib_it = std::find_if(libraries->result_.begin(), libraries->result_.end(), + auto lib_it = std::find_if(libraries->result_.begin(), libraries->result_.end(), [&hash](const auto& lib) { return lib->hash_.bits().equals(hash.cbits(), 256); }); if (lib_it == libraries->result_.end()) { return TonlibError::Internal("library is found in proof but not in response"); @@ -4699,7 +4694,7 @@ void TonlibClient::get_libraries(ton::BlockIdExt blkid, std::vector if (contents.ok()->get_hash() != libdescr.lib->get_hash()) { return TonlibError::Internal(PSLICE() << "library hash mismatch data " << lib->hash_.to_hex() << " != proof " << libdescr.lib->get_hash().to_hex()); } - + result_entries.push_back(tonlib_api::make_object(lib->hash_, lib->data_.as_slice().str())); self->libraries.set_ref(lib->hash_, contents.move_as_ok()); LOG(DEBUG) << "registered library " << lib->hash_.to_hex(); @@ -4800,6 +4795,7 @@ td::Status TonlibClient::do_request(const tonlib_api::smc_runGetMethod& request, } args.set_stack(std::move(stack)); args.set_balance(it->second->get_balance()); + args.set_extra_currencies(it->second->get_extra_currencies()); args.set_now(it->second->get_sync_time()); args.set_address(it->second->get_address()); @@ -5663,14 +5659,14 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_lookupBlock& reques auto blkid = ton::BlockId(request.id_->workchain_, request.id_->shard_, request.id_->seqno_); client_.with_last_block( [self = this, blkid, lite_block = std::move(lite_block), mode = request.mode_, lt = (td::uint64)request.lt_, - utime = (td::uint32)request.utime_, promise = std::move(promise)](td::Result r_last_block) mutable { + utime = (td::uint32)request.utime_, promise = std::move(promise)](td::Result r_last_block) mutable { if (r_last_block.is_error()) { promise.set_error(r_last_block.move_as_error_prefix(TonlibError::Internal("get last block failed "))); return; } self->client_.send_query(ton::lite_api::liteServer_lookupBlockWithProof(mode, std::move(lite_block), ton::create_tl_lite_block_id(r_last_block.ok().last_block_id), lt, utime), - promise.wrap([blkid, mode, utime, lt, last_block = r_last_block.ok().last_block_id](lite_api_ptr&& result) + promise.wrap([blkid, mode, utime, lt, last_block = r_last_block.ok().last_block_id](lite_api_ptr&& result) -> td::Result> { TRY_STATUS(check_lookup_block_proof(result, mode, blkid, last_block, lt, utime)); return to_tonlib_api(*result->id_); @@ -5683,6 +5679,26 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_lookupBlock& reques td::Status check_lookup_block_proof(lite_api_ptr& result, int mode, ton::BlockId blkid, ton::BlockIdExt client_mc_blkid, td::uint64 lt, td::uint32 utime) { try { ton::BlockIdExt cur_id = ton::create_block_id(result->mc_block_id_); + if (!cur_id.is_masterchain_ext()) { + return td::Status::Error("invalid response: mc block id is not from masterchain"); + } + if (client_mc_blkid != cur_id) { + auto state = block::check_extract_state_proof(client_mc_blkid, result->client_mc_state_proof_.as_slice(), + result->mc_block_proof_.as_slice()); + if (state.is_error()) { + LOG(WARNING) << "cannot check state proof: " << state.move_as_error().to_string(); + return state.move_as_error(); + } + auto state_root = state.move_as_ok(); + auto prev_blocks_dict = block::get_prev_blocks_dict(state_root); + if (!prev_blocks_dict) { + return td::Status::Error("cannot extract prev blocks dict from state"); + } + + if (!block::check_old_mc_block_id(*prev_blocks_dict, cur_id)) { + return td::Status::Error("couldn't check old mc block id"); + } + } try { for (auto& link : result->shard_links_) { ton::BlockIdExt prev_id = create_block_id(link->id_); @@ -5696,23 +5712,6 @@ td::Status check_lookup_block_proof(lite_api_ptrclient_mc_state_proof_.as_slice(), - result->mc_block_proof_.as_slice()); - if (state.is_error()) { - LOG(WARNING) << "cannot check state proof: " << state.move_as_error().to_string(); - return state.move_as_error(); - } - auto state_root = state.move_as_ok(); - auto prev_blocks_dict = block::get_prev_blocks_dict(state_root); - if (!prev_blocks_dict) { - return td::Status::Error("cannot extract prev blocks dict from state"); - } - - if (!block::check_old_mc_block_id(*prev_blocks_dict, cur_id)) { - return td::Status::Error("couldn't check old mc block id"); - } - } block::gen::Block::Record blk; block::gen::BlockExtra::Record extra; block::gen::McBlockExtra::Record mc_extra; @@ -5812,7 +5811,7 @@ td::Status check_lookup_block_proof(lite_api_ptr lt) { return td::Status::Error("prev header end_lt > lt"); @@ -5827,7 +5826,7 @@ td::Status check_lookup_block_proof(lite_api_ptr after; @@ -5941,7 +5940,7 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getTransactions& re if (!request.after_) { return td::Status::Error("Missing field `after`"); } - TRY_RESULT_ASSIGN(start_addr, to_bits256(request.after_->account_, "account")); + TRY_RESULT_ASSIGN(start_addr, to_bits256(request.after_->account_, "account")); start_lt = request.after_->lt_; after = ton::lite_api::make_object(start_addr, start_lt); } else { diff --git a/tonlib/tonlib/TonlibClient.h b/tonlib/tonlib/TonlibClient.h index 7db44324..28239a8a 100644 --- a/tonlib/tonlib/TonlibClient.h +++ b/tonlib/tonlib/TonlibClient.h @@ -33,6 +33,7 @@ #include "td/utils/optional.h" #include "smc-envelope/ManualDns.h" +#include "lite-client/ext-client.h" #include @@ -113,7 +114,7 @@ class TonlibClient : public td::actor::Actor { vm::Dictionary libraries{256}; // network - td::actor::ActorOwn raw_client_; + td::actor::ActorOwn raw_client_; td::actor::ActorId ext_client_outbound_; td::actor::ActorOwn raw_last_block_; td::actor::ActorOwn raw_last_config_; diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index 4567478e..2c7100f2 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -47,8 +47,6 @@ #include "tonlib/TonlibClient.h" #include "tonlib/TonlibCallback.h" -#include "tonlib/ExtClientLazy.h" - #include "smc-envelope/ManualDns.h" #include "smc-envelope/PaymentChannel.h" @@ -57,6 +55,7 @@ #include "crypto/util/Miner.h" #include "vm/boc.h" #include "vm/cells/CellBuilder.h" +#include "lite-client/ext-client.h" #include #include @@ -174,7 +173,7 @@ class TonlibCli : public td::actor::Actor { std::map>> query_handlers_; - td::actor::ActorOwn raw_client_; + td::actor::ActorOwn raw_client_; bool is_closing_{false}; td::uint32 ref_cnt_{1}; @@ -223,11 +222,7 @@ class TonlibCli : public td::actor::Actor { if (options_.use_callbacks_for_network) { auto config = tonlib::Config::parse(options_.config).move_as_ok(); - auto lite_clients_size = config.lite_clients.size(); - CHECK(lite_clients_size != 0); - auto lite_client_id = td::Random::fast(0, td::narrow_cast(lite_clients_size) - 1); - auto& lite_client = config.lite_clients[lite_client_id]; - class Callback : public tonlib::ExtClientLazy::Callback { + class Callback : public liteclient::ExtClient::Callback { public: explicit Callback(td::actor::ActorShared<> parent) : parent_(std::move(parent)) { } @@ -236,14 +231,14 @@ class TonlibCli : public td::actor::Actor { td::actor::ActorShared<> parent_; }; ref_cnt_++; - raw_client_ = tonlib::ExtClientLazy::create(lite_client.adnl_id, lite_client.address, + raw_client_ = liteclient::ExtClient::create(config.lite_servers, td::make_unique(td::actor::actor_shared())); } auto config = !options_.config.empty() - ? make_object(options_.config, options_.name, - options_.use_callbacks_for_network, options_.ignore_cache) - : nullptr; + ? make_object(options_.config, options_.name, + options_.use_callbacks_for_network, options_.ignore_cache) + : nullptr; tonlib_api::object_ptr ks_type; if (options_.in_memory) { @@ -386,7 +381,7 @@ class TonlibCli : public td::actor::Actor { td::TerminalIO::out() << "sendfile \tLoad a serialized message from and send it to server\n"; td::TerminalIO::out() << "setconfig|validateconfig [] [] [] - set or validate " "lite server config\n"; - td::TerminalIO::out() << "runmethod ...\tRuns GET method of account " + td::TerminalIO::out() << "runmethod ...\tRuns GET method of account " " with specified parameters\n"; td::TerminalIO::out() << "getstate \tget state of wallet with requested key\n"; td::TerminalIO::out() << "getstatebytransaction \tget state of wallet with requested key after transaction with local time and hash (base64url)\n"; @@ -1545,7 +1540,7 @@ class TonlibCli : public td::actor::Actor { auto update = tonlib_api::move_object_as(std::move(result)); CHECK(!raw_client_.empty()); snd_bytes_ += update->data_.size(); - send_closure(raw_client_, &ton::adnl::AdnlExtClient::send_query, "query", td::BufferSlice(update->data_), + send_closure(raw_client_, &liteclient::ExtClient::send_query, "query", td::BufferSlice(update->data_), td::Timestamp::in(5), [actor_id = actor_id(this), id = update->id_](td::Result res) { send_closure(actor_id, &TonlibCli::on_adnl_result, id, std::move(res)); diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index ec6eb013..e9ffd2a2 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -21,4 +21,7 @@ add_executable(opcode-timing opcode-timing.cpp ) target_link_libraries(opcode-timing ton_crypto) target_include_directories(pack-viewer PUBLIC $/..) -install(TARGETS generate-random-id RUNTIME DESTINATION bin) +add_executable(proxy-liteserver proxy-liteserver.cpp) +target_link_libraries(proxy-liteserver tdutils tdactor adnl dht tl_api ton_crypto git lite-client-common) + +install(TARGETS generate-random-id proxy-liteserver RUNTIME DESTINATION bin) diff --git a/utils/generate-random-id.cpp b/utils/generate-random-id.cpp index f606f358..a487ac17 100644 --- a/utils/generate-random-id.cpp +++ b/utils/generate-random-id.cpp @@ -84,6 +84,19 @@ int main(int argc, char *argv[]) { TRY_RESULT_PREFIX_ASSIGN(addr_list, ton::adnl::AdnlAddressList::create(addr_list_tl), "bad addr list: "); return td::Status::OK(); }); + p.add_checked_option('f', "path to file with addr-list", "addr list to sign", [&](td::Slice key) { + if (addr_list) { + return td::Status::Error("duplicate '-f' option"); + } + + td::BufferSlice bs(key); + TRY_RESULT_PREFIX(data, td::read_file(key.str()), "failed to read addr-list: "); + TRY_RESULT_PREFIX(as_json_value, td::json_decode(data.as_slice()), "bad addr list JSON: "); + ton::tl_object_ptr addr_list_tl; + TRY_STATUS_PREFIX(td::from_json(addr_list_tl, std::move(as_json_value)), "bad addr list TL: "); + TRY_RESULT_PREFIX_ASSIGN(addr_list, ton::adnl::AdnlAddressList::create(addr_list_tl), "bad addr list: "); + return td::Status::OK(); + }); p.add_checked_option('i', "network-id", "dht network id (default: -1)", [&](td::Slice key) { if (network_id_opt) { return td::Status::Error("duplicate '-i' option"); diff --git a/utils/opcode-timing.cpp b/utils/opcode-timing.cpp index 876ba109..47171eec 100644 --- a/utils/opcode-timing.cpp +++ b/utils/opcode-timing.cpp @@ -135,8 +135,8 @@ runInfo time_run_vm(td::Slice command, td::Ref stack) { CHECK(stack.is_unique()); try { vm::GasLimits gas_limit; - vm::VmState vm{vm::load_cell_slice_ref(cell), std::move(stack), gas_limit, 0, {}, vm::VmLog{}, {}, c7}; - vm.set_global_version(ton::SUPPORTED_VERSION); + vm::VmState vm{ + vm::load_cell_slice_ref(cell), ton::SUPPORTED_VERSION, std::move(stack), gas_limit, 0, {}, vm::VmLog{}, {}, c7}; std::clock_t cStart = std::clock(); int ret = ~vm.run(); std::clock_t cEnd = std::clock(); diff --git a/utils/proxy-liteserver.cpp b/utils/proxy-liteserver.cpp new file mode 100644 index 00000000..a9baa759 --- /dev/null +++ b/utils/proxy-liteserver.cpp @@ -0,0 +1,473 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#include "td/utils/filesystem.h" +#include "td/actor/actor.h" +#include "td/actor/MultiPromise.h" +#include "td/utils/OptionParser.h" +#include "td/utils/port/path.h" +#include "td/utils/port/signals.h" +#include "td/utils/port/IPAddress.h" +#include "td/utils/Random.h" +#include "td/utils/FileLog.h" +#include "git.h" +#include "auto/tl/ton_api.h" +#include "auto/tl/lite_api.h" +#include "tl-utils/lite-utils.hpp" +#include "auto/tl/ton_api_json.h" +#include "adnl/adnl.h" +#include "lite-client/ext-client.h" + +#if TD_DARWIN || TD_LINUX +#include +#endif +#include "td/utils/overloaded.h" + +#include +#include +#include +#include "td/utils/tl_storers.h" + +using namespace ton; + +class ProxyLiteserver : public td::actor::Actor { + public: + ProxyLiteserver(std::string global_config, std::string db_root, td::uint16 port, PublicKeyHash public_key_hash) + : global_config_(std::move(global_config)) + , db_root_(std::move(db_root)) + , port_(port) + , public_key_hash_(public_key_hash) { + } + + void start_up() override { + LOG_CHECK(!db_root_.empty()) << "db root is not set"; + td::mkdir(db_root_).ensure(); + db_root_ = td::realpath(db_root_).move_as_ok(); + keyring_ = keyring::Keyring::create(db_root_ + "/keyring"); + + if (public_key_hash_.is_zero()) { + id_ = {}; + run(); + } else { + td::actor::send_closure(keyring_, &keyring::Keyring::get_public_key, public_key_hash_, + [SelfId = actor_id(this)](td::Result R) mutable { + if (R.is_error()) { + LOG(FATAL) << "Failed to load public key: " << R.move_as_error(); + } + td::actor::send_closure(SelfId, &ProxyLiteserver::got_public_key, R.move_as_ok()); + }); + } + } + + void got_public_key(PublicKey pub) { + id_ = adnl::AdnlNodeIdFull{pub}; + run(); + } + + void run() { + td::Status S = prepare_local_config(); + if (S.is_error()) { + LOG(FATAL) << "Local config error: " << S; + } + + S = parse_global_config(); + if (S.is_error()) { + LOG(FATAL) << S; + } + + run_clients(); + create_ext_server(); + } + + td::Status prepare_local_config() { + auto r_conf_data = td::read_file(config_file()); + if (r_conf_data.is_ok()) { + auto conf_data = r_conf_data.move_as_ok(); + TRY_RESULT_PREFIX(conf_json, td::json_decode(conf_data.as_slice()), "failed to parse json: "); + TRY_STATUS_PREFIX(ton_api::from_json(*config_, conf_json.get_object()), "json does not fit TL scheme: "); + TRY_RESULT_PREFIX(cfg_port, td::narrow_cast_safe(config_->port_), "invalid port: "); + TRY_RESULT_PREFIX(cfg_id, adnl::AdnlNodeIdFull::create(config_->id_), "invalid id: "); + bool rewrite_config = false; + if (port_ == 0) { + port_ = cfg_port; + } else { + rewrite_config |= (port_ != cfg_port); + } + if (id_.empty()) { + id_ = std::move(cfg_id); + } else { + rewrite_config |= (id_ != cfg_id); + } + if (!rewrite_config) { + return td::Status::OK(); + } + } else { + LOG(WARNING) << "First launch, creating local config"; + } + if (port_ == 0) { + return td::Status::Error("port is not set"); + } + config_->port_ = port_; + if (id_.empty()) { + auto pk = PrivateKey{privkeys::Ed25519::random()}; + id_ = adnl::AdnlNodeIdFull{pk.compute_public_key()}; + td::actor::send_closure(keyring_, &keyring::Keyring::add_key, std::move(pk), false, [](td::Result R) { + if (R.is_error()) { + LOG(FATAL) << "Failed to store private key"; + } + }); + } + config_->id_ = id_.tl(); + + auto s = td::json_encode(td::ToJson(*config_), true); + TRY_STATUS_PREFIX(td::write_file(config_file(), s), "failed to write file: "); + LOG(WARNING) << "Writing config.json"; + return td::Status::OK(); + } + + td::Status parse_global_config() { + TRY_RESULT_PREFIX(global_config_data, td::read_file(global_config_), "Failed to read global config: "); + TRY_RESULT_PREFIX(global_config_json, td::json_decode(global_config_data.as_slice()), + "Failed to parse global config: "); + ton_api::liteclient_config_global gc; + TRY_STATUS_PREFIX(ton_api::from_json(gc, global_config_json.get_object()), "Failed to parse global config: "); + TRY_RESULT_PREFIX(servers, liteclient::LiteServerConfig::parse_global_config(gc), + "Falied to parse liteservers in global config: "); + if (servers.empty()) { + return td::Status::Error("No liteservers in global config"); + } + for (auto& s : servers) { + servers_.emplace_back(); + servers_.back().config = std::move(s); + } + return td::Status::OK(); + } + + void run_clients() { + class Callback : public adnl::AdnlExtClient::Callback { + public: + explicit Callback(td::actor::ActorId id, size_t idx) : id_(std::move(id)), idx_(idx) { + } + void on_ready() override { + td::actor::send_closure(id_, &ProxyLiteserver::on_client_status, idx_, true); + } + void on_stop_ready() override { + td::actor::send_closure(id_, &ProxyLiteserver::on_client_status, idx_, false); + } + + private: + td::actor::ActorId id_; + size_t idx_; + }; + + for (size_t i = 0; i < servers_.size(); ++i) { + Server& server = servers_[i]; + server.client = adnl::AdnlExtClient::create(server.config.adnl_id, server.config.addr, + std::make_unique(actor_id(this), i)); + server.alive = false; + } + } + + void on_client_status(size_t idx, bool ready) { + Server& server = servers_[idx]; + if (server.alive == ready) { + return; + } + server.alive = ready; + LOG(WARNING) << (ready ? "Connected to" : "Disconnected from") << " server #" << idx << " (" + << server.config.addr.get_ip_str() << ":" << server.config.addr.get_port() << ")"; + } + + void create_ext_server() { + adnl_ = adnl::Adnl::create("", keyring_.get()); + td::actor::send_closure(adnl_, &adnl::Adnl::add_id, id_, adnl::AdnlAddressList{}, (td::uint8)255); + + class AdnlCallback : public adnl::Adnl::Callback { + public: + explicit AdnlCallback(td::actor::ActorId id) : id_(id) { + } + + void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) override { + } + void receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, + td::Promise promise) override { + td::actor::send_closure(id_, &ProxyLiteserver::receive_query, std::move(data), std::move(promise)); + } + + private: + td::actor::ActorId id_; + }; + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, id_.compute_short_id(), + adnl::Adnl::int_to_bytestring(lite_api::liteServer_query::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_, &adnl::Adnl::create_ext_server, std::vector{id_.compute_short_id()}, + std::vector{port_}, + [SelfId = actor_id(this)](td::Result> R) { + R.ensure(); + td::actor::send_closure(SelfId, &ProxyLiteserver::created_ext_server, R.move_as_ok()); + }); + } + + void created_ext_server(td::actor::ActorOwn s) { + ext_server_ = std::move(s); + LOG(WARNING) << "Started proxy liteserver on port " << port_; + alarm(); + } + + td::Result select_server(const liteclient::QueryInfo& query_info) { + size_t best_idx = servers_.size(); + int cnt = 0; + for (size_t i = 0; i < servers_.size(); ++i) { + Server& server = servers_[i]; + if (!server.alive || !server.config.accepts_query(query_info)) { + continue; + } + ++cnt; + if (td::Random::fast(1, cnt) == 1) { + best_idx = i; + } + } + if (best_idx == servers_.size()) { + return td::Status::Error(PSTRING() << "no liteserver for query " << query_info.to_str()); + } + return best_idx; + } + + void receive_query(td::BufferSlice data, td::Promise promise) { + // Like in ValidatorManagerImpl::run_ext_query + auto F = fetch_tl_object(data, true); + if (F.is_ok()) { + data = std::move(F.move_as_ok()->data_); + } else { + auto G = fetch_tl_prefix(data, true); + if (G.is_error()) { + promise.set_error(G.move_as_error()); + return; + } + } + + tl_object_ptr wait_mc_seqno_obj; + auto E = fetch_tl_prefix(data, true); + if (E.is_ok()) { + wait_mc_seqno_obj = E.move_as_ok(); + } + liteclient::QueryInfo query_info = liteclient::get_query_info(data); + ++ls_stats_[query_info.query_id]; + promise = [promise = std::move(promise), query_info, timer = td::Timer(), + wait_mc_seqno = + (wait_mc_seqno_obj ? wait_mc_seqno_obj->seqno_ : 0)](td::Result R) mutable { + if (R.is_ok()) { + LOG(INFO) << "Query " << query_info.to_str() + << (wait_mc_seqno ? PSTRING() << " (wait seqno " << wait_mc_seqno << ")" : "") + << ": OK, time=" << timer.elapsed() << ", response_size=" << R.ok().size(); + promise.set_value(R.move_as_ok()); + return; + } + LOG(INFO) << "Query " << query_info.to_str() + << (wait_mc_seqno ? PSTRING() << " (wait seqno " << wait_mc_seqno << ")" : "") << ": " << R.error(); + promise.set_value(create_serialize_tl_object( + R.error().code(), "Gateway error: " + R.error().message().str())); + }; + + TRY_RESULT_PROMISE(promise, server_idx, select_server(query_info)); + Server& server = servers_[server_idx]; + LOG(INFO) << "Sending query " << query_info.to_str() + << (wait_mc_seqno_obj ? PSTRING() << " (wait seqno " << wait_mc_seqno_obj->seqno_ << ")" : "") + << ", size=" << data.size() << ", to server #" << server_idx << " (" << server.config.addr.get_ip_str() + << ":" << server.config.addr.get_port() << ")"; + + BlockSeqno wait_mc_seqno = wait_mc_seqno_obj ? wait_mc_seqno_obj->seqno_ : 0; + wait_mc_seqno = std::max(wait_mc_seqno, last_known_masterchain_seqno_); + if (server.last_known_masterchain_seqno < wait_mc_seqno) { + int timeout_ms = wait_mc_seqno_obj ? wait_mc_seqno_obj->timeout_ms_ : 8000; + data = serialize_tl_object(create_tl_object(wait_mc_seqno, timeout_ms), + true, std::move(data)); + } + data = create_serialize_tl_object(std::move(data)); + td::actor::send_closure(server.client, &adnl::AdnlExtClient::send_query, "q", std::move(data), + td::Timestamp::in(8.0), + [SelfId = actor_id(this), promise = std::move(promise), server_idx, + wait_mc_seqno](td::Result R) mutable { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &ProxyLiteserver::process_query_response, + R.ok().clone(), server_idx, wait_mc_seqno); + } + promise.set_result(std::move(R)); + }); + } + + void process_query_response(td::BufferSlice data, size_t server_idx, BlockSeqno wait_mc_seqno) { + auto F = fetch_tl_object(data, true); + if (F.is_error() || F.ok()->get_id() == lite_api::liteServer_error::ID) { + return; + } + BlockSeqno new_seqno = wait_mc_seqno; + lite_api::downcast_call(*F.ok_ref(), td::overloaded( + [&](lite_api::liteServer_masterchainInfo& f) { + new_seqno = std::max(new_seqno, f.last_->seqno_); + }, + [&](lite_api::liteServer_masterchainInfoExt& f) { + new_seqno = std::max(new_seqno, f.last_->seqno_); + }, + [&](lite_api::liteServer_accountState& f) { + if (f.id_->workchain_ == masterchainId) { + new_seqno = std::max(new_seqno, f.id_->seqno_); + } + }, + [&](auto& obj) {})); + servers_[server_idx].last_known_masterchain_seqno = + std::max(servers_[server_idx].last_known_masterchain_seqno, new_seqno); + if (new_seqno > last_known_masterchain_seqno_) { + last_known_masterchain_seqno_ = new_seqno; + LOG(INFO) << "Last known masterchain seqno = " << new_seqno; + } + } + + void alarm() override { + alarm_timestamp() = td::Timestamp::in(60.0); + if (!ls_stats_.empty()) { + td::StringBuilder sb; + sb << "Liteserver stats (1 minute):"; + td::uint32 total = 0; + for (const auto& p : ls_stats_) { + sb << " " << lite_query_name_by_id(p.first) << ":" << p.second; + total += p.second; + } + sb << " TOTAL:" << total; + LOG(WARNING) << sb.as_cslice(); + ls_stats_.clear(); + } + } + + private: + std::string global_config_; + std::string db_root_; + td::uint16 port_; + PublicKeyHash public_key_hash_; + + tl_object_ptr config_ = create_tl_object(); + adnl::AdnlNodeIdFull id_; + + td::actor::ActorOwn keyring_; + td::actor::ActorOwn adnl_; + td::actor::ActorOwn ext_server_; + + struct Server { + liteclient::LiteServerConfig config; + td::actor::ActorOwn client; + bool alive = false; + BlockSeqno last_known_masterchain_seqno = 0; + }; + std::vector servers_; + + std::map ls_stats_; // lite_api ID -> count, 0 for unknown + + BlockSeqno last_known_masterchain_seqno_ = 0; + tl_object_ptr last_masterchain_info_; + + std::string config_file() const { + return db_root_ + "/config.json"; + } +}; + +int main(int argc, char* argv[]) { + SET_VERBOSITY_LEVEL(verbosity_WARNING); + td::set_default_failure_signal_handler().ensure(); + + td::unique_ptr logger_; + SCOPE_EXIT { + td::log_interface = td::default_log_interface; + }; + + std::string global_config, db_root; + td::uint16 port = 0; + PublicKeyHash public_key_hash = PublicKeyHash::zero(); + td::uint32 threads = 4; + + td::OptionParser p; + p.set_description("Proxy liteserver: distributes incoming queries to servers in global config\n"); + p.add_option('v', "verbosity", "set verbosity level", [&](td::Slice arg) { + int v = VERBOSITY_NAME(FATAL) + (td::to_integer(arg)); + SET_VERBOSITY_LEVEL(v); + }); + p.add_option('V', "version", "show build information", [&]() { + std::cout << "proxy-liteserver build information: [ Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate() << "]\n"; + std::exit(0); + }); + p.add_option('h', "help", "print help", [&]() { + char b[10240]; + td::StringBuilder sb(td::MutableSlice{b, 10000}); + sb << p; + std::cout << sb.as_cslice().c_str(); + std::exit(2); + }); + p.add_checked_option('p', "port", "liteserver port (required only on first launch)", + [&](td::Slice arg) -> td::Status { + TRY_RESULT_ASSIGN(port, td::to_integer_safe(arg)); + return td::Status::OK(); + }); + p.add_checked_option( + 'A', "adnl-id", + "liteserver public key hash in hex (optional). The corresponding private key is required in /keyring/", + [&](td::Slice arg) -> td::Status { + td::Bits256 value; + if (value.from_hex(arg) != 256) { + return td::Status::Error("invalid adnl-id"); + } + public_key_hash = PublicKeyHash{value}; + return td::Status::OK(); + }); + p.add_option('C', "global-config", "global TON configuration file", + [&](td::Slice arg) { global_config = arg.str(); }); + p.add_option('D', "db", "db root", [&](td::Slice arg) { db_root = arg.str(); }); + p.add_option('d', "daemonize", "set SIGHUP", [&]() { + td::set_signal_handler(td::SignalType::HangUp, [](int) { +#if TD_DARWIN || TD_LINUX + close(0); + setsid(); +#endif + }).ensure(); + }); + p.add_option('l', "logname", "log to file", [&](td::Slice fname) { + logger_ = td::FileLog::create(fname.str()).move_as_ok(); + td::log_interface = logger_.get(); + }); + p.add_checked_option('t', "threads", PSTRING() << "number of threads (default=" << 4 << ")", + [&](td::Slice arg) -> td::Status { + TRY_RESULT_ASSIGN(threads, td::to_integer_safe(arg)); + return td::Status::OK(); + }); + + p.run(argc, argv).ensure(); + td::actor::Scheduler scheduler({threads}); + + scheduler.run_in_context([&] { + td::actor::create_actor("proxy-liteserver", global_config, db_root, port, public_key_hash) + .release(); + }); + while (scheduler.run(1)) { + } +} diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index ff4e6e04..d1110019 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1,4 +1,4 @@ -/* +/* This file is part of TON Blockchain source code. TON Blockchain is free software; you can redistribute it and/or @@ -35,6 +35,9 @@ #include "ton/ton-tl.hpp" #include "td/utils/JsonBuilder.h" #include "auto/tl/ton_api_json.h" +#include "keys/encryptor.h" +#include "td/utils/port/path.h" +#include "tl/tl_json.h" #include #include @@ -282,6 +285,66 @@ td::Status SignFileQuery::receive(td::BufferSlice data) { return td::Status::OK(); } +td::Status ExportAllPrivateKeysQuery::run() { + TRY_RESULT_ASSIGN(directory_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + client_pk_ = ton::privkeys::Ed25519::random(); + return td::Status::OK(); +} + +td::Status ExportAllPrivateKeysQuery::send() { + auto b = ton::create_serialize_tl_object( + client_pk_.compute_public_key().tl()); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status ExportAllPrivateKeysQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + // Private keys are encrypted using client-provided public key to avoid storing them in + // non-secure buffers (not td::SecureString) + TRY_RESULT_PREFIX(decryptor, client_pk_.create_decryptor(), "cannot create decryptor: "); + TRY_RESULT_PREFIX(keys_data, decryptor->decrypt(f->encrypted_data_.as_slice()), "cannot decrypt data: "); + SCOPE_EXIT { + keys_data.as_slice().fill_zero_secure(); + }; + td::Slice slice = keys_data.as_slice(); + if (slice.size() < 32) { + return td::Status::Error("data is too small"); + } + slice.remove_suffix(32); + std::vector private_keys; + while (!slice.empty()) { + if (slice.size() < 4) { + return td::Status::Error("unexpected end of data"); + } + td::uint32 size; + td::MutableSlice{reinterpret_cast(&size), 4}.copy_from(slice.substr(0, 4)); + if (size > slice.size()) { + return td::Status::Error("unexpected end of data"); + } + slice.remove_prefix(4); + TRY_RESULT_PREFIX(private_key, ton::PrivateKey::import(slice.substr(0, size)), "cannot parse private key: "); + if (!private_key.exportable()) { + return td::Status::Error("private key is not exportable"); + } + private_keys.push_back(std::move(private_key)); + slice.remove_prefix(size); + } + + TRY_STATUS_PREFIX(td::mkpath(directory_ + "/"), "cannot create directory " + directory_ + ": "); + td::TerminalIO::out() << "exported " << private_keys.size() << " private keys" << "\n"; + for (const ton::PrivateKey &private_key : private_keys) { + std::string hash_hex = private_key.compute_short_id().bits256_value().to_hex(); + TRY_STATUS_PREFIX(td::write_file(directory_ + "/" + hash_hex, private_key.export_as_slice()), + "failed to write file: "); + td::TerminalIO::out() << "pubkey_hash " << hash_hex << "\n"; + } + td::TerminalIO::out() << "written all files to " << directory_ << "\n"; + return td::Status::OK(); +} + td::Status AddAdnlAddrQuery::run() { TRY_RESULT_ASSIGN(key_hash_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(category_, tokenizer_.get_token()); @@ -948,8 +1011,18 @@ td::Status GetOverlaysStatsJsonQuery::receive(td::BufferSlice data) { sb << " \"" << t->key_ << "\": \"" << t->value_ << "\""; } - sb << "\n }\n"; - sb << "}\n"; + sb << "\n }"; + if (!s->extra_.empty()) { + sb << ",\n \"extra\": "; + for (char c : s->extra_) { + if (c == '\n') { + sb << "\n "; + } else { + sb << c; + } + } + } + sb << "\n}\n"; } sb << "]\n"; sb << std::flush; @@ -968,8 +1041,7 @@ td::Status ImportCertificateQuery::receive(td::BufferSlice data) { td::Status SignShardOverlayCertificateQuery::run() { - TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); - TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); TRY_RESULT_ASSIGN(key_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(expire_at_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(max_size_, tokenizer_.get_token()); @@ -979,8 +1051,9 @@ td::Status SignShardOverlayCertificateQuery::run() { } td::Status SignShardOverlayCertificateQuery::send() { - auto b = ton::create_serialize_tl_object - (wc_, shard_, ton::create_tl_object(key_.tl()), expire_at_, max_size_); + auto b = ton::create_serialize_tl_object( + shard_.workchain, shard_.shard, ton::create_tl_object(key_.tl()), + expire_at_, max_size_); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -998,8 +1071,7 @@ td::Status SignShardOverlayCertificateQuery::receive(td::BufferSlice data) { } td::Status ImportShardOverlayCertificateQuery::run() { - TRY_RESULT_ASSIGN(wc_, tokenizer_.get_token()); - TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token() ); + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(key_, tokenizer_.get_token()); TRY_RESULT_ASSIGN(in_file_, tokenizer_.get_token()); @@ -1010,8 +1082,9 @@ td::Status ImportShardOverlayCertificateQuery::send() { TRY_RESULT(data, td::read_file(in_file_)); TRY_RESULT_PREFIX(cert, ton::fetch_tl_object(data.as_slice(), true), "incorrect certificate"); - auto b = ton::create_serialize_tl_object - (wc_, shard_, ton::create_tl_object(key_.tl()), std::move(cert)); + auto b = ton::create_serialize_tl_object( + shard_.workchain, shard_.shard, ton::create_tl_object(key_.tl()), + std::move(cert)); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1100,14 +1173,12 @@ td::Status GetPerfTimerStatsJsonQuery::receive(td::BufferSlice data) { } td::Status GetShardOutQueueSizeQuery::run() { - TRY_RESULT_ASSIGN(block_id_.workchain, tokenizer_.get_token()); - TRY_RESULT_ASSIGN(block_id_.shard, tokenizer_.get_token()); + TRY_RESULT(shard, tokenizer_.get_token()); + block_id_.workchain = shard.workchain; + block_id_.shard = shard.shard; TRY_RESULT_ASSIGN(block_id_.seqno, tokenizer_.get_token()); if (!tokenizer_.endl()) { - ton::ShardIdFull dest; - TRY_RESULT_ASSIGN(dest.workchain, tokenizer_.get_token()); - TRY_RESULT_ASSIGN(dest.shard, tokenizer_.get_token()); - dest_ = dest; + TRY_RESULT_ASSIGN(dest_, tokenizer_.get_token()); } TRY_STATUS(tokenizer_.check_endl()); return td::Status::OK(); @@ -1115,8 +1186,7 @@ td::Status GetShardOutQueueSizeQuery::run() { td::Status GetShardOutQueueSizeQuery::send() { auto b = ton::create_serialize_tl_object( - dest_ ? 1 : 0, ton::create_tl_block_id_simple(block_id_), dest_ ? dest_.value().workchain : 0, - dest_ ? dest_.value().shard : 0); + dest_.is_valid() ? 1 : 0, ton::create_tl_block_id_simple(block_id_), dest_.workchain, dest_.shard); td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); return td::Status::OK(); } @@ -1216,6 +1286,12 @@ td::Status ShowCustomOverlaysQuery::receive(td::BufferSlice data) { : "") << (node->block_sender_ ? " (block sender)" : "") << "\n"; } + if (!overlay->sender_shards_.empty()) { + td::TerminalIO::out() << "Sender shards:\n"; + for (const auto &shard : overlay->sender_shards_) { + td::TerminalIO::out() << " " << ton::create_shard_id(shard).to_str() << "\n"; + } + } td::TerminalIO::out() << "\n"; } return td::Status::OK(); @@ -1482,3 +1558,41 @@ td::Status GetAdnlStatsQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << sb.as_cslice(); return td::Status::OK(); } + +td::Status AddShardQuery::run() { + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status AddShardQuery::send() { + auto b = ton::create_serialize_tl_object(ton::create_tl_shard_id(shard_)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status AddShardQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully added shard\n"; + return td::Status::OK(); +} + +td::Status DelShardQuery::run() { + TRY_RESULT_ASSIGN(shard_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status DelShardQuery::send() { + auto b = ton::create_serialize_tl_object(ton::create_tl_shard_id(shard_)); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status DelShardQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "successfully removed shard\n"; + return td::Status::OK(); +} \ No newline at end of file diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 0e21c9c2..817d70c9 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -36,6 +36,7 @@ #include "ton/ton-types.h" #include "keys/keys.hpp" +#include "td/utils/base64.h" class ValidatorEngineConsole; @@ -95,27 +96,25 @@ inline td::Result Tokenizer::get_token() { } template <> -inline td::Result Tokenizer::get_token() { - TRY_RESULT(S, get_raw_token()); - TRY_RESULT(F, td::hex_decode(S)); - if (F.size() == 32) { - return ton::PublicKeyHash{td::Slice{F}}; +inline td::Result Tokenizer::get_token() { + TRY_RESULT(word, get_raw_token()); + std::string data; + if (word.size() == 64) { + TRY_RESULT_ASSIGN(data, td::hex_decode(word)); + } else if (word.size() == 44) { + TRY_RESULT_ASSIGN(data, td::base64_decode(word)); } else { return td::Status::Error("cannot parse keyhash: bad length"); } + td::Bits256 v; + v.as_slice().copy_from(data); + return v; } template <> -inline td::Result Tokenizer::get_token() { - TRY_RESULT(S, get_raw_token()); - TRY_RESULT(F, td::hex_decode(S)); - if (F.size() == 32) { - td::Bits256 v; - v.as_slice().copy_from(F); - return v; - } else { - return td::Status::Error("cannot parse keyhash: bad length"); - } +inline td::Result Tokenizer::get_token() { + TRY_RESULT(x, get_token()); + return ton::PublicKeyHash{x}; } template <> @@ -146,6 +145,18 @@ inline td::Result> Tokenizer::get_token_vector() { } } +template <> +inline td::Result Tokenizer::get_token() { + TRY_RESULT(word, get_raw_token()); + auto r_wc = td::to_integer_safe(word); + if (r_wc.is_ok()) { + TRY_RESULT_ASSIGN(word, get_raw_token()); + TRY_RESULT(shard, td::to_integer_safe(word)); + return ton::ShardIdFull{r_wc.move_as_ok(), shard}; + } + return ton::ShardIdFull::parse(word); +} + class QueryRunner { public: virtual ~QueryRunner() = default; @@ -222,10 +233,10 @@ class GetTimeQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice R) override; static std::string get_name() { - return "gettime"; + return "get-time"; } static std::string get_help() { - return "gettime\tshows current server unixtime"; + return "get-time\tshows current server unixtime"; } std::string name() const override { return get_name(); @@ -287,10 +298,10 @@ class NewKeyQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice R) override; static std::string get_name() { - return "newkey"; + return "new-key"; } static std::string get_help() { - return "newkey\tgenerates new key pair on server"; + return "new-key\tgenerates new key pair on server"; } std::string name() const override { return get_name(); @@ -308,10 +319,10 @@ class ImportPrivateKeyFileQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice R) override; static std::string get_name() { - return "importf"; + return "import-f"; } static std::string get_help() { - return "importf \timport private key"; + return "import-f \timport private key"; } std::string name() const override { return get_name(); @@ -330,10 +341,10 @@ class ExportPublicKeyQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice R) override; static std::string get_name() { - return "exportpub"; + return "export-pub"; } static std::string get_help() { - return "exportpub \texports public key by key hash"; + return "export-pub \texports public key by key hash"; } std::string name() const override { return get_name(); @@ -352,10 +363,10 @@ class ExportPublicKeyFileQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice R) override; static std::string get_name() { - return "exportpubf"; + return "export-pubf"; } static std::string get_help() { - return "exportpubf \texports public key by key hash"; + return "export-pub-f \texports public key by key hash"; } std::string name() const override { return get_name(); @@ -398,10 +409,10 @@ class SignFileQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "signf"; + return "sign-f"; } static std::string get_help() { - return "signf \tsigns bytestring with privkey"; + return "sign-f \tsigns bytestring with privkey"; } std::string name() const override { return get_name(); @@ -413,6 +424,30 @@ class SignFileQuery : public Query { std::string out_file_; }; +class ExportAllPrivateKeysQuery : public Query { + public: + ExportAllPrivateKeysQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice R) override; + static std::string get_name() { + return "export-all-private-keys"; + } + static std::string get_help() { + return "export-all-private-keys \texports all private keys from validator engine and stores them to " + ""; + } + std::string name() const override { + return get_name(); + } + + private: + std::string directory_; + ton::PrivateKey client_pk_; +}; + class AddAdnlAddrQuery : public Query { public: AddAdnlAddrQuery(td::actor::ActorId console, Tokenizer tokenizer) @@ -422,10 +457,10 @@ class AddAdnlAddrQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addadnl"; + return "add-adnl"; } static std::string get_help() { - return "addadnl \tuse key as ADNL addr"; + return "add-adnl \tuse key as ADNL addr"; } std::string name() const override { return get_name(); @@ -445,10 +480,10 @@ class AddDhtIdQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "adddht"; + return "add-dht"; } static std::string get_help() { - return "adddht \tcreate DHT node with specified ADNL addr"; + return "add-dht \tcreate DHT node with specified ADNL addr"; } std::string name() const override { return get_name(); @@ -467,10 +502,10 @@ class AddValidatorPermanentKeyQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addpermkey"; + return "add-perm-key"; } static std::string get_help() { - return "addpermkey \tadd validator permanent key"; + return "add-perm-key \tadd validator permanent key"; } std::string name() const override { return get_name(); @@ -491,10 +526,10 @@ class AddValidatorTempKeyQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addtempkey"; + return "add-temp-key"; } static std::string get_help() { - return "addtempkey \tadd validator temp key"; + return "add-temp-key \tadd validator temp key"; } std::string name() const override { return get_name(); @@ -515,10 +550,10 @@ class AddValidatorAdnlAddrQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addvalidatoraddr"; + return "add-validator-addr"; } static std::string get_help() { - return "addvalidatoraddr \tadd validator ADNL addr"; + return "add-validator-addr \tadd validator ADNL addr"; } std::string name() const override { return get_name(); @@ -539,10 +574,10 @@ class ChangeFullNodeAdnlAddrQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "changefullnodeaddr"; + return "change-full-node-addr"; } static std::string get_help() { - return "changefullnodeaddr \tchanges fullnode ADNL address"; + return "change-full-node-addr \tchanges fullnode ADNL address"; } std::string name() const override { return get_name(); @@ -561,10 +596,10 @@ class AddLiteServerQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addliteserver"; + return "add-liteserver"; } static std::string get_help() { - return "addliteserver \tadd liteserver"; + return "add-liteserver \tadd liteserver"; } std::string name() const override { return get_name(); @@ -584,10 +619,10 @@ class DelAdnlAddrQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "deladnl"; + return "del-adnl"; } static std::string get_help() { - return "deladnl \tdel unused ADNL addr"; + return "del-adnl \tdel unused ADNL addr"; } std::string name() const override { return get_name(); @@ -606,10 +641,10 @@ class DelDhtIdQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "deldht"; + return "del-dht"; } static std::string get_help() { - return "deldht \tdel unused DHT node"; + return "del-dht \tdel unused DHT node"; } std::string name() const override { return get_name(); @@ -628,10 +663,10 @@ class DelValidatorPermanentKeyQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "delpermkey"; + return "del-perm-key"; } static std::string get_help() { - return "delpermkey \tforce del unused validator permanent key"; + return "del-perm-key \tforce del unused validator permanent key"; } std::string name() const override { return get_name(); @@ -650,10 +685,10 @@ class DelValidatorTempKeyQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "deltempkey"; + return "del-temp-key"; } static std::string get_help() { - return "deltempkey \tforce del unused validator temp key"; + return "del-temp-key \tforce del unused validator temp key"; } std::string name() const override { return get_name(); @@ -673,10 +708,10 @@ class DelValidatorAdnlAddrQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "delvalidatoraddr"; + return "del-validator-addr"; } static std::string get_help() { - return "delvalidatoraddr \tforce del unused validator ADNL addr"; + return "del-validator-addr \tforce del unused validator ADNL addr"; } std::string name() const override { return get_name(); @@ -696,10 +731,10 @@ class GetConfigQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getconfig"; + return "get-config"; } static std::string get_help() { - return "getconfig\tdownloads current config"; + return "get-config\tdownloads current config"; } std::string name() const override { return get_name(); @@ -717,10 +752,10 @@ class SetVerbosityQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "setverbosity"; + return "set-verbosity"; } static std::string get_help() { - return "setverbosity \tchanges verbosity level"; + return "set-verbosity \tchanges verbosity level"; } std::string name() const override { return get_name(); @@ -739,10 +774,10 @@ class GetStatsQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getstats"; + return "get-stats"; } static std::string get_help() { - return "getstats\tprints stats"; + return "get-stats\tprints stats"; } std::string name() const override { return get_name(); @@ -783,10 +818,10 @@ class AddNetworkAddressQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addaddr"; + return "add-addr"; } static std::string get_help() { - return "addaddr {cats...} {priocats...}\tadds ip address to address list"; + return "add-addr {cats...} {priocats...}\tadds ip address to address list"; } std::string name() const override { return get_name(); @@ -807,10 +842,10 @@ class AddNetworkProxyAddressQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addproxyaddr"; + return "add-proxy-addr"; } static std::string get_help() { - return "addproxyaddr {cats...} {priocats...}\tadds ip address to address list"; + return "add-proxy-addr {cats...} {priocats...}\tadds ip address to address list"; } std::string name() const override { return get_name(); @@ -834,10 +869,10 @@ class CreateElectionBidQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "createelectionbid"; + return "create-election-bid"; } static std::string get_help() { - return "createelectionbid \tcreate election bid"; + return "create-election-bid \tcreate election bid"; } std::string name() const override { return get_name(); @@ -859,10 +894,10 @@ class CreateProposalVoteQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "createproposalvote"; + return "create-proposal-vote"; } static std::string get_help() { - return "createproposalvote \tcreate proposal vote"; + return "create-proposal-vote \tcreate proposal vote"; } std::string name() const override { return get_name(); @@ -882,10 +917,10 @@ class CreateComplaintVoteQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "createcomplaintvote"; + return "create-complaint-vote"; } static std::string get_help() { - return "createcomplaintvote \tcreate proposal vote"; + return "create-complaint-vote \tcreate proposal vote"; } std::string name() const override { return get_name(); @@ -906,10 +941,10 @@ class CheckDhtServersQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "checkdht"; + return "check-dht"; } static std::string get_help() { - return "checkdht \tchecks, which root DHT servers are accessible from this ADNL addr"; + return "check-dht \tchecks, which root DHT servers are accessible from this ADNL addr"; } std::string name() const override { return get_name(); @@ -928,10 +963,10 @@ class GetOverlaysStatsQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getoverlaysstats"; + return "get-overlays-stats"; } static std::string get_help() { - return "getoverlaysstats\tgets stats for all overlays"; + return "get-overlays-stats\tgets stats for all overlays"; } std::string name() const override { return get_name(); @@ -947,10 +982,10 @@ class GetOverlaysStatsJsonQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getoverlaysstatsjson"; + return "get-overlays-stats-json"; } static std::string get_help() { - return "getoverlaysstatsjson \tgets stats for all overlays and writes to json file"; + return "get-overlays-stats-json \tgets stats for all overlays and writes to json file"; } std::string name() const override { return get_name(); @@ -969,10 +1004,11 @@ class SignCertificateQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "signcert"; + return "sign-cert"; } static std::string get_help() { - return "signcert \tsign overlay certificate by key"; + return "sign-cert \tsign overlay certificate by " + " key"; } std::string name() const override { return get_name(); @@ -1005,10 +1041,10 @@ class ImportCertificateQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "importcert"; + return "import-cert"; } static std::string get_help() { - return "importcert \timport overlay certificate for specific key"; + return "import-cert \timport overlay certificate for specific key"; } std::string name() const override { return get_name(); @@ -1030,10 +1066,11 @@ class SignShardOverlayCertificateQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "signshardoverlaycert"; + return "sign-shard-overlay-cert"; } static std::string get_help() { - return "signshardoverlaycert \tsign certificate for in currently active shard overlay"; + return "sign-shard-overlay-cert : \tsign certificate " + "for in currently active shard overlay"; } std::string name() const override { return get_name(); @@ -1041,8 +1078,7 @@ class SignShardOverlayCertificateQuery : public Query { private: - td::int32 wc_; - td::int64 shard_; + ton::ShardIdFull shard_; td::int32 expire_at_; ton::PublicKeyHash key_; td::uint32 max_size_; @@ -1059,10 +1095,11 @@ class ImportShardOverlayCertificateQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "importshardoverlaycert"; + return "import-shard-overlay-cert"; } static std::string get_help() { - return "importshardoverlaycert \timport certificate for in currently active shard overlay"; + return "import-shard-overlay-cert : \timport certificate for in " + "currently active shard overlay"; } std::string name() const override { return get_name(); @@ -1070,8 +1107,7 @@ class ImportShardOverlayCertificateQuery : public Query { private: - td::int32 wc_; - td::int64 shard_; + ton::ShardIdFull shard_; ton::PublicKeyHash key_; std::string in_file_; }; @@ -1085,10 +1121,10 @@ class GetActorStatsQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getactorstats"; + return "get-actor-stats"; } static std::string get_help() { - return "getactorstats []\tget actor stats and print it either in stdout or in "; + return "get-actor-stats []\tget actor stats and print it either in stdout or in "; } std::string name() const override { return get_name(); @@ -1107,10 +1143,11 @@ class GetPerfTimerStatsJsonQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getperftimerstatsjson"; + return "get-perf-timer-stats-json"; } static std::string get_help() { - return "getperftimerstatsjson \tgets min, average and max event processing time for last 60, 300 and 3600 seconds and writes to json file"; + return "get-perf-timer-stats-json \tgets min, average and max event processing time for last 60, 300 and " + "3600 seconds and writes to json file"; } std::string name() const override { return get_name(); @@ -1129,10 +1166,10 @@ class GetShardOutQueueSizeQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getshardoutqueuesize"; + return "get-shard-out-queue-size"; } static std::string get_help() { - return "getshardoutqueuesize [ ]\treturns number of messages in the " + return "get-shard-out-queue-size : [:]\treturns number of messages in the " "queue of the given shard. Destination shard is optional."; } std::string name() const override { @@ -1141,7 +1178,7 @@ class GetShardOutQueueSizeQuery : public Query { private: ton::BlockId block_id_; - td::optional dest_; + ton::ShardIdFull dest_ = ton::ShardIdFull{ton::workchainInvalid}; }; class SetExtMessagesBroadcastDisabledQuery : public Query { @@ -1153,11 +1190,11 @@ class SetExtMessagesBroadcastDisabledQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "setextmessagesbroadcastdisabled"; + return "set-ext-messages-broadcast-disabled"; } static std::string get_help() { - return "setextmessagesbroadcastdisabled \tdisable broadcasting and rebroadcasting ext messages; value is 0 " - "or 1."; + return "set-ext-messages-broadcast-disabled \tdisable broadcasting and rebroadcasting ext messages; value " + "is 0 or 1."; } std::string name() const override { return get_name(); @@ -1176,10 +1213,10 @@ class AddCustomOverlayQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "addcustomoverlay"; + return "add-custom-overlay"; } static std::string get_help() { - return "addcustomoverlay \tadd custom overlay with config from file "; + return "add-custom-overlay \tadd custom overlay with config from file "; } std::string name() const override { return get_name(); @@ -1198,10 +1235,10 @@ class DelCustomOverlayQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "delcustomoverlay"; + return "del-custom-overlay"; } static std::string get_help() { - return "delcustomoverlay \tdelete custom overlay with name "; + return "del-custom-overlay \tdelete custom overlay with name "; } std::string name() const override { return get_name(); @@ -1220,10 +1257,10 @@ class ShowCustomOverlaysQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "showcustomoverlays"; + return "show-custom-overlays"; } static std::string get_help() { - return "showcustomoverlays\tshow all custom overlays"; + return "show-custom-overlays\tshow all custom overlays"; } std::string name() const override { return get_name(); @@ -1239,10 +1276,10 @@ class SetStateSerializerEnabledQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "setstateserializerenabled"; + return "set-state-serializer-enabled"; } static std::string get_help() { - return "setstateserializerenabled \tdisable or enable persistent state serializer; value is 0 or 1"; + return "set-state-serializer-enabled \tdisable or enable persistent state serializer; value is 0 or 1"; } std::string name() const override { return get_name(); @@ -1261,10 +1298,10 @@ class SetCollatorOptionsJsonQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "setcollatoroptionsjson"; + return "set-collator-options-json"; } static std::string get_help() { - return "setcollatoroptionsjson \tset collator options from file "; + return "set-collator-options-json \tset collator options from file "; } std::string name() const override { return get_name(); @@ -1283,10 +1320,10 @@ class ResetCollatorOptionsQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "resetcollatoroptions"; + return "reset-collator-options"; } static std::string get_help() { - return "resetcollatoroptions\tset collator options to default values"; + return "reset-collator-options\tset collator options to default values"; } std::string name() const override { return get_name(); @@ -1302,10 +1339,10 @@ class GetCollatorOptionsJsonQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getcollatoroptionsjson"; + return "get-collator-options-json"; } static std::string get_help() { - return "getcollatoroptionsjson \tsave current collator options to file "; + return "get-collator-options-json \tsave current collator options to file "; } std::string name() const override { return get_name(); @@ -1324,11 +1361,11 @@ class GetAdnlStatsJsonQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getadnlstatsjson"; + return "get-adnl-stats-json"; } static std::string get_help() { - return "getadnlstatsjson [all]\tsave adnl stats to . all - returns all peers (default - only " - "peers with traffic in the last 10 minutes)"; + return "get-adnl-stats-json [all]\tsave adnl stats to . all - returns all peers (default - " + "only peers with traffic in the last 10 minutes)"; } std::string name() const override { return get_name(); @@ -1348,11 +1385,11 @@ class GetAdnlStatsQuery : public Query { td::Status send() override; td::Status receive(td::BufferSlice data) override; static std::string get_name() { - return "getadnlstats"; + return "get-adnl-stats"; } static std::string get_help() { - return "getadnlstats [all]\tdisplay adnl stats. all - returns all peers (default - only peers with traffic in the " - "last 10 minutes)"; + return "get-adnl-stats [all]\tdisplay adnl stats. all - returns all peers (default - only peers with traffic in " + "the last 10 minutes)"; } std::string name() const override { return get_name(); @@ -1362,3 +1399,47 @@ class GetAdnlStatsQuery : public Query { std::string file_name_; bool all_ = false; }; + +class AddShardQuery : public Query { + public: + AddShardQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "add-shard"; + } + static std::string get_help() { + return "add-shard :\tstart monitoring shard"; + } + std::string name() const override { + return get_name(); + } + + private: + ton::ShardIdFull shard_; +}; + +class DelShardQuery : public Query { + public: + DelShardQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "del-shard"; + } + static std::string get_help() { + return "del-shard :\tstop monitoring shard"; + } + std::string name() const override { + return get_name(); + } + + private: + ton::ShardIdFull shard_; +}; \ No newline at end of file diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 59e2f2e8..234cd6a5 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -112,6 +112,7 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); @@ -153,6 +154,8 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { @@ -203,9 +206,8 @@ void ValidatorEngineConsole::show_help(std::string command, td::Promisehelp() << "\n"; } } else { - auto it = query_runners_.find(command); - if (it != query_runners_.end()) { - td::TerminalIO::out() << it->second->help() << "\n"; + if (auto query = get_query(command)) { + td::TerminalIO::out() << query->help() << "\n"; } else { td::TerminalIO::out() << "unknown command '" << command << "'\n"; } @@ -229,10 +231,9 @@ void ValidatorEngineConsole::parse_line(td::BufferSlice data) { } auto name = tokenizer.get_token().move_as_ok(); - auto it = query_runners_.find(name); - if (it != query_runners_.end()) { + if (auto query = get_query(name)) { running_queries_++; - it->second->run(actor_id(this), std::move(tokenizer)); + query->run(actor_id(this), std::move(tokenizer)); } else { td::TerminalIO::out() << "unknown command '" << name << "'\n"; } diff --git a/validator-engine-console/validator-engine-console.h b/validator-engine-console/validator-engine-console.h index 7a384276..c802794e 100644 --- a/validator-engine-console/validator-engine-console.h +++ b/validator-engine-console/validator-engine-console.h @@ -57,9 +57,23 @@ class ValidatorEngineConsole : public td::actor::Actor { std::unique_ptr make_callback(); std::map> query_runners_; + std::map alternate_names_; + static std::string simplify_name(std::string name) { + std::erase_if(name, [](char c) { return c == '-'; }); + return name; + } void add_query_runner(std::unique_ptr runner) { auto name = runner->name(); query_runners_[name] = std::move(runner); + alternate_names_[simplify_name(name)] = name; + } + QueryRunner* get_query(std::string name) { + auto it = alternate_names_.find(name); + if (it != alternate_names_.end()) { + name = it->second; + } + auto it2 = query_runners_.find(name); + return it2 == query_runners_.end() ? nullptr : it2->second.get(); } public: diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 8b35b2b1..2ea04e18 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -54,6 +54,7 @@ #include "td/utils/Random.h" #include "auto/tl/lite_api.h" +#include "tl/tl_json.h" #include "memprof/memprof.h" @@ -64,10 +65,10 @@ #endif #include #include -#include #include #include #include +#include #include "git.h" #include "block-auto.h" #include "block-parse.h" @@ -84,7 +85,7 @@ Config::Config() { full_node = ton::PublicKeyHash::zero(); } -Config::Config(ton::ton_api::engine_validator_config &config) { +Config::Config(const ton::ton_api::engine_validator_config &config) { full_node = ton::PublicKeyHash::zero(); out_port = static_cast(config.out_port_); if (!out_port) { @@ -97,7 +98,7 @@ Config::Config(ton::ton_api::engine_validator_config &config) { std::vector categories; std::vector priority_categories; ton::ton_api::downcast_call( - *addr.get(), + *addr, td::overloaded( [&](const ton::ton_api::engine_addr &obj) { in_ip.init_ipv4_port(td::IPAddress::ipv4_to_str(obj.ip_), static_cast(obj.port_)).ensure(); @@ -179,6 +180,10 @@ Config::Config(ton::ton_api::engine_validator_config &config) { } } + for (auto &shard : config.shards_to_monitor_) { + config_add_shard(ton::create_shard_id(shard)).ensure(); + } + if (config.gc_) { for (auto &gc : config.gc_->ids_) { config_add_gc(ton::PublicKeyHash{gc}).ensure(); @@ -262,14 +267,21 @@ ton::tl_object_ptr Config::tl() const { std::move(control_proc_vec))); } + std::vector> shards_vec; + for (auto &shard : shards_to_monitor) { + shards_vec.push_back(ton::create_tl_shard_id(shard)); + } + auto gc_vec = ton::create_tl_object(std::vector{}); for (auto &id : gc) { gc_vec->ids_.push_back(id.tl()); } + return ton::create_tl_object( - out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), full_node.tl(), - std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(full_node_config_obj), - std::move(extra_config_obj), std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); + out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), + full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), + std::move(full_node_config_obj), std::move(extra_config_obj), std::move(liteserver_vec), std::move(control_vec), + std::move(shards_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, @@ -532,6 +544,32 @@ td::Result Config::config_add_control_process(ton::PublicKeyHash key, td:: } } +td::Result Config::config_add_shard(ton::ShardIdFull shard) { + if (shard.is_masterchain()) { + return td::Status::Error("masterchain is monitored by default"); + } + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << shard.to_str()); + } + if (std::find(shards_to_monitor.begin(), shards_to_monitor.end(), shard) != shards_to_monitor.end()) { + return false; + } + shards_to_monitor.push_back(shard); + return true; +} + +td::Result Config::config_del_shard(ton::ShardIdFull shard) { + if (!shard.is_valid_ext()) { + return td::Status::Error(PSTRING() << "invalid shard " << shard.to_str()); + } + auto it = std::find(shards_to_monitor.begin(), shards_to_monitor.end(), shard); + if (it == shards_to_monitor.end()) { + return false; + } + shards_to_monitor.erase(it); + return true; +} + td::Result Config::config_add_gc(ton::PublicKeyHash key) { return gc.insert(key).second; } @@ -1393,15 +1431,6 @@ td::Status ValidatorEngine::load_global_config() { } validator_options_ = ton::validator::ValidatorManagerOptions::create(zero_state, init_block); - validator_options_.write().set_shard_check_function( - [](ton::ShardIdFull shard, ton::CatchainSeqno cc_seqno, - ton::validator::ValidatorManagerOptions::ShardCheckMode mode) -> bool { - if (mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_monitor) { - return true; - } - CHECK(mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_validate); - return true; - }); if (state_ttl_ != 0) { validator_options_.write().set_state_ttl(state_ttl_); } @@ -1474,22 +1503,34 @@ td::Status ValidatorEngine::load_global_config() { h.push_back(b); } validator_options_.write().set_hardforks(std::move(h)); - - auto r_total_mem_stat = td::get_total_mem_stat(); - if (r_total_mem_stat.is_error()) { - LOG(ERROR) << "Failed to get total RAM size: " << r_total_mem_stat.move_as_error(); - } else { - td::uint64 total_ram = r_total_mem_stat.ok().total_ram; - LOG(WARNING) << "Total RAM = " << td::format::as_size(total_ram); - if (total_ram >= (90ULL << 30)) { - fast_state_serializer_enabled_ = true; - } - } validator_options_.write().set_fast_state_serializer_enabled(fast_state_serializer_enabled_); + validator_options_.write().set_catchain_broadcast_speed_multiplier(broadcast_speed_multiplier_catchain_); return td::Status::OK(); } +void ValidatorEngine::set_shard_check_function() { + if (!not_all_shards_) { + validator_options_.write().set_shard_check_function([](ton::ShardIdFull shard) -> bool { return true; }); + } else { + std::vector shards = {ton::ShardIdFull(ton::masterchainId)}; + for (const auto& s : config_.shards_to_monitor) { + shards.push_back(s); + } + std::sort(shards.begin(), shards.end()); + shards.erase(std::unique(shards.begin(), shards.end()), shards.end()); + validator_options_.write().set_shard_check_function( + [shards = std::move(shards)](ton::ShardIdFull shard) -> bool { + for (auto s : shards) { + if (shard_intersects(shard, s)) { + return true; + } + } + return false; + }); + } +} + void ValidatorEngine::load_empty_local_config(td::Promise promise) { auto ret_promise = td::PromiseCreator::lambda( [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { @@ -1531,6 +1572,14 @@ void ValidatorEngine::load_empty_local_config(td::Promise promise) { } void ValidatorEngine::load_local_config(td::Promise promise) { + for (ton::ShardIdFull shard : add_shard_cmds_) { + auto R = config_.config_add_shard(shard); + if (R.is_error()) { + LOG(WARNING) << "Cannot add shard " << shard.to_str() << " : " << R.move_as_error(); + } else if (R.ok()) { + LOG(WARNING) << "Adding shard to monitor " << shard.to_str(); + } + } if (local_config_.size() == 0) { load_empty_local_config(std::move(promise)); return; @@ -1746,6 +1795,15 @@ void ValidatorEngine::load_config(td::Promise promise) { td::actor::send_closure(keyring_, &ton::keyring::Keyring::add_key_short, key.first, get_key_promise(ig)); } + for (ton::ShardIdFull shard : add_shard_cmds_) { + auto R = config_.config_add_shard(shard); + if (R.is_error()) { + LOG(WARNING) << "Cannot add shard " << shard.to_str() << " : " << R.move_as_error(); + } else if (R.ok()) { + LOG(WARNING) << "Adding shard to monitor " << shard.to_str(); + } + } + write_config(ig.get_promise()); } @@ -1781,6 +1839,7 @@ void ValidatorEngine::got_key(ton::PublicKey key) { } void ValidatorEngine::start() { + set_shard_check_function(); read_config_ = true; start_adnl(); } @@ -1876,6 +1935,8 @@ void ValidatorEngine::started_dht() { void ValidatorEngine::start_rldp() { rldp_ = ton::rldp::Rldp::create(adnl_.get()); rldp2_ = ton::rldp2::Rldp::create(adnl_.get()); + td::actor::send_closure(rldp_, &ton::rldp::Rldp::set_default_mtu, 2048); + td::actor::send_closure(rldp2_, &ton::rldp2::Rldp::set_default_mtu, 2048); started_rldp(); } @@ -1897,7 +1958,8 @@ void ValidatorEngine::started_overlays() { void ValidatorEngine::start_validator() { validator_options_.write().set_allow_blockchain_init(config_.validators.size() > 0); - validator_options_.write().set_state_serializer_enabled(config_.state_serializer_enabled); + validator_options_.write().set_state_serializer_enabled(config_.state_serializer_enabled && + !state_serializer_disabled_flag_); load_collator_options(); validator_manager_ = ton::validator::ValidatorManagerFactory::create( @@ -1939,20 +2001,31 @@ void ValidatorEngine::start_full_node() { }; full_node_client_ = ton::adnl::AdnlExtMultiClient::create(std::move(vec), std::make_unique()); } + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + R.ensure(); + td::actor::send_closure(SelfId, &ValidatorEngine::started_full_node); + }); + ton::validator::fullnode::FullNodeOptions full_node_options{ + .config_ = config_.full_node_config, + .public_broadcast_speed_multiplier_ = broadcast_speed_multiplier_public_, + .private_broadcast_speed_multiplier_ = broadcast_speed_multiplier_private_}; full_node_ = ton::validator::fullnode::FullNode::create( short_id, ton::adnl::AdnlNodeIdShort{config_.full_node}, validator_options_->zero_block_id().file_hash, - config_.full_node_config, keyring_.get(), adnl_.get(), rldp_.get(), rldp2_.get(), + full_node_options, keyring_.get(), adnl_.get(), rldp_.get(), rldp2_.get(), default_dht_node_.is_zero() ? td::actor::ActorId{} : dht_nodes_[default_dht_node_].get(), - overlay_manager_.get(), validator_manager_.get(), full_node_client_.get(), db_root_); + overlay_manager_.get(), validator_manager_.get(), full_node_client_.get(), db_root_, std::move(P)); + for (auto &v : config_.validators) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_permanent_key, v.first, + [](td::Unit) {}); + } load_custom_overlays_config(); + if (!validator_telemetry_filename_.empty()) { + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::set_validator_telemetry_filename, + validator_telemetry_filename_); + } + } else { + started_full_node(); } - - for (auto &v : config_.validators) { - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_permanent_key, v.first, - [](td::Unit) {}); - } - - started_full_node(); } void ValidatorEngine::started_full_node() { @@ -3255,6 +3328,70 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_sign &que std::move(query.data_), std::move(P)); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_exportAllPrivateKeys &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_unsafe)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (keyring_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started keyring"))); + return; + } + + ton::PublicKey client_pubkey = ton::PublicKey{query.encryption_key_}; + if (!client_pubkey.is_ed25519()) { + promise.set_value( + create_control_query_error(td::Status::Error(ton::ErrorCode::protoviolation, "encryption key is not Ed25519"))); + return; + } + + td::actor::send_closure( + keyring_, &ton::keyring::Keyring::export_all_private_keys, + [promise = std::move(promise), + client_pubkey = std::move(client_pubkey)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + // Private keys are encrypted using client-provided public key to avoid storing them in + // non-secure buffers (not td::SecureString) + std::vector serialized_keys; + size_t data_size = 32; + for (const ton::PrivateKey &key : R.ok()) { + serialized_keys.push_back(key.export_as_slice()); + data_size += serialized_keys.back().size() + 4; + } + td::SecureString data{data_size}; + td::MutableSlice slice = data.as_mutable_slice(); + for (const td::SecureString &s : serialized_keys) { + td::uint32 size = td::narrow_cast_safe(s.size()).move_as_ok(); + CHECK(slice.size() >= size + 4); + slice.copy_from(td::Slice{reinterpret_cast(&size), 4}); + slice.remove_prefix(4); + slice.copy_from(s.as_slice()); + slice.remove_prefix(s.size()); + } + CHECK(slice.size() == 32); + td::Random::secure_bytes(slice); + + auto r_encryptor = client_pubkey.create_encryptor(); + if (r_encryptor.is_error()) { + promise.set_value(create_control_query_error(r_encryptor.move_as_error_prefix("cannot create encryptor: "))); + return; + } + auto encryptor = r_encryptor.move_as_ok(); + auto r_encrypted = encryptor->encrypt(data.as_slice()); + if (r_encryptor.is_error()) { + promise.set_value(create_control_query_error(r_encrypted.move_as_error_prefix("cannot encrypt data: "))); + return; + } + promise.set_value(ton::create_serialize_tl_object( + r_encrypted.move_as_ok())); + }); +} + void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setVerbosity &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { if (!(perm & ValidatorEnginePermissions::vep_default)) { @@ -3842,7 +3979,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setStateS promise.set_value(ton::create_serialize_tl_object()); return; } - validator_options_.write().set_state_serializer_enabled(query.enabled_); + validator_options_.write().set_state_serializer_enabled(query.enabled_ && !state_serializer_disabled_flag_); td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, validator_options_); config_.state_serializer_enabled = query.enabled_; @@ -3924,6 +4061,74 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getAdnlSt }); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addShard &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + auto shard = ton::create_shard_id(query.shard_); + auto R = config_.config_add_shard(shard); + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + set_shard_check_function(); + if (!validator_manager_.empty()) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + } + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delShard &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + + auto shard = ton::create_shard_id(query.shard_); + auto R = config_.config_del_shard(shard); + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + return; + } + if (!R.move_as_ok()) { + promise.set_value(create_control_query_error(td::Status::Error("No such shard"))); + return; + } + set_shard_check_function(); + if (!validator_manager_.empty()) { + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + } + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value(ton::serialize_tl_object(ton::create_tl_object(), true)); + } + }); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { @@ -3957,7 +4162,7 @@ void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNode } auto f = F.move_as_ok(); - ton::ton_api::downcast_call(*f.get(), [&](auto &obj) { + ton::ton_api::downcast_call(*f, [&](auto &obj) { run_control_query(obj, std::move(data), src.pubkey_hash(), it->second, std::move(promise)); }); } @@ -4216,6 +4421,23 @@ int main(int argc, char *argv[]) { }); return td::Status::OK(); }); + p.add_option('M', "not-all-shards", "monitor only a necessary set of shards instead of all", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_not_all_shards); }); + }); + p.add_checked_option( + '\0', "add-shard", "add shard to monitor (same as addshard in validator console), format: 0:8000000000000000", + [&](td::Slice arg) -> td::Status { + std::string str = arg.str(); + int wc; + unsigned long long shard; + if (sscanf(str.c_str(), "%d:%016llx", &wc, &shard) != 2) { + return td::Status::Error(PSTRING() << "invalid shard " << str); + } + acts.push_back([=, &x]() { + td::actor::send_closure(x, &ValidatorEngine::add_shard_cmd, ton::ShardIdFull{wc, (ton::ShardId)shard}); + }); + return td::Status::OK(); + }); td::uint32 threads = 7; p.add_checked_option( 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice arg) { @@ -4326,11 +4548,61 @@ int main(int argc, char *argv[]) { }); p.add_option( '\0', "fast-state-serializer", - "faster persistent state serializer, but requires more RAM (enabled automatically on machines with >= 90GB RAM)", + "faster persistent state serializer, but requires more RAM", [&]() { acts.push_back( [&x]() { td::actor::send_closure(x, &ValidatorEngine::set_fast_state_serializer_enabled, true); }); }); + p.add_option( + '\0', "collect-validator-telemetry", + "store validator telemetry from private block overlay to a given file (json format)", + [&](td::Slice s) { + acts.push_back( + [&x, s = s.str()]() { + td::actor::send_closure(x, &ValidatorEngine::set_validator_telemetry_filename, s); + }); + }); + p.add_option( + '\0', "disable-state-serializer", + "disable persistent state serializer (similar to set-state-serializer-enabled 0 in validator console)", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_state_serializer_disabled_flag); }); + }); + p.add_checked_option( + '\0', "broadcast-speed-catchain", + "multiplier for broadcast speed in catchain overlays (experimental, default is 1.0, which is ~300 KB/s)", + [&](td::Slice s) -> td::Status { + auto v = td::to_double(s); + if (v <= 0.0) { + return td::Status::Error("broadcast-speed-catchain should be positive"); + } + acts.push_back( + [&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_broadcast_speed_multiplier_catchain, v); }); + return td::Status::OK(); + }); + p.add_checked_option( + '\0', "broadcast-speed-public", + "multiplier for broadcast speed in public shard overlays (experimental, default is 1.0, which is ~300 KB/s)", + [&](td::Slice s) -> td::Status { + auto v = td::to_double(s); + if (v <= 0.0) { + return td::Status::Error("broadcast-speed-public should be positive"); + } + acts.push_back( + [&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_broadcast_speed_multiplier_public, v); }); + return td::Status::OK(); + }); + p.add_checked_option( + '\0', "broadcast-speed-private", + "multiplier for broadcast speed in private block overlays (experimental, default is 1.0, which is ~300 KB/s)", + [&](td::Slice s) -> td::Status { + auto v = td::to_double(s); + if (v <= 0.0) { + return td::Status::Error("broadcast-speed-private should be positive"); + } + acts.push_back( + [&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_broadcast_speed_multiplier_private, v); }); + return td::Status::OK(); + }); auto S = p.run(argc, argv); if (S.is_error()) { LOG(ERROR) << "failed to parse options: " << S.move_as_error(); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 50cd5a32..e0dc91f1 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -89,6 +89,7 @@ struct Config { ton::validator::fullnode::FullNodeConfig full_node_config; std::map controls; std::set gc; + std::vector shards_to_monitor; bool state_serializer_enabled = true; @@ -115,6 +116,8 @@ struct Config { td::Result config_add_control_interface(ton::PublicKeyHash key, td::int32 port); td::Result config_add_control_process(ton::PublicKeyHash key, td::int32 port, ton::PublicKeyHash id, td::uint32 permissions); + td::Result config_add_shard(ton::ShardIdFull shard); + td::Result config_del_shard(ton::ShardIdFull shard); td::Result config_add_gc(ton::PublicKeyHash key); td::Result config_del_network_addr(td::IPAddress addr, std::vector cats, std::vector prio_cats); @@ -132,7 +135,7 @@ struct Config { ton::tl_object_ptr tl() const; Config(); - Config(ton::ton_api::engine_validator_config &config); + Config(const ton::ton_api::engine_validator_config &config); }; class ValidatorEngine : public td::actor::Actor { @@ -222,6 +225,13 @@ class ValidatorEngine : public td::actor::Actor { ton::BlockSeqno truncate_seqno_{0}; std::string session_logs_file_; bool fast_state_serializer_enabled_ = false; + std::string validator_telemetry_filename_; + bool not_all_shards_ = false; + std::vector add_shard_cmds_; + bool state_serializer_disabled_flag_ = false; + double broadcast_speed_multiplier_catchain_ = 1.0; + double broadcast_speed_multiplier_public_ = 1.0; + double broadcast_speed_multiplier_private_ = 1.0; std::set unsafe_catchains_; std::map> unsafe_catchain_rotations_; @@ -310,6 +320,28 @@ class ValidatorEngine : public td::actor::Actor { void set_fast_state_serializer_enabled(bool value) { fast_state_serializer_enabled_ = value; } + void set_validator_telemetry_filename(std::string value) { + validator_telemetry_filename_ = std::move(value); + } + void set_not_all_shards() { + not_all_shards_ = true; + } + void add_shard_cmd(ton::ShardIdFull shard) { + add_shard_cmds_.push_back(shard); + } + void set_state_serializer_disabled_flag() { + state_serializer_disabled_flag_ = true; + } + void set_broadcast_speed_multiplier_catchain(double value) { + broadcast_speed_multiplier_catchain_ = value; + } + void set_broadcast_speed_multiplier_public(double value) { + broadcast_speed_multiplier_public_ = value; + } + void set_broadcast_speed_multiplier_private(double value) { + broadcast_speed_multiplier_private_ = value; + } + void start_up() override; ValidatorEngine() { } @@ -319,6 +351,7 @@ class ValidatorEngine : public td::actor::Actor { void load_empty_local_config(td::Promise promise); void load_local_config(td::Promise promise); void load_config(td::Promise promise); + void set_shard_check_function(); void start(); @@ -458,6 +491,8 @@ class ValidatorEngine : public td::actor::Actor { td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_sign &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_exportAllPrivateKeys &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_setVerbosity &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getStats &query, td::BufferSlice data, ton::PublicKeyHash src, @@ -480,6 +515,10 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getActorTextStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_addShard &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_delShard &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getPerfTimerStats &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getShardOutQueueSize &query, td::BufferSlice data, diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index d5ba00e3..5f5544b2 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -46,6 +46,7 @@ set(VALIDATOR_HEADERS interfaces/db.h interfaces/external-message.h interfaces/liteserver.h + interfaces/out-msg-queue-proof.h interfaces/proof.h interfaces/shard.h interfaces/signature-set.h @@ -56,6 +57,7 @@ set(VALIDATOR_HEADERS import-db-slice.hpp queue-size-counter.hpp + validator-telemetry.hpp manager-disk.h manager-disk.hpp @@ -82,6 +84,7 @@ set(VALIDATOR_SOURCE validator-group.cpp validator-options.cpp queue-size-counter.cpp + validator-telemetry.cpp downloaders/wait-block-data.cpp downloaders/wait-block-state.cpp diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index 71b3b73d..8c7cde17 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -624,7 +624,7 @@ void ArchiveManager::load_package(PackageId id) { } } - desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_, + desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, 0, db_root_, archive_lru_.get(), statistics_); m.emplace(id, std::move(desc)); @@ -659,7 +659,8 @@ const ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull FileDescription new_desc{id, false}; td::mkdir(db_root_ + id.path()).ensure(); std::string prefix = PSTRING() << db_root_ << id.path() << id.name(); - new_desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, db_root_, + new_desc.file = td::actor::create_actor("slice", id.id, id.key, id.temp, false, + id.key || id.temp ? 0 : cur_shard_split_depth_, db_root_, archive_lru_.get(), statistics_); const FileDescription &desc = f.emplace(id, std::move(new_desc)); if (!id.temp) { @@ -1132,14 +1133,16 @@ PackageId ArchiveManager::get_package_id_force(BlockSeqno masterchain_seqno, Sha return it->first; } -void ArchiveManager::get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) { +void ArchiveManager::get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) { auto F = get_file_desc_by_seqno(ShardIdFull{masterchainId}, masterchain_seqno, false); if (!F) { promise.set_error(td::Status::Error(ErrorCode::notready, "archive not found")); return; } - td::actor::send_closure(F->file_actor_id(), &ArchiveSlice::get_archive_id, masterchain_seqno, std::move(promise)); + td::actor::send_closure(F->file_actor_id(), &ArchiveSlice::get_archive_id, masterchain_seqno, shard_prefix, + std::move(promise)); } void ArchiveManager::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, @@ -1193,6 +1196,30 @@ void ArchiveManager::set_async_mode(bool mode, td::Promise promise) { } } +void ArchiveManager::prepare_stats(td::Promise>> promise) { + std::vector> stats; + { + std::map states; + for (auto &[key, file] : perm_states_) { + BlockSeqno seqno = key.first; + auto r_stat = td::stat(db_root_ + "/archive/states/" + file.filename_short()); + if (r_stat.is_error()) { + LOG(WARNING) << "Cannot stat persistent state file " << file.filename_short() << " : " << r_stat.move_as_error(); + } else { + states[seqno] += r_stat.move_as_ok().size_; + } + } + td::StringBuilder sb; + for (auto &[seqno, size] : states) { + sb << seqno << ":" << td::format::as_size(size) << " "; + } + if (!sb.as_cslice().empty()) { + stats.emplace_back("persistent_states", sb.as_cslice().str()); + } + } + promise.set_value(std::move(stats)); +} + void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise) { index_->begin_transaction().ensure(); td::MultiPromise mp; diff --git a/validator/db/archive-manager.hpp b/validator/db/archive-manager.hpp index 622969ec..d919e32e 100644 --- a/validator/db/archive-manager.hpp +++ b/validator/db/archive-manager.hpp @@ -67,7 +67,7 @@ class ArchiveManager : public td::actor::Actor { void get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt, td::Promise promise); void get_block_by_seqno(AccountIdPrefixFull account_id, BlockSeqno seqno, td::Promise promise); - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise); + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::Promise promise); void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise); @@ -77,6 +77,12 @@ class ArchiveManager : public td::actor::Actor { void commit_transaction(); void set_async_mode(bool mode, td::Promise promise); + void set_current_shard_split_depth(td::uint32 value) { + cur_shard_split_depth_ = value; + } + + void prepare_stats(td::Promise>> promise); + static constexpr td::uint32 archive_size() { return 20000; } @@ -175,6 +181,7 @@ class ArchiveManager : public td::actor::Actor { bool async_mode_ = false; bool huge_transaction_started_ = false; td::uint32 huge_transaction_size_ = 0; + td::uint32 cur_shard_split_depth_ = 0; DbStatistics statistics_; diff --git a/validator/db/archive-slice.cpp b/validator/db/archive-slice.cpp index 83fe144b..41ed5099 100644 --- a/validator/db/archive-slice.cpp +++ b/validator/db/archive-slice.cpp @@ -39,7 +39,7 @@ class PackageStatistics { void record_close(uint64_t count = 1) { close_count.fetch_add(count, std::memory_order_relaxed); } - + void record_read(double time, uint64_t bytes) { read_bytes.fetch_add(bytes, std::memory_order_relaxed); std::lock_guard guard(read_mutex); @@ -56,10 +56,10 @@ class PackageStatistics { std::stringstream ss; ss.setf(std::ios::fixed); ss.precision(6); - + ss << "ton.pack.open COUNT : " << open_count.exchange(0, std::memory_order_relaxed) << "\n"; ss << "ton.pack.close COUNT : " << close_count.exchange(0, std::memory_order_relaxed) << "\n"; - + ss << "ton.pack.read.bytes COUNT : " << read_bytes.exchange(0, std::memory_order_relaxed) << "\n"; ss << "ton.pack.write.bytes COUNT : " << write_bytes.exchange(0, std::memory_order_relaxed) << "\n"; @@ -118,7 +118,7 @@ void PackageWriter::append(std::string filename, td::BufferSlice data, return; } start = td::Timestamp::now(); - offset = p->append(std::move(filename), std::move(data), !async_mode_); + offset = p->append(std::move(filename), std::move(data), !async_mode_); end = td::Timestamp::now(); size = p->size(); } @@ -152,6 +152,21 @@ class PackageReader : public td::actor::Actor { std::shared_ptr statistics_; }; +static std::string get_package_file_name(PackageId p_id, ShardIdFull shard_prefix) { + td::StringBuilder sb; + sb << p_id.name(); + if (!shard_prefix.is_masterchain()) { + sb << "."; + sb << shard_prefix.workchain << ":" << shard_to_str(shard_prefix.shard); + } + sb << ".pack"; + return sb.as_cslice().str(); +} + +static std::string package_info_to_str(BlockSeqno seqno, ShardIdFull shard_prefix) { + return PSTRING() << seqno << "." << shard_prefix.workchain << ":" << shard_to_str(shard_prefix.shard); +} + void ArchiveSlice::add_handle(BlockHandle handle, td::Promise promise) { if (destroyed_) { promise.set_error(td::Status::Error(ErrorCode::notready, "package already gc'd")); @@ -271,7 +286,8 @@ void ArchiveSlice::add_file(BlockHandle handle, FileReference ref_id, td::Buffer TRY_RESULT_PROMISE( promise, p, choose_package( - handle ? handle->id().is_masterchain() ? handle->id().seqno() : handle->masterchain_ref_block() : 0, true)); + handle ? handle->id().is_masterchain() ? handle->id().seqno() : handle->masterchain_ref_block() : 0, + handle ? handle->id().shard_full() : ShardIdFull{masterchainId}, true)); std::string value; auto R = kv_->get(ref_id.hash().to_hex(), value); R.ensure(); @@ -376,7 +392,8 @@ void ArchiveSlice::get_file(ConstBlockHandle handle, FileReference ref_id, td::P TRY_RESULT_PROMISE( promise, p, choose_package( - handle ? handle->id().is_masterchain() ? handle->id().seqno() : handle->masterchain_ref_block() : 0, false)); + handle ? handle->id().is_masterchain() ? handle->id().seqno() : handle->masterchain_ref_block() : 0, + handle ? handle->id().shard_full() : ShardIdFull{masterchainId}, false)); promise = begin_async_query(std::move(promise)); auto P = td::PromiseCreator::lambda( [promise = std::move(promise)](td::Result> R) mutable { @@ -536,18 +553,32 @@ void ArchiveSlice::get_slice(td::uint64 archive_id, td::uint64 offset, td::uint3 } before_query(); auto value = static_cast(archive_id >> 32); - TRY_RESULT_PROMISE(promise, p, choose_package(value, false)); + PackageInfo *p; + if (shard_split_depth_ == 0) { + TRY_RESULT_PROMISE_ASSIGN(promise, p, choose_package(value, ShardIdFull{masterchainId}, false)); + } else { + if (value >= packages_.size()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "no such package")); + return; + } + p = &packages_[value]; + } promise = begin_async_query(std::move(promise)); td::actor::create_actor("readfile", p->path, offset, limit, 0, std::move(promise)).release(); } -void ArchiveSlice::get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) { +void ArchiveSlice::get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) { before_query(); if (!sliced_mode_) { promise.set_result(archive_id_); } else { - TRY_RESULT_PROMISE(promise, p, choose_package(masterchain_seqno, false)); - promise.set_result(p->id * (1ull << 32) + archive_id_); + TRY_RESULT_PROMISE(promise, p, choose_package(masterchain_seqno, shard_prefix, false)); + if (shard_split_depth_ == 0) { + promise.set_result(p->seqno * (1ull << 32) + archive_id_); + } else { + promise.set_result(p->idx * (1ull << 32) + archive_id_); + } } } @@ -573,9 +604,18 @@ void ArchiveSlice::before_query() { R2.ensure(); slice_size_ = td::to_integer(value); CHECK(slice_size_ > 0); + R2 = kv_->get("shard_split_depth", value); + R2.ensure(); + if (R2.move_as_ok() == td::KeyValue::GetStatus::Ok) { + shard_split_depth_ = td::to_integer(value); + CHECK(shard_split_depth_ <= 60); + } else { + shard_split_depth_ = 0; + } for (td::uint32 i = 0; i < tot; i++) { R2 = kv_->get(PSTRING() << "status." << i, value); R2.ensure(); + CHECK(R2.move_as_ok() == td::KeyValue::GetStatus::Ok); auto len = td::to_integer(value); R2 = kv_->get(PSTRING() << "version." << i, value); R2.ensure(); @@ -583,12 +623,24 @@ void ArchiveSlice::before_query() { if (R2.move_as_ok() == td::KeyValue::GetStatus::Ok) { ver = td::to_integer(value); } - auto v = archive_id_ + slice_size_ * i; - add_package(v, len, ver); + td::uint32 seqno; + ShardIdFull shard_prefix; + if (shard_split_depth_ == 0) { + seqno = archive_id_ + slice_size_ * i; + shard_prefix = ShardIdFull{masterchainId}; + } else { + R2 = kv_->get(PSTRING() << "info." << i, value); + R2.ensure(); + CHECK(R2.move_as_ok() == td::KeyValue::GetStatus::Ok); + unsigned long long shard; + CHECK(sscanf(value.c_str(), "%u.%d:%016llx", &seqno, &shard_prefix.workchain, &shard) == 3); + shard_prefix.shard = shard; + } + add_package(seqno, shard_prefix, len, ver); } } else { auto len = td::to_integer(value); - add_package(archive_id_, len, 0); + add_package(archive_id_, ShardIdFull{masterchainId}, len, 0); } } else { if (!temp_ && !key_blocks_only_) { @@ -599,13 +651,17 @@ void ArchiveSlice::before_query() { kv_->set("slice_size", td::to_string(slice_size_)).ensure(); kv_->set("status.0", "0").ensure(); kv_->set("version.0", td::to_string(default_package_version())).ensure(); + if (shard_split_depth_ > 0) { + kv_->set("info.0", package_info_to_str(archive_id_, ShardIdFull{masterchainId})).ensure(); + kv_->set("shard_split_depth", td::to_string(shard_split_depth_)).ensure(); + } kv_->commit_transaction().ensure(); - add_package(archive_id_, 0, default_package_version()); + add_package(archive_id_, ShardIdFull{masterchainId}, 0, default_package_version()); } else { kv_->begin_transaction().ensure(); kv_->set("status", "0").ensure(); kv_->commit_transaction().ensure(); - add_package(archive_id_, 0, 0); + add_package(archive_id_, ShardIdFull{masterchainId}, 0, 0); } } } @@ -642,6 +698,7 @@ void ArchiveSlice::do_close() { statistics_.pack_statistics->record_close(packages_.size()); } packages_.clear(); + id_to_package_.clear(); } template @@ -697,48 +754,61 @@ void ArchiveSlice::set_async_mode(bool mode, td::Promise promise) { } } -ArchiveSlice::ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, std::string db_root, +ArchiveSlice::ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, + td::uint32 shard_split_depth, std::string db_root, td::actor::ActorId archive_lru, DbStatistics statistics) : archive_id_(archive_id) , key_blocks_only_(key_blocks_only) , temp_(temp) , finalized_(finalized) , p_id_(archive_id_, key_blocks_only_, temp_) + , shard_split_depth_(temp || key_blocks_only ? 0 : shard_split_depth) , db_root_(std::move(db_root)) , archive_lru_(std::move(archive_lru)) , statistics_(statistics) { db_path_ = PSTRING() << db_root_ << p_id_.path() << p_id_.name() << ".index"; } -td::Result ArchiveSlice::choose_package(BlockSeqno masterchain_seqno, bool force) { +td::Result ArchiveSlice::choose_package(BlockSeqno masterchain_seqno, + ShardIdFull shard_prefix, bool force) { if (temp_ || key_blocks_only_ || !sliced_mode_) { return &packages_[0]; } if (masterchain_seqno < archive_id_) { return td::Status::Error(ErrorCode::notready, "too small masterchain seqno"); } - auto v = (masterchain_seqno - archive_id_) / slice_size_; - if (v >= packages_.size()) { + masterchain_seqno -= (masterchain_seqno - archive_id_) % slice_size_; + CHECK((masterchain_seqno - archive_id_) % slice_size_ == 0); + if (shard_split_depth_ == 0) { + shard_prefix = ShardIdFull{masterchainId}; + } else if (!shard_prefix.is_masterchain()) { + shard_prefix.shard |= 1; // In case length is < split depth + shard_prefix = ton::shard_prefix(shard_prefix, shard_split_depth_); + } + auto it = id_to_package_.find({masterchain_seqno, shard_prefix}); + if (it == id_to_package_.end()) { if (!force) { - return td::Status::Error(ErrorCode::notready, "too big masterchain seqno"); + return td::Status::Error(ErrorCode::notready, "no such package"); } - CHECK(v == packages_.size()); begin_transaction(); + size_t v = packages_.size(); kv_->set("slices", td::to_string(v + 1)).ensure(); kv_->set(PSTRING() << "status." << v, "0").ensure(); kv_->set(PSTRING() << "version." << v, td::to_string(default_package_version())).ensure(); + if (shard_split_depth_ > 0) { + kv_->set(PSTRING() << "info." << v, package_info_to_str(masterchain_seqno, shard_prefix)).ensure(); + } commit_transaction(); - CHECK((masterchain_seqno - archive_id_) % slice_size_ == 0); - add_package(masterchain_seqno, 0, default_package_version()); + add_package(masterchain_seqno, shard_prefix, 0, default_package_version()); return &packages_[v]; } else { - return &packages_[v]; + return &packages_[it->second]; } } -void ArchiveSlice::add_package(td::uint32 seqno, td::uint64 size, td::uint32 version) { +void ArchiveSlice::add_package(td::uint32 seqno, ShardIdFull shard_prefix, td::uint64 size, td::uint32 version) { PackageId p_id{seqno, key_blocks_only_, temp_}; - std::string path = PSTRING() << db_root_ << p_id.path() << p_id.name() << ".pack"; + std::string path = PSTRING() << db_root_ << p_id.path() << get_package_file_name(p_id, shard_prefix); auto R = Package::open(path, false, true); if (R.is_error()) { LOG(FATAL) << "failed to open/create archive '" << path << "': " << R.move_as_error(); @@ -748,8 +818,9 @@ void ArchiveSlice::add_package(td::uint32 seqno, td::uint64 size, td::uint32 ver statistics_.pack_statistics->record_open(); } auto idx = td::narrow_cast(packages_.size()); + id_to_package_[{seqno, shard_prefix}] = idx; if (finalized_) { - packages_.emplace_back(nullptr, td::actor::ActorOwn(), seqno, path, idx, version); + packages_.emplace_back(nullptr, td::actor::ActorOwn(), seqno, shard_prefix, path, idx, version); return; } auto pack = std::make_shared(R.move_as_ok()); @@ -757,7 +828,7 @@ void ArchiveSlice::add_package(td::uint32 seqno, td::uint64 size, td::uint32 ver pack->truncate(size).ensure(); } auto writer = td::actor::create_actor("writer", pack, async_mode_, statistics_.pack_statistics); - packages_.emplace_back(std::move(pack), std::move(writer), seqno, path, idx, version); + packages_.emplace_back(std::move(pack), std::move(writer), seqno, shard_prefix, path, idx, version); } namespace { @@ -790,6 +861,7 @@ void ArchiveSlice::destroy(td::Promise promise) { statistics_.pack_statistics->record_close(packages_.size()); } packages_.clear(); + id_to_package_.clear(); kv_ = nullptr; delay_action([name = db_path_, attempt = 0, @@ -861,7 +933,7 @@ void ArchiveSlice::move_handle(ConstBlockHandle handle, Package *old_pack, Packa move_file(fileref::Block{handle->id()}, old_pack, pack); } -bool ArchiveSlice::truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block_id, td::uint32 cutoff_idx, +bool ArchiveSlice::truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block_id, td::uint32 cutoff_seqno, Package *pack) { std::string value; auto R = kv_->get(get_db_key_block_info(block_id), value); @@ -876,18 +948,18 @@ bool ArchiveSlice::truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block return false; } - auto S = choose_package(seqno, false); + auto S = choose_package(seqno, block_id.shard_full(), false); S.ensure(); auto p = S.move_as_ok(); - CHECK(p->idx <= cutoff_idx); - if (p->idx == cutoff_idx) { + CHECK(p->seqno <= cutoff_seqno); + if (p->seqno == cutoff_seqno) { move_handle(std::move(handle), p->package.get(), pack); } return true; } -void ArchiveSlice::truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shard, td::uint32 cutoff_idx, +void ArchiveSlice::truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shard, td::uint32 cutoff_seqno, Package *pack) { auto key = get_db_key_lt_desc(shard); std::string value; @@ -913,7 +985,7 @@ void ArchiveSlice::truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shar E.ensure(); auto e = E.move_as_ok(); - if (truncate_block(masterchain_seqno, create_block_id(e->id_), cutoff_idx, pack)) { + if (truncate_block(masterchain_seqno, create_block_id(e->id_), cutoff_seqno, pack)) { CHECK(new_last_idx == i); new_last_idx = i + 1; } @@ -925,7 +997,7 @@ void ArchiveSlice::truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shar } } -void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise) { +void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle, td::Promise promise) { if (temp_ || archive_id_ > masterchain_seqno) { destroy(std::move(promise)); return; @@ -938,15 +1010,8 @@ void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handl return; } - auto cutoff = choose_package(masterchain_seqno, false); - cutoff.ensure(); - auto pack = cutoff.move_as_ok(); - CHECK(pack); - - auto pack_r = Package::open(pack->path + ".new", false, true); - pack_r.ensure(); - auto new_package = std::make_shared(pack_r.move_as_ok()); - new_package->truncate(0).ensure(); + std::map old_packages; + std::map> new_packages; std::string value; auto status_key = create_serialize_tl_object(); @@ -967,38 +1032,71 @@ void ArchiveSlice::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handl auto G = fetch_tl_object(value, true); G.ensure(); auto g = G.move_as_ok(); + ShardIdFull shard{g->workchain_, static_cast(g->shard_)}; - truncate_shard(masterchain_seqno, ShardIdFull{g->workchain_, static_cast(g->shard_)}, pack->idx, - new_package.get()); + auto package_r = choose_package(masterchain_seqno, shard, false); + if (package_r.is_error()) { + continue; + } + auto package = package_r.move_as_ok(); + CHECK(package); + if (!old_packages.count(package->shard_prefix)) { + old_packages[package->shard_prefix] = package; + auto new_package_r = Package::open(package->path + ".new", false, true); + new_package_r.ensure(); + auto new_package = std::make_shared(new_package_r.move_as_ok()); + new_package->truncate(0).ensure(); + new_packages[package->shard_prefix] = std::move(new_package); + } + truncate_shard(masterchain_seqno, shard, package->seqno, new_packages[package->shard_prefix].get()); } + for (auto& [shard_prefix, package] : old_packages) { + auto new_package = new_packages[shard_prefix]; + CHECK(new_package); + package->package = new_package; + package->writer.reset(); + td::unlink(package->path).ensure(); + td::rename(package->path + ".new", package->path).ensure(); + package->writer = td::actor::create_actor("writer", new_package, async_mode_); + } + + std::vector new_packages_info; + if (!sliced_mode_) { - kv_->set("status", td::to_string(new_package->size())).ensure(); + kv_->set("status", td::to_string(packages_.at(0).package->size())).ensure(); } else { - kv_->set(PSTRING() << "status." << pack->idx, td::to_string(new_package->size())).ensure(); - for (size_t i = pack->idx + 1; i < packages_.size(); i++) { + for (PackageInfo &package : packages_) { + if (package.seqno <= masterchain_seqno) { + new_packages_info.push_back(std::move(package)); + } else { + td::unlink(package.path).ensure(); + } + } + id_to_package_.clear(); + for (td::uint32 i = 0; i < new_packages_info.size(); ++i) { + PackageInfo &package = new_packages_info[i]; + package.idx = i; + kv_->set(PSTRING() << "status." << i, td::to_string(package.package->size())).ensure(); + kv_->set(PSTRING() << "version." << i, td::to_string(package.version)).ensure(); + if (shard_split_depth_ > 0) { + kv_->set(PSTRING() << "info." << i, package_info_to_str(package.seqno, package.shard_prefix)).ensure(); + } + id_to_package_[{package.seqno, package.shard_prefix}] = i; + } + for (size_t i = new_packages_info.size(); i < packages_.size(); i++) { kv_->erase(PSTRING() << "status." << i); kv_->erase(PSTRING() << "version." << i); + kv_->erase(PSTRING() << "info." << i); } - kv_->set("slices", td::to_string(pack->idx + 1)); + kv_->set("slices", td::to_string(new_packages_info.size())); + if (statistics_.pack_statistics) { + statistics_.pack_statistics->record_close(packages_.size() - new_packages_info.size()); + } + packages_ = std::move(new_packages_info); } - pack->package = new_package; - pack->writer.reset(); - td::unlink(pack->path).ensure(); - td::rename(pack->path + ".new", pack->path).ensure(); - pack->writer = td::actor::create_actor("writer", new_package, async_mode_); - - for (auto idx = pack->idx + 1; idx < packages_.size(); idx++) { - td::unlink(packages_[idx].path).ensure(); - } - if (statistics_.pack_statistics) { - statistics_.pack_statistics->record_close(packages_.size() - pack->idx - 1); - } - packages_.erase(packages_.begin() + pack->idx + 1, packages_.end()); - kv_->commit_transaction().ensure(); - promise.set_value(td::Unit()); } diff --git a/validator/db/archive-slice.hpp b/validator/db/archive-slice.hpp index faec2fb8..a027ec0f 100644 --- a/validator/db/archive-slice.hpp +++ b/validator/db/archive-slice.hpp @@ -96,10 +96,10 @@ class ArchiveLru; class ArchiveSlice : public td::actor::Actor { public: - ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, std::string db_root, - td::actor::ActorId archive_lru, DbStatistics statistics = {}); + ArchiveSlice(td::uint32 archive_id, bool key_blocks_only, bool temp, bool finalized, td::uint32 shard_split_depth, + std::string db_root, td::actor::ActorId archive_lru, DbStatistics statistics = {}); - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise); + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::Promise promise); void add_handle(BlockHandle handle, td::Promise promise); void update_handle(BlockHandle handle, td::Promise promise); @@ -159,6 +159,7 @@ class ArchiveSlice : public td::actor::Actor { bool sliced_mode_{false}; td::uint32 huge_transaction_size_ = 0; td::uint32 slice_size_{100}; + td::uint32 shard_split_depth_ = 0; enum Status { st_closed, st_open, st_want_close @@ -171,28 +172,31 @@ class ArchiveSlice : public td::actor::Actor { std::unique_ptr kv_; struct PackageInfo { - PackageInfo(std::shared_ptr package, td::actor::ActorOwn writer, BlockSeqno id, + PackageInfo(std::shared_ptr package, td::actor::ActorOwn writer, BlockSeqno seqno, ShardIdFull shard_prefix, std::string path, td::uint32 idx, td::uint32 version) : package(std::move(package)) , writer(std ::move(writer)) - , id(id) + , seqno(seqno) + , shard_prefix(shard_prefix) , path(std::move(path)) , idx(idx) , version(version) { } std::shared_ptr package; td::actor::ActorOwn writer; - BlockSeqno id; + BlockSeqno seqno; + ShardIdFull shard_prefix; std::string path; td::uint32 idx; td::uint32 version; }; std::vector packages_; + std::map, td::uint32> id_to_package_; - td::Result choose_package(BlockSeqno masterchain_seqno, bool force); - void add_package(BlockSeqno masterchain_seqno, td::uint64 size, td::uint32 version); - void truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shard, td::uint32 cutoff_idx, Package *pack); - bool truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block_id, td::uint32 cutoff_idx, Package *pack); + td::Result choose_package(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, bool force); + void add_package(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::uint64 size, td::uint32 version); + void truncate_shard(BlockSeqno masterchain_seqno, ShardIdFull shard, td::uint32 cutoff_seqno, Package *pack); + bool truncate_block(BlockSeqno masterchain_seqno, BlockIdExt block_id, td::uint32 cutoff_seqno, Package *pack); void delete_handle(ConstBlockHandle handle); void delete_file(FileReference ref_id); diff --git a/validator/db/archiver.cpp b/validator/db/archiver.cpp index c8cbd7b9..93ba18ca 100644 --- a/validator/db/archiver.cpp +++ b/validator/db/archiver.cpp @@ -25,11 +25,27 @@ namespace ton { namespace validator { BlockArchiver::BlockArchiver(BlockHandle handle, td::actor::ActorId archive_db, - td::Promise promise) - : handle_(std::move(handle)), archive_(archive_db), promise_(std::move(promise)) { + td::actor::ActorId db, td::Promise promise) + : handle_(std::move(handle)), archive_(archive_db), db_(std::move(db)), promise_(std::move(promise)) { } void BlockArchiver::start_up() { + if (handle_->id().is_masterchain()) { + td::actor::send_closure(db_, &Db::get_block_state, handle_, + [SelfId = actor_id(this), archive = archive_](td::Result> R) { + R.ensure(); + td::Ref state{R.move_as_ok()}; + td::uint32 monitor_min_split = state->monitor_min_split_depth(basechainId); + td::actor::send_closure(archive, &ArchiveManager::set_current_shard_split_depth, + monitor_min_split); + td::actor::send_closure(SelfId, &BlockArchiver::move_handle); + }); + } else { + move_handle(); + } +} + +void BlockArchiver::move_handle() { if (handle_->handle_moved_to_archive()) { moved_handle(); } else { diff --git a/validator/db/archiver.hpp b/validator/db/archiver.hpp index 859f269c..9498977f 100644 --- a/validator/db/archiver.hpp +++ b/validator/db/archiver.hpp @@ -33,11 +33,13 @@ class FileDb; class BlockArchiver : public td::actor::Actor { public: - BlockArchiver(BlockHandle handle, td::actor::ActorId archive_db, td::Promise promise); + BlockArchiver(BlockHandle handle, td::actor::ActorId archive_db, td::actor::ActorId db, + td::Promise promise); void abort_query(td::Status error); void start_up() override; + void move_handle(); void moved_handle(); void got_proof(td::BufferSlice data); void written_proof(); @@ -50,6 +52,7 @@ class BlockArchiver : public td::actor::Actor { private: BlockHandle handle_; td::actor::ActorId archive_; + td::actor::ActorId db_; td::Promise promise_; }; diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 9dcecdb3..e86a373d 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -158,6 +158,17 @@ void CellDbIn::start_up() { }, td::Timestamp::now()); } + + { + std::string key = "stats.last_deleted_mc_seqno", value; + auto R = cell_db_->get(td::as_slice(key), value); + R.ensure(); + if (R.ok() == td::KeyValue::GetStatus::Ok) { + auto r_value = td::to_integer_safe(value); + r_value.ensure(); + last_deleted_mc_state_ = r_value.move_as_ok(); + } + } } void CellDbIn::load_cell(RootHash hash, td::Promise> promise) { @@ -452,6 +463,11 @@ void CellDbIn::gc_cont2(BlockHandle handle) { cell_db_->erase(get_key(key_hash)).ensure(); set_block(F.prev, std::move(P)); set_block(F.next, std::move(N)); + if (handle->id().is_masterchain()) { + last_deleted_mc_state_ = handle->id().seqno(); + std::string key = "stats.last_deleted_mc_seqno", value = td::to_string(last_deleted_mc_state_); + cell_db_->set(td::as_slice(key), td::as_slice(value)); + } cell_db_->commit_write_batch().ensure(); alarm_timestamp() = td::Timestamp::now(); timer_write_batch.reset(); @@ -475,9 +491,6 @@ void CellDbIn::gc_cont2(BlockHandle handle) { if (!opts_->get_disable_rocksdb_stats()) { cell_db_statistics_.gc_cell_time_.insert(timer.elapsed() * 1e6); } - if (handle->id().is_masterchain()) { - last_deleted_mc_state_ = handle->id().seqno(); - } LOG(DEBUG) << "Deleted state " << handle->id().to_str(); timer_finish.reset(); timer_all.reset(); diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index bb5d767f..8d83e7a7 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -347,7 +347,8 @@ void RootDb::try_get_static_file(FileHash file_hash, td::Promise promise) { - td::actor::create_actor("archiver", std::move(handle), archive_db_.get(), std::move(promise)) + td::actor::create_actor("archiver", std::move(handle), archive_db_.get(), actor_id(this), + std::move(promise)) .release(); } @@ -421,7 +422,8 @@ void RootDb::start_up() { } void RootDb::archive(BlockHandle handle, td::Promise promise) { - td::actor::create_actor("archiveblock", std::move(handle), archive_db_.get(), std::move(promise)) + td::actor::create_actor("archiveblock", std::move(handle), archive_db_.get(), actor_id(this), + std::move(promise)) .release(); } @@ -436,6 +438,7 @@ void RootDb::allow_block_gc(BlockIdExt block_id, td::Promise promise) { void RootDb::prepare_stats(td::Promise>> promise) { auto merger = StatsMerger::create(std::move(promise)); td::actor::send_closure(cell_db_, &CellDb::prepare_stats, merger.make_promise("celldb.")); + td::actor::send_closure(archive_db_, &ArchiveManager::prepare_stats, merger.make_promise("archive.")); } void RootDb::truncate(BlockSeqno seqno, ConstBlockHandle handle, td::Promise promise) { @@ -501,8 +504,9 @@ void RootDb::check_key_block_proof_link_exists(BlockIdExt block_id, td::Promise< std::move(P)); } -void RootDb::get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) { - td::actor::send_closure(archive_db_, &ArchiveManager::get_archive_id, masterchain_seqno, std::move(promise)); +void RootDb::get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::Promise promise) { + td::actor::send_closure(archive_db_, &ArchiveManager::get_archive_id, masterchain_seqno, shard_prefix, + std::move(promise)); } void RootDb::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, @@ -519,6 +523,14 @@ void RootDb::run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) { td::actor::send_closure(archive_db_, &ArchiveManager::run_gc, mc_ts, gc_ts, archive_ttl); } +void RootDb::add_persistent_state_description(td::Ref desc, td::Promise promise) { + td::actor::send_closure(state_db_, &StateDb::add_persistent_state_description, std::move(desc), std::move(promise)); +} + +void RootDb::get_persistent_state_descriptions(td::Promise>> promise) { + td::actor::send_closure(state_db_, &StateDb::get_persistent_state_descriptions, std::move(promise)); +} + } // namespace validator } // namespace ton diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 755ff257..52f6098e 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -132,12 +132,15 @@ class RootDb : public Db { void check_key_block_proof_exists(BlockIdExt block_id, td::Promise promise) override; void check_key_block_proof_link_exists(BlockIdExt block_id, td::Promise promise) override; - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) override; + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::Promise promise) override; void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise) override; void set_async_mode(bool mode, td::Promise promise) override; void run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) override; + void add_persistent_state_description(td::Ref desc, td::Promise promise) override; + void get_persistent_state_descriptions(td::Promise>> promise) override; + private: td::actor::ActorId validator_manager_; diff --git a/validator/db/statedb.cpp b/validator/db/statedb.cpp index 5d49ae2b..fe3b9d73 100644 --- a/validator/db/statedb.cpp +++ b/validator/db/statedb.cpp @@ -240,6 +240,101 @@ void StateDb::start_up() { } } +void StateDb::add_persistent_state_description(td::Ref desc, + td::Promise promise) { + std::string value; + auto list_key = create_hash_tl_object(); + auto R = kv_->get(list_key.as_slice(), value); + R.ensure(); + tl_object_ptr list; + if (R.ok() == td::KeyValue::GetStatus::Ok) { + auto F = fetch_tl_object(value, true); + F.ensure(); + list = F.move_as_ok(); + } else { + list = create_tl_object( + std::vector>()); + } + for (const auto& obj : list->list_) { + if ((BlockSeqno)obj->masterchain_id_->seqno_ == desc->masterchain_id.seqno()) { + promise.set_error(td::Status::Error("duplicate masterchain seqno")); + return; + } + } + + auto now = (UnixTime)td::Clocks::system(); + size_t new_size = 0; + kv_->begin_write_batch().ensure(); + for (auto& obj : list->list_) { + auto end_time = (UnixTime)obj->end_time_; + if (end_time <= now) { + auto key = + create_hash_tl_object(obj->masterchain_id_->seqno_); + kv_->erase(key.as_slice()).ensure(); + } else { + list->list_[new_size++] = std::move(obj); + } + } + list->list_.resize(new_size); + + std::vector> shard_blocks; + for (const BlockIdExt& block_id : desc->shard_blocks) { + shard_blocks.push_back(create_tl_block_id(block_id)); + } + auto key = + create_hash_tl_object(desc->masterchain_id.seqno()); + kv_->set(key.as_slice(), + create_serialize_tl_object(std::move(shard_blocks)) + .as_slice()) + .ensure(); + + list->list_.push_back(create_tl_object( + create_tl_block_id(desc->masterchain_id), desc->start_time, desc->end_time)); + kv_->set(list_key.as_slice(), serialize_tl_object(list, true).as_slice()).ensure(); + + kv_->commit_write_batch().ensure(); + + promise.set_result(td::Unit()); +} + +void StateDb::get_persistent_state_descriptions(td::Promise>> promise) { + std::string value; + auto R = kv_->get(create_hash_tl_object().as_slice(), value); + R.ensure(); + if (R.ok() == td::KeyValue::GetStatus::NotFound) { + promise.set_value({}); + return; + } + auto F = fetch_tl_object(value, true); + F.ensure(); + std::vector> result; + auto now = (UnixTime)td::Clocks::system(); + for (const auto& obj : F.ok()->list_) { + auto end_time = (UnixTime)obj->end_time_; + if (end_time <= now) { + continue; + } + PersistentStateDescription desc; + desc.start_time = (UnixTime)obj->start_time_; + desc.end_time = end_time; + desc.masterchain_id = create_block_id(obj->masterchain_id_); + auto key = + create_hash_tl_object(desc.masterchain_id.seqno()); + auto R2 = kv_->get(key.as_slice(), value); + R2.ensure(); + if (R2.ok() == td::KeyValue::GetStatus::NotFound) { + continue; + } + auto F2 = fetch_tl_object(value, true); + F2.ensure(); + for (const auto& block_id : F2.ok()->shard_blocks_) { + desc.shard_blocks.push_back(create_block_id(block_id)); + } + result.push_back(td::Ref(true, std::move(desc))); + } + promise.set_result(std::move(result)); +} + void StateDb::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle handle, td::Promise promise) { { auto key = create_hash_tl_object(); diff --git a/validator/db/statedb.hpp b/validator/db/statedb.hpp index a7a00452..fe23898f 100644 --- a/validator/db/statedb.hpp +++ b/validator/db/statedb.hpp @@ -50,6 +50,9 @@ class StateDb : public td::actor::Actor { void update_hardforks(std::vector blocks, td::Promise promise); void get_hardforks(td::Promise> promise); + void add_persistent_state_description(td::Ref desc, td::Promise promise); + void get_persistent_state_descriptions(td::Promise>> promise); + StateDb(td::actor::ActorId root_db, std::string path); void start_up() override; diff --git a/validator/downloaders/download-state.cpp b/validator/downloaders/download-state.cpp index 32978ea5..8473cb22 100644 --- a/validator/downloaders/download-state.cpp +++ b/validator/downloaders/download-state.cpp @@ -38,6 +38,7 @@ DownloadShardState::DownloadShardState(BlockIdExt block_id, BlockIdExt mastercha } void DownloadShardState::start_up() { + status_ = ProcessStatus(manager_, "process.download_state"); alarm_timestamp() = timeout_; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { @@ -81,6 +82,7 @@ void DownloadShardState::download_state() { }); td::actor::send_closure(manager_, &ValidatorManager::send_get_block_proof_link_request, block_id_, priority_, std::move(P)); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading proof"); } void DownloadShardState::downloaded_proof_link(td::BufferSlice data) { @@ -123,6 +125,7 @@ void DownloadShardState::checked_proof_link() { td::actor::send_closure(manager_, &ValidatorManager::send_get_persistent_state_request, block_id_, masterchain_block_id_, priority_, std::move(P)); } + status_.set_status(PSTRING() << block_id_.id.to_str() << " : downloading state"); } void DownloadShardState::download_zero_state() { @@ -152,6 +155,7 @@ void DownloadShardState::downloaded_zero_state(td::BufferSlice data) { } void DownloadShardState::downloaded_shard_state(td::BufferSlice data) { + status_.set_status(PSTRING() << block_id_.id.to_str() << " : processing downloaded state"); auto S = create_shard_state(block_id_, data.clone()); if (S.is_error()) { fail_handler(actor_id(this), S.move_as_error()); @@ -174,6 +178,7 @@ void DownloadShardState::downloaded_shard_state(td::BufferSlice data) { } void DownloadShardState::checked_shard_state() { + status_.set_status(PSTRING() << block_id_.id.to_str() << " : storing state file"); LOG(WARNING) << "checked shard state " << block_id_.to_str(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); @@ -189,6 +194,7 @@ void DownloadShardState::checked_shard_state() { } void DownloadShardState::written_shard_state_file() { + status_.set_status(PSTRING() << block_id_.id.to_str() << " : storing state to celldb"); LOG(WARNING) << "written shard state file " << block_id_.to_str(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); @@ -198,6 +204,7 @@ void DownloadShardState::written_shard_state_file() { } void DownloadShardState::written_shard_state(td::Ref state) { + status_.set_status(PSTRING() << block_id_.id.to_str() << " : finishing"); state_ = std::move(state); handle_->set_unix_time(state_->get_unix_time()); handle_->set_is_key_block(block_id_.is_masterchain()); diff --git a/validator/downloaders/download-state.hpp b/validator/downloaders/download-state.hpp index 02984c53..bde80aae 100644 --- a/validator/downloaders/download-state.hpp +++ b/validator/downloaders/download-state.hpp @@ -19,6 +19,7 @@ #pragma once #include "validator/interfaces/validator-manager.h" +#include "stats-provider.h" namespace ton { @@ -67,6 +68,8 @@ class DownloadShardState : public td::actor::Actor { td::BufferSlice data_; td::Ref state_; + + ProcessStatus status_; }; } // namespace validator diff --git a/validator/downloaders/wait-block-data.cpp b/validator/downloaders/wait-block-data.cpp index 220a8a2c..53a3d351 100644 --- a/validator/downloaders/wait-block-data.cpp +++ b/validator/downloaders/wait-block-data.cpp @@ -106,13 +106,24 @@ void WaitBlockData::start() { }); td::actor::send_closure(manager_, &ValidatorManager::try_get_static_file, handle_->id().file_hash, std::move(P)); + } else if (try_get_candidate_) { + try_get_candidate_ = false; + td::actor::send_closure( + manager_, &ValidatorManager::get_candidate_data_by_block_id_from_db, handle_->id(), + [SelfId = actor_id(this), id = handle_->id()](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &WaitBlockData::start); + } else { + td::actor::send_closure(SelfId, &WaitBlockData::loaded_data, ReceivedBlock{id, R.move_as_ok()}); + } + }); } else { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &WaitBlockData::failed_to_get_block_data_from_net, R.move_as_error_prefix("net error: ")); } else { - td::actor::send_closure(SelfId, &WaitBlockData::got_data_from_net, R.move_as_ok()); + td::actor::send_closure(SelfId, &WaitBlockData::loaded_data, R.move_as_ok()); } }); @@ -137,16 +148,16 @@ void WaitBlockData::failed_to_get_block_data_from_net(td::Status reason) { td::Timestamp::in(0.1)); } -void WaitBlockData::got_data_from_net(ReceivedBlock block) { +void WaitBlockData::loaded_data(ReceivedBlock block) { auto X = create_block(std::move(block)); if (X.is_error()) { failed_to_get_block_data_from_net(X.move_as_error_prefix("bad block from net: ")); return; } - got_block_data_from_net(X.move_as_ok()); + loaded_block_data(X.move_as_ok()); } -void WaitBlockData::got_block_data_from_net(td::Ref block) { +void WaitBlockData::loaded_block_data(td::Ref block) { if (data_.not_null()) { return; } diff --git a/validator/downloaders/wait-block-data.hpp b/validator/downloaders/wait-block-data.hpp index 229b4bfc..f3b367d5 100644 --- a/validator/downloaders/wait-block-data.hpp +++ b/validator/downloaders/wait-block-data.hpp @@ -30,15 +30,16 @@ class ValidatorManager; class WaitBlockData : public td::actor::Actor { public: WaitBlockData(BlockHandle handle, td::uint32 priority, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise> promise) + td::Timestamp timeout, bool try_get_candidate, td::Promise> promise) : handle_(std::move(handle)) , priority_(priority) , manager_(manager) , timeout_(timeout) + , try_get_candidate_(try_get_candidate) , promise_(std::move(promise)) , perf_timer_("waitdata", 1.0, [manager](double duration) { - send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitdata", duration); - }) { + send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitdata", duration); + }) { } void update_timeout(td::Timestamp timeout, td::uint32 priority) { @@ -57,8 +58,8 @@ class WaitBlockData : public td::actor::Actor { void set_is_hardfork(bool value); void start(); void got_block_data_from_db(td::Ref data); - void got_data_from_net(ReceivedBlock data); - void got_block_data_from_net(td::Ref block); + void loaded_data(ReceivedBlock data); + void loaded_block_data(td::Ref block); void checked_proof_link(); void failed_to_get_block_data_from_net(td::Status reason); @@ -73,6 +74,7 @@ class WaitBlockData : public td::actor::Actor { td::actor::ActorId manager_; td::Timestamp timeout_; + bool try_get_candidate_; td::Promise> promise_; td::Ref data_; diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index f8d2cdcb..c80e7d89 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -21,6 +21,7 @@ #include "ton/ton-io.hpp" #include "common/checksum.h" #include "common/delay.h" +#include "validator/downloaders/download-state.hpp" namespace ton { @@ -66,7 +67,8 @@ void WaitBlockState::start() { if (reading_from_db_) { return; } - if (handle_->received_state()) { + bool inited_proof = handle_->id().is_masterchain() ? handle_->inited_proof() : handle_->inited_proof_link(); + if (handle_->received_state() && inited_proof) { reading_from_db_ = true; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { @@ -106,6 +108,19 @@ void WaitBlockState::start() { }); td::actor::send_closure(manager_, &ValidatorManager::send_get_zero_state_request, handle_->id(), priority_, std::move(P)); + } else if (check_persistent_state_desc() && !handle_->received_state()) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + LOG(WARNING) << "failed to get persistent state: " << R.move_as_error(); + td::actor::send_closure(SelfId, &WaitBlockState::start); + } else { + td::actor::send_closure(SelfId, &WaitBlockState::written_state, R.move_as_ok()); + } + }); + BlockIdExt masterchain_id = persistent_state_desc_->masterchain_id; + td::actor::create_actor("downloadstate", handle_->id(), masterchain_id, priority_, manager_, + timeout_, std::move(P)) + .release(); } else if (!handle_->inited_prev() || (!handle_->inited_proof() && !handle_->inited_proof_link())) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle = handle_](td::Result R) { if (R.is_error()) { diff --git a/validator/downloaders/wait-block-state.hpp b/validator/downloaders/wait-block-state.hpp index 4b484ca8..6a14d909 100644 --- a/validator/downloaders/wait-block-state.hpp +++ b/validator/downloaders/wait-block-state.hpp @@ -27,12 +27,14 @@ namespace validator { class WaitBlockState : public td::actor::Actor { public: WaitBlockState(BlockHandle handle, td::uint32 priority, td::actor::ActorId manager, - td::Timestamp timeout, td::Promise> promise) + td::Timestamp timeout, td::Promise> promise, + td::Ref persistent_state_desc = {}) : handle_(std::move(handle)) , priority_(priority) , manager_(manager) , timeout_(timeout) , promise_(std::move(promise)) + , persistent_state_desc_(std::move(persistent_state_desc)) , perf_timer_("waitstate", 1.0, [manager](double duration) { send_closure(manager, &ValidatorManager::add_perf_timer_stat, "waitstate", duration); }) { @@ -90,6 +92,7 @@ class WaitBlockState : public td::actor::Actor { td::actor::ActorId manager_; td::Timestamp timeout_; td::Promise> promise_; + td::Ref persistent_state_desc_; td::Ref prev_state_; td::Ref block_; @@ -99,7 +102,15 @@ class WaitBlockState : public td::actor::Actor { bool waiting_proof_ = false; td::Timestamp next_static_file_attempt_; - td::PerfWarningTimer perf_timer_; + td::PerfWarningTimer perf_timer_{"waitstate", 1.0}; + + bool check_persistent_state_desc() const { + if (persistent_state_desc_.is_null()) { + return false; + } + auto now = (UnixTime)td::Clocks::system(); + return persistent_state_desc_->end_time > now + 3600 && persistent_state_desc_->start_time < now - 6 * 3600; + } }; } // namespace validator diff --git a/validator/full-node-master.cpp b/validator/full-node-master.cpp index f26b1198..da49f0e2 100644 --- a/validator/full-node-master.cpp +++ b/validator/full-node-master.cpp @@ -371,7 +371,8 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise) { - promise.set_value(create_serialize_tl_object(proto_version(), proto_capabilities())); + promise.set_value( + create_serialize_tl_object(proto_version_major(), proto_version_minor(), 0)); } void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, @@ -385,7 +386,7 @@ void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNo } }); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_, - std::move(P)); + ShardIdFull{masterchainId}, std::move(P)); } void FullNodeMasterImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, diff --git a/validator/full-node-master.hpp b/validator/full-node-master.hpp index 5e9d22e1..ce0aedd3 100644 --- a/validator/full-node-master.hpp +++ b/validator/full-node-master.hpp @@ -28,10 +28,10 @@ namespace fullnode { class FullNodeMasterImpl : public FullNodeMaster { public: - static constexpr td::uint32 proto_version() { + static constexpr td::uint32 proto_version_major() { return 1; } - static constexpr td::uint64 proto_capabilities() { + static constexpr td::uint32 proto_version_minor() { return 0; } void start_up() override; diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index e5ea1f0b..f86323fc 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -19,6 +19,9 @@ #include "common/delay.h" #include "common/checksum.h" #include "full-node-serializer.hpp" +#include "auto/tl/ton_api_json.h" +#include "td/utils/JsonBuilder.h" +#include "tl/tl_json.h" namespace ton::validator::fullnode { @@ -85,15 +88,52 @@ void FullNodePrivateBlockOverlay::process_block_candidate_broadcast(PublicKeyHas validator_set_hash, std::move(data)); } +void FullNodePrivateBlockOverlay::process_telemetry_broadcast( + PublicKeyHash src, const tl_object_ptr& telemetry) { + if (telemetry->adnl_id_ != src.bits256_value()) { + VLOG(FULL_NODE_WARNING) << "Invalid telemetry broadcast from " << src << ": adnl_id mismatch"; + return; + } + auto now = (td::int32)td::Clocks::system(); + if (telemetry->timestamp_ < now - 60) { + VLOG(FULL_NODE_WARNING) << "Invalid telemetry broadcast from " << src << ": too old (" + << now - telemetry->timestamp_ << "s ago)"; + return; + } + if (telemetry->timestamp_ > now + 60) { + VLOG(FULL_NODE_WARNING) << "Invalid telemetry broadcast from " << src << ": too new (" + << telemetry->timestamp_ - now << "s in the future)"; + return; + } + VLOG(FULL_NODE_DEBUG) << "Got telemetry broadcast from " << src; + auto s = td::json_encode(td::ToJson(*telemetry), false); + std::erase_if(s, [](char c) { + return c == '\n' || c == '\r'; + }); + telemetry_file_ << s << "\n"; + telemetry_file_.flush(); + if (telemetry_file_.fail()) { + VLOG(FULL_NODE_WARNING) << "Failed to write telemetry to file"; + } +} + void FullNodePrivateBlockOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { if (adnl::AdnlNodeIdShort{src} == local_id_) { return; } auto B = fetch_tl_object(std::move(broadcast), true); if (B.is_error()) { + if (collect_telemetry_ && src != local_id_.pubkey_hash()) { + auto R = fetch_tl_prefix(broadcast, true); + if (R.is_ok()) { + process_telemetry_broadcast(src, R.ok()); + } + } return; } - ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto &obj) { Self->process_broadcast(src, obj); }); + ton_api::downcast_call(*B.move_as_ok(), [src, Self = this](auto& obj) { + Self->process_broadcast(src, obj); + }); } void FullNodePrivateBlockOverlay::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, @@ -144,6 +184,30 @@ void FullNodePrivateBlockOverlay::send_broadcast(BlockBroadcast broadcast) { local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); } +void FullNodePrivateBlockOverlay::send_validator_telemetry(tl_object_ptr telemetry) { + process_telemetry_broadcast(local_id_.pubkey_hash(), telemetry); + auto data = serialize_tl_object(telemetry, true); + if (data.size() <= overlay::Overlays::max_simple_broadcast_size()) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), 0, std::move(data)); + } else { + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), 0, std::move(data)); + } +} + +void FullNodePrivateBlockOverlay::collect_validator_telemetry(std::string filename) { + if (collect_telemetry_) { + telemetry_file_.close(); + } + collect_telemetry_ = true; + LOG(FULL_NODE_WARNING) << "Collecting validator telemetry to " << filename << " (local id: " << local_id_ << ")"; + telemetry_file_.open(filename, std::ios_base::app); + if (!telemetry_file_.is_open()) { + LOG(WARNING) << "Cannot open file " << filename << " for validator telemetry"; + } +} + void FullNodePrivateBlockOverlay::start_up() { std::sort(nodes_.begin(), nodes_.end()); nodes_.erase(std::unique(nodes_.begin(), nodes_.end()), nodes_.end()); @@ -200,8 +264,11 @@ void FullNodePrivateBlockOverlay::init() { overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(), overlay::CertificateFlags::AllowFec | overlay::CertificateFlags::Trusted, {}}; - td::actor::send_closure(overlays_, &overlay::Overlays::create_private_overlay, local_id_, overlay_id_full_.clone(), - nodes_, std::make_unique(actor_id(this)), rules, R"({ "type": "private-blocks" })"); + overlay::OverlayOptions overlay_options; + overlay_options.broadcast_speed_multiplier_ = opts_.private_broadcast_speed_multiplier_; + td::actor::send_closure(overlays_, &overlay::Overlays::create_private_overlay_ex, local_id_, overlay_id_full_.clone(), + nodes_, std::make_unique(actor_id(this)), rules, R"({ "type": "private-blocks" })", + overlay_options); td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_); @@ -302,7 +369,7 @@ void FullNodeCustomOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice } void FullNodeCustomOverlay::send_external_message(td::BufferSlice data) { - if (!inited_ || config_.ext_messages_broadcast_disabled_) { + if (!inited_ || opts_.config_.ext_messages_broadcast_disabled_) { return; } VLOG(FULL_NODE_DEBUG) << "Sending external message to custom overlay \"" << name_ << "\""; @@ -408,10 +475,13 @@ void FullNodeCustomOverlay::init() { authorized_keys[sender.pubkey_hash()] = overlay::Overlays::max_fec_broadcast_size(); } overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(), 0, std::move(authorized_keys)}; + overlay::OverlayOptions overlay_options; + overlay_options.broadcast_speed_multiplier_ = opts_.private_broadcast_speed_multiplier_; td::actor::send_closure( - overlays_, &overlay::Overlays::create_private_overlay, local_id_, overlay_id_full_.clone(), nodes_, + overlays_, &overlay::Overlays::create_private_overlay_ex, local_id_, overlay_id_full_.clone(), nodes_, std::make_unique(actor_id(this)), rules, - PSTRING() << R"({ "type": "custom-overlay", "name": ")" << td::format::Escaped{name_} << R"(" })"); + PSTRING() << R"({ "type": "custom-overlay", "name": ")" << td::format::Escaped{name_} << R"(" })", + overlay_options); td::actor::send_closure(rldp_, &rldp::Rldp::add_id, local_id_); td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, local_id_); diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp index a0022fa0..70e196ea 100644 --- a/validator/full-node-private-overlay.hpp +++ b/validator/full-node-private-overlay.hpp @@ -17,6 +17,7 @@ #pragma once #include "full-node.h" +#include namespace ton::validator::fullnode { @@ -32,6 +33,8 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + void process_telemetry_broadcast(PublicKeyHash src, const tl_object_ptr& telemetry); + template void process_broadcast(PublicKeyHash, T &) { VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast"; @@ -42,16 +45,19 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::BufferSlice data); void send_broadcast(BlockBroadcast broadcast); + void send_validator_telemetry(tl_object_ptr telemetry); + + void collect_validator_telemetry(std::string filename); void set_config(FullNodeConfig config) { - config_ = std::move(config); + opts_.config_ = std::move(config); } void start_up() override; void tear_down() override; FullNodePrivateBlockOverlay(adnl::AdnlNodeIdShort local_id, std::vector nodes, - FileHash zero_state_file_hash, FullNodeConfig config, + FileHash zero_state_file_hash, FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, @@ -60,7 +66,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { : local_id_(local_id) , nodes_(std::move(nodes)) , zero_state_file_hash_(zero_state_file_hash) - , config_(config) + , opts_(opts) , keyring_(keyring) , adnl_(adnl) , rldp_(rldp) @@ -74,7 +80,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { adnl::AdnlNodeIdShort local_id_; std::vector nodes_; FileHash zero_state_file_hash_; - FullNodeConfig config_; + FullNodeOptions opts_; bool enable_compression_ = true; td::actor::ActorId keyring_; @@ -91,6 +97,9 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { void try_init(); void init(); + + bool collect_telemetry_ = false; + std::ofstream telemetry_file_; }; class FullNodeCustomOverlay : public td::actor::Actor { @@ -117,14 +126,14 @@ class FullNodeCustomOverlay : public td::actor::Actor { td::BufferSlice data); void set_config(FullNodeConfig config) { - config_ = std::move(config); + opts_.config_ = std::move(config); } void start_up() override; void tear_down() override; FullNodeCustomOverlay(adnl::AdnlNodeIdShort local_id, CustomOverlayParams params, FileHash zero_state_file_hash, - FullNodeConfig config, td::actor::ActorId keyring, + FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, @@ -135,7 +144,7 @@ class FullNodeCustomOverlay : public td::actor::Actor { , msg_senders_(std::move(params.msg_senders_)) , block_senders_(std::move(params.block_senders_)) , zero_state_file_hash_(zero_state_file_hash) - , config_(config) + , opts_(opts) , keyring_(keyring) , adnl_(adnl) , rldp_(rldp) @@ -152,7 +161,7 @@ class FullNodeCustomOverlay : public td::actor::Actor { std::map msg_senders_; std::set block_senders_; FileHash zero_state_file_hash_; - FullNodeConfig config_; + FullNodeOptions opts_; td::actor::ActorId keyring_; td::actor::ActorId adnl_; diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index fbdbbfd7..ac0eb768 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -20,6 +20,7 @@ #include "checksum.h" #include "overlays.h" #include "td/utils/SharedSlice.h" +#include "td/utils/overloaded.h" #include "full-node-shard.hpp" #include "full-node-shard-queries.hpp" #include "full-node-serializer.hpp" @@ -27,7 +28,6 @@ #include "td/utils/buffer.h" #include "ton/ton-shard.h" #include "ton/ton-tl.hpp" -#include "ton/ton-io.hpp" #include "adnl/utils.hpp" #include "net/download-block-new.hpp" @@ -37,10 +37,14 @@ #include "net/download-proof.hpp" #include "net/get-next-key-blocks.hpp" #include "net/download-archive-slice.hpp" +#include "impl/out-msg-queue-proof.hpp" #include "td/utils/Random.h" #include "common/delay.h" +#include "td/utils/JsonBuilder.h" +#include "tl/tl_json.h" +#include "auto/tl/ton_api_json.h" namespace ton { @@ -50,9 +54,10 @@ namespace fullnode { Neighbour Neighbour::zero = Neighbour{adnl::AdnlNodeIdShort::zero()}; -void Neighbour::update_proto_version(const ton_api::tonNode_capabilities &q) { - proto_version = q.version_; - capabilities = q.capabilities_; +void Neighbour::update_proto_version(ton_api::tonNode_capabilities &q) { + version_major = q.version_major_; + version_minor = q.version_minor_; + flags = q.flags_; } void Neighbour::query_success(double t) { @@ -74,8 +79,9 @@ void Neighbour::update_roundtrip(double t) { void FullNodeShardImpl::create_overlay() { class Callback : public overlay::Overlays::Callback { public: - void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { - // just ignore + void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, + td::BufferSlice data) override { + td::actor::send_closure(node_, &FullNodeShardImpl::receive_message, src, std::move(data)); } void receive_query(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data, td::Promise promise) override { @@ -88,15 +94,23 @@ void FullNodeShardImpl::create_overlay() { td::Promise promise) override { td::actor::send_closure(node_, &FullNodeShardImpl::check_broadcast, src, std::move(data), std::move(promise)); } + void get_stats_extra(td::Promise promise) override { + td::actor::send_closure(node_, &FullNodeShardImpl::get_stats_extra, std::move(promise)); + } Callback(td::actor::ActorId node) : node_(node) { } private: td::actor::ActorId node_; }; - - td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay, adnl_id_, overlay_id_full_.clone(), - std::make_unique(actor_id(this)), rules_, PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() << ", \"workchain_id\": " << get_workchain() << " }"); + overlay::OverlayOptions opts; + opts.announce_self_ = active_; + opts.broadcast_speed_multiplier_ = opts_.public_broadcast_speed_multiplier_; + td::actor::send_closure(overlays_, &overlay::Overlays::create_public_overlay_ex, adnl_id_, overlay_id_full_.clone(), + std::make_unique(actor_id(this)), rules_, + PSTRING() << "{ \"type\": \"shard\", \"shard_id\": " << get_shard() + << ", \"workchain_id\": " << get_workchain() << " }", + opts); td::actor::send_closure(rldp_, &rldp::Rldp::add_id, adnl_id_); td::actor::send_closure(rldp2_, &rldp2::Rldp::add_id, adnl_id_); @@ -106,6 +120,9 @@ void FullNodeShardImpl::create_overlay() { } void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broadcast, td::Promise promise) { + if (!active_) { + return promise.set_error(td::Status::Error("cannot check broadcast: shard is not active")); + } auto B = fetch_tl_object(std::move(broadcast), true); if (B.is_error()) { return promise.set_error(B.move_as_error_prefix("failed to parse external message broadcast: ")); @@ -116,7 +133,7 @@ void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broad if (!processed_ext_msg_broadcasts_.insert(hash).second) { return promise.set_error(td::Status::Error("duplicate external message broadcast")); } - if (config_.ext_messages_broadcast_disabled_) { + if (opts_.config_.ext_messages_broadcast_disabled_) { promise.set_error(td::Status::Error("rebroadcasting external messages is disabled")); promise = [manager = validator_manager_, message = q->message_->data_.clone()](td::Result R) mutable { if (R.is_ok()) { @@ -134,6 +151,10 @@ void FullNodeShardImpl::check_broadcast(PublicKeyHash src, td::BufferSlice broad promise.wrap([](td::Ref) { return td::Unit(); })); } +void FullNodeShardImpl::remove_neighbour(adnl::AdnlNodeIdShort id) { + neighbours_.erase(id); +} + void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) { td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); adnl_id_ = adnl_id; @@ -141,6 +162,18 @@ void FullNodeShardImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promis create_overlay(); } +void FullNodeShardImpl::set_active(bool active) { + if (shard_.is_masterchain()) { + return; + } + if (active_ == active) { + return; + } + active_ = active; + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); + create_overlay(); +} + void FullNodeShardImpl::try_get_next_block(td::Timestamp timeout, td::Promise promise) { if (timeout.is_in_past()) { promise.set_error(td::Status::Error(ErrorCode::timeout, "timeout")); @@ -148,7 +181,7 @@ void FullNodeShardImpl::try_get_next_block(td::Timestamp timeout, td::Promise= 1) { + if (!b.adnl_id.is_zero() && b.version_major >= 1) { VLOG(FULL_NODE_DEBUG) << "using new download method with adnlid=" << b.adnl_id; td::actor::create_actor("downloadnext", adnl_id_, overlay_id_, handle_->id(), b.adnl_id, download_next_priority(), timeout, validator_manager_, rldp_, overlays_, @@ -187,7 +220,6 @@ void FullNodeShardImpl::got_next_block(td::Result R) { } void FullNodeShardImpl::get_next_block() { - //return; attempt_++; auto P = td::PromiseCreator::lambda([validator_manager = validator_manager_, attempt = attempt_, block_id = handle_->id(), SelfId = actor_id(this)](td::Result R) { @@ -591,7 +623,8 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getCapabilities &query, td::Promise promise) { VLOG(FULL_NODE_DEBUG) << "Got query getCapabilities from " << src; - promise.set_value(create_serialize_tl_object(proto_version(), proto_capabilities())); + promise.set_value( + create_serialize_tl_object(proto_version_major(), proto_version_minor(), 0)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, @@ -606,7 +639,24 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod }); VLOG(FULL_NODE_DEBUG) << "Got query getArchiveInfo " << query.masterchain_seqno_ << " from " << src; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_, - std::move(P)); + ShardIdFull{masterchainId}, std::move(P)); +} + +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getShardArchiveInfo &query, + td::Promise promise) { + auto P = td::PromiseCreator::lambda( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_serialize_tl_object()); + } else { + promise.set_value(create_serialize_tl_object(R.move_as_ok())); + } + }); + ShardIdFull shard_prefix = create_shard_id(query.shard_prefix_); + VLOG(FULL_NODE_DEBUG) << "Got query getShardArchiveInfo " << query.masterchain_seqno_ << " " << shard_prefix.to_str() + << " from " << src; + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_archive_id, query.masterchain_seqno_, + shard_prefix, std::move(P)); } void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, @@ -621,8 +671,70 @@ void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNod query.offset_, query.max_size_, std::move(promise)); } +void FullNodeShardImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getOutMsgQueueProof &query, + td::Promise promise) { + std::vector blocks; + for (const auto &x : query.blocks_) { + BlockIdExt id = create_block_id(x); + if (!id.is_valid_ext()) { + promise.set_error(td::Status::Error("invalid block_id")); + return; + } + if (!shard_is_ancestor(shard_, id.shard_full())) { + promise.set_error(td::Status::Error("query in wrong overlay")); + return; + } + blocks.push_back(create_block_id(x)); + } + ShardIdFull dst_shard = create_shard_id(query.dst_shard_); + if (!dst_shard.is_valid_ext()) { + promise.set_error(td::Status::Error("invalid shard")); + return; + } + block::ImportedMsgQueueLimits limits{(td::uint32)query.limits_->max_bytes_, (td::uint32)query.limits_->max_msgs_}; + if (limits.max_msgs > 512) { + promise.set_error(td::Status::Error("max_msgs is too big")); + return; + } + if (limits.max_bytes > (1 << 21)) { + promise.set_error(td::Status::Error("max_bytes is too big")); + return; + } + FLOG(DEBUG) { + sb << "Got query getOutMsgQueueProof to shard " << dst_shard.to_str() << " from blocks"; + for (const BlockIdExt &id : blocks) { + sb << " " << id.id.to_str(); + } + sb << " from " << src; + }; + td::actor::send_closure( + full_node_, &FullNode::get_out_msg_queue_query_token, + [=, manager = validator_manager_, blocks = std::move(blocks), + promise = std::move(promise)](td::Result> R) mutable { + TRY_RESULT_PROMISE(promise, token, std::move(R)); + auto P = + td::PromiseCreator::lambda([promise = std::move(promise), token = std::move(token)]( + td::Result> R) mutable { + if (R.is_error()) { + promise.set_result(create_serialize_tl_object()); + } else { + promise.set_result(serialize_tl_object(R.move_as_ok(), true)); + } + }); + td::actor::create_actor("buildqueueproof", dst_shard, std::move(blocks), limits, manager, + std::move(P)) + .release(); + }); +} + void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise) { + if (!active_) { + td::actor::send_closure(overlays_, &overlay::Overlays::send_message, src, adnl_id_, overlay_id_, + create_serialize_tl_object()); + promise.set_error(td::Status::Error("shard is inactive")); + return; + } auto B = fetch_tl_object(std::move(query), true); if (B.is_error()) { promise.set_error(td::Status::Error(ErrorCode::protoviolation, "cannot parse tonnode query")); @@ -631,6 +743,16 @@ void FullNodeShardImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice ton_api::downcast_call(*B.move_as_ok().get(), [&](auto &obj) { this->process_query(src, obj, std::move(promise)); }); } +void FullNodeShardImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) { + auto B = fetch_tl_object(std::move(data), true); + if (B.is_error()) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Got tonNode.forgetPeer from " << src; + neighbours_.erase(src); + td::actor::send_closure(overlays_, &overlay::Overlays::forget_peer, adnl_id_, overlay_id_, src); +} + void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query) { td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_ihr_message, std::move(query.message_->data_)); @@ -691,11 +813,19 @@ void FullNodeShardImpl::process_block_broadcast(PublicKeyHash src, ton_api::tonN LOG(DEBUG) << "dropped broadcast: " << B.move_as_error(); return; } + //if (!shard_is_ancestor(shard_, block_id.shard_full())) { + // LOG(FULL_NODE_WARNING) << "dropping block broadcast: shard mismatch. overlay=" << shard_.to_str() + // << " block=" << block_id.to_str(); + // return; + //} VLOG(FULL_NODE_DEBUG) << "Received block broadcast from " << src << ": " << B.ok().block_id.to_str(); td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); } void FullNodeShardImpl::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { + if (!active_) { + return; + } auto B = fetch_tl_object(std::move(broadcast), true); if (B.is_error()) { return; @@ -721,7 +851,7 @@ void FullNodeShardImpl::send_ihr_message(td::BufferSlice data) { } void FullNodeShardImpl::send_external_message(td::BufferSlice data) { - if (config_.ext_messages_broadcast_disabled_) { + if (opts_.config_.ext_messages_broadcast_disabled_) { return; } if (!client_.empty()) { @@ -804,7 +934,7 @@ void FullNodeShardImpl::send_broadcast(BlockBroadcast broadcast) { void FullNodeShardImpl::download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) { auto &b = choose_neighbour(); - if (!b.adnl_id.is_zero() && b.proto_version >= 1) { + if (!b.adnl_id.is_zero() && b.version_major >= 1) { VLOG(FULL_NODE_DEBUG) << "new block download"; td::actor::create_actor("downloadreq", id, adnl_id_, overlay_id_, b.adnl_id, priority, timeout, validator_manager_, rldp_, overlays_, adnl_, client_, @@ -863,15 +993,56 @@ void FullNodeShardImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp t .release(); } -void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) { +void FullNodeShardImpl::download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) { auto &b = choose_neighbour(); - td::actor::create_actor("archive", masterchain_seqno, std::move(tmp_dir), adnl_id_, overlay_id_, - b.adnl_id, timeout, validator_manager_, rldp2_, overlays_, adnl_, - client_, create_neighbour_promise(b, std::move(promise))) + td::actor::create_actor( + "archive", masterchain_seqno, shard_prefix, std::move(tmp_dir), adnl_id_, overlay_id_, b.adnl_id, timeout, + validator_manager_, rldp2_, overlays_, adnl_, client_, create_neighbour_promise(b, std::move(promise))) .release(); } +void FullNodeShardImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise) { + // TODO: maybe more complex download (like other requests here) + auto &b = choose_neighbour(); + if (b.adnl_id == adnl::AdnlNodeIdShort::zero()) { + promise.set_error(td::Status::Error(ErrorCode::notready, "no nodes")); + return; + } + std::vector> blocks_tl; + for (const BlockIdExt &id : blocks) { + blocks_tl.push_back(create_tl_block_id(id)); + } + td::BufferSlice query = create_serialize_tl_object( + create_tl_shard_id(dst_shard), std::move(blocks_tl), + create_tl_object(limits.max_bytes, limits.max_msgs)); + + auto P = td::PromiseCreator::lambda( + [=, promise = std::move(promise), blocks = std::move(blocks)](td::Result R) mutable { + if (R.is_error()) { + promise.set_result(R.move_as_error()); + return; + } + TRY_RESULT_PROMISE(promise, f, fetch_tl_object(R.move_as_ok(), true)); + ton_api::downcast_call( + *f, td::overloaded( + [&](ton_api::tonNode_outMsgQueueProofEmpty &x) { + promise.set_error(td::Status::Error("node doesn't have this block")); + }, + [&](ton_api::tonNode_outMsgQueueProof &x) { + delay_action( + [=, promise = std::move(promise), blocks = std::move(blocks), x = std::move(x)]() mutable { + promise.set_result(OutMsgQueueProof::fetch(dst_shard, blocks, limits, x)); + }, + td::Timestamp::now()); + })); + }); + td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, b.adnl_id, adnl_id_, overlay_id_, + "get_msg_queue", std::move(P), timeout, std::move(query), 1 << 22, rldp_); +} + void FullNodeShardImpl::set_handle(BlockHandle handle, td::Promise promise) { CHECK(!handle_); handle_ = std::move(handle); @@ -936,6 +1107,10 @@ void FullNodeShardImpl::start_up() { } } +void FullNodeShardImpl::tear_down() { + td::actor::send_closure(overlays_, &ton::overlay::Overlays::delete_overlay, adnl_id_, overlay_id_); +} + void FullNodeShardImpl::sign_new_certificate(PublicKeyHash sign_by) { if (sign_by.is_zero()) { return; @@ -1083,15 +1258,19 @@ const Neighbour &FullNodeShardImpl::choose_neighbour() const { return Neighbour::zero; } + double min_unreliability = 1e9; + for (auto &x : neighbours_) { + min_unreliability = std::min(min_unreliability, x.second.unreliability); + } const Neighbour *best = nullptr; td::uint32 sum = 0; for (auto &x : neighbours_) { - td::uint32 unr = static_cast(x.second.unreliability); + auto unr = static_cast(x.second.unreliability - min_unreliability); - if (x.second.proto_version < proto_version()) { + if (x.second.version_major < proto_version_major()) { unr += 4; - } else if (x.second.proto_version == proto_version() && x.second.capabilities < proto_capabilities()) { + } else if (x.second.version_major == proto_version_major() && x.second.version_minor < proto_version_minor()) { unr += 2; } @@ -1105,7 +1284,10 @@ const Neighbour &FullNodeShardImpl::choose_neighbour() const { } } } - return best ? *best : Neighbour::zero; + if (best) { + return *best; + } + return Neighbour::zero; } void FullNodeShardImpl::update_neighbour_stats(adnl::AdnlNodeIdShort adnl_id, double t, bool success) { @@ -1128,7 +1310,7 @@ void FullNodeShardImpl::got_neighbour_capabilities(adnl::AdnlNodeIdShort adnl_id if (F.is_error()) { it->second.query_failed(); } else { - it->second.update_proto_version(*F.move_as_ok().get()); + it->second.update_proto_version(*F.ok()); it->second.query_success(t); } } @@ -1157,7 +1339,7 @@ void FullNodeShardImpl::ping_neighbours() { td::Time::now() - start_time, R.move_as_ok()); } }); - auto q = create_serialize_tl_object(); + td::BufferSlice q = create_serialize_tl_object(); td::actor::send_closure(overlays_, &overlay::Overlays::send_query, it->first, adnl_id_, overlay_id_, "get_prepare_block", std::move(P), td::Timestamp::in(1.0), std::move(q)); @@ -1167,14 +1349,32 @@ void FullNodeShardImpl::ping_neighbours() { } } +void FullNodeShardImpl::get_stats_extra(td::Promise promise) { + auto res = create_tl_object(); + res->shard_ = shard_.to_str(); + res->active_ = active_; + for (const auto &p : neighbours_) { + const auto &n = p.second; + auto f = create_tl_object(); + f->id_ = n.adnl_id.bits256_value().to_hex(); + f->verison_major_ = n.version_major; + f->version_minor_ = n.version_minor; + f->flags_ = n.flags; + f->roundtrip_ = n.roundtrip; + f->unreliability_ = n.unreliability; + res->neighbours_.push_back(std::move(f)); + } + promise.set_result(td::json_encode(td::ToJson(*res), true)); +} + FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, FullNodeConfig config, + FileHash zero_state_file_hash, FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, td::actor::ActorId client, - td::actor::ActorId full_node) + td::actor::ActorId full_node, bool active) : shard_(shard) , local_id_(local_id) , adnl_id_(adnl_id) @@ -1187,18 +1387,19 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, , validator_manager_(validator_manager) , client_(client) , full_node_(full_node) - , config_(config) { + , active_(active) + , opts_(opts) { } td::actor::ActorOwn FullNodeShard::create( ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, + FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, td::actor::ActorId full_node) { - return td::actor::create_actor("tonnode", shard, local_id, adnl_id, zero_state_file_hash, config, - keyring, adnl, rldp, rldp2, overlays, validator_manager, client, - full_node); + td::actor::ActorId client, td::actor::ActorId full_node, bool active) { + return td::actor::create_actor(PSTRING() << "tonnode" << shard.to_str(), shard, local_id, adnl_id, + zero_state_file_hash, opts, keyring, adnl, rldp, rldp2, overlays, + validator_manager, client, full_node, active); } } // namespace fullnode diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index e89031fe..5898db80 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -36,6 +36,7 @@ class FullNodeShard : public td::actor::Actor { virtual ShardIdFull get_shard_full() const = 0; virtual void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; + virtual void set_active(bool active) = 0; virtual void set_config(FullNodeConfig config) = 0; virtual void send_ihr_message(td::BufferSlice data) = 0; @@ -45,9 +46,10 @@ class FullNodeShard : public td::actor::Actor { td::BufferSlice data) = 0; virtual void send_broadcast(BlockBroadcast broadcast) = 0; - virtual void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) = 0; - virtual void import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) = 0; - + virtual void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, + td::Promise promise) = 0; + virtual void import_overlay_certificate(PublicKeyHash signed_key, std::shared_ptr cert, + td::Promise promise) = 0; virtual void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; @@ -62,8 +64,11 @@ class FullNodeShard : public td::actor::Actor { td::Promise promise) = 0; virtual void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) = 0; - virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) = 0; + virtual void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) = 0; + virtual void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise) = 0; virtual void set_handle(BlockHandle handle, td::Promise promise) = 0; @@ -71,10 +76,10 @@ class FullNodeShard : public td::actor::Actor { static td::actor::ActorOwn create( ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, + FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, td::actor::ActorId full_node); + td::actor::ActorId client, td::actor::ActorId full_node, bool active); }; } // namespace fullnode diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index a7cf89ac..fb3eef76 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -32,16 +32,17 @@ namespace fullnode { struct Neighbour { adnl::AdnlNodeIdShort adnl_id; - td::uint32 proto_version = 0; - td::uint64 capabilities = 0; + td::uint32 version_major = 0; + td::uint32 version_minor = 0; + td::uint32 flags = 0; double roundtrip = 0; double roundtrip_relax_at = 0; double roundtrip_weight = 0; double unreliability = 0; - Neighbour(adnl::AdnlNodeIdShort adnl_id) : adnl_id(std::move(adnl_id)) { + explicit Neighbour(adnl::AdnlNodeIdShort adnl_id) : adnl_id(std::move(adnl_id)) { } - void update_proto_version(const ton_api::tonNode_capabilities &q); + void update_proto_version(ton_api::tonNode_capabilities &q); void query_success(double t); void query_failed(); void update_roundtrip(double t); @@ -64,12 +65,12 @@ class FullNodeShardImpl : public FullNodeShard { static constexpr td::uint32 download_next_priority() { return 1; } - static constexpr td::uint32 proto_version() { - return 2; - } - static constexpr td::uint64 proto_capabilities() { + static constexpr td::uint32 proto_version_major() { return 3; } + static constexpr td::uint32 proto_version_minor() { + return 0; + } static constexpr td::uint32 max_neighbours() { return 16; } @@ -82,14 +83,12 @@ class FullNodeShardImpl : public FullNodeShard { void create_overlay(); void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; + void set_active(bool active) override; void set_config(FullNodeConfig config) override { - config_ = config; + opts_.config_ = config; } - //td::Result fetch_block(td::BufferSlice data); - void prevalidate_block(BlockIdExt block_id, td::BufferSlice data, td::BufferSlice proof, - td::Promise promise); void try_get_next_block(td::Timestamp timestamp, td::Promise promise); void got_next_block(td::Result block); void get_next_block(); @@ -136,11 +135,14 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveInfo &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getShardArchiveInfo &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getArchiveSlice &query, td::Promise promise); - // void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_prepareNextKeyBlockProof &query, - // td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::tonNode_getOutMsgQueueProof &query, + td::Promise promise); void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice query, td::Promise promise); + void receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); @@ -156,6 +158,8 @@ class FullNodeShardImpl : public FullNodeShard { void receive_broadcast(PublicKeyHash src, td::BufferSlice query); void check_broadcast(PublicKeyHash src, td::BufferSlice query, td::Promise promise); + void get_stats_extra(td::Promise promise); + void remove_neighbour(adnl::AdnlNodeIdShort id); void send_ihr_message(td::BufferSlice data) override; void send_external_message(td::BufferSlice data) override; @@ -177,12 +181,16 @@ class FullNodeShardImpl : public FullNodeShard { td::Promise promise) override; void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) override; - void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) override; + void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override; + void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise) override; void set_handle(BlockHandle handle, td::Promise promise) override; void start_up() override; + void tear_down() override; void alarm() override; void update_validators(std::vector public_key_hashes, PublicKeyHash local_hash) override; @@ -214,11 +222,12 @@ class FullNodeShardImpl : public FullNodeShard { } FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, + FileHash zero_state_file_hash, FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, td::actor::ActorId full_node); + td::actor::ActorId client, td::actor::ActorId full_node, + bool active); private: bool use_new_download() const { @@ -258,7 +267,9 @@ class FullNodeShardImpl : public FullNodeShard { td::Timestamp ping_neighbours_at_; adnl::AdnlNodeIdShort last_pinged_neighbour_ = adnl::AdnlNodeIdShort::zero(); - FullNodeConfig config_; + bool active_; + + FullNodeOptions opts_; std::set my_ext_msg_broadcasts_; std::set processed_ext_msg_broadcasts_; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index bb6da2c8..e1951c36 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -17,10 +17,13 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "full-node.hpp" -#include "ton/ton-shard.h" #include "ton/ton-io.hpp" #include "td/actor/MultiPromise.h" #include "full-node.h" +#include "common/delay.h" +#include "impl/out-msg-queue-proof.hpp" +#include "td/utils/Random.h" +#include "ton/ton-tl.hpp" namespace ton { @@ -28,6 +31,8 @@ namespace validator { namespace fullnode { +static const double INACTIVE_SHARD_TTL = (double)overlay::Overlays::overlay_peer_ttl() + 60.0; + void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise promise) { if (local_keys_.count(key)) { promise.set_value(td::Unit()); @@ -52,7 +57,9 @@ void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise pr } for (auto &shard : shards_) { - td::actor::send_closure(shard.second, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + if (!shard.second.actor.empty()) { + td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + } } promise.set_value(td::Unit()); } @@ -64,6 +71,7 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise pr } local_keys_.erase(key); private_block_overlays_.erase(key); + update_validator_telemetry_collector(); for (auto &p : custom_overlays_) { update_custom_overlay(p.second); } @@ -81,30 +89,34 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise pr } for (auto &shard : shards_) { - td::actor::send_closure(shard.second, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + if (!shard.second.actor.empty()) { + td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + } } promise.set_value(td::Unit()); } -void FullNodeImpl::sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, - td::uint32 expiry_at, td::uint32 max_size, - td::Promise promise) { - auto it = shards_.find(shard_id); - if(it == shards_.end()) { - promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); - return; - } - td::actor::send_closure(it->second, &FullNodeShard::sign_overlay_certificate, signed_key, expiry_at, max_size, std::move(promise)); +void FullNodeImpl::sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, + td::uint32 max_size, td::Promise promise) { + auto it = shards_.find(shard_id); + if(it == shards_.end() || it->second.actor.empty()) { + promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); + return; + } + td::actor::send_closure(it->second.actor, &FullNodeShard::sign_overlay_certificate, signed_key, expiry_at, max_size, + std::move(promise)); } void FullNodeImpl::import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) { - auto it = shards_.find(shard_id); - if(it == shards_.end()) { - promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); - } - td::actor::send_closure(it->second, &FullNodeShard::import_overlay_certificate, signed_key, cert, std::move(promise)); + auto it = shards_.find(shard_id); + if(it == shards_.end() || it->second.actor.empty()) { + promise.set_error(td::Status::Error(ErrorCode::error, "shard not found")); + return; + } + td::actor::send_closure(it->second.actor, &FullNodeShard::import_overlay_certificate, signed_key, cert, + std::move(promise)); } void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) { @@ -115,7 +127,9 @@ void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promisesecond, &FullNodeShard::set_handle, top_handle, std::move(P)); + CHECK(it != shards_.end() && !it->second.actor.empty()); + td::actor::send_closure(it->second.actor, &FullNodeShard::set_handle, top_handle, std::move(P)); } -void FullNodeImpl::add_shard(ShardIdFull shard) { - while (true) { - if (shards_.count(shard) == 0) { - shards_.emplace(shard, - FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, config_, keyring_, adnl_, - rldp_, rldp2_, overlays_, validator_manager_, client_, actor_id(this))); - if (all_validators_.size() > 0) { - td::actor::send_closure(shards_[shard], &FullNodeShard::update_validators, all_validators_, sign_cert_by_); +void FullNodeImpl::on_new_masterchain_block(td::Ref state, std::set shards_to_monitor) { + CHECK(shards_to_monitor.count(ShardIdFull(masterchainId))); + bool join_all_overlays = !sign_cert_by_.is_zero(); + std::set all_shards; + std::set new_active; + all_shards.insert(ShardIdFull(masterchainId)); + std::set workchains; + wc_monitor_min_split_ = state->monitor_min_split_depth(basechainId); + auto cut_shard = [&](ShardIdFull shard) -> ShardIdFull { + return wc_monitor_min_split_ < shard.pfx_len() ? shard_prefix(shard, wc_monitor_min_split_) : shard; + }; + for (auto &info : state->get_shards()) { + workchains.insert(info->shard().workchain); + ShardIdFull shard = cut_shard(info->shard()); + while (true) { + all_shards.insert(shard); + if (shard.pfx_len() == 0) { + break; } + shard = shard_parent(shard); + } + } + for (const auto &[wc, winfo] : state->get_workchain_list()) { + if (!workchains.contains(wc) && winfo->active && winfo->enabled_since <= state->get_unix_time()) { + all_shards.insert(ShardIdFull(wc)); + } + } + for (ShardIdFull shard : shards_to_monitor) { + shard = cut_shard(shard); + while (true) { + new_active.insert(shard); + if (shard.pfx_len() == 0) { + break; + } + shard = shard_parent(shard); + } + } + + for (auto it = shards_.begin(); it != shards_.end(); ) { + if (all_shards.contains(it->first)) { + ++it; } else { - break; + it = shards_.erase(it); } - if (shard.shard == shardIdAll) { - break; + } + for (ShardIdFull shard : all_shards) { + bool active = new_active.contains(shard); + bool overlay_exists = !shards_[shard].actor.empty(); + if (active || join_all_overlays || overlay_exists) { + update_shard_actor(shard, active); + } + } + + for (auto &[_, shard_info] : shards_) { + if (!shard_info.active && shard_info.delete_at && shard_info.delete_at.is_in_past() && !join_all_overlays) { + shard_info.actor = {}; + shard_info.delete_at = td::Timestamp::never(); } - shard = shard_parent(shard); } } -void FullNodeImpl::del_shard(ShardIdFull shard) { - LOG(FATAL) << "deleting shards not implemented: shard=" << shard; - shards_.erase(shard); +void FullNodeImpl::update_shard_actor(ShardIdFull shard, bool active) { + ShardInfo &info = shards_[shard]; + if (info.actor.empty()) { + info.actor = FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, opts_, keyring_, adnl_, rldp_, + rldp2_, overlays_, validator_manager_, client_, actor_id(this), active); + if (!all_validators_.empty()) { + td::actor::send_closure(info.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + } + } else if (info.active != active) { + td::actor::send_closure(info.actor, &FullNodeShard::set_active, active); + } + info.active = active; + info.delete_at = active ? td::Timestamp::never() : td::Timestamp::in(INACTIVE_SHARD_TTL); } void FullNodeImpl::sync_completed() { @@ -205,7 +273,7 @@ void FullNodeImpl::sync_completed() { } void FullNodeImpl::send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) { - auto shard = get_shard(ShardIdFull{masterchainId}); + auto shard = get_shard(dst); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping OUT ihr message to unknown shard"; return; @@ -219,11 +287,12 @@ void FullNodeImpl::send_ext_message(AccountIdPrefixFull dst, td::BufferSlice dat VLOG(FULL_NODE_WARNING) << "dropping OUT ext message to unknown shard"; return; } - for (auto &private_overlay : custom_overlays_) { - for (auto &actor : private_overlay.second.actors_) { - auto local_id = actor.first; - if (private_overlay.second.params_.msg_senders_.count(local_id)) { - td::actor::send_closure(actor.second, &FullNodeCustomOverlay::send_external_message, data.clone()); + for (auto &[_, private_overlay] : custom_overlays_) { + if (private_overlay.params_.send_shard(dst.as_leaf_shard())) { + for (auto &[local_id, actor] : private_overlay.actors_) { + if (private_overlay.params_.msg_senders_.contains(local_id)) { + td::actor::send_closure(actor, &FullNodeCustomOverlay::send_external_message, data.clone()); + } } } } @@ -231,7 +300,7 @@ void FullNodeImpl::send_ext_message(AccountIdPrefixFull dst, td::BufferSlice dat } void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { - auto shard = get_shard(ShardIdFull{masterchainId, shardIdAll}); + auto shard = get_shard(ShardIdFull{masterchainId}); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; return; @@ -265,14 +334,16 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, int mode) { if (mode & broadcast_mode_custom) { send_block_broadcast_to_custom_overlays(broadcast); } - auto shard = get_shard(ShardIdFull{masterchainId}); + auto shard = get_shard(broadcast.block_id.shard_full()); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; return; } - if (!private_block_overlays_.empty() && (mode & broadcast_mode_private_block)) { - td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast, - broadcast.clone()); + if (mode & broadcast_mode_private_block) { + if (!private_block_overlays_.empty()) { + td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast, + broadcast.clone()); + } } if (mode & broadcast_mode_public) { td::actor::send_closure(shard, &FullNodeShard::send_broadcast, std::move(broadcast)); @@ -347,27 +418,61 @@ void FullNodeImpl::get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeou td::actor::send_closure(shard, &FullNodeShard::get_next_key_blocks, block_id, timeout, std::move(promise)); } -void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) { - auto shard = get_shard(ShardIdFull{masterchainId}); +void FullNodeImpl::download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) { + auto shard = get_shard(shard_prefix); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping download archive query to unknown shard"; + promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); + return; + } CHECK(!shard.empty()); - td::actor::send_closure(shard, &FullNodeShard::download_archive, masterchain_seqno, std::move(tmp_dir), timeout, - std::move(promise)); + td::actor::send_closure(shard, &FullNodeShard::download_archive, masterchain_seqno, shard_prefix, std::move(tmp_dir), + timeout, std::move(promise)); +} + +void FullNodeImpl::download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise) { + if (blocks.empty()) { + promise.set_value({}); + return; + } + // All blocks are expected to have the same minsplit shard prefix + auto shard = get_shard(blocks[0].shard_full()); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping download msg queue query to unknown shard"; + promise.set_error(td::Status::Error(ErrorCode::notready, "shard not ready")); + return; + } + td::actor::send_closure(shard, &FullNodeShard::download_out_msg_queue_proof, dst_shard, std::move(blocks), limits, + timeout, std::move(promise)); } td::actor::ActorId FullNodeImpl::get_shard(ShardIdFull shard) { - add_shard(ShardIdFull{shard.workchain, shardIdAll}); - while (shards_.count(shard) == 0) { - if (shard.shard == shardIdAll) { - return td::actor::ActorId{}; - } - shard = shard_parent(shard); + if (shard.is_masterchain()) { + return shards_[ShardIdFull{masterchainId}].actor.get(); } - return shards_[shard].get(); + if (shard.workchain != basechainId) { + return {}; + } + int pfx_len = shard.pfx_len(); + if (pfx_len > wc_monitor_min_split_) { + shard = shard_prefix(shard, wc_monitor_min_split_); + } + auto it = shards_.find(shard); + if (it != shards_.end()) { + update_shard_actor(shard, it->second.active); + return it->second.actor.get(); + } + + // Special case if shards_ was not yet initialized. + // This can happen briefly on node startup. + return shards_[ShardIdFull{masterchainId}].actor.get(); } td::actor::ActorId FullNodeImpl::get_shard(AccountIdPrefixFull dst) { - return get_shard(shard_prefix(dst, 60)); + return get_shard(shard_prefix(dst, max_shard_pfx_len)); } void FullNodeImpl::got_key_block_config(td::Ref config) { @@ -406,7 +511,9 @@ void FullNodeImpl::got_key_block_config(td::Ref config) { CHECK(all_validators_.size() > 0); for (auto &shard : shards_) { - td::actor::send_closure(shard.second, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + if (!shard.second.actor.empty()) { + td::actor::send_closure(shard.second.actor, &FullNodeShard::update_validators, all_validators_, sign_cert_by_); + } } } @@ -438,6 +545,15 @@ void FullNodeImpl::new_key_block(BlockHandle handle) { } } +void FullNodeImpl::send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) { + auto it = private_block_overlays_.find(key); + if (it == private_block_overlays_.end()) { + VLOG(FULL_NODE_INFO) << "Cannot send validator telemetry for " << key << " : no private block overlay"; + return; + } + td::actor::send_closure(it->second, &FullNodePrivateBlockOverlay::send_validator_telemetry, std::move(telemetry)); +} + void FullNodeImpl::process_block_broadcast(BlockBroadcast broadcast) { send_block_broadcast_to_custom_overlays(broadcast); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(broadcast), @@ -460,10 +576,33 @@ void FullNodeImpl::process_block_candidate_broadcast(BlockIdExt block_id, Catcha std::move(data)); } +void FullNodeImpl::get_out_msg_queue_query_token(td::Promise> promise) { + td::actor::send_closure(out_msg_queue_query_token_manager_, &TokenManager::get_token, 1, 0, td::Timestamp::in(10.0), + std::move(promise)); +} + +void FullNodeImpl::set_validator_telemetry_filename(std::string value) { + validator_telemetry_filename_ = std::move(value); + update_validator_telemetry_collector(); +} + +void FullNodeImpl::update_validator_telemetry_collector() { + if (validator_telemetry_filename_.empty() || private_block_overlays_.empty()) { + validator_telemetry_collector_key_ = PublicKeyHash::zero(); + return; + } + if (!private_block_overlays_.contains(validator_telemetry_collector_key_)) { + auto it = private_block_overlays_.begin(); + validator_telemetry_collector_key_ = it->first; + td::actor::send_closure(it->second, &FullNodePrivateBlockOverlay::collect_validator_telemetry, + validator_telemetry_filename_); + } +} + void FullNodeImpl::start_up() { - add_shard(ShardIdFull{masterchainId}); + update_shard_actor(ShardIdFull{masterchainId}, true); if (local_id_.is_zero()) { - if(adnl_id_.is_zero()) { + if (adnl_id_.is_zero()) { auto pk = ton::PrivateKey{ton::privkeys::Ed25519::random()}; local_id_ = pk.compute_short_id(); @@ -477,11 +616,9 @@ void FullNodeImpl::start_up() { void initial_read_complete(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::initial_read_complete, handle); } - void add_shard(ShardIdFull shard) override { - td::actor::send_closure(id_, &FullNodeImpl::add_shard, shard); - } - void del_shard(ShardIdFull shard) override { - td::actor::send_closure(id_, &FullNodeImpl::del_shard, shard); + void on_new_masterchain_block(td::Ref state, std::set shards_to_monitor) override { + td::actor::send_closure(id_, &FullNodeImpl::on_new_masterchain_block, std::move(state), + std::move(shards_to_monitor)); } void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) override { td::actor::send_closure(id_, &FullNodeImpl::send_ihr_message, dst, std::move(data)); @@ -527,26 +664,34 @@ void FullNodeImpl::start_up() { td::Promise> promise) override { td::actor::send_closure(id_, &FullNodeImpl::get_next_key_blocks, block_id, timeout, std::move(promise)); } - void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) override { - td::actor::send_closure(id_, &FullNodeImpl::download_archive, masterchain_seqno, std::move(tmp_dir), timeout, - std::move(promise)); + void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override { + td::actor::send_closure(id_, &FullNodeImpl::download_archive, masterchain_seqno, shard_prefix, std::move(tmp_dir), + timeout, std::move(promise)); + } + void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise) override { + td::actor::send_closure(id_, &FullNodeImpl::download_out_msg_queue_proof, dst_shard, std::move(blocks), limits, + timeout, std::move(promise)); } void new_key_block(BlockHandle handle) override { td::actor::send_closure(id_, &FullNodeImpl::new_key_block, std::move(handle)); } + void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override { + td::actor::send_closure(id_, &FullNodeImpl::send_validator_telemetry, key, std::move(telemetry)); + } - Callback(td::actor::ActorId id) : id_(id) { + explicit Callback(td::actor::ActorId id) : id_(id) { } private: td::actor::ActorId id_; }; - auto P = td::PromiseCreator::lambda([](td::Unit R) {}); td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::install_callback, - std::make_unique(actor_id(this)), std::move(P)); + std::make_unique(actor_id(this)), std::move(started_promise_)); } void FullNodeImpl::update_private_overlays() { @@ -555,6 +700,7 @@ void FullNodeImpl::update_private_overlays() { } private_block_overlays_.clear(); + update_validator_telemetry_collector(); if (local_keys_.empty()) { return; } @@ -571,8 +717,9 @@ void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { nodes.push_back(p.second); } private_block_overlays_[key] = td::actor::create_actor( - "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_, keyring_, + "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, opts_, keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_, actor_id(this)); + update_validator_telemetry_collector(); } } @@ -588,7 +735,7 @@ void FullNodeImpl::update_custom_overlay(CustomOverlayInfo &overlay) { old_actors.erase(it); } else { overlay.actors_[local_id] = td::actor::create_actor( - "CustomOverlay", local_id, params, zero_state_file_hash_, config_, keyring_, adnl_, rldp_, rldp2_, + "CustomOverlay", local_id, params, zero_state_file_hash_, opts_, keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_, actor_id(this)); } } @@ -602,7 +749,7 @@ void FullNodeImpl::update_custom_overlay(CustomOverlayInfo &overlay) { } } -void FullNodeImpl::send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast) { +void FullNodeImpl::send_block_broadcast_to_custom_overlays(const BlockBroadcast &broadcast) { if (!custom_overlays_sent_broadcasts_.insert(broadcast.block_id).second) { return; } @@ -611,11 +758,12 @@ void FullNodeImpl::send_block_broadcast_to_custom_overlays(const BlockBroadcast& custom_overlays_sent_broadcasts_.erase(custom_overlays_sent_broadcasts_lru_.front()); custom_overlays_sent_broadcasts_lru_.pop(); } - for (auto &private_overlay : custom_overlays_) { - for (auto &actor : private_overlay.second.actors_) { - auto local_id = actor.first; - if (private_overlay.second.params_.block_senders_.count(local_id)) { - td::actor::send_closure(actor.second, &FullNodeCustomOverlay::send_broadcast, broadcast.clone()); + for (auto &[_, private_overlay] : custom_overlays_) { + if (private_overlay.params_.send_shard(broadcast.block_id.shard_full())) { + for (auto &[local_id, actor] : private_overlay.actors_) { + if (private_overlay.params_.block_senders_.contains(local_id)) { + td::actor::send_closure(actor, &FullNodeCustomOverlay::send_broadcast, broadcast.clone()); + } } } } @@ -633,24 +781,26 @@ void FullNodeImpl::send_block_candidate_broadcast_to_custom_overlays(const Block custom_overlays_sent_broadcasts_.erase(custom_overlays_sent_broadcasts_lru_.front()); custom_overlays_sent_broadcasts_lru_.pop(); } - for (auto &private_overlay : custom_overlays_) { - for (auto &actor : private_overlay.second.actors_) { - auto local_id = actor.first; - if (private_overlay.second.params_.block_senders_.count(local_id)) { - td::actor::send_closure(actor.second, &FullNodeCustomOverlay::send_block_candidate, block_id, cc_seqno, - validator_set_hash, data.clone()); + for (auto &[_, private_overlay] : custom_overlays_) { + if (private_overlay.params_.send_shard(block_id.shard_full())) { + for (auto &[local_id, actor] : private_overlay.actors_) { + if (private_overlay.params_.block_senders_.contains(local_id)) { + td::actor::send_closure(actor, &FullNodeCustomOverlay::send_block_candidate, block_id, cc_seqno, + validator_set_hash, data.clone()); + } } } } } FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - FullNodeConfig config, td::actor::ActorId keyring, + FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId dht, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, std::string db_root) + td::actor::ActorId client, std::string db_root, + td::Promise started_promise) : local_id_(local_id) , adnl_id_(adnl_id) , zero_state_file_hash_(zero_state_file_hash) @@ -663,19 +813,19 @@ FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id , validator_manager_(validator_manager) , client_(client) , db_root_(db_root) - , config_(config) { + , started_promise_(std::move(started_promise)) + , opts_(opts) { } -td::actor::ActorOwn FullNode::create(ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, FullNodeConfig config, - td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId rldp, - td::actor::ActorId rldp2, td::actor::ActorId dht, - td::actor::ActorId overlays, - td::actor::ActorId validator_manager, - td::actor::ActorId client, std::string db_root) { - return td::actor::create_actor("fullnode", local_id, adnl_id, zero_state_file_hash, config, keyring, - adnl, rldp, rldp2, dht, overlays, validator_manager, client, db_root); +td::actor::ActorOwn FullNode::create( + ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeOptions opts, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId dht, + td::actor::ActorId overlays, td::actor::ActorId validator_manager, + td::actor::ActorId client, std::string db_root, td::Promise started_promise) { + return td::actor::create_actor("fullnode", local_id, adnl_id, zero_state_file_hash, opts, keyring, + adnl, rldp, rldp2, dht, overlays, validator_manager, client, db_root, + std::move(started_promise)); } FullNodeConfig::FullNodeConfig(const tl_object_ptr &obj) @@ -692,18 +842,27 @@ bool FullNodeConfig::operator!=(const FullNodeConfig &rhs) const { return !(*this == rhs); } +bool CustomOverlayParams::send_shard(const ShardIdFull &shard) const { + return sender_shards_.empty() || + std::any_of(sender_shards_.begin(), sender_shards_.end(), + [&](const ShardIdFull &our_shard) { return shard_intersects(shard, our_shard); }); +} + CustomOverlayParams CustomOverlayParams::fetch(const ton_api::engine_validator_customOverlay& f) { CustomOverlayParams c; c.name_ = f.name_; for (const auto &node : f.nodes_) { c.nodes_.emplace_back(node->adnl_id_); if (node->msg_sender_) { - c.msg_senders_[ton::adnl::AdnlNodeIdShort{node->adnl_id_}] = node->msg_sender_priority_; + c.msg_senders_[adnl::AdnlNodeIdShort{node->adnl_id_}] = node->msg_sender_priority_; } if (node->block_sender_) { c.block_senders_.emplace(node->adnl_id_); } } + for (const auto &shard : f.sender_shards_) { + c.sender_shards_.push_back(create_shard_id(shard)); + } return c; } diff --git a/validator/full-node.h b/validator/full-node.h index 621cdac0..555082dc 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -55,12 +55,20 @@ struct FullNodeConfig { bool ext_messages_broadcast_disabled_ = false; }; +struct FullNodeOptions { + FullNodeConfig config_; + double public_broadcast_speed_multiplier_ = 1.0; + double private_broadcast_speed_multiplier_ = 1.0; +}; + struct CustomOverlayParams { std::string name_; std::vector nodes_; std::map msg_senders_; std::set block_senders_; + std::vector sender_shards_; + bool send_shard(const ShardIdFull& shard) const; static CustomOverlayParams fetch(const ton_api::engine_validator_customOverlay& f); }; @@ -89,6 +97,9 @@ class FullNode : public td::actor::Actor { virtual void process_block_broadcast(BlockBroadcast broadcast) = 0; virtual void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::BufferSlice data) = 0; + virtual void get_out_msg_queue_query_token(td::Promise> promise) = 0; + + virtual void set_validator_telemetry_filename(std::string value) = 0; static constexpr td::uint32 max_block_size() { return 4 << 20; @@ -101,14 +112,12 @@ class FullNode : public td::actor::Actor { } enum { broadcast_mode_public = 1, broadcast_mode_private_block = 2, broadcast_mode_custom = 4 }; - static td::actor::ActorOwn create(ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, - FileHash zero_state_file_hash, FullNodeConfig config, - td::actor::ActorId keyring, - td::actor::ActorId adnl, td::actor::ActorId rldp, - td::actor::ActorId rldp2, td::actor::ActorId dht, - td::actor::ActorId overlays, - td::actor::ActorId validator_manager, - td::actor::ActorId client, std::string db_root); + static td::actor::ActorOwn create( + ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeOptions opts, + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId dht, + td::actor::ActorId overlays, td::actor::ActorId validator_manager, + td::actor::ActorId client, std::string db_root, td::Promise started_promise); }; } // namespace fullnode diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 584be5ee..b4c79363 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -28,6 +28,7 @@ #include #include #include +#include namespace ton { @@ -44,9 +45,8 @@ class FullNodeImpl : public FullNode { void add_permanent_key(PublicKeyHash key, td::Promise promise) override; void del_permanent_key(PublicKeyHash key, td::Promise promise) override; - void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, - td::uint32 expiry_at, td::uint32 max_size, - td::Promise promise) override; + void sign_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, td::uint32 expiry_at, + td::uint32 max_size, td::Promise promise) override; void import_shard_overlay_certificate(ShardIdFull shard_id, PublicKeyHash signed_key, std::shared_ptr cert, td::Promise promise) override; @@ -57,8 +57,7 @@ class FullNodeImpl : public FullNode { void add_custom_overlay(CustomOverlayParams params, td::Promise promise) override; void del_custom_overlay(std::string name, td::Promise promise) override; - void add_shard(ShardIdFull shard); - void del_shard(ShardIdFull shard); + void on_new_masterchain_block(td::Ref state, std::set shards_to_monitor); void sync_completed(); @@ -79,34 +78,50 @@ class FullNodeImpl : public FullNode { void download_block_proof_link(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise); - void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise); + void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise); + void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise); void got_key_block_config(td::Ref config); void new_key_block(BlockHandle handle); + void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry); void process_block_broadcast(BlockBroadcast broadcast) override; void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, td::BufferSlice data) override; + void get_out_msg_queue_query_token(td::Promise> promise) override; + + void set_validator_telemetry_filename(std::string value) override; void start_up() override; FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, - FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, + FullNodeOptions opts, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId dht, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client, std::string db_root); + td::actor::ActorId client, std::string db_root, + td::Promise started_promise); private: + struct ShardInfo { + td::actor::ActorOwn actor; + bool active = false; + td::Timestamp delete_at = td::Timestamp::never(); + }; + + void update_shard_actor(ShardIdFull shard, bool active); + PublicKeyHash local_id_; adnl::AdnlNodeIdShort adnl_id_; FileHash zero_state_file_hash_; td::actor::ActorId get_shard(AccountIdPrefixFull dst); - td::actor::ActorId get_shard(ShardIdFull dst); - - std::map> shards_; + td::actor::ActorId get_shard(ShardIdFull shard); + std::map shards_; + int wc_monitor_min_split_ = 0; td::actor::ActorId keyring_; td::actor::ActorId adnl_; @@ -124,7 +139,9 @@ class FullNodeImpl : public FullNode { std::map current_validators_; std::set local_keys_; - FullNodeConfig config_; + + td::Promise started_promise_; + FullNodeOptions opts_; std::map> private_block_overlays_; bool broadcast_block_candidates_in_public_overlay_ = false; @@ -143,6 +160,14 @@ class FullNodeImpl : public FullNode { void send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast); void send_block_candidate_broadcast_to_custom_overlays(const BlockIdExt& block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, const td::BufferSlice& data); + + std::string validator_telemetry_filename_; + PublicKeyHash validator_telemetry_collector_key_ = PublicKeyHash::zero(); + + void update_validator_telemetry_collector(); + + td::actor::ActorOwn out_msg_queue_query_token_manager_ = + td::actor::create_actor("tokens", /* max_tokens = */ 1); }; } // namespace fullnode diff --git a/validator/impl/CMakeLists.txt b/validator/impl/CMakeLists.txt index 9df6725d..978cf859 100644 --- a/validator/impl/CMakeLists.txt +++ b/validator/impl/CMakeLists.txt @@ -16,6 +16,7 @@ set(TON_VALIDATOR_SOURCE ihr-message.cpp liteserver.cpp message-queue.cpp + out-msg-queue-proof.cpp proof.cpp shard.cpp signature-set.cpp @@ -35,13 +36,13 @@ set(TON_VALIDATOR_SOURCE liteserver.hpp liteserver-cache.hpp message-queue.hpp + out-msg-queue-proof.hpp proof.hpp shard.hpp signature-set.hpp top-shard-descr.hpp validate-query.hpp - validator-set.hpp -) + validator-set.hpp) add_library(ton_validator STATIC ${TON_VALIDATOR_SOURCE}) diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index a9dd7fe2..de48626d 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -308,8 +308,11 @@ bool AcceptBlockQuery::create_new_proof() { } // 10. check resulting object if (!block::gen::t_BlockProof.validate_ref(bs_cell)) { - block::gen::t_BlockProof.print_ref(std::cerr, bs_cell); - vm::load_cell_slice(bs_cell).print_rec(std::cerr); + FLOG(WARNING) { + sb << "BlockProof object just created failed to pass automated consistency checks: "; + block::gen::t_BlockProof.print_ref(sb, bs_cell); + vm::load_cell_slice(bs_cell).print_rec(sb); + }; return fatal_error("BlockProof object just created failed to pass automated consistency checks"); } // 11. create a proof object from this cell @@ -851,15 +854,12 @@ bool AcceptBlockQuery::create_top_shard_block_description() { && (root.is_null() || cb.store_ref_bool(std::move(root))) && cb.finalize_to(td_cell))) { return fatal_error("cannot serialize ShardTopBlockDescription for the newly-accepted block "s + id_.to_str()); } - if (false) { - // debug output - std::cerr << "new ShardTopBlockDescription: "; - block::gen::t_TopBlockDescr.print_ref(std::cerr, td_cell); - vm::load_cell_slice(td_cell).print_rec(std::cerr); - } if (!block::gen::t_TopBlockDescr.validate_ref(td_cell)) { - block::gen::t_TopBlockDescr.print_ref(std::cerr, td_cell); - vm::load_cell_slice(td_cell).print_rec(std::cerr); + FLOG(WARNING) { + sb << "just created ShardTopBlockDescription is invalid: "; + block::gen::t_TopBlockDescr.print_ref(sb, td_cell); + vm::load_cell_slice(td_cell).print_rec(sb); + }; return fatal_error("just created ShardTopBlockDescription for "s + id_.to_str() + " is invalid"); } auto res = vm::std_boc_serialize(td_cell, 0); diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index b1594a4c..340e3a40 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -50,7 +50,7 @@ class Collator final : public td::actor::Actor { using LtCellRef = block::LtCellRef; using NewOutMsg = block::NewOutMsg; const ShardIdFull shard_; - ton::BlockId new_id; + ton::BlockId new_id{workchainInvalid, 0, 0}; bool busy_{false}; bool before_split_{false}; bool after_split_{false}; @@ -109,14 +109,11 @@ class Collator final : public td::actor::Actor { return 2; } - static td::Result> - impl_create_ordinary_transaction(Ref msg_root, - block::Account* acc, - UnixTime utime, LogicalTime lt, - block::StoragePhaseConfig* storage_phase_cfg, - block::ComputePhaseConfig* compute_phase_cfg, - block::ActionPhaseConfig* action_phase_cfg, - bool external, LogicalTime after_lt); + static td::Result> impl_create_ordinary_transaction( + Ref msg_root, block::Account* acc, UnixTime utime, LogicalTime lt, + block::StoragePhaseConfig* storage_phase_cfg, block::ComputePhaseConfig* compute_phase_cfg, + block::ActionPhaseConfig* action_phase_cfg, block::SerializeConfig* serialize_cfg, bool external, + LogicalTime after_lt); private: void start_up() override; @@ -177,9 +174,11 @@ class Collator final : public td::actor::Actor { block::StoragePhaseConfig storage_phase_cfg_{&storage_prices_}; block::ComputePhaseConfig compute_phase_cfg_; block::ActionPhaseConfig action_phase_cfg_; + block::SerializeConfig serialize_cfg_; td::RefInt256 masterchain_create_fee_, basechain_create_fee_; std::unique_ptr block_limits_; std::unique_ptr block_limit_status_; + int block_limit_class_ = 0; ton::LogicalTime min_new_msg_lt{std::numeric_limits::max()}; block::CurrencyCollection total_balance_, old_total_balance_, total_validator_fees_; block::CurrencyCollection global_balance_, old_global_balance_, import_created_{0}; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index 6b5d7614..2a6d7a2b 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -53,16 +53,6 @@ static constexpr int HIGH_PRIORITY_EXTERNAL = 10; // don't skip high priority e static constexpr int MAX_ATTEMPTS = 5; -#define DBG(__n) dbg(__n)&& -#define DSTART int __dcnt = 0; -#define DEB DBG(++__dcnt) - -static inline bool dbg(int c) TD_UNUSED; -static inline bool dbg(int c) { - std::cerr << '[' << (char)('0' + c / 10) << (char)('0' + c % 10) << ']'; - return true; -} - /** * Constructs a Collator object. * @@ -362,6 +352,8 @@ bool Collator::fatal_error(td::Status error) { attempt_idx_ + 1); } else { main_promise(std::move(error)); + td::actor::send_closure(manager, &ValidatorManager::record_collate_query_stats, BlockIdExt{new_id, RootHash::zero(), FileHash::zero()}, + work_timer_.elapsed(), cpu_work_timer_.elapsed(), td::optional{}); } busy_ = false; } @@ -761,8 +753,6 @@ bool Collator::unpack_last_mc_state() { << " (upgrade validator software?)"; } // TODO: extract start_lt and end_lt from prev_mc_block as well - // std::cerr << " block::gen::ShardState::print_ref(mc_state_root) = "; - // block::gen::t_ShardState.print_ref(std::cerr, mc_state_root, 2); return true; } @@ -888,8 +878,10 @@ void Collator::got_neighbor_out_queue(int i, td::Result> res) // unpack ProcessedUpto LOG(DEBUG) << "unpacking ProcessedUpto of neighbor " << descr.blk_.to_str(); if (verbosity >= 2) { - block::gen::t_ProcessedInfo.print(std::cerr, qinfo.proc_info); - qinfo.proc_info->print_rec(std::cerr); + FLOG(INFO) { + block::gen::t_ProcessedInfo.print(sb, qinfo.proc_info); + qinfo.proc_info->print_rec(sb); + }; } descr.processed_upto = block::MsgProcessedUptoCollection::unpack(descr.shard(), qinfo.proc_info); if (!descr.processed_upto) { @@ -1756,9 +1748,11 @@ bool Collator::import_new_shard_top_blocks() { shard_conf_adjusted_ = true; } if (tb_act && verbosity >= 0) { // DEBUG - LOG(INFO) << "updated shard block configuration to "; - auto csr = shard_conf_->get_root_csr(); - block::gen::t_ShardHashes.print(std::cerr, csr.write()); + FLOG(INFO) { + sb << "updated shard block configuration to "; + auto csr = shard_conf_->get_root_csr(); + block::gen::t_ShardHashes.print(sb, csr); + }; } block::gen::ShardFeeCreated::Record fc; if (!(tlb::csr_unpack(fees_import_dict_->get_root_extra(), @@ -2001,12 +1995,9 @@ bool Collator::init_lt() { * @returns True if the configuration parameters were successfully fetched and initialized, false otherwise. */ bool Collator::fetch_config_params() { - auto res = block::FetchConfigParams::fetch_config_params(*config_, - &old_mparams_, &storage_prices_, &storage_phase_cfg_, - &rand_seed_, &compute_phase_cfg_, &action_phase_cfg_, - &masterchain_create_fee_, &basechain_create_fee_, - workchain(), now_ - ); + auto res = block::FetchConfigParams::fetch_config_params( + *config_, &old_mparams_, &storage_prices_, &storage_phase_cfg_, &rand_seed_, &compute_phase_cfg_, + &action_phase_cfg_, &serialize_cfg_, &masterchain_create_fee_, &basechain_create_fee_, workchain(), now_); if (res.is_error()) { return fatal_error(res.move_as_error()); } @@ -2279,10 +2270,12 @@ bool Collator::dequeue_message(Ref msg_envelope, ton::LogicalTime deli bool Collator::out_msg_queue_cleanup() { LOG(INFO) << "cleaning outbound queue from messages already imported by neighbors"; if (verbosity >= 2) { - auto rt = out_msg_queue_->get_root(); - std::cerr << "old out_msg_queue is "; - block::gen::t_OutMsgQueue.print(std::cerr, *rt); - rt->print_rec(std::cerr); + FLOG(INFO) { + auto rt = out_msg_queue_->get_root(); + sb << "old out_msg_queue is "; + block::gen::t_OutMsgQueue.print(sb, rt); + rt->print_rec(sb); + }; } if (after_merge_) { @@ -2333,6 +2326,7 @@ bool Collator::out_msg_queue_cleanup() { register_out_msg_queue_op(); if (!block_limit_status_->fits(block::ParamLimits::cl_normal)) { block_full_ = true; + block_limit_class_ = std::max(block_limit_class_, block_limit_status_->classify()); } } return !delivered; @@ -2402,6 +2396,7 @@ bool Collator::out_msg_queue_cleanup() { register_out_msg_queue_op(); if (!block_limit_status_->fits(block::ParamLimits::cl_normal)) { block_full_ = true; + block_limit_class_ = std::max(block_limit_class_, block_limit_status_->classify()); } queue.next(); ++i; @@ -2420,10 +2415,12 @@ bool Collator::out_msg_queue_cleanup() { << out_msg_queue_size_; } if (verbosity >= 2) { - auto rt = out_msg_queue_->get_root(); - std::cerr << "new out_msg_queue is "; - block::gen::t_OutMsgQueue.print(std::cerr, *rt); - rt->print_rec(std::cerr); + FLOG(INFO) { + auto rt = out_msg_queue_->get_root(); + sb << "new out_msg_queue is "; + block::gen::t_OutMsgQueue.print(sb, rt); + rt->print_rec(sb); + }; } return register_out_msg_queue_op(true); } @@ -2522,19 +2519,27 @@ bool Collator::combine_account_transactions() { auto cell = cb.finalize(); auto csr = vm::load_cell_slice_ref(cell); if (verbosity > 2) { - std::cerr << "new AccountBlock for " << z.first.to_hex() << ": "; - block::gen::t_AccountBlock.print_ref(std::cerr, cell); - csr->print_rec(std::cerr); + FLOG(INFO) { + sb << "new AccountBlock for " << z.first.to_hex() << ": "; + block::gen::t_AccountBlock.print_ref(sb, cell); + csr->print_rec(sb); + }; } if (!block::gen::t_AccountBlock.validate_ref(100000, cell)) { - block::gen::t_AccountBlock.print_ref(std::cerr, cell); - csr->print_rec(std::cerr); + FLOG(WARNING) { + sb << "AccountBlock failed to pass automatic validation tests: "; + block::gen::t_AccountBlock.print_ref(sb, cell); + csr->print_rec(sb); + }; return fatal_error(std::string{"new AccountBlock for "} + z.first.to_hex() + " failed to pass automatic validation tests"); } if (!block::tlb::t_AccountBlock.validate_ref(100000, cell)) { - block::gen::t_AccountBlock.print_ref(std::cerr, cell); - csr->print_rec(std::cerr); + FLOG(WARNING) { + sb << "AccountBlock failed to pass handwritten validation tests: "; + block::gen::t_AccountBlock.print_ref(sb, cell); + csr->print_rec(sb); + }; return fatal_error(std::string{"new AccountBlock for "} + z.first.to_hex() + " failed to pass handwritten validation tests"); } @@ -2559,8 +2564,10 @@ bool Collator::combine_account_transactions() { } else if (acc.status == block::Account::acc_nonexist) { // account deleted if (verbosity > 2) { - std::cerr << "deleting account " << acc.addr.to_hex() << " with empty new value "; - block::gen::t_Account.print_ref(std::cerr, acc.total_state); + FLOG(INFO) { + sb << "deleting account " << acc.addr.to_hex() << " with empty new value "; + block::gen::t_Account.print_ref(sb, acc.total_state); + }; } if (account_dict->lookup_delete(acc.addr).is_null()) { return fatal_error(std::string{"cannot delete account "} + acc.addr.to_hex() + " from ShardAccounts"); @@ -2568,8 +2575,10 @@ bool Collator::combine_account_transactions() { } else { // existing account modified if (verbosity > 4) { - std::cerr << "modifying account " << acc.addr.to_hex() << " to "; - block::gen::t_Account.print_ref(std::cerr, acc.total_state); + FLOG(INFO) { + sb << "modifying account " << acc.addr.to_hex() << " to "; + block::gen::t_Account.print_ref(sb, acc.total_state); + }; } if (!(cb.store_ref_bool(acc.total_state) // account_descr$_ account:^Account && cb.store_bits_bool(acc.last_trans_hash_) // last_trans_hash:bits256 @@ -2592,9 +2601,11 @@ bool Collator::combine_account_transactions() { return fatal_error("cannot serialize ShardAccountBlocks"); } if (verbosity > 2) { - std::cerr << "new ShardAccountBlocks: "; - block::gen::t_ShardAccountBlocks.print_ref(std::cerr, shard_account_blocks_); - vm::load_cell_slice(shard_account_blocks_).print_rec(std::cerr); + FLOG(INFO) { + sb << "new ShardAccountBlocks: "; + block::gen::t_ShardAccountBlocks.print_ref(sb, shard_account_blocks_); + vm::load_cell_slice(shard_account_blocks_).print_rec(sb); + }; } if (!block::gen::t_ShardAccountBlocks.validate_ref(100000, shard_account_blocks_)) { return fatal_error("new ShardAccountBlocks failed to pass automatic validity tests"); @@ -2604,9 +2615,11 @@ bool Collator::combine_account_transactions() { } auto shard_accounts = account_dict->get_root(); if (verbosity > 2) { - std::cerr << "new ShardAccounts: "; - block::gen::t_ShardAccounts.print(std::cerr, *shard_accounts); - shard_accounts->print_rec(std::cerr); + FLOG(INFO) { + sb << "new ShardAccounts: "; + block::gen::t_ShardAccounts.print(sb, shard_accounts); + shard_accounts->print_rec(sb); + }; } if (verify >= 2) { LOG(INFO) << "verifying new ShardAccounts"; @@ -2657,7 +2670,9 @@ bool Collator::create_special_transaction(block::CurrencyCollection amount, Ref< addr.to_hex()); } if (verbosity >= 4) { - block::gen::t_Message_Any.print_ref(std::cerr, msg); + FLOG(INFO) { + block::gen::t_Message_Any.print_ref(sb, msg); + }; } CHECK(block::gen::t_Message_Any.validate_ref(msg)); CHECK(block::tlb::t_Message.validate_ref(msg)); @@ -2732,7 +2747,7 @@ bool Collator::create_ticktock_transaction(const ton::StdSmcAddress& smc_addr, t return fatal_error(td::Status::Error( -666, std::string{"cannot create action phase of a new transaction for smart contract "} + smc_addr.to_hex())); } - if (!trans->serialize()) { + if (!trans->serialize(serialize_cfg_)) { return fatal_error(td::Status::Error( -666, std::string{"cannot serialize new transaction for smart contract "} + smc_addr.to_hex())); } @@ -2816,7 +2831,7 @@ Ref Collator::create_ordinary_transaction(Ref msg_root, after_lt = std::max(after_lt, it->second); } auto res = impl_create_ordinary_transaction(msg_root, acc, now_, start_lt, &storage_phase_cfg_, &compute_phase_cfg_, - &action_phase_cfg_, external, after_lt); + &action_phase_cfg_, &serialize_cfg_, external, after_lt); if (res.is_error()) { auto error = res.move_as_error(); if (error.code() == -701) { @@ -2867,6 +2882,7 @@ Ref Collator::create_ordinary_transaction(Ref msg_root, * @param storage_phase_cfg The configuration for the storage phase of the transaction. * @param compute_phase_cfg The configuration for the compute phase of the transaction. * @param action_phase_cfg The configuration for the action phase of the transaction. + * @param serialize_cfg The configuration for the serialization of the transaction. * @param external Flag indicating if the message is external. * @param after_lt The logical time after which the transaction should occur. Used only for external messages. * @@ -2880,6 +2896,7 @@ td::Result> Collator::impl_crea block::StoragePhaseConfig* storage_phase_cfg, block::ComputePhaseConfig* compute_phase_cfg, block::ActionPhaseConfig* action_phase_cfg, + block::SerializeConfig* serialize_cfg, bool external, LogicalTime after_lt) { if (acc->last_trans_end_lt_ >= lt && acc->transactions.empty()) { return td::Status::Error(-669, PSTRING() << "last transaction time in the state of account " << acc->workchain @@ -2947,7 +2964,7 @@ td::Result> Collator::impl_crea return td::Status::Error( -669, "cannot create bounce phase of a new transaction for smart contract "s + acc->addr.to_hex()); } - if (!trans->serialize()) { + if (!trans->serialize(*serialize_cfg)) { return td::Status::Error(-669, "cannot serialize new transaction for smart contract "s + acc->addr.to_hex()); } return std::move(trans); @@ -3161,8 +3178,10 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R Ref msg_env; CHECK(block::tlb::pack_cell(msg_env, msg_env_rec)); if (verbosity > 2) { - std::cerr << "new (processed outbound) message envelope: "; - block::gen::t_MsgEnvelope.print_ref(std::cerr, msg_env); + FLOG(INFO) { + sb << "new (processed outbound) message envelope: "; + block::gen::t_MsgEnvelope.print_ref(sb, msg_env); + }; } // 3. create InMsg, referring to this MsgEnvelope and this Transaction vm::CellBuilder cb; @@ -3205,6 +3224,7 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R // 7. check whether the block is full now if (!block_limit_status_->fits(block::ParamLimits::cl_normal)) { block_full_ = true; + block_limit_class_ = std::max(block_limit_class_, block_limit_status_->classify()); return 3; } if (soft_timeout_.is_in_past(td::Timestamp::now())) { @@ -3283,16 +3303,20 @@ bool Collator::enqueue_transit_message(Ref msg, Ref old_msg_ Ref out_msg = cb.finalize(); // 4.1. insert OutMsg into OutMsgDescr if (verbosity > 2) { - std::cerr << "OutMsg for a transit message: "; - block::gen::t_OutMsg.print_ref(std::cerr, out_msg); + FLOG(INFO) { + sb << "OutMsg for a transit message: "; + block::gen::t_OutMsg.print_ref(sb, out_msg); + }; } if (!insert_out_msg(out_msg)) { return fatal_error("cannot insert a new OutMsg into OutMsgDescr"); } // 4.2. insert InMsg into InMsgDescr if (verbosity > 2) { - std::cerr << "InMsg for a transit message: "; - block::gen::t_InMsg.print_ref(std::cerr, in_msg); + FLOG(INFO) { + sb << "InMsg for a transit message: "; + block::gen::t_InMsg.print_ref(sb, in_msg); + }; } if (!insert_in_msg(in_msg)) { return fatal_error("cannot insert a new InMsg into InMsgDescr"); @@ -3363,7 +3387,10 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT if (enq_msg.is_null() || enq_msg->size_ext() != 0x10040 || (enqueued_lt = enq_msg->prefetch_ulong(64)) < /* 0 */ 1 * lt) { // DEBUG if (enq_msg.not_null()) { - block::gen::t_EnqueuedMsg.print(std::cerr, *enq_msg); + FLOG(WARNING) { + sb << "inbound internal message is not a valid EnqueuedMsg: "; + block::gen::t_EnqueuedMsg.print(sb, enq_msg); + }; } LOG(ERROR) << "inbound internal message is not a valid EnqueuedMsg (created lt " << lt << ", enqueued " << enqueued_lt << ")"; @@ -3568,6 +3595,7 @@ bool Collator::process_inbound_internal_messages() { block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); if (block_full_) { LOG(INFO) << "BLOCK FULL, stop processing inbound internal messages"; + block_limit_class_ = std::max(block_limit_class_, block_limit_status_->classify()); stats_.limits_log += PSTRING() << "INBOUND_INT_MESSAGES: " << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) << "\n"; break; @@ -3586,14 +3614,18 @@ bool Collator::process_inbound_internal_messages() { LOG(DEBUG) << "processing inbound message with (lt,hash)=(" << kv->lt << "," << kv->key.to_hex() << ") from neighbor #" << kv->source; if (verbosity > 2) { - std::cerr << "inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() << " msg="; - block::gen::t_EnqueuedMsg.print(std::cerr, *(kv->msg)); + FLOG(INFO) { + sb << "inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() << " msg="; + block::gen::t_EnqueuedMsg.print(sb, kv->msg); + }; } if (!process_inbound_message(kv->msg, kv->lt, kv->key.cbits(), neighbors_.at(kv->source))) { if (verbosity > 1) { - std::cerr << "invalid inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() - << " msg="; - block::gen::t_EnqueuedMsg.print(std::cerr, *(kv->msg)); + FLOG(INFO) { + sb << "invalid inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() + << " msg="; + block::gen::t_EnqueuedMsg.print(sb, kv->msg); + }; } return fatal_error("error processing inbound internal message"); } @@ -3659,6 +3691,7 @@ bool Collator::process_inbound_external_messages() { } if (r > 0) { full = !block_limit_status_->fits(block::ParamLimits::cl_soft); + block_limit_class_ = std::max(block_limit_class_, block_limit_status_->classify()); } auto it = ext_msg_map.find(hash); CHECK(it != ext_msg_map.end()); @@ -3761,6 +3794,7 @@ bool Collator::process_dispatch_queue() { block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); if (block_full_) { LOG(INFO) << "BLOCK FULL, stop processing dispatch queue"; + block_limit_class_ = std::max(block_limit_class_, block_limit_status_->classify()); stats_.limits_log += PSTRING() << "DISPATCH_QUEUE_STAGE_" << iter << ": " << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) << "\n"; @@ -3878,7 +3912,10 @@ bool Collator::process_deferred_message(Ref enq_msg, StdSmcAddres LogicalTime enqueued_lt = 0; if (enq_msg.is_null() || enq_msg->size_ext() != 0x10040 || (enqueued_lt = enq_msg->prefetch_ulong(64)) != lt) { if (enq_msg.not_null()) { - block::gen::t_EnqueuedMsg.print(std::cerr, *enq_msg); + FLOG(WARNING) { + sb << "internal message in DispatchQueue is not a valid EnqueuedMsg: "; + block::gen::t_EnqueuedMsg.print(sb, enq_msg); + }; } LOG(ERROR) << "internal message in DispatchQueue is not a valid EnqueuedMsg (created lt " << lt << ", enqueued " << enqueued_lt << ")"; @@ -3980,8 +4017,10 @@ bool Collator::process_deferred_message(Ref enq_msg, StdSmcAddres */ bool Collator::insert_in_msg(Ref in_msg) { if (verbosity > 2) { - std::cerr << "InMsg being inserted into InMsgDescr: "; - block::gen::t_InMsg.print_ref(std::cerr, in_msg); + FLOG(INFO) { + sb << "InMsg being inserted into InMsgDescr: "; + block::gen::t_InMsg.print_ref(sb, in_msg); + }; } auto cs = load_cell_slice(in_msg); if (!cs.size_refs()) { @@ -4022,8 +4061,10 @@ bool Collator::insert_in_msg(Ref in_msg) { */ bool Collator::insert_out_msg(Ref out_msg) { if (verbosity > 2) { - std::cerr << "OutMsg being inserted into OutMsgDescr: "; - block::gen::t_OutMsg.print_ref(std::cerr, out_msg); + FLOG(INFO) { + sb << "OutMsg being inserted into OutMsgDescr: "; + block::gen::t_OutMsg.print_ref(sb, out_msg); + }; } auto cs = load_cell_slice(out_msg); if (!cs.size_refs()) { @@ -4119,8 +4160,10 @@ bool Collator::enqueue_message(block::NewOutMsg msg, td::RefInt256 fwd_fees_rema } // 4. insert OutMsg into OutMsgDescr if (verbosity > 2) { - std::cerr << "OutMsg for a newly-generated message: "; - block::gen::t_OutMsg.print_ref(std::cerr, out_msg); + FLOG(INFO) { + sb << "OutMsg for a newly-generated message: "; + block::gen::t_OutMsg.print_ref(sb, out_msg); + }; } if (!insert_out_msg(out_msg)) { return fatal_error("cannot insert a new OutMsg into OutMsgDescr"); @@ -4413,9 +4456,12 @@ bool Collator::create_mc_state_extra() { bool ignore_cfg_changes = false; Ref cfg0; if (!block::valid_config_data(cfg_smc_config, config_addr, true, true, old_mparams_)) { - block::gen::t_Hashmap_32_Ref_Cell.print_ref(std::cerr, cfg_smc_config); LOG(ERROR) << "configuration smart contract "s + config_addr.to_hex() + " contains an invalid configuration in its data, IGNORING CHANGES"; + FLOG(WARNING) { + sb << "ignored configuration: "; + block::gen::t_Hashmap_32_Ref_Cell.print_ref(sb, cfg_smc_config); + }; ignore_cfg_changes = true; } else { cfg0 = cfg_dict.lookup_ref(td::BitArray<32>{(long long)0}); @@ -4453,34 +4499,26 @@ bool Collator::create_mc_state_extra() { return fatal_error(wset_res.move_as_error()); } bool update_shard_cc = is_key_block_ || (now_ / ccvc.shard_cc_lifetime > prev_now_ / ccvc.shard_cc_lifetime); - // temp debug - if (verbosity >= 3 * 1) { - auto csr = shard_conf_->get_root_csr(); - LOG(INFO) << "new shard configuration before post-processing is"; - std::ostringstream os; - csr->print_rec(os); - block::gen::t_ShardHashes.print(os, csr.write()); - LOG(INFO) << os.str(); - } - // end (temp debug) if (!update_shard_config(wset_res.move_as_ok(), ccvc, update_shard_cc)) { auto csr = shard_conf_->get_root_csr(); if (csr.is_null()) { LOG(WARNING) << "new shard configuration is null (!)"; } else { LOG(WARNING) << "invalid new shard configuration is"; - std::ostringstream os; - csr->print_rec(os); - block::gen::t_ShardHashes.print(os, csr.write()); - LOG(WARNING) << os.str(); + FLOG(WARNING) { + csr->print_rec(sb); + block::gen::t_ShardHashes.print(sb, csr); + }; } return fatal_error("cannot post-process shard configuration"); } // 3. save new shard_hashes state_extra.shard_hashes = shard_conf_->get_root_csr(); - if (verbosity >= 3 * 0) { // DEBUG - std::cerr << "updated shard configuration to "; - block::gen::t_ShardHashes.print(std::cerr, *state_extra.shard_hashes); + if (verbosity >= 3) { + FLOG(INFO) { + sb << "updated shard configuration to "; + block::gen::t_ShardHashes.print(sb, state_extra.shard_hashes); + }; } if (!block::gen::t_ShardHashes.validate_upto(10000, *state_extra.shard_hashes)) { return fatal_error("new ShardHashes is invalid"); @@ -4581,13 +4619,18 @@ bool Collator::create_mc_state_extra() { if (verify >= 2) { LOG(INFO) << "verifying new BlockCreateStats"; if (!block::gen::t_BlockCreateStats.validate_csr(100000, cs)) { - cs->print_rec(std::cerr); - block::gen::t_BlockCreateStats.print(std::cerr, *cs); + FLOG(WARNING) { + sb << "BlockCreateStats in the new masterchain state failed to pass automated validity checks: "; + cs->print_rec(sb); + block::gen::t_BlockCreateStats.print(sb, cs); + }; return fatal_error("BlockCreateStats in the new masterchain state failed to pass automated validity checks"); } } if (verbosity >= 4 * 1) { - block::gen::t_BlockCreateStats.print(std::cerr, *cs); + FLOG(INFO) { + block::gen::t_BlockCreateStats.print(sb, cs); + }; } } else { state_extra.r1.block_create_stats.clear(); @@ -4622,7 +4665,6 @@ bool Collator::update_block_creator_count(td::ConstBitPtr key, unsigned shard_in if (!block::unpack_CreatorStats(std::move(cs), mc_cnt, shard_cnt)) { return fatal_error("cannot unpack CreatorStats for "s + key.to_hex(256) + " from previous masterchain state"); } - // std::cerr << mc_cnt.to_str() << " " << shard_cnt.to_str() << std::endl; if (mc_incr && !mc_cnt.increase_by(mc_incr, now_)) { return fatal_error(PSTRING() << "cannot increase masterchain block counter in CreatorStats for " << key.to_hex(256) << " by " << mc_incr << " (old value is " << mc_cnt.to_str() << ")"); @@ -4787,11 +4829,11 @@ bool Collator::check_block_overload() { LOG(INFO) << "block load statistics: gas=" << block_limit_status_->gas_used << " lt_delta=" << block_limit_status_->cur_lt - block_limit_status_->limits.start_lt << " size_estimate=" << block_size_estimate_; - auto cl = block_limit_status_->classify(); - if (cl >= block::ParamLimits::cl_soft || dispatch_queue_total_limit_reached_) { + block_limit_class_ = std::max(block_limit_class_, block_limit_status_->classify()); + if (block_limit_class_ >= block::ParamLimits::cl_soft || dispatch_queue_total_limit_reached_) { std::string message = "block is overloaded "; - if (cl >= block::ParamLimits::cl_soft) { - message += PSTRING() << "(category " << cl << ")"; + if (block_limit_class_ >= block::ParamLimits::cl_soft) { + message += PSTRING() << "(category " << block_limit_class_ << ")"; } else { message += "(long dispatch queue processing)"; } @@ -4802,7 +4844,7 @@ bool Collator::check_block_overload() { overload_history_ |= 1; LOG(INFO) << message; } - } else if (cl <= block::ParamLimits::cl_underload) { + } else if (block_limit_class_ <= block::ParamLimits::cl_underload) { if (out_msg_queue_size_ > MERGE_MAX_QUEUE_SIZE) { LOG(INFO) << "block is underloaded, but don't set underload history because out_msg_queue size is too big to merge (" @@ -4993,9 +5035,11 @@ bool Collator::update_public_libraries() { } } if (libraries_changed_ && verbosity >= 2) { - std::cerr << "New public libraries: "; - block::gen::t_HashmapE_256_LibDescr.print(std::cerr, shard_libraries_->get_root()); - shard_libraries_->get_root()->print_rec(std::cerr); + FLOG(INFO) { + sb << "New public libraries: "; + block::gen::t_HashmapE_256_LibDescr.print(sb, shard_libraries_->get_root()); + shard_libraries_->get_root()->print_rec(sb); + }; } return true; } @@ -5118,9 +5162,11 @@ bool Collator::create_shard_state() { } LOG(DEBUG) << "min_ref_mc_seqno is " << min_ref_mc_seqno_; if (verbosity > 2) { - std::cerr << "new ShardState: "; - block::gen::t_ShardState.print_ref(std::cerr, state_root); - vm::load_cell_slice(state_root).print_rec(std::cerr); + FLOG(INFO) { + sb << "new ShardState: "; + block::gen::t_ShardState.print_ref(sb, state_root); + vm::load_cell_slice(state_root).print_rec(sb); + }; } if (verify >= 2) { LOG(INFO) << "verifying new ShardState"; @@ -5133,9 +5179,11 @@ bool Collator::create_shard_state() { return fatal_error("cannot create Merkle update for ShardState"); } if (verbosity > 2) { - std::cerr << "Merkle Update for ShardState: "; - vm::CellSlice cs{vm::NoVm{}, state_update}; - cs.print_rec(std::cerr); + FLOG(INFO) { + sb << "Merkle Update for ShardState: "; + vm::CellSlice cs{vm::NoVm{}, state_update}; + cs.print_rec(sb); + }; } LOG(INFO) << "updating block profile statistics"; block_limit_status_->add_proof(state_root); @@ -5180,10 +5228,12 @@ bool Collator::update_processed_upto() { */ bool Collator::compute_out_msg_queue_info(Ref& out_msg_queue_info) { if (verbosity >= 2) { - auto rt = out_msg_queue_->get_root(); - std::cerr << "resulting out_msg_queue is "; - block::gen::t_OutMsgQueue.print(std::cerr, *rt); - rt->print_rec(std::cerr); + FLOG(INFO) { + auto rt = out_msg_queue_->get_root(); + sb << "resulting out_msg_queue is "; + block::gen::t_OutMsgQueue.print(sb, rt); + rt->print_rec(sb); + }; } vm::CellBuilder cb; // out_msg_queue_extra#0 dispatch_queue:DispatchQueue out_queue_size:(Maybe uint48) = OutMsgQueueExtra; @@ -5233,8 +5283,10 @@ bool Collator::compute_total_balance() { } vm::CellSlice cs{*(in_msg_dict->get_root_extra())}; if (verbosity > 2) { - block::gen::t_ImportFees.print(std::cerr, vm::CellSlice{*(in_msg_dict->get_root_extra())}); - cs.print_rec(std::cerr); + FLOG(INFO) { + block::gen::t_ImportFees.print(sb, in_msg_dict->get_root_extra()); + cs.print_rec(sb); + }; } auto new_import_fees = block::tlb::t_Grams.as_integer_skip(cs); if (new_import_fees.is_null()) { @@ -5462,9 +5514,11 @@ bool Collator::create_block() { return fatal_error("cannot create new Block"); } if (verbosity >= 3 * 1) { - std::cerr << "new Block: "; - block::gen::t_Block.print_ref(std::cerr, new_block); - vm::load_cell_slice(new_block).print_rec(std::cerr); + FLOG(INFO) { + sb << "new Block: "; + block::gen::t_Block.print_ref(sb, new_block); + vm::load_cell_slice(new_block).print_rec(sb); + }; } if (verify >= 1) { LOG(INFO) << "verifying new Block"; @@ -5502,9 +5556,11 @@ Ref Collator::collate_shard_block_descr_set() { return {}; } if (verbosity >= 4 * 1) { - std::cerr << "serialized TopBlockDescrSet for collated data is: "; - block::gen::t_TopBlockDescrSet.print_ref(std::cerr, cell); - vm::load_cell_slice(cell).print_rec(std::cerr); + FLOG(INFO) { + sb << "serialized TopBlockDescrSet for collated data is: "; + block::gen::t_TopBlockDescrSet.print_ref(sb, cell); + vm::load_cell_slice(cell).print_rec(sb); + }; } return cell; } @@ -5711,8 +5767,10 @@ td::Result Collator::register_external_message_cell(Ref ext_msg, return td::Status::Error("inbound external message has destination address not in this shard"); } if (verbosity > 2) { - std::cerr << "registered external message: "; - block::gen::t_Message_Any.print_ref(std::cerr, ext_msg); + FLOG(INFO) { + sb << "registered external message: "; + block::gen::t_Message_Any.print_ref(sb, ext_msg); + }; } ext_msg_map.emplace(hash, 1); ext_msg_list_.push_back({std::move(ext_msg), ext_hash, priority}); diff --git a/validator/impl/external-message.cpp b/validator/impl/external-message.cpp index 2fdb491b..8b1f5eb7 100644 --- a/validator/impl/external-message.cpp +++ b/validator/impl/external-message.cpp @@ -136,13 +136,12 @@ td::Status ExtMessageQ::run_message_on_account(ton::WorkchainId wc, td::BitArray<256> rand_seed_; block::ComputePhaseConfig compute_phase_cfg_; block::ActionPhaseConfig action_phase_cfg_; + block::SerializeConfig serialize_config_; td::RefInt256 masterchain_create_fee, basechain_create_fee; - auto fetch_res = block::FetchConfigParams::fetch_config_params(*config, &old_mparams, - &storage_prices_, &storage_phase_cfg_, - &rand_seed_, &compute_phase_cfg_, - &action_phase_cfg_, &masterchain_create_fee, - &basechain_create_fee, wc, utime); + auto fetch_res = block::FetchConfigParams::fetch_config_params( + *config, &old_mparams, &storage_prices_, &storage_phase_cfg_, &rand_seed_, &compute_phase_cfg_, + &action_phase_cfg_, &serialize_config_, &masterchain_create_fee, &basechain_create_fee, wc, utime); if(fetch_res.is_error()) { auto error = fetch_res.move_as_error(); LOG(DEBUG) << "Cannot fetch config params: " << error.message(); @@ -152,10 +151,9 @@ td::Status ExtMessageQ::run_message_on_account(ton::WorkchainId wc, compute_phase_cfg_.with_vm_log = true; compute_phase_cfg_.stop_on_accept_message = true; - auto res = Collator::impl_create_ordinary_transaction(msg_root, acc, utime, lt, - &storage_phase_cfg_, &compute_phase_cfg_, - &action_phase_cfg_, - true, lt); + auto res = + Collator::impl_create_ordinary_transaction(msg_root, acc, utime, lt, &storage_phase_cfg_, &compute_phase_cfg_, + &action_phase_cfg_, &serialize_config_, true, lt); if(res.is_error()) { auto error = res.move_as_error(); LOG(DEBUG) << "Cannot run message on account: " << error.message(); diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index 6bd4e421..06f40b8f 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -85,19 +85,13 @@ void LiteQuery::abort_query(td::Status reason) { if (acc_state_promise_) { acc_state_promise_.set_error(std::move(reason)); } else if (promise_) { + td::actor::send_closure(manager_, &ValidatorManager::add_lite_query_stats, query_obj_ ? query_obj_->get_id() : 0, + false); promise_.set_error(std::move(reason)); } stop(); } -void LiteQuery::abort_query_ext(td::Status reason, std::string comment) { - LOG(INFO) << "aborted liteserver query: " << comment << " : " << reason.to_string(); - if (promise_) { - promise_.set_error(reason.move_as_error_prefix(comment + " : ")); - } - stop(); -} - bool LiteQuery::fatal_error(td::Status error) { abort_query(std::move(error)); return false; @@ -120,6 +114,8 @@ bool LiteQuery::finish_query(td::BufferSlice result, bool skip_cache_update) { td::actor::send_closure(cache_, &LiteServerCache::update, cache_key_, result.clone()); } if (promise_) { + td::actor::send_closure(manager_, &ValidatorManager::add_lite_query_stats, query_obj_ ? query_obj_->get_id() : 0, + true); promise_.set_result(std::move(result)); stop(); return true; @@ -139,7 +135,6 @@ void LiteQuery::start_up() { auto F = fetch_tl_object(query_, true); if (F.is_error()) { - td::actor::send_closure(manager_, &ValidatorManager::add_lite_query_stats, 0); // unknown abort_query(F.move_as_error()); return; } @@ -192,7 +187,6 @@ bool LiteQuery::use_cache() { } void LiteQuery::perform() { - td::actor::send_closure(manager_, &ValidatorManager::add_lite_query_stats, query_obj_->get_id()); lite_api::downcast_call( *query_obj_, td::overloaded( @@ -1520,11 +1514,17 @@ void LiteQuery::finish_runSmcMethod(td::BufferSlice shard_proof, td::BufferSlice libraries.push_back(acc_libs); } vm::GasLimits gas{gas_limit, gas_limit}; - vm::VmState vm{code, std::move(stack_), gas, 1, std::move(data), vm::VmLog::Null(), std::move(libraries)}; + vm::VmState vm{code, + config->get_global_version(), + std::move(stack_), + gas, + 1, + std::move(data), + vm::VmLog::Null(), + std::move(libraries)}; auto c7 = prepare_vm_c7(gen_utime, gen_lt, td::make_ref(acc.addr->clone()), balance, config.get(), std::move(code), due_payment); vm.set_c7(c7); // tuple with SmartContractInfo - vm.set_global_version(config->get_global_version()); // vm.incr_stack_trace(1); // enable stack dump after each step LOG(INFO) << "starting VM to run GET-method of smart contract " << acc_workchain_ << ":" << acc_addr_.to_hex(); // **** RUN VM **** @@ -1899,6 +1899,9 @@ void LiteQuery::continue_getConfigParams(int mode, std::vector param_list) } cfg = res.move_as_ok(); } else { + if (mode & block::ConfigInfo::needPrevBlocks) { + mode |= block::ConfigInfo::needCapabilities; + } auto res = block::ConfigInfo::extract_config(mpb.root(), mode); if (res.is_error()) { fatal_error(res.move_as_error()); diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index 447e1dad..fc873533 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -97,7 +97,6 @@ class LiteQuery : public td::actor::Actor { bool fatal_error(std::string err_msg, int err_code = -400); bool fatal_error(int err_code, std::string err_msg = ""); void abort_query(td::Status reason); - void abort_query_ext(td::Status reason, std::string err_msg); bool finish_query(td::BufferSlice result, bool skip_cache_update = false); void alarm() override; void start_up() override; diff --git a/validator/impl/out-msg-queue-proof.cpp b/validator/impl/out-msg-queue-proof.cpp new file mode 100644 index 00000000..95ad4a41 --- /dev/null +++ b/validator/impl/out-msg-queue-proof.cpp @@ -0,0 +1,294 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "out-msg-queue-proof.hpp" +#include "interfaces/proof.h" +#include "shard.hpp" +#include "vm/cells/MerkleProof.h" +#include "common/delay.h" +#include "interfaces/validator-manager.h" +#include "block/block-parse.h" +#include "block/block-auto.h" +#include "output-queue-merger.h" + +namespace ton { + +namespace validator { + +static td::Status check_no_prunned(const Ref& cell) { + if (cell.is_null()) { + return td::Status::OK(); + } + TRY_RESULT(loaded_cell, cell->load_cell()); + if (loaded_cell.data_cell->get_level() > 0) { + return td::Status::Error("prunned branch"); + } + return td::Status::OK(); +} + +static td::Status check_no_prunned(const vm::CellSlice& cs) { + for (unsigned i = 0; i < cs.size_refs(); ++i) { + TRY_STATUS(check_no_prunned(cs.prefetch_ref(i))); + } + return td::Status::OK(); +} + +static td::Result> process_queue( + ShardIdFull dst_shard, std::vector> blocks, + block::ImportedMsgQueueLimits limits) { + td::uint64 estimated_proof_size = 0; + + td::HashSet visited; + std::function dfs_cs; + auto dfs = [&](const Ref& cell) { + if (cell.is_null() || !visited.insert(cell->get_hash()).second) { + return; + } + dfs_cs(vm::CellSlice(vm::NoVm(), cell)); + }; + dfs_cs = [&](const vm::CellSlice& cs) { + // Based on BlockLimitStatus::estimate_block_size + estimated_proof_size += 12 + (cs.size() + 7) / 8 + cs.size_refs() * 3; + for (unsigned i = 0; i < cs.size_refs(); i++) { + dfs(cs.prefetch_ref(i)); + } + }; + std::vector neighbors; + for (auto& b : blocks) { + TRY_STATUS_PREFIX(check_no_prunned(*b.second.proc_info), "invalid proc_info proof: ") + dfs_cs(*b.second.proc_info); + neighbors.emplace_back(b.first, b.second.out_queue->prefetch_ref()); + } + + block::OutputQueueMerger queue_merger{dst_shard, std::move(neighbors)}; + std::vector msg_count(blocks.size()); + td::int32 msg_count_total = 0; + bool limit_reached = false; + + while (!queue_merger.is_eof()) { + auto kv = queue_merger.extract_cur(); + queue_merger.next(); + block::EnqueuedMsgDescr enq; + auto msg = kv->msg; + if (!enq.unpack(msg.write())) { + return td::Status::Error("cannot unpack EnqueuedMsgDescr"); + } + if (limit_reached) { + break; + } + ++msg_count[kv->source]; + ++msg_count_total; + + dfs_cs(*kv->msg); + TRY_STATUS_PREFIX(check_no_prunned(*kv->msg), "invalid message proof: ") + if (estimated_proof_size >= limits.max_bytes || msg_count_total >= (long long)limits.max_msgs) { + limit_reached = true; + } + } + if (!limit_reached) { + std::fill(msg_count.begin(), msg_count.end(), -1); + } + return msg_count; +} + +td::Result> OutMsgQueueProof::build( + ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits) { + if (!dst_shard.is_valid_ext()) { + return td::Status::Error("invalid shard"); + } + if (blocks.empty()) { + return create_tl_object(td::BufferSlice{}, td::BufferSlice{}, + std::vector{}); + } + + std::vector> block_state_proofs; + for (auto& block : blocks) { + if (block.id.seqno() != 0) { + if (block.block_root.is_null()) { + return td::Status::Error("block is null"); + } + TRY_RESULT(proof, create_block_state_proof(block.block_root)); + block_state_proofs.push_back(std::move(proof)); + } + if (!block::ShardConfig::is_neighbor(dst_shard, block.id.shard_full())) { + return td::Status::Error("shards are not neighbors"); + } + } + TRY_RESULT(block_state_proof, vm::std_boc_serialize_multi(block_state_proofs)); + + vm::Dictionary states_dict_pure{32}; + for (size_t i = 0; i < blocks.size(); ++i) { + if (blocks[i].state_root.is_null()) { + return td::Status::Error("state is null"); + } + states_dict_pure.set_ref(td::BitArray<32>{(long long)i}, blocks[i].state_root); + } + + vm::MerkleProofBuilder mpb{states_dict_pure.get_root_cell()}; + vm::Dictionary states_dict{mpb.root(), 32}; + std::vector> data(blocks.size()); + for (size_t i = 0; i < blocks.size(); ++i) { + data[i].first = blocks[i].id; + TRY_RESULT(state, ShardStateQ::fetch(blocks[i].id, {}, states_dict.lookup_ref(td::BitArray<32>{(long long)i}))); + TRY_RESULT(outq_descr, state->message_queue()); + block::gen::OutMsgQueueInfo::Record qinfo; + if (!tlb::unpack_cell(outq_descr->root_cell(), data[i].second)) { + return td::Status::Error("invalid message queue"); + } + } + TRY_RESULT(msg_count, process_queue(dst_shard, std::move(data), limits)); + + TRY_RESULT(proof, mpb.extract_proof()); + vm::Dictionary states_dict_proof{vm::CellSlice{vm::NoVm(), proof}.prefetch_ref(), 32}; + std::vector> state_proofs; + for (size_t i = 0; i < blocks.size(); ++i) { + td::Ref proof_raw = states_dict_proof.lookup_ref(td::BitArray<32>{(long long)i}); + CHECK(proof_raw.not_null()); + state_proofs.push_back(vm::CellBuilder::create_merkle_proof(proof_raw)); + } + TRY_RESULT(queue_proof, vm::std_boc_serialize_multi(state_proofs)); + return create_tl_object(std::move(queue_proof), std::move(block_state_proof), + std::move(msg_count)); +} + +td::Result>> OutMsgQueueProof::fetch(ShardIdFull dst_shard, + std::vector blocks, + block::ImportedMsgQueueLimits limits, + const ton_api::tonNode_outMsgQueueProof& f) { + try { + std::vector> res; + TRY_RESULT(queue_proofs, vm::std_boc_deserialize_multi(f.queue_proofs_, (int)blocks.size())); + TRY_RESULT(block_state_proofs, vm::std_boc_deserialize_multi(f.block_state_proofs_, (int)blocks.size())); + if (queue_proofs.size() != blocks.size()) { + return td::Status::Error("invalid size of queue_proofs"); + } + if (f.msg_counts_.size() != blocks.size()) { + return td::Status::Error("invalid size of msg_counts"); + } + size_t j = 0; + std::vector> data(blocks.size()); + for (size_t i = 0; i < blocks.size(); ++i) { + td::Bits256 state_root_hash; + Ref block_state_proof = {}; + if (blocks[i].seqno() == 0) { + state_root_hash = blocks[i].root_hash; + } else { + if (j == block_state_proofs.size()) { + return td::Status::Error("invalid size of block_state_proofs"); + } + block_state_proof = block_state_proofs[j++]; + TRY_RESULT_ASSIGN(state_root_hash, unpack_block_state_proof(blocks[i], block_state_proof)); + } + auto state_root = vm::MerkleProof::virtualize(queue_proofs[i], 1); + if (state_root->get_hash().as_slice() != state_root_hash.as_slice()) { + return td::Status::Error("state root hash mismatch"); + } + res.emplace_back(true, blocks[i], state_root, block_state_proof, f.msg_counts_[i]); + + data[i].first = blocks[i]; + TRY_RESULT(state, ShardStateQ::fetch(blocks[i], {}, state_root)); + TRY_RESULT(outq_descr, state->message_queue()); + block::gen::OutMsgQueueInfo::Record qinfo; + if (!tlb::unpack_cell(outq_descr->root_cell(), data[i].second)) { + return td::Status::Error("invalid message queue"); + } + } + if (j != block_state_proofs.size()) { + return td::Status::Error("invalid size of block_state_proofs"); + } + TRY_RESULT(msg_count, process_queue(dst_shard, std::move(data), limits)); + if (msg_count != f.msg_counts_) { + return td::Status::Error("incorrect msg_count"); + } + return res; + } catch (vm::VmVirtError& err) { + return td::Status::Error(PSTRING() << "invalid proof: " << err.get_msg()); + } +} + +void BuildOutMsgQueueProof::abort_query(td::Status reason) { + if (promise_) { + FLOG(DEBUG) { + sb << "failed to build msg queue proof to " << dst_shard_.to_str() << " from"; + for (const auto& block : blocks_) { + sb << " " << block.id.id.to_str(); + } + sb << ": " << reason; + }; + promise_.set_error( + reason.move_as_error_prefix(PSTRING() << "failed to build msg queue proof to " << dst_shard_.to_str() << ": ")); + } + stop(); +} + +void BuildOutMsgQueueProof::start_up() { + for (size_t i = 0; i < blocks_.size(); ++i) { + BlockIdExt id = blocks_[i].id; + ++pending; + td::actor::send_closure(manager_, &ValidatorManagerInterface::get_shard_state_from_db_short, id, + [SelfId = actor_id(this), i](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, + R.move_as_error_prefix("failed to get shard state: ")); + } else { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_state_root, i, + R.move_as_ok()->root_cell()); + } + }); + if (id.seqno() != 0) { + ++pending; + td::actor::send_closure(manager_, &ValidatorManagerInterface::get_block_data_from_db_short, id, + [SelfId = actor_id(this), i](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::abort_query, + R.move_as_error_prefix("failed to get block data: ")); + } else { + td::actor::send_closure(SelfId, &BuildOutMsgQueueProof::got_block_root, i, + R.move_as_ok()->root_cell()); + } + }); + } + } + if (pending == 0) { + build_proof(); + } +} + +void BuildOutMsgQueueProof::got_state_root(size_t i, Ref root) { + blocks_[i].state_root = std::move(root); + if (--pending == 0) { + build_proof(); + } +} + +void BuildOutMsgQueueProof::got_block_root(size_t i, Ref root) { + blocks_[i].block_root = std::move(root); + if (--pending == 0) { + build_proof(); + } +} + +void BuildOutMsgQueueProof::build_proof() { + auto result = OutMsgQueueProof::build(dst_shard_, std::move(blocks_), limits_); + if (result.is_error()) { + LOG(ERROR) << "Failed to build msg queue proof: " << result.error(); + } + promise_.set_result(std::move(result)); + stop(); +} + +} // namespace validator +} // namespace ton diff --git a/validator/impl/out-msg-queue-proof.hpp b/validator/impl/out-msg-queue-proof.hpp new file mode 100644 index 00000000..e28561e2 --- /dev/null +++ b/validator/impl/out-msg-queue-proof.hpp @@ -0,0 +1,64 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "vm/cells.h" +#include "ton/ton-types.h" +#include "auto/tl/ton_api.h" +#include "interfaces/out-msg-queue-proof.h" +#include "td/actor/actor.h" +#include "interfaces/shard.h" +#include "validator.h" + +namespace ton { + +namespace validator { +using td::Ref; + +class ValidatorManager; +class ValidatorManagerInterface; + +class BuildOutMsgQueueProof : public td::actor::Actor { + public: + BuildOutMsgQueueProof(ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, + td::actor::ActorId manager, + td::Promise> promise) + : dst_shard_(dst_shard), limits_(limits), manager_(manager), promise_(std::move(promise)) { + blocks_.resize(blocks.size()); + for (size_t i = 0; i < blocks_.size(); ++i) { + blocks_[i].id = blocks[i]; + } + } + + void abort_query(td::Status reason); + void start_up() override; + void got_state_root(size_t i, Ref root); + void got_block_root(size_t i, Ref root); + void build_proof(); + + private: + ShardIdFull dst_shard_; + std::vector blocks_; + block::ImportedMsgQueueLimits limits_; + + td::actor::ActorId manager_; + td::Promise> promise_; + + size_t pending = 0; +}; + +} // namespace validator +} // namespace ton diff --git a/validator/impl/proof.cpp b/validator/impl/proof.cpp index 033a1ab1..d7222211 100644 --- a/validator/impl/proof.cpp +++ b/validator/impl/proof.cpp @@ -162,5 +162,40 @@ td::Result> ProofQ::get_signatures_root() const { return proof.signatures->prefetch_ref(); } +td::Result> create_block_state_proof(td::Ref root) { + if (root.is_null()) { + return td::Status::Error("root is null"); + } + vm::MerkleProofBuilder mpb{std::move(root)}; + block::gen::Block::Record block; + if (!tlb::unpack_cell(mpb.root(), block) || block.state_update->load_cell().is_error()) { + return td::Status::Error("invalid block"); + } + TRY_RESULT(proof, mpb.extract_proof()); + if (proof.is_null()) { + return td::Status::Error("failed to create proof"); + } + return proof; +} + +td::Result unpack_block_state_proof(BlockIdExt block_id, td::Ref proof) { + auto virt_root = vm::MerkleProof::virtualize(proof, 1); + if (virt_root.is_null()) { + return td::Status::Error("invalid Merkle proof"); + } + if (virt_root->get_hash().as_slice() != block_id.root_hash.as_slice()) { + return td::Status::Error("hash mismatch"); + } + block::gen::Block::Record block; + if (!tlb::unpack_cell(virt_root, block)) { + return td::Status::Error("invalid block"); + } + vm::CellSlice upd_cs{vm::NoVmSpec(), block.state_update}; + if (!(upd_cs.is_special() && upd_cs.prefetch_long(8) == 4 && upd_cs.size_ext() == 0x20228)) { + return td::Status::Error("invalid Merkle update"); + } + return upd_cs.prefetch_ref(1)->get_hash(0).bits(); +} + } // namespace validator } // namespace ton diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index d07d533e..fa36e1e6 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -160,6 +160,9 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { return td::make_ref(config_); } } + block::WorkchainSet get_workchain_list() const override { + return config_ ? config_->get_workchain_list() : block::WorkchainSet(); + } private: ZeroStateIdExt zerostate_id_; diff --git a/validator/impl/signature-set.cpp b/validator/impl/signature-set.cpp index c7298216..0078a115 100644 --- a/validator/impl/signature-set.cpp +++ b/validator/impl/signature-set.cpp @@ -42,9 +42,6 @@ td::BufferSlice BlockSignatureSetQ::serialize() const { } Ref root; CHECK(serialize_to(root)); - //std::cerr << "serializing BlockSignatureSet: "; - //vm::CellSlice{vm::NoVm{}, root}.print_rec(std::cerr); - //std::cerr << std::endl; auto res = vm::std_boc_serialize(std::move(root)); LOG_CHECK(res.is_ok()) << res.move_as_error(); return res.move_as_ok(); diff --git a/validator/impl/top-shard-descr.cpp b/validator/impl/top-shard-descr.cpp index 8ff8862d..9eadeef3 100644 --- a/validator/impl/top-shard-descr.cpp +++ b/validator/impl/top-shard-descr.cpp @@ -175,9 +175,11 @@ td::Status ShardTopBlockDescrQ::unpack() { block::gen::TopBlockDescr::Record rec; if (!(block::gen::t_TopBlockDescr.force_validate_ref(root_) && tlb::unpack_cell(root_, rec) && block::tlb::t_BlockIdExt.unpack(rec.proof_for.write(), block_id_))) { - std::cerr << "invalid ShardTopBlockDescr: "; - block::gen::t_TopBlockDescr.print_ref(std::cerr, root_); - vm::load_cell_slice(root_).print_rec(std::cerr); + FLOG(INFO) { + sb << "invalid ShardTopBlockDescr: "; + block::gen::t_TopBlockDescr.print_ref(sb, root_); + vm::load_cell_slice(root_).print_rec(sb); + }; return td::Status::Error(-666, "Shard top block description is not a valid TopBlockDescr TL-B object"); } LOG(DEBUG) << "unpacking a ShardTopBlockDescr for " << block_id_.to_str() << " with " << rec.len << " links"; diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 8490567e..90966d82 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -115,7 +115,7 @@ bool ValidateQuery::reject_query(std::string error, td::BufferSlice reason) { error = error_ctx() + error; LOG(ERROR) << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error; if (main_promise) { - record_stats(); + record_stats(false); errorlog::ErrorLog::log(PSTRING() << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error << ": data=" << block_candidate.id.file_hash.to_hex() << " collated_data=" << block_candidate.collated_file_hash.to_hex()); @@ -153,7 +153,7 @@ bool ValidateQuery::soft_reject_query(std::string error, td::BufferSlice reason) error = error_ctx() + error; LOG(ERROR) << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error; if (main_promise) { - record_stats(); + record_stats(false); errorlog::ErrorLog::log(PSTRING() << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error << ": data=" << block_candidate.id.file_hash.to_hex() << " collated_data=" << block_candidate.collated_file_hash.to_hex()); @@ -176,7 +176,7 @@ bool ValidateQuery::fatal_error(td::Status error) { error.ensure_error(); LOG(ERROR) << "aborting validation of block candidate for " << shard_.to_str() << " : " << error.to_string(); if (main_promise) { - record_stats(); + record_stats(false); auto c = error.code(); if (c <= -667 && c >= -670) { errorlog::ErrorLog::log(PSTRING() << "FATAL ERROR: aborting validation of block candidate for " << shard_.to_str() @@ -234,7 +234,7 @@ bool ValidateQuery::fatal_error(std::string err_msg, int err_code) { */ void ValidateQuery::finish_query() { if (main_promise) { - record_stats(); + record_stats(true); LOG(WARNING) << "validate query done"; main_promise.set_result(now_); } @@ -346,16 +346,7 @@ void ValidateQuery::start_up() { // return; } } - // 2. learn latest masterchain state and block id - LOG(DEBUG) << "sending get_top_masterchain_state_block() to Manager"; - ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::get_top_masterchain_state_block, - [self = get_self()](td::Result, BlockIdExt>> res) { - LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; - td::actor::send_closure_later( - std::move(self), &ValidateQuery::after_get_latest_mc_state, std::move(res)); - }); - // 3. load state(s) corresponding to previous block(s) + // 2. load state(s) corresponding to previous block(s) prev_states.resize(prev_blocks.size()); for (int i = 0; (unsigned)i < prev_blocks.size(); i++) { // 3.1. load state @@ -368,21 +359,13 @@ void ValidateQuery::start_up() { std::move(self), &ValidateQuery::after_get_shard_state, i, std::move(res)); }); } - // 4. unpack block candidate (while necessary data is being loaded) + // 3. unpack block candidate (while necessary data is being loaded) if (!unpack_block_candidate()) { reject_query("error unpacking block candidate"); return; } - // 5. request masterchain state referred to in the block + // 4. request masterchain handle and state referred to in the block if (!is_masterchain()) { - ++pending; - td::actor::send_closure_later(manager, &ValidatorManager::wait_block_state_short, mc_blkid_, priority(), timeout, - [self = get_self()](td::Result> res) { - LOG(DEBUG) << "got answer to wait_block_state() query for masterchain block"; - td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_mc_state, - std::move(res)); - }); - // 5.1. request corresponding block handle ++pending; td::actor::send_closure_later(manager, &ValidatorManager::get_block_handle, mc_blkid_, true, [self = get_self()](td::Result res) { @@ -663,6 +646,19 @@ bool ValidateQuery::extract_collated_data() { return true; } +/** + * Send get_top_masterchain_state_block to manager, call after_get_latest_mc_state afterwards + */ +void ValidateQuery::request_latest_mc_state() { + ++pending; + td::actor::send_closure_later(manager, &ValidatorManager::get_top_masterchain_state_block, + [self = get_self()](td::Result, BlockIdExt>> res) { + LOG(DEBUG) << "got answer to get_top_masterchain_state_block"; + td::actor::send_closure_later( + std::move(self), &ValidateQuery::after_get_latest_mc_state, std::move(res)); + }); +} + /** * Callback function called after retrieving the latest masterchain state. * @@ -710,6 +706,7 @@ void ValidateQuery::after_get_latest_mc_state(td::Result> res) { + CHECK(!is_masterchain()); LOG(WARNING) << "in ValidateQuery::after_get_mc_state() for " << mc_blkid_.to_str(); --pending; if (res.is_error()) { @@ -720,6 +717,7 @@ void ValidateQuery::after_get_mc_state(td::Result> res) { fatal_error("cannot process masterchain state for "s + mc_blkid_.to_str()); return; } + request_latest_mc_state(); if (!pending) { if (!try_validate()) { fatal_error("cannot validate new block"); @@ -734,17 +732,21 @@ void ValidateQuery::after_get_mc_state(td::Result> res) { */ void ValidateQuery::got_mc_handle(td::Result res) { LOG(DEBUG) << "in ValidateQuery::got_mc_handle() for " << mc_blkid_.to_str(); - --pending; if (res.is_error()) { fatal_error(res.move_as_error()); return; } - auto handle = res.move_as_ok(); - if (!handle->inited_proof() && mc_blkid_.seqno()) { - fatal_error(-666, "reference masterchain block "s + mc_blkid_.to_str() + " for block " + id_.to_str() + - " does not have a valid proof"); - return; - } + auto mc_handle = res.move_as_ok(); + td::actor::send_closure_later( + manager, &ValidatorManager::wait_block_state, mc_handle, priority(), timeout, + [self = get_self(), id = id_, mc_handle](td::Result> res) { + LOG(DEBUG) << "got answer to wait_block_state() query for masterchain block"; + if (res.is_ok() && mc_handle->id().seqno() > 0 && !mc_handle->inited_proof()) { + res = td::Status::Error(-666, "reference masterchain block "s + mc_handle->id().to_str() + " for block " + + id.to_str() + " does not have a valid proof"); + } + td::actor::send_closure_later(std::move(self), &ValidateQuery::after_get_mc_state, std::move(res)); + }); } /** @@ -778,6 +780,9 @@ void ValidateQuery::after_get_shard_state(int idx, td::Result> r return; } } + if (is_masterchain()) { + request_latest_mc_state(); + } if (!pending) { if (!try_validate()) { fatal_error("cannot validate new block"); @@ -997,7 +1002,12 @@ bool ValidateQuery::fetch_config_params() { action_phase_cfg_.bounce_on_fail_enabled = config_->get_global_version() >= 4; action_phase_cfg_.message_skip_enabled = config_->get_global_version() >= 8; action_phase_cfg_.disable_custom_fess = config_->get_global_version() >= 8; + action_phase_cfg_.reserve_extra_enabled = config_->get_global_version() >= 9; action_phase_cfg_.mc_blackhole_addr = config_->get_burning_config().blackhole_addr; + action_phase_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; + } + { + serialize_cfg_.extra_currency_v2 = config_->get_global_version() >= 10; } { // fetch block_grams_created @@ -1547,8 +1557,10 @@ void ValidateQuery::got_neighbor_out_queue(int i, td::Result> // unpack ProcessedUpto LOG(DEBUG) << "unpacking ProcessedUpto of neighbor " << descr.blk_.to_str(); if (verbosity >= 2) { - block::gen::t_ProcessedInfo.print(std::cerr, qinfo.proc_info); - qinfo.proc_info->print_rec(std::cerr); + FLOG(INFO) { + block::gen::t_ProcessedInfo.print(sb, qinfo.proc_info); + qinfo.proc_info->print_rec(sb); + }; } descr.processed_upto = block::MsgProcessedUptoCollection::unpack(descr.shard(), qinfo.proc_info); if (!descr.processed_upto) { @@ -2650,7 +2662,6 @@ bool ValidateQuery::unpack_precheck_value_flow(Ref value_flow_root) { " but the sum over all accounts present in the new state is " + cc.to_str()); } auto msg_extra = in_msg_dict_->get_root_extra(); - // block::gen::t_ImportFees.print(std::cerr, msg_extra); if (!(block::tlb::t_Grams.as_integer_skip_to(msg_extra.write(), import_fees_) && cc.unpack(std::move(msg_extra)))) { return reject_query("cannot unpack ImportFees from the augmentation of the InMsgDescr dictionary"); } @@ -2754,20 +2765,22 @@ bool ValidateQuery::precheck_one_account_update(td::ConstBitPtr acc_id, Reflookup(acc_id, 256); if (acc_blk_root.is_null()) { if (verbosity >= 3 * 0) { - std::cerr << "state of account " << workchain() << ":" << acc_id.to_hex(256) - << " in the old shardchain state:" << std::endl; - if (old_value.not_null()) { - block::gen::t_ShardAccount.print(std::cerr, *old_value); - } else { - std::cerr << "" << std::endl; - } - std::cerr << "state of account " << workchain() << ":" << acc_id.to_hex(256) - << " in the new shardchain state:" << std::endl; - if (new_value.not_null()) { - block::gen::t_ShardAccount.print(std::cerr, *new_value); - } else { - std::cerr << "" << std::endl; - } + FLOG(INFO) { + sb << "state of account " << workchain() << ":" << acc_id.to_hex(256) + << " in the old shardchain state:" << "\n"; + if (old_value.not_null()) { + block::gen::t_ShardAccount.print(sb, old_value); + } else { + sb << "" << "\n"; + } + sb << "state of account " << workchain() << ":" << acc_id.to_hex(256) + << " in the new shardchain state:" << "\n"; + if (new_value.not_null()) { + block::gen::t_ShardAccount.print(sb, new_value); + } else { + sb << "" << "\n"; + } + }; } return reject_query("the state of account "s + acc_id.to_hex(256) + " changed in the new state with respect to the old state, but the block contains no " @@ -2925,8 +2938,6 @@ bool ValidateQuery::precheck_one_account_block(td::ConstBitPtr acc_id, Refprint_rec(std::cerr); - // block::gen::t_AccountBlock.print(std::cerr, acc_blk_root); block::gen::AccountBlock::Record acc_blk; block::gen::HASH_UPDATE::Record hash_upd; if (!(tlb::csr_unpack(acc_blk_root, acc_blk) && @@ -3854,7 +3865,9 @@ bool ValidateQuery::check_in_msg(td::ConstBitPtr key, Ref in_msg) ton::LogicalTime trans_lt; CHECK(block::get_transaction_id(transaction, trans_addr, trans_lt)); if (dest_addr != trans_addr) { - block::gen::t_InMsg.print(std::cerr, *in_msg); + FLOG(INFO) { + block::gen::t_InMsg.print(sb, in_msg); + }; return reject_query(PSTRING() << "InMsg corresponding to inbound message with hash " << key.to_hex(256) << " and destination address " << dest_addr.to_hex() << " claims that the message is processed by transaction " << trans_lt @@ -4402,7 +4415,9 @@ bool ValidateQuery::check_out_msg(td::ConstBitPtr key, Ref out_ms ton::LogicalTime trans_lt; CHECK(block::get_transaction_id(transaction, trans_addr, trans_lt)); if (src_addr != trans_addr) { - block::gen::t_OutMsg.print(std::cerr, *out_msg); + FLOG(INFO) { + block::gen::t_OutMsg.print(sb, out_msg); + }; return reject_query(PSTRING() << "OutMsg corresponding to outbound message with hash " << key.to_hex(256) << " and source address " << src_addr.to_hex() << " claims that the message was created by transaction " << trans_lt @@ -5016,15 +5031,19 @@ bool ValidateQuery::check_in_queue() { LOG(DEBUG) << "processing inbound message with (lt,hash)=(" << kv->lt << "," << kv->key.to_hex() << ") from neighbor #" << kv->source; if (verbosity > 3) { - std::cerr << "inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() << " msg="; - block::gen::t_EnqueuedMsg.print(std::cerr, *(kv->msg)); + FLOG(INFO) { + sb << "inbound message: lt=" << kv->lt << " from=" << kv->source << " key=" << kv->key.to_hex() << " msg="; + block::gen::t_EnqueuedMsg.print(sb, kv->msg); + }; } bool unprocessed = false; if (!check_neighbor_outbound_message(kv->msg, kv->lt, kv->key.cbits(), neighbors_.at(kv->source), unprocessed)) { if (verbosity > 1) { - std::cerr << "invalid neighbor outbound message: lt=" << kv->lt << " from=" << kv->source - << " key=" << kv->key.to_hex() << " msg="; - block::gen::t_EnqueuedMsg.print(std::cerr, *(kv->msg)); + FLOG(INFO) { + sb << "invalid neighbor outbound message: lt=" << kv->lt << " from=" << kv->source + << " key=" << kv->key.to_hex() << " msg="; + block::gen::t_EnqueuedMsg.print(sb, kv->msg); + }; } return reject_query("error processing outbound internal message "s + kv->key.to_hex() + " of neighbor " + neighbors_.at(kv->source).blk_.to_str()); @@ -5593,7 +5612,7 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT return reject_query(PSTRING() << "cannot re-create bounce phase of transaction " << lt << " for smart contract " << addr.to_hex()); } - if (!trs->serialize()) { + if (!trs->serialize(serialize_cfg_)) { return reject_query(PSTRING() << "cannot re-create the serialization of transaction " << lt << " for smart contract " << addr.to_hex()); } @@ -5630,10 +5649,12 @@ bool ValidateQuery::check_one_transaction(block::Account& account, ton::LogicalT // now compare the re-created transaction with the one we have if (trans_root2->get_hash() != trans_root->get_hash()) { if (verbosity >= 3 * 0) { - std::cerr << "original transaction " << lt << " of " << addr.to_hex() << ": "; - block::gen::t_Transaction.print_ref(std::cerr, trans_root); - std::cerr << "re-created transaction " << lt << " of " << addr.to_hex() << ": "; - block::gen::t_Transaction.print_ref(std::cerr, trans_root2); + FLOG(INFO) { + sb << "original transaction " << lt << " of " << addr.to_hex() << ": "; + block::gen::t_Transaction.print_ref(sb, trans_root); + sb << "re-created transaction " << lt << " of " << addr.to_hex() << ": "; + block::gen::t_Transaction.print_ref(sb, trans_root2); + }; } return reject_query(PSTRING() << "the transaction " << lt << " of " << addr.to_hex() << " has hash " << trans_root->get_hash().to_hex() @@ -6911,13 +6932,13 @@ void ValidateQuery::written_candidate() { /** * Sends validation work time to manager. */ -void ValidateQuery::record_stats() { +void ValidateQuery::record_stats(bool success) { double work_time = work_timer_.elapsed(); double cpu_work_time = cpu_work_timer_.elapsed(); LOG(WARNING) << "validation took " << perf_timer_.elapsed() << "s"; LOG(WARNING) << "Validate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; td::actor::send_closure(manager, &ValidatorManager::record_validate_query_stats, block_candidate.id, work_time, - cpu_work_time); + cpu_work_time, success); } } // namespace validator diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 52d4968a..60f0cc8a 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -205,6 +205,7 @@ class ValidateQuery : public td::actor::Actor { block::StoragePhaseConfig storage_phase_cfg_{&storage_prices_}; block::ComputePhaseConfig compute_phase_cfg_; block::ActionPhaseConfig action_phase_cfg_; + block::SerializeConfig serialize_cfg_; td::RefInt256 masterchain_create_fee_, basechain_create_fee_; std::vector neighbors_; @@ -284,6 +285,7 @@ class ValidateQuery : public td::actor::Actor { return actor_id(this); } + void request_latest_mc_state(); void after_get_latest_mc_state(td::Result, BlockIdExt>> res); void after_get_mc_state(td::Result> res); void got_mc_handle(td::Result res); @@ -399,7 +401,7 @@ class ValidateQuery : public td::actor::Actor { td::Timer work_timer_{true}; td::ThreadCpuTimer cpu_work_timer_{true}; - void record_stats(); + void record_stats(bool success); }; } // namespace validator diff --git a/validator/impl/validator-set.cpp b/validator/impl/validator-set.cpp index 629337cf..d4b0d647 100644 --- a/validator/impl/validator-set.cpp +++ b/validator/impl/validator-set.cpp @@ -28,14 +28,14 @@ namespace ton { namespace validator { using td::Ref; -const ValidatorDescr *ValidatorSetQ::find_validator(const NodeIdShort &id) const { +const ValidatorDescr *ValidatorSetQ::get_validator(const NodeIdShort &id) const { auto it = std::lower_bound(ids_map_.begin(), ids_map_.end(), id, [](const auto &p, const auto &x) { return p.first < x; }); return it < ids_map_.end() && it->first == id ? &ids_[it->second] : nullptr; } bool ValidatorSetQ::is_validator(NodeIdShort id) const { - return find_validator(id); + return get_validator(id); } td::Result ValidatorSetQ::check_signatures(RootHash root_hash, FileHash file_hash, @@ -53,7 +53,7 @@ td::Result ValidatorSetQ::check_signatures(RootHash root_hash, } nodes.insert(sig.node); - auto vdescr = find_validator(sig.node); + auto vdescr = get_validator(sig.node); if (!vdescr) { return td::Status::Error(ErrorCode::protoviolation, "unknown node to sign"); } @@ -84,7 +84,7 @@ td::Result ValidatorSetQ::check_approve_signatures(RootHash roo } nodes.insert(sig.node); - auto vdescr = find_validator(sig.node); + auto vdescr = get_validator(sig.node); if (!vdescr) { return td::Status::Error(ErrorCode::protoviolation, "unknown node to sign"); } diff --git a/validator/impl/validator-set.hpp b/validator/impl/validator-set.hpp index 3141f36c..951ca4b7 100644 --- a/validator/impl/validator-set.hpp +++ b/validator/impl/validator-set.hpp @@ -32,6 +32,7 @@ namespace validator { class ValidatorSetQ : public ValidatorSet { public: + const ValidatorDescr* get_validator(const NodeIdShort& id) const override; bool is_validator(NodeIdShort id) const override; CatchainSeqno get_catchain_seqno() const override { return cc_seqno_; @@ -62,8 +63,6 @@ class ValidatorSetQ : public ValidatorSet { ValidatorWeight total_weight_; std::vector ids_; std::vector> ids_map_; - - const ValidatorDescr* find_validator(const NodeIdShort& id) const; }; class ValidatorSetCompute { diff --git a/validator/import-db-slice.cpp b/validator/import-db-slice.cpp index a93fb05b..06573d34 100644 --- a/validator/import-db-slice.cpp +++ b/validator/import-db-slice.cpp @@ -17,6 +17,7 @@ Copyright 2019-2020 Telegram Systems LLP */ #include "import-db-slice.hpp" + #include "validator/db/fileref.hpp" #include "td/utils/overloaded.h" #include "validator/fabric.h" @@ -26,35 +27,91 @@ #include "ton/ton-io.hpp" #include "downloaders/download-state.hpp" +#include + namespace ton { namespace validator { -ArchiveImporter::ArchiveImporter(std::string path, td::Ref state, BlockSeqno shard_client_seqno, +ArchiveImporter::ArchiveImporter(std::string db_root, td::Ref state, BlockSeqno shard_client_seqno, td::Ref opts, td::actor::ActorId manager, - td::Promise> promise) - : path_(std::move(path)) - , state_(std::move(state)) + std::vector to_import_files, + td::Promise> promise) + : db_root_(std::move(db_root)) + , last_masterchain_state_(std::move(state)) , shard_client_seqno_(shard_client_seqno) + , start_import_seqno_(shard_client_seqno + 1) , opts_(std::move(opts)) , manager_(manager) + , to_import_files_(std::move(to_import_files)) + , use_imported_files_(!to_import_files_.empty()) , promise_(std::move(promise)) { } void ArchiveImporter::start_up() { - auto R = Package::open(path_, false, false); - if (R.is_error()) { - abort_query(R.move_as_error()); + if (use_imported_files_) { + LOG(INFO) << "Importing archive for masterchain seqno #" << start_import_seqno_ << " from disk"; + for (const std::string& path : to_import_files_) { + LOG(INFO) << "Importing file from disk " << path; + td::Status S = process_package(path, true); + if (S.is_error()) { + LOG(INFO) << "Error processing package " << path << ": " << S; + } + } + files_to_cleanup_.clear(); + processed_mc_archive(); return; } - package_ = std::make_shared(R.move_as_ok()); + LOG(INFO) << "Importing archive for masterchain seqno #" << start_import_seqno_ << " from net"; + td::actor::send_closure(manager_, &ValidatorManager::send_download_archive_request, start_import_seqno_, + ShardIdFull{masterchainId}, db_root_ + "/tmp/", td::Timestamp::in(3600.0), + [SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &ArchiveImporter::abort_query, R.move_as_error()); + } else { + td::actor::send_closure(SelfId, &ArchiveImporter::downloaded_mc_archive, R.move_as_ok()); + } + }); +} - bool fail = false; - package_->iterate([&](std::string filename, td::BufferSlice data, td::uint64 offset) -> bool { +void ArchiveImporter::downloaded_mc_archive(std::string path) { + td::Status S = process_package(path, true); + if (S.is_error()) { + abort_query(std::move(S)); + return; + } + processed_mc_archive(); +} + +void ArchiveImporter::processed_mc_archive() { + if (masterchain_blocks_.empty()) { + LOG(DEBUG) << "No masterhchain blocks in archive"; + last_masterchain_seqno_ = last_masterchain_state_->get_seqno(); + checked_all_masterchain_blocks(); + return; + } + + auto seqno = masterchain_blocks_.begin()->first; + LOG(DEBUG) << "First mc seqno in archive = " << seqno; + if (seqno > last_masterchain_state_->get_seqno() + 1) { + abort_query(td::Status::Error(ErrorCode::notready, "too big first masterchain seqno")); + return; + } + + check_masterchain_block(seqno); +} + +td::Status ArchiveImporter::process_package(std::string path, bool with_masterchain) { + LOG(DEBUG) << "Processing package " << path << " (with_masterchain=" << with_masterchain << ")"; + files_to_cleanup_.push_back(path); + TRY_RESULT(p, Package::open(path, false, false)); + auto package = std::make_shared(std::move(p)); + + td::Status S = td::Status::OK(); + package->iterate([&](std::string filename, td::BufferSlice, td::uint64 offset) -> bool { auto F = FileReference::create(filename); if (F.is_error()) { - abort_query(F.move_as_error()); - fail = true; + S = F.move_as_error(); return false; } auto f = F.move_as_ok(); @@ -79,33 +136,26 @@ void ArchiveImporter::start_up() { ignore = false; is_proof = false; }, - [&](const auto &p) { ignore = true; })); + [&](const auto &) { ignore = true; })); - if (!ignore) { - blocks_[b][is_proof ? 0 : 1] = offset; + if (!ignore && (with_masterchain || !b.is_masterchain())) { + if (is_proof) { + blocks_[b].proof_pkg = package; + blocks_[b].proof_offset = offset; + } else { + blocks_[b].data_pkg = package; + blocks_[b].data_offset = offset; + } if (b.is_masterchain()) { masterchain_blocks_[b.seqno()] = b; + last_masterchain_seqno_ = std::max(last_masterchain_seqno_, b.seqno()); + } else { + have_shard_blocks_ = true; } } return true; }); - - if (fail) { - return; - } - - if (masterchain_blocks_.size() == 0) { - abort_query(td::Status::Error(ErrorCode::notready, "archive does not contain any masterchain blocks")); - return; - } - - auto seqno = masterchain_blocks_.begin()->first; - if (seqno > state_->get_seqno() + 1) { - abort_query(td::Status::Error(ErrorCode::notready, "too big first masterchain seqno")); - return; - } - - check_masterchain_block(seqno); + return S; } void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { @@ -115,17 +165,17 @@ void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { abort_query(td::Status::Error(ErrorCode::notready, "no new blocks")); return; } - checked_all_masterchain_blocks(seqno - 1); + checked_all_masterchain_blocks(); return; } - while (seqno <= state_->get_block_id().seqno()) { - if (seqno < state_->get_block_id().seqno()) { - if (!state_->check_old_mc_block_id(it->second)) { + while (seqno <= last_masterchain_state_->get_block_id().seqno()) { + if (seqno < last_masterchain_state_->get_block_id().seqno()) { + if (!last_masterchain_state_->check_old_mc_block_id(it->second)) { abort_query(td::Status::Error(ErrorCode::protoviolation, "bad old masterchain block id")); return; } } else { - if (state_->get_block_id() != it->second) { + if (last_masterchain_state_->get_block_id() != it->second) { abort_query(td::Status::Error(ErrorCode::protoviolation, "bad old masterchain block id")); return; } @@ -133,18 +183,27 @@ void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { seqno++; it = masterchain_blocks_.find(seqno); if (it == masterchain_blocks_.end()) { - checked_all_masterchain_blocks(seqno - 1); + checked_all_masterchain_blocks(); return; } } - if (seqno != state_->get_block_id().seqno() + 1) { + LOG(DEBUG) << "Checking masterchain block #" << seqno; + if (seqno != last_masterchain_state_->get_block_id().seqno() + 1) { abort_query(td::Status::Error(ErrorCode::protoviolation, "hole in masterchain seqno")); return; } auto it2 = blocks_.find(it->second); CHECK(it2 != blocks_.end()); + if (!it2->second.proof_pkg) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block proof")); + return; + } + if (!it2->second.data_pkg) { + abort_query(td::Status::Error(ErrorCode::protoviolation, "no masterchain block data")); + return; + } - auto R1 = package_->read(it2->second[0]); + auto R1 = it2->second.proof_pkg->read(it2->second.proof_offset); if (R1.is_error()) { abort_query(R1.move_as_error()); return; @@ -156,7 +215,7 @@ void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { return; } - auto R2 = package_->read(it2->second[1]); + auto R2 = it2->second.data_pkg->read(it2->second.data_offset); if (R2.is_error()) { abort_query(R2.move_as_error()); return; @@ -175,7 +234,7 @@ void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { auto proof = proofR.move_as_ok(); auto data = dataR.move_as_ok(); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id = state_->get_block_id(), + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id = last_masterchain_state_->get_block_id(), data](td::Result R) mutable { if (R.is_error()) { td::actor::send_closure(SelfId, &ArchiveImporter::abort_query, R.move_as_error()); @@ -191,11 +250,12 @@ void ArchiveImporter::check_masterchain_block(BlockSeqno seqno) { td::actor::send_closure(SelfId, &ArchiveImporter::checked_masterchain_proof, std::move(handle), std::move(data)); }); - run_check_proof_query(it->second, std::move(proof), manager_, td::Timestamp::in(2.0), std::move(P), state_, - opts_->is_hardfork(it->second)); + run_check_proof_query(it->second, std::move(proof), manager_, td::Timestamp::in(2.0), std::move(P), + last_masterchain_state_, opts_->is_hardfork(it->second)); } void ArchiveImporter::checked_masterchain_proof(BlockHandle handle, td::Ref data) { + LOG(DEBUG) << "Checked proof for masterchain block #" << handle->id().seqno(); CHECK(data.not_null()); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result R) { R.ensure(); @@ -205,6 +265,7 @@ void ArchiveImporter::checked_masterchain_proof(BlockHandle handle, td::Refid().seqno(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); td::actor::send_closure(SelfId, &ArchiveImporter::got_new_materchain_state, @@ -214,22 +275,87 @@ void ArchiveImporter::applied_masterchain_block(BlockHandle handle) { } void ArchiveImporter::got_new_materchain_state(td::Ref state) { - state_ = std::move(state); - check_masterchain_block(state_->get_block_id().seqno() + 1); + last_masterchain_state_ = std::move(state); + imported_any_ = true; + check_masterchain_block(last_masterchain_state_->get_block_id().seqno() + 1); } -void ArchiveImporter::checked_all_masterchain_blocks(BlockSeqno seqno) { - check_next_shard_client_seqno(shard_client_seqno_ + 1); +void ArchiveImporter::checked_all_masterchain_blocks() { + LOG(DEBUG) << "Done importing masterchain blocks. Last block seqno = " << last_masterchain_seqno_; + if (start_import_seqno_ > last_masterchain_state_->get_seqno()) { + abort_query(td::Status::Error("no new masterchain blocks were imported")); + return; + } + BlockIdExt block_id; + CHECK(last_masterchain_state_->get_old_mc_block_id(start_import_seqno_, block_id)); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db_short, block_id, + [SelfId = actor_id(this)](td::Result> R) { + R.ensure(); + td::Ref state{R.move_as_ok()}; + td::actor::send_closure(SelfId, &ArchiveImporter::download_shard_archives, + std::move(state)); + }); +} + +void ArchiveImporter::download_shard_archives(td::Ref start_state) { + start_state_ = start_state; + td::uint32 monitor_min_split = start_state->monitor_min_split_depth(basechainId); + LOG(DEBUG) << "Monitor min split = " << monitor_min_split; + // If monitor_min_split == 0, we use the old archive format (packages are not separated by shard) + // If masterchain package has shard blocks then it's old archive format, don't need to download shards + if (monitor_min_split > 0 && !have_shard_blocks_ && !use_imported_files_) { + for (td::uint64 i = 0; i < (1ULL << monitor_min_split); ++i) { + ShardIdFull shard_prefix{basechainId, (i * 2 + 1) << (64 - monitor_min_split - 1)}; + if (opts_->need_monitor(shard_prefix, start_state)) { + ++pending_shard_archives_; + LOG(DEBUG) << "Downloading shard archive #" << start_import_seqno_ << " " << shard_prefix.to_str(); + download_shard_archive(shard_prefix); + } + } + } else { + LOG(DEBUG) << "Skip downloading shard archives"; + } + if (pending_shard_archives_ == 0) { + check_next_shard_client_seqno(shard_client_seqno_ + 1); + } +} + +void ArchiveImporter::download_shard_archive(ShardIdFull shard_prefix) { + td::actor::send_closure( + manager_, &ValidatorManager::send_download_archive_request, start_import_seqno_, shard_prefix, db_root_ + "/tmp/", + td::Timestamp::in(3600.0), + [SelfId = actor_id(this), seqno = start_import_seqno_, shard_prefix](td::Result R) { + if (R.is_error()) { + LOG(WARNING) << "Failed to download archive slice #" << seqno << " for shard " << shard_prefix.to_str(); + delay_action( + [=]() { td::actor::send_closure(SelfId, &ArchiveImporter::download_shard_archive, shard_prefix); }, + td::Timestamp::in(2.0)); + } else { + LOG(DEBUG) << "Downloaded shard archive #" << seqno << " " << shard_prefix.to_str(); + td::actor::send_closure(SelfId, &ArchiveImporter::downloaded_shard_archive, R.move_as_ok()); + } + }); +} + +void ArchiveImporter::downloaded_shard_archive(std::string path) { + td::Status S = process_package(path, false); + if (S.is_error()) { + LOG(INFO) << "Error processing package: " << S; + } + --pending_shard_archives_; + if (pending_shard_archives_ == 0) { + check_next_shard_client_seqno(shard_client_seqno_ + 1); + } } void ArchiveImporter::check_next_shard_client_seqno(BlockSeqno seqno) { - if (seqno > state_->get_seqno()) { + if (seqno > last_masterchain_state_->get_seqno() || seqno > last_masterchain_seqno_) { finish_query(); - } else if (seqno == state_->get_seqno()) { - got_masterchain_state(state_); + } else if (seqno == last_masterchain_state_->get_seqno()) { + got_masterchain_state(last_masterchain_state_); } else { BlockIdExt b; - bool f = state_->get_old_mc_block_id(seqno, b); + bool f = last_masterchain_state_->get_old_mc_block_id(seqno, b); CHECK(f); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); @@ -241,33 +367,38 @@ void ArchiveImporter::check_next_shard_client_seqno(BlockSeqno seqno) { } void ArchiveImporter::got_masterchain_state(td::Ref state) { + if (state->get_seqno() != start_import_seqno_ && state->is_key_state()) { + finish_query(); + return; + } + LOG(DEBUG) << "Applying shard client seqno " << state->get_seqno(); auto s = state->get_shards(); - - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), seqno = state->get_seqno()](td::Result R) { + td::MultiPromise mp; + auto ig = mp.init_guard(); + for (auto &shard : s) { + if (opts_->need_monitor(shard->shard(), state)) { + apply_shard_block(shard->top_block_id(), state->get_block_id(), ig.get_promise()); + } + } + ig.add_promise([SelfId = actor_id(this), seqno = state->get_seqno()](td::Result R) { if (R.is_error()) { td::actor::send_closure(SelfId, &ArchiveImporter::abort_query, R.move_as_error()); } else { td::actor::send_closure(SelfId, &ArchiveImporter::checked_shard_client_seqno, seqno); } }); - - td::MultiPromise mp; - auto ig = mp.init_guard(); - ig.add_promise(std::move(P)); - - for (auto &shard : s) { - apply_shard_block(shard->top_block_id(), state->get_block_id(), ig.get_promise()); - } } void ArchiveImporter::checked_shard_client_seqno(BlockSeqno seqno) { CHECK(shard_client_seqno_ + 1 == seqno); shard_client_seqno_++; + imported_any_ = true; check_next_shard_client_seqno(seqno + 1); } void ArchiveImporter::apply_shard_block(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::Promise promise) { + LOG(DEBUG) << "Applying shard block " << block_id.id.to_str(); auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), masterchain_block_id, promise = std::move(promise)](td::Result R) mutable { R.ensure(); @@ -286,7 +417,7 @@ void ArchiveImporter::apply_shard_block_cont1(BlockHandle handle, BlockIdExt mas if (handle->id().seqno() == 0) { auto P = td::PromiseCreator::lambda( - [promise = std::move(promise)](td::Result> R) mutable { promise.set_value(td::Unit()); }); + [promise = std::move(promise)](td::Result>) mutable { promise.set_value(td::Unit()); }); td::actor::create_actor("downloadstate", handle->id(), masterchain_block_id, 2, manager_, td::Timestamp::in(3600), std::move(P)) .release(); @@ -294,12 +425,13 @@ void ArchiveImporter::apply_shard_block_cont1(BlockHandle handle, BlockIdExt mas } auto it = blocks_.find(handle->id()); - if (it == blocks_.end()) { - promise.set_error(td::Status::Error(ErrorCode::notready, PSTRING() << "no proof for shard block " << handle->id())); + if (it == blocks_.end() || !it->second.proof_pkg || !it->second.data_pkg) { + promise.set_error( + td::Status::Error(ErrorCode::notready, PSTRING() << "no data/proof for shard block " << handle->id())); return; } - TRY_RESULT_PROMISE(promise, data, package_->read(it->second[0])); - TRY_RESULT_PROMISE(promise, proof, create_proof_link(handle->id(), std::move(data.second))); + TRY_RESULT_PROMISE(promise, proof_data, it->second.proof_pkg->read(it->second.proof_offset)); + TRY_RESULT_PROMISE(promise, proof, create_proof_link(handle->id(), std::move(proof_data.second))); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle, masterchain_block_id, promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { @@ -345,8 +477,8 @@ void ArchiveImporter::apply_shard_block_cont2(BlockHandle handle, BlockIdExt mas void ArchiveImporter::apply_shard_block_cont3(BlockHandle handle, BlockIdExt masterchain_block_id, td::Promise promise) { auto it = blocks_.find(handle->id()); - CHECK(it != blocks_.end()); - TRY_RESULT_PROMISE(promise, data, package_->read(it->second[1])); + CHECK(it != blocks_.end() && it->second.data_pkg); + TRY_RESULT_PROMISE(promise, data, it->second.data_pkg->read(it->second.data_offset)); if (sha256_bits256(data.second.as_slice()) != handle->id().file_hash) { promise.set_error(td::Status::Error(ErrorCode::protoviolation, "bad block file hash")); return; @@ -367,6 +499,7 @@ void ArchiveImporter::check_shard_block_applied(BlockIdExt block_id, td::Promise if (!handle->is_applied()) { promise.set_error(td::Status::Error(ErrorCode::notready, "not applied")); } else { + LOG(DEBUG) << "Applied shard block " << handle->id().id.to_str(); promise.set_value(td::Unit()); } } @@ -375,13 +508,24 @@ void ArchiveImporter::check_shard_block_applied(BlockIdExt block_id, td::Promise } void ArchiveImporter::abort_query(td::Status error) { - LOG(INFO) << error; + if (!imported_any_) { + for (const std::string &f : files_to_cleanup_) { + td::unlink(f).ignore(); + } + promise_.set_error(std::move(error)); + return; + } + LOG(INFO) << "Archive import: " << error; finish_query(); } + void ArchiveImporter::finish_query() { + for (const std::string &f : files_to_cleanup_) { + td::unlink(f).ignore(); + } if (promise_) { - promise_.set_value( - std::vector{state_->get_seqno(), std::min(state_->get_seqno(), shard_client_seqno_)}); + promise_.set_value({last_masterchain_state_->get_seqno(), + std::min(last_masterchain_state_->get_seqno(), shard_client_seqno_)}); } stop(); } diff --git a/validator/import-db-slice.hpp b/validator/import-db-slice.hpp index 0993d4bb..04f22642 100644 --- a/validator/import-db-slice.hpp +++ b/validator/import-db-slice.hpp @@ -19,6 +19,7 @@ #pragma once #include "td/actor/actor.h" +#include "td/utils/port/path.h" #include "validator/interfaces/validator-manager.h" #include "validator/db/package.hpp" @@ -28,19 +29,27 @@ namespace validator { class ArchiveImporter : public td::actor::Actor { public: - ArchiveImporter(std::string path, td::Ref state, BlockSeqno shard_client_seqno, + ArchiveImporter(std::string db_root, td::Ref state, BlockSeqno shard_client_seqno, td::Ref opts, td::actor::ActorId manager, - td::Promise> promise); + std::vector to_import_files, td::Promise> promise); void start_up() override; void abort_query(td::Status error); void finish_query(); + void downloaded_mc_archive(std::string path); + td::Status process_package(std::string path, bool with_masterchain); + + void processed_mc_archive(); void check_masterchain_block(BlockSeqno seqno); void checked_masterchain_proof(BlockHandle handle, td::Ref data); void applied_masterchain_block(BlockHandle handle); void got_new_materchain_state(td::Ref state); - void checked_all_masterchain_blocks(BlockSeqno seqno); + + void checked_all_masterchain_blocks(); + void download_shard_archives(td::Ref start_state); + void download_shard_archive(ShardIdFull shard_prefix); + void downloaded_shard_archive(std::string path); void check_next_shard_client_seqno(BlockSeqno seqno); void checked_shard_client_seqno(BlockSeqno seqno); @@ -52,19 +61,36 @@ class ArchiveImporter : public td::actor::Actor { void check_shard_block_applied(BlockIdExt block_id, td::Promise promise); private: - std::string path_; - td::Ref state_; + std::string db_root_; + td::Ref last_masterchain_state_; BlockSeqno shard_client_seqno_; + BlockSeqno start_import_seqno_; td::Ref opts_; - std::shared_ptr package_; - td::actor::ActorId manager_; - td::Promise> promise_; + + std::vector to_import_files_; + bool use_imported_files_; + td::Promise> promise_; std::map masterchain_blocks_; - std::map> blocks_; + BlockSeqno last_masterchain_seqno_ = 0; + + struct BlockInfo { + std::shared_ptr data_pkg; + td::uint64 data_offset = 0; + std::shared_ptr proof_pkg; + td::uint64 proof_offset = 0; + }; + std::map blocks_; + + td::Ref start_state_; + size_t pending_shard_archives_ = 0; + + bool imported_any_ = false; + bool have_shard_blocks_ = false; + std::vector files_to_cleanup_; }; } // namespace validator diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index 46620322..29ef715b 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -117,12 +117,18 @@ class Db : public td::actor::Actor { virtual void check_key_block_proof_exists(BlockIdExt block_id, td::Promise promise) = 0; virtual void check_key_block_proof_link_exists(BlockIdExt block_id, td::Promise promise) = 0; - virtual void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) = 0; + virtual void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) = 0; virtual void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise) = 0; virtual void set_async_mode(bool mode, td::Promise promise) = 0; virtual void run_gc(UnixTime mc_ts, UnixTime gc_ts, UnixTime archive_ttl) = 0; + + virtual void add_persistent_state_description(td::Ref desc, + td::Promise promise) = 0; + virtual void get_persistent_state_descriptions( + td::Promise>> promise) = 0; }; } // namespace validator diff --git a/validator/interfaces/out-msg-queue-proof.h b/validator/interfaces/out-msg-queue-proof.h new file mode 100644 index 00000000..c0aa5610 --- /dev/null +++ b/validator/interfaces/out-msg-queue-proof.h @@ -0,0 +1,57 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include "vm/cells.h" +#include "ton/ton-types.h" +#include "auto/tl/ton_api.h" +#include "block/block.h" + +namespace ton { + +namespace validator { +using td::Ref; + +struct OutMsgQueueProof : public td::CntObject { + OutMsgQueueProof(BlockIdExt block_id, Ref state_root, Ref block_state_proof, + td::int32 msg_count = -1) + : block_id_(block_id) + , state_root_(std::move(state_root)) + , block_state_proof_(std::move(block_state_proof)) + , msg_count_(msg_count) { + } + + BlockIdExt block_id_; + Ref state_root_; + Ref block_state_proof_; + td::int32 msg_count_; // -1 - no limit + + static td::Result>> fetch(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + const ton_api::tonNode_outMsgQueueProof &f); + + struct OneBlock { + BlockIdExt id; + Ref state_root; + Ref block_root; + }; + static td::Result> build(ShardIdFull dst_shard, + std::vector blocks, + block::ImportedMsgQueueLimits limits); +}; + +} // namespace validator +} // namespace ton diff --git a/validator/interfaces/proof.h b/validator/interfaces/proof.h index 99471a1f..6665ad08 100644 --- a/validator/interfaces/proof.h +++ b/validator/interfaces/proof.h @@ -48,6 +48,9 @@ class Proof : virtual public ProofLink { virtual td::Result> export_as_proof_link() const = 0; }; +td::Result> create_block_state_proof(td::Ref root); +td::Result unpack_block_state_proof(BlockIdExt block_id, td::Ref proof); + } // namespace validator } // namespace ton diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index 7a731ab5..64aea9b6 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -84,6 +84,7 @@ class MasterchainState : virtual public ShardState { ton::LogicalTime* end_lt = nullptr) const = 0; virtual bool check_old_mc_block_id(const ton::BlockIdExt& blkid, bool strict = false) const = 0; virtual td::Result> get_config_holder() const = 0; + virtual block::WorkchainSet get_workchain_list() const = 0; virtual td::Status prepare() { return td::Status::OK(); } diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index ce0c27e1..00fb77e1 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -66,6 +66,8 @@ using ValidateCandidateResult = td::Variant; class ValidatorManager : public ValidatorManagerInterface { public: + virtual void init_last_masterchain_state(td::Ref state) { + } virtual void set_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; virtual void get_cell_db_reader(td::Promise> promise) = 0; @@ -75,10 +77,6 @@ class ValidatorManager : public ValidatorManagerInterface { std::function write_data, td::Promise promise) = 0; virtual void store_zero_state_file(BlockIdExt block_id, td::BufferSlice state, td::Promise promise) = 0; - virtual void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) = 0; - virtual void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) = 0; virtual void set_block_data(BlockHandle handle, td::Ref data, td::Promise promise) = 0; virtual void wait_block_data(BlockHandle handle, td::uint32 priority, td::Timestamp, @@ -147,10 +145,15 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_ihr_message(td::Ref message) = 0; virtual void send_top_shard_block_description(td::Ref desc) = 0; virtual void send_block_broadcast(BlockBroadcast broadcast, int mode) = 0; + virtual void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) = 0; + virtual void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + td::Promise>> promise) = 0; + virtual void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) = 0; virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_shard_client_state(bool from_db, td::Promise promise) = 0; - virtual void subscribe_to_shard(ShardIdFull shard) = 0; virtual void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) = 0; virtual void get_async_serializer_state(td::Promise promise) = 0; @@ -202,15 +205,17 @@ class ValidatorManager : public ValidatorManagerInterface { td::optional shard, td::Promise> promise) = 0; - virtual void add_lite_query_stats(int lite_query_id) { + virtual void add_lite_query_stats(int lite_query_id, bool success) { } virtual void record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, - CollationStats stats) { + td::optional stats) { } - virtual void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) { + virtual void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, bool success) { } + virtual void add_persistent_state_description(td::Ref desc) = 0; + static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { return ts / (1 << 17) != prev_ts / (1 << 17); } diff --git a/validator/interfaces/validator-set.h b/validator/interfaces/validator-set.h index b71c0bfe..ad7fb9b5 100644 --- a/validator/interfaces/validator-set.h +++ b/validator/interfaces/validator-set.h @@ -30,6 +30,7 @@ namespace validator { class ValidatorSet : public td::CntObject { public: virtual ~ValidatorSet() = default; + virtual const ValidatorDescr* get_validator(const NodeIdShort& id) const = 0; virtual bool is_validator(NodeIdShort id) const = 0; virtual CatchainSeqno get_catchain_seqno() const = 0; virtual td::uint32 get_validator_set_hash() const = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 5287a387..cd06bf55 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -263,11 +263,20 @@ class ValidatorManagerImpl : public ValidatorManager { void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast, int mode) override { } + void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override { + } + void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + td::Promise>> promise) override { + UNREACHABLE(); + } + void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override { + UNREACHABLE(); + } void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; - void subscribe_to_shard(ShardIdFull shard) override { - } void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override { UNREACHABLE(); @@ -279,11 +288,12 @@ class ValidatorManagerImpl : public ValidatorManager { void try_get_static_file(FileHash file_hash, td::Promise promise) override; void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) override { + td::Promise> promise) override { promise.set_error(td::Status::Error(ErrorCode::error, "download disabled")); } - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) override { + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) override { UNREACHABLE(); } void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, @@ -400,8 +410,8 @@ class ValidatorManagerImpl : public ValidatorManager { } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { - queue_size_counter_ = - td::actor::create_actor("queuesizecounter", td::Ref{}, actor_id(this)); + queue_size_counter_ = td::actor::create_actor("queuesizecounter", td::Ref{}, + opts_, actor_id(this)); } td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::get_queue_size, block_id, std::move(promise)); } @@ -435,6 +445,8 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { promise.set_result(td::Status::Error("not implemented")); } + void add_persistent_state_description(td::Ref desc) override { + } void update_options(td::Ref opts) override { opts_ = std::move(opts); diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 2e703faf..0b8b9e73 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -20,6 +20,7 @@ #include "interfaces/validator-manager.h" #include "interfaces/db.h" +#include "ton/ton-types.h" #include "validator-group.hpp" #include "manager-init.h" #include "manager-hardfork.h" @@ -332,6 +333,17 @@ class ValidatorManagerImpl : public ValidatorManager { } void send_block_broadcast(BlockBroadcast broadcast, int mode) override { } + void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override { + } + void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + td::Promise>> promise) override { + UNREACHABLE(); + } + void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override { + UNREACHABLE(); + } void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override { UNREACHABLE(); @@ -339,8 +351,6 @@ class ValidatorManagerImpl : public ValidatorManager { void get_shard_client_state(bool from_db, td::Promise promise) override { UNREACHABLE(); } - void subscribe_to_shard(ShardIdFull shard) override { - } void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override { UNREACHABLE(); @@ -352,11 +362,12 @@ class ValidatorManagerImpl : public ValidatorManager { void try_get_static_file(FileHash file_hash, td::Promise promise) override; void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) override { + td::Promise> promise) override { promise.set_error(td::Status::Error(ErrorCode::error, "download disabled")); } - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) override { + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) override { UNREACHABLE(); } void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, @@ -464,8 +475,8 @@ class ValidatorManagerImpl : public ValidatorManager { } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { - queue_size_counter_ = - td::actor::create_actor("queuesizecounter", td::Ref{}, actor_id(this)); + queue_size_counter_ = td::actor::create_actor("queuesizecounter", td::Ref{}, + opts_, actor_id(this)); } td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::get_queue_size, block_id, std::move(promise)); } @@ -502,6 +513,8 @@ class ValidatorManagerImpl : public ValidatorManager { void update_options(td::Ref opts) override { opts_ = std::move(opts); } + void add_persistent_state_description(td::Ref desc) override { + } private: td::Ref opts_; diff --git a/validator/manager-init.cpp b/validator/manager-init.cpp index 64a0a547..6f304680 100644 --- a/validator/manager-init.cpp +++ b/validator/manager-init.cpp @@ -32,6 +32,8 @@ namespace ton { namespace validator { void ValidatorManagerMasterchainReiniter::start_up() { + status_ = ProcessStatus(manager_, "process.initial_sync"); + status_.set_status(PSTRING() << "starting, init block seqno " << block_id_.seqno()); LOG(INFO) << "init_block_id=" << block_id_; CHECK(block_id_.is_masterchain()); CHECK(block_id_.id.shard == shardIdAll); @@ -58,6 +60,7 @@ void ValidatorManagerMasterchainReiniter::got_masterchain_handle(BlockHandle han key_blocks_.push_back(handle_); if (opts_->initial_sync_disabled()) { + status_.set_status(PSTRING() << "downloading masterchain state " << handle_->id().seqno()); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { R.ensure(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::download_masterchain_state); @@ -181,6 +184,7 @@ void ValidatorManagerMasterchainReiniter::got_next_key_blocks(std::vector(key_blocks_.size()); key_blocks_.resize(key_blocks_.size() + vec.size(), nullptr); @@ -247,6 +251,7 @@ void ValidatorManagerMasterchainReiniter::choose_masterchain_state() { } void ValidatorManagerMasterchainReiniter::download_masterchain_state() { + status_.set_status(PSTRING() << "downloading masterchain state " << block_id_.seqno()); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { LOG(WARNING) << "failed to download masterchain state: " << R.move_as_error(); @@ -268,11 +273,13 @@ void ValidatorManagerMasterchainReiniter::downloaded_masterchain_state(td::Refreceived_state()); CHECK(handle_->is_applied()); LOG(INFO) << "downloaded masterchain state"; + td::actor::send_closure(manager_, &ValidatorManager::init_last_masterchain_state, state_); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { R.ensure(); td::actor::send_closure(SelfId, &ValidatorManagerMasterchainReiniter::downloaded_all_shards); }); client_ = td::actor::create_actor("shardclient", opts_, handle_, state_, manager_, std::move(P)); + status_.set_status(PSTRING() << "downloading all shard states, mc seqno " << block_id_.seqno()); } void ValidatorManagerMasterchainReiniter::downloaded_all_shards() { @@ -567,7 +574,7 @@ void ValidatorManagerMasterchainStarter::truncated() { truncate_shard_next(handle_->id(), ig.get_promise()); auto s = state_->get_shards(); for (auto &shard : s) { - if (opts_->need_monitor(shard->shard())) { + if (opts_->need_monitor(shard->shard(), state_)) { truncate_shard_next(shard->top_block_id(), ig.get_promise()); } } diff --git a/validator/manager-init.hpp b/validator/manager-init.hpp index 7dce4e47..901b826b 100644 --- a/validator/manager-init.hpp +++ b/validator/manager-init.hpp @@ -27,6 +27,8 @@ #include "manager-init.h" +#include + namespace ton { namespace validator { @@ -77,6 +79,8 @@ class ValidatorManagerMasterchainReiniter : public td::actor::Actor { td::uint32 pending_ = 0; td::actor::ActorOwn client_; + + ProcessStatus status_; }; class ValidatorManagerMasterchainStarter : public td::actor::Actor { diff --git a/validator/manager.cpp b/validator/manager.cpp index 67cd60a7..b0ac5409 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -20,11 +20,9 @@ #include "checksum.h" #include "td/utils/buffer.h" #include "validator-group.hpp" -#include "adnl/utils.hpp" #include "downloaders/wait-block-state.hpp" #include "downloaders/wait-block-state-merge.hpp" #include "downloaders/wait-block-data.hpp" -#include "validator-group.hpp" #include "fabric.h" #include "manager.h" #include "validate-broadcast.hpp" @@ -202,7 +200,7 @@ void ValidatorManagerImpl::validate_block(ReceivedBlock block, td::Promise R) mutable { + [SelfId = actor_id(this), promise = std::move(promise), id = blkid](td::Result R) mutable { if (R.is_error()) { promise.set_error(R.move_as_error()); } else { @@ -217,12 +215,26 @@ void ValidatorManagerImpl::prevalidate_block(BlockBroadcast broadcast, td::Promi promise.set_error(td::Status::Error(ErrorCode::notready, "node not started")); return; } + if (!need_monitor(broadcast.block_id.shard_full())) { + promise.set_error(td::Status::Error("not monitoring shard")); + return; + } + promise = [SelfId = actor_id(this), promise = std::move(promise), block_id = broadcast.block_id, + cc_seqno = broadcast.catchain_seqno](td::Result R) mutable { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::validated_block_broadcast, block_id, cc_seqno); + } + promise.set_result(std::move(R)); + }; td::actor::create_actor("broadcast", std::move(broadcast), last_masterchain_block_handle_, last_masterchain_state_, last_known_key_block_handle_, actor_id(this), td::Timestamp::in(2.0), std::move(promise)) .release(); } +void ValidatorManagerImpl::validated_block_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno) { +} + void ValidatorManagerImpl::sync_complete(td::Promise promise) { started_ = true; @@ -418,6 +430,10 @@ void ValidatorManagerImpl::add_external_message(td::Ref msg, int pri ext_messages_hashes_[id.hash] = {priority, id}; } void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Promise> promise) { + if (!started_) { + promise.set_error(td::Status::Error(ErrorCode::notready, "node not synced")); + return; + } auto state = do_get_last_liteserver_state(); if (state.is_null()) { promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); @@ -439,11 +455,9 @@ void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Prom promise = [self = this, wc, addr, promise = std::move(promise), SelfId = actor_id(this)](td::Result> R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - return; - } - td::actor::send_lambda(SelfId, [=, promise = std::move(promise), message = R.move_as_ok()]() mutable { + td::actor::send_lambda(SelfId, [=, promise = std::move(promise), R = std::move(R)]() mutable { + ++(R.is_ok() ? self->total_check_ext_messages_ok_ : self->total_check_ext_messages_error_); + TRY_RESULT_PROMISE(promise, message, std::move(R)); if (self->checked_ext_msg_counter_.inc_msg_count(wc, addr) > max_ext_msg_per_addr()) { promise.set_error( td::Status::Error(PSTRING() << "too many external messages to address " << wc << ":" << addr.to_hex())); @@ -474,7 +488,7 @@ void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { } void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) { - if (!is_validator()) { + if (!is_validator() && !cached_block_candidates_.count(block_id)) { return; } if (!last_masterchain_block_handle_) { @@ -508,37 +522,36 @@ void ValidatorManagerImpl::new_block_candidate(BlockIdExt block_id, td::BufferSl if (!started_) { return; } + if (!need_monitor(block_id.shard_full())) { + VLOG(VALIDATOR_DEBUG) << "dropping block candidate broadcast: not monitoring shard"; + return; + } add_cached_block_candidate(ReceivedBlock{block_id, std::move(data)}); } void ValidatorManagerImpl::add_shard_block_description(td::Ref desc) { - if (desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { - auto it = shard_blocks_.find(ShardTopBlockDescriptionId{desc->shard(), desc->catchain_seqno()}); - if (it != shard_blocks_.end() && desc->block_id().id.seqno <= it->second->block_id().id.seqno) { - VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block broadcast"; - return; - } - shard_blocks_[ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}] = desc; - VLOG(VALIDATOR_DEBUG) << "new shard block descr for " << desc->block_id(); - if (last_masterchain_block_handle_ && last_masterchain_seqno_ > 0 && - desc->generated_at() < last_masterchain_block_handle_->unix_time() + 60) { - delay_action( - [SelfId = actor_id(this), desc]() { - auto P = td::PromiseCreator::lambda([](td::Result> R) { - if (R.is_error()) { - auto S = R.move_as_error(); - if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { - VLOG(VALIDATOR_NOTICE) << "failed to get shard state: " << S; - } else { - VLOG(VALIDATOR_DEBUG) << "failed to get shard state: " << S; - } - } - }); - td::actor::send_closure(SelfId, &ValidatorManager::wait_block_state_short, desc->block_id(), 0, - td::Timestamp::in(60.0), std::move(P)); - }, - td::Timestamp::in(1.0)); - } + if (!desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { + return; + } + auto it = shard_blocks_.find(ShardTopBlockDescriptionId{desc->shard(), desc->catchain_seqno()}); + if (it != shard_blocks_.end() && desc->block_id().id.seqno <= it->second->block_id().id.seqno) { + VLOG(VALIDATOR_DEBUG) << "dropping duplicate shard block broadcast"; + return; + } + shard_blocks_[ShardTopBlockDescriptionId{desc->block_id().shard_full(), desc->catchain_seqno()}] = desc; + VLOG(VALIDATOR_DEBUG) << "new shard block descr for " << desc->block_id(); + if (need_monitor(desc->block_id().shard_full())) { + auto P = td::PromiseCreator::lambda([](td::Result> R) { + if (R.is_error()) { + auto S = R.move_as_error(); + if (S.code() != ErrorCode::timeout && S.code() != ErrorCode::notready) { + VLOG(VALIDATOR_NOTICE) << "failed to get shard state: " << S; + } else { + VLOG(VALIDATOR_DEBUG) << "failed to get shard state: " << S; + } + } + }); + wait_block_state_short(desc->block_id(), 0, td::Timestamp::in(60.0), std::move(P)); } } @@ -554,7 +567,7 @@ void ValidatorManagerImpl::add_cached_block_candidate(ReceivedBlock block) { if (it != wait_block_data_.end()) { auto r_block = create_block(cached_block_candidates_[id].clone()); if (r_block.is_ok()) { - td::actor::send_closure(it->second.actor_, &WaitBlockData::got_block_data_from_net, r_block.move_as_ok()); + td::actor::send_closure(it->second.actor_, &WaitBlockData::loaded_block_data, r_block.move_as_ok()); } } } @@ -672,6 +685,10 @@ void ValidatorManagerImpl::run_ext_query(td::BufferSlice data, td::Promise> promise) { + if (last_masterchain_state_.not_null() && !opts_->need_monitor(handle->id().shard_full(), last_masterchain_state_)) { + return promise.set_error( + td::Status::Error(PSTRING() << "not monitoring shard " << handle->id().shard_full().to_str())); + } auto it0 = block_state_cache_.find(handle->id()); if (it0 != block_state_cache_.end()) { it0->second.ttl_ = td::Timestamp::in(30.0); @@ -684,7 +701,8 @@ void ValidatorManagerImpl::wait_block_state(BlockHandle handle, td::uint32 prior td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); }); auto id = td::actor::create_actor("waitstate", handle, priority, actor_id(this), - td::Timestamp::in(10.0), std::move(P)) + td::Timestamp::at(timeout.at() + 10.0), std::move(P), + get_block_persistent_state_to_download(handle->id())) .release(); wait_state_[handle->id()].actor_ = id; it = wait_state_.find(handle->id()); @@ -717,7 +735,7 @@ void ValidatorManagerImpl::wait_block_data(BlockHandle handle, td::uint32 priori td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_data, handle, std::move(R)); }); auto id = td::actor::create_actor("waitdata", handle, priority, actor_id(this), - td::Timestamp::in(10.0), std::move(P)) + td::Timestamp::at(timeout.at() + 10.0), false, std::move(P)) .release(); wait_block_data_[handle->id()].actor_ = id; it = wait_block_data_.find(handle->id()); @@ -744,6 +762,10 @@ void ValidatorManagerImpl::wait_block_data_short(BlockIdExt block_id, td::uint32 void ValidatorManagerImpl::wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) { + if (last_masterchain_state_.not_null() && !opts_->need_monitor(left_id.shard_full(), last_masterchain_state_)) { + return promise.set_error( + td::Status::Error(PSTRING() << "not monitoring shard " << left_id.shard_full().to_str())); + } td::actor::create_actor("merge", left_id, right_id, priority, actor_id(this), timeout, std::move(promise)) .release(); @@ -1127,9 +1149,10 @@ void ValidatorManagerImpl::finished_wait_state(BlockHandle handle, td::Result> R) { td::actor::send_closure(SelfId, &ValidatorManagerImpl::finished_wait_state, handle, std::move(R)); }); - auto id = td::actor::create_actor("waitstate", handle, X.second, actor_id(this), X.first, - std::move(P)) - .release(); + auto id = + td::actor::create_actor("waitstate", handle, X.second, actor_id(this), X.first, + std::move(P), get_block_persistent_state_to_download(handle->id())) + .release(); it->second.actor_ = id; return; } @@ -1158,8 +1181,9 @@ void ValidatorManagerImpl::finished_wait_data(BlockHandle handle, td::Result("waitdata", handle, X.second, actor_id(this), X.first, std::move(P)) - .release(); + td::actor::create_actor("waitdata", handle, X.second, actor_id(this), X.first, false, + std::move(P)) + .release(); it->second.actor_ = id; return; } @@ -1321,7 +1345,7 @@ void ValidatorManagerImpl::written_handle(BlockHandle handle, td::Promisesecond.actor_, &WaitBlockData::force_read_from_db); } } - if (inited_state) { + if (inited_state && inited_proof) { auto it = wait_state_.find(handle->id()); if (it != wait_state_.end()) { td::actor::send_closure(it->second.actor_, &WaitBlockState::force_read_from_db); @@ -1617,6 +1641,7 @@ void ValidatorManagerImpl::send_top_shard_block_description(td::Refblock_id().shard_full(), desc->catchain_seqno()}] = desc; callback_->send_shard_block_info(desc->block_id(), desc->catchain_seqno(), desc->serialize()); + add_shard_block_description(desc); } } @@ -1624,6 +1649,24 @@ void ValidatorManagerImpl::send_block_broadcast(BlockBroadcast broadcast, int mo callback_->send_broadcast(std::move(broadcast), mode); } +void ValidatorManagerImpl::send_validator_telemetry(PublicKeyHash key, + tl_object_ptr telemetry) { + callback_->send_validator_telemetry(key, std::move(telemetry)); +} + +void ValidatorManagerImpl::send_get_out_msg_queue_proof_request( + ShardIdFull dst_shard, std::vector blocks, block::ImportedMsgQueueLimits limits, + td::Promise>> promise) { + callback_->download_out_msg_queue_proof(dst_shard, std::move(blocks), limits, td::Timestamp::in(10.0), + std::move(promise)); +} + +void ValidatorManagerImpl::send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, + std::string tmp_dir, td::Timestamp timeout, + td::Promise promise) { + callback_->download_archive(mc_seqno, shard_prefix, std::move(tmp_dir), timeout, std::move(promise)); +} + void ValidatorManagerImpl::start_up() { db_ = create_db_actor(actor_id(this), db_root_, opts_); actor_stats_ = td::actor::create_actor("actor_stats"); @@ -1659,7 +1702,6 @@ void ValidatorManagerImpl::start_up() { if (fname.substr(fname.size() - 5) != ".pack") { return; } - fname = fname.substr(0, fname.size() - 5); if (fname.substr(0, 8) != "archive.") { return; } @@ -1668,13 +1710,18 @@ void ValidatorManagerImpl::start_up() { while (fname.size() > 1 && fname[0] == '0') { fname.remove_prefix(1); } + auto i = fname.find('.'); + if (i == td::Slice::npos) { + return; + } + fname = fname.substr(0, i); auto v = td::to_integer_safe(fname); if (v.is_error()) { return; } - auto pos = v.move_as_ok(); - LOG(INFO) << "found archive slice '" << cfname << "' for position " << pos; - to_import_[pos] = std::make_pair(cfname.str(), true); + auto seqno = v.move_as_ok(); + LOG(INFO) << "found archive slice '" << cfname << "' for seqno " << seqno; + to_import_[seqno].push_back(cfname.str()); } }); if (S.is_error()) { @@ -1687,6 +1734,14 @@ void ValidatorManagerImpl::start_up() { alarm_timestamp().relax(check_waiters_at_); } +void ValidatorManagerImpl::init_last_masterchain_state(td::Ref state) { + if (last_masterchain_state_.not_null()) { + return; + } + last_masterchain_state_ = std::move(state); + update_shard_overlays(); +} + void ValidatorManagerImpl::started(ValidatorManagerInitResult R) { CHECK(R.handle); CHECK(R.state.not_null()); @@ -1724,6 +1779,18 @@ void ValidatorManagerImpl::started(ValidatorManagerInitResult R) { if (opts_->nonfinal_ls_queries_enabled()) { candidates_buffer_ = td::actor::create_actor("candidates-buffer", actor_id(this)); } + init_validator_telemetry(); + + auto Q = td::PromiseCreator::lambda( + [SelfId = actor_id(this)](td::Result>> R) { + if (R.is_error()) { + LOG(FATAL) << "db error: " << R.move_as_error(); + } else { + td::actor::send_closure(SelfId, &ValidatorManagerImpl::got_persistent_state_descriptions, R.move_as_ok()); + } + }); + td::actor::send_closure(db_, &Db::get_persistent_state_descriptions, std::move(Q)); + update_shard_overlays(); } void ValidatorManagerImpl::read_gc_list(std::vector list) { @@ -1826,61 +1893,35 @@ void ValidatorManagerImpl::download_next_archive() { } auto seqno = std::min(last_masterchain_seqno_, shard_client_handle_->id().seqno()); + std::vector to_import_files; auto it = to_import_.upper_bound(seqno + 1); if (it != to_import_.begin()) { - it--; - if (it->second.second) { - it->second.second = false; - downloaded_archive_slice(it->second.first, false); - return; - } + --it; + to_import_files = std::move(it->second); + it->second.clear(); } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { - LOG(INFO) << "failed to download archive slice: " << R.error(); + LOG(INFO) << "failed to download and import archive slice: " << R.error(); delay_action([SelfId]() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::download_next_archive); }, td::Timestamp::in(2.0)); } else { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::downloaded_archive_slice, R.move_as_ok(), true); + td::actor::send_closure(SelfId, &ValidatorManagerImpl::checked_archive_slice, R.ok().first, R.ok().second); } }); - callback_->download_archive(seqno + 1, db_root_ + "/tmp/", td::Timestamp::in(36000.0), std::move(P)); -} - -void ValidatorManagerImpl::downloaded_archive_slice(std::string name, bool is_tmp) { - LOG(INFO) << "downloaded archive slice: " << name; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), name, is_tmp](td::Result> R) { - if (is_tmp) { - td::unlink(name).ensure(); - } - if (R.is_error()) { - LOG(INFO) << "failed to check downloaded archive slice: " << R.error(); - delay_action([SelfId]() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::download_next_archive); }, - td::Timestamp::in(2.0)); - } else { - td::actor::send_closure(SelfId, &ValidatorManagerImpl::checked_archive_slice, R.move_as_ok()); - } - }); - - auto seqno = std::min(last_masterchain_seqno_, shard_client_handle_->id().seqno()); - - td::actor::create_actor("archiveimport", name, last_masterchain_state_, seqno, opts_, actor_id(this), - std::move(P)) + td::actor::create_actor("archiveimport", db_root_, last_masterchain_state_, seqno, opts_, + actor_id(this), std::move(to_import_files), std::move(P)) .release(); } -void ValidatorManagerImpl::checked_archive_slice(std::vector seqno) { - CHECK(seqno.size() == 2); - LOG(INFO) << "checked downloaded archive slice: mc_top_seqno=" << seqno[0] << " shard_top_seqno_=" << seqno[1]; - CHECK(seqno[0] <= last_masterchain_seqno_); - CHECK(seqno[1] <= last_masterchain_seqno_); +void ValidatorManagerImpl::checked_archive_slice(BlockSeqno new_last_mc_seqno, BlockSeqno new_shard_client_seqno) { + LOG(INFO) << "checked downloaded archive slice: mc_top_seqno=" << new_last_mc_seqno + << " shard_top_seqno_=" << new_shard_client_seqno; + CHECK(new_last_mc_seqno <= last_masterchain_seqno_); + CHECK(new_shard_client_seqno <= last_masterchain_seqno_); - BlockIdExt b; - if (seqno[1] < last_masterchain_seqno_) { - CHECK(last_masterchain_state_->get_old_mc_block_id(seqno[1], b)); - } else { - b = last_masterchain_block_id_; - } + BlockIdExt shard_client_block_id; + CHECK(last_masterchain_state_->get_old_mc_block_id(new_shard_client_seqno, shard_client_block_id)); auto P = td::PromiseCreator::lambda( [SelfId = actor_id(this), db = db_.get(), client = shard_client_.get()](td::Result R) { @@ -1896,7 +1937,7 @@ void ValidatorManagerImpl::checked_archive_slice(std::vector seqno) }); td::actor::send_closure(db, &Db::get_block_state, std::move(handle), std::move(P)); }); - get_block_handle(b, true, std::move(P)); + get_block_handle(shard_client_block_id, true, std::move(P)); } void ValidatorManagerImpl::finish_prestart_sync() { @@ -1930,8 +1971,10 @@ void ValidatorManagerImpl::new_masterchain_block() { td::actor::send_closure(serializer_, &AsyncStateSerializer::update_last_known_key_block_ts, last_key_block_handle_->unix_time()); } + init_validator_telemetry(); } + update_shard_overlays(); update_shards(); update_shard_blocks(); @@ -1945,6 +1988,26 @@ void ValidatorManagerImpl::new_masterchain_block() { } } +void ValidatorManagerImpl::update_shard_overlays() { + CHECK(last_masterchain_state_.not_null()); + std::set shards_to_monitor; + shards_to_monitor.insert(ShardIdFull{masterchainId}); + std::set workchains; + for (const auto& shard : last_masterchain_state_->get_shards()) { + workchains.insert(shard->shard().workchain); + if (opts_->need_monitor(shard->shard(),last_masterchain_state_)) { + shards_to_monitor.insert(shard->shard()); + } + } + for (const auto &[wc, desc] : last_masterchain_state_->get_workchain_list()) { + if (!workchains.count(wc) && desc->active && + opts_->need_monitor(ShardIdFull{wc, shardIdAll}, last_masterchain_state_)) { + shards_to_monitor.insert(ShardIdFull{wc, shardIdAll}); + } + } + callback_->on_new_masterchain_block(last_masterchain_state_, std::move(shards_to_monitor)); +} + void ValidatorManagerImpl::update_shards() { if ((last_masterchain_state_->rotated_all_shards() || last_masterchain_seqno_ == 0) && opts_->get_last_fork_masterchain_seqno() <= last_masterchain_seqno_) { @@ -2070,7 +2133,7 @@ void ValidatorManagerImpl::update_shards() { } } - bool validating_masterchain = false; + active_validator_groups_master_ = active_validator_groups_shard_ = 0; if (allow_validate_) { for (auto &desc : new_shards) { auto shard = desc.first; @@ -2087,9 +2150,7 @@ void ValidatorManagerImpl::update_shards() { auto validator_id = get_validator(shard, val_set); if (!validator_id.is_zero()) { - if (shard.is_masterchain()) { - validating_masterchain = true; - } + ++(shard.is_masterchain() ? active_validator_groups_master_ : active_validator_groups_shard_); auto val_group_id = get_validator_set_id(shard, val_set, opts_hash, key_seqno, opts); if (force_recover) { @@ -2261,7 +2322,8 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto validator_id = get_validator(shard, validator_set); CHECK(!validator_id.is_zero()); auto G = td::actor::create_actor( - "validatorgroup", shard, validator_id, session_id, validator_set, key_seqno, opts, keyring_, adnl_, rldp_, + PSTRING() << "valgroup" << shard.to_str(), shard, validator_id, session_id, validator_set, key_seqno, opts, + keyring_, adnl_, rldp_, overlays_, db_root_, actor_id(this), init_session, opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_); return G; @@ -2437,7 +2499,7 @@ void ValidatorManagerImpl::allow_block_state_gc(BlockIdExt block_id, td::Promise return; } auto shards = gc_masterchain_state_->get_shards(); - for (auto shard : shards) { + for (const auto &shard : shards) { if (shard_intersects(shard->shard(), block_id.shard_full())) { promise.set_result(block_id.id.seqno < shard->top_block_id().id.seqno); return; @@ -2675,10 +2737,6 @@ void ValidatorManagerImpl::get_shard_client_state(bool from_db, td::Promiseadd_shard(shard); -} - void ValidatorManagerImpl::update_async_serializer_state(AsyncSerializerState state, td::Promise promise) { td::actor::send_closure(db_, &Db::update_async_serializer_state, std::move(state), std::move(promise)); } @@ -2691,12 +2749,13 @@ void ValidatorManagerImpl::try_get_static_file(FileHash file_hash, td::Promise promise) { +void ValidatorManagerImpl::get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) { if (masterchain_seqno > last_masterchain_seqno_) { promise.set_error(td::Status::Error(ErrorCode::notready, "masterchain seqno too big")); return; } - td::actor::send_closure(db_, &Db::get_archive_id, masterchain_seqno, std::move(promise)); + td::actor::send_closure(db_, &Db::get_archive_id, masterchain_seqno, shard_prefix, std::move(promise)); } void ValidatorManagerImpl::get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, @@ -2708,10 +2767,13 @@ bool ValidatorManagerImpl::is_validator() { return temp_keys_.size() > 0 || permanent_keys_.size() > 0; } +bool ValidatorManagerImpl::validating_masterchain() { + return !get_validator(ShardIdFull(masterchainId), + last_masterchain_state_->get_validator_set(ShardIdFull(masterchainId))) + .is_zero(); +} + PublicKeyHash ValidatorManagerImpl::get_validator(ShardIdFull shard, td::Ref val_set) { - if (!opts_->need_validate(shard, val_set->get_catchain_seqno())) { - return PublicKeyHash::zero(); - } for (auto &key : temp_keys_) { if (val_set->is_validator(key.bits256_value())) { return key; @@ -2783,8 +2845,8 @@ void ValidatorManagerImpl::prepare_stats(td::Promiseid().to_str()); vec.emplace_back("rotatemasterchainblock", last_rotate_block_id_.to_str()); //vec.emplace_back("shardclientmasterchainseqno", td::to_string(min_confirmed_masterchain_seqno_)); - vec.emplace_back("stateserializermasterchainseqno", td::to_string(state_serializer_masterchain_seqno_)); } + td::NamedThreadSafeCounter::get_default().for_each([&](auto key, auto value) { vec.emplace_back("counter." + key, PSTRING() << value); }); @@ -2802,9 +2864,48 @@ void ValidatorManagerImpl::prepare_stats(td::Promiseget_state_serializer_enabled(); + if (is_validator() && last_masterchain_state_->get_global_id() == -239) { + serializer_enabled = false; + } + vec.emplace_back("stateserializerenabled", serializer_enabled ? "true" : "false"); + merger.make_promise("").set_value(std::move(vec)); + if (!serializer_.empty()) { + td::actor::send_closure(serializer_, &AsyncStateSerializer::prepare_stats, merger.make_promise("")); + } + td::actor::send_closure(db_, &Db::prepare_stats, merger.make_promise("db.")); + for (auto &[_, p] : stats_providers_) { + p.second(merger.make_promise(p.first)); + } } void ValidatorManagerImpl::prepare_perf_timer_stats(td::Promise> promise) { @@ -3217,10 +3318,15 @@ void ValidatorManagerImpl::get_validator_groups_info_for_litequery( } void ValidatorManagerImpl::update_options(td::Ref opts) { - // Currently options can be updated only to change state_serializer_enabled flag and collator_options + if (!shard_client_.empty()) { + td::actor::send_closure(shard_client_, &ShardClient::update_options, opts); + } if (!serializer_.empty()) { td::actor::send_closure(serializer_, &AsyncStateSerializer::update_options, opts); } + if (!queue_size_counter_.empty()) { + td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::update_options, opts); + } for (auto &group : validator_groups_) { td::actor::send_closure(group.second.actor, &ValidatorGroup::update_options, opts); } @@ -3230,6 +3336,60 @@ void ValidatorManagerImpl::update_options(td::Ref opts) opts_ = std::move(opts); } +void ValidatorManagerImpl::add_persistent_state_description(td::Ref desc) { + auto now = (UnixTime)td::Clocks::system(); + if (desc->end_time <= now) { + return; + } + td::actor::send_closure(db_, &Db::add_persistent_state_description, desc, [](td::Result) {}); + auto it = persistent_state_descriptions_.begin(); + while (it != persistent_state_descriptions_.end()) { + const auto &prev_desc = it->second; + if (prev_desc->end_time <= now) { + for (const BlockIdExt &block_id : prev_desc->shard_blocks) { + persistent_state_blocks_.erase(block_id); + } + it = persistent_state_descriptions_.erase(it); + } else { + ++it; + } + } + add_persistent_state_description_impl(std::move(desc)); +} + +void ValidatorManagerImpl::add_persistent_state_description_impl(td::Ref desc) { + if (!persistent_state_descriptions_.emplace(desc->masterchain_id.seqno(), desc).second) { + return; + } + LOG(DEBUG) << "Add persistent state description for mc block " << desc->masterchain_id.to_str() + << " start_time=" << desc->start_time << " end_time=" << desc->end_time; + for (const BlockIdExt &block_id : desc->shard_blocks) { + persistent_state_blocks_[block_id] = desc; + LOG(DEBUG) << "Persistent state description: shard block " << block_id.to_str(); + } +} + +void ValidatorManagerImpl::got_persistent_state_descriptions(std::vector> descs) { + for (auto &desc : descs) { + add_persistent_state_description_impl(std::move(desc)); + } +} + +td::Ref ValidatorManagerImpl::get_block_persistent_state_to_download(BlockIdExt block_id) { + if (block_id.is_masterchain()) { + return {}; + } + auto it = persistent_state_blocks_.find(block_id); + if (it == persistent_state_blocks_.end()) { + return {}; + } + if (it->second->masterchain_id.seqno() + 16 >= min_confirmed_masterchain_seqno_) { + // Do not download persistent states during ordinary shard client sync + return {}; + } + return it->second; +} + td::actor::ActorOwn ValidatorManagerFactory::create( td::Ref opts, std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, @@ -3239,17 +3399,28 @@ td::actor::ActorOwn ValidatorManagerFactory::create( } void ValidatorManagerImpl::record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, - CollationStats stats) { + td::optional stats) { + if (!stats) { + ++(block_id.is_masterchain() ? total_collated_blocks_master_error_ : total_collated_blocks_shard_error_); + return; + } auto &record = new_block_stats_record(block_id); record.collator_work_time_ = work_time; record.collator_cpu_work_time_ = cpu_work_time; - record.collator_stats_ = std::move(stats); + record.collator_stats_ = std::move(stats.value()); + ++(block_id.is_masterchain() ? total_collated_blocks_master_ok_ : total_collated_blocks_shard_ok_); } -void ValidatorManagerImpl::record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) { +void ValidatorManagerImpl::record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, + bool success) { auto &record = new_block_stats_record(block_id); record.validator_work_time_ = work_time; record.validator_cpu_work_time_ = cpu_work_time; + if (success) { + ++(block_id.is_masterchain() ? total_validated_blocks_master_ok_ : total_validated_blocks_shard_ok_); + } else { + ++(block_id.is_masterchain() ? total_validated_blocks_master_error_ : total_validated_blocks_shard_error_); + } } ValidatorManagerImpl::RecordedBlockStats &ValidatorManagerImpl::new_block_stats_record(BlockIdExt block_id) { @@ -3263,6 +3434,16 @@ ValidatorManagerImpl::RecordedBlockStats &ValidatorManagerImpl::new_block_stats_ return recorded_block_stats_[block_id]; } +void ValidatorManagerImpl::register_stats_provider( + td::uint64 idx, std::string prefix, + std::function>>)> callback) { + stats_providers_[idx] = {std::move(prefix), std::move(callback)}; +} + +void ValidatorManagerImpl::unregister_stats_provider(td::uint64 idx) { + stats_providers_.erase(idx); +} + size_t ValidatorManagerImpl::CheckedExtMsgCounter::get_msg_count(WorkchainId wc, StdSmcAddress addr) { before_query(); auto it1 = counter_cur_.find({wc, addr}); @@ -3286,6 +3467,41 @@ void ValidatorManagerImpl::CheckedExtMsgCounter::before_query() { } } +void ValidatorManagerImpl::init_validator_telemetry() { + if (last_masterchain_state_.is_null()) { + return; + } + td::Ref validator_set = last_masterchain_state_->get_total_validator_set(0); + if (validator_set.is_null()) { + validator_telemetry_.clear(); + return; + } + std::set processed; + for (auto& key : temp_keys_) { + if (const ValidatorDescr* desc = validator_set->get_validator(key.bits256_value())) { + processed.insert(key); + adnl::AdnlNodeIdShort adnl_id; + if (desc->addr.is_zero()) { + adnl_id = adnl::AdnlNodeIdShort{ValidatorFullId{desc->key}.compute_short_id()}; + } else { + adnl_id = adnl::AdnlNodeIdShort{desc->addr}; + } + auto& telemetry = validator_telemetry_[key]; + if (telemetry.empty()) { + telemetry = td::actor::create_actor( + "telemetry", key, adnl_id, opts_->zero_block_id().file_hash, actor_id(this)); + } + } + } + for (auto it = validator_telemetry_.begin(); it != validator_telemetry_.end();) { + if (processed.contains(it->first)) { + ++it; + } else { + it = validator_telemetry_.erase(it); + } + } +} + } // namespace validator } // namespace ton diff --git a/validator/manager.hpp b/validator/manager.hpp index 7410f75f..418deb35 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -34,6 +34,7 @@ #include "rldp/rldp.h" #include "token-manager.h" #include "queue-size-counter.hpp" +#include "validator-telemetry.hpp" #include "impl/candidates-buffer.hpp" #include @@ -188,6 +189,12 @@ class ValidatorManagerImpl : public ValidatorManager { waiting_.resize(j); } }; + template + struct WaitListCaching : public WaitList { + bool done_ = false; + ResType result_; + td::Timestamp remove_at_; + }; std::map>> wait_state_; std::map>> wait_block_data_; @@ -304,6 +311,7 @@ class ValidatorManagerImpl : public ValidatorManager { std::vector perf_timer_stats; void new_masterchain_block(); + void update_shard_overlays(); void update_shards(); void update_shard_blocks(); void written_destroyed_validator_sessions(std::vector> groups); @@ -322,8 +330,7 @@ class ValidatorManagerImpl : public ValidatorManager { void applied_hardfork(); void prestart_sync(); void download_next_archive(); - void downloaded_archive_slice(std::string name, bool is_tmp); - void checked_archive_slice(std::vector seqno); + void checked_archive_slice(BlockSeqno new_last_mc_seqno, BlockSeqno new_shard_client_seqno); void finish_prestart_sync(); void completed_prestart_sync(); @@ -339,6 +346,7 @@ class ValidatorManagerImpl : public ValidatorManager { } void add_temp_key(PublicKeyHash key, td::Promise promise) override { temp_keys_.insert(key); + init_validator_telemetry(); promise.set_value(td::Unit()); } void del_permanent_key(PublicKeyHash key, td::Promise promise) override { @@ -347,6 +355,7 @@ class ValidatorManagerImpl : public ValidatorManager { } void del_temp_key(PublicKeyHash key, td::Promise promise) override { temp_keys_.erase(key); + init_validator_telemetry(); promise.set_value(td::Unit()); } @@ -358,6 +367,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise promise) override; void validate_block(ReceivedBlock block, td::Promise promise) override; void prevalidate_block(BlockBroadcast broadcast, td::Promise promise) override; + void validated_block_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno); //void create_validate_block(BlockId block, td::BufferSlice data, td::Promise promise) = 0; void sync_complete(td::Promise promise) override; @@ -501,10 +511,15 @@ class ValidatorManagerImpl : public ValidatorManager { void send_ihr_message(td::Ref message) override; void send_top_shard_block_description(td::Ref desc) override; void send_block_broadcast(BlockBroadcast broadcast, int mode) override; + void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) override; + void send_get_out_msg_queue_proof_request(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, + td::Promise>> promise) override; + void send_download_archive_request(BlockSeqno mc_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) override; void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; - void subscribe_to_shard(ShardIdFull shard) override; void update_async_serializer_state(AsyncSerializerState state, td::Promise promise) override; void get_async_serializer_state(td::Promise promise) override; @@ -512,12 +527,12 @@ class ValidatorManagerImpl : public ValidatorManager { void try_get_static_file(FileHash file_hash, td::Promise promise) override; void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) override { - td::actor::send_closure(token_manager_, &TokenManager::get_download_token, download_size, priority, timeout, + td::Promise> promise) override { + td::actor::send_closure(token_manager_, &TokenManager::get_token, download_size, priority, timeout, std::move(promise)); } - void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) override; + void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, td::Promise promise) override; void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise) override; @@ -538,10 +553,12 @@ class ValidatorManagerImpl : public ValidatorManager { void finished_wait_data(BlockHandle handle, td::Result> R); void start_up() override; + void init_last_masterchain_state(td::Ref state) override; void started(ValidatorManagerInitResult result); void read_gc_list(std::vector list); bool is_validator(); + bool validating_masterchain(); PublicKeyHash get_validator(ShardIdFull shard, td::Ref val_set); ValidatorManagerImpl(td::Ref opts, std::string db_root, @@ -600,14 +617,20 @@ class ValidatorManagerImpl : public ValidatorManager { void update_options(td::Ref opts) override; + void add_persistent_state_description(td::Ref desc) override; + void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { if (last_masterchain_state_.is_null()) { promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); return; } - queue_size_counter_ = td::actor::create_actor("queuesizecounter", - last_masterchain_state_, actor_id(this)); + queue_size_counter_ = + td::actor::create_actor("queuesizecounter", last_masterchain_state_, opts_, actor_id(this)); + } + if (!opts_->need_monitor(block_id.shard_full(), last_masterchain_state_)) { + return promise.set_error( + td::Status::Error(PSTRING() << "not monitoring shard " << block_id.shard_full().to_str())); } td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::get_queue_size, block_id, std::move(promise)); } @@ -632,8 +655,9 @@ class ValidatorManagerImpl : public ValidatorManager { td::optional shard, td::Promise> promise) override; - void add_lite_query_stats(int lite_query_id) override { + void add_lite_query_stats(int lite_query_id, bool success) override { ++ls_stats_[lite_query_id]; + ++(success ? total_ls_queries_ok_ : total_ls_queries_error_)[lite_query_id]; } private: @@ -678,7 +702,7 @@ class ValidatorManagerImpl : public ValidatorManager { td::actor::ActorOwn serializer_; - std::map> to_import_; + std::map> to_import_; private: std::unique_ptr callback_; @@ -708,7 +732,15 @@ class ValidatorManagerImpl : public ValidatorManager { return 3 * 10; } + void got_persistent_state_descriptions(std::vector> descs); + void add_persistent_state_description_impl(td::Ref desc); + td::Ref get_block_persistent_state_to_download(BlockIdExt block_id); + private: + bool need_monitor(ShardIdFull shard) const { + return opts_->need_monitor(shard, last_masterchain_state_); + } + std::map> shard_client_waiters_; td::actor::ActorOwn queue_size_counter_; @@ -716,6 +748,16 @@ class ValidatorManagerImpl : public ValidatorManager { std::map ls_stats_; // lite_api ID -> count, 0 for unknown td::uint32 ls_stats_check_ext_messages_{0}; + UnixTime started_at_ = (UnixTime)td::Clocks::system(); + std::map total_ls_queries_ok_, total_ls_queries_error_; // lite_api ID -> count, 0 for unknown + td::uint64 total_check_ext_messages_ok_{0}, total_check_ext_messages_error_{0}; + td::uint64 total_collated_blocks_master_ok_{0}, total_collated_blocks_master_error_{0}; + td::uint64 total_validated_blocks_master_ok_{0}, total_validated_blocks_master_error_{0}; + td::uint64 total_collated_blocks_shard_ok_{0}, total_collated_blocks_shard_error_{0}; + td::uint64 total_validated_blocks_shard_ok_{0}, total_validated_blocks_shard_error_{0}; + + size_t active_validator_groups_master_{0}, active_validator_groups_shard_{0}; + td::actor::ActorOwn candidates_buffer_; struct RecordedBlockStats { @@ -729,9 +771,25 @@ class ValidatorManagerImpl : public ValidatorManager { std::queue recorded_block_stats_lru_; void record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, - CollationStats stats) override; - void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) override; + td::optional stats) override; + void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, bool success) override; RecordedBlockStats &new_block_stats_record(BlockIdExt block_id); + + void register_stats_provider( + td::uint64 idx, std::string prefix, + std::function>>)> callback) override; + void unregister_stats_provider(td::uint64 idx) override; + + std::map> validator_telemetry_; + + void init_validator_telemetry(); + + std::map> persistent_state_descriptions_; + std::map> persistent_state_blocks_; + + std::map>>)>>> + stats_providers_; }; } // namespace validator diff --git a/validator/net/download-archive-slice.cpp b/validator/net/download-archive-slice.cpp index 6235b8b0..c2f8ecea 100644 --- a/validator/net/download-archive-slice.cpp +++ b/validator/net/download-archive-slice.cpp @@ -20,6 +20,8 @@ #include "td/utils/port/path.h" #include "td/utils/overloaded.h" +#include + namespace ton { namespace validator { @@ -27,12 +29,13 @@ namespace validator { namespace fullnode { DownloadArchiveSlice::DownloadArchiveSlice( - BlockSeqno masterchain_seqno, std::string tmp_dir, adnl::AdnlNodeIdShort local_id, + BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, td::Timestamp timeout, td::actor::ActorId validator_manager, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId adnl, td::actor::ActorId client, td::Promise promise) : masterchain_seqno_(masterchain_seqno) + , shard_prefix_(shard_prefix) , tmp_dir_(std::move(tmp_dir)) , local_id_(local_id) , overlay_id_(overlay_id) @@ -114,7 +117,13 @@ void DownloadArchiveSlice::got_node_to_download(adnl::AdnlNodeIdShort download_f } }); - auto q = create_serialize_tl_object(masterchain_seqno_); + td::BufferSlice q; + if (shard_prefix_.is_masterchain()) { + q = create_serialize_tl_object(masterchain_seqno_); + } else { + q = create_serialize_tl_object(masterchain_seqno_, + create_tl_shard_id(shard_prefix_)); + } if (client_.empty()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_query, download_from_, local_id_, overlay_id_, "get_archive_info", std::move(P), td::Timestamp::in(3.0), std::move(q)); @@ -145,7 +154,8 @@ void DownloadArchiveSlice::got_archive_info(td::BufferSlice data) { } prev_logged_timer_ = td::Timer(); - LOG(INFO) << "downloading archive slice #" << masterchain_seqno_ << " from " << download_from_; + LOG(INFO) << "downloading archive slice #" << masterchain_seqno_ << " " << shard_prefix_.to_str() << " from " + << download_from_; get_archive_slice(); } @@ -186,13 +196,15 @@ void DownloadArchiveSlice::got_archive_slice(td::BufferSlice data) { double elapsed = prev_logged_timer_.elapsed(); if (elapsed > 10.0) { prev_logged_timer_ = td::Timer(); - LOG(INFO) << "downloading archive slice #" << masterchain_seqno_ << ": total=" << offset_ << " (" + LOG(INFO) << "downloading archive slice #" << masterchain_seqno_ << " " << shard_prefix_.to_str() + << ": total=" << offset_ << " (" << td::format::as_size((td::uint64)(double(offset_ - prev_logged_sum_) / elapsed)) << "/s)"; prev_logged_sum_ = offset_; } if (data.size() < slice_size()) { - LOG(INFO) << "finished downloading arcrive slice #" << masterchain_seqno_ << ": total=" << offset_; + LOG(INFO) << "finished downloading arcrive slice #" << masterchain_seqno_ << " " << shard_prefix_.to_str() + << ": total=" << offset_; finish_query(); } else { get_archive_slice(); diff --git a/validator/net/download-archive-slice.hpp b/validator/net/download-archive-slice.hpp index 0384ac8c..42fd715f 100644 --- a/validator/net/download-archive-slice.hpp +++ b/validator/net/download-archive-slice.hpp @@ -32,8 +32,9 @@ namespace fullnode { class DownloadArchiveSlice : public td::actor::Actor { public: - DownloadArchiveSlice(BlockSeqno masterchain_seqno, std::string tmp_dir, adnl::AdnlNodeIdShort local_id, - overlay::OverlayIdShort overlay_id, adnl::AdnlNodeIdShort download_from, td::Timestamp timeout, + DownloadArchiveSlice(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + adnl::AdnlNodeIdShort local_id, overlay::OverlayIdShort overlay_id, + adnl::AdnlNodeIdShort download_from, td::Timestamp timeout, td::actor::ActorId validator_manager, td::actor::ActorId rldp, td::actor::ActorId overlays, td::actor::ActorId adnl, @@ -55,6 +56,7 @@ class DownloadArchiveSlice : public td::actor::Actor { private: BlockSeqno masterchain_seqno_; + ShardIdFull shard_prefix_; std::string tmp_dir_; std::string tmp_name_; td::FileFd fd_; diff --git a/validator/net/download-block-new.cpp b/validator/net/download-block-new.cpp index e9a193b4..37580cef 100644 --- a/validator/net/download-block-new.cpp +++ b/validator/net/download-block-new.cpp @@ -144,7 +144,7 @@ void DownloadBlockNew::got_block_handle(BlockHandle handle) { return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &DownloadBlockNew::abort_query, R.move_as_error_prefix("failed to get download token: ")); @@ -156,7 +156,7 @@ void DownloadBlockNew::got_block_handle(BlockHandle handle) { std::move(P)); } -void DownloadBlockNew::got_download_token(std::unique_ptr token) { +void DownloadBlockNew::got_download_token(std::unique_ptr token) { token_ = std::move(token); if (download_from_.is_zero() && client_.empty()) { diff --git a/validator/net/download-block-new.hpp b/validator/net/download-block-new.hpp index d2a0e136..ecd062ee 100644 --- a/validator/net/download-block-new.hpp +++ b/validator/net/download-block-new.hpp @@ -49,7 +49,7 @@ class DownloadBlockNew : public td::actor::Actor { void start_up() override; void got_block_handle(BlockHandle handle); - void got_download_token(std::unique_ptr token); + void got_download_token(std::unique_ptr token); void got_node_to_download(adnl::AdnlNodeIdShort node); void got_data(td::BufferSlice data); void got_data_from_db(td::BufferSlice data); @@ -79,7 +79,7 @@ class DownloadBlockNew : public td::actor::Actor { bool allow_partial_proof_ = false; - std::unique_ptr token_; + std::unique_ptr token_; }; } // namespace fullnode diff --git a/validator/net/download-block.cpp b/validator/net/download-block.cpp index 9ca84be2..c60955ed 100644 --- a/validator/net/download-block.cpp +++ b/validator/net/download-block.cpp @@ -128,7 +128,7 @@ void DownloadBlock::got_block_handle(BlockHandle handle) { return; } - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &DownloadBlock::abort_query, R.move_as_error_prefix("failed to get download token: ")); @@ -140,7 +140,7 @@ void DownloadBlock::got_block_handle(BlockHandle handle) { std::move(P)); } -void DownloadBlock::got_download_token(std::unique_ptr token) { +void DownloadBlock::got_download_token(std::unique_ptr token) { token_ = std::move(token); if (download_from_.is_zero() && !short_ && client_.empty()) { diff --git a/validator/net/download-block.hpp b/validator/net/download-block.hpp index b1847d58..2e2a715b 100644 --- a/validator/net/download-block.hpp +++ b/validator/net/download-block.hpp @@ -49,7 +49,7 @@ class DownloadBlock : public td::actor::Actor { void start_up() override; void got_block_handle(BlockHandle handle); - void got_download_token(std::unique_ptr token); + void got_download_token(std::unique_ptr token); void got_node_to_download(adnl::AdnlNodeIdShort node); void got_block_proof_description(td::BufferSlice proof_description); void got_block_proof(td::BufferSlice data); @@ -86,7 +86,7 @@ class DownloadBlock : public td::actor::Actor { bool allow_partial_proof_ = false; - std::unique_ptr token_; + std::unique_ptr token_; }; } // namespace fullnode diff --git a/validator/net/download-proof.cpp b/validator/net/download-proof.cpp index 2ff95b88..784ecac2 100644 --- a/validator/net/download-proof.cpp +++ b/validator/net/download-proof.cpp @@ -107,7 +107,7 @@ void DownloadProof::start_up() { } void DownloadProof::checked_db() { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &DownloadProof::abort_query, R.move_as_error_prefix("failed to get download token: ")); @@ -119,7 +119,7 @@ void DownloadProof::checked_db() { std::move(P)); } -void DownloadProof::got_download_token(std::unique_ptr token) { +void DownloadProof::got_download_token(std::unique_ptr token) { token_ = std::move(token); if (download_from_.is_zero() && client_.empty()) { diff --git a/validator/net/download-proof.hpp b/validator/net/download-proof.hpp index 0739dcaf..9caf9c3a 100644 --- a/validator/net/download-proof.hpp +++ b/validator/net/download-proof.hpp @@ -45,7 +45,7 @@ class DownloadProof : public td::actor::Actor { void start_up() override; void checked_db(); - void got_download_token(std::unique_ptr token); + void got_download_token(std::unique_ptr token); void got_node_to_download(adnl::AdnlNodeIdShort node); void got_block_proof_description(td::BufferSlice proof_description); void got_block_proof(td::BufferSlice data); @@ -72,7 +72,7 @@ class DownloadProof : public td::actor::Actor { td::BufferSlice data_; - std::unique_ptr token_; + std::unique_ptr token_; }; } // namespace fullnode diff --git a/validator/net/download-state.cpp b/validator/net/download-state.cpp index 2b373ef3..6735a2b5 100644 --- a/validator/net/download-state.cpp +++ b/validator/net/download-state.cpp @@ -70,6 +70,7 @@ void DownloadState::finish_query() { } void DownloadState::start_up() { + status_ = ProcessStatus(validator_manager_, "process.download_state_net"); alarm_timestamp() = timeout_; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_persistent_state, block_id_, @@ -190,6 +191,7 @@ void DownloadState::got_block_state_description(td::BufferSlice data) { td::Timestamp::in(3.0), std::move(P)); } })); + status_.set_status(PSTRING() << block_id_.id.to_str() << " : 0 bytes, 0B/s"); } void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 requested_size) { @@ -198,14 +200,18 @@ void DownloadState::got_block_state_part(td::BufferSlice data, td::uint32 reques parts_.push_back(std::move(data)); double elapsed = prev_logged_timer_.elapsed(); - if (elapsed > 10.0) { + if (elapsed > 5.0) { prev_logged_timer_ = td::Timer(); + auto speed = (td::uint64)((double)(sum_ - prev_logged_sum_) / elapsed); LOG(WARNING) << "downloading state " << block_id_.to_str() << ": " << td::format::as_size(sum_) << " (" - << td::format::as_size((td::uint64)(double(sum_ - prev_logged_sum_) / elapsed)) << "/s)"; + << td::format::as_size(speed) << "/s)"; + status_.set_status(PSTRING() << block_id_.id.to_str() << " : " << sum_ << " bytes, " << td::format::as_size(speed) + << "/s"); prev_logged_sum_ = sum_; } if (last_part) { + status_.set_status(PSTRING() << block_id_.id.to_str() << " : " << sum_ << " bytes, finishing"); td::BufferSlice res{td::narrow_cast(sum_)}; auto S = res.as_slice(); for (auto &p : parts_) { diff --git a/validator/net/download-state.hpp b/validator/net/download-state.hpp index 19c44beb..470c5431 100644 --- a/validator/net/download-state.hpp +++ b/validator/net/download-state.hpp @@ -23,6 +23,8 @@ #include "validator/validator.h" #include "adnl/adnl-ext-client.h" +#include + namespace ton { namespace validator { @@ -75,6 +77,8 @@ class DownloadState : public td::actor::Actor { td::uint64 prev_logged_sum_ = 0; td::Timer prev_logged_timer_; + + ProcessStatus status_; }; } // namespace fullnode diff --git a/validator/net/get-next-key-blocks.cpp b/validator/net/get-next-key-blocks.cpp index 3354b005..2c12e495 100644 --- a/validator/net/get-next-key-blocks.cpp +++ b/validator/net/get-next-key-blocks.cpp @@ -84,7 +84,7 @@ void GetNextKeyBlocks::finish_query() { void GetNextKeyBlocks::start_up() { alarm_timestamp() = timeout_; - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { if (R.is_error()) { td::actor::send_closure(SelfId, &GetNextKeyBlocks::abort_query, R.move_as_error_prefix("failed to get download token: ")); @@ -96,7 +96,7 @@ void GetNextKeyBlocks::start_up() { std::move(P)); } -void GetNextKeyBlocks::got_download_token(std::unique_ptr token) { +void GetNextKeyBlocks::got_download_token(std::unique_ptr token) { token_ = std::move(token); if (download_from_.is_zero() && client_.empty()) { diff --git a/validator/net/get-next-key-blocks.hpp b/validator/net/get-next-key-blocks.hpp index 074289e2..14c040bd 100644 --- a/validator/net/get-next-key-blocks.hpp +++ b/validator/net/get-next-key-blocks.hpp @@ -44,7 +44,7 @@ class GetNextKeyBlocks : public td::actor::Actor { void finish_query(); void start_up() override; - void got_download_token(std::unique_ptr token); + void got_download_token(std::unique_ptr token); void got_node_to_download(adnl::AdnlNodeIdShort node); void send_request(); void got_result(td::BufferSlice res); @@ -75,7 +75,7 @@ class GetNextKeyBlocks : public td::actor::Actor { std::vector pending_; std::vector res_; - std::unique_ptr token_; + std::unique_ptr token_; }; } // namespace fullnode diff --git a/validator/queue-size-counter.cpp b/validator/queue-size-counter.cpp index eb858089..4fe55ae3 100644 --- a/validator/queue-size-counter.cpp +++ b/validator/queue-size-counter.cpp @@ -234,7 +234,9 @@ void QueueSizeCounter::process_top_shard_blocks_cont(td::Ref s last_top_blocks_.clear(); last_top_blocks_.push_back(state->get_block_id()); for (auto &shard : state->get_shards()) { - last_top_blocks_.push_back(shard->top_block_id()); + if (opts_->need_monitor(shard->shard(), state)) { + last_top_blocks_.push_back(shard->top_block_id()); + } } for (const BlockIdExt &block_id : last_top_blocks_) { get_queue_size_ex_retry(block_id, init, ig.get_promise()); diff --git a/validator/queue-size-counter.hpp b/validator/queue-size-counter.hpp index 4825a43c..7e5f7627 100644 --- a/validator/queue-size-counter.hpp +++ b/validator/queue-size-counter.hpp @@ -21,16 +21,22 @@ namespace ton::validator { class QueueSizeCounter : public td::actor::Actor { public: - QueueSizeCounter(td::Ref last_masterchain_state, td::actor::ActorId manager) - : init_masterchain_state_(last_masterchain_state), manager_(std::move(manager)) { + QueueSizeCounter(td::Ref last_masterchain_state, td::Ref opts, + td::actor::ActorId manager) + : init_masterchain_state_(last_masterchain_state), opts_(std::move(opts)), manager_(std::move(manager)) { } void start_up() override; void get_queue_size(BlockIdExt block_id, td::Promise promise); void alarm() override; + void update_options(td::Ref opts) { + opts_ = std::move(opts); + } + private: td::Ref init_masterchain_state_; + td::Ref opts_; td::actor::ActorId manager_; bool simple_mode_ = false; diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index 24dd77e8..ac86cf37 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -70,18 +70,13 @@ void ShardClient::got_init_handle_from_db(BlockHandle handle) { } void ShardClient::got_init_state_from_db(td::Ref state) { - masterchain_state_ = std::move(state); - build_shard_overlays(); - masterchain_state_.clear(); - saved_to_db(); } void ShardClient::start_up_init_mode() { - build_shard_overlays(); std::vector shards; for (const auto& s : masterchain_state_->get_shards()) { - if (opts_->need_monitor(s->shard())) { + if (opts_->need_monitor(s->shard(), masterchain_state_)) { shards.push_back(s->top_block_id()); } } @@ -166,7 +161,6 @@ void ShardClient::download_masterchain_state() { void ShardClient::got_masterchain_block_state(td::Ref state) { masterchain_state_ = std::move(state); - build_shard_overlays(); if (started_) { apply_all_shards(); } @@ -189,8 +183,10 @@ void ShardClient::apply_all_shards() { ig.add_promise(std::move(P)); auto vec = masterchain_state_->get_shards(); + std::set workchains; for (auto &shard : vec) { - if (opts_->need_monitor(shard->shard())) { + workchains.insert(shard->shard().workchain); + if (opts_->need_monitor(shard->shard(), masterchain_state_)) { auto Q = td::PromiseCreator::lambda([SelfId = actor_id(this), promise = ig.get_promise(), shard = shard->shard()](td::Result> R) mutable { if (R.is_error()) { @@ -200,7 +196,22 @@ void ShardClient::apply_all_shards() { } }); td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, shard->top_block_id(), - shard_client_priority(), td::Timestamp::in(600), std::move(Q)); + shard_client_priority(), td::Timestamp::in(1500), std::move(Q)); + } + } + for (const auto &[wc, desc] : masterchain_state_->get_workchain_list()) { + if (!workchains.count(wc) && desc->active && opts_->need_monitor(ShardIdFull{wc, shardIdAll}, masterchain_state_)) { + auto Q = td::PromiseCreator::lambda([SelfId = actor_id(this), promise = ig.get_promise(), + workchain = wc](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix(PSTRING() << "workchain " << workchain << ": ")); + } else { + td::actor::send_closure(SelfId, &ShardClient::downloaded_shard_state, R.move_as_ok(), std::move(promise)); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::wait_block_state_short, + BlockIdExt{wc, shardIdAll, 0, desc->zerostate_root_hash, desc->zerostate_file_hash}, + shard_client_priority(), td::Timestamp::in(1500), std::move(Q)); } } } @@ -223,7 +234,6 @@ void ShardClient::new_masterchain_block_notification(BlockHandle handle, td::Ref masterchain_block_handle_ = std::move(handle); masterchain_state_ = std::move(state); waiting_ = false; - build_shard_overlays(); apply_all_shards(); } @@ -244,26 +254,6 @@ void ShardClient::get_processed_masterchain_block_id(td::Promise pro } } -void ShardClient::build_shard_overlays() { - auto v = masterchain_state_->get_shards(); - - for (auto &x : v) { - auto shard = x->shard(); - if (opts_->need_monitor(shard)) { - auto d = masterchain_state_->monitor_min_split_depth(shard.workchain); - auto l = shard_prefix_length(shard.shard); - if (l > d) { - shard = shard_prefix(shard, d); - } - - if (created_overlays_.count(shard) == 0) { - created_overlays_.insert(shard); - td::actor::send_closure(manager_, &ValidatorManager::subscribe_to_shard, shard); - } - } - } -} - void ShardClient::force_update_shard_client(BlockHandle handle, td::Promise promise) { CHECK(!init_mode_); CHECK(!started_); @@ -294,10 +284,13 @@ void ShardClient::force_update_shard_client_ex(BlockHandle handle, td::Ref opts) { + opts_ = std::move(opts); +} + } // namespace validator } // namespace ton diff --git a/validator/shard-client.hpp b/validator/shard-client.hpp index c1676deb..7c2c978c 100644 --- a/validator/shard-client.hpp +++ b/validator/shard-client.hpp @@ -42,8 +42,6 @@ class ShardClient : public td::actor::Actor { td::Promise promise_; - std::set created_overlays_; - public: ShardClient(td::Ref opts, BlockHandle masterchain_block_handle, td::Ref masterchain_state, td::actor::ActorId manager, @@ -64,8 +62,6 @@ class ShardClient : public td::actor::Actor { return 2; } - void build_shard_overlays(); - void start_up() override; void start_up_init_mode(); void download_shard_states(BlockIdExt masterchain_block_id, std::vector shards, size_t idx); @@ -90,6 +86,8 @@ class ShardClient : public td::actor::Actor { void force_update_shard_client(BlockHandle handle, td::Promise promise); void force_update_shard_client_ex(BlockHandle handle, td::Ref state, td::Promise promise); + + void update_options(td::Ref opts); }; } // namespace validator diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index ab38a6e9..bc3d7b5e 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -18,7 +18,6 @@ */ #include "state-serializer.hpp" #include "td/utils/Random.h" -#include "adnl/utils.hpp" #include "ton/ton-io.hpp" #include "common/delay.h" #include "td/utils/filesystem.h" @@ -59,6 +58,12 @@ void AsyncStateSerializer::got_self_state(AsyncSerializerState state) { }); td::actor::send_closure(manager_, &ValidatorManager::get_block_handle, last_block_id_, true, std::move(P)); } + + inited_block_id_ = true; + for (auto& promise : wait_init_block_id_) { + promise.set_value(td::Unit()); + } + wait_init_block_id_.clear(); } void AsyncStateSerializer::got_init_handle(BlockHandle handle) { @@ -151,6 +156,21 @@ void AsyncStateSerializer::next_iteration() { CHECK(masterchain_handle_->id() == last_block_id_); if (attempt_ < max_attempt() && last_key_block_id_.id.seqno < last_block_id_.id.seqno && need_serialize(masterchain_handle_)) { + if (!stored_persistent_state_description_) { + LOG(INFO) << "storing persistent state description for " << masterchain_handle_->id().id; + running_ = true; + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &AsyncStateSerializer::fail_handler, + R.move_as_error_prefix("failed to get masterchain state: ")); + } else { + td::actor::send_closure(SelfId, &AsyncStateSerializer::store_persistent_state_description, + td::Ref(R.move_as_ok())); + } + }); + td::actor::send_closure(manager_, &ValidatorManager::get_shard_state_from_db, masterchain_handle_, std::move(P)); + return; + } if (!have_masterchain_state_ && !opts_->get_state_serializer_enabled()) { LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str() << ": serializer is disabled (by user)"; @@ -172,16 +192,15 @@ void AsyncStateSerializer::next_iteration() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_previous_state_files); }, td::Timestamp::in(delay)); + current_status_ = PSTRING() << "delay before serializing seqno=" << masterchain_handle_->id().seqno() << " " + << (int)delay << "s"; + current_status_ts_ = td::Timestamp::now(); return; } - while (next_idx_ < shards_.size()) { - if (!need_monitor(shards_[next_idx_].shard_full())) { - next_idx_++; - } else { - running_ = true; - request_shard_state(shards_[next_idx_]); - return; - } + if (next_idx_ < shards_.size()) { + running_ = true; + request_shard_state(shards_[next_idx_]); + return; } LOG(ERROR) << "finished serializing persistent state for " << masterchain_handle_->id().id.to_str(); } @@ -203,6 +222,7 @@ void AsyncStateSerializer::next_iteration() { if (masterchain_handle_->inited_next_left()) { last_block_id_ = masterchain_handle_->one_next(true); have_masterchain_state_ = false; + stored_persistent_state_description_ = false; masterchain_handle_ = nullptr; saved_to_db_ = false; shards_.clear(); @@ -217,6 +237,24 @@ void AsyncStateSerializer::got_top_masterchain_handle(BlockIdExt block_id) { } } +void AsyncStateSerializer::store_persistent_state_description(td::Ref state) { + stored_persistent_state_description_ = true; + attempt_ = 0; + running_ = false; + + PersistentStateDescription desc; + desc.masterchain_id = state->get_block_id(); + desc.start_time = state->get_unix_time(); + desc.end_time = ValidatorManager::persistent_state_ttl(desc.start_time); + for (const auto &v : state->get_shards()) { + desc.shard_blocks.push_back(v->top_block_id()); + } + td::actor::send_closure(manager_, &ValidatorManager::add_persistent_state_description, + td::Ref(true, std::move(desc))); + + next_iteration(); +} + void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) { CHECK(!masterchain_handle_); masterchain_handle_ = std::move(handle); @@ -318,8 +356,10 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state CHECK(shards_.size() == 0); auto vec = state->get_shards(); - for (auto& v : vec) { - shards_.push_back(v->top_block_id()); + for (auto &v : vec) { + if (opts_->need_monitor(v->shard(), state)) { + shards_.push_back(v->top_block_id()); + } } auto write_data = [shard = state->get_shard(), root = state->root_cell(), cell_db_reader, @@ -348,9 +388,14 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, masterchain_handle_->id(), masterchain_handle_->id(), write_data, std::move(P)); + + current_status_ = PSTRING() << "serializing masterchain state " << state->get_block_id().id.to_str(); + current_status_ts_ = td::Timestamp::now(); } void AsyncStateSerializer::stored_masterchain_state() { + current_status_ = "pending"; + current_status_ts_ = {}; LOG(ERROR) << "finished serializing masterchain state " << masterchain_handle_->id().id.to_str(); running_ = false; next_iteration(); @@ -413,9 +458,14 @@ void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Refid(), masterchain_handle_->id(), write_data, std::move(P)); + current_status_ = PSTRING() << "serializing shard state " << next_idx_ << "/" << shards_.size() << " " + << state->get_block_id().id.to_str(); + current_status_ts_ = td::Timestamp::now(); } void AsyncStateSerializer::fail_handler(td::Status reason) { + current_status_ = PSTRING() << "pending, " << reason; + current_status_ts_ = {}; VLOG(VALIDATOR_NOTICE) << "failure: " << reason; attempt_++; delay_action( @@ -429,6 +479,8 @@ void AsyncStateSerializer::fail_handler_cont() { } void AsyncStateSerializer::success_handler() { + current_status_ = "pending"; + current_status_ts_ = {}; running_ = false; next_iteration(); } @@ -447,9 +499,27 @@ void AsyncStateSerializer::auto_disable_serializer(bool disabled) { } } - -bool AsyncStateSerializer::need_monitor(ShardIdFull shard) { - return opts_->need_monitor(shard); +void AsyncStateSerializer::prepare_stats(td::Promise>> promise) { + if (!inited_block_id_) { + wait_init_block_id_.push_back( + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_STATUS_PROMISE(promise, R.move_as_status()); + td::actor::send_closure(SelfId, &AsyncStateSerializer::prepare_stats, std::move(promise)); + }); + return; + } + std::vector> vec; + vec.emplace_back("stateserializermasterchainseqno", td::to_string(last_block_id_.seqno())); + td::StringBuilder sb; + sb << current_status_; + if (current_status_ts_) { + sb << " (started " << (int)(td::Timestamp::now() - current_status_ts_) << "s ago)"; + } + if (!opts_->get_state_serializer_enabled() || auto_disabled_) { + sb << " (disabled)"; + } + vec.emplace_back("stateserializerstatus", sb.as_cslice().str()); + promise.set_result(std::move(vec)); } bool AsyncStateSerializer::need_serialize(BlockHandle handle) { diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index b38a216b..406ac350 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -36,6 +36,9 @@ class AsyncStateSerializer : public td::actor::Actor { UnixTime last_key_block_ts_ = 0; bool saved_to_db_ = true; + bool inited_block_id_ = false; + std::vector> wait_init_block_id_; + td::Ref opts_; bool auto_disabled_ = false; td::CancellationTokenSource cancellation_token_source_; @@ -46,6 +49,7 @@ class AsyncStateSerializer : public td::actor::Actor { td::uint32 next_idx_ = 0; BlockHandle masterchain_handle_; + bool stored_persistent_state_description_ = false; bool have_masterchain_state_ = false; std::vector shards_; @@ -69,7 +73,6 @@ class AsyncStateSerializer : public td::actor::Actor { } bool need_serialize(BlockHandle handle); - bool need_monitor(ShardIdFull shard); bool have_newer_persistent_state(UnixTime cur_ts); void alarm() override; @@ -84,6 +87,7 @@ class AsyncStateSerializer : public td::actor::Actor { void next_iteration(); void got_top_masterchain_handle(BlockIdExt block_id); + void store_persistent_state_description(td::Ref state); void got_masterchain_handle(BlockHandle handle_); void got_masterchain_state(td::Ref state, std::shared_ptr cell_db_reader); void stored_masterchain_state(); @@ -94,6 +98,8 @@ class AsyncStateSerializer : public td::actor::Actor { promise.set_result(last_block_id_.id.seqno); } + void prepare_stats(td::Promise>> promise); + void update_last_known_key_block_ts(UnixTime ts) { last_known_key_block_ts_ = std::max(last_known_key_block_ts_, ts); } @@ -110,6 +116,9 @@ class AsyncStateSerializer : public td::actor::Actor { void update_options(td::Ref opts); void auto_disable_serializer(bool disabled); + + std::string current_status_ = "pending"; + td::Timestamp current_status_ts_ = td::Timestamp::never(); }; } // namespace validator diff --git a/validator/stats-provider.h b/validator/stats-provider.h new file mode 100644 index 00000000..e0a7f565 --- /dev/null +++ b/validator/stats-provider.h @@ -0,0 +1,105 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once + +#include "validator.h" +#include "common/AtomicRef.h" + +#include + +namespace ton { + +namespace validator { + +class StatsProvider { + public: + StatsProvider() = default; + StatsProvider(td::actor::ActorId manager, std::string prefix, + std::function>>)> callback) + : inited_(true), manager_(std::move(manager)) { + static std::atomic cur_idx{0}; + idx_ = cur_idx.fetch_add(1); + td::actor::send_closure(manager_, &ValidatorManagerInterface::register_stats_provider, idx_, std::move(prefix), + std::move(callback)); + } + StatsProvider(const StatsProvider&) = delete; + StatsProvider(StatsProvider&& other) noexcept + : inited_(other.inited_), idx_(other.idx_), manager_(std::move(other.manager_)) { + other.inited_ = false; + } + ~StatsProvider() { + if (inited_) { + td::actor::send_closure(manager_, &ValidatorManagerInterface::unregister_stats_provider, idx_); + } + } + + StatsProvider& operator=(const StatsProvider&) = delete; + StatsProvider& operator=(StatsProvider&& other) noexcept { + if (this != &other) { + inited_ = other.inited_; + idx_ = other.idx_; + manager_ = std::move(other.manager_); + other.inited_ = false; + } + return *this; + } + + bool inited() const { + return inited_; + } + + private: + bool inited_ = false; + td::uint64 idx_ = 0; + td::actor::ActorId manager_; +}; + +class ProcessStatus { + public: + ProcessStatus() = default; + ProcessStatus(td::actor::ActorId manager, std::string name) + : stats_provider_(std::move(manager), std::move(name), [value = value_](auto promise) { + auto status = value->load(); + if (status.is_null()) { + promise.set_error(td::Status::Error("empty")); + return; + } + std::vector> vec; + vec.emplace_back("", *status); + promise.set_value(std::move(vec)); + }) { + } + ProcessStatus(const ProcessStatus&) = delete; + ProcessStatus(ProcessStatus&& other) noexcept = default; + ProcessStatus& operator=(const ProcessStatus&) = delete; + ProcessStatus& operator=(ProcessStatus&& other) noexcept = default; + + void set_status(std::string s) { + if (!value_) { + return; + } + value_->store(td::Ref>(true, std::move(s))); + } + + private: + std::shared_ptr>> value_ = std::make_shared>>(); + StatsProvider stats_provider_; +}; + +} // namespace validator + +} // namespace ton diff --git a/validator/token-manager.cpp b/validator/token-manager.cpp index 0bc4a9c6..8242f921 100644 --- a/validator/token-manager.cpp +++ b/validator/token-manager.cpp @@ -22,23 +22,23 @@ namespace ton { namespace validator { -void TokenManager::get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) { +void TokenManager::get_token(size_t size, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) { if (free_priority_tokens_ > 0 && priority > 0) { --free_priority_tokens_; - promise.set_value(gen_token(download_size, priority)); + promise.set_value(gen_token(size, priority)); return; } if (free_tokens_ > 0) { --free_tokens_; - promise.set_value(gen_token(download_size, priority)); + promise.set_value(gen_token(size, priority)); return; } - pending_.emplace(PendingPromiseKey{download_size, priority, seqno_++}, PendingPromise{timeout, std::move(promise)}); + pending_.emplace(PendingPromiseKey{size, priority, seqno_++}, PendingPromise{timeout, std::move(promise)}); } -void TokenManager::download_token_cleared(size_t download_size, td::uint32 priority) { +void TokenManager::token_cleared(size_t size, td::uint32 priority) { (priority ? free_priority_tokens_ : free_tokens_)++; if (free_priority_tokens_ > max_priority_tokens_) { free_priority_tokens_--; @@ -47,7 +47,7 @@ void TokenManager::download_token_cleared(size_t download_size, td::uint32 prior for (auto it = pending_.begin(); it != pending_.end();) { if (it->first.priority && (free_tokens_ || free_priority_tokens_)) { - it->second.promise.set_value(gen_token(download_size, priority)); + it->second.promise.set_value(gen_token(size, priority)); auto it2 = it++; pending_.erase(it2); if (free_priority_tokens_ > 0) { @@ -56,7 +56,7 @@ void TokenManager::download_token_cleared(size_t download_size, td::uint32 prior free_tokens_--; } } else if (!it->first.priority && free_tokens_) { - it->second.promise.set_value(gen_token(download_size, priority)); + it->second.promise.set_value(gen_token(size, priority)); auto it2 = it++; pending_.erase(it2); free_tokens_--; @@ -69,7 +69,7 @@ void TokenManager::download_token_cleared(size_t download_size, td::uint32 prior void TokenManager::alarm() { for (auto it = pending_.begin(); it != pending_.end();) { if (it->second.timeout.is_in_past()) { - it->second.promise.set_error(td::Status::Error(ErrorCode::timeout, "timeout in wait download token")); + it->second.promise.set_error(td::Status::Error(ErrorCode::timeout, "timeout in wait token")); it = pending_.erase(it); } else { it++; @@ -77,23 +77,23 @@ void TokenManager::alarm() { } } -std::unique_ptr TokenManager::gen_token(size_t download_size, td::uint32 priority) { - class Token : public DownloadToken { +std::unique_ptr TokenManager::gen_token(size_t size, td::uint32 priority) { + class TokenImpl : public ActionToken { public: - Token(size_t download_size, td::uint32 priority, td::actor::ActorId manager) - : download_size_(download_size), priority_(priority), manager_(manager) { + TokenImpl(size_t size, td::uint32 priority, td::actor::ActorId manager) + : size_(size), priority_(priority), manager_(manager) { } - ~Token() override { - td::actor::send_closure(manager_, &TokenManager::download_token_cleared, download_size_, priority_); + ~TokenImpl() override { + td::actor::send_closure(manager_, &TokenManager::token_cleared, size_, priority_); } private: - size_t download_size_; + size_t size_; td::uint32 priority_; td::actor::ActorId manager_; }; - return std::make_unique(download_size, priority, actor_id(this)); + return std::make_unique(size, priority, actor_id(this)); } } // namespace validator diff --git a/validator/token-manager.h b/validator/token-manager.h index 0d75710f..0fd0126a 100644 --- a/validator/token-manager.h +++ b/validator/token-manager.h @@ -31,16 +31,19 @@ class TokenManager : public td::actor::Actor { public: TokenManager() { } + explicit TokenManager(td::uint32 max_tokens) + : free_tokens_(max_tokens), free_priority_tokens_(max_tokens), max_priority_tokens_(max_tokens) { + } void alarm() override; - void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise); - void download_token_cleared(size_t download_size, td::uint32 priority); + void get_token(size_t size, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise); + void token_cleared(size_t size, td::uint32 priority); private: - std::unique_ptr gen_token(size_t download_size, td::uint32 priority); + std::unique_ptr gen_token(size_t size, td::uint32 priority); struct PendingPromiseKey { - size_t download_size; + size_t size; td::uint32 priority; td::uint64 seqno; @@ -50,7 +53,7 @@ class TokenManager : public td::actor::Actor { }; struct PendingPromise { td::Timestamp timeout; - td::Promise> promise; + td::Promise> promise; }; td::uint64 seqno_ = 0; std::map pending_; diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 1817180d..110ccd81 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -373,6 +373,7 @@ void ValidatorGroup::create_session() { } CHECK(found); + config_.catchain_opts.broadcast_speed_multiplier = opts_->get_catchain_broadcast_speed_multiplier(); if (!config_.new_catchain_ids) { session_ = validatorsession::ValidatorSession::create(session_id_, config_, local_id_, std::move(vec), make_validator_session_callback(), keyring_, adnl_, rldp_, diff --git a/validator/validator-options.cpp b/validator/validator-options.cpp index 93fe05e6..cb26fe44 100644 --- a/validator/validator-options.cpp +++ b/validator/validator-options.cpp @@ -26,7 +26,7 @@ namespace validator { td::Ref ValidatorManagerOptions::create( BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard, bool allow_blockchain_init, + std::function check_shard, bool allow_blockchain_init, double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) { return td::make_ref(zero_block_id, init_block_id, std::move(check_shard), diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 203aa5eb..ace6b106 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -32,11 +32,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { BlockIdExt init_block_id() const override { return init_block_id_; } - bool need_monitor(ShardIdFull shard) const override { - return check_shard_(shard, 0, ShardCheckMode::m_monitor); - } - bool need_validate(ShardIdFull shard, CatchainSeqno cc_seqno) const override { - return check_shard_(shard, cc_seqno, ShardCheckMode::m_validate); + bool need_monitor(ShardIdFull shard, const td::Ref& state) const override { + td::uint32 min_split = state->monitor_min_split_depth(shard.workchain); + return check_shard_((td::uint32)shard.pfx_len() <= min_split ? shard : shard_prefix(shard, min_split)); } bool allow_blockchain_init() const override { return allow_blockchain_init_; @@ -156,6 +154,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool get_fast_state_serializer_enabled() const override { return fast_state_serializer_enabled_; } + double get_catchain_broadcast_speed_multiplier() const override { + return catchain_broadcast_speed_multipliers_; + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -163,7 +164,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_init_block_id(BlockIdExt block_id) override { init_block_id_ = block_id; } - void set_shard_check_function(std::function check_shard) override { + void set_shard_check_function(std::function check_shard) override { check_shard_ = std::move(check_shard); } void set_allow_blockchain_init(bool value) override { @@ -251,17 +252,18 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_fast_state_serializer_enabled(bool value) override { fast_state_serializer_enabled_ = value; } + void set_catchain_broadcast_speed_multiplier(double value) override { + catchain_broadcast_speed_multipliers_ = value; + } ValidatorManagerOptionsImpl *make_copy() const override { return new ValidatorManagerOptionsImpl(*this); } ValidatorManagerOptionsImpl(BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard, - bool allow_blockchain_init, double sync_blocks_before, - double block_ttl, double state_ttl, double max_mempool_num, - double archive_ttl, double key_proof_ttl, - bool initial_sync_disabled) + std::function check_shard, bool allow_blockchain_init, + double sync_blocks_before, double block_ttl, double state_ttl, double max_mempool_num, + double archive_ttl, double key_proof_ttl, bool initial_sync_disabled) : zero_block_id_(zero_block_id) , init_block_id_(init_block_id) , check_shard_(std::move(check_shard)) @@ -278,7 +280,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { private: BlockIdExt zero_block_id_; BlockIdExt init_block_id_; - std::function check_shard_; + std::function check_shard_; bool allow_blockchain_init_; double sync_blocks_before_; double block_ttl_; @@ -306,6 +308,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool state_serializer_enabled_ = true; td::Ref collator_options_{true}; bool fast_state_serializer_enabled_ = false; + double catchain_broadcast_speed_multipliers_; }; } // namespace validator diff --git a/validator/validator-telemetry.cpp b/validator/validator-telemetry.cpp new file mode 100644 index 00000000..403dd6f9 --- /dev/null +++ b/validator/validator-telemetry.cpp @@ -0,0 +1,87 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#include "validator-telemetry.hpp" +#include "git.h" +#include "td/utils/Random.h" +#include "td/utils/port/uname.h" +#include "interfaces/validator-manager.h" + +namespace ton::validator { + +void ValidatorTelemetry::start_up() { + node_version_ = PSTRING() << "validator-engine, Commit: " << GitMetadata::CommitSHA1() + << ", Date: " << GitMetadata::CommitDate(); + + os_version_ = td::get_operating_system_version().str(); + + auto r_total_mem_stat = td::get_total_mem_stat(); + if (r_total_mem_stat.is_error()) { + LOG(WARNING) << "Cannot get RAM size: " << r_total_mem_stat.move_as_error(); + } else { + ram_size_ = r_total_mem_stat.ok().total_ram; + } + + auto r_cpu_cores = td::get_cpu_cores(); + if (r_cpu_cores.is_error()) { + LOG(WARNING) << "Cannot get CPU info: " << r_cpu_cores.move_as_error(); + } else { + cpu_cores_ = r_cpu_cores.move_as_ok(); + } + + LOG(DEBUG) << "Initializing validator telemetry, key = " << key_ << ", adnl_id = " << local_id_; + alarm_timestamp().relax(send_telemetry_at_ = td::Timestamp::in(td::Random::fast(30.0, 60.0))); +} + +void ValidatorTelemetry::alarm() { + if (send_telemetry_at_.is_in_past()) { + send_telemetry_at_ = td::Timestamp::never(); + send_telemetry(); + } + alarm_timestamp().relax(send_telemetry_at_); +} + +void ValidatorTelemetry::send_telemetry() { + send_telemetry_at_ = td::Timestamp::in(PERIOD); + + auto telemetry = create_tl_object(); + telemetry->flags_ = 0; + telemetry->timestamp_ = td::Clocks::system(); + telemetry->adnl_id_ = local_id_.bits256_value(); + telemetry->node_version_ = node_version_; + telemetry->os_version_ = os_version_; + telemetry->node_started_at_ = adnl::Adnl::adnl_start_time(); + telemetry->ram_size_ = ram_size_; + telemetry->cpu_cores_ = cpu_cores_; + telemetry->node_threads_ = (td::int32)td::actor::SchedulerContext::get() + ->scheduler_group() + ->schedulers.at(td::actor::SchedulerContext::get()->get_scheduler_id().value()) + .cpu_threads_count; + + LOG(DEBUG) << "Sending validator telemetry for adnl id " << local_id_; + td::actor::send_closure(manager_, &ValidatorManager::send_validator_telemetry, key_, std::move(telemetry)); +} + +} // namespace ton::validator diff --git a/validator/validator-telemetry.hpp b/validator/validator-telemetry.hpp new file mode 100644 index 00000000..73908bdd --- /dev/null +++ b/validator/validator-telemetry.hpp @@ -0,0 +1,66 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. +*/ +#pragma once +#include "overlay.h" +#include "td/actor/actor.h" +#include "adnl/adnl.h" +#include "interfaces/shard.h" + +namespace ton::validator { +class ValidatorManager; + +class ValidatorTelemetry : public td::actor::Actor { +public: + ValidatorTelemetry(PublicKeyHash key, adnl::AdnlNodeIdShort local_id, td::Bits256 zero_state_file_hash, + td::actor::ActorId manager) + : key_(key) + , local_id_(local_id) + , zero_state_file_hash_(zero_state_file_hash) + , manager_(std::move(manager)) { + } + + void start_up() override; + void alarm() override; + +private: + PublicKeyHash key_; + adnl::AdnlNodeIdShort local_id_; + td::Bits256 zero_state_file_hash_; + td::actor::ActorId manager_; + + std::string node_version_; + std::string os_version_; + td::uint32 cpu_cores_ = 0; + td::uint64 ram_size_ = 0; + + td::Timestamp send_telemetry_at_ = td::Timestamp::never(); + + void send_telemetry(); + + static constexpr double PERIOD = 600.0; + static constexpr td::uint32 MAX_SIZE = 8192; +}; +} // namespace ton::validator \ No newline at end of file diff --git a/validator/validator.h b/validator/validator.h index 9dbaa185..5d6c0173 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -20,6 +20,7 @@ #include #include +#include #include "td/actor/actor.h" @@ -35,15 +36,16 @@ #include "interfaces/proof.h" #include "interfaces/shard.h" #include "catchain/catchain-types.h" +#include "interfaces/out-msg-queue-proof.h" #include "interfaces/external-message.h" namespace ton { namespace validator { -class DownloadToken { +class ActionToken { public: - virtual ~DownloadToken() = default; + virtual ~ActionToken() = default; }; struct PerfTimerStats { @@ -77,8 +79,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual BlockIdExt zero_block_id() const = 0; virtual BlockIdExt init_block_id() const = 0; - virtual bool need_monitor(ShardIdFull shard) const = 0; - virtual bool need_validate(ShardIdFull shard, CatchainSeqno cc_seqno) const = 0; + virtual bool need_monitor(ShardIdFull shard, const td::Ref& state) const = 0; virtual bool allow_blockchain_init() const = 0; virtual double sync_blocks_before() const = 0; virtual double block_ttl() const = 0; @@ -115,11 +116,11 @@ struct ValidatorManagerOptions : public td::CntObject { virtual bool get_state_serializer_enabled() const = 0; virtual td::Ref get_collator_options() const = 0; virtual bool get_fast_state_serializer_enabled() const = 0; + virtual double get_catchain_broadcast_speed_multiplier() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; - virtual void set_shard_check_function( - std::function check_shard) = 0; + virtual void set_shard_check_function(std::function check_shard) = 0; virtual void set_allow_blockchain_init(bool value) = 0; virtual void set_sync_blocks_before(double value) = 0; virtual void set_block_ttl(double value) = 0; @@ -148,15 +149,15 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_state_serializer_enabled(bool value) = 0; virtual void set_collator_options(td::Ref value) = 0; virtual void set_fast_state_serializer_enabled(bool value) = 0; + virtual void set_catchain_broadcast_speed_multiplier(double value) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, - std::function check_shard = [](ShardIdFull, CatchainSeqno, - ShardCheckMode) { return true; }, + + std::function check_shard = [](ShardIdFull) { return true; }, bool allow_blockchain_init = false, double sync_blocks_before = 3600, double block_ttl = 86400, double state_ttl = 86400, double archive_ttl = 86400 * 7, double key_proof_ttl = 86400 * 3650, - double max_mempool_num = 999999, - bool initial_sync_disabled = false); + double max_mempool_num = 999999, bool initial_sync_disabled = false); }; class ValidatorManagerInterface : public td::actor::Actor { @@ -166,8 +167,8 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual ~Callback() = default; virtual void initial_read_complete(BlockHandle top_masterchain_blocks) = 0; - virtual void add_shard(ShardIdFull shard) = 0; - virtual void del_shard(ShardIdFull shard) = 0; + virtual void on_new_masterchain_block(td::Ref state, + std::set shards_to_monitor) = 0; virtual void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; @@ -187,10 +188,14 @@ class ValidatorManagerInterface : public td::actor::Actor { td::Promise promise) = 0; virtual void get_next_key_blocks(BlockIdExt block_id, td::Timestamp timeout, td::Promise> promise) = 0; - virtual void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, - td::Promise promise) = 0; + virtual void download_archive(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, std::string tmp_dir, + td::Timestamp timeout, td::Promise promise) = 0; + virtual void download_out_msg_queue_proof(ShardIdFull dst_shard, std::vector blocks, + block::ImportedMsgQueueLimits limits, td::Timestamp timeout, + td::Promise>> promise) = 0; virtual void new_key_block(BlockHandle handle) = 0; + virtual void send_validator_telemetry(PublicKeyHash key, tl_object_ptr telemetry) = 0; }; virtual ~ValidatorManagerInterface() = default; @@ -250,7 +255,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void add_ext_server_port(td::uint16 port) = 0; virtual void get_download_token(size_t download_size, td::uint32 priority, td::Timestamp timeout, - td::Promise> promise) = 0; + td::Promise> promise) = 0; virtual void get_block_data_from_db(ConstBlockHandle handle, td::Promise> promise) = 0; virtual void get_block_data_from_db_short(BlockIdExt block_id, td::Promise> promise) = 0; @@ -271,7 +276,13 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_block_by_seqno_from_db(AccountIdPrefixFull account, BlockSeqno seqno, td::Promise promise) = 0; - virtual void get_archive_id(BlockSeqno masterchain_seqno, td::Promise promise) = 0; + virtual void wait_block_state(BlockHandle handle, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) = 0; + virtual void wait_block_state_short(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, + td::Promise> promise) = 0; + + virtual void get_archive_id(BlockSeqno masterchain_seqno, ShardIdFull shard_prefix, + td::Promise promise) = 0; virtual void get_archive_slice(td::uint64 archive_id, td::uint64 offset, td::uint32 limit, td::Promise promise) = 0; @@ -284,6 +295,13 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) = 0; virtual void update_options(td::Ref opts) = 0; + + virtual void register_stats_provider( + td::uint64 idx, std::string prefix, + std::function>>)> callback) { + } + virtual void unregister_stats_provider(td::uint64 idx) { + } }; } // namespace validator