external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-02-13 03:32:22 +00:00
Code Issues Releases Wiki Activity

Merge pull request #584 from ton-blockchain/testnet

Browse source 
Merge 12.2022 updates
This commit is contained in:
EmelyanenkoK 2023-01-08 16:04:27 +03:00 committed by GitHub
commit f59c363ab9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
212 changed files with 13405 additions and 2653 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
.github/script/amd64-18.04.Dockerfile vendored Normal file
View file

@ -0,0 +1,19 @@
FROM ubuntu:18.04
RUN apt update
RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata
RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git curl libreadline-dev ccache libmicrohttpd-dev ninja-build
WORKDIR /
ARG BRANCH
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH
WORKDIR /ton
RUN mkdir /ton/build
WORKDIR /ton/build
ENV CC clang
ENV CXX clang++
ENV CCACHE_DISABLE 1
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client

19
.github/script/amd64-20.04.Dockerfile vendored Normal file
View file

@ -0,0 +1,19 @@
FROM ubuntu:20.04
RUN apt update
RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata
RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git curl libreadline-dev ccache libmicrohttpd-dev ninja-build
WORKDIR /
ARG BRANCH
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH
WORKDIR /ton
RUN mkdir /ton/build
WORKDIR /ton/build
ENV CC clang
ENV CXX clang++
ENV CCACHE_DISABLE 1
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client

19
.github/script/amd64-22.04.Dockerfile vendored Normal file
View file

@ -0,0 +1,19 @@
FROM ubuntu:22.04
RUN apt update
RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata
RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git curl libreadline-dev ccache libmicrohttpd-dev ninja-build
WORKDIR /
ARG BRANCH
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH
WORKDIR /ton
RUN mkdir /ton/build
WORKDIR /ton/build
ENV CC clang
ENV CXX clang++
ENV CCACHE_DISABLE 1
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client

19
.github/script/arm64-18.04.Dockerfile vendored Normal file
View file

@ -0,0 +1,19 @@
FROM ubuntu:18.04
RUN apt update
RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata
RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git curl libreadline-dev ccache libmicrohttpd-dev ninja-build
WORKDIR /
ARG BRANCH
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH
WORKDIR /ton
RUN mkdir /ton/build
WORKDIR /ton/build
ENV CC clang
ENV CXX clang++
ENV CCACHE_DISABLE 1
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= ..
RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id dht-server lite-client

19
.github/script/arm64-20.04.Dockerfile vendored Normal file
View file

@ -0,0 +1,19 @@
FROM ubuntu:20.04
RUN apt update
RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata
RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git curl libreadline-dev ccache libmicrohttpd-dev ninja-build
WORKDIR /
ARG BRANCH
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH
WORKDIR /ton
RUN mkdir /ton/build
WORKDIR /ton/build
ENV CC clang
ENV CXX clang++
ENV CCACHE_DISABLE 1
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= ..
RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id dht-server lite-client

19
.github/script/arm64-22.04.Dockerfile vendored Normal file
View file

@ -0,0 +1,19 @@
FROM ubuntu:22.04
RUN apt update
RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata
RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git curl libreadline-dev ccache libmicrohttpd-dev ninja-build
WORKDIR /
ARG BRANCH
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH
WORKDIR /ton
RUN mkdir /ton/build
WORKDIR /ton/build
ENV CC clang
ENV CXX clang++
ENV CCACHE_DISABLE 1
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= ..
RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id dht-server lite-client

167
.github/workflows/create-release.yml vendored Normal file
View file

@ -0,0 +1,167 @@
name: Create release
on: [workflow_dispatch]
permissions: write-all
jobs:
create-release:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
- name: Show all artifacts
run: |
mkdir artifacts
ls -lart artifacts
- name: Download Ubuntu x86-64 artifacts
uses: dawidd6/action-download-artifact@v2
with:
workflow: ubuntu-compile.yml
path: artifacts
workflow_conclusion: success
skip_unpack: true
- name: Download Ubuntu arm64 artifacts
uses: dawidd6/action-download-artifact@v2
with:
workflow: docker-compile-ubuntu.yml
path: artifacts
workflow_conclusion: success
skip_unpack: true
- name: Download MacOS 11.7 artifacts
uses: dawidd6/action-download-artifact@v2
with:
workflow: macos-11.7-compile.yml
path: artifacts
workflow_conclusion: success
skip_unpack: true
- name: Download MacOS 12.6 artifacts
uses: dawidd6/action-download-artifact@v2
with:
workflow: macos-12.6-compile.yml
path: artifacts
workflow_conclusion: success
skip_unpack: true
- name: Download Windows artifacts
uses: dawidd6/action-download-artifact@v2
with:
workflow: win-2019-compile.yml
path: artifacts
workflow_conclusion: success
skip_unpack: true
- name: Show all artifacts
run: |
tree artifacts
# create release
- name: Read Changelog.md and use it as a body of new release
id: read_release
shell: bash
run: |
r=$(cat Changelog.md)
r="${r//'%'/'%25'}"
r="${r//$'\n'/'%0A'}"
r="${r//$'\r'/'%0D'}"
echo "::set-output name=CHANGELOG_BODY::$r"
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y.%m')"
- name: Get registration token
id: getRegToken
run: |
curl -X POST -H \"Accept: application/vnd.github+json\" -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' https://api.github.com/repos/neodix42/HardTestDevelopment/actions/runners/registration-token
- name: Create release
id: create_release
uses: actions/create-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: v${{ steps.date.outputs.date }}
release_name: v${{ steps.date.outputs.date }}
body: |
${{ steps.read_release.outputs.CHANGELOG_BODY }}
draft: false
prerelease: false
- name: Upload Windows 2019 artifacts
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-win-binaries.zip
asset_name: ton-windows-2019-x86-64.zip
tag: v${{ steps.date.outputs.date }}
- name: Upload MacOS 11.7 x86-64 artifacts
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-macos-11.7.zip
asset_name: ton-macos-11.7-x86-64.zip
tag: v${{ steps.date.outputs.date }}
- name: Upload MacOS 12.6 x86-64 artifacts
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-macos-12.6.zip
asset_name: ton-macos-12.6-x86-64.zip
tag: v${{ steps.date.outputs.date }}
- name: Upload Ubuntu 18.04 x86-64 artifacts
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-binaries-ubuntu-18.04.zip
asset_name: ton-ubuntu-18.04-x86-64.zip
tag: v${{ steps.date.outputs.date }}
- name: Upload Ubuntu 20.04 x86-64 artifacts
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-binaries-ubuntu-20.04.zip
asset_name: ton-ubuntu-20.04-x86-64.zip
tag: v${{ steps.date.outputs.date }}
- name: Upload Ubuntu 22.04 x86-64 artifacts
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-binaries-ubuntu-22.04.zip
asset_name: ton-ubuntu-22.04-x86-64.zip
tag: v${{ steps.date.outputs.date }}
- name: Upload Ubuntu 18.04 arm64 artifacts
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-ubuntu-18.04-arm64.zip
asset_name: ton-ubuntu-18.04-arm64.zip
tag: v${{ steps.date.outputs.date }}
- name: Upload Ubuntu 20.04 arm64 artifacts
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-ubuntu-20.04-arm64.zip
asset_name: ton-ubuntu-20.04-arm64.zip
tag: v${{ steps.date.outputs.date }}
- name: Upload Ubuntu 22.04 arm64 artifacts
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-ubuntu-22.04-arm64.zip
asset_name: ton-ubuntu-22.04-arm64.zip
tag: v${{ steps.date.outputs.date }}

57
.github/workflows/docker-compile-ubuntu.yml vendored Normal file
View file

@ -0,0 +1,57 @@
name: Docker Ubuntu Compile arm64
on: [push,workflow_dispatch,workflow_call]
jobs:
build:
strategy:
fail-fast: false
max-parallel: 3
matrix:
arch: [arm64]
ver: [22.04, 18.04, 20.04 ]
runs-on: ubuntu-22.04
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Set output
id: vars
run: echo ::set-output name=short_ref::${GITHUB_REF#refs/*/}
- name: Check output
run: echo branch ${{ steps.vars.outputs.short_ref }}
- name: Build with docker buildx
run: |
mkdir build-${{matrix.ver}}-${{matrix.arch}}
docker buildx build --build-arg BRANCH=${{ steps.vars.outputs.short_ref }} --platform=linux/${{matrix.arch}} --progress=plain --load . -t build-${{matrix.ver}}-${{matrix.arch}} -f .github/script/${{matrix.arch}}-${{matrix.ver}}.Dockerfile
container_id=$(docker create --platform=linux/${{matrix.arch}} build-${{matrix.ver}}-${{matrix.arch}})
docker cp $container_id:/ton/build/dht-server/dht-server build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/validator-engine/validator-engine build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/validator-engine-console/validator-engine-console build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/storage/storage-daemon/storage-daemon build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/storage/storage-daemon/storage-daemon-cli build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/crypto/fift build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/crypto/func build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/crypto/create-state build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/blockchain-explorer/blockchain-explorer build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/lite-client/lite-client build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/utils/generate-random-id build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/tonlib/libtonlibjson.so.0.5 build-${{matrix.ver}}-${{matrix.arch}}/tonlibjson.so
docker cp -a $container_id:/ton/crypto/smartcont build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/crypto/fift/lib build-${{matrix.ver}}-${{matrix.arch}}/
- name: Upload artifacts
uses: actions/upload-artifact@v1
with:
name: ton-ubuntu-${{matrix.ver}}-${{matrix.arch}}
path: build-${{matrix.ver}}-${{matrix.arch}}

4
.github/workflows/docker-ubuntu-image.yml vendored
View file

@ -3,6 +3,8 @@ name: Docker Ubuntu 20.04 image
on:
workflow_dispatch:
push:
branches:
- 'master'
env:
REGISTRY: ghcr.io
@ -13,7 +15,7 @@ jobs:
runs-on: ubuntu-20.04
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v1

41
.github/workflows/macos-10.15-compile.yml vendored
View file

@ -1,41 +0,0 @@
name: C/C++ CI macOS-10.15 Compile
on:
workflow_dispatch:
push:
jobs:
build:
runs-on: macos-10.15
steps:
- name: Check out repository
uses: actions/checkout@v2
with:
submodules: 'recursive'
- name: Compile OpenSSL
run: |
git clone https://github.com/openssl/openssl openssl_1_1_1
cd openssl_1_1_1
git checkout OpenSSL_1_1_1-stable
./Configure --prefix=/usr/local/macos darwin64-x86_64-cc -static -mmacosx-version-min=10.15
make build_libs -j4
- name: Build all
run: |
rootPath=`pwd`
mkdir build
cd build
cmake -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.15 -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_BUILD_TYPE=Release ..
make -j4
- name: Find & copy binaries
run: |
rsync -r --exclude 'CMakeFiles' --exclude 'Makefile' --exclude '*.a' --exclude '*.cmake' --exclude 'third-party' --exclude 'test-*' --exclude '*.cc' --exclude '*.json' --exclude '*.txt' build/* artifacts/
- name: Upload artifacts
uses: actions/upload-artifact@master
with:
name: ton-macos-binaries
path: artifacts

60
.github/workflows/macos-10.15-tonlib-java.yml vendored
View file

@ -1,60 +0,0 @@
name: macOS-10.15 tonlib-java
on:
workflow_dispatch:
push:
branches:
- 'wallets'
jobs:
build:
runs-on: macos-10.15
steps:
- name: Check out repository
uses: actions/checkout@v2
with:
submodules: 'recursive'
- name: Compile OpenSSL
run: |
git clone https://github.com/openssl/openssl openssl_1_1_1
cd openssl_1_1_1
git checkout OpenSSL_1_1_1-stable
./Configure --prefix=/usr/local/macos darwin64-x86_64-cc -static -mmacosx-version-min=10.15
make build_libs -j4
- name: Configure & Build
run: |
rootPath=`pwd`
export CC=$(which clang)
export CXX=$(which clang++)
export CCACHE_DISABLE=1
export JAVA_AWT_LIBRARY=NotNeeded
export JAVA_JVM_LIBRARY=NotNeeded
export JAVA_INCLUDE_PATH=${JAVA_HOME}/include
export JAVA_AWT_INCLUDE_PATH=${JAVA_HOME}/include
export JAVA_INCLUDE_PATH2=${JAVA_HOME}/include/darwin
cd example/android/
mkdir build
cd build
cmake -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.15 -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DTON_ONLY_TONLIB=ON ..
cmake --build . --target prepare_cross_compiling
cmake --build . --target native-lib --config Release
- name: find & copy binaries
run: |
mkdir -p artifacts/tonlib-java
cp example/android/src/drinkless/org/ton/TonApi.java artifacts/tonlib-java/
cp example/android/build/libnative-lib.dylib artifacts/tonlib-java/
- name: Upload artifacts
uses: actions/upload-artifact@master
with:
name: tonlib-macos-java
path: artifacts

61
.github/workflows/macos-11.7-compile.yml vendored Normal file
View file

@ -0,0 +1,61 @@
name: MacOS 11.7 Big Sur x86-64 Compile
on: [push,workflow_dispatch,workflow_call]
jobs:
build:
runs-on: macos-11
steps:
- name: Check out repository
uses: actions/checkout@v3
with:
submodules: 'recursive'
- name: Compile OpenSSL
run: |
git clone https://github.com/openssl/openssl openssl_1_1_1
cd openssl_1_1_1
git checkout OpenSSL_1_1_1-stable
./Configure --prefix=/usr/local/macos darwin64-x86_64-cc -static -mmacosx-version-min=11.7
make build_libs -j4
- name: Build all
run: |
export NONINTERACTIVE=1
brew install ninja
rootPath=`pwd`
mkdir build
cd build
cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=11.7 -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_BUILD_TYPE=Release ..
ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc
- name: Find & copy binaries
run: |
mkdir artifacts
cp build/storage/storage-daemon/storage-daemon artifacts/
cp build/storage/storage-daemon/storage-daemon-cli artifacts/
cp build/crypto/fift artifacts/
cp build/crypto/func artifacts/
cp build/crypto/create-state artifacts/
cp build/crypto/tlbc artifacts/
cp build/validator-engine-console/validator-engine-console artifacts/
cp build/tonlib/tonlib-cli artifacts/
cp build/tonlib/libtonlibjson.0.5.dylib artifacts/
cp build/http/http-proxy artifacts/
cp build/rldp-http-proxy/rldp-http-proxy artifacts/
cp build/dht-server/dht-server artifacts/
cp build/lite-client/lite-client artifacts/
cp build/validator-engine/validator-engine artifacts/
cp build/utils/generate-random-id artifacts/
cp build/utils/json2tlo artifacts/
cp build/adnl/adnl-proxy artifacts/
rsync -r crypto/smartcont artifacts/
rsync -r crypto/fift/lib artifacts/
ls -laRt artifacts
- name: Upload artifacts
uses: actions/upload-artifact@master
with:
name: ton-macos-11.7
path: artifacts

61
.github/workflows/macos-12.6-compile.yml vendored Normal file
View file

@ -0,0 +1,61 @@
name: MacOS 12.6 Monterey x86-64 Compile
on: [push,workflow_dispatch,workflow_call]
jobs:
build:
runs-on: macos-12
steps:
- name: Check out repository
uses: actions/checkout@v3
with:
submodules: 'recursive'
- name: Compile OpenSSL
run: |
git clone https://github.com/openssl/openssl openssl_1_1_1
cd openssl_1_1_1
git checkout OpenSSL_1_1_1-stable
./Configure --prefix=/usr/local/macos darwin64-x86_64-cc -static -mmacosx-version-min=12.6
make build_libs -j4
- name: Build all
run: |
export NONINTERACTIVE=1
brew install ninja
rootPath=`pwd`
mkdir build
cd build
cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=12.6 -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_BUILD_TYPE=Release ..
ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc
- name: Find & copy binaries
run: |
mkdir artifacts
cp build/storage/storage-daemon/storage-daemon artifacts/
cp build/storage/storage-daemon/storage-daemon-cli artifacts/
cp build/crypto/fift artifacts/
cp build/crypto/func artifacts/
cp build/crypto/create-state artifacts/
cp build/crypto/tlbc artifacts/
cp build/validator-engine-console/validator-engine-console artifacts/
cp build/tonlib/tonlib-cli artifacts/
cp build/tonlib/libtonlibjson.0.5.dylib artifacts/
cp build/http/http-proxy artifacts/
cp build/rldp-http-proxy/rldp-http-proxy artifacts/
cp build/dht-server/dht-server artifacts/
cp build/lite-client/lite-client artifacts/
cp build/validator-engine/validator-engine artifacts/
cp build/utils/generate-random-id artifacts/
cp build/utils/json2tlo artifacts/
cp build/adnl/adnl-proxy artifacts/
rsync -r crypto/smartcont artifacts/
rsync -r crypto/fift/lib artifacts/
ls -laRt artifacts
- name: Upload artifacts
uses: actions/upload-artifact@master
with:
name: ton-macos-12.6
path: artifacts

56
.github/workflows/tonlib-android-jni.yml vendored Normal file
View file

@ -0,0 +1,56 @@
name: Tonlib Android JNI
on: [push,workflow_dispatch,workflow_call]
jobs:
build:
runs-on: ubuntu-22.04
steps:
- name: Check out repository
uses: actions/checkout@v3
with:
submodules: 'recursive'
- name: Install libraries
run: |
sudo apt update
sudo apt install -y build-essential git make cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev ninja-build
- name: Configure & Build
run: |
wget https://dl.google.com/android/repository/android-ndk-r25b-linux.zip
unzip android-ndk-r25b-linux.zip
export JAVA_AWT_LIBRARY=NotNeeded
export JAVA_JVM_LIBRARY=NotNeeded
export JAVA_INCLUDE_PATH=${JAVA_HOME}/include
export JAVA_AWT_INCLUDE_PATH=${JAVA_HOME}/include
export JAVA_INCLUDE_PATH2=${JAVA_HOME}/include/linux
export ANDROID_NDK_ROOT=$(pwd)/android-ndk-r25b
export OPENSSL_DIR=$(pwd)/example/android/third_party/crypto
rm -rf example/android/src/drinkless/org/ton/TonApi.java
cd example/android/
cmake -GNinja -DTON_ONLY_TONLIB=ON .
ninja prepare_cross_compiling
rm CMakeCache.txt
./build-all.sh
../../android-ndk-r25b/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip libs/x86/libnative-lib.so
../../android-ndk-r25b/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip libs/x86_64/libnative-lib.so
../../android-ndk-r25b/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip libs/armeabi-v7a/libnative-lib.so
../../android-ndk-r25b/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip libs/arm64-v8a/libnative-lib.so
find . -name "*.debug" -type f -delete
- name: Find & copy binaries
run: |
mkdir -p artifacts/tonlib-android-jni
cp example/android/src/drinkless/org/ton/TonApi.java artifacts/tonlib-android-jni/
cp -R example/android/libs/* artifacts/tonlib-android-jni/
- name: Upload artifacts
uses: actions/upload-artifact@master
with:
name: Tonlib JNI libraries for Android
path: artifacts

22
.github/workflows/ubuntu-18.04-compile.yml vendored
View file

@ -1,6 +1,6 @@
name: C/C++ CI Ubuntu 18.04 Compile
name: Ubuntu 18.04 Compile
on: [push,workflow_dispatch]
on: [push,workflow_dispatch,workflow_call]
jobs:
build:
@ -9,7 +9,7 @@ jobs:
steps:
- name: Check out repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
submodules: 'recursive'
@ -18,6 +18,10 @@ jobs:
sudo apt update
sudo apt install -y build-essential git make cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev ninja-build
- name: Show CPU flags
run: |
cat /proc/cpuinfo
- name: Configure & Build
run: |
export CC=$(which clang)
@ -25,16 +29,18 @@ jobs:
export CCACHE_DISABLE=1
mkdir build
cd build
cmake -GNinja -DCMAKE_BUILD_TYPE=Release ..
ninja fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork
cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state
- name: find & copy binaries
- name: Find & copy binaries
run: |
mkdir artifacts
cp build/crypto/fift build/crypto/tlbc build/crypto/func build/crypto/create-state build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli build/tonlib/libtonlibjson.so.0.5 build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy artifacts
cp build/storage/storage-daemon/storage-daemon build/storage/storage-daemon/storage-daemon-cli build/crypto/fift build/crypto/tlbc build/crypto/func build/crypto/create-state build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli build/tonlib/libtonlibjson.so.0.5 build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy artifacts
cp -R crypto/smartcont artifacts/
cp -R crypto/fift/lib artifacts/
- name: Upload artifacts
uses: actions/upload-artifact@master
with:
name: ton-binaries
name: ton-ubuntu-binaries
path: artifacts

8
.github/workflows/ubuntu-18.04-ton-ccpcheck.yml vendored
View file

@ -1,8 +1,6 @@
name: Ubuntu 18.04 TON ccpcheck
name: TON ccpcheck
on:
workflow_dispatch:
push:
on: [push,workflow_dispatch,workflow_call]
jobs:
build:
@ -11,7 +9,7 @@ jobs:
steps:
- name: Check out repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
submodules: 'recursive'

50
.github/workflows/ubuntu-18.04-tonlib-java.yml vendored
View file

@ -1,50 +0,0 @@
name: Ubuntu 18.04 tonlib-java
on:
workflow_dispatch:
push:
branches:
- 'wallets'
jobs:
build:
runs-on: ubuntu-18.04
steps:
- name: Check out repository
uses: actions/checkout@v2
with:
submodules: 'recursive'
- name: Install libraries
run: |
sudo apt update
sudo apt install -y build-essential git make cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev ninja-build
- name: Configure & Build
run: |
export JAVA_AWT_LIBRARY=NotNeeded
export JAVA_JVM_LIBRARY=NotNeeded
export JAVA_INCLUDE_PATH=${JAVA_HOME}/include
export JAVA_AWT_INCLUDE_PATH=${JAVA_HOME}/include
export JAVA_INCLUDE_PATH2=${JAVA_HOME}/include/linux
cd example/android/
mkdir build
cd build
cmake -DTON_ONLY_TONLIB=ON ..
cmake --build . --target prepare_cross_compiling
cmake --build . --target native-lib
- name: find & copy binaries
run: |
mkdir -p artifacts/tonlib-java
cp example/android/src/drinkless/org/ton/TonApi.java artifacts/tonlib-java/
cp example/android/build/libnative-lib.so artifacts/tonlib-java/
- name: Upload artifacts
uses: actions/upload-artifact@master
with:
name: tonlib-ubuntu-java
path: artifacts

22
.github/workflows/ubuntu-compile.yml vendored
View file

@ -1,6 +1,6 @@
name: Ubuntu Compile
name: Ubuntu Compile x86-64
on: [push,workflow_dispatch]
on: [push,workflow_dispatch,workflow_call]
jobs:
build:
@ -12,7 +12,7 @@ jobs:
steps:
- name: Check out repository
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
submodules: 'recursive'
@ -20,6 +20,11 @@ jobs:
run: |
sudo apt update
sudo apt install -y build-essential git make cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev ninja-build
- name: Show CPU flags
run: |
cat /proc/cpuinfo
- name: Configure & Build
run: |
export CC=$(which clang)
@ -27,12 +32,15 @@ jobs:
export CCACHE_DISABLE=1
mkdir build-${{ matrix.os }}
cd build-${{ matrix.os }}
cmake -GNinja -DCMAKE_BUILD_TYPE=Release ..
ninja fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork
- name: find & copy binaries
cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork
- name: Find & copy binaries
run: |
mkdir artifacts-${{ matrix.os }}
cp build-${{ matrix.os }}/crypto/fift build-${{ matrix.os }}/crypto/tlbc build-${{ matrix.os }}/crypto/func build-${{ matrix.os }}/crypto/create-state build-${{ matrix.os }}/validator-engine-console/validator-engine-console build-${{ matrix.os }}/tonlib/tonlib-cli build-${{ matrix.os }}/tonlib/libtonlibjson.so.0.5 build-${{ matrix.os }}/http/http-proxy build-${{ matrix.os }}/rldp-http-proxy/rldp-http-proxy build-${{ matrix.os }}/dht-server/dht-server build-${{ matrix.os }}/lite-client/lite-client build-${{ matrix.os }}/validator-engine/validator-engine build-${{ matrix.os }}/utils/generate-random-id build-${{ matrix.os }}/utils/json2tlo build-${{ matrix.os }}/adnl/adnl-proxy artifacts-${{ matrix.os }}
cp build-${{ matrix.os }}/storage/storage-daemon/storage-daemon build-${{ matrix.os }}/storage/storage-daemon/storage-daemon-cli build-${{ matrix.os }}/crypto/fift build-${{ matrix.os }}/crypto/tlbc build-${{ matrix.os }}/crypto/func build-${{ matrix.os }}/crypto/create-state build-${{ matrix.os }}/validator-engine-console/validator-engine-console build-${{ matrix.os }}/tonlib/tonlib-cli build-${{ matrix.os }}/tonlib/libtonlibjson.so.0.5 build-${{ matrix.os }}/http/http-proxy build-${{ matrix.os }}/rldp-http-proxy/rldp-http-proxy build-${{ matrix.os }}/dht-server/dht-server build-${{ matrix.os }}/lite-client/lite-client build-${{ matrix.os }}/validator-engine/validator-engine build-${{ matrix.os }}/utils/generate-random-id build-${{ matrix.os }}/utils/json2tlo build-${{ matrix.os }}/adnl/adnl-proxy artifacts-${{ matrix.os }}
cp -R crypto/smartcont artifacts-${{ matrix.os }}
cp -R crypto/fift/lib artifacts-${{ matrix.os }}
- name: Upload artifacts
uses: actions/upload-artifact@master
with:

88
.github/workflows/win-2019-compile.yml vendored Normal file
View file

@ -0,0 +1,88 @@
name: Windows Server 2019 x64 Compile
on: [push,workflow_dispatch,workflow_call]
defaults:
run:
shell: cmd
jobs:
build:
runs-on: windows-2019
steps:
- name: Get Current OS version
run: |
systeminfo | findstr /B /C:"OS Name" /C:"OS Version"
- name: Check out current repository
uses: actions/checkout@v3
with:
submodules: 'recursive'
- name: Check out zlib repository
uses: actions/checkout@v3
with:
repository: desktop-app/zlib
path: zlib
- name: Setup msbuild.exe
uses: microsoft/setup-msbuild@v1.0.2
- name: Compile zlib Win64
run: |
cd zlib\contrib\vstudio\vc14
msbuild zlibstat.vcxproj /p:Configuration=ReleaseWithoutAsm /p:platform=x64 -p:PlatformToolset=v142
- name: Install pre-compiled OpenSSL Win64
run: |
curl -Lo openssl-1.1.1o.zip https://github.com/neodiX42/precompiled-openssl-win64/raw/main/openssl-1.1.1o.zip
jar xf openssl-1.1.1o.zip
- name: Install pre-compiled libmicrohttpd Win64
run: |
curl -Lo libmicrohttpd-latest-w32-bin.zip https://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-latest-w32-bin.zip
unzip libmicrohttpd-latest-w32-bin.zip
- name: Install pre-compiled Readline Win64
run: |
curl -Lo readline-5.0-1-lib.zip https://github.com/neodiX42/precompiled-openssl-win64/raw/main/readline-5.0-1-lib.zip
unzip readline-5.0-1-lib.zip
- name: Compile
run: |
set root=%cd%
echo %root%
mkdir build
cd build
cmake -DREADLINE_INCLUDE_DIR=%root%\readline-5.0-1-lib\include\readline -DREADLINE_LIBRARY=%root%\readline-5.0-1-lib\lib\readline.lib -DZLIB_FOUND=1 -DMHD_FOUND=1 -DMHD_LIBRARY=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static\libmicrohttpd.lib -DMHD_INCLUDE_DIR=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static -DZLIB_INCLUDE_DIR=%root%\zlib -DZLIB_LIBRARY=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=%root%/openssl-1.1/x64/include -DOPENSSL_CRYPTO_LIBRARY=%root%/openssl-1.1/x64/lib/libcrypto.lib -DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj /W0" ..
cmake --build . --target storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork --config Release
- name: Show executables
run: |
cd build
del Release\test-*
dir *.exe /a-D /S /B
dir *.dll /a-D /S /B
- name: Check if validator-engine.exe exists
run: |
set root=%cd%
copy %root%\build\validator-engine\Release\validator-engine.exe test
- name: Find & copy binaries
run: |
mkdir artifacts
mkdir artifacts\smartcont
mkdir artifacts\lib
for %%I in (build\storage\storage-daemon\Release\storage-daemon.exe build\storage\storage-daemon\Release\storage-daemon-cli.exe build\crypto\Release\fift.exe build\crypto\Release\tlbc.exe build\crypto\Release\func.exe build\crypto\Release\create-state.exe build\validator-engine-console\Release\validator-engine-console.exe build\tonlib\Release\tonlib-cli.exe build\tonlib\Release\tonlibjson.dll build\http\Release\http-proxy.exe build\rldp-http-proxy\Release\rldp-http-proxy.exe build\dht-server\Release\dht-server.exe build\lite-client\Release\lite-client.exe build\validator-engine\Release\validator-engine.exe build\utils\Release\generate-random-id.exe build\utils\Release\json2tlo.exe build\adnl\Release\adnl-proxy.exe) do copy %%I artifacts\
xcopy /e /k /h /i crypto\smartcont artifacts\smartcont
xcopy /e /k /h /i crypto\fift\lib artifacts\lib
- name: Upload artifacts
uses: actions/upload-artifact@master
with:
name: ton-win-binaries
path: artifacts

80
.github/workflows/windows2019x64-compile.yml vendored
View file

@ -1,80 +0,0 @@
name: Windows Server 2019 x64 Compile
on:
workflow_dispatch:
push:
defaults:
run:
shell: cmd
jobs:
build:
runs-on: windows-2019
steps:
- name: Get Current OS version
run: |
systeminfo | findstr /B /C:"OS Name" /C:"OS Version"
- name: Check out current repository
uses: actions/checkout@v2
with:
submodules: 'recursive'
- name: Check out zlib repository
uses: actions/checkout@v2
with:
repository: desktop-app/zlib
path: zlib
- name: Setup msbuild.exe
uses: microsoft/setup-msbuild@v1.0.2
- name: Compile zlib Win64
run: |
cd zlib\contrib\vstudio\vc14
msbuild zlibstat.vcxproj /p:Configuration=ReleaseWithoutAsm /p:platform=x64 -p:PlatformToolset=v142
- name: Install pre-compiled OpenSSL Win64
run: |
curl -Lo openssl-1.1.1o.zip https://github.com/neodiX42/precompiled-openssl-win64/raw/main/openssl-1.1.1o.zip
jar xf openssl-1.1.1o.zip
- name: Configure
run: |
set root=%cd%
echo %root%
mkdir build
cd build
cmake -DZLIB_FOUND=1 -DZLIB_INCLUDE_DIR=%root%\zlib -DZLIB_LIBRARY=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=%root%/openssl-1.1/x64/include -DOPENSSL_CRYPTO_LIBRARY=%root%/openssl-1.1/x64/lib/libcrypto.lib -DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj /W0" ..
- name: Build
run: |
cd build
cmake --build . --target fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork --config Release
- name: Show executables
run: |
cd build
del Release\test-*
dir *.exe /a-D /S /B
dir *.dll /a-D /S /B
- name: Check if validator-engine.exe exists
run: |
set root=%cd%
copy %root%\build\validator-engine\Release\validator-engine.exe test
- name: Find & copy binaries
run: |
mkdir artifacts
for /f %%a in ('dir *.exe /b /a /s') do copy /Y %%a artifacts
copy build\tonlib\Release\tonlibjson.dll artifacts
- name: Upload artifacts
uses: actions/upload-artifact@master
with:
name: ton-win64-binaries
path: artifacts

71
.github/workflows/windows2019x64-tonlib-java.yml vendored
View file

@ -1,71 +0,0 @@
name: Windows 2019 tonlib-java
on:
workflow_dispatch:
push:
branches:
- 'wallets'
defaults:
run:
shell: cmd
jobs:
build:
runs-on: windows-2019
steps:
- name: Get Current OS version
run: |
systeminfo | findstr /B /C:"OS Name" /C:"OS Version"
- name: Check out current repository
uses: actions/checkout@v2
with:
submodules: 'recursive'
- name: Check out zlib repository
uses: actions/checkout@v2
with:
repository: desktop-app/zlib
path: zlib
- name: Setup msbuild.exe
uses: microsoft/setup-msbuild@v1.0.2
- name: Compile zlib Win64
run: |
cd zlib\contrib\vstudio\vc14
msbuild zlibstat.vcxproj /p:Configuration=ReleaseWithoutAsm /p:platform=x64 -p:PlatformToolset=v142
- name: Install precompiled OpenSSL Win64
run: |
curl -Lo openssl-1.1.1o.zip https://github.com/neodiX42/precompiled-openssl-win64/raw/main/openssl-1.1.1o.zip
jar xf openssl-1.1.1o.zip
- name: Configure & Build
run: |
set JAVA_AWT_LIBRARY=NotNeeded
set JAVA_JVM_LIBRARY=NotNeeded
set JAVA_INCLUDE_PATH=${JAVA_HOME}/include
set JAVA_AWT_INCLUDE_PATH=${JAVA_HOME}/include
set JAVA_INCLUDE_PATH2=${JAVA_HOME}/include/win32
set root=%cd%
echo %root%
cd example/android
mkdir build
cd build
cmake -DZLIB_FOUND=1 -DZLIB_INCLUDE_DIR=%root%\zlib -DZLIB_LIBRARY=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=%root%/openssl-1.1/x64/include -DOPENSSL_CRYPTO_LIBRARY=%root%/openssl-1.1/x64/lib/libcrypto.lib -DTON_ONLY_TONLIB=ON -DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj /W0" ..
cmake --build . --target native-lib --config Release
- name: Find & copy binaries
run: |
mkdir tonlib-java
cp example/android/build/Release/native-lib.dll tonlib-java/
- name: Upload artifacts
uses: actions/upload-artifact@master
with:
name: tonlib-win64-java
path: tonlib-java

3
CMakeLists.txt
View file

@ -290,8 +290,9 @@ if (MSVC)
endif()
if (NOT MSVC)
add_cxx_compiler_flag("-Wall")
add_cxx_compiler_flag("-Wextra")
endif()
add_cxx_compiler_flag("-Wextra")
add_cxx_compiler_flag("-Wimplicit-fallthrough=2")
add_cxx_compiler_flag("-Wpointer-arith")
add_cxx_compiler_flag("-Wcast-qual")

15
Changelog.md
View file

@ -38,3 +38,18 @@ Besides the work of the core team, this update is based on the efforts of @aweso
* Added build of FunC and Fift to WASM
Besides the work of the core team, this update is based on the efforts of @tvorogme (debug improvements), @AlexeyFSL (WASM builds) and third-party security auditors.
## 12.2022 Update
Node update:
1. Improvements of ton-proxy: fixed few bugs, improved stability
2. Improved collator/validator checks, added optimization of storage stat calculation, generation and validation of new blocks is made safer
3. Some previously hard-coded parameters such as split/merge timings, max sizes and depths of internal and external messages, and others now can be updated by validators through setting ConfigParams. Max contract size added to configs.
4. Tonlib: updated raw.getTransactions (now it contains InitState), fixed long bytestrings truncation
5. abseil-cpp is updated to newer versions
6. Added configs for Token Bridge
7. LiteServers: a few bug fixes, added liteServer.getAccountStatePrunned method, improved work with not yet applied blocks.
8. Improved DHT: works for some NAT configurations, optimized excessive requests, added option for DHT network segregation.
9. FunC v0.4.0: added try/catch statements, added throw_arg functions, allowed in-place modification of global variables, forbidden ambiguous modification of local variables after it's usage in the same expression.
10. TON Storage: added storage-daemon (create, download bag of Files, storage-provider staff), added storage-daemon-cli
Besides the work of the core team, this update is based on the efforts of @vtamara (help with abseil-cpp upgrade), @krigga(in-place modification of global variables) and third-party security auditors.

10
README.md
View file

@ -1,3 +1,11 @@
<div align="center">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="https://ton.org/download/ton_logo_dark_background.svg">
<img alt="TON logo" src="https://ton.org/download/ton_logo_light_background.svg">
</picture>
<hr/>
</div>
[![TON Overflow Group][ton-overflow-badge]][ton-overflow-url]
[![Stack Overflow Group][stack-overflow-badge]][stack-overflow-url]
[![Telegram Foundation Group][telegram-foundation-badge]][telegram-foundation-url]
@ -17,8 +25,6 @@
# TON
Main TON monorepo, which includes the code of the node/validator, lite-client, tonlib, FunC compiler, etc.
## Updates flow:

13
adnl/adnl-address-list.cpp
View file

@ -185,7 +185,8 @@ td::Ref<AdnlAddressImpl> AdnlAddressImpl::create(const tl_object_ptr<ton_api::ad
*const_cast<ton_api::adnl_Address *>(addr.get()),
td::overloaded([&](const ton_api::adnl_address_udp &obj) { res = td::make_ref<AdnlAddressUdp>(obj); },
[&](const ton_api::adnl_address_udp6 &obj) { res = td::make_ref<AdnlAddressUdp6>(obj); },
[&](const ton_api::adnl_address_tunnel &obj) { res = td::make_ref<AdnlAddressTunnel>(obj); }));
[&](const ton_api::adnl_address_tunnel &obj) { res = td::make_ref<AdnlAddressTunnel>(obj); },
[&](const ton_api::adnl_address_reverse &obj) { res = td::make_ref<AdnlAddressReverse>(); }));
return res;
}
@ -202,7 +203,12 @@ AdnlAddressList::AdnlAddressList(const tl_object_ptr<ton_api::adnl_addressList>
version_ = static_cast<td::uint32>(addrs->version_);
std::vector<td::Ref<AdnlAddressImpl>> vec;
for (auto &addr : addrs->addrs_) {
vec.push_back(AdnlAddressImpl::create(addr));
auto obj = AdnlAddressImpl::create(addr);
if (obj->is_reverse()) {
has_reverse_ = true;
} else {
vec.push_back(std::move(obj));
}
}
addrs_ = std::move(vec);
reinit_date_ = addrs->reinit_date_;
@ -215,6 +221,9 @@ tl_object_ptr<ton_api::adnl_addressList> AdnlAddressList::tl() const {
for (auto &v : addrs_) {
addrs.emplace_back(v->tl());
}
if (has_reverse_) {
addrs.push_back(create_tl_object<ton_api::adnl_address_reverse>());
}
return create_tl_object<ton_api::adnl_addressList>(std::move(addrs), version_, reinit_date_, priority_, expire_at_);
}

11
adnl/adnl-address-list.h
View file

@ -39,6 +39,9 @@ class AdnlAddressImpl : public td::CntObject {
virtual td::actor::ActorOwn<AdnlNetworkConnection> create_connection(
td::actor::ActorId<AdnlNetworkManager> network_manager, td::actor::ActorId<Adnl> adnl,
std::unique_ptr<AdnlNetworkConnection::Callback> callback) const = 0;
virtual bool is_reverse() const {
return false;
}
static td::Ref<AdnlAddressImpl> create(const tl_object_ptr<ton_api::adnl_Address> &addr);
};
@ -54,6 +57,7 @@ class AdnlAddressList {
td::int32 priority_;
td::int32 expire_at_;
std::vector<AdnlAddress> addrs_;
bool has_reverse_{false};
public:
static constexpr td::uint32 max_serialized_size() {
@ -102,6 +106,13 @@ class AdnlAddressList {
static td::Result<AdnlAddressList> create(const tl_object_ptr<ton_api::adnl_addressList> &addr_list);
td::Status add_udp_address(td::IPAddress addr);
void set_reverse(bool x = true) {
has_reverse_ = x;
}
bool has_reverse() const {
return has_reverse_;
}
};
} // namespace adnl

25
adnl/adnl-address-list.hpp
View file

@ -116,6 +116,31 @@ class AdnlAddressTunnel : public AdnlAddressImpl {
std::unique_ptr<AdnlNetworkConnection::Callback> callback) const override;
};
class AdnlAddressReverse : public AdnlAddressImpl {
public:
AdnlAddressReverse *make_copy() const override {
return new AdnlAddressReverse();
}
bool is_public() const override {
return true;
}
td::uint32 serialized_size() const override {
return 4;
}
tl_object_ptr<ton_api::adnl_Address> tl() const override {
return create_tl_object<ton_api::adnl_address_reverse>();
}
td::actor::ActorOwn<AdnlNetworkConnection> create_connection(
td::actor::ActorId<AdnlNetworkManager> network_manager, td::actor::ActorId<Adnl> adnl,
std::unique_ptr<AdnlNetworkConnection::Callback> callback) const override {
LOG(ERROR) << "Cannot create connection for AdnlAddressReverse";
return {};
}
bool is_reverse() const override {
return true;
}
};
} // namespace adnl
} // namespace ton

3
adnl/adnl-ext-client.hpp
View file

@ -80,6 +80,9 @@ class AdnlExtClientImpl : public AdnlExtClient {
if (!conn_.empty() && conn_.get() == conn) {
callback_->on_stop_ready();
conn_ = {};
for (auto& q : out_queries_) {
td::actor::send_closure(q.second, &AdnlQuery::set_error, td::Status::Error(ErrorCode::cancelled));
}
alarm_timestamp() = next_create_at_;
try_stop();
}

13
adnl/adnl-local-id.cpp
View file

@ -121,7 +121,7 @@ void AdnlLocalId::update_address_list(AdnlAddressList addr_list) {
}
void AdnlLocalId::publish_address_list() {
if (dht_node_.empty() || addr_list_.empty() || addr_list_.size() == 0) {
if (dht_node_.empty() || addr_list_.empty() || (addr_list_.size() == 0 && !addr_list_.has_reverse())) {
VLOG(ADNL_NOTICE) << this << ": skipping public addr list, because localid (or dht node) not fully initialized";
return;
}
@ -175,6 +175,17 @@ void AdnlLocalId::publish_address_list() {
td::actor::send_closure(keyring_, &keyring::Keyring::sign_message, short_id_.pubkey_hash(), std::move(B),
std::move(P));
if (addr_list_.has_reverse()) {
td::actor::send_closure(
dht_node_, &dht::Dht::register_reverse_connection, id_, [print_id = print_id()](td::Result<td::Unit> R) {
if (R.is_error()) {
VLOG(ADNL_NOTICE) << print_id << ": failed to register reverse connection in DHT: " << R.move_as_error();
} else {
VLOG(ADNL_INFO) << print_id << ": registered reverse connection";
}
});
}
}
AdnlLocalId::AdnlLocalId(AdnlNodeIdFull id, AdnlAddressList addr_list, td::uint32 mode,

59
adnl/adnl-peer.cpp
View file

@ -113,6 +113,7 @@ void AdnlPeerPairImpl::discover() {
}
void AdnlPeerPairImpl::receive_packet_checked(AdnlPacket packet) {
request_reverse_ping_after_ = td::Timestamp::in(15.0);
auto d = Adnl::adnl_start_time();
if (packet.dst_reinit_date() > d) {
VLOG(ADNL_WARNING) << this << ": dropping IN message: too new our reinit date " << packet.dst_reinit_date();
@ -653,10 +654,15 @@ td::Result<std::pair<td::actor::ActorId<AdnlNetworkConnection>, bool>> AdnlPeerP
}
if (conns_.size() == 0 && priority_conns_.size() == 0) {
return td::Status::Error(ErrorCode::notready, PSTRING()
<< "empty network information: version=" << addr_list_.version()
<< " reinit_date=" << addr_list_.reinit_date()
<< " real_reinit_date=" << reinit_date_);
if (has_reverse_addr_) {
request_reverse_ping();
return td::Status::Error(ErrorCode::notready, "waiting for reverse ping");
} else {
return td::Status::Error(ErrorCode::notready, PSTRING()
<< "empty network information: version=" << addr_list_.version()
<< " reinit_date=" << addr_list_.reinit_date()
<< " real_reinit_date=" << reinit_date_);
}
}
for (auto &conn : priority_conns_) {
@ -704,11 +710,18 @@ void AdnlPeerPairImpl::update_addr_list(AdnlAddressList addr_list) {
VLOG(ADNL_INFO) << this << ": updating addr list to version " << addr_list.version() << " size=" << addr_list.size();
const auto addrs = addr_list.addrs();
has_reverse_addr_ = addr_list.has_reverse();
if (has_reverse_addr_ && addrs.empty()) {
return;
}
std::vector<Conn> conns;
auto &old_conns = priority ? priority_conns_ : conns_;
size_t idx = 0;
for (const auto &addr : addrs) {
if (addr->is_reverse()) {
continue;
}
if ((mode_ & static_cast<td::uint32>(AdnlLocalIdMode::direct_only)) && !addr->is_public()) {
continue;
}
@ -730,7 +743,7 @@ void AdnlPeerPairImpl::get_conn_ip_str(td::Promise<td::string> promise) {
promise.set_value("undefined");
return;
}
for (auto &conn : priority_conns_) {
if (conn.ready()) {
td::actor::send_closure(conn.conn, &AdnlNetworkConnection::get_ip_str, std::move(promise));
@ -743,7 +756,7 @@ void AdnlPeerPairImpl::get_conn_ip_str(td::Promise<td::string> promise) {
return;
}
}
promise.set_value("undefined");
}
@ -811,7 +824,7 @@ void AdnlPeerImpl::receive_packet(AdnlNodeIdShort dst, td::uint32 dst_mode, td::
}
}
td::actor::send_closure(it->second.get(), &AdnlPeerPair::receive_packet_checked, std::move(packet));
td::actor::send_closure(it->second.get(), &AdnlPeerPair::receive_packet, std::move(packet));
}
void AdnlPeerImpl::send_messages(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId<AdnlLocalId> src_actor,
@ -868,7 +881,7 @@ void AdnlPeerImpl::get_conn_ip_str(AdnlNodeIdShort l_id, td::Promise<td::string>
if (it == peer_pairs_.end()) {
promise.set_value("undefined");
return;
}
}
td::actor::send_closure(it->second, &AdnlPeerPair::get_conn_ip_str, std::move(promise));
}
@ -944,6 +957,36 @@ void AdnlPeerPairImpl::update_peer_id(AdnlNodeIdFull id) {
CHECK(!peer_id_.empty());
}
void AdnlPeerPairImpl::request_reverse_ping() {
if (request_reverse_ping_active_ || !request_reverse_ping_after_.is_in_past()) {
return;
}
VLOG(ADNL_INFO) << this << ": requesting reverse ping";
request_reverse_ping_after_ = td::Timestamp::in(15.0);
request_reverse_ping_active_ = true;
td::actor::send_closure(
local_actor_, &AdnlLocalId::get_self_node,
[SelfId = actor_id(this), peer = peer_id_short_, dht = dht_node_](td::Result<AdnlNode> R) {
if (R.is_error()) {
td::actor::send_closure(SelfId, &AdnlPeerPairImpl::request_reverse_ping_result, R.move_as_error());
return;
}
td::actor::send_closure(
dht, &dht::Dht::request_reverse_ping, R.move_as_ok(), peer, [SelfId](td::Result<td::Unit> R) {
td::actor::send_closure(SelfId, &AdnlPeerPairImpl::request_reverse_ping_result, std::move(R));
});
});
}
void AdnlPeerPairImpl::request_reverse_ping_result(td::Result<td::Unit> R) {
request_reverse_ping_active_ = false;
if (R.is_ok()) {
VLOG(ADNL_INFO) << this << ": reverse ping requested";
} else {
VLOG(ADNL_INFO) << this << ": failed to request reverse ping: " << R.move_as_error();
}
}
} // namespace adnl
} // namespace ton

7
adnl/adnl-peer.hpp
View file

@ -153,6 +153,9 @@ class AdnlPeerPairImpl : public AdnlPeerPair {
}
}
void request_reverse_ping();
void request_reverse_ping_result(td::Result<td::Unit> R);
struct Conn {
class ConnCallback : public AdnlNetworkConnection::Callback {
public:
@ -250,6 +253,10 @@ class AdnlPeerPairImpl : public AdnlPeerPair {
td::Timestamp next_dht_query_at_ = td::Timestamp::never();
td::Timestamp next_db_update_at_ = td::Timestamp::never();
td::Timestamp retry_send_at_ = td::Timestamp::never();
bool has_reverse_addr_ = false;
td::Timestamp request_reverse_ping_after_ = td::Timestamp::now();
bool request_reverse_ping_active_ = false;
};
class AdnlPeerImpl : public AdnlPeer {

7
adnl/adnl-query.cpp
View file

@ -25,13 +25,16 @@ namespace ton {
namespace adnl {
void AdnlQuery::alarm() {
promise_.set_error(td::Status::Error(ErrorCode::timeout, "adnl query timeout"));
stop();
set_error(td::Status::Error(ErrorCode::timeout, "adnl query timeout"));
}
void AdnlQuery::result(td::BufferSlice data) {
promise_.set_value(std::move(data));
stop();
}
void AdnlQuery::set_error(td::Status error) {
promise_.set_error(std::move(error));
stop();
}
AdnlQueryId AdnlQuery::random_query_id() {
AdnlQueryId q_id;

1
adnl/adnl-query.h
View file

@ -48,6 +48,7 @@ class AdnlQuery : public td::actor::Actor {
}
void alarm() override;
void result(td::BufferSlice data);
void set_error(td::Status error);
void start_up() override {
alarm_timestamp() = timeout_;
}

7
crypto/CMakeLists.txt
View file

@ -275,7 +275,12 @@ target_include_directories(ton_crypto PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_S
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>)
target_link_libraries(ton_crypto PUBLIC ${OPENSSL_CRYPTO_LIBRARY} tdutils tddb_utils)
if (NOT WIN32)
target_link_libraries(ton_crypto PUBLIC dl z)
find_library(DL dl)
if (DL)
target_link_libraries(ton_crypto PUBLIC dl z)
else()
target_link_libraries(ton_crypto PUBLIC z)
endif()
endif()
target_include_directories(ton_crypto SYSTEM PUBLIC $<BUILD_INTERFACE:${OPENSSL_INCLUDE_DIR}>)

47
crypto/block/block.tlb
View file

@ -113,7 +113,10 @@ var_uint$_ {n:#} len:(#< n) value:(uint (len * 8))
= VarUInteger n;
var_int$_ {n:#} len:(#< n) value:(int (len * 8))
= VarInteger n;
nanograms$_ amount:(VarUInteger 16) = Grams;
nanograms$_ amount:(VarUInteger 16) = Grams;
_ grams:Grams = Coins;
//
extra_currencies$_ dict:(HashmapE 32 (VarUInteger 32))
= ExtraCurrencyCollection;
@ -611,15 +614,29 @@ wfmt_ext#0 min_addr_len:(## 12) max_addr_len:(## 12) addr_len_step:(## 12)
workchain_type_id:(## 32) { workchain_type_id >= 1 }
= WorkchainFormat 0;
workchain#a6 enabled_since:uint32 actual_min_split:(## 8)
min_split:(## 8) max_split:(## 8) { actual_min_split <= min_split }
wc_split_merge_timings#0
split_merge_delay:uint32 split_merge_interval:uint32
min_split_merge_interval:uint32 max_split_merge_delay:uint32
= WcSplitMergeTimings;
//workchain#a5 enabled_since:uint32 min_split:(## 8) max_split:(## 8)
// { min_split <= max_split } { max_split <= 60 }
workchain#a6 enabled_since:uint32 actual_min_split:(## 8)
min_split:(## 8) max_split:(## 8) { actual_min_split <= min_split }
basic:(## 1) active:Bool accept_msgs:Bool flags:(## 13) { flags = 0 }
zerostate_root_hash:bits256 zerostate_file_hash:bits256
version:uint32 format:(WorkchainFormat basic)
= WorkchainDescr;
workchain_v2#a7 enabled_since:uint32 actual_min_split:(## 8)
min_split:(## 8) max_split:(## 8) { actual_min_split <= min_split }
basic:(## 1) active:Bool accept_msgs:Bool flags:(## 13) { flags = 0 }
zerostate_root_hash:bits256 zerostate_file_hash:bits256
version:uint32 format:(WorkchainFormat basic)
split_merge_timings:WcSplitMergeTimings
= WorkchainDescr;
_ workchains:(HashmapE 32 WorkchainDescr) = ConfigParam 12;
complaint_prices#1a deposit:Grams bit_price:Grams cell_price:Grams = ComplaintPricing;
@ -734,11 +751,32 @@ misbehaviour_punishment_config_v1#01
= MisbehaviourPunishmentConfig;
_ MisbehaviourPunishmentConfig = ConfigParam 40;
size_limits_config#01 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16
max_ext_msg_size:uint32 max_ext_msg_depth:uint16 = SizeLimitsConfig;
size_limits_config_v2#02 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16
max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_acc_state_bits:uint32 = SizeLimitsConfig;
_ SizeLimitsConfig = ConfigParam 43;
oracle_bridge_params#_ bridge_address:bits256 oracle_mutlisig_address:bits256 oracles:(HashmapE 256 uint256) external_chain_address:bits256 = OracleBridgeParams;
_ OracleBridgeParams = ConfigParam 71; // Ethereum bridge
_ OracleBridgeParams = ConfigParam 72; // Binance Smart Chain bridge
_ OracleBridgeParams = ConfigParam 73; // Polygon bridge
// Note that chains in which bridge, minter and jetton-wallet operate are fixated
jetton_bridge_prices#_ bridge_burn_fee:Coins bridge_mint_fee:Coins
wallet_min_tons_for_storage:Coins
wallet_gas_consumption:Coins
minter_min_tons_for_storage:Coins
discover_gas_consumption:Coins = JettonBridgePrices;
jetton_bridge_params_v0#00 bridge_address:bits256 oracles_address:bits256 oracles:(HashmapE 256 uint256) state_flags:uint8 burn_bridge_fee:Coins = JettonBridgeParams;
jetton_bridge_params_v1#01 bridge_address:bits256 oracles_address:bits256 oracles:(HashmapE 256 uint256) state_flags:uint8 prices:^JettonBridgePrices external_chain_address:bits256 = JettonBridgeParams;
_ JettonBridgeParams = ConfigParam 79; // ETH->TON token bridge
_ JettonBridgeParams = ConfigParam 80; // BNB->TON token bridge
_ JettonBridgeParams = ConfigParam 81; // Polygon->TON token bridge
//
// PROOFS
//
@ -840,6 +878,8 @@ cap_method_pubkey#71f4 = SmcCapability;
cap_is_wallet#2177 = SmcCapability;
cap_name#ff name:Text = SmcCapability;
dns_storage_address#7473 bag_id:bits256 = DNSRecord;
//
// PAYMENT CHANNELS
//
@ -865,4 +905,3 @@ chan_op_cmd#912838d1 msg:ChanSignedMsg = ChanOp;
chan_data$_ config:^ChanConfig state:^ChanState = ChanData;

105
crypto/block/mc-config.cpp
View file

@ -1913,6 +1913,38 @@ std::vector<ton::ValidatorDescr> Config::compute_total_validator_set(int next) c
return res.move_as_ok()->export_validator_set();
}
td::Result<SizeLimitsConfig> Config::get_size_limits_config() const {
SizeLimitsConfig limits;
td::Ref<vm::Cell> param = get_config_param(43);
if (param.is_null()) {
return limits;
}
auto unpack_v1 = [&](auto& rec) {
limits.max_msg_bits = rec.max_msg_bits;
limits.max_msg_cells = rec.max_msg_cells;
limits.max_library_cells = rec.max_library_cells;
limits.max_vm_data_depth = static_cast<td::uint16>(rec.max_vm_data_depth);
limits.ext_msg_limits.max_size = rec.max_ext_msg_size;
limits.ext_msg_limits.max_depth = static_cast<td::uint16>(rec.max_ext_msg_depth);
};
auto unpack_v2 = [&](auto& rec) {
unpack_v1(rec);
limits.max_acc_state_bits = rec.max_acc_state_bits;
limits.max_acc_state_cells = rec.max_acc_state_cells;
};
gen::SizeLimitsConfig::Record_size_limits_config rec_v1;
gen::SizeLimitsConfig::Record_size_limits_config_v2 rec_v2;
if (tlb::unpack_cell(param, rec_v1)) {
unpack_v1(rec_v1);
} else if (tlb::unpack_cell(param, rec_v2)) {
unpack_v2(rec_v2);
} else {
return td::Status::Error("configuration parameter 43 is invalid");
}
return limits;
}
td::Result<std::pair<ton::UnixTime, ton::UnixTime>> Config::unpack_validator_set_start_stop(Ref<vm::Cell> vset_root) {
if (vset_root.is_null()) {
return td::Status::Error("validator set absent");
@ -1942,31 +1974,58 @@ bool WorkchainInfo::unpack(ton::WorkchainId wc, vm::CellSlice& cs) {
if (wc == ton::workchainInvalid) {
return false;
}
block::gen::WorkchainDescr::Record info;
if (!tlb::unpack(cs, info)) {
return false;
}
enabled_since = info.enabled_since;
actual_min_split = info.actual_min_split;
min_split = info.min_split;
max_split = info.max_split;
basic = info.basic;
active = info.active;
accept_msgs = info.accept_msgs;
flags = info.flags;
zerostate_root_hash = info.zerostate_root_hash;
zerostate_file_hash = info.zerostate_file_hash;
version = info.version;
if (basic) {
min_addr_len = max_addr_len = addr_len_step = 256;
} else {
block::gen::WorkchainFormat::Record_wfmt_ext ext;
if (!tlb::type_unpack(cs, block::gen::WorkchainFormat{basic}, ext)) {
auto unpack_v1 = [this](auto& info) {
enabled_since = info.enabled_since;
actual_min_split = info.actual_min_split;
min_split = info.min_split;
max_split = info.max_split;
basic = info.basic;
active = info.active;
accept_msgs = info.accept_msgs;
flags = info.flags;
zerostate_root_hash = info.zerostate_root_hash;
zerostate_file_hash = info.zerostate_file_hash;
version = info.version;
if (basic) {
min_addr_len = max_addr_len = addr_len_step = 256;
} else {
block::gen::WorkchainFormat::Record_wfmt_ext ext;
if (!tlb::csr_type_unpack(info.format, block::gen::WorkchainFormat{basic}, ext)) {
return false;
}
min_addr_len = ext.min_addr_len;
max_addr_len = ext.max_addr_len;
addr_len_step = ext.addr_len_step;
}
return true;
};
auto unpack_v2 = [&, this](auto& info) {
if (!unpack_v1(info)) {
return false;
}
min_addr_len = ext.min_addr_len;
max_addr_len = ext.max_addr_len;
addr_len_step = ext.addr_len_step;
block::gen::WcSplitMergeTimings::Record rec;
if (!tlb::csr_unpack(info.split_merge_timings, rec)) {
return false;
}
split_merge_delay = rec.split_merge_delay;
split_merge_interval = rec.split_merge_interval;
min_split_merge_interval = rec.min_split_merge_interval;
max_split_merge_delay = rec.max_split_merge_delay;
return true;
};
block::gen::WorkchainDescr::Record_workchain info_v1;
block::gen::WorkchainDescr::Record_workchain_v2 info_v2;
vm::CellSlice cs0 = cs;
if (tlb::unpack(cs, info_v1)) {
if (!unpack_v1(info_v1)) {
return false;
}
} else if (tlb::unpack(cs = cs0, info_v2)) {
if (!unpack_v2(info_v2)) {
return false;
}
} else {
return false;
}
workchain = wc;
LOG(DEBUG) << "unpacked info for workchain " << wc << ": basic=" << basic << ", active=" << active

23
crypto/block/mc-config.h
View file

@ -376,6 +376,21 @@ struct MsgPrices {
td::RefInt256 get_next_part(td::RefInt256 total) const;
};
struct SizeLimitsConfig {
// Default values are used when not present in global config
struct ExtMsgLimits {
td::uint32 max_size = 65535;
td::uint16 max_depth = 512;
};
td::uint32 max_msg_bits = 1 << 21;
td::uint32 max_msg_cells = 1 << 13;
td::uint32 max_library_cells = 1000;
td::uint16 max_vm_data_depth = 512;
ExtMsgLimits ext_msg_limits;
td::uint32 max_acc_state_cells = 1 << 16;
td::uint32 max_acc_state_bits = (1 << 16) * 1023;
};
struct CatchainValidatorsConfig {
td::uint32 mc_cc_lifetime, shard_cc_lifetime, shard_val_lifetime, shard_val_num;
bool shuffle_mc_val;
@ -402,6 +417,13 @@ struct WorkchainInfo : public td::CntObject {
ton::RootHash zerostate_root_hash;
ton::FileHash zerostate_file_hash;
int min_addr_len, max_addr_len, addr_len_step;
// Default values are used when split_merge_timings is not set in config
unsigned split_merge_delay = 100; // prepare (delay) split/merge for 100 seconds
unsigned split_merge_interval = 100; // split/merge is enabled during 60 second interval
unsigned min_split_merge_interval = 30; // split/merge interval must be at least 30 seconds
unsigned max_split_merge_delay = 1000; // end of split/merge interval must be at most 1000 seconds in the future
bool is_valid() const {
return workchain != ton::workchainInvalid;
}
@ -593,6 +615,7 @@ class Config {
std::vector<ton::ValidatorDescr> compute_validator_set(ton::ShardIdFull shard, ton::UnixTime time,
ton::CatchainSeqno cc_seqno) const;
std::vector<ton::ValidatorDescr> compute_total_validator_set(int next) const;
td::Result<SizeLimitsConfig> get_size_limits_config() const;
static std::vector<ton::ValidatorDescr> do_compute_validator_set(const block::CatchainValidatorsConfig& ccv_conf,
ton::ShardIdFull shard,
const block::ValidatorSet& vset, ton::UnixTime time,

135
crypto/block/transaction.cpp
View file

@ -24,6 +24,7 @@
#include "td/utils/uint128.h"
#include "ton/ton-shard.h"
#include "vm/vm.h"
#include "td/utils/Timer.h"
namespace {
class StringLoggerTail : public td::LogInterface {
@ -345,7 +346,7 @@ bool Account::unpack(Ref<vm::CellSlice> shard_account, Ref<vm::CellSlice> extra,
block::gen::AccountStorage::Record storage;
if (!(tlb::unpack_exact(acc_cs, acc) && (my_addr = acc.addr).not_null() && unpack_address(acc.addr.write()) &&
compute_my_addr() && unpack_storage_info(acc.storage_stat.write()) &&
tlb::csr_unpack(std::move(acc.storage), storage) &&
tlb::csr_unpack(this->storage = std::move(acc.storage), storage) &&
std::max(storage.last_trans_lt, 1ULL) > acc_info.last_trans_lt && balance.unpack(std::move(storage.balance)))) {
return false;
}
@ -459,7 +460,6 @@ bool Account::deactivate() {
return true;
}
bool Account::belongs_to_shard(ton::ShardIdFull shard) const {
return workchain == shard.workchain && ton::shard_is_ancestor(shard.shard, addr);
}
@ -593,7 +593,7 @@ bool Transaction::unpack_input_msg(bool ihr_delivered, const ActionPhaseConfig*
sstat.bits -= cs.size(); // bits in the root cells are free
sstat.cells--; // the root cell itself is not counted as a cell
LOG(DEBUG) << "storage paid for a message: " << sstat.cells << " cells, " << sstat.bits << " bits";
if (sstat.bits > max_msg_bits || sstat.cells > max_msg_cells) {
if (sstat.bits > cfg->size_limits.max_msg_bits || sstat.cells > cfg->size_limits.max_msg_cells) {
LOG(DEBUG) << "inbound external message too large, invalid";
return false;
}
@ -1043,12 +1043,15 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) {
vm_log.log_options = td::LogOptions(VERBOSITY_NAME(DEBUG), true, false);
}
vm::VmState vm{new_code, std::move(stack), gas, 1, new_data, vm_log, compute_vm_libraries(cfg)};
vm.set_max_data_depth(cfg.max_vm_data_depth);
vm.set_c7(prepare_vm_c7(cfg)); // tuple with SmartContractInfo
// vm.incr_stack_trace(1); // enable stack dump after each step
LOG(DEBUG) << "starting VM";
cp.vm_init_state_hash = vm.get_state_hash();
td::Timer timer;
cp.exit_code = ~vm.run();
double elapsed = timer.elapsed();
LOG(DEBUG) << "VM terminated with exit code " << cp.exit_code;
cp.out_of_gas = (cp.exit_code == ~(int)vm::Excno::out_of_gas);
cp.vm_final_state_hash = vm.get_final_state_hash(cp.exit_code);
@ -1064,7 +1067,8 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) {
}
LOG(INFO) << "steps: " << vm.get_steps_count() << " gas: used=" << gas.gas_consumed() << ", max=" << gas.gas_max
<< ", limit=" << gas.gas_limit << ", credit=" << gas.gas_credit;
LOG(INFO) << "out_of_gas=" << cp.out_of_gas << ", accepted=" << cp.accepted << ", success=" << cp.success;
LOG(INFO) << "out_of_gas=" << cp.out_of_gas << ", accepted=" << cp.accepted << ", success=" << cp.success
<< ", time=" << elapsed << "s";
if (logger != nullptr) {
cp.vm_log = logger->get_log();
}
@ -1121,6 +1125,25 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) {
ap.total_action_fees = td::zero_refint();
ap.reserved_balance.set_zero();
td::Ref<vm::Cell> old_code = new_code, old_data = new_data, old_library = new_library;
auto enforce_state_size_limits = [&]() {
if (account.is_special) {
return true;
}
if (!check_state_size_limit(cfg)) {
// Rollback changes to state, fail action phase
LOG(INFO) << "Account state size exceeded limits";
new_storage_stat.clear();
new_code = old_code;
new_data = old_data;
new_library = old_library;
ap.result_code = 50;
ap.state_size_too_big = true;
return false;
}
return true;
};
int n = 0;
while (true) {
ap.action_list.push_back(list);
@ -1196,9 +1219,21 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) {
ap.no_funds = true;
}
LOG(DEBUG) << "invalid action " << ap.result_arg << " in action list: error code " << ap.result_code;
// This is reuqired here because changes to libraries are applied even if actipn phase fails
enforce_state_size_limits();
return true;
}
}
end_lt = ap.end_lt;
if (ap.new_code.not_null()) {
new_code = ap.new_code;
}
new_data = compute_phase->new_data; // tentative persistent data update applied
if (!enforce_state_size_limits()) {
return true;
}
ap.result_arg = 0;
ap.result_code = 0;
CHECK(ap.remaining_balance.grams->sgn() >= 0);
@ -1212,12 +1247,7 @@ bool Transaction::prepare_action_phase(const ActionPhaseConfig& cfg) {
was_deleted = true;
}
ap.success = true;
end_lt = ap.end_lt;
out_msgs = std::move(ap.out_msgs);
if (ap.new_code.not_null()) {
new_code = ap.new_code;
}
new_data = compute_phase->new_data; // tentative persistent data update applied
total_fees +=
ap.total_action_fees; // NB: forwarding fees are not accounted here (they are not collected by the validators in this transaction)
balance = ap.remaining_balance;
@ -1272,6 +1302,11 @@ int Transaction::try_action_change_library(vm::CellSlice& cs, ActionPhase& ap, c
// library code not found
return 41;
}
vm::CellStorageStat sstat;
sstat.compute_used_storage(lib_ref);
if (sstat.cells > cfg.size_limits.max_library_cells) {
return 43;
}
vm::CellBuilder cb;
CHECK(cb.store_bool_bool(rec.mode >> 1) && cb.store_ref_bool(std::move(lib_ref)));
CHECK(dict.set_builder(hash, cb));
@ -1546,7 +1581,7 @@ int Transaction::try_action_send_msg(const vm::CellSlice& cs0, ActionPhase& ap,
sstat.add_used_storage(info.value->prefetch_ref());
}
LOG(DEBUG) << "storage paid for a message: " << sstat.cells << " cells, " << sstat.bits << " bits";
if (sstat.bits > max_msg_bits || sstat.cells > max_msg_cells) {
if (sstat.bits > cfg.size_limits.max_msg_bits || sstat.cells > cfg.size_limits.max_msg_cells) {
LOG(DEBUG) << "message too large, invalid";
return skip_invalid ? 0 : 40;
}
@ -1801,6 +1836,35 @@ int Transaction::try_action_reserve_currency(vm::CellSlice& cs, ActionPhase& ap,
return 0;
}
bool Transaction::check_state_size_limit(const ActionPhaseConfig& cfg) {
auto cell_equal = [](const td::Ref<vm::Cell>& a, const td::Ref<vm::Cell>& b) -> bool {
if (a.is_null()) {
return b.is_null();
}
if (b.is_null()) {
return false;
}
return a->get_hash() == b->get_hash();
};
if (cell_equal(account.code, new_code) && cell_equal(account.data, new_data) &&
cell_equal(account.library, new_library)) {
return true;
}
// new_storage_stat is used here beause these stats will be reused in compute_state()
new_storage_stat.limit_cells = cfg.size_limits.max_acc_state_cells;
new_storage_stat.limit_bits = cfg.size_limits.max_acc_state_bits;
new_storage_stat.add_used_storage(new_code);
new_storage_stat.add_used_storage(new_data);
new_storage_stat.add_used_storage(new_library);
if (acc_status == Account::acc_active) {
new_storage_stat.clear_limit();
} else {
new_storage_stat.clear();
}
return new_storage_stat.cells <= cfg.size_limits.max_acc_state_cells &&
new_storage_stat.bits <= cfg.size_limits.max_acc_state_bits;
}
bool Transaction::prepare_bounce_phase(const ActionPhaseConfig& cfg) {
if (in_msg.is_null() || !bounce_enabled) {
return false;
@ -1924,6 +1988,32 @@ bool Account::store_acc_status(vm::CellBuilder& cb, int acc_status) const {
return cb.store_long_bool(v, 2);
}
static td::optional<vm::CellStorageStat> try_update_storage_stat(const vm::CellStorageStat& old_stat,
td::Ref<vm::CellSlice> old_cs,
td::Ref<vm::Cell> new_cell) {
if (old_stat.cells == 0 || old_cs.is_null()) {
return {};
}
vm::CellSlice new_cs = vm::CellSlice(vm::NoVm(), new_cell);
if (old_cs->size_refs() != new_cs.size_refs()) {
return {};
}
for (unsigned i = 0; i < old_cs->size_refs(); ++i) {
if (old_cs->prefetch_ref(i)->get_hash() != new_cs.prefetch_ref(i)->get_hash()) {
return {};
}
}
if (old_stat.bits < old_cs->size()) {
return {};
}
vm::CellStorageStat new_stat;
new_stat.cells = old_stat.cells;
new_stat.bits = old_stat.bits - old_cs->size() + new_cs.size();
new_stat.public_cells = old_stat.public_cells;
return new_stat;
}
bool Transaction::compute_state() {
if (new_total_state.not_null()) {
return true;
@ -1985,6 +2075,7 @@ bool Transaction::compute_state() {
// code:(Maybe ^Cell) data:(Maybe ^Cell) library:(HashmapE 256 SimpleLib)
}
auto storage = cb.finalize();
new_storage = td::Ref<vm::CellSlice>(true, vm::NoVm(), storage);
if (si_pos) {
auto cs_ref = load_cell_slice_ref(storage);
CHECK(cs_ref.unique_write().skip_ext(si_pos));
@ -1993,7 +2084,16 @@ bool Transaction::compute_state() {
new_inner_state.clear();
}
vm::CellStorageStat& stats = new_storage_stat;
CHECK(stats.compute_used_storage(Ref<vm::Cell>(storage)));
auto new_stats = try_update_storage_stat(account.storage_stat, account.storage, storage);
if (new_stats) {
stats = new_stats.unwrap();
} else {
td::Timer timer;
CHECK(stats.add_used_storage(Ref<vm::Cell>(storage)));
if (timer.elapsed() > 0.1) {
LOG(INFO) << "Compute used storage took " << timer.elapsed() << "s";
}
}
CHECK(cb.store_long_bool(1, 1) // account$1
&& cb.append_cellslice_bool(account.my_addr) // addr:MsgAddressInt
&& block::store_UInt7(cb, stats.cells) // storage_used$_ cells:(VarUInteger 7)
@ -2265,9 +2365,15 @@ bool Transaction::would_fit(unsigned cls, const block::BlockLimitStatus& blimst)
return blimst.would_fit(cls, end_lt, gas_used(), &extra);
}
bool Transaction::update_limits(block::BlockLimitStatus& blimst) const {
return blimst.update_lt(end_lt) && blimst.update_gas(gas_used()) && blimst.add_proof(new_total_state) &&
blimst.add_cell(root) && blimst.add_transaction() && blimst.add_account(is_first);
bool Transaction::update_limits(block::BlockLimitStatus& blimst, bool with_size) const {
if (!(blimst.update_lt(end_lt) && blimst.update_gas(gas_used()))) {
return false;
}
if (with_size) {
return blimst.add_proof(new_total_state) && blimst.add_cell(root) && blimst.add_transaction() &&
blimst.add_account(is_first);
}
return true;
}
/*
@ -2296,6 +2402,7 @@ Ref<vm::Cell> Transaction::commit(Account& acc) {
acc.last_trans_hash_ = root->get_hash().bits();
acc.last_paid = last_paid;
acc.storage_stat = new_storage_stat;
acc.storage = new_storage;
acc.balance = std::move(balance);
acc.due_payment = std::move(due_payment);
acc.total_state = std::move(new_total_state);

9
crypto/block/transaction.h
View file

@ -107,6 +107,7 @@ struct ComputePhaseConfig {
Ref<vm::Cell> global_config;
td::BitArray<256> block_rand_seed;
bool with_vm_log{false};
td::uint16 max_vm_data_depth = 512;
ComputePhaseConfig(td::uint64 _gas_price = 0, td::uint64 _gas_limit = 0, td::uint64 _gas_credit = 0)
: gas_price(_gas_price), gas_limit(_gas_limit), special_gas_limit(_gas_limit), gas_credit(_gas_credit) {
compute_threshold();
@ -143,6 +144,7 @@ struct ActionPhaseConfig {
int bounce_msg_body{0}; // usually 0 or 256 bits
MsgPrices fwd_std;
MsgPrices fwd_mc; // from/to masterchain
SizeLimitsConfig size_limits;
const WorkchainSet* workchains{nullptr};
const MsgPrices& fetch_msg_prices(bool is_masterchain) const {
return is_masterchain ? fwd_mc : fwd_std;
@ -182,6 +184,7 @@ struct ActionPhase {
bool code_changed{false};
bool action_list_invalid{false};
bool acc_delete_req{false};
bool state_size_too_big{false};
enum { acst_unchanged = 0, acst_frozen = 2, acst_deleted = 3 };
int acc_status_change{acst_unchanged};
td::RefInt256 total_fwd_fees; // all fees debited from the account
@ -235,6 +238,7 @@ struct Account {
td::RefInt256 due_payment;
Ref<vm::Cell> orig_total_state; // ^Account
Ref<vm::Cell> total_state; // ^Account
Ref<vm::CellSlice> storage; // AccountStorage
Ref<vm::CellSlice> inner_state; // StateInit
ton::Bits256 state_hash; // hash of StateInit for frozen accounts
Ref<vm::Cell> code, data, library, orig_library;
@ -283,7 +287,6 @@ struct Account {
};
struct Transaction {
static constexpr unsigned max_msg_bits = (1 << 21), max_msg_cells = (1 << 13);
enum {
tr_none,
tr_ord,
@ -323,6 +326,7 @@ struct Transaction {
ton::UnixTime last_paid;
Ref<vm::Cell> root;
Ref<vm::Cell> new_total_state;
Ref<vm::CellSlice> new_storage;
Ref<vm::CellSlice> new_inner_state;
Ref<vm::Cell> new_code, new_data, new_library;
Ref<vm::Cell> in_msg, in_msg_state;
@ -348,6 +352,7 @@ struct Transaction {
std::vector<Ref<vm::Cell>> compute_vm_libraries(const ComputePhaseConfig& cfg);
bool prepare_compute_phase(const ComputePhaseConfig& cfg);
bool prepare_action_phase(const ActionPhaseConfig& cfg);
bool check_state_size_limit(const ActionPhaseConfig& cfg);
bool prepare_bounce_phase(const ActionPhaseConfig& cfg);
bool compute_state();
bool serialize();
@ -359,7 +364,7 @@ struct Transaction {
const vm::NewCellStorageStat& store_stat, const vm::CellUsageTree* usage_tree) const;
bool update_block_storage_profile(vm::NewCellStorageStat& store_stat, const vm::CellUsageTree* usage_tree) const;
bool would_fit(unsigned cls, const block::BlockLimitStatus& blk_lim_st) const;
bool update_limits(block::BlockLimitStatus& blk_lim_st) const;
bool update_limits(block::BlockLimitStatus& blk_lim_st, bool with_size = true) const;
Ref<vm::Cell> commit(Account& _account); // _account should point to the same account
LtCellRef extract_out_msg(unsigned i);

22
crypto/func/analyzer.cpp
View file

@ -520,6 +520,14 @@ bool Op::compute_used_vars(const CodeBlob& code, bool edit) {
} while (changes <= edit);
return set_var_info(std::move(new_var_info));
}
case _TryCatch: {
code.compute_used_code_vars(block0, next_var_info, edit);
code.compute_used_code_vars(block1, next_var_info, edit);
VarDescrList merge_info = block0->var_info + block1->var_info + next_var_info;
merge_info -= left;
merge_info.clear_last();
return set_var_info(std::move(merge_info));
}
default:
std::cerr << "fatal: unknown operation <??" << cl << "> in compute_used_vars()\n";
throw src::ParseError{where, "unknown operation"};
@ -645,6 +653,10 @@ bool prune_unreachable(std::unique_ptr<Op>& ops) {
reach = true;
break;
}
case Op::_TryCatch: {
reach = prune_unreachable(op.block0) | prune_unreachable(op.block1);
break;
}
default:
std::cerr << "fatal: unknown operation <??" << op.cl << ">\n";
throw src::ParseError{op.where, "unknown operation in prune_unreachable()"};
@ -798,6 +810,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) {
break;
}
case _While: {
auto values0 = values;
values = block0->fwd_analyze(values);
if (values[left[0]] && values[left[0]]->always_false()) {
// block1 never executed
@ -805,7 +818,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) {
break;
}
while (true) {
VarDescrList next_values = values | block0->fwd_analyze(block1->fwd_analyze(values));
VarDescrList next_values = values | block0->fwd_analyze(values0 | block1->fwd_analyze(values));
if (same_values(next_values, values)) {
break;
}
@ -825,6 +838,12 @@ VarDescrList Op::fwd_analyze(VarDescrList values) {
values = block0->fwd_analyze(values);
break;
}
case _TryCatch: {
VarDescrList val1 = block0->fwd_analyze(values);
VarDescrList val2 = block1->fwd_analyze(std::move(values));
values = val1 | val2;
break;
}
default:
std::cerr << "fatal: unknown operation <??" << cl << ">\n";
throw src::ParseError{where, "unknown operation in fwd_analyze()"};
@ -866,6 +885,7 @@ bool Op::mark_noreturn() {
case _Return:
return set_noreturn(true);
case _If:
case _TryCatch:
return set_noreturn((block0->mark_noreturn() & (block1 && block1->mark_noreturn())) | next->mark_noreturn());
case _Again:
block0->mark_noreturn();

74
crypto/func/auto-tests/stress_tester.py
View file

@ -13,7 +13,7 @@ def getenv(name, default=None):
print("Environemnt variable", name, "is not set", file=sys.stderr)
exit(1)
VAR_CNT = 5
VAR_CNT = 10
TMP_DIR = tempfile.mkdtemp()
FUNC_EXECUTABLE = getenv("FUNC_EXECUTABLE", "func")
FIFT_EXECUTABLE = getenv("FIFT_EXECUTABLE", "fift")
@ -31,6 +31,15 @@ class State:
self.x = x
self.vs = [0] * VAR_CNT
def copy(self):
s = State(self.x)
s.vs = self.vs.copy()
return s
def copy_from(self, s):
self.x = s.x
self.vs = s.vs.copy()
class Code:
pass
@ -128,13 +137,51 @@ class CodeRepeat(Code):
self.c.write(f, indent + 1)
print(" " * (indent + 1) + "%s += 1;" % var, file=f)
print(" " * indent + "}", file=f)
else:
elif self.loop_type == 2:
var = gen_var_name()
print(" " * indent + "int %s = 0;" % var, file=f)
print(" " * indent + "do {", file=f)
self.c.write(f, indent + 1)
print(" " * (indent + 1) + "%s += 1;" % var, file=f)
print(" " * indent + "} until (%s >= %d);" % (var, self.n), file=f)
else:
var = gen_var_name()
print(" " * indent + "int %s = %d;" % (var, self.n - 1), file=f)
print(" " * indent + "while (%s >= 0) {" % var, file=f)
self.c.write(f, indent + 1)
print(" " * (indent + 1) + "%s -= 1;" % var, file=f)
print(" " * indent + "}", file=f)
class CodeThrow(Code):
def __init__(self):
pass
def execute(self, state):
return "EXCEPTION"
def write(self, f, indent=0):
print(" " * indent + "throw(42);", file=f)
class CodeTryCatch(Code):
def __init__(self, c1, c2):
self.c1 = c1
self.c2 = c2
def execute(self, state):
state0 = state.copy()
res = self.c1.execute(state)
if res == "EXCEPTION":
state.copy_from(state0)
return self.c2.execute(state)
else:
return res
def write(self, f, indent=0):
print(" " * indent + "try {", file=f)
self.c1.write(f, indent + 1)
print(" " * indent + "} catch (_, _) {", file=f)
self.c2.write(f, indent + 1)
print(" " * indent + "}", file=f)
def write_function(f, name, body, inline=False, inline_ref=False, method_id=None):
print("_ %s(int x)" % name, file=f, end="")
@ -147,31 +194,37 @@ def write_function(f, name, body, inline=False, inline_ref=False, method_id=None
print(" {", file=f)
for i in range(VAR_CNT):
print(" int v%d = 0;" % i, file=f)
body.write(f, 1);
body.write(f, 1)
print("}", file=f)
def gen_code(xl, xr, with_return, loop_depth=0):
def gen_code(xl, xr, with_return, loop_depth=0, try_catch_depth=0, can_throw=False):
if try_catch_depth < 3 and random.randint(0, 5) == 0:
c1 = gen_code(xl, xr, with_return, loop_depth, try_catch_depth + 1, random.randint(0, 1) == 0)
c2 = gen_code(xl, xr, with_return, loop_depth, try_catch_depth + 1, can_throw)
return CodeTryCatch(c1, c2)
code = []
for _ in range(random.randint(0, 2)):
if random.randint(0, 3) == 0 and loop_depth < 3:
c = gen_code(xl, xr, False, loop_depth + 1)
code.append(CodeRepeat(random.randint(0, 3), c, random.randint(0, 2)))
c = gen_code(xl, xr, False, loop_depth + 1, try_catch_depth, can_throw)
code.append(CodeRepeat(random.randint(0, 3), c, random.randint(0, 3)))
elif xr - xl > 1:
xmid = random.randrange(xl + 1, xr)
ret = random.choice((0, 0, 0, 0, 0, 1, 2))
c1 = gen_code(xl, xmid, ret == 1, loop_depth)
c1 = gen_code(xl, xmid, ret == 1, loop_depth, try_catch_depth, can_throw)
if random.randrange(5) == 0:
c2 = CodeEmpty()
else:
c2 = gen_code(xmid, xr, ret == 2, loop_depth)
c2 = gen_code(xmid, xr, ret == 2, loop_depth, try_catch_depth, can_throw)
code.append(CodeIfRange(xl, xmid, c1, c2))
if xr - xl == 1 and can_throw and random.randint(0, 5) == 0:
code.append(CodeThrow())
if with_return:
if xr - xl == 1:
code.append(CodeReturn(random.randrange(10**9)))
else:
xmid = random.randrange(xl + 1, xr)
c1 = gen_code(xl, xmid, True, loop_depth)
c2 = gen_code(xmid, xr, True, loop_depth)
c1 = gen_code(xl, xmid, True, loop_depth, try_catch_depth, can_throw)
c2 = gen_code(xmid, xr, True, loop_depth, try_catch_depth, can_throw)
code.append(CodeIfRange(xl, xmid, c1, c2))
for _ in range(random.randint(0, 3)):
pos = random.randint(0, len(code))
@ -203,6 +256,7 @@ def runvm(compiled_fif, xl, xr):
output.append(list(map(int, s.split())))
return output
cnt_ok = 0
cnt_fail = 0
for test_id in range(0, 1000000):

37
crypto/func/builtins.cpp
View file

@ -986,6 +986,38 @@ AsmOp compile_cond_throw(std::vector<VarDescr>& res, std::vector<VarDescr>& args
}
}
AsmOp compile_throw_arg(std::vector<VarDescr>& res, std::vector<VarDescr>& args) {
assert(res.empty() && args.size() == 2);
VarDescr &x = args[1];
if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) {
x.unused();
return exec_arg_op("THROWARG", x.int_const, 1, 0);
} else {
return exec_op("THROWARGANY", 2, 0);
}
}
AsmOp compile_cond_throw_arg(std::vector<VarDescr>& res, std::vector<VarDescr>& args, bool mode) {
assert(res.empty() && args.size() == 3);
VarDescr &x = args[1], &y = args[2];
std::string suff = (mode ? "IF" : "IFNOT");
bool skip_cond = false;
if (y.always_true() || y.always_false()) {
y.unused();
skip_cond = true;
if (y.always_true() != mode) {
x.unused();
return AsmOp::Nop();
}
}
if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) {
x.unused();
return skip_cond ? exec_arg_op("THROWARG", x.int_const, 1, 0) : exec_arg_op("THROWARG"s + suff, x.int_const, 2, 0);
} else {
return skip_cond ? exec_op("THROWARGANY", 2, 0) : exec_op("THROWARGANY"s + suff, 3, 0);
}
}
AsmOp compile_bool_const(std::vector<VarDescr>& res, std::vector<VarDescr>& args, bool val) {
assert(res.size() == 1 && args.empty());
VarDescr& r = res[0];
@ -1111,6 +1143,8 @@ void define_builtins() {
auto fetch_slice_op = TypeExpr::new_map(SliceInt, TypeExpr::new_tensor({Slice, Slice}));
auto prefetch_slice_op = TypeExpr::new_map(SliceInt, Slice);
//auto arith_null_op = TypeExpr::new_map(TypeExpr::new_unit(), Int);
auto throw_arg_op = TypeExpr::new_forall({X}, TypeExpr::new_map(TypeExpr::new_tensor({X, Int}), Unit));
auto cond_throw_arg_op = TypeExpr::new_forall({X}, TypeExpr::new_map(TypeExpr::new_tensor({X, Int, Int}), Unit));
define_builtin_func("_+_", arith_bin_op, compile_add);
define_builtin_func("_-_", arith_bin_op, compile_sub);
define_builtin_func("-_", arith_un_op, compile_negate);
@ -1170,6 +1204,9 @@ void define_builtins() {
define_builtin_func("throw", impure_un_op, compile_throw, true);
define_builtin_func("throw_if", impure_bin_op, std::bind(compile_cond_throw, _1, _2, true), true);
define_builtin_func("throw_unless", impure_bin_op, std::bind(compile_cond_throw, _1, _2, false), true);
define_builtin_func("throw_arg", throw_arg_op, compile_throw_arg, true);
define_builtin_func("throw_arg_if", cond_throw_arg_op, std::bind(compile_cond_throw_arg, _1, _2, true), true);
define_builtin_func("throw_arg_unless", cond_throw_arg_op, std::bind(compile_cond_throw_arg, _1, _2, false), true);
define_builtin_func("load_int", fetch_int_op, std::bind(compile_fetch_int, _1, _2, true, true), {}, {1, 0});
define_builtin_func("load_uint", fetch_int_op, std::bind(compile_fetch_int, _1, _2, true, false), {}, {1, 0});
define_builtin_func("preload_int", prefetch_int_op, std::bind(compile_fetch_int, _1, _2, false, true));

71
crypto/func/codegen.cpp
View file

@ -782,6 +782,77 @@ bool Op::generate_code_step(Stack& stack) {
return false;
}
}
case _TryCatch: {
if (block0->is_empty() && block1->is_empty()) {
return true;
}
if (block0->noreturn() || block1->noreturn()) {
stack.o.retalt_ = true;
}
Stack catch_stack{stack.o};
std::vector<var_idx_t> catch_vars;
std::vector<bool> catch_last;
for (const VarDescr& var : block1->var_info.list) {
if (stack.find(var.idx) >= 0) {
catch_vars.push_back(var.idx);
catch_last.push_back(!block0->var_info[var.idx]);
}
}
const size_t block_size = 255;
for (size_t begin = catch_vars.size(), end = begin; end > 0; end = begin) {
begin = end >= block_size ? end - block_size : 0;
for (size_t i = begin; i < end; ++i) {
catch_stack.push_new_var(catch_vars[i]);
}
}
catch_stack.push_new_var(left[0]);
catch_stack.push_new_var(left[1]);
stack.rearrange_top(catch_vars, catch_last);
stack.opt_show();
stack.o << "c4 PUSH";
stack.o << "c5 PUSH";
stack.o << "c7 PUSH";
stack.o << "<{";
stack.o.indent();
if (block1->noreturn()) {
catch_stack.mode |= Stack::_NeedRetAlt;
}
block1->generate_code_all(catch_stack);
catch_stack.drop_vars_except(next->var_info);
catch_stack.opt_show();
stack.o.undent();
stack.o << "}>CONT";
stack.o << "c7 SETCONT";
stack.o << "c5 SETCONT";
stack.o << "c4 SETCONT";
for (size_t begin = catch_vars.size(), end = begin; end > 0; end = begin) {
begin = end >= block_size ? end - block_size : 0;
stack.o << std::to_string(end - begin) + " PUSHINT";
stack.o << "-1 PUSHINT";
stack.o << "SETCONTVARARGS";
}
stack.s.erase(stack.s.end() - catch_vars.size(), stack.s.end());
stack.modified();
stack.o << "<{";
stack.o.indent();
if (block0->noreturn()) {
stack.mode |= Stack::_NeedRetAlt;
}
block0->generate_code_all(stack);
if (block0->noreturn()) {
stack.s = std::move(catch_stack.s);
} else if (!block1->noreturn()) {
stack.merge_state(catch_stack);
}
stack.opt_show();
stack.o.undent();
stack.o << "}>CONT";
stack.o << "c1 PUSH";
stack.o << "COMPOSALT";
stack.o << "SWAP";
stack.o << "TRY";
return true;
}
default:
std::cerr << "fatal: unknown operation <??" << cl << ">\n";
throw src::ParseError{where, "unknown operation in generate_code()"};

33
crypto/func/func.h
View file

@ -39,7 +39,7 @@ extern std::string generated_from;
constexpr int optimize_depth = 20;
const std::string func_version{"0.3.0"};
const std::string func_version{"0.4.0"};
enum Keyword {
_Eof = -1,
@ -53,6 +53,8 @@ enum Keyword {
_Do,
_While,
_Until,
_Try,
_Catch,
_If,
_Ifnot,
_Then,
@ -304,10 +306,16 @@ struct TmpVar {
sym_idx_t name;
int coord;
std::unique_ptr<SrcLocation> where;
size_t modify_forbidden = 0;
TmpVar(var_idx_t _idx, int _cls, TypeExpr* _type = 0, SymDef* sym = 0, const SrcLocation* loc = 0);
void show(std::ostream& os, int omit_idx = 0) const;
void dump(std::ostream& os) const;
void set_location(const SrcLocation& loc);
std::string to_string() const {
std::ostringstream s;
show(s, 2);
return s.str();
}
};
struct VarDescr {
@ -537,6 +545,7 @@ struct Op {
_Until,
_Repeat,
_Again,
_TryCatch,
_SliceConst
};
int cl;
@ -719,6 +728,22 @@ struct CodeBlob {
void mark_noreturn();
void generate_code(AsmOpList& out_list, int mode = 0);
void generate_code(std::ostream& os, int mode = 0, int indent = 0);
void mark_modify_forbidden(var_idx_t idx) {
++vars.at(idx).modify_forbidden;
}
void unmark_modify_forbidden(var_idx_t idx) {
assert(vars.at(idx).modify_forbidden > 0);
--vars.at(idx).modify_forbidden;
}
void check_modify_forbidden(var_idx_t idx, const SrcLocation& here) const {
if (vars.at(idx).modify_forbidden) {
throw src::ParseError{here, PSTRING() << "Modifying local variable " << vars[idx].to_string()
<< " after using it in the same expression"};
}
}
};
/*
@ -922,7 +947,7 @@ struct Expr {
}
int define_new_vars(CodeBlob& code);
int predefine_vars();
std::vector<var_idx_t> pre_compile(CodeBlob& code, bool lval = false) const;
std::vector<var_idx_t> pre_compile(CodeBlob& code, std::vector<std::pair<SymDef*, var_idx_t>>* lval_globs = nullptr) const;
static std::vector<var_idx_t> pre_compile_let(CodeBlob& code, Expr* lhs, Expr* rhs, const SrcLocation& here);
var_idx_t new_tmp(CodeBlob& code) const;
std::vector<var_idx_t> new_tmp_vect(CodeBlob& code) const {
@ -1559,6 +1584,9 @@ struct Stack {
int find_outside(var_idx_t var, int from, int to) const;
void forget_const();
void validate(int i) const {
if (i > 255) {
throw src::Fatal{"Too deep stack"};
}
assert(i >= 0 && i < depth() && "invalid stack reference");
}
void modified() {
@ -1593,6 +1621,7 @@ struct Stack {
void apply_wrappers() {
if (o.retalt_) {
o.insert(0, "SAMEALTSAVE");
o.insert(0, "c2 SAVE");
if (mode & _InlineFunc) {
o.indent_all();
o.insert(0, "CONT:<{");

62
crypto/func/gen-abscode.cpp
View file

@ -221,6 +221,13 @@ var_idx_t Expr::new_tmp(CodeBlob& code) const {
return code.create_tmp_var(e_type, &here);
}
void add_set_globs(CodeBlob& code, std::vector<std::pair<SymDef*, var_idx_t>>& globs, const SrcLocation& here) {
for (const auto& p : globs) {
auto& op = code.emplace_back(here, Op::_SetGlob, std::vector<var_idx_t>{}, std::vector<var_idx_t>{ p.second }, p.first);
op.flags |= Op::_Impure;
}
}
std::vector<var_idx_t> Expr::pre_compile_let(CodeBlob& code, Expr* lhs, Expr* rhs, const SrcLocation& here) {
while (lhs->is_type_apply()) {
lhs = lhs->args.at(0);
@ -245,19 +252,18 @@ std::vector<var_idx_t> Expr::pre_compile_let(CodeBlob& code, Expr* lhs, Expr* rh
return tmp;
}
auto right = rhs->pre_compile(code);
if (lhs->cls == Expr::_GlobVar) {
assert(lhs->sym);
auto& op = code.emplace_back(here, Op::_SetGlob, std::vector<var_idx_t>{}, right, lhs->sym);
op.flags |= Op::_Impure;
} else {
auto left = lhs->pre_compile(code, true);
code.emplace_back(here, Op::_Let, std::move(left), right);
std::vector<std::pair<SymDef*, var_idx_t>> globs;
auto left = lhs->pre_compile(code, &globs);
for (var_idx_t v : left) {
code.check_modify_forbidden(v, here);
}
code.emplace_back(here, Op::_Let, std::move(left), right);
add_set_globs(code, globs, here);
return right;
}
std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, bool lval) const {
if (lval && !(cls == _Tensor || cls == _Var || cls == _Hole || cls == _TypeApply)) {
std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, std::vector<std::pair<SymDef*, var_idx_t>>* lval_globs) const {
if (lval_globs && !(cls == _Tensor || cls == _Var || cls == _Hole || cls == _TypeApply || cls == _GlobVar)) {
std::cerr << "lvalue expression constructor is " << cls << std::endl;
throw src::Fatal{"cannot compile lvalue expression with unknown constructor"};
}
@ -265,9 +271,15 @@ std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, bool lval) const {
case _Tensor: {
std::vector<var_idx_t> res;
for (const auto& x : args) {
auto add = x->pre_compile(code, lval);
auto add = x->pre_compile(code, lval_globs);
for (var_idx_t v : add) {
code.mark_modify_forbidden(v);
}
res.insert(res.end(), add.cbegin(), add.cend());
}
for (var_idx_t v : res) {
code.unmark_modify_forbidden(v);
}
return res;
}
case _Apply: {
@ -279,6 +291,9 @@ std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, bool lval) const {
std::vector<std::vector<var_idx_t>> add_list(args.size());
for (int i : func->arg_order) {
add_list[i] = args[i]->pre_compile(code);
for (var_idx_t v : add_list[i]) {
code.mark_modify_forbidden(v);
}
}
for (const auto& add : add_list) {
res.insert(res.end(), add.cbegin(), add.cend());
@ -286,9 +301,15 @@ std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, bool lval) const {
} else {
for (const auto& x : args) {
auto add = x->pre_compile(code);
for (var_idx_t v : add) {
code.mark_modify_forbidden(v);
}
res.insert(res.end(), add.cbegin(), add.cend());
}
}
for (var_idx_t v : res) {
code.unmark_modify_forbidden(v);
}
auto rvect = new_tmp_vect(code);
auto& op = code.emplace_back(here, Op::_Call, rvect, std::move(res), sym);
if (flags & _IsImpure) {
@ -297,7 +318,7 @@ std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, bool lval) const {
return rvect;
}
case _TypeApply:
return args[0]->pre_compile(code, lval);
return args[0]->pre_compile(code, lval_globs);
case _Var:
case _Hole:
return {val};
@ -329,8 +350,13 @@ std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, bool lval) const {
case _Glob:
case _GlobVar: {
auto rvect = new_tmp_vect(code);
code.emplace_back(here, Op::_GlobVar, rvect, std::vector<var_idx_t>{}, sym);
return rvect;
if (lval_globs) {
lval_globs->push_back({ sym, rvect[0] });
return rvect;
} else {
code.emplace_back(here, Op::_GlobVar, rvect, std::vector<var_idx_t>{}, sym);
return rvect;
}
}
case _Letop: {
return pre_compile_let(code, args.at(0), args.at(1), here);
@ -338,9 +364,17 @@ std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, bool lval) const {
case _LetFirst: {
auto rvect = new_tmp_vect(code);
auto right = args[1]->pre_compile(code);
auto left = args[0]->pre_compile(code, true);
std::vector<std::pair<SymDef*, var_idx_t>> local_globs;
if (!lval_globs) {
lval_globs = &local_globs;
}
auto left = args[0]->pre_compile(code, lval_globs);
left.push_back(rvect[0]);
for (var_idx_t v : left) {
code.check_modify_forbidden(v, here);
}
code.emplace_back(here, Op::_Let, std::move(left), std::move(right));
add_set_globs(code, local_globs, here);
return rvect;
}
case _MkTuple: {

2
crypto/func/keywords.cpp
View file

@ -97,6 +97,8 @@ void define_keywords() {
.add_keyword("do", Kw::_Do)
.add_keyword("while", Kw::_While)
.add_keyword("until", Kw::_Until)
.add_keyword("try", Kw::_Try)
.add_keyword("catch", Kw::_Catch)
.add_keyword("if", Kw::_If)
.add_keyword("ifnot", Kw::_Ifnot)
.add_keyword("then", Kw::_Then)

32
crypto/func/parse-func.cpp
View file

@ -1102,6 +1102,36 @@ blk_fl::val parse_do_stmt(Lexer& lex, CodeBlob& code) {
return res & ~blk_fl::empty;
}
blk_fl::val parse_try_catch_stmt(Lexer& lex, CodeBlob& code) {
lex.expect(_Try);
Op& try_catch_op = code.emplace_back(lex.cur().loc, Op::_TryCatch);
code.push_set_cur(try_catch_op.block0);
blk_fl::val res0 = parse_block_stmt(lex, code);
code.close_pop_cur(lex.cur().loc);
lex.expect(_Catch);
code.push_set_cur(try_catch_op.block1);
sym::open_scope(lex);
Expr* expr = parse_expr(lex, code, true);
expr->chk_lvalue(lex.cur());
TypeExpr* tvm_error_type = TypeExpr::new_tensor(TypeExpr::new_var(), TypeExpr::new_atomic(_Int));
try {
unify(expr->e_type, tvm_error_type);
} catch (UnifyError& ue) {
std::ostringstream os;
os << "`catch` arguments have incorrect type " << expr->e_type << ": " << ue;
lex.cur().error(os.str());
}
expr->predefine_vars();
expr->define_new_vars(code);
try_catch_op.left = expr->pre_compile(code);
assert(try_catch_op.left.size() == 2);
blk_fl::val res1 = parse_block_stmt(lex, code);
sym::close_scope(lex);
code.close_pop_cur(lex.cur().loc);
blk_fl::combine_parallel(res0, res1);
return res0;
}
blk_fl::val parse_if_stmt(Lexer& lex, CodeBlob& code, int first_lex = _If) {
SrcLocation loc{lex.cur().loc};
lex.expect(first_lex);
@ -1165,6 +1195,8 @@ blk_fl::val parse_stmt(Lexer& lex, CodeBlob& code) {
return parse_do_stmt(lex, code);
case _While:
return parse_while_stmt(lex, code);
case _Try:
return parse_try_catch_stmt(lex, code);
default: {
auto expr = parse_expr(lex, code);
expr->chk_rvalue(lex.cur());

113
crypto/func/test/tc1.fc Normal file
View file

@ -0,0 +1,113 @@
() test1() impure {
int i = 3;
repeat (3) {
try {
int j = i;
i *= 2;
throw_unless(500, j <= 10);
} catch (x, e) {
i -= 2;
}
i += i + 1;
}
throw_unless(501, i == 43);
}
int divide_by_ten(int num) {
try {
throw_unless(500, num < 10);
} catch (x, e) {
return divide_by_ten(num - 10) + 1;
}
return 0;
}
() test2() impure {
int n = divide_by_ten(37);
throw_unless(502, n == 3);
}
(int, int) swap_int(int a, int b) {
try {
a = a * b;
b = a / b;
a = a / b;
return (a, b);
} catch (x, e) {
throw_unless(500, b == 0);
}
return (0, a);
}
() test3() impure {
int a = 0;
int b = 57;
try {
(a, b) = swap_int(a, b);
} catch (x, e) {
throw_unless(500, a == 0);
a = b;
b = 0;
}
throw_unless(503, (a == 57) & (b == 0));
}
int get_x(int x, int y) {
try {
} catch (x, e) {
return -1;
}
return x;
}
int get_y(int x, int y) {
try {
return -1;
} catch (x, e) {
}
return y;
}
() test4() impure {
throw_unless(504, get_x(3, 4) == 3);
throw_unless(504, get_y(3, 4) == -1);
}
(int, int, int, int, int) foo(int a, int b, int c, int d, int e) {
try {
throw(11);
} catch (x, y) {
a += 1;
b += 2;
c += 3;
d += 4;
e += 5;
}
return (a, b, c, d, e);
}
() test5() impure {
var (a, b, c, d, e) = foo(10, 20, 30, 40, 50);
throw_unless(505, (a == 11) & (b == 22) & (c == 33) & (d == 44) & (e == 55));
}
() test6() impure {
int a = 0;
int b = 0;
int c = 0;
try {
b = 3;
} catch (x, y) {
b = 12;
}
throw_unless(506, (a == 0) & (b == 3) & (c == 0));
}
() main() {
test1();
test2();
test3();
test4();
test5();
test6();
}

84
crypto/func/test/tc2.fc Normal file
View file

@ -0,0 +1,84 @@
forall X -> int cast_to_int(X x) asm "NOP";
forall X -> builder cast_to_builder(X x) asm "NOP";
_ test1_body() {
int a = 3;
builder b = begin_cell();
int c = 1;
try {
c = 3;
throw_arg(b, 100);
} catch (x, y) {
return (a + c + y, cast_to_builder(x));
}
return (0, null());
}
() test1() impure {
var (x, y) = test1_body();
throw_unless(101, x == 104);
throw_unless(102, y.builder_refs() == y.builder_bits());
}
_ test2_body(int a, int b, int c) {
try {
try {
try {
try {
throw_arg_if(1, 201, a + b + c == 3);
throw_arg_if(2, 201, a == 3);
throw_arg_unless(1, 202, b == 4);
return 1;
} catch (y, x) {
int y = y.cast_to_int();
throw_arg_unless(y, x, x == 202);
throw_arg(y + 1, 200);
}
} catch (y, x) {
int y = y.cast_to_int();
throw_arg_if(y, x, x == 200);
throw_arg_if(y + 2, x, y < 2);
throw_arg_if(y + 3, 203, a + b + c == 4);
throw_arg_unless(y + 4, 204, b == 4);
return 3;
}
} catch (y, x) {
int y = y.cast_to_int();
try {
throw_arg_if(y, x, x == 200);
throw_arg_if(y + 1, 200, x == 201);
throw_arg_if(x - 203, 200, x == 202);
throw_arg_if(y, 200, x == 203);
throw_arg_if(a + 4, 205, a + b + c == 5);
throw_arg(7, 200);
} catch (v, u) {
int v = v.cast_to_int();
throw_arg_unless(v, u, u == 205);
if (c == 0) {
return b + 4;
}
throw_arg(v + 1, 200);
}
}
} catch (y, x) {
throw_unless(x, x == 200);
return y.cast_to_int();
}
return null();
}
() test2() impure {
throw_unless(201, test2_body(0, 4, 0) == 1);
throw_unless(202, test2_body(0, 5, 0) == 2);
throw_unless(203, test2_body(3, 4, 0) == 3);
throw_unless(204, test2_body(3, 0, 0) == 4);
throw_unless(205, test2_body(3, 1, 0) == 5);
throw_unless(206, test2_body(3, 2, 0) == 6);
throw_unless(207, test2_body(3, 1, 2) == 7);
throw_unless(208, test2_body(3, 1, 1) == 8);
}
() main() {
test1();
test2();
}

2
crypto/smartcont/stdlib.fc
View file

@ -611,7 +611,7 @@ int rand(int range) impure asm "RAND";
;;; Returns the current random seed as an unsigned 256-bit Integer.
int get_seed() impure asm "RANDSEED";
;;; Sets the random seed to unsigned 256-bit seed.
int set_seed() impure asm "SETRAND";
() set_seed(int) impure asm "SETRAND";
;;; Mixes unsigned 256-bit integer x into the random seed r by setting the random seed to sha256 of the concatenation of two 32-byte strings: the first with the big-endian representation of the old seed r, and the second with the big-endian representation of x.
() randomize(int x) impure asm "ADDRAND";
;;; Equivalent to randomize(cur_lt());.

28
crypto/smc-envelope/ManualDns.cpp
View file

@ -47,6 +47,8 @@ td::StringBuilder& operator<<(td::StringBuilder& sb, const ManualDns::EntryData&
.move_as_ok();
case ManualDns::EntryData::Type::SmcAddress:
return sb << "SMC:" << data.data.get<ManualDns::EntryDataSmcAddress>().smc_address.rserialize();
case ManualDns::EntryData::Type::StorageAddress:
return sb << "STORAGE:" << data.data.get<ManualDns::EntryDataStorageAddress>().bag_id.to_hex();
}
return sb << "<unknown>";
}
@ -93,6 +95,11 @@ td::Result<td::Ref<vm::Cell>> DnsInterface::EntryData::as_cell() const {
smc_address.smc_address.addr);
dns.smc_addr = vm::load_cell_slice_ref(cb.finalize());
tlb::pack_cell(res, dns);
},
[&](const EntryDataStorageAddress& storage_address) {
block::gen::DNSRecord::Record_dns_storage_address dns;
dns.bag_id = storage_address.bag_id;
tlb::pack_cell(res, dns);
}));
if (error.is_error()) {
return error;
@ -142,6 +149,11 @@ td::Result<DnsInterface::EntryData> DnsInterface::EntryData::from_cellslice(vm::
}
return EntryData::smc_address(block::StdAddress(wc, addr));
}
case block::gen::DNSRecord::dns_storage_address: {
block::gen::DNSRecord::Record_dns_storage_address dns;
tlb::unpack(cs, dns);
return EntryData::storage_address(dns.bag_id);
}
}
return td::Status::Error("Unknown entry data");
}
@ -536,10 +548,12 @@ std::string DnsInterface::decode_name(td::Slice name) {
std::string ManualDns::serialize_data(const EntryData& data) {
std::string res;
data.data.visit(td::overloaded([&](const ton::ManualDns::EntryDataText& text) { res = "UNSUPPORTED"; },
[&](const ton::ManualDns::EntryDataNextResolver& resolver) { res = "UNSUPPORTED"; },
[&](const ton::ManualDns::EntryDataAdnlAddress& adnl_address) { res = "UNSUPPORTED"; },
[&](const ton::ManualDns::EntryDataSmcAddress& text) { res = "UNSUPPORTED"; }));
data.data.visit(
td::overloaded([&](const ton::ManualDns::EntryDataText& text) { res = "UNSUPPORTED"; },
[&](const ton::ManualDns::EntryDataNextResolver& resolver) { res = "UNSUPPORTED"; },
[&](const ton::ManualDns::EntryDataAdnlAddress& adnl_address) { res = "UNSUPPORTED"; },
[&](const ton::ManualDns::EntryDataSmcAddress& text) { res = "UNSUPPORTED"; },
[&](const ton::ManualDns::EntryDataStorageAddress& storage_address) { res = "UNSUPPORTED"; }));
return res;
}
@ -559,6 +573,12 @@ td::Result<td::optional<ManualDns::EntryData>> ManualDns::parse_data(td::Slice c
} else if (type == "NEXT") {
TRY_RESULT(address, block::StdAddress::parse(parser.read_all()));
return ManualDns::EntryData::next_resolver(address);
} else if (type == "STORAGE") {
td::Bits256 bag_id;
if (bag_id.from_hex(parser.read_all(), false) != 256) {
return td::Status::Error("failed to parse bag id");
}
return ManualDns::EntryData::storage_address(bag_id);
} else if (parser.data() == "DELETED") {
return {};
}

17
crypto/smc-envelope/ManualDns.h
View file

@ -66,9 +66,19 @@ class DnsInterface {
// TODO: capability
};
struct EntryDataStorageAddress {
ton::Bits256 bag_id;
// TODO: proto
bool operator==(const EntryDataStorageAddress& other) const {
return bag_id == other.bag_id;
}
};
struct EntryData {
enum Type { Empty, Text, NextResolver, AdnlAddress, SmcAddress } type{Empty};
td::Variant<EntryDataText, EntryDataNextResolver, EntryDataAdnlAddress, EntryDataSmcAddress> data;
enum Type { Empty, Text, NextResolver, AdnlAddress, SmcAddress, StorageAddress } type{Empty};
td::Variant<EntryDataText, EntryDataNextResolver, EntryDataAdnlAddress, EntryDataSmcAddress,
EntryDataStorageAddress>
data;
static EntryData text(std::string text) {
return {Text, EntryDataText{text}};
@ -82,6 +92,9 @@ class DnsInterface {
static EntryData smc_address(block::StdAddress smc_address) {
return {SmcAddress, EntryDataSmcAddress{smc_address}};
}
static EntryData storage_address(ton::Bits256 bag_id) {
return {StorageAddress, EntryDataStorageAddress{bag_id}};
}
bool operator==(const EntryData& other) const {
return data == other.data;

12
crypto/vm/boc.cpp
View file

@ -1039,9 +1039,15 @@ bool CellStorageStat::add_used_storage(Ref<vm::CellSlice> cs_ref, bool kill_dup,
bool CellStorageStat::add_used_storage(const CellSlice& cs, bool kill_dup, unsigned skip_count_root) {
if (!(skip_count_root & 1)) {
++cells;
if (cells > limit_cells) {
return false;
}
}
if (!(skip_count_root & 2)) {
bits += cs.size();
if (bits > limit_bits) {
return false;
}
}
for (unsigned i = 0; i < cs.size_refs(); i++) {
if (!add_used_storage(cs.prefetch_ref(i), kill_dup)) {
@ -1054,9 +1060,15 @@ bool CellStorageStat::add_used_storage(const CellSlice& cs, bool kill_dup, unsig
bool CellStorageStat::add_used_storage(CellSlice&& cs, bool kill_dup, unsigned skip_count_root) {
if (!(skip_count_root & 1)) {
++cells;
if (cells > limit_cells) {
return false;
}
}
if (!(skip_count_root & 2)) {
bits += cs.size();
if (bits > limit_bits) {
return false;
}
}
while (cs.size_refs()) {
if (!add_used_storage(cs.fetch_ref(), kill_dup)) {

8
crypto/vm/boc.h
View file

@ -117,8 +117,13 @@ struct CellStorageStat {
}
void clear() {
cells = bits = public_cells = 0;
clear_limit();
clear_seen();
}
void clear_limit() {
limit_cells = std::numeric_limits<unsigned long long>::max();
limit_bits = std::numeric_limits<unsigned long long>::max();
}
bool compute_used_storage(Ref<vm::CellSlice> cs_ref, bool kill_dup = true, unsigned skip_count_root = 0);
bool compute_used_storage(const CellSlice& cs, bool kill_dup = true, unsigned skip_count_root = 0);
bool compute_used_storage(CellSlice&& cs, bool kill_dup = true, unsigned skip_count_root = 0);
@ -128,6 +133,9 @@ struct CellStorageStat {
bool add_used_storage(const CellSlice& cs, bool kill_dup = true, unsigned skip_count_root = 0);
bool add_used_storage(CellSlice&& cs, bool kill_dup = true, unsigned skip_count_root = 0);
bool add_used_storage(Ref<vm::Cell> cell, bool kill_dup = true, unsigned skip_count_root = 0);
unsigned long long limit_cells = std::numeric_limits<unsigned long long>::max();
unsigned long long limit_bits = std::numeric_limits<unsigned long long>::max();
};
struct VmStorageStat {

4
crypto/vm/db/DynamicBagOfCellsDb.cpp
View file

@ -293,7 +293,9 @@ class DynamicBagOfCellsDbImpl : public DynamicBagOfCellsDb, private ExtCellCreat
return db_->load_cell(hash);
}
TRY_RESULT(load_result, cell_loader_->load(hash, true, *this));
CHECK(load_result.status == CellLoader::LoadResult::Ok);
if (load_result.status != CellLoader::LoadResult::Ok) {
return td::Status::Error("cell not found");
}
return std::move(load_result.cell());
}

7
crypto/vm/vm.h
View file

@ -97,6 +97,7 @@ class VmState final : public VmStateInterface {
int stack_trace{0}, debug_off{0};
bool chksig_always_succeed{false};
td::ConstBitPtr missing_library{0};
td::uint16 max_data_depth = 512; // Default value
public:
enum {
@ -108,8 +109,7 @@ class VmState final : public VmStateInterface {
implicit_jmpref_gas_price = 10,
implicit_ret_gas_price = 5,
free_stack_depth = 32,
stack_entry_gas_price = 1,
max_data_depth = 512
stack_entry_gas_price = 1
};
VmState();
VmState(Ref<CellSlice> _code);
@ -325,6 +325,9 @@ class VmState final : public VmStateInterface {
td::ConstBitPtr get_missing_library() const {
return missing_library;
}
void set_max_data_depth(td::uint16 depth) {
max_data_depth = depth;
}
private:
void init_cregs(bool same_c3 = false, bool push_0 = true);

5
dht/CMakeLists.txt
View file

@ -30,3 +30,8 @@ target_include_directories(dht PUBLIC
)
target_link_libraries(dht PRIVATE tdutils tdactor adnl tl_api)
add_executable(dht-ping-servers utils/dht-ping-servers.cpp)
target_link_libraries(dht-ping-servers PRIVATE tdutils tdactor adnl dht terminal)
add_executable(dht-resolve utils/dht-resolve.cpp)
target_link_libraries(dht-resolve PRIVATE tdutils tdactor adnl dht terminal)

83
dht/dht-bucket.cpp
View file

@ -66,38 +66,66 @@ td::uint32 DhtBucket::active_cnt() {
}
td::Status DhtBucket::add_full_node(DhtKeyId id, DhtNode newnode, td::actor::ActorId<adnl::Adnl> adnl,
adnl::AdnlNodeIdShort self_id) {
adnl::AdnlNodeIdShort self_id, td::int32 our_network_id, bool set_active) {
for (auto &node : active_nodes_) {
if (node && node->get_key() == id) {
return node->update_value(std::move(newnode), adnl, self_id);
if (set_active) {
return node->receive_ping(std::move(newnode), adnl, self_id);
} else {
return node->update_value(std::move(newnode), adnl, self_id);
}
}
}
for (auto &node : backup_nodes_) {
for (size_t i = 0; i < backup_nodes_.size(); ++i) {
auto &node = backup_nodes_[i];
if (node && node->get_key() == id) {
return node->update_value(std::move(newnode), adnl, self_id);
if (set_active) {
TRY_STATUS(node->receive_ping(std::move(newnode), adnl, self_id));
if (node->is_ready()) {
promote_node(i);
}
return td::Status::OK();
} else {
return node->update_value(std::move(newnode), adnl, self_id);
}
}
}
TRY_RESULT_PREFIX(N, DhtRemoteNode::create(std::move(newnode), max_missed_pings_), "failed to add new node: ");
for (auto &node : backup_nodes_) {
if (node == nullptr) {
node = std::move(N);
return td::Status::OK();
TRY_RESULT_PREFIX(N, DhtRemoteNode::create(std::move(newnode), max_missed_pings_, our_network_id),
"failed to add new node: ");
if (set_active) {
for (auto &node : active_nodes_) {
if (node == nullptr) {
node = std::move(N);
node->receive_ping();
return td::Status::OK();
}
}
}
for (auto &node : backup_nodes_) {
CHECK(node);
if (node->ready_from() == 0 && node->failed_from() + 60 < td::Time::now_cached()) {
node = std::move(N);
return td::Status::OK();
}
size_t idx = select_backup_node_to_drop();
if (idx < backup_nodes_.size()) {
backup_nodes_[idx] = std::move(N);
}
return td::Status::OK();
}
size_t DhtBucket::select_backup_node_to_drop() const {
size_t result = backup_nodes_.size();
for (size_t idx = 0; idx < backup_nodes_.size(); ++idx) {
const auto &node = backup_nodes_[idx];
if (node == nullptr) {
return idx;
}
if (node->ready_from() == 0 && node->failed_from() + 60 < td::Time::now_cached()) {
if (result == backup_nodes_.size() || node->failed_from() < backup_nodes_[result]->failed_from()) {
result = idx;
}
}
}
return result;
}
void DhtBucket::receive_ping(DhtKeyId id, DhtNode result, td::actor::ActorId<adnl::Adnl> adnl,
adnl::AdnlNodeIdShort self_id) {
for (auto &node : active_nodes_) {
@ -119,17 +147,9 @@ void DhtBucket::receive_ping(DhtKeyId id, DhtNode result, td::actor::ActorId<adn
}
void DhtBucket::demote_node(size_t idx) {
for (auto &node : backup_nodes_) {
if (node == nullptr) {
node = std::move(active_nodes_[idx]);
return;
}
}
for (auto &node : backup_nodes_) {
if (node->ready_from() == 0 && node->failed_from() + 60 < td::Time::now_cached()) {
node = std::move(active_nodes_[idx]);
return;
}
size_t new_idx = select_backup_node_to_drop();
if (new_idx < backup_nodes_.size()) {
backup_nodes_[new_idx] = std::move(active_nodes_[idx]);
}
active_nodes_[idx] = nullptr;
}
@ -150,7 +170,7 @@ void DhtBucket::check(bool client_only, td::actor::ActorId<adnl::Adnl> adnl, td:
size_t have_space = 0;
for (size_t i = 0; i < active_nodes_.size(); i++) {
auto &node = active_nodes_[i];
if (node && td::Time::now_cached() - node->last_ping_at() > ping_timeout_) {
if (node && td::Time::now_cached() - node->last_ping_at() > node->ping_interval()) {
node->send_ping(client_only, adnl, dht, src);
if (node->ready_from() == 0) {
demote_node(i);
@ -162,7 +182,7 @@ void DhtBucket::check(bool client_only, td::actor::ActorId<adnl::Adnl> adnl, td:
}
for (size_t i = 0; i < backup_nodes_.size(); i++) {
auto &node = backup_nodes_[i];
if (node && td::Time::now_cached() - node->last_ping_at() > ping_timeout_) {
if (node && td::Time::now_cached() - node->last_ping_at() > node->ping_interval()) {
node->send_ping(client_only, adnl, dht, src);
}
if (node && have_space > 0 && node->is_ready()) {
@ -200,6 +220,9 @@ DhtNodesList DhtBucket::export_nodes() const {
list.push_back(node->get_node());
}
}
if (list.size() > k_) {
list.list().resize(k_);
}
return list;
}

4
dht/dht-bucket.hpp
View file

@ -31,7 +31,6 @@ class DhtMember;
class DhtBucket {
private:
double ping_timeout_ = 60;
td::uint32 max_missed_pings_ = 3;
std::vector<std::unique_ptr<DhtRemoteNode>> active_nodes_;
@ -43,6 +42,7 @@ class DhtBucket {
// const DhtMember::PrintId &print_id);
void demote_node(size_t idx);
void promote_node(size_t idx);
size_t select_backup_node_to_drop() const;
public:
DhtBucket(td::uint32 k) : k_(k) {
@ -51,7 +51,7 @@ class DhtBucket {
}
td::uint32 active_cnt();
td::Status add_full_node(DhtKeyId id, DhtNode node, td::actor::ActorId<adnl::Adnl> adnl,
adnl::AdnlNodeIdShort self_id);
adnl::AdnlNodeIdShort self_id, td::int32 our_network_id, bool set_active = false);
void check(bool client_only, td::actor::ActorId<adnl::Adnl> adnl, td::actor::ActorId<DhtMember> node,
adnl::AdnlNodeIdShort src);
void receive_ping(DhtKeyId id, DhtNode result, td::actor::ActorId<adnl::Adnl> adnl, adnl::AdnlNodeIdShort self_id);

38
dht/dht-in.hpp
View file

@ -46,6 +46,7 @@ class DhtMemberImpl : public DhtMember {
DhtKeyId key_;
td::uint32 k_;
td::uint32 a_;
td::int32 network_id_{-1};
td::uint32 max_cache_time_ = 60;
td::uint32 max_cache_size_ = 100;
@ -66,6 +67,15 @@ class DhtMemberImpl : public DhtMember {
DhtKeyId last_republish_key_ = DhtKeyId::zero();
DhtKeyId last_check_key_ = DhtKeyId::zero();
adnl::AdnlNodeIdShort last_check_reverse_conn_ = adnl::AdnlNodeIdShort::zero();
struct ReverseConnection {
adnl::AdnlNodeIdShort dht_node_;
DhtKeyId key_id_;
td::Timestamp ttl_;
};
std::map<adnl::AdnlNodeIdShort, ReverseConnection> reverse_connections_;
std::set<adnl::AdnlNodeIdShort> our_reverse_connections_;
class Callback : public adnl::Adnl::Callback {
public:
@ -122,17 +132,33 @@ class DhtMemberImpl : public DhtMember {
void process_query(adnl::AdnlNodeIdShort src, ton_api::dht_store &query, td::Promise<td::BufferSlice> promise);
void process_query(adnl::AdnlNodeIdShort src, ton_api::dht_getSignedAddressList &query,
td::Promise<td::BufferSlice> promise);
void process_query(adnl::AdnlNodeIdShort src, ton_api::dht_registerReverseConnection &query,
td::Promise<td::BufferSlice> promise);
void process_query(adnl::AdnlNodeIdShort src, ton_api::dht_requestReversePing &query,
td::Promise<td::BufferSlice> promise);
public:
DhtMemberImpl(adnl::AdnlNodeIdShort id, std::string db_root, td::actor::ActorId<keyring::Keyring> keyring,
td::actor::ActorId<adnl::Adnl> adnl, td::uint32 k, td::uint32 a = 3, bool client_only = false)
: id_(id), key_{id_}, k_(k), a_(a), db_root_(db_root), keyring_(keyring), adnl_(adnl), client_only_(client_only) {
td::actor::ActorId<adnl::Adnl> adnl, td::int32 network_id, td::uint32 k, td::uint32 a = 3,
bool client_only = false)
: id_(id)
, key_{id_}
, k_(k)
, a_(a)
, network_id_(network_id)
, db_root_(db_root)
, keyring_(keyring)
, adnl_(adnl)
, client_only_(client_only) {
for (size_t i = 0; i < 256; i++) {
buckets_.emplace_back(k_);
}
}
void add_full_node(DhtKeyId id, DhtNode node) override;
void add_full_node(DhtKeyId id, DhtNode node) override {
add_full_node_impl(id, std::move(node));
}
void add_full_node_impl(DhtKeyId id, DhtNode node, bool set_active = false);
adnl::AdnlNodeIdShort get_id() const override {
return id_;
@ -143,6 +169,12 @@ class DhtMemberImpl : public DhtMember {
void set_value(DhtValue key_value, td::Promise<td::Unit> result) override;
td::uint32 distance(DhtKeyId key_id, td::uint32 max_value);
void register_reverse_connection(adnl::AdnlNodeIdFull client, td::Promise<td::Unit> promise) override;
void request_reverse_ping(adnl::AdnlNode target, adnl::AdnlNodeIdShort client,
td::Promise<td::Unit> promise) override;
void request_reverse_ping_cont(adnl::AdnlNode target, td::BufferSlice signature, adnl::AdnlNodeIdShort client,
td::Promise<td::Unit> promise);
td::Status store_in(DhtValue value) override;
void send_store(DhtValue value, td::Promise<td::Unit> promise);

39
dht/dht-node.cpp
View file

@ -23,31 +23,46 @@ namespace ton {
namespace dht {
td::Status DhtNode::update(tl_object_ptr<ton_api::dht_node> obj) {
td::Status DhtNode::update(tl_object_ptr<ton_api::dht_node> obj, td::int32 our_network_id) {
if (version_ && obj->version_ <= version_) {
return td::Status::Error(ErrorCode::notready, "too old version");
}
auto signature = std::move(obj->signature_);
auto B = serialize_tl_object(obj, true);
td::BufferSlice signature;
td::int32 network_id = -1;
if (obj->signature_.size() == 64) {
signature = std::move(obj->signature_);
} else if (obj->signature_.size() == 64 + 4) {
signature = td::BufferSlice{obj->signature_.as_slice().remove_prefix(4)};
network_id = *(td::uint32 *)obj->signature_.as_slice().remove_suffix(64).data();
} else {
return td::Status::Error(ErrorCode::notready, "invalid length of signature");
}
if (network_id != our_network_id && network_id != -1 && our_network_id != -1) {
// Remove (network_id != -1 && our_network_id != -1) after network update
return td::Status::Error(ErrorCode::notready, PSTRING() << "wrong network id (expected " << our_network_id
<< ", found " << network_id << ")");
}
TRY_RESULT(pub, adnl::AdnlNodeIdFull::create(obj->id_));
TRY_RESULT(addr_list, adnl::AdnlAddressList::create(std::move(obj->addr_list_)));
if (!addr_list.public_only()) {
return td::Status::Error(ErrorCode::notready, "dht node must have only public addresses");
}
if (!addr_list.size()) {
return td::Status::Error(ErrorCode::notready, "dht node must have >0 addresses");
}
DhtNode new_node{std::move(pub), std::move(addr_list), obj->version_, network_id, std::move(signature)};
TRY_STATUS(new_node.check_signature());
TRY_RESULT(E, pub.pubkey().create_encryptor());
TRY_STATUS(E->check_signature(B.as_slice(), signature.as_slice()));
id_ = pub;
addr_list_ = addr_list;
version_ = obj->version_;
signature_ = td::SharedSlice(signature.as_slice());
*this = std::move(new_node);
return td::Status::OK();
}
td::Status DhtNode::check_signature() const {
TRY_RESULT(enc, id_.pubkey().create_encryptor());
auto node2 = clone();
node2.signature_ = {};
TRY_STATUS_PREFIX(enc->check_signature(serialize_tl_object(node2.tl(), true).as_slice(), signature_.as_slice()),
"bad node signature: ");
return td::Status::OK();
}

42
dht/dht-node.hpp
View file

@ -22,6 +22,8 @@
#include "adnl/adnl-address-list.hpp"
#include "dht-types.h"
#include "auto/tl/ton_api.hpp"
#include "td/utils/overloaded.h"
namespace ton {
@ -32,26 +34,26 @@ class DhtNode {
adnl::AdnlNodeIdFull id_;
adnl::AdnlAddressList addr_list_;
td::int32 version_{0};
td::int32 network_id_{-1};
td::SharedSlice signature_;
public:
DhtNode() {
DhtNode() = default;
DhtNode(adnl::AdnlNodeIdFull id, adnl::AdnlAddressList addr_list, td::int32 version, td::int32 network_id, td::BufferSlice signature)
: id_(id), addr_list_(addr_list), version_(version), network_id_(network_id), signature_(signature.as_slice()) {
}
DhtNode(adnl::AdnlNodeIdFull id, adnl::AdnlAddressList addr_list, td::int32 version, td::BufferSlice signature)
: id_(id), addr_list_(addr_list), version_(version), signature_(signature.as_slice()) {
DhtNode(adnl::AdnlNodeIdFull id, adnl::AdnlAddressList addr_list, td::int32 version, td::int32 network_id, td::SharedSlice signature)
: id_(id), addr_list_(addr_list), version_(version), network_id_(network_id), signature_(std::move(signature)) {
}
DhtNode(adnl::AdnlNodeIdFull id, adnl::AdnlAddressList addr_list, td::int32 version, td::SharedSlice signature)
: id_(id), addr_list_(addr_list), version_(version), signature_(std::move(signature)) {
}
static td::Result<DhtNode> create(tl_object_ptr<ton_api::dht_node> obj) {
static td::Result<DhtNode> create(tl_object_ptr<ton_api::dht_node> obj, td::int32 our_network_id) {
if (obj->version_ == 0) {
return td::Status::Error(ErrorCode::protoviolation, "zero version");
}
DhtNode n;
TRY_STATUS(n.update(std::move(obj)));
TRY_STATUS(n.update(std::move(obj), our_network_id));
return std::move(n);
}
td::Status update(tl_object_ptr<ton_api::dht_node> obj);
td::Status update(tl_object_ptr<ton_api::dht_node> obj, td::int32 our_network_id);
DhtKeyId get_key() const {
CHECK(!id_.empty());
return DhtKeyId{id_.compute_short_id()};
@ -68,20 +70,30 @@ class DhtNode {
}
tl_object_ptr<ton_api::dht_node> tl() const {
return create_tl_object<ton_api::dht_node>(id_.tl(), addr_list_.tl(), version_, signature_.clone_as_buffer_slice());
td::BufferSlice signature_ext;
if (network_id_ == -1) {
signature_ext = signature_.clone_as_buffer_slice();
} else {
signature_ext = td::BufferSlice{4 + signature_.size()};
td::MutableSlice s = signature_ext.as_slice();
s.copy_from(td::Slice(reinterpret_cast<const td::uint8 *>(&network_id_), 4));
s.remove_prefix(4);
s.copy_from(signature_.as_slice());
}
return create_tl_object<ton_api::dht_node>(id_.tl(), addr_list_.tl(), version_, std::move(signature_ext));
}
DhtNode clone() const {
return DhtNode{id_, addr_list_, version_, signature_.clone()};
return DhtNode{id_, addr_list_, version_, network_id_, signature_.clone()};
}
td::Status check_signature() const;
};
class DhtNodesList {
public:
DhtNodesList() {
}
DhtNodesList(tl_object_ptr<ton_api::dht_nodes> R) {
DhtNodesList() = default;
DhtNodesList(tl_object_ptr<ton_api::dht_nodes> R, td::int32 our_network_id) {
for (auto &n : R->nodes_) {
auto N = DhtNode::create(std::move(n));
auto N = DhtNode::create(std::move(n), our_network_id);
if (N.is_ok()) {
list_.emplace_back(N.move_as_ok());
} else {

218
dht/dht-query.cpp
View file

@ -20,14 +20,11 @@
#include "td/utils/tl_storers.h"
#include "td/utils/crypto.h"
#include "td/utils/tl_parsers.h"
#include "td/utils/Random.h"
#include "td/utils/overloaded.h"
#include "td/utils/format.h"
#include "keys/encryptor.h"
#include "auto/tl/ton_api.hpp"
#include "dht-query.hpp"
@ -123,7 +120,7 @@ void DhtQueryFindNodes::on_result(td::Result<td::BufferSlice> R, adnl::AdnlNodeI
VLOG(DHT_WARNING) << this << ": incorrect result on dht.findNodes query from " << dst << ": "
<< Res.move_as_error();
} else {
add_nodes(DhtNodesList{Res.move_as_ok()});
add_nodes(DhtNodesList{Res.move_as_ok(), our_network_id()});
}
finish_query();
}
@ -149,6 +146,23 @@ void DhtQueryFindValue::send_one_query(adnl::AdnlNodeIdShort id) {
td::Timestamp::in(2.0 + td::Random::fast(0, 20) * 0.1), std::move(B));
}
void DhtQueryFindValue::send_one_query_nodes(adnl::AdnlNodeIdShort id) {
auto P = create_serialize_tl_object<ton_api::dht_findNode>(get_key().tl(), get_k());
td::BufferSlice B;
if (client_only_) {
B = std::move(P);
} else {
B = create_serialize_tl_object_suffix<ton_api::dht_query>(P.as_slice(), self_.tl());
}
auto Pr = td::PromiseCreator::lambda([SelfId = actor_id(this), dst = id](td::Result<td::BufferSlice> R) {
td::actor::send_closure(SelfId, &DhtQueryFindValue::on_result_nodes, std::move(R), dst);
});
td::actor::send_closure(adnl_, &adnl::Adnl::send_query, get_src(), id, "dht findValue", std::move(Pr),
td::Timestamp::in(2.0 + td::Random::fast(0, 20) * 0.1), std::move(B));
}
void DhtQueryFindValue::on_result(td::Result<td::BufferSlice> R, adnl::AdnlNodeIdShort dst) {
if (R.is_error()) {
VLOG(DHT_INFO) << this << ": failed find value query " << get_src() << "->" << dst << ": " << R.move_as_error();
@ -164,44 +178,72 @@ void DhtQueryFindValue::on_result(td::Result<td::BufferSlice> R, adnl::AdnlNodeI
}
bool need_stop = false;
bool send_get_nodes = false;
auto A = Res.move_as_ok();
ton_api::downcast_call(
*A.get(), td::overloaded(
[&](ton_api::dht_valueFound &v) {
auto valueR = DhtValue::create(std::move(v.value_), true);
if (valueR.is_error()) {
VLOG(DHT_WARNING) << this << ": received incorrect dht answer on find value query from " << dst
<< ": " << valueR.move_as_error();
return;
}
auto value = valueR.move_as_ok();
if (value.key_id() != key_) {
VLOG(DHT_WARNING) << this << ": received value for bad key on find value query from " << dst;
return;
}
promise_.set_value(std::move(value));
need_stop = true;
},
[&](ton_api::dht_valueNotFound &v) { add_nodes(DhtNodesList{std::move(v.nodes_)}); }));
*A, td::overloaded(
[&](ton_api::dht_valueFound &v) {
auto valueR = DhtValue::create(std::move(v.value_), true);
if (valueR.is_error()) {
VLOG(DHT_WARNING) << this << ": received incorrect dht answer on find value query from " << dst
<< ": " << valueR.move_as_error();
return;
}
auto value = valueR.move_as_ok();
if (value.key_id() != key_) {
VLOG(DHT_WARNING) << this << ": received value for bad key on find value query from " << dst;
return;
}
if (!value.check_is_acceptable()) {
send_get_nodes = true;
return;
}
promise_.set_value(std::move(value));
need_stop = true;
},
[&](ton_api::dht_valueNotFound &v) {
add_nodes(DhtNodesList{std::move(v.nodes_), our_network_id()});
}));
if (need_stop) {
stop();
} else if (send_get_nodes) {
send_one_query_nodes(dst);
} else {
finish_query();
}
}
void DhtQueryFindValue::on_result_nodes(td::Result<td::BufferSlice> R, adnl::AdnlNodeIdShort dst) {
if (R.is_error()) {
VLOG(DHT_INFO) << this << ": failed find nodes query " << get_src() << "->" << dst << ": " << R.move_as_error();
finish_query();
return;
}
auto Res = fetch_tl_object<ton_api::dht_nodes>(R.move_as_ok(), true);
if (Res.is_error()) {
VLOG(DHT_WARNING) << this << ": dropping incorrect answer on dht.findNodes query from " << dst << ": "
<< Res.move_as_error();
finish_query();
return;
}
auto r = Res.move_as_ok();
add_nodes(DhtNodesList{create_tl_object<ton_api::dht_nodes>(std::move(r->nodes_)), our_network_id()});
finish_query();
}
void DhtQueryFindValue::finish(DhtNodesList list) {
promise_.set_error(td::Status::Error(ErrorCode::notready, "dht key not found"));
}
DhtQueryStore::DhtQueryStore(DhtValue key_value, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src,
DhtNodesList list, td::uint32 k, td::uint32 a, DhtNode self, bool client_only,
td::actor::ActorId<DhtMember> node, td::actor::ActorId<adnl::Adnl> adnl,
DhtNodesList list, td::uint32 k, td::uint32 a, td::int32 our_network_id, DhtNode self,
bool client_only, td::actor::ActorId<DhtMember> node, td::actor::ActorId<adnl::Adnl> adnl,
td::Promise<td::Unit> promise)
: print_id_(print_id)
, k_(k)
, a_(a)
, our_network_id_(our_network_id)
, promise_(std::move(promise))
, value_(std::move(key_value))
, list_(std::move(list))
@ -219,7 +261,8 @@ void DhtQueryStore::start_up() {
auto key = value_.key_id();
auto A = td::actor::create_actor<DhtQueryFindNodes>("FindNodesQuery", key, print_id_, src_, std::move(list_), k_, a_,
self_.clone(), client_only_, node_, adnl_, std::move(P));
our_network_id_, self_.clone(), client_only_, node_, adnl_,
std::move(P));
A.release();
}
@ -279,6 +322,133 @@ void DhtQueryStore::store_ready(td::Result<td::BufferSlice> R) {
}
}
DhtQueryRegisterReverseConnection::DhtQueryRegisterReverseConnection(
DhtKeyId key_id, adnl::AdnlNodeIdFull client, td::uint32 ttl, td::BufferSlice signature,
DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, DhtNodesList list, td::uint32 k, td::uint32 a,
td::int32 our_network_id, DhtNode self, bool client_only, td::actor::ActorId<DhtMember> node,
td::actor::ActorId<adnl::Adnl> adnl, td::Promise<td::Unit> promise)
: print_id_(print_id)
, k_(k)
, a_(a)
, our_network_id_(our_network_id)
, promise_(std::move(promise))
, key_id_(key_id)
, list_(std::move(list))
, self_(std::move(self))
, client_only_(client_only) {
node_ = node;
adnl_ = adnl;
src_ = src;
query_ = create_serialize_tl_object<ton_api::dht_registerReverseConnection>(client.tl(), ttl, std::move(signature));
}
void DhtQueryRegisterReverseConnection::start_up() {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<DhtNodesList> res) {
td::actor::send_closure(SelfId, &DhtQueryRegisterReverseConnection::send_queries, std::move(res));
});
auto A = td::actor::create_actor<DhtQueryFindNodes>("FindNodesQuery", key_id_, print_id_, src_, std::move(list_), k_,
a_, our_network_id_, self_.clone(), client_only_, node_, adnl_,
std::move(P));
A.release();
}
void DhtQueryRegisterReverseConnection::send_queries(td::Result<DhtNodesList> R) {
if (R.is_error()) {
auto S = R.move_as_error();
VLOG(DHT_NOTICE) << this << ": failed to get nearest nodes to " << key_id_ << ": " << S;
promise_.set_error(std::move(S));
stop();
return;
}
auto list = R.move_as_ok();
remaining_ = static_cast<td::uint32>(list.size());
if (remaining_ == 0) {
VLOG(DHT_NOTICE) << this << ": failed to get nearest nodes to " << key_id_ << ": no nodes";
promise_.set_error(td::Status::Error("no dht nodes"));
stop();
return;
}
for (auto &node : list.list()) {
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
td::actor::send_closure(SelfId, &DhtQueryRegisterReverseConnection::ready, std::move(R));
});
td::actor::send_closure(adnl_, &adnl::Adnl::send_query, src_, node.adnl_id().compute_short_id(), "dht regrevcon",
std::move(P), td::Timestamp::in(2.0 + td::Random::fast(0, 20) * 0.1), query_.clone());
}
}
void DhtQueryRegisterReverseConnection::ready(td::Result<td::BufferSlice> R) {
if (R.is_error()) {
fail_++;
VLOG(DHT_INFO) << this << ": failed register reverse connection query: " << R.move_as_error();
} else {
auto R2 = fetch_tl_object<ton_api::dht_stored>(R.move_as_ok(), true);
if (R2.is_error()) {
fail_++;
VLOG(DHT_WARNING) << this << ": can not parse answer (expected dht.stored): " << R2.move_as_error();
} else {
success_++;
}
}
CHECK(remaining_ > 0);
remaining_--;
if (remaining_ == 0) {
if (success_ > 0) {
promise_.set_value(td::Unit());
} else {
promise_.set_result(td::Status::Error("failed to make actual query"));
}
stop();
}
}
void DhtQueryRequestReversePing::send_one_query(adnl::AdnlNodeIdShort id) {
td::BufferSlice B;
if (client_only_) {
B = query_.clone();
} else {
B = create_serialize_tl_object_suffix<ton_api::dht_query>(query_.as_slice(), self_.tl());
}
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), dst = id](td::Result<td::BufferSlice> R) {
td::actor::send_closure(SelfId, &DhtQueryRequestReversePing::on_result, std::move(R), dst);
});
td::actor::send_closure(adnl_, &adnl::Adnl::send_query, get_src(), id, "dht requestReversePing", std::move(P),
td::Timestamp::in(2.0 + td::Random::fast(0, 20) * 0.1), std::move(B));
}
void DhtQueryRequestReversePing::on_result(td::Result<td::BufferSlice> R, adnl::AdnlNodeIdShort dst) {
if (R.is_error()) {
VLOG(DHT_INFO) << this << ": failed reverse ping query " << get_src() << "->" << dst << ": " << R.move_as_error();
finish_query();
return;
}
auto Res = fetch_tl_object<ton_api::dht_ReversePingResult>(R.move_as_ok(), true);
if (Res.is_error()) {
VLOG(DHT_WARNING) << this << ": dropping incorrect answer on dht.requestReversePing query from " << dst << ": "
<< Res.move_as_error();
finish_query();
return;
}
auto A = Res.move_as_ok();
ton_api::downcast_call(*A, td::overloaded(
[&](ton_api::dht_reversePingOk &v) {
promise_.set_value(td::Unit());
stop();
},
[&](ton_api::dht_clientNotFound &v) {
add_nodes(DhtNodesList{std::move(v.nodes_), our_network_id()});
finish_query();
}));
}
void DhtQueryRequestReversePing::finish(DhtNodesList list) {
promise_.set_error(td::Status::Error(ErrorCode::notready, "dht key not found"));
}
} // namespace dht
} // namespace ton

84
dht/dht-query.hpp
View file

@ -45,7 +45,7 @@ class DhtQuery : public td::actor::Actor {
public:
DhtQuery(DhtKeyId key, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, DhtNodesList list, td::uint32 k,
td::uint32 a, DhtNode self, bool client_only, td::actor::ActorId<DhtMember> node,
td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only, td::actor::ActorId<DhtMember> node,
td::actor::ActorId<adnl::Adnl> adnl)
: key_(key)
, self_(std::move(self))
@ -54,6 +54,7 @@ class DhtQuery : public td::actor::Actor {
, src_(src)
, k_(k)
, a_(a)
, our_network_id_(our_network_id)
, node_(node)
, adnl_(adnl) {
add_nodes(std::move(list));
@ -77,6 +78,9 @@ class DhtQuery : public td::actor::Actor {
td::uint32 get_k() const {
return k_;
}
td::int32 our_network_id() const {
return our_network_id_;
}
void start_up() override {
send_queries();
}
@ -91,6 +95,7 @@ class DhtQuery : public td::actor::Actor {
std::set<DhtKeyId> pending_ids_;
td::uint32 k_;
td::uint32 a_;
td::int32 our_network_id_;
td::actor::ActorId<DhtMember> node_;
td::uint32 active_queries_ = 0;
@ -104,9 +109,10 @@ class DhtQueryFindNodes : public DhtQuery {
public:
DhtQueryFindNodes(DhtKeyId key, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, DhtNodesList list,
td::uint32 k, td::uint32 a, DhtNode self, bool client_only, td::actor::ActorId<DhtMember> node,
td::actor::ActorId<adnl::Adnl> adnl, td::Promise<DhtNodesList> promise)
: DhtQuery(key, print_id, src, std::move(list), k, a, std::move(self), client_only, node, adnl)
td::uint32 k, td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only,
td::actor::ActorId<DhtMember> node, td::actor::ActorId<adnl::Adnl> adnl,
td::Promise<DhtNodesList> promise)
: DhtQuery(key, print_id, src, std::move(list), k, a, our_network_id, std::move(self), client_only, node, adnl)
, promise_(std::move(promise)) {
}
void send_one_query(adnl::AdnlNodeIdShort id) override;
@ -123,13 +129,16 @@ class DhtQueryFindValue : public DhtQuery {
public:
DhtQueryFindValue(DhtKeyId key, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, DhtNodesList list,
td::uint32 k, td::uint32 a, DhtNode self, bool client_only, td::actor::ActorId<DhtMember> node,
td::actor::ActorId<adnl::Adnl> adnl, td::Promise<DhtValue> promise)
: DhtQuery(key, print_id, src, std::move(list), k, a, std::move(self), client_only, node, adnl)
td::uint32 k, td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only,
td::actor::ActorId<DhtMember> node, td::actor::ActorId<adnl::Adnl> adnl,
td::Promise<DhtValue> promise)
: DhtQuery(key, print_id, src, std::move(list), k, a, our_network_id, std::move(self), client_only, node, adnl)
, promise_(std::move(promise)) {
}
void send_one_query(adnl::AdnlNodeIdShort id) override;
void send_one_query_nodes(adnl::AdnlNodeIdShort id);
void on_result(td::Result<td::BufferSlice> R, adnl::AdnlNodeIdShort dst);
void on_result_nodes(td::Result<td::BufferSlice> R, adnl::AdnlNodeIdShort dst);
void finish(DhtNodesList list) override;
std::string get_name() const override {
return "find value";
@ -141,6 +150,7 @@ class DhtQueryStore : public td::actor::Actor {
DhtMember::PrintId print_id_;
td::uint32 k_;
td::uint32 a_;
td::int32 our_network_id_;
td::Promise<td::Unit> promise_;
td::actor::ActorId<DhtMember> node_;
td::actor::ActorId<adnl::Adnl> adnl_;
@ -155,7 +165,7 @@ class DhtQueryStore : public td::actor::Actor {
public:
DhtQueryStore(DhtValue key_value, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, DhtNodesList list,
td::uint32 k, td::uint32 a, DhtNode self, bool client_only, td::actor::ActorId<DhtMember> node,
td::uint32 k, td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only, td::actor::ActorId<DhtMember> node,
td::actor::ActorId<adnl::Adnl> adnl, td::Promise<td::Unit> promise);
void send_stores(td::Result<DhtNodesList> res);
void store_ready(td::Result<td::BufferSlice> res);
@ -165,6 +175,64 @@ class DhtQueryStore : public td::actor::Actor {
}
};
class DhtQueryRegisterReverseConnection : public td::actor::Actor {
private:
DhtMember::PrintId print_id_;
td::uint32 k_;
td::uint32 a_;
td::int32 our_network_id_;
td::Promise<td::Unit> promise_;
td::actor::ActorId<DhtMember> node_;
td::actor::ActorId<adnl::Adnl> adnl_;
adnl::AdnlNodeIdShort src_;
DhtKeyId key_id_;
td::BufferSlice query_;
td::uint32 success_ = 0;
td::uint32 fail_ = 0;
td::uint32 remaining_;
DhtNodesList list_;
DhtNode self_;
bool client_only_;
public:
DhtQueryRegisterReverseConnection(DhtKeyId key_id, adnl::AdnlNodeIdFull client, td::uint32 ttl,
td::BufferSlice signature, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src,
DhtNodesList list, td::uint32 k, td::uint32 a, td::int32 our_network_id,
DhtNode self, bool client_only, td::actor::ActorId<DhtMember> node,
td::actor::ActorId<adnl::Adnl> adnl, td::Promise<td::Unit> promise);
void send_queries(td::Result<DhtNodesList> R);
void ready(td::Result<td::BufferSlice> R);
void start_up() override;
DhtMember::PrintId print_id() const {
return print_id_;
}
};
class DhtQueryRequestReversePing : public DhtQuery {
private:
td::Promise<td::Unit> promise_;
td::BufferSlice query_;
public:
DhtQueryRequestReversePing(adnl::AdnlNodeIdShort client, adnl::AdnlNode target, td::BufferSlice signature,
DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, DhtNodesList list, td::uint32 k,
td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only,
td::actor::ActorId<DhtMember> node, td::actor::ActorId<adnl::Adnl> adnl,
td::Promise<td::Unit> promise)
: DhtQuery(DhtMember::get_reverse_connection_key(client).compute_key_id(), print_id, src, std::move(list), k, a,
our_network_id, std::move(self), client_only, node, adnl)
, promise_(std::move(promise))
, query_(create_serialize_tl_object<ton_api::dht_requestReversePing>(target.tl(), std::move(signature),
client.bits256_value(), k)) {
}
void send_one_query(adnl::AdnlNodeIdShort id) override;
void on_result(td::Result<td::BufferSlice> R, adnl::AdnlNodeIdShort dst);
void finish(DhtNodesList list) override;
std::string get_name() const override {
return "request remote ping";
}
};
inline td::StringBuilder &operator<<(td::StringBuilder &sb, const DhtQuery &dht) {
sb << dht.print_id();
return sb;

56
dht/dht-remote-node.cpp
View file

@ -20,13 +20,10 @@
#include "td/utils/tl_storers.h"
#include "td/utils/crypto.h"
#include "td/utils/tl_parsers.h"
#include "td/utils/Random.h"
#include "td/utils/format.h"
#include "keys/encryptor.h"
#include "auto/tl/ton_api.hpp"
#include "dht-remote-node.hpp"
@ -35,28 +32,43 @@ namespace ton {
namespace dht {
static const double PING_INTERVAL_DEFAULT = 60.0;
static const double PING_INTERVAL_MULTIPLIER = 1.1;
static const double PING_INTERVAL_MAX = 3600.0 * 4;
DhtRemoteNode::DhtRemoteNode(DhtNode node, td::uint32 max_missed_pings, td::int32 our_network_id)
: node_(std::move(node))
, max_missed_pings_(max_missed_pings)
, our_network_id_(our_network_id)
, ping_interval_(PING_INTERVAL_DEFAULT) {
failed_from_ = td::Time::now_cached();
id_ = node_.get_key();
}
td::Status DhtRemoteNode::receive_ping(DhtNode node, td::actor::ActorId<adnl::Adnl> adnl,
adnl::AdnlNodeIdShort self_id) {
TRY_STATUS(update_value(std::move(node), adnl, self_id));
receive_ping();
return td::Status::OK();
}
void DhtRemoteNode::receive_ping() {
missed_pings_ = 0;
ping_interval_ = PING_INTERVAL_DEFAULT;
if (ready_from_ == 0) {
ready_from_ = td::Time::now_cached();
}
return td::Status::OK();
}
td::Status DhtRemoteNode::update_value(DhtNode node, td::actor::ActorId<adnl::Adnl> adnl,
adnl::AdnlNodeIdShort self_id) {
CHECK(node.adnl_id() == node_.adnl_id());
if (node.adnl_id() != node_.adnl_id()) {
return td::Status::Error("Wrong adnl id");
}
if (node.version() <= node_.version()) {
return td::Status::OK();
}
TRY_RESULT(enc, node.adnl_id().pubkey().create_encryptor());
auto tl = node.tl();
auto sig = std::move(tl->signature_);
TRY_STATUS_PREFIX(enc->check_signature(serialize_tl_object(tl, true).as_slice(), sig.as_slice()),
"bad node signature: ");
TRY_STATUS(node.check_signature());
node_ = std::move(node);
td::actor::send_closure(adnl, &adnl::Adnl::add_peer, self_id, node_.adnl_id(), node_.addr_list());
@ -66,22 +78,25 @@ td::Status DhtRemoteNode::update_value(DhtNode node, td::actor::ActorId<adnl::Ad
void DhtRemoteNode::send_ping(bool client_only, td::actor::ActorId<adnl::Adnl> adnl, td::actor::ActorId<DhtMember> node,
adnl::AdnlNodeIdShort src) {
missed_pings_++;
if (missed_pings_ > max_missed_pings_ && ready_from_ > 0) {
ready_from_ = 0;
failed_from_ = td::Time::now_cached();
if (missed_pings_ > max_missed_pings_) {
ping_interval_ = std::min(ping_interval_ * PING_INTERVAL_MULTIPLIER, PING_INTERVAL_MAX);
if (ready_from_ > 0) {
ready_from_ = 0;
failed_from_ = td::Time::now_cached();
}
}
last_ping_at_ = td::Time::now_cached();
td::actor::send_closure(adnl, &adnl::Adnl::add_peer, src, node_.adnl_id(), node_.addr_list());
auto P = td::PromiseCreator::lambda([key = id_, id = node_.adnl_id().compute_short_id(), client_only, node, src,
adnl](td::Result<DhtNode> R) mutable {
auto P = td::PromiseCreator::lambda([key = id_, id = node_.adnl_id().compute_short_id(), client_only, node, src, adnl,
our_network_id = our_network_id_](td::Result<DhtNode> R) mutable {
if (R.is_error()) {
LOG(ERROR) << "[dht]: failed to get self node";
return;
}
auto P = td::PromiseCreator::lambda([key, node, adnl](td::Result<td::BufferSlice> R) {
auto P = td::PromiseCreator::lambda([key, node, adnl, our_network_id](td::Result<td::BufferSlice> R) {
if (R.is_error()) {
VLOG(DHT_INFO) << "[dht]: received error for query to " << key << ": " << R.move_as_error();
return;
@ -89,7 +104,7 @@ void DhtRemoteNode::send_ping(bool client_only, td::actor::ActorId<adnl::Adnl> a
auto F = fetch_tl_object<ton_api::dht_node>(R.move_as_ok(), true);
if (F.is_ok()) {
auto N = DhtNode::create(F.move_as_ok());
auto N = DhtNode::create(F.move_as_ok(), our_network_id);
if (N.is_ok()) {
td::actor::send_closure(node, &DhtMember::receive_ping, key, N.move_as_ok());
} else {
@ -123,7 +138,8 @@ adnl::AdnlNodeIdFull DhtRemoteNode::get_full_id() const {
return node_.adnl_id();
}
td::Result<std::unique_ptr<DhtRemoteNode>> DhtRemoteNode::create(DhtNode node, td::uint32 max_missed_pings) {
td::Result<std::unique_ptr<DhtRemoteNode>> DhtRemoteNode::create(DhtNode node, td::uint32 max_missed_pings,
td::int32 our_network_id) {
TRY_RESULT(enc, node.adnl_id().pubkey().create_encryptor());
auto tl = node.tl();
auto sig = std::move(tl->signature_);
@ -131,7 +147,7 @@ td::Result<std::unique_ptr<DhtRemoteNode>> DhtRemoteNode::create(DhtNode node, t
TRY_STATUS_PREFIX(enc->check_signature(serialize_tl_object(tl, true).as_slice(), sig.as_slice()),
"bad node signature: ");
return std::make_unique<DhtRemoteNode>(std::move(node), max_missed_pings);
return std::make_unique<DhtRemoteNode>(std::move(node), max_missed_pings, our_network_id);
}
} // namespace dht

15
dht/dht-remote-node.hpp
View file

@ -40,19 +40,18 @@ class DhtRemoteNode {
DhtNode node_;
td::uint32 max_missed_pings_;
td::int32 our_network_id_;
td::uint32 missed_pings_ = 0;
double last_ping_at_ = 0;
double ready_from_ = 0;
double failed_from_ = 0;
double ping_interval_;
td::int32 version_;
public:
DhtRemoteNode(DhtNode node, td::uint32 max_missed_pings)
: node_(std::move(node)), max_missed_pings_(max_missed_pings) {
failed_from_ = td::Time::now_cached();
id_ = node_.get_key();
}
static td::Result<std::unique_ptr<DhtRemoteNode>> create(DhtNode node, td::uint32 max_missed_pings);
DhtRemoteNode(DhtNode node, td::uint32 max_missed_pings, td::int32 our_network_id);
static td::Result<std::unique_ptr<DhtRemoteNode>> create(DhtNode node, td::uint32 max_missed_pings,
td::int32 our_network_id);
DhtNode get_node() const {
return node_.clone();
}
@ -76,9 +75,13 @@ class DhtRemoteNode {
double last_ping_at() const {
return last_ping_at_;
}
double ping_interval() const {
return ping_interval_;
}
void send_ping(bool client_only, td::actor::ActorId<adnl::Adnl> adnl, td::actor::ActorId<DhtMember> node,
adnl::AdnlNodeIdShort src);
td::Status receive_ping(DhtNode node, td::actor::ActorId<adnl::Adnl> adnl, adnl::AdnlNodeIdShort self_id);
void receive_ping();
td::Status update_value(DhtNode node, td::actor::ActorId<adnl::Adnl> adnl, adnl::AdnlNodeIdShort self_id);
};

19
dht/dht-types.cpp
View file

@ -209,6 +209,10 @@ td::Status DhtValue::check() const {
return key_.update_rule()->check_value(*this);
}
bool DhtValue::check_is_acceptable() const {
return key_.update_rule()->check_is_acceptable(*this);
}
DhtKeyId DhtValue::key_id() const {
return key_.key().compute_key_id();
}
@ -360,6 +364,21 @@ td::Status DhtUpdateRuleOverlayNodes::update_value(DhtValue &value, DhtValue &&n
return td::Status::OK();
}
bool DhtUpdateRuleOverlayNodes::check_is_acceptable(const ton::dht::DhtValue &value) {
auto F = fetch_tl_object<ton_api::overlay_nodes>(value.value().clone_as_buffer_slice(), true);
if (F.is_error()) {
return false;
}
auto L = F.move_as_ok();
auto now = td::Clocks::system();
for (auto &node : L->nodes_) {
if (node->version_ + 600 > now) {
return true;
}
}
return false;
}
tl_object_ptr<ton_api::dht_UpdateRule> DhtUpdateRuleOverlayNodes::tl() const {
return create_tl_object<ton_api::dht_updateRule_overlayNodes>();
}

5
dht/dht-types.h
View file

@ -119,6 +119,9 @@ class DhtUpdateRule {
virtual td::Status check_value(const DhtValue &value) = 0;
virtual td::Status update_value(DhtValue &value, DhtValue &&new_value) = 0;
virtual bool need_republish() const = 0;
virtual bool check_is_acceptable(const DhtValue &value) {
return true;
}
virtual tl_object_ptr<ton_api::dht_UpdateRule> tl() const = 0;
static td::Result<std::shared_ptr<DhtUpdateRule>> create(tl_object_ptr<ton_api::dht_UpdateRule> obj);
};
@ -210,6 +213,7 @@ class DhtValue {
void update_signature(td::BufferSlice signature);
void update_signature(td::SharedSlice signature);
td::Status check() const;
bool check_is_acceptable() const;
DhtKeyId key_id() const;
@ -249,6 +253,7 @@ class DhtUpdateRuleOverlayNodes : public DhtUpdateRule {
bool need_republish() const override {
return false;
}
bool check_is_acceptable(const DhtValue &value) override;
tl_object_ptr<ton_api::dht_UpdateRule> tl() const override;
static td::Result<std::shared_ptr<DhtUpdateRule>> create();
};

305
dht/dht.cpp
View file

@ -20,7 +20,6 @@
#include "td/utils/tl_storers.h"
#include "td/utils/crypto.h"
#include "td/utils/tl_parsers.h"
#include "td/utils/Random.h"
#include "td/utils/base64.h"
@ -28,9 +27,6 @@
#include "td/db/RocksDb.h"
#include "keys/encryptor.h"
#include "adnl/utils.hpp"
#include "auto/tl/ton_api.hpp"
#include "dht.h"
@ -44,10 +40,9 @@ namespace dht {
td::actor::ActorOwn<DhtMember> DhtMember::create(adnl::AdnlNodeIdShort id, std::string db_root,
td::actor::ActorId<keyring::Keyring> keyring,
td::actor::ActorId<adnl::Adnl> adnl, td::uint32 k, td::uint32 a,
bool client_only) {
return td::actor::ActorOwn<DhtMember>(
td::actor::create_actor<DhtMemberImpl>("dht", id, db_root, keyring, adnl, k, a, client_only));
td::actor::ActorId<adnl::Adnl> adnl, td::int32 network_id,
td::uint32 k, td::uint32 a, bool client_only) {
return td::actor::create_actor<DhtMemberImpl>("dht", id, db_root, keyring, adnl, network_id, k, a, client_only);
}
td::Result<td::actor::ActorOwn<Dht>> Dht::create(adnl::AdnlNodeIdShort id, std::string db_root,
@ -57,7 +52,7 @@ td::Result<td::actor::ActorOwn<Dht>> Dht::create(adnl::AdnlNodeIdShort id, std::
CHECK(conf->get_k() > 0);
CHECK(conf->get_a() > 0);
auto D = DhtMember::create(id, db_root, keyring, adnl, conf->get_k(), conf->get_a());
auto D = DhtMember::create(id, db_root, keyring, adnl, conf->get_network_id(), conf->get_k(), conf->get_a());
auto &nodes = conf->nodes();
for (auto &node : nodes.list()) {
@ -74,7 +69,7 @@ td::Result<td::actor::ActorOwn<Dht>> Dht::create_client(adnl::AdnlNodeIdShort id
CHECK(conf->get_k() > 0);
CHECK(conf->get_a() > 0);
auto D = DhtMember::create(id, db_root, keyring, adnl, conf->get_k(), conf->get_a(), true);
auto D = DhtMember::create(id, db_root, keyring, adnl, conf->get_network_id(), conf->get_k(), conf->get_a(), true);
auto &nodes = conf->nodes();
for (auto &node : nodes.list()) {
@ -90,8 +85,11 @@ void DhtMemberImpl::start_up() {
ton_api::dht_findValue::ID,
ton_api::dht_store::ID,
ton_api::dht_ping::ID,
ton_api::dht_registerReverseConnection::ID,
ton_api::dht_requestReversePing::ID,
ton_api::dht_query::ID,
ton_api::dht_message::ID};
ton_api::dht_message::ID,
ton_api::dht_requestReversePingCont::ID};
for (auto it : methods) {
td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, id_, adnl::Adnl::int_to_bytestring(it),
@ -112,12 +110,12 @@ void DhtMemberImpl::start_up() {
V.ensure();
auto nodes = std::move(V.move_as_ok()->nodes_);
auto s = nodes->nodes_.size();
DhtNodesList list{std::move(nodes)};
CHECK(list.size() == s);
DhtNodesList list{std::move(nodes), network_id_};
CHECK(list.size() <= s); // Some nodes can be dropped due to a wrong network id
auto &B = buckets_[bit];
for (auto &node : list.list()) {
auto key = node.get_key();
B.add_full_node(key, std::move(node), adnl_, id_);
B.add_full_node(key, std::move(node), adnl_, id_, network_id_);
}
}
}
@ -131,8 +129,11 @@ void DhtMemberImpl::tear_down() {
ton_api::dht_findValue::ID,
ton_api::dht_store::ID,
ton_api::dht_ping::ID,
ton_api::dht_registerReverseConnection::ID,
ton_api::dht_requestReversePing::ID,
ton_api::dht_query::ID,
ton_api::dht_message::ID};
ton_api::dht_message::ID,
ton_api::dht_requestReversePingCont::ID};
for (auto it : methods) {
td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, id_, adnl::Adnl::int_to_bytestring(it));
@ -299,6 +300,61 @@ void DhtMemberImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::dht_getSig
get_self_node(std::move(P));
}
static td::BufferSlice register_reverse_connection_to_sign(adnl::AdnlNodeIdShort client, adnl::AdnlNodeIdShort dht_id,
td::uint32 ttl) {
td::BufferSlice result(32 + 32 + 4);
td::MutableSlice s = result.as_slice();
s.copy_from(client.as_slice());
s.remove_prefix(32);
s.copy_from(dht_id.as_slice());
s.remove_prefix(32);
s.copy_from(std::string(reinterpret_cast<char *>(&ttl), 4));
return result;
}
void DhtMemberImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::dht_registerReverseConnection &query,
td::Promise<td::BufferSlice> promise) {
td::uint32 ttl = query.ttl_, now = (td::uint32)td::Clocks::system();
if (ttl <= now) {
return;
}
PublicKey pub{query.node_};
adnl::AdnlNodeIdShort client_id{pub.compute_short_id()};
td::BufferSlice to_sign = register_reverse_connection_to_sign(client_id, src, ttl);
TRY_RESULT_PROMISE(promise, encryptor, pub.create_encryptor());
TRY_STATUS_PROMISE(promise, encryptor->check_signature(to_sign, query.signature_));
DhtKeyId key_id = get_reverse_connection_key(client_id).compute_key_id();
reverse_connections_[client_id] = ReverseConnection{src, key_id, td::Timestamp::at_unix(std::min(ttl, now + 300))};
promise.set_value(create_serialize_tl_object<ton_api::dht_stored>());
}
void DhtMemberImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::dht_requestReversePing &query,
td::Promise<td::BufferSlice> promise) {
adnl::AdnlNodeIdShort client{query.client_};
auto it = reverse_connections_.find(client);
if (it != reverse_connections_.end()) {
if (it->second.ttl_.is_in_past()) {
reverse_connections_.erase(it);
} else {
PublicKey pub{query.target_->id_};
TRY_RESULT_PROMISE(promise, encryptor, pub.create_encryptor());
TRY_STATUS_PROMISE(promise,
encryptor->check_signature(serialize_tl_object(query.target_, true), query.signature_));
td::actor::send_closure(adnl_, &adnl::Adnl::send_message, id_, it->second.dht_node_,
create_serialize_tl_object<ton_api::dht_requestReversePingCont>(
std::move(query.target_), std::move(query.signature_), query.client_));
promise.set_result(create_serialize_tl_object<ton_api::dht_reversePingOk>());
return;
}
}
auto k = static_cast<td::uint32>(query.k_);
if (k > max_k()) {
k = max_k();
}
auto R = get_nearest_nodes(get_reverse_connection_key(client).compute_key_id(), k);
promise.set_value(create_serialize_tl_object<ton_api::dht_clientNotFound>(R.tl()));
}
void DhtMemberImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data,
td::Promise<td::BufferSlice> promise) {
if (client_only_) {
@ -307,11 +363,15 @@ void DhtMemberImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice dat
{
auto R = fetch_tl_prefix<ton_api::dht_query>(data, true);
if (R.is_ok()) {
auto N = DhtNode::create(std::move(R.move_as_ok()->node_));
auto N = DhtNode::create(std::move(R.move_as_ok()->node_), network_id_);
if (N.is_ok()) {
auto node = N.move_as_ok();
auto key = node.get_key();
add_full_node(key, std::move(node));
if (node.adnl_id().compute_short_id() == src) {
auto key = node.get_key();
add_full_node_impl(key, std::move(node), true);
} else {
VLOG(DHT_WARNING) << this << ": dropping bad node: unexpected adnl id";
}
} else {
VLOG(DHT_WARNING) << this << ": dropping bad node " << N.move_as_error();
}
@ -335,10 +395,10 @@ void DhtMemberImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice dat
VLOG(DHT_EXTRA_DEBUG) << this << ": query to DHT from " << src << ": " << ton_api::to_string(Q);
ton_api::downcast_call(*Q.get(), [&](auto &object) { this->process_query(src, object, std::move(promise)); });
ton_api::downcast_call(*Q, [&](auto &object) { this->process_query(src, object, std::move(promise)); });
}
void DhtMemberImpl::add_full_node(DhtKeyId key, DhtNode node) {
void DhtMemberImpl::add_full_node_impl(DhtKeyId key, DhtNode node, bool set_active) {
VLOG(DHT_EXTRA_DEBUG) << this << ": adding full node " << key;
auto eid = key ^ key_;
@ -350,7 +410,7 @@ void DhtMemberImpl::add_full_node(DhtKeyId key, DhtNode node) {
#endif
if (bit < 256) {
CHECK(key.get_bit(bit) != key_.get_bit(bit));
buckets_[bit].add_full_node(key, std::move(node), adnl_, id_);
buckets_[bit].add_full_node(key, std::move(node), adnl_, id_, network_id_, set_active);
} else {
CHECK(key == key_);
}
@ -369,6 +429,27 @@ void DhtMemberImpl::receive_ping(DhtKeyId key, DhtNode result) {
}
void DhtMemberImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) {
auto F = fetch_tl_object<ton_api::dht_requestReversePingCont>(data, true);
if (F.is_ok()) {
auto S = [&]() -> td::Status {
auto f = F.move_as_ok();
adnl::AdnlNodeIdShort client{f->client_};
if (!our_reverse_connections_.count(client)) {
return td::Status::Error(PSTRING() << ": unknown id for reverse ping: " << client);
}
TRY_RESULT_PREFIX(node, adnl::AdnlNode::create(f->target_), "failed to parse node: ");
TRY_RESULT_PREFIX(encryptor, node.pub_id().pubkey().create_encryptor(), "failed to create encryptor: ");
TRY_STATUS_PREFIX(encryptor->check_signature(serialize_tl_object(f->target_, true), f->signature_),
"invalid signature: ");
VLOG(DHT_INFO) << this << ": sending reverse ping to " << node.compute_short_id();
td::actor::send_closure(adnl_, &adnl::Adnl::add_peer, client, node.pub_id(), node.addr_list());
td::actor::send_closure(adnl_, &adnl::Adnl::send_message, client, node.compute_short_id(), td::BufferSlice());
return td::Status::OK();
}();
if (S.is_error()) {
VLOG(DHT_INFO) << this << ": " << S;
}
}
}
void DhtMemberImpl::set_value(DhtValue value, td::Promise<td::Unit> promise) {
@ -385,10 +466,11 @@ void DhtMemberImpl::set_value(DhtValue value, td::Promise<td::Unit> promise) {
void DhtMemberImpl::get_value_in(DhtKeyId key, td::Promise<DhtValue> result) {
auto P = td::PromiseCreator::lambda([key, promise = std::move(result), SelfId = actor_id(this), print_id = print_id(),
adnl = adnl_, list = get_nearest_nodes(key, k_), k = k_, a = a_, id = id_,
adnl = adnl_, list = get_nearest_nodes(key, k_), k = k_, a = a_,
network_id = network_id_, id = id_,
client_only = client_only_](td::Result<DhtNode> R) mutable {
R.ensure();
td::actor::create_actor<DhtQueryFindValue>("FindValueQuery", key, print_id, id, std::move(list), k, a,
td::actor::create_actor<DhtQueryFindValue>("FindValueQuery", key, print_id, id, std::move(list), k, a, network_id,
R.move_as_ok(), client_only, SelfId, adnl, std::move(promise))
.release();
});
@ -396,6 +478,70 @@ void DhtMemberImpl::get_value_in(DhtKeyId key, td::Promise<DhtValue> result) {
get_self_node(std::move(P));
}
void DhtMemberImpl::register_reverse_connection(adnl::AdnlNodeIdFull client, td::Promise<td::Unit> promise) {
auto client_short = client.compute_short_id();
td::uint32 ttl = (td::uint32)td::Clocks::system() + 300;
our_reverse_connections_.insert(client_short);
auto key_id = get_reverse_connection_key(client_short).compute_key_id();
td::actor::send_closure(keyring_, &keyring::Keyring::sign_message, client_short.pubkey_hash(),
register_reverse_connection_to_sign(client_short, id_, ttl),
[=, print_id = print_id(), list = get_nearest_nodes(key_id, k_), SelfId = actor_id(this),
promise = std::move(promise)](td::Result<td::BufferSlice> R) mutable {
TRY_RESULT_PROMISE_PREFIX(promise, signature, std::move(R), "Failed to sign: ");
td::actor::send_closure(SelfId, &DhtMemberImpl::get_self_node,
[=, list = std::move(list), signature = std::move(signature),
promise = std::move(promise)](td::Result<DhtNode> R) mutable {
R.ensure();
td::actor::create_actor<DhtQueryRegisterReverseConnection>(
"RegisterReverseQuery", key_id, std::move(client), ttl,
std::move(signature), print_id, id_, std::move(list), k_, a_,
network_id_, R.move_as_ok(), client_only_, SelfId, adnl_,
std::move(promise))
.release();
});
});
}
void DhtMemberImpl::request_reverse_ping(adnl::AdnlNode target, adnl::AdnlNodeIdShort client,
td::Promise<td::Unit> promise) {
auto pubkey_hash = target.compute_short_id().pubkey_hash();
td::BufferSlice to_sign = serialize_tl_object(target.tl(), true);
td::actor::send_closure(keyring_, &keyring::Keyring::sign_message, pubkey_hash, std::move(to_sign),
[SelfId = actor_id(this), promise = std::move(promise), target = std::move(target),
client](td::Result<td::BufferSlice> R) mutable {
TRY_RESULT_PROMISE(promise, signature, std::move(R));
td::actor::send_closure(SelfId, &DhtMemberImpl::request_reverse_ping_cont,
std::move(target), std::move(signature), client,
std::move(promise));
});
}
void DhtMemberImpl::request_reverse_ping_cont(adnl::AdnlNode target, td::BufferSlice signature,
adnl::AdnlNodeIdShort client, td::Promise<td::Unit> promise) {
auto it = reverse_connections_.find(client);
if (it != reverse_connections_.end()) {
if (it->second.ttl_.is_in_past()) {
reverse_connections_.erase(it);
} else {
td::actor::send_closure(adnl_, &adnl::Adnl::send_message, id_, it->second.dht_node_,
create_serialize_tl_object<ton_api::dht_requestReversePingCont>(
target.tl(), std::move(signature), client.bits256_value()));
promise.set_result(td::Unit());
return;
}
}
auto key_id = get_reverse_connection_key(client).compute_key_id();
get_self_node([=, target = std::move(target), signature = std::move(signature), promise = std::move(promise),
SelfId = actor_id(this), print_id = print_id(), list = get_nearest_nodes(key_id, k_),
client_only = client_only_](td::Result<DhtNode> R) mutable {
R.ensure();
td::actor::create_actor<DhtQueryRequestReversePing>(
"RequestReversePing", client, std::move(target), std::move(signature), print_id, id_, std::move(list), k_, a_,
network_id_, R.move_as_ok(), client_only, SelfId, adnl_, std::move(promise))
.release();
});
}
void DhtMemberImpl::check() {
VLOG(DHT_INFO) << this << ": ping=" << ping_queries_ << " fnode=" << find_node_queries_
<< " fvalue=" << find_value_queries_ << " store=" << store_queries_
@ -454,6 +600,16 @@ void DhtMemberImpl::check() {
}
}
}
if (reverse_connections_.size() > 0) {
auto it = reverse_connections_.upper_bound(last_check_reverse_conn_);
if (it == reverse_connections_.end()) {
it = reverse_connections_.begin();
}
last_check_reverse_conn_ = it->first;
if (it->second.ttl_.is_in_past()) {
reverse_connections_.erase(it);
}
}
if (republish_att_.is_in_past()) {
auto it = our_values_.lower_bound(last_republish_key_);
@ -496,9 +652,10 @@ void DhtMemberImpl::check() {
DhtKeyId key{x};
auto P = td::PromiseCreator::lambda([key, promise = std::move(promise), SelfId = actor_id(this),
print_id = print_id(), adnl = adnl_, list = get_nearest_nodes(key, k_), k = k_,
a = a_, id = id_, client_only = client_only_](td::Result<DhtNode> R) mutable {
a = a_, network_id = network_id_, id = id_,
client_only = client_only_](td::Result<DhtNode> R) mutable {
R.ensure();
td::actor::create_actor<DhtQueryFindNodes>("FindNodesQuery", key, print_id, id, std::move(list), k, a,
td::actor::create_actor<DhtQueryFindNodes>("FindNodesQuery", key, print_id, id, std::move(list), k, a, network_id,
R.move_as_ok(), client_only, SelfId, adnl, std::move(promise))
.release();
});
@ -519,63 +676,71 @@ void DhtMemberImpl::send_store(DhtValue value, td::Promise<td::Unit> promise) {
value.check().ensure();
auto key_id = value.key_id();
auto P =
td::PromiseCreator::lambda([value = std::move(value), print_id = print_id(), id = id_, client_only = client_only_,
list = get_nearest_nodes(key_id, k_), k = k_, a = a_, SelfId = actor_id(this),
adnl = adnl_, promise = std::move(promise)](td::Result<DhtNode> R) mutable {
R.ensure();
td::actor::create_actor<DhtQueryStore>("StoreQuery", std::move(value), print_id, id, std::move(list), k, a,
R.move_as_ok(), client_only, SelfId, adnl, std::move(promise))
.release();
});
auto P = td::PromiseCreator::lambda([value = std::move(value), print_id = print_id(), id = id_,
client_only = client_only_, list = get_nearest_nodes(key_id, k_), k = k_, a = a_,
network_id = network_id_, SelfId = actor_id(this), adnl = adnl_,
promise = std::move(promise)](td::Result<DhtNode> R) mutable {
R.ensure();
td::actor::create_actor<DhtQueryStore>("StoreQuery", std::move(value), print_id, id, std::move(list), k, a,
network_id, R.move_as_ok(), client_only, SelfId, adnl, std::move(promise))
.release();
});
get_self_node(std::move(P));
}
void DhtMemberImpl::get_self_node(td::Promise<DhtNode> promise) {
auto P =
td::PromiseCreator::lambda([promise = std::move(promise), print_id = print_id(), id = id_, keyring = keyring_,
client_only = client_only_](td::Result<adnl::AdnlNode> R) mutable {
R.ensure();
auto node = R.move_as_ok();
auto version = static_cast<td::int32>(td::Clocks::system());
auto B = create_serialize_tl_object<ton_api::dht_node>(node.pub_id().tl(), node.addr_list().tl(), version,
td::BufferSlice());
if (!client_only) {
CHECK(node.addr_list().size() > 0);
}
auto P = td::PromiseCreator::lambda(
[promise = std::move(promise), node = std::move(node), version](td::Result<td::BufferSlice> R) mutable {
R.ensure();
DhtNode n{node.pub_id(), node.addr_list(), version, R.move_as_ok()};
promise.set_result(std::move(n));
});
td::actor::send_closure(keyring, &keyring::Keyring::sign_message, id.pubkey_hash(), std::move(B), std::move(P));
});
auto P = td::PromiseCreator::lambda([promise = std::move(promise), print_id = print_id(), id = id_,
keyring = keyring_, client_only = client_only_,
network_id = network_id_](td::Result<adnl::AdnlNode> R) mutable {
R.ensure();
auto node = R.move_as_ok();
auto version = static_cast<td::int32>(td::Clocks::system());
td::BufferSlice B = serialize_tl_object(
DhtNode{node.pub_id(), node.addr_list(), version, network_id, td::BufferSlice{}}.tl(), true);
if (!client_only) {
CHECK(node.addr_list().size() > 0);
}
auto P = td::PromiseCreator::lambda([promise = std::move(promise), node = std::move(node), version,
network_id](td::Result<td::BufferSlice> R) mutable {
R.ensure();
DhtNode n{node.pub_id(), node.addr_list(), version, network_id, R.move_as_ok()};
promise.set_result(std::move(n));
});
td::actor::send_closure(keyring, &keyring::Keyring::sign_message, id.pubkey_hash(), std::move(B), std::move(P));
});
td::actor::send_closure(adnl_, &adnl::Adnl::get_self_node, id_, std::move(P));
}
td::Result<std::shared_ptr<DhtGlobalConfig>> Dht::create_global_config(tl_object_ptr<ton_api::dht_config_global> conf) {
td::uint32 k;
if (conf->k_ == 0) {
td::Result<std::shared_ptr<DhtGlobalConfig>> Dht::create_global_config(tl_object_ptr<ton_api::dht_config_Global> conf) {
td::uint32 k = 0, a = 0;
td::int32 network_id = -1;
tl_object_ptr<ton_api::dht_nodes> static_nodes;
ton_api::downcast_call(*conf, td::overloaded(
[&](ton_api::dht_config_global &f) {
k = f.k_;
a = f.a_;
network_id = -1;
static_nodes = std::move(f.static_nodes_);
},
[&](ton_api::dht_config_global_v2 &f) {
k = f.k_;
a = f.a_;
network_id = f.network_id_;
static_nodes = std::move(f.static_nodes_);
}));
if (k == 0) {
k = DhtMember::default_k();
} else if (conf->k_ > 0 && static_cast<td::uint32>(conf->k_) <= DhtMember::max_k()) {
k = conf->k_;
} else {
return td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad value k=" << conf->k_);
} else if (k > DhtMember::max_k()) {
return td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad value k=" << k);
}
td::uint32 a;
if (conf->a_ == 0) {
if (a == 0) {
a = DhtMember::default_a();
} else if (conf->a_ > 0 && static_cast<td::uint32>(conf->a_) <= DhtMember::max_a()) {
a = conf->a_;
} else {
return td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad value a=" << conf->a_);
} else if (a > DhtMember::max_a()) {
return td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad value a=" << a);
}
DhtNodesList l{std::move(conf->static_nodes_)};
return std::make_shared<DhtGlobalConfig>(k, a, std::move(l));
DhtNodesList l{std::move(static_nodes), network_id};
return std::make_shared<DhtGlobalConfig>(k, a, network_id, std::move(l));
}
} // namespace dht

6
dht/dht.h
View file

@ -47,13 +47,17 @@ class Dht : public td::actor::Actor {
td::actor::ActorId<keyring::Keyring> keyring,
td::actor::ActorId<adnl::Adnl> adnl);
static td::Result<std::shared_ptr<DhtGlobalConfig>> create_global_config(
tl_object_ptr<ton_api::dht_config_global> conf);
tl_object_ptr<ton_api::dht_config_Global> conf);
virtual adnl::AdnlNodeIdShort get_id() const = 0;
virtual void set_value(DhtValue key_value, td::Promise<td::Unit> result) = 0;
virtual void get_value(DhtKey key, td::Promise<DhtValue> result) = 0;
virtual void register_reverse_connection(adnl::AdnlNodeIdFull client, td::Promise<td::Unit> promise) = 0;
virtual void request_reverse_ping(adnl::AdnlNode target, adnl::AdnlNodeIdShort client,
td::Promise<td::Unit> promise) = 0;
virtual void dump(td::StringBuilder &sb) const = 0;
virtual ~Dht() = default;

15
dht/dht.hpp
View file

@ -52,15 +52,20 @@ class DhtGlobalConfig {
auto get_a() const {
return a_;
}
auto get_network_id() const {
return network_id_;
}
const auto &nodes() const {
return static_nodes_;
}
DhtGlobalConfig(td::uint32 k, td::uint32 a, DhtNodesList nodes) : k_(k), a_(a), static_nodes_(std::move(nodes)) {
DhtGlobalConfig(td::uint32 k, td::uint32 a, td::int32 network_id, DhtNodesList nodes)
: k_(k), a_(a), network_id_(network_id), static_nodes_(std::move(nodes)) {
}
private:
td::uint32 k_;
td::uint32 a_;
td::int32 network_id_;
DhtNodesList static_nodes_;
};
@ -85,8 +90,8 @@ class DhtMember : public Dht {
static td::actor::ActorOwn<DhtMember> create(adnl::AdnlNodeIdShort id, std::string db_root,
td::actor::ActorId<keyring::Keyring> keyring,
td::actor::ActorId<adnl::Adnl> adnl, td::uint32 k = 10, td::uint32 a = 3,
bool client_only = false);
td::actor::ActorId<adnl::Adnl> adnl, td::int32 network_id,
td::uint32 k = 10, td::uint32 a = 3, bool client_only = false);
//virtual void update_addr_list(tl_object_ptr<ton_api::adnl_addressList> addr_list) = 0;
//virtual void add_node(adnl::AdnlNodeIdShort id) = 0;
@ -101,6 +106,10 @@ class DhtMember : public Dht {
virtual void get_self_node(td::Promise<DhtNode> promise) = 0;
virtual PrintId print_id() const = 0;
static DhtKey get_reverse_connection_key(adnl::AdnlNodeIdShort node) {
return DhtKey{node.pubkey_hash(), "address", 0};
}
};
inline td::StringBuilder &operator<<(td::StringBuilder &sb, const DhtMember::PrintId &id) {

303
dht/test/dht-test-ping.cpp
View file

@ -1,303 +0,0 @@
/*
This file is part of TON Blockchain source code.
TON Blockchain is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
TON Blockchain is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with TON Blockchain. If not, see <http://www.gnu.org/licenses/>.
In addition, as a special exception, the copyright holders give permission
to link the code of portions of this program with the OpenSSL library.
You must obey the GNU General Public License in all respects for all
of the code used other than OpenSSL. If you modify file(s) with this
exception, you may extend this exception to your version of the file(s),
but you are not obligated to do so. If you do not wish to do so, delete this
exception statement from your version. If you delete this exception statement
from all source files in the program, then also delete it here.
Copyright 2017-2020 Telegram Systems LLP
*/
#include "adnl/adnl-network-manager.h"
#include "adnl/adnl-peer-table.h"
#include "adnl/utils.hpp"
#include "keys/encryptor.h"
#include "td/utils/Time.h"
#include "td/utils/format.h"
#include "td/utils/OptionParser.h"
#include "td/utils/filesystem.h"
#include "dht/dht.h"
#include "auto/tl/ton_api_json.h"
#include <iostream>
#include <sstream>
template <std::size_t size>
std::ostream &operator<<(std::ostream &stream, const td::UInt<size> &x) {
for (size_t i = 0; i < size / 8; i++) {
stream << td::format::hex_digit((x.raw[i] >> 4) & 15) << td::format::hex_digit(x.raw[i] & 15);
}
return stream;
}
class adnl::AdnlNode : public td::actor::Actor {
private:
std::vector<td::UInt256> ping_ids_;
td::actor::ActorOwn<ton::adnl::AdnlNetworkManager> network_manager_;
td::actor::ActorOwn<ton::adnl::AdnlPeerTable> peer_table_;
td::actor::ActorOwn<ton::DhtNode> dht_node_;
td::UInt256 local_id_;
bool local_id_set_ = false;
std::string host_ = "127.0.0.1";
td::uint32 ip_ = 0x7f000001;
td::uint16 port_ = 2380;
std::string local_config_ = "ton-local.config";
std::string global_config_ = "ton-global.config";
void receive_message(td::UInt256 src, td::UInt256 dst, td::BufferSlice data) {
std::cout << "MESSAGE FROM " << src << " to " << dst << " of size " << std::to_string(data.size()) << "\n";
}
void receive_query(td::UInt256 src, td::UInt256 dst, td::uint64 query_id, td::BufferSlice data) {
std::cout << "QUERY " << std::to_string(query_id) << " FROM " << src << " to " << dst << " of size "
<< std::to_string(data.size()) << "\n";
td::actor::send_closure(peer_table_, &ton::adnl::AdnlPeerTable::answer_query, dst, src, query_id,
ton::create_tl_object<ton::ton_api::testObject>());
}
std::unique_ptr<ton::adnl::AdnlPeerTable::Callback> make_callback() {
class Callback : public ton::adnl::AdnlPeerTable::Callback {
public:
void receive_message(td::UInt256 src, td::UInt256 dst, td::BufferSlice data) override {
td::actor::send_closure(id_, &adnl::AdnlNode::receive_message, src, dst, std::move(data));
}
void receive_query(td::UInt256 src, td::UInt256 dst, td::uint64 query_id, td::BufferSlice data) override {
td::actor::send_closure(id_, &adnl::AdnlNode::receive_query, src, dst, query_id, std::move(data));
}
Callback(td::actor::ActorId<adnl::AdnlNode> id) : id_(std::move(id)) {
}
private:
td::actor::ActorId<adnl::AdnlNode> id_;
};
return std::make_unique<Callback>(td::actor::actor_id(this));
}
public:
void set_local_config(std::string str) {
local_config_ = str;
}
void set_global_config(std::string str) {
global_config_ = str;
}
void start_up() override {
alarm_timestamp() = td::Timestamp::in(1);
}
adnl::AdnlNode() {
network_manager_ = ton::adnl::AdnlNetworkManager::create();
peer_table_ = ton::adnl::AdnlPeerTable::create();
td::actor::send_closure(network_manager_, &ton::adnl::AdnlNetworkManager::register_peer_table, peer_table_.get());
td::actor::send_closure(peer_table_, &ton::adnl::AdnlPeerTable::register_network_manager, network_manager_.get());
}
void listen_udp(td::uint16 port) {
td::actor::send_closure(network_manager_, &ton::adnl::AdnlNetworkManager::add_listening_udp_port, "0.0.0.0", port);
port_ = port;
}
void run() {
auto L = td::read_file(local_config_);
if (L.is_error()) {
LOG(FATAL) << "can not read local config: " << L.move_as_error();
}
auto L2 = td::json_decode(L.move_as_ok().as_slice());
if (L2.is_error()) {
LOG(FATAL) << "can not parse local config: " << L2.move_as_error();
}
auto lc_j = L2.move_as_ok();
if (lc_j.type() != td::JsonValue::Type::Object) {
LOG(FATAL) << "can not parse local config: expected json object";
}
ton::ton_api::config_local lc;
auto rl = ton::ton_api::from_json(lc, lc_j.get_object());
if (rl.is_error()) {
LOG(FATAL) << "can not interpret local config: " << rl.move_as_error();
}
auto G = td::read_file(global_config_);
if (G.is_error()) {
LOG(FATAL) << "can not read global config: " << G.move_as_error();
}
auto G2 = td::json_decode(G.move_as_ok().as_slice());
if (G2.is_error()) {
LOG(FATAL) << "can not parse global config: " << G2.move_as_error();
}
auto gc_j = G2.move_as_ok();
if (gc_j.type() != td::JsonValue::Type::Object) {
LOG(FATAL) << "can not parse global config: expected json object";
}
ton::ton_api::config_global gc;
auto rg = ton::ton_api::from_json(gc, gc_j.get_object());
if (rg.is_error()) {
LOG(FATAL) << "can not interpret local config: " << rg.move_as_error();
}
if (gc.adnl_) {
auto it = gc.adnl_->static_nodes_.begin();
while (it != gc.adnl_->static_nodes_.end()) {
auto R = ton::adnl_validate_full_id(std::move((*it)->id_));
if (R.is_error()) {
LOG(FATAL) << "can not apply global config: " << R.move_as_error();
}
auto R2 = ton::adnl_validate_addr_list(std::move((*it)->addr_list_));
if (R2.is_error()) {
LOG(FATAL) << "can not apply global config: " << R2.move_as_error();
}
td::actor::send_closure(peer_table_, &ton::adnl::AdnlPeerTable::add_peer, R.move_as_ok(), R2.move_as_ok());
it++;
}
}
if (!gc.dht_) {
LOG(FATAL) << "global config does not contain dht section";
}
if (lc.dht_.size() != 1) {
LOG(FATAL) << "local config must contain exactly one dht section";
}
auto R = ton::DhtNode::create_from_json(std::move(gc.dht_), std::move(lc.dht_[0]), peer_table_.get());
if (R.is_error()) {
LOG(FATAL) << "fail creating dht node: " << R.move_as_error();
}
dht_node_ = R.move_as_ok();
}
/*
void set_host(td::IPAddress ip, std::string host) {
ip_ = ip.get_ipv4();
host_ = host;
}
void send_pings_to(td::UInt256 id) {
std::cout << "send pings to " << id << "\n";
ping_ids_.push_back(id);
}
void add_local_id(ton::tl_object_ptr<ton::ton_api::adnl_id_Pk> pk_) {
auto pub_ = ton::get_public_key(pk_);
local_id_ = ton::adnl_short_id(pub_);
std::cout << "local_id = '" << local_id_ << "'\n";
auto x = ton::create_tl_object<ton::ton_api::adnl_address_udp>(ip_, port_);
auto v = std::vector<ton::tl_object_ptr<ton::ton_api::adnl_Address>>();
v.push_back(ton::move_tl_object_as<ton::ton_api::adnl_Address>(x));
auto y =
ton::create_tl_object<ton::ton_api::adnl_addressList>(std::move(v), static_cast<td::int32>(td::Time::now()));
LOG(INFO) << "local_addr_list: " << ton::ton_api::to_string(y);
td::actor::send_closure(peer_table_, &ton::adnl::AdnlPeerTable::add_id, ton::clone_tl_object(pk_),
ton::clone_tl_object(y));
td::actor::send_closure(peer_table_, &ton::adnl::AdnlPeerTable::subscribe_custom, local_id_, "TEST", make_callback());
local_id_set_ = true;
dht_node_ = ton::DhtNode::create(std::move(pk_), peer_table_.get());
td::actor::send_closure(dht_node_, &ton::DhtNode::update_addr_list, std::move(y));
}
void add_static_dht_node(ton::tl_object_ptr<ton::ton_api::adnl_id_Full> id,
ton::tl_object_ptr<ton::ton_api::adnl_addressList> addr_list,
td::BufferSlice signature) {
auto Id = ton::adnl_short_id(id);
td::actor::send_closure(
dht_node_, &ton::DhtNode::add_full_node, Id,
ton::create_tl_object<ton::ton_api::dht_node>(std::move(id), std::move(addr_list), signature.as_slice().str()));
}
void add_foreign(ton::tl_object_ptr<ton::ton_api::adnl_id_Full> id,
ton::tl_object_ptr<ton::ton_api::adnl_addressList> addr_list) {
std::cout << ton::adnl_short_id(id) << "\n";
td::actor::send_closure(peer_table_, &ton::adnl::AdnlPeerTable::add_peer, std::move(id), std::move(addr_list));
}
void alarm() override {
std::cout << "alarm\n";
if (local_id_set_) {
for (auto it = ping_ids_.begin(); it != ping_ids_.end(); it++) {
auto P = td::PromiseCreator::lambda([](td::Result<ton::tl_object_ptr<ton::ton_api::adnl_Message>> result) {
if (result.is_error()) {
std::cout << "received error " << result.move_as_error().to_string() << "\n";
} else {
auto message = result.move_as_ok();
std::cout << "received answer to query\n";
}
});
td::actor::send_closure(peer_table_, &ton::adnl::AdnlPeerTable::send_query, local_id_, *it, std::move(P),
td::Timestamp::in(5),
ton::move_tl_object_as<ton::ton_api::adnl_Message>(
ton::create_tl_object<ton::ton_api::adnl_message_custom>("TEST")));
}
}
}
*/
};
td::Result<td::UInt256> get_uint256(std::string str) {
if (str.size() != 64) {
return td::Status::Error("uint256 must have 64 bytes");
}
td::UInt256 res;
for (size_t i = 0; i < 32; i++) {
res.raw[i] = static_cast<td::uint8>(td::hex_to_int(str[2 * i]) * 16 + td::hex_to_int(str[2 * i + 1]));
}
return res;
}
int main(int argc, char *argv[]) {
td::actor::ActorOwn<adnl::AdnlNode> x;
td::OptionParser p;
p.set_description("test basic adnl functionality");
p.add_option('h', "help", "prints_help", [&]() {
char b[10240];
td::StringBuilder sb({b, 10000});
sb << p;
std::cout << sb.as_cslice().c_str();
std::exit(2);
return td::Status::OK();
});
p.add_option('p', "port", "sets udp port", [&](td::Slice port) {
td::actor::send_closure(x, &adnl::AdnlNode::listen_udp, static_cast<td::uint16>(std::stoi(port.str())));
return td::Status::OK();
});
p.add_option('C', "global-config", "file to read global config", [&](td::Slice fname) {
td::actor::send_closure(x, &adnl::AdnlNode::set_global_config, fname.str());
return td::Status::OK();
});
p.add_option('c', "local-config", "file to read local config", [&](td::Slice fname) {
td::actor::send_closure(x, &adnl::AdnlNode::set_local_config, fname.str());
return td::Status::OK();
});
td::actor::Scheduler scheduler({2});
scheduler.run_in_context([&] {
x = td::actor::create_actor<adnl::AdnlNode>(td::actor::ActorInfoCreator::Options().with_name("A").with_poll());
});
scheduler.run_in_context([&] { p.run(argc, argv).ensure(); });
scheduler.run_in_context([&] { td::actor::send_closure(x, &adnl::AdnlNode::run); });
scheduler.run();
return 0;
}

214
dht/utils/dht-ping-servers.cpp Normal file
View file

@ -0,0 +1,214 @@
/*
This file is part of TON Blockchain source code.
TON Blockchain is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
TON Blockchain is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with TON Blockchain. If not, see <http://www.gnu.org/licenses/>.
In addition, as a special exception, the copyright holders give permission
to link the code of portions of this program with the OpenSSL library.
You must obey the GNU General Public License in all respects for all
of the code used other than OpenSSL. If you modify file(s) with this
exception, you may extend this exception to your version of the file(s),
but you are not obligated to do so. If you do not wish to do so, delete this
exception statement from your version. If you delete this exception statement
from all source files in the program, then also delete it here.
Copyright 2017-2020 Telegram Systems LLP
*/
#include "adnl/adnl-network-manager.h"
#include "adnl/adnl.h"
#include "adnl/utils.hpp"
#include "keys/encryptor.h"
#include "td/utils/Time.h"
#include "td/utils/format.h"
#include "td/utils/OptionParser.h"
#include "td/utils/filesystem.h"
#include "dht/dht.hpp"
#include "auto/tl/ton_api_json.h"
#include "common/delay.h"
#include "td/utils/Random.h"
#include "terminal/terminal.h"
#include <iostream>
class AdnlNode : public td::actor::Actor {
private:
td::actor::ActorOwn<ton::adnl::AdnlNetworkManager> network_manager_;
td::actor::ActorOwn<ton::adnl::Adnl> adnl_;
td::actor::ActorOwn<ton::keyring::Keyring> keyring_;
ton::adnl::AdnlNodeIdShort local_id_;
std::string host_ = "127.0.0.1";
td::uint16 port_ = 2380;
std::string global_config_ = "ton-global.config";
struct NodeInfo {
ton::adnl::AdnlNodeIdShort id;
td::uint32 sent = 0, received = 0;
double sum_time = 0.0;
explicit NodeInfo(ton::adnl::AdnlNodeIdShort id) : id(id) {
}
};
std::vector<NodeInfo> nodes_;
td::uint32 pings_remaining_ = 4;
td::uint32 pending_ = 1;
public:
void set_global_config(std::string str) {
global_config_ = str;
}
void listen_udp(td::uint16 port) {
port_ = port;
}
AdnlNode() {
}
void run() {
network_manager_ = ton::adnl::AdnlNetworkManager::create(port_);
keyring_ = ton::keyring::Keyring::create("");
adnl_ = ton::adnl::Adnl::create("", keyring_.get());
td::actor::send_closure(adnl_, &ton::adnl::Adnl::register_network_manager, network_manager_.get());
td::IPAddress addr;
addr.init_host_port(host_, port_).ensure();
ton::adnl::AdnlCategoryMask mask;
mask[0] = true;
td::actor::send_closure(network_manager_, &ton::adnl::AdnlNetworkManager::add_self_addr, addr, mask, 0);
auto pk = ton::privkeys::Ed25519::random();
td::actor::send_closure(keyring_, &ton::keyring::Keyring::add_key, pk, true, [](td::Result<td::Unit>) {});
ton::adnl::AdnlNodeIdFull local_id_full(pk.pub());
ton::adnl::AdnlAddressList addr_list;
addr_list.set_version(static_cast<td::int32>(td::Clocks::system()));
addr_list.set_reinit_date(ton::adnl::Adnl::adnl_start_time());
td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_id, local_id_full, std::move(addr_list), (td::uint8)0);
local_id_ = local_id_full.compute_short_id();
auto r_dht = get_dht_config();
if (r_dht.is_error()) {
LOG(FATAL) << "Cannot get dht config: " << r_dht.move_as_error();
}
auto dht = r_dht.move_as_ok();
ton::adnl::AdnlNodesList static_nodes;
for (const auto &node : dht->nodes().list()) {
LOG(INFO) << "Node #" << nodes_.size() << " : " << node.adnl_id().compute_short_id();
nodes_.emplace_back(node.adnl_id().compute_short_id());
static_nodes.push(ton::adnl::AdnlNode(node.adnl_id(), node.addr_list()));
}
td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_static_nodes_from_config, std::move(static_nodes));
ton::delay_action([SelfId = actor_id(this)]() { td::actor::send_closure(SelfId, &AdnlNode::send_pings); },
td::Timestamp::in(1.0));
}
td::Result<std::shared_ptr<ton::dht::DhtGlobalConfig>> get_dht_config() {
TRY_RESULT_PREFIX(conf_data, td::read_file(global_config_), "failed to read: ");
TRY_RESULT_PREFIX(conf_json, td::json_decode(conf_data.as_slice()), "failed to parse json: ");
ton::ton_api::config_global conf;
TRY_STATUS_PREFIX(ton::ton_api::from_json(conf, conf_json.get_object()), "json does not fit TL scheme: ");
if (!conf.dht_) {
return td::Status::Error(ton::ErrorCode::error, "does not contain [dht] section");
}
TRY_RESULT_PREFIX(dht, ton::dht::Dht::create_global_config(std::move(conf.dht_)), "bad [dht] section: ");
return std::move(dht);
}
void send_pings() {
CHECK(pings_remaining_);
--pings_remaining_;
for (size_t i = 0; i < nodes_.size(); ++i) {
auto id = nodes_[i].id;
LOG(INFO) << "Sending ping to " << id;
++pending_;
td::actor::send_closure(
adnl_, &ton::adnl::Adnl::send_query, local_id_, id, "ping",
[SelfId = actor_id(this), i, timer = td::Timer()](td::Result<td::BufferSlice> R) {
td::actor::send_closure(SelfId, &AdnlNode::on_pong, i, timer.elapsed(), R.is_ok());
}, td::Timestamp::in(5.0),
ton::create_serialize_tl_object<ton::ton_api::dht_ping>(td::Random::fast_uint64()));
}
if (pings_remaining_ == 0) {
--pending_;
try_finish();
} else {
ton::delay_action([SelfId = actor_id(this)]() { td::actor::send_closure(SelfId, &AdnlNode::send_pings); },
td::Timestamp::in(1.0));
}
}
void on_pong(size_t i, double time, bool success) {
auto &node = nodes_[i];
++node.sent;
if (success) {
++node.received;
node.sum_time += time;
LOG(INFO) << "Pong from " << node.id << " in " << time << "s";
} else {
LOG(INFO) << "Pong from " << node.id << " : timeout";
}
--pending_;
try_finish();
}
void try_finish() {
if (pending_) {
return;
}
td::TerminalIO::out() << "Pinged " << nodes_.size() << " nodes:\n";
for (const auto& node : nodes_) {
td::TerminalIO::out() << node.id << " : " << node.received << "/" << node.sent;
if (node.received > 0) {
td::TerminalIO::out() << " (avg. time = " << node.sum_time / node.received << ")";
}
td::TerminalIO::out() << "\n";
}
std::exit(0);
}
};
int main(int argc, char *argv[]) {
td::actor::ActorOwn<AdnlNode> x;
td::OptionParser p;
p.set_description("ping dht servers from config");
p.add_option('h', "help", "print help", [&]() {
char b[10240];
td::StringBuilder sb(td::MutableSlice{b, 10000});
sb << p;
std::cout << sb.as_cslice().c_str();
std::exit(2);
});
p.add_option('p', "port", "set udp port", [&](td::Slice port) {
td::actor::send_closure(x, &AdnlNode::listen_udp, static_cast<td::uint16>(std::stoi(port.str())));
});
p.add_option('C', "global-config", "file to read global config from",
[&](td::Slice fname) { td::actor::send_closure(x, &AdnlNode::set_global_config, fname.str()); });
p.add_option('v', "verbosity", "set verbosity", [&](td::Slice arg) {
int v = VERBOSITY_NAME(FATAL) + (td::to_integer<int>(arg));
SET_VERBOSITY_LEVEL(v);
});
td::actor::Scheduler scheduler({2});
scheduler.run_in_context([&] { x = td::actor::create_actor<AdnlNode>("AdnlNode"); });
scheduler.run_in_context([&] { p.run(argc, argv).ensure(); });
scheduler.run_in_context([&] { td::actor::send_closure(x, &AdnlNode::run); });
scheduler.run();
return 0;
}

219
dht/utils/dht-resolve.cpp Normal file
View file

@ -0,0 +1,219 @@
/*
This file is part of TON Blockchain source code.
TON Blockchain is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
TON Blockchain is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with TON Blockchain. If not, see <http://www.gnu.org/licenses/>.
In addition, as a special exception, the copyright holders give permission
to link the code of portions of this program with the OpenSSL library.
You must obey the GNU General Public License in all respects for all
of the code used other than OpenSSL. If you modify file(s) with this
exception, you may extend this exception to your version of the file(s),
but you are not obligated to do so. If you do not wish to do so, delete this
exception statement from your version. If you delete this exception statement
from all source files in the program, then also delete it here.
Copyright 2017-2020 Telegram Systems LLP
*/
#include "adnl/adnl-network-manager.h"
#include "adnl/adnl.h"
#include "adnl/utils.hpp"
#include "keys/encryptor.h"
#include "td/utils/Time.h"
#include "td/utils/format.h"
#include "td/utils/OptionParser.h"
#include "td/utils/filesystem.h"
#include "dht/dht.hpp"
#include "auto/tl/ton_api_json.h"
#include "common/delay.h"
#include "td/utils/Random.h"
#include "terminal/terminal.h"
#include "common/util.h"
#include <iostream>
class Resolver : public td::actor::Actor {
private:
td::actor::ActorOwn<ton::adnl::AdnlNetworkManager> network_manager_;
td::actor::ActorOwn<ton::adnl::Adnl> adnl_;
td::actor::ActorOwn<ton::keyring::Keyring> keyring_;
ton::adnl::AdnlNodeIdShort local_id_;
td::actor::ActorOwn<ton::dht::Dht> dht_;
std::string global_config_;
int server_idx_;
std::string host_ = "127.0.0.1";
td::uint16 port_;
ton::dht::DhtKey key_;
double timeout_;
public:
Resolver(std::string global_config, int server_idx, td::uint16 port, ton::dht::DhtKey key, double timeout)
: global_config_(global_config), server_idx_(server_idx), port_(port), key_(std::move(key)), timeout_(timeout) {
}
void run() {
network_manager_ = ton::adnl::AdnlNetworkManager::create(port_);
keyring_ = ton::keyring::Keyring::create("");
adnl_ = ton::adnl::Adnl::create("", keyring_.get());
td::actor::send_closure(adnl_, &ton::adnl::Adnl::register_network_manager, network_manager_.get());
td::IPAddress addr;
addr.init_host_port(host_, port_).ensure();
ton::adnl::AdnlCategoryMask mask;
mask[0] = true;
td::actor::send_closure(network_manager_, &ton::adnl::AdnlNetworkManager::add_self_addr, addr, mask, 0);
auto pk = ton::privkeys::Ed25519::random();
td::actor::send_closure(keyring_, &ton::keyring::Keyring::add_key, pk, true, [](td::Result<td::Unit>) {});
ton::adnl::AdnlNodeIdFull local_id_full(pk.pub());
ton::adnl::AdnlAddressList addr_list;
addr_list.set_version(static_cast<td::int32>(td::Clocks::system()));
addr_list.set_reinit_date(ton::adnl::Adnl::adnl_start_time());
td::actor::send_closure(adnl_, &ton::adnl::Adnl::add_id, local_id_full, std::move(addr_list), (td::uint8)0);
local_id_ = local_id_full.compute_short_id();
auto dht_config = get_dht_config();
if (dht_config.is_error()) {
LOG(FATAL) << "Failed to load dht config: " << dht_config.move_as_error();
}
auto D = ton::dht::Dht::create_client(local_id_, "", dht_config.move_as_ok(), keyring_.get(), adnl_.get());
if (D.is_error()) {
LOG(FATAL) << "Failed to init dht client: " << D.move_as_error();
}
dht_ = D.move_as_ok();
LOG(INFO) << "Get value " << key_.public_key_hash() << " " << key_.name() << " " << key_.idx();
send_query();
alarm_timestamp() = td::Timestamp::in(timeout_);
}
void send_query() {
td::actor::send_closure(dht_, &ton::dht::Dht::get_value, key_,
[SelfId = actor_id(this)](td::Result<ton::dht::DhtValue> R) {
td::actor::send_closure(SelfId, &Resolver::got_result, std::move(R));
});
}
void got_result(td::Result<ton::dht::DhtValue> R) {
if (R.is_error()) {
LOG(WARNING) << "Failed to get value, retrying: " << R.move_as_error();
ton::delay_action([SelfId = actor_id(this)]() { td::actor::send_closure(SelfId, &Resolver::send_query); },
td::Timestamp::in(0.25));
return;
}
auto r = R.move_as_ok();
LOG(INFO) << "Got result";
td::TerminalIO::out() << "KEY: " << td::base64_encode(ton::serialize_tl_object(r.key().public_key().tl(), true))
<< "\n";
td::TerminalIO::out() << "VALUE: " << td::base64_encode(r.value().as_slice()) << "\n";
std::exit(0);
}
void alarm() override {
LOG(FATAL) << "Failed to get value: timeout";
}
td::Result<std::shared_ptr<ton::dht::DhtGlobalConfig>> get_dht_config() {
TRY_RESULT_PREFIX(conf_data, td::read_file(global_config_), "failed to read: ");
TRY_RESULT_PREFIX(conf_json, td::json_decode(conf_data.as_slice()), "failed to parse json: ");
ton::ton_api::config_global conf;
TRY_STATUS_PREFIX(ton::ton_api::from_json(conf, conf_json.get_object()), "json does not fit TL scheme: ");
if (!conf.dht_) {
return td::Status::Error(ton::ErrorCode::error, "does not contain [dht] section");
}
ton::ton_api::dht_nodes* static_nodes = nullptr;
ton::ton_api::downcast_call(*conf.dht_, [&](auto &f) { static_nodes = f.static_nodes_.get(); });
auto &nodes = static_nodes->nodes_;
if (server_idx_ >= 0) {
CHECK(server_idx_ < (int)nodes.size());
LOG(INFO) << "Using server #" << server_idx_;
std::swap(nodes[0], nodes[server_idx_]);
nodes.resize(1);
} else {
LOG(INFO) << "Using all " << nodes.size() << " servers";
}
TRY_RESULT_PREFIX(dht, ton::dht::Dht::create_global_config(std::move(conf.dht_)), "bad [dht] section: ");
return std::move(dht);
}
};
td::Result<td::Bits256> parse_bits256(td::Slice s) {
td::BufferSlice str = td::base64_decode(s, true);
if (str.size() != 32) {
return td::Status::Error("Invalid bits256");
}
return td::Bits256(td::BitPtr((unsigned char *)str.data()));
}
int main(int argc, char *argv[]) {
td::actor::ActorOwn<Resolver> x;
td::optional<std::string> global_config;
int server_idx = -1;
td::uint16 port = 2380;
td::optional<td::Bits256> key_id;
td::optional<std::string> key_name;
td::uint32 key_idx = 0;
double timeout = 5.0;
td::OptionParser p;
p.set_description("find value in dht by the given key (key-id, key-name, ket-idx)");
p.add_option('h', "help", "print help", [&]() {
char b[10240];
td::StringBuilder sb(td::MutableSlice{b, 10000});
sb << p;
std::cout << sb.as_cslice().c_str();
std::exit(2);
});
p.add_option('C', "global-config", "global config", [&](td::Slice arg) { global_config = arg.str(); });
p.add_checked_option('s', "server-idx", "index of dht server from global config (default: all)", [&](td::Slice arg) {
TRY_RESULT_ASSIGN(server_idx, td::to_integer_safe<int>(arg));
return td::Status::OK();
});
p.add_checked_option('p', "port", "set udp port", [&](td::Slice arg) {
TRY_RESULT_ASSIGN(port, td::to_integer_safe<td::uint16>(arg));
return td::Status::OK();
});
p.add_option('v', "verbosity", "set verbosity", [&](td::Slice arg) {
int v = VERBOSITY_NAME(FATAL) + (td::to_integer<int>(arg));
SET_VERBOSITY_LEVEL(v);
});
p.add_checked_option('k', "key-id", "set key id (256-bit, base64)", [&](td::Slice arg) {
TRY_RESULT_ASSIGN(key_id, parse_bits256(arg));
return td::Status::OK();
});
p.add_option('n', "key-name", "set key name", [&](td::Slice arg) { key_name = arg.str(); });
p.add_checked_option('i', "key-idx", "set key idx (default: 0)", [&](td::Slice arg) {
TRY_RESULT_ASSIGN(key_idx, td::to_integer_safe<td::uint32>(arg));
return td::Status::OK();
});
p.add_option('t', "timeout", "set timeout (default: 5s)", [&](td::Slice arg) { timeout = td::to_double(arg); });
td::actor::Scheduler scheduler({2});
scheduler.run_in_context([&] { p.run(argc, argv).ensure(); });
scheduler.run_in_context([&] {
LOG_IF(FATAL, !global_config) << "global config is not set";
LOG_IF(FATAL, !key_id) << "key-id is not set";
LOG_IF(FATAL, !key_name) << "key-name is not set";
x = td::actor::create_actor<Resolver>(
"Resolver", global_config.value(), server_idx, port,
ton::dht::DhtKey{ton::PublicKeyHash(key_id.value()), key_name.value(), key_idx}, timeout);
});
scheduler.run_in_context([&] { td::actor::send_closure(x, &Resolver::run); });
scheduler.run();
return 0;
}

9
docker/Dockerfile
View file

@ -11,16 +11,19 @@ WORKDIR /ton
RUN mkdir build && \
cd build && \
cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. && \
ninja tonlibjson fift func validator-engine validator-engine-console generate-random-id dht-server lite-client
cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. && \
ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console generate-random-id dht-server lite-client
FROM ubuntu:20.04
RUN apt-get update && \
apt-get install -y openssl wget libatomic1 && \
rm -rf /var/lib/apt/lists/*
RUN mkdir -p /var/ton-work/db && \
mkdir -p /var/ton-work/db/static
COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon /usr/local/bin/
COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon-cli /usr/local/bin/
COPY --from=builder /ton/build/lite-client/lite-client /usr/local/bin/
COPY --from=builder /ton/build/validator-engine/validator-engine /usr/local/bin/
COPY --from=builder /ton/build/validator-engine-console/validator-engine-console /usr/local/bin/
@ -30,4 +33,4 @@ WORKDIR /var/ton-work/db
COPY init.sh control.template ./
RUN chmod +x init.sh
ENTRYPOINT ["/var/ton-work/db/init.sh"]
ENTRYPOINT ["/var/ton-work/db/init.sh"]

22
example/android/CMakeLists.txt
View file

@ -7,8 +7,14 @@ cmake_minimum_required(VERSION 3.4.1 FATAL_ERROR)
option(TONLIB_ENABLE_JNI "Enable JNI-compatible TonLib API" ON)
string(APPEND CMAKE_CXX_FLAGS " -std=c++14 -Wall -Wextra -Wno-unused-parameter -Wno-deprecated-declarations -Wconversion -Wno-sign-conversion -fno-omit-frame-pointer -ffunction-sections -fdata-sections")
string(APPEND CMAKE_SHARED_LINKER_FLAGS " -Wl,--gc-sections -Wl,--exclude-libs,ALL")
if (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
string(APPEND CMAKE_CXX_FLAGS " -std=c++14 -fno-omit-frame-pointer -ffunction-sections -fdata-sections")
else()
string(APPEND CMAKE_CXX_FLAGS " -std=c++14 -Wall -Wextra -Wno-unused-parameter -Wno-deprecated-declarations -Wconversion -Wno-sign-conversion -fno-omit-frame-pointer -ffunction-sections -fdata-sections")
endif()
if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
string(APPEND CMAKE_SHARED_LINKER_FLAGS " -Wl,--gc-sections -Wl,--exclude-libs,ALL")
endif()
# Creates and names a library, sets it as either STATIC
# or SHARED, and provides the relative paths to its source code.
@ -37,9 +43,15 @@ target_link_libraries(native-lib tonlib)
set(TONLIB_API_JAVA_PACKAGE "drinkless/org/ton")
target_compile_definitions(native-lib PRIVATE PACKAGE_NAME="${TONLIB_API_JAVA_PACKAGE}")
add_custom_command(TARGET native-lib POST_BUILD
COMMAND ${CMAKE_COMMAND} -E rename $<TARGET_FILE:native-lib> $<TARGET_FILE:native-lib>.debug
COMMAND ${CMAKE_STRIP} --strip-debug --strip-unneeded $<TARGET_FILE:native-lib>.debug -o $<TARGET_FILE:native-lib>)
if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
add_custom_command(TARGET native-lib POST_BUILD
COMMAND ${CMAKE_COMMAND} -E rename $<TARGET_FILE:native-lib> $<TARGET_FILE:native-lib>.debug
COMMAND ${CMAKE_STRIP} -S $<TARGET_FILE:native-lib>.debug -o $<TARGET_FILE:native-lib>)
elseif (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
add_custom_command(TARGET native-lib POST_BUILD
COMMAND ${CMAKE_COMMAND} -E rename $<TARGET_FILE:native-lib> $<TARGET_FILE:native-lib>.debug
COMMAND ${CMAKE_STRIP} --strip-debug --strip-unneeded $<TARGET_FILE:native-lib>.debug -o $<TARGET_FILE:native-lib>)
endif()
if (NOT CMAKE_CROSSCOMPILING)
set(TONLIB_API_JAVA_PATH ${CMAKE_CURRENT_SOURCE_DIR}/src/)

48
example/android/README.md Normal file
View file

@ -0,0 +1,48 @@
# Generation of Tonlib libraries for Android OS
**Tl;dr** Download the latest version of Tonlib libraries for Android from TON release page or check the artifacts from [Android JNI GitHub action](https://github.com/ton-blockchain/ton/actions/workflows/tonlib-android-jni.yml).
## Compile Tonlib for Android manually
Prerequisite: installed Java and set environment variable JAVA_HOME.
```bash
git clone --recursive https://github.com/ton-blockchain/ton.git
cd ton
wget https://dl.google.com/android/repository/android-ndk-r25b-linux.zip
unzip android-ndk-r25b-linux.zip
export JAVA_AWT_LIBRARY=NotNeeded
export JAVA_JVM_LIBRARY=NotNeeded
export JAVA_INCLUDE_PATH=${JAVA_HOME}/include
export JAVA_AWT_INCLUDE_PATH=${JAVA_HOME}/include
export JAVA_INCLUDE_PATH2=${JAVA_HOME}/include/linux
export ANDROID_NDK_ROOT=$(pwd)/android-ndk-r25b
export OPENSSL_DIR=$(pwd)/example/android/third_party/crypto
rm -rf example/android/src/drinkless/org/ton/TonApi.java
cd example/android/
cmake -GNinja -DTON_ONLY_TONLIB=ON .
ninja prepare_cross_compiling
rm CMakeCache.txt
./build-all.sh
```
# Generation of Tonlib libraries for iOS in Xcode
1. Clone repository https://github.com/labraburn/tonlib-xcframework
2. Open repository directory in Terminal
3. Run command:
```bash
swift run builder --output ./build --clean
```
5. Run command:
```bash
echo ./build/TON.xcframework/* | xargs -n 1 cp -R ./Resources/Headers
````
7. Import **OpenSSL.xcframework** and **TON.xcframework** in XCode in section _"Frameworks, Libraries, and Embedded Content"_
8. Now you can start using Tonlib client by importing it in C or Objective-C source files:
```objective-c
#import <tonlib/tonlib_client_json.h>
```
# Generation of Tonlib libraries for Desktop applications
You can use Tonlib compiled in an ordinary way for desktop applications. If you use Java you can load the library using JNA.
The latest Tonlib library can be found among other TON artifacts either on TON release page or inside the [appropriate GitHub action](https://github.com/ton-blockchain/ton/actions/).

3
example/android/build.sh
View file

@ -33,8 +33,7 @@ mkdir -p build-$ARCH
cd build-$ARCH
cmake .. \
-DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake -DCMAKE_BUILD_TYPE=Release -GNinja -DANDROID_ABI=${ABI} -DOPENSSL_ROOT_DIR=/Users/arseny30/Code/td_android/libtd/src/main/jni/third_party/crypto/${ARCH} -DTON_ARCH="" -DTON_ONLY_TONLIB=ON || exit 1
cmake .. -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake -DCMAKE_BUILD_TYPE=Release -GNinja -DANDROID_ABI=${ABI} -DOPENSSL_ROOT_DIR=${OPENSSL_DIR}/${ARCH} -DTON_ARCH="" -DTON_ONLY_TONLIB=ON || exit 1
ninja native-lib || exit 1
popd

2
http/http-inbound-connection.cpp
View file

@ -79,10 +79,10 @@ td::Status HttpInboundConnection::receive(td::ChainBufferReader &input) {
send_client_error();
return td::Status::OK();
}
cur_request_ = R.move_as_ok();
if (exit_loop) {
return td::Status::OK();
}
cur_request_ = R.move_as_ok();
}
auto payload = cur_request_->create_empty_payload().move_as_ok();

2
http/http-outbound-connection.cpp
View file

@ -42,10 +42,10 @@ td::Status HttpOutboundConnection::receive(td::ChainBufferReader &input) {
answer_error(HttpStatusCode::status_bad_request, "", std::move(promise_));
return td::Status::OK();
}
cur_response_ = R.move_as_ok();
if (exit_loop) {
return td::Status::OK();
}
cur_response_ = R.move_as_ok();
}
if (cur_response_->code() == 100) {

45
http/http.cpp
View file

@ -279,25 +279,20 @@ td::Status HttpPayload::parse(td::ChainBufferReader &input) {
} break;
case ParseState::reading_chunk_data: {
if (cur_chunk_size_ == 0) {
switch (type_) {
case PayloadType::pt_empty:
UNREACHABLE();
case PayloadType::pt_eof:
case PayloadType::pt_tunnel:
cur_chunk_size_ = 1LL << 60;
break;
case PayloadType::pt_chunked:
state_ = ParseState::reading_crlf;
break;
case PayloadType::pt_content_length: {
LOG(INFO) << "payload parse success";
const std::lock_guard<std::mutex> lock{mutex_};
state_ = ParseState::completed;
run_callbacks();
return td::Status::OK();
} break;
if (type_ == PayloadType::pt_eof || type_ == PayloadType::pt_tunnel) {
cur_chunk_size_ = 1LL << 60;
} else if (type_ == PayloadType::pt_chunked) {
state_ = ParseState::reading_crlf;
break;
} else if (type_ == PayloadType::pt_content_length) {
LOG(INFO) << "payload parse success";
const std::lock_guard<std::mutex> lock{mutex_};
state_ = ParseState::completed;
run_callbacks();
return td::Status::OK();
} else {
UNREACHABLE();
}
break;
}
if (input.size() == 0) {
return td::Status::OK();
@ -502,7 +497,7 @@ bool HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, Htt
char buf[64];
::sprintf(buf, "%lx\r\n", s.size());
auto slice = td::Slice(buf, strlen(buf));
wrote |= !slice.empty();
wrote = true;
output.append(slice);
}
@ -514,7 +509,8 @@ bool HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, Htt
wrote = true;
}
}
if (chunks_.size() != 0 || !parse_completed()) {
auto cur_state = state_.load(std::memory_order_consume);
if (chunks_.size() != 0 || (cur_state != ParseState::reading_trailer && cur_state != ParseState::completed)) {
return wrote;
}
if (!written_zero_chunk_) {
@ -531,7 +527,7 @@ bool HttpPayload::store_http(td::ChainBufferWriter &output, size_t max_size, Htt
}
while (max_size > 0) {
auto cur_state = state_.load(std::memory_order_consume);
cur_state = state_.load(std::memory_order_consume);
HttpHeader h = get_header();
if (h.empty()) {
if (cur_state != ParseState::completed) {
@ -587,7 +583,8 @@ tl_object_ptr<ton_api::http_payloadPart> HttpPayload::store_tl(size_t max_size)
max_size -= s.size();
}
obj->data_.truncate(obj->data_.size() - S.size());
if (chunks_.size() != 0) {
auto cur_state = state_.load(std::memory_order_consume);
if (chunks_.size() != 0 || (cur_state != ParseState::reading_trailer && cur_state != ParseState::completed)) {
return obj;
}
if (!written_zero_chunk_) {
@ -597,7 +594,7 @@ tl_object_ptr<ton_api::http_payloadPart> HttpPayload::store_tl(size_t max_size)
LOG(INFO) << "data completed";
while (max_size > 0) {
auto cur_state = state_.load(std::memory_order_consume);
cur_state = state_.load(std::memory_order_consume);
HttpHeader h = get_header();
if (h.empty()) {
if (cur_state != ParseState::completed) {
@ -869,7 +866,7 @@ td::Status HttpHeader::basic_check() {
}
for (auto &c : value) {
if (c == '\r' || c == '\n') {
return td::Status::Error("bad character in header name");
return td::Status::Error("bad character in header value");
}
}
return td::Status::OK();

90
lite-client/lite-client.cpp
View file

@ -1005,11 +1005,12 @@ bool TestNode::do_parse_line() {
return eoln() && get_server_mc_block_id();
} else if (word == "sendfile") {
return !eoln() && set_error(send_ext_msg_from_filename(get_line_tail()));
} else if (word == "getaccount") {
} else if (word == "getaccount" || word == "getaccountprunned") {
bool prunned = word == "getaccountprunned";
return parse_account_addr_ext(workchain, addr, addr_ext) &&
(seekeoln()
? get_account_state(workchain, addr, mc_last_id_, addr_ext)
: parse_block_id_ext(blkid) && seekeoln() && get_account_state(workchain, addr, blkid, addr_ext));
(seekeoln() ? get_account_state(workchain, addr, mc_last_id_, addr_ext, "", -1, prunned)
: parse_block_id_ext(blkid) && seekeoln() &&
get_account_state(workchain, addr, blkid, addr_ext, "", -1, prunned));
} else if (word == "saveaccount" || word == "saveaccountcode" || word == "saveaccountdata") {
std::string filename;
int mode = ((word.c_str()[11] >> 1) & 3);
@ -1173,7 +1174,7 @@ td::Status TestNode::send_ext_msg_from_filename(std::string filename) {
}
bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::BlockIdExt ref_blkid,
int addr_ext, std::string filename, int mode) {
int addr_ext, std::string filename, int mode, bool prunned) {
if (!ref_blkid.is_valid()) {
return set_error("must obtain last block information before making other queries");
}
@ -1181,35 +1182,44 @@ bool TestNode::get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress
return set_error("server connection not ready");
}
if (addr_ext) {
return get_special_smc_addr(addr_ext, [this, ref_blkid, filename, mode](td::Result<ton::StdSmcAddress> res) {
if (res.is_error()) {
LOG(ERROR) << "cannot resolve special smart contract address: " << res.move_as_error();
} else {
get_account_state(ton::masterchainId, res.move_as_ok(), ref_blkid, 0, filename, mode);
}
});
return get_special_smc_addr(
addr_ext, [this, ref_blkid, filename, mode, prunned](td::Result<ton::StdSmcAddress> res) {
if (res.is_error()) {
LOG(ERROR) << "cannot resolve special smart contract address: " << res.move_as_error();
} else {
get_account_state(ton::masterchainId, res.move_as_ok(), ref_blkid, 0, filename, mode, prunned);
}
});
}
auto a = ton::create_tl_object<ton::lite_api::liteServer_accountId>(workchain, addr);
auto b = ton::serialize_tl_object(ton::create_tl_object<ton::lite_api::liteServer_getAccountState>(
ton::create_tl_lite_block_id(ref_blkid), std::move(a)),
true);
LOG(INFO) << "requesting account state for " << workchain << ":" << addr.to_hex() << " with respect to "
<< ref_blkid.to_str() << " with savefile `" << filename << "` and mode " << mode;
return envelope_send_query(
std::move(b), [Self = actor_id(this), workchain, addr, ref_blkid, filename, mode](td::Result<td::BufferSlice> R) {
if (R.is_error()) {
return;
}
auto F = ton::fetch_tl_object<ton::lite_api::liteServer_accountState>(R.move_as_ok(), true);
if (F.is_error()) {
LOG(ERROR) << "cannot parse answer to liteServer.getAccountState";
} else {
auto f = F.move_as_ok();
td::actor::send_closure_later(Self, &TestNode::got_account_state, ref_blkid, ton::create_block_id(f->id_),
ton::create_block_id(f->shardblk_), std::move(f->shard_proof_),
std::move(f->proof_), std::move(f->state_), workchain, addr, filename, mode);
}
});
td::BufferSlice b;
if (prunned) {
b = ton::serialize_tl_object(ton::create_tl_object<ton::lite_api::liteServer_getAccountStatePrunned>(
ton::create_tl_lite_block_id(ref_blkid), std::move(a)),
true);
} else {
b = ton::serialize_tl_object(ton::create_tl_object<ton::lite_api::liteServer_getAccountState>(
ton::create_tl_lite_block_id(ref_blkid), std::move(a)),
true);
}
LOG(INFO) << "requesting " << (prunned ? "prunned " : "") << "account state for " << workchain << ":" << addr.to_hex()
<< " with respect to " << ref_blkid.to_str() << " with savefile `" << filename << "` and mode " << mode;
return envelope_send_query(std::move(b), [Self = actor_id(this), workchain, addr, ref_blkid, filename, mode,
prunned](td::Result<td::BufferSlice> R) {
if (R.is_error()) {
return;
}
auto F = ton::fetch_tl_object<ton::lite_api::liteServer_accountState>(R.move_as_ok(), true);
if (F.is_error()) {
LOG(ERROR) << "cannot parse answer to liteServer.getAccountState";
} else {
auto f = F.move_as_ok();
td::actor::send_closure_later(Self, &TestNode::got_account_state, ref_blkid, ton::create_block_id(f->id_),
ton::create_block_id(f->shardblk_), std::move(f->shard_proof_),
std::move(f->proof_), std::move(f->state_), workchain, addr, filename, mode,
prunned);
}
});
}
td::int64 TestNode::compute_method_id(std::string method) {
@ -1739,6 +1749,13 @@ bool TestNode::show_dns_record(std::ostream& os, td::Bits256 cat, Ref<vm::CellSl
}
break;
}
case block::gen::DNSRecord::dns_storage_address: {
block::gen::DNSRecord::Record_dns_storage_address rec;
if (tlb::unpack_exact(cs, rec)) {
os << "\tstorage address " << rec.bag_id.to_hex();
}
break;
}
case block::gen::DNSRecord::dns_next_resolver: {
block::gen::DNSRecord::Record_dns_next_resolver rec;
if (tlb::unpack_exact(cs, rec) && block::tlb::t_MsgAddressInt.extract_std_address(rec.resolver, wc, addr)) {
@ -1901,15 +1918,18 @@ bool TestNode::get_last_transactions(ton::WorkchainId workchain, ton::StdSmcAddr
void TestNode::got_account_state(ton::BlockIdExt ref_blk, ton::BlockIdExt blk, ton::BlockIdExt shard_blk,
td::BufferSlice shard_proof, td::BufferSlice proof, td::BufferSlice state,
ton::WorkchainId workchain, ton::StdSmcAddress addr, std::string filename, int mode) {
LOG(INFO) << "got account state for " << workchain << ":" << addr.to_hex() << " with respect to blocks "
<< blk.to_str() << (shard_blk == blk ? "" : std::string{" and "} + shard_blk.to_str());
ton::WorkchainId workchain, ton::StdSmcAddress addr, std::string filename, int mode,
bool prunned) {
LOG(INFO) << "got " << (prunned ? "prunned " : "") << "account state for " << workchain << ":" << addr.to_hex()
<< " with respect to blocks " << blk.to_str()
<< (shard_blk == blk ? "" : std::string{" and "} + shard_blk.to_str());
block::AccountState account_state;
account_state.blk = blk;
account_state.shard_blk = shard_blk;
account_state.shard_proof = std::move(shard_proof);
account_state.proof = std::move(proof);
account_state.state = std::move(state);
account_state.is_virtualized = prunned;
auto r_info = account_state.validate(ref_blk, block::StdAddress(workchain, addr));
if (r_info.is_error()) {
LOG(ERROR) << r_info.error().message();

5
lite-client/lite-client.h
View file

@ -191,10 +191,11 @@ class TestNode : public td::actor::Actor {
td::Status send_ext_msg_from_filename(std::string filename);
td::Status save_db_file(ton::FileHash file_hash, td::BufferSlice data);
bool get_account_state(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::BlockIdExt ref_blkid,
int addr_ext = 0, std::string filename = "", int mode = -1);
int addr_ext = 0, std::string filename = "", int mode = -1, bool prunned = false);
void got_account_state(ton::BlockIdExt ref_blk, ton::BlockIdExt blk, ton::BlockIdExt shard_blk,
td::BufferSlice shard_proof, td::BufferSlice proof, td::BufferSlice state,
ton::WorkchainId workchain, ton::StdSmcAddress addr, std::string filename, int mode);
ton::WorkchainId workchain, ton::StdSmcAddress addr, std::string filename, int mode,
bool prunned);
bool parse_run_method(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::BlockIdExt ref_blkid, int addr_ext,
std::string method_name, bool ext_mode);
bool after_parse_run_method(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::BlockIdExt ref_blkid,

12
overlay/overlay-manager.cpp
View file

@ -90,12 +90,20 @@ void OverlayManager::delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdSho
}
void OverlayManager::create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules, td::string scope) {
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules,
td::string scope) {
create_public_overlay_ex(local_id, std::move(overlay_id), std::move(callback), std::move(rules), std::move(scope),
true);
}
void OverlayManager::create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules,
td::string scope, bool announce_self) {
CHECK(!dht_node_.empty());
auto id = overlay_id.compute_short_id();
register_overlay(local_id, id,
Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id),
std::move(callback), std::move(rules), scope));
std::move(callback), std::move(rules), scope, announce_self));
}
void OverlayManager::create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,

3
overlay/overlay-manager.h
View file

@ -52,6 +52,9 @@ class OverlayManager : public Overlays {
void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules, td::string scope) override;
void create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules, td::string scope,
bool announce_self) override;
void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Callback> callback,
OverlayPrivacyRules rules) override;

76
overlay/overlay-peers.cpp
View file

@ -43,6 +43,7 @@ void OverlayImpl::del_peer(adnl::AdnlNodeIdShort id) {
P->set_neighbour(false);
}
peers_.remove(id);
bad_peers_.erase(id);
update_neighbours(0);
}
@ -51,7 +52,16 @@ void OverlayImpl::del_some_peers() {
return;
}
while (peers_.size() > max_peers()) {
auto P = get_random_peer();
OverlayPeer *P;
if (bad_peers_.empty()) {
P = get_random_peer();
} else {
auto it = bad_peers_.upper_bound(next_bad_peer_);
if (it == bad_peers_.end()) {
it = bad_peers_.begin();
}
P = peers_.get(next_bad_peer_ = *it);
}
if (P) {
auto id = P->get_id();
del_peer(id);
@ -118,16 +128,35 @@ void OverlayImpl::add_peer(OverlayNode P) {
add_peer_in(std::move(P));
}
void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::BufferSlice data) {
void OverlayImpl::on_ping_result(adnl::AdnlNodeIdShort peer, bool success) {
if (!public_) {
return;
}
if (OverlayPeer *p = peers_.get(peer)) {
p->on_ping_result(success);
if (p->is_alive()) {
bad_peers_.erase(peer);
} else {
bad_peers_.insert(peer);
}
}
}
void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::Result<td::BufferSlice> R) {
CHECK(public_);
auto R = fetch_tl_object<ton_api::overlay_nodes>(std::move(data), true);
on_ping_result(src, R.is_ok());
if (R.is_error()) {
VLOG(OVERLAY_NOTICE) << this << ": failed getRandomPeers query: " << R.move_as_error();
return;
}
auto R2 = fetch_tl_object<ton_api::overlay_nodes>(R.move_as_ok(), true);
if (R2.is_error()) {
VLOG(OVERLAY_WARNING) << this << ": dropping incorrect answer to overlay.getRandomPeers query from " << src << ": "
<< R.move_as_error();
<< R2.move_as_error();
return;
}
auto res = R.move_as_ok();
auto res = R2.move_as_ok();
std::vector<OverlayNode> nodes;
for (auto &n : res->nodes_) {
@ -142,10 +171,12 @@ void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::BufferSlic
void OverlayImpl::send_random_peers_cont(adnl::AdnlNodeIdShort src, OverlayNode node,
td::Promise<td::BufferSlice> promise) {
std::vector<tl_object_ptr<ton_api::overlay_node>> vec;
vec.emplace_back(node.tl());
if (announce_self_) {
vec.emplace_back(node.tl());
}
for (td::uint32 i = 0; i < nodes_to_send(); i++) {
auto P = get_random_peer();
auto P = get_random_peer(true);
if (P) {
vec.emplace_back(P->get().tl());
} else {
@ -159,11 +190,7 @@ void OverlayImpl::send_random_peers_cont(adnl::AdnlNodeIdShort src, OverlayNode
} else {
auto P =
td::PromiseCreator::lambda([SelfId = actor_id(this), src, oid = print_id()](td::Result<td::BufferSlice> res) {
if (res.is_error()) {
VLOG(OVERLAY_NOTICE) << oid << ": failed getRandomPeers query: " << res.move_as_error();
return;
}
td::actor::send_closure(SelfId, &OverlayImpl::receive_random_peers, src, res.move_as_ok());
td::actor::send_closure(SelfId, &OverlayImpl::receive_random_peers, src, std::move(res));
});
auto Q =
create_tl_object<ton_api::overlay_getRandomPeers>(create_tl_object<ton_api::overlay_nodes>(std::move(vec)));
@ -216,6 +243,7 @@ void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) {
neighbours_.pop_back();
X->set_neighbour(false);
}
bad_peers_.erase(X->get_id());
peers_.remove(X->get_id());
continue;
}
@ -244,15 +272,25 @@ void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) {
}
}
OverlayPeer *OverlayImpl::get_random_peer() {
while (peers_.size() > 0) {
OverlayPeer *OverlayImpl::get_random_peer(bool only_alive) {
size_t skip_bad = 3;
while (peers_.size() > (only_alive ? bad_peers_.size() : 0)) {
auto P = peers_.get_random();
if (public_ && P->get_version() + 3600 < td::Clocks::system()) {
VLOG(OVERLAY_INFO) << this << ": deleting outdated peer " << P->get_id();
del_peer(P->get_id());
} else {
return P;
continue;
}
if (!P->is_alive()) {
if (only_alive) {
continue;
}
if (skip_bad > 0) {
--skip_bad;
continue;
}
}
return P;
}
return nullptr;
}
@ -261,17 +299,17 @@ void OverlayImpl::get_overlay_random_peers(td::uint32 max_peers,
td::Promise<std::vector<adnl::AdnlNodeIdShort>> promise) {
std::vector<adnl::AdnlNodeIdShort> v;
auto t = td::Clocks::system();
while (peers_.size() > v.size()) {
while (v.size() < max_peers && v.size() < peers_.size() - bad_peers_.size()) {
auto P = peers_.get_random();
if (P->get_version() + 3600 < t) {
VLOG(OVERLAY_INFO) << this << ": deleting outdated peer " << P->get_id();
del_peer(P->get_id());
} else {
} else if (P->is_alive()) {
bool dup = false;
for (auto &n : v) {
if (n == P->get_id()) {
dup = true;
continue;
break;
}
}
if (!dup) {

17
overlay/overlay.cpp
View file

@ -37,10 +37,10 @@ td::actor::ActorOwn<Overlay> Overlay::create(td::actor::ActorId<keyring::Keyring
td::actor::ActorId<OverlayManager> manager,
td::actor::ActorId<dht::Dht> dht_node, adnl::AdnlNodeIdShort local_id,
OverlayIdFull overlay_id, std::unique_ptr<Overlays::Callback> callback,
OverlayPrivacyRules rules, td::string scope) {
OverlayPrivacyRules rules, td::string scope, bool announce_self) {
auto R = td::actor::create_actor<OverlayImpl>("overlay", keyring, adnl, manager, dht_node, local_id,
std::move(overlay_id), true, std::vector<adnl::AdnlNodeIdShort>(),
std::move(callback), std::move(rules), scope);
std::move(callback), std::move(rules), scope, announce_self);
return td::actor::ActorOwn<Overlay>(std::move(R));
}
@ -60,7 +60,7 @@ OverlayImpl::OverlayImpl(td::actor::ActorId<keyring::Keyring> keyring, td::actor
td::actor::ActorId<OverlayManager> manager, td::actor::ActorId<dht::Dht> dht_node,
adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub,
std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Overlays::Callback> callback,
OverlayPrivacyRules rules, td::string scope)
OverlayPrivacyRules rules, td::string scope, bool announce_self)
: keyring_(keyring)
, adnl_(adnl)
, manager_(manager)
@ -70,7 +70,8 @@ OverlayImpl::OverlayImpl(td::actor::ActorId<keyring::Keyring> keyring, td::actor
, callback_(std::move(callback))
, public_(pub)
, rules_(std::move(rules))
, scope_(scope) {
, scope_(scope)
, announce_self_(announce_self) {
overlay_id_ = id_full_.compute_short_id();
VLOG(OVERLAY_INFO) << this << ": creating " << (public_ ? "public" : "private");
@ -144,6 +145,8 @@ void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data,
promise.set_error(td::Status::Error(ErrorCode::protoviolation, "overlay is private"));
return;
}
} else {
on_ping_result(src, true);
}
auto R = fetch_tl_object<ton_api::Function>(data.clone(), true);
@ -222,6 +225,8 @@ void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice dat
VLOG(OVERLAY_WARNING) << this << ": received query in private overlay from unknown source " << src;
return;
}
} else {
on_ping_result(src, true);
}
auto X = fetch_tl_object<ton_api::overlay_Broadcast>(data.clone(), true);
if (X.is_error()) {
@ -324,6 +329,10 @@ void OverlayImpl::receive_dht_nodes(td::Result<dht::DhtValue> res, bool dummy) {
VLOG(OVERLAY_NOTICE) << this << ": can not get value from DHT: " << res.move_as_error();
}
if (!announce_self_) {
return;
}
VLOG(OVERLAY_INFO) << this << ": adding self node to DHT overlay's nodes";
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), oid = print_id()](td::Result<OverlayNode> R) {
if (R.is_error()) {

2
overlay/overlay.h
View file

@ -42,7 +42,7 @@ class Overlay : public td::actor::Actor {
td::actor::ActorId<OverlayManager> manager,
td::actor::ActorId<dht::Dht> dht_node, adnl::AdnlNodeIdShort local_id,
OverlayIdFull overlay_id, std::unique_ptr<Overlays::Callback> callback,
OverlayPrivacyRules rules, td::string scope);
OverlayPrivacyRules rules, td::string scope, bool announce_self = true);
static td::actor::ActorOwn<Overlay> create(td::actor::ActorId<keyring::Keyring> keyring,
td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<OverlayManager> manager,

23
overlay/overlay.hpp
View file

@ -79,7 +79,17 @@ class OverlayPeer {
td::int32 get_version() const {
return node_.version();
}
void on_ping_result(bool success) {
if (success) {
missed_pings_ = 0;
} else {
++missed_pings_;
}
}
bool is_alive() const {
return missed_pings_ < 3;
}
td::uint32 throughput_out_bytes = 0;
td::uint32 throughput_in_bytes = 0;
@ -105,6 +115,7 @@ class OverlayPeer {
adnl::AdnlNodeIdShort id_;
bool is_neighbour_ = false;
size_t missed_pings_ = 0;
};
class OverlayImpl : public Overlay {
@ -113,7 +124,7 @@ class OverlayImpl : public Overlay {
td::actor::ActorId<OverlayManager> manager, td::actor::ActorId<dht::Dht> dht_node,
adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub,
std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Overlays::Callback> callback,
OverlayPrivacyRules rules, td::string scope = "{ \"type\": \"undefined\" }");
OverlayPrivacyRules rules, td::string scope = "{ \"type\": \"undefined\" }", bool announce_self = true);
void update_dht_node(td::actor::ActorId<dht::Dht> dht) override {
dht_node_ = dht;
}
@ -138,7 +149,8 @@ class OverlayImpl : public Overlay {
alarm_timestamp() = td::Timestamp::in(1);
}
void receive_random_peers(adnl::AdnlNodeIdShort src, td::BufferSlice data);
void on_ping_result(adnl::AdnlNodeIdShort peer, bool success);
void receive_random_peers(adnl::AdnlNodeIdShort src, td::Result<td::BufferSlice> R);
void send_random_peers(adnl::AdnlNodeIdShort dst, td::Promise<td::BufferSlice> promise);
void send_random_peers_cont(adnl::AdnlNodeIdShort dst, OverlayNode node, td::Promise<td::BufferSlice> promise);
void get_overlay_random_peers(td::uint32 max_peers, td::Promise<std::vector<adnl::AdnlNodeIdShort>> promise) override;
@ -281,7 +293,7 @@ class OverlayImpl : public Overlay {
void add_peers(std::vector<OverlayNode> nodes);
void del_some_peers();
void del_peer(adnl::AdnlNodeIdShort id);
OverlayPeer *get_random_peer();
OverlayPeer *get_random_peer(bool only_alive = false);
td::actor::ActorId<keyring::Keyring> keyring_;
td::actor::ActorId<adnl::Adnl> adnl_;
@ -296,6 +308,8 @@ class OverlayImpl : public Overlay {
td::Timestamp update_db_at_;
td::Timestamp update_throughput_at_;
td::Timestamp last_throughput_update_;
std::set<adnl::AdnlNodeIdShort> bad_peers_;
adnl::AdnlNodeIdShort next_bad_peer_ = adnl::AdnlNodeIdShort::zero();
std::unique_ptr<Overlays::Callback> callback_;
@ -352,6 +366,7 @@ class OverlayImpl : public Overlay {
bool semi_public_ = false;
OverlayPrivacyRules rules_;
td::string scope_;
bool announce_self_ = true;
std::map<PublicKeyHash, std::shared_ptr<Certificate>> certs_;
class CachedEncryptor : public td::ListNode {

6
overlay/overlays.h
View file

@ -193,7 +193,11 @@ class Overlays : public td::actor::Actor {
virtual void update_dht_node(td::actor::ActorId<dht::Dht> dht) = 0;
virtual void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules, td::string scope) = 0;
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules,
td::string scope) = 0;
virtual void create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Callback> callback, OverlayPrivacyRules rules,
td::string scope, bool announce_self) = 0;
virtual void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Callback> callback,
OverlayPrivacyRules rules) = 0;

2
rldp-http-proxy/CMakeLists.txt
View file

@ -1,5 +1,5 @@
cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR)
add_executable(rldp-http-proxy rldp-http-proxy.cpp DNSResolver.h TonlibClient.h TonlibClient.cpp DNSResolver.cpp)
add_executable(rldp-http-proxy rldp-http-proxy.cpp DNSResolver.h DNSResolver.cpp)
target_include_directories(rldp-http-proxy PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>)
target_link_libraries(rldp-http-proxy PRIVATE tonhttp rldp dht tonlib git)

93
rldp-http-proxy/DNSResolver.cpp
View file

@ -25,26 +25,39 @@
*/
#include "DNSResolver.h"
#include "td/utils/overloaded.h"
#include "common/delay.h"
static const double CACHE_TIMEOUT_HARD = 300.0;
static const double CACHE_TIMEOUT_SOFT = 270.0;
DNSResolver::DNSResolver(td::actor::ActorId<TonlibClient> tonlib_client) : tonlib_client_(std::move(tonlib_client)) {
DNSResolver::DNSResolver(td::actor::ActorId<tonlib::TonlibClientWrapper> tonlib_client)
: tonlib_client_(std::move(tonlib_client)) {
}
void DNSResolver::start_up() {
auto obj = tonlib_api::make_object<tonlib_api::sync>();
auto P = td::PromiseCreator::lambda([](td::Result<tonlib_api::object_ptr<tonlib_api::Object>>) {});
td::actor::send_closure(tonlib_client_, &TonlibClient::send_request, std::move(obj), std::move(P));
sync();
}
void DNSResolver::resolve(std::string host, td::Promise<ton::adnl::AdnlNodeIdShort> promise) {
void DNSResolver::sync() {
auto obj = tonlib_api::make_object<tonlib_api::sync>();
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](
td::Result<tonlib_api::object_ptr<tonlib_api::ton_blockIdExt>> R) {
if (R.is_error()) {
LOG(WARNING) << "Sync error: " << R.move_as_error();
ton::delay_action([SelfId]() { td::actor::send_closure(SelfId, &DNSResolver::sync); }, td::Timestamp::in(5.0));
}
});
td::actor::send_closure(tonlib_client_, &tonlib::TonlibClientWrapper::send_request<tonlib_api::sync>, std::move(obj),
std::move(P));
}
void DNSResolver::resolve(std::string host, td::Promise<std::string> promise) {
auto it = cache_.find(host);
if (it != cache_.end()) {
const CacheEntry &entry = it->second;
double now = td::Time::now();
if (now < entry.created_at_ + CACHE_TIMEOUT_HARD) {
promise.set_result(entry.id_);
promise.set_result(entry.address_);
promise.reset();
if (now < entry.created_at_ + CACHE_TIMEOUT_SOFT) {
return;
@ -55,51 +68,47 @@ void DNSResolver::resolve(std::string host, td::Promise<ton::adnl::AdnlNodeIdSho
td::Bits256 category = td::sha256_bits256(td::Slice("site", 4));
auto obj = tonlib_api::make_object<tonlib_api::dns_resolve>(nullptr, host, category, 16);
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), promise = std::move(promise), host = std::move(host)](
td::Result<tonlib_api::object_ptr<tonlib_api::Object>> R) mutable {
td::Result<tonlib_api::object_ptr<tonlib_api::dns_resolved>> R) mutable {
if (R.is_error()) {
if (promise) {
promise.set_result(R.move_as_error());
}
} else {
auto v = R.move_as_ok();
auto obj = dynamic_cast<tonlib_api::dns_resolved *>(v.get());
if (obj == nullptr) {
promise.set_result(td::Status::Error("invalid response from tonlib"));
return;
}
ton::adnl::AdnlNodeIdShort id;
td::uint32 cnt = 0;
for (auto &e : obj->entries_) {
tonlib_api::downcast_call(*e->entry_.get(),
td::overloaded(
[&](tonlib_api::dns_entryDataAdnlAddress &x) {
if (td::Random::fast(0, cnt) == 0) {
auto R = ton::adnl::AdnlNodeIdShort::parse(x.adnl_address_->adnl_address_);
if (R.is_ok()) {
id = R.move_as_ok();
cnt++;
}
}
},
[&](auto &x) {}));
}
if (cnt == 0) {
if (promise) {
promise.set_error(td::Status::Error("no DNS entries"));
}
} else {
td::actor::send_closure(SelfId, &DNSResolver::save_to_cache, std::move(host), id);
if (promise) {
promise.set_result(id);
}
return;
}
auto obj = R.move_as_ok();
std::string result;
if (!obj->entries_.empty()) {
tonlib_api::downcast_call(*obj->entries_[0]->entry_,
td::overloaded(
[&](tonlib_api::dns_entryDataAdnlAddress &x) {
auto R = ton::adnl::AdnlNodeIdShort::parse(x.adnl_address_->adnl_address_);
if (R.is_ok()) {
ton::adnl::AdnlNodeIdShort id = R.move_as_ok();
result = id.serialize() + ".adnl";
}
},
[&](tonlib_api::dns_entryDataStorageAddress &x) {
result = td::to_lower(x.bag_id_.to_hex()) + ".bag";
},
[&](auto &x) {}));
}
if (result.empty()) {
if (promise) {
promise.set_error(td::Status::Error("no DNS entries"));
}
return;
}
td::actor::send_closure(SelfId, &DNSResolver::save_to_cache, std::move(host), result);
if (promise) {
promise.set_result(std::move(result));
}
});
td::actor::send_closure(tonlib_client_, &TonlibClient::send_request, std::move(obj), std::move(P));
td::actor::send_closure(tonlib_client_, &tonlib::TonlibClientWrapper::send_request<tonlib_api::dns_resolve>,
std::move(obj), std::move(P));
}
void DNSResolver::save_to_cache(std::string host, ton::adnl::AdnlNodeIdShort id) {
void DNSResolver::save_to_cache(std::string host, std::string address) {
CacheEntry &entry = cache_[host];
entry.id_ = id;
entry.address_ = address;
entry.created_at_ = td::Time::now();
}

13
rldp-http-proxy/DNSResolver.h
View file

@ -25,24 +25,25 @@
*/
#pragma once
#include "td/actor/actor.h"
#include "TonlibClient.h"
#include "tonlib/tonlib/TonlibClientWrapper.h"
#include "adnl/adnl.h"
#include "td/actor/PromiseFuture.h"
class DNSResolver : public td::actor::Actor {
public:
explicit DNSResolver(td::actor::ActorId<TonlibClient> tonlib_client);
explicit DNSResolver(td::actor::ActorId<tonlib::TonlibClientWrapper> tonlib_client);
void start_up() override;
void resolve(std::string host, td::Promise<ton::adnl::AdnlNodeIdShort> promise);
void resolve(std::string host, td::Promise<std::string> promise);
private:
void save_to_cache(std::string host, ton::adnl::AdnlNodeIdShort id);
void sync();
void save_to_cache(std::string host, std::string address);
td::actor::ActorId<TonlibClient> tonlib_client_;
td::actor::ActorId<tonlib::TonlibClientWrapper> tonlib_client_;
struct CacheEntry {
ton::adnl::AdnlNodeIdShort id_;
std::string address_;
double created_at_;
};
std::map<std::string, CacheEntry> cache_;

426
rldp-http-proxy/rldp-http-proxy.cpp
View file

@ -33,7 +33,6 @@
#include "td/utils/FileLog.h"
#include "td/utils/Random.h"
#include "td/utils/filesystem.h"
#include "td/utils/overloaded.h"
#include "auto/tl/ton_api_json.h"
#include "auto/tl/tonlib_api.hpp"
@ -55,7 +54,7 @@
#include "td/utils/BufferedFd.h"
#include "common/delay.h"
#include "TonlibClient.h"
#include "tonlib/tonlib/TonlibClientWrapper.h"
#include "DNSResolver.h"
#if TD_DARWIN || TD_LINUX
@ -117,7 +116,7 @@ class HttpRemote : public td::actor::Actor {
}
});
td::actor::send_closure(client_, &ton::http::HttpClient::send_request, std::move(request), std::move(payload),
td::Timestamp::in(30.0), std::move(P));
td::Timestamp::never(), std::move(P));
} else {
ton::http::answer_error(ton::http::HttpStatusCode::status_bad_request, "", std::move(promise));
}
@ -134,6 +133,15 @@ td::BufferSlice create_error_response(const std::string &proto_version, int code
proto_version, code, reason, std::vector<ton::tl_object_ptr<ton::ton_api::http_header>>(), true);
}
const std::string PROXY_SITE_VERISON_HEADER_NAME = "Ton-Proxy-Site-Version";
const std::string PROXY_ENTRY_VERISON_HEADER_NAME = "Ton-Proxy-Entry-Version";
const std::string PROXY_VERSION_HEADER = PSTRING() << "Commit: " << GitMetadata::CommitSHA1()
<< ", Date: " << GitMetadata::CommitDate();
using RegisteredPayloadSenderGuard =
std::unique_ptr<std::pair<td::actor::ActorId<RldpHttpProxy>, td::Bits256>,
std::function<void(std::pair<td::actor::ActorId<RldpHttpProxy>, td::Bits256> *)>>;
class HttpRldpPayloadReceiver : public td::actor::Actor {
public:
HttpRldpPayloadReceiver(std::shared_ptr<ton::http::HttpPayload> payload, td::Bits256 transfer_id,
@ -244,10 +252,10 @@ class HttpRldpPayloadReceiver : public td::actor::Actor {
private:
static constexpr size_t watermark() {
return 1 << 15;
return (1 << 21) - (1 << 11);
}
static constexpr size_t chunk_size() {
return 1 << 17;
return (1 << 21) - (1 << 11);
}
std::shared_ptr<ton::http::HttpPayload> payload_;
@ -268,12 +276,14 @@ class HttpRldpPayloadSender : public td::actor::Actor {
public:
HttpRldpPayloadSender(std::shared_ptr<ton::http::HttpPayload> payload, td::Bits256 transfer_id,
ton::adnl::AdnlNodeIdShort local_id, td::actor::ActorId<ton::adnl::Adnl> adnl,
td::actor::ActorId<ton::rldp::Rldp> rldp, bool is_tunnel = false)
td::actor::ActorId<ton::rldp::Rldp> rldp, td::actor::ActorId<RldpHttpProxy> proxy,
bool is_tunnel = false)
: payload_(std::move(payload))
, id_(transfer_id)
, local_id_(local_id)
, adnl_(adnl)
, rldp_(rldp)
, proxy_(proxy)
, is_tunnel_(is_tunnel) {
}
@ -289,52 +299,10 @@ class HttpRldpPayloadSender : public td::actor::Actor {
return x;
}
void start_up() override {
class AdnlCb : public ton::adnl::Adnl::Callback {
public:
AdnlCb(td::actor::ActorId<HttpRldpPayloadSender> id) : self_id_(id) {
}
void receive_message(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst,
td::BufferSlice data) override {
LOG(INFO) << "http payload sender: dropping message";
}
void receive_query(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data,
td::Promise<td::BufferSlice> promise) override {
td::actor::send_closure(self_id_, &HttpRldpPayloadSender::receive_query, std::move(data), std::move(promise));
}
void start_up() override;
private:
td::actor::ActorId<HttpRldpPayloadSender> self_id_;
};
td::actor::send_closure(adnl_, &ton::adnl::Adnl::subscribe, local_id_, generate_prefix(),
std::make_unique<AdnlCb>(actor_id(this)));
class Cb : public ton::http::HttpPayload::Callback {
public:
Cb(td::actor::ActorId<HttpRldpPayloadSender> id, size_t watermark) : self_id_(id), watermark_(watermark) {
}
void run(size_t ready_bytes) override {
if (!reached_ && ready_bytes >= watermark_) {
reached_ = true;
td::actor::send_closure(self_id_, &HttpRldpPayloadSender::try_answer_query, false);
} else if (reached_ && ready_bytes < watermark_) {
reached_ = false;
}
}
void completed() override {
td::actor::send_closure(self_id_, &HttpRldpPayloadSender::try_answer_query, false);
}
private:
bool reached_ = false;
td::actor::ActorId<HttpRldpPayloadSender> self_id_;
size_t watermark_;
};
payload_->add_callback(
std::make_unique<Cb>(actor_id(this), is_tunnel_ ? 1 : ton::http::HttpRequest::low_watermark()));
alarm_timestamp() = td::Timestamp::in(is_tunnel_ ? 60.0 : 10.0);
void registered_sender(RegisteredPayloadSenderGuard guard) {
guard_ = std::move(guard);
}
void try_answer_query(bool from_timer = false) {
@ -391,13 +359,9 @@ class HttpRldpPayloadSender : public td::actor::Actor {
try_answer_query(false);
}
void receive_query(td::BufferSlice data, td::Promise<td::BufferSlice> promise) {
auto F = ton::fetch_tl_object<ton::ton_api::http_getNextPayloadPart>(data, true);
if (F.is_error()) {
LOG(INFO) << "failed to parse query: " << F.move_as_error();
return;
}
send_data(F.move_as_ok(), std::move(promise));
void receive_query(ton::tl_object_ptr<ton::ton_api::http_getNextPayloadPart> f,
td::Promise<td::BufferSlice> promise) {
send_data(std::move(f), std::move(promise));
}
void alarm() override {
@ -429,24 +393,22 @@ class HttpRldpPayloadSender : public td::actor::Actor {
stop();
}
void tear_down() override {
td::actor::send_closure(adnl_, &ton::adnl::Adnl::unsubscribe, local_id_, generate_prefix());
}
private:
static constexpr size_t watermark() {
return 1 << 15;
return (1 << 21) - (1 << 11);
}
std::shared_ptr<ton::http::HttpPayload> payload_;
td::Bits256 id_;
RegisteredPayloadSenderGuard guard_;
td::int32 seqno_ = 0;
ton::adnl::AdnlNodeIdShort local_id_;
td::actor::ActorId<ton::adnl::Adnl> adnl_;
td::actor::ActorId<ton::rldp::Rldp> rldp_;
td::actor::ActorId<RldpHttpProxy> proxy_;
size_t cur_query_size_;
td::Promise<td::BufferSlice> cur_query_promise_;
@ -462,7 +424,9 @@ class TcpToRldpRequestSender : public td::actor::Actor {
std::shared_ptr<ton::http::HttpPayload> request_payload,
td::Promise<std::pair<std::unique_ptr<ton::http::HttpResponse>, std::shared_ptr<ton::http::HttpPayload>>> promise,
td::actor::ActorId<ton::adnl::Adnl> adnl, td::actor::ActorId<ton::dht::Dht> dht,
td::actor::ActorId<ton::rldp::Rldp> rldp, td::actor::ActorId<DNSResolver> dns_resolver)
td::actor::ActorId<ton::rldp::Rldp> rldp, td::actor::ActorId<RldpHttpProxy> proxy,
td::actor::ActorId<DNSResolver> dns_resolver,
ton::adnl::AdnlNodeIdShort storage_gateway)
: local_id_(local_id)
, host_(std::move(host))
, request_(std::move(request))
@ -471,17 +435,21 @@ class TcpToRldpRequestSender : public td::actor::Actor {
, adnl_(adnl)
, dht_(dht)
, rldp_(rldp)
, dns_resolver_(dns_resolver) {
}
void start_up() override {
resolve();
, proxy_(proxy)
, dns_resolver_(dns_resolver)
, storage_gateway_(storage_gateway) {
}
void resolve();
void start_up() override {
td::Random::secure_bytes(id_.as_slice());
request_tl_ = request_->store_tl(id_);
resolve(host_);
}
void resolve(std::string host);
void resolved(ton::adnl::AdnlNodeIdShort id) {
dst_ = id;
td::Random::secure_bytes(id_.as_slice());
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
if (R.is_error()) {
@ -492,10 +460,10 @@ class TcpToRldpRequestSender : public td::actor::Actor {
});
td::actor::create_actor<HttpRldpPayloadSender>("HttpPayloadSender", request_payload_, id_, local_id_, adnl_, rldp_,
is_tunnel())
proxy_, is_tunnel())
.release();
auto f = ton::serialize_tl_object(request_->store_tl(id_), true);
auto f = ton::serialize_tl_object(request_tl_, true);
td::actor::send_closure(rldp_, &ton::rldp::Rldp::send_query_ex, local_id_, dst_, "http request over rldp",
std::move(P), td::Timestamp::in(30.0), std::move(f), 16 << 10);
}
@ -523,6 +491,7 @@ class TcpToRldpRequestSender : public td::actor::Actor {
}
response_->add_header(std::move(h));
}
response_->add_header({PROXY_ENTRY_VERISON_HEADER_NAME, PROXY_VERSION_HEADER});
auto S = response_->complete_parse_header();
if (S.is_error()) {
abort_query(S.move_as_error());
@ -573,13 +542,18 @@ class TcpToRldpRequestSender : public td::actor::Actor {
std::unique_ptr<ton::http::HttpRequest> request_;
std::shared_ptr<ton::http::HttpPayload> request_payload_;
ton::tl_object_ptr<ton::ton_api::http_request> request_tl_;
td::Promise<std::pair<std::unique_ptr<ton::http::HttpResponse>, std::shared_ptr<ton::http::HttpPayload>>> promise_;
td::actor::ActorId<ton::adnl::Adnl> adnl_;
td::actor::ActorId<ton::dht::Dht> dht_;
td::actor::ActorId<ton::rldp::Rldp> rldp_;
td::actor::ActorId<RldpHttpProxy> proxy_;
td::actor::ActorId<DNSResolver> dns_resolver_;
ton::adnl::AdnlNodeIdShort storage_gateway_ = ton::adnl::AdnlNodeIdShort::zero();
bool dns_resolve_sent_ = false;
std::unique_ptr<ton::http::HttpResponse> response_;
std::shared_ptr<ton::http::HttpPayload> response_payload_;
@ -588,47 +562,28 @@ class TcpToRldpRequestSender : public td::actor::Actor {
class RldpTcpTunnel : public td::actor::Actor, private td::ObserverBase {
public:
RldpTcpTunnel(td::Bits256 transfer_id, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort local_id,
td::actor::ActorId<ton::adnl::Adnl> adnl, td::actor::ActorId<ton::rldp::Rldp> rldp, td::SocketFd fd)
td::actor::ActorId<ton::adnl::Adnl> adnl, td::actor::ActorId<ton::rldp::Rldp> rldp,
td::actor::ActorId<RldpHttpProxy> proxy, td::SocketFd fd)
: id_(transfer_id)
, src_(src)
, local_id_(local_id)
, adnl_(std::move(adnl))
, rldp_(std::move(rldp))
, proxy_(std::move(proxy))
, fd_(std::move(fd)) {
}
void start_up() override {
self_ = actor_id(this);
td::actor::SchedulerContext::get()->get_poll().subscribe(fd_.get_poll_info().extract_pollable_fd(this),
td::PollFlags::ReadWrite());
class Cb : public ton::adnl::Adnl::Callback {
public:
explicit Cb(td::actor::ActorId<RldpTcpTunnel> id) : self_id_(std::move(id)) {
}
void receive_message(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst,
td::BufferSlice data) override {
LOG(INFO) << "rldp tcp tunnel: dropping message";
}
void receive_query(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data,
td::Promise<td::BufferSlice> promise) override {
td::actor::send_closure(self_id_, &RldpTcpTunnel::receive_query, std::move(data), std::move(promise));
}
private:
td::actor::ActorId<RldpTcpTunnel> self_id_;
};
td::actor::send_closure(adnl_, &ton::adnl::Adnl::subscribe, local_id_, generate_prefix(),
std::make_unique<Cb>(actor_id(this)));
process();
}
void start_up() override;
void tear_down() override {
LOG(INFO) << "RldpTcpTunnel: tear_down";
td::actor::send_closure(adnl_, &ton::adnl::Adnl::unsubscribe, local_id_, generate_prefix());
td::actor::SchedulerContext::get()->get_poll().unsubscribe(fd_.get_poll_info().get_pollable_fd_ref());
}
void registered_sender(RegisteredPayloadSenderGuard guard) {
guard_ = std::move(guard);
}
void notify() override {
td::actor::send_closure(self_, &RldpTcpTunnel::process);
}
@ -642,19 +597,14 @@ class RldpTcpTunnel : public td::actor::Actor, private td::ObserverBase {
td::actor::send_closure(SelfId, &RldpTcpTunnel::got_data_from_rldp, std::move(R));
});
auto f = ton::create_serialize_tl_object<ton::ton_api::http_getNextPayloadPart>(id_, out_seqno_++, 1 << 17);
auto f = ton::create_serialize_tl_object<ton::ton_api::http_getNextPayloadPart>(id_, out_seqno_++,
(1 << 21) - (1 << 11));
td::actor::send_closure(rldp_, &ton::rldp::Rldp::send_query_ex, local_id_, src_, "payload part", std::move(P),
td::Timestamp::in(60.0), std::move(f), (1 << 18) + 1024);
td::Timestamp::in(60.0), std::move(f), (1 << 21) + 1024);
}
void receive_query(td::BufferSlice data, td::Promise<td::BufferSlice> promise) {
auto F = ton::fetch_tl_object<ton::ton_api::http_getNextPayloadPart>(data, true);
if (F.is_error()) {
LOG(INFO) << "failed to parse query: " << F.error();
promise.set_error(F.move_as_error());
return;
}
auto f = F.move_as_ok();
void receive_query(ton::tl_object_ptr<ton::ton_api::http_getNextPayloadPart> f,
td::Promise<td::BufferSlice> promise) {
if (cur_promise_) {
LOG(INFO) << "failed to process query: previous query is active";
promise.set_error(td::Status::Error("previous query is active"));
@ -772,11 +722,13 @@ class RldpTcpTunnel : public td::actor::Actor, private td::ObserverBase {
}
td::Bits256 id_;
RegisteredPayloadSenderGuard guard_;
ton::adnl::AdnlNodeIdShort src_;
ton::adnl::AdnlNodeIdShort local_id_;
td::actor::ActorId<ton::adnl::Adnl> adnl_;
td::actor::ActorId<ton::rldp::Rldp> rldp_;
td::actor::ActorId<RldpHttpProxy> proxy_;
td::BufferedFd<td::SocketFd> fd_;
@ -795,15 +747,17 @@ class RldpToTcpRequestSender : public td::actor::Actor {
std::unique_ptr<ton::http::HttpRequest> request,
std::shared_ptr<ton::http::HttpPayload> request_payload, td::Promise<td::BufferSlice> promise,
td::actor::ActorId<ton::adnl::Adnl> adnl, td::actor::ActorId<ton::rldp::Rldp> rldp,
td::actor::ActorId<HttpRemote> remote)
td::actor::ActorId<RldpHttpProxy> proxy, td::actor::ActorId<HttpRemote> remote)
: id_(id)
, local_id_(local_id)
, dst_(dst)
, request_(std::move(request))
, request_payload_(std::move(request_payload))
, proto_version_(request_->proto_version())
, promise_(std::move(promise))
, adnl_(adnl)
, rldp_(rldp)
, proxy_(proxy)
, remote_(std::move(remote)) {
}
void start_up() override {
@ -824,11 +778,10 @@ class RldpToTcpRequestSender : public td::actor::Actor {
}
void got_result(std::pair<std::unique_ptr<ton::http::HttpResponse>, std::shared_ptr<ton::http::HttpPayload>> R) {
if (R.first->need_payload()) {
td::actor::create_actor<HttpRldpPayloadSender>("HttpPayloadSender(R)", std::move(R.second), id_, local_id_, adnl_,
rldp_)
.release();
}
td::actor::create_actor<HttpRldpPayloadSender>("HttpPayloadSender(R)", std::move(R.second), id_, local_id_, adnl_,
rldp_, proxy_)
.release();
R.first->add_header({PROXY_SITE_VERISON_HEADER_NAME, PROXY_VERSION_HEADER});
auto f = ton::serialize_tl_object(R.first->store_tl(), true);
promise_.set_value(std::move(f));
stop();
@ -836,7 +789,7 @@ class RldpToTcpRequestSender : public td::actor::Actor {
void abort_query(td::Status error) {
LOG(INFO) << "aborting http over rldp query: " << error;
promise_.set_result(create_error_response(request_->proto_version(), 502, "Bad Gateway"));
promise_.set_result(create_error_response(proto_version_, 502, "Bad Gateway"));
stop();
}
@ -848,11 +801,13 @@ class RldpToTcpRequestSender : public td::actor::Actor {
std::unique_ptr<ton::http::HttpRequest> request_;
std::shared_ptr<ton::http::HttpPayload> request_payload_;
std::string proto_version_;
td::Promise<td::BufferSlice> promise_;
td::actor::ActorId<ton::adnl::Adnl> adnl_;
td::actor::ActorId<ton::rldp::Rldp> rldp_;
td::actor::ActorId<RldpHttpProxy> proxy_;
td::actor::ActorId<HttpRemote> remote_;
};
@ -972,7 +927,7 @@ class RldpHttpProxy : public td::actor::Actor {
auto tonlib_options = tonlib_api::make_object<tonlib_api::options>(
tonlib_api::make_object<tonlib_api::config>(conf_dataR.move_as_ok().as_slice().str(), "", false, false),
tonlib_api::make_object<tonlib_api::keyStoreTypeInMemory>());
tonlib_client_ = td::actor::create_actor<TonlibClient>("tonlibclient", std::move(tonlib_options));
tonlib_client_ = td::actor::create_actor<tonlib::TonlibClientWrapper>("tonlibclient", std::move(tonlib_options));
dns_resolver_ = td::actor::create_actor<DNSResolver>("dnsresolver", tonlib_client_.get());
}
@ -1036,15 +991,9 @@ class RldpHttpProxy : public td::actor::Actor {
}
}
{
if (is_client_) {
auto D = ton::dht::Dht::create_client(dht_id_, "", dht_config_, keyring_.get(), adnl_.get());
D.ensure();
dht_ = D.move_as_ok();
} else {
auto D = ton::dht::Dht::create(dht_id_, db_root_, dht_config_, keyring_.get(), adnl_.get());
D.ensure();
dht_ = D.move_as_ok();
}
auto D = ton::dht::Dht::create_client(dht_id_, "", dht_config_, keyring_.get(), adnl_.get());
D.ensure();
dht_ = D.move_as_ok();
td::actor::send_closure(adnl_, &ton::adnl::Adnl::register_dht_node, dht_.get());
}
if (port_) {
@ -1067,6 +1016,22 @@ class RldpHttpProxy : public td::actor::Actor {
server_ = ton::http::HttpServer::create(port_, std::make_shared<Cb>(actor_id(this)));
}
class AdnlPayloadCb : public ton::adnl::Adnl::Callback {
public:
AdnlPayloadCb(td::actor::ActorId<RldpHttpProxy> id) : self_id_(id) {
}
void receive_message(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst,
td::BufferSlice data) override {
}
void receive_query(ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data,
td::Promise<td::BufferSlice> promise) override {
td::actor::send_closure(self_id_, &RldpHttpProxy::receive_payload_part_request, std::move(data),
std::move(promise));
}
private:
td::actor::ActorId<RldpHttpProxy> self_id_;
};
for (auto &serv_id : server_ids_) {
class AdnlCb : public ton::adnl::Adnl::Callback {
public:
@ -1087,9 +1052,18 @@ class RldpHttpProxy : public td::actor::Actor {
td::actor::send_closure(adnl_, &ton::adnl::Adnl::subscribe, serv_id,
ton::adnl::Adnl::int_to_bytestring(ton::ton_api::http_request::ID),
std::make_unique<AdnlCb>(actor_id(this)));
if (local_id_ != serv_id) {
td::actor::send_closure(adnl_, &ton::adnl::Adnl::subscribe, serv_id,
ton::adnl::Adnl::int_to_bytestring(ton::ton_api::http_getNextPayloadPart::ID),
std::make_unique<AdnlPayloadCb>(actor_id(this)));
}
}
td::actor::send_closure(adnl_, &ton::adnl::Adnl::subscribe, local_id_,
ton::adnl::Adnl::int_to_bytestring(ton::ton_api::http_getNextPayloadPart::ID),
std::make_unique<AdnlPayloadCb>(actor_id(this)));
rldp_ = ton::rldp::Rldp::create(adnl_.get());
td::actor::send_closure(rldp_, &ton::rldp::Rldp::set_default_mtu, 16 << 10);
td::actor::send_closure(rldp_, &ton::rldp::Rldp::add_id, local_id_);
for (auto &serv_id : server_ids_) {
td::actor::send_closure(rldp_, &ton::rldp::Rldp::add_id, serv_id);
@ -1132,15 +1106,20 @@ class RldpHttpProxy : public td::actor::Actor {
}
}
std::transform(host.begin(), host.end(), host.begin(), [](unsigned char c) { return std::tolower(c); });
if (!proxy_all_ &&
(host.size() < 5 || (host.substr(host.size() - 4) != ".ton" && host.substr(host.size() - 5) != ".adnl"))) {
bool allow = proxy_all_;
for (const char* suffix : {".adnl", ".ton", ".bag"}) {
if (td::ends_with(host, td::Slice(suffix))) {
allow = true;
}
}
if (!allow) {
promise.set_error(td::Status::Error(ton::ErrorCode::error, "bad server name"));
return;
}
td::actor::create_actor<TcpToRldpRequestSender>("outboundreq", local_id_, host, std::move(request),
std::move(payload), std::move(promise), adnl_.get(), dht_.get(),
rldp_.get(), dns_resolver_.get())
rldp_.get(), actor_id(this), dns_resolver_.get(), storage_gateway_)
.release();
}
@ -1236,7 +1215,7 @@ class RldpHttpProxy : public td::actor::Actor {
LOG(INFO) << "starting HTTP over RLDP request";
td::actor::create_actor<RldpToTcpRequestSender>("inboundreq", f->id_, dst, src, std::move(request),
payload.move_as_ok(), std::move(promise), adnl_.get(), rldp_.get(),
server.http_remote_.get())
actor_id(this), server.http_remote_.get())
.release();
}
@ -1248,10 +1227,52 @@ class RldpHttpProxy : public td::actor::Actor {
return;
}
td::actor::create_actor<RldpTcpTunnel>(td::actor::ActorOptions().with_name("tunnel").with_poll(), id, src, local_id,
adnl_.get(), rldp_.get(), fd.move_as_ok()).release();
adnl_.get(), rldp_.get(), actor_id(this), fd.move_as_ok())
.release();
std::vector<ton::tl_object_ptr<ton::ton_api::http_header>> headers;
headers.push_back(
ton::create_tl_object<ton::ton_api::http_header>(PROXY_SITE_VERISON_HEADER_NAME, PROXY_VERSION_HEADER));
promise.set_result(ton::create_serialize_tl_object<ton::ton_api::http_response>(
http_version, 200, "Connection Established", std::vector<ton::tl_object_ptr<ton::ton_api::http_header>>(),
false));
http_version, 200, "Connection Established", std::move(headers), false));
}
void receive_payload_part_request(td::BufferSlice data, td::Promise<td::BufferSlice> promise) {
auto F = ton::fetch_tl_object<ton::ton_api::http_getNextPayloadPart>(data, true);
if (F.is_error()) {
LOG(INFO) << "failed to parse query: " << F.error();
promise.set_error(F.move_as_error());
return;
}
auto f = F.move_as_ok();
auto it = payload_senders_.find(f->id_);
if (it == payload_senders_.end()) {
LOG(INFO) << "failed to answer query: unknown request id";
promise.set_error(td::Status::Error("unknown request id"));
return;
}
it->second(std::move(f), std::move(promise));
}
void register_payload_sender(
td::Bits256 id,
std::function<void(ton::tl_object_ptr<ton::ton_api::http_getNextPayloadPart>, td::Promise<td::BufferSlice>)> f,
td::Promise<RegisteredPayloadSenderGuard> promise) {
auto &f1 = payload_senders_[id];
if (f1) {
promise.set_error(td::Status::Error("duplicate id"));
return;
}
f1 = std::move(f);
promise.set_result(RegisteredPayloadSenderGuard(
new std::pair<td::actor::ActorId<RldpHttpProxy>, td::Bits256>(actor_id(this), id),
[](std::pair<td::actor::ActorId<RldpHttpProxy>, td::Bits256> *x) {
td::actor::send_closure(x->first, &RldpHttpProxy::unregister_payload_sender, x->second);
delete x;
}));
}
void unregister_payload_sender(td::Bits256 id) {
payload_senders_.erase(id);
}
void add_adnl_addr(ton::adnl::AdnlNodeIdShort id) {
@ -1266,6 +1287,10 @@ class RldpHttpProxy : public td::actor::Actor {
proxy_all_ = value;
}
void set_storage_gateway(ton::adnl::AdnlNodeIdShort id) {
storage_gateway_ = id;
}
private:
struct Host {
struct Server {
@ -1301,13 +1326,45 @@ class RldpHttpProxy : public td::actor::Actor {
std::string db_root_ = ".";
bool proxy_all_ = false;
td::actor::ActorOwn<TonlibClient> tonlib_client_;
td::actor::ActorOwn<tonlib::TonlibClientWrapper> tonlib_client_;
td::actor::ActorOwn<DNSResolver> dns_resolver_;
ton::adnl::AdnlNodeIdShort storage_gateway_ = ton::adnl::AdnlNodeIdShort::zero();
std::map<td::Bits256,
std::function<void(ton::tl_object_ptr<ton::ton_api::http_getNextPayloadPart>, td::Promise<td::BufferSlice>)>>
payload_senders_;
};
void TcpToRldpRequestSender::resolve() {
auto S = td::Slice(host_);
if (S.size() >= 5 && S.substr(S.size() - 5) == ".adnl") {
void TcpToRldpRequestSender::resolve(std::string host) {
auto S = td::Slice(host);
if (td::ends_with(S, ".bag")) {
if (storage_gateway_.is_zero()) {
abort_query(td::Status::Error("storage gateway is not set"));
return;
}
td::Slice bag_id = S.substr(0, S.size() - 4);
td::Slice url = request_tl_->url_;
if (td::begins_with(url, "http://")) {
url.remove_prefix(7);
}
size_t pos = url.find('/');
if (pos == td::Slice::npos) {
url = "/";
} else {
url = url.substr(pos);
}
request_tl_->url_ = (PSTRING() << "/gateway/" << bag_id << url);
host = storage_gateway_.serialize() + ".adnl";
for (auto& header : request_tl_->headers_) {
if (td::to_lower(header->name_) == "host") {
header->value_ = host;
break;
}
}
resolved(storage_gateway_);
return;
}
if (td::ends_with(S, ".adnl")) {
S.truncate(S.size() - 5);
auto R = ton::adnl::AdnlNodeIdShort::parse(S);
if (R.is_error()) {
@ -1317,15 +1374,81 @@ void TcpToRldpRequestSender::resolve() {
resolved(R.move_as_ok());
return;
}
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<ton::adnl::AdnlNodeIdShort> R) {
if (R.is_error()) {
td::actor::send_closure(SelfId, &TcpToRldpRequestSender::abort_query,
R.move_as_error_prefix("failed to resolve: "));
} else {
td::actor::send_closure(SelfId, &TcpToRldpRequestSender::resolved, R.move_as_ok());
if (dns_resolve_sent_) {
abort_query(td::Status::Error(PSTRING() << "unexpected dns result: " << host));
return;
}
dns_resolve_sent_ = true;
td::actor::send_closure(dns_resolver_, &DNSResolver::resolve, host,
[SelfId = actor_id(this)](td::Result<std::string> R) {
if (R.is_error()) {
td::actor::send_closure(SelfId, &TcpToRldpRequestSender::abort_query,
R.move_as_error_prefix("failed to resolve: "));
} else {
td::actor::send_closure(SelfId, &TcpToRldpRequestSender::resolve, R.move_as_ok());
}
});
}
void HttpRldpPayloadSender::start_up() {
td::actor::send_closure(
proxy_, &RldpHttpProxy::register_payload_sender, id_,
[SelfId = actor_id(this)](ton::tl_object_ptr<ton::ton_api::http_getNextPayloadPart> f,
td::Promise<td::BufferSlice> promise) {
td::actor::send_closure(SelfId, &HttpRldpPayloadSender::receive_query, std::move(f), std::move(promise));
},
[SelfId = actor_id(this)](td::Result<RegisteredPayloadSenderGuard> R) {
if (R.is_error()) {
LOG(INFO) << "Failed to register request sender: " << R.move_as_error();
}
td::actor::send_closure(SelfId, &HttpRldpPayloadSender::registered_sender, R.move_as_ok());
});
class Cb : public ton::http::HttpPayload::Callback {
public:
Cb(td::actor::ActorId<HttpRldpPayloadSender> id, size_t watermark) : self_id_(id), watermark_(watermark) {
}
});
td::actor::send_closure(dns_resolver_, &DNSResolver::resolve, host_, std::move(P));
void run(size_t ready_bytes) override {
if (!reached_ && ready_bytes >= watermark_) {
reached_ = true;
td::actor::send_closure(self_id_, &HttpRldpPayloadSender::try_answer_query, false);
} else if (reached_ && ready_bytes < watermark_) {
reached_ = false;
}
}
void completed() override {
td::actor::send_closure(self_id_, &HttpRldpPayloadSender::try_answer_query, false);
}
private:
bool reached_ = false;
td::actor::ActorId<HttpRldpPayloadSender> self_id_;
size_t watermark_;
};
payload_->add_callback(
std::make_unique<Cb>(actor_id(this), is_tunnel_ ? 1 : ton::http::HttpRequest::low_watermark()));
alarm_timestamp() = td::Timestamp::in(is_tunnel_ ? 60.0 : 10.0);
}
void RldpTcpTunnel::start_up() {
self_ = actor_id(this);
td::actor::SchedulerContext::get()->get_poll().subscribe(fd_.get_poll_info().extract_pollable_fd(this),
td::PollFlags::ReadWrite());
td::actor::send_closure(
proxy_, &RldpHttpProxy::register_payload_sender, id_,
[SelfId = actor_id(this)](ton::tl_object_ptr<ton::ton_api::http_getNextPayloadPart> f,
td::Promise<td::BufferSlice> promise) {
td::actor::send_closure(SelfId, &RldpTcpTunnel::receive_query, std::move(f), std::move(promise));
},
[SelfId = actor_id(this)](td::Result<RegisteredPayloadSenderGuard> R) {
if (R.is_error()) {
LOG(INFO) << "Failed to register request sender: " << R.move_as_error();
}
td::actor::send_closure(SelfId, &RldpTcpTunnel::registered_sender, R.move_as_ok());
});
process();
}
int main(int argc, char *argv[]) {
@ -1339,7 +1462,7 @@ int main(int argc, char *argv[]) {
td::log_interface = td::default_log_interface;
};
auto add_local_host = [&](const std::string& local, const std::string& remote) -> td::Status {
auto add_local_host = [&](const std::string &local, const std::string &remote) -> td::Status {
std::string host;
std::vector<td::uint16> ports;
auto p = local.find(':');
@ -1356,7 +1479,7 @@ int main(int argc, char *argv[]) {
}
try {
ports.push_back((td::uint16)std::stoul(local.substr(p, p2 - p)));
} catch (const std::logic_error& e) {
} catch (const std::logic_error &e) {
return td::Status::Error(PSLICE() << "Invalid port: " << local.substr(p, p2 - p));
}
p = p2 + 1;
@ -1426,9 +1549,7 @@ int main(int argc, char *argv[]) {
p.add_checked_option('L', "local",
"<hosthame>:<ports>, hostname that will be proxied to localhost\n"
"<ports> is a comma-separated list of ports (may be omitted, default: 80, 443)\n",
[&](td::Slice arg) -> td::Status {
return add_local_host(arg.str(), "127.0.0.1");
});
[&](td::Slice arg) -> td::Status { return add_local_host(arg.str(), "127.0.0.1"); });
p.add_option('D', "db", "db root",
[&](td::Slice arg) { td::actor::send_closure(x, &RldpHttpProxy::set_db_root, arg.str()); });
p.add_checked_option(
@ -1455,6 +1576,11 @@ int main(int argc, char *argv[]) {
logger_ = td::FileLog::create(fname.str()).move_as_ok();
td::log_interface = logger_.get();
});
p.add_checked_option('S', "storage-gateway", "adnl address of ton storage gateway", [&](td::Slice arg) -> td::Status {
TRY_RESULT(adnl, ton::adnl::AdnlNodeIdShort::parse(arg));
td::actor::send_closure(x, &RldpHttpProxy::set_storage_gateway, adnl);
return td::Status::OK();
});
p.add_checked_option('P', "proxy-all", "value=[YES|NO]. proxy all HTTP requests (default only *.ton and *.adnl)",
[&](td::Slice value) {
if (value == "YES" || value == "yes") {

9
rldp/rldp-in.hpp
View file

@ -71,7 +71,7 @@ class RldpIn : public RldpImpl {
void send_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, std::string name,
td::Promise<td::BufferSlice> promise, td::Timestamp timeout, td::BufferSlice data) override {
send_query_ex(src, dst, name, std::move(promise), timeout, std::move(data), default_mtu());
send_query_ex(src, dst, name, std::move(promise), timeout, std::move(data), default_mtu_);
}
void send_query_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, std::string name,
td::Promise<td::BufferSlice> promise, td::Timestamp timeout, td::BufferSlice data,
@ -96,11 +96,15 @@ class RldpIn : public RldpImpl {
void receive_message(adnl::AdnlNodeIdShort source, adnl::AdnlNodeIdShort local_id, TransferId transfer_id,
td::BufferSlice data);
void in_transfer_completed(TransferId transfer_id);
void in_transfer_completed(TransferId transfer_id, bool success);
void add_id(adnl::AdnlNodeIdShort local_id) override;
void get_conn_ip_str(adnl::AdnlNodeIdShort l_id, adnl::AdnlNodeIdShort p_id, td::Promise<td::string> promise) override;
void set_default_mtu(td::uint64 mtu) override {
default_mtu_ = mtu;
}
RldpIn(td::actor::ActorId<adnl::AdnlPeerTable> adnl) : adnl_(adnl) {
}
@ -116,6 +120,7 @@ class RldpIn : public RldpImpl {
std::set<TransferId> lru_set_;
RldpLru lru_;
td::uint32 lru_size_ = 0;
td::uint64 default_mtu_ = adnl::Adnl::get_mtu();
std::map<TransferId, td::uint64> max_size_;

11
rldp/rldp.cpp
View file

@ -116,9 +116,9 @@ void RldpIn::process_message_part(adnl::AdnlNodeIdShort source, adnl::AdnlNodeId
}
auto ite = max_size_.find(part.transfer_id_);
if (ite == max_size_.end()) {
if (static_cast<td::uint64>(part.total_size_) > default_mtu()) {
if (static_cast<td::uint64>(part.total_size_) > default_mtu_) {
VLOG(RLDP_NOTICE) << "dropping too big rldp packet of size=" << part.total_size_
<< " default_mtu=" << default_mtu();
<< " default_mtu=" << default_mtu_;
return;
}
} else {
@ -134,11 +134,11 @@ void RldpIn::process_message_part(adnl::AdnlNodeIdShort source, adnl::AdnlNodeId
}
auto P = td::PromiseCreator::lambda(
[SelfId = actor_id(this), source, local_id, transfer_id = part.transfer_id_](td::Result<td::BufferSlice> R) {
td::actor::send_closure(SelfId, &RldpIn::in_transfer_completed, transfer_id, R.is_ok());
if (R.is_error()) {
VLOG(RLDP_INFO) << "failed to receive: " << R.move_as_error();
return;
}
td::actor::send_closure(SelfId, &RldpIn::in_transfer_completed, transfer_id);
td::actor::send_closure(SelfId, &RldpIn::receive_message, source, local_id, transfer_id, R.move_as_ok());
});
@ -228,8 +228,9 @@ void RldpIn::transfer_completed(TransferId transfer_id) {
VLOG(RLDP_DEBUG) << "rldp: completed transfer " << transfer_id << "; " << senders_.size() << " out transfer pending ";
}
void RldpIn::in_transfer_completed(TransferId transfer_id) {
if (lru_set_.count(transfer_id) == 1) {
void RldpIn::in_transfer_completed(TransferId transfer_id, bool success) {
receivers_.erase(transfer_id);
if (!success || lru_set_.count(transfer_id) == 1) {
return;
}
while (lru_size_ >= lru_size()) {

6
rldp/rldp.h
View file

@ -28,15 +28,13 @@ class Rldp : public adnl::AdnlSenderInterface {
public:
virtual ~Rldp() = default;
static constexpr td::uint64 default_mtu() {
return adnl::Adnl::get_mtu();
}
virtual void add_id(adnl::AdnlNodeIdShort local_id) = 0;
virtual void send_message_ex(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::Timestamp timeout,
td::BufferSlice data) = 0;
virtual void set_default_mtu(td::uint64 mtu) = 0;
static td::actor::ActorOwn<Rldp> create(td::actor::ActorId<adnl::Adnl> adnl);
};

3
rldp2/BdwStats.cpp
View file

@ -18,6 +18,7 @@
*/
#include "BdwStats.h"
#include "rldp.hpp"
namespace ton {
namespace rldp2 {
@ -39,7 +40,7 @@ void BdwStats::on_packet_ack(const PacketInfo &info, td::Timestamp sent_at, td::
auto ack_passed = now.at() - info.delivered_now.at();
auto passed = td::max(sent_passed, ack_passed);
if (passed < 0.01) {
LOG(ERROR) << "Invalid passed " << passed;
VLOG(RLDP_INFO) << "Invalid passed " << passed;
}
auto delivered = delivered_count - info.delivered_count;
on_rate_sample((double)delivered / passed, now, info.is_paused);

7
rldp2/CMakeLists.txt
View file

@ -7,6 +7,9 @@ endif()
if (NOT GSL_FOUND)
find_package(GSL)
endif()
if (NOT BLAS_FOUND)
find_package(BLAS)
endif()
set(RLDP_SOURCE
Ack.cpp
@ -50,8 +53,8 @@ target_include_directories(rldp PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>/..
${OPENSSL_INCLUDE_DIR}
)
if (GSL_FOUND)
target_link_libraries(rldp2 PRIVATE gsl)
if (GSL_FOUND AND BLAS_FOUND)
target_link_libraries(rldp2 PRIVATE gsl blas)
target_compile_definitions(rldp2 PRIVATE -DTON_HAVE_GSL=1)
endif()
target_link_libraries(rldp2 PUBLIC tdutils tdactor fec adnl tl_api)

20
rldp2/RldpConnection.cpp
View file

@ -18,6 +18,7 @@
*/
#include "RldpConnection.h"
#include "rldp.hpp"
#include "td/utils/overloaded.h"
#include "td/utils/Random.h"
@ -83,7 +84,7 @@ td::Timestamp RldpConnection::loop_limits(td::Timestamp now) {
outbound_transfers_.erase(it);
to_on_sent_.emplace_back(limit->transfer_id, std::move(error));
} else {
LOG(ERROR) << "Timeout on unknown transfer " << limit->transfer_id.to_hex();
VLOG(RLDP_WARNING) << "Timeout on unknown transfer " << limit->transfer_id.to_hex();
}
}
limits_set_.erase(*limit);
@ -113,7 +114,7 @@ void RldpConnection::send(TransferId transfer_id, td::BufferSlice data, td::Time
td::Random::secure_bytes(transfer_id.as_slice());
} else {
if (outbound_transfers_.find(transfer_id) != outbound_transfers_.end()) {
LOG(WARNING) << "Skip resend of " << transfer_id.to_hex();
VLOG(RLDP_WARNING) << "Skip resend of " << transfer_id.to_hex();
return;
}
}
@ -143,17 +144,6 @@ void RldpConnection::loop_bbr(td::Timestamp now) {
double speed = bbr_.get_rate();
td::uint32 congestion_window = bbr_.get_window_size();
static td::Timestamp next;
//FIXME: remove this UNSAFE debug output
if (next.is_in_past(now)) {
next = td::Timestamp::in(1, now);
if (td::actor::core::ActorExecuteContext::get()->actor().get_actor_info_ptr()->get_name() == "Alice") {
LOG(ERROR) << "speed=" << td::format::as_size((td::int64)speed * 768) << " "
<< "cgw=" << td::format::as_size((td::int64)congestion_window * 768) << " "
<< "loss=" << loss_stats_.loss * 100 << "%";
}
}
pacer_.set_speed(speed);
congestion_window_ = congestion_window;
}
@ -301,7 +291,7 @@ void RldpConnection::receive_raw_obj(ton::ton_api::rldp2_messagePart &part) {
max_size = limit_it->max_size;
}
if (total_size > max_size) {
LOG(INFO) << "Drop too big rldp query " << part.total_size_ << " > " << max_size;
VLOG(RLDP_INFO) << "Drop too big rldp query " << part.total_size_ << " > " << max_size;
return;
}
@ -324,7 +314,7 @@ void RldpConnection::receive_raw_obj(ton::ton_api::rldp2_messagePart &part) {
}
return {};
}
if (in_part->receiver.on_received(part.seqno_, td::Timestamp::now())) {
if (in_part->receiver.on_received(part.seqno_ + 1, td::Timestamp::now())) {
TRY_STATUS_PREFIX(in_part->decoder->add_symbol({static_cast<td::uint32>(part.seqno_), std::move(part.data_)}),
td::Status::Error(ErrorCode::protoviolation, "invalid symbol"));
if (in_part->decoder->may_try_decode()) {

Some files were not shown because too many files have changed in this diff Show more