external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-02-15 04:32:21 +00:00
Code Issues Releases Wiki Activity

Merge branch 'testnet' into block-generation

Browse source
This commit is contained in:
SpyCheese 2023-01-11 16:10:57 +03:00
commit 1ccf25d6b7
60 changed files with 706 additions and 236 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.github/script/amd64-18.04.Dockerfile vendored
View file

@ -6,7 +6,8 @@ RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gpe
WORKDIR / WORKDIR /
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git ARG BRANCH
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH
WORKDIR /ton WORKDIR /ton
RUN mkdir /ton/build RUN mkdir /ton/build
@ -14,5 +15,5 @@ WORKDIR /ton/build
ENV CC clang ENV CC clang
ENV CXX clang++ ENV CXX clang++
ENV CCACHE_DISABLE 1 ENV CCACHE_DISABLE 1
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
RUN ninja tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client

7
.github/script/amd64-20.04.Dockerfile vendored
View file

@ -6,7 +6,8 @@ RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gpe
WORKDIR / WORKDIR /
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git ARG BRANCH
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH
WORKDIR /ton WORKDIR /ton
RUN mkdir /ton/build RUN mkdir /ton/build
@ -14,5 +15,5 @@ WORKDIR /ton/build
ENV CC clang ENV CC clang
ENV CXX clang++ ENV CXX clang++
ENV CCACHE_DISABLE 1 ENV CCACHE_DISABLE 1
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
RUN ninja tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client

7
.github/script/amd64-22.04.Dockerfile vendored
View file

@ -6,7 +6,8 @@ RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gpe
WORKDIR / WORKDIR /
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git ARG BRANCH
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH
WORKDIR /ton WORKDIR /ton
RUN mkdir /ton/build RUN mkdir /ton/build
@ -14,5 +15,5 @@ WORKDIR /ton/build
ENV CC clang ENV CC clang
ENV CXX clang++ ENV CXX clang++
ENV CCACHE_DISABLE 1 ENV CCACHE_DISABLE 1
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
RUN ninja tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client

7
.github/script/arm64-18.04.Dockerfile vendored
View file

@ -6,7 +6,8 @@ RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gpe
WORKDIR / WORKDIR /
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git ARG BRANCH
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH
WORKDIR /ton WORKDIR /ton
RUN mkdir /ton/build RUN mkdir /ton/build
@ -14,5 +15,5 @@ WORKDIR /ton/build
ENV CC clang ENV CC clang
ENV CXX clang++ ENV CXX clang++
ENV CCACHE_DISABLE 1 ENV CCACHE_DISABLE 1
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DTON_ARCH= .. RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= ..
RUN ninja tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id dht-server lite-client RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id dht-server lite-client

7
.github/script/arm64-20.04.Dockerfile vendored
View file

@ -6,7 +6,8 @@ RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gpe
WORKDIR / WORKDIR /
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git ARG BRANCH
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH
WORKDIR /ton WORKDIR /ton
RUN mkdir /ton/build RUN mkdir /ton/build
@ -14,5 +15,5 @@ WORKDIR /ton/build
ENV CC clang ENV CC clang
ENV CXX clang++ ENV CXX clang++
ENV CCACHE_DISABLE 1 ENV CCACHE_DISABLE 1
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DTON_ARCH= .. RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= ..
RUN ninja tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id dht-server lite-client RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id dht-server lite-client

7
.github/script/arm64-22.04.Dockerfile vendored
View file

@ -6,7 +6,8 @@ RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gpe
WORKDIR / WORKDIR /
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git ARG BRANCH
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH
WORKDIR /ton WORKDIR /ton
RUN mkdir /ton/build RUN mkdir /ton/build
@ -14,5 +15,5 @@ WORKDIR /ton/build
ENV CC clang ENV CC clang
ENV CXX clang++ ENV CXX clang++
ENV CCACHE_DISABLE 1 ENV CCACHE_DISABLE 1
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DTON_ARCH= .. RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= ..
RUN ninja tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id dht-server lite-client RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id dht-server lite-client

12
.github/workflows/docker-compile-ubuntu.yml vendored
View file

@ -22,14 +22,24 @@ jobs:
- name: Set up Docker Buildx - name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2 uses: docker/setup-buildx-action@v2
- name: Set output
id: vars
run: echo ::set-output name=short_ref::${GITHUB_REF#refs/*/}
- name: Check output
run: echo branch ${{ steps.vars.outputs.short_ref }}
- name: Build with docker buildx - name: Build with docker buildx
run: | run: |
mkdir build-${{matrix.ver}}-${{matrix.arch}} mkdir build-${{matrix.ver}}-${{matrix.arch}}
docker buildx build --platform=linux/${{matrix.arch}} --progress=plain --load . -t build-${{matrix.ver}}-${{matrix.arch}} -f .github/script/${{matrix.arch}}-${{matrix.ver}}.Dockerfile
docker buildx build --build-arg BRANCH=${{ steps.vars.outputs.short_ref }} --platform=linux/${{matrix.arch}} --progress=plain --load . -t build-${{matrix.ver}}-${{matrix.arch}} -f .github/script/${{matrix.arch}}-${{matrix.ver}}.Dockerfile
container_id=$(docker create --platform=linux/${{matrix.arch}} build-${{matrix.ver}}-${{matrix.arch}}) container_id=$(docker create --platform=linux/${{matrix.arch}} build-${{matrix.ver}}-${{matrix.arch}})
docker cp $container_id:/ton/build/dht-server/dht-server build-${{matrix.ver}}-${{matrix.arch}}/ docker cp $container_id:/ton/build/dht-server/dht-server build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/validator-engine/validator-engine build-${{matrix.ver}}-${{matrix.arch}}/ docker cp -a $container_id:/ton/build/validator-engine/validator-engine build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/validator-engine-console/validator-engine-console build-${{matrix.ver}}-${{matrix.arch}}/ docker cp -a $container_id:/ton/build/validator-engine-console/validator-engine-console build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/storage/storage-daemon/storage-daemon build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/storage/storage-daemon/storage-daemon-cli build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/crypto/fift build-${{matrix.ver}}-${{matrix.arch}}/ docker cp -a $container_id:/ton/build/crypto/fift build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/crypto/func build-${{matrix.ver}}-${{matrix.arch}}/ docker cp -a $container_id:/ton/build/crypto/func build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/crypto/create-state build-${{matrix.ver}}-${{matrix.arch}}/ docker cp -a $container_id:/ton/build/crypto/create-state build-${{matrix.ver}}-${{matrix.arch}}/

2
.github/workflows/docker-ubuntu-image.yml vendored
View file

@ -3,6 +3,8 @@ name: Docker Ubuntu 20.04 image
on: on:
workflow_dispatch: workflow_dispatch:
push: push:
branches:
- 'master'
env: env:
REGISTRY: ghcr.io REGISTRY: ghcr.io

4
.github/workflows/macos-11.7-compile.yml vendored
View file

@ -28,11 +28,13 @@ jobs:
mkdir build mkdir build
cd build cd build
cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=11.7 -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_BUILD_TYPE=Release .. cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=11.7 -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_BUILD_TYPE=Release ..
ninja fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc
- name: Find & copy binaries - name: Find & copy binaries
run: | run: |
mkdir artifacts mkdir artifacts
cp build/storage/storage-daemon/storage-daemon artifacts/
cp build/storage/storage-daemon/storage-daemon-cli artifacts/
cp build/crypto/fift artifacts/ cp build/crypto/fift artifacts/
cp build/crypto/func artifacts/ cp build/crypto/func artifacts/
cp build/crypto/create-state artifacts/ cp build/crypto/create-state artifacts/

4
.github/workflows/macos-12.6-compile.yml vendored
View file

@ -28,11 +28,13 @@ jobs:
mkdir build mkdir build
cd build cd build
cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=12.6 -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_BUILD_TYPE=Release .. cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=12.6 -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_BUILD_TYPE=Release ..
ninja fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc
- name: Find & copy binaries - name: Find & copy binaries
run: | run: |
mkdir artifacts mkdir artifacts
cp build/storage/storage-daemon/storage-daemon artifacts/
cp build/storage/storage-daemon/storage-daemon-cli artifacts/
cp build/crypto/fift artifacts/ cp build/crypto/fift artifacts/
cp build/crypto/func artifacts/ cp build/crypto/func artifacts/
cp build/crypto/create-state artifacts/ cp build/crypto/create-state artifacts/

12
.github/workflows/ubuntu-18.04-compile.yml vendored
View file

@ -18,6 +18,10 @@ jobs:
sudo apt update sudo apt update
sudo apt install -y build-essential git make cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev ninja-build sudo apt install -y build-essential git make cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev ninja-build
- name: Show CPU flags
run: |
cat /proc/cpuinfo
- name: Configure & Build - name: Configure & Build
run: | run: |
export CC=$(which clang) export CC=$(which clang)
@ -25,13 +29,13 @@ jobs:
export CCACHE_DISABLE=1 export CCACHE_DISABLE=1
mkdir build mkdir build
cd build cd build
cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
ninja fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state
- name: find & copy binaries - name: Find & copy binaries
run: | run: |
mkdir artifacts mkdir artifacts
cp build/crypto/fift build/crypto/tlbc build/crypto/func build/crypto/create-state build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli build/tonlib/libtonlibjson.so.0.5 build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy artifacts cp build/storage/storage-daemon/storage-daemon build/storage/storage-daemon/storage-daemon-cli build/crypto/fift build/crypto/tlbc build/crypto/func build/crypto/create-state build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli build/tonlib/libtonlibjson.so.0.5 build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy artifacts
cp -R crypto/smartcont artifacts/ cp -R crypto/smartcont artifacts/
cp -R crypto/fift/lib artifacts/ cp -R crypto/fift/lib artifacts/

12
.github/workflows/ubuntu-compile.yml vendored
View file

@ -20,6 +20,11 @@ jobs:
run: | run: |
sudo apt update sudo apt update
sudo apt install -y build-essential git make cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev ninja-build sudo apt install -y build-essential git make cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev ninja-build
- name: Show CPU flags
run: |
cat /proc/cpuinfo
- name: Configure & Build - name: Configure & Build
run: | run: |
export CC=$(which clang) export CC=$(which clang)
@ -27,12 +32,13 @@ jobs:
export CCACHE_DISABLE=1 export CCACHE_DISABLE=1
mkdir build-${{ matrix.os }} mkdir build-${{ matrix.os }}
cd build-${{ matrix.os }} cd build-${{ matrix.os }}
cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
ninja fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork
- name: Find & copy binaries - name: Find & copy binaries
run: | run: |
mkdir artifacts-${{ matrix.os }} mkdir artifacts-${{ matrix.os }}
cp build-${{ matrix.os }}/crypto/fift build-${{ matrix.os }}/crypto/tlbc build-${{ matrix.os }}/crypto/func build-${{ matrix.os }}/crypto/create-state build-${{ matrix.os }}/validator-engine-console/validator-engine-console build-${{ matrix.os }}/tonlib/tonlib-cli build-${{ matrix.os }}/tonlib/libtonlibjson.so.0.5 build-${{ matrix.os }}/http/http-proxy build-${{ matrix.os }}/rldp-http-proxy/rldp-http-proxy build-${{ matrix.os }}/dht-server/dht-server build-${{ matrix.os }}/lite-client/lite-client build-${{ matrix.os }}/validator-engine/validator-engine build-${{ matrix.os }}/utils/generate-random-id build-${{ matrix.os }}/utils/json2tlo build-${{ matrix.os }}/adnl/adnl-proxy artifacts-${{ matrix.os }} cp build-${{ matrix.os }}/storage/storage-daemon/storage-daemon build-${{ matrix.os }}/storage/storage-daemon/storage-daemon-cli build-${{ matrix.os }}/crypto/fift build-${{ matrix.os }}/crypto/tlbc build-${{ matrix.os }}/crypto/func build-${{ matrix.os }}/crypto/create-state build-${{ matrix.os }}/validator-engine-console/validator-engine-console build-${{ matrix.os }}/tonlib/tonlib-cli build-${{ matrix.os }}/tonlib/libtonlibjson.so.0.5 build-${{ matrix.os }}/http/http-proxy build-${{ matrix.os }}/rldp-http-proxy/rldp-http-proxy build-${{ matrix.os }}/dht-server/dht-server build-${{ matrix.os }}/lite-client/lite-client build-${{ matrix.os }}/validator-engine/validator-engine build-${{ matrix.os }}/utils/generate-random-id build-${{ matrix.os }}/utils/json2tlo build-${{ matrix.os }}/adnl/adnl-proxy artifacts-${{ matrix.os }}
cp -R crypto/smartcont artifacts-${{ matrix.os }} cp -R crypto/smartcont artifacts-${{ matrix.os }}
cp -R crypto/fift/lib artifacts-${{ matrix.os }} cp -R crypto/fift/lib artifacts-${{ matrix.os }}
- name: Upload artifacts - name: Upload artifacts

4
.github/workflows/win-2019-compile.yml vendored
View file

@ -57,7 +57,7 @@ jobs:
mkdir build mkdir build
cd build cd build
cmake -DREADLINE_INCLUDE_DIR=%root%\readline-5.0-1-lib\include\readline -DREADLINE_LIBRARY=%root%\readline-5.0-1-lib\lib\readline.lib -DZLIB_FOUND=1 -DMHD_FOUND=1 -DMHD_LIBRARY=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static\libmicrohttpd.lib -DMHD_INCLUDE_DIR=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static -DZLIB_INCLUDE_DIR=%root%\zlib -DZLIB_LIBRARY=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=%root%/openssl-1.1/x64/include -DOPENSSL_CRYPTO_LIBRARY=%root%/openssl-1.1/x64/lib/libcrypto.lib -DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj /W0" .. cmake -DREADLINE_INCLUDE_DIR=%root%\readline-5.0-1-lib\include\readline -DREADLINE_LIBRARY=%root%\readline-5.0-1-lib\lib\readline.lib -DZLIB_FOUND=1 -DMHD_FOUND=1 -DMHD_LIBRARY=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static\libmicrohttpd.lib -DMHD_INCLUDE_DIR=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static -DZLIB_INCLUDE_DIR=%root%\zlib -DZLIB_LIBRARY=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=%root%/openssl-1.1/x64/include -DOPENSSL_CRYPTO_LIBRARY=%root%/openssl-1.1/x64/lib/libcrypto.lib -DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj /W0" ..
cmake --build . --target fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork --config Release cmake --build . --target storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork --config Release
- name: Show executables - name: Show executables
run: | run: |
@ -77,7 +77,7 @@ jobs:
mkdir artifacts\smartcont mkdir artifacts\smartcont
mkdir artifacts\lib mkdir artifacts\lib
for %%I in (build\crypto\Release\fift.exe build\crypto\Release\tlbc.exe build\crypto\Release\func.exe build\crypto\Release\create-state.exe build\validator-engine-console\Release\validator-engine-console.exe build\tonlib\Release\tonlib-cli.exe build\tonlib\Release\tonlibjson.dll build\http\Release\http-proxy.exe build\rldp-http-proxy\Release\rldp-http-proxy.exe build\dht-server\Release\dht-server.exe build\lite-client\Release\lite-client.exe build\validator-engine\Release\validator-engine.exe build\utils\Release\generate-random-id.exe build\utils\Release\json2tlo.exe build\adnl\Release\adnl-proxy.exe) do copy %%I artifacts\ for %%I in (build\storage\storage-daemon\Release\storage-daemon.exe build\storage\storage-daemon\Release\storage-daemon-cli.exe build\crypto\Release\fift.exe build\crypto\Release\tlbc.exe build\crypto\Release\func.exe build\crypto\Release\create-state.exe build\validator-engine-console\Release\validator-engine-console.exe build\tonlib\Release\tonlib-cli.exe build\tonlib\Release\tonlibjson.dll build\http\Release\http-proxy.exe build\rldp-http-proxy\Release\rldp-http-proxy.exe build\dht-server\Release\dht-server.exe build\lite-client\Release\lite-client.exe build\validator-engine\Release\validator-engine.exe build\utils\Release\generate-random-id.exe build\utils\Release\json2tlo.exe build\adnl\Release\adnl-proxy.exe) do copy %%I artifacts\
xcopy /e /k /h /i crypto\smartcont artifacts\smartcont xcopy /e /k /h /i crypto\smartcont artifacts\smartcont
xcopy /e /k /h /i crypto\fift\lib artifacts\lib xcopy /e /k /h /i crypto\fift\lib artifacts\lib

15
Changelog.md
View file

@ -38,3 +38,18 @@ Besides the work of the core team, this update is based on the efforts of @aweso
* Added build of FunC and Fift to WASM * Added build of FunC and Fift to WASM
Besides the work of the core team, this update is based on the efforts of @tvorogme (debug improvements), @AlexeyFSL (WASM builds) and third-party security auditors. Besides the work of the core team, this update is based on the efforts of @tvorogme (debug improvements), @AlexeyFSL (WASM builds) and third-party security auditors.
## 12.2022 Update
Node update:
1. Improvements of ton-proxy: fixed few bugs, improved stability
2. Improved collator/validator checks, added optimization of storage stat calculation, generation and validation of new blocks is made safer
3. Some previously hard-coded parameters such as split/merge timings, max sizes and depths of internal and external messages, and others now can be updated by validators through setting ConfigParams. Max contract size added to configs.
4. Tonlib: updated raw.getTransactions (now it contains InitState), fixed long bytestrings truncation
5. abseil-cpp is updated to newer versions
6. Added configs for Token Bridge
7. LiteServers: a few bug fixes, added liteServer.getAccountStatePrunned method, improved work with not yet applied blocks.
8. Improved DHT: works for some NAT configurations, optimized excessive requests, added option for DHT network segregation.
9. FunC v0.4.0: added try/catch statements, added throw_arg functions, allowed in-place modification of global variables, forbidden ambiguous modification of local variables after it's usage in the same expression.
10. TON Storage: added storage-daemon (create, download bag of Files, storage-provider staff), added storage-daemon-cli
Besides the work of the core team, this update is based on the efforts of @vtamara (help with abseil-cpp upgrade), @krigga(in-place modification of global variables) and third-party security auditors.

12
crypto/block/block-parse.h
View file

@ -653,16 +653,20 @@ struct TrComputeInternal1 final : TLB_Complex {
}; };
struct ComputeSkipReason final : TLB { struct ComputeSkipReason final : TLB {
enum { cskip_no_state = 0, cskip_bad_state = 1, cskip_no_gas = 2 }; enum { cskip_no_state = 0, cskip_bad_state = 1, cskip_no_gas = 2, cskip_suspended = 3 };
int get_size(const vm::CellSlice& cs) const override { int get_size(const vm::CellSlice& cs) const override {
return 2; return cs.prefetch_ulong(2) == 3 ? 3 : 2;
} }
bool validate_skip(int* ops, vm::CellSlice& cs, bool weak = false) const override { bool validate_skip(int* ops, vm::CellSlice& cs, bool weak = false) const override {
return get_tag(cs) >= 0 && cs.advance(2); int tag = get_tag(cs);
return tag >= 0 && cs.advance(tag == 3 ? 3 : 2);
} }
int get_tag(const vm::CellSlice& cs) const override { int get_tag(const vm::CellSlice& cs) const override {
int t = (int)cs.prefetch_ulong(2); int t = (int)cs.prefetch_ulong(2);
return t < 3 ? t : -1; if (t == 3 && cs.prefetch_ulong(3) != 0b110) {
return -1;
}
return t;
} }
}; };

7
crypto/block/block.tlb
View file

@ -309,6 +309,7 @@ tr_phase_compute_vm$1 success:Bool msg_state_used:Bool
cskip_no_state$00 = ComputeSkipReason; cskip_no_state$00 = ComputeSkipReason;
cskip_bad_state$01 = ComputeSkipReason; cskip_bad_state$01 = ComputeSkipReason;
cskip_no_gas$10 = ComputeSkipReason; cskip_no_gas$10 = ComputeSkipReason;
cskip_suspended$110 = ComputeSkipReason;
tr_phase_action$_ success:Bool valid:Bool no_funds:Bool tr_phase_action$_ success:Bool valid:Bool no_funds:Bool
status_change:AccStatusChange status_change:AccStatusChange
@ -762,6 +763,10 @@ size_limits_config_v2#02 max_msg_bits:uint32 max_msg_cells:uint32 max_library_ce
max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_acc_state_bits:uint32 = SizeLimitsConfig; max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_acc_state_bits:uint32 = SizeLimitsConfig;
_ SizeLimitsConfig = ConfigParam 43; _ SizeLimitsConfig = ConfigParam 43;
// key is [ wc:int32 addr:uint256 ]
suspended_address_list#00 addresses:(HashmapE 288 Unit) suspended_until:uint32 = SuspendedAddressList;
_ SuspendedAddressList = ConfigParam 44;
oracle_bridge_params#_ bridge_address:bits256 oracle_mutlisig_address:bits256 oracles:(HashmapE 256 uint256) external_chain_address:bits256 = OracleBridgeParams; oracle_bridge_params#_ bridge_address:bits256 oracle_mutlisig_address:bits256 oracles:(HashmapE 256 uint256) external_chain_address:bits256 = OracleBridgeParams;
_ OracleBridgeParams = ConfigParam 71; // Ethereum bridge _ OracleBridgeParams = ConfigParam 71; // Ethereum bridge
_ OracleBridgeParams = ConfigParam 72; // Binance Smart Chain bridge _ OracleBridgeParams = ConfigParam 72; // Binance Smart Chain bridge
@ -883,6 +888,8 @@ cap_method_pubkey#71f4 = SmcCapability;
cap_is_wallet#2177 = SmcCapability; cap_is_wallet#2177 = SmcCapability;
cap_name#ff name:Text = SmcCapability; cap_name#ff name:Text = SmcCapability;
dns_storage_address#7473 bag_id:bits256 = DNSRecord;
// //
// PAYMENT CHANNELS // PAYMENT CHANNELS
// //

9
crypto/block/mc-config.cpp
View file

@ -1945,6 +1945,15 @@ td::Result<SizeLimitsConfig> Config::get_size_limits_config() const {
return limits; return limits;
} }
std::unique_ptr<vm::Dictionary> Config::get_suspended_addresses(ton::UnixTime now) const {
td::Ref<vm::Cell> param = get_config_param(44);
gen::SuspendedAddressList::Record rec;
if (param.is_null() || !tlb::unpack_cell(param, rec) || rec.suspended_until <= now) {
return {};
}
return std::make_unique<vm::Dictionary>(rec.addresses->prefetch_ref(), 288);
}
td::Result<std::pair<ton::UnixTime, ton::UnixTime>> Config::unpack_validator_set_start_stop(Ref<vm::Cell> vset_root) { td::Result<std::pair<ton::UnixTime, ton::UnixTime>> Config::unpack_validator_set_start_stop(Ref<vm::Cell> vset_root) {
if (vset_root.is_null()) { if (vset_root.is_null()) {
return td::Status::Error("validator set absent"); return td::Status::Error("validator set absent");

1
crypto/block/mc-config.h
View file

@ -628,6 +628,7 @@ class Config {
std::vector<ton::ValidatorDescr> compute_total_validator_set(int next) const; std::vector<ton::ValidatorDescr> compute_total_validator_set(int next) const;
CollatorConfig get_collator_config(bool need_collator_nodes) const; CollatorConfig get_collator_config(bool need_collator_nodes) const;
td::Result<SizeLimitsConfig> get_size_limits_config() const; td::Result<SizeLimitsConfig> get_size_limits_config() const;
std::unique_ptr<vm::Dictionary> get_suspended_addresses(ton::UnixTime now) const;
static std::vector<ton::ValidatorDescr> do_compute_validator_set(const block::CatchainValidatorsConfig& ccv_conf, static std::vector<ton::ValidatorDescr> do_compute_validator_set(const block::CatchainValidatorsConfig& ccv_conf,
ton::ShardIdFull shard, ton::ShardIdFull shard,
const block::ValidatorSet& vset, ton::UnixTime time, const block::ValidatorSet& vset, ton::UnixTime time,

21
crypto/block/transaction.cpp
View file

@ -794,6 +794,20 @@ bool ComputePhaseConfig::parse_GasLimitsPrices_internal(Ref<vm::CellSlice> cs, t
return true; return true;
} }
bool ComputePhaseConfig::is_address_suspended(ton::WorkchainId wc, td::Bits256 addr) const {
if (!suspended_addresses) {
return false;
}
try {
vm::CellBuilder key;
key.store_long_bool(wc, 32);
key.store_bits_bool(addr);
return !suspended_addresses->lookup(key.data_bits(), 288).is_null();
} catch (vm::VmError) {
return false;
}
}
void ComputePhaseConfig::compute_threshold() { void ComputePhaseConfig::compute_threshold() {
gas_price256 = td::make_refint(gas_price); gas_price256 = td::make_refint(gas_price);
if (gas_limit > flat_gas_limit) { if (gas_limit > flat_gas_limit) {
@ -1006,6 +1020,11 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) {
if (in_msg_state.not_null() && if (in_msg_state.not_null() &&
(acc_status == Account::acc_uninit || (acc_status == Account::acc_uninit ||
(acc_status == Account::acc_frozen && account.state_hash == in_msg_state->get_hash().bits()))) { (acc_status == Account::acc_frozen && account.state_hash == in_msg_state->get_hash().bits()))) {
if (acc_status == Account::acc_uninit && cfg.is_address_suspended(account.workchain, account.addr)) {
LOG(DEBUG) << "address is suspended, skipping compute phase";
cp.skip_reason = ComputePhase::sk_suspended;
return true;
}
use_msg_state = true; use_msg_state = true;
if (!(unpack_msg_state() && account.check_split_depth(new_split_depth))) { if (!(unpack_msg_state() && account.check_split_depth(new_split_depth))) {
LOG(DEBUG) << "cannot unpack in_msg_state, or it has bad split_depth; cannot init account state"; LOG(DEBUG) << "cannot unpack in_msg_state, or it has bad split_depth; cannot init account state";
@ -2263,6 +2282,8 @@ bool Transaction::serialize_compute_phase(vm::CellBuilder& cb) {
return cb.store_long_bool(1, 3); // cskip_bad_state$01 = ComputeSkipReason; return cb.store_long_bool(1, 3); // cskip_bad_state$01 = ComputeSkipReason;
case ComputePhase::sk_no_gas: case ComputePhase::sk_no_gas:
return cb.store_long_bool(2, 3); // cskip_no_gas$10 = ComputeSkipReason; return cb.store_long_bool(2, 3); // cskip_no_gas$10 = ComputeSkipReason;
case ComputePhase::sk_suspended:
return cb.store_long_bool(0b0110, 4); // cskip_suspended$110 = ComputeSkipReason;
case ComputePhase::sk_none: case ComputePhase::sk_none:
break; break;
default: default:

4
crypto/block/transaction.h
View file

@ -108,6 +108,7 @@ struct ComputePhaseConfig {
td::BitArray<256> block_rand_seed; td::BitArray<256> block_rand_seed;
bool with_vm_log{false}; bool with_vm_log{false};
td::uint16 max_vm_data_depth = 512; td::uint16 max_vm_data_depth = 512;
std::unique_ptr<vm::Dictionary> suspended_addresses;
ComputePhaseConfig(td::uint64 _gas_price = 0, td::uint64 _gas_limit = 0, td::uint64 _gas_credit = 0) ComputePhaseConfig(td::uint64 _gas_price = 0, td::uint64 _gas_limit = 0, td::uint64 _gas_credit = 0)
: gas_price(_gas_price), gas_limit(_gas_limit), special_gas_limit(_gas_limit), gas_credit(_gas_credit) { : gas_price(_gas_price), gas_limit(_gas_limit), special_gas_limit(_gas_limit), gas_credit(_gas_credit) {
compute_threshold(); compute_threshold();
@ -132,6 +133,7 @@ struct ComputePhaseConfig {
} }
bool parse_GasLimitsPrices(Ref<vm::CellSlice> cs, td::RefInt256& freeze_due_limit, td::RefInt256& delete_due_limit); bool parse_GasLimitsPrices(Ref<vm::CellSlice> cs, td::RefInt256& freeze_due_limit, td::RefInt256& delete_due_limit);
bool parse_GasLimitsPrices(Ref<vm::Cell> cell, td::RefInt256& freeze_due_limit, td::RefInt256& delete_due_limit); bool parse_GasLimitsPrices(Ref<vm::Cell> cell, td::RefInt256& freeze_due_limit, td::RefInt256& delete_due_limit);
bool is_address_suspended(ton::WorkchainId wc, td::Bits256 addr) const;
private: private:
bool parse_GasLimitsPrices_internal(Ref<vm::CellSlice> cs, td::RefInt256& freeze_due_limit, bool parse_GasLimitsPrices_internal(Ref<vm::CellSlice> cs, td::RefInt256& freeze_due_limit,
@ -157,7 +159,7 @@ struct CreditPhase {
}; };
struct ComputePhase { struct ComputePhase {
enum { sk_none, sk_no_state, sk_bad_state, sk_no_gas }; enum { sk_none, sk_no_state, sk_bad_state, sk_no_gas, sk_suspended };
int skip_reason{sk_none}; int skip_reason{sk_none};
bool success{false}; bool success{false};
bool msg_state_used{false}; bool msg_state_used{false};

3
crypto/func/analyzer.cpp
View file

@ -810,6 +810,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) {
break; break;
} }
case _While: { case _While: {
auto values0 = values;
values = block0->fwd_analyze(values); values = block0->fwd_analyze(values);
if (values[left[0]] && values[left[0]]->always_false()) { if (values[left[0]] && values[left[0]]->always_false()) {
// block1 never executed // block1 never executed
@ -817,7 +818,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) {
break; break;
} }
while (true) { while (true) {
VarDescrList next_values = values | block0->fwd_analyze(block1->fwd_analyze(values)); VarDescrList next_values = values | block0->fwd_analyze(values0 | block1->fwd_analyze(values));
if (same_values(next_values, values)) { if (same_values(next_values, values)) {
break; break;
} }

11
crypto/func/auto-tests/stress_tester.py
View file

@ -137,13 +137,20 @@ class CodeRepeat(Code):
self.c.write(f, indent + 1) self.c.write(f, indent + 1)
print(" " * (indent + 1) + "%s += 1;" % var, file=f) print(" " * (indent + 1) + "%s += 1;" % var, file=f)
print(" " * indent + "}", file=f) print(" " * indent + "}", file=f)
else: elif self.loop_type == 2:
var = gen_var_name() var = gen_var_name()
print(" " * indent + "int %s = 0;" % var, file=f) print(" " * indent + "int %s = 0;" % var, file=f)
print(" " * indent + "do {", file=f) print(" " * indent + "do {", file=f)
self.c.write(f, indent + 1) self.c.write(f, indent + 1)
print(" " * (indent + 1) + "%s += 1;" % var, file=f) print(" " * (indent + 1) + "%s += 1;" % var, file=f)
print(" " * indent + "} until (%s >= %d);" % (var, self.n), file=f) print(" " * indent + "} until (%s >= %d);" % (var, self.n), file=f)
else:
var = gen_var_name()
print(" " * indent + "int %s = %d;" % (var, self.n - 1), file=f)
print(" " * indent + "while (%s >= 0) {" % var, file=f)
self.c.write(f, indent + 1)
print(" " * (indent + 1) + "%s -= 1;" % var, file=f)
print(" " * indent + "}", file=f)
class CodeThrow(Code): class CodeThrow(Code):
def __init__(self): def __init__(self):
@ -199,7 +206,7 @@ def gen_code(xl, xr, with_return, loop_depth=0, try_catch_depth=0, can_throw=Fal
for _ in range(random.randint(0, 2)): for _ in range(random.randint(0, 2)):
if random.randint(0, 3) == 0 and loop_depth < 3: if random.randint(0, 3) == 0 and loop_depth < 3:
c = gen_code(xl, xr, False, loop_depth + 1, try_catch_depth, can_throw) c = gen_code(xl, xr, False, loop_depth + 1, try_catch_depth, can_throw)
code.append(CodeRepeat(random.randint(0, 3), c, random.randint(0, 2))) code.append(CodeRepeat(random.randint(0, 3), c, random.randint(0, 3)))
elif xr - xl > 1: elif xr - xl > 1:
xmid = random.randrange(xl + 1, xr) xmid = random.randrange(xl + 1, xr)
ret = random.choice((0, 0, 0, 0, 0, 1, 2)) ret = random.choice((0, 0, 0, 0, 0, 1, 2))

26
crypto/func/func.h
View file

@ -39,7 +39,7 @@ extern std::string generated_from;
constexpr int optimize_depth = 20; constexpr int optimize_depth = 20;
const std::string func_version{"0.3.0"}; const std::string func_version{"0.4.0"};
enum Keyword { enum Keyword {
_Eof = -1, _Eof = -1,
@ -306,10 +306,16 @@ struct TmpVar {
sym_idx_t name; sym_idx_t name;
int coord; int coord;
std::unique_ptr<SrcLocation> where; std::unique_ptr<SrcLocation> where;
size_t modify_forbidden = 0;
TmpVar(var_idx_t _idx, int _cls, TypeExpr* _type = 0, SymDef* sym = 0, const SrcLocation* loc = 0); TmpVar(var_idx_t _idx, int _cls, TypeExpr* _type = 0, SymDef* sym = 0, const SrcLocation* loc = 0);
void show(std::ostream& os, int omit_idx = 0) const; void show(std::ostream& os, int omit_idx = 0) const;
void dump(std::ostream& os) const; void dump(std::ostream& os) const;
void set_location(const SrcLocation& loc); void set_location(const SrcLocation& loc);
std::string to_string() const {
std::ostringstream s;
show(s, 2);
return s.str();
}
}; };
struct VarDescr { struct VarDescr {
@ -722,6 +728,22 @@ struct CodeBlob {
void mark_noreturn(); void mark_noreturn();
void generate_code(AsmOpList& out_list, int mode = 0); void generate_code(AsmOpList& out_list, int mode = 0);
void generate_code(std::ostream& os, int mode = 0, int indent = 0); void generate_code(std::ostream& os, int mode = 0, int indent = 0);
void mark_modify_forbidden(var_idx_t idx) {
++vars.at(idx).modify_forbidden;
}
void unmark_modify_forbidden(var_idx_t idx) {
assert(vars.at(idx).modify_forbidden > 0);
--vars.at(idx).modify_forbidden;
}
void check_modify_forbidden(var_idx_t idx, const SrcLocation& here) const {
if (vars.at(idx).modify_forbidden) {
throw src::ParseError{here, PSTRING() << "Modifying local variable " << vars[idx].to_string()
<< " after using it in the same expression"};
}
}
}; };
/* /*
@ -925,7 +947,7 @@ struct Expr {
} }
int define_new_vars(CodeBlob& code); int define_new_vars(CodeBlob& code);
int predefine_vars(); int predefine_vars();
std::vector<var_idx_t> pre_compile(CodeBlob& code, bool lval = false) const; std::vector<var_idx_t> pre_compile(CodeBlob& code, std::vector<std::pair<SymDef*, var_idx_t>>* lval_globs = nullptr) const;
static std::vector<var_idx_t> pre_compile_let(CodeBlob& code, Expr* lhs, Expr* rhs, const SrcLocation& here); static std::vector<var_idx_t> pre_compile_let(CodeBlob& code, Expr* lhs, Expr* rhs, const SrcLocation& here);
var_idx_t new_tmp(CodeBlob& code) const; var_idx_t new_tmp(CodeBlob& code) const;
std::vector<var_idx_t> new_tmp_vect(CodeBlob& code) const { std::vector<var_idx_t> new_tmp_vect(CodeBlob& code) const {

58
crypto/func/gen-abscode.cpp
View file

@ -221,6 +221,13 @@ var_idx_t Expr::new_tmp(CodeBlob& code) const {
return code.create_tmp_var(e_type, &here); return code.create_tmp_var(e_type, &here);
} }
void add_set_globs(CodeBlob& code, std::vector<std::pair<SymDef*, var_idx_t>>& globs, const SrcLocation& here) {
for (const auto& p : globs) {
auto& op = code.emplace_back(here, Op::_SetGlob, std::vector<var_idx_t>{}, std::vector<var_idx_t>{ p.second }, p.first);
op.flags |= Op::_Impure;
}
}
std::vector<var_idx_t> Expr::pre_compile_let(CodeBlob& code, Expr* lhs, Expr* rhs, const SrcLocation& here) { std::vector<var_idx_t> Expr::pre_compile_let(CodeBlob& code, Expr* lhs, Expr* rhs, const SrcLocation& here) {
while (lhs->is_type_apply()) { while (lhs->is_type_apply()) {
lhs = lhs->args.at(0); lhs = lhs->args.at(0);
@ -245,19 +252,18 @@ std::vector<var_idx_t> Expr::pre_compile_let(CodeBlob& code, Expr* lhs, Expr* rh
return tmp; return tmp;
} }
auto right = rhs->pre_compile(code); auto right = rhs->pre_compile(code);
if (lhs->cls == Expr::_GlobVar) { std::vector<std::pair<SymDef*, var_idx_t>> globs;
assert(lhs->sym); auto left = lhs->pre_compile(code, &globs);
auto& op = code.emplace_back(here, Op::_SetGlob, std::vector<var_idx_t>{}, right, lhs->sym); for (var_idx_t v : left) {
op.flags |= Op::_Impure; code.check_modify_forbidden(v, here);
} else {
auto left = lhs->pre_compile(code, true);
code.emplace_back(here, Op::_Let, std::move(left), right);
} }
code.emplace_back(here, Op::_Let, std::move(left), right);
add_set_globs(code, globs, here);
return right; return right;
} }
std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, bool lval) const { std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, std::vector<std::pair<SymDef*, var_idx_t>>* lval_globs) const {
if (lval && !(cls == _Tensor || cls == _Var || cls == _Hole || cls == _TypeApply)) { if (lval_globs && !(cls == _Tensor || cls == _Var || cls == _Hole || cls == _TypeApply || cls == _GlobVar)) {
std::cerr << "lvalue expression constructor is " << cls << std::endl; std::cerr << "lvalue expression constructor is " << cls << std::endl;
throw src::Fatal{"cannot compile lvalue expression with unknown constructor"}; throw src::Fatal{"cannot compile lvalue expression with unknown constructor"};
} }
@ -265,9 +271,15 @@ std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, bool lval) const {
case _Tensor: { case _Tensor: {
std::vector<var_idx_t> res; std::vector<var_idx_t> res;
for (const auto& x : args) { for (const auto& x : args) {
auto add = x->pre_compile(code, lval); auto add = x->pre_compile(code, lval_globs);
for (var_idx_t v : add) {
code.mark_modify_forbidden(v);
}
res.insert(res.end(), add.cbegin(), add.cend()); res.insert(res.end(), add.cbegin(), add.cend());
} }
for (var_idx_t v : res) {
code.unmark_modify_forbidden(v);
}
return res; return res;
} }
case _Apply: { case _Apply: {
@ -279,6 +291,9 @@ std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, bool lval) const {
std::vector<std::vector<var_idx_t>> add_list(args.size()); std::vector<std::vector<var_idx_t>> add_list(args.size());
for (int i : func->arg_order) { for (int i : func->arg_order) {
add_list[i] = args[i]->pre_compile(code); add_list[i] = args[i]->pre_compile(code);
for (var_idx_t v : add_list[i]) {
code.mark_modify_forbidden(v);
}
} }
for (const auto& add : add_list) { for (const auto& add : add_list) {
res.insert(res.end(), add.cbegin(), add.cend()); res.insert(res.end(), add.cbegin(), add.cend());
@ -286,9 +301,15 @@ std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, bool lval) const {
} else { } else {
for (const auto& x : args) { for (const auto& x : args) {
auto add = x->pre_compile(code); auto add = x->pre_compile(code);
for (var_idx_t v : add) {
code.mark_modify_forbidden(v);
}
res.insert(res.end(), add.cbegin(), add.cend()); res.insert(res.end(), add.cbegin(), add.cend());
} }
} }
for (var_idx_t v : res) {
code.unmark_modify_forbidden(v);
}
auto rvect = new_tmp_vect(code); auto rvect = new_tmp_vect(code);
auto& op = code.emplace_back(here, Op::_Call, rvect, std::move(res), sym); auto& op = code.emplace_back(here, Op::_Call, rvect, std::move(res), sym);
if (flags & _IsImpure) { if (flags & _IsImpure) {
@ -297,7 +318,7 @@ std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, bool lval) const {
return rvect; return rvect;
} }
case _TypeApply: case _TypeApply:
return args[0]->pre_compile(code, lval); return args[0]->pre_compile(code, lval_globs);
case _Var: case _Var:
case _Hole: case _Hole:
return {val}; return {val};
@ -329,18 +350,31 @@ std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, bool lval) const {
case _Glob: case _Glob:
case _GlobVar: { case _GlobVar: {
auto rvect = new_tmp_vect(code); auto rvect = new_tmp_vect(code);
if (lval_globs) {
lval_globs->push_back({ sym, rvect[0] });
return rvect;
} else {
code.emplace_back(here, Op::_GlobVar, rvect, std::vector<var_idx_t>{}, sym); code.emplace_back(here, Op::_GlobVar, rvect, std::vector<var_idx_t>{}, sym);
return rvect; return rvect;
} }
}
case _Letop: { case _Letop: {
return pre_compile_let(code, args.at(0), args.at(1), here); return pre_compile_let(code, args.at(0), args.at(1), here);
} }
case _LetFirst: { case _LetFirst: {
auto rvect = new_tmp_vect(code); auto rvect = new_tmp_vect(code);
auto right = args[1]->pre_compile(code); auto right = args[1]->pre_compile(code);
auto left = args[0]->pre_compile(code, true); std::vector<std::pair<SymDef*, var_idx_t>> local_globs;
if (!lval_globs) {
lval_globs = &local_globs;
}
auto left = args[0]->pre_compile(code, lval_globs);
left.push_back(rvect[0]); left.push_back(rvect[0]);
for (var_idx_t v : left) {
code.check_modify_forbidden(v, here);
}
code.emplace_back(here, Op::_Let, std::move(left), std::move(right)); code.emplace_back(here, Op::_Let, std::move(left), std::move(right));
add_set_globs(code, local_globs, here);
return rvect; return rvect;
} }
case _MkTuple: { case _MkTuple: {

24
crypto/smc-envelope/ManualDns.cpp
View file

@ -47,6 +47,8 @@ td::StringBuilder& operator<<(td::StringBuilder& sb, const ManualDns::EntryData&
.move_as_ok(); .move_as_ok();
case ManualDns::EntryData::Type::SmcAddress: case ManualDns::EntryData::Type::SmcAddress:
return sb << "SMC:" << data.data.get<ManualDns::EntryDataSmcAddress>().smc_address.rserialize(); return sb << "SMC:" << data.data.get<ManualDns::EntryDataSmcAddress>().smc_address.rserialize();
case ManualDns::EntryData::Type::StorageAddress:
return sb << "STORAGE:" << data.data.get<ManualDns::EntryDataStorageAddress>().bag_id.to_hex();
} }
return sb << "<unknown>"; return sb << "<unknown>";
} }
@ -93,6 +95,11 @@ td::Result<td::Ref<vm::Cell>> DnsInterface::EntryData::as_cell() const {
smc_address.smc_address.addr); smc_address.smc_address.addr);
dns.smc_addr = vm::load_cell_slice_ref(cb.finalize()); dns.smc_addr = vm::load_cell_slice_ref(cb.finalize());
tlb::pack_cell(res, dns); tlb::pack_cell(res, dns);
},
[&](const EntryDataStorageAddress& storage_address) {
block::gen::DNSRecord::Record_dns_storage_address dns;
dns.bag_id = storage_address.bag_id;
tlb::pack_cell(res, dns);
})); }));
if (error.is_error()) { if (error.is_error()) {
return error; return error;
@ -142,6 +149,11 @@ td::Result<DnsInterface::EntryData> DnsInterface::EntryData::from_cellslice(vm::
} }
return EntryData::smc_address(block::StdAddress(wc, addr)); return EntryData::smc_address(block::StdAddress(wc, addr));
} }
case block::gen::DNSRecord::dns_storage_address: {
block::gen::DNSRecord::Record_dns_storage_address dns;
tlb::unpack(cs, dns);
return EntryData::storage_address(dns.bag_id);
}
} }
return td::Status::Error("Unknown entry data"); return td::Status::Error("Unknown entry data");
} }
@ -536,10 +548,12 @@ std::string DnsInterface::decode_name(td::Slice name) {
std::string ManualDns::serialize_data(const EntryData& data) { std::string ManualDns::serialize_data(const EntryData& data) {
std::string res; std::string res;
data.data.visit(td::overloaded([&](const ton::ManualDns::EntryDataText& text) { res = "UNSUPPORTED"; }, data.data.visit(
td::overloaded([&](const ton::ManualDns::EntryDataText& text) { res = "UNSUPPORTED"; },
[&](const ton::ManualDns::EntryDataNextResolver& resolver) { res = "UNSUPPORTED"; }, [&](const ton::ManualDns::EntryDataNextResolver& resolver) { res = "UNSUPPORTED"; },
[&](const ton::ManualDns::EntryDataAdnlAddress& adnl_address) { res = "UNSUPPORTED"; }, [&](const ton::ManualDns::EntryDataAdnlAddress& adnl_address) { res = "UNSUPPORTED"; },
[&](const ton::ManualDns::EntryDataSmcAddress& text) { res = "UNSUPPORTED"; })); [&](const ton::ManualDns::EntryDataSmcAddress& text) { res = "UNSUPPORTED"; },
[&](const ton::ManualDns::EntryDataStorageAddress& storage_address) { res = "UNSUPPORTED"; }));
return res; return res;
} }
@ -559,6 +573,12 @@ td::Result<td::optional<ManualDns::EntryData>> ManualDns::parse_data(td::Slice c
} else if (type == "NEXT") { } else if (type == "NEXT") {
TRY_RESULT(address, block::StdAddress::parse(parser.read_all())); TRY_RESULT(address, block::StdAddress::parse(parser.read_all()));
return ManualDns::EntryData::next_resolver(address); return ManualDns::EntryData::next_resolver(address);
} else if (type == "STORAGE") {
td::Bits256 bag_id;
if (bag_id.from_hex(parser.read_all(), false) != 256) {
return td::Status::Error("failed to parse bag id");
}
return ManualDns::EntryData::storage_address(bag_id);
} else if (parser.data() == "DELETED") { } else if (parser.data() == "DELETED") {
return {}; return {};
} }

17
crypto/smc-envelope/ManualDns.h
View file

@ -66,9 +66,19 @@ class DnsInterface {
// TODO: capability // TODO: capability
}; };
struct EntryDataStorageAddress {
ton::Bits256 bag_id;
// TODO: proto
bool operator==(const EntryDataStorageAddress& other) const {
return bag_id == other.bag_id;
}
};
struct EntryData { struct EntryData {
enum Type { Empty, Text, NextResolver, AdnlAddress, SmcAddress } type{Empty}; enum Type { Empty, Text, NextResolver, AdnlAddress, SmcAddress, StorageAddress } type{Empty};
td::Variant<EntryDataText, EntryDataNextResolver, EntryDataAdnlAddress, EntryDataSmcAddress> data; td::Variant<EntryDataText, EntryDataNextResolver, EntryDataAdnlAddress, EntryDataSmcAddress,
EntryDataStorageAddress>
data;
static EntryData text(std::string text) { static EntryData text(std::string text) {
return {Text, EntryDataText{text}}; return {Text, EntryDataText{text}};
@ -82,6 +92,9 @@ class DnsInterface {
static EntryData smc_address(block::StdAddress smc_address) { static EntryData smc_address(block::StdAddress smc_address) {
return {SmcAddress, EntryDataSmcAddress{smc_address}}; return {SmcAddress, EntryDataSmcAddress{smc_address}};
} }
static EntryData storage_address(ton::Bits256 bag_id) {
return {StorageAddress, EntryDataStorageAddress{bag_id}};
}
bool operator==(const EntryData& other) const { bool operator==(const EntryData& other) const {
return data == other.data; return data == other.data;

7
docker/Dockerfile
View file

@ -11,16 +11,19 @@ WORKDIR /ton
RUN mkdir build && \ RUN mkdir build && \
cd build && \ cd build && \
cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. && \ cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. && \
ninja tonlibjson fift func validator-engine validator-engine-console generate-random-id dht-server lite-client ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console generate-random-id dht-server lite-client
FROM ubuntu:20.04 FROM ubuntu:20.04
RUN apt-get update && \ RUN apt-get update && \
apt-get install -y openssl wget libatomic1 && \ apt-get install -y openssl wget libatomic1 && \
rm -rf /var/lib/apt/lists/* rm -rf /var/lib/apt/lists/*
RUN mkdir -p /var/ton-work/db && \ RUN mkdir -p /var/ton-work/db && \
mkdir -p /var/ton-work/db/static mkdir -p /var/ton-work/db/static
COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon /usr/local/bin/
COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon-cli /usr/local/bin/
COPY --from=builder /ton/build/lite-client/lite-client /usr/local/bin/ COPY --from=builder /ton/build/lite-client/lite-client /usr/local/bin/
COPY --from=builder /ton/build/validator-engine/validator-engine /usr/local/bin/ COPY --from=builder /ton/build/validator-engine/validator-engine /usr/local/bin/
COPY --from=builder /ton/build/validator-engine-console/validator-engine-console /usr/local/bin/ COPY --from=builder /ton/build/validator-engine-console/validator-engine-console /usr/local/bin/

7
lite-client/lite-client.cpp
View file

@ -1749,6 +1749,13 @@ bool TestNode::show_dns_record(std::ostream& os, td::Bits256 cat, Ref<vm::CellSl
} }
break; break;
} }
case block::gen::DNSRecord::dns_storage_address: {
block::gen::DNSRecord::Record_dns_storage_address rec;
if (tlb::unpack_exact(cs, rec)) {
os << "\tstorage address " << rec.bag_id.to_hex();
}
break;
}
case block::gen::DNSRecord::dns_next_resolver: { case block::gen::DNSRecord::dns_next_resolver: {
block::gen::DNSRecord::Record_dns_next_resolver rec; block::gen::DNSRecord::Record_dns_next_resolver rec;
if (tlb::unpack_exact(cs, rec) && block::tlb::t_MsgAddressInt.extract_std_address(rec.resolver, wc, addr)) { if (tlb::unpack_exact(cs, rec) && block::tlb::t_MsgAddressInt.extract_std_address(rec.resolver, wc, addr)) {

38
rldp-http-proxy/DNSResolver.cpp
View file

@ -51,13 +51,13 @@ void DNSResolver::sync() {
std::move(P)); std::move(P));
} }
void DNSResolver::resolve(std::string host, td::Promise<ton::adnl::AdnlNodeIdShort> promise) { void DNSResolver::resolve(std::string host, td::Promise<std::string> promise) {
auto it = cache_.find(host); auto it = cache_.find(host);
if (it != cache_.end()) { if (it != cache_.end()) {
const CacheEntry &entry = it->second; const CacheEntry &entry = it->second;
double now = td::Time::now(); double now = td::Time::now();
if (now < entry.created_at_ + CACHE_TIMEOUT_HARD) { if (now < entry.created_at_ + CACHE_TIMEOUT_HARD) {
promise.set_result(entry.id_); promise.set_result(entry.address_);
promise.reset(); promise.reset();
if (now < entry.created_at_ + CACHE_TIMEOUT_SOFT) { if (now < entry.created_at_ + CACHE_TIMEOUT_SOFT) {
return; return;
@ -73,42 +73,42 @@ void DNSResolver::resolve(std::string host, td::Promise<ton::adnl::AdnlNodeIdSho
if (promise) { if (promise) {
promise.set_result(R.move_as_error()); promise.set_result(R.move_as_error());
} }
} else { return;
}
auto obj = R.move_as_ok(); auto obj = R.move_as_ok();
ton::adnl::AdnlNodeIdShort id; std::string result;
td::uint32 cnt = 0; if (!obj->entries_.empty()) {
for (auto &e : obj->entries_) { tonlib_api::downcast_call(*obj->entries_[0]->entry_,
tonlib_api::downcast_call(*e->entry_.get(),
td::overloaded( td::overloaded(
[&](tonlib_api::dns_entryDataAdnlAddress &x) { [&](tonlib_api::dns_entryDataAdnlAddress &x) {
if (td::Random::fast(0, cnt) == 0) {
auto R = ton::adnl::AdnlNodeIdShort::parse(x.adnl_address_->adnl_address_); auto R = ton::adnl::AdnlNodeIdShort::parse(x.adnl_address_->adnl_address_);
if (R.is_ok()) { if (R.is_ok()) {
id = R.move_as_ok(); ton::adnl::AdnlNodeIdShort id = R.move_as_ok();
cnt++; result = id.serialize() + ".adnl";
}
} }
}, },
[&](tonlib_api::dns_entryDataStorageAddress &x) {
result = td::to_lower(x.bag_id_.to_hex()) + ".bag";
},
[&](auto &x) {})); [&](auto &x) {}));
} }
if (cnt == 0) { if (result.empty()) {
if (promise) { if (promise) {
promise.set_error(td::Status::Error("no DNS entries")); promise.set_error(td::Status::Error("no DNS entries"));
} }
} else { return;
td::actor::send_closure(SelfId, &DNSResolver::save_to_cache, std::move(host), id); }
td::actor::send_closure(SelfId, &DNSResolver::save_to_cache, std::move(host), result);
if (promise) { if (promise) {
promise.set_result(id); promise.set_result(std::move(result));
}
}
} }
}); });
td::actor::send_closure(tonlib_client_, &tonlib::TonlibClientWrapper::send_request<tonlib_api::dns_resolve>, td::actor::send_closure(tonlib_client_, &tonlib::TonlibClientWrapper::send_request<tonlib_api::dns_resolve>,
std::move(obj), std::move(P)); std::move(obj), std::move(P));
} }
void DNSResolver::save_to_cache(std::string host, ton::adnl::AdnlNodeIdShort id) { void DNSResolver::save_to_cache(std::string host, std::string address) {
CacheEntry &entry = cache_[host]; CacheEntry &entry = cache_[host];
entry.id_ = id; entry.address_ = address;
entry.created_at_ = td::Time::now(); entry.created_at_ = td::Time::now();
} }

6
rldp-http-proxy/DNSResolver.h
View file

@ -34,16 +34,16 @@ class DNSResolver : public td::actor::Actor {
explicit DNSResolver(td::actor::ActorId<tonlib::TonlibClientWrapper> tonlib_client); explicit DNSResolver(td::actor::ActorId<tonlib::TonlibClientWrapper> tonlib_client);
void start_up() override; void start_up() override;
void resolve(std::string host, td::Promise<ton::adnl::AdnlNodeIdShort> promise); void resolve(std::string host, td::Promise<std::string> promise);
private: private:
void sync(); void sync();
void save_to_cache(std::string host, ton::adnl::AdnlNodeIdShort id); void save_to_cache(std::string host, std::string address);
td::actor::ActorId<tonlib::TonlibClientWrapper> tonlib_client_; td::actor::ActorId<tonlib::TonlibClientWrapper> tonlib_client_;
struct CacheEntry { struct CacheEntry {
ton::adnl::AdnlNodeIdShort id_; std::string address_;
double created_at_; double created_at_;
}; };
std::map<std::string, CacheEntry> cache_; std::map<std::string, CacheEntry> cache_;

95
rldp-http-proxy/rldp-http-proxy.cpp
View file

@ -425,7 +425,8 @@ class TcpToRldpRequestSender : public td::actor::Actor {
td::Promise<std::pair<std::unique_ptr<ton::http::HttpResponse>, std::shared_ptr<ton::http::HttpPayload>>> promise, td::Promise<std::pair<std::unique_ptr<ton::http::HttpResponse>, std::shared_ptr<ton::http::HttpPayload>>> promise,
td::actor::ActorId<ton::adnl::Adnl> adnl, td::actor::ActorId<ton::dht::Dht> dht, td::actor::ActorId<ton::adnl::Adnl> adnl, td::actor::ActorId<ton::dht::Dht> dht,
td::actor::ActorId<ton::rldp::Rldp> rldp, td::actor::ActorId<RldpHttpProxy> proxy, td::actor::ActorId<ton::rldp::Rldp> rldp, td::actor::ActorId<RldpHttpProxy> proxy,
td::actor::ActorId<DNSResolver> dns_resolver) td::actor::ActorId<DNSResolver> dns_resolver,
ton::adnl::AdnlNodeIdShort storage_gateway)
: local_id_(local_id) : local_id_(local_id)
, host_(std::move(host)) , host_(std::move(host))
, request_(std::move(request)) , request_(std::move(request))
@ -435,17 +436,20 @@ class TcpToRldpRequestSender : public td::actor::Actor {
, dht_(dht) , dht_(dht)
, rldp_(rldp) , rldp_(rldp)
, proxy_(proxy) , proxy_(proxy)
, dns_resolver_(dns_resolver) { , dns_resolver_(dns_resolver)
} , storage_gateway_(storage_gateway) {
void start_up() override {
resolve();
} }
void resolve(); void start_up() override {
td::Random::secure_bytes(id_.as_slice());
request_tl_ = request_->store_tl(id_);
resolve(host_);
}
void resolve(std::string host);
void resolved(ton::adnl::AdnlNodeIdShort id) { void resolved(ton::adnl::AdnlNodeIdShort id) {
dst_ = id; dst_ = id;
td::Random::secure_bytes(id_.as_slice());
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
if (R.is_error()) { if (R.is_error()) {
@ -459,7 +463,7 @@ class TcpToRldpRequestSender : public td::actor::Actor {
proxy_, is_tunnel()) proxy_, is_tunnel())
.release(); .release();
auto f = ton::serialize_tl_object(request_->store_tl(id_), true); auto f = ton::serialize_tl_object(request_tl_, true);
td::actor::send_closure(rldp_, &ton::rldp::Rldp::send_query_ex, local_id_, dst_, "http request over rldp", td::actor::send_closure(rldp_, &ton::rldp::Rldp::send_query_ex, local_id_, dst_, "http request over rldp",
std::move(P), td::Timestamp::in(30.0), std::move(f), 16 << 10); std::move(P), td::Timestamp::in(30.0), std::move(f), 16 << 10);
} }
@ -538,6 +542,7 @@ class TcpToRldpRequestSender : public td::actor::Actor {
std::unique_ptr<ton::http::HttpRequest> request_; std::unique_ptr<ton::http::HttpRequest> request_;
std::shared_ptr<ton::http::HttpPayload> request_payload_; std::shared_ptr<ton::http::HttpPayload> request_payload_;
ton::tl_object_ptr<ton::ton_api::http_request> request_tl_;
td::Promise<std::pair<std::unique_ptr<ton::http::HttpResponse>, std::shared_ptr<ton::http::HttpPayload>>> promise_; td::Promise<std::pair<std::unique_ptr<ton::http::HttpResponse>, std::shared_ptr<ton::http::HttpPayload>>> promise_;
@ -546,6 +551,9 @@ class TcpToRldpRequestSender : public td::actor::Actor {
td::actor::ActorId<ton::rldp::Rldp> rldp_; td::actor::ActorId<ton::rldp::Rldp> rldp_;
td::actor::ActorId<RldpHttpProxy> proxy_; td::actor::ActorId<RldpHttpProxy> proxy_;
td::actor::ActorId<DNSResolver> dns_resolver_; td::actor::ActorId<DNSResolver> dns_resolver_;
ton::adnl::AdnlNodeIdShort storage_gateway_ = ton::adnl::AdnlNodeIdShort::zero();
bool dns_resolve_sent_ = false;
std::unique_ptr<ton::http::HttpResponse> response_; std::unique_ptr<ton::http::HttpResponse> response_;
std::shared_ptr<ton::http::HttpPayload> response_payload_; std::shared_ptr<ton::http::HttpPayload> response_payload_;
@ -983,15 +991,9 @@ class RldpHttpProxy : public td::actor::Actor {
} }
} }
{ {
if (is_client_) {
auto D = ton::dht::Dht::create_client(dht_id_, "", dht_config_, keyring_.get(), adnl_.get()); auto D = ton::dht::Dht::create_client(dht_id_, "", dht_config_, keyring_.get(), adnl_.get());
D.ensure(); D.ensure();
dht_ = D.move_as_ok(); dht_ = D.move_as_ok();
} else {
auto D = ton::dht::Dht::create(dht_id_, db_root_, dht_config_, keyring_.get(), adnl_.get());
D.ensure();
dht_ = D.move_as_ok();
}
td::actor::send_closure(adnl_, &ton::adnl::Adnl::register_dht_node, dht_.get()); td::actor::send_closure(adnl_, &ton::adnl::Adnl::register_dht_node, dht_.get());
} }
if (port_) { if (port_) {
@ -1104,15 +1106,20 @@ class RldpHttpProxy : public td::actor::Actor {
} }
} }
std::transform(host.begin(), host.end(), host.begin(), [](unsigned char c) { return std::tolower(c); }); std::transform(host.begin(), host.end(), host.begin(), [](unsigned char c) { return std::tolower(c); });
if (!proxy_all_ && bool allow = proxy_all_;
(host.size() < 5 || (host.substr(host.size() - 4) != ".ton" && host.substr(host.size() - 5) != ".adnl"))) { for (const char* suffix : {".adnl", ".ton", ".bag"}) {
if (td::ends_with(host, td::Slice(suffix))) {
allow = true;
}
}
if (!allow) {
promise.set_error(td::Status::Error(ton::ErrorCode::error, "bad server name")); promise.set_error(td::Status::Error(ton::ErrorCode::error, "bad server name"));
return; return;
} }
td::actor::create_actor<TcpToRldpRequestSender>("outboundreq", local_id_, host, std::move(request), td::actor::create_actor<TcpToRldpRequestSender>("outboundreq", local_id_, host, std::move(request),
std::move(payload), std::move(promise), adnl_.get(), dht_.get(), std::move(payload), std::move(promise), adnl_.get(), dht_.get(),
rldp_.get(), actor_id(this), dns_resolver_.get()) rldp_.get(), actor_id(this), dns_resolver_.get(), storage_gateway_)
.release(); .release();
} }
@ -1280,6 +1287,10 @@ class RldpHttpProxy : public td::actor::Actor {
proxy_all_ = value; proxy_all_ = value;
} }
void set_storage_gateway(ton::adnl::AdnlNodeIdShort id) {
storage_gateway_ = id;
}
private: private:
struct Host { struct Host {
struct Server { struct Server {
@ -1317,15 +1328,43 @@ class RldpHttpProxy : public td::actor::Actor {
td::actor::ActorOwn<tonlib::TonlibClientWrapper> tonlib_client_; td::actor::ActorOwn<tonlib::TonlibClientWrapper> tonlib_client_;
td::actor::ActorOwn<DNSResolver> dns_resolver_; td::actor::ActorOwn<DNSResolver> dns_resolver_;
ton::adnl::AdnlNodeIdShort storage_gateway_ = ton::adnl::AdnlNodeIdShort::zero();
std::map<td::Bits256, std::map<td::Bits256,
std::function<void(ton::tl_object_ptr<ton::ton_api::http_getNextPayloadPart>, td::Promise<td::BufferSlice>)>> std::function<void(ton::tl_object_ptr<ton::ton_api::http_getNextPayloadPart>, td::Promise<td::BufferSlice>)>>
payload_senders_; payload_senders_;
}; };
void TcpToRldpRequestSender::resolve() { void TcpToRldpRequestSender::resolve(std::string host) {
auto S = td::Slice(host_); auto S = td::Slice(host);
if (S.size() >= 5 && S.substr(S.size() - 5) == ".adnl") { if (td::ends_with(S, ".bag")) {
if (storage_gateway_.is_zero()) {
abort_query(td::Status::Error("storage gateway is not set"));
return;
}
td::Slice bag_id = S.substr(0, S.size() - 4);
td::Slice url = request_tl_->url_;
if (td::begins_with(url, "http://")) {
url.remove_prefix(7);
}
size_t pos = url.find('/');
if (pos == td::Slice::npos) {
url = "/";
} else {
url = url.substr(pos);
}
request_tl_->url_ = (PSTRING() << "/gateway/" << bag_id << url);
host = storage_gateway_.serialize() + ".adnl";
for (auto& header : request_tl_->headers_) {
if (td::to_lower(header->name_) == "host") {
header->value_ = host;
break;
}
}
resolved(storage_gateway_);
return;
}
if (td::ends_with(S, ".adnl")) {
S.truncate(S.size() - 5); S.truncate(S.size() - 5);
auto R = ton::adnl::AdnlNodeIdShort::parse(S); auto R = ton::adnl::AdnlNodeIdShort::parse(S);
if (R.is_error()) { if (R.is_error()) {
@ -1335,15 +1374,20 @@ void TcpToRldpRequestSender::resolve() {
resolved(R.move_as_ok()); resolved(R.move_as_ok());
return; return;
} }
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<ton::adnl::AdnlNodeIdShort> R) { if (dns_resolve_sent_) {
abort_query(td::Status::Error(PSTRING() << "unexpected dns result: " << host));
return;
}
dns_resolve_sent_ = true;
td::actor::send_closure(dns_resolver_, &DNSResolver::resolve, host,
[SelfId = actor_id(this)](td::Result<std::string> R) {
if (R.is_error()) { if (R.is_error()) {
td::actor::send_closure(SelfId, &TcpToRldpRequestSender::abort_query, td::actor::send_closure(SelfId, &TcpToRldpRequestSender::abort_query,
R.move_as_error_prefix("failed to resolve: ")); R.move_as_error_prefix("failed to resolve: "));
} else { } else {
td::actor::send_closure(SelfId, &TcpToRldpRequestSender::resolved, R.move_as_ok()); td::actor::send_closure(SelfId, &TcpToRldpRequestSender::resolve, R.move_as_ok());
} }
}); });
td::actor::send_closure(dns_resolver_, &DNSResolver::resolve, host_, std::move(P));
} }
void HttpRldpPayloadSender::start_up() { void HttpRldpPayloadSender::start_up() {
@ -1532,6 +1576,11 @@ int main(int argc, char *argv[]) {
logger_ = td::FileLog::create(fname.str()).move_as_ok(); logger_ = td::FileLog::create(fname.str()).move_as_ok();
td::log_interface = logger_.get(); td::log_interface = logger_.get();
}); });
p.add_checked_option('S', "storage-gateway", "adnl address of ton storage gateway", [&](td::Slice arg) -> td::Status {
TRY_RESULT(adnl, ton::adnl::AdnlNodeIdShort::parse(arg));
td::actor::send_closure(x, &RldpHttpProxy::set_storage_gateway, adnl);
return td::Status::OK();
});
p.add_checked_option('P', "proxy-all", "value=[YES|NO]. proxy all HTTP requests (default only *.ton and *.adnl)", p.add_checked_option('P', "proxy-all", "value=[YES|NO]. proxy all HTTP requests (default only *.ton and *.adnl)",
[&](td::Slice value) { [&](td::Slice value) {
if (value == "YES" || value == "yes") { if (value == "YES" || value == "yes") {

2
rldp2/BdwStats.cpp
View file

@ -40,7 +40,7 @@ void BdwStats::on_packet_ack(const PacketInfo &info, td::Timestamp sent_at, td::
auto ack_passed = now.at() - info.delivered_now.at(); auto ack_passed = now.at() - info.delivered_now.at();
auto passed = td::max(sent_passed, ack_passed); auto passed = td::max(sent_passed, ack_passed);
if (passed < 0.01) { if (passed < 0.01) {
VLOG(RLDP_WARNING) << "Invalid passed " << passed; VLOG(RLDP_INFO) << "Invalid passed " << passed;
} }
auto delivered = delivered_count - info.delivered_count; auto delivered = delivered_count - info.delivered_count;
on_rate_sample((double)delivered / passed, now, info.is_paused); on_rate_sample((double)delivered / passed, now, info.is_paused);

7
rldp2/CMakeLists.txt
View file

@ -7,6 +7,9 @@ endif()
if (NOT GSL_FOUND) if (NOT GSL_FOUND)
find_package(GSL) find_package(GSL)
endif() endif()
if (NOT BLAS_FOUND)
find_package(BLAS)
endif()
set(RLDP_SOURCE set(RLDP_SOURCE
Ack.cpp Ack.cpp
@ -50,8 +53,8 @@ target_include_directories(rldp PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>/.. $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>/..
${OPENSSL_INCLUDE_DIR} ${OPENSSL_INCLUDE_DIR}
) )
if (GSL_FOUND) if (GSL_FOUND AND BLAS_FOUND)
target_link_libraries(rldp2 PRIVATE gsl) target_link_libraries(rldp2 PRIVATE gsl blas)
target_compile_definitions(rldp2 PRIVATE -DTON_HAVE_GSL=1) target_compile_definitions(rldp2 PRIVATE -DTON_HAVE_GSL=1)
endif() endif()
target_link_libraries(rldp2 PUBLIC tdutils tdactor fec adnl tl_api) target_link_libraries(rldp2 PUBLIC tdutils tdactor fec adnl tl_api)

4
rldp2/RttStats.cpp
View file

@ -25,11 +25,11 @@ namespace ton {
namespace rldp2 { namespace rldp2 {
void RttStats::on_rtt_sample(double rtt_sample, double ack_delay, td::Timestamp now) { void RttStats::on_rtt_sample(double rtt_sample, double ack_delay, td::Timestamp now) {
if (rtt_sample < 0.001 || rtt_sample > 10) { if (rtt_sample < 0.001 || rtt_sample > 10) {
VLOG(RLDP_WARNING) << "Suspicious rtt sample " << rtt_sample; VLOG(RLDP_INFO) << "Suspicious rtt sample " << rtt_sample;
return; return;
} }
if (ack_delay < -1e-9 || ack_delay > 10) { if (ack_delay < -1e-9 || ack_delay > 10) {
VLOG(RLDP_WARNING) << "Suspicious ack_delay " << ack_delay; VLOG(RLDP_INFO) << "Suspicious ack_delay " << ack_delay;
return; return;
} }
rtt_sample = td::max(0.01, rtt_sample); rtt_sample = td::max(0.01, rtt_sample);

2
storage/LoadSpeed.cpp
View file

@ -22,7 +22,7 @@
#include "td/utils/format.h" #include "td/utils/format.h"
namespace ton { namespace ton {
void LoadSpeed::add(std::size_t size, td::Timestamp now) { void LoadSpeed::add(td::uint64 size, td::Timestamp now) {
total_size_ += size; total_size_ += size;
events_.push(Event{size, now}); events_.push(Event{size, now});
update(now); update(now);

49
storage/NodeActor.cpp
View file

@ -32,24 +32,27 @@
namespace ton { namespace ton {
NodeActor::NodeActor(PeerId self_id, Torrent torrent, td::unique_ptr<Callback> callback, NodeActor::NodeActor(PeerId self_id, Torrent torrent, td::unique_ptr<Callback> callback,
td::unique_ptr<NodeCallback> node_callback, std::shared_ptr<db::DbType> db, bool should_download)
: self_id_(self_id)
, torrent_(std::move(torrent))
, callback_(std::move(callback))
, node_callback_(std::move(node_callback))
, db_(std::move(db))
, should_download_(should_download) {
}
NodeActor::NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr<Callback> callback,
td::unique_ptr<NodeCallback> node_callback, std::shared_ptr<db::DbType> db, bool should_download, td::unique_ptr<NodeCallback> node_callback, std::shared_ptr<db::DbType> db, bool should_download,
DbInitialData db_initial_data) bool should_upload)
: self_id_(self_id) : self_id_(self_id)
, torrent_(std::move(torrent)) , torrent_(std::move(torrent))
, callback_(std::move(callback)) , callback_(std::move(callback))
, node_callback_(std::move(node_callback)) , node_callback_(std::move(node_callback))
, db_(std::move(db)) , db_(std::move(db))
, should_download_(should_download) , should_download_(should_download)
, should_upload_(should_upload) {
}
NodeActor::NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr<Callback> callback,
td::unique_ptr<NodeCallback> node_callback, std::shared_ptr<db::DbType> db, bool should_download,
bool should_upload, DbInitialData db_initial_data)
: self_id_(self_id)
, torrent_(std::move(torrent))
, callback_(std::move(callback))
, node_callback_(std::move(node_callback))
, db_(std::move(db))
, should_download_(should_download)
, should_upload_(should_upload)
, pending_set_file_priority_(std::move(db_initial_data.priorities)) , pending_set_file_priority_(std::move(db_initial_data.priorities))
, pieces_in_db_(std::move(db_initial_data.pieces_in_db)) { , pieces_in_db_(std::move(db_initial_data.pieces_in_db)) {
} }
@ -199,7 +202,7 @@ void NodeActor::loop_will_upload() {
} }
for (auto &it : peers_) { for (auto &it : peers_) {
auto will_upload = peers_set.count(it.first) > 0; auto will_upload = peers_set.count(it.first) > 0 && should_upload_;
auto &state = it.second.state; auto &state = it.second.state;
auto node_state = state->node_state_.load(); auto node_state = state->node_state_.load();
if (node_state.will_upload != will_upload) { if (node_state.will_upload != will_upload) {
@ -399,6 +402,15 @@ void NodeActor::set_should_download(bool should_download) {
yield(); yield();
} }
void NodeActor::set_should_upload(bool should_upload) {
if (should_upload == should_upload_) {
return;
}
should_upload_ = should_upload;
db_store_torrent();
yield();
}
void NodeActor::load_from(td::optional<TorrentMeta> meta, std::string files_path, td::Promise<td::Unit> promise) { void NodeActor::load_from(td::optional<TorrentMeta> meta, std::string files_path, td::Promise<td::Unit> promise) {
auto S = [&]() -> td::Status { auto S = [&]() -> td::Status {
if (meta) { if (meta) {
@ -442,6 +454,12 @@ void NodeActor::load_from(td::optional<TorrentMeta> meta, std::string files_path
loop(); loop();
} }
void NodeActor::copy_to_new_root_dir(std::string new_root_dir, td::Promise<td::Unit> promise) {
TRY_STATUS_PROMISE(promise, torrent_.copy_to(new_root_dir));
db_store_torrent();
promise.set_result(td::Unit());
}
void NodeActor::tear_down() { void NodeActor::tear_down() {
for (auto &promise : wait_for_completion_) { for (auto &promise : wait_for_completion_) {
promise.set_error(td::Status::Error("Torrent closed")); promise.set_error(td::Status::Error("Torrent closed"));
@ -573,7 +591,7 @@ void NodeActor::loop_peer(const PeerId &peer_id, Peer &peer) {
for (td::uint32 part_id : state->peer_queries_.read()) { for (td::uint32 part_id : state->peer_queries_.read()) {
should_notify_peer = true; should_notify_peer = true;
auto res = [&]() -> td::Result<PeerState::Part> { auto res = [&]() -> td::Result<PeerState::Part> {
if (!node_state.will_upload) { if (!node_state.will_upload || !should_upload_) {
return td::Status::Error("Won't upload"); return td::Status::Error("Won't upload");
} }
TRY_RESULT(proof, torrent_.get_piece_proof(part_id)); TRY_RESULT(proof, torrent_.get_piece_proof(part_id));
@ -685,6 +703,7 @@ void NodeActor::db_store_torrent() {
} }
auto obj = create_tl_object<ton_api::storage_db_torrent>(); auto obj = create_tl_object<ton_api::storage_db_torrent>();
obj->active_download_ = should_download_; obj->active_download_ = should_download_;
obj->active_upload_ = should_upload_;
obj->root_dir_ = torrent_.get_root_dir(); obj->root_dir_ = torrent_.get_root_dir();
db_->set(create_hash_tl_object<ton_api::storage_db_key_torrent>(torrent_.get_hash()), serialize_tl_object(obj, true), db_->set(create_hash_tl_object<ton_api::storage_db_key_torrent>(torrent_.get_hash()), serialize_tl_object(obj, true),
[](td::Result<td::Unit> R) { [](td::Result<td::Unit> R) {
@ -837,6 +856,7 @@ void NodeActor::load_from_db(std::shared_ptr<db::DbType> db, td::Bits256 hash, t
void got_torrent(tl_object_ptr<ton_api::storage_db_torrent> obj) { void got_torrent(tl_object_ptr<ton_api::storage_db_torrent> obj) {
root_dir_ = std::move(obj->root_dir_); root_dir_ = std::move(obj->root_dir_);
active_download_ = obj->active_download_; active_download_ = obj->active_download_;
active_upload_ = obj->active_upload_;
db_->get(create_hash_tl_object<ton_api::storage_db_key_torrentMeta>(hash_), db_->get(create_hash_tl_object<ton_api::storage_db_key_torrentMeta>(hash_),
[SelfId = actor_id(this)](td::Result<db::DbType::GetResult> R) { [SelfId = actor_id(this)](td::Result<db::DbType::GetResult> R) {
if (R.is_error()) { if (R.is_error()) {
@ -962,7 +982,7 @@ void NodeActor::load_from_db(std::shared_ptr<db::DbType> db, td::Bits256 hash, t
data.pieces_in_db = std::move(pieces_in_db_); data.pieces_in_db = std::move(pieces_in_db_);
finish(td::actor::create_actor<NodeActor>("Node", 1, torrent_.unwrap(), std::move(callback_), finish(td::actor::create_actor<NodeActor>("Node", 1, torrent_.unwrap(), std::move(callback_),
std::move(node_callback_), std::move(db_), active_download_, std::move(node_callback_), std::move(db_), active_download_,
std::move(data))); active_upload_, std::move(data)));
} }
private: private:
@ -974,6 +994,7 @@ void NodeActor::load_from_db(std::shared_ptr<db::DbType> db, td::Bits256 hash, t
std::string root_dir_; std::string root_dir_;
bool active_download_{false}; bool active_download_{false};
bool active_upload_{false};
td::optional<Torrent> torrent_; td::optional<Torrent> torrent_;
std::vector<PendingSetFilePriority> priorities_; std::vector<PendingSetFilePriority> priorities_;
std::set<td::uint64> pieces_in_db_; std::set<td::uint64> pieces_in_db_;

13
storage/NodeActor.h
View file

@ -63,32 +63,36 @@ class NodeActor : public td::actor::Actor {
}; };
NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr<Callback> callback, NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr<Callback> callback,
td::unique_ptr<NodeCallback> node_callback, std::shared_ptr<db::DbType> db, bool should_download = true); td::unique_ptr<NodeCallback> node_callback, std::shared_ptr<db::DbType> db, bool should_download = true,
bool should_upload = true);
NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr<Callback> callback, NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr<Callback> callback,
td::unique_ptr<NodeCallback> node_callback, std::shared_ptr<db::DbType> db, bool should_download, td::unique_ptr<NodeCallback> node_callback, std::shared_ptr<db::DbType> db, bool should_download,
DbInitialData db_initial_data); bool should_upload, DbInitialData db_initial_data);
void start_peer(PeerId peer_id, td::Promise<td::actor::ActorId<PeerActor>> promise); void start_peer(PeerId peer_id, td::Promise<td::actor::ActorId<PeerActor>> promise);
struct NodeState { struct NodeState {
Torrent &torrent; Torrent &torrent;
bool active_download; bool active_download;
bool active_upload;
double download_speed; double download_speed;
double upload_speed; double upload_speed;
const std::vector<td::uint8> &file_priority; const std::vector<td::uint8> &file_priority;
}; };
void with_torrent(td::Promise<NodeState> promise) { void with_torrent(td::Promise<NodeState> promise) {
promise.set_value( promise.set_value(NodeState{torrent_, should_download_, should_upload_, download_speed_.speed(),
NodeState{torrent_, should_download_, download_speed_.speed(), upload_speed_.speed(), file_priority_}); upload_speed_.speed(), file_priority_});
} }
std::string get_stats_str(); std::string get_stats_str();
void set_should_download(bool should_download); void set_should_download(bool should_download);
void set_should_upload(bool should_upload);
void set_all_files_priority(td::uint8 priority, td::Promise<bool> promise); void set_all_files_priority(td::uint8 priority, td::Promise<bool> promise);
void set_file_priority_by_idx(size_t i, td::uint8 priority, td::Promise<bool> promise); void set_file_priority_by_idx(size_t i, td::uint8 priority, td::Promise<bool> promise);
void set_file_priority_by_name(std::string name, td::uint8 priority, td::Promise<bool> promise); void set_file_priority_by_name(std::string name, td::uint8 priority, td::Promise<bool> promise);
void load_from(td::optional<TorrentMeta> meta, std::string files_path, td::Promise<td::Unit> promise); void load_from(td::optional<TorrentMeta> meta, std::string files_path, td::Promise<td::Unit> promise);
void copy_to_new_root_dir(std::string new_root_dir, td::Promise<td::Unit> promise);
void wait_for_completion(td::Promise<td::Unit> promise); void wait_for_completion(td::Promise<td::Unit> promise);
void get_peers_info(td::Promise<tl_object_ptr<ton_api::storage_daemon_peerList>> promise); void get_peers_info(td::Promise<tl_object_ptr<ton_api::storage_daemon_peerList>> promise);
@ -107,6 +111,7 @@ class NodeActor : public td::actor::Actor {
td::unique_ptr<NodeCallback> node_callback_; td::unique_ptr<NodeCallback> node_callback_;
std::shared_ptr<db::DbType> db_; std::shared_ptr<db::DbType> db_;
bool should_download_{false}; bool should_download_{false};
bool should_upload_{false};
class Notifier : public td::actor::Actor { class Notifier : public td::actor::Actor {
public: public:

33
storage/Torrent.cpp
View file

@ -698,4 +698,37 @@ void Torrent::load_from_files(std::string files_path) {
} }
} }
td::Status Torrent::copy_to(const std::string &new_root_dir) {
if (!is_completed() || included_size_ != info_.file_size) {
return td::Status::Error("Torrent::copy_to is allowed only for fully completed torrents");
}
auto get_new_chunk_path = [&](td::Slice name) -> std::string {
return PSTRING() << new_root_dir << TD_DIR_SLASH << header_.value().dir_name << TD_DIR_SLASH << name;
};
std::vector<td::BlobView> new_blobs;
for (size_t i = 1; i < chunks_.size(); ++i) {
auto &chunk = chunks_[i];
std::string new_path = get_new_chunk_path(chunk.name);
TRY_STATUS(td::mkpath(new_path));
TRY_RESULT(new_blob, td::FileNoCacheBlobView::create(new_path, chunk.size, true));
static const td::uint64 BUF_SIZE = 1 << 17;
td::BufferSlice buf(BUF_SIZE);
for (td::uint64 l = 0; l < chunk.size; l += BUF_SIZE) {
td::uint64 r = std::min(chunk.size, l + BUF_SIZE);
TRY_RESULT_PREFIX(s, chunk.data.view(buf.as_slice().substr(0, r - l), l),
PSTRING() << "Failed to read " << chunk.name << ": ");
if (s.size() != r - l) {
return td::Status::Error(PSTRING() << "Failed to read " << chunk.name);
}
TRY_STATUS_PREFIX(new_blob.write(s, l), PSTRING() << "Failed to write " << chunk.name << ": ");
}
new_blobs.push_back(std::move(new_blob));
}
root_dir_ = new_root_dir;
for (size_t i = 1; i < chunks_.size(); ++i) {
chunks_[i].data = std::move(new_blobs[i - 1]);
}
return td::Status::OK();
}
} // namespace ton } // namespace ton

2
storage/Torrent.h
View file

@ -159,6 +159,8 @@ class Torrent {
void load_from_files(std::string files_path); void load_from_files(std::string files_path);
td::Status copy_to(const std::string& new_root_dir);
private: private:
td::Bits256 hash_; td::Bits256 hash_;
bool inited_info_ = false; bool inited_info_ = false;

3
storage/TorrentCreator.cpp
View file

@ -98,6 +98,9 @@ TD_WARN_UNUSED_RESULT td::Status Torrent::Creator::add_file(td::Slice name, td::
} }
td::Result<Torrent> Torrent::Creator::finalize() { td::Result<Torrent> Torrent::Creator::finalize() {
if (files_.empty()) {
return td::Status::Error("No files");
}
TorrentHeader header; TorrentHeader header;
TRY_RESULT(files_count, td::narrow_cast_safe<td::uint32>(files_.size())); TRY_RESULT(files_count, td::narrow_cast_safe<td::uint32>(files_.size()));
header.files_count = files_count; header.files_count = files_count;

3
storage/TorrentHeader.cpp
View file

@ -100,6 +100,9 @@ td::Status TorrentHeader::validate(td::uint64 total_size, td::uint64 header_size
if (serialization_size() != header_size) { if (serialization_size() != header_size) {
return td::Status::Error("Invalid size"); return td::Status::Error("Invalid size");
} }
if (files_count == 0) {
return td::Status::Error("No files");
}
for (size_t i = 0; i + 1 < files_count; ++i) { for (size_t i = 0; i + 1 < files_count; ++i) {
if (name_index[i] > name_index[i + 1]) { if (name_index[i] > name_index[i + 1]) {
return td::Status::Error("Invalid name offset"); return td::Status::Error("Invalid name offset");

45
storage/storage-daemon/StorageManager.cpp
View file

@ -21,6 +21,8 @@
#include "td/db/RocksDb.h" #include "td/db/RocksDb.h"
#include "td/actor/MultiPromise.h" #include "td/actor/MultiPromise.h"
namespace ton {
static overlay::OverlayIdFull get_overlay_id(td::Bits256 hash) { static overlay::OverlayIdFull get_overlay_id(td::Bits256 hash) {
td::BufferSlice hash_str(hash.as_slice()); td::BufferSlice hash_str(hash.as_slice());
return overlay::OverlayIdFull(std::move(hash_str)); return overlay::OverlayIdFull(std::move(hash_str));
@ -126,16 +128,33 @@ td::unique_ptr<NodeActor::Callback> StorageManager::create_callback(
return td::make_unique<Callback>(actor_id(this), hash, std::move(closing_state)); return td::make_unique<Callback>(actor_id(this), hash, std::move(closing_state));
} }
void StorageManager::add_torrent(Torrent torrent, bool start_download, td::Promise<td::Unit> promise) { void StorageManager::add_torrent(Torrent torrent, bool start_download, bool allow_upload, bool copy_inside,
TRY_STATUS_PROMISE(promise, add_torrent_impl(std::move(torrent), start_download)); td::Promise<td::Unit> promise) {
td::Bits256 hash = torrent.get_hash();
TRY_STATUS_PROMISE(promise, add_torrent_impl(std::move(torrent), start_download, allow_upload));
db_store_torrent_list(); db_store_torrent_list();
if (!copy_inside) {
promise.set_result(td::Unit()); promise.set_result(td::Unit());
return;
}
TorrentEntry& entry = torrents_[hash];
std::string new_dir = db_root_ + "/torrent-files/" + hash.to_hex();
LOG(INFO) << "Copy torrent to " << new_dir;
td::actor::send_closure(
entry.actor, &NodeActor::copy_to_new_root_dir, std::move(new_dir),
[SelfId = actor_id(this), hash, promise = std::move(promise)](td::Result<td::Unit> R) mutable {
if (R.is_error()) {
LOG(WARNING) << "Copy torrent: " << R.error();
td::actor::send_closure(SelfId, &StorageManager::remove_torrent, hash, false, [](td::Result<td::Unit> R) {});
}
promise.set_result(std::move(R));
});
} }
td::Status StorageManager::add_torrent_impl(Torrent torrent, bool start_download) { td::Status StorageManager::add_torrent_impl(Torrent torrent, bool start_download, bool allow_upload) {
td::Bits256 hash = torrent.get_hash(); td::Bits256 hash = torrent.get_hash();
if (torrents_.count(hash)) { if (torrents_.count(hash)) {
return td::Status::Error("Cannot add torrent: duplicate hash"); return td::Status::Error(PSTRING() << "Cannot add torrent: duplicate hash " << hash.to_hex());
} }
TorrentEntry& entry = torrents_[hash]; TorrentEntry& entry = torrents_[hash];
entry.hash = hash; entry.hash = hash;
@ -145,25 +164,25 @@ td::Status StorageManager::add_torrent_impl(Torrent torrent, bool start_download
LOG(INFO) << "Added torrent " << hash.to_hex() << " , root_dir = " << torrent.get_root_dir(); LOG(INFO) << "Added torrent " << hash.to_hex() << " , root_dir = " << torrent.get_root_dir();
entry.actor = entry.actor =
td::actor::create_actor<NodeActor>("Node", 1, std::move(torrent), create_callback(hash, entry.closing_state), td::actor::create_actor<NodeActor>("Node", 1, std::move(torrent), create_callback(hash, entry.closing_state),
std::move(context), db_, start_download); std::move(context), db_, start_download, allow_upload);
return td::Status::OK(); return td::Status::OK();
} }
void StorageManager::add_torrent_by_meta(TorrentMeta meta, std::string root_dir, bool start_download, void StorageManager::add_torrent_by_meta(TorrentMeta meta, std::string root_dir, bool start_download, bool allow_upload,
td::Promise<td::Unit> promise) { td::Promise<td::Unit> promise) {
td::Bits256 hash(meta.info.get_hash()); td::Bits256 hash(meta.info.get_hash());
Torrent::Options options; Torrent::Options options;
options.root_dir = root_dir.empty() ? db_root_ + "/torrent-files/" + hash.to_hex() : root_dir; options.root_dir = root_dir.empty() ? db_root_ + "/torrent-files/" + hash.to_hex() : root_dir;
TRY_RESULT_PROMISE(promise, torrent, Torrent::open(std::move(options), std::move(meta))); TRY_RESULT_PROMISE(promise, torrent, Torrent::open(std::move(options), std::move(meta)));
add_torrent(std::move(torrent), start_download, std::move(promise)); add_torrent(std::move(torrent), start_download, allow_upload, false, std::move(promise));
} }
void StorageManager::add_torrent_by_hash(td::Bits256 hash, std::string root_dir, bool start_download, void StorageManager::add_torrent_by_hash(td::Bits256 hash, std::string root_dir, bool start_download, bool allow_upload,
td::Promise<td::Unit> promise) { td::Promise<td::Unit> promise) {
Torrent::Options options; Torrent::Options options;
options.root_dir = root_dir.empty() ? db_root_ + "/torrent-files/" + hash.to_hex() : root_dir; options.root_dir = root_dir.empty() ? db_root_ + "/torrent-files/" + hash.to_hex() : root_dir;
TRY_RESULT_PROMISE(promise, torrent, Torrent::open(std::move(options), hash)); TRY_RESULT_PROMISE(promise, torrent, Torrent::open(std::move(options), hash));
add_torrent(std::move(torrent), start_download, std::move(promise)); add_torrent(std::move(torrent), start_download, allow_upload, false, std::move(promise));
} }
void StorageManager::set_active_download(td::Bits256 hash, bool active, td::Promise<td::Unit> promise) { void StorageManager::set_active_download(td::Bits256 hash, bool active, td::Promise<td::Unit> promise) {
@ -172,6 +191,12 @@ void StorageManager::set_active_download(td::Bits256 hash, bool active, td::Prom
promise.set_result(td::Unit()); promise.set_result(td::Unit());
} }
void StorageManager::set_active_upload(td::Bits256 hash, bool active, td::Promise<td::Unit> promise) {
TRY_RESULT_PROMISE(promise, entry, get_torrent(hash));
td::actor::send_closure(entry->actor, &NodeActor::set_should_upload, active);
promise.set_result(td::Unit());
}
void StorageManager::with_torrent(td::Bits256 hash, td::Promise<NodeActor::NodeState> promise) { void StorageManager::with_torrent(td::Bits256 hash, td::Promise<NodeActor::NodeState> promise) {
TRY_RESULT_PROMISE(promise, entry, get_torrent(hash)); TRY_RESULT_PROMISE(promise, entry, get_torrent(hash));
td::actor::send_closure(entry->actor, &NodeActor::with_torrent, std::move(promise)); td::actor::send_closure(entry->actor, &NodeActor::with_torrent, std::move(promise));
@ -266,3 +291,5 @@ void StorageManager::get_peers_info(td::Bits256 hash,
TRY_RESULT_PROMISE(promise, entry, get_torrent(hash)); TRY_RESULT_PROMISE(promise, entry, get_torrent(hash));
td::actor::send_closure(entry->actor, &NodeActor::get_peers_info, std::move(promise)); td::actor::send_closure(entry->actor, &NodeActor::get_peers_info, std::move(promise));
} }
} // namespace ton

16
storage/storage-daemon/StorageManager.h
View file

@ -23,7 +23,7 @@
#include "storage/PeerManager.h" #include "storage/PeerManager.h"
#include "storage/db.h" #include "storage/db.h"
using namespace ton; namespace ton {
class StorageManager : public td::actor::Actor { class StorageManager : public td::actor::Actor {
public: public:
@ -39,11 +39,15 @@ class StorageManager : public td::actor::Actor {
void start_up() override; void start_up() override;
void add_torrent(Torrent torrent, bool start_download, td::Promise<td::Unit> promise); void add_torrent(Torrent torrent, bool start_download, bool allow_upload, bool copy_inside,
void add_torrent_by_meta(TorrentMeta meta, std::string root_dir, bool start_download, td::Promise<td::Unit> promise); td::Promise<td::Unit> promise);
void add_torrent_by_hash(td::Bits256 hash, std::string root_dir, bool start_download, td::Promise<td::Unit> promise); void add_torrent_by_meta(TorrentMeta meta, std::string root_dir, bool start_download, bool allow_upload,
td::Promise<td::Unit> promise);
void add_torrent_by_hash(td::Bits256 hash, std::string root_dir, bool start_download, bool allow_upload,
td::Promise<td::Unit> promise);
void set_active_download(td::Bits256 hash, bool active, td::Promise<td::Unit> promise); void set_active_download(td::Bits256 hash, bool active, td::Promise<td::Unit> promise);
void set_active_upload(td::Bits256 hash, bool active, td::Promise<td::Unit> promise);
void with_torrent(td::Bits256 hash, td::Promise<NodeActor::NodeState> promise); void with_torrent(td::Bits256 hash, td::Promise<NodeActor::NodeState> promise);
void get_all_torrents(td::Promise<std::vector<td::Bits256>> promise); void get_all_torrents(td::Promise<std::vector<td::Bits256>> promise);
@ -85,7 +89,7 @@ class StorageManager : public td::actor::Actor {
std::map<td::Bits256, TorrentEntry> torrents_; std::map<td::Bits256, TorrentEntry> torrents_;
td::Status add_torrent_impl(Torrent torrent, bool start_download); td::Status add_torrent_impl(Torrent torrent, bool start_download, bool allow_upload);
td::Result<TorrentEntry*> get_torrent(td::Bits256 hash) { td::Result<TorrentEntry*> get_torrent(td::Bits256 hash) {
auto it = torrents_.find(hash); auto it = torrents_.find(hash);
@ -105,3 +109,5 @@ class StorageManager : public td::actor::Actor {
void on_torrent_closed(Torrent torrent, std::shared_ptr<TorrentEntry::ClosingState> closing_state); void on_torrent_closed(Torrent torrent, std::shared_ptr<TorrentEntry::ClosingState> closing_state);
}; };
} // namespace ton

34
storage/storage-daemon/StorageProvider.cpp
View file

@ -24,6 +24,8 @@
#include "common/delay.h" #include "common/delay.h"
#include "td/actor/MultiPromise.h" #include "td/actor/MultiPromise.h"
namespace ton {
td::Result<ProviderParams> ProviderParams::create(const tl_object_ptr<ton_api::storage_daemon_provider_params>& obj) { td::Result<ProviderParams> ProviderParams::create(const tl_object_ptr<ton_api::storage_daemon_provider_params>& obj) {
ProviderParams p; ProviderParams p;
p.accept_new_contracts = obj->accept_new_contracts_; p.accept_new_contracts = obj->accept_new_contracts_;
@ -155,7 +157,7 @@ void StorageProvider::start_up() {
init_new_storage_contract(address, contract); init_new_storage_contract(address, contract);
break; break;
case StorageContract::st_downloaded: case StorageContract::st_downloaded:
check_contract_active(address); after_contract_downloaded(address);
break; break;
case StorageContract::st_active: case StorageContract::st_active:
contract.check_next_proof_at = td::Timestamp::now(); contract.check_next_proof_at = td::Timestamp::now();
@ -382,7 +384,7 @@ void StorageProvider::db_update_microchunk_tree(const ContractAddress& address)
void StorageProvider::init_new_storage_contract(ContractAddress address, StorageContract& contract) { void StorageProvider::init_new_storage_contract(ContractAddress address, StorageContract& contract) {
CHECK(contract.state == StorageContract::st_downloading); CHECK(contract.state == StorageContract::st_downloading);
td::actor::send_closure(storage_manager_, &StorageManager::add_torrent_by_hash, contract.torrent_hash, "", false, td::actor::send_closure(storage_manager_, &StorageManager::add_torrent_by_hash, contract.torrent_hash, "", false,
[](td::Result<td::Unit> R) { false, [](td::Result<td::Unit> R) {
// Ignore errors: error can mean that the torrent already exists, other errors will be caught later // Ignore errors: error can mean that the torrent already exists, other errors will be caught later
if (R.is_error()) { if (R.is_error()) {
LOG(DEBUG) << "Add torrent: " << R.move_as_error(); LOG(DEBUG) << "Add torrent: " << R.move_as_error();
@ -449,11 +451,25 @@ void StorageProvider::downloaded_torrent(ContractAddress address, MicrochunkTree
contract.microchunk_tree = std::make_shared<MicrochunkTree>(std::move(microchunk_tree)); contract.microchunk_tree = std::make_shared<MicrochunkTree>(std::move(microchunk_tree));
db_update_microchunk_tree(address); db_update_microchunk_tree(address);
db_update_storage_contract(address, false); db_update_storage_contract(address, false);
check_contract_active(address); after_contract_downloaded(address);
} }
void StorageProvider::check_contract_active(ContractAddress address, td::Timestamp retry_until, void StorageProvider::after_contract_downloaded(ContractAddress address, td::Timestamp retry_until,
td::Timestamp retry_false_until) { td::Timestamp retry_false_until) {
auto it = contracts_.find(address);
if (it == contracts_.end()) {
LOG(WARNING) << "Contract " << address.to_string() << " does not exist anymore";
return;
}
auto& contract = it->second;
td::actor::send_closure(storage_manager_, &StorageManager::set_active_upload, contract.torrent_hash, true,
[SelfId = actor_id(this), address](td::Result<td::Unit> R) {
if (R.is_error()) {
LOG(ERROR) << "Set active upload: " << R.move_as_error();
return;
}
LOG(DEBUG) << "Set active upload: OK";
});
get_storage_contract_data(address, tonlib_client_, get_storage_contract_data(address, tonlib_client_,
[=, SelfId = actor_id(this)](td::Result<StorageContractData> R) mutable { [=, SelfId = actor_id(this)](td::Result<StorageContractData> R) mutable {
if (R.is_error()) { if (R.is_error()) {
@ -461,7 +477,7 @@ void StorageProvider::check_contract_active(ContractAddress address, td::Timesta
if (retry_until && retry_until.is_in_past()) { if (retry_until && retry_until.is_in_past()) {
delay_action( delay_action(
[=]() { [=]() {
td::actor::send_closure(SelfId, &StorageProvider::check_contract_active, td::actor::send_closure(SelfId, &StorageProvider::after_contract_downloaded,
address, retry_until, retry_false_until); address, retry_until, retry_false_until);
}, },
td::Timestamp::in(5.0)); td::Timestamp::in(5.0));
@ -473,8 +489,8 @@ void StorageProvider::check_contract_active(ContractAddress address, td::Timesta
} else if (retry_false_until && retry_false_until.is_in_past()) { } else if (retry_false_until && retry_false_until.is_in_past()) {
delay_action( delay_action(
[=]() { [=]() {
td::actor::send_closure(SelfId, &StorageProvider::check_contract_active, address, td::actor::send_closure(SelfId, &StorageProvider::after_contract_downloaded,
retry_until, retry_false_until); address, retry_until, retry_false_until);
}, },
td::Timestamp::in(5.0)); td::Timestamp::in(5.0));
} else { } else {
@ -497,7 +513,7 @@ void StorageProvider::activate_contract_cont(ContractAddress address) {
td::Timestamp::in(10.0)); td::Timestamp::in(10.0));
return; return;
} }
td::actor::send_closure(SelfId, &StorageProvider::check_contract_active, address, td::Timestamp::in(60.0), td::actor::send_closure(SelfId, &StorageProvider::after_contract_downloaded, address, td::Timestamp::in(60.0),
td::Timestamp::in(40.0)); td::Timestamp::in(40.0));
}); });
} }
@ -838,3 +854,5 @@ StorageProvider::Config::Config(const tl_object_ptr<ton_api::storage_daemon_prov
tl_object_ptr<ton_api::storage_daemon_providerConfig> StorageProvider::Config::tl() const { tl_object_ptr<ton_api::storage_daemon_providerConfig> StorageProvider::Config::tl() const {
return create_tl_object<ton_api::storage_daemon_providerConfig>(max_contracts, max_total_size); return create_tl_object<ton_api::storage_daemon_providerConfig>(max_contracts, max_total_size);
} }
} // namespace ton

6
storage/storage-daemon/StorageProvider.h
View file

@ -24,7 +24,7 @@
#include "smc-util.h" #include "smc-util.h"
#include "storage/MicrochunkTree.h" #include "storage/MicrochunkTree.h"
using namespace ton; namespace ton {
struct ProviderParams { struct ProviderParams {
bool accept_new_contracts = false; bool accept_new_contracts = false;
@ -109,7 +109,7 @@ class StorageProvider : public td::actor::Actor {
void on_new_storage_contract_cont(ContractAddress address, StorageContractData data, td::Promise<td::Unit> promise); void on_new_storage_contract_cont(ContractAddress address, StorageContractData data, td::Promise<td::Unit> promise);
void init_new_storage_contract(ContractAddress address, StorageContract& contract); void init_new_storage_contract(ContractAddress address, StorageContract& contract);
void downloaded_torrent(ContractAddress address, MicrochunkTree microchunk_tree); void downloaded_torrent(ContractAddress address, MicrochunkTree microchunk_tree);
void check_contract_active(ContractAddress address, td::Timestamp retry_until = td::Timestamp::in(30.0), void after_contract_downloaded(ContractAddress address, td::Timestamp retry_until = td::Timestamp::in(30.0),
td::Timestamp retry_false_until = td::Timestamp::never()); td::Timestamp retry_false_until = td::Timestamp::never());
void activate_contract_cont(ContractAddress address); void activate_contract_cont(ContractAddress address);
void activated_storage_contract(ContractAddress address); void activated_storage_contract(ContractAddress address);
@ -125,3 +125,5 @@ class StorageProvider : public td::actor::Actor {
void got_next_proof(ContractAddress address, td::Result<td::Ref<vm::Cell>> R); void got_next_proof(ContractAddress address, td::Result<td::Ref<vm::Cell>> R);
void sent_next_proof(ContractAddress address); void sent_next_proof(ContractAddress address);
}; };
} // namespace ton

6
storage/storage-daemon/smc-util.cpp
View file

@ -20,6 +20,8 @@
#include "keys/encryptor.h" #include "keys/encryptor.h"
#include "smartcont/provider-code.h" #include "smartcont/provider-code.h"
namespace ton {
static void smc_forget(td::actor::ActorId<tonlib::TonlibClientWrapper> client, td::int64 id) { static void smc_forget(td::actor::ActorId<tonlib::TonlibClientWrapper> client, td::int64 id) {
auto query = create_tl_object<tonlib_api::smc_forget>(id); auto query = create_tl_object<tonlib_api::smc_forget>(id);
td::actor::send_closure(client, &tonlib::TonlibClientWrapper::send_request<tonlib_api::smc_forget>, std::move(query), td::actor::send_closure(client, &tonlib::TonlibClientWrapper::send_request<tonlib_api::smc_forget>, std::move(query),
@ -226,7 +228,7 @@ void FabricContractWrapper::loaded_last_transactions(
void FabricContractWrapper::run_get_method( void FabricContractWrapper::run_get_method(
std::string method, std::vector<tl_object_ptr<tonlib_api::tvm_StackEntry>> args, std::string method, std::vector<tl_object_ptr<tonlib_api::tvm_StackEntry>> args,
td::Promise<std::vector<tl_object_ptr<tonlib_api::tvm_StackEntry>>> promise) { td::Promise<std::vector<tl_object_ptr<tonlib_api::tvm_StackEntry>>> promise) {
::run_get_method(address_, client_, std::move(method), std::move(args), std::move(promise)); ton::run_get_method(address_, client_, std::move(method), std::move(args), std::move(promise));
} }
void FabricContractWrapper::send_internal_message(ContractAddress dest, td::RefInt256 coins, vm::CellSlice body, void FabricContractWrapper::send_internal_message(ContractAddress dest, td::RefInt256 coins, vm::CellSlice body,
@ -485,3 +487,5 @@ void get_storage_contract_data(ContractAddress address, td::actor::ActorId<tonli
rate_per_mb_day, max_span, last_proof_time, torrent_hash}; rate_per_mb_day, max_span, last_proof_time, torrent_hash};
})); }));
} }
} // namespace ton

4
storage/storage-daemon/smc-util.h
View file

@ -24,7 +24,7 @@
#include <queue> #include <queue>
#include "keyring/keyring.h" #include "keyring/keyring.h"
using namespace ton; namespace ton {
struct ContractAddress { struct ContractAddress {
WorkchainId wc = workchainIdNotYet; WorkchainId wc = workchainIdNotYet;
@ -182,3 +182,5 @@ struct StorageContractData {
void get_storage_contract_data(ContractAddress address, td::actor::ActorId<tonlib::TonlibClientWrapper> client, void get_storage_contract_data(ContractAddress address, td::actor::ActorId<tonlib::TonlibClientWrapper> client,
td::Promise<StorageContractData> promise); td::Promise<StorageContractData> promise);
} // namespace ton

128
storage/storage-daemon/storage-daemon-cli.cpp
View file

@ -39,7 +39,7 @@
#include "common/refint.h" #include "common/refint.h"
#include "crypto/block/block.h" #include "crypto/block/block.h"
using namespace ton; namespace ton {
bool is_whitespace(char c) { bool is_whitespace(char c) {
return strchr(" \t\n\r", c) != nullptr; return strchr(" \t\n\r", c) != nullptr;
@ -298,10 +298,13 @@ class StorageDaemonCli : public td::actor::Actor {
} else if (tokens[0] == "create") { } else if (tokens[0] == "create") {
std::string path; std::string path;
bool found_path = false; bool found_path = false;
bool upload = true;
bool copy = false;
std::string description; std::string description;
bool json = false; bool json = false;
bool no_more_flags = false;
for (size_t i = 1; i < tokens.size(); ++i) { for (size_t i = 1; i < tokens.size(); ++i) {
if (!tokens[i].empty() && tokens[i][0] == '-') { if (!tokens[i].empty() && tokens[i][0] == '-' && !no_more_flags) {
if (tokens[i] == "-d") { if (tokens[i] == "-d") {
++i; ++i;
if (i == tokens.size()) { if (i == tokens.size()) {
@ -310,10 +313,22 @@ class StorageDaemonCli : public td::actor::Actor {
description = tokens[i]; description = tokens[i];
continue; continue;
} }
if (tokens[i] == "--no-upload") {
upload = false;
continue;
}
if (tokens[i] == "--copy") {
copy = true;
continue;
}
if (tokens[i] == "--json") { if (tokens[i] == "--json") {
json = true; json = true;
continue; continue;
} }
if (tokens[i] == "--") {
no_more_flags = true;
continue;
}
return td::Status::Error(PSTRING() << "Unknown flag " << tokens[i]); return td::Status::Error(PSTRING() << "Unknown flag " << tokens[i]);
} }
if (found_path) { if (found_path) {
@ -325,15 +340,17 @@ class StorageDaemonCli : public td::actor::Actor {
if (!found_path) { if (!found_path) {
return td::Status::Error("Unexpected EOLN"); return td::Status::Error("Unexpected EOLN");
} }
return execute_create(std::move(path), std::move(description), json); return execute_create(std::move(path), std::move(description), upload, copy, json);
} else if (tokens[0] == "add-by-hash" || tokens[0] == "add-by-meta") { } else if (tokens[0] == "add-by-hash" || tokens[0] == "add-by-meta") {
td::optional<std::string> param; td::optional<std::string> param;
std::string root_dir; std::string root_dir;
bool paused = false; bool paused = false;
bool upload = true;
bool json = false; bool json = false;
td::optional<std::vector<std::string>> partial; td::optional<std::vector<std::string>> partial;
bool no_more_flags = false;
for (size_t i = 1; i < tokens.size(); ++i) { for (size_t i = 1; i < tokens.size(); ++i) {
if (!tokens[i].empty() && tokens[i][0] == '-') { if (!tokens[i].empty() && tokens[i][0] == '-' && !no_more_flags) {
if (tokens[i] == "-d") { if (tokens[i] == "-d") {
++i; ++i;
if (i == tokens.size()) { if (i == tokens.size()) {
@ -346,6 +363,10 @@ class StorageDaemonCli : public td::actor::Actor {
paused = true; paused = true;
continue; continue;
} }
if (tokens[i] == "--no-upload") {
upload = false;
continue;
}
if (tokens[i] == "--json") { if (tokens[i] == "--json") {
json = true; json = true;
continue; continue;
@ -354,6 +375,10 @@ class StorageDaemonCli : public td::actor::Actor {
partial = std::vector<std::string>(tokens.begin() + i + 1, tokens.end()); partial = std::vector<std::string>(tokens.begin() + i + 1, tokens.end());
break; break;
} }
if (tokens[i] == "--") {
no_more_flags = true;
continue;
}
return td::Status::Error(PSTRING() << "Unknown flag " << tokens[i]); return td::Status::Error(PSTRING() << "Unknown flag " << tokens[i]);
} }
if (param) { if (param) {
@ -366,9 +391,9 @@ class StorageDaemonCli : public td::actor::Actor {
} }
if (tokens[0] == "add-by-hash") { if (tokens[0] == "add-by-hash") {
TRY_RESULT(hash, parse_hash(param.value())); TRY_RESULT(hash, parse_hash(param.value()));
return execute_add_by_hash(hash, std::move(root_dir), paused, std::move(partial), json); return execute_add_by_hash(hash, std::move(root_dir), paused, upload, std::move(partial), json);
} else { } else {
return execute_add_by_meta(param.value(), std::move(root_dir), paused, std::move(partial), json); return execute_add_by_meta(param.value(), std::move(root_dir), paused, upload, std::move(partial), json);
} }
} else if (tokens[0] == "list") { } else if (tokens[0] == "list") {
bool with_hashes = false; bool with_hashes = false;
@ -441,6 +466,12 @@ class StorageDaemonCli : public td::actor::Actor {
} }
TRY_RESULT(hash, parse_torrent(tokens[1])); TRY_RESULT(hash, parse_torrent(tokens[1]));
return execute_set_active_download(hash, tokens[0] == "download-resume"); return execute_set_active_download(hash, tokens[0] == "download-resume");
} else if (tokens[0] == "upload-pause" || tokens[0] == "upload-resume") {
if (tokens.size() != 2) {
return td::Status::Error("Expected bag");
}
TRY_RESULT(hash, parse_torrent(tokens[1]));
return execute_set_active_upload(hash, tokens[0] == "upload-resume");
} else if (tokens[0] == "priority-all") { } else if (tokens[0] == "priority-all") {
if (tokens.size() != 3) { if (tokens.size() != 3) {
return td::Status::Error("Expected bag and priority"); return td::Status::Error("Expected bag and priority");
@ -521,8 +552,9 @@ class StorageDaemonCli : public td::actor::Actor {
td::optional<std::string> provider_address; td::optional<std::string> provider_address;
td::optional<std::string> rate; td::optional<std::string> rate;
td::optional<td::uint32> max_span; td::optional<td::uint32> max_span;
bool no_more_flags = false;
for (size_t i = 1; i < tokens.size(); ++i) { for (size_t i = 1; i < tokens.size(); ++i) {
if (!tokens[i].empty() && tokens[i][0] == '-') { if (!tokens[i].empty() && tokens[i][0] == '-' && !no_more_flags) {
if (tokens[i] == "--query-id") { if (tokens[i] == "--query-id") {
++i; ++i;
TRY_RESULT_PREFIX_ASSIGN(query_id, td::to_integer_safe<td::uint64>(tokens[i]), "Invalid query id: "); TRY_RESULT_PREFIX_ASSIGN(query_id, td::to_integer_safe<td::uint64>(tokens[i]), "Invalid query id: ");
@ -543,6 +575,10 @@ class StorageDaemonCli : public td::actor::Actor {
TRY_RESULT_PREFIX_ASSIGN(max_span, td::to_integer_safe<td::uint8>(tokens[i]), "Invalid max span: "); TRY_RESULT_PREFIX_ASSIGN(max_span, td::to_integer_safe<td::uint8>(tokens[i]), "Invalid max span: ");
continue; continue;
} }
if (tokens[i] == "--") {
no_more_flags = true;
continue;
}
return td::Status::Error(PSTRING() << "Unknown flag " << tokens[i]); return td::Status::Error(PSTRING() << "Unknown flag " << tokens[i]);
} }
if (cnt == 0) { if (cnt == 0) {
@ -731,19 +767,22 @@ class StorageDaemonCli : public td::actor::Actor {
td::Status execute_help() { td::Status execute_help() {
td::TerminalIO::out() << "help\tPrint this help\n"; td::TerminalIO::out() << "help\tPrint this help\n";
td::TerminalIO::out() << "create [-d description] [--json] <file/dir>\tCreate bag of files from <file/dir>\n";
td::TerminalIO::out() << "\t-d - Description will be stored in torrent info.\n";
td::TerminalIO::out() << "\t--json\tOutput in json\n";
td::TerminalIO::out() td::TerminalIO::out()
<< "add-by-hash <bag-id> [-d root_dir] [--paused] [--json] [--partial file1 file2 ...]\tAdd bag " << "create [-d description] [--no-upload] [--copy] [--json] <file/dir>\tCreate bag of files from <file/dir>\n";
"with given BagID (in hex)\n"; td::TerminalIO::out() << "\t-d\tDescription will be stored in torrent info\n";
td::TerminalIO::out() << "\t--no-upload\tDon't share bag with peers\n";
td::TerminalIO::out() << "\t--copy\tFiles will be copied to an internal directory of storage-daemon\n";
td::TerminalIO::out() << "\t--json\tOutput in json\n";
td::TerminalIO::out() << "add-by-hash <bag-id> [-d root_dir] [--paused] [--no-upload] [--json] [--partial file1 "
"file2 ...]\tAdd bag with given BagID (in hex)\n";
td::TerminalIO::out() << "\t-d\tTarget directory, default is an internal directory of storage-daemon\n"; td::TerminalIO::out() << "\t-d\tTarget directory, default is an internal directory of storage-daemon\n";
td::TerminalIO::out() << "\t--paused\tDon't start download immediately\n"; td::TerminalIO::out() << "\t--paused\tDon't start download immediately\n";
td::TerminalIO::out() << "\t--no-upload\tDon't share bag with peers\n";
td::TerminalIO::out() td::TerminalIO::out()
<< "\t--partial\tEverything after this flag is a list of filenames. Only these files will be downloaded.\n"; << "\t--partial\tEverything after this flag is a list of filenames. Only these files will be downloaded.\n";
td::TerminalIO::out() << "\t--json\tOutput in json\n"; td::TerminalIO::out() << "\t--json\tOutput in json\n";
td::TerminalIO::out() << "add-by-meta <meta> [-d root_dir] [--paused] [--json] [--partial file1 file2 ...]\tLoad " td::TerminalIO::out() << "add-by-meta <meta> [-d root_dir] [--paused] [--no-upload] [--json] [--partial file1 "
"meta from file and add bag\n"; "file2 ...]\tLoad meta from file and add bag\n";
td::TerminalIO::out() << "\tFlags are the same as in add-by-hash\n"; td::TerminalIO::out() << "\tFlags are the same as in add-by-hash\n";
td::TerminalIO::out() << "list [--hashes] [--json]\tPrint list of bags\n"; td::TerminalIO::out() << "list [--hashes] [--json]\tPrint list of bags\n";
td::TerminalIO::out() << "\t--hashes\tPrint full BagID\n"; td::TerminalIO::out() << "\t--hashes\tPrint full BagID\n";
@ -756,6 +795,8 @@ class StorageDaemonCli : public td::actor::Actor {
td::TerminalIO::out() << "\t--json\tOutput in json\n"; td::TerminalIO::out() << "\t--json\tOutput in json\n";
td::TerminalIO::out() << "download-pause <bag>\tPause download of <bag>\n"; td::TerminalIO::out() << "download-pause <bag>\tPause download of <bag>\n";
td::TerminalIO::out() << "download-resume <bag>\tResume download of <bag>\n"; td::TerminalIO::out() << "download-resume <bag>\tResume download of <bag>\n";
td::TerminalIO::out() << "upload-pause <bag>\tPause upload of <bag>\n";
td::TerminalIO::out() << "upload-resume <bag>\tResume upload of <bag>\n";
td::TerminalIO::out() << "priority-all <bag> <p>\tSet priority of all files in <bag> to <p>\n"; td::TerminalIO::out() << "priority-all <bag> <p>\tSet priority of all files in <bag> to <p>\n";
td::TerminalIO::out() << "\tPriority is in [0..255], 0 - don't download\n"; td::TerminalIO::out() << "\tPriority is in [0..255], 0 - don't download\n";
td::TerminalIO::out() << "priority-idx <bag> <idx> <p>\tSet priority of file #<idx> in <bag> to <p>\n"; td::TerminalIO::out() << "priority-idx <bag> <idx> <p>\tSet priority of file #<idx> in <bag> to <p>\n";
@ -828,9 +869,9 @@ class StorageDaemonCli : public td::actor::Actor {
return td::Status::OK(); return td::Status::OK();
} }
td::Status execute_create(std::string path, std::string description, bool json) { td::Status execute_create(std::string path, std::string description, bool upload, bool copy, bool json) {
TRY_RESULT_PREFIX_ASSIGN(path, td::realpath(path), "Invalid path: "); TRY_RESULT_PREFIX_ASSIGN(path, td::realpath(path), "Invalid path: ");
auto query = create_tl_object<ton_api::storage_daemon_createTorrent>(path, description); auto query = create_tl_object<ton_api::storage_daemon_createTorrent>(path, description, upload, copy);
send_query(std::move(query), send_query(std::move(query),
[=, SelfId = actor_id(this)](td::Result<tl_object_ptr<ton_api::storage_daemon_torrentFull>> R) { [=, SelfId = actor_id(this)](td::Result<tl_object_ptr<ton_api::storage_daemon_torrentFull>> R) {
if (R.is_error()) { if (R.is_error()) {
@ -838,6 +879,7 @@ class StorageDaemonCli : public td::actor::Actor {
} }
if (json) { if (json) {
print_json(R.ok()); print_json(R.ok());
td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK());
return; return;
} }
td::TerminalIO::out() << "Bag created\n"; td::TerminalIO::out() << "Bag created\n";
@ -847,7 +889,7 @@ class StorageDaemonCli : public td::actor::Actor {
return td::Status::OK(); return td::Status::OK();
} }
td::Status execute_add_by_hash(td::Bits256 hash, std::string root_dir, bool paused, td::Status execute_add_by_hash(td::Bits256 hash, std::string root_dir, bool paused, bool upload,
td::optional<std::vector<std::string>> partial, bool json) { td::optional<std::vector<std::string>> partial, bool json) {
if (!root_dir.empty()) { if (!root_dir.empty()) {
TRY_STATUS_PREFIX(td::mkpath(root_dir), "Failed to create directory: "); TRY_STATUS_PREFIX(td::mkpath(root_dir), "Failed to create directory: ");
@ -861,8 +903,8 @@ class StorageDaemonCli : public td::actor::Actor {
priorities.push_back(create_tl_object<ton_api::storage_priorityAction_name>(std::move(f), 1)); priorities.push_back(create_tl_object<ton_api::storage_priorityAction_name>(std::move(f), 1));
} }
} }
auto query = auto query = create_tl_object<ton_api::storage_daemon_addByHash>(hash, std::move(root_dir), !paused, upload,
create_tl_object<ton_api::storage_daemon_addByHash>(hash, std::move(root_dir), !paused, std::move(priorities)); std::move(priorities));
send_query(std::move(query), send_query(std::move(query),
[=, SelfId = actor_id(this)](td::Result<tl_object_ptr<ton_api::storage_daemon_torrentFull>> R) { [=, SelfId = actor_id(this)](td::Result<tl_object_ptr<ton_api::storage_daemon_torrentFull>> R) {
if (R.is_error()) { if (R.is_error()) {
@ -870,6 +912,7 @@ class StorageDaemonCli : public td::actor::Actor {
} }
if (json) { if (json) {
print_json(R.ok()); print_json(R.ok());
td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK());
return; return;
} }
td::TerminalIO::out() << "Bag added\n"; td::TerminalIO::out() << "Bag added\n";
@ -879,7 +922,7 @@ class StorageDaemonCli : public td::actor::Actor {
return td::Status::OK(); return td::Status::OK();
} }
td::Status execute_add_by_meta(std::string meta_file, std::string root_dir, bool paused, td::Status execute_add_by_meta(std::string meta_file, std::string root_dir, bool paused, bool upload,
td::optional<std::vector<std::string>> partial, bool json) { td::optional<std::vector<std::string>> partial, bool json) {
TRY_RESULT_PREFIX(meta, td::read_file(meta_file), "Failed to read meta: "); TRY_RESULT_PREFIX(meta, td::read_file(meta_file), "Failed to read meta: ");
if (!root_dir.empty()) { if (!root_dir.empty()) {
@ -895,7 +938,7 @@ class StorageDaemonCli : public td::actor::Actor {
} }
} }
auto query = create_tl_object<ton_api::storage_daemon_addByMeta>(std::move(meta), std::move(root_dir), !paused, auto query = create_tl_object<ton_api::storage_daemon_addByMeta>(std::move(meta), std::move(root_dir), !paused,
std::move(priorities)); upload, std::move(priorities));
send_query(std::move(query), send_query(std::move(query),
[=, SelfId = actor_id(this)](td::Result<tl_object_ptr<ton_api::storage_daemon_torrentFull>> R) { [=, SelfId = actor_id(this)](td::Result<tl_object_ptr<ton_api::storage_daemon_torrentFull>> R) {
if (R.is_error()) { if (R.is_error()) {
@ -903,6 +946,7 @@ class StorageDaemonCli : public td::actor::Actor {
} }
if (json) { if (json) {
print_json(R.ok()); print_json(R.ok());
td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK());
return; return;
} }
td::TerminalIO::out() << "Bag added\n"; td::TerminalIO::out() << "Bag added\n";
@ -921,6 +965,7 @@ class StorageDaemonCli : public td::actor::Actor {
} }
if (json) { if (json) {
print_json(R.ok()); print_json(R.ok());
td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK());
return; return;
} }
td::actor::send_closure(SelfId, &StorageDaemonCli::print_torrent_list, R.move_as_ok(), with_hashes); td::actor::send_closure(SelfId, &StorageDaemonCli::print_torrent_list, R.move_as_ok(), with_hashes);
@ -938,6 +983,7 @@ class StorageDaemonCli : public td::actor::Actor {
} }
if (json) { if (json) {
print_json(R.ok()); print_json(R.ok());
td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK());
return; return;
} }
td::actor::send_closure(SelfId, &StorageDaemonCli::print_torrent_full, R.move_as_ok()); td::actor::send_closure(SelfId, &StorageDaemonCli::print_torrent_full, R.move_as_ok());
@ -976,6 +1022,7 @@ class StorageDaemonCli : public td::actor::Actor {
} }
if (json) { if (json) {
print_json(R.ok()); print_json(R.ok());
td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK());
return; return;
} }
auto obj = R.move_as_ok(); auto obj = R.move_as_ok();
@ -1020,6 +1067,19 @@ class StorageDaemonCli : public td::actor::Actor {
return td::Status::OK(); return td::Status::OK();
} }
td::Status execute_set_active_upload(td::Bits256 hash, bool active) {
auto query = create_tl_object<ton_api::storage_daemon_setActiveUpload>(hash, active);
send_query(std::move(query),
[SelfId = actor_id(this)](td::Result<tl_object_ptr<ton_api::storage_daemon_success>> R) {
if (R.is_error()) {
return;
}
td::TerminalIO::out() << "Success\n";
td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK());
});
return td::Status::OK();
}
td::Status execute_set_priority_all(td::Bits256 hash, td::uint8 priority) { td::Status execute_set_priority_all(td::Bits256 hash, td::uint8 priority) {
auto query = create_tl_object<ton_api::storage_daemon_setFilePriorityAll>(hash, priority); auto query = create_tl_object<ton_api::storage_daemon_setFilePriorityAll>(hash, priority);
send_query(std::move(query), send_query(std::move(query),
@ -1229,6 +1289,7 @@ class StorageDaemonCli : public td::actor::Actor {
} }
if (json) { if (json) {
print_json(R.ok()); print_json(R.ok());
td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK());
return; return;
} }
auto params = R.move_as_ok(); auto params = R.move_as_ok();
@ -1294,6 +1355,7 @@ class StorageDaemonCli : public td::actor::Actor {
} }
if (json) { if (json) {
print_json(R.ok()); print_json(R.ok());
td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK());
return; return;
} }
auto info = R.move_as_ok(); auto info = R.move_as_ok();
@ -1601,7 +1663,12 @@ class StorageDaemonCli : public td::actor::Actor {
} else { } else {
td::TerminalIO::out() << "Download paused\n"; td::TerminalIO::out() << "Download paused\n";
} }
td::TerminalIO::out() << "Upload speed: " << td::format::as_size((td::uint64)obj.torrent_->upload_speed_) << "/s\n"; if (obj.torrent_->active_upload_) {
td::TerminalIO::out() << "Upload speed: " << td::format::as_size((td::uint64)obj.torrent_->upload_speed_)
<< "/s\n";
} else {
td::TerminalIO::out() << "Upload paused\n";
}
td::TerminalIO::out() << "Root dir: " << obj.torrent_->root_dir_ << "\n"; td::TerminalIO::out() << "Root dir: " << obj.torrent_->root_dir_ << "\n";
if (obj.torrent_->flags_ & 2) { // header ready if (obj.torrent_->flags_ & 2) { // header ready
td::TerminalIO::out() << obj.files_.size() << " files:\n"; td::TerminalIO::out() << obj.files_.size() << " files:\n";
@ -1616,7 +1683,7 @@ class StorageDaemonCli : public td::actor::Actor {
} }
snprintf(str, sizeof(str), "%6u: (%s) %7s/%-7s %s ", i, priority, snprintf(str, sizeof(str), "%6u: (%s) %7s/%-7s %s ", i, priority,
f->priority_ == 0 ? "---" : size_to_str(f->downloaded_size_).c_str(), size_to_str(f->size_).c_str(), f->priority_ == 0 ? "---" : size_to_str(f->downloaded_size_).c_str(), size_to_str(f->size_).c_str(),
(f->downloaded_size_ == f->size_ ? "+" : " ")); ((f->downloaded_size_ == f->size_ && f->priority_ > 0) ? "+" : " "));
td::TerminalIO::out() << str << f->name_ << "\n"; td::TerminalIO::out() << str << f->name_ << "\n";
++i; ++i;
} }
@ -1639,7 +1706,7 @@ class StorageDaemonCli : public td::actor::Actor {
}); });
td::TerminalIO::out() << obj.torrents_.size() << " bags\n"; td::TerminalIO::out() << obj.torrents_.size() << " bags\n";
std::vector<std::vector<std::string>> table; std::vector<std::vector<std::string>> table;
table.push_back({"#####", "BagID", "Description", "Downloaded", "Total", "Speed"}); table.push_back({"#####", "BagID", "Description", "Downloaded", "Total", "Download", "Upload"});
for (const auto& torrent : obj.torrents_) { for (const auto& torrent : obj.torrents_) {
std::vector<std::string> row; std::vector<std::string> row;
row.push_back(std::to_string(hash_to_id_[torrent->hash_])); row.push_back(std::to_string(hash_to_id_[torrent->hash_]));
@ -1683,18 +1750,21 @@ class StorageDaemonCli : public td::actor::Actor {
row.push_back(downloaded_size.append("/").append(included_size)); row.push_back(downloaded_size.append("/").append(included_size));
row.push_back(total_size); row.push_back(total_size);
row.push_back(status); row.push_back(status);
row.push_back(torrent->active_upload_ ? size_to_str((td::uint64)torrent->upload_speed_) + "/s" : "Paused");
table.push_back(std::move(row)); table.push_back(std::move(row));
} }
print_table(table, {2}); print_table(table, {2});
} }
}; };
} // namespace ton
int main(int argc, char* argv[]) { int main(int argc, char* argv[]) {
SET_VERBOSITY_LEVEL(verbosity_INFO); SET_VERBOSITY_LEVEL(verbosity_INFO);
td::set_default_failure_signal_handler(); td::set_default_failure_signal_handler();
td::IPAddress ip_addr; td::IPAddress ip_addr;
PrivateKey client_private_key; ton::PrivateKey client_private_key;
PublicKey server_public_key; ton::PublicKey server_public_key;
std::vector<std::string> commands; std::vector<std::string> commands;
td::OptionParser p; td::OptionParser p;
p.set_description("command-line interface for storage-daemon"); p.set_description("command-line interface for storage-daemon");
@ -1719,12 +1789,12 @@ int main(int argc, char* argv[]) {
p.add_option('c', "cmd", "execute command", [&](td::Slice arg) { commands.push_back(arg.str()); }); p.add_option('c', "cmd", "execute command", [&](td::Slice arg) { commands.push_back(arg.str()); });
p.add_checked_option('k', "key", "private key", [&](td::Slice arg) { p.add_checked_option('k', "key", "private key", [&](td::Slice arg) {
TRY_RESULT_PREFIX(data, td::read_file(arg.str()), "failed to read: "); TRY_RESULT_PREFIX(data, td::read_file(arg.str()), "failed to read: ");
TRY_RESULT_ASSIGN(client_private_key, PrivateKey::import(data)); TRY_RESULT_ASSIGN(client_private_key, ton::PrivateKey::import(data));
return td::Status::OK(); return td::Status::OK();
}); });
p.add_checked_option('p', "pub", "server public key", [&](td::Slice arg) { p.add_checked_option('p', "pub", "server public key", [&](td::Slice arg) {
TRY_RESULT_PREFIX(data, td::read_file(arg.str()), "failed to read: "); TRY_RESULT_PREFIX(data, td::read_file(arg.str()), "failed to read: ");
TRY_RESULT_ASSIGN(server_public_key, PublicKey::import(data)); TRY_RESULT_ASSIGN(server_public_key, ton::PublicKey::import(data));
return td::Status::OK(); return td::Status::OK();
}); });
@ -1738,7 +1808,7 @@ int main(int argc, char* argv[]) {
td::actor::Scheduler scheduler({0}); td::actor::Scheduler scheduler({0});
scheduler.run_in_context([&] { scheduler.run_in_context([&] {
td::actor::create_actor<StorageDaemonCli>("console", ip_addr, client_private_key, server_public_key, td::actor::create_actor<ton::StorageDaemonCli>("console", ip_addr, client_private_key, server_public_key,
std::move(commands)) std::move(commands))
.release(); .release();
}); });

34
storage/storage-daemon/storage-daemon.cpp
View file

@ -44,7 +44,7 @@
#endif #endif
#include <iostream> #include <iostream>
using namespace ton; namespace ton {
td::BufferSlice create_query_error(td::CSlice message) { td::BufferSlice create_query_error(td::CSlice message) {
return create_serialize_tl_object<ton_api::storage_daemon_queryError>(message.str()); return create_serialize_tl_object<ton_api::storage_daemon_queryError>(message.str());
@ -69,6 +69,7 @@ class StorageDaemon : public td::actor::Actor {
void start_up() override { void start_up() override {
CHECK(db_root_ != ""); CHECK(db_root_ != "");
td::mkdir(db_root_).ensure(); td::mkdir(db_root_).ensure();
db_root_ = td::realpath(db_root_).move_as_ok();
keyring_ = keyring::Keyring::create(db_root_ + "/keyring"); keyring_ = keyring::Keyring::create(db_root_ + "/keyring");
{ {
auto S = load_global_config(); auto S = load_global_config();
@ -285,19 +286,21 @@ class StorageDaemon : public td::actor::Actor {
void run_control_query(ton_api::storage_daemon_createTorrent &query, td::Promise<td::BufferSlice> promise) { void run_control_query(ton_api::storage_daemon_createTorrent &query, td::Promise<td::BufferSlice> promise) {
// Run in a separate thread // Run in a separate thread
delay_action( delay_action(
[promise = std::move(promise), manager = manager_.get(), query = std::move(query)]() mutable { [promise = std::move(promise), manager = manager_.get(), db_root = db_root_,
query = std::move(query)]() mutable {
Torrent::Creator::Options options; Torrent::Creator::Options options;
options.piece_size = 128 * 1024; options.piece_size = 128 * 1024;
options.description = std::move(query.description_); options.description = std::move(query.description_);
TRY_RESULT_PROMISE(promise, torrent, Torrent::Creator::create_from_path(std::move(options), query.path_)); TRY_RESULT_PROMISE(promise, torrent, Torrent::Creator::create_from_path(std::move(options), query.path_));
td::Bits256 hash = torrent.get_hash(); td::Bits256 hash = torrent.get_hash();
td::actor::send_closure(manager, &StorageManager::add_torrent, std::move(torrent), false, td::actor::send_closure(manager, &StorageManager::add_torrent, std::move(torrent), false, query.allow_upload_,
query.copy_inside_,
[manager, hash, promise = std::move(promise)](td::Result<td::Unit> R) mutable { [manager, hash, promise = std::move(promise)](td::Result<td::Unit> R) mutable {
if (R.is_error()) { if (R.is_error()) {
promise.set_error(R.move_as_error()); promise.set_error(R.move_as_error());
} else { return;
get_torrent_info_full_serialized(manager, hash, std::move(promise));
} }
get_torrent_info_full_serialized(manager, hash, std::move(promise));
}); });
}, },
td::Timestamp::now()); td::Timestamp::now());
@ -308,6 +311,7 @@ class StorageDaemon : public td::actor::Actor {
bool start_download_now = query.start_download_ && query.priorities_.empty(); bool start_download_now = query.start_download_ && query.priorities_.empty();
td::actor::send_closure( td::actor::send_closure(
manager_, &StorageManager::add_torrent_by_hash, hash, std::move(query.root_dir_), start_download_now, manager_, &StorageManager::add_torrent_by_hash, hash, std::move(query.root_dir_), start_download_now,
query.allow_upload_,
query_add_torrent_cont(hash, query.start_download_, std::move(query.priorities_), std::move(promise))); query_add_torrent_cont(hash, query.start_download_, std::move(query.priorities_), std::move(promise)));
} }
@ -317,6 +321,7 @@ class StorageDaemon : public td::actor::Actor {
bool start_download_now = query.start_download_ && query.priorities_.empty(); bool start_download_now = query.start_download_ && query.priorities_.empty();
td::actor::send_closure( td::actor::send_closure(
manager_, &StorageManager::add_torrent_by_meta, std::move(meta), std::move(query.root_dir_), start_download_now, manager_, &StorageManager::add_torrent_by_meta, std::move(meta), std::move(query.root_dir_), start_download_now,
query.allow_upload_,
query_add_torrent_cont(hash, query.start_download_, std::move(query.priorities_), std::move(promise))); query_add_torrent_cont(hash, query.start_download_, std::move(query.priorities_), std::move(promise)));
} }
@ -362,6 +367,12 @@ class StorageDaemon : public td::actor::Actor {
promise.wrap([](td::Unit &&) { return create_serialize_tl_object<ton_api::storage_daemon_success>(); })); promise.wrap([](td::Unit &&) { return create_serialize_tl_object<ton_api::storage_daemon_success>(); }));
} }
void run_control_query(ton_api::storage_daemon_setActiveUpload &query, td::Promise<td::BufferSlice> promise) {
td::actor::send_closure(
manager_, &StorageManager::set_active_upload, query.hash_, query.active_,
promise.wrap([](td::Unit &&) { return create_serialize_tl_object<ton_api::storage_daemon_success>(); }));
}
void run_control_query(ton_api::storage_daemon_getTorrents &query, td::Promise<td::BufferSlice> promise) { void run_control_query(ton_api::storage_daemon_getTorrents &query, td::Promise<td::BufferSlice> promise) {
td::actor::send_closure( td::actor::send_closure(
manager_, &StorageManager::get_all_torrents, manager_, &StorageManager::get_all_torrents,
@ -644,9 +655,8 @@ class StorageDaemon : public td::actor::Actor {
promise.set_error(td::Status::Error("No storage provider")); promise.set_error(td::Status::Error("No storage provider"));
return; return;
} }
td::actor::send_closure(provider_, &StorageProvider::get_params, promise.wrap([](ProviderParams params) { td::actor::send_closure(provider_, &StorageProvider::get_params,
return serialize_tl_object(params.tl(), true); promise.wrap([](ProviderParams params) { return serialize_tl_object(params.tl(), true); }));
}));
} }
void run_control_query(ton_api::storage_daemon_setProviderParams &query, td::Promise<td::BufferSlice> promise) { void run_control_query(ton_api::storage_daemon_setProviderParams &query, td::Promise<td::BufferSlice> promise) {
@ -785,6 +795,7 @@ class StorageDaemon : public td::actor::Actor {
auto obj = create_tl_object<ton_api::storage_daemon_torrent>(); auto obj = create_tl_object<ton_api::storage_daemon_torrent>();
fill_torrent_info_short(state.torrent, *obj); fill_torrent_info_short(state.torrent, *obj);
obj->active_download_ = state.active_download; obj->active_download_ = state.active_download;
obj->active_upload_ = state.active_upload;
obj->download_speed_ = state.download_speed; obj->download_speed_ = state.download_speed;
obj->upload_speed_ = state.upload_speed; obj->upload_speed_ = state.upload_speed;
promise.set_result(std::move(obj)); promise.set_result(std::move(obj));
@ -802,6 +813,7 @@ class StorageDaemon : public td::actor::Actor {
auto obj = create_tl_object<ton_api::storage_daemon_torrentFull>(); auto obj = create_tl_object<ton_api::storage_daemon_torrentFull>();
fill_torrent_info_full(state.torrent, *obj); fill_torrent_info_full(state.torrent, *obj);
obj->torrent_->active_download_ = state.active_download; obj->torrent_->active_download_ = state.active_download;
obj->torrent_->active_upload_ = state.active_upload;
obj->torrent_->download_speed_ = state.download_speed; obj->torrent_->download_speed_ = state.download_speed;
obj->torrent_->upload_speed_ = state.upload_speed; obj->torrent_->upload_speed_ = state.upload_speed;
for (size_t i = 0; i < obj->files_.size(); ++i) { for (size_t i = 0; i < obj->files_.size(); ++i) {
@ -856,6 +868,8 @@ class StorageDaemon : public td::actor::Actor {
} }
}; };
} // namespace ton
int main(int argc, char *argv[]) { int main(int argc, char *argv[]) {
SET_VERBOSITY_LEVEL(verbosity_WARNING); SET_VERBOSITY_LEVEL(verbosity_WARNING);
td::set_default_failure_signal_handler().ensure(); td::set_default_failure_signal_handler().ensure();
@ -928,8 +942,8 @@ int main(int argc, char *argv[]) {
scheduler.run_in_context([&] { scheduler.run_in_context([&] {
p.run(argc, argv).ensure(); p.run(argc, argv).ensure();
td::actor::create_actor<StorageDaemon>("storage-daemon", ip_addr, client_mode, global_config, db_root, control_port, td::actor::create_actor<ton::StorageDaemon>("storage-daemon", ip_addr, client_mode, global_config, db_root,
enable_storage_provider) control_port, enable_storage_provider)
.release(); .release();
}); });
while (scheduler.run(1)) { while (scheduler.run(1)) {

11
tl/generate/scheme/ton_api.tl
View file

@ -795,7 +795,7 @@ storage.db.key.piecesInDb hash:int256 = storage.db.key.PiecesInDb;
storage.db.key.pieceInDb hash:int256 idx:long = storage.db.key.PieceInDb; storage.db.key.pieceInDb hash:int256 idx:long = storage.db.key.PieceInDb;
storage.db.torrentList torrents:(vector int256) = storage.db.TorrentList; storage.db.torrentList torrents:(vector int256) = storage.db.TorrentList;
storage.db.torrent root_dir:string active_download:Bool = storage.db.TorrentShort; storage.db.torrent root_dir:string active_download:Bool active_upload:Bool = storage.db.TorrentShort;
storage.db.priorities actions:(vector storage.PriorityAction) = storage.db.Priorities; storage.db.priorities actions:(vector storage.PriorityAction) = storage.db.Priorities;
storage.db.piecesInDb pieces:(vector long) = storage.db.PiecesInDb; storage.db.piecesInDb pieces:(vector long) = storage.db.PiecesInDb;
@ -831,7 +831,7 @@ storage.daemon.torrent
total_size:flags.0?long description:flags.0?string total_size:flags.0?long description:flags.0?string
files_count:flags.1?long included_size:flags.1?long dir_name:flags.1?string files_count:flags.1?long included_size:flags.1?long dir_name:flags.1?string
downloaded_size:long downloaded_size:long
root_dir:string active_download:Bool completed:Bool root_dir:string active_download:Bool active_upload:Bool completed:Bool
download_speed:double upload_speed:double download_speed:double upload_speed:double
fatal_error:flags.2?string fatal_error:flags.2?string
= storage.daemon.Torrent; = storage.daemon.Torrent;
@ -866,10 +866,11 @@ storage.daemon.providerAddress address:string = storage.daemon.ProviderAddress;
---functions--- ---functions---
storage.daemon.setVerbosity verbosity:int = storage.daemon.Success; storage.daemon.setVerbosity verbosity:int = storage.daemon.Success;
storage.daemon.createTorrent path:string description:string = storage.daemon.TorrentFull; storage.daemon.createTorrent path:string description:string allow_upload:Bool copy_inside:Bool = storage.daemon.TorrentFull;
storage.daemon.addByHash hash:int256 root_dir:string start_download:Bool priorities:(vector storage.PriorityAction) = storage.daemon.TorrentFull; storage.daemon.addByHash hash:int256 root_dir:string start_download:Bool allow_upload:Bool priorities:(vector storage.PriorityAction) = storage.daemon.TorrentFull;
storage.daemon.addByMeta meta:bytes root_dir:string start_download:Bool priorities:(vector storage.PriorityAction) = storage.daemon.TorrentFull; storage.daemon.addByMeta meta:bytes root_dir:string start_download:Bool allow_upload:Bool priorities:(vector storage.PriorityAction) = storage.daemon.TorrentFull;
storage.daemon.setActiveDownload hash:int256 active:Bool = storage.daemon.Success; storage.daemon.setActiveDownload hash:int256 active:Bool = storage.daemon.Success;
storage.daemon.setActiveUpload hash:int256 active:Bool = storage.daemon.Success;
storage.daemon.getTorrents = storage.daemon.TorrentList; storage.daemon.getTorrents = storage.daemon.TorrentList;
storage.daemon.getTorrentFull hash:int256 = storage.daemon.TorrentFull; storage.daemon.getTorrentFull hash:int256 = storage.daemon.TorrentFull;

BIN
tl/generate/scheme/ton_api.tlo
View file

Binary file not shown.

1
tl/generate/scheme/tonlib_api.tl
View file

@ -119,6 +119,7 @@ dns.entryDataText text:string = dns.EntryData;
dns.entryDataNextResolver resolver:AccountAddress = dns.EntryData; dns.entryDataNextResolver resolver:AccountAddress = dns.EntryData;
dns.entryDataSmcAddress smc_address:AccountAddress = dns.EntryData; dns.entryDataSmcAddress smc_address:AccountAddress = dns.EntryData;
dns.entryDataAdnlAddress adnl_address:AdnlAddress = dns.EntryData; dns.entryDataAdnlAddress adnl_address:AdnlAddress = dns.EntryData;
dns.entryDataStorageAddress bag_id:int256 = dns.EntryData;
dns.entry name:string category:int256 entry:dns.EntryData = dns.Entry; dns.entry name:string category:int256 entry:dns.EntryData = dns.Entry;

BIN
tl/generate/scheme/tonlib_api.tlo
View file

Binary file not shown.

6
tonlib/tonlib/TonlibClient.cpp
View file

@ -2919,6 +2919,9 @@ td::Result<ton::ManualDns::EntryData> to_dns_entry_data(tonlib_api::dns_EntryDat
TRY_RESULT(address, get_adnl_address(adnl_address.adnl_address_->adnl_address_)); TRY_RESULT(address, get_adnl_address(adnl_address.adnl_address_->adnl_address_));
return ton::ManualDns::EntryData::adnl_address(std::move(address)); return ton::ManualDns::EntryData::adnl_address(std::move(address));
}, },
[&](tonlib_api::dns_entryDataStorageAddress& storage_address) -> R {
return ton::ManualDns::EntryData::storage_address(storage_address.bag_id_);
},
[&](tonlib_api::dns_entryDataText& text) -> R { return ton::ManualDns::EntryData::text(text.text_); })); [&](tonlib_api::dns_entryDataText& text) -> R { return ton::ManualDns::EntryData::text(text.text_); }));
} }
@ -4108,6 +4111,9 @@ td::Result<tonlib_api::object_ptr<tonlib_api::dns_EntryData>> to_tonlib_api(
[&](const ton::ManualDns::EntryDataSmcAddress& smc_address) { [&](const ton::ManualDns::EntryDataSmcAddress& smc_address) {
res = tonlib_api::make_object<tonlib_api::dns_entryDataSmcAddress>( res = tonlib_api::make_object<tonlib_api::dns_entryDataSmcAddress>(
tonlib_api::make_object<tonlib_api::accountAddress>(smc_address.smc_address.rserialize(true))); tonlib_api::make_object<tonlib_api::accountAddress>(smc_address.smc_address.rserialize(true)));
},
[&](const ton::ManualDns::EntryDataStorageAddress& storage_address) {
res = tonlib_api::make_object<tonlib_api::dns_entryDataStorageAddress>(storage_address.bag_id);
})); }));
return res; return res;
} }

2
tonlib/tonlib/tonlib-cli.cpp
View file

@ -317,7 +317,7 @@ class TonlibCli : public td::actor::Actor {
td::TerminalIO::out() << "dns cmdfile <key_id> <file>\n"; td::TerminalIO::out() << "dns cmdfile <key_id> <file>\n";
td::TerminalIO::out() << "\t<dns_cmd> = set <name> <category> <data> | delete.name <name> | delete.all\n"; td::TerminalIO::out() << "\t<dns_cmd> = set <name> <category> <data> | delete.name <name> | delete.all\n";
td::TerminalIO::out() << "\t<data> = DELETED | EMPTY | TEXT:<text> | NEXT:<smc-address> | SMC:<smc-address> | " td::TerminalIO::out() << "\t<data> = DELETED | EMPTY | TEXT:<text> | NEXT:<smc-address> | SMC:<smc-address> | "
"ADNL:<adnl-address>\n"; "ADNL:<adnl-address> | STORAGE:<bag-id>\n";
} }
void pchan_help() { void pchan_help() {

2
validator/impl/collator-impl.h
View file

@ -113,7 +113,7 @@ class Collator final : public td::actor::Actor {
block::ActionPhaseConfig* action_phase_cfg, block::ActionPhaseConfig* action_phase_cfg,
td::RefInt256* masterchain_create_fee, td::RefInt256* masterchain_create_fee,
td::RefInt256* basechain_create_fee, td::RefInt256* basechain_create_fee,
WorkchainId wc); WorkchainId wc, UnixTime now);
static td::Result<std::unique_ptr<block::Transaction>> static td::Result<std::unique_ptr<block::Transaction>>
impl_create_ordinary_transaction(Ref<vm::Cell> msg_root, impl_create_ordinary_transaction(Ref<vm::Cell> msg_root,

5
validator/impl/collator.cpp
View file

@ -1596,7 +1596,7 @@ bool Collator::init_lt() {
bool Collator::fetch_config_params() { bool Collator::fetch_config_params() {
auto res = impl_fetch_config_params(std::move(config_), &old_mparams_, &storage_prices_, &storage_phase_cfg_, auto res = impl_fetch_config_params(std::move(config_), &old_mparams_, &storage_prices_, &storage_phase_cfg_,
&rand_seed_, &compute_phase_cfg_, &action_phase_cfg_, &masterchain_create_fee_, &rand_seed_, &compute_phase_cfg_, &action_phase_cfg_, &masterchain_create_fee_,
&basechain_create_fee_, workchain()); &basechain_create_fee_, workchain(), now_);
if (res.is_error()) { if (res.is_error()) {
return fatal_error(res.move_as_error()); return fatal_error(res.move_as_error());
} }
@ -1609,7 +1609,7 @@ td::Result<std::unique_ptr<block::ConfigInfo>> Collator::impl_fetch_config_param
std::vector<block::StoragePrices>* storage_prices, block::StoragePhaseConfig* storage_phase_cfg, std::vector<block::StoragePrices>* storage_prices, block::StoragePhaseConfig* storage_phase_cfg,
td::BitArray<256>* rand_seed, block::ComputePhaseConfig* compute_phase_cfg, td::BitArray<256>* rand_seed, block::ComputePhaseConfig* compute_phase_cfg,
block::ActionPhaseConfig* action_phase_cfg, td::RefInt256* masterchain_create_fee, block::ActionPhaseConfig* action_phase_cfg, td::RefInt256* masterchain_create_fee,
td::RefInt256* basechain_create_fee, WorkchainId wc) { td::RefInt256* basechain_create_fee, WorkchainId wc, UnixTime now) {
*old_mparams = config->get_config_param(9); *old_mparams = config->get_config_param(9);
{ {
auto res = config->get_storage_prices(); auto res = config->get_storage_prices();
@ -1638,6 +1638,7 @@ td::Result<std::unique_ptr<block::ConfigInfo>> Collator::impl_fetch_config_param
compute_phase_cfg->libraries = std::make_unique<vm::Dictionary>(config->get_libraries_root(), 256); compute_phase_cfg->libraries = std::make_unique<vm::Dictionary>(config->get_libraries_root(), 256);
compute_phase_cfg->max_vm_data_depth = size_limits.max_vm_data_depth; compute_phase_cfg->max_vm_data_depth = size_limits.max_vm_data_depth;
compute_phase_cfg->global_config = config->get_root_cell(); compute_phase_cfg->global_config = config->get_root_cell();
compute_phase_cfg->suspended_addresses = config->get_suspended_addresses(now);
} }
{ {
// compute action_phase_cfg // compute action_phase_cfg

2
validator/impl/external-message.cpp
View file

@ -147,7 +147,7 @@ td::Status ExtMessageQ::run_message_on_account(ton::WorkchainId wc,
&storage_prices_, &storage_phase_cfg_, &storage_prices_, &storage_phase_cfg_,
&rand_seed_, &compute_phase_cfg_, &rand_seed_, &compute_phase_cfg_,
&action_phase_cfg_, &masterchain_create_fee, &action_phase_cfg_, &masterchain_create_fee,
&basechain_create_fee, wc); &basechain_create_fee, wc, utime);
if(fetch_res.is_error()) { if(fetch_res.is_error()) {
auto error = fetch_res.move_as_error(); auto error = fetch_res.move_as_error();
LOG(DEBUG) << "Cannot fetch config params: " << error.message(); LOG(DEBUG) << "Cannot fetch config params: " << error.message();

2
validator/impl/liteserver.cpp
View file

@ -424,7 +424,7 @@ void LiteQuery::perform_getState(BlockIdExt blkid) {
fatal_error("invalid BlockIdExt"); fatal_error("invalid BlockIdExt");
return; return;
} }
if (blkid.is_masterchain() && blkid.id.seqno > 1000) { if (blkid.id.seqno > 1000) {
fatal_error("cannot request total state: possibly too large"); fatal_error("cannot request total state: possibly too large");
return; return;
} }

1
validator/impl/validate-query.cpp
View file

@ -789,6 +789,7 @@ bool ValidateQuery::fetch_config_params() {
compute_phase_cfg_.libraries = std::make_unique<vm::Dictionary>(config_->get_libraries_root(), 256); compute_phase_cfg_.libraries = std::make_unique<vm::Dictionary>(config_->get_libraries_root(), 256);
compute_phase_cfg_.max_vm_data_depth = size_limits.max_vm_data_depth; compute_phase_cfg_.max_vm_data_depth = size_limits.max_vm_data_depth;
compute_phase_cfg_.global_config = config_->get_root_cell(); compute_phase_cfg_.global_config = config_->get_root_cell();
compute_phase_cfg_.suspended_addresses = config_->get_suspended_addresses(now_);
} }
{ {
// compute action_phase_cfg // compute action_phase_cfg