external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-02-15 04:32:21 +00:00
Code Issues Releases Wiki Activity

Merge branch 'testnet' into block-generation

Browse source
This commit is contained in:
SpyCheese 2023-01-11 16:10:57 +03:00
commit 1ccf25d6b7
60 changed files with 706 additions and 236 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.github/script/amd64-18.04.Dockerfile vendored
View file

@ -6,7 +6,8 @@ RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gpe
WORKDIR /
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git
ARG BRANCH
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH
WORKDIR /ton
RUN mkdir /ton/build
@ -14,5 +15,5 @@ WORKDIR /ton/build
ENV CC clang
ENV CXX clang++
ENV CCACHE_DISABLE 1
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release ..
RUN ninja tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client

7
.github/script/amd64-20.04.Dockerfile vendored
View file

@ -6,7 +6,8 @@ RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gpe
WORKDIR /
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git
ARG BRANCH
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH
WORKDIR /ton
RUN mkdir /ton/build
@ -14,5 +15,5 @@ WORKDIR /ton/build
ENV CC clang
ENV CXX clang++
ENV CCACHE_DISABLE 1
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release ..
RUN ninja tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client

7
.github/script/amd64-22.04.Dockerfile vendored
View file

@ -6,7 +6,8 @@ RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gpe
WORKDIR /
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git
ARG BRANCH
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH
WORKDIR /ton
RUN mkdir /ton/build
@ -14,5 +15,5 @@ WORKDIR /ton/build
ENV CC clang
ENV CXX clang++
ENV CCACHE_DISABLE 1
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release ..
RUN ninja tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id create-hardfork dht-server lite-client

7
.github/script/arm64-18.04.Dockerfile vendored
View file

@ -6,7 +6,8 @@ RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gpe
WORKDIR /
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git
ARG BRANCH
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH
WORKDIR /ton
RUN mkdir /ton/build
@ -14,5 +15,5 @@ WORKDIR /ton/build
ENV CC clang
ENV CXX clang++
ENV CCACHE_DISABLE 1
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DTON_ARCH= ..
RUN ninja tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id dht-server lite-client
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= ..
RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id dht-server lite-client

7
.github/script/arm64-20.04.Dockerfile vendored
View file

@ -6,7 +6,8 @@ RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gpe
WORKDIR /
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git
ARG BRANCH
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH
WORKDIR /ton
RUN mkdir /ton/build
@ -14,5 +15,5 @@ WORKDIR /ton/build
ENV CC clang
ENV CXX clang++
ENV CCACHE_DISABLE 1
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DTON_ARCH= ..
RUN ninja tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id dht-server lite-client
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= ..
RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id dht-server lite-client

7
.github/script/arm64-22.04.Dockerfile vendored
View file

@ -6,7 +6,8 @@ RUN apt install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gpe
WORKDIR /
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git
ARG BRANCH
RUN git clone --recurse-submodules https://github.com/ton-blockchain/ton.git && cd ton && git checkout $BRANCH
WORKDIR /ton
RUN mkdir /ton/build
@ -14,5 +15,5 @@ WORKDIR /ton/build
ENV CC clang
ENV CXX clang++
ENV CCACHE_DISABLE 1
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DTON_ARCH= ..
RUN ninja tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id dht-server lite-client
RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= ..
RUN ninja storage-daemon storage-daemon-cli tonlibjson blockchain-explorer fift func validator-engine validator-engine-console create-state generate-random-id dht-server lite-client

12
.github/workflows/docker-compile-ubuntu.yml vendored
View file

@ -22,14 +22,24 @@ jobs:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Set output
id: vars
run: echo ::set-output name=short_ref::${GITHUB_REF#refs/*/}
- name: Check output
run: echo branch ${{ steps.vars.outputs.short_ref }}
- name: Build with docker buildx
run: |
mkdir build-${{matrix.ver}}-${{matrix.arch}}
docker buildx build --platform=linux/${{matrix.arch}} --progress=plain --load . -t build-${{matrix.ver}}-${{matrix.arch}} -f .github/script/${{matrix.arch}}-${{matrix.ver}}.Dockerfile
docker buildx build --build-arg BRANCH=${{ steps.vars.outputs.short_ref }} --platform=linux/${{matrix.arch}} --progress=plain --load . -t build-${{matrix.ver}}-${{matrix.arch}} -f .github/script/${{matrix.arch}}-${{matrix.ver}}.Dockerfile
container_id=$(docker create --platform=linux/${{matrix.arch}} build-${{matrix.ver}}-${{matrix.arch}})
docker cp $container_id:/ton/build/dht-server/dht-server build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/validator-engine/validator-engine build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/validator-engine-console/validator-engine-console build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/storage/storage-daemon/storage-daemon build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/storage/storage-daemon/storage-daemon-cli build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/crypto/fift build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/crypto/func build-${{matrix.ver}}-${{matrix.arch}}/
docker cp -a $container_id:/ton/build/crypto/create-state build-${{matrix.ver}}-${{matrix.arch}}/

2
.github/workflows/docker-ubuntu-image.yml vendored
View file

@ -3,6 +3,8 @@ name: Docker Ubuntu 20.04 image
on:
workflow_dispatch:
push:
branches:
- 'master'
env:
REGISTRY: ghcr.io

4
.github/workflows/macos-11.7-compile.yml vendored
View file

@ -28,11 +28,13 @@ jobs:
mkdir build
cd build
cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=11.7 -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_BUILD_TYPE=Release ..
ninja fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc
ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc
- name: Find & copy binaries
run: |
mkdir artifacts
cp build/storage/storage-daemon/storage-daemon artifacts/
cp build/storage/storage-daemon/storage-daemon-cli artifacts/
cp build/crypto/fift artifacts/
cp build/crypto/func artifacts/
cp build/crypto/create-state artifacts/

4
.github/workflows/macos-12.6-compile.yml vendored
View file

@ -28,11 +28,13 @@ jobs:
mkdir build
cd build
cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=12.6 -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_BUILD_TYPE=Release ..
ninja fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc
ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork tlbc
- name: Find & copy binaries
run: |
mkdir artifacts
cp build/storage/storage-daemon/storage-daemon artifacts/
cp build/storage/storage-daemon/storage-daemon-cli artifacts/
cp build/crypto/fift artifacts/
cp build/crypto/func artifacts/
cp build/crypto/create-state artifacts/

12
.github/workflows/ubuntu-18.04-compile.yml vendored
View file

@ -18,6 +18,10 @@ jobs:
sudo apt update
sudo apt install -y build-essential git make cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev ninja-build
- name: Show CPU flags
run: |
cat /proc/cpuinfo
- name: Configure & Build
run: |
export CC=$(which clang)
@ -25,13 +29,13 @@ jobs:
export CCACHE_DISABLE=1
mkdir build
cd build
cmake -GNinja -DCMAKE_BUILD_TYPE=Release ..
ninja fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork
cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state
- name: find & copy binaries
- name: Find & copy binaries
run: |
mkdir artifacts
cp build/crypto/fift build/crypto/tlbc build/crypto/func build/crypto/create-state build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli build/tonlib/libtonlibjson.so.0.5 build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy artifacts
cp build/storage/storage-daemon/storage-daemon build/storage/storage-daemon/storage-daemon-cli build/crypto/fift build/crypto/tlbc build/crypto/func build/crypto/create-state build/validator-engine-console/validator-engine-console build/tonlib/tonlib-cli build/tonlib/libtonlibjson.so.0.5 build/http/http-proxy build/rldp-http-proxy/rldp-http-proxy build/dht-server/dht-server build/lite-client/lite-client build/validator-engine/validator-engine build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy artifacts
cp -R crypto/smartcont artifacts/
cp -R crypto/fift/lib artifacts/

12
.github/workflows/ubuntu-compile.yml vendored
View file

@ -20,6 +20,11 @@ jobs:
run: |
sudo apt update
sudo apt install -y build-essential git make cmake clang libgflags-dev zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev ninja-build
- name: Show CPU flags
run: |
cat /proc/cpuinfo
- name: Configure & Build
run: |
export CC=$(which clang)
@ -27,12 +32,13 @@ jobs:
export CCACHE_DISABLE=1
mkdir build-${{ matrix.os }}
cd build-${{ matrix.os }}
cmake -GNinja -DCMAKE_BUILD_TYPE=Release ..
ninja fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork
cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork
- name: Find & copy binaries
run: |
mkdir artifacts-${{ matrix.os }}
cp build-${{ matrix.os }}/crypto/fift build-${{ matrix.os }}/crypto/tlbc build-${{ matrix.os }}/crypto/func build-${{ matrix.os }}/crypto/create-state build-${{ matrix.os }}/validator-engine-console/validator-engine-console build-${{ matrix.os }}/tonlib/tonlib-cli build-${{ matrix.os }}/tonlib/libtonlibjson.so.0.5 build-${{ matrix.os }}/http/http-proxy build-${{ matrix.os }}/rldp-http-proxy/rldp-http-proxy build-${{ matrix.os }}/dht-server/dht-server build-${{ matrix.os }}/lite-client/lite-client build-${{ matrix.os }}/validator-engine/validator-engine build-${{ matrix.os }}/utils/generate-random-id build-${{ matrix.os }}/utils/json2tlo build-${{ matrix.os }}/adnl/adnl-proxy artifacts-${{ matrix.os }}
cp build-${{ matrix.os }}/storage/storage-daemon/storage-daemon build-${{ matrix.os }}/storage/storage-daemon/storage-daemon-cli build-${{ matrix.os }}/crypto/fift build-${{ matrix.os }}/crypto/tlbc build-${{ matrix.os }}/crypto/func build-${{ matrix.os }}/crypto/create-state build-${{ matrix.os }}/validator-engine-console/validator-engine-console build-${{ matrix.os }}/tonlib/tonlib-cli build-${{ matrix.os }}/tonlib/libtonlibjson.so.0.5 build-${{ matrix.os }}/http/http-proxy build-${{ matrix.os }}/rldp-http-proxy/rldp-http-proxy build-${{ matrix.os }}/dht-server/dht-server build-${{ matrix.os }}/lite-client/lite-client build-${{ matrix.os }}/validator-engine/validator-engine build-${{ matrix.os }}/utils/generate-random-id build-${{ matrix.os }}/utils/json2tlo build-${{ matrix.os }}/adnl/adnl-proxy artifacts-${{ matrix.os }}
cp -R crypto/smartcont artifacts-${{ matrix.os }}
cp -R crypto/fift/lib artifacts-${{ matrix.os }}
- name: Upload artifacts

4
.github/workflows/win-2019-compile.yml vendored
View file

@ -57,7 +57,7 @@ jobs:
mkdir build
cd build
cmake -DREADLINE_INCLUDE_DIR=%root%\readline-5.0-1-lib\include\readline -DREADLINE_LIBRARY=%root%\readline-5.0-1-lib\lib\readline.lib -DZLIB_FOUND=1 -DMHD_FOUND=1 -DMHD_LIBRARY=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static\libmicrohttpd.lib -DMHD_INCLUDE_DIR=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static -DZLIB_INCLUDE_DIR=%root%\zlib -DZLIB_LIBRARY=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=%root%/openssl-1.1/x64/include -DOPENSSL_CRYPTO_LIBRARY=%root%/openssl-1.1/x64/lib/libcrypto.lib -DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj /W0" ..
cmake --build . --target fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork --config Release
cmake --build . --target storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork --config Release
- name: Show executables
run: |
@ -77,7 +77,7 @@ jobs:
mkdir artifacts\smartcont
mkdir artifacts\lib
for %%I in (build\crypto\Release\fift.exe build\crypto\Release\tlbc.exe build\crypto\Release\func.exe build\crypto\Release\create-state.exe build\validator-engine-console\Release\validator-engine-console.exe build\tonlib\Release\tonlib-cli.exe build\tonlib\Release\tonlibjson.dll build\http\Release\http-proxy.exe build\rldp-http-proxy\Release\rldp-http-proxy.exe build\dht-server\Release\dht-server.exe build\lite-client\Release\lite-client.exe build\validator-engine\Release\validator-engine.exe build\utils\Release\generate-random-id.exe build\utils\Release\json2tlo.exe build\adnl\Release\adnl-proxy.exe) do copy %%I artifacts\
for %%I in (build\storage\storage-daemon\Release\storage-daemon.exe build\storage\storage-daemon\Release\storage-daemon-cli.exe build\crypto\Release\fift.exe build\crypto\Release\tlbc.exe build\crypto\Release\func.exe build\crypto\Release\create-state.exe build\validator-engine-console\Release\validator-engine-console.exe build\tonlib\Release\tonlib-cli.exe build\tonlib\Release\tonlibjson.dll build\http\Release\http-proxy.exe build\rldp-http-proxy\Release\rldp-http-proxy.exe build\dht-server\Release\dht-server.exe build\lite-client\Release\lite-client.exe build\validator-engine\Release\validator-engine.exe build\utils\Release\generate-random-id.exe build\utils\Release\json2tlo.exe build\adnl\Release\adnl-proxy.exe) do copy %%I artifacts\
xcopy /e /k /h /i crypto\smartcont artifacts\smartcont
xcopy /e /k /h /i crypto\fift\lib artifacts\lib

15
Changelog.md
View file

@ -38,3 +38,18 @@ Besides the work of the core team, this update is based on the efforts of @aweso
* Added build of FunC and Fift to WASM
Besides the work of the core team, this update is based on the efforts of @tvorogme (debug improvements), @AlexeyFSL (WASM builds) and third-party security auditors.
## 12.2022 Update
Node update:
1. Improvements of ton-proxy: fixed few bugs, improved stability
2. Improved collator/validator checks, added optimization of storage stat calculation, generation and validation of new blocks is made safer
3. Some previously hard-coded parameters such as split/merge timings, max sizes and depths of internal and external messages, and others now can be updated by validators through setting ConfigParams. Max contract size added to configs.
4. Tonlib: updated raw.getTransactions (now it contains InitState), fixed long bytestrings truncation
5. abseil-cpp is updated to newer versions
6. Added configs for Token Bridge
7. LiteServers: a few bug fixes, added liteServer.getAccountStatePrunned method, improved work with not yet applied blocks.
8. Improved DHT: works for some NAT configurations, optimized excessive requests, added option for DHT network segregation.
9. FunC v0.4.0: added try/catch statements, added throw_arg functions, allowed in-place modification of global variables, forbidden ambiguous modification of local variables after it's usage in the same expression.
10. TON Storage: added storage-daemon (create, download bag of Files, storage-provider staff), added storage-daemon-cli
Besides the work of the core team, this update is based on the efforts of @vtamara (help with abseil-cpp upgrade), @krigga(in-place modification of global variables) and third-party security auditors.

12
crypto/block/block-parse.h
View file

@ -653,16 +653,20 @@ struct TrComputeInternal1 final : TLB_Complex {
};
struct ComputeSkipReason final : TLB {
enum { cskip_no_state = 0, cskip_bad_state = 1, cskip_no_gas = 2 };
enum { cskip_no_state = 0, cskip_bad_state = 1, cskip_no_gas = 2, cskip_suspended = 3 };
int get_size(const vm::CellSlice& cs) const override {
return 2;
return cs.prefetch_ulong(2) == 3 ? 3 : 2;
}
bool validate_skip(int* ops, vm::CellSlice& cs, bool weak = false) const override {
return get_tag(cs) >= 0 && cs.advance(2);
int tag = get_tag(cs);
return tag >= 0 && cs.advance(tag == 3 ? 3 : 2);
}
int get_tag(const vm::CellSlice& cs) const override {
int t = (int)cs.prefetch_ulong(2);
return t < 3 ? t : -1;
if (t == 3 && cs.prefetch_ulong(3) != 0b110) {
return -1;
}
return t;
}
};

7
crypto/block/block.tlb
View file

@ -309,6 +309,7 @@ tr_phase_compute_vm$1 success:Bool msg_state_used:Bool
cskip_no_state$00 = ComputeSkipReason;
cskip_bad_state$01 = ComputeSkipReason;
cskip_no_gas$10 = ComputeSkipReason;
cskip_suspended$110 = ComputeSkipReason;
tr_phase_action$_ success:Bool valid:Bool no_funds:Bool
status_change:AccStatusChange
@ -762,6 +763,10 @@ size_limits_config_v2#02 max_msg_bits:uint32 max_msg_cells:uint32 max_library_ce
max_ext_msg_size:uint32 max_ext_msg_depth:uint16 max_acc_state_cells:uint32 max_acc_state_bits:uint32 = SizeLimitsConfig;
_ SizeLimitsConfig = ConfigParam 43;
// key is [ wc:int32 addr:uint256 ]
suspended_address_list#00 addresses:(HashmapE 288 Unit) suspended_until:uint32 = SuspendedAddressList;
_ SuspendedAddressList = ConfigParam 44;
oracle_bridge_params#_ bridge_address:bits256 oracle_mutlisig_address:bits256 oracles:(HashmapE 256 uint256) external_chain_address:bits256 = OracleBridgeParams;
_ OracleBridgeParams = ConfigParam 71; // Ethereum bridge
_ OracleBridgeParams = ConfigParam 72; // Binance Smart Chain bridge
@ -883,6 +888,8 @@ cap_method_pubkey#71f4 = SmcCapability;
cap_is_wallet#2177 = SmcCapability;
cap_name#ff name:Text = SmcCapability;
dns_storage_address#7473 bag_id:bits256 = DNSRecord;
//
// PAYMENT CHANNELS
//

9
crypto/block/mc-config.cpp
View file

@ -1945,6 +1945,15 @@ td::Result<SizeLimitsConfig> Config::get_size_limits_config() const {
return limits;
}
std::unique_ptr<vm::Dictionary> Config::get_suspended_addresses(ton::UnixTime now) const {
td::Ref<vm::Cell> param = get_config_param(44);
gen::SuspendedAddressList::Record rec;
if (param.is_null() || !tlb::unpack_cell(param, rec) || rec.suspended_until <= now) {
return {};
}
return std::make_unique<vm::Dictionary>(rec.addresses->prefetch_ref(), 288);
}
td::Result<std::pair<ton::UnixTime, ton::UnixTime>> Config::unpack_validator_set_start_stop(Ref<vm::Cell> vset_root) {
if (vset_root.is_null()) {
return td::Status::Error("validator set absent");

1
crypto/block/mc-config.h
View file

@ -628,6 +628,7 @@ class Config {
std::vector<ton::ValidatorDescr> compute_total_validator_set(int next) const;
CollatorConfig get_collator_config(bool need_collator_nodes) const;
td::Result<SizeLimitsConfig> get_size_limits_config() const;
std::unique_ptr<vm::Dictionary> get_suspended_addresses(ton::UnixTime now) const;
static std::vector<ton::ValidatorDescr> do_compute_validator_set(const block::CatchainValidatorsConfig& ccv_conf,
ton::ShardIdFull shard,
const block::ValidatorSet& vset, ton::UnixTime time,

21
crypto/block/transaction.cpp
View file

@ -794,6 +794,20 @@ bool ComputePhaseConfig::parse_GasLimitsPrices_internal(Ref<vm::CellSlice> cs, t
return true;
}
bool ComputePhaseConfig::is_address_suspended(ton::WorkchainId wc, td::Bits256 addr) const {
if (!suspended_addresses) {
return false;
}
try {
vm::CellBuilder key;
key.store_long_bool(wc, 32);
key.store_bits_bool(addr);
return !suspended_addresses->lookup(key.data_bits(), 288).is_null();
} catch (vm::VmError) {
return false;
}
}
void ComputePhaseConfig::compute_threshold() {
gas_price256 = td::make_refint(gas_price);
if (gas_limit > flat_gas_limit) {
@ -1006,6 +1020,11 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) {
if (in_msg_state.not_null() &&
(acc_status == Account::acc_uninit ||
(acc_status == Account::acc_frozen && account.state_hash == in_msg_state->get_hash().bits()))) {
if (acc_status == Account::acc_uninit && cfg.is_address_suspended(account.workchain, account.addr)) {
LOG(DEBUG) << "address is suspended, skipping compute phase";
cp.skip_reason = ComputePhase::sk_suspended;
return true;
}
use_msg_state = true;
if (!(unpack_msg_state() && account.check_split_depth(new_split_depth))) {
LOG(DEBUG) << "cannot unpack in_msg_state, or it has bad split_depth; cannot init account state";
@ -2263,6 +2282,8 @@ bool Transaction::serialize_compute_phase(vm::CellBuilder& cb) {
return cb.store_long_bool(1, 3); // cskip_bad_state$01 = ComputeSkipReason;
case ComputePhase::sk_no_gas:
return cb.store_long_bool(2, 3); // cskip_no_gas$10 = ComputeSkipReason;
case ComputePhase::sk_suspended:
return cb.store_long_bool(0b0110, 4); // cskip_suspended$110 = ComputeSkipReason;
case ComputePhase::sk_none:
break;
default:

4
crypto/block/transaction.h
View file

@ -108,6 +108,7 @@ struct ComputePhaseConfig {
td::BitArray<256> block_rand_seed;
bool with_vm_log{false};
td::uint16 max_vm_data_depth = 512;
std::unique_ptr<vm::Dictionary> suspended_addresses;
ComputePhaseConfig(td::uint64 _gas_price = 0, td::uint64 _gas_limit = 0, td::uint64 _gas_credit = 0)
: gas_price(_gas_price), gas_limit(_gas_limit), special_gas_limit(_gas_limit), gas_credit(_gas_credit) {
compute_threshold();
@ -132,6 +133,7 @@ struct ComputePhaseConfig {
}
bool parse_GasLimitsPrices(Ref<vm::CellSlice> cs, td::RefInt256& freeze_due_limit, td::RefInt256& delete_due_limit);
bool parse_GasLimitsPrices(Ref<vm::Cell> cell, td::RefInt256& freeze_due_limit, td::RefInt256& delete_due_limit);
bool is_address_suspended(ton::WorkchainId wc, td::Bits256 addr) const;
private:
bool parse_GasLimitsPrices_internal(Ref<vm::CellSlice> cs, td::RefInt256& freeze_due_limit,
@ -157,7 +159,7 @@ struct CreditPhase {
};
struct ComputePhase {
enum { sk_none, sk_no_state, sk_bad_state, sk_no_gas };
enum { sk_none, sk_no_state, sk_bad_state, sk_no_gas, sk_suspended };
int skip_reason{sk_none};
bool success{false};
bool msg_state_used{false};

3
crypto/func/analyzer.cpp
View file

@ -810,6 +810,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) {
break;
}
case _While: {
auto values0 = values;
values = block0->fwd_analyze(values);
if (values[left[0]] && values[left[0]]->always_false()) {
// block1 never executed
@ -817,7 +818,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) {
break;
}
while (true) {
VarDescrList next_values = values | block0->fwd_analyze(block1->fwd_analyze(values));
VarDescrList next_values = values | block0->fwd_analyze(values0 | block1->fwd_analyze(values));
if (same_values(next_values, values)) {
break;
}

11
crypto/func/auto-tests/stress_tester.py
View file

@ -137,13 +137,20 @@ class CodeRepeat(Code):
self.c.write(f, indent + 1)
print(" " * (indent + 1) + "%s += 1;" % var, file=f)
print(" " * indent + "}", file=f)
else:
elif self.loop_type == 2:
var = gen_var_name()
print(" " * indent + "int %s = 0;" % var, file=f)
print(" " * indent + "do {", file=f)
self.c.write(f, indent + 1)
print(" " * (indent + 1) + "%s += 1;" % var, file=f)
print(" " * indent + "} until (%s >= %d);" % (var, self.n), file=f)
else:
var = gen_var_name()
print(" " * indent + "int %s = %d;" % (var, self.n - 1), file=f)
print(" " * indent + "while (%s >= 0) {" % var, file=f)
self.c.write(f, indent + 1)
print(" " * (indent + 1) + "%s -= 1;" % var, file=f)
print(" " * indent + "}", file=f)
class CodeThrow(Code):
def __init__(self):
@ -199,7 +206,7 @@ def gen_code(xl, xr, with_return, loop_depth=0, try_catch_depth=0, can_throw=Fal
for _ in range(random.randint(0, 2)):
if random.randint(0, 3) == 0 and loop_depth < 3:
c = gen_code(xl, xr, False, loop_depth + 1, try_catch_depth, can_throw)
code.append(CodeRepeat(random.randint(0, 3), c, random.randint(0, 2)))
code.append(CodeRepeat(random.randint(0, 3), c, random.randint(0, 3)))
elif xr - xl > 1:
xmid = random.randrange(xl + 1, xr)
ret = random.choice((0, 0, 0, 0, 0, 1, 2))

26
crypto/func/func.h
View file

@ -39,7 +39,7 @@ extern std::string generated_from;
constexpr int optimize_depth = 20;
const std::string func_version{"0.3.0"};
const std::string func_version{"0.4.0"};
enum Keyword {
_Eof = -1,
@ -306,10 +306,16 @@ struct TmpVar {
sym_idx_t name;
int coord;
std::unique_ptr<SrcLocation> where;
size_t modify_forbidden = 0;
TmpVar(var_idx_t _idx, int _cls, TypeExpr* _type = 0, SymDef* sym = 0, const SrcLocation* loc = 0);
void show(std::ostream& os, int omit_idx = 0) const;
void dump(std::ostream& os) const;
void set_location(const SrcLocation& loc);
std::string to_string() const {
std::ostringstream s;
show(s, 2);
return s.str();
}
};
struct VarDescr {
@ -722,6 +728,22 @@ struct CodeBlob {
void mark_noreturn();
void generate_code(AsmOpList& out_list, int mode = 0);
void generate_code(std::ostream& os, int mode = 0, int indent = 0);
void mark_modify_forbidden(var_idx_t idx) {
++vars.at(idx).modify_forbidden;
}
void unmark_modify_forbidden(var_idx_t idx) {
assert(vars.at(idx).modify_forbidden > 0);
--vars.at(idx).modify_forbidden;
}
void check_modify_forbidden(var_idx_t idx, const SrcLocation& here) const {
if (vars.at(idx).modify_forbidden) {
throw src::ParseError{here, PSTRING() << "Modifying local variable " << vars[idx].to_string()
<< " after using it in the same expression"};
}
}
};
/*
@ -925,7 +947,7 @@ struct Expr {
}
int define_new_vars(CodeBlob& code);
int predefine_vars();
std::vector<var_idx_t> pre_compile(CodeBlob& code, bool lval = false) const;
std::vector<var_idx_t> pre_compile(CodeBlob& code, std::vector<std::pair<SymDef*, var_idx_t>>* lval_globs = nullptr) const;
static std::vector<var_idx_t> pre_compile_let(CodeBlob& code, Expr* lhs, Expr* rhs, const SrcLocation& here);
var_idx_t new_tmp(CodeBlob& code) const;
std::vector<var_idx_t> new_tmp_vect(CodeBlob& code) const {

58
crypto/func/gen-abscode.cpp
View file

@ -221,6 +221,13 @@ var_idx_t Expr::new_tmp(CodeBlob& code) const {
return code.create_tmp_var(e_type, &here);
}
void add_set_globs(CodeBlob& code, std::vector<std::pair<SymDef*, var_idx_t>>& globs, const SrcLocation& here) {
for (const auto& p : globs) {
auto& op = code.emplace_back(here, Op::_SetGlob, std::vector<var_idx_t>{}, std::vector<var_idx_t>{ p.second }, p.first);
op.flags |= Op::_Impure;
}
}
std::vector<var_idx_t> Expr::pre_compile_let(CodeBlob& code, Expr* lhs, Expr* rhs, const SrcLocation& here) {
while (lhs->is_type_apply()) {
lhs = lhs->args.at(0);
@ -245,19 +252,18 @@ std::vector<var_idx_t> Expr::pre_compile_let(CodeBlob& code, Expr* lhs, Expr* rh
return tmp;
}
auto right = rhs->pre_compile(code);
if (lhs->cls == Expr::_GlobVar) {
assert(lhs->sym);
auto& op = code.emplace_back(here, Op::_SetGlob, std::vector<var_idx_t>{}, right, lhs->sym);
op.flags |= Op::_Impure;
} else {
auto left = lhs->pre_compile(code, true);
code.emplace_back(here, Op::_Let, std::move(left), right);
std::vector<std::pair<SymDef*, var_idx_t>> globs;
auto left = lhs->pre_compile(code, &globs);
for (var_idx_t v : left) {
code.check_modify_forbidden(v, here);
}
code.emplace_back(here, Op::_Let, std::move(left), right);
add_set_globs(code, globs, here);
return right;
}
std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, bool lval) const {
if (lval && !(cls == _Tensor || cls == _Var || cls == _Hole || cls == _TypeApply)) {
std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, std::vector<std::pair<SymDef*, var_idx_t>>* lval_globs) const {
if (lval_globs && !(cls == _Tensor || cls == _Var || cls == _Hole || cls == _TypeApply || cls == _GlobVar)) {
std::cerr << "lvalue expression constructor is " << cls << std::endl;
throw src::Fatal{"cannot compile lvalue expression with unknown constructor"};
}
@ -265,9 +271,15 @@ std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, bool lval) const {
case _Tensor: {
std::vector<var_idx_t> res;
for (const auto& x : args) {
auto add = x->pre_compile(code, lval);
auto add = x->pre_compile(code, lval_globs);
for (var_idx_t v : add) {
code.mark_modify_forbidden(v);
}
res.insert(res.end(), add.cbegin(), add.cend());
}
for (var_idx_t v : res) {
code.unmark_modify_forbidden(v);
}
return res;
}
case _Apply: {
@ -279,6 +291,9 @@ std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, bool lval) const {
std::vector<std::vector<var_idx_t>> add_list(args.size());
for (int i : func->arg_order) {
add_list[i] = args[i]->pre_compile(code);
for (var_idx_t v : add_list[i]) {
code.mark_modify_forbidden(v);
}
}
for (const auto& add : add_list) {
res.insert(res.end(), add.cbegin(), add.cend());
@ -286,9 +301,15 @@ std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, bool lval) const {
} else {
for (const auto& x : args) {
auto add = x->pre_compile(code);
for (var_idx_t v : add) {
code.mark_modify_forbidden(v);
}
res.insert(res.end(), add.cbegin(), add.cend());
}
}
for (var_idx_t v : res) {
code.unmark_modify_forbidden(v);
}
auto rvect = new_tmp_vect(code);
auto& op = code.emplace_back(here, Op::_Call, rvect, std::move(res), sym);
if (flags & _IsImpure) {
@ -297,7 +318,7 @@ std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, bool lval) const {
return rvect;
}
case _TypeApply:
return args[0]->pre_compile(code, lval);
return args[0]->pre_compile(code, lval_globs);
case _Var:
case _Hole:
return {val};
@ -329,18 +350,31 @@ std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, bool lval) const {
case _Glob:
case _GlobVar: {
auto rvect = new_tmp_vect(code);
if (lval_globs) {
lval_globs->push_back({ sym, rvect[0] });
return rvect;
} else {
code.emplace_back(here, Op::_GlobVar, rvect, std::vector<var_idx_t>{}, sym);
return rvect;
}
}
case _Letop: {
return pre_compile_let(code, args.at(0), args.at(1), here);
}
case _LetFirst: {
auto rvect = new_tmp_vect(code);
auto right = args[1]->pre_compile(code);
auto left = args[0]->pre_compile(code, true);
std::vector<std::pair<SymDef*, var_idx_t>> local_globs;
if (!lval_globs) {
lval_globs = &local_globs;
}
auto left = args[0]->pre_compile(code, lval_globs);
left.push_back(rvect[0]);
for (var_idx_t v : left) {
code.check_modify_forbidden(v, here);
}
code.emplace_back(here, Op::_Let, std::move(left), std::move(right));
add_set_globs(code, local_globs, here);
return rvect;
}
case _MkTuple: {

24
crypto/smc-envelope/ManualDns.cpp
View file

@ -47,6 +47,8 @@ td::StringBuilder& operator<<(td::StringBuilder& sb, const ManualDns::EntryData&
.move_as_ok();
case ManualDns::EntryData::Type::SmcAddress:
return sb << "SMC:" << data.data.get<ManualDns::EntryDataSmcAddress>().smc_address.rserialize();
case ManualDns::EntryData::Type::StorageAddress:
return sb << "STORAGE:" << data.data.get<ManualDns::EntryDataStorageAddress>().bag_id.to_hex();
}
return sb << "<unknown>";
}
@ -93,6 +95,11 @@ td::Result<td::Ref<vm::Cell>> DnsInterface::EntryData::as_cell() const {
smc_address.smc_address.addr);
dns.smc_addr = vm::load_cell_slice_ref(cb.finalize());
tlb::pack_cell(res, dns);
},
[&](const EntryDataStorageAddress& storage_address) {
block::gen::DNSRecord::Record_dns_storage_address dns;
dns.bag_id = storage_address.bag_id;
tlb::pack_cell(res, dns);
}));
if (error.is_error()) {
return error;
@ -142,6 +149,11 @@ td::Result<DnsInterface::EntryData> DnsInterface::EntryData::from_cellslice(vm::
}
return EntryData::smc_address(block::StdAddress(wc, addr));
}
case block::gen::DNSRecord::dns_storage_address: {
block::gen::DNSRecord::Record_dns_storage_address dns;
tlb::unpack(cs, dns);
return EntryData::storage_address(dns.bag_id);
}
}
return td::Status::Error("Unknown entry data");
}
@ -536,10 +548,12 @@ std::string DnsInterface::decode_name(td::Slice name) {
std::string ManualDns::serialize_data(const EntryData& data) {
std::string res;
data.data.visit(td::overloaded([&](const ton::ManualDns::EntryDataText& text) { res = "UNSUPPORTED"; },
data.data.visit(
td::overloaded([&](const ton::ManualDns::EntryDataText& text) { res = "UNSUPPORTED"; },
[&](const ton::ManualDns::EntryDataNextResolver& resolver) { res = "UNSUPPORTED"; },
[&](const ton::ManualDns::EntryDataAdnlAddress& adnl_address) { res = "UNSUPPORTED"; },
[&](const ton::ManualDns::EntryDataSmcAddress& text) { res = "UNSUPPORTED"; }));
[&](const ton::ManualDns::EntryDataSmcAddress& text) { res = "UNSUPPORTED"; },
[&](const ton::ManualDns::EntryDataStorageAddress& storage_address) { res = "UNSUPPORTED"; }));
return res;
}
@ -559,6 +573,12 @@ td::Result<td::optional<ManualDns::EntryData>> ManualDns::parse_data(td::Slice c
} else if (type == "NEXT") {
TRY_RESULT(address, block::StdAddress::parse(parser.read_all()));
return ManualDns::EntryData::next_resolver(address);
} else if (type == "STORAGE") {
td::Bits256 bag_id;
if (bag_id.from_hex(parser.read_all(), false) != 256) {
return td::Status::Error("failed to parse bag id");
}
return ManualDns::EntryData::storage_address(bag_id);
} else if (parser.data() == "DELETED") {
return {};
}

17
crypto/smc-envelope/ManualDns.h
View file

@ -66,9 +66,19 @@ class DnsInterface {
// TODO: capability
};
struct EntryDataStorageAddress {
ton::Bits256 bag_id;
// TODO: proto
bool operator==(const EntryDataStorageAddress& other) const {
return bag_id == other.bag_id;
}
};
struct EntryData {
enum Type { Empty, Text, NextResolver, AdnlAddress, SmcAddress } type{Empty};
td::Variant<EntryDataText, EntryDataNextResolver, EntryDataAdnlAddress, EntryDataSmcAddress> data;
enum Type { Empty, Text, NextResolver, AdnlAddress, SmcAddress, StorageAddress } type{Empty};
td::Variant<EntryDataText, EntryDataNextResolver, EntryDataAdnlAddress, EntryDataSmcAddress,
EntryDataStorageAddress>
data;
static EntryData text(std::string text) {
return {Text, EntryDataText{text}};
@ -82,6 +92,9 @@ class DnsInterface {
static EntryData smc_address(block::StdAddress smc_address) {
return {SmcAddress, EntryDataSmcAddress{smc_address}};
}
static EntryData storage_address(ton::Bits256 bag_id) {
return {StorageAddress, EntryDataStorageAddress{bag_id}};
}
bool operator==(const EntryData& other) const {
return data == other.data;

7
docker/Dockerfile
View file

@ -11,16 +11,19 @@ WORKDIR /ton
RUN mkdir build && \
cd build && \
cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. && \
ninja tonlibjson fift func validator-engine validator-engine-console generate-random-id dht-server lite-client
cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" .. && \
ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console generate-random-id dht-server lite-client
FROM ubuntu:20.04
RUN apt-get update && \
apt-get install -y openssl wget libatomic1 && \
rm -rf /var/lib/apt/lists/*
RUN mkdir -p /var/ton-work/db && \
mkdir -p /var/ton-work/db/static
COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon /usr/local/bin/
COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon-cli /usr/local/bin/
COPY --from=builder /ton/build/lite-client/lite-client /usr/local/bin/
COPY --from=builder /ton/build/validator-engine/validator-engine /usr/local/bin/
COPY --from=builder /ton/build/validator-engine-console/validator-engine-console /usr/local/bin/

7
lite-client/lite-client.cpp
View file

@ -1749,6 +1749,13 @@ bool TestNode::show_dns_record(std::ostream& os, td::Bits256 cat, Ref<vm::CellSl
}
break;
}
case block::gen::DNSRecord::dns_storage_address: {
block::gen::DNSRecord::Record_dns_storage_address rec;
if (tlb::unpack_exact(cs, rec)) {
os << "\tstorage address " << rec.bag_id.to_hex();
}
break;
}
case block::gen::DNSRecord::dns_next_resolver: {
block::gen::DNSRecord::Record_dns_next_resolver rec;
if (tlb::unpack_exact(cs, rec) && block::tlb::t_MsgAddressInt.extract_std_address(rec.resolver, wc, addr)) {

38
rldp-http-proxy/DNSResolver.cpp
View file

@ -51,13 +51,13 @@ void DNSResolver::sync() {
std::move(P));
}
void DNSResolver::resolve(std::string host, td::Promise<ton::adnl::AdnlNodeIdShort> promise) {
void DNSResolver::resolve(std::string host, td::Promise<std::string> promise) {
auto it = cache_.find(host);
if (it != cache_.end()) {
const CacheEntry &entry = it->second;
double now = td::Time::now();
if (now < entry.created_at_ + CACHE_TIMEOUT_HARD) {
promise.set_result(entry.id_);
promise.set_result(entry.address_);
promise.reset();
if (now < entry.created_at_ + CACHE_TIMEOUT_SOFT) {
return;
@ -73,42 +73,42 @@ void DNSResolver::resolve(std::string host, td::Promise<ton::adnl::AdnlNodeIdSho
if (promise) {
promise.set_result(R.move_as_error());
}
} else {
return;
}
auto obj = R.move_as_ok();
ton::adnl::AdnlNodeIdShort id;
td::uint32 cnt = 0;
for (auto &e : obj->entries_) {
tonlib_api::downcast_call(*e->entry_.get(),
std::string result;
if (!obj->entries_.empty()) {
tonlib_api::downcast_call(*obj->entries_[0]->entry_,
td::overloaded(
[&](tonlib_api::dns_entryDataAdnlAddress &x) {
if (td::Random::fast(0, cnt) == 0) {
auto R = ton::adnl::AdnlNodeIdShort::parse(x.adnl_address_->adnl_address_);
if (R.is_ok()) {
id = R.move_as_ok();
cnt++;
}
ton::adnl::AdnlNodeIdShort id = R.move_as_ok();
result = id.serialize() + ".adnl";
}
},
[&](tonlib_api::dns_entryDataStorageAddress &x) {
result = td::to_lower(x.bag_id_.to_hex()) + ".bag";
},
[&](auto &x) {}));
}
if (cnt == 0) {
if (result.empty()) {
if (promise) {
promise.set_error(td::Status::Error("no DNS entries"));
}
} else {
td::actor::send_closure(SelfId, &DNSResolver::save_to_cache, std::move(host), id);
return;
}
td::actor::send_closure(SelfId, &DNSResolver::save_to_cache, std::move(host), result);
if (promise) {
promise.set_result(id);
}
}
promise.set_result(std::move(result));
}
});
td::actor::send_closure(tonlib_client_, &tonlib::TonlibClientWrapper::send_request<tonlib_api::dns_resolve>,
std::move(obj), std::move(P));
}
void DNSResolver::save_to_cache(std::string host, ton::adnl::AdnlNodeIdShort id) {
void DNSResolver::save_to_cache(std::string host, std::string address) {
CacheEntry &entry = cache_[host];
entry.id_ = id;
entry.address_ = address;
entry.created_at_ = td::Time::now();
}

6
rldp-http-proxy/DNSResolver.h
View file

@ -34,16 +34,16 @@ class DNSResolver : public td::actor::Actor {
explicit DNSResolver(td::actor::ActorId<tonlib::TonlibClientWrapper> tonlib_client);
void start_up() override;
void resolve(std::string host, td::Promise<ton::adnl::AdnlNodeIdShort> promise);
void resolve(std::string host, td::Promise<std::string> promise);
private:
void sync();
void save_to_cache(std::string host, ton::adnl::AdnlNodeIdShort id);
void save_to_cache(std::string host, std::string address);
td::actor::ActorId<tonlib::TonlibClientWrapper> tonlib_client_;
struct CacheEntry {
ton::adnl::AdnlNodeIdShort id_;
std::string address_;
double created_at_;
};
std::map<std::string, CacheEntry> cache_;

95
rldp-http-proxy/rldp-http-proxy.cpp
View file

@ -425,7 +425,8 @@ class TcpToRldpRequestSender : public td::actor::Actor {
td::Promise<std::pair<std::unique_ptr<ton::http::HttpResponse>, std::shared_ptr<ton::http::HttpPayload>>> promise,
td::actor::ActorId<ton::adnl::Adnl> adnl, td::actor::ActorId<ton::dht::Dht> dht,
td::actor::ActorId<ton::rldp::Rldp> rldp, td::actor::ActorId<RldpHttpProxy> proxy,
td::actor::ActorId<DNSResolver> dns_resolver)
td::actor::ActorId<DNSResolver> dns_resolver,
ton::adnl::AdnlNodeIdShort storage_gateway)
: local_id_(local_id)
, host_(std::move(host))
, request_(std::move(request))
@ -435,17 +436,20 @@ class TcpToRldpRequestSender : public td::actor::Actor {
, dht_(dht)
, rldp_(rldp)
, proxy_(proxy)
, dns_resolver_(dns_resolver) {
}
void start_up() override {
resolve();
, dns_resolver_(dns_resolver)
, storage_gateway_(storage_gateway) {
}
void resolve();
void start_up() override {
td::Random::secure_bytes(id_.as_slice());
request_tl_ = request_->store_tl(id_);
resolve(host_);
}
void resolve(std::string host);
void resolved(ton::adnl::AdnlNodeIdShort id) {
dst_ = id;
td::Random::secure_bytes(id_.as_slice());
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<td::BufferSlice> R) {
if (R.is_error()) {
@ -459,7 +463,7 @@ class TcpToRldpRequestSender : public td::actor::Actor {
proxy_, is_tunnel())
.release();
auto f = ton::serialize_tl_object(request_->store_tl(id_), true);
auto f = ton::serialize_tl_object(request_tl_, true);
td::actor::send_closure(rldp_, &ton::rldp::Rldp::send_query_ex, local_id_, dst_, "http request over rldp",
std::move(P), td::Timestamp::in(30.0), std::move(f), 16 << 10);
}
@ -538,6 +542,7 @@ class TcpToRldpRequestSender : public td::actor::Actor {
std::unique_ptr<ton::http::HttpRequest> request_;
std::shared_ptr<ton::http::HttpPayload> request_payload_;
ton::tl_object_ptr<ton::ton_api::http_request> request_tl_;
td::Promise<std::pair<std::unique_ptr<ton::http::HttpResponse>, std::shared_ptr<ton::http::HttpPayload>>> promise_;
@ -546,6 +551,9 @@ class TcpToRldpRequestSender : public td::actor::Actor {
td::actor::ActorId<ton::rldp::Rldp> rldp_;
td::actor::ActorId<RldpHttpProxy> proxy_;
td::actor::ActorId<DNSResolver> dns_resolver_;
ton::adnl::AdnlNodeIdShort storage_gateway_ = ton::adnl::AdnlNodeIdShort::zero();
bool dns_resolve_sent_ = false;
std::unique_ptr<ton::http::HttpResponse> response_;
std::shared_ptr<ton::http::HttpPayload> response_payload_;
@ -983,15 +991,9 @@ class RldpHttpProxy : public td::actor::Actor {
}
}
{
if (is_client_) {
auto D = ton::dht::Dht::create_client(dht_id_, "", dht_config_, keyring_.get(), adnl_.get());
D.ensure();
dht_ = D.move_as_ok();
} else {
auto D = ton::dht::Dht::create(dht_id_, db_root_, dht_config_, keyring_.get(), adnl_.get());
D.ensure();
dht_ = D.move_as_ok();
}
td::actor::send_closure(adnl_, &ton::adnl::Adnl::register_dht_node, dht_.get());
}
if (port_) {
@ -1104,15 +1106,20 @@ class RldpHttpProxy : public td::actor::Actor {
}
}
std::transform(host.begin(), host.end(), host.begin(), [](unsigned char c) { return std::tolower(c); });
if (!proxy_all_ &&
(host.size() < 5 || (host.substr(host.size() - 4) != ".ton" && host.substr(host.size() - 5) != ".adnl"))) {
bool allow = proxy_all_;
for (const char* suffix : {".adnl", ".ton", ".bag"}) {
if (td::ends_with(host, td::Slice(suffix))) {
allow = true;
}
}
if (!allow) {
promise.set_error(td::Status::Error(ton::ErrorCode::error, "bad server name"));
return;
}
td::actor::create_actor<TcpToRldpRequestSender>("outboundreq", local_id_, host, std::move(request),
std::move(payload), std::move(promise), adnl_.get(), dht_.get(),
rldp_.get(), actor_id(this), dns_resolver_.get())
rldp_.get(), actor_id(this), dns_resolver_.get(), storage_gateway_)
.release();
}
@ -1280,6 +1287,10 @@ class RldpHttpProxy : public td::actor::Actor {
proxy_all_ = value;
}
void set_storage_gateway(ton::adnl::AdnlNodeIdShort id) {
storage_gateway_ = id;
}
private:
struct Host {
struct Server {
@ -1317,15 +1328,43 @@ class RldpHttpProxy : public td::actor::Actor {
td::actor::ActorOwn<tonlib::TonlibClientWrapper> tonlib_client_;
td::actor::ActorOwn<DNSResolver> dns_resolver_;
ton::adnl::AdnlNodeIdShort storage_gateway_ = ton::adnl::AdnlNodeIdShort::zero();
std::map<td::Bits256,
std::function<void(ton::tl_object_ptr<ton::ton_api::http_getNextPayloadPart>, td::Promise<td::BufferSlice>)>>
payload_senders_;
};
void TcpToRldpRequestSender::resolve() {
auto S = td::Slice(host_);
if (S.size() >= 5 && S.substr(S.size() - 5) == ".adnl") {
void TcpToRldpRequestSender::resolve(std::string host) {
auto S = td::Slice(host);
if (td::ends_with(S, ".bag")) {
if (storage_gateway_.is_zero()) {
abort_query(td::Status::Error("storage gateway is not set"));
return;
}
td::Slice bag_id = S.substr(0, S.size() - 4);
td::Slice url = request_tl_->url_;
if (td::begins_with(url, "http://")) {
url.remove_prefix(7);
}
size_t pos = url.find('/');
if (pos == td::Slice::npos) {
url = "/";
} else {
url = url.substr(pos);
}
request_tl_->url_ = (PSTRING() << "/gateway/" << bag_id << url);
host = storage_gateway_.serialize() + ".adnl";
for (auto& header : request_tl_->headers_) {
if (td::to_lower(header->name_) == "host") {
header->value_ = host;
break;
}
}
resolved(storage_gateway_);
return;
}
if (td::ends_with(S, ".adnl")) {
S.truncate(S.size() - 5);
auto R = ton::adnl::AdnlNodeIdShort::parse(S);
if (R.is_error()) {
@ -1335,15 +1374,20 @@ void TcpToRldpRequestSender::resolve() {
resolved(R.move_as_ok());
return;
}
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result<ton::adnl::AdnlNodeIdShort> R) {
if (dns_resolve_sent_) {
abort_query(td::Status::Error(PSTRING() << "unexpected dns result: " << host));
return;
}
dns_resolve_sent_ = true;
td::actor::send_closure(dns_resolver_, &DNSResolver::resolve, host,
[SelfId = actor_id(this)](td::Result<std::string> R) {
if (R.is_error()) {
td::actor::send_closure(SelfId, &TcpToRldpRequestSender::abort_query,
R.move_as_error_prefix("failed to resolve: "));
} else {
td::actor::send_closure(SelfId, &TcpToRldpRequestSender::resolved, R.move_as_ok());
td::actor::send_closure(SelfId, &TcpToRldpRequestSender::resolve, R.move_as_ok());
}
});
td::actor::send_closure(dns_resolver_, &DNSResolver::resolve, host_, std::move(P));
}
void HttpRldpPayloadSender::start_up() {
@ -1532,6 +1576,11 @@ int main(int argc, char *argv[]) {
logger_ = td::FileLog::create(fname.str()).move_as_ok();
td::log_interface = logger_.get();
});
p.add_checked_option('S', "storage-gateway", "adnl address of ton storage gateway", [&](td::Slice arg) -> td::Status {
TRY_RESULT(adnl, ton::adnl::AdnlNodeIdShort::parse(arg));
td::actor::send_closure(x, &RldpHttpProxy::set_storage_gateway, adnl);
return td::Status::OK();
});
p.add_checked_option('P', "proxy-all", "value=[YES|NO]. proxy all HTTP requests (default only *.ton and *.adnl)",
[&](td::Slice value) {
if (value == "YES" || value == "yes") {

2
rldp2/BdwStats.cpp
View file

@ -40,7 +40,7 @@ void BdwStats::on_packet_ack(const PacketInfo &info, td::Timestamp sent_at, td::
auto ack_passed = now.at() - info.delivered_now.at();
auto passed = td::max(sent_passed, ack_passed);
if (passed < 0.01) {
VLOG(RLDP_WARNING) << "Invalid passed " << passed;
VLOG(RLDP_INFO) << "Invalid passed " << passed;
}
auto delivered = delivered_count - info.delivered_count;
on_rate_sample((double)delivered / passed, now, info.is_paused);

7
rldp2/CMakeLists.txt
View file

@ -7,6 +7,9 @@ endif()
if (NOT GSL_FOUND)
find_package(GSL)
endif()
if (NOT BLAS_FOUND)
find_package(BLAS)
endif()
set(RLDP_SOURCE
Ack.cpp
@ -50,8 +53,8 @@ target_include_directories(rldp PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>/..
${OPENSSL_INCLUDE_DIR}
)
if (GSL_FOUND)
target_link_libraries(rldp2 PRIVATE gsl)
if (GSL_FOUND AND BLAS_FOUND)
target_link_libraries(rldp2 PRIVATE gsl blas)
target_compile_definitions(rldp2 PRIVATE -DTON_HAVE_GSL=1)
endif()
target_link_libraries(rldp2 PUBLIC tdutils tdactor fec adnl tl_api)

4
rldp2/RttStats.cpp
View file

@ -25,11 +25,11 @@ namespace ton {
namespace rldp2 {
void RttStats::on_rtt_sample(double rtt_sample, double ack_delay, td::Timestamp now) {
if (rtt_sample < 0.001 || rtt_sample > 10) {
VLOG(RLDP_WARNING) << "Suspicious rtt sample " << rtt_sample;
VLOG(RLDP_INFO) << "Suspicious rtt sample " << rtt_sample;
return;
}
if (ack_delay < -1e-9 || ack_delay > 10) {
VLOG(RLDP_WARNING) << "Suspicious ack_delay " << ack_delay;
VLOG(RLDP_INFO) << "Suspicious ack_delay " << ack_delay;
return;
}
rtt_sample = td::max(0.01, rtt_sample);

2
storage/LoadSpeed.cpp
View file

@ -22,7 +22,7 @@
#include "td/utils/format.h"
namespace ton {
void LoadSpeed::add(std::size_t size, td::Timestamp now) {
void LoadSpeed::add(td::uint64 size, td::Timestamp now) {
total_size_ += size;
events_.push(Event{size, now});
update(now);

49
storage/NodeActor.cpp
View file

@ -32,24 +32,27 @@
namespace ton {
NodeActor::NodeActor(PeerId self_id, Torrent torrent, td::unique_ptr<Callback> callback,
td::unique_ptr<NodeCallback> node_callback, std::shared_ptr<db::DbType> db, bool should_download)
: self_id_(self_id)
, torrent_(std::move(torrent))
, callback_(std::move(callback))
, node_callback_(std::move(node_callback))
, db_(std::move(db))
, should_download_(should_download) {
}
NodeActor::NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr<Callback> callback,
td::unique_ptr<NodeCallback> node_callback, std::shared_ptr<db::DbType> db, bool should_download,
DbInitialData db_initial_data)
bool should_upload)
: self_id_(self_id)
, torrent_(std::move(torrent))
, callback_(std::move(callback))
, node_callback_(std::move(node_callback))
, db_(std::move(db))
, should_download_(should_download)
, should_upload_(should_upload) {
}
NodeActor::NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr<Callback> callback,
td::unique_ptr<NodeCallback> node_callback, std::shared_ptr<db::DbType> db, bool should_download,
bool should_upload, DbInitialData db_initial_data)
: self_id_(self_id)
, torrent_(std::move(torrent))
, callback_(std::move(callback))
, node_callback_(std::move(node_callback))
, db_(std::move(db))
, should_download_(should_download)
, should_upload_(should_upload)
, pending_set_file_priority_(std::move(db_initial_data.priorities))
, pieces_in_db_(std::move(db_initial_data.pieces_in_db)) {
}
@ -199,7 +202,7 @@ void NodeActor::loop_will_upload() {
}
for (auto &it : peers_) {
auto will_upload = peers_set.count(it.first) > 0;
auto will_upload = peers_set.count(it.first) > 0 && should_upload_;
auto &state = it.second.state;
auto node_state = state->node_state_.load();
if (node_state.will_upload != will_upload) {
@ -399,6 +402,15 @@ void NodeActor::set_should_download(bool should_download) {
yield();
}
void NodeActor::set_should_upload(bool should_upload) {
if (should_upload == should_upload_) {
return;
}
should_upload_ = should_upload;
db_store_torrent();
yield();
}
void NodeActor::load_from(td::optional<TorrentMeta> meta, std::string files_path, td::Promise<td::Unit> promise) {
auto S = [&]() -> td::Status {
if (meta) {
@ -442,6 +454,12 @@ void NodeActor::load_from(td::optional<TorrentMeta> meta, std::string files_path
loop();
}
void NodeActor::copy_to_new_root_dir(std::string new_root_dir, td::Promise<td::Unit> promise) {
TRY_STATUS_PROMISE(promise, torrent_.copy_to(new_root_dir));
db_store_torrent();
promise.set_result(td::Unit());
}
void NodeActor::tear_down() {
for (auto &promise : wait_for_completion_) {
promise.set_error(td::Status::Error("Torrent closed"));
@ -573,7 +591,7 @@ void NodeActor::loop_peer(const PeerId &peer_id, Peer &peer) {
for (td::uint32 part_id : state->peer_queries_.read()) {
should_notify_peer = true;
auto res = [&]() -> td::Result<PeerState::Part> {
if (!node_state.will_upload) {
if (!node_state.will_upload || !should_upload_) {
return td::Status::Error("Won't upload");
}
TRY_RESULT(proof, torrent_.get_piece_proof(part_id));
@ -685,6 +703,7 @@ void NodeActor::db_store_torrent() {
}
auto obj = create_tl_object<ton_api::storage_db_torrent>();
obj->active_download_ = should_download_;
obj->active_upload_ = should_upload_;
obj->root_dir_ = torrent_.get_root_dir();
db_->set(create_hash_tl_object<ton_api::storage_db_key_torrent>(torrent_.get_hash()), serialize_tl_object(obj, true),
[](td::Result<td::Unit> R) {
@ -837,6 +856,7 @@ void NodeActor::load_from_db(std::shared_ptr<db::DbType> db, td::Bits256 hash, t
void got_torrent(tl_object_ptr<ton_api::storage_db_torrent> obj) {
root_dir_ = std::move(obj->root_dir_);
active_download_ = obj->active_download_;
active_upload_ = obj->active_upload_;
db_->get(create_hash_tl_object<ton_api::storage_db_key_torrentMeta>(hash_),
[SelfId = actor_id(this)](td::Result<db::DbType::GetResult> R) {
if (R.is_error()) {
@ -962,7 +982,7 @@ void NodeActor::load_from_db(std::shared_ptr<db::DbType> db, td::Bits256 hash, t
data.pieces_in_db = std::move(pieces_in_db_);
finish(td::actor::create_actor<NodeActor>("Node", 1, torrent_.unwrap(), std::move(callback_),
std::move(node_callback_), std::move(db_), active_download_,
std::move(data)));
active_upload_, std::move(data)));
}
private:
@ -974,6 +994,7 @@ void NodeActor::load_from_db(std::shared_ptr<db::DbType> db, td::Bits256 hash, t
std::string root_dir_;
bool active_download_{false};
bool active_upload_{false};
td::optional<Torrent> torrent_;
std::vector<PendingSetFilePriority> priorities_;
std::set<td::uint64> pieces_in_db_;

13
storage/NodeActor.h
View file

@ -63,32 +63,36 @@ class NodeActor : public td::actor::Actor {
};
NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr<Callback> callback,
td::unique_ptr<NodeCallback> node_callback, std::shared_ptr<db::DbType> db, bool should_download = true);
td::unique_ptr<NodeCallback> node_callback, std::shared_ptr<db::DbType> db, bool should_download = true,
bool should_upload = true);
NodeActor(PeerId self_id, ton::Torrent torrent, td::unique_ptr<Callback> callback,
td::unique_ptr<NodeCallback> node_callback, std::shared_ptr<db::DbType> db, bool should_download,
DbInitialData db_initial_data);
bool should_upload, DbInitialData db_initial_data);
void start_peer(PeerId peer_id, td::Promise<td::actor::ActorId<PeerActor>> promise);
struct NodeState {
Torrent &torrent;
bool active_download;
bool active_upload;
double download_speed;
double upload_speed;
const std::vector<td::uint8> &file_priority;
};
void with_torrent(td::Promise<NodeState> promise) {
promise.set_value(
NodeState{torrent_, should_download_, download_speed_.speed(), upload_speed_.speed(), file_priority_});
promise.set_value(NodeState{torrent_, should_download_, should_upload_, download_speed_.speed(),
upload_speed_.speed(), file_priority_});
}
std::string get_stats_str();
void set_should_download(bool should_download);
void set_should_upload(bool should_upload);
void set_all_files_priority(td::uint8 priority, td::Promise<bool> promise);
void set_file_priority_by_idx(size_t i, td::uint8 priority, td::Promise<bool> promise);
void set_file_priority_by_name(std::string name, td::uint8 priority, td::Promise<bool> promise);
void load_from(td::optional<TorrentMeta> meta, std::string files_path, td::Promise<td::Unit> promise);
void copy_to_new_root_dir(std::string new_root_dir, td::Promise<td::Unit> promise);
void wait_for_completion(td::Promise<td::Unit> promise);
void get_peers_info(td::Promise<tl_object_ptr<ton_api::storage_daemon_peerList>> promise);
@ -107,6 +111,7 @@ class NodeActor : public td::actor::Actor {
td::unique_ptr<NodeCallback> node_callback_;
std::shared_ptr<db::DbType> db_;
bool should_download_{false};
bool should_upload_{false};
class Notifier : public td::actor::Actor {
public:

33
storage/Torrent.cpp
View file

@ -698,4 +698,37 @@ void Torrent::load_from_files(std::string files_path) {
}
}
td::Status Torrent::copy_to(const std::string &new_root_dir) {
if (!is_completed() || included_size_ != info_.file_size) {
return td::Status::Error("Torrent::copy_to is allowed only for fully completed torrents");
}
auto get_new_chunk_path = [&](td::Slice name) -> std::string {
return PSTRING() << new_root_dir << TD_DIR_SLASH << header_.value().dir_name << TD_DIR_SLASH << name;
};
std::vector<td::BlobView> new_blobs;
for (size_t i = 1; i < chunks_.size(); ++i) {
auto &chunk = chunks_[i];
std::string new_path = get_new_chunk_path(chunk.name);
TRY_STATUS(td::mkpath(new_path));
TRY_RESULT(new_blob, td::FileNoCacheBlobView::create(new_path, chunk.size, true));
static const td::uint64 BUF_SIZE = 1 << 17;
td::BufferSlice buf(BUF_SIZE);
for (td::uint64 l = 0; l < chunk.size; l += BUF_SIZE) {
td::uint64 r = std::min(chunk.size, l + BUF_SIZE);
TRY_RESULT_PREFIX(s, chunk.data.view(buf.as_slice().substr(0, r - l), l),
PSTRING() << "Failed to read " << chunk.name << ": ");
if (s.size() != r - l) {
return td::Status::Error(PSTRING() << "Failed to read " << chunk.name);
}
TRY_STATUS_PREFIX(new_blob.write(s, l), PSTRING() << "Failed to write " << chunk.name << ": ");
}
new_blobs.push_back(std::move(new_blob));
}
root_dir_ = new_root_dir;
for (size_t i = 1; i < chunks_.size(); ++i) {
chunks_[i].data = std::move(new_blobs[i - 1]);
}
return td::Status::OK();
}
} // namespace ton

2
storage/Torrent.h
View file

@ -159,6 +159,8 @@ class Torrent {
void load_from_files(std::string files_path);
td::Status copy_to(const std::string& new_root_dir);
private:
td::Bits256 hash_;
bool inited_info_ = false;

3
storage/TorrentCreator.cpp
View file

@ -98,6 +98,9 @@ TD_WARN_UNUSED_RESULT td::Status Torrent::Creator::add_file(td::Slice name, td::
}
td::Result<Torrent> Torrent::Creator::finalize() {
if (files_.empty()) {
return td::Status::Error("No files");
}
TorrentHeader header;
TRY_RESULT(files_count, td::narrow_cast_safe<td::uint32>(files_.size()));
header.files_count = files_count;

3
storage/TorrentHeader.cpp
View file

@ -100,6 +100,9 @@ td::Status TorrentHeader::validate(td::uint64 total_size, td::uint64 header_size
if (serialization_size() != header_size) {
return td::Status::Error("Invalid size");
}
if (files_count == 0) {
return td::Status::Error("No files");
}
for (size_t i = 0; i + 1 < files_count; ++i) {
if (name_index[i] > name_index[i + 1]) {
return td::Status::Error("Invalid name offset");

45
storage/storage-daemon/StorageManager.cpp
View file

@ -21,6 +21,8 @@
#include "td/db/RocksDb.h"
#include "td/actor/MultiPromise.h"
namespace ton {
static overlay::OverlayIdFull get_overlay_id(td::Bits256 hash) {
td::BufferSlice hash_str(hash.as_slice());
return overlay::OverlayIdFull(std::move(hash_str));
@ -126,16 +128,33 @@ td::unique_ptr<NodeActor::Callback> StorageManager::create_callback(
return td::make_unique<Callback>(actor_id(this), hash, std::move(closing_state));
}
void StorageManager::add_torrent(Torrent torrent, bool start_download, td::Promise<td::Unit> promise) {
TRY_STATUS_PROMISE(promise, add_torrent_impl(std::move(torrent), start_download));
void StorageManager::add_torrent(Torrent torrent, bool start_download, bool allow_upload, bool copy_inside,
td::Promise<td::Unit> promise) {
td::Bits256 hash = torrent.get_hash();
TRY_STATUS_PROMISE(promise, add_torrent_impl(std::move(torrent), start_download, allow_upload));
db_store_torrent_list();
if (!copy_inside) {
promise.set_result(td::Unit());
return;
}
TorrentEntry& entry = torrents_[hash];
std::string new_dir = db_root_ + "/torrent-files/" + hash.to_hex();
LOG(INFO) << "Copy torrent to " << new_dir;
td::actor::send_closure(
entry.actor, &NodeActor::copy_to_new_root_dir, std::move(new_dir),
[SelfId = actor_id(this), hash, promise = std::move(promise)](td::Result<td::Unit> R) mutable {
if (R.is_error()) {
LOG(WARNING) << "Copy torrent: " << R.error();
td::actor::send_closure(SelfId, &StorageManager::remove_torrent, hash, false, [](td::Result<td::Unit> R) {});
}
promise.set_result(std::move(R));
});
}
td::Status StorageManager::add_torrent_impl(Torrent torrent, bool start_download) {
td::Status StorageManager::add_torrent_impl(Torrent torrent, bool start_download, bool allow_upload) {
td::Bits256 hash = torrent.get_hash();
if (torrents_.count(hash)) {
return td::Status::Error("Cannot add torrent: duplicate hash");
return td::Status::Error(PSTRING() << "Cannot add torrent: duplicate hash " << hash.to_hex());
}
TorrentEntry& entry = torrents_[hash];
entry.hash = hash;
@ -145,25 +164,25 @@ td::Status StorageManager::add_torrent_impl(Torrent torrent, bool start_download
LOG(INFO) << "Added torrent " << hash.to_hex() << " , root_dir = " << torrent.get_root_dir();
entry.actor =
td::actor::create_actor<NodeActor>("Node", 1, std::move(torrent), create_callback(hash, entry.closing_state),
std::move(context), db_, start_download);
std::move(context), db_, start_download, allow_upload);
return td::Status::OK();
}
void StorageManager::add_torrent_by_meta(TorrentMeta meta, std::string root_dir, bool start_download,
void StorageManager::add_torrent_by_meta(TorrentMeta meta, std::string root_dir, bool start_download, bool allow_upload,
td::Promise<td::Unit> promise) {
td::Bits256 hash(meta.info.get_hash());
Torrent::Options options;
options.root_dir = root_dir.empty() ? db_root_ + "/torrent-files/" + hash.to_hex() : root_dir;
TRY_RESULT_PROMISE(promise, torrent, Torrent::open(std::move(options), std::move(meta)));
add_torrent(std::move(torrent), start_download, std::move(promise));
add_torrent(std::move(torrent), start_download, allow_upload, false, std::move(promise));
}
void StorageManager::add_torrent_by_hash(td::Bits256 hash, std::string root_dir, bool start_download,
void StorageManager::add_torrent_by_hash(td::Bits256 hash, std::string root_dir, bool start_download, bool allow_upload,
td::Promise<td::Unit> promise) {
Torrent::Options options;
options.root_dir = root_dir.empty() ? db_root_ + "/torrent-files/" + hash.to_hex() : root_dir;
TRY_RESULT_PROMISE(promise, torrent, Torrent::open(std::move(options), hash));
add_torrent(std::move(torrent), start_download, std::move(promise));
add_torrent(std::move(torrent), start_download, allow_upload, false, std::move(promise));
}
void StorageManager::set_active_download(td::Bits256 hash, bool active, td::Promise<td::Unit> promise) {
@ -172,6 +191,12 @@ void StorageManager::set_active_download(td::Bits256 hash, bool active, td::Prom
promise.set_result(td::Unit());
}
void StorageManager::set_active_upload(td::Bits256 hash, bool active, td::Promise<td::Unit> promise) {
TRY_RESULT_PROMISE(promise, entry, get_torrent(hash));
td::actor::send_closure(entry->actor, &NodeActor::set_should_upload, active);
promise.set_result(td::Unit());
}
void StorageManager::with_torrent(td::Bits256 hash, td::Promise<NodeActor::NodeState> promise) {
TRY_RESULT_PROMISE(promise, entry, get_torrent(hash));
td::actor::send_closure(entry->actor, &NodeActor::with_torrent, std::move(promise));
@ -266,3 +291,5 @@ void StorageManager::get_peers_info(td::Bits256 hash,
TRY_RESULT_PROMISE(promise, entry, get_torrent(hash));
td::actor::send_closure(entry->actor, &NodeActor::get_peers_info, std::move(promise));
}
} // namespace ton

16
storage/storage-daemon/StorageManager.h
View file

@ -23,7 +23,7 @@
#include "storage/PeerManager.h"
#include "storage/db.h"
using namespace ton;
namespace ton {
class StorageManager : public td::actor::Actor {
public:
@ -39,11 +39,15 @@ class StorageManager : public td::actor::Actor {
void start_up() override;
void add_torrent(Torrent torrent, bool start_download, td::Promise<td::Unit> promise);
void add_torrent_by_meta(TorrentMeta meta, std::string root_dir, bool start_download, td::Promise<td::Unit> promise);
void add_torrent_by_hash(td::Bits256 hash, std::string root_dir, bool start_download, td::Promise<td::Unit> promise);
void add_torrent(Torrent torrent, bool start_download, bool allow_upload, bool copy_inside,
td::Promise<td::Unit> promise);
void add_torrent_by_meta(TorrentMeta meta, std::string root_dir, bool start_download, bool allow_upload,
td::Promise<td::Unit> promise);
void add_torrent_by_hash(td::Bits256 hash, std::string root_dir, bool start_download, bool allow_upload,
td::Promise<td::Unit> promise);
void set_active_download(td::Bits256 hash, bool active, td::Promise<td::Unit> promise);
void set_active_upload(td::Bits256 hash, bool active, td::Promise<td::Unit> promise);
void with_torrent(td::Bits256 hash, td::Promise<NodeActor::NodeState> promise);
void get_all_torrents(td::Promise<std::vector<td::Bits256>> promise);
@ -85,7 +89,7 @@ class StorageManager : public td::actor::Actor {
std::map<td::Bits256, TorrentEntry> torrents_;
td::Status add_torrent_impl(Torrent torrent, bool start_download);
td::Status add_torrent_impl(Torrent torrent, bool start_download, bool allow_upload);
td::Result<TorrentEntry*> get_torrent(td::Bits256 hash) {
auto it = torrents_.find(hash);
@ -105,3 +109,5 @@ class StorageManager : public td::actor::Actor {
void on_torrent_closed(Torrent torrent, std::shared_ptr<TorrentEntry::ClosingState> closing_state);
};
} // namespace ton

34
storage/storage-daemon/StorageProvider.cpp
View file

@ -24,6 +24,8 @@
#include "common/delay.h"
#include "td/actor/MultiPromise.h"
namespace ton {
td::Result<ProviderParams> ProviderParams::create(const tl_object_ptr<ton_api::storage_daemon_provider_params>& obj) {
ProviderParams p;
p.accept_new_contracts = obj->accept_new_contracts_;
@ -155,7 +157,7 @@ void StorageProvider::start_up() {
init_new_storage_contract(address, contract);
break;
case StorageContract::st_downloaded:
check_contract_active(address);
after_contract_downloaded(address);
break;
case StorageContract::st_active:
contract.check_next_proof_at = td::Timestamp::now();
@ -382,7 +384,7 @@ void StorageProvider::db_update_microchunk_tree(const ContractAddress& address)
void StorageProvider::init_new_storage_contract(ContractAddress address, StorageContract& contract) {
CHECK(contract.state == StorageContract::st_downloading);
td::actor::send_closure(storage_manager_, &StorageManager::add_torrent_by_hash, contract.torrent_hash, "", false,
[](td::Result<td::Unit> R) {
false, [](td::Result<td::Unit> R) {
// Ignore errors: error can mean that the torrent already exists, other errors will be caught later
if (R.is_error()) {
LOG(DEBUG) << "Add torrent: " << R.move_as_error();
@ -449,11 +451,25 @@ void StorageProvider::downloaded_torrent(ContractAddress address, MicrochunkTree
contract.microchunk_tree = std::make_shared<MicrochunkTree>(std::move(microchunk_tree));
db_update_microchunk_tree(address);
db_update_storage_contract(address, false);
check_contract_active(address);
after_contract_downloaded(address);
}
void StorageProvider::check_contract_active(ContractAddress address, td::Timestamp retry_until,
void StorageProvider::after_contract_downloaded(ContractAddress address, td::Timestamp retry_until,
td::Timestamp retry_false_until) {
auto it = contracts_.find(address);
if (it == contracts_.end()) {
LOG(WARNING) << "Contract " << address.to_string() << " does not exist anymore";
return;
}
auto& contract = it->second;
td::actor::send_closure(storage_manager_, &StorageManager::set_active_upload, contract.torrent_hash, true,
[SelfId = actor_id(this), address](td::Result<td::Unit> R) {
if (R.is_error()) {
LOG(ERROR) << "Set active upload: " << R.move_as_error();
return;
}
LOG(DEBUG) << "Set active upload: OK";
});
get_storage_contract_data(address, tonlib_client_,
[=, SelfId = actor_id(this)](td::Result<StorageContractData> R) mutable {
if (R.is_error()) {
@ -461,7 +477,7 @@ void StorageProvider::check_contract_active(ContractAddress address, td::Timesta
if (retry_until && retry_until.is_in_past()) {
delay_action(
[=]() {
td::actor::send_closure(SelfId, &StorageProvider::check_contract_active,
td::actor::send_closure(SelfId, &StorageProvider::after_contract_downloaded,
address, retry_until, retry_false_until);
},
td::Timestamp::in(5.0));
@ -473,8 +489,8 @@ void StorageProvider::check_contract_active(ContractAddress address, td::Timesta
} else if (retry_false_until && retry_false_until.is_in_past()) {
delay_action(
[=]() {
td::actor::send_closure(SelfId, &StorageProvider::check_contract_active, address,
retry_until, retry_false_until);
td::actor::send_closure(SelfId, &StorageProvider::after_contract_downloaded,
address, retry_until, retry_false_until);
},
td::Timestamp::in(5.0));
} else {
@ -497,7 +513,7 @@ void StorageProvider::activate_contract_cont(ContractAddress address) {
td::Timestamp::in(10.0));
return;
}
td::actor::send_closure(SelfId, &StorageProvider::check_contract_active, address, td::Timestamp::in(60.0),
td::actor::send_closure(SelfId, &StorageProvider::after_contract_downloaded, address, td::Timestamp::in(60.0),
td::Timestamp::in(40.0));
});
}
@ -838,3 +854,5 @@ StorageProvider::Config::Config(const tl_object_ptr<ton_api::storage_daemon_prov
tl_object_ptr<ton_api::storage_daemon_providerConfig> StorageProvider::Config::tl() const {
return create_tl_object<ton_api::storage_daemon_providerConfig>(max_contracts, max_total_size);
}
} // namespace ton

6
storage/storage-daemon/StorageProvider.h
View file

@ -24,7 +24,7 @@
#include "smc-util.h"
#include "storage/MicrochunkTree.h"
using namespace ton;
namespace ton {
struct ProviderParams {
bool accept_new_contracts = false;
@ -109,7 +109,7 @@ class StorageProvider : public td::actor::Actor {
void on_new_storage_contract_cont(ContractAddress address, StorageContractData data, td::Promise<td::Unit> promise);
void init_new_storage_contract(ContractAddress address, StorageContract& contract);
void downloaded_torrent(ContractAddress address, MicrochunkTree microchunk_tree);
void check_contract_active(ContractAddress address, td::Timestamp retry_until = td::Timestamp::in(30.0),
void after_contract_downloaded(ContractAddress address, td::Timestamp retry_until = td::Timestamp::in(30.0),
td::Timestamp retry_false_until = td::Timestamp::never());
void activate_contract_cont(ContractAddress address);
void activated_storage_contract(ContractAddress address);
@ -125,3 +125,5 @@ class StorageProvider : public td::actor::Actor {
void got_next_proof(ContractAddress address, td::Result<td::Ref<vm::Cell>> R);
void sent_next_proof(ContractAddress address);
};
} // namespace ton

6
storage/storage-daemon/smc-util.cpp
View file

@ -20,6 +20,8 @@
#include "keys/encryptor.h"
#include "smartcont/provider-code.h"
namespace ton {
static void smc_forget(td::actor::ActorId<tonlib::TonlibClientWrapper> client, td::int64 id) {
auto query = create_tl_object<tonlib_api::smc_forget>(id);
td::actor::send_closure(client, &tonlib::TonlibClientWrapper::send_request<tonlib_api::smc_forget>, std::move(query),
@ -226,7 +228,7 @@ void FabricContractWrapper::loaded_last_transactions(
void FabricContractWrapper::run_get_method(
std::string method, std::vector<tl_object_ptr<tonlib_api::tvm_StackEntry>> args,
td::Promise<std::vector<tl_object_ptr<tonlib_api::tvm_StackEntry>>> promise) {
::run_get_method(address_, client_, std::move(method), std::move(args), std::move(promise));
ton::run_get_method(address_, client_, std::move(method), std::move(args), std::move(promise));
}
void FabricContractWrapper::send_internal_message(ContractAddress dest, td::RefInt256 coins, vm::CellSlice body,
@ -485,3 +487,5 @@ void get_storage_contract_data(ContractAddress address, td::actor::ActorId<tonli
rate_per_mb_day, max_span, last_proof_time, torrent_hash};
}));
}
} // namespace ton

4
storage/storage-daemon/smc-util.h
View file

@ -24,7 +24,7 @@
#include <queue>
#include "keyring/keyring.h"
using namespace ton;
namespace ton {
struct ContractAddress {
WorkchainId wc = workchainIdNotYet;
@ -182,3 +182,5 @@ struct StorageContractData {
void get_storage_contract_data(ContractAddress address, td::actor::ActorId<tonlib::TonlibClientWrapper> client,
td::Promise<StorageContractData> promise);
} // namespace ton

128
storage/storage-daemon/storage-daemon-cli.cpp
View file

@ -39,7 +39,7 @@
#include "common/refint.h"
#include "crypto/block/block.h"
using namespace ton;
namespace ton {
bool is_whitespace(char c) {
return strchr(" \t\n\r", c) != nullptr;
@ -298,10 +298,13 @@ class StorageDaemonCli : public td::actor::Actor {
} else if (tokens[0] == "create") {
std::string path;
bool found_path = false;
bool upload = true;
bool copy = false;
std::string description;
bool json = false;
bool no_more_flags = false;
for (size_t i = 1; i < tokens.size(); ++i) {
if (!tokens[i].empty() && tokens[i][0] == '-') {
if (!tokens[i].empty() && tokens[i][0] == '-' && !no_more_flags) {
if (tokens[i] == "-d") {
++i;
if (i == tokens.size()) {
@ -310,10 +313,22 @@ class StorageDaemonCli : public td::actor::Actor {
description = tokens[i];
continue;
}
if (tokens[i] == "--no-upload") {
upload = false;
continue;
}
if (tokens[i] == "--copy") {
copy = true;
continue;
}
if (tokens[i] == "--json") {
json = true;
continue;
}
if (tokens[i] == "--") {
no_more_flags = true;
continue;
}
return td::Status::Error(PSTRING() << "Unknown flag " << tokens[i]);
}
if (found_path) {
@ -325,15 +340,17 @@ class StorageDaemonCli : public td::actor::Actor {
if (!found_path) {
return td::Status::Error("Unexpected EOLN");
}
return execute_create(std::move(path), std::move(description), json);
return execute_create(std::move(path), std::move(description), upload, copy, json);
} else if (tokens[0] == "add-by-hash" || tokens[0] == "add-by-meta") {
td::optional<std::string> param;
std::string root_dir;
bool paused = false;
bool upload = true;
bool json = false;
td::optional<std::vector<std::string>> partial;
bool no_more_flags = false;
for (size_t i = 1; i < tokens.size(); ++i) {
if (!tokens[i].empty() && tokens[i][0] == '-') {
if (!tokens[i].empty() && tokens[i][0] == '-' && !no_more_flags) {
if (tokens[i] == "-d") {
++i;
if (i == tokens.size()) {
@ -346,6 +363,10 @@ class StorageDaemonCli : public td::actor::Actor {
paused = true;
continue;
}
if (tokens[i] == "--no-upload") {
upload = false;
continue;
}
if (tokens[i] == "--json") {
json = true;
continue;
@ -354,6 +375,10 @@ class StorageDaemonCli : public td::actor::Actor {
partial = std::vector<std::string>(tokens.begin() + i + 1, tokens.end());
break;
}
if (tokens[i] == "--") {
no_more_flags = true;
continue;
}
return td::Status::Error(PSTRING() << "Unknown flag " << tokens[i]);
}
if (param) {
@ -366,9 +391,9 @@ class StorageDaemonCli : public td::actor::Actor {
}
if (tokens[0] == "add-by-hash") {
TRY_RESULT(hash, parse_hash(param.value()));
return execute_add_by_hash(hash, std::move(root_dir), paused, std::move(partial), json);
return execute_add_by_hash(hash, std::move(root_dir), paused, upload, std::move(partial), json);
} else {
return execute_add_by_meta(param.value(), std::move(root_dir), paused, std::move(partial), json);
return execute_add_by_meta(param.value(), std::move(root_dir), paused, upload, std::move(partial), json);
}
} else if (tokens[0] == "list") {
bool with_hashes = false;
@ -441,6 +466,12 @@ class StorageDaemonCli : public td::actor::Actor {
}
TRY_RESULT(hash, parse_torrent(tokens[1]));
return execute_set_active_download(hash, tokens[0] == "download-resume");
} else if (tokens[0] == "upload-pause" || tokens[0] == "upload-resume") {
if (tokens.size() != 2) {
return td::Status::Error("Expected bag");
}
TRY_RESULT(hash, parse_torrent(tokens[1]));
return execute_set_active_upload(hash, tokens[0] == "upload-resume");
} else if (tokens[0] == "priority-all") {
if (tokens.size() != 3) {
return td::Status::Error("Expected bag and priority");
@ -521,8 +552,9 @@ class StorageDaemonCli : public td::actor::Actor {
td::optional<std::string> provider_address;
td::optional<std::string> rate;
td::optional<td::uint32> max_span;
bool no_more_flags = false;
for (size_t i = 1; i < tokens.size(); ++i) {
if (!tokens[i].empty() && tokens[i][0] == '-') {
if (!tokens[i].empty() && tokens[i][0] == '-' && !no_more_flags) {
if (tokens[i] == "--query-id") {
++i;
TRY_RESULT_PREFIX_ASSIGN(query_id, td::to_integer_safe<td::uint64>(tokens[i]), "Invalid query id: ");
@ -543,6 +575,10 @@ class StorageDaemonCli : public td::actor::Actor {
TRY_RESULT_PREFIX_ASSIGN(max_span, td::to_integer_safe<td::uint8>(tokens[i]), "Invalid max span: ");
continue;
}
if (tokens[i] == "--") {
no_more_flags = true;
continue;
}
return td::Status::Error(PSTRING() << "Unknown flag " << tokens[i]);
}
if (cnt == 0) {
@ -731,19 +767,22 @@ class StorageDaemonCli : public td::actor::Actor {
td::Status execute_help() {
td::TerminalIO::out() << "help\tPrint this help\n";
td::TerminalIO::out() << "create [-d description] [--json] <file/dir>\tCreate bag of files from <file/dir>\n";
td::TerminalIO::out() << "\t-d - Description will be stored in torrent info.\n";
td::TerminalIO::out() << "\t--json\tOutput in json\n";
td::TerminalIO::out()
<< "add-by-hash <bag-id> [-d root_dir] [--paused] [--json] [--partial file1 file2 ...]\tAdd bag "
"with given BagID (in hex)\n";
<< "create [-d description] [--no-upload] [--copy] [--json] <file/dir>\tCreate bag of files from <file/dir>\n";
td::TerminalIO::out() << "\t-d\tDescription will be stored in torrent info\n";
td::TerminalIO::out() << "\t--no-upload\tDon't share bag with peers\n";
td::TerminalIO::out() << "\t--copy\tFiles will be copied to an internal directory of storage-daemon\n";
td::TerminalIO::out() << "\t--json\tOutput in json\n";
td::TerminalIO::out() << "add-by-hash <bag-id> [-d root_dir] [--paused] [--no-upload] [--json] [--partial file1 "
"file2 ...]\tAdd bag with given BagID (in hex)\n";
td::TerminalIO::out() << "\t-d\tTarget directory, default is an internal directory of storage-daemon\n";
td::TerminalIO::out() << "\t--paused\tDon't start download immediately\n";
td::TerminalIO::out() << "\t--no-upload\tDon't share bag with peers\n";
td::TerminalIO::out()
<< "\t--partial\tEverything after this flag is a list of filenames. Only these files will be downloaded.\n";
td::TerminalIO::out() << "\t--json\tOutput in json\n";
td::TerminalIO::out() << "add-by-meta <meta> [-d root_dir] [--paused] [--json] [--partial file1 file2 ...]\tLoad "
"meta from file and add bag\n";
td::TerminalIO::out() << "add-by-meta <meta> [-d root_dir] [--paused] [--no-upload] [--json] [--partial file1 "
"file2 ...]\tLoad meta from file and add bag\n";
td::TerminalIO::out() << "\tFlags are the same as in add-by-hash\n";
td::TerminalIO::out() << "list [--hashes] [--json]\tPrint list of bags\n";
td::TerminalIO::out() << "\t--hashes\tPrint full BagID\n";
@ -756,6 +795,8 @@ class StorageDaemonCli : public td::actor::Actor {
td::TerminalIO::out() << "\t--json\tOutput in json\n";
td::TerminalIO::out() << "download-pause <bag>\tPause download of <bag>\n";
td::TerminalIO::out() << "download-resume <bag>\tResume download of <bag>\n";
td::TerminalIO::out() << "upload-pause <bag>\tPause upload of <bag>\n";
td::TerminalIO::out() << "upload-resume <bag>\tResume upload of <bag>\n";
td::TerminalIO::out() << "priority-all <bag> <p>\tSet priority of all files in <bag> to <p>\n";
td::TerminalIO::out() << "\tPriority is in [0..255], 0 - don't download\n";
td::TerminalIO::out() << "priority-idx <bag> <idx> <p>\tSet priority of file #<idx> in <bag> to <p>\n";
@ -828,9 +869,9 @@ class StorageDaemonCli : public td::actor::Actor {
return td::Status::OK();
}
td::Status execute_create(std::string path, std::string description, bool json) {
td::Status execute_create(std::string path, std::string description, bool upload, bool copy, bool json) {
TRY_RESULT_PREFIX_ASSIGN(path, td::realpath(path), "Invalid path: ");
auto query = create_tl_object<ton_api::storage_daemon_createTorrent>(path, description);
auto query = create_tl_object<ton_api::storage_daemon_createTorrent>(path, description, upload, copy);
send_query(std::move(query),
[=, SelfId = actor_id(this)](td::Result<tl_object_ptr<ton_api::storage_daemon_torrentFull>> R) {
if (R.is_error()) {
@ -838,6 +879,7 @@ class StorageDaemonCli : public td::actor::Actor {
}
if (json) {
print_json(R.ok());
td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK());
return;
}
td::TerminalIO::out() << "Bag created\n";
@ -847,7 +889,7 @@ class StorageDaemonCli : public td::actor::Actor {
return td::Status::OK();
}
td::Status execute_add_by_hash(td::Bits256 hash, std::string root_dir, bool paused,
td::Status execute_add_by_hash(td::Bits256 hash, std::string root_dir, bool paused, bool upload,
td::optional<std::vector<std::string>> partial, bool json) {
if (!root_dir.empty()) {
TRY_STATUS_PREFIX(td::mkpath(root_dir), "Failed to create directory: ");
@ -861,8 +903,8 @@ class StorageDaemonCli : public td::actor::Actor {
priorities.push_back(create_tl_object<ton_api::storage_priorityAction_name>(std::move(f), 1));
}
}
auto query =
create_tl_object<ton_api::storage_daemon_addByHash>(hash, std::move(root_dir), !paused, std::move(priorities));
auto query = create_tl_object<ton_api::storage_daemon_addByHash>(hash, std::move(root_dir), !paused, upload,
std::move(priorities));
send_query(std::move(query),
[=, SelfId = actor_id(this)](td::Result<tl_object_ptr<ton_api::storage_daemon_torrentFull>> R) {
if (R.is_error()) {
@ -870,6 +912,7 @@ class StorageDaemonCli : public td::actor::Actor {
}
if (json) {
print_json(R.ok());
td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK());
return;
}
td::TerminalIO::out() << "Bag added\n";
@ -879,7 +922,7 @@ class StorageDaemonCli : public td::actor::Actor {
return td::Status::OK();
}
td::Status execute_add_by_meta(std::string meta_file, std::string root_dir, bool paused,
td::Status execute_add_by_meta(std::string meta_file, std::string root_dir, bool paused, bool upload,
td::optional<std::vector<std::string>> partial, bool json) {
TRY_RESULT_PREFIX(meta, td::read_file(meta_file), "Failed to read meta: ");
if (!root_dir.empty()) {
@ -895,7 +938,7 @@ class StorageDaemonCli : public td::actor::Actor {
}
}
auto query = create_tl_object<ton_api::storage_daemon_addByMeta>(std::move(meta), std::move(root_dir), !paused,
std::move(priorities));
upload, std::move(priorities));
send_query(std::move(query),
[=, SelfId = actor_id(this)](td::Result<tl_object_ptr<ton_api::storage_daemon_torrentFull>> R) {
if (R.is_error()) {
@ -903,6 +946,7 @@ class StorageDaemonCli : public td::actor::Actor {
}
if (json) {
print_json(R.ok());
td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK());
return;
}
td::TerminalIO::out() << "Bag added\n";
@ -921,6 +965,7 @@ class StorageDaemonCli : public td::actor::Actor {
}
if (json) {
print_json(R.ok());
td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK());
return;
}
td::actor::send_closure(SelfId, &StorageDaemonCli::print_torrent_list, R.move_as_ok(), with_hashes);
@ -938,6 +983,7 @@ class StorageDaemonCli : public td::actor::Actor {
}
if (json) {
print_json(R.ok());
td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK());
return;
}
td::actor::send_closure(SelfId, &StorageDaemonCli::print_torrent_full, R.move_as_ok());
@ -976,6 +1022,7 @@ class StorageDaemonCli : public td::actor::Actor {
}
if (json) {
print_json(R.ok());
td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK());
return;
}
auto obj = R.move_as_ok();
@ -1020,6 +1067,19 @@ class StorageDaemonCli : public td::actor::Actor {
return td::Status::OK();
}
td::Status execute_set_active_upload(td::Bits256 hash, bool active) {
auto query = create_tl_object<ton_api::storage_daemon_setActiveUpload>(hash, active);
send_query(std::move(query),
[SelfId = actor_id(this)](td::Result<tl_object_ptr<ton_api::storage_daemon_success>> R) {
if (R.is_error()) {
return;
}
td::TerminalIO::out() << "Success\n";
td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK());
});
return td::Status::OK();
}
td::Status execute_set_priority_all(td::Bits256 hash, td::uint8 priority) {
auto query = create_tl_object<ton_api::storage_daemon_setFilePriorityAll>(hash, priority);
send_query(std::move(query),
@ -1229,6 +1289,7 @@ class StorageDaemonCli : public td::actor::Actor {
}
if (json) {
print_json(R.ok());
td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK());
return;
}
auto params = R.move_as_ok();
@ -1294,6 +1355,7 @@ class StorageDaemonCli : public td::actor::Actor {
}
if (json) {
print_json(R.ok());
td::actor::send_closure(SelfId, &StorageDaemonCli::command_finished, td::Status::OK());
return;
}
auto info = R.move_as_ok();
@ -1601,7 +1663,12 @@ class StorageDaemonCli : public td::actor::Actor {
} else {
td::TerminalIO::out() << "Download paused\n";
}
td::TerminalIO::out() << "Upload speed: " << td::format::as_size((td::uint64)obj.torrent_->upload_speed_) << "/s\n";
if (obj.torrent_->active_upload_) {
td::TerminalIO::out() << "Upload speed: " << td::format::as_size((td::uint64)obj.torrent_->upload_speed_)
<< "/s\n";
} else {
td::TerminalIO::out() << "Upload paused\n";
}
td::TerminalIO::out() << "Root dir: " << obj.torrent_->root_dir_ << "\n";
if (obj.torrent_->flags_ & 2) { // header ready
td::TerminalIO::out() << obj.files_.size() << " files:\n";
@ -1616,7 +1683,7 @@ class StorageDaemonCli : public td::actor::Actor {
}
snprintf(str, sizeof(str), "%6u: (%s) %7s/%-7s %s ", i, priority,
f->priority_ == 0 ? "---" : size_to_str(f->downloaded_size_).c_str(), size_to_str(f->size_).c_str(),
(f->downloaded_size_ == f->size_ ? "+" : " "));
((f->downloaded_size_ == f->size_ && f->priority_ > 0) ? "+" : " "));
td::TerminalIO::out() << str << f->name_ << "\n";
++i;
}
@ -1639,7 +1706,7 @@ class StorageDaemonCli : public td::actor::Actor {
});
td::TerminalIO::out() << obj.torrents_.size() << " bags\n";
std::vector<std::vector<std::string>> table;
table.push_back({"#####", "BagID", "Description", "Downloaded", "Total", "Speed"});
table.push_back({"#####", "BagID", "Description", "Downloaded", "Total", "Download", "Upload"});
for (const auto& torrent : obj.torrents_) {
std::vector<std::string> row;
row.push_back(std::to_string(hash_to_id_[torrent->hash_]));
@ -1683,18 +1750,21 @@ class StorageDaemonCli : public td::actor::Actor {
row.push_back(downloaded_size.append("/").append(included_size));
row.push_back(total_size);
row.push_back(status);
row.push_back(torrent->active_upload_ ? size_to_str((td::uint64)torrent->upload_speed_) + "/s" : "Paused");
table.push_back(std::move(row));
}
print_table(table, {2});
}
};
} // namespace ton
int main(int argc, char* argv[]) {
SET_VERBOSITY_LEVEL(verbosity_INFO);
td::set_default_failure_signal_handler();
td::IPAddress ip_addr;
PrivateKey client_private_key;
PublicKey server_public_key;
ton::PrivateKey client_private_key;
ton::PublicKey server_public_key;
std::vector<std::string> commands;
td::OptionParser p;
p.set_description("command-line interface for storage-daemon");
@ -1719,12 +1789,12 @@ int main(int argc, char* argv[]) {
p.add_option('c', "cmd", "execute command", [&](td::Slice arg) { commands.push_back(arg.str()); });
p.add_checked_option('k', "key", "private key", [&](td::Slice arg) {
TRY_RESULT_PREFIX(data, td::read_file(arg.str()), "failed to read: ");
TRY_RESULT_ASSIGN(client_private_key, PrivateKey::import(data));
TRY_RESULT_ASSIGN(client_private_key, ton::PrivateKey::import(data));
return td::Status::OK();
});
p.add_checked_option('p', "pub", "server public key", [&](td::Slice arg) {
TRY_RESULT_PREFIX(data, td::read_file(arg.str()), "failed to read: ");
TRY_RESULT_ASSIGN(server_public_key, PublicKey::import(data));
TRY_RESULT_ASSIGN(server_public_key, ton::PublicKey::import(data));
return td::Status::OK();
});
@ -1738,7 +1808,7 @@ int main(int argc, char* argv[]) {
td::actor::Scheduler scheduler({0});
scheduler.run_in_context([&] {
td::actor::create_actor<StorageDaemonCli>("console", ip_addr, client_private_key, server_public_key,
td::actor::create_actor<ton::StorageDaemonCli>("console", ip_addr, client_private_key, server_public_key,
std::move(commands))
.release();
});

34
storage/storage-daemon/storage-daemon.cpp
View file

@ -44,7 +44,7 @@
#endif
#include <iostream>
using namespace ton;
namespace ton {
td::BufferSlice create_query_error(td::CSlice message) {
return create_serialize_tl_object<ton_api::storage_daemon_queryError>(message.str());
@ -69,6 +69,7 @@ class StorageDaemon : public td::actor::Actor {
void start_up() override {
CHECK(db_root_ != "");
td::mkdir(db_root_).ensure();
db_root_ = td::realpath(db_root_).move_as_ok();
keyring_ = keyring::Keyring::create(db_root_ + "/keyring");
{
auto S = load_global_config();
@ -285,19 +286,21 @@ class StorageDaemon : public td::actor::Actor {
void run_control_query(ton_api::storage_daemon_createTorrent &query, td::Promise<td::BufferSlice> promise) {
// Run in a separate thread
delay_action(
[promise = std::move(promise), manager = manager_.get(), query = std::move(query)]() mutable {
[promise = std::move(promise), manager = manager_.get(), db_root = db_root_,
query = std::move(query)]() mutable {
Torrent::Creator::Options options;
options.piece_size = 128 * 1024;
options.description = std::move(query.description_);
TRY_RESULT_PROMISE(promise, torrent, Torrent::Creator::create_from_path(std::move(options), query.path_));
td::Bits256 hash = torrent.get_hash();
td::actor::send_closure(manager, &StorageManager::add_torrent, std::move(torrent), false,
td::actor::send_closure(manager, &StorageManager::add_torrent, std::move(torrent), false, query.allow_upload_,
query.copy_inside_,
[manager, hash, promise = std::move(promise)](td::Result<td::Unit> R) mutable {
if (R.is_error()) {
promise.set_error(R.move_as_error());
} else {
get_torrent_info_full_serialized(manager, hash, std::move(promise));
return;
}
get_torrent_info_full_serialized(manager, hash, std::move(promise));
});
},
td::Timestamp::now());
@ -308,6 +311,7 @@ class StorageDaemon : public td::actor::Actor {
bool start_download_now = query.start_download_ && query.priorities_.empty();
td::actor::send_closure(
manager_, &StorageManager::add_torrent_by_hash, hash, std::move(query.root_dir_), start_download_now,
query.allow_upload_,
query_add_torrent_cont(hash, query.start_download_, std::move(query.priorities_), std::move(promise)));
}
@ -317,6 +321,7 @@ class StorageDaemon : public td::actor::Actor {
bool start_download_now = query.start_download_ && query.priorities_.empty();
td::actor::send_closure(
manager_, &StorageManager::add_torrent_by_meta, std::move(meta), std::move(query.root_dir_), start_download_now,
query.allow_upload_,
query_add_torrent_cont(hash, query.start_download_, std::move(query.priorities_), std::move(promise)));
}
@ -362,6 +367,12 @@ class StorageDaemon : public td::actor::Actor {
promise.wrap([](td::Unit &&) { return create_serialize_tl_object<ton_api::storage_daemon_success>(); }));
}
void run_control_query(ton_api::storage_daemon_setActiveUpload &query, td::Promise<td::BufferSlice> promise) {
td::actor::send_closure(
manager_, &StorageManager::set_active_upload, query.hash_, query.active_,
promise.wrap([](td::Unit &&) { return create_serialize_tl_object<ton_api::storage_daemon_success>(); }));
}
void run_control_query(ton_api::storage_daemon_getTorrents &query, td::Promise<td::BufferSlice> promise) {
td::actor::send_closure(
manager_, &StorageManager::get_all_torrents,
@ -644,9 +655,8 @@ class StorageDaemon : public td::actor::Actor {
promise.set_error(td::Status::Error("No storage provider"));
return;
}
td::actor::send_closure(provider_, &StorageProvider::get_params, promise.wrap([](ProviderParams params) {
return serialize_tl_object(params.tl(), true);
}));
td::actor::send_closure(provider_, &StorageProvider::get_params,
promise.wrap([](ProviderParams params) { return serialize_tl_object(params.tl(), true); }));
}
void run_control_query(ton_api::storage_daemon_setProviderParams &query, td::Promise<td::BufferSlice> promise) {
@ -785,6 +795,7 @@ class StorageDaemon : public td::actor::Actor {
auto obj = create_tl_object<ton_api::storage_daemon_torrent>();
fill_torrent_info_short(state.torrent, *obj);
obj->active_download_ = state.active_download;
obj->active_upload_ = state.active_upload;
obj->download_speed_ = state.download_speed;
obj->upload_speed_ = state.upload_speed;
promise.set_result(std::move(obj));
@ -802,6 +813,7 @@ class StorageDaemon : public td::actor::Actor {
auto obj = create_tl_object<ton_api::storage_daemon_torrentFull>();
fill_torrent_info_full(state.torrent, *obj);
obj->torrent_->active_download_ = state.active_download;
obj->torrent_->active_upload_ = state.active_upload;
obj->torrent_->download_speed_ = state.download_speed;
obj->torrent_->upload_speed_ = state.upload_speed;
for (size_t i = 0; i < obj->files_.size(); ++i) {
@ -856,6 +868,8 @@ class StorageDaemon : public td::actor::Actor {
}
};
} // namespace ton
int main(int argc, char *argv[]) {
SET_VERBOSITY_LEVEL(verbosity_WARNING);
td::set_default_failure_signal_handler().ensure();
@ -928,8 +942,8 @@ int main(int argc, char *argv[]) {
scheduler.run_in_context([&] {
p.run(argc, argv).ensure();
td::actor::create_actor<StorageDaemon>("storage-daemon", ip_addr, client_mode, global_config, db_root, control_port,
enable_storage_provider)
td::actor::create_actor<ton::StorageDaemon>("storage-daemon", ip_addr, client_mode, global_config, db_root,
control_port, enable_storage_provider)
.release();
});
while (scheduler.run(1)) {

11
tl/generate/scheme/ton_api.tl
View file

@ -795,7 +795,7 @@ storage.db.key.piecesInDb hash:int256 = storage.db.key.PiecesInDb;
storage.db.key.pieceInDb hash:int256 idx:long = storage.db.key.PieceInDb;
storage.db.torrentList torrents:(vector int256) = storage.db.TorrentList;
storage.db.torrent root_dir:string active_download:Bool = storage.db.TorrentShort;
storage.db.torrent root_dir:string active_download:Bool active_upload:Bool = storage.db.TorrentShort;
storage.db.priorities actions:(vector storage.PriorityAction) = storage.db.Priorities;
storage.db.piecesInDb pieces:(vector long) = storage.db.PiecesInDb;
@ -831,7 +831,7 @@ storage.daemon.torrent
total_size:flags.0?long description:flags.0?string
files_count:flags.1?long included_size:flags.1?long dir_name:flags.1?string
downloaded_size:long
root_dir:string active_download:Bool completed:Bool
root_dir:string active_download:Bool active_upload:Bool completed:Bool
download_speed:double upload_speed:double
fatal_error:flags.2?string
= storage.daemon.Torrent;
@ -866,10 +866,11 @@ storage.daemon.providerAddress address:string = storage.daemon.ProviderAddress;
---functions---
storage.daemon.setVerbosity verbosity:int = storage.daemon.Success;
storage.daemon.createTorrent path:string description:string = storage.daemon.TorrentFull;
storage.daemon.addByHash hash:int256 root_dir:string start_download:Bool priorities:(vector storage.PriorityAction) = storage.daemon.TorrentFull;
storage.daemon.addByMeta meta:bytes root_dir:string start_download:Bool priorities:(vector storage.PriorityAction) = storage.daemon.TorrentFull;
storage.daemon.createTorrent path:string description:string allow_upload:Bool copy_inside:Bool = storage.daemon.TorrentFull;
storage.daemon.addByHash hash:int256 root_dir:string start_download:Bool allow_upload:Bool priorities:(vector storage.PriorityAction) = storage.daemon.TorrentFull;
storage.daemon.addByMeta meta:bytes root_dir:string start_download:Bool allow_upload:Bool priorities:(vector storage.PriorityAction) = storage.daemon.TorrentFull;
storage.daemon.setActiveDownload hash:int256 active:Bool = storage.daemon.Success;
storage.daemon.setActiveUpload hash:int256 active:Bool = storage.daemon.Success;
storage.daemon.getTorrents = storage.daemon.TorrentList;
storage.daemon.getTorrentFull hash:int256 = storage.daemon.TorrentFull;

BIN
tl/generate/scheme/ton_api.tlo
View file

Binary file not shown.

1
tl/generate/scheme/tonlib_api.tl
View file

@ -119,6 +119,7 @@ dns.entryDataText text:string = dns.EntryData;
dns.entryDataNextResolver resolver:AccountAddress = dns.EntryData;
dns.entryDataSmcAddress smc_address:AccountAddress = dns.EntryData;
dns.entryDataAdnlAddress adnl_address:AdnlAddress = dns.EntryData;
dns.entryDataStorageAddress bag_id:int256 = dns.EntryData;
dns.entry name:string category:int256 entry:dns.EntryData = dns.Entry;

BIN
tl/generate/scheme/tonlib_api.tlo
View file

Binary file not shown.

6
tonlib/tonlib/TonlibClient.cpp
View file

@ -2919,6 +2919,9 @@ td::Result<ton::ManualDns::EntryData> to_dns_entry_data(tonlib_api::dns_EntryDat
TRY_RESULT(address, get_adnl_address(adnl_address.adnl_address_->adnl_address_));
return ton::ManualDns::EntryData::adnl_address(std::move(address));
},
[&](tonlib_api::dns_entryDataStorageAddress& storage_address) -> R {
return ton::ManualDns::EntryData::storage_address(storage_address.bag_id_);
},
[&](tonlib_api::dns_entryDataText& text) -> R { return ton::ManualDns::EntryData::text(text.text_); }));
}
@ -4108,6 +4111,9 @@ td::Result<tonlib_api::object_ptr<tonlib_api::dns_EntryData>> to_tonlib_api(
[&](const ton::ManualDns::EntryDataSmcAddress& smc_address) {
res = tonlib_api::make_object<tonlib_api::dns_entryDataSmcAddress>(
tonlib_api::make_object<tonlib_api::accountAddress>(smc_address.smc_address.rserialize(true)));
},
[&](const ton::ManualDns::EntryDataStorageAddress& storage_address) {
res = tonlib_api::make_object<tonlib_api::dns_entryDataStorageAddress>(storage_address.bag_id);
}));
return res;
}

2
tonlib/tonlib/tonlib-cli.cpp
View file

@ -317,7 +317,7 @@ class TonlibCli : public td::actor::Actor {
td::TerminalIO::out() << "dns cmdfile <key_id> <file>\n";
td::TerminalIO::out() << "\t<dns_cmd> = set <name> <category> <data> | delete.name <name> | delete.all\n";
td::TerminalIO::out() << "\t<data> = DELETED | EMPTY | TEXT:<text> | NEXT:<smc-address> | SMC:<smc-address> | "
"ADNL:<adnl-address>\n";
"ADNL:<adnl-address> | STORAGE:<bag-id>\n";
}
void pchan_help() {

2
validator/impl/collator-impl.h
View file

@ -113,7 +113,7 @@ class Collator final : public td::actor::Actor {
block::ActionPhaseConfig* action_phase_cfg,
td::RefInt256* masterchain_create_fee,
td::RefInt256* basechain_create_fee,
WorkchainId wc);
WorkchainId wc, UnixTime now);
static td::Result<std::unique_ptr<block::Transaction>>
impl_create_ordinary_transaction(Ref<vm::Cell> msg_root,

5
validator/impl/collator.cpp
View file

@ -1596,7 +1596,7 @@ bool Collator::init_lt() {
bool Collator::fetch_config_params() {
auto res = impl_fetch_config_params(std::move(config_), &old_mparams_, &storage_prices_, &storage_phase_cfg_,
&rand_seed_, &compute_phase_cfg_, &action_phase_cfg_, &masterchain_create_fee_,
&basechain_create_fee_, workchain());
&basechain_create_fee_, workchain(), now_);
if (res.is_error()) {
return fatal_error(res.move_as_error());
}
@ -1609,7 +1609,7 @@ td::Result<std::unique_ptr<block::ConfigInfo>> Collator::impl_fetch_config_param
std::vector<block::StoragePrices>* storage_prices, block::StoragePhaseConfig* storage_phase_cfg,
td::BitArray<256>* rand_seed, block::ComputePhaseConfig* compute_phase_cfg,
block::ActionPhaseConfig* action_phase_cfg, td::RefInt256* masterchain_create_fee,
td::RefInt256* basechain_create_fee, WorkchainId wc) {
td::RefInt256* basechain_create_fee, WorkchainId wc, UnixTime now) {
*old_mparams = config->get_config_param(9);
{
auto res = config->get_storage_prices();
@ -1638,6 +1638,7 @@ td::Result<std::unique_ptr<block::ConfigInfo>> Collator::impl_fetch_config_param
compute_phase_cfg->libraries = std::make_unique<vm::Dictionary>(config->get_libraries_root(), 256);
compute_phase_cfg->max_vm_data_depth = size_limits.max_vm_data_depth;
compute_phase_cfg->global_config = config->get_root_cell();
compute_phase_cfg->suspended_addresses = config->get_suspended_addresses(now);
}
{
// compute action_phase_cfg

2
validator/impl/external-message.cpp
View file

@ -147,7 +147,7 @@ td::Status ExtMessageQ::run_message_on_account(ton::WorkchainId wc,
&storage_prices_, &storage_phase_cfg_,
&rand_seed_, &compute_phase_cfg_,
&action_phase_cfg_, &masterchain_create_fee,
&basechain_create_fee, wc);
&basechain_create_fee, wc, utime);
if(fetch_res.is_error()) {
auto error = fetch_res.move_as_error();
LOG(DEBUG) << "Cannot fetch config params: " << error.message();

2
validator/impl/liteserver.cpp
View file

@ -424,7 +424,7 @@ void LiteQuery::perform_getState(BlockIdExt blkid) {
fatal_error("invalid BlockIdExt");
return;
}
if (blkid.is_masterchain() && blkid.id.seqno > 1000) {
if (blkid.id.seqno > 1000) {
fatal_error("cannot request total state: possibly too large");
return;
}

1
validator/impl/validate-query.cpp
View file

@ -789,6 +789,7 @@ bool ValidateQuery::fetch_config_params() {
compute_phase_cfg_.libraries = std::make_unique<vm::Dictionary>(config_->get_libraries_root(), 256);
compute_phase_cfg_.max_vm_data_depth = size_limits.max_vm_data_depth;
compute_phase_cfg_.global_config = config_->get_root_cell();
compute_phase_cfg_.suspended_addresses = config_->get_suspended_addresses(now_);
}
{
// compute action_phase_cfg