external/ton
1
0
Fork 0
mirror of https://github.com/ton-blockchain/ton synced 2025-03-09 15:40:10 +00:00
Code Issues Releases Wiki Activity

Merge pull request #701 from ton-blockchain/testnet

Browse source 
Merge developer branch
This commit is contained in:
EmelyanenkoK 2023-05-16 13:31:17 +03:00 • committed by GitHub
commit 3bc81bd0de
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
57 changed files with 2313 additions and 1722 deletions
Download patch file Download diff file Expand all files Collapse all files

177
.github/workflows/create-release.yml vendored
View file

@ -27,21 +27,21 @@ jobs:
workflow_conclusion: success
skip_unpack: false
# - name: Download Linux arm64 artifacts
# uses: dawidd6/action-download-artifact@v2
# with:
# workflow: ton-aarch64-linux.yml
# path: artifacts
# workflow_conclusion: success
# skip_unpack: true
#
# - name: Download and unzip Linux arm64 artifacts
# uses: dawidd6/action-download-artifact@v2
# with:
# workflow: ton-aarch64-linux.yml
# path: artifacts
# workflow_conclusion: success
# skip_unpack: false
# - name: Download Linux arm64 artifacts
# uses: dawidd6/action-download-artifact@v2
# with:
# workflow: ton-aarch64-linux.yml
# path: artifacts
# workflow_conclusion: success
# skip_unpack: true
#
# - name: Download and unzip Linux arm64 artifacts
# uses: dawidd6/action-download-artifact@v2
# with:
# workflow: ton-aarch64-linux.yml
# path: artifacts
# workflow_conclusion: success
# skip_unpack: false
- name: Download Mac x86-64 artifacts
uses: dawidd6/action-download-artifact@v2
@ -59,21 +59,21 @@ jobs:
workflow_conclusion: success
skip_unpack: false
# - name: Download Mac arm64 artifacts
# uses: dawidd6/action-download-artifact@v2
# with:
# workflow: ton-aarch64-macos.yml
# path: artifacts
# workflow_conclusion: success
# skip_unpack: true
#
# - name: Download and unzip Mac arm64 artifacts
# uses: dawidd6/action-download-artifact@v2
# with:
# workflow: ton-aarch64-macos.yml
# path: artifacts
# workflow_conclusion: success
# skip_unpack: false
# - name: Download Mac arm64 artifacts
# uses: dawidd6/action-download-artifact@v2
# with:
# workflow: ton-aarch64-macos.yml
# path: artifacts
# workflow_conclusion: success
# skip_unpack: true
#
# - name: Download and unzip Mac arm64 artifacts
# uses: dawidd6/action-download-artifact@v2
# with:
# workflow: ton-aarch64-macos.yml
# path: artifacts
# workflow_conclusion: success
# skip_unpack: false
- name: Download Windows artifacts
uses: dawidd6/action-download-artifact@v2
@ -104,21 +104,30 @@ jobs:
tree artifacts
# create release
# create release
- name: Read Changelog.md and use it as a body of new release
id: read_release
shell: bash
run: |
r=$(cat Changelog.md)
r=$(cat recent_changelog.md)
r="${r//'%'/'%25'}"
r="${r//$'\n'/'%0A'}"
r="${r//$'\r'/'%0D'}"
echo "::set-output name=CHANGELOG_BODY::$r"
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y.%m')"
- name: Get next tag
id: tag
run: |
git fetch --all --tags
git tag -l
NEW_TAG=v$(date +'%Y.%m')
FOUND=$(git tag -l | grep $NEW_TAG | wc -l)
if [ $FOUND -eq 0 ]; then
echo "TAG=$NEW_TAG" >> $GITHUB_OUTPUT
else
echo "TAG=$NEW_TAG-$FOUND" >> $GITHUB_OUTPUT
fi
- name: Get registration token
id: getRegToken
@ -131,16 +140,16 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: v${{ steps.date.outputs.date }}
release_name: v${{ steps.date.outputs.date }}
tag_name: ${{ steps.tag.outputs.TAG }}
release_name: TON ${{ steps.tag.outputs.TAG }}
body: |
${{ steps.read_release.outputs.CHANGELOG_BODY }}
draft: false
prerelease: false
# upload
# upload
# win
# win
- name: Upload Windows 2019 artifacts
uses: svenstaro/upload-release-action@v2
@ -148,7 +157,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-win-binaries.zip
asset_name: ton-win-x86-64.zip
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Windows 2019 single artifact - fift
uses: svenstaro/upload-release-action@v2
@ -156,7 +165,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-win-binaries/fift.exe
asset_name: fift.exe
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Windows 2019 single artifact - func
uses: svenstaro/upload-release-action@v2
@ -164,7 +173,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-win-binaries/func.exe
asset_name: func.exe
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Windows 2019 single artifact - lite-client
uses: svenstaro/upload-release-action@v2
@ -172,7 +181,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-win-binaries/lite-client.exe
asset_name: lite-client.exe
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Windows 2019 single artifact - rldp-http-proxy
uses: svenstaro/upload-release-action@v2
@ -180,7 +189,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-win-binaries/rldp-http-proxy.exe
asset_name: rldp-http-proxy.exe
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Windows 2019 single artifact - http-proxy
uses: svenstaro/upload-release-action@v2
@ -188,7 +197,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-win-binaries/http-proxy.exe
asset_name: http-proxy.exe
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Windows 2019 single artifact - storage-daemon-cli
uses: svenstaro/upload-release-action@v2
@ -196,7 +205,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-win-binaries/storage-daemon-cli.exe
asset_name: storage-daemon-cli.exe
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Windows 2019 single artifact - tonlibjson
uses: svenstaro/upload-release-action@v2
@ -204,7 +213,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-win-binaries/tonlibjson.dll
asset_name: tonlibjson.dll
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Windows 2019 single artifact - tonlib-cli
uses: svenstaro/upload-release-action@v2
@ -212,9 +221,9 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-win-binaries/tonlib-cli.exe
asset_name: tonlib-cli.exe
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
# mac x86-64
# mac x86-64
- name: Upload Mac x86-64 artifacts
uses: svenstaro/upload-release-action@v2
@ -222,7 +231,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-x86_64-macos-binaries.zip
asset_name: ton-mac-x86-64.zip
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Mac x86-64 single artifact - fift
uses: svenstaro/upload-release-action@v2
@ -230,7 +239,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-x86_64-macos-binaries/fift
asset_name: fift-mac-x86-64
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Mac x86-64 single artifact - func
uses: svenstaro/upload-release-action@v2
@ -238,7 +247,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-x86_64-macos-binaries/func
asset_name: func-mac-x86-64
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Mac x86-64 single artifact - lite-client
uses: svenstaro/upload-release-action@v2
@ -246,7 +255,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-x86_64-macos-binaries/lite-client
asset_name: lite-client-mac-x86-64
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Mac x86-64 single artifact - rldp-http-proxy
uses: svenstaro/upload-release-action@v2
@ -254,7 +263,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-x86_64-macos-binaries/rldp-http-proxy
asset_name: rldp-http-proxy-mac-x86-64
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Mac x86-64 single artifact - http-proxy
uses: svenstaro/upload-release-action@v2
@ -262,7 +271,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-x86_64-macos-binaries/http-proxy
asset_name: http-proxy-mac-x86-64
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Mac x86-64 single artifact - storage-daemon-cli
uses: svenstaro/upload-release-action@v2
@ -270,7 +279,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-x86_64-macos-binaries/storage-daemon-cli
asset_name: storage-daemon-cli-mac-x86-64
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Mac x86-64 single artifact - tonlibjson
uses: svenstaro/upload-release-action@v2
@ -278,7 +287,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-x86_64-macos-binaries/libtonlibjson.dylib
asset_name: tonlibjson-mac-x86-64.dylib
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Mac x86-64 single artifact - tonlib-cli
uses: svenstaro/upload-release-action@v2
@ -286,9 +295,9 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-x86_64-macos-binaries/tonlib-cli
asset_name: tonlib-cli-mac-x86-64
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
# linux x86-64
# linux x86-64
- name: Upload Linux x86-64 artifacts
uses: svenstaro/upload-release-action@v2
@ -296,7 +305,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-x86_64-linux-binaries.zip
asset_name: ton-linux-x86_64.zip
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Linux x86-64 single artifact - fift
uses: svenstaro/upload-release-action@v2
@ -304,7 +313,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-x86_64-linux-binaries/fift
asset_name: fift-linux-x86_64
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Linux x86-64 single artifact - func
uses: svenstaro/upload-release-action@v2
@ -312,7 +321,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-x86_64-linux-binaries/func
asset_name: func-linux-x86_64
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Linux x86-64 single artifact - lite-client
uses: svenstaro/upload-release-action@v2
@ -320,7 +329,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-x86_64-linux-binaries/lite-client
asset_name: lite-client-linux-x86_64
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Linux x86-64 single artifact - rldp-http-proxy
uses: svenstaro/upload-release-action@v2
@ -328,7 +337,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-x86_64-linux-binaries/rldp-http-proxy
asset_name: rldp-http-proxy-linux-x86_64
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Linux x86-64 single artifact - http-proxy
uses: svenstaro/upload-release-action@v2
@ -336,7 +345,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-x86_64-linux-binaries/http-proxy
asset_name: http-proxy-linux-x86_64
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Linux x86-64 single artifact - storage-daemon-cli
uses: svenstaro/upload-release-action@v2
@ -344,7 +353,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-x86_64-linux-binaries/storage-daemon-cli
asset_name: storage-daemon-cli-linux-x86_64
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Linux x86-64 single artifact - tonlibjson
uses: svenstaro/upload-release-action@v2
@ -352,7 +361,7 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-x86_64-linux-binaries/libtonlibjson.so.0.5
asset_name: tonlibjson-linux-x86_64.so
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
- name: Upload Linux x86-64 single artifact - tonlib-cli
uses: svenstaro/upload-release-action@v2
@ -360,23 +369,23 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-x86_64-linux-binaries/tonlib-cli
asset_name: tonlib-cli-linux-x86_64
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}
# - name: Upload Linux arm64 artifacts
# uses: svenstaro/upload-release-action@v2
# with:
# repo_token: ${{ secrets.GITHUB_TOKEN }}
# file: artifacts/ton-aarch64-linux-binaries.zip
# asset_name: ton-linux-arm64.zip
# tag: v${{ steps.date.outputs.date }}
#
# - name: Upload Mac arm64 artifacts
# uses: svenstaro/upload-release-action@v2
# with:
# repo_token: ${{ secrets.GITHUB_TOKEN }}
# file: artifacts/ton-aarch64-macos-binaries
# asset_name: ton-mac-arm64.zip
# tag: v${{ steps.date.outputs.date }}
# - name: Upload Linux arm64 artifacts
# uses: svenstaro/upload-release-action@v2
# with:
# repo_token: ${{ secrets.GITHUB_TOKEN }}
# file: artifacts/ton-aarch64-linux-binaries.zip
# asset_name: ton-linux-arm64.zip
# tag: ${{ steps.tag.outputs.TAG }}
#
# - name: Upload Mac arm64 artifacts
# uses: svenstaro/upload-release-action@v2
# with:
# repo_token: ${{ secrets.GITHUB_TOKEN }}
# file: artifacts/ton-aarch64-macos-binaries
# asset_name: ton-mac-arm64.zip
# tag: ${{ steps.tag.outputs.TAG }}
- name: Upload WASM artifacts
uses: svenstaro/upload-release-action@v2
@ -384,4 +393,4 @@ jobs:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file: artifacts/ton-wasm-binaries.zip
asset_name: ton-wasm-binaries.zip
tag: v${{ steps.date.outputs.date }}
tag: ${{ steps.tag.outputs.TAG }}

12
.github/workflows/ubuntu-22.04-compile.yml vendored
View file

@ -27,9 +27,19 @@ jobs:
export CC=$(which clang)
export CXX=$(which clang++)
export CCACHE_DISABLE=1
git clone https://github.com/openssl/openssl openssl_1_1_1
cd openssl_1_1_1
git checkout OpenSSL_1_1_1-stable
./config
make build_libs -j4
cd ..
rootPath=`pwd`
mkdir build
cd build
cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$rootPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$rootPath/openssl_1_1_1/libcrypto.a -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state
- name: Find & copy binaries

13
.github/workflows/ubuntu-compile.yml vendored
View file

@ -30,9 +30,20 @@ jobs:
export CC=$(which clang)
export CXX=$(which clang++)
export CCACHE_DISABLE=1
mkdir build-${{ matrix.os }}
cd build-${{ matrix.os }}
cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
git clone https://github.com/openssl/openssl openssl_1_1_1
cd openssl_1_1_1
git checkout OpenSSL_1_1_1-stable
./config
make build_libs -j4
cd ..
buildPath=`pwd`
cmake -GNinja -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=$buildPath/openssl_1_1_1/include -DOPENSSL_CRYPTO_LIBRARY=$buildPath/openssl_1_1_1/libcrypto.a -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DCMAKE_CXX_FLAGS="-mavx2" ..
ninja storage-daemon storage-daemon-cli fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork
- name: Find & copy binaries

7
.github/workflows/win-2019-compile.yml vendored
View file

@ -48,18 +48,13 @@ jobs:
curl -Lo libmicrohttpd-latest-w32-bin.zip https://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-latest-w32-bin.zip
unzip libmicrohttpd-latest-w32-bin.zip
- name: Install pre-compiled Readline Win64
run: |
curl -Lo readline-5.0-1-lib.zip https://github.com/neodiX42/precompiled-openssl-win64/raw/main/readline-5.0-1-lib.zip
unzip readline-5.0-1-lib.zip
- name: Compile
run: |
set root=%cd%
echo %root%
mkdir build
cd build
cmake -DREADLINE_INCLUDE_DIR=%root%\readline-5.0-1-lib\include\readline -DREADLINE_LIBRARY=%root%\readline-5.0-1-lib\lib\readline.lib -DZLIB_FOUND=1 -DMHD_FOUND=1 -DMHD_LIBRARY=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static\libmicrohttpd.lib -DMHD_INCLUDE_DIR=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static -DZLIB_INCLUDE_DIR=%root%\zlib -DZLIB_LIBRARY=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=%root%/openssl-1.1/x64/include -DOPENSSL_CRYPTO_LIBRARY=%root%/openssl-1.1/x64/lib/libcrypto.lib -DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj /W0" ..
cmake -DPORTABLE=1 -DZLIB_FOUND=1 -DMHD_FOUND=1 -DMHD_LIBRARY=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static\libmicrohttpd.lib -DMHD_INCLUDE_DIR=%root%\libmicrohttpd-0.9.75-w32-bin\x86_64\VS2019\Release-static -DZLIB_INCLUDE_DIR=%root%\zlib -DZLIB_LIBRARY=%root%\zlib\contrib\vstudio\vc14\x64\ZlibStatReleaseWithoutAsm\zlibstat.lib -DOPENSSL_FOUND=1 -DOPENSSL_INCLUDE_DIR=%root%/openssl-1.1/x64/include -DOPENSSL_CRYPTO_LIBRARY=%root%/openssl-1.1/x64/lib/libcrypto.lib -DCMAKE_CXX_FLAGS="/DTD_WINDOWS=1 /EHsc /bigobj /W0" ..
cmake --build . --target storage-daemon storage-daemon-cli blockchain-explorer fift func tonlib tonlibjson tonlib-cli validator-engine lite-client pow-miner validator-engine-console generate-random-id json2tlo dht-server http-proxy rldp-http-proxy adnl-proxy create-state create-hardfork --config Release
- name: Show executables

9
Changelog.md
View file

@ -1,3 +1,12 @@
## 2023.05 Update
1. Archive manager optimization
2. A series of catchain (basic consensus protocol) security improvements
3. Update for Fift libraries and FunC: better error-handling, fixes for `catch` stack recovery
4. A series of out message queue handling optimization (already deployed during emergency upgrades between releases)
5. Improvement of binaries portability
Besides the work of the core team, this update is based on the efforts of @aleksej-paschenko (portability improvement), [Disintar team](https://github.com/disintar/) (archive manager optimization) and [sec3-service](https://github.com/sec3-service) security auditors (funC improvements).
## 2023.04 Update
1. CPU load optimization: previous DHT reconnect policy was too aggressive
2. Network throughput improvements: granular control on external message broadcast, optimize celldb GC, adjust state serialization and block downloading timings, rldp2 for states and archives

1
catchain/catchain-receiver-interface.h
View file

@ -52,6 +52,7 @@ class CatChainReceiverInterface : public td::actor::Actor {
td::BufferSlice query, td::uint64 max_answer_size,
td::actor::ActorId<adnl::AdnlSenderInterface> via) = 0;
virtual void send_custom_message_data(const PublicKeyHash &dst, td::BufferSlice query) = 0;
virtual void on_blame_processed(td::uint32 source_id) = 0;
virtual void destroy() = 0;

15
catchain/catchain-receiver-source.cpp
View file

@ -60,7 +60,7 @@ td::Result<std::unique_ptr<CatChainReceiverSource>> CatChainReceiverSource::crea
void CatChainReceiverSourceImpl::blame(td::uint32 fork, CatChainBlockHeight height) {
blame();
if (!blamed_heights_.empty()) {
// if (!blamed_heights_.empty()) {
if (blamed_heights_.size() <= fork) {
blamed_heights_.resize(fork + 1, 0);
}
@ -68,7 +68,7 @@ void CatChainReceiverSourceImpl::blame(td::uint32 fork, CatChainBlockHeight heig
VLOG(CATCHAIN_INFO) << this << ": blamed at " << fork << " " << height;
blamed_heights_[fork] = height;
}
}
// }
}
void CatChainReceiverSourceImpl::blame() {
@ -144,7 +144,7 @@ void CatChainReceiverSourceImpl::on_new_block(CatChainReceivedBlock *block) {
on_found_fork_proof(create_serialize_tl_object<ton_api::catchain_block_data_fork>(block->export_tl_dep(),
it->second->export_tl_dep())
.as_slice());
chain_->add_prepared_event(fork_proof());
chain_->on_found_fork_proof(id_, fork_proof());
}
blame();
return;
@ -162,6 +162,15 @@ void CatChainReceiverSourceImpl::on_found_fork_proof(const td::Slice &proof) {
}
}
bool CatChainReceiverSourceImpl::allow_send_block(CatChainBlockHash hash) {
td::uint32 count = ++block_requests_count_[hash];
if (count > MAX_BLOCK_REQUESTS) {
VLOG(CATCHAIN_INFO) << this << ": node requested block " << hash << " " << count << " times";
return false;
}
return true;
}
} // namespace catchain
} // namespace ton

3
catchain/catchain-receiver-source.h
View file

@ -61,6 +61,9 @@ class CatChainReceiverSource {
virtual td::BufferSlice fork_proof() const = 0;
virtual bool fork_is_found() const = 0;
// One block can be sent to one node only a limited number of times to prevent DoS
virtual bool allow_send_block(CatChainBlockHash hash) = 0;
static td::Result<std::unique_ptr<CatChainReceiverSource>> create(CatChainReceiver *chain, PublicKey pub_key,
adnl::AdnlNodeIdShort adnl_id, td::uint32 id);

7
catchain/catchain-receiver-source.hpp
View file

@ -111,6 +111,8 @@ class CatChainReceiverSourceImpl : public CatChainReceiverSource {
return chain_;
}
bool allow_send_block(CatChainBlockHash hash) override;
CatChainReceiverSourceImpl(CatChainReceiver *chain, PublicKey source, adnl::AdnlNodeIdShort adnl_id, td::uint32 id);
private:
@ -130,6 +132,11 @@ class CatChainReceiverSourceImpl : public CatChainReceiverSource {
CatChainBlockHeight delivered_height_ = 0;
CatChainBlockHeight received_height_ = 0;
std::map<CatChainBlockHash, td::uint32> block_requests_count_;
// One block can be sent to one node up to 5 times
static const td::uint32 MAX_BLOCK_REQUESTS = 5;
};
} // namespace catchain

124
catchain/catchain-receiver.cpp
View file

@ -168,13 +168,6 @@ void CatChainReceiverImpl::receive_message_from_overlay(adnl::AdnlNodeIdShort sr
return;
}
/*auto S = get_source_by_hash(src);
CHECK(S != nullptr);
if (S->blamed()) {
VLOG(CATCHAIN_INFO) << this << ": dropping block update from blamed source " << src;
return;
}*/
if (data.size() > opts_.max_serialized_block_size) {
VLOG(CATCHAIN_WARNING) << this << ": dropping broken block from " << src << ": too big (size="
<< data.size() << ", limit=" << opts_.max_serialized_block_size << ")";
@ -197,19 +190,6 @@ void CatChainReceiverImpl::receive_broadcast_from_overlay(const PublicKeyHash &s
callback_->on_broadcast(src, std::move(data));
}
/*void CatChainReceiverImpl::send_block(const PublicKeyHash &src, tl_object_ptr<ton_api::catchain_block> block,
td::BufferSlice payload) {
CHECK(read_db_);
CHECK(src == local_id_);
validate_block_sync(block, payload.as_slice()).ensure();
auto B = create_block(std::move(block), td::SharedSlice{payload.as_slice()});
CHECK(B != nullptr);
run_scheduler();
CHECK(B->delivered());
}*/
CatChainReceivedBlock *CatChainReceiverImpl::create_block(tl_object_ptr<ton_api::catchain_block> block,
td::SharedSlice payload) {
if (block->height_ == 0) {
@ -267,10 +247,8 @@ td::Status CatChainReceiverImpl::validate_block_sync(const tl_object_ptr<ton_api
td::Status CatChainReceiverImpl::validate_block_sync(const tl_object_ptr<ton_api::catchain_block> &block,
const td::Slice &payload) const {
//LOG(INFO) << ton_api::to_string(block);
TRY_STATUS_PREFIX(CatChainReceivedBlock::pre_validate_block(this, block, payload), "failed to validate block: ");
if (block->height_ > 0) {
// After pre_validate_block, block->height_ > 0
auto id = CatChainReceivedBlock::block_id(this, block, payload);
td::BufferSlice B = serialize_tl_object(id, true);
@ -279,9 +257,6 @@ td::Status CatChainReceiverImpl::validate_block_sync(const tl_object_ptr<ton_api
Encryptor *E = S->get_encryptor_sync();
CHECK(E != nullptr);
return E->check_signature(B.as_slice(), block->signature_.as_slice());
} else {
return td::Status::OK();
}
}
void CatChainReceiverImpl::run_scheduler() {
@ -515,7 +490,6 @@ CatChainReceiverImpl::CatChainReceiverImpl(std::unique_ptr<Callback> callback,
}
CHECK(local_idx_ != static_cast<td::uint32>(ids.size()));
//std::sort(short_ids.begin(), short_ids.end());
auto F = create_tl_object<ton_api::catchain_firstblock>(unique_hash, std::move(short_ids));
overlay_full_id_ = overlay::OverlayIdFull{serialize_tl_object(F, true)};
@ -527,6 +501,8 @@ CatChainReceiverImpl::CatChainReceiverImpl(std::unique_ptr<Callback> callback,
blocks_[root_block_->get_hash()] = std::move(R);
last_sent_block_ = root_block_;
blame_processed_.resize(sources_.size(), false);
choose_neighbours();
}
@ -704,7 +680,6 @@ void CatChainReceiverImpl::receive_query_from_overlay(adnl::AdnlNodeIdShort src,
auto F = fetch_tl_object<ton_api::Function>(data.clone(), true);
if (F.is_error()) {
callback_->on_custom_query(get_source_by_adnl_id(src)->get_hash(), std::move(data), std::move(promise));
//LOG(WARNING) << this << ": unknown query from " << src;
return;
}
auto f = F.move_as_ok();
@ -717,68 +692,15 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat
if (it == blocks_.end() || it->second->get_height() == 0 || !it->second->initialized()) {
promise.set_value(serialize_tl_object(create_tl_object<ton_api::catchain_blockNotFound>(), true));
} else {
CatChainReceiverSource *S = get_source_by_adnl_id(src);
CHECK(S != nullptr);
if (S->allow_send_block(it->second->get_hash())) {
promise.set_value(serialize_tl_object(create_tl_object<ton_api::catchain_blockResult>(it->second->export_tl()),
true, it->second->get_payload().as_slice()));
}
}
void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlocks query,
td::Promise<td::BufferSlice> promise) {
if (query.blocks_.size() > MAX_QUERY_BLOCKS) {
promise.set_error(td::Status::Error(ErrorCode::protoviolation, "too many blocks"));
return;
}
td::int32 cnt = 0;
for (const CatChainBlockHash &b : query.blocks_) {
auto it = blocks_.find(b);
if (it != blocks_.end() && it->second->get_height() > 0) {
auto block = create_tl_object<ton_api::catchain_blockUpdate>(it->second->export_tl());
CHECK(!it->second->get_payload().empty());
td::BufferSlice B = serialize_tl_object(block, true, it->second->get_payload().clone());
CHECK(B.size() <= opts_.max_serialized_block_size);
td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_message, src,
get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(B));
cnt++;
} else {
promise.set_error(td::Status::Error("block was requested too many times"));
}
}
promise.set_value(serialize_tl_object(create_tl_object<ton_api::catchain_sent>(cnt), true));
}
void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlockHistory query,
td::Promise<td::BufferSlice> promise) {
int64_t h = query.height_;
if (h <= 0) {
promise.set_error(td::Status::Error(ErrorCode::protoviolation, "not-positive height"));
return;
}
if (h > MAX_QUERY_HEIGHT) {
h = MAX_QUERY_HEIGHT;
}
std::set<CatChainBlockHash> s{query.stop_if_.begin(), query.stop_if_.end()};
CatChainReceivedBlock *B = get_block(query.block_);
if (B == nullptr) {
promise.set_value(serialize_tl_object(create_tl_object<ton_api::catchain_sent>(0), true));
return;
}
if (static_cast<CatChainBlockHeight>(h) > B->get_height()) {
h = B->get_height();
}
td::uint32 cnt = 0;
while (h-- > 0) {
if (s.find(B->get_hash()) != s.end()) {
break;
}
auto block = create_tl_object<ton_api::catchain_blockUpdate>(B->export_tl());
CHECK(!B->get_payload().empty());
td::BufferSlice BB = serialize_tl_object(block, true, B->get_payload().as_slice());
CHECK(BB.size() <= opts_.max_serialized_block_size);
td::actor::send_closure(overlay_manager_, &overlay::Overlays::send_message, src,
get_source(local_idx_)->get_adnl_id(), overlay_id_, std::move(BB));
B = B->get_prev();
cnt++;
}
promise.set_value(serialize_tl_object(create_tl_object<ton_api::catchain_sent>(cnt), true));
}
void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getDifference query,
@ -832,6 +754,8 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat
}
}
CHECK(right > 0);
CatChainReceiverSource *S0 = get_source_by_adnl_id(src);
CHECK(S0 != nullptr);
for (td::uint32 i = 0; i < get_sources_cnt(); i++) {
if (vt[i] >= 0 && my_vt[i] > vt[i]) {
CatChainReceiverSource *S = get_source(i);
@ -839,6 +763,7 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat
while (t-- > 0) {
CatChainReceivedBlock *M = S->get_block(++vt[i]);
CHECK(M != nullptr);
if (S0->allow_send_block(M->get_hash())) {
auto block = create_tl_object<ton_api::catchain_blockUpdate>(M->export_tl());
CHECK(!M->get_payload().empty());
td::BufferSlice BB = serialize_tl_object(block, true, M->get_payload().as_slice());
@ -848,6 +773,7 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat
}
}
}
}
promise.set_value(serialize_tl_object(create_tl_object<ton_api::catchain_difference>(std::move(vt)), true));
}
@ -1031,10 +957,15 @@ void CatChainReceiverImpl::written_unsafe_root_block(CatChainReceivedBlock *bloc
void CatChainReceiverImpl::alarm() {
alarm_timestamp() = td::Timestamp::never();
if (next_sync_ && next_sync_.is_in_past()) {
if (next_sync_ && next_sync_.is_in_past() && get_sources_cnt() > 1) {
next_sync_ = td::Timestamp::in(td::Random::fast(SYNC_INTERVAL_MIN, SYNC_INTERVAL_MAX));
for (unsigned i = 0; i < SYNC_ITERATIONS; i++) {
CatChainReceiverSource *S = get_source(td::Random::fast(0, static_cast<td::int32>(get_sources_cnt()) - 1));
auto idx = td::Random::fast(1, static_cast<td::int32>(get_sources_cnt()) - 1);
if (idx == static_cast<td::int32>(local_idx_)) {
idx = 0;
}
// idx is a random number in [0, get_sources_cnt-1] not equal to local_idx
CatChainReceiverSource *S = get_source(idx);
CHECK(S != nullptr);
if (!S->blamed()) {
synchronize_with(S);
@ -1117,6 +1048,23 @@ static void destroy_db(const std::string& name, td::uint32 attempt) {
}
}
void CatChainReceiverImpl::on_found_fork_proof(td::uint32 source_id, td::BufferSlice data) {
if (blame_processed_[source_id]) {
add_block(std::move(data), std::vector<CatChainBlockHash>());
} else {
pending_fork_proofs_[source_id] = std::move(data);
}
}
void CatChainReceiverImpl::on_blame_processed(td::uint32 source_id) {
blame_processed_[source_id] = true;
auto it = pending_fork_proofs_.find(source_id);
if (it != pending_fork_proofs_.end()) {
add_block(std::move(it->second), std::vector<CatChainBlockHash>());
pending_fork_proofs_.erase(it);
}
}
void CatChainReceiverImpl::destroy() {
auto name = db_root_ + "/catchainreceiver" + db_suffix_ + td::base64url_encode(as_slice(incarnation_));
delay_action([name]() { destroy_db(name, 0); }, td::Timestamp::in(DESTROY_DB_DELAY));

2
catchain/catchain-receiver.h
View file

@ -56,7 +56,7 @@ class CatChainReceiver : public CatChainReceiverInterface {
virtual void run_block(CatChainReceivedBlock *block) = 0;
virtual void deliver_block(CatChainReceivedBlock *block) = 0;
virtual td::uint32 add_fork() = 0;
virtual void add_prepared_event(td::BufferSlice data) = 0;
virtual void on_found_fork_proof(td::uint32 source_id, td::BufferSlice data) = 0;
virtual void on_blame(td::uint32 source_id) = 0;
virtual const CatChainOptions &opts() const = 0;

17
catchain/catchain-receiver.hpp
View file

@ -39,9 +39,6 @@ class CatChainReceiverImpl final : public CatChainReceiver {
return PrintId{incarnation_, local_id_};
}
void add_prepared_event(td::BufferSlice data) override {
add_block(std::move(data), std::vector<CatChainBlockHash>());
}
CatChainSessionId get_incarnation() const override {
return incarnation_;
}
@ -73,15 +70,10 @@ class CatChainReceiverImpl final : public CatChainReceiver {
void receive_query_from_overlay(adnl::AdnlNodeIdShort src, td::BufferSlice data,
td::Promise<td::BufferSlice> promise);
void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlock query, td::Promise<td::BufferSlice> promise);
void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlocks query,
td::Promise<td::BufferSlice> promise);
void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getBlockHistory query,
td::Promise<td::BufferSlice> promise);
void process_query(adnl::AdnlNodeIdShort src, ton_api::catchain_getDifference query,
td::Promise<td::BufferSlice> promise);
template <class T>
void process_query(adnl::AdnlNodeIdShort src, const T &query, td::Promise<td::BufferSlice> promise) {
//LOG(WARNING) << this << ": unknown query from " << src;
callback_->on_custom_query(get_source_by_adnl_id(src)->get_hash(), serialize_tl_object(&query, true),
std::move(promise));
}
@ -89,7 +81,6 @@ class CatChainReceiverImpl final : public CatChainReceiver {
void receive_block(adnl::AdnlNodeIdShort src, tl_object_ptr<ton_api::catchain_block> block, td::BufferSlice payload);
void receive_block_answer(adnl::AdnlNodeIdShort src, td::BufferSlice);
//void send_block(const PublicKeyHash &src, tl_object_ptr<ton_api::catchain_block> block, td::BufferSlice payload);
CatChainReceivedBlock *create_block(tl_object_ptr<ton_api::catchain_block> block, td::SharedSlice payload) override;
CatChainReceivedBlock *create_block(tl_object_ptr<ton_api::catchain_block_dep> block) override;
@ -117,6 +108,8 @@ class CatChainReceiverImpl final : public CatChainReceiver {
void on_blame(td::uint32 src) override {
callback_->blame(src);
}
void on_found_fork_proof(td::uint32 source_id, td::BufferSlice data) override;
void on_blame_processed(td::uint32 source_id) override;
const CatChainOptions &opts() const override {
return opts_;
}
@ -204,9 +197,6 @@ class CatChainReceiverImpl final : public CatChainReceiver {
std::vector<td::uint32> neighbours_;
//std::queue<tl_object_ptr<ton_api::catchain_block_inner_Data>> events_;
//std::queue<td::BufferSlice> raw_events_;
td::actor::ActorId<keyring::Keyring> keyring_;
td::actor::ActorId<adnl::Adnl> adnl_;
td::actor::ActorId<overlay::Overlays> overlay_manager_;
@ -231,6 +221,9 @@ class CatChainReceiverImpl final : public CatChainReceiver {
bool started_{false};
std::list<CatChainReceivedBlock *> to_run_;
std::vector<bool> blame_processed_;
std::map<td::uint32, td::BufferSlice> pending_fork_proofs_;
};
} // namespace catchain

13
catchain/catchain.cpp
View file

@ -108,11 +108,12 @@ void CatChainImpl::need_new_block(td::Timestamp t) {
if (!receiver_started_) {
return;
}
if (!force_process_) {
if (!force_process_ || !active_process_) {
VLOG(CATCHAIN_INFO) << this << ": forcing creation of new block";
}
if (active_process_) {
force_process_ = true;
if (!active_process_) {
} else {
alarm_timestamp().relax(t);
}
}
@ -197,6 +198,14 @@ void CatChainImpl::on_blame(td::uint32 src_id) {
}
}
}
for (td::uint32 i = 0; i < process_deps_.size(); ++i) {
if (blocks_[process_deps_[i]]->source() == src_id) {
process_deps_[i] = process_deps_.back();
process_deps_.pop_back();
--i;
}
}
td::actor::send_closure(receiver_, &CatChainReceiverInterface::on_blame_processed, src_id);
}
void CatChainImpl::on_custom_query(const PublicKeyHash &src, td::BufferSlice data, td::Promise<td::BufferSlice> promise) {

4
crypto/block/block.tlb
View file

@ -783,8 +783,8 @@ jetton_bridge_params_v0#00 bridge_address:bits256 oracles_address:bits256 oracle
jetton_bridge_params_v1#01 bridge_address:bits256 oracles_address:bits256 oracles:(HashmapE 256 uint256) state_flags:uint8 prices:^JettonBridgePrices external_chain_address:bits256 = JettonBridgeParams;
_ JettonBridgeParams = ConfigParam 79; // ETH->TON token bridge
_ JettonBridgeParams = ConfigParam 80; // BNB->TON token bridge
_ JettonBridgeParams = ConfigParam 81; // Polygon->TON token bridge
_ JettonBridgeParams = ConfigParam 81; // BNB->TON token bridge
_ JettonBridgeParams = ConfigParam 82; // Polygon->TON token bridge
//

9
crypto/fift/lib/Asm.fif
View file

@ -2,7 +2,7 @@ library TVM_Asm
// simple TVM Assembler
namespace Asm
Asm definitions
"0.4.3" constant asm-fif-version
"0.4.4" constant asm-fif-version
variable @atend
variable @was-split
@ -578,7 +578,7 @@ x{CF3F} @Defop BCHKBITREFSQ
x{CF40} @Defop STZEROES
x{CF41} @Defop STONES
x{CF42} @Defop STSAME
{ tuck sbitrefs swap 15 + swap @havebitrefs not
{ tuck sbitrefs swap 22 + swap @havebitrefs not
{ swap PUSHSLICE STSLICER }
{ over sbitrefs 2dup 57 3 2x<=
{ rot x{CFC_} s, swap 2 u, over 6 + 3 >> tuck 3 u, 3 roll s,
@ -1194,6 +1194,11 @@ x{FA01} @Defop LDVARINT16
x{FA02} dup @Defop STGRAMS @Defop STVARUINT16
x{FA03} @Defop STVARINT16
x{FA04} @Defop LDVARUINT32 // (s -- x s')
x{FA05} @Defop LDVARINT32 // (s -- x s')
x{FA06} @Defop STVARUINT32 // (b x -- b')
x{FA07} @Defop STVARINT32 // (b x -- b')
x{FA40} @Defop LDMSGADDR
x{FA41} @Defop LDMSGADDRQ
x{FA42} @Defop PARSEMSGADDR

2
crypto/fift/words.cpp
View file

@ -235,7 +235,7 @@ void interpret_cmp(vm::Stack& stack, const char opt[3]) {
}
void interpret_sgn(vm::Stack& stack, const char opt[3]) {
auto x = stack.pop_int();
auto x = stack.pop_int_finite();
int r = x->sgn();
assert((unsigned)(r + 1) <= 2);
stack.push_smallint(((const signed char*)opt)[r + 1]);

3
crypto/func/abscode.cpp
View file

@ -38,6 +38,9 @@ TmpVar::TmpVar(var_idx_t _idx, int _cls, TypeExpr* _type, SymDef* sym, const Src
if (!_type) {
v_type = TypeExpr::new_hole();
}
if (cls == _Named) {
undefined = true;
}
}
void TmpVar::set_location(const SrcLocation& loc) {

28
crypto/func/analyzer.cpp
View file

@ -42,14 +42,14 @@ int CodeBlob::split_vars(bool strict) {
}
std::vector<TypeExpr*> comp_types;
int k = var.v_type->extract_components(comp_types);
assert(k <= 254 && n <= 0x7fff00);
assert((unsigned)k == comp_types.size());
func_assert(k <= 254 && n <= 0x7fff00);
func_assert((unsigned)k == comp_types.size());
if (k != 1) {
var.coord = ~((n << 8) + k);
for (int i = 0; i < k; i++) {
auto v = create_var(vars[j].cls, comp_types[i], 0, vars[j].where.get());
assert(v == n + i);
assert(vars[v].idx == v);
func_assert(v == n + i);
func_assert(vars[v].idx == v);
vars[v].name = vars[j].name;
vars[v].coord = ((int)j << 8) + i + 1;
}
@ -75,9 +75,9 @@ bool CodeBlob::compute_used_code_vars() {
}
bool CodeBlob::compute_used_code_vars(std::unique_ptr<Op>& ops_ptr, const VarDescrList& var_info, bool edit) const {
assert(ops_ptr);
func_assert(ops_ptr);
if (!ops_ptr->next) {
assert(ops_ptr->cl == Op::_Nop);
func_assert(ops_ptr->cl == Op::_Nop);
return ops_ptr->set_var_info(var_info);
}
return compute_used_code_vars(ops_ptr->next, var_info, edit) | ops_ptr->compute_used_vars(*this, edit);
@ -346,7 +346,7 @@ bool Op::std_compute_used_vars(bool disabled) {
}
bool Op::compute_used_vars(const CodeBlob& code, bool edit) {
assert(next);
func_assert(next);
const VarDescrList& next_var_info = next->var_info;
if (cl == _Nop) {
return set_var_info_except(next_var_info, left);
@ -379,7 +379,7 @@ bool Op::compute_used_vars(const CodeBlob& code, bool edit) {
case _Let: {
// left = right
std::size_t cnt = next_var_info.count_used(left);
assert(left.size() == right.size());
func_assert(left.size() == right.size());
auto l_it = left.cbegin(), r_it = right.cbegin();
VarDescrList new_var_info{next_var_info};
new_var_info -= left;
@ -500,7 +500,7 @@ bool Op::compute_used_vars(const CodeBlob& code, bool edit) {
}
changes = (new_var_info.size() == n);
} while (changes <= edit);
assert(left.size() == 1);
func_assert(left.size() == 1);
bool last = new_var_info.count_used(left) == 0;
new_var_info += left;
if (last) {
@ -655,7 +655,7 @@ bool prune_unreachable(std::unique_ptr<Op>& ops) {
ops = std::move(op.block0);
return false;
}
reach = true;
reach = (op.cl != Op::_Again);
break;
}
case Op::_TryCatch: {
@ -684,7 +684,7 @@ void CodeBlob::prune_unreachable_code() {
void CodeBlob::fwd_analyze() {
VarDescrList values;
assert(ops && ops->cl == Op::_Import);
func_assert(ops && ops->cl == Op::_Import);
for (var_idx_t i : ops->left) {
values += i;
if (vars[i].v_type->is_int()) {
@ -765,7 +765,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) {
break;
case _Let: {
std::vector<VarDescr> old_val;
assert(left.size() == right.size());
func_assert(left.size() == right.size());
for (std::size_t i = 0; i < right.size(); i++) {
const VarDescr* ov = values[right[i]];
if (!ov && verbosity >= 5) {
@ -780,7 +780,7 @@ VarDescrList Op::fwd_analyze(VarDescrList values) {
}
std::cerr << std::endl;
}
// assert(ov);
// func_assert(ov);
if (ov) {
old_val.push_back(*ov);
} else {
@ -894,7 +894,7 @@ bool Op::mark_noreturn() {
return set_noreturn((block0->mark_noreturn() & (block1 && block1->mark_noreturn())) | next->mark_noreturn());
case _Again:
block0->mark_noreturn();
return set_noreturn(false);
return set_noreturn(true);
case _Until:
return set_noreturn(block0->mark_noreturn() | next->mark_noreturn());
case _While:

6
crypto/func/asmops.cpp
View file

@ -166,7 +166,7 @@ AsmOp AsmOp::UnTuple(int a) {
AsmOp AsmOp::IntConst(td::RefInt256 x) {
if (x->signed_fits_bits(8)) {
return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT", x);
return AsmOp::Const(dec_string(x) + " PUSHINT", x);
}
if (!x->is_valid()) {
return AsmOp::Const("PUSHNAN", x);
@ -184,9 +184,9 @@ AsmOp AsmOp::IntConst(td::RefInt256 x) {
return AsmOp::Const(k, "PUSHNEGPOW2", x);
}
if (!x->mod_pow2_short(23)) {
return AsmOp::Const(dec_string(std::move(x)) + " PUSHINTX", x);
return AsmOp::Const(dec_string(x) + " PUSHINTX", x);
}
return AsmOp::Const(dec_string(std::move(x)) + " PUSHINT", x);
return AsmOp::Const(dec_string(x) + " PUSHINT", x);
}
AsmOp AsmOp::BoolConst(bool f) {

27
crypto/func/auto-tests/legacy_tester.js Normal file
View file

@ -0,0 +1,27 @@
const fs = require('fs/promises');
const { compileWasm, compileFile } = require('./wasm_tests_common');
async function main() {
const tests = JSON.parse((await fs.readFile('../legacy_tests.json')).toString('utf-8'))
for (const [filename, hashstr] of tests) {
if (filename.includes('storage-provider')) continue;
const mod = await compileWasm()
const response = await compileFile(mod, filename);
if (response.status !== 'ok') {
console.error(response);
throw new Error('Could not compile ' + filename);
}
if (BigInt('0x' + response.codeHashHex) !== BigInt(hashstr)) {
throw new Error('Compilation result is different for ' + filename);
}
console.log(filename, 'ok');
}
}
main()

1
crypto/func/auto-tests/legacy_tests.json Normal file
View file

@ -0,0 +1 @@
[["elector/elector-code.fc", "115226404411715505328583639896096915745686314074575650766750648324043316883483"], ["config/config-code.fc", "10913070768607625342121305745084703121685937915388357634624451844356456145601"], ["eth-bridge-multisig/multisig-code.fc", "101509909129354488841890823627011033360100627957439967918234053299675481277954"], ["bsc-bridge-collector/votes-collector.fc", "62190447221288642706570413295807615918589884489514159926097051017036969900417"], ["uni-lock-wallet/uni-lockup-wallet.fc", "61959738324779104851267145467044677651344601417998258530238254441977103654381"], ["nft-collection/nft-collection-editable.fc", "45561997735512210616567774035540357815786262097548276229169737015839077731274"], ["dns-collection/nft-collection.fc", "107999822699841936063083742021519765435859194241091312445235370766165379261859"], ["tele-nft-item/nft-item.fc", "69777543125381987786450436977742010705076866061362104025338034583422166453344"], ["storage/storage-contract.fc", "91377830060355733016937375216020277778264560226873154627574229667513068328151"], ["storage/storage-provider.fc", "13618336676213331164384407184540461509022654507176709588621016553953760588122"], ["nominator-pool/pool.fc", "69767057279163099864792356875696330339149706521019810113334238732928422055375"], ["jetton-minter/jetton-minter.fc", "9028309926287301331466371999814928201427184114165428257502393474125007156494"], ["gg-marketplace/nft-marketplace-v2.fc", "92199806964112524639740773542356508485601908152150843819273107618799016205930"], ["jetton-wallet/jetton-wallet.fc", "86251125787443633057458168028617933212663498001665054651523310772884328206542"], ["whales-nominators/nominators.fc", "8941364499854379927692172316865293429893094891593442801401542636695127885153"], ["tact-examples/treasure_Treasure.code.fc", "13962538639825790677138656603323869918938565499584297120566680287245364723897"], ["tact-examples/jetton_SampleJetton.code.fc", "94076762218493729104783735200107713211245710256802265203823917715299139499110"], ["tact-examples/jetton_JettonDefaultWallet.code.fc", "29421313492520031238091587108198906058157443241743283101866538036369069620563"], ["tact-examples/maps_MapTestContract.code.fc", "22556550222249123835909180266811414538971143565993192846012583552876721649744"]]

77
crypto/func/auto-tests/run_tests.js Normal file
View file

@ -0,0 +1,77 @@
const fs = require('fs/promises');
const os = require('os');
const path = require('path');
const { compileWasm, compileFile } = require('./wasm_tests_common');
const { execSync } = require('child_process');
async function main() {
const compiledPath = path.join(os.tmpdir(), 'compiled.fif');
const runnerPath = path.join(os.tmpdir(), 'runner.fif');
const tests = (await fs.readdir('.')).filter(f => f.endsWith('.fc')).sort();
const mathChars = '0x123456789()+-*/<>'.split('')
for (const testFile of tests) {
const mod = await compileWasm()
const result = await compileFile(mod, testFile)
if (result.status !== 'ok') {
console.error(result);
throw new Error('Could not compile ' + filename);
}
const fileLines = (await fs.readFile(testFile)).toString('utf-8').split('\n');
const testCases = [];
for (const line of fileLines) {
const parts = line.split('|').map(c => c.trim());
if (parts.length !== 4 || parts[0] !== 'TESTCASE') continue;
const processedInputs = [];
for (const input of parts[2].split(' ')) {
if (input.includes('x{')) {
processedInputs.push(input);
continue;
}
if (input.length === 0) {
continue
}
const replacedInput = input.split('').filter(c => mathChars.includes(c)).join('').replace('//', '/').replace(/([0-9a-f])($|[^0-9a-fx])/gmi, '$1n$2')
processedInputs.push(eval(replacedInput).toString());
}
testCases.push([parts[1], processedInputs.join(' '), parts[3]]);
}
await fs.writeFile(compiledPath, '"Asm.fif" include\n' + JSON.parse('"' + result.fiftCode + '"'));
await fs.writeFile(runnerPath, `"${compiledPath}" include <s constant code\n${testCases.map(t => `${t[1]} ${t[0]} code 1 runvmx abort"exitcode is not 0" .s cr { drop } depth 1- times`).join('\n')}`)
const fiftResult = execSync(`${process.env.FIFT_EXECUTABLE || 'fift'} -I ${process.env.FIFT_LIBS} /tmp/runner.fif`, {
stdio: ['pipe', 'pipe', 'ignore']
}).toString('utf-8')
const testResults = fiftResult.split('\n').map(s => s.trim()).filter(s => s.length > 0)
if (testResults.length !== testCases.length) {
throw new Error(`Got ${testResults.length} results but there are ${testCases.length} cases`)
}
for (let i = 0; i < testResults.length; i++) {
if (testResults[i] !== testCases[i][2]) {
throw new Error(`Unequal result ${testResults[i]} and case ${testCases[i][2]}`)
}
}
console.log(testFile, 'ok')
}
}
main()

109
crypto/func/auto-tests/tests/try-func.fc Normal file
View file

@ -0,0 +1,109 @@
int foo(int x) method_id(11) {
try {
if (x == 7) {
throw(44);
}
return x;
} catch (_, _) {
return 2;
}
}
int foo_inline(int x) inline method_id(12) {
try {
if (x == 7) {
throw(44);
}
return x;
} catch (_, _) {
return 2;
}
}
int foo_inlineref(int x) inline_ref method_id(13) {
try {
if (x == 7) {
throw(44);
}
return x;
} catch (_, _) {
return 2;
}
}
int test(int x, int y, int z) method_id(1) {
y = foo(y);
return x * 100 + y * 10 + z;
}
int test_inline(int x, int y, int z) method_id(2) {
y = foo_inline(y);
return x * 100 + y * 10 + z;
}
int test_inlineref(int x, int y, int z) method_id(3) {
y = foo_inlineref(y);
return x * 100 + y * 10 + z;
}
int foo_inline_big(
int x1, int x2, int x3, int x4, int x5, int x6, int x7, int x8, int x9, int x10,
int x11, int x12, int x13, int x14, int x15, int x16, int x17, int x18, int x19, int x20
) inline method_id(14) {
try {
if (x1 == 7) {
throw(44);
}
return x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + x10 + x11 + x12 + x13 + x14 + x15 + x16 + x17 + x18 + x19 + x20;
} catch (_, _) {
return 1;
}
}
int test_inline_big(int x, int y, int z) method_id(4) {
y = foo_inline_big(
y, y + 1, y + 2, y + 3, y + 4, y + 5, y + 6, y + 7, y + 8, y + 9,
y + 10, y + 11, y + 12, y + 13, y + 14, y + 15, y + 16, y + 17, y + 18, y + 19);
return x * 1000000 + y * 1000 + z;
}
int foo_big(
int x1, int x2, int x3, int x4, int x5, int x6, int x7, int x8, int x9, int x10,
int x11, int x12, int x13, int x14, int x15, int x16, int x17, int x18, int x19, int x20
) method_id(15) {
try {
if (x1 == 7) {
throw(44);
}
return x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + x10 + x11 + x12 + x13 + x14 + x15 + x16 + x17 + x18 + x19 + x20;
} catch (_, _) {
return 1;
}
}
int test_big(int x, int y, int z) method_id(5) {
y = foo_big(
y, y + 1, y + 2, y + 3, y + 4, y + 5, y + 6, y + 7, y + 8, y + 9,
y + 10, y + 11, y + 12, y + 13, y + 14, y + 15, y + 16, y + 17, y + 18, y + 19);
return x * 1000000 + y * 1000 + z;
}
() main() {
}
{-
method_id | in | out
TESTCASE | 1 | 1 2 3 | 123
TESTCASE | 1 | 3 8 9 | 389
TESTCASE | 1 | 3 7 9 | 329
TESTCASE | 2 | 1 2 3 | 123
TESTCASE | 2 | 3 8 9 | 389
TESTCASE | 2 | 3 7 9 | 329
TESTCASE | 3 | 1 2 3 | 123
TESTCASE | 3 | 3 8 9 | 389
TESTCASE | 3 | 3 7 9 | 329
TESTCASE | 4 | 4 8 9 | 4350009
TESTCASE | 4 | 4 7 9 | 4001009
TESTCASE | 5 | 4 8 9 | 4350009
TESTCASE | 5 | 4 7 9 | 4001009
-}

63
crypto/func/auto-tests/wasm_tests_common.js Normal file
View file

@ -0,0 +1,63 @@
const fsSync = require('fs');
const copyToCString = (mod, str) => {
const len = mod.lengthBytesUTF8(str) + 1;
const ptr = mod._malloc(len);
mod.stringToUTF8(str, ptr, len);
return ptr;
};
const copyToCStringPtr = (mod, str, ptr) => {
const allocated = copyToCString(mod, str);
mod.setValue(ptr, allocated, '*');
return allocated;
};
const copyFromCString = (mod, ptr) => {
return mod.UTF8ToString(ptr);
};
async function compileFile(mod, filename) {
const callbackPtr = mod.addFunction((_kind, _data, contents, error) => {
const kind = copyFromCString(mod, _kind);
const data = copyFromCString(mod, _data);
if (kind === 'realpath') {
copyToCStringPtr(mod, fsSync.realpathSync(data), contents);
} else if (kind === 'source') {
const path = fsSync.realpathSync(data);
try {
copyToCStringPtr(mod, fsSync.readFileSync(path).toString('utf-8'), contents);
} catch (err) {
copyToCStringPtr(mod, e.message, error);
}
} else {
copyToCStringPtr(mod, 'Unknown callback kind ' + kind, error);
}
}, 'viiii');
const config = {
optLevel: 2,
sources: [filename]
};
const configPtr = copyToCString(mod, JSON.stringify(config));
const responsePtr = mod._func_compile(configPtr, callbackPtr);
return JSON.parse(copyFromCString(mod, responsePtr));
}
const wasmModule = require(process.env.FUNCFIFTLIB_MODULE)
const wasmBinary = new Uint8Array(fsSync.readFileSync(process.env.FUNCFIFTLIB_WASM))
async function compileWasm() {
const mod = await wasmModule({ wasmBinary })
return mod
}
module.exports = {
compileFile,
compileWasm
}

54
crypto/func/builtins.cpp
View file

@ -429,7 +429,7 @@ AsmOp push_const(td::RefInt256 x) {
}
AsmOp compile_add(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const SrcLocation& where) {
assert(res.size() == 1 && args.size() == 2);
func_assert(res.size() == 1 && args.size() == 2);
VarDescr &r = res[0], &x = args[0], &y = args[1];
if (x.is_int_const() && y.is_int_const()) {
r.set_const(x.int_const + y.int_const);
@ -471,7 +471,7 @@ AsmOp compile_add(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const
}
AsmOp compile_sub(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const SrcLocation& where) {
assert(res.size() == 1 && args.size() == 2);
func_assert(res.size() == 1 && args.size() == 2);
VarDescr &r = res[0], &x = args[0], &y = args[1];
if (x.is_int_const() && y.is_int_const()) {
r.set_const(x.int_const - y.int_const);
@ -504,7 +504,7 @@ AsmOp compile_sub(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const
}
AsmOp compile_negate(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const SrcLocation& where) {
assert(res.size() == 1 && args.size() == 1);
func_assert(res.size() == 1 && args.size() == 1);
VarDescr &r = res[0], &x = args[0];
if (x.is_int_const()) {
r.set_const(-x.int_const);
@ -519,7 +519,7 @@ AsmOp compile_negate(std::vector<VarDescr>& res, std::vector<VarDescr>& args, co
}
AsmOp compile_and(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const SrcLocation& where) {
assert(res.size() == 1 && args.size() == 2);
func_assert(res.size() == 1 && args.size() == 2);
VarDescr &r = res[0], &x = args[0], &y = args[1];
if (x.is_int_const() && y.is_int_const()) {
r.set_const(x.int_const & y.int_const);
@ -532,7 +532,7 @@ AsmOp compile_and(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const
}
AsmOp compile_or(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const SrcLocation& where) {
assert(res.size() == 1 && args.size() == 2);
func_assert(res.size() == 1 && args.size() == 2);
VarDescr &r = res[0], &x = args[0], &y = args[1];
if (x.is_int_const() && y.is_int_const()) {
r.set_const(x.int_const | y.int_const);
@ -545,7 +545,7 @@ AsmOp compile_or(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const
}
AsmOp compile_xor(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const SrcLocation& where) {
assert(res.size() == 1 && args.size() == 2);
func_assert(res.size() == 1 && args.size() == 2);
VarDescr &r = res[0], &x = args[0], &y = args[1];
if (x.is_int_const() && y.is_int_const()) {
r.set_const(x.int_const ^ y.int_const);
@ -558,7 +558,7 @@ AsmOp compile_xor(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const
}
AsmOp compile_not(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const SrcLocation& where) {
assert(res.size() == 1 && args.size() == 1);
func_assert(res.size() == 1 && args.size() == 1);
VarDescr &r = res[0], &x = args[0];
if (x.is_int_const()) {
r.set_const(~x.int_const);
@ -638,12 +638,12 @@ AsmOp compile_mul_internal(VarDescr& r, VarDescr& x, VarDescr& y, const SrcLocat
}
AsmOp compile_mul(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const SrcLocation& where) {
assert(res.size() == 1 && args.size() == 2);
func_assert(res.size() == 1 && args.size() == 2);
return compile_mul_internal(res[0], args[0], args[1], where);
}
AsmOp compile_lshift(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const SrcLocation& where) {
assert(res.size() == 1 && args.size() == 2);
func_assert(res.size() == 1 && args.size() == 2);
VarDescr &r = res[0], &x = args[0], &y = args[1];
if (y.is_int_const()) {
auto yv = y.int_const->to_long();
@ -686,7 +686,7 @@ AsmOp compile_lshift(std::vector<VarDescr>& res, std::vector<VarDescr>& args, co
AsmOp compile_rshift(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const SrcLocation& where,
int round_mode) {
assert(res.size() == 1 && args.size() == 2);
func_assert(res.size() == 1 && args.size() == 2);
VarDescr &r = res[0], &x = args[0], &y = args[1];
if (y.is_int_const()) {
auto yv = y.int_const->to_long();
@ -755,13 +755,13 @@ AsmOp compile_div_internal(VarDescr& r, VarDescr& x, VarDescr& y, const SrcLocat
}
AsmOp compile_div(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const SrcLocation& where, int round_mode) {
assert(res.size() == 1 && args.size() == 2);
func_assert(res.size() == 1 && args.size() == 2);
return compile_div_internal(res[0], args[0], args[1], where, round_mode);
}
AsmOp compile_mod(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const src::SrcLocation& where,
int round_mode) {
assert(res.size() == 1 && args.size() == 2);
func_assert(res.size() == 1 && args.size() == 2);
VarDescr &r = res[0], &x = args[0], &y = args[1];
if (x.is_int_const() && y.is_int_const()) {
r.set_const(mod(x.int_const, y.int_const, round_mode));
@ -802,7 +802,7 @@ AsmOp compile_mod(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const
AsmOp compile_muldiv(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const SrcLocation& where,
int round_mode) {
assert(res.size() == 1 && args.size() == 3);
func_assert(res.size() == 1 && args.size() == 3);
VarDescr &r = res[0], &x = args[0], &y = args[1], &z = args[2];
if (x.is_int_const() && y.is_int_const() && z.is_int_const()) {
r.set_const(muldiv(x.int_const, y.int_const, z.int_const, round_mode));
@ -923,8 +923,8 @@ int compute_compare(const VarDescr& x, const VarDescr& y, int mode) {
}
AsmOp compile_cmp_int(std::vector<VarDescr>& res, std::vector<VarDescr>& args, int mode) {
assert(mode >= 1 && mode <= 7);
assert(res.size() == 1 && args.size() == 2);
func_assert(mode >= 1 && mode <= 7);
func_assert(res.size() == 1 && args.size() == 2);
VarDescr &r = res[0], &x = args[0], &y = args[1];
if (x.is_int_const() && y.is_int_const()) {
int v = compute_compare(x.int_const, y.int_const, mode);
@ -935,7 +935,7 @@ AsmOp compile_cmp_int(std::vector<VarDescr>& res, std::vector<VarDescr>& args, i
}
int v = compute_compare(x, y, mode);
// std::cerr << "compute_compare(" << x << ", " << y << ", " << mode << ") = " << v << std::endl;
assert(v);
func_assert(v);
if (!(v & (v - 1))) {
r.set_const(v - (v >> 2) - 2);
x.unused();
@ -971,7 +971,7 @@ AsmOp compile_cmp_int(std::vector<VarDescr>& res, std::vector<VarDescr>& args, i
}
AsmOp compile_throw(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const SrcLocation&) {
assert(res.empty() && args.size() == 1);
func_assert(res.empty() && args.size() == 1);
VarDescr& x = args[0];
if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) {
x.unused();
@ -982,7 +982,7 @@ AsmOp compile_throw(std::vector<VarDescr>& res, std::vector<VarDescr>& args, con
}
AsmOp compile_cond_throw(std::vector<VarDescr>& res, std::vector<VarDescr>& args, bool mode) {
assert(res.empty() && args.size() == 2);
func_assert(res.empty() && args.size() == 2);
VarDescr &x = args[0], &y = args[1];
std::string suff = (mode ? "IF" : "IFNOT");
bool skip_cond = false;
@ -1003,7 +1003,7 @@ AsmOp compile_cond_throw(std::vector<VarDescr>& res, std::vector<VarDescr>& args
}
AsmOp compile_throw_arg(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const SrcLocation&) {
assert(res.empty() && args.size() == 2);
func_assert(res.empty() && args.size() == 2);
VarDescr &x = args[1];
if (x.is_int_const() && x.int_const->unsigned_fits_bits(11)) {
x.unused();
@ -1014,7 +1014,7 @@ AsmOp compile_throw_arg(std::vector<VarDescr>& res, std::vector<VarDescr>& args,
}
AsmOp compile_cond_throw_arg(std::vector<VarDescr>& res, std::vector<VarDescr>& args, bool mode) {
assert(res.empty() && args.size() == 3);
func_assert(res.empty() && args.size() == 3);
VarDescr &x = args[1], &y = args[2];
std::string suff = (mode ? "IF" : "IFNOT");
bool skip_cond = false;
@ -1035,7 +1035,7 @@ AsmOp compile_cond_throw_arg(std::vector<VarDescr>& res, std::vector<VarDescr>&
}
AsmOp compile_bool_const(std::vector<VarDescr>& res, std::vector<VarDescr>& args, bool val) {
assert(res.size() == 1 && args.empty());
func_assert(res.size() == 1 && args.empty());
VarDescr& r = res[0];
r.set_const(val ? -1 : 0);
return AsmOp::Const(val ? "TRUE" : "FALSE");
@ -1046,7 +1046,7 @@ AsmOp compile_bool_const(std::vector<VarDescr>& res, std::vector<VarDescr>& args
// int preload_int(slice s, int len) asm "PLDIX";
// int preload_uint(slice s, int len) asm "PLDUX";
AsmOp compile_fetch_int(std::vector<VarDescr>& res, std::vector<VarDescr>& args, bool fetch, bool sgnd) {
assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch);
func_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch);
auto &y = args[1], &r = res.back();
r.val = (sgnd ? VarDescr::FiniteInt : VarDescr::FiniteUInt);
int v = -1;
@ -1069,7 +1069,7 @@ AsmOp compile_fetch_int(std::vector<VarDescr>& res, std::vector<VarDescr>& args,
// builder store_uint(builder b, int x, int len) asm(x b len) "STUX";
// builder store_int(builder b, int x, int len) asm(x b len) "STIX";
AsmOp compile_store_int(std::vector<VarDescr>& res, std::vector<VarDescr>& args, bool sgnd) {
assert(args.size() == 3 && res.size() == 1);
func_assert(args.size() == 3 && res.size() == 1);
auto& z = args[2];
if (z.is_int_const() && z.int_const > 0 && z.int_const <= 256) {
z.unused();
@ -1079,7 +1079,7 @@ AsmOp compile_store_int(std::vector<VarDescr>& res, std::vector<VarDescr>& args,
}
AsmOp compile_fetch_slice(std::vector<VarDescr>& res, std::vector<VarDescr>& args, bool fetch) {
assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch);
func_assert(args.size() == 2 && res.size() == 1 + (unsigned)fetch);
auto& y = args[1];
int v = -1;
if (y.is_int_const() && y.int_const > 0 && y.int_const <= 256) {
@ -1094,7 +1094,7 @@ AsmOp compile_fetch_slice(std::vector<VarDescr>& res, std::vector<VarDescr>& arg
// <type> <type>_at(tuple t, int index) asm "INDEXVAR";
AsmOp compile_tuple_at(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const SrcLocation&) {
assert(args.size() == 2 && res.size() == 1);
func_assert(args.size() == 2 && res.size() == 1);
auto& y = args[1];
if (y.is_int_const() && y.int_const >= 0 && y.int_const < 16) {
y.unused();
@ -1105,7 +1105,7 @@ AsmOp compile_tuple_at(std::vector<VarDescr>& res, std::vector<VarDescr>& args,
// int null?(X arg)
AsmOp compile_is_null(std::vector<VarDescr>& res, std::vector<VarDescr>& args, const SrcLocation&) {
assert(args.size() == 1 && res.size() == 1);
func_assert(args.size() == 1 && res.size() == 1);
auto &x = args[0], &r = res[0];
if (x.always_null() || x.always_not_null()) {
x.unused();
@ -1118,7 +1118,7 @@ AsmOp compile_is_null(std::vector<VarDescr>& res, std::vector<VarDescr>& args, c
bool compile_run_method(AsmOpList& code, std::vector<VarDescr>& res, std::vector<VarDescr>& args, int n,
bool has_value) {
assert(args.size() == (unsigned)n + 1 && res.size() == (unsigned)has_value);
func_assert(args.size() == (unsigned)n + 1 && res.size() == (unsigned)has_value);
auto& x = args[0];
if (x.is_int_const() && x.int_const->unsigned_fits_bits(14)) {
x.unused();

93
crypto/func/codegen.cpp
View file

@ -165,7 +165,7 @@ void Stack::push_new_const(var_idx_t idx, const_idx_t cidx) {
void Stack::assign_var(var_idx_t new_idx, var_idx_t old_idx) {
int i = find(old_idx);
assert(i >= 0 && "variable not found in stack");
func_assert(i >= 0 && "variable not found in stack");
if (new_idx != old_idx) {
at(i).first = new_idx;
modified();
@ -174,10 +174,10 @@ void Stack::assign_var(var_idx_t new_idx, var_idx_t old_idx) {
void Stack::do_copy_var(var_idx_t new_idx, var_idx_t old_idx) {
int i = find(old_idx);
assert(i >= 0 && "variable not found in stack");
func_assert(i >= 0 && "variable not found in stack");
if (find(old_idx, i + 1) < 0) {
issue_push(i);
assert(at(0).first == old_idx);
func_assert(at(0).first == old_idx);
}
assign_var(new_idx, old_idx);
}
@ -199,21 +199,21 @@ void Stack::enforce_state(const StackLayout& req_stack) {
j = 0;
}
issue_xchg(j, depth() - i - 1);
assert(s[i].first == x);
func_assert(s[i].first == x);
}
while (depth() > k) {
issue_pop(0);
}
assert(depth() == k);
func_assert(depth() == k);
for (int i = 0; i < k; i++) {
assert(s[i].first == req_stack[i]);
func_assert(s[i].first == req_stack[i]);
}
}
void Stack::merge_const(const Stack& req_stack) {
assert(s.size() == req_stack.s.size());
func_assert(s.size() == req_stack.s.size());
for (std::size_t i = 0; i < s.size(); i++) {
assert(s[i].first == req_stack.s[i].first);
func_assert(s[i].first == req_stack.s[i].first);
if (s[i].second != req_stack.s[i].second) {
s[i].second = not_const;
}
@ -251,15 +251,15 @@ void Stack::rearrange_top(const StackLayout& top, std::vector<bool> last) {
if (last[i]) {
// rearrange x to be at s(ss-1)
issue_xchg(--ss, j);
assert(get(ss).first == x);
func_assert(get(ss).first == x);
} else {
// create a new copy of x
issue_push(j);
issue_xchg(0, ss);
assert(get(ss).first == x);
func_assert(get(ss).first == x);
}
}
assert(!ss);
func_assert(!ss);
}
void Stack::rearrange_top(var_idx_t top, bool last) {
@ -269,7 +269,7 @@ void Stack::rearrange_top(var_idx_t top, bool last) {
} else {
issue_push(i);
}
assert(get(0).first == top);
func_assert(get(0).first == top);
}
bool Op::generate_code_step(Stack& stack) {
@ -300,7 +300,7 @@ bool Op::generate_code_step(Stack& stack) {
stack.o << push_const(int_const);
stack.push_new_const(left[0], cidx);
} else {
assert(stack.at(i).second == cidx);
func_assert(stack.at(i).second == cidx);
stack.do_copy_var(left[0], stack[i]);
}
return true;
@ -329,7 +329,7 @@ bool Op::generate_code_step(Stack& stack) {
std::string name = sym::symbols.get_name(fun_ref->sym_idx);
stack.o << AsmOp::Custom(name + " GETGLOB", 0, 1);
if (left.size() != 1) {
assert(left.size() <= 15);
func_assert(left.size() <= 15);
stack.o << AsmOp::UnTuple((int)left.size());
}
for (auto i : left) {
@ -337,7 +337,7 @@ bool Op::generate_code_step(Stack& stack) {
}
return true;
} else {
assert(left.size() == 1);
func_assert(left.size() == 1);
auto p = next->var_info[left[0]];
if (!p || p->is_unused() || disabled()) {
return true;
@ -349,10 +349,10 @@ bool Op::generate_code_step(Stack& stack) {
// TODO: create and compile a true lambda instead of this (so that arg_order and ret_order would work correctly)
std::vector<VarDescr> args0, res;
TypeExpr::remove_indirect(func->sym_type);
assert(func->get_type()->is_map());
func_assert(func->get_type()->is_map());
auto wr = func->get_type()->args.at(0)->get_width();
auto wl = func->get_type()->args.at(1)->get_width();
assert(wl >= 0 && wr >= 0);
func_assert(wl >= 0 && wr >= 0);
for (int i = 0; i < wl; i++) {
res.emplace_back(0);
}
@ -370,7 +370,7 @@ bool Op::generate_code_step(Stack& stack) {
return true;
}
case _Let: {
assert(left.size() == right.size());
func_assert(left.size() == right.size());
int i = 0;
std::vector<bool> active;
active.reserve(left.size());
@ -420,13 +420,13 @@ bool Op::generate_code_step(Stack& stack) {
stack.rearrange_top(right, std::move(last));
stack.opt_show();
int k = (int)stack.depth() - (int)right.size();
assert(k >= 0);
func_assert(k >= 0);
if (cl == _Tuple) {
stack.o << AsmOp::Tuple((int)right.size());
assert(left.size() == 1);
func_assert(left.size() == 1);
} else {
stack.o << AsmOp::UnTuple((int)left.size());
assert(right.size() == 1);
func_assert(right.size() == 1);
}
stack.s.resize(k);
for (int i = 0; i < (int)left.size(); i++) {
@ -442,16 +442,16 @@ bool Op::generate_code_step(Stack& stack) {
SymValFunc* func = (fun_ref ? dynamic_cast<SymValFunc*>(fun_ref->value) : nullptr);
auto arg_order = (func ? func->get_arg_order() : nullptr);
auto ret_order = (func ? func->get_ret_order() : nullptr);
assert(!arg_order || arg_order->size() == right.size());
assert(!ret_order || ret_order->size() == left.size());
func_assert(!arg_order || arg_order->size() == right.size());
func_assert(!ret_order || ret_order->size() == left.size());
std::vector<var_idx_t> right1;
if (args.size()) {
assert(args.size() == right.size());
func_assert(args.size() == right.size());
for (int i = 0; i < (int)right.size(); i++) {
int j = arg_order ? arg_order->at(i) : i;
const VarDescr& arg = args.at(j);
if (!arg.is_unused()) {
assert(var_info[arg.idx] && !var_info[arg.idx]->is_unused());
func_assert(var_info[arg.idx] && !var_info[arg.idx]->is_unused());
right1.push_back(arg.idx);
}
}
@ -469,17 +469,25 @@ bool Op::generate_code_step(Stack& stack) {
stack.rearrange_top(right1, std::move(last));
stack.opt_show();
int k = (int)stack.depth() - (int)right1.size();
assert(k >= 0);
func_assert(k >= 0);
for (int i = 0; i < (int)right1.size(); i++) {
if (stack.s[k + i].first != right1[i]) {
std::cerr << stack.o;
}
assert(stack.s[k + i].first == right1[i]);
func_assert(stack.s[k + i].first == right1[i]);
}
auto exec_callxargs = [&](int args, int ret) {
if (args <= 15 && ret <= 15) {
stack.o << exec_arg2_op("CALLXARGS", args, ret, args + 1, ret);
} else {
func_assert(args <= 254 && ret <= 254);
stack.o << AsmOp::Const(PSTRING() << args << " PUSHINT");
stack.o << AsmOp::Const(PSTRING() << ret << " PUSHINT");
stack.o << AsmOp::Custom("CALLXVARARGS", args + 3, ret);
}
};
if (cl == _CallInd) {
// TODO: replace with exec_arg2_op()
stack.o << exec_arg2_op("CALLXARGS", (int)right.size() - 1, (int)left.size(), (int)right.size(),
(int)left.size());
exec_callxargs((int)right.size() - 1, (int)left.size());
} else {
auto func = dynamic_cast<const SymValAsmFunc*>(fun_ref->value);
if (func) {
@ -493,8 +501,14 @@ bool Op::generate_code_step(Stack& stack) {
auto fv = dynamic_cast<const SymValCodeFunc*>(fun_ref->value);
std::string name = sym::symbols.get_name(fun_ref->sym_idx);
bool is_inline = (fv && (fv->flags & 3));
stack.o << AsmOp::Custom(name + (is_inline ? " INLINECALLDICT" : " CALLDICT"), (int)right.size(),
(int)left.size());
if (is_inline) {
stack.o << AsmOp::Custom(name + " INLINECALLDICT", (int)right.size(), (int)left.size());
} else if (fv && fv->code && fv->code->require_callxargs) {
stack.o << AsmOp::Custom(name + (" PREPAREDICT"), 0, 2);
exec_callxargs((int)right.size() + 1, (int)left.size());
} else {
stack.o << AsmOp::Custom(name + " CALLDICT", (int)right.size(), (int)left.size());
}
}
}
stack.s.resize(k);
@ -505,7 +519,7 @@ bool Op::generate_code_step(Stack& stack) {
return true;
}
case _SetGlob: {
assert(fun_ref && dynamic_cast<const SymValGlobVar*>(fun_ref->value));
func_assert(fun_ref && dynamic_cast<const SymValGlobVar*>(fun_ref->value));
std::vector<bool> last;
for (var_idx_t x : right) {
last.push_back(var_info[x] && var_info[x]->is_last());
@ -513,12 +527,12 @@ bool Op::generate_code_step(Stack& stack) {
stack.rearrange_top(right, std::move(last));
stack.opt_show();
int k = (int)stack.depth() - (int)right.size();
assert(k >= 0);
func_assert(k >= 0);
for (int i = 0; i < (int)right.size(); i++) {
if (stack.s[k + i].first != right[i]) {
std::cerr << stack.o;
}
assert(stack.s[k + i].first == right[i]);
func_assert(stack.s[k + i].first == right[i]);
}
if (right.size() > 1) {
stack.o << AsmOp::Tuple((int)right.size());
@ -539,7 +553,7 @@ bool Op::generate_code_step(Stack& stack) {
}
var_idx_t x = left[0];
stack.rearrange_top(x, var_info[x] && var_info[x]->is_last());
assert(stack[0] == x);
func_assert(stack[0] == x);
stack.opt_show();
stack.s.pop_back();
stack.modified();
@ -651,7 +665,7 @@ bool Op::generate_code_step(Stack& stack) {
var_idx_t x = left[0];
//stack.drop_vars_except(block0->var_info, x);
stack.rearrange_top(x, var_info[x] && var_info[x]->is_last());
assert(stack[0] == x);
func_assert(stack[0] == x);
stack.opt_show();
stack.s.pop_back();
stack.modified();
@ -877,12 +891,13 @@ void Op::generate_code_all(Stack& stack) {
void CodeBlob::generate_code(AsmOpList& out, int mode) {
Stack stack{out, mode};
assert(ops && ops->cl == Op::_Import);
func_assert(ops && ops->cl == Op::_Import);
auto args = (int)ops->left.size();
for (var_idx_t x : ops->left) {
stack.push_new_var(x);
}
ops->generate_code_all(stack);
stack.apply_wrappers();
stack.apply_wrappers(require_callxargs && (mode & Stack::_InlineAny) ? args : -1);
if (!(mode & Stack::_DisableOpt)) {
optimize_code(out);
}

9
crypto/func/func.cpp
View file

@ -72,7 +72,7 @@ td::Result<std::string> fs_read_callback(ReadCallback::Kind kind, const char* qu
void generate_output_func(SymDef* func_sym, std::ostream &outs, std::ostream &errs) {
SymValCodeFunc* func_val = dynamic_cast<SymValCodeFunc*>(func_sym->value);
assert(func_val);
func_assert(func_val);
std::string name = sym::symbols.get_name(func_sym->sym_idx);
if (verbosity >= 2) {
errs << "\n\n=========================\nfunction " << name << " : " << func_val->get_type() << std::endl;
@ -145,6 +145,9 @@ void generate_output_func(SymDef* func_sym, std::ostream &outs, std::ostream &er
if (fv && (fv->flags & 1) && code.ops->noreturn()) {
mode |= Stack::_InlineFunc;
}
if (fv && (fv->flags & 3)) {
mode |= Stack::_InlineAny;
}
code.generate_code(outs, mode, indent + 1);
outs << std::string(indent * 2, ' ') << "}>\n";
if (verbosity >= 2) {
@ -163,7 +166,7 @@ int generate_output(std::ostream &outs, std::ostream &errs) {
}
for (SymDef* func_sym : glob_func) {
SymValCodeFunc* func_val = dynamic_cast<SymValCodeFunc*>(func_sym->value);
assert(func_val);
func_assert(func_val);
std::string name = sym::symbols.get_name(func_sym->sym_idx);
outs << std::string(indent * 2, ' ');
if (func_val->method_id.is_null()) {
@ -173,7 +176,7 @@ int generate_output(std::ostream &outs, std::ostream &errs) {
}
}
for (SymDef* gvar_sym : glob_vars) {
assert(dynamic_cast<SymValGlobVar*>(gvar_sym->value));
func_assert(dynamic_cast<SymValGlobVar*>(gvar_sym->value));
std::string name = sym::symbols.get_name(gvar_sym->sym_idx);
outs << std::string(indent * 2, ' ') << "DECLGLOBVAR " << name << "\n";
}

33
crypto/func/func.h
View file

@ -33,6 +33,10 @@
#include "parser/symtable.h"
#include "td/utils/Status.h"
#define func_assert(expr) \
(bool(expr) ? void(0) \
: throw src::Fatal(PSTRING() << "Assertion failed at " << __FILE__ << ":" << __LINE__ << ": " << #expr))
namespace funC {
extern int verbosity;
@ -41,7 +45,7 @@ extern std::string generated_from;
constexpr int optimize_depth = 20;
const std::string func_version{"0.4.3"};
const std::string func_version{"0.4.4"};
enum Keyword {
_Eof = -1,
@ -159,6 +163,7 @@ struct TypeExpr {
int minw, maxw;
static constexpr int w_inf = 1023;
std::vector<TypeExpr*> args;
bool was_forall_var = false;
TypeExpr(te_type _constr, int _val = 0) : constr(_constr), value(_val), minw(0), maxw(w_inf) {
}
TypeExpr(te_type _constr, int _val, int width) : constr(_constr), value(_val), minw(width), maxw(width) {
@ -265,7 +270,7 @@ struct TypeExpr {
return new TypeExpr{te_ForAll, body, std::move(list)};
}
static bool remove_indirect(TypeExpr*& te, TypeExpr* forbidden = nullptr);
static bool remove_forall(TypeExpr*& te);
static std::vector<TypeExpr*> remove_forall(TypeExpr*& te);
static bool remove_forall_in(TypeExpr*& te, TypeExpr* te2, const std::vector<TypeExpr*>& new_vars);
};
@ -309,6 +314,7 @@ struct TmpVar {
int coord;
std::unique_ptr<SrcLocation> where;
std::vector<std::function<void(const SrcLocation &)>> on_modification;
bool undefined = false;
TmpVar(var_idx_t _idx, int _cls, TypeExpr* _type = 0, SymDef* sym = 0, const SrcLocation* loc = 0);
void show(std::ostream& os, int omit_idx = 0) const;
void dump(std::ostream& os) const;
@ -693,6 +699,7 @@ struct CodeBlob {
std::unique_ptr<Op>* cur_ops;
std::stack<std::unique_ptr<Op>*> cur_ops_stack;
int flags = 0;
bool require_callxargs = false;
CodeBlob(TypeExpr* ret = nullptr) : var_cnt(0), in_var_cnt(0), op_cnt(0), ret_type(ret), cur_ops(&ops) {
}
template <typename... Args>
@ -1176,7 +1183,7 @@ struct AsmOpList {
}
template <typename... Args>
AsmOpList& add(Args&&... args) {
list_.emplace_back(std::forward<Args>(args)...);
append(AsmOp(std::forward<Args>(args)...));
adjust_last();
return *this;
}
@ -1563,8 +1570,8 @@ struct Stack {
AsmOpList& o;
enum {
_StkCmt = 1, _CptStkCmt = 2, _DisableOpt = 4, _DisableOut = 128, _Shown = 256,
_InlineFunc = 512, _NeedRetAlt = 1024,
_ModeSave = _InlineFunc | _NeedRetAlt,
_InlineFunc = 512, _NeedRetAlt = 1024, _InlineAny = 2048,
_ModeSave = _InlineFunc | _NeedRetAlt | _InlineAny,
_Garbage = -0x10000
};
int mode;
@ -1611,7 +1618,7 @@ struct Stack {
if (i > 255) {
throw src::Fatal{"Too deep stack"};
}
assert(i >= 0 && i < depth() && "invalid stack reference");
func_assert(i >= 0 && i < depth() && "invalid stack reference");
}
void modified() {
mode &= ~_Shown;
@ -1642,14 +1649,24 @@ struct Stack {
bool operator==(const Stack& y) const & {
return s == y.s;
}
void apply_wrappers() {
void apply_wrappers(int callxargs_count) {
bool is_inline = mode & _InlineFunc;
if (o.retalt_) {
o.insert(0, "SAMEALTSAVE");
o.insert(0, "c2 SAVE");
if (mode & _InlineFunc) {
}
if (callxargs_count != -1 || (is_inline && o.retalt_)) {
o.indent_all();
o.insert(0, "CONT:<{");
o << "}>";
if (callxargs_count != -1) {
if (callxargs_count <= 15) {
o << AsmOp::Custom(PSTRING() << callxargs_count << " -1 CALLXARGS");
} else {
func_assert(callxargs_count <= 254);
o << AsmOp::Custom(PSTRING() << callxargs_count << " PUSHINT -1 PUSHINT CALLXVARARGS");
}
} else {
o << "EXECUTE";
}
}

18
crypto/func/gen-abscode.cpp
View file

@ -92,7 +92,7 @@ bool Expr::deduce_type(const Lexem& lem) {
return true;
}
case _VarApply: {
assert(args.size() == 2);
func_assert(args.size() == 2);
TypeExpr* fun_type = TypeExpr::new_map(args[1]->e_type, TypeExpr::new_hole());
try {
unify(fun_type, args[0]->e_type);
@ -107,7 +107,7 @@ bool Expr::deduce_type(const Lexem& lem) {
return true;
}
case _Letop: {
assert(args.size() == 2);
func_assert(args.size() == 2);
try {
// std::cerr << "in assignment: " << args[0]->e_type << " from " << args[1]->e_type << std::endl;
unify(args[0]->e_type, args[1]->e_type);
@ -122,7 +122,7 @@ bool Expr::deduce_type(const Lexem& lem) {
return true;
}
case _LetFirst: {
assert(args.size() == 2);
func_assert(args.size() == 2);
TypeExpr* rhs_type = TypeExpr::new_tensor({args[0]->e_type, TypeExpr::new_hole()});
try {
// std::cerr << "in implicit assignment of a modifying method: " << rhs_type << " and " << args[1]->e_type << std::endl;
@ -140,7 +140,7 @@ bool Expr::deduce_type(const Lexem& lem) {
return true;
}
case _CondExpr: {
assert(args.size() == 3);
func_assert(args.size() == 3);
auto flag_type = TypeExpr::new_atomic(_Int);
try {
unify(args[0]->e_type, flag_type);
@ -204,7 +204,7 @@ int Expr::predefine_vars() {
}
case _Var:
if (!sym) {
assert(val < 0 && here.defined());
func_assert(val < 0 && here.defined());
if (prohibited_var_names.count(sym::symbols.get_name(~val))) {
throw src::ParseError{
here, PSTRING() << "symbol `" << sym::symbols.get_name(~val) << "` cannot be redefined as a variable"};
@ -274,7 +274,7 @@ std::vector<var_idx_t> pre_compile_tensor(const std::vector<Expr *> args, CodeBl
arg_order.resize(args.size());
std::iota(arg_order.begin(), arg_order.end(), 0);
}
assert(args.size() == arg_order.size());
func_assert(args.size() == arg_order.size());
std::vector<std::vector<var_idx_t>> res_lists(args.size());
struct ModifiedVar {
@ -310,7 +310,7 @@ std::vector<var_idx_t> pre_compile_tensor(const std::vector<Expr *> args, CodeBl
}
for (const auto& list : res_lists) {
for (var_idx_t v : list) {
assert(!code.vars.at(v).on_modification.empty());
func_assert(!code.vars.at(v).on_modification.empty());
code.vars.at(v).on_modification.pop_back();
}
}
@ -339,7 +339,7 @@ std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, std::vector<std::pair<S
return pre_compile_tensor(args, code, lval_globs, {});
}
case _Apply: {
assert(sym);
func_assert(sym);
auto func = dynamic_cast<SymValFunc*>(sym->value);
std::vector<var_idx_t> res;
if (func && func->arg_order.size() == args.size() && !(code.flags & CodeBlob::_ComputeAsmLtr)) {
@ -426,7 +426,7 @@ std::vector<var_idx_t> Expr::pre_compile(CodeBlob& code, std::vector<std::pair<S
}
case _CondExpr: {
auto cond = args[0]->pre_compile(code);
assert(cond.size() == 1);
func_assert(cond.size() == 1);
auto rvect = new_tmp_vect(code);
Op& if_op = code.emplace_back(here, Op::_If, cond);
code.push_set_cur(if_op.block0);

28
crypto/func/optimize.cpp
View file

@ -61,9 +61,9 @@ void Optimizer::apply() {
if (!p_ && !q_) {
return;
}
assert(p_ > 0 && p_ <= l_ && q_ >= 0 && q_ <= n && l_ <= n);
func_assert(p_ > 0 && p_ <= l_ && q_ >= 0 && q_ <= n && l_ <= n);
for (int i = p_; i < l_; i++) {
assert(op_[i]);
func_assert(op_[i]);
op_cons_[i]->car = std::move(op_[i]);
op_cons_[i] = nullptr;
}
@ -71,7 +71,7 @@ void Optimizer::apply() {
code_ = std::move(code_->cdr);
}
for (int j = q_ - 1; j >= 0; j--) {
assert(oq_[j]);
func_assert(oq_[j]);
oq_[j]->indent = indent_;
code_ = AsmOpCons::cons(std::move(oq_[j]), std::move(code_));
}
@ -246,7 +246,7 @@ bool Optimizer::rewrite_const_push_xchgs() {
}
show_left();
auto c_op = std::move(op_[0]);
assert(c_op->is_gconst());
func_assert(c_op->is_gconst());
StackTransform t;
q_ = 0;
int pos = 0;
@ -265,31 +265,31 @@ bool Optimizer::rewrite_const_push_xchgs() {
if (b > pos) {
oq_[q_]->b = b - 1;
}
assert(apply_op(t, *oq_[q_]));
func_assert(apply_op(t, *oq_[q_]));
++q_;
}
} else {
assert(op_[i]->is_push(&a));
assert(a != pos);
func_assert(op_[i]->is_push(&a));
func_assert(a != pos);
oq_[q_] = std::move(op_[i]);
if (a > pos) {
oq_[q_]->a = a - 1;
}
assert(apply_op(t, *oq_[q_]));
func_assert(apply_op(t, *oq_[q_]));
++q_;
++pos;
}
}
assert(!pos);
func_assert(!pos);
t.apply_push_newconst();
assert(t <= tr_[p_ - 1]);
func_assert(t <= tr_[p_ - 1]);
oq_[q_++] = std::move(c_op);
show_right();
return true;
}
bool Optimizer::rewrite(int p, AsmOp&& new_op) {
assert(p > 0 && p <= l_);
func_assert(p > 0 && p <= l_);
p_ = p;
q_ = 1;
show_left();
@ -300,7 +300,7 @@ bool Optimizer::rewrite(int p, AsmOp&& new_op) {
}
bool Optimizer::rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2) {
assert(p > 1 && p <= l_);
func_assert(p > 1 && p <= l_);
p_ = p;
q_ = 2;
show_left();
@ -313,7 +313,7 @@ bool Optimizer::rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2) {
}
bool Optimizer::rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2, AsmOp&& new_op3) {
assert(p > 2 && p <= l_);
func_assert(p > 2 && p <= l_);
p_ = p;
q_ = 3;
show_left();
@ -328,7 +328,7 @@ bool Optimizer::rewrite(int p, AsmOp&& new_op1, AsmOp&& new_op2, AsmOp&& new_op3
}
bool Optimizer::rewrite_nop() {
assert(p_ > 0 && p_ <= l_);
func_assert(p_ > 0 && p_ <= l_);
q_ = 0;
show_left();
show_right();

9
crypto/func/parse-func.cpp
View file

@ -399,7 +399,7 @@ bool check_global_func(const Lexem& cur, sym_idx_t func_name = 0) {
cur.loc.show_error(std::string{"undefined function `"} + symbols.get_name(func_name) +
"`, defining a global function of unknown type");
def = sym::define_global_symbol(func_name, 0, cur.loc);
assert(def && "cannot define global function");
func_assert(def && "cannot define global function");
++undef_func_cnt;
make_new_glob_func(def, TypeExpr::new_func()); // was: ... ::new_func()
return true;
@ -1111,6 +1111,7 @@ blk_fl::val parse_do_stmt(Lexer& lex, CodeBlob& code) {
}
blk_fl::val parse_try_catch_stmt(Lexer& lex, CodeBlob& code) {
code.require_callxargs = true;
lex.expect(_Try);
Op& try_catch_op = code.emplace_back(lex.cur().loc, Op::_TryCatch);
code.push_set_cur(try_catch_op.block0);
@ -1132,7 +1133,7 @@ blk_fl::val parse_try_catch_stmt(Lexer& lex, CodeBlob& code) {
expr->predefine_vars();
expr->define_new_vars(code);
try_catch_op.left = expr->pre_compile(code);
assert(try_catch_op.left.size() == 2);
func_assert(try_catch_op.left.size() == 2 || try_catch_op.left.size() == 1);
blk_fl::val res1 = parse_block_stmt(lex, code);
sym::close_scope(lex);
code.close_pop_cur(lex.cur().loc);
@ -1295,7 +1296,7 @@ SymValAsmFunc* parse_asm_func_body(Lexer& lex, TypeExpr* func_type, const Formal
}
lex.next();
}
assert(arg_order.size() == (unsigned)tot_width);
func_assert(arg_order.size() == (unsigned)tot_width);
}
if (lex.tp() == _Mapsto) {
lex.expect(_Mapsto);
@ -1487,7 +1488,7 @@ void parse_func_def(Lexer& lex) {
std::cerr << "function " << func_name.str << " : " << func_type << std::endl;
}
SymDef* func_sym = sym::define_global_symbol(func_name.val, 0, loc);
assert(func_sym);
func_assert(func_sym);
SymValFunc* func_sym_val = dynamic_cast<SymValFunc*>(func_sym->value);
if (func_sym->value) {
if (func_sym->value->type != SymVal::_Func || !func_sym_val) {

2
crypto/func/stack-transform.cpp
View file

@ -183,7 +183,7 @@ bool StackTransform::is_permutation() const {
if (!is_valid() || d) {
return false;
}
assert(n <= max_n);
func_assert(n <= max_n);
std::array<int, max_n> X, Y;
for (int i = 0; i < n; i++) {
X[i] = A[i].first;

49
crypto/func/unify-types.cpp
View file

@ -132,7 +132,7 @@ void TypeExpr::replace_with(TypeExpr* te2) {
}
bool TypeExpr::remove_indirect(TypeExpr*& te, TypeExpr* forbidden) {
assert(te);
func_assert(te);
while (te->constr == te_Indirect) {
te = te->args[0];
}
@ -146,12 +146,9 @@ bool TypeExpr::remove_indirect(TypeExpr*& te, TypeExpr* forbidden) {
return res;
}
bool TypeExpr::remove_forall(TypeExpr*& te) {
assert(te);
if (te->constr != te_ForAll) {
return false;
}
assert(te->args.size() >= 1);
std::vector<TypeExpr*> TypeExpr::remove_forall(TypeExpr*& te) {
func_assert(te && te->constr == te_ForAll);
func_assert(te->args.size() >= 1);
std::vector<TypeExpr*> new_vars;
for (std::size_t i = 1; i < te->args.size(); i++) {
new_vars.push_back(new_hole(1));
@ -161,12 +158,12 @@ bool TypeExpr::remove_forall(TypeExpr*& te) {
te = te->args[0];
remove_forall_in(te, te2, new_vars);
// std::cerr << "-> " << te << std::endl;
return true;
return new_vars;
}
bool TypeExpr::remove_forall_in(TypeExpr*& te, TypeExpr* te2, const std::vector<TypeExpr*>& new_vars) {
assert(te);
assert(te2 && te2->constr == te_ForAll);
func_assert(te);
func_assert(te2 && te2->constr == te_ForAll);
if (te->constr == te_Var) {
for (std::size_t i = 0; i < new_vars.size(); i++) {
if (te == te2->args[i + 1]) {
@ -280,7 +277,7 @@ std::ostream& TypeExpr::print(std::ostream& os, int lex_level) {
return os << "]";
}
case te_Map: {
assert(args.size() == 2);
func_assert(args.size() == 2);
if (lex_level > 0) {
os << "(";
}
@ -293,7 +290,7 @@ std::ostream& TypeExpr::print(std::ostream& os, int lex_level) {
return os;
}
case te_ForAll: {
assert(args.size() >= 1);
func_assert(args.size() >= 1);
if (lex_level > 0) {
os << '(';
}
@ -346,11 +343,11 @@ void check_update_widths(TypeExpr* te1, TypeExpr* te2) {
check_width_compat(te1, te2);
te1->minw = te2->minw = std::max(te1->minw, te2->minw);
te1->maxw = te2->maxw = std::min(te1->maxw, te2->maxw);
assert(te1->minw <= te1->maxw);
func_assert(te1->minw <= te1->maxw);
}
void unify(TypeExpr*& te1, TypeExpr*& te2) {
assert(te1 && te2);
func_assert(te1 && te2);
// std::cerr << "unify( " << te1 << " , " << te2 << " )\n";
while (te1->constr == TypeExpr::te_Indirect) {
te1 = te1->args[0];
@ -363,23 +360,37 @@ void unify(TypeExpr*& te1, TypeExpr*& te2) {
}
if (te1->constr == TypeExpr::te_ForAll) {
TypeExpr* te = te1;
if (!TypeExpr::remove_forall(te)) {
throw UnifyError{te1, te2, "cannot remove universal type quantifier while performing type unification"};
std::vector<TypeExpr*> new_vars = TypeExpr::remove_forall(te);
for (TypeExpr* t : new_vars) {
t->was_forall_var = true;
}
unify(te, te2);
for (TypeExpr* t : new_vars) {
t->was_forall_var = false;
}
return;
}
if (te2->constr == TypeExpr::te_ForAll) {
TypeExpr* te = te2;
if (!TypeExpr::remove_forall(te)) {
throw UnifyError{te2, te1, "cannot remove universal type quantifier while performing type unification"};
std::vector<TypeExpr*> new_vars = TypeExpr::remove_forall(te);
for (TypeExpr* t : new_vars) {
t->was_forall_var = true;
}
unify(te1, te);
for (TypeExpr* t : new_vars) {
t->was_forall_var = false;
}
return;
}
if (te1->was_forall_var && te2->constr == TypeExpr::te_Tensor) {
throw UnifyError{te1, te2, "cannot unify generic type and tensor"};
}
if (te2->was_forall_var && te1->constr == TypeExpr::te_Tensor) {
throw UnifyError{te2, te1, "cannot unify generic type and tensor"};
}
if (te1->constr == TypeExpr::te_Unknown) {
if (te2->constr == TypeExpr::te_Unknown) {
assert(te1->value != te2->value);
func_assert(te1->value != te2->value);
}
if (!TypeExpr::remove_indirect(te2, te1)) {
throw UnifyError{te1, te2, "type unification results in an infinite cyclic type"};

1
crypto/funcfiftlib/funcfiftlib.cpp
View file

@ -92,6 +92,7 @@ td::Result<std::string> compile_internal(char *config_json) {
result_obj("status", "ok");
result_obj("codeBoc", td::base64_encode(boc));
result_obj("fiftCode", escape_json(outs.str()));
result_obj("codeHashHex", code_cell->get_hash().to_hex());
result_obj.leave();
outs.clear();

2
doc/catchain.tex
View file

@ -243,7 +243,7 @@ Notice that an external punishment for creating catchain forks may be used in th
Once a fork (created by~$i$) is detected (by another process~$j$), i.e.\ $j$ learns about two different messages $m_{i,s}$ and $m'_{i,s}$ created by $i$ and having same height $s$ (usually this happens while recursively downloading dependencies of some other messages), $j$ starts ignoring~$i$ and all of its subsequent messages. They are not accepted and not broadcast further. However, messages created by~$i$ prior to the fork detection may be still downloaded if they are referred to in messages (blocks) created by processes that did not see this fork before referring to such messages created by~$i$.
\nxsubpoint\emb{Accepting messages from a ``bad'' process is bad}\label{sp:no.bad.accept}
Furthermore, if process $i$ learns about a fork created by process $j$, then $i$ shows this to its neighbors by creating a new service broadcast message that contains the corresponding fork proof (cf.~\ptref{sp:fork.proofs}). Afterwards, this and all subsequent messages of $j$ cannot directly depend on any messages by the known ``bad'' producer $i$ (but they still can refer to messages from another party $k$ that directly or indirectly refer to messages of~$i$ if no fork by~$i$ was known to $k$ at the time when the referring message was created). If $j$ violates this restriction and creates messages with such invalid references, these messages will be discarded by all honest processes in the group.
Furthermore, if process $i$ learns about a fork created by process $j$, then $i$ shows this to its neighbors by creating a new service broadcast message that contains the corresponding fork proof (cf.~\ptref{sp:fork.proofs}). Afterwards, this and all subsequent messages of $i$ cannot directly depend on any messages by the known ``bad'' producer $j$ (but they still can refer to messages from another party $k$ that directly or indirectly refer to messages of~$i$ if no fork by~$i$ was known to $k$ at the time when the referring message was created). If $i$ violates this restriction and creates messages with such invalid references, these messages will be discarded by all honest processes in the group.
\nxsubpoint\emb{The set of ``bad'' group members is a part of the intrinsic state}\label{sp:bad.proc.set}
Each process~$i$ keeps its own copy of the set of known ``bad'' processes in the group, i.e., those processes that have created at least one fork or have violated \ptref{sp:no.bad.accept}. This set is updated by adding~$j$ into it as soon as $i$ learns about a fork created by~$j$ (or about a violation of~\ptref{sp:no.bad.accept} by $j$); after that, a callback provided by the higher-level protocol is invoked. This set is used when a new broadcast message arrives: if the sender is bad, then the message is ignored and discarded.

9
emulator/CMakeLists.txt
View file

@ -1,5 +1,7 @@
cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR)
option(BUILD_SHARED_LIBS "Use \"OFF\" for a static build." ON)
if (NOT OPENSSL_FOUND)
find_package(OpenSSL REQUIRED)
endif()
@ -27,7 +29,12 @@ include(GenerateExportHeader)
add_library(emulator_static STATIC ${EMULATOR_STATIC_SOURCE})
target_link_libraries(emulator_static PUBLIC ton_crypto ton_block smc-envelope)
add_library(emulator STATIC ${EMULATOR_SOURCE} ${EMULATOR_HEADERS})
if (NOT USE_EMSCRIPTEN AND BUILD_SHARED_LIBS)
add_library(emulator SHARED ${EMULATOR_SOURCE} ${EMULATOR_HEADERS})
else()
add_library(emulator STATIC ${EMULATOR_SOURCE} ${EMULATOR_HEADERS})
endif()
target_link_libraries(emulator PUBLIC emulator_static)
generate_export_header(emulator EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/emulator_export.h)
target_include_directories(emulator PUBLIC

3
flake.nix
View file

@ -50,6 +50,9 @@
"-static-libstdc++"
]);
GIT_REVISION = if self ? rev then self.rev else "dirty";
GIT_REVISION_DATE = (builtins.concatStringsSep "-" (builtins.match "(.{4})(.{2})(.{2}).*" self.lastModifiedDate)) + " " + (builtins.concatStringsSep ":" (builtins.match "^........(.{2})(.{2})(.{2}).*" self.lastModifiedDate));
postInstall = ''
moveToOutput bin "$bin"
'';

4
git_watcher.cmake
View file

@ -139,6 +139,8 @@ function(GetGitState _working_dir)
RunGitCommand(show -s "--format=%H" ${object})
if(exit_code EQUAL 0)
set(ENV{GIT_HEAD_SHA1} ${output})
else()
set(ENV{GIT_HEAD_SHA1} "$ENV{GIT_REVISION}")
endif()
RunGitCommand(show -s "--format=%an" ${object})
@ -154,6 +156,8 @@ function(GetGitState _working_dir)
RunGitCommand(show -s "--format=%ci" ${object})
if(exit_code EQUAL 0)
set(ENV{GIT_COMMIT_DATE_ISO8601} "${output}")
else()
set(ENV{GIT_COMMIT_DATE_ISO8601} "$ENV{GIT_REVISION_DATE}")
endif()
RunGitCommand(show -s "--format=%s" ${object})

8
recent_changelog.md Normal file
View file

@ -0,0 +1,8 @@
## 2023.05 Update
1. Archive manager optimization
2. A series of catchain (basic consensus protocol) security improvements
3. Update for Fift libraries and FunC: better error-handling, fixes for `catch` stack recovery
4. A series of out message queue handling optimization (already deployed during emergency upgrades between releases)
5. Improvement of binaries portability
Besides the work of the core team, this update is based on the efforts of @aleksej-paschenko (portability improvement), [Disintar team](https://github.com/disintar/) (archive manager optimization) and [sec3-service](https://github.com/sec3-service) security auditors (funC improvements).

7
tdutils/td/utils/Timer.cpp
View file

@ -78,9 +78,12 @@ void PerfWarningTimer::reset() {
return;
}
double duration = Time::now() - start_at_;
//LOG_IF(WARNING, duration > max_duration_)
//<< "SLOW: " << tag("name", name_) << tag("duration", format::as_time(duration));
if (callback_) {
callback_(duration);
} else {
LOG_IF(WARNING, duration > max_duration_)
<< "SLOW: " << tag("name", name_) << tag("duration", format::as_time(duration));
}
start_at_ = 0;
}

2
tdutils/td/utils/Timer.h
View file

@ -46,7 +46,7 @@ class Timer {
class PerfWarningTimer {
public:
explicit PerfWarningTimer(string name, double max_duration = 0.1, std::function<void(double)>&& callback = [] (double) {});
explicit PerfWarningTimer(string name, double max_duration = 0.1, std::function<void(double)>&& callback = {});
PerfWarningTimer(const PerfWarningTimer &) = delete;
PerfWarningTimer &operator=(const PerfWarningTimer &) = delete;
PerfWarningTimer(PerfWarningTimer &&other);

4
tl/generate/scheme/ton_api.tl
View file

@ -277,14 +277,10 @@ catchain.differenceFork left:catchain.block.dep right:catchain.block.dep = catch
catchain.blockNotFound = catchain.BlockResult;
catchain.blockResult block:catchain.block = catchain.BlockResult;
catchain.sent cnt:int = catchain.Sent;
---functions---
catchain.getBlock block:int256 = catchain.BlockResult;
catchain.getBlocks blocks:(vector int256) = catchain.Sent;
catchain.getDifference rt:(vector int) = catchain.Difference;
catchain.getBlockHistory block:int256 height:long stop_if:(vector int256) = catchain.Sent;
//catchain.getForkDifference src:int fork:catchain.fork = catchain.ForkDifference;
---types---

BIN
tl/generate/scheme/ton_api.tlo
View file

Binary file not shown.

42
validator-engine/validator-engine.cpp
View file

@ -1319,9 +1319,6 @@ td::Status ValidatorEngine::load_global_config() {
}
CHECK(mode == ton::validator::ValidatorManagerOptions::ShardCheckMode::m_validate);
return true;
/*ton::ShardIdFull p{ton::basechainId, ((cc_seqno * 1ull % 4) << 62) + 1};
auto s = ton::shard_prefix(p, 2);
return shard.is_masterchain() || ton::shard_intersects(shard, s);*/
});
if (state_ttl_ != 0) {
validator_options_.write().set_state_ttl(state_ttl_);
@ -3647,33 +3644,60 @@ int main(int argc, char *argv[]) {
logger_ = td::TsFileLog::create(fname.str()).move_as_ok();
td::log_interface = logger_.get();
});
p.add_option('s', "state-ttl", "state will be gc'd after this time (in seconds) default=3600", [&](td::Slice fname) {
p.add_checked_option('s', "state-ttl", "state will be gc'd after this time (in seconds) default=3600",
[&](td::Slice fname) {
auto v = td::to_double(fname);
if (v <= 0) {
return td::Status::Error("state-ttl should be positive");
}
acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_state_ttl, v); });
return td::Status::OK();
});
p.add_option('m', "mempool-num", "Maximal number of mempool external message", [&](td::Slice fname) {
p.add_checked_option('m', "mempool-num", "Maximal number of mempool external message", [&](td::Slice fname) {
auto v = td::to_double(fname);
if (v < 0) {
return td::Status::Error("mempool-num should be non-negative");
}
acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_mempool_num, v); });
return td::Status::OK();
});
p.add_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=7*86400",
p.add_checked_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=7*86400",
[&](td::Slice fname) {
auto v = td::to_double(fname);
if (v <= 0) {
return td::Status::Error("block-ttl should be positive");
}
acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_block_ttl, v); });
return td::Status::OK();
});
p.add_option('A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=365*86400",
p.add_checked_option(
'A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=365*86400",
[&](td::Slice fname) {
auto v = td::to_double(fname);
if (v <= 0) {
return td::Status::Error("archive-ttl should be positive");
}
acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_archive_ttl, v); });
return td::Status::OK();
});
p.add_option('K', "key-proof-ttl", "key blocks will be deleted after this time (in seconds) default=365*86400*10",
p.add_checked_option(
'K', "key-proof-ttl", "key blocks will be deleted after this time (in seconds) default=365*86400*10",
[&](td::Slice fname) {
auto v = td::to_double(fname);
if (v <= 0) {
return td::Status::Error("key-proof-ttl should be positive");
}
acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_key_proof_ttl, v); });
return td::Status::OK();
});
p.add_option('S', "sync-before", "in initial sync download all blocks for last given seconds default=3600",
p.add_checked_option('S', "sync-before", "in initial sync download all blocks for last given seconds default=3600",
[&](td::Slice fname) {
auto v = td::to_double(fname);
if (v <= 0) {
return td::Status::Error("sync-before should be positive");
}
acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_sync_ttl, v); });
return td::Status::OK();
});
p.add_option('T', "truncate-db", "truncate db (with specified seqno as new top masterchain block seqno)",
[&](td::Slice fname) {

3
validator-session/CMakeLists.txt
View file

@ -9,6 +9,7 @@ set(VALIDATOR_SESSION_SOURCE
validator-session-description.cpp
validator-session-state.cpp
validator-session.cpp
validator-session-round-attempt-state.cpp
persistent-vector.h
validator-session-description.h
@ -16,7 +17,7 @@ set(VALIDATOR_SESSION_SOURCE
validator-session-state.h
validator-session.h
validator-session.hpp
)
validator-session-round-attempt-state.h)
add_library(validatorsession STATIC ${VALIDATOR_SESSION_SOURCE})

7
validator-session/persistent-vector.h
View file

@ -322,7 +322,6 @@ class CntVector : public ValidatorSessionDescription::RootObject {
CHECK(idx < size());
return data_[idx];
}
//const T& at(size_t idx) const;
private:
const td::uint32 data_size_;
@ -546,7 +545,6 @@ class CntVector<bool> : public ValidatorSessionDescription::RootObject {
CHECK(idx < max_size());
return get_bit(data_, idx);
}
//const T& at(size_t idx) const;
private:
const td::uint32 data_size_;
@ -734,10 +732,6 @@ class CntSortedVector : public ValidatorSessionDescription::RootObject {
return CntSortedVector::create(desc, std::move(v));
}
/*static const CntSortedVector* merge(ValidatorSessionDescription& desc, const CntSortedVector* l,
const CntSortedVector* r) {
return merge(desc, l, r, [](T l, T r) { return l; });
}*/
static const CntSortedVector* push(ValidatorSessionDescription& desc, const CntSortedVector* v, T value) {
if (!v) {
return create(desc, std::vector<T>{value});
@ -795,7 +789,6 @@ class CntSortedVector : public ValidatorSessionDescription::RootObject {
CHECK(idx < size());
return data_[idx];
}
//const T& at(size_t idx) const;
private:
const td::uint32 data_size_;

95
validator-session/validator-session-description.cpp
View file

@ -51,13 +51,6 @@ ValidatorSessionDescriptionImpl::ValidatorSessionDescriptionImpl(ValidatorSessio
CHECK(it != rev_sources_.end());
self_idx_ = it->second;
pdata_temp_ptr_ = 0;
pdata_temp_size_ = 1 << 27;
pdata_temp_ = new td::uint8[pdata_temp_size_];
pdata_perm_size_ = 1ull << 27;
pdata_perm_ptr_ = 0;
for (auto &el : cache_) {
Cached v{nullptr};
el.store(v, std::memory_order_relaxed);
@ -162,43 +155,11 @@ void ValidatorSessionDescriptionImpl::update_hash(const RootObject *obj, HashTyp
}
void *ValidatorSessionDescriptionImpl::alloc(size_t size, size_t align, bool temp) {
CHECK(align && !(align & (align - 1))); // align should be a power of 2
auto get_padding = [&](const uint8_t* ptr) {
return (-(size_t)ptr) & (align - 1);
};
if (temp) {
pdata_temp_ptr_ += get_padding(pdata_temp_ + pdata_temp_ptr_);
auto s = pdata_temp_ptr_;
pdata_temp_ptr_ += size;
CHECK(s + size <= pdata_temp_size_);
return static_cast<void *>(pdata_temp_ + s);
} else {
while (true) {
size_t idx = pdata_perm_ptr_ / pdata_perm_size_;
if (idx < pdata_perm_.size()) {
auto ptr = pdata_perm_[idx] + (pdata_perm_ptr_ % pdata_perm_size_);
pdata_perm_ptr_ += get_padding(ptr);
ptr += get_padding(ptr);
pdata_perm_ptr_ += size;
if (pdata_perm_ptr_ <= pdata_perm_.size() * pdata_perm_size_) {
return static_cast<void *>(ptr);
}
}
pdata_perm_.push_back(new td::uint8[pdata_perm_size_]);
}
}
return (temp ? mem_temp_ : mem_perm_).alloc(size, align);
}
bool ValidatorSessionDescriptionImpl::is_persistent(const void *ptr) const {
if (ptr == nullptr) {
return true;
}
for (auto &v : pdata_perm_) {
if (ptr >= v && ptr <= v + pdata_perm_size_) {
return true;
}
}
return false;
return mem_perm_.contains(ptr);
}
std::unique_ptr<ValidatorSessionDescription> ValidatorSessionDescription::create(
@ -206,6 +167,58 @@ std::unique_ptr<ValidatorSessionDescription> ValidatorSessionDescription::create
return std::make_unique<ValidatorSessionDescriptionImpl>(std::move(opts), nodes, local_id);
}
ValidatorSessionDescriptionImpl::MemPool::MemPool(size_t chunk_size) : chunk_size_(chunk_size) {
}
ValidatorSessionDescriptionImpl::MemPool::~MemPool() {
for (auto &v : data_) {
delete[] v;
}
}
void *ValidatorSessionDescriptionImpl::MemPool::alloc(size_t size, size_t align) {
CHECK(align && !(align & (align - 1))); // align should be a power of 2
CHECK(size + align <= chunk_size_);
auto get_padding = [&](const uint8_t* ptr) {
return (-(size_t)ptr) & (align - 1);
};
while (true) {
size_t idx = ptr_ / chunk_size_;
if (idx < data_.size()) {
auto ptr = data_[idx] + (ptr_ % chunk_size_);
ptr_ += get_padding(ptr);
ptr += get_padding(ptr);
ptr_ += size;
if (ptr_ <= data_.size() * chunk_size_) {
return static_cast<void *>(ptr);
} else {
ptr_ = data_.size() * chunk_size_;
}
}
data_.push_back(new td::uint8[chunk_size_]);
}
}
void ValidatorSessionDescriptionImpl::MemPool::clear() {
while (data_.size() > 1) {
delete[] data_.back();
data_.pop_back();
}
ptr_ = 0;
}
bool ValidatorSessionDescriptionImpl::MemPool::contains(const void* ptr) const {
if (ptr == nullptr) {
return true;
}
for (auto &v : data_) {
if (ptr >= v && ptr <= v + chunk_size_) {
return true;
}
}
return false;
}
} // namespace validatorsession
} // namespace ton

32
validator-session/validator-session-description.hpp
View file

@ -50,20 +50,30 @@ class ValidatorSessionDescriptionImpl : public ValidatorSessionDescription {
td::uint32 self_idx_;
static constexpr td::uint32 cache_size = (1 << 20);
static constexpr size_t mem_chunk_size_perm = (1 << 27);
static constexpr size_t mem_chunk_size_temp = (1 << 27);
struct Cached {
const RootObject *ptr;
};
std::array<std::atomic<Cached>, cache_size> cache_;
//std::array<std::atomic<Cached>, cache_size> temp_cache_;
td::uint8 *pdata_temp_;
size_t pdata_temp_ptr_;
size_t pdata_temp_size_;
class MemPool {
public:
explicit MemPool(size_t chunk_size);
~MemPool();
void *alloc(size_t size, size_t align);
void clear();
bool contains(const void* ptr) const;
private:
size_t chunk_size_;
std::vector<td::uint8 *> data_;
size_t ptr_ = 0;
};
MemPool mem_perm_ = MemPool(mem_chunk_size_perm);
MemPool mem_temp_ = MemPool(mem_chunk_size_temp);
size_t pdata_perm_size_;
std::vector<td::uint8 *> pdata_perm_;
size_t pdata_perm_ptr_;
std::atomic<td::uint64> reuse_{0};
public:
@ -116,7 +126,7 @@ class ValidatorSessionDescriptionImpl : public ValidatorSessionDescription {
void update_hash(const RootObject *obj, HashType hash) override;
void *alloc(size_t size, size_t align, bool temp) override;
void clear_temp_memory() override {
pdata_temp_ptr_ = 0;
mem_temp_.clear();
}
bool is_persistent(const void *ptr) const override;
HashType compute_hash(td::Slice data) const override;
@ -152,12 +162,6 @@ class ValidatorSessionDescriptionImpl : public ValidatorSessionDescription {
const ValidatorSessionOptions &opts() const override {
return opts_;
}
~ValidatorSessionDescriptionImpl() {
delete[] pdata_temp_;
for (auto &x : pdata_perm_) {
delete[] x;
}
}
};
} // namespace validatorsession

486
validator-session/validator-session-round-attempt-state.cpp Normal file
View file

@ -0,0 +1,486 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
Copyright 2017-2020 Telegram Systems LLP
*/
#include "validator-session-state.h"
#include "td/utils/Random.h"
#include "auto/tl/ton_api.hpp"
#include <set>
namespace ton {
namespace validatorsession {
static const SessionVoteCandidate* get_candidate(const VoteVector* vec, ValidatorSessionCandidateId id) {
if (!vec) {
return nullptr;
}
auto size = vec->size();
auto v = vec->data();
for (td::uint32 i = 0; i < size; i++) {
if (v[i]->get_id() == id) {
return v[i];
}
}
return nullptr;
}
//
//
// SessionBlockCandidateSignature
//
//
const SessionBlockCandidateSignature* SessionBlockCandidateSignature::merge(ValidatorSessionDescription& desc,
const SessionBlockCandidateSignature* l,
const SessionBlockCandidateSignature* r) {
if (!l) {
return r;
}
if (!r) {
return l;
}
if (l == r) {
return l;
}
if (l->as_slice() < r->as_slice()) {
return l;
} else {
return r;
}
}
//
//
// SessionBlockCandidate
//
//
bool SessionBlockCandidate::check_block_is_approved(ValidatorSessionDescription& desc) const {
ValidatorWeight w = 0;
for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) {
if (approved_by_->at(i)) {
w += desc.get_node_weight(i);
if (w >= desc.get_cutoff_weight()) {
return true;
}
}
}
return false;
}
const SessionBlockCandidate* SessionBlockCandidate::merge(ValidatorSessionDescription& desc,
const SessionBlockCandidate* l,
const SessionBlockCandidate* r) {
if (!l) {
return r;
}
if (!r) {
return l;
}
if (l == r) {
return l;
}
CHECK(l->get_id() == r->get_id());
auto v = SessionBlockCandidateSignatureVector::merge(
desc, l->approved_by_, r->approved_by_,
[&](const SessionBlockCandidateSignature* l, const SessionBlockCandidateSignature* r) {
return SessionBlockCandidateSignature::merge(desc, l, r);
});
return SessionBlockCandidate::create(desc, l->block_, std::move(v));
}
//
//
// SessionVoteCandidate
//
//
bool SessionVoteCandidate::check_block_is_voted(ValidatorSessionDescription& desc) const {
ValidatorWeight w = 0;
for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) {
if (voted_by_->at(i)) {
w += desc.get_node_weight(i);
if (w >= desc.get_cutoff_weight()) {
return true;
}
}
}
return false;
}
const SessionVoteCandidate* SessionVoteCandidate::merge(ValidatorSessionDescription& desc,
const SessionVoteCandidate* l, const SessionVoteCandidate* r) {
if (!l) {
return r;
}
if (!r) {
return l;
}
if (l == r) {
return l;
}
CHECK(l->get_id() == r->get_id());
auto v = CntVector<bool>::merge(desc, l->voted_by_, r->voted_by_);
return SessionVoteCandidate::create(desc, l->block_, std::move(v));
}
//
//
// ATTEMPT STATE
//
//
const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::merge(
ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* left,
const ValidatorSessionRoundAttemptState* right) {
if (!left) {
return right;
}
if (!right) {
return left;
}
if (left == right) {
return left;
}
CHECK(left->seqno_ == right->seqno_);
const SentBlock* vote_for = nullptr;
bool vote_for_inited = false;
if (!left->vote_for_inited_) {
vote_for = right->vote_for_;
vote_for_inited = right->vote_for_inited_;
} else if (!right->vote_for_inited_) {
vote_for = left->vote_for_;
vote_for_inited = left->vote_for_inited_;
} else if (left->vote_for_ == right->vote_for_) {
vote_for_inited = true;
vote_for = left->vote_for_;
} else {
auto l = SentBlock::get_block_id(left->vote_for_);
auto r = SentBlock::get_block_id(right->vote_for_);
vote_for_inited = true;
if (l < r) {
vote_for = left->vote_for_;
} else {
vote_for = right->vote_for_;
}
}
auto precommitted = CntVector<bool>::merge(desc, left->precommitted_, right->precommitted_);
auto votes = VoteVector::merge(desc, left->votes_, right->votes_,
[&](const SessionVoteCandidate* l, const SessionVoteCandidate* r) {
return SessionVoteCandidate::merge(desc, l, r);
});
return ValidatorSessionRoundAttemptState::create(desc, left->seqno_, std::move(votes), std::move(precommitted),
vote_for, vote_for_inited);
}
const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action(
ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx,
td::uint32 att, const ton_api::validatorSession_message_voteFor& act, const ValidatorSessionRoundState* round) {
if (state->vote_for_inited_) {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act
<< "]: invalid message: duplicate VOTEFOR";
return state;
}
if (src_idx != desc.get_vote_for_author(att)) {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act
<< "]: invalid message: bad VOTEFOR author";
return state;
}
if (round->get_first_attempt(src_idx) == 0 && desc.opts().max_round_attempts > 0) {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act
<< "]: invalid message: too early for VOTEFOR";
return state;
}
if (round->get_first_attempt(src_idx) + desc.opts().max_round_attempts > att && desc.opts().max_round_attempts > 0) {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act
<< "]: invalid message: too early for VOTEFOR";
return state;
}
auto x = round->get_block(act.candidate_);
if (!x) {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act
<< "]: invalid message: VOTEFOR for not submitted block";
return state;
}
if (!x->check_block_is_approved(desc)) {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act
<< "]: invalid message: VOTEFOR for not approved block";
return state;
}
return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, state->votes_, state->precommitted_,
x->get_block(), true);
}
const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action(
ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx,
td::uint32 att, const ton_api::validatorSession_message_vote& act, const ValidatorSessionRoundState* round) {
bool made;
return make_one(desc, state, src_idx, att, round, &act, made);
}
const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action(
ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx,
td::uint32 att, const ton_api::validatorSession_message_precommit& act, const ValidatorSessionRoundState* round) {
bool made;
return make_one(desc, state, src_idx, att, round, &act, made);
}
const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action(
ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx,
td::uint32 att, const ton_api::validatorSession_message_empty& act, const ValidatorSessionRoundState* round) {
bool made;
return make_one(desc, state, src_idx, att, round, &act, made);
}
const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::try_vote(
ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx,
td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act,
bool& made) {
made = false;
if (state->check_vote_received_from(src_idx)) {
return state;
}
auto found = false;
auto block = round->choose_block_to_vote(desc, src_idx, att, state->vote_for_, state->vote_for_inited_, found);
if (!found) {
return state;
}
auto block_id = SentBlock::get_block_id(block);
made = true;
if (act) {
if (act->get_id() != ton_api::validatorSession_message_vote::ID) {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act
<< "]: expected VOTE(" << block_id << ")";
} else {
auto x = static_cast<const ton_api::validatorSession_message_vote*>(act);
if (x->candidate_ != block_id) {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act
<< "]: expected VOTE(" << block_id << ")";
}
}
} else {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx)
<< "]: making implicit VOTE(" << block_id << ")";
}
auto candidate = get_candidate(state->votes_, block_id);
if (!candidate) {
candidate = SessionVoteCandidate::create(desc, block);
}
candidate = SessionVoteCandidate::push(desc, candidate, src_idx);
auto v = VoteVector::push(desc, state->votes_, candidate);
return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, std::move(v), state->precommitted_,
state->vote_for_, state->vote_for_inited_);
}
const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::try_precommit(
ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx,
td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act,
bool& made) {
made = false;
if (state->check_precommit_received_from(src_idx)) {
return state;
}
bool found;
auto block = state->get_voted_block(desc, found);
if (!found) {
return state;
}
made = true;
auto block_id = SentBlock::get_block_id(block);
if (act) {
if (act->get_id() != ton_api::validatorSession_message_precommit::ID) {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act
<< "]: expected PRECOMMIT(" << block_id << ")";
} else {
auto x = static_cast<const ton_api::validatorSession_message_precommit*>(act);
if (x->candidate_ != block_id) {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act
<< "]: expected PRECOMMIT(" << block_id << ")";
}
}
} else {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx)
<< "]: making implicit PRECOMMIT(" << block_id << ")";
}
auto v = CntVector<bool>::change(desc, state->precommitted_, src_idx, true);
return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, state->votes_, std::move(v), state->vote_for_,
state->vote_for_inited_);
}
const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::make_one(
ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx,
td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act,
bool& made) {
made = false;
state = try_vote(desc, state, src_idx, att, round, act, made);
if (made) {
return state;
}
state = try_precommit(desc, state, src_idx, att, round, act, made);
if (made) {
return state;
}
if (act && act->get_id() != ton_api::validatorSession_message_empty::ID) {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act
<< "]: invalid message: expected EMPTY";
}
return state;
}
const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action(
ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx,
td::uint32 att, const ton_api::validatorSession_round_Message* act, const ValidatorSessionRoundState* round) {
ton_api::downcast_call(*const_cast<ton_api::validatorSession_round_Message*>(act),
[&](auto& obj) { state = action(desc, state, src_idx, att, obj, round); });
return state;
}
bool ValidatorSessionRoundAttemptState::check_vote_received_from(td::uint32 src_idx) const {
if (!votes_) {
return false;
}
auto size = votes_->size();
auto v = votes_->data();
for (td::uint32 i = 0; i < size; i++) {
if (v[i]->check_block_is_voted_by(src_idx)) {
return true;
}
}
return false;
}
bool ValidatorSessionRoundAttemptState::check_precommit_received_from(td::uint32 src_idx) const {
return precommitted_->at(src_idx);
}
const SentBlock* ValidatorSessionRoundAttemptState::get_voted_block(ValidatorSessionDescription& desc, bool& f) const {
f = false;
if (!votes_) {
return nullptr;
}
auto size = votes_->size();
auto v = votes_->data();
for (td::uint32 i = 0; i < size; i++) {
if (v[i]->check_block_is_voted(desc)) {
f = true;
return v[i]->get_block();
}
}
return nullptr;
}
bool ValidatorSessionRoundAttemptState::check_attempt_is_precommitted(ValidatorSessionDescription& desc) const {
ValidatorWeight weight = 0;
for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) {
if (precommitted_->at(i)) {
weight += desc.get_node_weight(i);
if (weight >= desc.get_cutoff_weight()) {
return true;
}
}
}
return false;
}
tl_object_ptr<ton_api::validatorSession_round_Message> ValidatorSessionRoundAttemptState::create_action(
ValidatorSessionDescription& desc, const ValidatorSessionRoundState* round, td::uint32 src_idx,
td::uint32 att) const {
if (!check_vote_received_from(src_idx)) {
auto found = false;
auto B = round->choose_block_to_vote(desc, src_idx, att, vote_for_, vote_for_inited_, found);
if (found) {
auto block_id = SentBlock::get_block_id(B);
return create_tl_object<ton_api::validatorSession_message_vote>(round->get_seqno(), seqno_, block_id);
}
}
if (!check_precommit_received_from(src_idx)) {
bool f = false;
auto B = get_voted_block(desc, f);
if (f) {
auto block_id = SentBlock::get_block_id(B);
return create_tl_object<ton_api::validatorSession_message_precommit>(round->get_seqno(), seqno_, block_id);
}
}
return create_tl_object<ton_api::validatorSession_message_empty>(round->get_seqno(), seqno_);
}
void ValidatorSessionRoundAttemptState::dump(ValidatorSessionDescription& desc, td::StringBuilder& sb) const {
sb << "attempt=" << seqno_ << "\n";
sb << ">>>>\n";
if (vote_for_inited_) {
sb << "vote_for=" << (vote_for_ ? vote_for_->get_src_idx() : std::numeric_limits<td::uint32>::max()) << "\n";
} else {
sb << "vote_for=NONE\n";
}
if (votes_) {
auto s = votes_->size();
sb << "votes: ";
std::vector<td::int32> R;
R.resize(desc.get_total_nodes(), -1);
for (td::uint32 i = 0; i < s; i++) {
const auto e = votes_->at(i);
const auto& x = e->get_voters_list();
for (td::uint32 j = 0; j < desc.get_total_nodes(); j++) {
if (x->at(j)) {
R[j] = e->get_src_idx();
}
}
}
for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) {
sb << R[i] << " ";
}
sb << "\n";
} else {
sb << "votes: EMPTY\n";
}
sb << "precommits: ";
for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) {
const auto e = precommitted_->at(i);
if (e) {
sb << "+ ";
} else {
sb << "- ";
}
}
sb << "\n";
sb << "<<<<\n";
}
} // namespace validatorsession
} // namespace ton

620
validator-session/validator-session-round-attempt-state.h Normal file
View file

@ -0,0 +1,620 @@
/*
This file is part of TON Blockchain Library.
TON Blockchain Library is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
TON Blockchain Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "td/utils/int_types.h"
#include "td/utils/buffer.h"
#include "adnl/utils.hpp"
#include "common/io.hpp"
#include "persistent-vector.h"
#include "validator-session-description.h"
#include "validator-session-types.h"
#include <limits>
namespace ton {
namespace validatorsession {
using HashType = ValidatorSessionDescription::HashType;
struct SessionBlockCandidateSignature : public ValidatorSessionDescription::RootObject {
public:
static auto create_hash(ValidatorSessionDescription& desc, td::Slice data) {
auto obj = create_tl_object<ton_api::hashable_blockSignature>(desc.compute_hash(data));
return desc.compute_hash(serialize_tl_object(obj, true).as_slice());
}
static bool compare(const RootObject* r, td::Slice data, HashType hash) {
if (!r || r->get_size() < sizeof(SessionBlockCandidateSignature)) {
return false;
}
auto R = static_cast<const SessionBlockCandidateSignature*>(r);
return R->hash_ == hash && R->data_.ubegin() == data.ubegin() && R->data_.size() == data.size();
}
static auto lookup(ValidatorSessionDescription& desc, td::Slice data, HashType hash, bool temp) {
auto r = desc.get_by_hash(hash, temp);
if (compare(r, data, hash)) {
desc.on_reuse();
return static_cast<const SessionBlockCandidateSignature*>(r);
}
return static_cast<const SessionBlockCandidateSignature*>(nullptr);
}
static SessionBlockCandidateSignature* create(ValidatorSessionDescription& desc, td::BufferSlice value) {
auto hash = create_hash(desc, value.as_slice());
auto d = static_cast<td::uint8*>(desc.alloc(value.size(), 8, false));
td::MutableSlice s{d, value.size()};
s.copy_from(value.as_slice());
return new (desc, true) SessionBlockCandidateSignature{desc, s, hash};
}
static const SessionBlockCandidateSignature* move_to_persistent(ValidatorSessionDescription& desc,
const SessionBlockCandidateSignature* b) {
if (desc.is_persistent(b)) {
return b;
}
CHECK(desc.is_persistent(b->data_.ubegin()));
auto r = lookup(desc, b->data_, b->hash_, false);
if (r) {
return r;
}
return new (desc, false) SessionBlockCandidateSignature{desc, b->data_, b->hash_};
}
static const SessionBlockCandidateSignature* merge(ValidatorSessionDescription& desc,
const SessionBlockCandidateSignature* l,
const SessionBlockCandidateSignature* r);
SessionBlockCandidateSignature(ValidatorSessionDescription& desc, td::Slice data, HashType hash)
: RootObject(sizeof(SessionBlockCandidateSignature)), data_{data}, hash_(std::move(hash)) {
desc.update_hash(this, hash_);
}
td::BufferSlice value() const {
return td::BufferSlice{data_};
}
td::Slice as_slice() const {
return data_;
}
auto get_hash(ValidatorSessionDescription& desc) const {
return hash_;
}
private:
const td::Slice data_;
const HashType hash_;
};
using SessionBlockCandidateSignatureVector = CntVector<const SessionBlockCandidateSignature*>;
class SentBlock : public ValidatorSessionDescription::RootObject {
public:
static HashType create_hash(ValidatorSessionDescription& desc, td::uint32 src_idx, ValidatorSessionRootHash root_hash,
ValidatorSessionFileHash file_hash,
ValidatorSessionCollatedDataFileHash collated_data_file_hash) {
auto obj = create_tl_object<ton_api::hashable_sentBlock>(src_idx, get_vs_hash(desc, root_hash),
get_vs_hash(desc, file_hash),
get_vs_hash(desc, collated_data_file_hash));
return desc.compute_hash(serialize_tl_object(obj, true).as_slice());
}
static bool compare(const RootObject* root_object, td::uint32 src_idx, const ValidatorSessionRootHash& root_hash,
const ValidatorSessionFileHash& file_hash,
const ValidatorSessionCollatedDataFileHash& collated_data_file_hash, HashType hash) {
if (!root_object || root_object->get_size() < sizeof(SentBlock)) {
return false;
}
auto obj = static_cast<const SentBlock*>(root_object);
return obj->src_idx_ == src_idx && obj->root_hash_ == root_hash && obj->file_hash_ == file_hash &&
obj->collated_data_file_hash_ == collated_data_file_hash && obj->hash_ == hash;
}
static auto lookup(ValidatorSessionDescription& desc, td::uint32 src_idx, const ValidatorSessionRootHash& root_hash,
const ValidatorSessionFileHash& file_hash,
const ValidatorSessionCollatedDataFileHash& collated_data_file_hash, HashType hash, bool temp) {
auto r = desc.get_by_hash(hash, temp);
if (compare(r, src_idx, root_hash, file_hash, collated_data_file_hash, hash)) {
desc.on_reuse();
return static_cast<const SentBlock*>(r);
}
return static_cast<const SentBlock*>(nullptr);
}
static const SentBlock* create(ValidatorSessionDescription& desc, td::uint32 src_idx,
const ValidatorSessionRootHash& root_hash, const ValidatorSessionFileHash& file_hash,
const ValidatorSessionCollatedDataFileHash& collated_data_file_hash) {
auto hash = create_hash(desc, src_idx, root_hash, file_hash, collated_data_file_hash);
auto r = lookup(desc, src_idx, root_hash, file_hash, collated_data_file_hash, hash, true);
if (r) {
return r;
}
auto candidate_id = desc.candidate_id(src_idx, root_hash, file_hash, collated_data_file_hash);
return new (desc, true) SentBlock{desc, src_idx, root_hash, file_hash, collated_data_file_hash, candidate_id, hash};
}
static const SentBlock* create(ValidatorSessionDescription& desc, const ValidatorSessionCandidateId& zero) {
CHECK(zero.is_zero());
auto hash = create_hash(desc, 0, ValidatorSessionRootHash::zero(), ValidatorSessionFileHash::zero(),
ValidatorSessionCollatedDataFileHash::zero());
return new (desc, true) SentBlock{desc,
0,
ValidatorSessionRootHash::zero(),
ValidatorSessionFileHash::zero(),
ValidatorSessionCollatedDataFileHash::zero(),
zero,
hash};
}
static const SentBlock* move_to_persistent(ValidatorSessionDescription& desc, const SentBlock* b) {
if (desc.is_persistent(b)) {
return b;
}
auto r = lookup(desc, b->src_idx_, b->root_hash_, b->file_hash_, b->collated_data_file_hash_, b->hash_, false);
if (r) {
return r;
}
return new (desc, false) SentBlock{
desc, b->src_idx_, b->root_hash_, b->file_hash_, b->collated_data_file_hash_, b->candidate_id_, b->hash_};
}
SentBlock(ValidatorSessionDescription& desc, td::uint32 src_idx, ValidatorSessionRootHash root_hash,
ValidatorSessionFileHash file_hash, ValidatorSessionCollatedDataFileHash collated_data_file_hash,
ValidatorSessionCandidateId candidate_id, HashType hash)
: RootObject(sizeof(SentBlock))
, src_idx_(src_idx)
, root_hash_(std::move(root_hash))
, file_hash_(std::move(file_hash))
, collated_data_file_hash_(std::move(collated_data_file_hash))
, candidate_id_(candidate_id)
, hash_(std::move(hash)) {
desc.update_hash(this, hash_);
}
auto get_src_idx() const {
return src_idx_;
}
auto get_root_hash() const {
return root_hash_;
}
auto get_file_hash() const {
return file_hash_;
}
auto get_collated_data_file_hash() const {
return collated_data_file_hash_;
}
static ValidatorSessionCandidateId get_block_id(const SentBlock* block) {
return block ? block->candidate_id_ : skip_round_candidate_id();
}
HashType get_hash(ValidatorSessionDescription& desc) const {
return hash_;
}
bool operator<(const SentBlock& block) const {
if (src_idx_ < block.src_idx_) {
return true;
}
if (src_idx_ > block.src_idx_) {
return false;
}
if (candidate_id_ < block.candidate_id_) {
return true;
}
return false;
}
struct Compare {
bool operator()(const SentBlock* a, const SentBlock* b) const {
return *a < *b;
}
};
private:
const td::uint32 src_idx_;
const ValidatorSessionRootHash root_hash_;
const ValidatorSessionFileHash file_hash_;
const ValidatorSessionCollatedDataFileHash collated_data_file_hash_;
const ValidatorSessionCandidateId candidate_id_;
const HashType hash_;
};
class SessionBlockCandidate : public ValidatorSessionDescription::RootObject {
public:
static HashType create_hash(ValidatorSessionDescription& desc, HashType block, HashType approved) {
auto obj = create_tl_object<ton_api::hashable_blockCandidate>(block, approved);
return desc.compute_hash(serialize_tl_object(obj, true).as_slice());
}
static bool compare(const RootObject* r, const SentBlock* block, const SessionBlockCandidateSignatureVector* approved,
HashType hash) {
if (!r || r->get_size() < sizeof(SessionBlockCandidate)) {
return false;
}
auto R = static_cast<const SessionBlockCandidate*>(r);
return R->block_ == block && R->approved_by_ == approved && R->hash_ == hash;
}
static auto lookup(ValidatorSessionDescription& desc, const SentBlock* block,
const SessionBlockCandidateSignatureVector* approved, HashType hash, bool temp) {
auto r = desc.get_by_hash(hash, temp);
if (compare(r, block, approved, hash)) {
desc.on_reuse();
return static_cast<const SessionBlockCandidate*>(r);
}
return static_cast<const SessionBlockCandidate*>(nullptr);
}
static const SessionBlockCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block,
const SessionBlockCandidateSignatureVector* approved) {
auto hash = create_hash(desc, get_vs_hash(desc, block), get_vs_hash(desc, approved));
auto r = lookup(desc, block, approved, hash, true);
if (r) {
return r;
}
return new (desc, true) SessionBlockCandidate(desc, block, approved, hash);
}
static const SessionBlockCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block) {
std::vector<const SessionBlockCandidateSignature*> v;
v.resize(desc.get_total_nodes(), nullptr);
auto vec = SessionBlockCandidateSignatureVector::create(desc, std::move(v));
return create(desc, block, vec);
}
static const SessionBlockCandidate* move_to_persistent(ValidatorSessionDescription& desc,
const SessionBlockCandidate* b) {
if (desc.is_persistent(b)) {
return b;
}
auto block = SentBlock::move_to_persistent(desc, b->block_);
auto approved = SessionBlockCandidateSignatureVector::move_to_persistent(desc, b->approved_by_);
auto r = lookup(desc, block, approved, b->hash_, false);
if (r) {
return r;
}
return new (desc, false) SessionBlockCandidate{desc, block, approved, b->hash_};
}
SessionBlockCandidate(ValidatorSessionDescription& desc, const SentBlock* block,
const SessionBlockCandidateSignatureVector* approved, HashType hash)
: RootObject{sizeof(SessionBlockCandidate)}, block_(block), approved_by_(approved), hash_(hash) {
desc.update_hash(this, hash_);
}
static const SessionBlockCandidate* merge(ValidatorSessionDescription& desc, const SessionBlockCandidate* l,
const SessionBlockCandidate* r);
auto get_block() const {
return block_;
}
auto get_id() const {
return SentBlock::get_block_id(block_);
}
auto get_src_idx() const {
return block_ ? block_->get_src_idx() : std::numeric_limits<td::uint32>::max();
}
bool check_block_is_approved_by(td::uint32 src_idx) const {
return approved_by_->at(src_idx);
}
bool check_block_is_approved(ValidatorSessionDescription& desc) const;
auto get_hash(ValidatorSessionDescription& desc) const {
return hash_;
}
auto get_approvers_list() const {
return approved_by_;
}
static const SessionBlockCandidate* push(ValidatorSessionDescription& desc, const SessionBlockCandidate* state,
td::uint32 src_idx, const SessionBlockCandidateSignature* sig) {
CHECK(state);
if (state->approved_by_->at(src_idx)) {
return state;
}
return create(desc, state->block_,
SessionBlockCandidateSignatureVector::change(desc, state->approved_by_, src_idx, sig));
}
class Compare {
public:
bool operator()(const SessionBlockCandidate* l, const SessionBlockCandidate* r) {
return l->get_id() < r->get_id();
}
};
private:
const SentBlock* block_;
const SessionBlockCandidateSignatureVector* approved_by_;
const HashType hash_;
};
class SessionVoteCandidate : public ValidatorSessionDescription::RootObject {
public:
static HashType create_hash(ValidatorSessionDescription& desc, HashType block, HashType voted) {
auto obj = create_tl_object<ton_api::hashable_blockVoteCandidate>(block, voted);
return desc.compute_hash(serialize_tl_object(obj, true).as_slice());
}
static bool compare(const RootObject* r, const SentBlock* block, const CntVector<bool>* voted, HashType hash) {
if (!r || r->get_size() < sizeof(SessionVoteCandidate)) {
return false;
}
auto R = static_cast<const SessionVoteCandidate*>(r);
return R->block_ == block && R->voted_by_ == voted && R->hash_ == hash;
}
static auto lookup(ValidatorSessionDescription& desc, const SentBlock* block, const CntVector<bool>* voted,
HashType hash, bool temp) {
auto r = desc.get_by_hash(hash, temp);
if (compare(r, block, voted, hash)) {
desc.on_reuse();
return static_cast<const SessionVoteCandidate*>(r);
}
return static_cast<const SessionVoteCandidate*>(nullptr);
}
static const SessionVoteCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block,
const CntVector<bool>* voted) {
auto hash = create_hash(desc, get_vs_hash(desc, block), get_vs_hash(desc, voted));
auto r = lookup(desc, block, voted, hash, true);
if (r) {
return r;
}
return new (desc, true) SessionVoteCandidate(desc, block, voted, hash);
}
static const SessionVoteCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block) {
std::vector<bool> v;
v.resize(desc.get_total_nodes(), false);
auto vec = CntVector<bool>::create(desc, std::move(v));
return create(desc, block, vec);
}
static const SessionVoteCandidate* move_to_persistent(ValidatorSessionDescription& desc,
const SessionVoteCandidate* b) {
if (desc.is_persistent(b)) {
return b;
}
auto block = SentBlock::move_to_persistent(desc, b->block_);
auto voted = CntVector<bool>::move_to_persistent(desc, b->voted_by_);
auto r = lookup(desc, block, voted, b->hash_, false);
if (r) {
return r;
}
return new (desc, false) SessionVoteCandidate{desc, block, voted, b->hash_};
}
SessionVoteCandidate(ValidatorSessionDescription& desc, const SentBlock* block, const CntVector<bool>* voted,
HashType hash)
: RootObject{sizeof(SessionVoteCandidate)}, block_(block), voted_by_(voted), hash_(hash) {
desc.update_hash(this, hash_);
}
static const SessionVoteCandidate* merge(ValidatorSessionDescription& desc, const SessionVoteCandidate* l,
const SessionVoteCandidate* r);
auto get_block() const {
return block_;
}
auto get_id() const {
return SentBlock::get_block_id(block_);
}
auto get_src_idx() const {
return block_ ? block_->get_src_idx() : std::numeric_limits<td::uint32>::max();
}
bool check_block_is_voted_by(td::uint32 src_idx) const {
return voted_by_->at(src_idx);
}
bool check_block_is_voted(ValidatorSessionDescription& desc) const;
auto get_hash(ValidatorSessionDescription& desc) const {
return hash_;
}
auto get_voters_list() const {
return voted_by_;
}
static const SessionVoteCandidate* push(ValidatorSessionDescription& desc, const SessionVoteCandidate* state,
td::uint32 src_idx) {
CHECK(state);
if (state->voted_by_->at(src_idx)) {
return state;
}
return create(desc, state->block_, CntVector<bool>::change(desc, state->voted_by_, src_idx, true));
}
class Compare {
public:
bool operator()(const SessionVoteCandidate* l, const SessionVoteCandidate* r) {
return l->get_id() < r->get_id();
}
};
private:
const SentBlock* block_;
const CntVector<bool>* voted_by_;
const HashType hash_;
};
using VoteVector = CntSortedVector<const SessionVoteCandidate*, SessionVoteCandidate::Compare>;
using ApproveVector = CntSortedVector<const SessionBlockCandidate*, SessionBlockCandidate::Compare>;
class ValidatorSessionRoundState;
class ValidatorSessionRoundAttemptState : public ValidatorSessionDescription::RootObject {
public:
static HashType create_hash(ValidatorSessionDescription& desc, td::uint32 seqno, HashType votes,
HashType precommitted, bool vote_for_inited, HashType vote_for) {
auto obj = create_tl_object<ton_api::hashable_validatorSessionRoundAttempt>(seqno, votes, precommitted,
vote_for_inited, vote_for);
return desc.compute_hash(serialize_tl_object(obj, true).as_slice());
}
static bool compare(const RootObject* r, td::uint32 seqno, const VoteVector* votes,
const CntVector<bool>* precommitted, const SentBlock* vote_for, bool vote_for_inited,
HashType hash) {
if (!r || r->get_size() < sizeof(ValidatorSessionRoundAttemptState)) {
return false;
}
auto R = static_cast<const ValidatorSessionRoundAttemptState*>(r);
return R->seqno_ == seqno && R->votes_ == votes && R->precommitted_ == precommitted && R->vote_for_ == vote_for &&
R->vote_for_inited_ == vote_for_inited && R->hash_ == hash;
}
static auto lookup(ValidatorSessionDescription& desc, td::uint32 seqno, const VoteVector* votes,
const CntVector<bool>* precommitted, const SentBlock* vote_for, bool vote_for_inited,
HashType hash, bool temp) {
auto r = desc.get_by_hash(hash, temp);
if (compare(r, seqno, votes, precommitted, vote_for, vote_for_inited, hash)) {
desc.on_reuse();
return static_cast<const ValidatorSessionRoundAttemptState*>(r);
}
return static_cast<const ValidatorSessionRoundAttemptState*>(nullptr);
}
static const ValidatorSessionRoundAttemptState* move_to_persistent(ValidatorSessionDescription& desc,
const ValidatorSessionRoundAttemptState* b) {
if (desc.is_persistent(b)) {
return b;
}
auto votes = VoteVector::move_to_persistent(desc, b->votes_);
auto precommitted = CntVector<bool>::move_to_persistent(desc, b->precommitted_);
auto vote_for = SentBlock::move_to_persistent(desc, b->vote_for_);
auto r = lookup(desc, b->seqno_, votes, precommitted, vote_for, b->vote_for_inited_, b->hash_, false);
if (r) {
return r;
}
return new (desc, false) ValidatorSessionRoundAttemptState{desc, b->seqno_, votes, precommitted,
vote_for, b->vote_for_inited_, b->hash_};
}
static const ValidatorSessionRoundAttemptState* create(ValidatorSessionDescription& desc, td::uint32 seqno,
const VoteVector* votes, const CntVector<bool>* precommitted,
const SentBlock* vote_for, bool vote_for_inited) {
auto hash = create_hash(desc, seqno, get_vs_hash(desc, votes), get_vs_hash(desc, precommitted),
get_vs_hash(desc, vote_for), vote_for_inited);
auto r = lookup(desc, seqno, votes, precommitted, vote_for, vote_for_inited, hash, true);
if (r) {
return r;
}
return new (desc, true)
ValidatorSessionRoundAttemptState(desc, seqno, votes, precommitted, vote_for, vote_for_inited, hash);
}
static const ValidatorSessionRoundAttemptState* create(ValidatorSessionDescription& desc, td::uint32 seqno) {
std::vector<bool> x;
x.resize(desc.get_total_nodes(), false);
auto p = CntVector<bool>::create(desc, std::move(x));
return create(desc, seqno, nullptr, p, nullptr, false);
}
ValidatorSessionRoundAttemptState(ValidatorSessionDescription& desc, td::uint32 seqno, const VoteVector* votes,
const CntVector<bool>* precommitted, const SentBlock* vote_for,
bool vote_for_inited, HashType hash)
: RootObject{sizeof(ValidatorSessionRoundAttemptState)}
, seqno_(seqno)
, votes_(votes)
, precommitted_(precommitted)
, vote_for_(vote_for)
, vote_for_inited_(vote_for_inited)
, hash_(std::move(hash)) {
desc.update_hash(this, hash_);
}
auto get_hash(ValidatorSessionDescription& desc) const {
return hash_;
}
auto get_seqno() const {
return seqno_;
}
auto get_votes() const {
return votes_;
}
auto get_precommits() const {
return precommitted_;
}
const SentBlock* get_voted_block(ValidatorSessionDescription& desc, bool& f) const;
const SentBlock* get_vote_for_block(ValidatorSessionDescription& desc, bool& f) const {
f = vote_for_inited_;
return vote_for_;
}
bool check_attempt_is_precommitted(ValidatorSessionDescription& desc) const;
bool check_vote_received_from(td::uint32 src_idx) const;
bool check_precommit_received_from(td::uint32 src_idx) const;
bool operator<(const ValidatorSessionRoundAttemptState& right) const {
return seqno_ < right.seqno_;
}
struct Compare {
bool operator()(const ValidatorSessionRoundAttemptState* a, const ValidatorSessionRoundAttemptState* b) const {
return *a < *b;
}
};
static const ValidatorSessionRoundAttemptState* merge(ValidatorSessionDescription& desc,
const ValidatorSessionRoundAttemptState* left,
const ValidatorSessionRoundAttemptState* right);
static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc,
const ValidatorSessionRoundAttemptState* state,
td::uint32 src_idx, td::uint32 att,
const ton_api::validatorSession_message_voteFor& act,
const ValidatorSessionRoundState* round);
static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc,
const ValidatorSessionRoundAttemptState* state,
td::uint32 src_idx, td::uint32 att,
const ton_api::validatorSession_message_vote& act,
const ValidatorSessionRoundState* round);
static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc,
const ValidatorSessionRoundAttemptState* state,
td::uint32 src_idx, td::uint32 att,
const ton_api::validatorSession_message_precommit& act,
const ValidatorSessionRoundState* round);
static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc,
const ValidatorSessionRoundAttemptState* state,
td::uint32 src_idx, td::uint32 att,
const ton_api::validatorSession_message_empty& act,
const ValidatorSessionRoundState* round);
static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc,
const ValidatorSessionRoundAttemptState* state,
td::uint32 src_idx, td::uint32 att,
const ton_api::validatorSession_round_Message* action,
const ValidatorSessionRoundState* round);
template <class T>
static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc,
const ValidatorSessionRoundAttemptState* state,
td::uint32 src_idx, td::uint32 att, const T& action,
const ValidatorSessionRoundState* round) {
UNREACHABLE();
}
static const ValidatorSessionRoundAttemptState* try_vote(ValidatorSessionDescription& desc,
const ValidatorSessionRoundAttemptState* state,
td::uint32 src_idx, td::uint32 att,
const ValidatorSessionRoundState* round,
const ton_api::validatorSession_round_Message* cmp,
bool& made);
static const ValidatorSessionRoundAttemptState* try_precommit(ValidatorSessionDescription& desc,
const ValidatorSessionRoundAttemptState* state,
td::uint32 src_idx, td::uint32 att,
const ValidatorSessionRoundState* round,
const ton_api::validatorSession_round_Message* cmp,
bool& made);
static const ValidatorSessionRoundAttemptState* make_one(ValidatorSessionDescription& desc,
const ValidatorSessionRoundAttemptState* state,
td::uint32 src_idx, td::uint32 att,
const ValidatorSessionRoundState* round,
const ton_api::validatorSession_round_Message* cmp,
bool& made);
tl_object_ptr<ton_api::validatorSession_round_Message> create_action(ValidatorSessionDescription& desc,
const ValidatorSessionRoundState* round,
td::uint32 src_idx, td::uint32 att) const;
void dump(ValidatorSessionDescription& desc, td::StringBuilder& sb) const;
private:
const td::uint32 seqno_;
const VoteVector* votes_;
const CntVector<bool>* precommitted_;
const SentBlock* vote_for_;
const bool vote_for_inited_;
const HashType hash_;
};
using AttemptVector =
CntSortedVector<const ValidatorSessionRoundAttemptState*, ValidatorSessionRoundAttemptState::Compare>;
} // namespace validatorsession
} // namespace ton

473
validator-session/validator-session-state.cpp
View file

@ -67,14 +67,6 @@ namespace ton {
namespace validatorsession {
static td::uint32 get_round_id(const ton_api::validatorSession_round_Message* message) {
td::uint32 round = 0;
bool is_called = ton_api::downcast_call(*const_cast<ton_api::validatorSession_round_Message*>(message),
[&](auto& obj) { round = obj.round_; });
CHECK(is_called);
return round;
}
static const ValidatorSessionRoundAttemptState* get_attempt(const AttemptVector* vec, td::uint32 seqno) {
if (!vec) {
return nullptr;
@ -108,118 +100,12 @@ static const SessionBlockCandidate* get_candidate(const ApproveVector* vec, Vali
return nullptr;
}
static const SessionVoteCandidate* get_candidate(const VoteVector* vec, ValidatorSessionCandidateId id) {
if (!vec) {
return nullptr;
}
auto size = vec->size();
auto v = vec->data();
for (td::uint32 i = 0; i < size; i++) {
if (v[i]->get_id() == id) {
return v[i];
}
}
return nullptr;
}
//
//
// SessionBlockCandidateSignature
//
//
const SessionBlockCandidateSignature* SessionBlockCandidateSignature::merge(ValidatorSessionDescription& desc,
const SessionBlockCandidateSignature* l,
const SessionBlockCandidateSignature* r) {
if (!l) {
return r;
}
if (!r) {
return l;
}
if (l == r) {
return l;
}
if (l->as_slice() < r->as_slice()) {
return l;
} else {
return r;
}
}
//
//
// SessionBlockCandidate
//
//
bool SessionBlockCandidate::check_block_is_approved(ValidatorSessionDescription& desc) const {
ValidatorWeight w = 0;
for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) {
if (approved_by_->at(i)) {
w += desc.get_node_weight(i);
if (w >= desc.get_cutoff_weight()) {
return true;
}
}
}
return false;
}
const SessionBlockCandidate* SessionBlockCandidate::merge(ValidatorSessionDescription& desc,
const SessionBlockCandidate* l,
const SessionBlockCandidate* r) {
if (!l) {
return r;
}
if (!r) {
return l;
}
if (l == r) {
return l;
}
CHECK(l->get_id() == r->get_id());
auto v = SessionBlockCandidateSignatureVector::merge(
desc, l->approved_by_, r->approved_by_,
[&](const SessionBlockCandidateSignature* l, const SessionBlockCandidateSignature* r) {
return SessionBlockCandidateSignature::merge(desc, l, r);
});
return SessionBlockCandidate::create(desc, l->block_, std::move(v));
}
//
//
// SessionVoteCandidate
//
//
bool SessionVoteCandidate::check_block_is_voted(ValidatorSessionDescription& desc) const {
ValidatorWeight w = 0;
for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) {
if (voted_by_->at(i)) {
w += desc.get_node_weight(i);
if (w >= desc.get_cutoff_weight()) {
return true;
}
}
}
return false;
}
const SessionVoteCandidate* SessionVoteCandidate::merge(ValidatorSessionDescription& desc,
const SessionVoteCandidate* l, const SessionVoteCandidate* r) {
if (!l) {
return r;
}
if (!r) {
return l;
}
if (l == r) {
return l;
}
CHECK(l->get_id() == r->get_id());
auto v = CntVector<bool>::merge(desc, l->voted_by_, r->voted_by_);
return SessionVoteCandidate::create(desc, l->block_, std::move(v));
static td::uint32 get_round_id(const ton_api::validatorSession_round_Message* message) {
td::uint32 round = 0;
bool is_called = ton_api::downcast_call(*const_cast<ton_api::validatorSession_round_Message*>(message),
[&](auto& obj) { round = obj.round_; });
CHECK(is_called);
return round;
}
//
@ -374,346 +260,6 @@ const ValidatorSessionOldRoundState* ValidatorSessionOldRoundState::action(
return state;
}
//
//
// ATTEMPT STATE
//
//
const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::merge(
ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* left,
const ValidatorSessionRoundAttemptState* right) {
if (!left) {
return right;
}
if (!right) {
return left;
}
if (left == right) {
return left;
}
CHECK(left->seqno_ == right->seqno_);
const SentBlock* vote_for = nullptr;
bool vote_for_inited = false;
if (!left->vote_for_inited_) {
vote_for = right->vote_for_;
vote_for_inited = right->vote_for_inited_;
} else if (!right->vote_for_inited_) {
vote_for = left->vote_for_;
vote_for_inited = left->vote_for_inited_;
} else if (left->vote_for_ == right->vote_for_) {
vote_for_inited = true;
vote_for = left->vote_for_;
} else {
auto l = SentBlock::get_block_id(left->vote_for_);
auto r = SentBlock::get_block_id(right->vote_for_);
vote_for_inited = true;
if (l < r) {
vote_for = left->vote_for_;
} else {
vote_for = right->vote_for_;
}
}
auto precommitted = CntVector<bool>::merge(desc, left->precommitted_, right->precommitted_);
auto votes = VoteVector::merge(desc, left->votes_, right->votes_,
[&](const SessionVoteCandidate* l, const SessionVoteCandidate* r) {
return SessionVoteCandidate::merge(desc, l, r);
});
return ValidatorSessionRoundAttemptState::create(desc, left->seqno_, std::move(votes), std::move(precommitted),
vote_for, vote_for_inited);
}
const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action(
ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx,
td::uint32 att, const ton_api::validatorSession_message_voteFor& act, const ValidatorSessionRoundState* round) {
if (state->vote_for_inited_) {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act
<< "]: invalid message: duplicate VOTEFOR";
return state;
}
if (src_idx != desc.get_vote_for_author(att)) {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act
<< "]: invalid message: bad VOTEFOR author";
return state;
}
if (round->get_first_attempt(src_idx) == 0 && desc.opts().max_round_attempts > 0) {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act
<< "]: invalid message: too early for VOTEFOR";
return state;
}
if (round->get_first_attempt(src_idx) + desc.opts().max_round_attempts > att && desc.opts().max_round_attempts == 0) {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act
<< "]: invalid message: too early for VOTEFOR";
return state;
}
auto x = round->get_block(act.candidate_);
if (!x) {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act
<< "]: invalid message: VOTEFOR for not approved block";
return state;
}
if (!x->check_block_is_approved(desc)) {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act
<< "]: invalid message: VOTEFOR for not approved block";
return state;
}
return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, state->votes_, state->precommitted_,
x->get_block(), true);
}
const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action(
ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx,
td::uint32 att, const ton_api::validatorSession_message_vote& act, const ValidatorSessionRoundState* round) {
bool made;
return make_one(desc, state, src_idx, att, round, &act, made);
}
const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action(
ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx,
td::uint32 att, const ton_api::validatorSession_message_precommit& act, const ValidatorSessionRoundState* round) {
bool made;
return make_one(desc, state, src_idx, att, round, &act, made);
}
const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action(
ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx,
td::uint32 att, const ton_api::validatorSession_message_empty& act, const ValidatorSessionRoundState* round) {
bool made;
return make_one(desc, state, src_idx, att, round, &act, made);
}
const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::try_vote(
ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx,
td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act,
bool& made) {
made = false;
if (state->check_vote_received_from(src_idx)) {
return state;
}
auto found = false;
auto block = round->choose_block_to_vote(desc, src_idx, att, state->vote_for_, state->vote_for_inited_, found);
if (!found) {
return state;
}
auto block_id = SentBlock::get_block_id(block);
made = true;
if (act) {
if (act->get_id() != ton_api::validatorSession_message_vote::ID) {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act
<< "]: expected VOTE(" << block_id << ")";
} else {
auto x = static_cast<const ton_api::validatorSession_message_vote*>(act);
if (x->candidate_ != block_id) {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act
<< "]: expected VOTE(" << block_id << ")";
}
}
} else {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx)
<< "]: making implicit VOTE(" << block_id << ")";
}
auto candidate = get_candidate(state->votes_, block_id);
if (!candidate) {
candidate = SessionVoteCandidate::create(desc, block);
}
candidate = SessionVoteCandidate::push(desc, candidate, src_idx);
auto v = VoteVector::push(desc, state->votes_, candidate);
return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, std::move(v), state->precommitted_,
state->vote_for_, state->vote_for_inited_);
}
const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::try_precommit(
ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx,
td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act,
bool& made) {
made = false;
if (state->check_precommit_received_from(src_idx)) {
return state;
}
bool found;
auto block = state->get_voted_block(desc, found);
if (!found) {
return state;
}
made = true;
auto block_id = SentBlock::get_block_id(block);
if (act) {
if (act->get_id() != ton_api::validatorSession_message_precommit::ID) {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act
<< "]: expected PRECOMMIT(" << block_id << ")";
} else {
auto x = static_cast<const ton_api::validatorSession_message_precommit*>(act);
if (x->candidate_ != block_id) {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act
<< "]: expected PRECOMMIT(" << block_id << ")";
}
}
} else {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx)
<< "]: making implicit PRECOMMIT(" << block_id << ")";
}
auto v = CntVector<bool>::change(desc, state->precommitted_, src_idx, true);
return ValidatorSessionRoundAttemptState::create(desc, state->seqno_, state->votes_, std::move(v), state->vote_for_,
state->vote_for_inited_);
}
const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::make_one(
ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx,
td::uint32 att, const ValidatorSessionRoundState* round, const ton_api::validatorSession_round_Message* act,
bool& made) {
made = false;
state = try_vote(desc, state, src_idx, att, round, act, made);
if (made) {
return state;
}
state = try_precommit(desc, state, src_idx, att, round, act, made);
if (made) {
return state;
}
if (act && act->get_id() != ton_api::validatorSession_message_empty::ID) {
VLOG(VALIDATOR_SESSION_WARNING) << "[validator session][node " << desc.get_source_id(src_idx) << "][" << act
<< "]: invalid message: expected EMPTY";
}
return state;
}
const ValidatorSessionRoundAttemptState* ValidatorSessionRoundAttemptState::action(
ValidatorSessionDescription& desc, const ValidatorSessionRoundAttemptState* state, td::uint32 src_idx,
td::uint32 att, const ton_api::validatorSession_round_Message* act, const ValidatorSessionRoundState* round) {
ton_api::downcast_call(*const_cast<ton_api::validatorSession_round_Message*>(act),
[&](auto& obj) { state = action(desc, state, src_idx, att, obj, round); });
return state;
}
bool ValidatorSessionRoundAttemptState::check_vote_received_from(td::uint32 src_idx) const {
if (!votes_) {
return false;
}
auto size = votes_->size();
auto v = votes_->data();
for (td::uint32 i = 0; i < size; i++) {
if (v[i]->check_block_is_voted_by(src_idx)) {
return true;
}
}
return false;
}
bool ValidatorSessionRoundAttemptState::check_precommit_received_from(td::uint32 src_idx) const {
return precommitted_->at(src_idx);
}
const SentBlock* ValidatorSessionRoundAttemptState::get_voted_block(ValidatorSessionDescription& desc, bool& f) const {
f = false;
if (!votes_) {
return nullptr;
}
auto size = votes_->size();
auto v = votes_->data();
for (td::uint32 i = 0; i < size; i++) {
if (v[i]->check_block_is_voted(desc)) {
f = true;
return v[i]->get_block();
}
}
return nullptr;
}
bool ValidatorSessionRoundAttemptState::check_attempt_is_precommitted(ValidatorSessionDescription& desc) const {
ValidatorWeight weight = 0;
for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) {
if (precommitted_->at(i)) {
weight += desc.get_node_weight(i);
if (weight >= desc.get_cutoff_weight()) {
return true;
}
}
}
return false;
}
tl_object_ptr<ton_api::validatorSession_round_Message> ValidatorSessionRoundAttemptState::create_action(
ValidatorSessionDescription& desc, const ValidatorSessionRoundState* round, td::uint32 src_idx,
td::uint32 att) const {
if (!check_vote_received_from(src_idx)) {
auto found = false;
auto B = round->choose_block_to_vote(desc, src_idx, att, vote_for_, vote_for_inited_, found);
if (found) {
auto block_id = SentBlock::get_block_id(B);
return create_tl_object<ton_api::validatorSession_message_vote>(round->get_seqno(), seqno_, block_id);
}
}
if (!check_precommit_received_from(src_idx)) {
bool f = false;
auto B = get_voted_block(desc, f);
if (f) {
auto block_id = SentBlock::get_block_id(B);
return create_tl_object<ton_api::validatorSession_message_precommit>(round->get_seqno(), seqno_, block_id);
}
}
return create_tl_object<ton_api::validatorSession_message_empty>(round->get_seqno(), seqno_);
}
void ValidatorSessionRoundAttemptState::dump(ValidatorSessionDescription& desc, td::StringBuilder& sb) const {
sb << "attempt=" << seqno_ << "\n";
sb << ">>>>\n";
if (vote_for_inited_) {
sb << "vote_for=" << (vote_for_ ? vote_for_->get_src_idx() : std::numeric_limits<td::uint32>::max()) << "\n";
} else {
sb << "vote_for=NONE\n";
}
if (votes_) {
auto s = votes_->size();
sb << "votes: ";
std::vector<td::int32> R;
R.resize(desc.get_total_nodes(), -1);
for (td::uint32 i = 0; i < s; i++) {
const auto e = votes_->at(i);
const auto& x = e->get_voters_list();
for (td::uint32 j = 0; j < desc.get_total_nodes(); j++) {
if (x->at(j)) {
R[j] = e->get_src_idx();
}
}
}
for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) {
sb << R[i] << " ";
}
sb << "\n";
} else {
sb << "votes: EMPTY\n";
}
sb << "precommits: ";
for (td::uint32 i = 0; i < desc.get_total_nodes(); i++) {
const auto e = precommitted_->at(i);
if (e) {
sb << "+ ";
} else {
sb << "- ";
}
}
sb << "\n";
sb << "<<<<\n";
}
//
//
// ROUND STATE
@ -821,8 +367,6 @@ const ValidatorSessionRoundState* ValidatorSessionRoundState::merge(ValidatorSes
return SessionBlockCandidateSignature::merge(desc, l, r);
});
//auto sent_vec = SentBlockVector::merge(desc, left->sent_blocks_, right->sent_blocks_);
auto sent = ApproveVector::merge(desc, left->sent_blocks_, right->sent_blocks_,
[&](const SessionBlockCandidate* l, const SessionBlockCandidate* r) {
return SessionBlockCandidate::merge(desc, l, r);
@ -985,9 +529,6 @@ const ValidatorSessionRoundState* ValidatorSessionRoundState::forward_action_to_
attempt = ValidatorSessionRoundAttemptState::create(desc, att);
}
bool had_voted_block;
attempt->get_voted_block(desc, had_voted_block);
ton_api::downcast_call(*const_cast<ton_api::validatorSession_round_Message*>(act), [&](auto& obj) {
attempt = ValidatorSessionRoundAttemptState::action(desc, attempt, src_idx, att, obj, state);
});
@ -1302,6 +843,8 @@ std::vector<const SentBlock*> ValidatorSessionRoundState::choose_blocks_to_appro
CHECK(prio >= 0);
td::uint32 blk_src_idx = B->get_src_idx();
if (was_source.count(blk_src_idx) > 0) {
// Any honest validator submits at most one block in a round
// Therefore, we can ignore all blocks from a node if it submits more than one
x[prio] = nullptr;
} else {
was_source.insert(blk_src_idx);

590
validator-session/validator-session-state.h
View file

@ -24,15 +24,15 @@
#include "common/io.hpp"
#include "persistent-vector.h"
#include "validator-session-description.h"
#include "validator-session-types.h"
#include "validator-session-round-attempt-state.h"
#include <limits>
namespace td {
td::StringBuilder& operator<<(td::StringBuilder& sb, const ton::ton_api::validatorSession_round_Message& message);
td::StringBuilder& operator<<(td::StringBuilder& sb, const ton::ton_api::validatorSession_round_Message* message);
}
@ -41,410 +41,6 @@ namespace ton {
namespace validatorsession {
using HashType = ValidatorSessionDescription::HashType;
constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_WARNING) = verbosity_WARNING;
constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_NOTICE) = verbosity_DEBUG;
constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_INFO) = verbosity_DEBUG;
constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_DEBUG) = verbosity_DEBUG;
constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_EXTRA_DEBUG) = verbosity_DEBUG + 1;
struct SessionBlockCandidateSignature : public ValidatorSessionDescription::RootObject {
public:
static auto create_hash(ValidatorSessionDescription& desc, td::Slice data) {
auto obj = create_tl_object<ton_api::hashable_blockSignature>(desc.compute_hash(data));
return desc.compute_hash(serialize_tl_object(obj, true).as_slice());
}
static bool compare(const RootObject* r, td::Slice data, HashType hash) {
if (!r || r->get_size() < sizeof(SessionBlockCandidateSignature)) {
return false;
}
auto R = static_cast<const SessionBlockCandidateSignature*>(r);
return R->hash_ == hash && R->data_.ubegin() == data.ubegin() && R->data_.size() == data.size();
}
static auto lookup(ValidatorSessionDescription& desc, td::Slice data, HashType hash, bool temp) {
auto r = desc.get_by_hash(hash, temp);
if (compare(r, data, hash)) {
desc.on_reuse();
return static_cast<const SessionBlockCandidateSignature*>(r);
}
return static_cast<const SessionBlockCandidateSignature*>(nullptr);
}
static SessionBlockCandidateSignature* create(ValidatorSessionDescription& desc, td::BufferSlice value) {
auto hash = create_hash(desc, value.as_slice());
auto d = static_cast<td::uint8*>(desc.alloc(value.size(), 8, false));
td::MutableSlice s{d, value.size()};
s.copy_from(value.as_slice());
return new (desc, true) SessionBlockCandidateSignature{desc, s, hash};
}
static const SessionBlockCandidateSignature* move_to_persistent(ValidatorSessionDescription& desc,
const SessionBlockCandidateSignature* b) {
if (desc.is_persistent(b)) {
return b;
}
CHECK(desc.is_persistent(b->data_.ubegin()));
auto r = lookup(desc, b->data_, b->hash_, false);
if (r) {
return r;
}
return new (desc, false) SessionBlockCandidateSignature{desc, b->data_, b->hash_};
}
static const SessionBlockCandidateSignature* merge(ValidatorSessionDescription& desc,
const SessionBlockCandidateSignature* l,
const SessionBlockCandidateSignature* r);
SessionBlockCandidateSignature(ValidatorSessionDescription& desc, td::Slice data, HashType hash)
: RootObject(sizeof(SessionBlockCandidateSignature)), data_{data}, hash_(std::move(hash)) {
desc.update_hash(this, hash_);
}
td::BufferSlice value() const {
return td::BufferSlice{data_};
}
td::Slice as_slice() const {
return data_;
}
auto get_hash(ValidatorSessionDescription& desc) const {
return hash_;
}
private:
const td::Slice data_;
const HashType hash_;
};
using SessionBlockCandidateSignatureVector = CntVector<const SessionBlockCandidateSignature*>;
class SentBlock : public ValidatorSessionDescription::RootObject {
public:
static HashType create_hash(ValidatorSessionDescription& desc, td::uint32 src_idx, ValidatorSessionRootHash root_hash,
ValidatorSessionFileHash file_hash,
ValidatorSessionCollatedDataFileHash collated_data_file_hash) {
auto obj = create_tl_object<ton_api::hashable_sentBlock>(src_idx, get_vs_hash(desc, root_hash),
get_vs_hash(desc, file_hash),
get_vs_hash(desc, collated_data_file_hash));
return desc.compute_hash(serialize_tl_object(obj, true).as_slice());
}
static bool compare(const RootObject* root_object, td::uint32 src_idx, const ValidatorSessionRootHash& root_hash,
const ValidatorSessionFileHash& file_hash,
const ValidatorSessionCollatedDataFileHash& collated_data_file_hash, HashType hash) {
if (!root_object || root_object->get_size() < sizeof(SentBlock)) {
return false;
}
auto obj = static_cast<const SentBlock*>(root_object);
return obj->src_idx_ == src_idx && obj->root_hash_ == root_hash && obj->file_hash_ == file_hash &&
obj->collated_data_file_hash_ == collated_data_file_hash && obj->hash_ == hash;
}
static auto lookup(ValidatorSessionDescription& desc, td::uint32 src_idx, const ValidatorSessionRootHash& root_hash,
const ValidatorSessionFileHash& file_hash,
const ValidatorSessionCollatedDataFileHash& collated_data_file_hash, HashType hash, bool temp) {
auto r = desc.get_by_hash(hash, temp);
if (compare(r, src_idx, root_hash, file_hash, collated_data_file_hash, hash)) {
desc.on_reuse();
return static_cast<const SentBlock*>(r);
}
return static_cast<const SentBlock*>(nullptr);
}
static const SentBlock* create(ValidatorSessionDescription& desc, td::uint32 src_idx,
const ValidatorSessionRootHash& root_hash, const ValidatorSessionFileHash& file_hash,
const ValidatorSessionCollatedDataFileHash& collated_data_file_hash) {
auto hash = create_hash(desc, src_idx, root_hash, file_hash, collated_data_file_hash);
auto r = lookup(desc, src_idx, root_hash, file_hash, collated_data_file_hash, hash, true);
if (r) {
return r;
}
auto candidate_id = desc.candidate_id(src_idx, root_hash, file_hash, collated_data_file_hash);
return new (desc, true) SentBlock{desc, src_idx, root_hash, file_hash, collated_data_file_hash, candidate_id, hash};
}
static const SentBlock* create(ValidatorSessionDescription& desc, const ValidatorSessionCandidateId& zero) {
CHECK(zero.is_zero());
auto hash = create_hash(desc, 0, ValidatorSessionRootHash::zero(), ValidatorSessionFileHash::zero(),
ValidatorSessionCollatedDataFileHash::zero());
return new (desc, true) SentBlock{desc,
0,
ValidatorSessionRootHash::zero(),
ValidatorSessionFileHash::zero(),
ValidatorSessionCollatedDataFileHash::zero(),
zero,
hash};
}
static const SentBlock* move_to_persistent(ValidatorSessionDescription& desc, const SentBlock* b) {
if (desc.is_persistent(b)) {
return b;
}
auto r = lookup(desc, b->src_idx_, b->root_hash_, b->file_hash_, b->collated_data_file_hash_, b->hash_, false);
if (r) {
return r;
}
return new (desc, false) SentBlock{
desc, b->src_idx_, b->root_hash_, b->file_hash_, b->collated_data_file_hash_, b->candidate_id_, b->hash_};
}
SentBlock(ValidatorSessionDescription& desc, td::uint32 src_idx, ValidatorSessionRootHash root_hash,
ValidatorSessionFileHash file_hash, ValidatorSessionCollatedDataFileHash collated_data_file_hash,
ValidatorSessionCandidateId candidate_id, HashType hash)
: RootObject(sizeof(SentBlock))
, src_idx_(src_idx)
, root_hash_(std::move(root_hash))
, file_hash_(std::move(file_hash))
, collated_data_file_hash_(std::move(collated_data_file_hash))
, candidate_id_(candidate_id)
, hash_(std::move(hash)) {
desc.update_hash(this, hash_);
}
auto get_src_idx() const {
return src_idx_;
}
auto get_root_hash() const {
return root_hash_;
}
auto get_file_hash() const {
return file_hash_;
}
auto get_collated_data_file_hash() const {
return collated_data_file_hash_;
}
static ValidatorSessionCandidateId get_block_id(const SentBlock* block) {
return block ? block->candidate_id_ : skip_round_candidate_id();
}
HashType get_hash(ValidatorSessionDescription& desc) const {
return hash_;
}
bool operator<(const SentBlock& block) const {
if (src_idx_ < block.src_idx_) {
return true;
}
if (src_idx_ > block.src_idx_) {
return false;
}
if (candidate_id_ < block.candidate_id_) {
return true;
}
return false;
}
struct Compare {
bool operator()(const SentBlock* a, const SentBlock* b) const {
return *a < *b;
}
};
private:
const td::uint32 src_idx_;
const ValidatorSessionRootHash root_hash_;
const ValidatorSessionFileHash file_hash_;
const ValidatorSessionCollatedDataFileHash collated_data_file_hash_;
const ValidatorSessionCandidateId candidate_id_;
const HashType hash_;
};
class SessionBlockCandidate : public ValidatorSessionDescription::RootObject {
public:
static HashType create_hash(ValidatorSessionDescription& desc, HashType block, HashType approved) {
auto obj = create_tl_object<ton_api::hashable_blockCandidate>(block, approved);
return desc.compute_hash(serialize_tl_object(obj, true).as_slice());
}
static bool compare(const RootObject* r, const SentBlock* block, const SessionBlockCandidateSignatureVector* approved,
HashType hash) {
if (!r || r->get_size() < sizeof(SessionBlockCandidate)) {
return false;
}
auto R = static_cast<const SessionBlockCandidate*>(r);
return R->block_ == block && R->approved_by_ == approved && R->hash_ == hash;
}
static auto lookup(ValidatorSessionDescription& desc, const SentBlock* block,
const SessionBlockCandidateSignatureVector* approved, HashType hash, bool temp) {
auto r = desc.get_by_hash(hash, temp);
if (compare(r, block, approved, hash)) {
desc.on_reuse();
return static_cast<const SessionBlockCandidate*>(r);
}
return static_cast<const SessionBlockCandidate*>(nullptr);
}
static const SessionBlockCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block,
const SessionBlockCandidateSignatureVector* approved) {
auto hash = create_hash(desc, get_vs_hash(desc, block), get_vs_hash(desc, approved));
auto r = lookup(desc, block, approved, hash, true);
if (r) {
return r;
}
return new (desc, true) SessionBlockCandidate(desc, block, approved, hash);
}
static const SessionBlockCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block) {
std::vector<const SessionBlockCandidateSignature*> v;
v.resize(desc.get_total_nodes(), nullptr);
auto vec = SessionBlockCandidateSignatureVector::create(desc, std::move(v));
return create(desc, block, vec);
}
static const SessionBlockCandidate* move_to_persistent(ValidatorSessionDescription& desc,
const SessionBlockCandidate* b) {
if (desc.is_persistent(b)) {
return b;
}
auto block = SentBlock::move_to_persistent(desc, b->block_);
auto approved = SessionBlockCandidateSignatureVector::move_to_persistent(desc, b->approved_by_);
auto r = lookup(desc, block, approved, b->hash_, false);
if (r) {
return r;
}
return new (desc, false) SessionBlockCandidate{desc, block, approved, b->hash_};
}
SessionBlockCandidate(ValidatorSessionDescription& desc, const SentBlock* block,
const SessionBlockCandidateSignatureVector* approved, HashType hash)
: RootObject{sizeof(SessionBlockCandidate)}, block_(block), approved_by_(approved), hash_(hash) {
desc.update_hash(this, hash_);
}
static const SessionBlockCandidate* merge(ValidatorSessionDescription& desc, const SessionBlockCandidate* l,
const SessionBlockCandidate* r);
auto get_block() const {
return block_;
}
auto get_id() const {
return SentBlock::get_block_id(block_);
}
auto get_src_idx() const {
return block_ ? block_->get_src_idx() : std::numeric_limits<td::uint32>::max();
}
bool check_block_is_approved_by(td::uint32 src_idx) const {
return approved_by_->at(src_idx);
}
bool check_block_is_approved(ValidatorSessionDescription& desc) const;
auto get_hash(ValidatorSessionDescription& desc) const {
return hash_;
}
auto get_approvers_list() const {
return approved_by_;
}
static const SessionBlockCandidate* push(ValidatorSessionDescription& desc, const SessionBlockCandidate* state,
td::uint32 src_idx, const SessionBlockCandidateSignature* sig) {
CHECK(state);
if (state->approved_by_->at(src_idx)) {
return state;
}
return create(desc, state->block_,
SessionBlockCandidateSignatureVector::change(desc, state->approved_by_, src_idx, sig));
}
class Compare {
public:
bool operator()(const SessionBlockCandidate* l, const SessionBlockCandidate* r) {
return l->get_id() < r->get_id();
}
};
private:
const SentBlock* block_;
const SessionBlockCandidateSignatureVector* approved_by_;
const HashType hash_;
};
class SessionVoteCandidate : public ValidatorSessionDescription::RootObject {
public:
static HashType create_hash(ValidatorSessionDescription& desc, HashType block, HashType voted) {
auto obj = create_tl_object<ton_api::hashable_blockVoteCandidate>(block, voted);
return desc.compute_hash(serialize_tl_object(obj, true).as_slice());
}
static bool compare(const RootObject* r, const SentBlock* block, const CntVector<bool>* voted, HashType hash) {
if (!r || r->get_size() < sizeof(SessionVoteCandidate)) {
return false;
}
auto R = static_cast<const SessionVoteCandidate*>(r);
return R->block_ == block && R->voted_by_ == voted && R->hash_ == hash;
}
static auto lookup(ValidatorSessionDescription& desc, const SentBlock* block, const CntVector<bool>* voted,
HashType hash, bool temp) {
auto r = desc.get_by_hash(hash, temp);
if (compare(r, block, voted, hash)) {
desc.on_reuse();
return static_cast<const SessionVoteCandidate*>(r);
}
return static_cast<const SessionVoteCandidate*>(nullptr);
}
static const SessionVoteCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block,
const CntVector<bool>* voted) {
auto hash = create_hash(desc, get_vs_hash(desc, block), get_vs_hash(desc, voted));
auto r = lookup(desc, block, voted, hash, true);
if (r) {
return r;
}
return new (desc, true) SessionVoteCandidate(desc, block, voted, hash);
}
static const SessionVoteCandidate* create(ValidatorSessionDescription& desc, const SentBlock* block) {
std::vector<bool> v;
v.resize(desc.get_total_nodes(), false);
auto vec = CntVector<bool>::create(desc, std::move(v));
return create(desc, block, vec);
}
static const SessionVoteCandidate* move_to_persistent(ValidatorSessionDescription& desc,
const SessionVoteCandidate* b) {
if (desc.is_persistent(b)) {
return b;
}
auto block = SentBlock::move_to_persistent(desc, b->block_);
auto voted = CntVector<bool>::move_to_persistent(desc, b->voted_by_);
auto r = lookup(desc, block, voted, b->hash_, false);
if (r) {
return r;
}
return new (desc, false) SessionVoteCandidate{desc, block, voted, b->hash_};
}
SessionVoteCandidate(ValidatorSessionDescription& desc, const SentBlock* block, const CntVector<bool>* voted,
HashType hash)
: RootObject{sizeof(SessionVoteCandidate)}, block_(block), voted_by_(voted), hash_(hash) {
desc.update_hash(this, hash_);
}
static const SessionVoteCandidate* merge(ValidatorSessionDescription& desc, const SessionVoteCandidate* l,
const SessionVoteCandidate* r);
auto get_block() const {
return block_;
}
auto get_id() const {
return SentBlock::get_block_id(block_);
}
auto get_src_idx() const {
return block_ ? block_->get_src_idx() : std::numeric_limits<td::uint32>::max();
}
bool check_block_is_voted_by(td::uint32 src_idx) const {
return voted_by_->at(src_idx);
}
bool check_block_is_voted(ValidatorSessionDescription& desc) const;
auto get_hash(ValidatorSessionDescription& desc) const {
return hash_;
}
auto get_voters_list() const {
return voted_by_;
}
static const SessionVoteCandidate* push(ValidatorSessionDescription& desc, const SessionVoteCandidate* state,
td::uint32 src_idx) {
CHECK(state);
if (state->voted_by_->at(src_idx)) {
return state;
}
return create(desc, state->block_, CntVector<bool>::change(desc, state->voted_by_, src_idx, true));
}
class Compare {
public:
bool operator()(const SessionVoteCandidate* l, const SessionVoteCandidate* r) {
return l->get_id() < r->get_id();
}
};
private:
const SentBlock* block_;
const CntVector<bool>* voted_by_;
const HashType hash_;
};
//using SentBlockVector = CntSortedVector<const SentBlock*, SentBlock::Compare>;
class ValidatorSessionRoundState;
class ValidatorSessionOldRoundState : public ValidatorSessionDescription::RootObject {
public:
static HashType create_hash(ValidatorSessionDescription& desc, td::uint32 seqno, HashType block, HashType signatures,
@ -582,188 +178,6 @@ class ValidatorSessionOldRoundState : public ValidatorSessionDescription::RootOb
const HashType hash_;
};
using VoteVector = CntSortedVector<const SessionVoteCandidate*, SessionVoteCandidate::Compare>;
using ApproveVector = CntSortedVector<const SessionBlockCandidate*, SessionBlockCandidate::Compare>;
class ValidatorSessionRoundAttemptState : public ValidatorSessionDescription::RootObject {
public:
static HashType create_hash(ValidatorSessionDescription& desc, td::uint32 seqno, HashType votes,
HashType precommitted, bool vote_for_inited, HashType vote_for) {
auto obj = create_tl_object<ton_api::hashable_validatorSessionRoundAttempt>(seqno, votes, precommitted,
vote_for_inited, vote_for);
return desc.compute_hash(serialize_tl_object(obj, true).as_slice());
}
static bool compare(const RootObject* r, td::uint32 seqno, const VoteVector* votes,
const CntVector<bool>* precommitted, const SentBlock* vote_for, bool vote_for_inited,
HashType hash) {
if (!r || r->get_size() < sizeof(ValidatorSessionRoundAttemptState)) {
return false;
}
auto R = static_cast<const ValidatorSessionRoundAttemptState*>(r);
return R->seqno_ == seqno && R->votes_ == votes && R->precommitted_ == precommitted && R->vote_for_ == vote_for &&
R->vote_for_inited_ == vote_for_inited && R->hash_ == hash;
}
static auto lookup(ValidatorSessionDescription& desc, td::uint32 seqno, const VoteVector* votes,
const CntVector<bool>* precommitted, const SentBlock* vote_for, bool vote_for_inited,
HashType hash, bool temp) {
auto r = desc.get_by_hash(hash, temp);
if (compare(r, seqno, votes, precommitted, vote_for, vote_for_inited, hash)) {
desc.on_reuse();
return static_cast<const ValidatorSessionRoundAttemptState*>(r);
}
return static_cast<const ValidatorSessionRoundAttemptState*>(nullptr);
}
static const ValidatorSessionRoundAttemptState* move_to_persistent(ValidatorSessionDescription& desc,
const ValidatorSessionRoundAttemptState* b) {
if (desc.is_persistent(b)) {
return b;
}
auto votes = VoteVector::move_to_persistent(desc, b->votes_);
auto precommitted = CntVector<bool>::move_to_persistent(desc, b->precommitted_);
auto vote_for = SentBlock::move_to_persistent(desc, b->vote_for_);
auto r = lookup(desc, b->seqno_, votes, precommitted, vote_for, b->vote_for_inited_, b->hash_, false);
if (r) {
return r;
}
return new (desc, false) ValidatorSessionRoundAttemptState{desc, b->seqno_, votes, precommitted,
vote_for, b->vote_for_inited_, b->hash_};
}
static const ValidatorSessionRoundAttemptState* create(ValidatorSessionDescription& desc, td::uint32 seqno,
const VoteVector* votes, const CntVector<bool>* precommitted,
const SentBlock* vote_for, bool vote_for_inited) {
auto hash = create_hash(desc, seqno, get_vs_hash(desc, votes), get_vs_hash(desc, precommitted),
get_vs_hash(desc, vote_for), vote_for_inited);
auto r = lookup(desc, seqno, votes, precommitted, vote_for, vote_for_inited, hash, true);
if (r) {
return r;
}
return new (desc, true)
ValidatorSessionRoundAttemptState(desc, seqno, votes, precommitted, vote_for, vote_for_inited, hash);
}
static const ValidatorSessionRoundAttemptState* create(ValidatorSessionDescription& desc, td::uint32 seqno) {
std::vector<bool> x;
x.resize(desc.get_total_nodes(), false);
auto p = CntVector<bool>::create(desc, std::move(x));
return create(desc, seqno, nullptr, p, nullptr, false);
}
ValidatorSessionRoundAttemptState(ValidatorSessionDescription& desc, td::uint32 seqno, const VoteVector* votes,
const CntVector<bool>* precommitted, const SentBlock* vote_for,
bool vote_for_inited, HashType hash)
: RootObject{sizeof(ValidatorSessionRoundAttemptState)}
, seqno_(seqno)
, votes_(votes)
, precommitted_(precommitted)
, vote_for_(vote_for)
, vote_for_inited_(vote_for_inited)
, hash_(std::move(hash)) {
desc.update_hash(this, hash_);
}
auto get_hash(ValidatorSessionDescription& desc) const {
return hash_;
}
auto get_seqno() const {
return seqno_;
}
auto get_votes() const {
return votes_;
}
auto get_precommits() const {
return precommitted_;
}
const SentBlock* get_voted_block(ValidatorSessionDescription& desc, bool& f) const;
const SentBlock* get_vote_for_block(ValidatorSessionDescription& desc, bool& f) const {
f = vote_for_inited_;
return vote_for_;
}
bool check_attempt_is_precommitted(ValidatorSessionDescription& desc) const;
bool check_vote_received_from(td::uint32 src_idx) const;
bool check_precommit_received_from(td::uint32 src_idx) const;
bool operator<(const ValidatorSessionRoundAttemptState& right) const {
return seqno_ < right.seqno_;
}
struct Compare {
bool operator()(const ValidatorSessionRoundAttemptState* a, const ValidatorSessionRoundAttemptState* b) const {
return *a < *b;
}
};
static const ValidatorSessionRoundAttemptState* merge(ValidatorSessionDescription& desc,
const ValidatorSessionRoundAttemptState* left,
const ValidatorSessionRoundAttemptState* right);
static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc,
const ValidatorSessionRoundAttemptState* state,
td::uint32 src_idx, td::uint32 att,
const ton_api::validatorSession_message_voteFor& act,
const ValidatorSessionRoundState* round);
static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc,
const ValidatorSessionRoundAttemptState* state,
td::uint32 src_idx, td::uint32 att,
const ton_api::validatorSession_message_vote& act,
const ValidatorSessionRoundState* round);
static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc,
const ValidatorSessionRoundAttemptState* state,
td::uint32 src_idx, td::uint32 att,
const ton_api::validatorSession_message_precommit& act,
const ValidatorSessionRoundState* round);
static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc,
const ValidatorSessionRoundAttemptState* state,
td::uint32 src_idx, td::uint32 att,
const ton_api::validatorSession_message_empty& act,
const ValidatorSessionRoundState* round);
static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc,
const ValidatorSessionRoundAttemptState* state,
td::uint32 src_idx, td::uint32 att,
const ton_api::validatorSession_round_Message* action,
const ValidatorSessionRoundState* round);
template <class T>
static const ValidatorSessionRoundAttemptState* action(ValidatorSessionDescription& desc,
const ValidatorSessionRoundAttemptState* state,
td::uint32 src_idx, td::uint32 att, const T& action,
const ValidatorSessionRoundState* round) {
UNREACHABLE();
}
static const ValidatorSessionRoundAttemptState* try_vote(ValidatorSessionDescription& desc,
const ValidatorSessionRoundAttemptState* state,
td::uint32 src_idx, td::uint32 att,
const ValidatorSessionRoundState* round,
const ton_api::validatorSession_round_Message* cmp,
bool& made);
static const ValidatorSessionRoundAttemptState* try_precommit(ValidatorSessionDescription& desc,
const ValidatorSessionRoundAttemptState* state,
td::uint32 src_idx, td::uint32 att,
const ValidatorSessionRoundState* round,
const ton_api::validatorSession_round_Message* cmp,
bool& made);
static const ValidatorSessionRoundAttemptState* make_one(ValidatorSessionDescription& desc,
const ValidatorSessionRoundAttemptState* state,
td::uint32 src_idx, td::uint32 att,
const ValidatorSessionRoundState* round,
const ton_api::validatorSession_round_Message* cmp,
bool& made);
tl_object_ptr<ton_api::validatorSession_round_Message> create_action(ValidatorSessionDescription& desc,
const ValidatorSessionRoundState* round,
td::uint32 src_idx, td::uint32 att) const;
void dump(ValidatorSessionDescription& desc, td::StringBuilder& sb) const;
private:
const td::uint32 seqno_;
const VoteVector* votes_;
const CntVector<bool>* precommitted_;
const SentBlock* vote_for_;
const bool vote_for_inited_;
const HashType hash_;
};
using AttemptVector =
CntSortedVector<const ValidatorSessionRoundAttemptState*, ValidatorSessionRoundAttemptState::Compare>;
class ValidatorSessionRoundState : public ValidatorSessionDescription::RootObject {
public:

6
validator-session/validator-session-types.h
View file

@ -27,6 +27,12 @@ namespace ton {
namespace validatorsession {
constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_WARNING) = verbosity_WARNING;
constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_NOTICE) = verbosity_DEBUG;
constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_INFO) = verbosity_DEBUG;
constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_DEBUG) = verbosity_DEBUG;
constexpr int VERBOSITY_NAME(VALIDATOR_SESSION_EXTRA_DEBUG) = verbosity_DEBUG + 1;
using ValidatorSessionRootHash = td::Bits256;
using ValidatorSessionFileHash = td::Bits256;
using ValidatorSessionCollatedDataFileHash = td::Bits256;

58
validator-session/validator-session.cpp
View file

@ -203,7 +203,23 @@ void ValidatorSessionImpl::preprocess_block(catchain::CatChainBlock *block) {
<< "ms: state=" << state->get_hash(description());
}
void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice data) {
bool ValidatorSessionImpl::ensure_candidate_unique(td::uint32 src_idx, td::uint32 round,
ValidatorSessionCandidateId block_id) {
auto it = src_round_candidate_[src_idx].find(round);
if (it != src_round_candidate_[src_idx].end() && it->second != block_id) {
VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << description_->get_source_adnl_id(src_idx) << "][candidate "
<< block_id << "]: this node already has candidate in round " << round;
return false;
}
src_round_candidate_[src_idx][round] = block_id;
return true;
}
void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice data,
td::optional<ValidatorSessionCandidateId> expected_id,
bool is_overlay_broadcast) {
// Note: src is not necessarily equal to the sender of this message:
// If requested using get_broadcast_p2p, src is the creator of the block, sender possibly is some other node.
auto src_idx = description().get_source_idx(src);
auto R = fetch_tl_object<ton_api::validatorSession_candidate>(data.clone(), true);
if (R.is_error()) {
@ -230,6 +246,12 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice
auto block_round = static_cast<td::uint32>(candidate->round_);
auto block_id = description().candidate_id(src_idx, candidate->root_hash_, file_hash, collated_data_file_hash);
if (expected_id && expected_id.value() != block_id) {
VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice())
<< "]: id mismatch";
return;
}
if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK ||
block_round >= cur_round_ + MAX_FUTURE_ROUND_BLOCK) {
VLOG(VALIDATOR_SESSION_NOTICE) << this << "[node " << src << "][broadcast " << block_id
@ -250,6 +272,10 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice
return;
}
if (is_overlay_broadcast && !ensure_candidate_unique(src_idx, block_round, block_id)) {
return;
}
blocks_[block_id] = std::move(candidate);
VLOG(VALIDATOR_SESSION_WARNING) << this << ": received broadcast " << block_id;
@ -508,7 +534,11 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) {
}
if (block) {
auto T = td::Timestamp::at(round_started_at_.at() + description().get_delay(block->get_src_idx()) + 2.0);
if (!ensure_candidate_unique(block->get_src_idx(), cur_round_, SentBlock::get_block_id(block))) {
return;
}
auto T = td::Timestamp::at(round_started_at_.at() + description().get_delay(block->get_src_idx()) +
REQUEST_BROADCAST_P2P_DELAY);
auto it = blocks_.find(block_id);
if (it != blocks_.end()) {
@ -545,14 +575,16 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) {
auto id = description().get_source_id(v[td::Random::fast(0, static_cast<td::int32>(v.size() - 1))]);
auto src_id = description().get_source_id(block->get_src_idx());
active_requests_.insert(block_id);
auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), id, src_id, print_id = print_id(),
hash = block_id, round = cur_round_](td::Result<td::BufferSlice> R) {
auto P = td::PromiseCreator::lambda(
[SelfId = actor_id(this), id, src_id, print_id = print_id(), hash = block_id, round = cur_round_,
candidate_id = SentBlock::get_block_id(block)](td::Result<td::BufferSlice> R) {
td::actor::send_closure(SelfId, &ValidatorSessionImpl::end_request, round, hash);
if (R.is_error()) {
VLOG(VALIDATOR_SESSION_WARNING)
<< print_id << ": failed to get candidate " << hash << " from " << id << ": " << R.move_as_error();
VLOG(VALIDATOR_SESSION_WARNING) << print_id << ": failed to get candidate " << hash << " from " << id
<< ": " << R.move_as_error();
} else {
td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src_id, R.move_as_ok());
td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src_id, R.move_as_ok(),
candidate_id, false);
}
});
@ -586,9 +618,10 @@ void ValidatorSessionImpl::get_broadcast_p2p(PublicKeyHash node, ValidatorSessio
round,
create_tl_object<ton_api::validatorSession_candidateId>(src.tl(), root_hash, file_hash, collated_data_file_hash));
td::actor::send_closure(catchain_, &catchain::CatChain::send_query_via, node, "download candidate",
std::move(promise), timeout, serialize_tl_object(obj, true),
description().opts().max_block_size + description().opts().max_collated_data_size + 1024,
td::actor::send_closure(
catchain_, &catchain::CatChain::send_query_via, node, "download candidate", std::move(promise), timeout,
serialize_tl_object(obj, true),
description().opts().max_block_size + description().opts().max_collated_data_size + MAX_CANDIDATE_EXTRA_SIZE,
rldp_);
}
@ -745,7 +778,6 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) {
while (cur_round_ < round) {
auto block = real_state_->get_committed_block(description(), cur_round_);
//CHECK(block);
auto sigs = real_state_->get_committed_block_signatures(description(), cur_round_);
CHECK(sigs);
auto approve_sigs = real_state_->get_committed_block_approve_signatures(description(), cur_round_);
@ -838,7 +870,8 @@ void ValidatorSessionImpl::on_catchain_started() {
auto broadcast = create_tl_object<ton_api::validatorSession_candidate>(
src.tl(), round, root_hash, std::move(B.data), std::move(B.collated_data));
td::actor::send_closure(SelfId, &ValidatorSessionImpl::process_broadcast, src,
serialize_tl_object(broadcast, true));
serialize_tl_object(broadcast, true), td::optional<ValidatorSessionCandidateId>(),
false);
}
});
callback_->get_approved_candidate(description().get_source_public_key(x->get_src_idx()), x->get_root_hash(),
@ -866,6 +899,7 @@ ValidatorSessionImpl::ValidatorSessionImpl(catchain::CatChainSessionId session_i
, overlay_manager_(overlays)
, allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) {
description_ = ValidatorSessionDescription::create(std::move(opts), nodes, local_id);
src_round_candidate_.resize(description_->get_total_nodes());
}
void ValidatorSessionImpl::start() {

11
validator-session/validator-session.hpp
View file

@ -74,6 +74,8 @@ class ValidatorSessionImpl : public ValidatorSession {
td::BufferSlice signature_;
std::map<ValidatorSessionCandidateId, tl_object_ptr<ton_api::validatorSession_candidate>> blocks_;
// src_round_candidate_[src_id][round] -> candidate id
std::vector<std::map<td::uint32, ValidatorSessionCandidateId>> src_round_candidate_;
catchain::CatChainSessionId unique_hash_;
@ -110,7 +112,8 @@ class ValidatorSessionImpl : public ValidatorSession {
td::actor::send_closure(id_, &ValidatorSessionImpl::preprocess_block, block);
}
void process_broadcast(const PublicKeyHash &src, td::BufferSlice data) override {
td::actor::send_closure(id_, &ValidatorSessionImpl::process_broadcast, src, std::move(data));
td::actor::send_closure(id_, &ValidatorSessionImpl::process_broadcast, src, std::move(data),
td::optional<ValidatorSessionCandidateId>(), true);
}
void process_message(const PublicKeyHash &src, td::BufferSlice data) override {
td::actor::send_closure(id_, &ValidatorSessionImpl::process_message, src, std::move(data));
@ -174,7 +177,9 @@ class ValidatorSessionImpl : public ValidatorSession {
void process_blocks(std::vector<catchain::CatChainBlock *> blocks);
void finished_processing();
void preprocess_block(catchain::CatChainBlock *block);
void process_broadcast(PublicKeyHash src, td::BufferSlice data);
bool ensure_candidate_unique(td::uint32 src_idx, td::uint32 round, ValidatorSessionCandidateId block_id);
void process_broadcast(PublicKeyHash src, td::BufferSlice data, td::optional<ValidatorSessionCandidateId> expected_id,
bool is_overlay_broadcast);
void process_message(PublicKeyHash src, td::BufferSlice data);
void process_query(PublicKeyHash src, td::BufferSlice data, td::Promise<td::BufferSlice> promise);
@ -206,6 +211,8 @@ class ValidatorSessionImpl : public ValidatorSession {
static const size_t MAX_REJECT_REASON_SIZE = 1024;
static const td::int32 MAX_FUTURE_ROUND_BLOCK = 100;
static const td::int32 MAX_PAST_ROUND_BLOCK = 20;
constexpr static const double REQUEST_BROADCAST_P2P_DELAY = 2.0;
static const td::uint32 MAX_CANDIDATE_EXTRA_SIZE = 1024;
};
} // namespace validatorsession

321
validator/db/archive-manager.cpp
View file

@ -74,7 +74,7 @@ void ArchiveManager::add_handle(BlockHandle handle, td::Promise<td::Unit> promis
}
void ArchiveManager::update_handle(BlockHandle handle, td::Promise<td::Unit> promise) {
FileDescription *f;
const FileDescription *f;
if (handle->handle_moved_to_archive()) {
CHECK(handle->inited_unix_time());
if (!handle->need_flush()) {
@ -439,15 +439,15 @@ void ArchiveManager::check_persistent_state(BlockIdExt block_id, BlockIdExt mast
void ArchiveManager::get_block_by_unix_time(AccountIdPrefixFull account_id, UnixTime ts,
td::Promise<ConstBlockHandle> promise) {
auto f = get_file_desc_by_unix_time(account_id, ts, false);
if (f) {
auto n = f;
do {
n = get_next_file_desc(n);
} while (n != nullptr && !n->has_account_prefix(account_id));
auto f1 = get_file_desc_by_unix_time(account_id, ts, false);
auto f2 = get_next_file_desc(f1, account_id, false);
if (!f1) {
std::swap(f1, f2);
}
if (f1) {
td::actor::ActorId<ArchiveSlice> aid;
if (n) {
aid = n->file_actor_id();
if (f2) {
aid = f2->file_actor_id();
}
auto P = td::PromiseCreator::lambda(
[aid, account_id, ts, promise = std::move(promise)](td::Result<ConstBlockHandle> R) mutable {
@ -457,7 +457,7 @@ void ArchiveManager::get_block_by_unix_time(AccountIdPrefixFull account_id, Unix
td::actor::send_closure(aid, &ArchiveSlice::get_block_by_unix_time, account_id, ts, std::move(promise));
}
});
td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_block_by_unix_time, account_id, ts, std::move(P));
td::actor::send_closure(f1->file_actor_id(), &ArchiveSlice::get_block_by_unix_time, account_id, ts, std::move(P));
} else {
promise.set_error(td::Status::Error(ErrorCode::notready, "ts not in db"));
}
@ -465,15 +465,15 @@ void ArchiveManager::get_block_by_unix_time(AccountIdPrefixFull account_id, Unix
void ArchiveManager::get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime lt,
td::Promise<ConstBlockHandle> promise) {
auto f = get_file_desc_by_lt(account_id, lt, false);
if (f) {
auto n = f;
do {
n = get_next_file_desc(n);
} while (n != nullptr && !n->has_account_prefix(account_id));
auto f1 = get_file_desc_by_lt(account_id, lt, false);
auto f2 = get_next_file_desc(f1, account_id, false);
if (!f1) {
std::swap(f1, f2);
}
if (f1) {
td::actor::ActorId<ArchiveSlice> aid;
if (n) {
aid = n->file_actor_id();
if (f2) {
aid = f2->file_actor_id();
}
auto P = td::PromiseCreator::lambda(
[aid, account_id, lt, promise = std::move(promise)](td::Result<ConstBlockHandle> R) mutable {
@ -483,7 +483,7 @@ void ArchiveManager::get_block_by_lt(AccountIdPrefixFull account_id, LogicalTime
td::actor::send_closure(aid, &ArchiveSlice::get_block_by_lt, account_id, lt, std::move(promise));
}
});
td::actor::send_closure(f->file_actor_id(), &ArchiveSlice::get_block_by_lt, account_id, lt, std::move(P));
td::actor::send_closure(f1->file_actor_id(), &ArchiveSlice::get_block_by_lt, account_id, lt, std::move(P));
} else {
promise.set_error(td::Status::Error(ErrorCode::notready, "lt not in db"));
}
@ -558,11 +558,16 @@ void ArchiveManager::deleted_package(PackageId id, td::Promise<td::Unit> promise
auto it = m.find(id);
CHECK(it != m.end());
CHECK(it->second.deleted);
it->second.clear_actor_id();
it->second.file.reset();
promise.set_value(td::Unit());
}
void ArchiveManager::load_package(PackageId id) {
auto &m = get_file_map(id);
if (m.count(id)) {
LOG(WARNING) << "Duplicate id " << id.name();
return;
}
auto key = create_serialize_tl_object<ton_api::db_files_package_key>(id.id, id.key, id.temp);
std::string value;
@ -595,10 +600,10 @@ void ArchiveManager::load_package(PackageId id) {
desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_);
get_file_map(id).emplace(id, std::move(desc));
m.emplace(id, std::move(desc));
}
ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno,
const ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno,
UnixTime ts, LogicalTime lt, bool force) {
auto &f = get_file_map(id);
auto it = f.find(id);
@ -607,7 +612,7 @@ ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard
return nullptr;
}
if (force && !id.temp) {
update_desc(it->second, shard, seqno, ts, lt);
update_desc(f, it->second, shard, seqno, ts, lt);
}
return &it->second;
}
@ -618,17 +623,18 @@ ArchiveManager::FileDescription *ArchiveManager::get_file_desc(ShardIdFull shard
return add_file_desc(shard, id, seqno, ts, lt);
}
ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno,
const ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno,
UnixTime ts, LogicalTime lt) {
auto &f = get_file_map(id);
CHECK(f.count(id) == 0);
FileDescription desc{id, false};
FileDescription new_desc{id, false};
td::mkdir(db_root_ + id.path()).ensure();
std::string prefix = PSTRING() << db_root_ << id.path() << id.name();
desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_);
new_desc.file = td::actor::create_actor<ArchiveSlice>("slice", id.id, id.key, id.temp, false, db_root_);
const FileDescription &desc = f.emplace(id, std::move(new_desc));
if (!id.temp) {
update_desc(desc, shard, seqno, ts, lt);
update_desc(f, desc, shard, seqno, ts, lt);
}
std::vector<tl_object_ptr<ton_api::db_files_package_firstBlock>> vec;
@ -652,7 +658,6 @@ ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull shard
for (auto &e : temp_files_) {
tt.push_back(e.first.id);
}
(id.temp ? tt : (id.key ? tk : t)).push_back(id.id);
index_
->set(create_serialize_tl_object<ton_api::db_files_index_key>().as_slice(),
create_serialize_tl_object<ton_api::db_files_index_value>(std::move(t), std::move(tk), std::move(tt))
@ -668,17 +673,16 @@ ArchiveManager::FileDescription *ArchiveManager::add_file_desc(ShardIdFull shard
.ensure();
}
index_->commit_transaction().ensure();
return &f.emplace(id, std::move(desc)).first->second;
return &desc;
}
void ArchiveManager::update_desc(FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, UnixTime ts,
LogicalTime lt) {
void ArchiveManager::update_desc(FileMap &f, const FileDescription &desc, ShardIdFull shard, BlockSeqno seqno,
UnixTime ts, LogicalTime lt) {
auto it = desc.first_blocks.find(shard);
if (it != desc.first_blocks.end() && it->second.seqno <= seqno) {
return;
}
desc.first_blocks[shard] = FileDescription::Desc{seqno, ts, lt};
f.set_shard_first_block(desc, shard, FileDescription::Desc{seqno, ts, lt});
std::vector<tl_object_ptr<ton_api::db_files_package_firstBlock>> vec;
for (auto &e : desc.first_blocks) {
vec.push_back(create_tl_object<ton_api::db_files_package_firstBlock>(e.first.workchain, e.first.shard,
@ -694,150 +698,91 @@ void ArchiveManager::update_desc(FileDescription &desc, ShardIdFull shard, Block
index_->commit_transaction().ensure();
}
ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno,
const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno,
bool key_block) {
auto &f = get_file_map(PackageId{0, key_block, false});
for (auto it = f.rbegin(); it != f.rend(); it++) {
auto i = it->second.first_blocks.find(shard);
if (i != it->second.first_blocks.end() && i->second.seqno <= seqno) {
if (it->second.deleted) {
return nullptr;
} else {
return &it->second;
}
}
}
return nullptr;
return get_file_map(PackageId{0, key_block, false}).get_file_desc_by_seqno(shard, seqno);
}
ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts,
const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts,
bool key_block) {
auto &f = get_file_map(PackageId{0, key_block, false});
for (auto it = f.rbegin(); it != f.rend(); it++) {
auto i = it->second.first_blocks.find(shard);
if (i != it->second.first_blocks.end() && i->second.ts <= ts) {
if (it->second.deleted) {
return nullptr;
} else {
return &it->second;
}
}
}
return nullptr;
return get_file_map(PackageId{0, key_block, false}).get_file_desc_by_unix_time(shard, ts);
}
ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt,
const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt,
bool key_block) {
auto &f = get_file_map(PackageId{0, key_block, false});
for (auto it = f.rbegin(); it != f.rend(); it++) {
auto i = it->second.first_blocks.find(shard);
if (i != it->second.first_blocks.end() && i->second.lt <= lt) {
if (it->second.deleted) {
return nullptr;
} else {
return &it->second;
}
}
}
return nullptr;
return get_file_map(PackageId{0, key_block, false}).get_file_desc_by_lt(shard, lt);
}
ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(AccountIdPrefixFull account, BlockSeqno seqno,
bool key_block) {
auto &f = get_file_map(PackageId{0, key_block, false});
const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_seqno(AccountIdPrefixFull account,
BlockSeqno seqno, bool key_block) {
if (account.is_masterchain()) {
return get_file_desc_by_seqno(ShardIdFull{masterchainId}, seqno, key_block);
}
for (auto it = f.rbegin(); it != f.rend(); it++) {
if (it->second.deleted) {
continue;
}
bool found = false;
for (int i = 0; i < 60; i++) {
auto shard = shard_prefix(account, i);
auto it2 = it->second.first_blocks.find(shard);
if (it2 != it->second.first_blocks.end()) {
if (it2->second.seqno <= seqno) {
return &it->second;
}
found = true;
} else if (found) {
auto &f = get_file_map(PackageId{0, key_block, false});
const FileDescription *result = nullptr;
for (int i = 0; i <= 60; i++) {
const FileDescription *desc = f.get_file_desc_by_seqno(shard_prefix(account, i), seqno);
if (desc && (!result || result->id < desc->id)) {
result = desc;
} else if (result && (!desc || desc->id < result->id)) {
break;
}
}
}
return nullptr;
return result;
}
ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(AccountIdPrefixFull account, UnixTime ts,
bool key_block) {
auto &f = get_file_map(PackageId{0, key_block, false});
const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_unix_time(AccountIdPrefixFull account,
UnixTime ts, bool key_block) {
if (account.is_masterchain()) {
return get_file_desc_by_unix_time(ShardIdFull{masterchainId}, ts, key_block);
}
for (auto it = f.rbegin(); it != f.rend(); it++) {
if (it->second.deleted) {
continue;
}
bool found = false;
for (int i = 0; i < 60; i++) {
auto shard = shard_prefix(account, i);
auto it2 = it->second.first_blocks.find(shard);
if (it2 != it->second.first_blocks.end()) {
if (it2->second.ts <= ts) {
return &it->second;
}
found = true;
} else if (found) {
auto &f = get_file_map(PackageId{0, key_block, false});
const FileDescription *result = nullptr;
for (int i = 0; i <= 60; i++) {
const FileDescription *desc = f.get_file_desc_by_unix_time(shard_prefix(account, i), ts);
if (desc && (!result || result->id < desc->id)) {
result = desc;
} else if (result && (!desc || desc->id < result->id)) {
break;
}
}
}
return nullptr;
return result;
}
ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(AccountIdPrefixFull account, LogicalTime lt,
const ArchiveManager::FileDescription *ArchiveManager::get_file_desc_by_lt(AccountIdPrefixFull account, LogicalTime lt,
bool key_block) {
auto &f = get_file_map(PackageId{0, key_block, false});
if (account.is_masterchain()) {
return get_file_desc_by_lt(ShardIdFull{masterchainId}, lt, key_block);
}
for (auto it = f.rbegin(); it != f.rend(); it++) {
if (it->second.deleted) {
continue;
}
bool found = false;
for (int i = 0; i < 60; i++) {
auto shard = shard_prefix(account, i);
auto it2 = it->second.first_blocks.find(shard);
if (it2 != it->second.first_blocks.end()) {
if (it2->second.lt <= lt) {
return &it->second;
}
found = true;
} else if (found) {
auto &f = get_file_map(PackageId{0, key_block, false});
const FileDescription *result = nullptr;
for (int i = 0; i <= 60; i++) {
const FileDescription *desc = f.get_file_desc_by_lt(shard_prefix(account, i), lt);
if (desc && (!result || result->id < desc->id)) {
result = desc;
} else if (result && (!desc || desc->id < result->id)) {
break;
}
}
}
return nullptr;
return result;
}
ArchiveManager::FileDescription *ArchiveManager::get_next_file_desc(FileDescription *f) {
auto &m = get_file_map(f->id);
auto it = m.find(f->id);
CHECK(it != m.end());
while (true) {
it++;
if (it == m.end()) {
return nullptr;
} else if (!it->second.deleted) {
return &it->second;
const ArchiveManager::FileDescription *ArchiveManager::get_next_file_desc(const FileDescription *f,
AccountIdPrefixFull shard, bool key_block) {
auto &m = get_file_map(PackageId{0, key_block, false});
const FileDescription *result = nullptr;
for (int i = 0; i <= 60; i++) {
const FileDescription *desc = m.get_next_file_desc(shard_prefix(shard, i), f);
if (desc && (!result || desc->id < result->id)) {
result = desc;
} else if (result && (!desc || result->id < desc->id)) {
break;
}
}
return result;
}
ArchiveManager::FileDescription *ArchiveManager::get_temp_file_desc_by_idx(PackageId idx) {
const ArchiveManager::FileDescription *ArchiveManager::get_temp_file_desc_by_idx(PackageId idx) {
auto it = temp_files_.find(idx);
if (it != temp_files_.end()) {
if (it->second.deleted) {
@ -1257,14 +1202,100 @@ void ArchiveManager::truncate(BlockSeqno masterchain_seqno, ConstBlockHandle han
}
}
bool ArchiveManager::FileDescription::has_account_prefix(AccountIdPrefixFull account_id) const {
for (int i = 0; i < 60; i++) {
auto shard = shard_prefix(account_id, i);
if (first_blocks.count(shard)) {
return true;
void ArchiveManager::FileMap::shard_index_add(const FileDescription &desc) {
for (const auto &p : desc.first_blocks) {
ShardIndex &s = shards_[p.first];
s.seqno_index_[p.second.seqno] = &desc;
s.lt_index_[p.second.lt] = &desc;
s.unix_time_index_[p.second.ts] = &desc;
s.packages_index_[desc.id] = &desc;
}
}
void ArchiveManager::FileMap::shard_index_del(const FileDescription &desc) {
for (const auto &p : desc.first_blocks) {
ShardIndex &s = shards_[p.first];
s.seqno_index_.erase(p.second.seqno);
s.lt_index_.erase(p.second.lt);
s.unix_time_index_.erase(p.second.ts);
s.packages_index_.erase(desc.id);
}
return false;
}
void ArchiveManager::FileMap::set_shard_first_block(const FileDescription &desc, ShardIdFull shard,
FileDescription::Desc v) {
ShardIndex &s = shards_[shard];
auto &d = const_cast<FileDescription &>(desc);
auto it = d.first_blocks.find(shard);
if (it != d.first_blocks.end()) {
s.seqno_index_.erase(it->second.seqno);
s.lt_index_.erase(it->second.lt);
s.unix_time_index_.erase(it->second.ts);
}
d.first_blocks[shard] = v;
s.seqno_index_[v.seqno] = &d;
s.lt_index_[v.lt] = &d;
s.unix_time_index_[v.ts] = &d;
s.packages_index_[d.id] = &d;
}
const ArchiveManager::FileDescription *ArchiveManager::FileMap::get_file_desc_by_seqno(ShardIdFull shard,
BlockSeqno seqno) const {
auto it = shards_.find(shard);
if (it == shards_.end()) {
return nullptr;
}
const auto &map = it->second.seqno_index_;
auto it2 = map.upper_bound(seqno);
if (it2 == map.begin()) {
return nullptr;
}
--it2;
return it2->second->deleted ? nullptr : it2->second;
}
const ArchiveManager::FileDescription *ArchiveManager::FileMap::get_file_desc_by_lt(ShardIdFull shard,
LogicalTime lt) const {
auto it = shards_.find(shard);
if (it == shards_.end()) {
return nullptr;
}
const auto &map = it->second.lt_index_;
auto it2 = map.upper_bound(lt);
if (it2 == map.begin()) {
return nullptr;
}
--it2;
return it2->second->deleted ? nullptr : it2->second;
}
const ArchiveManager::FileDescription *ArchiveManager::FileMap::get_file_desc_by_unix_time(ShardIdFull shard,
UnixTime ts) const {
auto it = shards_.find(shard);
if (it == shards_.end()) {
return nullptr;
}
const auto &map = it->second.unix_time_index_;
auto it2 = map.upper_bound(ts);
if (it2 == map.begin()) {
return nullptr;
}
--it2;
return it2->second->deleted ? nullptr : it2->second;
}
const ArchiveManager::FileDescription *ArchiveManager::FileMap::get_next_file_desc(ShardIdFull shard,
const FileDescription *desc) const {
auto it = shards_.find(shard);
if (it == shards_.end()) {
return nullptr;
}
const auto &map = it->second.packages_index_;
auto it2 = desc ? map.upper_bound(desc->id) : map.begin();
if (it2 == map.end()) {
return nullptr;
}
return it2->second->deleted ? nullptr : it2->second;
}
} // namespace validator

99
validator/db/archive-manager.hpp
View file

@ -71,7 +71,6 @@ class ArchiveManager : public td::actor::Actor {
void start_up() override;
void begin_transaction();
void commit_transaction();
void set_async_mode(bool mode, td::Promise<td::Unit> promise);
@ -94,26 +93,83 @@ class ArchiveManager : public td::actor::Actor {
auto file_actor_id() const {
return file.get();
}
void clear_actor_id() {
file.reset();
}
bool has_account_prefix(AccountIdPrefixFull account_id) const;
PackageId id;
bool deleted;
mutable bool deleted;
std::map<ShardIdFull, Desc> first_blocks;
td::actor::ActorOwn<ArchiveSlice> file;
mutable td::actor::ActorOwn<ArchiveSlice> file;
};
class FileMap {
public:
std::map<PackageId, FileDescription>::const_iterator begin() const {
return files_.cbegin();
}
std::map<PackageId, FileDescription>::const_iterator end() const {
return files_.cend();
}
std::map<PackageId, FileDescription>::const_reverse_iterator rbegin() const {
return files_.crbegin();
}
std::map<PackageId, FileDescription>::const_reverse_iterator rend() const {
return files_.crend();
}
std::map<PackageId, FileDescription>::const_iterator find(PackageId x) const {
return files_.find(x);
}
size_t count(const PackageId &x) const {
return files_.count(x);
}
size_t size() const {
return files_.size();
}
std::map<PackageId, FileDescription>::const_iterator lower_bound(const PackageId &x) const {
return files_.lower_bound(x);
}
std::map<PackageId, FileDescription>::const_iterator upper_bound(const PackageId &x) const {
return files_.upper_bound(x);
}
void clear() {
files_.clear();
shards_.clear();
}
const FileDescription &emplace(const PackageId &id, FileDescription desc) {
auto it = files_.emplace(id, std::move(desc));
if (it.second) {
shard_index_add(it.first->second);
}
return it.first->second;
}
void erase(std::map<PackageId, FileDescription>::const_iterator it) {
shard_index_del(it->second);
files_.erase(it);
}
void set_shard_first_block(const FileDescription &desc, ShardIdFull shard, FileDescription::Desc v);
const FileDescription *get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno) const;
const FileDescription *get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt) const;
const FileDescription *get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts) const;
const FileDescription *get_next_file_desc(ShardIdFull shard, const FileDescription *desc) const;
private:
std::map<PackageId, FileDescription> files_;
std::map<PackageId, FileDescription> key_files_;
std::map<PackageId, FileDescription> temp_files_;
struct ShardIndex {
std::map<BlockSeqno, const FileDescription *> seqno_index_;
std::map<LogicalTime, const FileDescription *> lt_index_;
std::map<UnixTime, const FileDescription *> unix_time_index_;
std::map<PackageId, const FileDescription *> packages_index_;
};
std::map<ShardIdFull, ShardIndex> shards_;
void shard_index_add(const FileDescription &desc);
void shard_index_del(const FileDescription &desc);
};
FileMap files_, key_files_, temp_files_;
BlockSeqno finalized_up_to_{0};
bool async_mode_ = false;
bool huge_transaction_started_ = false;
td::uint32 huge_transaction_size_ = 0;
auto &get_file_map(const PackageId &p) {
FileMap &get_file_map(const PackageId &p) {
return p.key ? key_files_ : p.temp ? temp_files_ : files_;
}
@ -126,18 +182,19 @@ class ArchiveManager : public td::actor::Actor {
void get_handle_finish(BlockHandle handle, td::Promise<BlockHandle> promise);
void get_file_short_cont(FileReference ref_id, PackageId idx, td::Promise<td::BufferSlice> promise);
FileDescription *get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt,
const FileDescription *get_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt,
bool force);
FileDescription *add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt);
void update_desc(FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, UnixTime ts, LogicalTime lt);
FileDescription *get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno, bool key_block);
FileDescription *get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt, bool key_block);
FileDescription *get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts, bool key_block);
FileDescription *get_file_desc_by_seqno(AccountIdPrefixFull shard, BlockSeqno seqno, bool key_block);
FileDescription *get_file_desc_by_lt(AccountIdPrefixFull shard, LogicalTime lt, bool key_block);
FileDescription *get_file_desc_by_unix_time(AccountIdPrefixFull shard, UnixTime ts, bool key_block);
FileDescription *get_next_file_desc(FileDescription *f);
FileDescription *get_temp_file_desc_by_idx(PackageId idx);
const FileDescription *add_file_desc(ShardIdFull shard, PackageId id, BlockSeqno seqno, UnixTime ts, LogicalTime lt);
void update_desc(FileMap &f, const FileDescription &desc, ShardIdFull shard, BlockSeqno seqno, UnixTime ts,
LogicalTime lt);
const FileDescription *get_file_desc_by_seqno(ShardIdFull shard, BlockSeqno seqno, bool key_block);
const FileDescription *get_file_desc_by_lt(ShardIdFull shard, LogicalTime lt, bool key_block);
const FileDescription *get_file_desc_by_unix_time(ShardIdFull shard, UnixTime ts, bool key_block);
const FileDescription *get_file_desc_by_seqno(AccountIdPrefixFull shard, BlockSeqno seqno, bool key_block);
const FileDescription *get_file_desc_by_lt(AccountIdPrefixFull shard, LogicalTime lt, bool key_block);
const FileDescription *get_file_desc_by_unix_time(AccountIdPrefixFull shard, UnixTime ts, bool key_block);
const FileDescription *get_next_file_desc(const FileDescription *f, AccountIdPrefixFull shard, bool key_block);
const FileDescription *get_temp_file_desc_by_idx(PackageId idx);
PackageId get_max_temp_file_desc_idx();
PackageId get_prev_temp_file_desc_idx(PackageId id);