From db505f45b2f3a4f6469e0cbb55a3fc3ffcef59c5 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Sun, 12 May 2024 04:48:14 -0500 Subject: [PATCH 1/5] Use jemalloc (#987) * use jemalloc * install system jemalloc in github action * check if jemalloc is actually used --------- Co-authored-by: neodiX --- .github/workflows/build-ton-linux-x86-64-shared.yml | 2 +- CMake/{FindJeMalloc.cmake => Findjemalloc.cmake} | 0 CMakeLists.txt | 2 +- Dockerfile | 6 +++--- assembly/native/build-ubuntu-shared.sh | 6 ++++-- 5 files changed, 9 insertions(+), 7 deletions(-) rename CMake/{FindJeMalloc.cmake => Findjemalloc.cmake} (100%) diff --git a/.github/workflows/build-ton-linux-x86-64-shared.yml b/.github/workflows/build-ton-linux-x86-64-shared.yml index ce0ade64..34f92d93 100644 --- a/.github/workflows/build-ton-linux-x86-64-shared.yml +++ b/.github/workflows/build-ton-linux-x86-64-shared.yml @@ -19,7 +19,7 @@ jobs: - name: Install system libraries run: | sudo apt-get update - sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev + sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev - name: Install clang-16 run: | diff --git a/CMake/FindJeMalloc.cmake b/CMake/Findjemalloc.cmake similarity index 100% rename from CMake/FindJeMalloc.cmake rename to CMake/Findjemalloc.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 89be3238..658eab70 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -234,7 +234,7 @@ if (THREADS_HAVE_PTHREAD_ARG) endif() if (TON_USE_JEMALLOC) - find_package(JeMalloc REQUIRED) + find_package(jemalloc REQUIRED) endif() set(MEMPROF "" CACHE STRING "Use one of \"ON\", \"FAST\" or \"SAFE\" to enable memory profiling. \ diff --git a/Dockerfile b/Dockerfile index 76c06b35..c5120e83 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:22.04 as builder RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool libjemalloc-dev && \ rm -rf /var/lib/apt/lists/* ENV CC clang ENV CXX clang++ @@ -14,12 +14,12 @@ COPY ./ ./ RUN mkdir build && \ cd build && \ - cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= .. && \ + cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DTON_USE_JEMALLOC=ON .. && \ ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console generate-random-id dht-server lite-client FROM ubuntu:22.04 RUN apt-get update && \ - apt-get install -y wget libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev && \ + apt-get install -y wget libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev libjemalloc-dev && \ rm -rf /var/lib/apt/lists/* RUN mkdir -p /var/ton-work/db && \ diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh index 7d5326a4..ec868ecd 100644 --- a/assembly/native/build-ubuntu-shared.sh +++ b/assembly/native/build-ubuntu-shared.sh @@ -1,7 +1,7 @@ #/bin/bash #sudo apt-get update -#sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev +#sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev with_tests=false with_artifacts=false @@ -42,7 +42,7 @@ else echo "Using compiled openssl_3" fi -cmake -GNinja .. \ +cmake -GNinja -DTON_USE_JEMALLOC=ON .. \ -DCMAKE_BUILD_TYPE=Release \ -DOPENSSL_ROOT_DIR=$opensslPath \ -DOPENSSL_INCLUDE_DIR=$opensslPath/include \ @@ -96,6 +96,8 @@ test $? -eq 0 || { echo "Can't strip final binaries"; exit 1; } ./lite-client/lite-client -V || exit 1 ./crypto/fift -V || exit 1 +ldd ./validator-engine/validator-engine || exit 1 + cd .. if [ "$with_artifacts" = true ]; then From 5186c4755c39205260bc2b01b3eef325d9feb09c Mon Sep 17 00:00:00 2001 From: neodix42 Date: Fri, 7 Jun 2024 17:50:11 +0200 Subject: [PATCH 2/5] Change fift path separator for FIFTPATH and -I argument (#1014) * add github action for macOS 14 (arm64, M1) * add github action (portable) for macOS 14 (arm64, M1) * rename macOS arm64 output artifact * Colon cannot be used as a path separator for FIFTPATH or -I argument in fift on Windows when absolute paths are used (e.g. C:\path\lib:C:\path\smartcont). Suggestion to use @ as a new path separator on Windows. --------- Co-authored-by: neodiX --- .../build-ton-macos-arm64-shared.yml | 25 ++++ .github/workflows/create-release.yml | 107 ++++++++++++++++++ .github/workflows/ton-arm64-macos.yml | 37 ++++++ crypto/CMakeLists.txt | 9 +- crypto/block/create-state.cpp | 9 +- crypto/fift/fift-main.cpp | 11 +- 6 files changed, 192 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/build-ton-macos-arm64-shared.yml create mode 100644 .github/workflows/ton-arm64-macos.yml diff --git a/.github/workflows/build-ton-macos-arm64-shared.yml b/.github/workflows/build-ton-macos-arm64-shared.yml new file mode 100644 index 00000000..2a68272c --- /dev/null +++ b/.github/workflows/build-ton-macos-arm64-shared.yml @@ -0,0 +1,25 @@ +name: MacOS TON build (shared, arm64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + runs-on: macos-14 + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Build TON + run: | + cp assembly/native/build-macos-shared.sh . + chmod +x build-macos-shared.sh + ./build-macos-shared.sh -t -a + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-binaries-macos-14 + path: artifacts diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 50d2661b..367dd663 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -35,6 +35,14 @@ jobs: workflow_conclusion: success skip_unpack: true + - name: Download Mac arm64 artifacts + uses: dawidd6/action-download-artifact@v2 + with: + workflow: ton-arm64-macos.yml + path: artifacts + workflow_conclusion: success + skip_unpack: true + - name: Download and unzip Mac x86-64 artifacts uses: dawidd6/action-download-artifact@v2 with: @@ -43,6 +51,14 @@ jobs: workflow_conclusion: success skip_unpack: false + - name: Download and unzip arm64 artifacts + uses: dawidd6/action-download-artifact@v2 + with: + workflow: ton-arm64-macos.yml + path: artifacts + workflow_conclusion: success + skip_unpack: false + - name: Download Windows artifacts uses: dawidd6/action-download-artifact@v2 with: @@ -297,6 +313,97 @@ jobs: asset_name: tonlib-cli-mac-x86-64 tag: ${{ steps.tag.outputs.TAG }} + + # mac arm64 + + - name: Upload Mac arm64 artifacts + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries.zip + asset_name: ton-mac-arm64.zip + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - fift + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/fift + asset_name: fift-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - func + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/func + asset_name: func-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - lite-client + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/lite-client + asset_name: lite-client-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - rldp-http-proxy + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/rldp-http-proxy + asset_name: rldp-http-proxy-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - http-proxy + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/http-proxy + asset_name: http-proxy-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - storage-daemon-cli + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/storage-daemon-cli + asset_name: storage-daemon-cli-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - storage-daemon + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/storage-daemon + asset_name: storage-daemon-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - tonlibjson + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/libtonlibjson.dylib + asset_name: tonlibjson-mac-arm64.dylib + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - libemulator + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/libemulator.dylib + asset_name: libemulator-mac-arm64.dylib + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - tonlib-cli + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/tonlib-cli + asset_name: tonlib-cli-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + # linux x86-64 - name: Upload Linux x86-64 artifacts diff --git a/.github/workflows/ton-arm64-macos.yml b/.github/workflows/ton-arm64-macos.yml new file mode 100644 index 00000000..9e8302e8 --- /dev/null +++ b/.github/workflows/ton-arm64-macos.yml @@ -0,0 +1,37 @@ +name: MacOS TON build (portable, arm64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + runs-on: macos-14 + + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - uses: cachix/install-nix-action@v23 + with: + extra_nix_config: | + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + + - name: Build TON + run: | + cp assembly/nix/build-macos-nix.sh . + chmod +x build-macos-nix.sh + ./build-macos-nix.sh -t + + - name: Simple binaries test + run: | + sudo mv /nix/store /nix/store2 + artifacts/validator-engine -V + artifacts/lite-client -V + artifacts/fift -V + artifacts/func -V + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-arm64-macos-binaries + path: artifacts diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 29b95466..30619440 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -479,10 +479,17 @@ if (NOT CMAKE_CROSSCOMPILING OR USE_EMSCRIPTEN) OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/${ARG_DEST_FIF} ) set(ARG_DEST_CPP "${ARG_DEST}.cpp") + + if (WIN32) + set(ARG_LIB_DIR "fift/lib@smartcont") + else() + set(ARG_LIB_DIR "fift/lib:smartcont") + endif() + add_custom_command( COMMENT "Generate ${ARG_DEST_CPP}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - COMMAND fift -Ifift/lib:smartcont -s asm-to-cpp.fif ${ARG_DEST_FIF} ${ARG_DEST_CPP} ${ARG_NAME} + COMMAND fift -I${ARG_LIB_DIR} -s asm-to-cpp.fif ${ARG_DEST_FIF} ${ARG_DEST_CPP} ${ARG_NAME} MAIN_DEPENDENCY ${ARG_SOURCE} DEPENDS fift ${ARG_DEST_FIF} smartcont/asm-to-cpp.fif fift/lib/Asm.fif OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/${ARG_DEST_CPP} diff --git a/crypto/block/create-state.cpp b/crypto/block/create-state.cpp index 183da0a7..348377e9 100644 --- a/crypto/block/create-state.cpp +++ b/crypto/block/create-state.cpp @@ -814,11 +814,16 @@ void usage(const char* progname) { void parse_include_path_set(std::string include_path_set, std::vector& res) { td::Parser parser(include_path_set); while (!parser.empty()) { - auto path = parser.read_till_nofail(':'); + #if TD_WINDOWS + auto path_separator = '@'; + #else + auto path_separator = ':'; + #endif + auto path = parser.read_till_nofail(path_separator); if (!path.empty()) { res.push_back(path.str()); } - parser.skip_nofail(':'); + parser.skip_nofail(path_separator); } } diff --git a/crypto/fift/fift-main.cpp b/crypto/fift/fift-main.cpp index ef833f43..fd424e8c 100644 --- a/crypto/fift/fift-main.cpp +++ b/crypto/fift/fift-main.cpp @@ -62,7 +62,7 @@ void usage(const char* progname) { << " [-i] [-n] [-I ] {-L } ...\n"; std::cerr << "\t-n\tDo not preload standard preamble file `Fift.fif`\n" "\t-i\tForce interactive mode even if explicit source file names are indicated\n" - "\t-I\tSets colon-separated library source include path. If not indicated, " + "\t-I\tSets colon-separated (unix) or at-separated (windows) library source include path. If not indicated, " "$FIFTPATH is used instead.\n" "\t-L\tPre-loads a library source file\n" "\t-d\tUse a ton database\n" @@ -75,11 +75,16 @@ void usage(const char* progname) { void parse_include_path_set(std::string include_path_set, std::vector& res) { td::Parser parser(include_path_set); while (!parser.empty()) { - auto path = parser.read_till_nofail(':'); + #if TD_WINDOWS + auto path_separator = '@'; + #else + auto path_separator = ':'; + #endif + auto path = parser.read_till_nofail(path_separator); if (!path.empty()) { res.push_back(path.str()); } - parser.skip_nofail(':'); + parser.skip_nofail(path_separator); } } From 0a0a92c6b04c77b9a59d65ba503c3c847dce885f Mon Sep 17 00:00:00 2001 From: neodix42 Date: Fri, 7 Jun 2024 17:52:11 +0200 Subject: [PATCH 3/5] Use jemalloc in portable artifacts (#1003) * use jemalloc in portable builds; also avoid mixing musl with glibc artifacts in nix builds; * minor fix for ubuntu arm nix build --- assembly/nix/build-linux-arm64-nix.sh | 4 +-- assembly/nix/build-linux-x86-64-nix.sh | 4 +-- assembly/nix/build-macos-nix.sh | 2 ++ assembly/nix/linux-arm64-static.nix | 44 +++++++++++++++-------- assembly/nix/linux-arm64-tonlib.nix | 26 ++++++++++---- assembly/nix/linux-x86-64-static.nix | 44 +++++++++++++++-------- assembly/nix/linux-x86-64-tonlib.nix | 48 ++++++++++++++++++-------- assembly/nix/macos-static.nix | 4 +-- assembly/nix/microhttpd.nix | 28 --------------- assembly/nix/openssl.nix | 30 ---------------- 10 files changed, 122 insertions(+), 112 deletions(-) delete mode 100644 assembly/nix/microhttpd.nix delete mode 100644 assembly/nix/openssl.nix diff --git a/assembly/nix/build-linux-arm64-nix.sh b/assembly/nix/build-linux-arm64-nix.sh index 8e5c367c..7e85a871 100644 --- a/assembly/nix/build-linux-arm64-nix.sh +++ b/assembly/nix/build-linux-arm64-nix.sh @@ -15,8 +15,6 @@ while getopts 't' flag; do done cp assembly/nix/linux-arm64* . -cp assembly/nix/microhttpd.nix . -cp assembly/nix/openssl.nix . export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz if [ "$with_tests" = true ]; then @@ -30,7 +28,9 @@ cp ./result/bin/* artifacts/ test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; } chmod +x artifacts/* rm -rf result + nix-build linux-arm64-tonlib.nix + cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so cp ./result/lib/libemulator.so artifacts/ cp ./result/lib/fift/* artifacts/lib/ diff --git a/assembly/nix/build-linux-x86-64-nix.sh b/assembly/nix/build-linux-x86-64-nix.sh index 38431ca4..c1f1dcf3 100644 --- a/assembly/nix/build-linux-x86-64-nix.sh +++ b/assembly/nix/build-linux-x86-64-nix.sh @@ -15,8 +15,6 @@ while getopts 't' flag; do done cp assembly/nix/linux-x86-64* . -cp assembly/nix/microhttpd.nix . -cp assembly/nix/openssl.nix . export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz if [ "$with_tests" = true ]; then @@ -30,7 +28,9 @@ cp ./result/bin/* artifacts/ test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; } chmod +x artifacts/* rm -rf result + nix-build linux-x86-64-tonlib.nix + cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so cp ./result/lib/libemulator.so artifacts/ cp ./result/lib/fift/* artifacts/lib/ diff --git a/assembly/nix/build-macos-nix.sh b/assembly/nix/build-macos-nix.sh index 12977745..c75ca042 100644 --- a/assembly/nix/build-macos-nix.sh +++ b/assembly/nix/build-macos-nix.sh @@ -28,7 +28,9 @@ cp ./result-bin/bin/* artifacts/ test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; } chmod +x artifacts/* rm -rf result-bin + nix-build macos-tonlib.nix + cp ./result/lib/libtonlibjson.dylib artifacts/ cp ./result/lib/libemulator.dylib artifacts/ cp ./result/lib/fift/* artifacts/lib/ diff --git a/assembly/nix/linux-arm64-static.nix b/assembly/nix/linux-arm64-static.nix index 8c2749b0..536152d2 100644 --- a/assembly/nix/linux-arm64-static.nix +++ b/assembly/nix/linux-arm64-static.nix @@ -6,9 +6,23 @@ , testing ? false }: let - microhttpdmy = (import ./microhttpd.nix) {}; + staticOptions = pkg: pkg.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ]; + }); + + secp256k1Static = (staticOptions pkgs.secp256k1); + libsodiumStatic = (staticOptions pkgs.libsodium); + jemallocStatic = (staticOptions pkgs.jemalloc); + + microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ]; + }); + in -with import microhttpdmy; stdenv.mkDerivation { pname = "ton"; version = "dev-bin"; @@ -16,31 +30,33 @@ stdenv.mkDerivation { src = ./.; nativeBuildInputs = with pkgs; - [ - cmake ninja git pkg-config - ]; + [ cmake ninja git pkg-config ]; buildInputs = with pkgs; [ - pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 glibc.static pkgsStatic.lz4 + (openssl.override { static = true; }).dev + microhttpdStatic.dev + (zlib.override { shared = false; }).dev + (lz4.override { enableStatic = true; enableShared = false; }).dev + jemallocStatic + secp256k1Static + libsodiumStatic.dev + glibc.static ]; - makeStatic = true; - doCheck = testing; - cmakeFlags = [ "-DTON_USE_ABSEIL=OFF" "-DNIX=ON" "-DBUILD_SHARED_LIBS=OFF" "-DCMAKE_LINK_SEARCH_START_STATIC=ON" "-DCMAKE_LINK_SEARCH_END_STATIC=ON" - "-DMHD_FOUND=1" - "-DMHD_INCLUDE_DIR=${microhttpdmy}/usr/local/include" - "-DMHD_LIBRARY=${microhttpdmy}/usr/local/lib/libmicrohttpd.a" - "-DCMAKE_CTEST_ARGUMENTS=--timeout;1800" + "-DTON_USE_JEMALLOC=ON" ]; + makeStatic = true; + doCheck = testing; + LDFLAGS = [ - "-static-libgcc" "-static-libstdc++" "-static" + "-static-libgcc" "-static-libstdc++" "-static" ]; } diff --git a/assembly/nix/linux-arm64-tonlib.nix b/assembly/nix/linux-arm64-tonlib.nix index ae62ca26..a051e34c 100644 --- a/assembly/nix/linux-arm64-tonlib.nix +++ b/assembly/nix/linux-arm64-tonlib.nix @@ -5,9 +5,21 @@ , stdenv ? pkgs.stdenv }: let - microhttpdmy = (import ./microhttpd.nix) {}; + staticOptions = pkg: pkg.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ]; + }); + + secp256k1Static = (staticOptions pkgs.secp256k1); + libsodiumStatic = (staticOptions pkgs.libsodium); + + microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ]; + }); in -with import microhttpdmy; pkgs.llvmPackages_16.stdenv.mkDerivation { pname = "ton"; version = "dev-lib"; @@ -21,7 +33,12 @@ pkgs.llvmPackages_16.stdenv.mkDerivation { buildInputs = with pkgs; [ - pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 pkgsStatic.lz4 + (openssl.override { static = true; }).dev + microhttpdStatic.dev + (zlib.override { shared = false; }).dev + (lz4.override { enableStatic = true; enableShared = false; }).dev + secp256k1Static + libsodiumStatic.dev ]; dontAddStaticConfigureFlags = false; @@ -29,9 +46,6 @@ pkgs.llvmPackages_16.stdenv.mkDerivation { cmakeFlags = [ "-DTON_USE_ABSEIL=OFF" "-DNIX=ON" - "-DMHD_FOUND=1" - "-DMHD_INCLUDE_DIR=${microhttpdmy}/usr/local/include" - "-DMHD_LIBRARY=${microhttpdmy}/usr/local/lib/libmicrohttpd.a" ]; LDFLAGS = [ diff --git a/assembly/nix/linux-x86-64-static.nix b/assembly/nix/linux-x86-64-static.nix index 8c2749b0..a96a9e86 100644 --- a/assembly/nix/linux-x86-64-static.nix +++ b/assembly/nix/linux-x86-64-static.nix @@ -6,9 +6,23 @@ , testing ? false }: let - microhttpdmy = (import ./microhttpd.nix) {}; + staticOptions = pkg: pkg.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ]; + }); + + secp256k1Static = (staticOptions pkgs.secp256k1); + libsodiumStatic = (staticOptions pkgs.libsodium); + jemallocStatic = (staticOptions pkgs.jemalloc); + + microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ]; + }); + in -with import microhttpdmy; stdenv.mkDerivation { pname = "ton"; version = "dev-bin"; @@ -16,31 +30,33 @@ stdenv.mkDerivation { src = ./.; nativeBuildInputs = with pkgs; - [ - cmake ninja git pkg-config - ]; + [ cmake ninja git pkg-config ]; buildInputs = with pkgs; [ - pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 glibc.static pkgsStatic.lz4 + (openssl.override { static = true; }).dev + microhttpdStatic.dev + (zlib.override { shared = false; }).dev + (lz4.override { enableStatic = true; enableShared = false; }).dev + jemallocStatic + secp256k1Static + libsodiumStatic.dev + glibc.static ]; - makeStatic = true; - doCheck = testing; - cmakeFlags = [ "-DTON_USE_ABSEIL=OFF" "-DNIX=ON" "-DBUILD_SHARED_LIBS=OFF" "-DCMAKE_LINK_SEARCH_START_STATIC=ON" "-DCMAKE_LINK_SEARCH_END_STATIC=ON" - "-DMHD_FOUND=1" - "-DMHD_INCLUDE_DIR=${microhttpdmy}/usr/local/include" - "-DMHD_LIBRARY=${microhttpdmy}/usr/local/lib/libmicrohttpd.a" - "-DCMAKE_CTEST_ARGUMENTS=--timeout;1800" + "-DTON_USE_JEMALLOC=ON" ]; + makeStatic = true; + doCheck = testing; + LDFLAGS = [ - "-static-libgcc" "-static-libstdc++" "-static" + "-static-libgcc" "-static-libstdc++" "-fPIC" ]; } diff --git a/assembly/nix/linux-x86-64-tonlib.nix b/assembly/nix/linux-x86-64-tonlib.nix index 5a6e43e8..afcbe3ba 100644 --- a/assembly/nix/linux-x86-64-tonlib.nix +++ b/assembly/nix/linux-x86-64-tonlib.nix @@ -7,20 +7,35 @@ , stdenv ? pkgs.stdenv }: let - system = builtins.currentSystem; + system = builtins.currentSystem; - nixos1909 = (import (builtins.fetchTarball { - url = "https://channels.nixos.org/nixos-19.09/nixexprs.tar.xz"; - sha256 = "1vp1h2gkkrckp8dzkqnpcc6xx5lph5d2z46sg2cwzccpr8ay58zy"; - }) { inherit system; }); - glibc227 = nixos1909.glibc // { pname = "glibc"; }; - stdenv227 = let - cc = pkgs.wrapCCWith { - cc = nixos1909.buildPackages.gcc-unwrapped; - libc = glibc227; - bintools = pkgs.binutils.override { libc = glibc227; }; - }; - in (pkgs.overrideCC pkgs.stdenv cc); + staticOptions = pkg: pkg.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ]; + }); + + secp256k1Static = (staticOptions pkgs.secp256k1); + libsodiumStatic = (staticOptions pkgs.libsodium); + + microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ]; + }); + + nixos1909 = (import (builtins.fetchTarball { + url = "https://channels.nixos.org/nixos-19.09/nixexprs.tar.xz"; + sha256 = "1vp1h2gkkrckp8dzkqnpcc6xx5lph5d2z46sg2cwzccpr8ay58zy"; + }) { inherit system; }); + glibc227 = nixos1909.glibc // { pname = "glibc"; }; + stdenv227 = let + cc = pkgs.wrapCCWith { + cc = nixos1909.buildPackages.gcc-unwrapped; + libc = glibc227; + bintools = pkgs.binutils.override { libc = glibc227; }; + }; + in (pkgs.overrideCC pkgs.stdenv cc); in stdenv227.mkDerivation { @@ -34,7 +49,12 @@ stdenv227.mkDerivation { buildInputs = with pkgs; [ - pkgsStatic.openssl pkgsStatic.zlib pkgsStatic.libmicrohttpd.dev pkgsStatic.libsodium.dev pkgsStatic.secp256k1 pkgsStatic.lz4 + (openssl.override { static = true; }).dev + microhttpdStatic.dev + (zlib.override { shared = false; }).dev + (lz4.override { enableStatic = true; enableShared = false; }).dev + secp256k1Static + libsodiumStatic.dev ]; dontAddStaticConfigureFlags = false; diff --git a/assembly/nix/macos-static.nix b/assembly/nix/macos-static.nix index be15579c..2fd0b3a6 100644 --- a/assembly/nix/macos-static.nix +++ b/assembly/nix/macos-static.nix @@ -17,7 +17,7 @@ pkgs.llvmPackages_14.stdenv.mkDerivation { buildInputs = with pkgs; lib.forEach [ - secp256k1 libsodium.dev libmicrohttpd.dev gmp.dev nettle.dev libtasn1.dev libidn2.dev libunistring.dev gettext (gnutls.override { withP11-kit = false; }).dev + secp256k1 libsodium.dev libmicrohttpd.dev gmp.dev nettle.dev libtasn1.dev libidn2.dev libunistring.dev gettext jemalloc (gnutls.override { withP11-kit = false; }).dev ] (x: x.overrideAttrs(oldAttrs: rec { configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-shared" "--disable-tests" ]; dontDisableStatic = true; })) ++ [ @@ -38,13 +38,13 @@ pkgs.llvmPackages_14.stdenv.mkDerivation { cmakeFlags = [ "-DTON_USE_ABSEIL=OFF" "-DNIX=ON" + "-DTON_USE_JEMALLOC=ON" "-DCMAKE_CROSSCOMPILING=OFF" "-DCMAKE_LINK_SEARCH_START_STATIC=ON" "-DCMAKE_LINK_SEARCH_END_STATIC=ON" "-DBUILD_SHARED_LIBS=OFF" "-DCMAKE_CXX_FLAGS=-stdlib=libc++" "-DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=11.3" - "-DCMAKE_CTEST_ARGUMENTS=--timeout;1800" ]; LDFLAGS = [ diff --git a/assembly/nix/microhttpd.nix b/assembly/nix/microhttpd.nix deleted file mode 100644 index 4f871425..00000000 --- a/assembly/nix/microhttpd.nix +++ /dev/null @@ -1,28 +0,0 @@ -{ pkgs ? import { system = builtins.currentSystem; } -, stdenv ? pkgs.stdenv -, fetchgit ? pkgs.fetchgit -}: - -stdenv.mkDerivation rec { - name = "microhttpdmy"; - - - src = fetchgit { - url = "https://git.gnunet.org/libmicrohttpd.git"; - rev = "refs/tags/v0.9.77"; - sha256 = "sha256-x+nfB07PbZwBlFc6kZZFYiRpk0a3QN/ByHB+hC8na/o="; - }; - - nativeBuildInputs = with pkgs; [ automake libtool autoconf texinfo ]; - - buildInputs = with pkgs; [ ]; - - configurePhase = '' - ./autogen.sh - ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic - ''; - - installPhase = '' - make install DESTDIR=$out - ''; -} diff --git a/assembly/nix/openssl.nix b/assembly/nix/openssl.nix deleted file mode 100644 index 8d30aa50..00000000 --- a/assembly/nix/openssl.nix +++ /dev/null @@ -1,30 +0,0 @@ -{ pkgs ? import { system = builtins.currentSystem; } -, stdenv ? pkgs.stdenv -, fetchFromGitHub ? pkgs.fetchFromGitHub -}: - -stdenv.mkDerivation rec { - name = "opensslmy"; - - src = fetchFromGitHub { - owner = "openssl"; - repo = "openssl"; - rev = "refs/tags/openssl-3.1.4"; - sha256 = "sha256-Vvf1wiNb4ikg1lIS9U137aodZ2JzM711tSWMJFYWtWI="; - }; - - nativeBuildInputs = with pkgs; [ perl ]; - - buildInputs = with pkgs; [ ]; - - postPatch = '' - patchShebangs Configure - ''; - - configurePhase = '' - ./Configure no-shared - ''; - installPhase = '' - make install DESTDIR=$out - ''; -} From 7841d751c08d218597c933628bcd0d0b751f6be8 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Sat, 8 Jun 2024 17:57:38 +0300 Subject: [PATCH 5/5] Add changelog --- Changelog.md | 13 +++++++++++++ recent_changelog.md | 20 +++++++++----------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/Changelog.md b/Changelog.md index a0241055..effe339f 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,16 @@ +## 2024.06 Update + +1. Make Jemalloc default allocator +2. Add candidate broadcasting and caching +3. Limit per address speed for external messages broadcast by reasonably large number +4. Overlay improvements: fix dropping peers in small custom overlays, fix wrong certificate on missed keyblocks +5. Extended statistics and logs for celldb usage, session stats, persistent state serialization +6. Tonlib and explorer fixes +7. Flags for precize control of Celldb: `--celldb-cache-size`, `--celldb-direct-io` and `--celldb-preload-all` +8. Add valiator-console command to stop persistent state serialization +9. Use `@` path separator for defining include path in fift and create-state utilities on Windows only. + + ## 2024.04 Update 1. Emulator: Single call optimized runGetMethod added diff --git a/recent_changelog.md b/recent_changelog.md index f0b029ee..25a93c18 100644 --- a/recent_changelog.md +++ b/recent_changelog.md @@ -1,13 +1,11 @@ ## 2024.04 Update -1. Emulator: Single call optimized runGetMethod added -2. Tonlib: a series of proof improvements, also breaking Change in `liteServer.getAllShardsInfo` method (see below) -3. DB: usage statistics now collected, outdated persistent states are not serialized -4. LS: fast `getOutMsgQueueSizes` added, preliminary support of non-final block requests -5. Network: lz4 compression of block candidates (disabled by default). -6. Overlays: add custom overlays -7. Transaction Executor: fixed issue with due_payment collection - -* `liteServer.getAllShardsInfo` method was updated for better efficiency. Previously, field proof contained BoC with two roots: one for BlockState from block's root and another for ShardHashes from BlockState. Now, it returns a single-root proof BoC, specifically the merkle proof of ShardHashes directly from the block's root, streamlining data access and integrity. Checking of the proof requires to check that ShardHashes in the `data` correspond to ShardHashes from the block. - -Besides the work of the core team, this update is based on the efforts of @akifoq (due_payment issue). +1. Make Jemalloc default allocator +2. Add candidate broadcasting and caching +3. Limit per address speed for external messages broadcast by reasonably large number +4. Overlay improvements: fix dropping peers in small custom overlays, fix wrong certificate on missed keyblocks +5. Extended statistics and logs for celldb usage, session stats, persistent state serialization +6. Tonlib and explorer fixes +7. Flags for precize control of Celldb: `--celldb-cache-size`, `--celldb-direct-io` and `--celldb-preload-all` +8. Add valiator-console command to stop persistent state serialization +9. Use `@` path separator for defining include path in fift and create-state utilities on Windows only.