lz4 support in multiple builds (#946)

* Compress block candidates in validator-session * Compress blocks in full-node (disabled for now) * test pipeline with lz4 * tonlib compilation required lz4; try win compile; * install lz4 on mac. * wip, test builds * remove FindLZ4.cmake * fix typo * fix wasm lz4 path * increase groovy timeout to 120 sec * add lz4 for android and emscripten builds * add lz4 for android and emscripten builds * fix win build include path for lz4 * add precompiled lz4 for android * cleanup * adjust android include dir for lz4 * fix path for android arm of lz4 * cleanup * minor fix --------- Co-authored-by: SpyCheese <mikle98@yandex.ru>
2025-03-09 15:40:10 +00:00 · 2024-03-28 11:18:11 +04:00 · 2024-03-28 11:18:11 +04:00 · b7849249c6
commit b7849249c6
parent 10487b1c71
38 changed files with 11004 additions and 68 deletions
--- a/assembly/cicd/jenkins/test-builds.groovy
+++ b/assembly/cicd/jenkins/test-builds.groovy
@ -8,7 +8,7 @@ pipeline {
                        label 'Ubuntu_x86-64'
                    }
                    steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                            sh '''
                                cp assembly/native/build-ubuntu-shared.sh .
                                chmod +x build-ubuntu-shared.sh
@ -27,7 +27,7 @@ pipeline {
                        label 'Ubuntu_x86-64'
                    }
                    steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                            sh '''
                                cp assembly/nix/build-linux-x86-64-nix.sh .
                                chmod +x build-linux-x86-64-nix.sh
@ -46,7 +46,7 @@ pipeline {
                        label 'Ubuntu_arm64'
                    }
                    steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                            sh '''
                                cp assembly/native/build-ubuntu-shared.sh .
                                chmod +x build-ubuntu-shared.sh
@ -65,7 +65,7 @@ pipeline {
                        label 'Ubuntu_arm64'
                    }
                    steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                            sh '''
                                cp assembly/nix/build-linux-arm64-nix.sh .
                                chmod +x build-linux-arm64-nix.sh
@ -84,7 +84,7 @@ pipeline {
                        label 'macOS_12.7_x86-64'
                    }
                    steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                            sh '''
                                cp assembly/native/build-macos-shared.sh .
                                chmod +x build-macos-shared.sh
@ -103,7 +103,7 @@ pipeline {
                        label 'macOS_12.7_x86-64'
                    }
                    steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                            sh '''
                                cp assembly/nix/build-macos-nix.sh .
                                chmod +x build-macos-nix.sh
@ -122,7 +122,7 @@ pipeline {
                        label 'macOS_12.6-arm64-m1'
                    }
                    steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                            sh '''
                                cp assembly/native/build-macos-shared.sh .
                                chmod +x build-macos-shared.sh
@ -141,7 +141,7 @@ pipeline {
                        label 'macOS_12.6-arm64-m1'
                    }
                    steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                            sh '''
                                cp assembly/nix/build-macos-nix.sh .
                                chmod +x build-macos-nix.sh
@ -160,7 +160,7 @@ pipeline {
                        label 'macOS_13.2-arm64-m2'
                    }
                    steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                            sh '''
                                cp assembly/native/build-macos-shared.sh .
                                chmod +x build-macos-shared.sh
@ -179,7 +179,7 @@ pipeline {
                        label 'Windows_x86-64'
                    }
                    steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                            bat '''
                                copy assembly\\native\\build-windows.bat .
                                build-windows.bat -t
@ -197,7 +197,7 @@ pipeline {
                        label 'Ubuntu_x86-64'
                    }
                    steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                            sh '''
                                cp assembly/android/build-android-tonlib.sh .
                                chmod +x build-android-tonlib.sh
@ -216,7 +216,7 @@ pipeline {
                        label 'Ubuntu_x86-64'
                    }
                    steps {
-                        timeout(time: 90, unit: 'MINUTES') {
+                        timeout(time: 180, unit: 'MINUTES') {
                            sh '''
                                cd assembly/wasm
                                chmod +x fift-func-wasm-build-ubuntu.sh
--- a/assembly/native/build-macos-portable.sh
+++ b/assembly/native/build-macos-portable.sh
@ -37,6 +37,21 @@ else
 fi
 export CCACHE_DISABLE=1

+if [ ! -d "lz4" ]; then
+git clone https://github.com/lz4/lz4.git
+cd lz4
+lz4Path=`pwd`
+git checkout v1.9.4
+make -j12
+test $? -eq 0 || { echo "Can't compile lz4"; exit 1; }
+cd ..
+# ./lib/liblz4.a
+# ./lib
+else
+  lz4Path=$(pwd)/lz4
+  echo "Using compiled lz4"
+fi
+
 if [ ! -d "secp256k1" ]; then
 git clone https://github.com/bitcoin-core/secp256k1.git
 cd secp256k1
@ -128,7 +143,10 @@ cmake -GNinja .. \
 -DSODIUM_LIBRARY_RELEASE=$sodiumPath/src/libsodium/.libs/libsodium.a \
 -DMHD_FOUND=1 \
 -DMHD_INCLUDE_DIR=$libmicrohttpdPath/src/include \
-DMHD_LIBRARY=$libmicrohttpdPath/src/microhttpd/.libs/libmicrohttpd.a
+-DMHD_LIBRARY=$libmicrohttpdPath/src/microhttpd/.libs/libmicrohttpd.a \
+-DLZ4_FOUND=1 \
+-DLZ4_INCLUDE_DIRS=$lz4Path/lib \
+-DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a


 test $? -eq 0 || { echo "Can't configure ton"; exit 1; }
--- a/assembly/native/build-macos-shared.sh
+++ b/assembly/native/build-macos-shared.sh
@ -51,6 +51,19 @@ else
  echo "Using compiled secp256k1"
 fi

+if [ ! -d "lz4" ]; then
+  git clone https://github.com/lz4/lz4
+  cd lz4
+  lz4Path=`pwd`
+  git checkout v1.9.4
+  make -j12
+  test $? -eq 0 || { echo "Can't compile lz4"; exit 1; }
+  cd ..
+else
+  lz4Path=$(pwd)/lz4
+  echo "Using compiled lz4"
+fi
+
 brew unlink openssl@1.1
 brew install openssl@3
 brew unlink openssl@3 &&  brew link --overwrite openssl@3
@ -59,7 +72,10 @@ cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
 -DCMAKE_CXX_FLAGS="-stdlib=libc++" \
 -DSECP256K1_FOUND=1 \
 -DSECP256K1_INCLUDE_DIR=$secp256k1Path/include \
-DSECP256K1_LIBRARY=$secp256k1Path/.libs/libsecp256k1.a
+-DSECP256K1_LIBRARY=$secp256k1Path/.libs/libsecp256k1.a \
+-DLZ4_FOUND=1 \
+-DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a \
+-DLZ4_INCLUDE_DIRS=$lz4Path/lib

 test $? -eq 0 || { echo "Can't configure ton"; exit 1; }

--- a/assembly/native/build-ubuntu-portable.sh
+++ b/assembly/native/build-ubuntu-portable.sh
@ -28,6 +28,20 @@ export CC=$(which clang-16)
 export CXX=$(which clang++-16)
 export CCACHE_DISABLE=1

+if [ ! -d "lz4" ]; then
+git clone https://github.com/lz4/lz4.git
+cd lz4
+lz4Path=`pwd`
+git checkout v1.9.4
+make -j12
+test $? -eq 0 || { echo "Can't compile lz4"; exit 1; }
+cd ..
+# ./lib/liblz4.a
+# ./lib
+else
+  lz4Path=$(pwd)/lz4
+  echo "Using compiled lz4"
+fi

 if [ ! -d "secp256k1" ]; then
 git clone https://github.com/bitcoin-core/secp256k1.git
@ -120,7 +134,11 @@ cmake -GNinja .. \
 -DSODIUM_LIBRARY_RELEASE=$sodiumPath/src/libsodium/.libs/libsodium.a \
 -DMHD_FOUND=1 \
 -DMHD_INCLUDE_DIR=$libmicrohttpdPath/src/include \
-DMHD_LIBRARY=$libmicrohttpdPath/src/microhttpd/.libs/libmicrohttpd.a
+-DMHD_LIBRARY=$libmicrohttpdPath/src/microhttpd/.libs/libmicrohttpd.a \
+-DLZ4_FOUND=1 \
+-DLZ4_INCLUDE_DIRS=$lz4Path/lib \
+-DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a
+


 test $? -eq 0 || { echo "Can't configure ton"; exit 1; }
--- a/assembly/native/build-ubuntu-shared.sh
+++ b/assembly/native/build-ubuntu-shared.sh
@ -1,7 +1,7 @@
 #/bin/bash

 #sudo apt-get update
-#sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev
+#sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev

 with_tests=false
 with_artifacts=false
@ -119,4 +119,4 @@ if [ "$with_tests" = true ]; then
  cd build
 #  ctest --output-on-failure -E "test-catchain|test-actors|test-smartcont|test-adnl|test-validator-session-state|test-dht|test-rldp"
  ctest --output-on-failure --timeout 1800
-fi
+fi
--- a/assembly/native/build-windows.bat
+++ b/assembly/native/build-windows.bat
@ -42,6 +42,22 @@ cd ..\..\..\..
 echo Using zlib...
 )

+if not exist "lz4" (
+git clone https://github.com/lz4/lz4.git
+cd lz4
+git checkout v1.9.4
+cd build\VS2017\liblz4
+msbuild liblz4.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v143
+dir /s
+IF %errorlevel% NEQ 0 (
+  echo Can't install lz4
+  exit /b %errorlevel%
+)
+cd ..\..\..\..
+) else (
+echo Using lz4...
+)
+
 if not exist "secp256k1" (
 git clone https://github.com/bitcoin-core/secp256k1.git
 cd secp256k1
@ -119,6 +135,9 @@ cmake -GNinja  -DCMAKE_BUILD_TYPE=Release ^
 -DSECP256K1_FOUND=1 ^
 -DSECP256K1_INCLUDE_DIR=%root%\secp256k1\include ^
 -DSECP256K1_LIBRARY=%root%\secp256k1\build\src\Release\libsecp256k1.lib ^
+-DLZ4_FOUND=1 ^
+-DLZ4_INCLUDE_DIRS=%root%\lz4\lib ^
+-DLZ4_LIBRARIES=%root%\lz4\build\VS2017\liblz4\bin\x64_Release\liblz4_static.lib ^
 -DMHD_FOUND=1 ^
 -DMHD_LIBRARY=%root%\libmicrohttpd-0.9.77-w32-bin\x86_64\VS2019\Release-static\libmicrohttpd.lib ^
 -DMHD_INCLUDE_DIR=%root%\libmicrohttpd-0.9.77-w32-bin\x86_64\VS2019\Release-static ^
--- a/assembly/nix/linux-arm64-static.nix
+++ b/assembly/nix/linux-arm64-static.nix
@ -22,7 +22,7 @@ stdenv.mkDerivation {

  buildInputs = with pkgs;
    [
-      pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 glibc.static
+      pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 glibc.static pkgsStatic.lz4
    ];

  makeStatic = true;
--- a/assembly/nix/linux-arm64-tonlib.nix
+++ b/assembly/nix/linux-arm64-tonlib.nix
@ -21,7 +21,7 @@ pkgs.llvmPackages_16.stdenv.mkDerivation {

  buildInputs = with pkgs;
    [
-      pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1
+      pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 pkgsStatic.lz4
    ];

  dontAddStaticConfigureFlags = false;
--- a/assembly/nix/linux-x86-64-static.nix
+++ b/assembly/nix/linux-x86-64-static.nix
@ -22,7 +22,7 @@ stdenv.mkDerivation {

  buildInputs = with pkgs;
    [
-      pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 glibc.static
+      pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 glibc.static pkgsStatic.lz4
    ];

  makeStatic = true;
--- a/assembly/nix/linux-x86-64-tonlib.nix
+++ b/assembly/nix/linux-x86-64-tonlib.nix
@ -34,7 +34,7 @@ stdenv227.mkDerivation {

  buildInputs = with pkgs;
    [
-      pkgsStatic.openssl pkgsStatic.zlib pkgsStatic.libmicrohttpd.dev pkgsStatic.libsodium.dev pkgsStatic.secp256k1
+      pkgsStatic.openssl pkgsStatic.zlib pkgsStatic.libmicrohttpd.dev pkgsStatic.libsodium.dev pkgsStatic.secp256k1 pkgsStatic.lz4
    ];

  dontAddStaticConfigureFlags = false;
--- a/assembly/nix/macos-static.nix
+++ b/assembly/nix/macos-static.nix
@ -25,6 +25,7 @@ pkgs.llvmPackages_14.stdenv.mkDerivation {
      (openssl.override { static = true; }).dev
      (zlib.override { shared = false; }).dev
      (libiconv.override { enableStatic = true; enableShared = false; })
+      (lz4.override { enableStatic = true; enableShared = false; }).dev
   ];


--- a/assembly/nix/macos-tonlib.nix
+++ b/assembly/nix/macos-tonlib.nix
@ -23,6 +23,7 @@ pkgs.llvmPackages_14.stdenv.mkDerivation {
      (openssl.override { static = true; }).dev
      (zlib.override { shared = false; }).dev
      (libiconv.override { enableStatic = true; enableShared = false; })
+      (lz4.override { enableStatic = true; enableShared = false; }).dev
    ];

  dontAddStaticConfigureFlags = true;
@ -52,4 +53,4 @@ pkgs.llvmPackages_14.stdenv.mkDerivation {
        install_name_tool -change "$(otool -L "$fn" | grep libc++abi.1 | cut -d' ' -f1 | xargs)" libc++abi.dylib "$fn"
     done
  '';
-}
+}
--- a/assembly/wasm/fift-func-wasm-build-ubuntu.sh
+++ b/assembly/wasm/fift-func-wasm-build-ubuntu.sh
@ -41,6 +41,11 @@ cd zlib
 ZLIB_DIR=`pwd`
 cd ..

+git clone https://github.com/lz4/lz4.git
+cd lz4
+LZ4_DIR=`pwd`
+cd ..
+
 git clone https://github.com/bitcoin-core/secp256k1.git
 cd secp256k1
 ./autogen.sh
@ -101,7 +106,10 @@ cd ../zlib
 emconfigure ./configure --static
 emmake make -j16
 test $? -eq 0 || { echo "Can't compile zlib with emmake "; exit 1; }
-ZLIB_DIR=`pwd`
+
+cd ../lz4
+emmake make -j16
+test $? -eq 0 || { echo "Can't compile lz4 with emmake "; exit 1; }

 cd ../secp256k1

@ -121,6 +129,9 @@ emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release \
 -DZLIB_FOUND=1 \
 -DZLIB_LIBRARIES=$ZLIB_DIR/libz.a \
 -DZLIB_INCLUDE_DIR=$ZLIB_DIR \
+-DLZ4_FOUND=1 \
+-DLZ4_LIBRARIES=$LZ4_DIR/lib/liblz4.a \
+-DLZ4_INCLUDE_DIRS=$LZ4_DIR/lib \
 -DOPENSSL_FOUND=1 \
 -DOPENSSL_ROOT_DIR=$OPENSSL_DIR \
 -DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \