lz4 support in multiple builds (#946)

* Compress block candidates in validator-session

* Compress blocks in full-node (disabled for now)

* test pipeline with lz4

* tonlib compilation required lz4;
try win compile;

* install lz4 on mac.

* wip, test builds

* remove FindLZ4.cmake

* fix typo

* fix wasm lz4 path

* increase groovy timeout to 120 sec

* add lz4 for android and emscripten builds

* add lz4 for android and emscripten builds

* fix win build include path for lz4

* add precompiled lz4 for android

* cleanup

* adjust android include dir for lz4

* fix path for android arm of lz4

* cleanup

* minor fix

---------

Co-authored-by: SpyCheese <mikle98@yandex.ru>

assembly/native/build-macos-portable.sh

@@ -37,6 +37,21 @@ else
 fi
 export CCACHE_DISABLE=1
 
+if [ ! -d "lz4" ]; then
+git clone https://github.com/lz4/lz4.git
+cd lz4
+lz4Path=`pwd`
+git checkout v1.9.4
+make -j12
+test $? -eq 0 || { echo "Can't compile lz4"; exit 1; }
+cd ..
+# ./lib/liblz4.a
+# ./lib
+else
+  lz4Path=$(pwd)/lz4
+  echo "Using compiled lz4"
+fi
+
 if [ ! -d "secp256k1" ]; then
 git clone https://github.com/bitcoin-core/secp256k1.git
 cd secp256k1
@@ -128,7 +143,10 @@ cmake -GNinja .. \
 -DSODIUM_LIBRARY_RELEASE=$sodiumPath/src/libsodium/.libs/libsodium.a \
 -DMHD_FOUND=1 \
 -DMHD_INCLUDE_DIR=$libmicrohttpdPath/src/include \
--DMHD_LIBRARY=$libmicrohttpdPath/src/microhttpd/.libs/libmicrohttpd.a
+-DMHD_LIBRARY=$libmicrohttpdPath/src/microhttpd/.libs/libmicrohttpd.a \
+-DLZ4_FOUND=1 \
+-DLZ4_INCLUDE_DIRS=$lz4Path/lib \
+-DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a
 
 
 test $? -eq 0 || { echo "Can't configure ton"; exit 1; }

assembly/native/build-macos-shared.sh

@@ -51,6 +51,19 @@ else
   echo "Using compiled secp256k1"
 fi
 
+if [ ! -d "lz4" ]; then
+  git clone https://github.com/lz4/lz4
+  cd lz4
+  lz4Path=`pwd`
+  git checkout v1.9.4
+  make -j12
+  test $? -eq 0 || { echo "Can't compile lz4"; exit 1; }
+  cd ..
+else
+  lz4Path=$(pwd)/lz4
+  echo "Using compiled lz4"
+fi
+
 brew unlink openssl@1.1
 brew install openssl@3
 brew unlink openssl@3 &&  brew link --overwrite openssl@3
@@ -59,7 +72,10 @@ cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
 -DCMAKE_CXX_FLAGS="-stdlib=libc++" \
 -DSECP256K1_FOUND=1 \
 -DSECP256K1_INCLUDE_DIR=$secp256k1Path/include \
--DSECP256K1_LIBRARY=$secp256k1Path/.libs/libsecp256k1.a
+-DSECP256K1_LIBRARY=$secp256k1Path/.libs/libsecp256k1.a \
+-DLZ4_FOUND=1 \
+-DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a \
+-DLZ4_INCLUDE_DIRS=$lz4Path/lib
 
 test $? -eq 0 || { echo "Can't configure ton"; exit 1; }
 

assembly/native/build-ubuntu-portable.sh

@@ -28,6 +28,20 @@ export CC=$(which clang-16)
 export CXX=$(which clang++-16)
 export CCACHE_DISABLE=1
 
+if [ ! -d "lz4" ]; then
+git clone https://github.com/lz4/lz4.git
+cd lz4
+lz4Path=`pwd`
+git checkout v1.9.4
+make -j12
+test $? -eq 0 || { echo "Can't compile lz4"; exit 1; }
+cd ..
+# ./lib/liblz4.a
+# ./lib
+else
+  lz4Path=$(pwd)/lz4
+  echo "Using compiled lz4"
+fi
 
 if [ ! -d "secp256k1" ]; then
 git clone https://github.com/bitcoin-core/secp256k1.git
@@ -120,7 +134,11 @@ cmake -GNinja .. \
 -DSODIUM_LIBRARY_RELEASE=$sodiumPath/src/libsodium/.libs/libsodium.a \
 -DMHD_FOUND=1 \
 -DMHD_INCLUDE_DIR=$libmicrohttpdPath/src/include \
--DMHD_LIBRARY=$libmicrohttpdPath/src/microhttpd/.libs/libmicrohttpd.a
+-DMHD_LIBRARY=$libmicrohttpdPath/src/microhttpd/.libs/libmicrohttpd.a \
+-DLZ4_FOUND=1 \
+-DLZ4_INCLUDE_DIRS=$lz4Path/lib \
+-DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a
+
 
 
 test $? -eq 0 || { echo "Can't configure ton"; exit 1; }

assembly/native/build-ubuntu-shared.sh

@@ -1,7 +1,7 @@
 #/bin/bash
 
 #sudo apt-get update
-#sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev
+#sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev
 
 with_tests=false
 with_artifacts=false
@@ -119,4 +119,4 @@ if [ "$with_tests" = true ]; then
   cd build
 #  ctest --output-on-failure -E "test-catchain|test-actors|test-smartcont|test-adnl|test-validator-session-state|test-dht|test-rldp"
   ctest --output-on-failure --timeout 1800
-fi
+fi

assembly/native/build-windows.bat

@@ -42,6 +42,22 @@ cd ..\..\..\..
 echo Using zlib...
 )
 
+if not exist "lz4" (
+git clone https://github.com/lz4/lz4.git
+cd lz4
+git checkout v1.9.4
+cd build\VS2017\liblz4
+msbuild liblz4.vcxproj /p:Configuration=Release /p:platform=x64 -p:PlatformToolset=v143
+dir /s
+IF %errorlevel% NEQ 0 (
+  echo Can't install lz4
+  exit /b %errorlevel%
+)
+cd ..\..\..\..
+) else (
+echo Using lz4...
+)
+
 if not exist "secp256k1" (
 git clone https://github.com/bitcoin-core/secp256k1.git
 cd secp256k1
@@ -119,6 +135,9 @@ cmake -GNinja  -DCMAKE_BUILD_TYPE=Release ^
 -DSECP256K1_FOUND=1 ^
 -DSECP256K1_INCLUDE_DIR=%root%\secp256k1\include ^
 -DSECP256K1_LIBRARY=%root%\secp256k1\build\src\Release\libsecp256k1.lib ^
+-DLZ4_FOUND=1 ^
+-DLZ4_INCLUDE_DIRS=%root%\lz4\lib ^
+-DLZ4_LIBRARIES=%root%\lz4\build\VS2017\liblz4\bin\x64_Release\liblz4_static.lib ^
 -DMHD_FOUND=1 ^
 -DMHD_LIBRARY=%root%\libmicrohttpd-0.9.77-w32-bin\x86_64\VS2019\Release-static\libmicrohttpd.lib ^
 -DMHD_INCLUDE_DIR=%root%\libmicrohttpd-0.9.77-w32-bin\x86_64\VS2019\Release-static ^