Use jemalloc in portable artifacts (#1003)

* use jemalloc in portable builds;
also avoid mixing musl with glibc artifacts in nix builds;

* minor fix for ubuntu arm nix build

assembly/nix/build-linux-arm64-nix.sh

@@ -15,8 +15,6 @@ while getopts 't' flag; do
 done
 
 cp assembly/nix/linux-arm64* .
-cp assembly/nix/microhttpd.nix .
-cp assembly/nix/openssl.nix .
 export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz
 
 if [ "$with_tests" = true ]; then
@@ -30,7 +28,9 @@ cp ./result/bin/* artifacts/
 test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; }
 chmod +x artifacts/*
 rm -rf result
+
 nix-build linux-arm64-tonlib.nix
+
 cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so
 cp ./result/lib/libemulator.so artifacts/
 cp ./result/lib/fift/* artifacts/lib/

assembly/nix/build-linux-x86-64-nix.sh

@@ -15,8 +15,6 @@ while getopts 't' flag; do
 done
 
 cp assembly/nix/linux-x86-64* .
-cp assembly/nix/microhttpd.nix .
-cp assembly/nix/openssl.nix .
 export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz
 
 if [ "$with_tests" = true ]; then
@@ -30,7 +28,9 @@ cp ./result/bin/* artifacts/
 test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; }
 chmod +x artifacts/*
 rm -rf result
+
 nix-build linux-x86-64-tonlib.nix
+
 cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so
 cp ./result/lib/libemulator.so artifacts/
 cp ./result/lib/fift/* artifacts/lib/

assembly/nix/build-macos-nix.sh

@@ -28,7 +28,9 @@ cp ./result-bin/bin/* artifacts/
 test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; }
 chmod +x artifacts/*
 rm -rf result-bin
+
 nix-build macos-tonlib.nix
+
 cp ./result/lib/libtonlibjson.dylib artifacts/
 cp ./result/lib/libemulator.dylib artifacts/
 cp ./result/lib/fift/* artifacts/lib/

assembly/nix/linux-arm64-static.nix

@@ -6,9 +6,23 @@
 , testing ? false
 }:
 let
-  microhttpdmy = (import ./microhttpd.nix) {};
+  staticOptions = pkg: pkg.overrideAttrs(oldAttrs: {
+    dontDisableStatic = true;
+    enableSharedExecutables = false;
+    configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ];
+  });
+
+  secp256k1Static = (staticOptions pkgs.secp256k1);
+  libsodiumStatic = (staticOptions pkgs.libsodium);
+  jemallocStatic = (staticOptions pkgs.jemalloc);
+
+  microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: {
+    dontDisableStatic = true;
+    enableSharedExecutables = false;
+    configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ];
+  });
+
 in
-with import microhttpdmy;
 stdenv.mkDerivation {
   pname = "ton";
   version = "dev-bin";
@@ -16,31 +30,33 @@ stdenv.mkDerivation {
   src = ./.;
 
   nativeBuildInputs = with pkgs;
-    [
+    [  cmake ninja git pkg-config ];
-      cmake ninja git pkg-config
-    ];
 
   buildInputs = with pkgs;
     [
-      pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 glibc.static pkgsStatic.lz4
+      (openssl.override { static = true; }).dev
+      microhttpdStatic.dev
+      (zlib.override { shared = false; }).dev
+      (lz4.override { enableStatic = true; enableShared = false; }).dev
+      jemallocStatic
+      secp256k1Static
+      libsodiumStatic.dev
+      glibc.static
     ];
 
-  makeStatic = true;
-  doCheck = testing;
-
   cmakeFlags = [
     "-DTON_USE_ABSEIL=OFF"
     "-DNIX=ON"
     "-DBUILD_SHARED_LIBS=OFF"
     "-DCMAKE_LINK_SEARCH_START_STATIC=ON"
     "-DCMAKE_LINK_SEARCH_END_STATIC=ON"
-    "-DMHD_FOUND=1"
+    "-DTON_USE_JEMALLOC=ON"
-    "-DMHD_INCLUDE_DIR=${microhttpdmy}/usr/local/include"
-    "-DMHD_LIBRARY=${microhttpdmy}/usr/local/lib/libmicrohttpd.a"
-    "-DCMAKE_CTEST_ARGUMENTS=--timeout;1800"
   ];
 
+  makeStatic = true;
+  doCheck = testing;
+
   LDFLAGS = [
-     "-static-libgcc" "-static-libstdc++" "-static"
+    "-static-libgcc" "-static-libstdc++" "-static"
   ];
 }

assembly/nix/linux-arm64-tonlib.nix

@@ -5,9 +5,21 @@
 , stdenv ? pkgs.stdenv
 }:
 let
-     microhttpdmy = (import ./microhttpd.nix) {};
+  staticOptions = pkg: pkg.overrideAttrs(oldAttrs: {
+    dontDisableStatic = true;
+    enableSharedExecutables = false;
+    configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ];
+  });
+
+  secp256k1Static = (staticOptions pkgs.secp256k1);
+  libsodiumStatic = (staticOptions pkgs.libsodium);
+
+  microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: {
+    dontDisableStatic = true;
+    enableSharedExecutables = false;
+    configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ];
+  });
 in
-with import microhttpdmy;
 pkgs.llvmPackages_16.stdenv.mkDerivation {
   pname = "ton";
   version = "dev-lib";
@@ -21,7 +33,12 @@ pkgs.llvmPackages_16.stdenv.mkDerivation {
 
   buildInputs = with pkgs;
     [
-      pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 pkgsStatic.lz4
+      (openssl.override { static = true; }).dev
+      microhttpdStatic.dev
+      (zlib.override { shared = false; }).dev
+      (lz4.override { enableStatic = true; enableShared = false; }).dev
+      secp256k1Static
+      libsodiumStatic.dev
     ];
 
   dontAddStaticConfigureFlags = false;
@@ -29,9 +46,6 @@ pkgs.llvmPackages_16.stdenv.mkDerivation {
   cmakeFlags = [
     "-DTON_USE_ABSEIL=OFF"
     "-DNIX=ON"
-    "-DMHD_FOUND=1"
-    "-DMHD_INCLUDE_DIR=${microhttpdmy}/usr/local/include"
-    "-DMHD_LIBRARY=${microhttpdmy}/usr/local/lib/libmicrohttpd.a"
   ];
 
   LDFLAGS = [

assembly/nix/linux-x86-64-static.nix

@@ -6,9 +6,23 @@
 , testing ? false
 }:
 let
-  microhttpdmy = (import ./microhttpd.nix) {};
+  staticOptions = pkg: pkg.overrideAttrs(oldAttrs: {
+    dontDisableStatic = true;
+    enableSharedExecutables = false;
+    configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ];
+  });
+
+  secp256k1Static = (staticOptions pkgs.secp256k1);
+  libsodiumStatic = (staticOptions pkgs.libsodium);
+  jemallocStatic = (staticOptions pkgs.jemalloc);
+
+  microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: {
+    dontDisableStatic = true;
+    enableSharedExecutables = false;
+    configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ];
+  });
+
 in
-with import microhttpdmy;
 stdenv.mkDerivation {
   pname = "ton";
   version = "dev-bin";
@@ -16,31 +30,33 @@ stdenv.mkDerivation {
   src = ./.;
 
   nativeBuildInputs = with pkgs;
-    [
+    [  cmake ninja git pkg-config ];
-      cmake ninja git pkg-config
-    ];
 
   buildInputs = with pkgs;
     [
-      pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 glibc.static pkgsStatic.lz4
+      (openssl.override { static = true; }).dev
+      microhttpdStatic.dev
+      (zlib.override { shared = false; }).dev
+      (lz4.override { enableStatic = true; enableShared = false; }).dev
+      jemallocStatic
+      secp256k1Static
+      libsodiumStatic.dev
+      glibc.static
     ];
 
-  makeStatic = true;
-  doCheck = testing;
-
   cmakeFlags = [
     "-DTON_USE_ABSEIL=OFF"
     "-DNIX=ON"
     "-DBUILD_SHARED_LIBS=OFF"
     "-DCMAKE_LINK_SEARCH_START_STATIC=ON"
     "-DCMAKE_LINK_SEARCH_END_STATIC=ON"
-    "-DMHD_FOUND=1"
+    "-DTON_USE_JEMALLOC=ON"
-    "-DMHD_INCLUDE_DIR=${microhttpdmy}/usr/local/include"
-    "-DMHD_LIBRARY=${microhttpdmy}/usr/local/lib/libmicrohttpd.a"
-    "-DCMAKE_CTEST_ARGUMENTS=--timeout;1800"
   ];
 
+  makeStatic = true;
+  doCheck = testing;
+
   LDFLAGS = [
-     "-static-libgcc" "-static-libstdc++" "-static"
+    "-static-libgcc" "-static-libstdc++" "-fPIC"
   ];
 }

assembly/nix/linux-x86-64-tonlib.nix

@@ -7,20 +7,35 @@
 , stdenv ? pkgs.stdenv
 }:
 let
-      system = builtins.currentSystem;
+  system = builtins.currentSystem;
 
-          nixos1909 = (import (builtins.fetchTarball {
+  staticOptions = pkg: pkg.overrideAttrs(oldAttrs: {
-            url = "https://channels.nixos.org/nixos-19.09/nixexprs.tar.xz";
+    dontDisableStatic = true;
-            sha256 = "1vp1h2gkkrckp8dzkqnpcc6xx5lph5d2z46sg2cwzccpr8ay58zy";
+    enableSharedExecutables = false;
-          }) { inherit system; });
+    configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ];
-          glibc227 = nixos1909.glibc // { pname = "glibc"; };
+  });
-          stdenv227 = let
+
-            cc = pkgs.wrapCCWith {
+  secp256k1Static = (staticOptions pkgs.secp256k1);
-              cc = nixos1909.buildPackages.gcc-unwrapped;
+  libsodiumStatic = (staticOptions pkgs.libsodium);
-              libc = glibc227;
+
-              bintools = pkgs.binutils.override { libc = glibc227; };
+  microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: {
-            };
+    dontDisableStatic = true;
-          in (pkgs.overrideCC pkgs.stdenv cc);
+    enableSharedExecutables = false;
+    configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ];
+  });
+
+  nixos1909 = (import (builtins.fetchTarball {
+    url = "https://channels.nixos.org/nixos-19.09/nixexprs.tar.xz";
+    sha256 = "1vp1h2gkkrckp8dzkqnpcc6xx5lph5d2z46sg2cwzccpr8ay58zy";
+  }) { inherit system; });
+  glibc227 = nixos1909.glibc // { pname = "glibc"; };
+  stdenv227 = let
+    cc = pkgs.wrapCCWith {
+      cc = nixos1909.buildPackages.gcc-unwrapped;
+      libc = glibc227;
+      bintools = pkgs.binutils.override { libc = glibc227; };
+    };
+  in (pkgs.overrideCC pkgs.stdenv cc);
 
 in
 stdenv227.mkDerivation {
@@ -34,7 +49,12 @@ stdenv227.mkDerivation {
 
   buildInputs = with pkgs;
     [
-      pkgsStatic.openssl pkgsStatic.zlib pkgsStatic.libmicrohttpd.dev pkgsStatic.libsodium.dev pkgsStatic.secp256k1 pkgsStatic.lz4
+      (openssl.override { static = true; }).dev
+      microhttpdStatic.dev
+      (zlib.override { shared = false; }).dev
+      (lz4.override { enableStatic = true; enableShared = false; }).dev
+      secp256k1Static
+      libsodiumStatic.dev
     ];
 
   dontAddStaticConfigureFlags = false;

assembly/nix/macos-static.nix

@@ -17,7 +17,7 @@ pkgs.llvmPackages_14.stdenv.mkDerivation {
 
   buildInputs = with pkgs;
    lib.forEach [
-        secp256k1 libsodium.dev libmicrohttpd.dev gmp.dev nettle.dev libtasn1.dev libidn2.dev libunistring.dev gettext (gnutls.override { withP11-kit = false; }).dev
+        secp256k1 libsodium.dev libmicrohttpd.dev gmp.dev nettle.dev libtasn1.dev libidn2.dev libunistring.dev gettext jemalloc (gnutls.override { withP11-kit = false; }).dev
       ]
       (x: x.overrideAttrs(oldAttrs: rec { configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-shared" "--disable-tests" ]; dontDisableStatic = true; }))
     ++ [
@@ -38,13 +38,13 @@ pkgs.llvmPackages_14.stdenv.mkDerivation {
   cmakeFlags = [
     "-DTON_USE_ABSEIL=OFF"
     "-DNIX=ON"
+    "-DTON_USE_JEMALLOC=ON"
     "-DCMAKE_CROSSCOMPILING=OFF"
     "-DCMAKE_LINK_SEARCH_START_STATIC=ON"
     "-DCMAKE_LINK_SEARCH_END_STATIC=ON"
     "-DBUILD_SHARED_LIBS=OFF"
     "-DCMAKE_CXX_FLAGS=-stdlib=libc++"
     "-DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=11.3"
-    "-DCMAKE_CTEST_ARGUMENTS=--timeout;1800"
   ];
 
   LDFLAGS = [

assembly/nix/microhttpd.nix

@@ -1,28 +0,0 @@
-{ pkgs ? import <nixpkgs> { system = builtins.currentSystem; }
-, stdenv ? pkgs.stdenv
-, fetchgit ? pkgs.fetchgit
-}:
-
-stdenv.mkDerivation rec {
-  name = "microhttpdmy";
-
-
-  src = fetchgit {
-    url = "https://git.gnunet.org/libmicrohttpd.git";
-    rev = "refs/tags/v0.9.77";
-    sha256 = "sha256-x+nfB07PbZwBlFc6kZZFYiRpk0a3QN/ByHB+hC8na/o=";
-  };
-
-  nativeBuildInputs = with pkgs; [ automake libtool autoconf texinfo ];
-
-  buildInputs = with pkgs; [ ];
-
-  configurePhase = ''
-   ./autogen.sh
-   ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic
-  '';
-
-  installPhase = ''
-    make install DESTDIR=$out
-  '';
-}

assembly/nix/openssl.nix

@@ -1,30 +0,0 @@
-{ pkgs ? import <nixpkgs> { system = builtins.currentSystem; }
-, stdenv ? pkgs.stdenv
-, fetchFromGitHub ? pkgs.fetchFromGitHub
-}:
-
-stdenv.mkDerivation rec {
-  name = "opensslmy";
-
-  src = fetchFromGitHub {
-    owner = "openssl";
-    repo = "openssl";
-    rev = "refs/tags/openssl-3.1.4";
-    sha256 = "sha256-Vvf1wiNb4ikg1lIS9U137aodZ2JzM711tSWMJFYWtWI=";
-  };
-
-  nativeBuildInputs = with pkgs; [ perl ];
-
-  buildInputs = with pkgs; [ ];
-
-  postPatch = ''
-    patchShebangs Configure
-  '';
-
-  configurePhase = ''
-   ./Configure no-shared
-  '';
-  installPhase = ''
-    make install DESTDIR=$out
-  '';
-}