summary refs log tree commit diff
diff options
context:
space:
mode:
authorSergei Trofimovich <slyich@gmail.com>2023-02-21 19:50:48 +0000
committerSergei Trofimovich <slyich@gmail.com>2023-02-26 22:02:09 +0000
commit69cf5181c31c691e9fb225ada9a33f54208d914b (patch)
treed9fcfa94ee535efc424a566105d1d4993f6af108
parent203e1cb49986de7452cb907417fedc80a43a6e70 (diff)
downloadnixpkgs-69cf5181c31c691e9fb225ada9a33f54208d914b.tar
nixpkgs-69cf5181c31c691e9fb225ada9a33f54208d914b.tar.gz
nixpkgs-69cf5181c31c691e9fb225ada9a33f54208d914b.tar.bz2
nixpkgs-69cf5181c31c691e9fb225ada9a33f54208d914b.tar.lz
nixpkgs-69cf5181c31c691e9fb225ada9a33f54208d914b.tar.xz
nixpkgs-69cf5181c31c691e9fb225ada9a33f54208d914b.tar.zst
nixpkgs-69cf5181c31c691e9fb225ada9a33f54208d914b.zip
stdenv/generic/setup.sh: enable parallel installs by default
The primary motivating example is openssl:

Before the change full package build took 1m54s minutes.
After the change full package build takes 59s.

About a 2x speedup.

The difference is visible because openssl builds hundreds of manpages
spawning a perl process per manual in `install` phase. Such a workload
is very easy to parallelize.

Another example would be `autotools`+`libtool` based build system where
install step requires relinking. The more binaries there are to relink
the more gain it will be to do it in parallel.

The change enables parallel installs by default only for buiilds that
already have parallel builds enabled. There is a high chance those build
systems already handle parallelism well but some packages will fail.

Consistently propagated the enableParallelBuilding to:
- cmake (enabled by default, similar to builds)
- ninja (set parallelism explicitly, don't rely on default)
- bmake (enable when requested)
- scons (enable when requested)
- meson (set parallelism explicitly, don't rely on default)
- waf (set parallelism explicitly, don't rely on default)
- qmake-4/5/6 (enable by default, similar to builds)
- xorg (always enable, similar to builds)
-rw-r--r--pkgs/development/libraries/qt-4.x/4.8/qmake-hook.sh5
-rw-r--r--pkgs/development/libraries/qt-5/hooks/qmake-hook.sh5
-rw-r--r--pkgs/development/libraries/qt-6/hooks/qmake-hook.sh5
-rw-r--r--pkgs/development/tools/build-managers/bmake/setup-hook.sh1
-rwxr-xr-xpkgs/development/tools/build-managers/cmake/setup-hook.sh5
-rw-r--r--pkgs/development/tools/build-managers/meson/setup-hook.sh5
-rw-r--r--pkgs/development/tools/build-managers/ninja/setup-hook.sh8
-rw-r--r--pkgs/development/tools/build-managers/scons/setup-hook.sh1
-rw-r--r--pkgs/development/tools/build-managers/wafHook/setup-hook.sh6
-rw-r--r--pkgs/servers/x11/xorg/builder.sh1
-rw-r--r--pkgs/stdenv/generic/make-derivation.nix1
-rw-r--r--pkgs/stdenv/generic/setup.sh1
12 files changed, 44 insertions, 0 deletions
diff --git a/pkgs/development/libraries/qt-4.x/4.8/qmake-hook.sh b/pkgs/development/libraries/qt-4.x/4.8/qmake-hook.sh
index f288e99dd12..b024f52f204 100644
--- a/pkgs/development/libraries/qt-4.x/4.8/qmake-hook.sh
+++ b/pkgs/development/libraries/qt-4.x/4.8/qmake-hook.sh
@@ -8,6 +8,11 @@ qmakeConfigurePhase() {
         echo "qmake4Hook: enabled parallel building"
     fi
 
+    if ! [[ -v enableParallelInstalling ]]; then
+        enableParallelInstalling=1
+        echo "qmake: enabled parallel installing"
+    fi
+
     runHook postConfigure
 }
 
diff --git a/pkgs/development/libraries/qt-5/hooks/qmake-hook.sh b/pkgs/development/libraries/qt-5/hooks/qmake-hook.sh
index 741225a5aa8..0d30f0e2665 100644
--- a/pkgs/development/libraries/qt-5/hooks/qmake-hook.sh
+++ b/pkgs/development/libraries/qt-5/hooks/qmake-hook.sh
@@ -38,6 +38,11 @@ qmakeConfigurePhase() {
         echo "qmake: enabled parallel building"
     fi
 
+    if ! [[ -v enableParallelInstalling ]]; then
+        enableParallelInstalling=1
+        echo "qmake: enabled parallel installing"
+    fi
+
     runHook postConfigure
 }
 
diff --git a/pkgs/development/libraries/qt-6/hooks/qmake-hook.sh b/pkgs/development/libraries/qt-6/hooks/qmake-hook.sh
index ec1d2ea6124..8c4ce096443 100644
--- a/pkgs/development/libraries/qt-6/hooks/qmake-hook.sh
+++ b/pkgs/development/libraries/qt-6/hooks/qmake-hook.sh
@@ -40,6 +40,11 @@ qmakeConfigurePhase() {
         echo "qmake: enabled parallel building"
     fi
 
+    if ! [[ -v enableParallelInstalling ]]; then
+        enableParallelInstalling=1
+        echo "qmake: enabled parallel installing"
+    fi
+
     runHook postConfigure
 }
 
diff --git a/pkgs/development/tools/build-managers/bmake/setup-hook.sh b/pkgs/development/tools/build-managers/bmake/setup-hook.sh
index c5ca27dd910..f477a96d57a 100644
--- a/pkgs/development/tools/build-managers/bmake/setup-hook.sh
+++ b/pkgs/development/tools/build-managers/bmake/setup-hook.sh
@@ -76,6 +76,7 @@ bmakeInstallPhase() {
 
     # shellcheck disable=SC2086
     local flagsArray=(
+        ${enableParallelInstalling:+-j${NIX_BUILD_CORES}}
         SHELL=$SHELL
         # Old bash empty array hack
         $makeFlags ${makeFlagsArray+"${makeFlagsArray[@]}"}
diff --git a/pkgs/development/tools/build-managers/cmake/setup-hook.sh b/pkgs/development/tools/build-managers/cmake/setup-hook.sh
index 1d25887754a..b28ed42b689 100755
--- a/pkgs/development/tools/build-managers/cmake/setup-hook.sh
+++ b/pkgs/development/tools/build-managers/cmake/setup-hook.sh
@@ -130,6 +130,11 @@ cmakeConfigurePhase() {
         echo "cmake: enabled parallel building"
     fi
 
+    if ! [[ -v enableParallelInstalling ]]; then
+        enableParallelInstalling=1
+        echo "cmake: enabled parallel installing"
+    fi
+
     runHook postConfigure
 }
 
diff --git a/pkgs/development/tools/build-managers/meson/setup-hook.sh b/pkgs/development/tools/build-managers/meson/setup-hook.sh
index ded9c3c63b8..6305a405af2 100644
--- a/pkgs/development/tools/build-managers/meson/setup-hook.sh
+++ b/pkgs/development/tools/build-managers/meson/setup-hook.sh
@@ -28,6 +28,11 @@ mesonConfigurePhase() {
         echo "meson: enabled parallel building"
     fi
 
+    if ! [[ -v enableParallelInstalling ]]; then
+        enableParallelInstalling=1
+        echo "meson: enabled parallel installing"
+    fi
+
     runHook postConfigure
 }
 
diff --git a/pkgs/development/tools/build-managers/ninja/setup-hook.sh b/pkgs/development/tools/build-managers/ninja/setup-hook.sh
index ecaa8b88c1a..7fa5e4675f3 100644
--- a/pkgs/development/tools/build-managers/ninja/setup-hook.sh
+++ b/pkgs/development/tools/build-managers/ninja/setup-hook.sh
@@ -53,8 +53,16 @@ ninjaCheckPhase() {
 ninjaInstallPhase() {
     runHook preInstall
 
+    local buildCores=1
+
+    # Parallel building is enabled by default.
+    if [ "${enableParallelInstalling-1}" ]; then
+        buildCores="$NIX_BUILD_CORES"
+    fi
+
     # shellcheck disable=SC2086
     local flagsArray=(
+        -j$buildCores
         $ninjaFlags "${ninjaFlagsArray[@]}"
         ${installTargets:-install}
     )
diff --git a/pkgs/development/tools/build-managers/scons/setup-hook.sh b/pkgs/development/tools/build-managers/scons/setup-hook.sh
index 0b908f68286..67358ee843d 100644
--- a/pkgs/development/tools/build-managers/scons/setup-hook.sh
+++ b/pkgs/development/tools/build-managers/scons/setup-hook.sh
@@ -33,6 +33,7 @@ sconsInstallPhase() {
     fi
 
     local flagsArray=(
+        ${enableParallelInstalling:+-j${NIX_BUILD_CORES}}
         $sconsFlags ${sconsFlagsArray[@]}
         $installFlags ${installFlagsArray[@]}
         ${installTargets:-install}
diff --git a/pkgs/development/tools/build-managers/wafHook/setup-hook.sh b/pkgs/development/tools/build-managers/wafHook/setup-hook.sh
index e2f6714a32d..4d95c28de1f 100644
--- a/pkgs/development/tools/build-managers/wafHook/setup-hook.sh
+++ b/pkgs/development/tools/build-managers/wafHook/setup-hook.sh
@@ -30,6 +30,11 @@ wafConfigurePhase() {
         echo "waf: enabled parallel building"
     fi
 
+    if ! [[ -v enableParallelInstalling ]]; then
+        enableParallelInstalling=1
+        echo "waf: enabled parallel installing"
+    fi
+
     runHook postConfigure
 }
 
@@ -68,6 +73,7 @@ wafInstallPhase() {
     fi
 
     local flagsArray=(
+        ${enableParallelInstalling:+-j ${NIX_BUILD_CORES}}
         $wafFlags ${wafFlagsArray[@]}
         $installFlags ${installFlagsArray[@]}
         ${installTargets:-install}
diff --git a/pkgs/servers/x11/xorg/builder.sh b/pkgs/servers/x11/xorg/builder.sh
index 9ee81091584..a9c607ae35e 100644
--- a/pkgs/servers/x11/xorg/builder.sh
+++ b/pkgs/servers/x11/xorg/builder.sh
@@ -37,5 +37,6 @@ fi
 
 
 enableParallelBuilding=1
+enableParallelInstalling=1
 
 genericBuild
diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix
index 6bf319d0730..50487b9f2e2 100644
--- a/pkgs/stdenv/generic/make-derivation.nix
+++ b/pkgs/stdenv/generic/make-derivation.nix
@@ -433,6 +433,7 @@ else let
     } // lib.optionalAttrs (enableParallelBuilding) {
       inherit enableParallelBuilding;
       enableParallelChecking = attrs.enableParallelChecking or true;
+      enableParallelInstalling = attrs.enableParallelInstalling or true;
     } // lib.optionalAttrs (hardeningDisable != [] || hardeningEnable != [] || stdenv.hostPlatform.isMusl) {
       NIX_HARDENING_ENABLE = enabledHardeningOptions;
     } // lib.optionalAttrs (stdenv.hostPlatform.isx86_64 && stdenv.hostPlatform ? gcc.arch) {
diff --git a/pkgs/stdenv/generic/setup.sh b/pkgs/stdenv/generic/setup.sh
index 1a637bf13fd..734abb890c2 100644
--- a/pkgs/stdenv/generic/setup.sh
+++ b/pkgs/stdenv/generic/setup.sh
@@ -1376,6 +1376,7 @@ installPhase() {
 
     # shellcheck disable=SC2086
     local flagsArray=(
+        ${enableParallelInstalling:+-j${NIX_BUILD_CORES}}
         SHELL=$SHELL
     )
     _accumFlagsArray makeFlags makeFlagsArray installFlags installFlagsArray