nixos/udev: compress all firmware if supported

This should be a significant disk space saving for most NixOS
installations.  This method is a bit more complicated than doing it in
the postInstall for the firmware derivations, but this way it's
automatic, so each firmware package doesn't have to separately
implement its compression.

Currently, only xz compression is supported, but it's likely that
future versions of Linux will additionally support zstd, so I've
written the code in such a way that it would be very easy to implement
zstd compression for those kernels when they arrive, falling back to
xz for older (current) kernels.

I chose the highest possible level of compression (xz -9) because even
at this level, decompression time is negligible.  Here's how long it took
to decompress every firmware file my laptop uses:

	i915/kbl_dmc_ver1_04.bin                  	2ms
	regulatory.db                             	4ms
	regulatory.db.p7s                         	3ms
	iwlwifi-7265D-29.ucode                    	62ms
	9d71-GOOGLE-EVEMAX-0-tplg.bin             	22ms
	intel/dsp_fw_kbl.bin                      	65ms
	dsp_lib_dsm_core_spt_release.bin          	6ms
	intel/ibt-hw-37.8.10-fw-22.50.19.14.f.bseq	7ms

And since booting NixOS is a parallel process, it's unlikely (but
difficult to measure) that the time to user interaction was held up at
all by most of these.

Fixes (partially?) #148197

4 files changed, 30 insertions, 3 deletions

diff --git a/nixos/modules/services/hardware/udev.nix b/nixos/modules/services/hardware/udev.nix
index 2f386617187..2e9deebbb74 100644
--- a/nixos/modules/services/hardware/udev.nix
+++ b/nixos/modules/services/hardware/udev.nix
@@ -171,6 +171,11 @@ let
       mv etc/udev/hwdb.bin $out
     '';
 
+  compressFirmware = if config.boot.kernelPackages.kernelAtLeast "5.3" then
+    pkgs.compressFirmwareXz
+  else
+    id;
+
   # Udev has a 512-character limit for ENV{PATH}, so create a symlink
   # tree to work around this.
   udevPath = pkgs.buildEnv {
@@ -267,7 +272,7 @@ in
       '';
       apply = list: pkgs.buildEnv {
         name = "firmware";
-        paths = list;
+        paths = map compressFirmware list;
         pathsToLink = [ "/lib/firmware" ];
         ignoreCollisions = true;
       };
diff --git a/pkgs/build-support/kernel/compress-firmware-xz.nix b/pkgs/build-support/kernel/compress-firmware-xz.nix
new file mode 100644
index 00000000000..56595131c89
--- /dev/null
+++ b/pkgs/build-support/kernel/compress-firmware-xz.nix
@@ -0,0 +1,16 @@
+{ runCommand }:
+
+firmware:
+
+runCommand "${firmware.name}-xz" {} ''
+  mkdir -p $out/lib
+  (cd ${firmware} && find lib/firmware -type d -print0) |
+      (cd $out && xargs -0 mkdir -v --)
+  (cd ${firmware} && find lib/firmware -type f -print0) |
+      (cd $out && xargs -0tP "$NIX_BUILD_CORES" -n1 \
+          sh -c 'xz -9c -T1 -C crc32 --lzma2=dict=2MiB "${firmware}/$1" > "$1.xz"' --)
+  (cd ${firmware} && find lib/firmware -type l) | while read link; do
+      target="$(readlink "${firmware}/$link")"
+      ln -vs -- "''${target/^${firmware}/$out}.xz" "$out/$link.xz"
+  done
+''
diff --git a/pkgs/build-support/kernel/modules-closure.sh b/pkgs/build-support/kernel/modules-closure.sh
index 3b3a38ea1d3..74bc490eb15 100644
--- a/pkgs/build-support/kernel/modules-closure.sh
+++ b/pkgs/build-support/kernel/modules-closure.sh
@@ -81,8 +81,12 @@ for module in $(cat closure); do
     for i in $(modinfo -b $kernel --set-version "$version" -F firmware $module | grep -v '^name:'); do
         mkdir -p "$out/lib/firmware/$(dirname "$i")"
         echo "firmware for $module: $i"
-        cp "$firmware/lib/firmware/$i" "$out/lib/firmware/$i" 2>/dev/null \
-            || echo "WARNING: missing firmware $i for module $module"
+        for name in "$i" "$i.xz" ""; do
+            [ -z "$name" ] && echo "WARNING: missing firmware $i for module $module"
+            if cp "$firmware/lib/firmware/$name" "$out/lib/firmware/$name" 2>/dev/null; then
+                break
+            fi
+        done
     done
 done
 
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index e89aa5c04a8..f1170aae20d 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -808,6 +808,8 @@ with pkgs;
       sanitizers = [ ];
     };
 
+  compressFirmwareXz = callPackage ../build-support/kernel/compress-firmware-xz.nix { };
+
   makeModulesClosure = { kernel, firmware, rootModules, allowMissing ? false }:
     callPackage ../build-support/kernel/modules-closure.nix {
       inherit kernel firmware rootModules allowMissing;