dockerTools: Calculate tarsum's on the fly

Calculating the tarsum after creating a layer is inefficient, since
we have to read the tarball we've just written from the disk.

This commit simultaneously calculates the tarsum while creating the
tarball.

2 files changed, 30 insertions, 41 deletions

diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix
index bee6e37cccb..7fd6389aabf 100644
--- a/pkgs/build-support/docker/default.nix
+++ b/pkgs/build-support/docker/default.nix
@@ -392,14 +392,10 @@ rec {
         (cd layer; eval "$extraCommands")
       fi
 
-      # Tar up the layer and throw it into 'layer.tar'.
+      # Tar up the layer and throw it into 'layer.tar', while calculating its checksum.
       echo "Packing layer..."
       mkdir $out
-      tar --transform='s|^\./||' -C layer --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf $out/layer.tar .
-
-      # Compute a checksum of the tarball.
-      echo "Computing layer checksum..."
-      tarhash=$(tarsum < $out/layer.tar)
+      tarhash=$(tar --transform='s|^\./||' -C layer --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf - . | tee $out/layer.tar | tarsum)
 
       # Add a 'checksum' field to the JSON, with the value set to the
       # checksum of the tarball.
@@ -449,11 +445,7 @@ rec {
       # Tar up the layer and throw it into 'layer.tar'.
       echo "Packing layer..."
       mkdir $out
-      tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf $out/layer.tar .
-
-      # Compute a checksum of the tarball.
-      echo "Computing layer checksum..."
-      tarhash=$(tarsum < $out/layer.tar)
+      tarhash=$(tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=${toString uid} --group=${toString gid} -cf - . | tee $out/layer.tar | tarsum)
 
       # Add a 'checksum' field to the JSON, with the value set to the
       # checksum of the tarball.
@@ -537,11 +529,10 @@ rec {
 
         echo "Packing layer..."
         mkdir -p $out
-        tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" -cf $out/layer.tar .
+        tarhash=$(tar -C layer --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" -cf - . |
+                    tee $out/layer.tar |
+                    ${tarsum}/bin/tarsum)
 
-        # Compute the tar checksum and add it to the output json.
-        echo "Computing checksum..."
-        tarhash=$(${tarsum}/bin/tarsum < $out/layer.tar)
         cat ${baseJson} | jshon -s "$tarhash" -i checksum > $out/json
         # Indicate to docker that we're using schema version 1.0.
         echo -n "1.0" > $out/VERSION
diff --git a/pkgs/build-support/docker/store-path-to-layer.sh b/pkgs/build-support/docker/store-path-to-layer.sh
index e9fa22e4b99..d834716e4b2 100755
--- a/pkgs/build-support/docker/store-path-to-layer.sh
+++ b/pkgs/build-support/docker/store-path-to-layer.sh
@@ -11,37 +11,35 @@ echo "Creating layer #$layerNumber for $@"
 mkdir -p "$layerPath"
 
 # Make sure /nix and /nix/store appear first in the archive.
+#
 # We create the directories here and use them because
 # when there are other things being added to the
 # nix store, tar could fail, saying,
 # "tar: /nix/store: file changed as we read it"
 mkdir -p nix/store
-tar -cf "$layerPath/layer.tar"  \
-    --mtime="@$SOURCE_DATE_EPOCH" \
-    --owner=0 --group=0 \
-    --transform='s,nix,/nix,' \
-    nix
-
-# We change into the /nix/store in order to avoid a similar
-# "file changed as we read it" error as above. Namely,
-# if we use the absolute path of /nix/store/123-pkg
-# and something new is added to the nix store while tar
-# is running, it will detect a change to /nix/store and
-# fail. Instead, if we cd into the nix store and copy
-# the relative nix store path, tar will ignore changes
-# to /nix/store. In order to create the correct structure
-# in the tar file, we transform the relative nix store
-# path to the absolute store path.
-basename -a "$@" |
-  tar -C /nix/store -rpf "$layerPath/layer.tar" \
-    --verbatim-files-from --files-from - \
-    --hard-dereference --sort=name \
-    --mtime="@$SOURCE_DATE_EPOCH" \
-    --owner=0 --group=0 \
-    --transform="flags=rS;s,^,/nix/store/,"
-
-# Compute a checksum of the tarball.
-tarhash=$(tarsum < $layerPath/layer.tar)
+
+# Then we change into the /nix/store in order to
+# avoid a similar "file changed as we read it" error
+# as above. Namely, if we use the absolute path of
+# /nix/store/123-pkg and something new is added to the nix
+# store while tar is running, it will detect a change to
+# /nix/store and fail. Instead, if we cd into the nix store
+# and copy the relative nix store path, tar will ignore
+# changes to /nix/store. In order to create the correct
+# structure in the tar file, we transform the relative nix
+# store path to the absolute store path.
+tarhash=$(
+  basename -a "$@" |
+    tar -cp nix \
+      -C /nix/store --verbatim-files-from --files-from - \
+      --hard-dereference --sort=name \
+      --mtime="@$SOURCE_DATE_EPOCH" \
+      --owner=0 --group=0 \
+      --transform 's,^nix(/|$),/nix/,' \
+      --transform 's,^[^/],/nix/store/\0,rS' |
+    tee "$layerPath/layer.tar" |
+    tarsum
+)
 
 # Add a 'checksum' field to the JSON, with the value set to the
 # checksum of the tarball.