From 3be767593b69ed334606ef077425b800ce1c0f71 Mon Sep 17 00:00:00 2001 From: Richard Wallace Date: Mon, 30 Dec 2019 14:45:10 -0700 Subject: dockerTools.buildLayeredImage: fix building layered images in parallel when tar'ing store paths into layered archives when building layered images, don't use the absolute nix store path so that tar won't complain if something new is added to the nix store when building the final docker image, ignore any file changes tar detects in the layers. they are all immutable and the only thing that might change is the number of hard links due to store optimization --- pkgs/build-support/docker/default.nix | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) (limited to 'pkgs/build-support/docker/default.nix') diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix index e10ff269950..a6304d9c064 100644 --- a/pkgs/build-support/docker/default.nix +++ b/pkgs/build-support/docker/default.nix @@ -325,7 +325,6 @@ rec { | jshon -d config \ | jshon -s "1970-01-01T00:00:01Z" -i created > generic.json - # WARNING! # The following code is fiddly w.r.t. ensuring every layer is # created, and that no paths are missed. If you change the @@ -625,7 +624,22 @@ rec { -i "$imageName" > image/repositories echo "Cooking the image..." - tar -C image --dereference --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --mode=a-w --xform s:'^./':: -c . | pigz -nT > $out + # tar exits with an exit code of 1 if files changed while it was + # reading them. it considers a change in the number of hard links + # to be a "change", which can cause this to fail if images are being + # built concurrently and auto-optimise-store is turned on. since + # know the contents of these files will not change, we can reasonably + # ignore this exit code + set +e + tar -C image --dereference --hard-dereference --sort=name \ + --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 \ + --mode=a-w --xform s:'^./':: --use-compress-program='pigz -nT' \ + --warning=no-file-changed -cf $out . + RET=$? + if [ $RET -ne 0 ] && [ $RET -ne 1 ]; then + exit $RET + fi + set -e echo "Finished." ''; -- cgit 1.4.1 From da261e36316b5b32009e8aa936e32df4b91e11bd Mon Sep 17 00:00:00 2001 From: Antoine Eiche Date: Sat, 11 Jan 2020 09:02:30 +0100 Subject: dockerTools.buildLayeredImage: fix typo in comments --- pkgs/build-support/docker/default.nix | 8 ++++---- pkgs/build-support/docker/store-path-to-layer.sh | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) (limited to 'pkgs/build-support/docker/default.nix') diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix index a6304d9c064..3fcae13e20d 100644 --- a/pkgs/build-support/docker/default.nix +++ b/pkgs/build-support/docker/default.nix @@ -625,11 +625,11 @@ rec { echo "Cooking the image..." # tar exits with an exit code of 1 if files changed while it was - # reading them. it considers a change in the number of hard links + # reading them. It considers a change in the number of hard links # to be a "change", which can cause this to fail if images are being - # built concurrently and auto-optimise-store is turned on. since - # know the contents of these files will not change, we can reasonably - # ignore this exit code + # built concurrently and the auto-optimise-store nix option is turned on. + # Since the contents of these files will not change, we can reasonably + # ignore this exit code. set +e tar -C image --dereference --hard-dereference --sort=name \ --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 \ diff --git a/pkgs/build-support/docker/store-path-to-layer.sh b/pkgs/build-support/docker/store-path-to-layer.sh index c7850154c7e..c808abab7a8 100755 --- a/pkgs/build-support/docker/store-path-to-layer.sh +++ b/pkgs/build-support/docker/store-path-to-layer.sh @@ -13,8 +13,8 @@ echo "Creating layer #$layerNumber for $storePath" mkdir -p "$layerPath" -# make sure /nix and /nix/store appear first in the archive. -# we create the directories here and use them because +# Make sure /nix and /nix/store appear first in the archive. +# We create the directories here and use them because # when there are other things being added to the # nix store, tar could fail, saying, # "tar: /nix/store: file changed as we read it" @@ -25,16 +25,16 @@ tar -cf "$layerPath/layer.tar" \ --transform='s,nix,/nix,' \ nix -# we change into the /nix/store in order to avoid a similar +# We change into the /nix/store in order to avoid a similar # "file changed as we read it" error as above. Namely, # if we use the absolute path of /nix/store/123-pkg -# and something new it added to the nix store while tar +# and something new is added to the nix store while tar # is running, it will detect a change to /nix/store and # fail. Instead, if we cd into the nix store and copy # the relative nix store path, tar will ignore changes # to /nix/store. In order to create the correct structure # in the tar file, we transform the relative nix store -# path to the absolute store path +# path to the absolute store path. n=$(basename "$storePath") tar -C /nix/store -rpf "$layerPath/layer.tar" \ --hard-dereference --sort=name \ -- cgit 1.4.1