From f6d75f550ebcea9145b5b63473ce2652e580a44e Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 16 Dec 2019 12:36:45 -0500 Subject: dockerTools.buildLayeredImage: tweak formatting on contentsEnv --- pkgs/build-support/docker/default.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix index 2a25ac04d40..0781c7dd4f1 100644 --- a/pkgs/build-support/docker/default.nix +++ b/pkgs/build-support/docker/default.nix @@ -536,7 +536,12 @@ rec { }: let baseName = baseNameOf name; - contentsEnv = symlinkJoin { name = "bulk-layers"; paths = (if builtins.isList contents then contents else [ contents ]); }; + contentsEnv = symlinkJoin { + name = "bulk-layers"; + paths = if builtins.isList contents + then contents + else [ contents ]; + }; configJson = let pure = writeText "${baseName}-config.json" (builtins.toJSON { -- cgit 1.4.1 From aec80dddc0416722a421330316003f27cf0c566b Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 16 Dec 2019 12:47:47 -0500 Subject: dockerTools.buildLayeredImage: pass a list of closures to mkManyPureLayers so it can exclude the top-most level Before, every docker image had three extra layers: 1. A `closure` layer which is an internal implementation detail of calculating the closure of the container 2. a `name-config.json` layer which is the images' run-time configuration, and has no business being *in* the image as a layer. 3. a "bulk-layers" layer which is again and implementation detail around collecting the image's closure. None of these layers need to be in the final product. --- pkgs/build-support/docker/default.nix | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix index 0781c7dd4f1..77edbb76601 100644 --- a/pkgs/build-support/docker/default.nix +++ b/pkgs/build-support/docker/default.nix @@ -290,7 +290,7 @@ rec { mkManyPureLayers = { name, # Files to add to the layer. - closure, + closures, configJson, # Docker has a 125-layer maximum, we pick 100 to ensure there is # plenty of room for extension. @@ -303,10 +303,12 @@ rec { isExecutable = true; src = ./store-path-to-layer.sh; }; + + overallClosure = writeText "closure" (lib.concatStringsSep " " closures); in runCommand "${name}-granular-docker-layers" { inherit maxLayers; - paths = referencesByPopularity closure; + paths = referencesByPopularity overallClosure; nativeBuildInputs = [ jshon rsync tarsum ]; enableParallelBuilding = true; } @@ -558,7 +560,7 @@ rec { bulkLayers = mkManyPureLayers { name = baseName; - closure = writeText "closure" "${contentsEnv} ${configJson}"; + closures = [ contentsEnv configJson ]; # One layer will be taken up by the customisationLayer, so # take up one less. maxLayers = maxLayers - 1; -- cgit 1.4.1 From 700f4c538885bb9c1290d1723ec5380c2fe6e910 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 16 Dec 2019 12:57:04 -0500 Subject: dockerTools.buildLayeredImage: prepare to exclude some paths Without changing behavior, since this code is fiddly, make it possible to add a filtering step before packaging individual paths. --- pkgs/build-support/docker/default.nix | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix index 77edbb76601..eb3863bcfba 100644 --- a/pkgs/build-support/docker/default.nix +++ b/pkgs/build-support/docker/default.nix @@ -319,15 +319,20 @@ rec { | jshon -d config \ | jshon -s "1970-01-01T00:00:01Z" -i created > generic.json + # WARNING! # The following code is fiddly w.r.t. ensuring every layer is # created, and that no paths are missed. If you change the # following head and tail call lines, double-check that your # code behaves properly when the number of layers equals: # maxLayers-1, maxLayers, and maxLayers+1 - head -n $((maxLayers - 1)) $paths | cat -n | xargs -P$NIX_BUILD_CORES -n2 ${storePathToLayer} - if [ $(cat $paths | wc -l) -ge $maxLayers ]; then - tail -n+$maxLayers $paths | xargs ${storePathToLayer} $maxLayers + paths() { + cat $paths + } + + paths | head -n $((maxLayers - 1)) | cat -n | xargs -P$NIX_BUILD_CORES -n2 ${storePathToLayer} + if [ $(paths | wc -l) -ge $maxLayers ]; then + paths | tail -n+$maxLayers | xargs ${storePathToLayer} $maxLayers fi echo "Finished building layer '$name'" -- cgit 1.4.1 From 12e24163803c2bd0051d93fe2957351eb8046735 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 16 Dec 2019 12:58:27 -0500 Subject: dockerTools.buildLayeredImage: Exclude top level implementation detail layers --- pkgs/build-support/docker/default.nix | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix index eb3863bcfba..bdc4f6434ed 100644 --- a/pkgs/build-support/docker/default.nix +++ b/pkgs/build-support/docker/default.nix @@ -287,6 +287,12 @@ rec { # unless there are more paths than $maxLayers. In that case, create # $maxLayers-1 for the most popular layers, and smush the remainaing # store paths in to one final layer. + # + # NOTE: the `closures` parameter is a list of closures to include. + # The TOP LEVEL store paths themselves will never be present in the + # resulting image. At this time (2019-12-16) none of these layers + # are appropriate to include, as they are all created as + # implementation details of dockerTools. mkManyPureLayers = { name, # Files to add to the layer. @@ -327,7 +333,7 @@ rec { # code behaves properly when the number of layers equals: # maxLayers-1, maxLayers, and maxLayers+1 paths() { - cat $paths + cat $paths ${lib.concatMapStringsSep " " (path: "| grep -v ${path}") (closures ++ [ overallClosure ])} } paths | head -n $((maxLayers - 1)) | cat -n | xargs -P$NIX_BUILD_CORES -n2 ${storePathToLayer} -- cgit 1.4.1