diff options
author | Graham Christensen <graham@grahamc.com> | 2018-12-04 12:18:06 -0500 |
---|---|---|
committer | Graham Christensen <graham@grahamc.com> | 2018-12-05 14:25:54 -0500 |
commit | c88337c9aca9d91804da7d1d05960c88e17455c9 (patch) | |
tree | 2e34f1058b8cab8bc27e69cee22c981e19b4ef99 /pkgs/build-support/docker | |
parent | 33b9aa4f35fdc0220987c7f87c1204b5b76f21ad (diff) | |
download | nixpkgs-c88337c9aca9d91804da7d1d05960c88e17455c9.tar nixpkgs-c88337c9aca9d91804da7d1d05960c88e17455c9.tar.gz nixpkgs-c88337c9aca9d91804da7d1d05960c88e17455c9.tar.bz2 nixpkgs-c88337c9aca9d91804da7d1d05960c88e17455c9.tar.lz nixpkgs-c88337c9aca9d91804da7d1d05960c88e17455c9.tar.xz nixpkgs-c88337c9aca9d91804da7d1d05960c88e17455c9.tar.zst nixpkgs-c88337c9aca9d91804da7d1d05960c88e17455c9.zip |
dockerTools.buildImage: support using a layered image in fromImage
Docker images used to be, essentially, a linked list of layers. Each layer would have a tarball and a json document pointing to its parent, and the image pointed to the top layer: imageA ----> layerA | v layerB | v layerC The current image spec changed this format to where the Image defined the order and set of layers: imageA ---> layerA |--> layerB `--> layerC For backwards compatibility, docker produces images which follow both specs: layers point to parents, and images also point to the entire list: imageA ---> layerA | | | v |--> layerB | | | v `--> layerC This is nice for tooling which supported the older version and never updated to support the newer format. Our `buildImage` code only supported the old version, so in order for `buildImage` to properly generate an image based on another image with `fromImage`, the parent image's layers must fully support the old mechanism. This is not a problem in general, but is a problem with `buildLayeredImage`. `buildLayeredImage` creates images with newer image spec, because individual store paths don't have a guaranteed parent layer. Including a specific parent ID in the layer's json makes the output less likely to cache hit when published or pulled. This means until now, `buildLayeredImage` could not be the input to `buildImage`. The changes in this PR change `buildImage` to only use the layer's manifest when locating parent IDs. This does break buildImage on extremely old Docker images, though I do wonder how many of these exist. This work has been sponsored by Target.
Diffstat (limited to 'pkgs/build-support/docker')
-rw-r--r-- | pkgs/build-support/docker/default.nix | 46 | ||||
-rw-r--r-- | pkgs/build-support/docker/examples.nix | 19 |
2 files changed, 50 insertions, 15 deletions
diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix index 890f64a9d3b..f16cb7cec13 100644 --- a/pkgs/build-support/docker/default.nix +++ b/pkgs/build-support/docker/default.nix @@ -188,22 +188,27 @@ rec { # Use the name and tag to get the parent ID field. parentID=$(jshon -e $fromImageName -e $fromImageTag -u \ < image/repositories) + + cat ./image/manifest.json | jq -r '.[0].Layers | .[]' > layer-list + else + touch layer-list fi # Unpack all of the parent layers into the image. lowerdir="" - while [[ -n "$parentID" ]]; do - echo "Unpacking layer $parentID" - mkdir -p image/$parentID/layer - tar -C image/$parentID/layer -xpf image/$parentID/layer.tar - rm image/$parentID/layer.tar + extractionID=0 + for layerTar in $(cat layer-list); do + echo "Unpacking layer $layerTar" + extractionID=$((extractionID + 1)) + + mkdir -p image/$extractionID/layer + tar -C image/$extractionID/layer -xpf $layerTar + rm $layerTar - find image/$parentID/layer -name ".wh.*" -exec bash -c 'name="$(basename {}|sed "s/^.wh.//")"; mknod "$(dirname {})/$name" c 0 0; rm {}' \; + find image/$extractionID/layer -name ".wh.*" -exec bash -c 'name="$(basename {}|sed "s/^.wh.//")"; mknod "$(dirname {})/$name" c 0 0; rm {}' \; # Get the next lower directory and continue the loop. - lowerdir=$lowerdir''${lowerdir:+:}image/$parentID/layer - parentID=$(cat image/$parentID/json \ - | (jshon -e parent -u 2>/dev/null || true)) + lowerdir=$lowerdir''${lowerdir:+:}image/$extractionID/layer done mkdir work @@ -673,6 +678,9 @@ rec { if [[ -n "$fromImage" ]]; then echo "Unpacking base image..." tar -C image -xpf "$fromImage" + + cat ./image/manifest.json | jq -r '.[0].Layers | .[]' > layer-list + # Do not import the base image configuration and manifest chmod a+w image image/*.json rm -f image/*.json @@ -690,6 +698,8 @@ rec { for l in image/*/layer.tar; do ls_tar $l >> baseFiles done + else + touch layer-list fi chmod -R ug+rw image @@ -742,17 +752,23 @@ rec { # Use the temp folder we've been working on to create a new image. mv temp image/$layerID + # Add the new layer ID to the beginning of the layer list + ( + # originally this used `sed -i "1i$layerID" layer-list`, but + # would fail if layer-list was completely empty. + echo "$layerID/layer.tar" + cat layer-list + ) | ${pkgs.moreutils}/bin/sponge layer-list + # Create image json and image manifest imageJson=$(cat ${baseJson} | jq ". + {\"rootfs\": {\"diff_ids\": [], \"type\": \"layers\"}}") manifestJson=$(jq -n "[{\"RepoTags\":[\"$imageName:$imageTag\"]}]") - currentID=$layerID - while [[ -n "$currentID" ]]; do - layerChecksum=$(sha256sum image/$currentID/layer.tar | cut -d ' ' -f1) + + for layerTar in $(cat ./layer-list); do + layerChecksum=$(sha256sum image/$layerTar | cut -d ' ' -f1) imageJson=$(echo "$imageJson" | jq ".history |= [{\"created\": \"$(jq -r .created ${baseJson})\"}] + .") imageJson=$(echo "$imageJson" | jq ".rootfs.diff_ids |= [\"sha256:$layerChecksum\"] + .") - manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= [\"$currentID/layer.tar\"] + .") - - currentID=$(cat image/$currentID/json | (jshon -e parent -u 2>/dev/null || true)) + manifestJson=$(echo "$manifestJson" | jq ".[0].Layers |= [\"$layerTar\"] + .") done imageJsonChecksum=$(echo "$imageJson" | sha256sum | cut -d ' ' -f1) diff --git a/pkgs/build-support/docker/examples.nix b/pkgs/build-support/docker/examples.nix index 003e7429a81..090bfafa085 100644 --- a/pkgs/build-support/docker/examples.nix +++ b/pkgs/build-support/docker/examples.nix @@ -156,5 +156,24 @@ rec { name = "layered-image"; tag = "latest"; config.Cmd = [ "${pkgs.hello}/bin/hello" ]; + contents = [ pkgs.hello pkgs.bash pkgs.coreutils ]; + }; + + # 11. Create an image on top of a layered image + layered-on-top = pkgs.dockerTools.buildImage { + name = "layered-on-top"; + tag = "latest"; + fromImage = layered-image; + extraCommands = '' + mkdir ./example-output + chmod 777 ./example-output + ''; + config = { + Env = [ "PATH=${pkgs.coreutils}/bin/" ]; + WorkingDir = "/example-output"; + Cmd = [ + "${pkgs.bash}/bin/bash" "-c" "echo hello > foo; cat foo" + ]; + }; }; } |