docker tools: fix pull image function

3 files changed, 56 insertions, 119 deletions

diff --git a/pkgs/build-support/docker/examples.nix b/pkgs/build-support/docker/examples.nix
index 05b4a9b4f2d..49cbb7a9817 100644
--- a/pkgs/build-support/docker/examples.nix
+++ b/pkgs/build-support/docker/examples.nix
@@ -83,16 +83,12 @@ rec {
   };
 
   # 4. example of pulling an image. could be used as a base for other images
-  #
-  # ***** Currently broken, getting 404s. Perhaps the docker API has changed?
-  #
-  #
-  # debian = pullImage {
-  #   imageName = "debian";
-  #   imageTag = "jessie";
-  #   # this hash will need change if the tag is updated at docker hub
-  #   sha256 = "18kd495lc2k35h03bpcbdjnix17nlqbwf6nmq3sb161blf0dk14q";
-  # };
+  nix = pullImage {
+    imageName = "nixos/nix";
+    imageTag = "1.11";
+    # this hash will need change if the tag is updated at docker hub
+    sha256 = "1gk4bq05vl3rj3mh4mlbl4iicgndmimlv8jvkhdk4hrv0r44bwr3";
+  };
 
   # 5. example of multiple contents, emacs and vi happily coexisting
   editors = buildImage {
diff --git a/pkgs/build-support/docker/pull.nix b/pkgs/build-support/docker/pull.nix
index 0e1b147f6e1..5ccd0a41c5e 100644
--- a/pkgs/build-support/docker/pull.nix
+++ b/pkgs/build-support/docker/pull.nix
@@ -1,41 +1,32 @@
-{ stdenv, lib, curl, jshon, python, runCommand }:
-
-# Inspired and simplified version of fetchurl.
+{ stdenv, lib, docker, vmTools, utillinux, curl, kmod, dhcp, cacert, e2fsprogs }:
+let
+  nameReplace = name: builtins.replaceStrings ["/" ":"] ["-" "-"] name;
+in
 # For simplicity we only support sha256.
+{ imageName, imageTag ? "latest", imageId ? "${imageName}:${imageTag}"
+, sha256, name ? (nameReplace "docker-image-${imageName}-${imageTag}.tar") }:
+let
+  pullImage = vmTools.runInLinuxVM (
+    stdenv.mkDerivation {
+      inherit name imageId;
 
-# Currently only registry v1 is supported, compatible with Docker Hub.
-
-{ imageName, imageTag ? "latest", imageId ? null
-, sha256, name ? "${imageName}-${imageTag}"
-, indexUrl ? "https://index.docker.io"
-, registryVersion ? "v1"
-, curlOpts ? "" }:
-
-assert registryVersion == "v1";
-
-let layer = stdenv.mkDerivation {
-  inherit name imageName imageTag imageId
-          indexUrl registryVersion curlOpts;
+      certs = "${cacert}/etc/ssl/certs/ca-bundle.crt";
 
-  builder = ./pull.sh;
-  detjson = ./detjson.py;
+      builder = ./pull.sh;
 
-  buildInputs = [ curl jshon python ];
+      buildInputs = [ curl utillinux docker kmod dhcp cacert e2fsprogs ];
 
-  outputHashAlgo = "sha256";
-  outputHash = sha256;
-  outputHashMode = "recursive";
+      outputHashAlgo = "sha256";
+      outputHash = sha256;
 
-  impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [
-    # This variable allows the user to pass additional options to curl
-    "NIX_CURL_FLAGS"
-  ];
+      impureEnvVars = lib.fetchers.proxyImpureEnvVars;
 
-  # Doing the download on a remote machine just duplicates network
-  # traffic, so don't do that.
-  preferLocalBuild = true;
-};
+      preVM = vmTools.createEmptyImage {
+        size = 2048;
+        fullName = "${name}-disk";
+      };
 
-in runCommand "${name}.tar.gz" {} ''
-  tar -C ${layer} -czf $out .
-''
+      QEMU_OPTS = "-netdev user,id=net0 -device virtio-net-pci,netdev=net0";
+    });
+in
+  pullImage
diff --git a/pkgs/build-support/docker/pull.sh b/pkgs/build-support/docker/pull.sh
index 7ba146e9de0..db643f5e88e 100644
--- a/pkgs/build-support/docker/pull.sh
+++ b/pkgs/build-support/docker/pull.sh
@@ -1,86 +1,36 @@
-# Reference: docker src contrib/download-frozen-image.sh
-
 source $stdenv/setup
 
-# Curl flags to handle redirects, not use EPSV, handle cookies for
-# servers to need them during redirects, and work on SSL without a
-# certificate (this isn't a security problem because we check the
-# cryptographic hash of the output anyway).
-curl=$(command -v curl)
-curl() {
-  [[ -n ${token:-} ]] && set -- -H "Authorization: Token $token" "$@"
-  $curl \
-    --location --max-redirs 20 \
-    --retry 3 \
-    --fail \
-    --disable-epsv \
-    --cookie-jar cookies \
-    --insecure \
-    $curlOpts \
-    $NIX_CURL_FLAGS \
-    "$@"
-}
-
-fetchLayer() {
-    local url="$1"
-    local dest="$2"
-    local curlexit=18;
-
-    # if we get error code 18, resume partial download
-    while [ $curlexit -eq 18 ]; do
-        # keep this inside an if statement, since on failure it doesn't abort the script
-        if curl -C - "$url" --output "$dest"; then
-            return 0
-        else
-            curlexit=$?;
-        fi
-    done
-
-    return $curlexit
-}
-
-headers=$(curl -o /dev/null -D- -H 'X-Docker-Token: true' \
-          "$indexUrl/$registryVersion/repositories/$imageName/images")
+mkdir -p /var/lib/docker
+mkfs.ext4 /dev/vda
+mount -t ext4 /dev/vda /var/lib/docker
 
-header() {
-  grep $1 <<< "$headers" | tr -d '\r' | cut -d ' ' -f 2
-}
+modprobe virtio_net
+dhclient eth0
 
-# this only takes the first endpoint, more may be provided
-# https://docs.docker.com/v1.6/reference/api/docker-io_api/
-if ! registryUrl=$(header X-Docker-Endpoints); then
-  echo "error: index returned no endpoint"
-  exit 1
-fi
-baseUrl="https://$registryUrl/$registryVersion"
+mkdir -p /etc/ssl/certs/
+cp "$certs" "/etc/ssl/certs/"
 
-token="$(header X-Docker-Token || true)";
-
-if [ -z "$imageId" ]; then
-    imageId="$(curl "$baseUrl/repositories/$imageName/tags/$imageTag")"
-    imageId="${imageId//\"/}"
-    if [ -z "$imageId" ]; then
-        echo "error: no image ID found for ${imageName}:${imageTag}"
-        exit 1
+# from https://github.com/tianon/cgroupfs-mount/blob/master/cgroupfs-mount
+mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup
+cd /sys/fs/cgroup
+for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do
+  mkdir -p $sys
+  if ! mountpoint -q $sys; then
+    if ! mount -n -t cgroup -o $sys cgroup $sys; then
+      rmdir $sys || true
     fi
+  fi
+done
 
-    echo "found image ${imageName}:${imageTag}@$imageId"
-fi
-
-mkdir -p $out
+# run docker daemon
+dockerd -H tcp://127.0.0.1:5555 -H unix:///var/run/docker.sock &
 
-jshon -n object \
-  -n object -s "$imageId" -i "$imageTag" \
-  -i "$imageName" > $out/repositories
+until $(curl --output /dev/null --silent --connect-timeout 2 http://127.0.0.1:5555); do
+  printf '.'
+  sleep 1
+done
 
-curl "$baseUrl/images/$imageId/ancestry" -o ancestry.json
+rm -r $out
 
-layerIds=$(jshon -a -u < ancestry.json)
-for layerId in $layerIds; do
-    echo "fetching layer $layerId"
-    
-    mkdir "$out/$layerId"
-    echo '1.0' > "$out/$layerId/VERSION"
-    curl "$baseUrl/images/$layerId/json" | python $detjson > "$out/$layerId/json"
-    fetchLayer "$baseUrl/images/$layerId/layer" "$out/$layerId/layer.tar"
-done
+docker pull ${imageId}
+docker save ${imageId} > $out