docker: init fetchdocker nix code for docker2nix

This change adds granular, non-docker daemon docker image fetchers and
a docker image layer compositor to be used in conjunction with the
`docker2nix` utility provided by the `haskellPackages.hocker` package.

This change includes a hackage package version bump and updated sha256
for recent fixes released to `hocker` resulting from formulating this
patch.

6 files changed, 250 insertions, 0 deletions

diff --git a/pkgs/build-support/fetchdocker/credentials.nix b/pkgs/build-support/fetchdocker/credentials.nix
new file mode 100644
index 00000000000..c01288dbf53
--- /dev/null
+++ b/pkgs/build-support/fetchdocker/credentials.nix
@@ -0,0 +1,38 @@
+# We provide three paths to get the credentials into the builder's
+# environment:
+#
+# 1. Via impureEnvVars. This method is difficult for multi-user Nix
+#    installations (but works very well for single-user Nix
+#    installations!) because it requires setting the environment
+#    variables on the nix-daemon which is either complicated or unsafe
+#    (i.e: configuring via Nix means the secrets will be persisted
+#    into the store)
+#
+# 2. If the DOCKER_CREDENTIALS key with a path to a credentials file
+#    is added to the NIX_PATH (usually via the '-I ' argument to most
+#    Nix tools) then an attempt will be made to read credentials from
+#    it. The semantics are simple, the file should contain two lines
+#    for the username and password based authentication:
+#
+# $ cat ./credentials-file.txt
+# DOCKER_USER=myusername
+# DOCKER_PASS=mypassword
+#
+#    ... and a single line for the token based authentication:
+#
+# $ cat ./credentials-file.txt
+# DOCKER_TOKEN=mytoken
+#
+# 3. A credential file at /etc/nix-docker-credentials.txt with the
+#    same format as the file described in #2 can also be used to
+#    communicate credentials to the builder. This is necessary for
+#    situations (like Hydra) where you cannot customize the NIX_PATH
+#    given to the nix-build invocation to provide it with the
+#    DOCKER_CREDENTIALS path
+let
+  pathParts =
+   (builtins.filter
+    ({path, prefix}: "DOCKER_CREDENTIALS" == prefix)
+    builtins.nixPath);
+in
+  if (pathParts != []) then (builtins.head pathParts).path else ""
diff --git a/pkgs/build-support/fetchdocker/default.nix b/pkgs/build-support/fetchdocker/default.nix
new file mode 100644
index 00000000000..ae3ae4185e0
--- /dev/null
+++ b/pkgs/build-support/fetchdocker/default.nix
@@ -0,0 +1,61 @@
+{ stdenv, lib, coreutils, bash, gnutar, jq, writeText }:
+let
+  stripScheme =
+    builtins.replaceStrings [ "https://" "http://" ] [ "" "" ];
+  stripNixStore =
+    s: lib.removePrefix "/nix/store/" s;
+in
+{ name
+, registry         ? "https://registry-1.docker.io/v2/"
+, repository       ? "library"
+, imageName
+, tag
+, imageLayers
+, imageConfig
+, image            ? "${stripScheme registry}/${repository}/${imageName}:${tag}"
+}:
+
+# Make sure there are *no* slashes in the repository or container
+# names since we use these to make the output derivation name for the
+# nix-store path.
+assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters repository);
+assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters imageName);
+
+let
+  # Abuse `builtins.toPath` to collapse possible double slashes
+  repoTag0 = builtins.toString (builtins.toPath "/${stripScheme registry}/${repository}/${imageName}");
+  repoTag1 = lib.removePrefix "/" repoTag0;
+
+  layers = builtins.map stripNixStore imageLayers;
+
+  manifest =
+    writeText "manifest.json" (builtins.toJSON [
+      { Config   = stripNixStore imageConfig;
+        Layers   = layers;
+        RepoTags = [ "${repoTag1}:${tag}" ];
+      }]);
+
+  repositories =
+    writeText "repositories" (builtins.toJSON {
+      "${repoTag1}" = {
+        "${tag}" = lib.last layers;
+      };
+    });
+
+  imageFileStorePaths =
+    writeText "imageFileStorePaths.txt"
+      (lib.concatStringsSep "\n" ((lib.unique imageLayers) ++ [imageConfig]));
+in
+stdenv.mkDerivation {
+  builder     = ./fetchdocker-builder.sh;
+  buildInputs = [ coreutils ];
+  preferLocalBuild = true;
+
+  inherit name imageName repository tag;
+  inherit bash gnutar manifest repositories;
+  inherit imageFileStorePaths;
+
+  passthru = {
+    inherit image;
+  };
+}
diff --git a/pkgs/build-support/fetchdocker/fetchDockerConfig.nix b/pkgs/build-support/fetchdocker/fetchDockerConfig.nix
new file mode 100644
index 00000000000..9fd813bfa57
--- /dev/null
+++ b/pkgs/build-support/fetchdocker/fetchDockerConfig.nix
@@ -0,0 +1,13 @@
+pkgargs@{ stdenv, lib, haskellPackages, writeText, gawk }:
+let
+  generic-fetcher =
+    import ./generic-fetcher.nix pkgargs;
+in
+
+args@{ repository ? "library", imageName, tag, ... }:
+
+generic-fetcher ({
+  fetcher = "hocker-config";
+  name    = "${repository}_${imageName}_${tag}-config.json";
+  tag     = "unused";
+} // args)
diff --git a/pkgs/build-support/fetchdocker/fetchDockerLayer.nix b/pkgs/build-support/fetchdocker/fetchDockerLayer.nix
new file mode 100644
index 00000000000..869ba637429
--- /dev/null
+++ b/pkgs/build-support/fetchdocker/fetchDockerLayer.nix
@@ -0,0 +1,13 @@
+pkgargs@{ stdenv, lib, haskellPackages, writeText, gawk }:
+let
+  generic-fetcher =
+    import ./generic-fetcher.nix pkgargs;
+in
+
+args@{ layerDigest, ... }:
+
+generic-fetcher ({
+  fetcher = "hocker-layer";
+  name    = "docker-layer-${layerDigest}.tar.gz";
+  tag     = "unused";
+} // args)
diff --git a/pkgs/build-support/fetchdocker/fetchdocker-builder.sh b/pkgs/build-support/fetchdocker/fetchdocker-builder.sh
new file mode 100644
index 00000000000..7443591e656
--- /dev/null
+++ b/pkgs/build-support/fetchdocker/fetchdocker-builder.sh
@@ -0,0 +1,28 @@
+source "${stdenv}/setup"
+header "exporting ${repository}/${imageName} (tag: ${tag}) into ${out}"
+mkdir -p "${out}"
+
+cat <<EOF > "${out}/compositeImage.sh"
+#! ${bash}/bin/bash
+#
+# Create a tar archive of a docker image's layers, docker image config
+# json, manifest.json, and repositories json; this streams directly to
+# stdout and is intended to be used in concert with docker load, i.e:
+# 
+# ${out}/compositeImage.sh | docker load
+
+# The first character follow the 's' command for sed becomes the
+# delimiter sed will use; this makes the transformation regex easy to
+# read. We feed tar a file listing the files we want in the archive,
+# because the paths are absolute and docker load wants them flattened in
+# the archive, we need to transform all of the paths going in by
+# stripping everything *including* the last solidus so that we end up
+# with the basename of the path.
+${gnutar}/bin/tar \
+  --transform='s=.*/==' \
+  --transform="s=.*-manifest.json=manifest.json=" \
+  --transform="s=.*-repositories=repositories=" \
+  -c "${manifest}" "${repositories}" -T "${imageFileStorePaths}"
+EOF
+chmod +x "${out}/compositeImage.sh"
+stopNest
diff --git a/pkgs/build-support/fetchdocker/generic-fetcher.nix b/pkgs/build-support/fetchdocker/generic-fetcher.nix
new file mode 100644
index 00000000000..e051cee0843
--- /dev/null
+++ b/pkgs/build-support/fetchdocker/generic-fetcher.nix
@@ -0,0 +1,97 @@
+{ stdenv, lib, haskellPackages, writeText, gawk }:
+let
+  awk                   = "${gawk}/bin/awk";
+  dockerCredentialsFile = import ./credentials.nix;
+  stripScheme           =
+    builtins.replaceStrings [ "https://" "http://" ] [ "" "" ];
+in
+{ fetcher
+, name
+ , registry    ? "https://registry-1.docker.io/v2/"
+ , repository  ? "library"
+ , imageName
+ , sha256
+ , tag         ? ""
+ , layerDigest ? ""
+}:
+
+# There must be no slashes in the repository or container names since
+# we use these to make the output derivation name for the nix store
+# path
+assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters repository);
+assert null == lib.findFirst (c: "/"==c) null (lib.stringToCharacters imageName);
+
+# Only allow hocker-config and hocker-layer as fetchers for now
+assert (builtins.elem fetcher ["hocker-config" "hocker-layer"]);
+
+# If layerDigest is non-empty then it must not have a 'sha256:' prefix!
+assert
+  (if layerDigest != ""
+   then !lib.hasPrefix "sha256:" layerDigest
+   else true);
+
+let
+  layerDigestFlag =
+    lib.optionalString (layerDigest != "") "--layer ${layerDigest}";
+in
+stdenv.mkDerivation {
+  inherit name;
+  builder = writeText "${fetcher}-builder.sh" ''
+    source "$stdenv/setup"
+    header "${fetcher} exporting to $out"
+
+    declare -A creds
+
+    # This is a hack for Hydra since we have no way of adding values
+    # to the NIX_PATH for Hydra jobsets!!
+    staticCredentialsFile="/etc/nix-docker-credentials.txt"
+    if [ ! -f "$dockerCredentialsFile" -a -f "$staticCredentialsFile" ]; then
+      echo "credentials file not set, falling back on static credentials file at: $staticCredentialsFile"
+      dockerCredentialsFile=$staticCredentialsFile
+    fi
+
+    if [ -f "$dockerCredentialsFile" ]; then
+      header "using credentials from $dockerCredentialsFile"
+
+      CREDSFILE=$(cat "$dockerCredentialsFile")
+      creds[token]=$(${awk} -F'=' '/DOCKER_TOKEN/ {print $2}' <<< "$CREDSFILE" | head -n1)
+
+      # Prefer DOCKER_TOKEN over the username and password
+      # authentication method
+      if [ -z "''${creds[token]}" ]; then
+        creds[user]=$(${awk} -F'=' '/DOCKER_USER/  {print $2}' <<< "$CREDSFILE" | head -n1)
+        creds[pass]=$(${awk} -F'=' '/DOCKER_PASS/  {print $2}' <<< "$CREDSFILE" | head -n1)
+      fi
+    fi
+
+    # These variables will be filled in first by the impureEnvVars, if
+    # those variables are empty then they will default to the
+    # credentials that may have been read in from the 'DOCKER_CREDENTIALS'
+    DOCKER_USER="''${DOCKER_USER:-''${creds[user]}}"
+    DOCKER_PASS="''${DOCKER_PASS:-''${creds[pass]}}"
+    DOCKER_TOKEN="''${DOCKER_TOKEN:-''${creds[token]}}"
+
+    ${fetcher} --out="$out" \
+      ''${registry:+--registry "$registry"} \
+      ''${DOCKER_USER:+--username "$DOCKER_USER"} \
+      ''${DOCKER_PASS:+--password "$DOCKER_PASS"} \
+      ''${DOCKER_TOKEN:+--token "$DOCKER_TOKEN"} \
+      ${layerDigestFlag} \
+      "${repository}/${imageName}" \
+      "${tag}"
+
+    stopNest
+  '';
+
+  buildInputs = [ haskellPackages.hocker ];
+
+  outputHashAlgo = "sha256";
+  outputHashMode = "flat";
+  outputHash = sha256;
+
+  preferLocalBuild = true;
+
+  impureEnvVars = [ "DOCKER_USER" "DOCKER_PASS" "DOCKER_TOKEN" ];
+
+  inherit registry dockerCredentialsFile;
+}