diff options
Diffstat (limited to 'pkgs/build-support/docker/default.nix')
| -rw-r--r-- | pkgs/build-support/docker/default.nix | 365 |
1 files changed, 365 insertions, 0 deletions
diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix new file mode 100644 index 00000000000..55344aad566 --- /dev/null +++ b/pkgs/build-support/docker/default.nix @@ -0,0 +1,365 @@ +{ stdenv, lib, callPackage, runCommand, writeReferencesToFile, writeText, vmTools, writeScript +, docker, shadow, utillinux, coreutils, jshon, e2fsprogs, goPackages }: + +# WARNING: this API is unstable and may be subject to backwards-incompatible changes in the future. + +rec { + + pullImage = callPackage ./pull.nix {}; + + # We need to sum layer.tar, not a directory, hence tarsum instead of nix-hash. + # And we cannot untar it, because then we cannot preserve permissions ecc. + tarsum = runCommand "tarsum" { + buildInputs = [ goPackages.go ]; + } '' + mkdir tarsum + cd tarsum + + cp ${./tarsum.go} tarsum.go + export GOPATH=$(pwd) + mkdir src + ln -sT ${docker.src}/pkg/tarsum src/tarsum + go build + + cp tarsum $out + ''; + + # buildEnv creates symlinks to dirs, which is hard to edit inside the overlay VM + mergeDrvs = { drvs, onlyDeps ? false }: + runCommand "merge-drvs" { + inherit drvs onlyDeps; + } '' + if [ -n "$onlyDeps" ]; then + echo $drvs > $out + exit 0 + fi + + mkdir $out + for drv in $drvs; do + echo Merging $drv + if [ -d "$drv" ]; then + cp -drf --preserve=mode -f $drv/* $out/ + else + tar -C $out -xpf $drv || true + fi + done + ''; + + mkTarball = { name ? "docker-tar", drv, onlyDeps ? false }: + runCommand "${name}.tar.gz" rec { + inherit drv onlyDeps; + + drvClosure = writeReferencesToFile drv; + + } '' + while read dep; do + echo Copying $dep + dir="$(dirname "$dep")" + mkdir -p "rootfs/$dir" + cp -drf --preserve=mode $dep "rootfs/$dir/" + done < "$drvClosure" + + if [ -z "$onlyDeps" ]; then + cp -drf --preserve=mode $drv/* rootfs/ + fi + + tar -C rootfs/ -cpzf $out . + ''; + + shellScript = text: + writeScript "script.sh" '' + #!${stdenv.shell} + set -e + export PATH=${coreutils}/bin:/bin + + ${text} + ''; + + shadowSetup = '' + export PATH=${shadow}/bin:$PATH + mkdir -p /etc/pam.d + if [ ! -f /etc/passwd ]; then + echo "root:x:0:0::/root:/bin/sh" > /etc/passwd + echo "root:!x:::::::" > /etc/shadow + fi + if [ ! -f /etc/group ]; then + echo "root:x:0:" > /etc/group + echo "root:x::" > /etc/gshadow + fi + if [ ! -f /etc/pam.d/other ]; then + cat > /etc/pam.d/other <<EOF +account sufficient pam_unix.so +auth sufficient pam_rootok.so +password requisite pam_unix.so nullok sha512 +session required pam_unix.so +EOF + fi + if [ ! -f /etc/login.defs ]; then + touch /etc/login.defs + fi + ''; + + # Append to tar instead of unpacking + mergeTarballs = tarballs: + runCommand "merge-tars" { inherit tarballs; } '' + mkdir tmp + for tb in $tarballs; do + tar -C tmp -xkpf $tb + done + tar -C tmp -cpzf $out . + ''; + + runWithOverlay = { name , fromImage ? null, fromImageName ? null, fromImageTag ? null + , diskSize ? 1024, preMount ? "", postMount ? "", postUmount ? "" }: + vmTools.runInLinuxVM ( + runCommand name { + preVM = vmTools.createEmptyImage { size = diskSize; fullName = "docker-run-disk"; }; + + inherit fromImage fromImageName fromImageTag; + + buildInputs = [ utillinux e2fsprogs jshon ]; + } '' + rm -rf $out + + mkdir disk + mkfs /dev/${vmTools.hd} + mount /dev/${vmTools.hd} disk + cd disk + + if [ -n "$fromImage" ]; then + echo Unpacking base image + mkdir image + tar -C image -xpf "$fromImage" + + if [ -z "$fromImageName" ]; then + fromImageName=$(jshon -k < image/repositories|head -n1) + fi + if [ -z "$fromImageTag" ]; then + fromImageTag=$(jshon -e $fromImageName -k < image/repositories|head -n1) + fi + parentID=$(jshon -e $fromImageName -e $fromImageTag -u < image/repositories) + fi + + lowerdir="" + while [ -n "$parentID" ]; do + echo Unpacking layer $parentID + mkdir -p image/$parentID/layer + tar -C image/$parentID/layer -xpf image/$parentID/layer.tar + rm image/$parentID/layer.tar + + find image/$parentID/layer -name ".wh.*" -exec bash -c 'name="$(basename {}|sed "s/^.wh.//")"; mknod "$(dirname {})/$name" c 0 0; rm {}' \; + + lowerdir=$lowerdir''${lowerdir:+:}image/$parentID/layer + parentID=$(cat image/$parentID/json|(jshon -e parent -u 2>/dev/null || true)) + done + + mkdir work + mkdir layer + mkdir mnt + + ${preMount} + + if [ -n "$lowerdir" ]; then + mount -t overlay overlay -olowerdir=$lowerdir,workdir=work,upperdir=layer mnt + else + mount --bind layer mnt + fi + + ${postMount} + + umount mnt + + pushd layer + find . -type c -exec bash -c 'name="$(basename {})"; touch "$(dirname {})/.wh.$name"; rm "{}"' \; + popd + + ${postUmount} + ''); + + exportImage = { name ? fromImage.name, fromImage, fromImageName ? null, fromImageTag ? null, diskSize ? 1024 }: + runWithOverlay { + inherit name fromImage fromImageName fromImageTag diskSize; + + postMount = '' + echo Packing raw image + tar -C mnt -czf $out . + ''; + }; + + mkPureLayer = { baseJson, contents ? null, extraCommands ? "" }: + runCommand "docker-layer" { + inherit baseJson contents extraCommands; + + buildInputs = [ jshon ]; + } '' + mkdir layer + if [ -n "$contents" ]; then + echo Adding contents + for c in $contents; do + cp -drf $c/* layer/ + chmod -R ug+w layer/ + done + fi + + pushd layer + ${extraCommands} + popd + + echo Packing layer + mkdir $out + tar -C layer -cf $out/layer.tar . + ts=$(${tarsum} < $out/layer.tar) + cat ${baseJson} | jshon -s "$ts" -i checksum > $out/json + echo -n "1.0" > $out/VERSION + ''; + + mkRootLayer = { runAsRoot, baseJson, fromImage ? null, fromImageName ? null, fromImageTag ? null + , diskSize ? 1024, contents ? null, extraCommands ? "" }: + let runAsRootScript = writeScript "run-as-root.sh" runAsRoot; + in runWithOverlay { + name = "docker-layer"; + + inherit fromImage fromImageName fromImageTag diskSize; + + preMount = lib.optionalString (contents != null) '' + echo Adding contents + for c in ${builtins.toString contents}; do + cp -drf $c/* layer/ + chmod -R ug+w layer/ + done + ''; + + postMount = '' + mkdir -p mnt/{dev,proc,sys,nix/store} + mount --rbind /dev mnt/dev + mount --rbind /sys mnt/sys + mount --rbind /nix/store mnt/nix/store + + unshare -imnpuf --mount-proc chroot mnt ${runAsRootScript} + umount -R mnt/dev mnt/sys mnt/nix/store + rmdir --ignore-fail-on-non-empty mnt/dev mnt/proc mnt/sys mnt/nix/store mnt/nix + ''; + + postUmount = '' + pushd layer + ${extraCommands} + popd + + echo Packing layer + mkdir $out + tar -C layer -cf $out/layer.tar . + ts=$(${tarsum} < $out/layer.tar) + cat ${baseJson} | jshon -s "$ts" -i checksum > $out/json + echo -n "1.0" > $out/VERSION + ''; + }; + + # 1. extract the base image + # 2. create the layer + # 3. add layer deps to the layer itself, diffing with the base image + # 4. compute the layer id + # 5. put the layer in the image + # 6. repack the image + buildImage = args@{ name, tag ? "latest" + , fromImage ? null, fromImageName ? null, fromImageTag ? null + , contents ? null, tarballs ? [], config ? null + , runAsRoot ? null, diskSize ? 1024, extraCommands ? "" }: + + let + + baseJson = writeText "${name}-config.json" (builtins.toJSON { + created = "1970-01-01T00:00:01Z"; + architecture = "amd64"; + os = "linux"; + config = config; + }); + + layer = (if runAsRoot == null + then mkPureLayer { inherit baseJson contents extraCommands; } + else mkRootLayer { inherit baseJson fromImage fromImageName fromImageTag contents runAsRoot diskSize extraCommands; }); + depsTarball = mkTarball { name = "${name}-deps"; + drv = layer; + onlyDeps = true; }; + + result = runCommand "${name}.tar.gz" { + buildInputs = [ jshon ]; + + imageName = name; + imageTag = tag; + inherit fromImage baseJson; + + mergedTarball = if tarballs == [] then depsTarball else mergeTarballs ([ depsTarball ] ++ tarballs); + + passthru = { + buildArgs = args; + }; + } '' + mkdir image + touch baseFiles + if [ -n "$fromImage" ]; then + echo Unpacking base image + tar -C image -xpf "$fromImage" + + if [ -z "$fromImageName" ]; then + fromImageName=$(jshon -k < image/repositories|head -n1) + fi + if [ -z "$fromImageTag" ]; then + fromImageTag=$(jshon -e $fromImageName -k < image/repositories|head -n1) + fi + parentID=$(jshon -e $fromImageName -e $fromImageTag -u < image/repositories) + + for l in image/*/layer.tar; do + tar -tf $l >> baseFiles + done + fi + + chmod -R ug+rw image + + mkdir temp + cp ${layer}/* temp/ + chmod ug+w temp/* + + echo Adding dependencies + tar -tf temp/layer.tar >> baseFiles + tar -tf "$mergedTarball" | grep -v ${layer} > layerFiles + if [ "$(wc -l layerFiles|cut -d ' ' -f 1)" -gt 3 ]; then + sed -i -e 's|^[\./]\+||' baseFiles layerFiles + comm <(sort -n baseFiles|uniq) <(sort -n layerFiles|uniq) -1 -3 > newFiles + mkdir deps + pushd deps + tar -xpf "$mergedTarball" --no-recursion --files-from ../newFiles 2>/dev/null || true + tar -rf ../temp/layer.tar --no-recursion --files-from ../newFiles 2>/dev/null || true + popd + else + echo No new deps, no diffing needed + fi + + echo Adding meta + + if [ -n "$parentID" ]; then + cat temp/json | jshon -s "$parentID" -i parent > tmpjson + mv tmpjson temp/json + fi + + layerID=$(sha256sum temp/json|cut -d ' ' -f 1) + size=$(stat --printf="%s" temp/layer.tar) + cat temp/json | jshon -s "$layerID" -i id -n $size -i Size > tmpjson + mv tmpjson temp/json + + mv temp image/$layerID + + jshon -n object \ + -n object -s "$layerID" -i "$imageTag" \ + -i "$imageName" > image/repositories + + chmod -R a-w image + + echo Cooking the image + tar -C image -czf $out . + ''; + + in + + result; + +} |