nixos-rebuild: Add option for building and/or deploying on a remote host

This commit adds the options --build-host and --target-host to nixos-rebuild.

--build-host instructs nixos-rebuild to perform all nix builds on the
specified host (via ssh). Build results are then copied back to the
local machine and used when activating the system.

--build-target instructs nixos-rebuild to activate the configuration
not on the local machine but on the specified remote host. Build
results are copied to the target machine and then activated there (via ssh).

It is possible to combine the usage of --build-host and --target-host,
in which case you can perform the build on one remote machine and deploy
the configuration to another remote machine. The only requirement is that
the build host has a working ssh connection to the target host (if the
target is not local), and that the local machine can connect to both
the target and the build host. Also, your user must be allowed to copy
nix closures between the local machine and the target and host machines.

At no point in time are the configuration sources (the nix files) copied
anywhere. Instead, nix evaluation always happens locally
(with nix-instantiate). The drv-file is then copied and realised remotely
(with nix-store).

As a convenience, if only --target-host is specified, --build-host is
implicitly set to that host too. So if you want to build locally and deploy
remotely you have to explicitly set "--build-host localhost".

To activate (test, boot or switch) you need to have root access to the
target host. You can specify this by "--target-host root@myhost".

I have tested the obvious scenarios and they are working. Some of the
combinations of --build-host and --target-host and the various actions might
not make much sense, and should maybe be forbidden (like setting a remote
target host when building a VM), and some combinations might not work at all.

1 files changed, 160 insertions, 27 deletions

diff --git a/nixos/modules/installer/tools/nixos-rebuild.sh b/nixos/modules/installer/tools/nixos-rebuild.sh
index 6792690b4c3..1071460097e 100644
--- a/nixos/modules/installer/tools/nixos-rebuild.sh
+++ b/nixos/modules/installer/tools/nixos-rebuild.sh
@@ -19,6 +19,8 @@ rollback=
 upgrade=
 repair=
 profile=/nix/var/nix/profiles/system
+buildHost=
+targetHost=
 
 while [ "$#" -gt 0 ]; do
     i="$1"; shift 1
@@ -73,6 +75,14 @@ while [ "$#" -gt 0 ]; do
         fi
         shift 1
         ;;
+      --build-host|h)
+        buildHost="$1"
+        shift 1
+        ;;
+      --target-host|t)
+        targetHost="$1"
+        shift 1
+        ;;
       *)
         echo "$0: unknown option \`$i'"
         exit 1
@@ -80,6 +90,90 @@ while [ "$#" -gt 0 ]; do
     esac
 done
 
+
+if [ -z "$buildHost" -a -n "$targetHost" ]; then
+    buildHost="$targetHost"
+fi
+if [ "$targetHost" = localhost ]; then
+    targetHost=
+fi
+if [ "$buildHost" = localhost ]; then
+    buildHost=
+fi
+
+buildHostCmd() {
+    if [ -z "$buildHost" ]; then
+        "$@"
+    elif [ -n "$remoteNix" ]; then
+        ssh $SSHOPTS "$buildHost" PATH="$remoteNix:$PATH" "$@"
+    else
+        ssh $SSHOPTS "$buildHost" "$@"
+    fi
+}
+
+targetHostCmd() {
+    if [ -z "$targetHost" ]; then
+        "$@"
+    else
+        ssh $SSHOPTS "$targetHost" "$@"
+    fi
+}
+
+copyToTarget() {
+    if ! [ "$targetHost" = "$buildHost" ]; then
+        if [ -z "$targetHost" ]; then
+            NIX_SSHOPTS=$SSH_OPTS nix-copy-closure --from "$buildHost" "$1"
+        elif [ -z "$buildHost" ]; then
+            NIX_SSHOPTS=$SSH_OPTS nix-copy-closure --to "$targetHost" "$1"
+        else
+            buildHostCmd nix-copy-closure --to "$targetHost" "$1"
+        fi
+    fi
+}
+
+nixBuild() {
+    if [ -z "$buildHost" ]; then
+        nix-build "$@"
+    else
+        local instArgs=()
+        local buildArgs=()
+
+        while [ "$#" -gt 0 ]; do
+            local i="$1"; shift 1
+            case "$i" in
+              -o)
+                local out="$1"; shift 1
+                buildArgs+=("--add-root" "$out" "--indirect")
+                ;;
+              -A)
+                local j="$1"; shift 1
+                instArgs+=("$i" "$j")
+                ;;
+              -I)
+                # We don't want this in buildArgs
+                shift 1
+                ;;
+              "<"*) # nix paths
+                instArgs+=("$i")
+                ;;
+              *)
+                buildArgs+=("$i")
+                ;;
+            esac
+        done
+
+        local drv="$(nix-instantiate "${instArgs[@]}" "${extraBuildFlags[@]}")"
+        if [ -a "$drv" ]; then
+            NIX_SSHOPTS=$SSH_OPTS nix-copy-closure --to "$buildHost" "$drv"
+            buildHostCmd nix-store -r "$drv" "${buildArgs[@]}"
+        else
+            echo "nix-instantiate failed"
+            exit 1
+        fi
+  fi
+}
+
+
 if [ -z "$action" ]; then showSyntax; fi
 
 # Only run shell scripts from the Nixpkgs tree if the action is
@@ -128,7 +222,16 @@ fi
 
 
 tmpDir=$(mktemp -t -d nixos-rebuild.XXXXXX)
-trap 'rm -rf "$tmpDir"' EXIT
+SSHOPTS="$NIX_SSHOPTS -o ControlMaster=auto -o ControlPath=$tmpDir/ssh-%n -o ControlPersist=60"
+
+cleanup() {
+    for ctrl in "$tmpDir"/ssh-*; do
+        ssh -o ControlPath="$ctrl" -O exit dummyhost 2>/dev/null || true
+    done
+    rm -rf "$tmpDir"
+}
+trap cleanup EXIT
+
 
 
 # If the Nix daemon is running, then use it.  This allows us to use
@@ -150,30 +253,56 @@ if [ -n "$rollback" -o "$action" = dry-build ]; then
     buildNix=
 fi
 
+prebuiltNix() {
+    machine="$1"
+    if [ "$machine" = x86_64 ]; then
+        return /nix/store/xryr9g56h8yjddp89d6dw12anyb4ch7c-nix-1.10
+    elif [[ "$machine" =~ i.86 ]]; then
+        return /nix/store/2w92k5wlpspf0q2k9mnf2z42prx3bwmv-nix-1.10
+    else
+        echo "$0: unsupported platform"
+        exit 1
+    fi
+}
+
+remotePATH=
+
 if [ -n "$buildNix" ]; then
     echo "building Nix..." >&2
-    if ! nix-build '<nixpkgs/nixos>' -A config.nix.package -o $tmpDir/nix "${extraBuildFlags[@]}" > /dev/null; then
-        if ! nix-build '<nixpkgs/nixos>' -A nixFallback -o $tmpDir/nix "${extraBuildFlags[@]}" > /dev/null; then
-            if ! nix-build '<nixpkgs>' -A nix -o $tmpDir/nix "${extraBuildFlags[@]}" > /dev/null; then
-                machine="$(uname -m)"
-                if [ "$machine" = x86_64 ]; then
-                    nixStorePath=/nix/store/xryr9g56h8yjddp89d6dw12anyb4ch7c-nix-1.10
-                elif [[ "$machine" =~ i.86 ]]; then
-                    nixStorePath=/nix/store/2w92k5wlpspf0q2k9mnf2z42prx3bwmv-nix-1.10
-                else
-                    echo "$0: unsupported platform"
-                    exit 1
-                fi
+    nixDrv=
+    if ! nixDrv="$(nix-instantiate '<nixpkgs/nixos>' --add-root $tmpDir/nixdrv --indirect -A config.nix.package "${extraBuildFlags[@]}")"; then
+        if ! nixDrv="$(nix-instantiate '<nixpkgs/nixos>' --add-root $tmpDir/nixdrv --indirect -A nixFallback "${extraBuildFlags[@]}")"; then
+            if ! nixDrv="$(nix-instantiate '<nixpkgs>' --add-root $tmpDir/nixdrv --indirect -A nix "${extraBuildFlags[@]}")"; then
+                nixStorePath="$(prebuiltNix "$(uname -m)")"
                 if ! nix-store -r $nixStorePath --add-root $tmpDir/nix --indirect \
                     --option extra-binary-caches https://cache.nixos.org/; then
                     echo "warning: don't know how to get latest Nix" >&2
                 fi
                 # Older version of nix-store -r don't support --add-root.
                 [ -e $tmpDir/nix ] || ln -sf $nixStorePath $tmpDir/nix
+                if [ -n "$buildHost" ]; then
+                    remoteNixStorePath="$(prebuiltNix "$(buildHostCmd uname -m)")"
+                    remoteNix="$remoteNixStorePath/bin"
+                    if ! buildHostCmd nix-store -r $remoteNixStorePath \
+                      --option extra-binary-caches https://cache.nixos.org/ >/dev/null; then
+                        remoteNix=
+                        echo "warning: don't know how to get latest Nix" >&2
+                    fi
+                fi
             fi
         fi
     fi
-    PATH=$tmpDir/nix/bin:$PATH
+    if [ -a "$nixDrv" ]; then
+        nix-store -r "$nixDrv" --add-root $tmpDir/nix --indirect >/dev/null
+        if [ -n "$buildHost" ]; then
+            nix-copy-closure --to "$buildHost" "$nixDrv"
+            # The nix build produces multiple outputs, we add them all to the remote path
+            for p in $(buildHostCmd nix-store -r "$(readlink "$nixDrv")" "${buildArgs[@]}"); do
+                remoteNix="$remoteNix${remoteNix:+:}$p/bin"
+            done
+        fi
+    fi
+    PATH="$tmpDir/nix/bin:$PATH"
 fi
 
 
@@ -200,31 +329,35 @@ fi
 if [ -z "$rollback" ]; then
     echo "building the system configuration..." >&2
     if [ "$action" = switch -o "$action" = boot ]; then
-        nix-env "${extraBuildFlags[@]}" -p "$profile" -f '<nixpkgs/nixos>' --set -A system
-        pathToConfig="$profile"
+        pathToConfig="$(nixBuild '<nixpkgs/nixos>' -A system "${extraBuildFlags[@]}")"
+        copyToTarget "$pathToConfig"
+        targetHostCmd nix-env -p "$profile" --set "$pathToConfig"
     elif [ "$action" = test -o "$action" = build -o "$action" = dry-build -o "$action" = dry-activate ]; then
-        nix-build '<nixpkgs/nixos>' -A system -k "${extraBuildFlags[@]}" > /dev/null
-        pathToConfig=./result
+        pathToConfig="$(nixBuild '<nixpkgs/nixos>' -A system -k "${extraBuildFlags[@]}")"
     elif [ "$action" = build-vm ]; then
-        nix-build '<nixpkgs/nixos>' -A vm -k "${extraBuildFlags[@]}" > /dev/null
-        pathToConfig=./result
+        pathToConfig="$(nixBuild '<nixpkgs/nixos>' -A vm -k "${extraBuildFlags[@]}")"
     elif [ "$action" = build-vm-with-bootloader ]; then
-        nix-build '<nixpkgs/nixos>' -A vmWithBootLoader -k "${extraBuildFlags[@]}" > /dev/null
-        pathToConfig=./result
+        pathToConfig="$(nixBuild '<nixpkgs/nixos>' -A vmWithBootLoader -k "${extraBuildFlags[@]}")"
     else
         showSyntax
     fi
+    # Copy build to target host if we haven't already done it
+    if ! [ "$action" = switch -o "$action" = boot ]; then
+        copyToTarget "$pathToConfig"
+    fi
 else # [ -n "$rollback" ]
     if [ "$action" = switch -o "$action" = boot ]; then
-        nix-env --rollback -p "$profile"
+        targetHostCmd nix-env --rollback -p "$profile"
         pathToConfig="$profile"
     elif [ "$action" = test -o "$action" = build ]; then
         systemNumber=$(
-            nix-env -p "$profile" --list-generations |
+            targetHostCmd nix-env -p "$profile" --list-generations |
             sed -n '/current/ {g; p;}; s/ *\([0-9]*\).*/\1/; h'
         )
-        ln -sT "$profile"-${systemNumber}-link ./result
-        pathToConfig=./result
+        pathToConfig="$profile"-${systemNumber}-link
+        if [ -z "$targetHost" ]; then
+            ln -sT "$pathToConfig" ./result
+        fi
     else
         showSyntax
     fi
@@ -234,7 +367,7 @@ fi
 # If we're not just building, then make the new configuration the boot
 # default and/or activate it now.
 if [ "$action" = switch -o "$action" = boot -o "$action" = test -o "$action" = dry-activate ]; then
-    if ! $pathToConfig/bin/switch-to-configuration "$action"; then
+    if ! targetHostCmd $pathToConfig/bin/switch-to-configuration "$action"; then
         echo "warning: error(s) occurred while switching to the new configuration" >&2
         exit 1
     fi