Refactor nixos-install to separate out filesystem build logic

The key distinction I'm drawing is that there's a component that deals
with the store of the machine being built, and another component for
the store building it. The inner part of it assumes nothing from the
builder (doesn't need chroot or root powers) so it can run comfortably
inside a Nix build, as well as nixos-rebuild. I have some upcoming work
that will use that to significantly speed up and streamline image builds
for NixOS, especially on virtualized hosts like EC2, but it's also a
reasonable speedup on native hosts.

4 files changed, 161 insertions, 128 deletions

diff --git a/nixos/modules/installer/tools/nixos-install.sh b/nixos/modules/installer/tools/nixos-install.sh
index 57bc249360e..e2ae2ee9fdf 100644
--- a/nixos/modules/installer/tools/nixos-install.sh
+++ b/nixos/modules/installer/tools/nixos-install.sh
@@ -87,38 +87,6 @@ if ! test -e "$mountPoint"; then
     exit 1
 fi
 
-
-# Mount some stuff in the target root directory.
-mkdir -m 0755 -p $mountPoint/dev $mountPoint/proc $mountPoint/sys $mountPoint/etc $mountPoint/run $mountPoint/home
-mkdir -m 01777 -p $mountPoint/tmp
-mkdir -m 0755 -p $mountPoint/tmp/root
-mkdir -m 0755 -p $mountPoint/var
-mkdir -m 0700 -p $mountPoint/root
-mount --rbind /dev $mountPoint/dev
-mount --rbind /proc $mountPoint/proc
-mount --rbind /sys $mountPoint/sys
-mount --rbind / $mountPoint/tmp/root
-mount -t tmpfs -o "mode=0755" none $mountPoint/run
-rm -rf $mountPoint/var/run
-ln -s /run $mountPoint/var/run
-for f in /etc/resolv.conf /etc/hosts; do rm -f $mountPoint/$f; [ -f "$f" ] && cp -Lf $f $mountPoint/etc/; done
-for f in /etc/passwd /etc/group;      do touch $mountPoint/$f; [ -f "$f" ] && mount --rbind -o ro $f $mountPoint/$f; done
-
-cp -Lf "@cacert@" "$mountPoint/tmp/ca-cert.crt"
-export SSL_CERT_FILE=/tmp/ca-cert.crt
-# For Nix 1.7
-export CURL_CA_BUNDLE=/tmp/ca-cert.crt
-
-if [ -n "$runChroot" ]; then
-    if ! [ -L $mountPoint/nix/var/nix/profiles/system ]; then
-        echo "$0: installation not finished; cannot chroot into installation directory"
-        exit 1
-    fi
-    ln -s /nix/var/nix/profiles/system $mountPoint/run/current-system
-    exec chroot $mountPoint "${chrootCommand[@]}"
-fi
-
-
 # Get the path of the NixOS configuration file.
 if test -z "$NIXOS_CONFIG"; then
     NIXOS_CONFIG=/etc/nixos/configuration.nix
@@ -130,121 +98,60 @@ if [ ! -e "$mountPoint/$NIXOS_CONFIG" ] && [ -z "$closure" ]; then
 fi
 
 
-# Create the necessary Nix directories on the target device, if they
-# don't already exist.
-mkdir -m 0755 -p \
-    $mountPoint/nix/var/nix/gcroots \
-    $mountPoint/nix/var/nix/temproots \
-    $mountPoint/nix/var/nix/userpool \
-    $mountPoint/nix/var/nix/profiles \
-    $mountPoint/nix/var/nix/db \
-    $mountPoint/nix/var/log/nix/drvs
-
-mkdir -m 1775 -p $mountPoint/nix/store
-chown @root_uid@:@nixbld_gid@ $mountPoint/nix/store
-
-
-# There is no daemon in the chroot.
-unset NIX_REMOTE
-
-
-# We don't have locale-archive in the chroot, so clear $LANG.
-export LANG=
-export LC_ALL=
-export LC_TIME=
-
-
 # Builds will use users that are members of this group
 extraBuildFlags+=(--option "build-users-group" "$buildUsersGroup")
 
-
 # Inherit binary caches from the host
+# TODO: will this still work with Nix 1.12 now that it has no perl? Probably not... 
 binary_caches="$(@perl@/bin/perl -I @nix@/lib/perl5/site_perl/*/* -e 'use Nix::Config; Nix::Config::readConfig; print $Nix::Config::config{"binary-caches"};')"
 extraBuildFlags+=(--option "binary-caches" "$binary_caches")
 
+nixpkgs="$(readlink -f "$(nix-instantiate --find-file nixpkgs)")"
+export NIX_PATH="nixpkgs=$nixpkgs:nixos-config=$mountPoint/$NIXOS_CONFIG"
+unset NIXOS_CONFIG
 
-# Copy Nix to the Nix store on the target device, unless it's already there.
-if ! NIX_DB_DIR=$mountPoint/nix/var/nix/db nix-store --check-validity @nix@ 2> /dev/null; then
-    echo "copying Nix to $mountPoint...."
-    for i in $(@perl@/bin/perl @pathsFromGraph@ @nixClosure@); do
-        echo "  $i"
-        chattr -R -i $mountPoint/$i 2> /dev/null || true # clear immutable bit
-        @rsync@/bin/rsync -a $i $mountPoint/nix/store/
-    done
-
-    # Register the paths in the Nix closure as valid.  This is necessary
-    # to prevent them from being deleted the first time we install
-    # something.  (I.e., Nix will see that, e.g., the glibc path is not
-    # valid, delete it to get it out of the way, but as a result nothing
-    # will work anymore.)
-    chroot $mountPoint @nix@/bin/nix-store --register-validity < @nixClosure@
-fi
-
-
-# Create the required /bin/sh symlink; otherwise lots of things
-# (notably the system() function) won't work.
-mkdir -m 0755 -p $mountPoint/bin
-# !!! assuming that @shell@ is in the closure
-ln -sf @shell@ $mountPoint/bin/sh
+# TODO: do I need to set NIX_SUBSTITUTERS here or is the --option binary-caches above enough?
 
 
-# Build hooks likely won't function correctly in the minimal chroot; just disable them.
-unset NIX_BUILD_HOOK
+# A place to drop temporary closures
+trap "rm -rf $tmpdir" EXIT
+tmpdir="$(mktemp -d)"
 
-# Make the build below copy paths from the CD if possible.  Note that
-# /tmp/root in the chroot is the root of the CD.
-export NIX_OTHER_STORES=/tmp/root/nix:$NIX_OTHER_STORES
-
-p=@nix@/libexec/nix/substituters
-export NIX_SUBSTITUTERS=$p/copy-from-other-stores.pl:$p/download-from-binary-cache.pl
+# Build a closure (on the host; we then copy it into the guest)
+function closure() {
+    nix-build "${extraBuildFlags[@]}" --no-out-link -E "with import <nixpkgs> {}; runCommand \"closure\" { exportReferencesGraph = [ \"x\" (buildEnv { name = \"env\"; paths = [ ($1) stdenv ]; }) ]; } \"cp x \$out\""
+}
 
+system_closure="$tmpdir/system.closure"
 
 if [ -z "$closure" ]; then
-    # Get the absolute path to the NixOS/Nixpkgs sources.
-    nixpkgs="$(readlink -f $(nix-instantiate --find-file nixpkgs))"
-
-    nixEnvAction="-f <nixpkgs/nixos> --set -A system"
+    expr="(import <nixpkgs/nixos> {}).system"
+    system_root="$(nix-build -E "$expr")"
+    system_closure="$(closure "$expr")"
 else
-    nixpkgs=""
-    nixEnvAction="--set $closure"
+    system_root=$closure
+    # Create a temporary file ending in .closure (so nixos-prepare-root knows to --import it) to transport the store closure
+    # to the filesytem we're preparing. Also delete it on exit!
+    nix-store --export $(nix-store -qR $closure) > $system_closure
 fi
 
-# Build the specified Nix expression in the target store and install
-# it into the system configuration profile.
-echo "building the system configuration..."
-NIX_PATH="nixpkgs=/tmp/root/$nixpkgs:nixos-config=$NIXOS_CONFIG" NIXOS_CONFIG= \
-    chroot $mountPoint @nix@/bin/nix-env \
-    "${extraBuildFlags[@]}" -p /nix/var/nix/profiles/system $nixEnvAction
+channel_root="$(nix-env -p /nix/var/nix/profiles/per-user/root/channels -q nixos --no-name --out-path 2>/dev/null || echo -n "")"
+channel_closure="$tmpdir/channel.closure"
+nix-store --export $channel_root > $channel_closure
 
+# Populate the target root directory with the basics
+@prepare_root@/bin/nixos-prepare-root $mountPoint $channel_root $system_root @nixClosure@ $system_closure $channel_closure
 
-# Copy the NixOS/Nixpkgs sources to the target as the initial contents
-# of the NixOS channel.
-mkdir -m 0755 -p $mountPoint/nix/var/nix/profiles
-mkdir -m 1777 -p $mountPoint/nix/var/nix/profiles/per-user
-mkdir -m 0755 -p $mountPoint/nix/var/nix/profiles/per-user/root
-srcs=$(nix-env "${extraBuildFlags[@]}" -p /nix/var/nix/profiles/per-user/root/channels -q nixos --no-name --out-path 2>/dev/null || echo -n "")
-if [ -z "$noChannelCopy" ] && [ -n "$srcs" ]; then
-    echo "copying NixOS/Nixpkgs sources..."
-    chroot $mountPoint @nix@/bin/nix-env \
-        "${extraBuildFlags[@]}" -p /nix/var/nix/profiles/per-user/root/channels -i "$srcs" --quiet
-fi
-mkdir -m 0700 -p $mountPoint/root/.nix-defexpr
-ln -sfn /nix/var/nix/profiles/per-user/root/channels $mountPoint/root/.nix-defexpr/channels
-
-
-# Get rid of the /etc bind mounts.
-for f in /etc/passwd /etc/group; do [ -f "$f" ] && umount $mountPoint/$f; done
+# nixos-prepare-root doesn't currently do anything with file ownership, so we set it up here instead
+chown @root_uid@:@nixbld_gid@ $mountPoint/nix/store
 
+mount --rbind /dev $mountPoint/dev
+mount --rbind /proc $mountPoint/proc
+mount --rbind /sys $mountPoint/sys
 
 # Grub needs an mtab.
 ln -sfn /proc/mounts $mountPoint/etc/mtab
 
-
-# Mark the target as a NixOS installation, otherwise
-# switch-to-configuration will chicken out.
-touch $mountPoint/etc/NIXOS
-
-
 # Switch to the new system configuration.  This will install Grub with
 # a menu default pointing at the kernel/initrd/etc of the new
 # configuration.
diff --git a/nixos/modules/installer/tools/nixos-prepare-root.sh b/nixos/modules/installer/tools/nixos-prepare-root.sh
new file mode 100644
index 00000000000..c374330f846
--- /dev/null
+++ b/nixos/modules/installer/tools/nixos-prepare-root.sh
@@ -0,0 +1,105 @@
+#! @shell@
+
+# This script's goal is to perform all "static" setup of a filesystem structure from pre-built store paths. Everything
+# in here should run in a non-root context and inside a Nix builder. It's designed primarily to be called from image-
+# building scripts and from nixos-install, but because it makes very few assumptions about the context in which it runs,
+# it could be useful in other contexts as well.
+#
+# Current behavior:
+#  - set up basic filesystem structure
+#  - make Nix store etc.
+#  - copy Nix, system, channel, and misceallaneous closures to target Nix store
+#  - register validity of all paths in the target store
+#  - set up channel and system profiles
+
+# Ensure a consistent umask.
+umask 0022
+
+set -e
+
+mountPoint="$1"
+channel="$2"
+system="$3"
+shift 3
+closures="$@"
+
+PATH="@coreutils@/bin:@nix@/bin:@perl@/bin:@utillinux@/bin:@rsync@/bin"
+
+if ! test -e "$mountPoint"; then
+    echo "mount point $mountPoint doesn't exist"
+    exit 1
+fi
+
+# Create a few of the standard directories in the target root directory.
+mkdir -m 0755 -p $mountPoint/dev $mountPoint/proc $mountPoint/sys $mountPoint/etc $mountPoint/run $mountPoint/home
+mkdir -m 01777 -p $mountPoint/tmp
+mkdir -m 0755 -p $mountPoint/tmp/root
+mkdir -m 0755 -p $mountPoint/var
+mkdir -m 0700 -p $mountPoint/root
+
+ln -s /run $mountPoint/var/run
+
+# Create the necessary Nix directories on the target device
+mkdir -m 0755 -p \
+    $mountPoint/nix/var/nix/gcroots \
+    $mountPoint/nix/var/nix/temproots \
+    $mountPoint/nix/var/nix/userpool \
+    $mountPoint/nix/var/nix/profiles \
+    $mountPoint/nix/var/nix/db \
+    $mountPoint/nix/var/log/nix/drvs
+
+mkdir -m 1775 -p $mountPoint/nix/store
+
+# All Nix operations below should operate on our target store, not /nix/store.
+# N.B: this relies on Nix 1.12 or higher
+export NIX_REMOTE=local?root=$mountPoint
+
+# Copy our closures to the Nix store on the target mount point, unless they're already there.
+for i in $closures; do
+    # We support closures both in the format produced by `nix-store --export` and by `exportReferencesGraph`,
+    # mostly because there doesn't seem to be a single format that can be produced outside of a nix build and
+    # inside one. See https://github.com/NixOS/nix/issues/1242 for more discussion.
+    if [[ "$i" =~ \.closure$ ]]; then
+        echo "importing serialized closure $i to $mountPoint..."
+        nix-store --import < $i
+    else
+        # There has to be a better way to do this, right?
+        echo "copying closure $i to $mountPoint..."
+        for j in $(perl @pathsFromGraph@ $i); do
+            echo "  $j... "
+            rsync -a $j $mountPoint/nix/store/
+        done
+
+        nix-store --register-validity < $i
+    fi
+done
+
+# Create the required /bin/sh symlink; otherwise lots of things
+# (notably the system() function) won't work.
+if [ ! -x $mountPoint/@shell@ ]; then
+    echo "Error: @shell@ wasn't included in the closure" >&2
+    exit 1
+fi
+mkdir -m 0755 -p $mountPoint/bin
+ln -sf @shell@ $mountPoint/bin/sh
+
+echo "setting the system closure to '$system'..."
+nix-env "${extraBuildFlags[@]}" -p $mountPoint/nix/var/nix/profiles/system --set "$system"
+
+ln -sfn /nix/var/nix/profiles/system $mountPoint/run/current-system
+
+# Copy the NixOS/Nixpkgs sources to the target as the initial contents of the NixOS channel.
+mkdir -m 0755 -p $mountPoint/nix/var/nix/profiles
+mkdir -m 1777 -p $mountPoint/nix/var/nix/profiles/per-user
+mkdir -m 0755 -p $mountPoint/nix/var/nix/profiles/per-user/root
+
+if [ -z "$noChannelCopy" ] && [ -n "$channel" ]; then
+    echo "copying channel..."
+    nix-env --option build-use-substitutes false "${extraBuildFlags[@]}" -p $mountPoint/nix/var/nix/profiles/per-user/root/channels --set "$channel" --quiet
+fi
+mkdir -m 0700 -p $mountPoint/root/.nix-defexpr
+ln -sfn /nix/var/nix/profiles/per-user/root/channels $mountPoint/root/.nix-defexpr/channels
+
+# Mark the target as a NixOS installation, otherwise switch-to-configuration will chicken out.
+touch $mountPoint/etc/NIXOS
+
diff --git a/nixos/modules/installer/tools/tools.nix b/nixos/modules/installer/tools/tools.nix
index a35f6ad8ae5..a3bae78c0ff 100644
--- a/nixos/modules/installer/tools/tools.nix
+++ b/nixos/modules/installer/tools/tools.nix
@@ -4,7 +4,6 @@
 { config, pkgs, modulesPath, ... }:
 
 let
-
   cfg = config.installer;
 
   makeProg = args: pkgs.substituteAll (args // {
@@ -17,6 +16,14 @@ let
     src = ./nixos-build-vms/nixos-build-vms.sh;
   };
 
+  nixos-prepare-root = makeProg {
+    name = "nixos-prepare-root";
+    src = ./nixos-prepare-root.sh;
+
+    nix = pkgs.nixUnstable;
+    inherit (pkgs) perl pathsFromGraph rsync utillinux coreutils;
+  };
+
   nixos-install = makeProg {
     name = "nixos-install";
     src = ./nixos-install.sh;
@@ -26,6 +33,7 @@ let
     cacert = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
     root_uid = config.ids.uids.root;
     nixbld_gid = config.ids.gids.nixbld;
+    prepare_root = nixos-prepare-root;
 
     nixClosure = pkgs.runCommand "closure"
       { exportReferencesGraph = ["refs" config.nix.package.out]; }
@@ -69,6 +77,7 @@ in
 
     environment.systemPackages =
       [ nixos-build-vms
+        nixos-prepare-root
         nixos-install
         nixos-rebuild
         nixos-generate-config
@@ -77,7 +86,7 @@ in
       ];
 
     system.build = {
-      inherit nixos-install nixos-generate-config nixos-option nixos-rebuild;
+      inherit nixos-install nixos-prepare-root nixos-generate-config nixos-option nixos-rebuild;
     };
 
   };
diff --git a/nixos/tests/installer.nix b/nixos/tests/installer.nix
index 35dd00fe630..3ab3c1bac48 100644
--- a/nixos/tests/installer.nix
+++ b/nixos/tests/installer.nix
@@ -34,6 +34,12 @@ let
           boot.loader.systemd-boot.enable = true;
         ''}
 
+        users.extraUsers.alice = {
+          isNormalUser = true;
+          home = "/home/alice";
+          description = "Alice Foobar";
+        };
+
         hardware.enableAllFirmware = lib.mkForce false;
 
         ${replaceChars ["\n"] ["\n  "] extraConfig}
@@ -96,7 +102,7 @@ let
       $machine->shutdown;
 
       # Now see if we can boot the installation.
-      $machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}" });
+      $machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}", name => "boot-after-install" });
 
       # For example to enter LUKS passphrase.
       ${preBootCommands}
@@ -118,11 +124,17 @@ let
       $machine->waitForUnit("swap.target");
       $machine->succeed("cat /proc/swaps | grep -q /dev");
 
+      # Check that the store is in good shape
+      $machine->succeed("nix-store --verify --check-contents >&2");
+
       # Check whether the channel works.
       $machine->succeed("nix-env -iA nixos.procps >&2");
       $machine->succeed("type -tP ps | tee /dev/stderr") =~ /.nix-profile/
           or die "nix-env failed";
 
+      # Check that the daemon works, and that non-root users can run builds (this will build a new profile generation through the daemon)
+      $machine->succeed("su alice -l -c 'nix-env -iA nixos.procps' >&2");
+
       # We need to a writable nix-store on next boot.
       $machine->copyFileFromHost(
           "${ makeConfig { inherit bootLoader grubVersion grubDevice grubIdentifier extraConfig; forceGrubReinstallCount = 1; } }",
@@ -139,7 +151,7 @@ let
       $machine->shutdown;
 
       # Check whether a writable store build works
-      $machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}" });
+      $machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}", name => "rebuild-switch" });
       ${preBootCommands}
       $machine->waitForUnit("multi-user.target");
       $machine->copyFileFromHost(
@@ -150,7 +162,7 @@ let
 
       # And just to be sure, check that the machine still boots after
       # "nixos-rebuild switch".
-      $machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}" });
+      $machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}", "boot-after-rebuild-switch" });
       ${preBootCommands}
       $machine->waitForUnit("network.target");
       $machine->shutdown;