NixOS/amazonImageZfs: init

Introduce an AWS EC2 AMI which supports aarch64 and x86_64 with a ZFS
root.

This uses `make-zfs-image` which implies two EBS volumes are needed
inside EC2, one for boot, one for root. It should not matter which
is identified `xvda` and which is `xvdb`, though I have always
uploaded `boot` as `xvda`.

5 files changed, 167 insertions, 30 deletions

diff --git a/nixos/maintainers/scripts/ec2/amazon-image-zfs.nix b/nixos/maintainers/scripts/ec2/amazon-image-zfs.nix
new file mode 100644
index 00000000000..32dd96a7cb7
--- /dev/null
+++ b/nixos/maintainers/scripts/ec2/amazon-image-zfs.nix
@@ -0,0 +1,12 @@
+{
+  imports = [ ./amazon-image.nix ];
+  ec2.zfs = {
+    enable = true;
+    datasets = {
+      "tank/system/root".mount = "/";
+      "tank/system/var".mount = "/var";
+      "tank/local/nix".mount = "/nix";
+      "tank/user/home".mount = "/home";
+    };
+  };
+}
diff --git a/nixos/maintainers/scripts/ec2/amazon-image.nix b/nixos/maintainers/scripts/ec2/amazon-image.nix
index 677aff4421e..cb9fbfd8603 100644
--- a/nixos/maintainers/scripts/ec2/amazon-image.nix
+++ b/nixos/maintainers/scripts/ec2/amazon-image.nix
@@ -4,6 +4,7 @@ with lib;
 
 let
   cfg = config.amazonImage;
+
 in {
 
   imports = [ ../../../modules/virtualisation/amazon-image.nix ];
@@ -53,15 +54,7 @@ in {
     };
   };
 
-  config.system.build.amazonImage = import ../../../lib/make-disk-image.nix {
-    inherit lib config;
-    inherit (cfg) contents format name;
-    pkgs = import ../../../.. { inherit (pkgs) system; }; # ensure we use the regular qemu-kvm package
-    partitionTableType = if config.ec2.efi then "efi"
-                         else if config.ec2.hvm then "legacy+gpt"
-                         else "none";
-    diskSize = cfg.sizeMB;
-    fsType = "ext4";
+  config.system.build.amazonImage = let
     configFile = pkgs.writeText "configuration.nix"
       ''
         { modulesPath, ... }: {
@@ -72,24 +65,84 @@ in {
           ${optionalString config.ec2.efi ''
             ec2.efi = true;
           ''}
+          ${optionalString config.ec2.zfs.enable ''
+            ec2.zfs.enable = true;
+            networking.hostId = "${config.networking.hostId}";
+          ''}
         }
       '';
-    postVM = ''
-      extension=''${diskImage##*.}
-      friendlyName=$out/${cfg.name}.$extension
-      mv "$diskImage" "$friendlyName"
-      diskImage=$friendlyName
-
-      mkdir -p $out/nix-support
-      echo "file ${cfg.format} $diskImage" >> $out/nix-support/hydra-build-products
-
-      ${pkgs.jq}/bin/jq -n \
-        --arg label ${lib.escapeShellArg config.system.nixos.label} \
-        --arg system ${lib.escapeShellArg pkgs.stdenv.hostPlatform.system} \
-        --arg logical_bytes "$(${pkgs.qemu}/bin/qemu-img info --output json "$diskImage" | ${pkgs.jq}/bin/jq '."virtual-size"')" \
-        --arg file "$diskImage" \
-        '$ARGS.named' \
-        > $out/nix-support/image-info.json
-    '';
-  };
+
+    zfsBuilder = import ../../../lib/make-zfs-image.nix {
+      inherit lib config configFile;
+      inherit (cfg) contents format name;
+      pkgs = import ../../../.. { inherit (pkgs) system; }; # ensure we use the regular qemu-kvm package
+
+      includeChannel = true;
+
+      bootSize = 1000; # 1G is the minimum EBS volume
+
+      rootSize = cfg.sizeMB;
+      rootPoolProperties = {
+        ashift = 12;
+        autoexpand = "on";
+      };
+
+      datasets = config.ec2.zfs.datasets;
+
+      postVM = ''
+        extension=''${rootDiskImage##*.}
+        friendlyName=$out/${cfg.name}
+        rootDisk="$friendlyName.root.$extension"
+        bootDisk="$friendlyName.boot.$extension"
+        mv "$rootDiskImage" "$rootDisk"
+        mv "$bootDiskImage" "$bootDisk"
+
+        mkdir -p $out/nix-support
+        echo "file ${cfg.format} $bootDisk" >> $out/nix-support/hydra-build-products
+        echo "file ${cfg.format} $rootDisk" >> $out/nix-support/hydra-build-products
+
+       ${pkgs.jq}/bin/jq -n \
+         --arg label ${lib.escapeShellArg config.system.nixos.label} \
+         --arg system ${lib.escapeShellArg pkgs.stdenv.hostPlatform.system} \
+         --arg root_logical_bytes "$(${pkgs.qemu}/bin/qemu-img info --output json "$bootDisk" | ${pkgs.jq}/bin/jq '."virtual-size"')" \
+         --arg boot_logical_bytes "$(${pkgs.qemu}/bin/qemu-img info --output json "$rootDisk" | ${pkgs.jq}/bin/jq '."virtual-size"')" \
+         --arg root "$rootDisk" \
+         --arg boot "$bootDisk" \
+         '$ARGS.named' \
+         > $out/nix-support/image-info.json
+      '';
+    };
+
+    extBuilder = import ../../../lib/make-disk-image.nix {
+      inherit lib config configFile;
+
+      inherit (cfg) contents format name;
+      pkgs = import ../../../.. { inherit (pkgs) system; }; # ensure we use the regular qemu-kvm package
+
+      fsType = "ext4";
+      partitionTableType = if config.ec2.efi then "efi"
+                           else if config.ec2.hvm then "legacy+gpt"
+                           else "none";
+
+      diskSize = cfg.sizeMB;
+
+      postVM = ''
+        extension=''${diskImage##*.}
+        friendlyName=$out/${cfg.name}.$extension
+        mv "$diskImage" "$friendlyName"
+        diskImage=$friendlyName
+
+        mkdir -p $out/nix-support
+        echo "file ${cfg.format} $diskImage" >> $out/nix-support/hydra-build-products
+
+       ${pkgs.jq}/bin/jq -n \
+         --arg label ${lib.escapeShellArg config.system.nixos.label} \
+         --arg system ${lib.escapeShellArg pkgs.stdenv.hostPlatform.system} \
+         --arg logical_bytes "$(${pkgs.qemu}/bin/qemu-img info --output json "$diskImage" | ${pkgs.jq}/bin/jq '."virtual-size"')" \
+         --arg file "$diskImage" \
+         '$ARGS.named' \
+         > $out/nix-support/image-info.json
+      '';
+    };
+  in if config.ec2.zfs.enable then zfsBuilder else extBuilder;
 }
diff --git a/nixos/modules/virtualisation/amazon-image.nix b/nixos/modules/virtualisation/amazon-image.nix
index bf5c04543a7..fe248a94488 100644
--- a/nixos/modules/virtualisation/amazon-image.nix
+++ b/nixos/modules/virtualisation/amazon-image.nix
@@ -41,17 +41,23 @@ in
 
     boot.growPartition = cfg.hvm;
 
-    fileSystems."/" = {
+    fileSystems."/" = mkIf (!cfg.zfs.enable) {
       device = "/dev/disk/by-label/nixos";
       fsType = "ext4";
       autoResize = true;
     };
 
-    fileSystems."/boot" = mkIf cfg.efi {
+    fileSystems."/boot" = mkIf (cfg.efi || cfg.zfs.enable) {
+      # The ZFS image uses a partition labeled ESP whether or not we're
+      # booting with EFI.
       device = "/dev/disk/by-label/ESP";
       fsType = "vfat";
     };
 
+    services.zfs.expandOnBoot = mkIf cfg.zfs.enable "all";
+
+    boot.zfs.devNodes = mkIf cfg.zfs.enable "/dev/";
+
     boot.extraModulePackages = [
       config.boot.kernelPackages.ena
     ];
diff --git a/nixos/modules/virtualisation/amazon-options.nix b/nixos/modules/virtualisation/amazon-options.nix
index 2e807131e93..698edcd835a 100644
--- a/nixos/modules/virtualisation/amazon-options.nix
+++ b/nixos/modules/virtualisation/amazon-options.nix
@@ -1,7 +1,46 @@
 { config, lib, pkgs, ... }:
-{
+let
+  inherit (lib) types;
+in {
   options = {
     ec2 = {
+      zfs = {
+        enable = lib.mkOption {
+          default = false;
+          internal = true;
+          description = ''
+            Whether the EC2 instance uses a ZFS root.
+          '';
+        };
+
+        datasets = lib.mkOption {
+          description = ''
+            Datasets to create under the `tank` and `boot` zpools.
+
+            **NOTE:** This option is used only at image creation time, and
+            does not attempt to declaratively create or manage datasets
+            on an existing system.
+          '';
+
+          default = {};
+
+          type = types.attrsOf (types.submodule {
+            options = {
+              mount = lib.mkOption {
+                description = "Where to mount this dataset.";
+                type = types.nullOr types.string;
+                default = null;
+              };
+
+              properties = lib.mkOption {
+                description = "Properties to set on this dataset.";
+                type = types.attrsOf types.string;
+                default = {};
+              };
+            };
+          });
+        };
+      };
       hvm = lib.mkOption {
         default = lib.versionAtLeast config.system.stateVersion "17.03";
         internal = true;
@@ -18,4 +57,17 @@
       };
     };
   };
+
+  config = lib.mkIf config.ec2.zfs.enable {
+    networking.hostId = lib.mkDefault "00000000";
+
+    fileSystems = let
+      mountable = lib.filterAttrs (_: value: ((value.mount or null) != null)) config.ec2.zfs.datasets;
+    in lib.mapAttrs'
+      (dataset: opts: lib.nameValuePair opts.mount {
+        device = dataset;
+        fsType = "zfs";
+      })
+      mountable;
+  };
 }
diff --git a/nixos/release.nix b/nixos/release.nix
index 2367e79e4ad..264d82bacc8 100644
--- a/nixos/release.nix
+++ b/nixos/release.nix
@@ -217,6 +217,20 @@ in rec {
     }).config.system.build.amazonImage)
 
   );
+  amazonImageZfs = forMatchingSystems [ "x86_64-linux" "aarch64-linux" ] (system:
+
+    with import ./.. { inherit system; };
+
+    hydraJob ((import lib/eval-config.nix {
+      inherit system;
+      modules =
+        [ configuration
+          versionModule
+          ./maintainers/scripts/ec2/amazon-image-zfs.nix
+        ];
+    }).config.system.build.amazonImage)
+
+  );
 
 
   # Test job for https://github.com/NixOS/nixpkgs/issues/121354 to test