1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
|
# Configuration for Amazon EC2 instances. (Note that this file is a
# misnomer - it should be "amazon-config.nix" or so, not
# "amazon-image.nix", since it's used not only to build images but
# also to reconfigure instances. However, we can't rename it because
# existing "configuration.nix" files on EC2 instances refer to it.)
{ config, lib, pkgs, ... }:
with lib;
let
cfg = config.ec2;
metadataFetcher = import ./ec2-metadata-fetcher.nix {
inherit (pkgs) curl;
targetRoot = "$targetRoot/";
wgetExtraOptions = "-q";
};
in
{
imports = [
../profiles/headless.nix
# Note: While we do use the headless profile, we also explicitly
# turn on the serial console on ttyS0 below. This is because
# AWS does support accessing the serial console:
# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configure-access-to-serial-console.html
./ec2-data.nix
./amazon-init.nix
];
config = {
assertions = [
{ assertion = cfg.hvm;
message = "Paravirtualized EC2 instances are no longer supported.";
}
{ assertion = cfg.efi -> cfg.hvm;
message = "EC2 instances using EFI must be HVM instances.";
}
{ assertion = versionOlder config.boot.kernelPackages.kernel.version "5.15";
message = "ENA driver fails to build with kernel >= 5.15";
}
];
boot.kernelPackages = pkgs.linuxKernel.packages.linux_5_10;
boot.growPartition = cfg.hvm;
fileSystems."/" = mkIf (!cfg.zfs.enable) {
device = "/dev/disk/by-label/nixos";
fsType = "ext4";
autoResize = true;
};
fileSystems."/boot" = mkIf (cfg.efi || cfg.zfs.enable) {
# The ZFS image uses a partition labeled ESP whether or not we're
# booting with EFI.
device = "/dev/disk/by-label/ESP";
fsType = "vfat";
};
services.zfs.expandOnBoot = mkIf cfg.zfs.enable "all";
boot.zfs.devNodes = mkIf cfg.zfs.enable "/dev/";
boot.extraModulePackages = [
config.boot.kernelPackages.ena
];
boot.initrd.kernelModules = [ "xen-blkfront" "xen-netfront" ];
boot.initrd.availableKernelModules = [ "ixgbevf" "ena" "nvme" ];
boot.kernelParams = mkIf cfg.hvm [ "console=ttyS0,115200n8" "random.trust_cpu=on" ];
# Prevent the nouveau kernel module from being loaded, as it
# interferes with the nvidia/nvidia-uvm modules needed for CUDA.
# Also blacklist xen_fbfront to prevent a 30 second delay during
# boot.
boot.blacklistedKernelModules = [ "nouveau" "xen_fbfront" ];
# Generate a GRUB menu. Amazon's pv-grub uses this to boot our kernel/initrd.
boot.loader.grub.version = if cfg.hvm then 2 else 1;
boot.loader.grub.device = if (cfg.hvm && !cfg.efi) then "/dev/xvda" else "nodev";
boot.loader.grub.extraPerEntryConfig = mkIf (!cfg.hvm) "root (hd0)";
boot.loader.grub.efiSupport = cfg.efi;
boot.loader.grub.efiInstallAsRemovable = cfg.efi;
boot.loader.timeout = 1;
boot.loader.grub.extraConfig = ''
serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1
terminal_output console serial
terminal_input console serial
'';
boot.initrd.network.enable = true;
# Mount all formatted ephemeral disks and activate all swap devices.
# We cannot do this with the ‘fileSystems’ and ‘swapDevices’ options
# because the set of devices is dependent on the instance type
# (e.g. "m1.small" has one ephemeral filesystem and one swap device,
# while "m1.large" has two ephemeral filesystems and no swap
# devices). Also, put /tmp and /var on /disk0, since it has a lot
# more space than the root device. Similarly, "move" /nix to /disk0
# by layering a unionfs-fuse mount on top of it so we have a lot more space for
# Nix operations.
boot.initrd.postMountCommands =
''
${metadataFetcher}
diskNr=0
diskForUnionfs=
for device in /dev/xvd[abcde]*; do
if [ "$device" = /dev/xvda -o "$device" = /dev/xvda1 ]; then continue; fi
fsType=$(blkid -o value -s TYPE "$device" || true)
if [ "$fsType" = swap ]; then
echo "activating swap device $device..."
swapon "$device" || true
elif [ "$fsType" = ext3 ]; then
mp="/disk$diskNr"
diskNr=$((diskNr + 1))
if mountFS "$device" "$mp" "" ext3; then
if [ -z "$diskForUnionfs" ]; then diskForUnionfs="$mp"; fi
fi
else
echo "skipping unknown device type $device"
fi
done
if [ -n "$diskForUnionfs" ]; then
mkdir -m 755 -p $targetRoot/$diskForUnionfs/root
mkdir -m 1777 -p $targetRoot/$diskForUnionfs/root/tmp $targetRoot/tmp
mount --bind $targetRoot/$diskForUnionfs/root/tmp $targetRoot/tmp
if [ "$(cat "$metaDir/ami-manifest-path")" != "(unknown)" ]; then
mkdir -m 755 -p $targetRoot/$diskForUnionfs/root/var $targetRoot/var
mount --bind $targetRoot/$diskForUnionfs/root/var $targetRoot/var
mkdir -p /unionfs-chroot/ro-nix
mount --rbind $targetRoot/nix /unionfs-chroot/ro-nix
mkdir -m 755 -p $targetRoot/$diskForUnionfs/root/nix
mkdir -p /unionfs-chroot/rw-nix
mount --rbind $targetRoot/$diskForUnionfs/root/nix /unionfs-chroot/rw-nix
unionfs -o allow_other,cow,nonempty,chroot=/unionfs-chroot,max_files=32768 /rw-nix=RW:/ro-nix=RO $targetRoot/nix
fi
fi
'';
boot.initrd.extraUtilsCommands =
''
# We need swapon in the initrd.
copy_bin_and_libs ${pkgs.util-linux}/sbin/swapon
'';
# Allow root logins only using the SSH key that the user specified
# at instance creation time.
services.openssh.enable = true;
services.openssh.permitRootLogin = "prohibit-password";
# Enable the serial console on ttyS0
systemd.services."serial-getty@ttyS0".enable = true;
# Creates symlinks for block device names.
services.udev.packages = [ pkgs.amazon-ec2-utils ];
# Force getting the hostname from EC2.
networking.hostName = mkDefault "";
# Always include cryptsetup so that Charon can use it.
environment.systemPackages = [ pkgs.cryptsetup ];
boot.initrd.supportedFilesystems = [ "unionfs-fuse" ];
# EC2 has its own NTP server provided by the hypervisor
networking.timeServers = [ "169.254.169.123" ];
# udisks has become too bloated to have in a headless system
# (e.g. it depends on GTK).
services.udisks2.enable = false;
};
}
|