pyrox.dev
1# Configuration for Amazon EC2 instances. (Note that this file is a
2# misnomer - it should be "amazon-config.nix" or so, not
3# "amazon-image.nix", since it's used not only to build images but
4# also to reconfigure instances. However, we can't rename it because
5# existing "configuration.nix" files on EC2 instances refer to it.)
6
7{ config, lib, pkgs, ... }:
8
9with lib;
10
11let
12 cfg = config.ec2;
13 metadataFetcher = import ./ec2-metadata-fetcher.nix {
14 inherit (pkgs) curl;
15 targetRoot = "$targetRoot/";
16 wgetExtraOptions = "-q";
17 };
18in
19
20{
21 imports = [
22 ../profiles/headless.nix
23 # Note: While we do use the headless profile, we also explicitly
24 # turn on the serial console on ttyS0 below. This is because
25 # AWS does support accessing the serial console:
26 # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configure-access-to-serial-console.html
27 ./ec2-data.nix
28 ./amazon-init.nix
29 ];
30
31 config = {
32
33 assertions = [
34 { assertion = cfg.hvm;
35 message = "Paravirtualized EC2 instances are no longer supported.";
36 }
37 { assertion = cfg.efi -> cfg.hvm;
38 message = "EC2 instances using EFI must be HVM instances.";
39 }
40 { assertion = versionOlder config.boot.kernelPackages.kernel.version "5.17";
41 message = "ENA driver fails to build with kernel >= 5.17";
42 }
43 ];
44
45 boot.growPartition = cfg.hvm;
46
47 fileSystems."/" = mkIf (!cfg.zfs.enable) {
48 device = "/dev/disk/by-label/nixos";
49 fsType = "ext4";
50 autoResize = true;
51 };
52
53 fileSystems."/boot" = mkIf (cfg.efi || cfg.zfs.enable) {
54 # The ZFS image uses a partition labeled ESP whether or not we're
55 # booting with EFI.
56 device = "/dev/disk/by-label/ESP";
57 fsType = "vfat";
58 };
59
60 services.zfs.expandOnBoot = mkIf cfg.zfs.enable "all";
61
62 boot.zfs.devNodes = mkIf cfg.zfs.enable "/dev/";
63
64 boot.extraModulePackages = [
65 config.boot.kernelPackages.ena
66 ];
67 boot.initrd.kernelModules = [ "xen-blkfront" "xen-netfront" ];
68 boot.initrd.availableKernelModules = [ "ixgbevf" "ena" "nvme" ];
69 boot.kernelParams = mkIf cfg.hvm [ "console=ttyS0,115200n8" "random.trust_cpu=on" ];
70
71 # Prevent the nouveau kernel module from being loaded, as it
72 # interferes with the nvidia/nvidia-uvm modules needed for CUDA.
73 # Also blacklist xen_fbfront to prevent a 30 second delay during
74 # boot.
75 boot.blacklistedKernelModules = [ "nouveau" "xen_fbfront" ];
76
77 # Generate a GRUB menu. Amazon's pv-grub uses this to boot our kernel/initrd.
78 boot.loader.grub.version = if cfg.hvm then 2 else 1;
79 boot.loader.grub.device = if (cfg.hvm && !cfg.efi) then "/dev/xvda" else "nodev";
80 boot.loader.grub.extraPerEntryConfig = mkIf (!cfg.hvm) "root (hd0)";
81 boot.loader.grub.efiSupport = cfg.efi;
82 boot.loader.grub.efiInstallAsRemovable = cfg.efi;
83 boot.loader.timeout = 1;
84 boot.loader.grub.extraConfig = ''
85 serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1
86 terminal_output console serial
87 terminal_input console serial
88 '';
89
90 boot.initrd.network.enable = true;
91
92 # Mount all formatted ephemeral disks and activate all swap devices.
93 # We cannot do this with the ‘fileSystems’ and ‘swapDevices’ options
94 # because the set of devices is dependent on the instance type
95 # (e.g. "m1.small" has one ephemeral filesystem and one swap device,
96 # while "m1.large" has two ephemeral filesystems and no swap
97 # devices). Also, put /tmp and /var on /disk0, since it has a lot
98 # more space than the root device. Similarly, "move" /nix to /disk0
99 # by layering a unionfs-fuse mount on top of it so we have a lot more space for
100 # Nix operations.
101 boot.initrd.postMountCommands =
102 ''
103 ${metadataFetcher}
104
105 diskNr=0
106 diskForUnionfs=
107 for device in /dev/xvd[abcde]*; do
108 if [ "$device" = /dev/xvda -o "$device" = /dev/xvda1 ]; then continue; fi
109 fsType=$(blkid -o value -s TYPE "$device" || true)
110 if [ "$fsType" = swap ]; then
111 echo "activating swap device $device..."
112 swapon "$device" || true
113 elif [ "$fsType" = ext3 ]; then
114 mp="/disk$diskNr"
115 diskNr=$((diskNr + 1))
116 if mountFS "$device" "$mp" "" ext3; then
117 if [ -z "$diskForUnionfs" ]; then diskForUnionfs="$mp"; fi
118 fi
119 else
120 echo "skipping unknown device type $device"
121 fi
122 done
123
124 if [ -n "$diskForUnionfs" ]; then
125 mkdir -m 755 -p $targetRoot/$diskForUnionfs/root
126
127 mkdir -m 1777 -p $targetRoot/$diskForUnionfs/root/tmp $targetRoot/tmp
128 mount --bind $targetRoot/$diskForUnionfs/root/tmp $targetRoot/tmp
129
130 if [ "$(cat "$metaDir/ami-manifest-path")" != "(unknown)" ]; then
131 mkdir -m 755 -p $targetRoot/$diskForUnionfs/root/var $targetRoot/var
132 mount --bind $targetRoot/$diskForUnionfs/root/var $targetRoot/var
133
134 mkdir -p /unionfs-chroot/ro-nix
135 mount --rbind $targetRoot/nix /unionfs-chroot/ro-nix
136
137 mkdir -m 755 -p $targetRoot/$diskForUnionfs/root/nix
138 mkdir -p /unionfs-chroot/rw-nix
139 mount --rbind $targetRoot/$diskForUnionfs/root/nix /unionfs-chroot/rw-nix
140
141 unionfs -o allow_other,cow,nonempty,chroot=/unionfs-chroot,max_files=32768 /rw-nix=RW:/ro-nix=RO $targetRoot/nix
142 fi
143 fi
144 '';
145
146 boot.initrd.extraUtilsCommands =
147 ''
148 # We need swapon in the initrd.
149 copy_bin_and_libs ${pkgs.util-linux}/sbin/swapon
150 '';
151
152 # Allow root logins only using the SSH key that the user specified
153 # at instance creation time.
154 services.openssh.enable = true;
155 services.openssh.permitRootLogin = "prohibit-password";
156
157 # Enable the serial console on ttyS0
158 systemd.services."serial-getty@ttyS0".enable = true;
159
160 # Creates symlinks for block device names.
161 services.udev.packages = [ pkgs.amazon-ec2-utils ];
162
163 # Force getting the hostname from EC2.
164 networking.hostName = mkDefault "";
165
166 # Always include cryptsetup so that Charon can use it.
167 environment.systemPackages = [ pkgs.cryptsetup ];
168
169 boot.initrd.supportedFilesystems = [ "unionfs-fuse" ];
170
171 # EC2 has its own NTP server provided by the hypervisor
172 networking.timeServers = [ "169.254.169.123" ];
173
174 # udisks has become too bloated to have in a headless system
175 # (e.g. it depends on GTK).
176 services.udisks2.enable = false;
177 };
178}