pyrox.dev
1# Configuration for Amazon EC2 instances. (Note that this file is a
2# misnomer - it should be "amazon-config.nix" or so, not
3# "amazon-image.nix", since it's used not only to build images but
4# also to reconfigure instances. However, we can't rename it because
5# existing "configuration.nix" files on EC2 instances refer to it.)
6
7{ config, lib, pkgs, ... }:
8
9with lib;
10
11let
12 cfg = config.ec2;
13 metadataFetcher = import ./ec2-metadata-fetcher.nix {
14 inherit (pkgs) curl;
15 targetRoot = "$targetRoot/";
16 wgetExtraOptions = "-q";
17 };
18in
19
20{
21 imports = [
22 ../profiles/headless.nix
23 # Note: While we do use the headless profile, we also explicitly
24 # turn on the serial console on ttyS0 below. This is because
25 # AWS does support accessing the serial console:
26 # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configure-access-to-serial-console.html
27 ./ec2-data.nix
28 ./amazon-init.nix
29 ];
30
31 config = {
32
33 assertions = [
34 { assertion = cfg.hvm;
35 message = "Paravirtualized EC2 instances are no longer supported.";
36 }
37 { assertion = cfg.efi -> cfg.hvm;
38 message = "EC2 instances using EFI must be HVM instances.";
39 }
40 ];
41
42 boot.growPartition = cfg.hvm;
43
44 fileSystems."/" = mkIf (!cfg.zfs.enable) {
45 device = "/dev/disk/by-label/nixos";
46 fsType = "ext4";
47 autoResize = true;
48 };
49
50 fileSystems."/boot" = mkIf (cfg.efi || cfg.zfs.enable) {
51 # The ZFS image uses a partition labeled ESP whether or not we're
52 # booting with EFI.
53 device = "/dev/disk/by-label/ESP";
54 fsType = "vfat";
55 };
56
57 services.zfs.expandOnBoot = mkIf cfg.zfs.enable "all";
58
59 boot.zfs.devNodes = mkIf cfg.zfs.enable "/dev/";
60
61 boot.extraModulePackages = [
62 config.boot.kernelPackages.ena
63 ];
64 boot.initrd.kernelModules = [ "xen-blkfront" "xen-netfront" ];
65 boot.initrd.availableKernelModules = [ "ixgbevf" "ena" "nvme" ];
66 boot.kernelParams = mkIf cfg.hvm [ "console=ttyS0,115200n8" "random.trust_cpu=on" ];
67
68 # Prevent the nouveau kernel module from being loaded, as it
69 # interferes with the nvidia/nvidia-uvm modules needed for CUDA.
70 # Also blacklist xen_fbfront to prevent a 30 second delay during
71 # boot.
72 boot.blacklistedKernelModules = [ "nouveau" "xen_fbfront" ];
73
74 # Generate a GRUB menu. Amazon's pv-grub uses this to boot our kernel/initrd.
75 boot.loader.grub.version = if cfg.hvm then 2 else 1;
76 boot.loader.grub.device = if (cfg.hvm && !cfg.efi) then "/dev/xvda" else "nodev";
77 boot.loader.grub.extraPerEntryConfig = mkIf (!cfg.hvm) "root (hd0)";
78 boot.loader.grub.efiSupport = cfg.efi;
79 boot.loader.grub.efiInstallAsRemovable = cfg.efi;
80 boot.loader.timeout = 1;
81 boot.loader.grub.extraConfig = ''
82 serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1
83 terminal_output console serial
84 terminal_input console serial
85 '';
86
87 boot.initrd.network.enable = true;
88
89 # Mount all formatted ephemeral disks and activate all swap devices.
90 # We cannot do this with the ‘fileSystems’ and ‘swapDevices’ options
91 # because the set of devices is dependent on the instance type
92 # (e.g. "m1.small" has one ephemeral filesystem and one swap device,
93 # while "m1.large" has two ephemeral filesystems and no swap
94 # devices). Also, put /tmp and /var on /disk0, since it has a lot
95 # more space than the root device. Similarly, "move" /nix to /disk0
96 # by layering a unionfs-fuse mount on top of it so we have a lot more space for
97 # Nix operations.
98 boot.initrd.postMountCommands =
99 ''
100 ${metadataFetcher}
101
102 diskNr=0
103 diskForUnionfs=
104 for device in /dev/xvd[abcde]*; do
105 if [ "$device" = /dev/xvda -o "$device" = /dev/xvda1 ]; then continue; fi
106 fsType=$(blkid -o value -s TYPE "$device" || true)
107 if [ "$fsType" = swap ]; then
108 echo "activating swap device $device..."
109 swapon "$device" || true
110 elif [ "$fsType" = ext3 ]; then
111 mp="/disk$diskNr"
112 diskNr=$((diskNr + 1))
113 if mountFS "$device" "$mp" "" ext3; then
114 if [ -z "$diskForUnionfs" ]; then diskForUnionfs="$mp"; fi
115 fi
116 else
117 echo "skipping unknown device type $device"
118 fi
119 done
120
121 if [ -n "$diskForUnionfs" ]; then
122 mkdir -m 755 -p $targetRoot/$diskForUnionfs/root
123
124 mkdir -m 1777 -p $targetRoot/$diskForUnionfs/root/tmp $targetRoot/tmp
125 mount --bind $targetRoot/$diskForUnionfs/root/tmp $targetRoot/tmp
126
127 if [ "$(cat "$metaDir/ami-manifest-path")" != "(unknown)" ]; then
128 mkdir -m 755 -p $targetRoot/$diskForUnionfs/root/var $targetRoot/var
129 mount --bind $targetRoot/$diskForUnionfs/root/var $targetRoot/var
130
131 mkdir -p /unionfs-chroot/ro-nix
132 mount --rbind $targetRoot/nix /unionfs-chroot/ro-nix
133
134 mkdir -m 755 -p $targetRoot/$diskForUnionfs/root/nix
135 mkdir -p /unionfs-chroot/rw-nix
136 mount --rbind $targetRoot/$diskForUnionfs/root/nix /unionfs-chroot/rw-nix
137
138 unionfs -o allow_other,cow,nonempty,chroot=/unionfs-chroot,max_files=32768 /rw-nix=RW:/ro-nix=RO $targetRoot/nix
139 fi
140 fi
141 '';
142
143 boot.initrd.extraUtilsCommands =
144 ''
145 # We need swapon in the initrd.
146 copy_bin_and_libs ${pkgs.util-linux}/sbin/swapon
147 '';
148
149 # Allow root logins only using the SSH key that the user specified
150 # at instance creation time.
151 services.openssh.enable = true;
152 services.openssh.permitRootLogin = "prohibit-password";
153
154 # Enable the serial console on ttyS0
155 systemd.services."serial-getty@ttyS0".enable = true;
156
157 # Creates symlinks for block device names.
158 services.udev.packages = [ pkgs.ec2-utils ];
159
160 # Force getting the hostname from EC2.
161 networking.hostName = mkDefault "";
162
163 # Always include cryptsetup so that Charon can use it.
164 environment.systemPackages = [ pkgs.cryptsetup ];
165
166 boot.initrd.supportedFilesystems = [ "unionfs-fuse" ];
167
168 # EC2 has its own NTP server provided by the hypervisor
169 networking.timeServers = [ "169.254.169.123" ];
170
171 # udisks has become too bloated to have in a headless system
172 # (e.g. it depends on GTK).
173 services.udisks2.enable = false;
174 };
175}