pyrox.dev
1# Configuration for Amazon EC2 instances. (Note that this file is a
2# misnomer - it should be "amazon-config.nix" or so, not
3# "amazon-image.nix", since it's used not only to build images but
4# also to reconfigure instances. However, we can't rename it because
5# existing "configuration.nix" files on EC2 instances refer to it.)
6
7{ config, lib, pkgs, ... }:
8
9with lib;
10
11let cfg = config.ec2; in
12
13{
14 imports = [ ../profiles/headless.nix ./ec2-data.nix ./amazon-init.nix ];
15
16 config = {
17
18 assertions = [
19 { assertion = cfg.hvm;
20 message = "Paravirtualized EC2 instances are no longer supported.";
21 }
22 ];
23
24 boot.growPartition = cfg.hvm;
25
26 fileSystems."/" = {
27 device = "/dev/disk/by-label/nixos";
28 autoResize = true;
29 };
30
31 boot.extraModulePackages =
32 [ config.boot.kernelPackages.ixgbevf
33 config.boot.kernelPackages.ena
34 ];
35 boot.initrd.kernelModules = [ "xen-blkfront" "xen-netfront" ];
36 boot.initrd.availableKernelModules = [ "ixgbevf" "ena" "nvme" ];
37 boot.kernelParams = mkIf cfg.hvm [ "console=ttyS0" ];
38
39 # Prevent the nouveau kernel module from being loaded, as it
40 # interferes with the nvidia/nvidia-uvm modules needed for CUDA.
41 # Also blacklist xen_fbfront to prevent a 30 second delay during
42 # boot.
43 boot.blacklistedKernelModules = [ "nouveau" "xen_fbfront" ];
44
45 # Generate a GRUB menu. Amazon's pv-grub uses this to boot our kernel/initrd.
46 boot.loader.grub.version = if cfg.hvm then 2 else 1;
47 boot.loader.grub.device = if cfg.hvm then "/dev/xvda" else "nodev";
48 boot.loader.grub.extraPerEntryConfig = mkIf (!cfg.hvm) "root (hd0)";
49 boot.loader.timeout = 0;
50
51 boot.initrd.postDeviceCommands =
52 ''
53 # Force udev to exit to prevent random "Device or resource busy
54 # while trying to open /dev/xvda" errors from fsck.
55 udevadm control --exit || true
56 '';
57
58 boot.initrd.network.enable = true;
59
60 # Mount all formatted ephemeral disks and activate all swap devices.
61 # We cannot do this with the ‘fileSystems’ and ‘swapDevices’ options
62 # because the set of devices is dependent on the instance type
63 # (e.g. "m1.large" has one ephemeral filesystem and one swap device,
64 # while "m1.large" has two ephemeral filesystems and no swap
65 # devices). Also, put /tmp and /var on /disk0, since it has a lot
66 # more space than the root device. Similarly, "move" /nix to /disk0
67 # by layering a unionfs-fuse mount on top of it so we have a lot more space for
68 # Nix operations.
69 boot.initrd.postMountCommands =
70 ''
71 metaDir=$targetRoot/etc/ec2-metadata
72 mkdir -m 0755 -p "$metaDir"
73
74 echo "getting EC2 instance metadata..."
75
76 if ! [ -e "$metaDir/ami-manifest-path" ]; then
77 wget -q -O "$metaDir/ami-manifest-path" http://169.254.169.254/1.0/meta-data/ami-manifest-path
78 fi
79
80 if ! [ -e "$metaDir/user-data" ]; then
81 wget -q -O "$metaDir/user-data" http://169.254.169.254/1.0/user-data && chmod 600 "$metaDir/user-data"
82 fi
83
84 if ! [ -e "$metaDir/hostname" ]; then
85 wget -q -O "$metaDir/hostname" http://169.254.169.254/1.0/meta-data/hostname
86 fi
87
88 if ! [ -e "$metaDir/public-keys-0-openssh-key" ]; then
89 wget -q -O "$metaDir/public-keys-0-openssh-key" http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key
90 fi
91
92 diskNr=0
93 diskForUnionfs=
94 for device in /dev/xvd[abcde]*; do
95 if [ "$device" = /dev/xvda -o "$device" = /dev/xvda1 ]; then continue; fi
96 fsType=$(blkid -o value -s TYPE "$device" || true)
97 if [ "$fsType" = swap ]; then
98 echo "activating swap device $device..."
99 swapon "$device" || true
100 elif [ "$fsType" = ext3 ]; then
101 mp="/disk$diskNr"
102 diskNr=$((diskNr + 1))
103 if mountFS "$device" "$mp" "" ext3; then
104 if [ -z "$diskForUnionfs" ]; then diskForUnionfs="$mp"; fi
105 fi
106 else
107 echo "skipping unknown device type $device"
108 fi
109 done
110
111 if [ -n "$diskForUnionfs" ]; then
112 mkdir -m 755 -p $targetRoot/$diskForUnionfs/root
113
114 mkdir -m 1777 -p $targetRoot/$diskForUnionfs/root/tmp $targetRoot/tmp
115 mount --bind $targetRoot/$diskForUnionfs/root/tmp $targetRoot/tmp
116
117 if [ "$(cat "$metaDir/ami-manifest-path")" != "(unknown)" ]; then
118 mkdir -m 755 -p $targetRoot/$diskForUnionfs/root/var $targetRoot/var
119 mount --bind $targetRoot/$diskForUnionfs/root/var $targetRoot/var
120
121 mkdir -p /unionfs-chroot/ro-nix
122 mount --rbind $targetRoot/nix /unionfs-chroot/ro-nix
123
124 mkdir -m 755 -p $targetRoot/$diskForUnionfs/root/nix
125 mkdir -p /unionfs-chroot/rw-nix
126 mount --rbind $targetRoot/$diskForUnionfs/root/nix /unionfs-chroot/rw-nix
127
128 unionfs -o allow_other,cow,nonempty,chroot=/unionfs-chroot,max_files=32768 /rw-nix=RW:/ro-nix=RO $targetRoot/nix
129 fi
130 fi
131 '';
132
133 boot.initrd.extraUtilsCommands =
134 ''
135 # We need swapon in the initrd.
136 copy_bin_and_libs ${pkgs.utillinux}/sbin/swapon
137 '';
138
139 # Don't put old configurations in the GRUB menu. The user has no
140 # way to select them anyway.
141 boot.loader.grub.configurationLimit = 0;
142
143 # Allow root logins only using the SSH key that the user specified
144 # at instance creation time.
145 services.openssh.enable = true;
146 services.openssh.permitRootLogin = "prohibit-password";
147
148 # Force getting the hostname from EC2.
149 networking.hostName = mkDefault "";
150
151 # Always include cryptsetup so that Charon can use it.
152 environment.systemPackages = [ pkgs.cryptsetup ];
153
154 boot.initrd.supportedFilesystems = [ "unionfs-fuse" ];
155
156 # EC2 has its own NTP server provided by the hypervisor
157 networking.timeServers = [ "169.254.169.123" ];
158 };
159}