pyrox.dev
1{ config, lib, pkgs, ... }:
2
3with lib;
4let
5 cfg = config.ec2;
6in
7{
8 imports = [ ../profiles/headless.nix ./ec2-data.nix ];
9
10 config = {
11 system.build.amazonImage =
12 pkgs.vmTools.runInLinuxVM (
13 pkgs.runCommand "amazon-image"
14 { preVM =
15 ''
16 mkdir $out
17 diskImage=$out/nixos.img
18 ${pkgs.vmTools.qemu}/bin/qemu-img create -f raw $diskImage "8G"
19 mv closure xchg/
20 '';
21 buildInputs = [ pkgs.utillinux pkgs.perl ];
22 exportReferencesGraph =
23 [ "closure" config.system.build.toplevel ];
24 }
25 ''
26 ${if cfg.hvm then ''
27 # Create a single / partition.
28 ${pkgs.parted}/sbin/parted /dev/vda mklabel msdos
29 ${pkgs.parted}/sbin/parted /dev/vda -- mkpart primary ext2 1M -1s
30 . /sys/class/block/vda1/uevent
31 mknod /dev/vda1 b $MAJOR $MINOR
32
33 # Create an empty filesystem and mount it.
34 ${pkgs.e2fsprogs}/sbin/mkfs.ext4 -L nixos /dev/vda1
35 ${pkgs.e2fsprogs}/sbin/tune2fs -c 0 -i 0 /dev/vda1
36 mkdir /mnt
37 mount /dev/vda1 /mnt
38 '' else ''
39 # Create an empty filesystem and mount it.
40 ${pkgs.e2fsprogs}/sbin/mkfs.ext4 -L nixos /dev/vda
41 ${pkgs.e2fsprogs}/sbin/tune2fs -c 0 -i 0 /dev/vda
42 mkdir /mnt
43 mount /dev/vda /mnt
44 ''}
45
46 # The initrd expects these directories to exist.
47 mkdir /mnt/dev /mnt/proc /mnt/sys
48
49 mount -o bind /proc /mnt/proc
50 mount -o bind /dev /mnt/dev
51 mount -o bind /sys /mnt/sys
52
53 # Copy all paths in the closure to the filesystem.
54 storePaths=$(perl ${pkgs.pathsFromGraph} /tmp/xchg/closure)
55
56 mkdir -p /mnt/nix/store
57 echo "copying everything (will take a while)..."
58 cp -prd $storePaths /mnt/nix/store/
59
60 # Register the paths in the Nix database.
61 printRegistration=1 perl ${pkgs.pathsFromGraph} /tmp/xchg/closure | \
62 chroot /mnt ${config.nix.package}/bin/nix-store --load-db --option build-users-group ""
63
64 # Create the system profile to allow nixos-rebuild to work.
65 chroot /mnt ${config.nix.package}/bin/nix-env --option build-users-group "" \
66 -p /nix/var/nix/profiles/system --set ${config.system.build.toplevel}
67
68 # `nixos-rebuild' requires an /etc/NIXOS.
69 mkdir -p /mnt/etc
70 touch /mnt/etc/NIXOS
71
72 # `switch-to-configuration' requires a /bin/sh
73 mkdir -p /mnt/bin
74 ln -s ${config.system.build.binsh}/bin/sh /mnt/bin/sh
75
76 # Install a configuration.nix.
77 mkdir -p /mnt/etc/nixos
78 cp ${./amazon-config.nix} /mnt/etc/nixos/configuration.nix
79
80 # Generate the GRUB menu.
81 ln -s vda /dev/xvda
82 chroot /mnt ${config.system.build.toplevel}/bin/switch-to-configuration boot
83
84 umount /mnt/proc /mnt/dev /mnt/sys
85 umount /mnt
86 ''
87 );
88
89 fileSystems."/".device = "/dev/disk/by-label/nixos";
90
91 boot.initrd.kernelModules = [ "xen-blkfront" ];
92 boot.kernelModules = [ "xen-netfront" ];
93
94 # Prevent the nouveau kernel module from being loaded, as it
95 # interferes with the nvidia/nvidia-uvm modules needed for CUDA.
96 boot.blacklistedKernelModules = [ "nouveau" ];
97
98 # Generate a GRUB menu. Amazon's pv-grub uses this to boot our kernel/initrd.
99 boot.loader.grub.version = if cfg.hvm then 2 else 1;
100 boot.loader.grub.device = if cfg.hvm then "/dev/xvda" else "nodev";
101 boot.loader.grub.timeout = 0;
102 boot.loader.grub.extraPerEntryConfig = "root (hd0${lib.optionalString cfg.hvm ",0"})";
103
104 boot.initrd.postDeviceCommands =
105 ''
106 # Force udev to exit to prevent random "Device or resource busy
107 # while trying to open /dev/xvda" errors from fsck.
108 udevadm control --exit || true
109 kill -9 -1
110 '';
111
112 # Mount all formatted ephemeral disks and activate all swap devices.
113 # We cannot do this with the ‘fileSystems’ and ‘swapDevices’ options
114 # because the set of devices is dependent on the instance type
115 # (e.g. "m1.large" has one ephemeral filesystem and one swap device,
116 # while "m1.large" has two ephemeral filesystems and no swap
117 # devices). Also, put /tmp and /var on /disk0, since it has a lot
118 # more space than the root device. Similarly, "move" /nix to /disk0
119 # by layering a unionfs-fuse mount on top of it so we have a lot more space for
120 # Nix operations.
121 boot.initrd.postMountCommands =
122 ''
123 diskNr=0
124 diskForUnionfs=
125 for device in /dev/xvd[abcde]*; do
126 if [ "$device" = /dev/xvda -o "$device" = /dev/xvda1 ]; then continue; fi
127 fsType=$(blkid -o value -s TYPE "$device" || true)
128 if [ "$fsType" = swap ]; then
129 echo "activating swap device $device..."
130 swapon "$device" || true
131 elif [ "$fsType" = ext3 ]; then
132 mp="/disk$diskNr"
133 diskNr=$((diskNr + 1))
134 echo "mounting $device on $mp..."
135 if mountFS "$device" "$mp" "" ext3; then
136 if [ -z "$diskForUnionfs" ]; then diskForUnionfs="$mp"; fi
137 fi
138 else
139 echo "skipping unknown device type $device"
140 fi
141 done
142
143 if [ -n "$diskForUnionfs" ]; then
144 mkdir -m 755 -p $targetRoot/$diskForUnionfs/root
145
146 mkdir -m 1777 -p $targetRoot/$diskForUnionfs/root/tmp $targetRoot/tmp
147 mount --bind $targetRoot/$diskForUnionfs/root/tmp $targetRoot/tmp
148
149 if [ ! -e $targetRoot/.ebs ]; then
150 mkdir -m 755 -p $targetRoot/$diskForUnionfs/root/var $targetRoot/var
151 mount --bind $targetRoot/$diskForUnionfs/root/var $targetRoot/var
152
153 mkdir -p /unionfs-chroot/ro-nix
154 mount --rbind $targetRoot/nix /unionfs-chroot/ro-nix
155
156 mkdir -m 755 -p $targetRoot/$diskForUnionfs/root/nix
157 mkdir -p /unionfs-chroot/rw-nix
158 mount --rbind $targetRoot/$diskForUnionfs/root/nix /unionfs-chroot/rw-nix
159
160 unionfs -o allow_other,cow,nonempty,chroot=/unionfs-chroot,max_files=32768 /rw-nix=RW:/ro-nix=RO $targetRoot/nix
161 fi
162 fi
163 '';
164
165 boot.initrd.extraUtilsCommands =
166 ''
167 # We need swapon in the initrd.
168 copy_bin_and_libs ${pkgs.utillinux}/sbin/swapon
169 '';
170
171 # Don't put old configurations in the GRUB menu. The user has no
172 # way to select them anyway.
173 boot.loader.grub.configurationLimit = 0;
174
175 # Allow root logins only using the SSH key that the user specified
176 # at instance creation time.
177 services.openssh.enable = true;
178 services.openssh.permitRootLogin = "without-password";
179
180 # Force getting the hostname from EC2.
181 networking.hostName = mkDefault "";
182
183 # Always include cryptsetup so that Charon can use it.
184 environment.systemPackages = [ pkgs.cryptsetup ];
185
186 boot.initrd.supportedFilesystems = [ "unionfs-fuse" ];
187 };
188}