pyrox.dev
1{ config, lib, options, ... }:
2
3let
4 keysDirectory = "/var/keys";
5
6 user = "builder";
7
8 keyType = "ed25519";
9
10 cfg = config.virtualisation.darwin-builder;
11
12in
13
14{
15 imports = [
16 ../virtualisation/qemu-vm.nix
17
18 # Avoid a dependency on stateVersion
19 {
20 disabledModules = [
21 ../virtualisation/nixos-containers.nix
22 ../services/x11/desktop-managers/xterm.nix
23 ];
24 # swraid's default depends on stateVersion
25 config.boot.swraid.enable = false;
26 options.boot.isContainer = lib.mkOption { default = false; internal = true; };
27 }
28 ];
29
30 options.virtualisation.darwin-builder = with lib; {
31 diskSize = mkOption {
32 default = 20 * 1024;
33 type = types.int;
34 example = 30720;
35 description = "The maximum disk space allocated to the runner in MB";
36 };
37 memorySize = mkOption {
38 default = 3 * 1024;
39 type = types.int;
40 example = 8192;
41 description = "The runner's memory in MB";
42 };
43 min-free = mkOption {
44 default = 1024 * 1024 * 1024;
45 type = types.int;
46 example = 1073741824;
47 description = ''
48 The threshold (in bytes) of free disk space left at which to
49 start garbage collection on the runner
50 '';
51 };
52 max-free = mkOption {
53 default = 3 * 1024 * 1024 * 1024;
54 type = types.int;
55 example = 3221225472;
56 description = ''
57 The threshold (in bytes) of free disk space left at which to
58 stop garbage collection on the runner
59 '';
60 };
61 workingDirectory = mkOption {
62 default = ".";
63 type = types.str;
64 example = "/var/lib/darwin-builder";
65 description = ''
66 The working directory to use to run the script. When running
67 as part of a flake will need to be set to a non read-only filesystem.
68 '';
69 };
70 hostPort = mkOption {
71 default = 31022;
72 type = types.int;
73 example = 22;
74 description = ''
75 The localhost host port to forward TCP to the guest port.
76 '';
77 };
78 };
79
80 config = {
81 # The builder is not intended to be used interactively
82 documentation.enable = false;
83
84 environment.etc = {
85 "ssh/ssh_host_ed25519_key" = {
86 mode = "0600";
87
88 source = ./keys/ssh_host_ed25519_key;
89 };
90
91 "ssh/ssh_host_ed25519_key.pub" = {
92 mode = "0644";
93
94 source = ./keys/ssh_host_ed25519_key.pub;
95 };
96 };
97
98 # DNS fails for QEMU user networking (SLiRP) on macOS. See:
99 #
100 # https://github.com/utmapp/UTM/issues/2353
101 #
102 # This works around that by using a public DNS server other than the DNS
103 # server that QEMU provides (normally 10.0.2.3)
104 networking.nameservers = [ "8.8.8.8" ];
105
106 # The linux builder is a lightweight VM for remote building; not evaluation.
107 nix.channel.enable = false;
108 # remote builder uses `nix-daemon` (ssh-ng:) or `nix-store --serve` (ssh:)
109 # --force: do not complain when missing
110 # TODO: install a store-only nix
111 # https://github.com/NixOS/rfcs/blob/master/rfcs/0134-nix-store-layer.md#detailed-design
112 environment.extraSetup = ''
113 rm --force $out/bin/{nix-instantiate,nix-build,nix-shell,nix-prefetch*,nix}
114 '';
115 # Deployment is by image.
116 # TODO system.switch.enable = false;?
117 system.disableInstallerTools = true;
118
119 nix.settings = {
120 auto-optimise-store = true;
121
122 min-free = cfg.min-free;
123
124 max-free = cfg.max-free;
125
126 trusted-users = [ "root" user ];
127 };
128
129 services = {
130 getty.autologinUser = user;
131
132 openssh = {
133 enable = true;
134
135 authorizedKeysFiles = [ "${keysDirectory}/%u_${keyType}.pub" ];
136 };
137 };
138
139 system.build.macos-builder-installer =
140 let
141 privateKey = "/etc/nix/${user}_${keyType}";
142
143 publicKey = "${privateKey}.pub";
144
145 # This installCredentials script is written so that it's as easy as
146 # possible for a user to audit before confirming the `sudo`
147 installCredentials = hostPkgs.writeShellScript "install-credentials" ''
148 set -euo pipefail
149
150 KEYS="''${1}"
151 INSTALL=${hostPkgs.coreutils}/bin/install
152 "''${INSTALL}" -g nixbld -m 600 "''${KEYS}/${user}_${keyType}" ${privateKey}
153 "''${INSTALL}" -g nixbld -m 644 "''${KEYS}/${user}_${keyType}.pub" ${publicKey}
154 '';
155
156 hostPkgs = config.virtualisation.host.pkgs;
157
158 script = hostPkgs.writeShellScriptBin "create-builder" (
159 ''
160 set -euo pipefail
161 '' +
162 # When running as non-interactively as part of a DarwinConfiguration the working directory
163 # must be set to a writeable directory.
164 (if cfg.workingDirectory != "." then ''
165 ${hostPkgs.coreutils}/bin/mkdir --parent "${cfg.workingDirectory}"
166 cd "${cfg.workingDirectory}"
167 '' else "") + ''
168 KEYS="''${KEYS:-./keys}"
169 ${hostPkgs.coreutils}/bin/mkdir --parent "''${KEYS}"
170 PRIVATE_KEY="''${KEYS}/${user}_${keyType}"
171 PUBLIC_KEY="''${PRIVATE_KEY}.pub"
172 if [ ! -e "''${PRIVATE_KEY}" ] || [ ! -e "''${PUBLIC_KEY}" ]; then
173 ${hostPkgs.coreutils}/bin/rm --force -- "''${PRIVATE_KEY}" "''${PUBLIC_KEY}"
174 ${hostPkgs.openssh}/bin/ssh-keygen -q -f "''${PRIVATE_KEY}" -t ${keyType} -N "" -C 'builder@localhost'
175 fi
176 if ! ${hostPkgs.diffutils}/bin/cmp "''${PUBLIC_KEY}" ${publicKey}; then
177 (set -x; sudo --reset-timestamp ${installCredentials} "''${KEYS}")
178 fi
179 KEYS="$(${hostPkgs.nix}/bin/nix-store --add "$KEYS")" ${lib.getExe config.system.build.vm}
180 '');
181
182 in
183 script.overrideAttrs (old: {
184 pos = __curPos; # sets meta.position to point here; see script binding above for package definition
185 meta = (old.meta or { }) // {
186 platforms = lib.platforms.darwin;
187 };
188 passthru = (old.passthru or { }) // {
189 # Let users in the repl inspect the config
190 nixosConfig = config;
191 nixosOptions = options;
192 };
193 });
194
195 system = {
196 # To prevent gratuitous rebuilds on each change to Nixpkgs
197 nixos.revision = null;
198
199 stateVersion = lib.mkDefault (throw ''
200 The macOS linux builder should not need a stateVersion to be set, but a module
201 has accessed stateVersion nonetheless.
202 Please inspect the trace of the following command to figure out which module
203 has a dependency on stateVersion.
204
205 nix-instantiate --attr darwin.linux-builder --show-trace
206 '');
207 };
208
209 users.users."${user}" = {
210 isNormalUser = true;
211 };
212
213 security.polkit.enable = true;
214
215 security.polkit.extraConfig = ''
216 polkit.addRule(function(action, subject) {
217 if (action.id === "org.freedesktop.login1.power-off" && subject.user === "${user}") {
218 return "yes";
219 } else {
220 return "no";
221 }
222 })
223 '';
224
225 virtualisation = {
226 diskSize = cfg.diskSize;
227
228 memorySize = cfg.memorySize;
229
230 forwardPorts = [
231 { from = "host"; guest.port = 22; host.port = cfg.hostPort; }
232 ];
233
234 # Disable graphics for the builder since users will likely want to run it
235 # non-interactively in the background.
236 graphics = false;
237
238 sharedDirectories.keys = {
239 source = "\"$KEYS\"";
240 target = keysDirectory;
241 };
242
243 # If we don't enable this option then the host will fail to delegate builds
244 # to the guest, because:
245 #
246 # - The host will lock the path to build
247 # - The host will delegate the build to the guest
248 # - The guest will attempt to lock the same path and fail because
249 # the lockfile on the host is visible on the guest
250 #
251 # Snapshotting the host's /nix/store as an image isolates the guest VM's
252 # /nix/store from the host's /nix/store, preventing this problem.
253 useNixStoreImage = true;
254
255 # Obviously the /nix/store needs to be writable on the guest in order for it
256 # to perform builds.
257 writableStore = true;
258
259 # This ensures that anything built on the guest isn't lost when the guest is
260 # restarted.
261 writableStoreUseTmpfs = false;
262
263 # Pass certificates from host to the guest otherwise when custom CA certificates
264 # are required we can't use the cached builder.
265 useHostCerts = true;
266 };
267 };
268}