pyrox.dev
1/*
2 This profile uses NixOS to create a remote builder VM to build Linux packages,
3 which can be used to build packages for Linux on other operating systems;
4 primarily macOS.
5
6 It contains both the relevant guest settings as well as an installer script
7 that manages it as a QEMU virtual machine on the host.
8*/
9{
10 config,
11 lib,
12 options,
13 ...
14}:
15
16let
17 keysDirectory = "/var/keys";
18
19 user = "builder";
20
21 keyType = "ed25519";
22
23 cfg = config.virtualisation.darwin-builder;
24
25in
26
27{
28 imports = [
29 ../virtualisation/qemu-vm.nix
30
31 # Avoid a dependency on stateVersion
32 {
33 disabledModules = [
34 ../virtualisation/nixos-containers.nix
35 ../services/x11/desktop-managers/xterm.nix
36 ];
37 # swraid's default depends on stateVersion
38 config.boot.swraid.enable = false;
39 options.boot.isContainer = lib.mkOption {
40 default = false;
41 internal = true;
42 };
43 }
44 ];
45
46 options.virtualisation.darwin-builder = with lib; {
47 diskSize = mkOption {
48 default = 20 * 1024;
49 type = types.int;
50 example = 30720;
51 description = "The maximum disk space allocated to the runner in MiB (1024×1024 bytes).";
52 };
53 memorySize = mkOption {
54 default = 3 * 1024;
55 type = types.int;
56 example = 8192;
57 description = "The runner's memory in MiB (1024×1024 bytes).";
58 };
59 min-free = mkOption {
60 default = 1024 * 1024 * 1024;
61 type = types.int;
62 example = 1073741824;
63 description = ''
64 The threshold (in bytes) of free disk space left at which to
65 start garbage collection on the runner
66 '';
67 };
68 max-free = mkOption {
69 default = 3 * 1024 * 1024 * 1024;
70 type = types.int;
71 example = 3221225472;
72 description = ''
73 The threshold (in bytes) of free disk space left at which to
74 stop garbage collection on the runner
75 '';
76 };
77 workingDirectory = mkOption {
78 default = ".";
79 type = types.str;
80 example = "/var/lib/darwin-builder";
81 description = ''
82 The working directory to use to run the script. When running
83 as part of a flake will need to be set to a non read-only filesystem.
84 '';
85 };
86 hostPort = mkOption {
87 default = 31022;
88 type = types.port;
89 example = 22;
90 description = ''
91 The localhost host port to forward TCP to the guest port.
92 '';
93 };
94 };
95
96 config = {
97 # The builder is not intended to be used interactively
98 documentation.enable = false;
99
100 environment.etc = {
101 "ssh/ssh_host_ed25519_key" = {
102 mode = "0600";
103
104 source = ./keys/ssh_host_ed25519_key;
105 };
106
107 "ssh/ssh_host_ed25519_key.pub" = {
108 mode = "0644";
109
110 source = ./keys/ssh_host_ed25519_key.pub;
111 };
112 };
113
114 # DNS fails for QEMU user networking (SLiRP) on macOS. See:
115 #
116 # https://github.com/utmapp/UTM/issues/2353
117 #
118 # This works around that by using a public DNS server other than the DNS
119 # server that QEMU provides (normally 10.0.2.3)
120 networking.nameservers = [ "8.8.8.8" ];
121
122 # The linux builder is a lightweight VM for remote building; not evaluation.
123 nix.channel.enable = false;
124
125 # Deployment is by image.
126 # TODO system.switch.enable = false;?
127 system.disableInstallerTools = true;
128
129 # Allow the system derivation to be substituted, so that
130 # users are less likely to run into a state where they need
131 # the builder running to build the builder if they just want
132 # to make a tweak that only affects the macOS side of things,
133 # like changing the QEMU args.
134 #
135 # TODO(winter): Move to qemu-vm? Trying it here for now as a
136 # low impact change that'll probably improve people's experience.
137 #
138 # (I have no clue what is going on in https://github.com/nix-darwin/nix-darwin/issues/1081
139 # though, as this fix would only apply to one person in that thread... hopefully someone
140 # comes across with a reproducer if this doesn't do it.)
141 system.systemBuilderArgs.allowSubstitutes = true;
142
143 nix.settings = {
144 min-free = cfg.min-free;
145
146 max-free = cfg.max-free;
147
148 trusted-users = [ user ];
149 };
150
151 services = {
152 getty.autologinUser = user;
153
154 openssh = {
155 enable = true;
156
157 authorizedKeysFiles = [ "${keysDirectory}/%u_${keyType}.pub" ];
158 };
159 };
160
161 system.build.macos-builder-installer =
162 let
163 privateKey = "/etc/nix/${user}_${keyType}";
164
165 publicKey = "${privateKey}.pub";
166
167 # This installCredentials script is written so that it's as easy as
168 # possible for a user to audit before confirming the `sudo`
169 installCredentials = hostPkgs.writeShellScript "install-credentials" ''
170 set -euo pipefail
171
172 KEYS="''${1}"
173 INSTALL=${hostPkgs.coreutils}/bin/install
174 "''${INSTALL}" -g nixbld -m 600 "''${KEYS}/${user}_${keyType}" ${privateKey}
175 "''${INSTALL}" -g nixbld -m 644 "''${KEYS}/${user}_${keyType}.pub" ${publicKey}
176 '';
177
178 hostPkgs = config.virtualisation.host.pkgs;
179
180 add-keys = hostPkgs.writeShellScriptBin "add-keys" (
181 ''
182 set -euo pipefail
183 ''
184 +
185 # When running as non-interactively as part of a DarwinConfiguration the working directory
186 # must be set to a writeable directory.
187 (
188 if cfg.workingDirectory != "." then
189 ''
190 ${hostPkgs.coreutils}/bin/mkdir --parent "${cfg.workingDirectory}"
191 cd "${cfg.workingDirectory}"
192 ''
193 else
194 ""
195 )
196 + ''
197 KEYS="''${KEYS:-./keys}"
198 ${hostPkgs.coreutils}/bin/mkdir --parent "''${KEYS}"
199 PRIVATE_KEY="''${KEYS}/${user}_${keyType}"
200 PUBLIC_KEY="''${PRIVATE_KEY}.pub"
201 if [ ! -e "''${PRIVATE_KEY}" ] || [ ! -e "''${PUBLIC_KEY}" ]; then
202 ${hostPkgs.coreutils}/bin/rm --force -- "''${PRIVATE_KEY}" "''${PUBLIC_KEY}"
203 ${hostPkgs.openssh}/bin/ssh-keygen -q -f "''${PRIVATE_KEY}" -t ${keyType} -N "" -C 'builder@localhost'
204 fi
205 if ! ${hostPkgs.diffutils}/bin/cmp "''${PUBLIC_KEY}" ${publicKey}; then
206 (set -x; sudo --reset-timestamp ${installCredentials} "''${KEYS}")
207 fi
208 ''
209 );
210
211 run-builder = hostPkgs.writeShellScriptBin "run-builder" (''
212 set -euo pipefail
213 KEYS="''${KEYS:-./keys}"
214 KEYS="$(${hostPkgs.nix}/bin/nix-store --add "$KEYS")" ${lib.getExe config.system.build.vm}
215 '');
216
217 script = hostPkgs.writeShellScriptBin "create-builder" (''
218 set -euo pipefail
219 export KEYS="''${KEYS:-./keys}"
220 ${lib.getExe add-keys}
221 ${lib.getExe run-builder}
222 '');
223
224 in
225 script.overrideAttrs (old: {
226 pos = __curPos; # sets meta.position to point here; see script binding above for package definition
227 meta = (old.meta or { }) // {
228 platforms = lib.platforms.darwin;
229 };
230 passthru = (old.passthru or { }) // {
231 # Let users in the repl inspect the config
232 nixosConfig = config;
233 nixosOptions = options;
234
235 inherit add-keys run-builder;
236 };
237 });
238
239 system = {
240 # To prevent gratuitous rebuilds on each change to Nixpkgs
241 nixos.revision = null;
242
243 # to be updated by module maintainers, see nixpkgs#325610
244 stateVersion = "24.05";
245 };
246
247 users.users."${user}" = {
248 isNormalUser = true;
249 };
250
251 security.polkit.enable = true;
252
253 security.polkit.extraConfig = ''
254 polkit.addRule(function(action, subject) {
255 if (action.id === "org.freedesktop.login1.power-off" && subject.user === "${user}") {
256 return "yes";
257 } else {
258 return "no";
259 }
260 })
261 '';
262
263 virtualisation = {
264 diskSize = cfg.diskSize;
265
266 memorySize = cfg.memorySize;
267
268 forwardPorts = [
269 {
270 from = "host";
271 guest.port = 22;
272 host.port = cfg.hostPort;
273 }
274 ];
275
276 # Disable graphics for the builder since users will likely want to run it
277 # non-interactively in the background.
278 graphics = false;
279
280 sharedDirectories.keys = {
281 source = "\"$KEYS\"";
282 target = keysDirectory;
283 };
284
285 # If we don't enable this option then the host will fail to delegate builds
286 # to the guest, because:
287 #
288 # - The host will lock the path to build
289 # - The host will delegate the build to the guest
290 # - The guest will attempt to lock the same path and fail because
291 # the lockfile on the host is visible on the guest
292 #
293 # Snapshotting the host's /nix/store as an image isolates the guest VM's
294 # /nix/store from the host's /nix/store, preventing this problem.
295 useNixStoreImage = true;
296
297 # Obviously the /nix/store needs to be writable on the guest in order for it
298 # to perform builds.
299 writableStore = true;
300
301 # This ensures that anything built on the guest isn't lost when the guest is
302 # restarted.
303 writableStoreUseTmpfs = false;
304
305 # Pass certificates from host to the guest otherwise when custom CA certificates
306 # are required we can't use the cached builder.
307 useHostCerts = true;
308 };
309 };
310}