pyrox.dev
1/*
2 This profile uses NixOS to create a remote builder VM to build Linux packages,
3 which can be used to build packages for Linux on other operating systems;
4 primarily macOS.
5
6 It contains both the relevant guest settings as well as an installer script
7 that manages it as a QEMU virtual machine on the host.
8*/
9{
10 config,
11 lib,
12 options,
13 ...
14}:
15
16let
17 keysDirectory = "/var/keys";
18
19 user = "builder";
20
21 keyType = "ed25519";
22
23 cfg = config.virtualisation.darwin-builder;
24
25in
26
27{
28 imports = [
29 ../virtualisation/qemu-vm.nix
30
31 # Avoid a dependency on stateVersion
32 {
33 disabledModules = [
34 ../virtualisation/nixos-containers.nix
35 ../services/x11/desktop-managers/xterm.nix
36 ];
37 # swraid's default depends on stateVersion
38 config.boot.swraid.enable = false;
39 options.boot.isContainer = lib.mkOption {
40 default = false;
41 internal = true;
42 };
43 }
44 ];
45
46 options.virtualisation.darwin-builder = with lib; {
47 diskSize = mkOption {
48 default = 20 * 1024;
49 type = types.int;
50 example = 30720;
51 description = "The maximum disk space allocated to the runner in MB";
52 };
53 memorySize = mkOption {
54 default = 3 * 1024;
55 type = types.int;
56 example = 8192;
57 description = "The runner's memory in MB";
58 };
59 min-free = mkOption {
60 default = 1024 * 1024 * 1024;
61 type = types.int;
62 example = 1073741824;
63 description = ''
64 The threshold (in bytes) of free disk space left at which to
65 start garbage collection on the runner
66 '';
67 };
68 max-free = mkOption {
69 default = 3 * 1024 * 1024 * 1024;
70 type = types.int;
71 example = 3221225472;
72 description = ''
73 The threshold (in bytes) of free disk space left at which to
74 stop garbage collection on the runner
75 '';
76 };
77 workingDirectory = mkOption {
78 default = ".";
79 type = types.str;
80 example = "/var/lib/darwin-builder";
81 description = ''
82 The working directory to use to run the script. When running
83 as part of a flake will need to be set to a non read-only filesystem.
84 '';
85 };
86 hostPort = mkOption {
87 default = 31022;
88 type = types.int;
89 example = 22;
90 description = ''
91 The localhost host port to forward TCP to the guest port.
92 '';
93 };
94 };
95
96 config = {
97 # The builder is not intended to be used interactively
98 documentation.enable = false;
99
100 environment.etc = {
101 "ssh/ssh_host_ed25519_key" = {
102 mode = "0600";
103
104 source = ./keys/ssh_host_ed25519_key;
105 };
106
107 "ssh/ssh_host_ed25519_key.pub" = {
108 mode = "0644";
109
110 source = ./keys/ssh_host_ed25519_key.pub;
111 };
112 };
113
114 # DNS fails for QEMU user networking (SLiRP) on macOS. See:
115 #
116 # https://github.com/utmapp/UTM/issues/2353
117 #
118 # This works around that by using a public DNS server other than the DNS
119 # server that QEMU provides (normally 10.0.2.3)
120 networking.nameservers = [ "8.8.8.8" ];
121
122 # The linux builder is a lightweight VM for remote building; not evaluation.
123 nix.channel.enable = false;
124
125 # Deployment is by image.
126 # TODO system.switch.enable = false;?
127 system.disableInstallerTools = true;
128
129 nix.settings = {
130 min-free = cfg.min-free;
131
132 max-free = cfg.max-free;
133
134 trusted-users = [ user ];
135 };
136
137 services = {
138 getty.autologinUser = user;
139
140 openssh = {
141 enable = true;
142
143 authorizedKeysFiles = [ "${keysDirectory}/%u_${keyType}.pub" ];
144 };
145 };
146
147 system.build.macos-builder-installer =
148 let
149 privateKey = "/etc/nix/${user}_${keyType}";
150
151 publicKey = "${privateKey}.pub";
152
153 # This installCredentials script is written so that it's as easy as
154 # possible for a user to audit before confirming the `sudo`
155 installCredentials = hostPkgs.writeShellScript "install-credentials" ''
156 set -euo pipefail
157
158 KEYS="''${1}"
159 INSTALL=${hostPkgs.coreutils}/bin/install
160 "''${INSTALL}" -g nixbld -m 600 "''${KEYS}/${user}_${keyType}" ${privateKey}
161 "''${INSTALL}" -g nixbld -m 644 "''${KEYS}/${user}_${keyType}.pub" ${publicKey}
162 '';
163
164 hostPkgs = config.virtualisation.host.pkgs;
165
166 add-keys = hostPkgs.writeShellScriptBin "add-keys" (
167 ''
168 set -euo pipefail
169 ''
170 +
171 # When running as non-interactively as part of a DarwinConfiguration the working directory
172 # must be set to a writeable directory.
173 (
174 if cfg.workingDirectory != "." then
175 ''
176 ${hostPkgs.coreutils}/bin/mkdir --parent "${cfg.workingDirectory}"
177 cd "${cfg.workingDirectory}"
178 ''
179 else
180 ""
181 )
182 + ''
183 KEYS="''${KEYS:-./keys}"
184 ${hostPkgs.coreutils}/bin/mkdir --parent "''${KEYS}"
185 PRIVATE_KEY="''${KEYS}/${user}_${keyType}"
186 PUBLIC_KEY="''${PRIVATE_KEY}.pub"
187 if [ ! -e "''${PRIVATE_KEY}" ] || [ ! -e "''${PUBLIC_KEY}" ]; then
188 ${hostPkgs.coreutils}/bin/rm --force -- "''${PRIVATE_KEY}" "''${PUBLIC_KEY}"
189 ${hostPkgs.openssh}/bin/ssh-keygen -q -f "''${PRIVATE_KEY}" -t ${keyType} -N "" -C 'builder@localhost'
190 fi
191 if ! ${hostPkgs.diffutils}/bin/cmp "''${PUBLIC_KEY}" ${publicKey}; then
192 (set -x; sudo --reset-timestamp ${installCredentials} "''${KEYS}")
193 fi
194 ''
195 );
196
197 run-builder = hostPkgs.writeShellScriptBin "run-builder" (''
198 set -euo pipefail
199 KEYS="''${KEYS:-./keys}"
200 KEYS="$(${hostPkgs.nix}/bin/nix-store --add "$KEYS")" ${lib.getExe config.system.build.vm}
201 '');
202
203 script = hostPkgs.writeShellScriptBin "create-builder" (''
204 set -euo pipefail
205 export KEYS="''${KEYS:-./keys}"
206 ${lib.getExe add-keys}
207 ${lib.getExe run-builder}
208 '');
209
210 in
211 script.overrideAttrs (old: {
212 pos = __curPos; # sets meta.position to point here; see script binding above for package definition
213 meta = (old.meta or { }) // {
214 platforms = lib.platforms.darwin;
215 };
216 passthru = (old.passthru or { }) // {
217 # Let users in the repl inspect the config
218 nixosConfig = config;
219 nixosOptions = options;
220
221 inherit add-keys run-builder;
222 };
223 });
224
225 system = {
226 # To prevent gratuitous rebuilds on each change to Nixpkgs
227 nixos.revision = null;
228
229 # to be updated by module maintainers, see nixpkgs#325610
230 stateVersion = "24.05";
231 };
232
233 users.users."${user}" = {
234 isNormalUser = true;
235 };
236
237 security.polkit.enable = true;
238
239 security.polkit.extraConfig = ''
240 polkit.addRule(function(action, subject) {
241 if (action.id === "org.freedesktop.login1.power-off" && subject.user === "${user}") {
242 return "yes";
243 } else {
244 return "no";
245 }
246 })
247 '';
248
249 virtualisation = {
250 diskSize = cfg.diskSize;
251
252 memorySize = cfg.memorySize;
253
254 forwardPorts = [
255 {
256 from = "host";
257 guest.port = 22;
258 host.port = cfg.hostPort;
259 }
260 ];
261
262 # Disable graphics for the builder since users will likely want to run it
263 # non-interactively in the background.
264 graphics = false;
265
266 sharedDirectories.keys = {
267 source = "\"$KEYS\"";
268 target = keysDirectory;
269 };
270
271 # If we don't enable this option then the host will fail to delegate builds
272 # to the guest, because:
273 #
274 # - The host will lock the path to build
275 # - The host will delegate the build to the guest
276 # - The guest will attempt to lock the same path and fail because
277 # the lockfile on the host is visible on the guest
278 #
279 # Snapshotting the host's /nix/store as an image isolates the guest VM's
280 # /nix/store from the host's /nix/store, preventing this problem.
281 useNixStoreImage = true;
282
283 # Obviously the /nix/store needs to be writable on the guest in order for it
284 # to perform builds.
285 writableStore = true;
286
287 # This ensures that anything built on the guest isn't lost when the guest is
288 # restarted.
289 writableStoreUseTmpfs = false;
290
291 # Pass certificates from host to the guest otherwise when custom CA certificates
292 # are required we can't use the cached builder.
293 useHostCerts = true;
294 };
295 };
296}