pyrox.dev
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}:
7
8let
9 cfg = config.virtualisation.incus;
10 preseedFormat = pkgs.formats.yaml { };
11
12 serverBinPath = ''/run/wrappers/bin:${pkgs.qemu_kvm}/libexec:${
13 lib.makeBinPath (
14 with pkgs;
15 [
16 cfg.package
17
18 acl
19 attr
20 bash
21 btrfs-progs
22 cdrkit
23 coreutils
24 criu
25 dnsmasq
26 e2fsprogs
27 findutils
28 getent
29 gnugrep
30 gnused
31 gnutar
32 gptfdisk
33 gzip
34 iproute2
35 iptables
36 iw
37 kmod
38 libnvidia-container
39 libxfs
40 lvm2
41 minio
42 minio-client
43 nftables
44 qemu-utils
45 qemu_kvm
46 rsync
47 squashfs-tools-ng
48 squashfsTools
49 sshfs
50 swtpm
51 systemd
52 thin-provisioning-tools
53 util-linux
54 virtiofsd
55 xdelta
56 xz
57 ]
58 ++ lib.optionals config.security.apparmor.enable [
59 apparmor-bin-utils
60
61 (writeShellScriptBin "apparmor_parser" ''
62 exec '${apparmor-parser}/bin/apparmor_parser' -I '${apparmor-profiles}/etc/apparmor.d' "$@"
63 '')
64 ]
65 ++ lib.optionals config.services.ceph.client.enable [ ceph-client ]
66 ++ lib.optionals config.virtualisation.vswitch.enable [ config.virtualisation.vswitch.package ]
67 ++ lib.optionals config.boot.zfs.enabled [
68 config.boot.zfs.package
69 "${config.boot.zfs.package}/lib/udev"
70 ]
71 )
72 }'';
73
74 # https://github.com/lxc/incus/blob/cff35a29ee3d7a2af1f937cbb6cf23776941854b/internal/server/instance/drivers/driver_qemu.go#L123
75 OVMF2MB = pkgs.OVMF.override {
76 secureBoot = true;
77 fdSize2MB = true;
78 };
79 ovmf-prefix = if pkgs.stdenv.hostPlatform.isAarch64 then "AAVMF" else "OVMF";
80 ovmf = pkgs.linkFarm "incus-ovmf" [
81 # 2MB must remain the default or existing VMs will fail to boot. New VMs will prefer 4MB
82 {
83 name = "OVMF_CODE.fd";
84 path = "${OVMF2MB.fd}/FV/${ovmf-prefix}_CODE.fd";
85 }
86 {
87 name = "OVMF_VARS.fd";
88 path = "${OVMF2MB.fd}/FV/${ovmf-prefix}_VARS.fd";
89 }
90 {
91 name = "OVMF_VARS.ms.fd";
92 path = "${OVMF2MB.fd}/FV/${ovmf-prefix}_VARS.fd";
93 }
94
95 {
96 name = "OVMF_CODE.4MB.fd";
97 path = "${pkgs.OVMFFull.fd}/FV/${ovmf-prefix}_CODE.fd";
98 }
99 {
100 name = "OVMF_VARS.4MB.fd";
101 path = "${pkgs.OVMFFull.fd}/FV/${ovmf-prefix}_VARS.fd";
102 }
103 {
104 name = "OVMF_VARS.4MB.ms.fd";
105 path = "${pkgs.OVMFFull.fd}/FV/${ovmf-prefix}_VARS.fd";
106 }
107 ];
108
109 environment = lib.mkMerge [
110 {
111 INCUS_LXC_TEMPLATE_CONFIG = "${pkgs.lxcfs}/share/lxc/config";
112 INCUS_OVMF_PATH = ovmf;
113 INCUS_USBIDS_PATH = "${pkgs.hwdata}/share/hwdata/usb.ids";
114 PATH = lib.mkForce serverBinPath;
115 }
116 (lib.mkIf (cfg.ui.enable) { "INCUS_UI" = cfg.ui.package; })
117 ];
118
119 incus-startup = pkgs.writeShellScript "incus-startup" ''
120 case "$1" in
121 start)
122 systemctl is-active incus.service -q && exit 0
123 exec incusd activateifneeded
124 ;;
125
126 stop)
127 systemctl is-active incus.service -q || exit 0
128 exec incusd shutdown
129 ;;
130
131 *)
132 echo "unknown argument \`$1'" >&2
133 exit 1
134 ;;
135 esac
136
137 exit 0
138 '';
139in
140{
141 meta = {
142 maintainers = lib.teams.lxc.members;
143 };
144
145 options = {
146 virtualisation.incus = {
147 enable = lib.mkEnableOption ''
148 incusd, a daemon that manages containers and virtual machines.
149
150 Users in the "incus-admin" group can interact with
151 the daemon (e.g. to start or stop containers) using the
152 {command}`incus` command line tool, among others.
153 '';
154
155 package = lib.mkPackageOption pkgs "incus-lts" { };
156
157 lxcPackage = lib.mkOption {
158 type = lib.types.package;
159 default = config.virtualisation.lxc.package;
160 defaultText = lib.literalExpression "config.virtualisation.lxc.package";
161 description = "The lxc package to use.";
162 };
163
164 clientPackage = lib.mkOption {
165 type = lib.types.package;
166 default = cfg.package.client;
167 defaultText = lib.literalExpression "config.virtualisation.incus.package.client";
168 description = "The incus client package to use. This package is added to PATH.";
169 };
170
171 softDaemonRestart = lib.mkOption {
172 type = lib.types.bool;
173 default = true;
174 description = ''
175 Allow for incus.service to be stopped without affecting running instances.
176 '';
177 };
178
179 preseed = lib.mkOption {
180 type = lib.types.nullOr (lib.types.submodule { freeformType = preseedFormat.type; });
181
182 default = null;
183
184 description = ''
185 Configuration for Incus preseed, see
186 <https://linuxcontainers.org/incus/docs/main/howto/initialize/#non-interactive-configuration>
187 for supported values.
188
189 Changes to this will be re-applied to Incus which will overwrite existing entities or create missing ones,
190 but entities will *not* be removed by preseed.
191 '';
192
193 example = {
194 networks = [
195 {
196 name = "incusbr0";
197 type = "bridge";
198 config = {
199 "ipv4.address" = "10.0.100.1/24";
200 "ipv4.nat" = "true";
201 };
202 }
203 ];
204 profiles = [
205 {
206 name = "default";
207 devices = {
208 eth0 = {
209 name = "eth0";
210 network = "incusbr0";
211 type = "nic";
212 };
213 root = {
214 path = "/";
215 pool = "default";
216 size = "35GiB";
217 type = "disk";
218 };
219 };
220 }
221 ];
222 storage_pools = [
223 {
224 name = "default";
225 driver = "dir";
226 config = {
227 source = "/var/lib/incus/storage-pools/default";
228 };
229 }
230 ];
231 };
232 };
233
234 socketActivation = lib.mkEnableOption (''
235 socket-activation for starting incus.service. Enabling this option
236 will stop incus.service from starting automatically on boot.
237 '');
238
239 startTimeout = lib.mkOption {
240 type = lib.types.ints.unsigned;
241 default = 600;
242 apply = toString;
243 description = ''
244 Time to wait (in seconds) for incusd to become ready to process requests.
245 If incusd does not reply within the configured time, `incus.service` will be
246 considered failed and systemd will attempt to restart it.
247 '';
248 };
249
250 ui = {
251 enable = lib.mkEnableOption "(experimental) Incus UI";
252
253 package = lib.mkPackageOption pkgs [
254 "incus"
255 "ui"
256 ] { };
257 };
258 };
259 };
260
261 config = lib.mkIf cfg.enable {
262 assertions = [
263 {
264 assertion =
265 !(
266 config.networking.firewall.enable
267 && !config.networking.nftables.enable
268 && config.virtualisation.incus.enable
269 );
270 message = "Incus on NixOS is unsupported using iptables. Set `networking.nftables.enable = true;`";
271 }
272 ];
273
274 # https://github.com/lxc/incus/blob/f145309929f849b9951658ad2ba3b8f10cbe69d1/doc/reference/server_settings.md
275 boot.kernel.sysctl = {
276 "fs.aio-max-nr" = lib.mkDefault 524288;
277 "fs.inotify.max_queued_events" = lib.mkDefault 1048576;
278 "fs.inotify.max_user_instances" = lib.mkOverride 1050 1048576; # override in case conflict nixos/modules/services/x11/xserver.nix
279 "fs.inotify.max_user_watches" = lib.mkOverride 1050 1048576; # override in case conflict nixos/modules/services/x11/xserver.nix
280 "kernel.dmesg_restrict" = lib.mkDefault 1;
281 "kernel.keys.maxbytes" = lib.mkDefault 2000000;
282 "kernel.keys.maxkeys" = lib.mkDefault 2000;
283 "net.core.bpf_jit_limit" = lib.mkDefault 1000000000;
284 "net.ipv4.neigh.default.gc_thresh3" = lib.mkDefault 8192;
285 "net.ipv6.neigh.default.gc_thresh3" = lib.mkDefault 8192;
286 # vm.max_map_count is set higher in nixos/modules/config/sysctl.nix
287 };
288
289 boot.kernelModules = [
290 "veth"
291 "xt_comment"
292 "xt_CHECKSUM"
293 "xt_MASQUERADE"
294 "vhost_vsock"
295 ] ++ lib.optionals (!config.networking.nftables.enable) [ "iptable_mangle" ];
296
297 environment.systemPackages = [
298 cfg.clientPackage
299
300 # gui console support
301 pkgs.spice-gtk
302 ];
303
304 # Note: the following options are also declared in virtualisation.lxc, but
305 # the latter can't be simply enabled to reuse the formers, because it
306 # does a bunch of unrelated things.
307 systemd.tmpfiles.rules = [ "d /var/lib/lxc/rootfs 0755 root root -" ];
308
309 security.apparmor = {
310 packages = [ cfg.lxcPackage ];
311 policies = {
312 "bin.lxc-start".profile = ''
313 include ${cfg.lxcPackage}/etc/apparmor.d/usr.bin.lxc-start
314 '';
315 "lxc-containers".profile = ''
316 include ${cfg.lxcPackage}/etc/apparmor.d/lxc-containers
317 '';
318 };
319 };
320
321 systemd.services.incus = {
322 description = "Incus Container and Virtual Machine Management Daemon";
323
324 inherit environment;
325
326 wantedBy = lib.mkIf (!cfg.socketActivation) [ "multi-user.target" ];
327 after = [
328 "network-online.target"
329 "lxcfs.service"
330 "incus.socket"
331 ] ++ lib.optionals config.virtualisation.vswitch.enable [ "ovs-vswitchd.service" ];
332
333 requires = [
334 "lxcfs.service"
335 "incus.socket"
336 ] ++ lib.optionals config.virtualisation.vswitch.enable [ "ovs-vswitchd.service" ];
337
338 wants = [ "network-online.target" ];
339
340 serviceConfig = {
341 ExecStart = "${cfg.package}/bin/incusd --group incus-admin";
342 ExecStartPost = "${cfg.package}/bin/incusd waitready --timeout=${cfg.startTimeout}";
343 ExecStop = lib.optionalString (!cfg.softDaemonRestart) "${cfg.package}/bin/incus admin shutdown";
344
345 KillMode = "process"; # when stopping, leave the containers alone
346 Delegate = "yes";
347 LimitMEMLOCK = "infinity";
348 LimitNOFILE = "1048576";
349 LimitNPROC = "infinity";
350 TasksMax = "infinity";
351
352 Restart = "on-failure";
353 TimeoutStartSec = "${cfg.startTimeout}s";
354 TimeoutStopSec = "30s";
355 };
356 };
357
358 systemd.services.incus-startup = lib.mkIf cfg.softDaemonRestart {
359 description = "Incus Instances Startup/Shutdown";
360
361 inherit environment;
362
363 after = [
364 "incus.service"
365 "incus.socket"
366 ];
367 requires = [ "incus.socket" ];
368
369 serviceConfig = {
370 ExecStart = "${incus-startup} start";
371 ExecStop = "${incus-startup} stop";
372 RemainAfterExit = true;
373 TimeoutStartSec = "600s";
374 TimeoutStopSec = "600s";
375 Type = "oneshot";
376 };
377 };
378
379 systemd.sockets.incus = {
380 description = "Incus UNIX socket";
381 wantedBy = [ "sockets.target" ];
382
383 socketConfig = {
384 ListenStream = "/var/lib/incus/unix.socket";
385 SocketMode = "0660";
386 SocketGroup = "incus-admin";
387 };
388 };
389
390 systemd.services.incus-preseed = lib.mkIf (cfg.preseed != null) {
391 description = "Incus initialization with preseed file";
392
393 wantedBy = [ "incus.service" ];
394 after = [ "incus.service" ];
395 bindsTo = [ "incus.service" ];
396 partOf = [ "incus.service" ];
397
398 script = ''
399 ${cfg.package}/bin/incus admin init --preseed <${preseedFormat.generate "incus-preseed.yaml" cfg.preseed}
400 '';
401
402 serviceConfig = {
403 Type = "oneshot";
404 RemainAfterExit = true;
405 };
406 };
407
408 users.groups.incus-admin = { };
409
410 users.users.root = {
411 # match documented default ranges https://linuxcontainers.org/incus/docs/main/userns-idmap/#allowed-ranges
412 subUidRanges = [
413 {
414 startUid = 1000000;
415 count = 1000000000;
416 }
417 ];
418 subGidRanges = [
419 {
420 startGid = 1000000;
421 count = 1000000000;
422 }
423 ];
424 };
425
426 virtualisation.lxc.lxcfs.enable = true;
427 };
428}