pyrox.dev
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}:
7
8let
9 cfg = config.virtualisation.incus;
10 preseedFormat = pkgs.formats.yaml { };
11
12 nvidiaEnabled = (lib.elem "nvidia" config.services.xserver.videoDrivers);
13
14 serverBinPath = ''/run/wrappers/bin:${pkgs.qemu_kvm}/libexec:${
15 lib.makeBinPath (
16 with pkgs;
17 [
18 cfg.package
19
20 acl
21 attr
22 bash
23 btrfs-progs
24 cdrkit
25 coreutils
26 criu
27 dnsmasq
28 e2fsprogs
29 findutils
30 getent
31 gawk
32 gnugrep
33 gnused
34 gnutar
35 gptfdisk
36 gzip
37 iproute2
38 iptables
39 iw
40 kmod
41 libxfs
42 lvm2
43 lxcfs
44 minio
45 minio-client
46 nftables
47 qemu-utils
48 qemu_kvm
49 rsync
50 squashfs-tools-ng
51 squashfsTools
52 sshfs
53 swtpm
54 systemd
55 thin-provisioning-tools
56 util-linux
57 virtiofsd
58 xdelta
59 xz
60 ]
61 ++ lib.optionals (lib.versionAtLeast cfg.package.version "6.3.0") [
62 skopeo
63 umoci
64 ]
65 ++ lib.optionals (lib.versionAtLeast cfg.package.version "6.11.0") [
66 lego
67 ]
68 ++ lib.optionals config.security.apparmor.enable [
69 apparmor-bin-utils
70
71 (writeShellScriptBin "apparmor_parser" ''
72 exec '${apparmor-parser}/bin/apparmor_parser' -I '${apparmor-profiles}/etc/apparmor.d' "$@"
73 '')
74 ]
75 ++ lib.optionals config.services.ceph.client.enable [ ceph-client ]
76 ++ lib.optionals config.virtualisation.vswitch.enable [ config.virtualisation.vswitch.package ]
77 ++ lib.optionals config.boot.zfs.enabled [
78 config.boot.zfs.package
79 "${config.boot.zfs.package}/lib/udev"
80 ]
81 ++ lib.optionals nvidiaEnabled [
82 libnvidia-container
83 ]
84 )
85 }'';
86
87 # https://github.com/lxc/incus/blob/cff35a29ee3d7a2af1f937cbb6cf23776941854b/internal/server/instance/drivers/driver_qemu.go#L123
88 OVMF2MB = pkgs.OVMF.override {
89 secureBoot = true;
90 fdSize2MB = true;
91 };
92 ovmf-prefix = if pkgs.stdenv.hostPlatform.isAarch64 then "AAVMF" else "OVMF";
93 ovmf = pkgs.linkFarm "incus-ovmf" (
94 [
95 # 2MB must remain the default or existing VMs will fail to boot. New VMs will prefer 4MB
96 {
97 name = "OVMF_CODE.fd";
98 path = "${OVMF2MB.fd}/FV/${ovmf-prefix}_CODE.fd";
99 }
100 {
101 name = "OVMF_VARS.fd";
102 path = "${OVMF2MB.fd}/FV/${ovmf-prefix}_VARS.fd";
103 }
104 {
105 name = "OVMF_VARS.ms.fd";
106 path = "${OVMF2MB.fd}/FV/${ovmf-prefix}_VARS.fd";
107 }
108
109 {
110 name = "OVMF_CODE.4MB.fd";
111 path = "${pkgs.OVMFFull.fd}/FV/${ovmf-prefix}_CODE.fd";
112 }
113 {
114 name = "OVMF_VARS.4MB.fd";
115 path = "${pkgs.OVMFFull.fd}/FV/${ovmf-prefix}_VARS.fd";
116 }
117 {
118 name = "OVMF_VARS.4MB.ms.fd";
119 path = "${pkgs.OVMFFull.fd}/FV/${ovmf-prefix}_VARS.fd";
120 }
121 ]
122 ++ lib.optionals pkgs.stdenv.hostPlatform.isx86_64 [
123 {
124 name = "seabios.bin";
125 path = "${pkgs.seabios-qemu}/share/seabios/bios.bin";
126 }
127 ]
128 );
129
130 environment = lib.mkMerge [
131 {
132 INCUS_DOCUMENTATION = "${cfg.package.doc}/html";
133 INCUS_EDK2_PATH = ovmf;
134 INCUS_LXC_HOOK = "${cfg.lxcPackage}/share/lxc/hooks";
135 INCUS_LXC_TEMPLATE_CONFIG = "${pkgs.lxcfs}/share/lxc/config";
136 INCUS_USBIDS_PATH = "${pkgs.hwdata}/share/hwdata/usb.ids";
137 INCUS_AGENT_PATH = "${cfg.package}/share/agent";
138 PATH = lib.mkForce serverBinPath;
139 }
140 (lib.mkIf (cfg.ui.enable) { "INCUS_UI" = cfg.ui.package; })
141 ];
142
143 incus-startup = pkgs.writeShellScript "incus-startup" ''
144 case "$1" in
145 start)
146 systemctl is-active incus.service -q && exit 0
147 exec incusd activateifneeded
148 ;;
149
150 stop)
151 systemctl is-active incus.service -q || exit 0
152 exec incusd shutdown
153 ;;
154
155 *)
156 echo "unknown argument \`$1'" >&2
157 exit 1
158 ;;
159 esac
160
161 exit 0
162 '';
163in
164{
165 meta = {
166 maintainers = lib.teams.lxc.members;
167 };
168
169 options = {
170 virtualisation.incus = {
171 enable = lib.mkEnableOption ''
172 incusd, a daemon that manages containers and virtual machines.
173
174 Users in the "incus-admin" group can interact with
175 the daemon (e.g. to start or stop containers) using the
176 {command}`incus` command line tool, among others.
177 Users in the "incus" group can also interact with
178 the daemon, but with lower permissions
179 (i.e. administrative operations are forbidden).
180 '';
181
182 package = lib.mkPackageOption pkgs "incus-lts" { };
183
184 lxcPackage = lib.mkOption {
185 type = lib.types.package;
186 default = config.virtualisation.lxc.package;
187 defaultText = lib.literalExpression "config.virtualisation.lxc.package";
188 description = "The lxc package to use.";
189 };
190
191 clientPackage = lib.mkOption {
192 type = lib.types.package;
193 default = cfg.package.client;
194 defaultText = lib.literalExpression "config.virtualisation.incus.package.client";
195 description = "The incus client package to use. This package is added to PATH.";
196 };
197
198 softDaemonRestart = lib.mkOption {
199 type = lib.types.bool;
200 default = true;
201 description = ''
202 Allow for incus.service to be stopped without affecting running instances.
203 '';
204 };
205
206 preseed = lib.mkOption {
207 type = lib.types.nullOr (lib.types.submodule { freeformType = preseedFormat.type; });
208
209 default = null;
210
211 description = ''
212 Configuration for Incus preseed, see
213 <https://linuxcontainers.org/incus/docs/main/howto/initialize/#non-interactive-configuration>
214 for supported values.
215
216 Changes to this will be re-applied to Incus which will overwrite existing entities or create missing ones,
217 but entities will *not* be removed by preseed.
218 '';
219
220 example = {
221 networks = [
222 {
223 name = "incusbr0";
224 type = "bridge";
225 config = {
226 "ipv4.address" = "10.0.100.1/24";
227 "ipv4.nat" = "true";
228 };
229 }
230 ];
231 profiles = [
232 {
233 name = "default";
234 devices = {
235 eth0 = {
236 name = "eth0";
237 network = "incusbr0";
238 type = "nic";
239 };
240 root = {
241 path = "/";
242 pool = "default";
243 size = "35GiB";
244 type = "disk";
245 };
246 };
247 }
248 ];
249 storage_pools = [
250 {
251 name = "default";
252 driver = "dir";
253 config = {
254 source = "/var/lib/incus/storage-pools/default";
255 };
256 }
257 ];
258 };
259 };
260
261 socketActivation = lib.mkEnableOption (''
262 socket-activation for starting incus.service. Enabling this option
263 will stop incus.service from starting automatically on boot.
264 '');
265
266 startTimeout = lib.mkOption {
267 type = lib.types.ints.unsigned;
268 default = 600;
269 apply = toString;
270 description = ''
271 Time to wait (in seconds) for incusd to become ready to process requests.
272 If incusd does not reply within the configured time, `incus.service` will be
273 considered failed and systemd will attempt to restart it.
274 '';
275 };
276
277 ui = {
278 enable = lib.mkEnableOption "Incus Web UI";
279
280 package = lib.mkPackageOption pkgs [ "incus-ui-canonical" ] { };
281 };
282 };
283 };
284
285 config = lib.mkIf cfg.enable {
286 assertions = [
287 {
288 assertion =
289 !(
290 config.networking.firewall.enable
291 && !config.networking.nftables.enable
292 && config.virtualisation.incus.enable
293 );
294 message = "Incus on NixOS is unsupported using iptables. Set `networking.nftables.enable = true;`";
295 }
296 ];
297
298 # https://github.com/lxc/incus/blob/f145309929f849b9951658ad2ba3b8f10cbe69d1/doc/reference/server_settings.md
299 boot.kernel.sysctl = {
300 "fs.aio-max-nr" = lib.mkDefault 524288;
301 "fs.inotify.max_queued_events" = lib.mkDefault 1048576;
302 "fs.inotify.max_user_instances" = lib.mkOverride 1050 1048576; # override in case conflict nixos/modules/services/x11/xserver.nix
303 "fs.inotify.max_user_watches" = lib.mkOverride 1050 1048576; # override in case conflict nixos/modules/services/x11/xserver.nix
304 "kernel.dmesg_restrict" = lib.mkDefault 1;
305 "kernel.keys.maxbytes" = lib.mkDefault 2000000;
306 "kernel.keys.maxkeys" = lib.mkDefault 2000;
307 "net.core.bpf_jit_limit" = lib.mkDefault 1000000000;
308 "net.ipv4.neigh.default.gc_thresh3" = lib.mkDefault 8192;
309 "net.ipv6.neigh.default.gc_thresh3" = lib.mkDefault 8192;
310 # vm.max_map_count is set higher in nixos/modules/config/sysctl.nix
311 };
312
313 boot.kernelModules = [
314 "br_netfilter"
315 "veth"
316 "xt_comment"
317 "xt_CHECKSUM"
318 "xt_MASQUERADE"
319 "vhost_vsock"
320 ]
321 ++ lib.optionals nvidiaEnabled [ "nvidia_uvm" ];
322
323 environment.systemPackages = [
324 cfg.clientPackage
325
326 # gui console support
327 pkgs.spice-gtk
328 ];
329
330 # Note: the following options are also declared in virtualisation.lxc, but
331 # the latter can't be simply enabled to reuse the formers, because it
332 # does a bunch of unrelated things.
333 systemd.tmpfiles.rules = [ "d /var/lib/lxc/rootfs 0755 root root -" ];
334
335 security.apparmor = {
336 packages = [ cfg.lxcPackage ];
337 policies = {
338 "bin.lxc-start".profile = ''
339 include ${cfg.lxcPackage}/etc/apparmor.d/usr.bin.lxc-start
340 '';
341 "lxc-containers".profile = ''
342 include ${cfg.lxcPackage}/etc/apparmor.d/lxc-containers
343 '';
344 "incusd".profile = ''
345 # This profile allows everything and only exists to give the
346 # application a name instead of having the label "unconfined"
347
348 abi <abi/4.0>,
349 include <tunables/global>
350
351 profile incusd ${lib.getExe' config.virtualisation.incus.package "incusd"} flags=(unconfined) {
352 userns,
353
354 include "/var/lib/incus/security/apparmor/cache"
355
356 # Site-specific additions and overrides. See local/README for details.
357 include if exists <local/incusd>
358 }
359
360 include "/var/lib/incus/security/apparmor/profiles"
361 '';
362 };
363 includes."abstractions/base" = ''
364 # Allow incusd's various AA profiles to load dynamic libraries from Nix store
365 # https://discuss.linuxcontainers.org/t/creating-new-containers-vms-blocked-by-apparmor-on-nixos/21908/6
366 mr /nix/store/*/lib/*.so*,
367 r ${pkgs.stdenv.cc.libc}/lib/gconv/gconv-modules,
368 r ${pkgs.stdenv.cc.libc}/lib/gconv/gconv-modules.d/,
369 r ${pkgs.stdenv.cc.libc}/lib/gconv/gconv-modules.d/gconv-modules-extra.conf,
370
371 # Support use of VM instance
372 mrix ${pkgs.qemu_kvm}/bin/*,
373 k ${OVMF2MB.fd}/FV/*.fd,
374 k ${pkgs.OVMFFull.fd}/FV/*.fd,
375 ''
376 + lib.optionalString pkgs.stdenv.hostPlatform.isx86_64 ''
377 k ${pkgs.seabios-qemu}/share/seabios/bios.bin,
378 '';
379 };
380
381 systemd.services.incus = {
382 description = "Incus Container and Virtual Machine Management Daemon";
383
384 inherit environment;
385
386 wantedBy = lib.mkIf (!cfg.socketActivation) [ "multi-user.target" ];
387 after = [
388 "network-online.target"
389 "lxcfs.service"
390 "incus.socket"
391 ]
392 ++ lib.optionals config.virtualisation.vswitch.enable [ "ovs-vswitchd.service" ];
393
394 requires = [
395 "lxcfs.service"
396 "incus.socket"
397 ]
398 ++ lib.optionals config.virtualisation.vswitch.enable [ "ovs-vswitchd.service" ];
399
400 wants = [ "network-online.target" ];
401
402 serviceConfig = {
403 ExecStart = "${cfg.package}/bin/incusd --group incus-admin";
404 ExecStartPost = "${cfg.package}/bin/incusd waitready --timeout=${cfg.startTimeout}";
405 ExecStop = lib.optionalString (!cfg.softDaemonRestart) "${cfg.package}/bin/incus admin shutdown";
406
407 KillMode = "process"; # when stopping, leave the containers alone
408 Delegate = "yes";
409 LimitMEMLOCK = "infinity";
410 LimitNOFILE = "1048576";
411 LimitNPROC = "infinity";
412 TasksMax = "infinity";
413
414 Restart = "on-failure";
415 TimeoutStartSec = "${cfg.startTimeout}s";
416 TimeoutStopSec = "30s";
417 };
418 };
419
420 systemd.services.incus-user = {
421 description = "Incus Container and Virtual Machine Management User Daemon";
422
423 inherit environment;
424
425 after = [
426 "incus.service"
427 "incus-user.socket"
428 ];
429
430 requires = [
431 "incus-user.socket"
432 ];
433
434 serviceConfig = {
435 ExecStart = "${cfg.package}/bin/incus-user --group incus";
436
437 Restart = "on-failure";
438 };
439 };
440
441 systemd.services.incus-startup = lib.mkIf cfg.softDaemonRestart {
442 description = "Incus Instances Startup/Shutdown";
443
444 inherit environment;
445
446 after = [
447 "incus.service"
448 "incus.socket"
449 ];
450 requires = [ "incus.socket" ];
451 wantedBy = config.systemd.services.incus.wantedBy;
452
453 # restarting this service will affect instances
454 restartIfChanged = false;
455
456 serviceConfig = {
457 ExecStart = "${incus-startup} start";
458 ExecStop = "${incus-startup} stop";
459 RemainAfterExit = true;
460 TimeoutStartSec = "600s";
461 TimeoutStopSec = "600s";
462 Type = "oneshot";
463 };
464 };
465
466 systemd.sockets.incus = {
467 description = "Incus UNIX socket";
468 wantedBy = [ "sockets.target" ];
469
470 socketConfig = {
471 ListenStream = "/var/lib/incus/unix.socket";
472 SocketMode = "0660";
473 SocketGroup = "incus-admin";
474 };
475 };
476
477 systemd.sockets.incus-user = {
478 description = "Incus user UNIX socket";
479 wantedBy = [ "sockets.target" ];
480
481 socketConfig = {
482 ListenStream = "/var/lib/incus/unix.socket.user";
483 SocketMode = "0660";
484 SocketGroup = "incus";
485 };
486 };
487
488 systemd.services.incus-preseed = lib.mkIf (cfg.preseed != null) {
489 description = "Incus initialization with preseed file";
490
491 wantedBy = [ "incus.service" ];
492 after = [ "incus.service" ];
493 bindsTo = [ "incus.service" ];
494 partOf = [ "incus.service" ];
495
496 script = ''
497 ${cfg.package}/bin/incus admin init --preseed <${preseedFormat.generate "incus-preseed.yaml" cfg.preseed}
498 '';
499
500 serviceConfig = {
501 Type = "oneshot";
502 RemainAfterExit = true;
503 };
504 };
505
506 users.groups.incus = { };
507 users.groups.incus-admin = { };
508
509 users.users.root = {
510 # match documented default ranges https://linuxcontainers.org/incus/docs/main/userns-idmap/#allowed-ranges
511 subUidRanges = [
512 {
513 startUid = 1000000;
514 count = 1000000000;
515 }
516 ];
517 subGidRanges = [
518 {
519 startGid = 1000000;
520 count = 1000000000;
521 }
522 ];
523 };
524
525 virtualisation.lxc.lxcfs.enable = true;
526 };
527}