pyrox.dev
1{
2 config,
3 lib,
4 pkgs,
5 ...
6}:
7
8let
9 cfg = config.virtualisation.incus;
10 preseedFormat = pkgs.formats.yaml { };
11
12 nvidiaEnabled = (lib.elem "nvidia" config.services.xserver.videoDrivers);
13
14 serverBinPath = ''/run/wrappers/bin:${pkgs.qemu_kvm}/libexec:${
15 lib.makeBinPath (
16 with pkgs;
17 [
18 cfg.package
19
20 acl
21 attr
22 bash
23 btrfs-progs
24 cdrkit
25 coreutils
26 criu
27 dnsmasq
28 e2fsprogs
29 findutils
30 getent
31 gawk
32 gnugrep
33 gnused
34 gnutar
35 gptfdisk
36 gzip
37 iproute2
38 iptables
39 iw
40 kmod
41 libxfs
42 lvm2
43 lxcfs
44 minio
45 minio-client
46 nftables
47 qemu-utils
48 qemu_kvm
49 rsync
50 squashfs-tools-ng
51 squashfsTools
52 sshfs
53 swtpm
54 systemd
55 thin-provisioning-tools
56 util-linux
57 virtiofsd
58 xdelta
59 xz
60 ]
61 ++ lib.optionals (lib.versionAtLeast cfg.package.version "6.3.0") [
62 skopeo
63 umoci
64 ]
65 ++ lib.optionals (lib.versionAtLeast cfg.package.version "6.11.0") [
66 lego
67 ]
68 ++ lib.optionals config.security.apparmor.enable [
69 apparmor-bin-utils
70
71 (writeShellScriptBin "apparmor_parser" ''
72 exec '${apparmor-parser}/bin/apparmor_parser' -I '${apparmor-profiles}/etc/apparmor.d' "$@"
73 '')
74 ]
75 ++ lib.optionals config.services.ceph.client.enable [ ceph-client ]
76 ++ lib.optionals config.virtualisation.vswitch.enable [ config.virtualisation.vswitch.package ]
77 ++ lib.optionals config.boot.zfs.enabled [
78 config.boot.zfs.package
79 "${config.boot.zfs.package}/lib/udev"
80 ]
81 ++ lib.optionals nvidiaEnabled [
82 libnvidia-container
83 ]
84 )
85 }'';
86
87 # https://github.com/lxc/incus/blob/cff35a29ee3d7a2af1f937cbb6cf23776941854b/internal/server/instance/drivers/driver_qemu.go#L123
88 OVMF2MB = pkgs.OVMF.override {
89 secureBoot = true;
90 fdSize2MB = true;
91 };
92 ovmf-prefix = if pkgs.stdenv.hostPlatform.isAarch64 then "AAVMF" else "OVMF";
93 ovmf = pkgs.linkFarm "incus-ovmf" (
94 [
95 # 2MB must remain the default or existing VMs will fail to boot. New VMs will prefer 4MB
96 {
97 name = "OVMF_CODE.fd";
98 path = "${OVMF2MB.fd}/FV/${ovmf-prefix}_CODE.fd";
99 }
100 {
101 name = "OVMF_VARS.fd";
102 path = "${OVMF2MB.fd}/FV/${ovmf-prefix}_VARS.fd";
103 }
104 {
105 name = "OVMF_VARS.ms.fd";
106 path = "${OVMF2MB.fd}/FV/${ovmf-prefix}_VARS.fd";
107 }
108
109 {
110 name = "OVMF_CODE.4MB.fd";
111 path = "${pkgs.OVMFFull.fd}/FV/${ovmf-prefix}_CODE.fd";
112 }
113 {
114 name = "OVMF_VARS.4MB.fd";
115 path = "${pkgs.OVMFFull.fd}/FV/${ovmf-prefix}_VARS.fd";
116 }
117 {
118 name = "OVMF_VARS.4MB.ms.fd";
119 path = "${pkgs.OVMFFull.fd}/FV/${ovmf-prefix}_VARS.fd";
120 }
121 ]
122 ++ lib.optionals pkgs.stdenv.hostPlatform.isx86_64 [
123 {
124 name = "seabios.bin";
125 path = "${pkgs.seabios-qemu}/share/seabios/bios.bin";
126 }
127 ]
128 );
129
130 environment = lib.mkMerge [
131 {
132 INCUS_DOCUMENTATION = "${cfg.package.doc}/html";
133 INCUS_EDK2_PATH = ovmf;
134 INCUS_LXC_HOOK = "${cfg.lxcPackage}/share/lxc/hooks";
135 INCUS_LXC_TEMPLATE_CONFIG = "${pkgs.lxcfs}/share/lxc/config";
136 INCUS_USBIDS_PATH = "${pkgs.hwdata}/share/hwdata/usb.ids";
137 PATH = lib.mkForce serverBinPath;
138 }
139 (lib.mkIf (cfg.ui.enable) { "INCUS_UI" = cfg.ui.package; })
140 ];
141
142 incus-startup = pkgs.writeShellScript "incus-startup" ''
143 case "$1" in
144 start)
145 systemctl is-active incus.service -q && exit 0
146 exec incusd activateifneeded
147 ;;
148
149 stop)
150 systemctl is-active incus.service -q || exit 0
151 exec incusd shutdown
152 ;;
153
154 *)
155 echo "unknown argument \`$1'" >&2
156 exit 1
157 ;;
158 esac
159
160 exit 0
161 '';
162in
163{
164 meta = {
165 maintainers = lib.teams.lxc.members;
166 };
167
168 options = {
169 virtualisation.incus = {
170 enable = lib.mkEnableOption ''
171 incusd, a daemon that manages containers and virtual machines.
172
173 Users in the "incus-admin" group can interact with
174 the daemon (e.g. to start or stop containers) using the
175 {command}`incus` command line tool, among others.
176 Users in the "incus" group can also interact with
177 the daemon, but with lower permissions
178 (i.e. administrative operations are forbidden).
179 '';
180
181 package = lib.mkPackageOption pkgs "incus-lts" { };
182
183 lxcPackage = lib.mkOption {
184 type = lib.types.package;
185 default = config.virtualisation.lxc.package;
186 defaultText = lib.literalExpression "config.virtualisation.lxc.package";
187 description = "The lxc package to use.";
188 };
189
190 clientPackage = lib.mkOption {
191 type = lib.types.package;
192 default = cfg.package.client;
193 defaultText = lib.literalExpression "config.virtualisation.incus.package.client";
194 description = "The incus client package to use. This package is added to PATH.";
195 };
196
197 softDaemonRestart = lib.mkOption {
198 type = lib.types.bool;
199 default = true;
200 description = ''
201 Allow for incus.service to be stopped without affecting running instances.
202 '';
203 };
204
205 preseed = lib.mkOption {
206 type = lib.types.nullOr (lib.types.submodule { freeformType = preseedFormat.type; });
207
208 default = null;
209
210 description = ''
211 Configuration for Incus preseed, see
212 <https://linuxcontainers.org/incus/docs/main/howto/initialize/#non-interactive-configuration>
213 for supported values.
214
215 Changes to this will be re-applied to Incus which will overwrite existing entities or create missing ones,
216 but entities will *not* be removed by preseed.
217 '';
218
219 example = {
220 networks = [
221 {
222 name = "incusbr0";
223 type = "bridge";
224 config = {
225 "ipv4.address" = "10.0.100.1/24";
226 "ipv4.nat" = "true";
227 };
228 }
229 ];
230 profiles = [
231 {
232 name = "default";
233 devices = {
234 eth0 = {
235 name = "eth0";
236 network = "incusbr0";
237 type = "nic";
238 };
239 root = {
240 path = "/";
241 pool = "default";
242 size = "35GiB";
243 type = "disk";
244 };
245 };
246 }
247 ];
248 storage_pools = [
249 {
250 name = "default";
251 driver = "dir";
252 config = {
253 source = "/var/lib/incus/storage-pools/default";
254 };
255 }
256 ];
257 };
258 };
259
260 socketActivation = lib.mkEnableOption (''
261 socket-activation for starting incus.service. Enabling this option
262 will stop incus.service from starting automatically on boot.
263 '');
264
265 startTimeout = lib.mkOption {
266 type = lib.types.ints.unsigned;
267 default = 600;
268 apply = toString;
269 description = ''
270 Time to wait (in seconds) for incusd to become ready to process requests.
271 If incusd does not reply within the configured time, `incus.service` will be
272 considered failed and systemd will attempt to restart it.
273 '';
274 };
275
276 ui = {
277 enable = lib.mkEnableOption "Incus Web UI";
278
279 package = lib.mkPackageOption pkgs [ "incus-ui-canonical" ] { };
280 };
281 };
282 };
283
284 config = lib.mkIf cfg.enable {
285 assertions = [
286 {
287 assertion =
288 !(
289 config.networking.firewall.enable
290 && !config.networking.nftables.enable
291 && config.virtualisation.incus.enable
292 );
293 message = "Incus on NixOS is unsupported using iptables. Set `networking.nftables.enable = true;`";
294 }
295 ];
296
297 # https://github.com/lxc/incus/blob/f145309929f849b9951658ad2ba3b8f10cbe69d1/doc/reference/server_settings.md
298 boot.kernel.sysctl = {
299 "fs.aio-max-nr" = lib.mkDefault 524288;
300 "fs.inotify.max_queued_events" = lib.mkDefault 1048576;
301 "fs.inotify.max_user_instances" = lib.mkOverride 1050 1048576; # override in case conflict nixos/modules/services/x11/xserver.nix
302 "fs.inotify.max_user_watches" = lib.mkOverride 1050 1048576; # override in case conflict nixos/modules/services/x11/xserver.nix
303 "kernel.dmesg_restrict" = lib.mkDefault 1;
304 "kernel.keys.maxbytes" = lib.mkDefault 2000000;
305 "kernel.keys.maxkeys" = lib.mkDefault 2000;
306 "net.core.bpf_jit_limit" = lib.mkDefault 1000000000;
307 "net.ipv4.neigh.default.gc_thresh3" = lib.mkDefault 8192;
308 "net.ipv6.neigh.default.gc_thresh3" = lib.mkDefault 8192;
309 # vm.max_map_count is set higher in nixos/modules/config/sysctl.nix
310 };
311
312 boot.kernelModules = [
313 "br_netfilter"
314 "veth"
315 "xt_comment"
316 "xt_CHECKSUM"
317 "xt_MASQUERADE"
318 "vhost_vsock"
319 ] ++ lib.optionals nvidiaEnabled [ "nvidia_uvm" ];
320
321 environment.systemPackages = [
322 cfg.clientPackage
323
324 # gui console support
325 pkgs.spice-gtk
326 ];
327
328 # Note: the following options are also declared in virtualisation.lxc, but
329 # the latter can't be simply enabled to reuse the formers, because it
330 # does a bunch of unrelated things.
331 systemd.tmpfiles.rules = [ "d /var/lib/lxc/rootfs 0755 root root -" ];
332
333 security.apparmor = {
334 packages = [ cfg.lxcPackage ];
335 policies = {
336 "bin.lxc-start".profile = ''
337 include ${cfg.lxcPackage}/etc/apparmor.d/usr.bin.lxc-start
338 '';
339 "lxc-containers".profile = ''
340 include ${cfg.lxcPackage}/etc/apparmor.d/lxc-containers
341 '';
342 "incusd".profile = ''
343 # This profile allows everything and only exists to give the
344 # application a name instead of having the label "unconfined"
345
346 abi <abi/4.0>,
347 include <tunables/global>
348
349 profile incusd ${lib.getExe' config.virtualisation.incus.package "incusd"} flags=(unconfined) {
350 userns,
351 </var/lib/incus/security/apparmor/cache>
352 </var/lib/incus/security/apparmor/profiles>
353
354 # Site-specific additions and overrides. See local/README for details.
355 include if exists <local/incusd>
356 }
357 '';
358 };
359 includes."abstractions/base" =
360 ''
361 # Allow incusd's various AA profiles to load dynamic libraries from Nix store
362 # https://discuss.linuxcontainers.org/t/creating-new-containers-vms-blocked-by-apparmor-on-nixos/21908/6
363 mr /nix/store/*/lib/*.so*,
364 r ${pkgs.stdenv.cc.libc}/lib/gconv/gconv-modules,
365 r ${pkgs.stdenv.cc.libc}/lib/gconv/gconv-modules.d/,
366 r ${pkgs.stdenv.cc.libc}/lib/gconv/gconv-modules.d/gconv-modules-extra.conf,
367
368 # Support use of VM instance
369 mrix ${pkgs.qemu_kvm}/bin/*,
370 k ${OVMF2MB.fd}/FV/*.fd,
371 k ${pkgs.OVMFFull.fd}/FV/*.fd,
372 ''
373 + lib.optionalString pkgs.stdenv.hostPlatform.isx86_64 ''
374 k ${pkgs.seabios-qemu}/share/seabios/bios.bin,
375 '';
376 };
377
378 systemd.services.incus = {
379 description = "Incus Container and Virtual Machine Management Daemon";
380
381 inherit environment;
382
383 wantedBy = lib.mkIf (!cfg.socketActivation) [ "multi-user.target" ];
384 after = [
385 "network-online.target"
386 "lxcfs.service"
387 "incus.socket"
388 ] ++ lib.optionals config.virtualisation.vswitch.enable [ "ovs-vswitchd.service" ];
389
390 requires = [
391 "lxcfs.service"
392 "incus.socket"
393 ] ++ lib.optionals config.virtualisation.vswitch.enable [ "ovs-vswitchd.service" ];
394
395 wants = [ "network-online.target" ];
396
397 serviceConfig = {
398 ExecStart = "${cfg.package}/bin/incusd --group incus-admin";
399 ExecStartPost = "${cfg.package}/bin/incusd waitready --timeout=${cfg.startTimeout}";
400 ExecStop = lib.optionalString (!cfg.softDaemonRestart) "${cfg.package}/bin/incus admin shutdown";
401
402 KillMode = "process"; # when stopping, leave the containers alone
403 Delegate = "yes";
404 LimitMEMLOCK = "infinity";
405 LimitNOFILE = "1048576";
406 LimitNPROC = "infinity";
407 TasksMax = "infinity";
408
409 Restart = "on-failure";
410 TimeoutStartSec = "${cfg.startTimeout}s";
411 TimeoutStopSec = "30s";
412 };
413 };
414
415 systemd.services.incus-user = {
416 description = "Incus Container and Virtual Machine Management User Daemon";
417
418 inherit environment;
419
420 after = [
421 "incus.service"
422 "incus-user.socket"
423 ];
424
425 requires = [
426 "incus-user.socket"
427 ];
428
429 serviceConfig = {
430 ExecStart = "${cfg.package}/bin/incus-user --group incus";
431
432 Restart = "on-failure";
433 };
434 };
435
436 systemd.services.incus-startup = lib.mkIf cfg.softDaemonRestart {
437 description = "Incus Instances Startup/Shutdown";
438
439 inherit environment;
440
441 after = [
442 "incus.service"
443 "incus.socket"
444 ];
445 requires = [ "incus.socket" ];
446 wantedBy = config.systemd.services.incus.wantedBy;
447
448 serviceConfig = {
449 ExecStart = "${incus-startup} start";
450 ExecStop = "${incus-startup} stop";
451 RemainAfterExit = true;
452 TimeoutStartSec = "600s";
453 TimeoutStopSec = "600s";
454 Type = "oneshot";
455 };
456 };
457
458 systemd.sockets.incus = {
459 description = "Incus UNIX socket";
460 wantedBy = [ "sockets.target" ];
461
462 socketConfig = {
463 ListenStream = "/var/lib/incus/unix.socket";
464 SocketMode = "0660";
465 SocketGroup = "incus-admin";
466 };
467 };
468
469 systemd.sockets.incus-user = {
470 description = "Incus user UNIX socket";
471 wantedBy = [ "sockets.target" ];
472
473 socketConfig = {
474 ListenStream = "/var/lib/incus/unix.socket.user";
475 SocketMode = "0660";
476 SocketGroup = "incus";
477 };
478 };
479
480 systemd.services.incus-preseed = lib.mkIf (cfg.preseed != null) {
481 description = "Incus initialization with preseed file";
482
483 wantedBy = [ "incus.service" ];
484 after = [ "incus.service" ];
485 bindsTo = [ "incus.service" ];
486 partOf = [ "incus.service" ];
487
488 script = ''
489 ${cfg.package}/bin/incus admin init --preseed <${preseedFormat.generate "incus-preseed.yaml" cfg.preseed}
490 '';
491
492 serviceConfig = {
493 Type = "oneshot";
494 RemainAfterExit = true;
495 };
496 };
497
498 users.groups.incus = { };
499 users.groups.incus-admin = { };
500
501 users.users.root = {
502 # match documented default ranges https://linuxcontainers.org/incus/docs/main/userns-idmap/#allowed-ranges
503 subUidRanges = [
504 {
505 startUid = 1000000;
506 count = 1000000000;
507 }
508 ];
509 subGidRanges = [
510 {
511 startGid = 1000000;
512 count = 1000000000;
513 }
514 ];
515 };
516
517 virtualisation.lxc.lxcfs.enable = true;
518 };
519}