pyrox.dev
1{ config, lib, pkgs, ... }:
2
3let
4 cfg = config.virtualisation.incus;
5 preseedFormat = pkgs.formats.yaml { };
6in
7{
8 meta.maintainers = [ lib.maintainers.adamcstephens ];
9
10 options = {
11 virtualisation.incus = {
12 enable = lib.mkEnableOption (lib.mdDoc ''
13 incusd, a daemon that manages containers and virtual machines.
14
15 Users in the "incus-admin" group can interact with
16 the daemon (e.g. to start or stop containers) using the
17 {command}`incus` command line tool, among others.
18 '');
19
20 package = lib.mkPackageOptionMD pkgs "incus" { };
21
22 lxcPackage = lib.mkPackageOptionMD pkgs "lxc" { };
23
24 preseed = lib.mkOption {
25 type = lib.types.nullOr (
26 lib.types.submodule { freeformType = preseedFormat.type; }
27 );
28
29 default = null;
30
31 description = lib.mdDoc ''
32 Configuration for Incus preseed, see
33 <https://linuxcontainers.org/incus/docs/main/howto/initialize/#non-interactive-configuration>
34 for supported values.
35
36 Changes to this will be re-applied to Incus which will overwrite existing entities or create missing ones,
37 but entities will *not* be removed by preseed.
38 '';
39
40 example = {
41 networks = [
42 {
43 name = "incusbr0";
44 type = "bridge";
45 config = {
46 "ipv4.address" = "10.0.100.1/24";
47 "ipv4.nat" = "true";
48 };
49 }
50 ];
51 profiles = [
52 {
53 name = "default";
54 devices = {
55 eth0 = {
56 name = "eth0";
57 network = "incusbr0";
58 type = "nic";
59 };
60 root = {
61 path = "/";
62 pool = "default";
63 size = "35GiB";
64 type = "disk";
65 };
66 };
67 }
68 ];
69 storage_pools = [
70 {
71 name = "default";
72 driver = "dir";
73 config = {
74 source = "/var/lib/incus/storage-pools/default";
75 };
76 }
77 ];
78 };
79 };
80
81 socketActivation = lib.mkEnableOption (
82 lib.mdDoc ''
83 socket-activation for starting incus.service. Enabling this option
84 will stop incus.service from starting automatically on boot.
85 ''
86 );
87
88 startTimeout = lib.mkOption {
89 type = lib.types.ints.unsigned;
90 default = 600;
91 apply = toString;
92 description = lib.mdDoc ''
93 Time to wait (in seconds) for incusd to become ready to process requests.
94 If incusd does not reply within the configured time, `incus.service` will be
95 considered failed and systemd will attempt to restart it.
96 '';
97 };
98 };
99 };
100
101 config = lib.mkIf cfg.enable {
102 # https://github.com/lxc/incus/blob/f145309929f849b9951658ad2ba3b8f10cbe69d1/doc/reference/server_settings.md
103 boot.kernel.sysctl = {
104 "fs.aio-max-nr" = lib.mkDefault 524288;
105 "fs.inotify.max_queued_events" = lib.mkDefault 1048576;
106 "fs.inotify.max_user_instances" = lib.mkOverride 1050 1048576; # override in case conflict nixos/modules/services/x11/xserver.nix
107 "fs.inotify.max_user_watches" = lib.mkOverride 1050 1048576; # override in case conflict nixos/modules/services/x11/xserver.nix
108 "kernel.dmesg_restrict" = lib.mkDefault 1;
109 "kernel.keys.maxbytes" = lib.mkDefault 2000000;
110 "kernel.keys.maxkeys" = lib.mkDefault 2000;
111 "net.core.bpf_jit_limit" = lib.mkDefault 1000000000;
112 "net.ipv4.neigh.default.gc_thresh3" = lib.mkDefault 8192;
113 "net.ipv6.neigh.default.gc_thresh3" = lib.mkDefault 8192;
114 # vm.max_map_count is set higher in nixos/modules/config/sysctl.nix
115 };
116
117 boot.kernelModules = [
118 "veth"
119 "xt_comment"
120 "xt_CHECKSUM"
121 "xt_MASQUERADE"
122 "vhost_vsock"
123 ] ++ lib.optionals (!config.networking.nftables.enable) [ "iptable_mangle" ];
124
125 environment.systemPackages = [ cfg.package ];
126
127 # Note: the following options are also declared in virtualisation.lxc, but
128 # the latter can't be simply enabled to reuse the formers, because it
129 # does a bunch of unrelated things.
130 systemd.tmpfiles.rules = [ "d /var/lib/lxc/rootfs 0755 root root -" ];
131
132 security.apparmor = {
133 packages = [ cfg.lxcPackage ];
134 policies = {
135 "bin.lxc-start".profile = ''
136 include ${cfg.lxcPackage}/etc/apparmor.d/usr.bin.lxc-start
137 '';
138 "lxc-containers".profile = ''
139 include ${cfg.lxcPackage}/etc/apparmor.d/lxc-containers
140 '';
141 };
142 };
143
144 systemd.services.incus = {
145 description = "Incus Container and Virtual Machine Management Daemon";
146
147 wantedBy = lib.mkIf (!cfg.socketActivation) [ "multi-user.target" ];
148 after = [
149 "network-online.target"
150 "lxcfs.service"
151 ] ++ (lib.optional cfg.socketActivation "incus.socket");
152 requires = [
153 "lxcfs.service"
154 ] ++ (lib.optional cfg.socketActivation "incus.socket");
155 wants = [
156 "network-online.target"
157 ];
158
159 path = lib.mkIf config.boot.zfs.enabled [ config.boot.zfs.package ];
160
161 environment = {
162 # Override Path to the LXC template configuration directory
163 INCUS_LXC_TEMPLATE_CONFIG = "${pkgs.lxcfs}/share/lxc/config";
164 };
165
166 serviceConfig = {
167 ExecStart = "${cfg.package}/bin/incusd --group incus-admin";
168 ExecStartPost = "${cfg.package}/bin/incusd waitready --timeout=${cfg.startTimeout}";
169 ExecStop = "${cfg.package}/bin/incus admin shutdown";
170
171 KillMode = "process"; # when stopping, leave the containers alone
172 Delegate = "yes";
173 LimitMEMLOCK = "infinity";
174 LimitNOFILE = "1048576";
175 LimitNPROC = "infinity";
176 TasksMax = "infinity";
177
178 Restart = "on-failure";
179 TimeoutStartSec = "${cfg.startTimeout}s";
180 TimeoutStopSec = "30s";
181 };
182 };
183
184 systemd.sockets.incus = lib.mkIf cfg.socketActivation {
185 description = "Incus UNIX socket";
186 wantedBy = [ "sockets.target" ];
187
188 socketConfig = {
189 ListenStream = "/var/lib/incus/unix.socket";
190 SocketMode = "0660";
191 SocketGroup = "incus-admin";
192 Service = "incus.service";
193 };
194 };
195
196 systemd.services.incus-preseed = lib.mkIf (cfg.preseed != null) {
197 description = "Incus initialization with preseed file";
198
199 wantedBy = ["incus.service"];
200 after = ["incus.service"];
201 bindsTo = ["incus.service"];
202 partOf = ["incus.service"];
203
204 script = ''
205 ${cfg.package}/bin/incus admin init --preseed <${
206 preseedFormat.generate "incus-preseed.yaml" cfg.preseed
207 }
208 '';
209
210 serviceConfig = {
211 Type = "oneshot";
212 RemainAfterExit = true;
213 };
214 };
215
216 users.groups.incus-admin = { };
217
218 users.users.root = {
219 # match documented default ranges https://linuxcontainers.org/incus/docs/main/userns-idmap/#allowed-ranges
220 subUidRanges = [
221 {
222 startUid = 1000000;
223 count = 1000000000;
224 }
225 ];
226 subGidRanges = [
227 {
228 startGid = 1000000;
229 count = 1000000000;
230 }
231 ];
232 };
233
234 virtualisation.lxc.lxcfs.enable = true;
235 };
236}