pyrox.dev
1import ../make-test-python.nix (
2 {
3 pkgs,
4 lib,
5
6 lts ? true,
7
8 allTests ? false,
9
10 appArmor ? false,
11 featureUser ? allTests,
12 initLegacy ? true,
13 initSystemd ? true,
14 instanceContainer ? allTests,
15 instanceVm ? allTests,
16 networkOvs ? allTests,
17 storageLvm ? allTests,
18 storageZfs ? allTests,
19 ...
20 }:
21
22 let
23 releases =
24 init:
25 import ../../release.nix {
26 configuration = {
27 # Building documentation makes the test unnecessarily take a longer time:
28 documentation.enable = lib.mkForce false;
29
30 boot.initrd.systemd.enable = init == "systemd";
31
32 # Arbitrary sysctl modification to ensure containers can update sysctl
33 boot.kernel.sysctl."net.ipv4.ip_forward" = "1";
34 };
35 };
36
37 images = init: {
38 container = {
39 metadata =
40 (releases init).incusContainerMeta.${pkgs.stdenv.hostPlatform.system}
41 + "/tarball/nixos-image-lxc-*-${pkgs.stdenv.hostPlatform.system}.tar.xz";
42
43 rootfs =
44 (releases init).incusContainerImage.${pkgs.stdenv.hostPlatform.system}
45 + "/nixos-lxc-image-${pkgs.stdenv.hostPlatform.system}.squashfs";
46 };
47
48 virtual-machine = {
49 metadata =
50 (releases init).incusVirtualMachineImageMeta.${pkgs.stdenv.hostPlatform.system} + "/*/*.tar.xz";
51 disk = (releases init).incusVirtualMachineImage.${pkgs.stdenv.hostPlatform.system} + "/nixos.qcow2";
52 };
53 };
54
55 initVariants = lib.optionals initLegacy [ "legacy" ] ++ lib.optionals initSystemd [ "systemd" ];
56
57 canTestVm = instanceVm && pkgs.stdenv.isLinux && pkgs.stdenv.isx86_64;
58 in
59 {
60 name = "incus" + lib.optionalString lts "-lts";
61
62 meta = {
63 maintainers = lib.teams.lxc.members;
64 };
65
66 nodes.machine = {
67 virtualisation = {
68 cores = 2;
69 memorySize = 2048;
70 diskSize = 12 * 1024;
71 emptyDiskImages = [
72 # vdb for zfs
73 2048
74 # vdc for lvm
75 2048
76 ];
77
78 incus = {
79 enable = true;
80 package = if lts then pkgs.incus-lts else pkgs.incus;
81
82 preseed = {
83 networks =
84 [
85 {
86 name = "incusbr0";
87 type = "bridge";
88 config = {
89 "ipv4.address" = "10.0.10.1/24";
90 "ipv4.nat" = "true";
91 };
92 }
93 ]
94 ++ lib.optionals networkOvs [
95 {
96 name = "ovsbr0";
97 type = "bridge";
98 config = {
99 "bridge.driver" = "openvswitch";
100 "ipv4.address" = "10.0.20.1/24";
101 "ipv4.nat" = "true";
102 };
103 }
104 ];
105 profiles = [
106 {
107 name = "default";
108 devices = {
109 eth0 = {
110 name = "eth0";
111 network = "incusbr0";
112 type = "nic";
113 };
114 root = {
115 path = "/";
116 pool = "default";
117 size = "35GiB";
118 type = "disk";
119 };
120 };
121 }
122 ];
123 storage_pools = [
124 {
125 name = "default";
126 driver = "dir";
127 }
128 ];
129 };
130 };
131
132 vswitch.enable = networkOvs;
133 };
134
135 boot.supportedFilesystems = lib.optionals storageZfs [ "zfs" ];
136 boot.zfs.forceImportRoot = false;
137
138 environment.systemPackages = [ pkgs.parted ];
139
140 networking.hostId = "01234567";
141 networking.firewall.trustedInterfaces = [ "incusbr0" ];
142
143 security.apparmor.enable = appArmor;
144 services.dbus.apparmor = (if appArmor then "enabled" else "disabled");
145
146 services.lvm = {
147 boot.thin.enable = storageLvm;
148 dmeventd.enable = storageLvm;
149 };
150
151 networking.nftables.enable = true;
152
153 users.users.testuser = {
154 isNormalUser = true;
155 shell = pkgs.bashInteractive;
156 group = "incus";
157 uid = 1000;
158 };
159 };
160
161 testScript = # python
162 ''
163 import json
164
165 def wait_for_instance(name: str, project: str = "default"):
166 machine.wait_until_succeeds(f"incus exec {name} --disable-stdin --force-interactive --project {project} -- /run/current-system/sw/bin/systemctl is-system-running")
167
168
169 def wait_incus_exec_success(name: str, command: str, timeout: int = 900, project: str = "default"):
170 def check_command(_) -> bool:
171 status, _ = machine.execute(f"incus exec {name} --disable-stdin --force-interactive --project {project} -- {command}")
172 return status == 0
173
174 with machine.nested(f"Waiting for successful exec: {command}"):
175 retry(check_command, timeout)
176
177
178 def set_config(name: str, config: str, restart: bool = False, unset: bool = False):
179 if restart:
180 machine.succeed(f"incus stop {name}")
181
182 if unset:
183 machine.succeed(f"incus config unset {name} {config}")
184 else:
185 machine.succeed(f"incus config set {name} {config}")
186
187 if restart:
188 machine.succeed(f"incus start {name}")
189 wait_for_instance(name)
190 else:
191 # give a moment to settle
192 machine.sleep(1)
193
194
195 def cleanup():
196 # avoid conflict between preseed and cleanup operations
197 machine.execute("systemctl kill incus-preseed.service")
198
199 instances = json.loads(machine.succeed("incus list --format json --all-projects"))
200 with subtest("Stopping all running instances"):
201 for instance in [a for a in instances if a['status'] == 'Running']:
202 machine.execute(f"incus stop --force {instance['name']} --project {instance['project']}")
203 machine.execute(f"incus delete --force {instance['name']} --project {instance['project']}")
204
205
206 def check_sysctl(name: str):
207 with subtest("systemd sysctl settings are applied"):
208 machine.succeed(f"incus exec {name} -- systemctl status systemd-sysctl")
209 sysctl = machine.succeed(f"incus exec {name} -- sysctl net.ipv4.ip_forward").strip().split(" ")[-1]
210 assert "1" == sysctl, f"systemd-sysctl configuration not correctly applied, {sysctl} != 1"
211
212
213 with subtest("Wait for startup"):
214 machine.wait_for_unit("incus.service")
215 machine.wait_for_unit("incus-preseed.service")
216
217
218 with subtest("Verify preseed resources created"):
219 machine.succeed("incus profile show default")
220 machine.succeed("incus network info incusbr0")
221 machine.succeed("incus storage show default")
222
223 ''
224 + lib.optionalString instanceContainer (
225 lib.foldl (
226 acc: variant:
227 acc
228 # python
229 + ''
230 metadata = "${(images variant).container.metadata}"
231 rootfs = "${(images variant).container.rootfs}"
232 alias = "nixos/container/${variant}"
233 variant = "${variant}"
234
235 with subtest("container image can be imported"):
236 machine.succeed(f"incus image import {metadata} {rootfs} --alias {alias}")
237
238
239 with subtest("container can be launched and managed"):
240 machine.succeed(f"incus launch {alias} container-{variant}1")
241 wait_for_instance(f"container-{variant}1")
242
243
244 with subtest("container mounts lxcfs overlays"):
245 machine.succeed(f"incus exec container-{variant}1 mount | grep 'lxcfs on /proc/cpuinfo type fuse.lxcfs'")
246 machine.succeed(f"incus exec container-{variant}1 mount | grep 'lxcfs on /proc/meminfo type fuse.lxcfs'")
247
248
249 with subtest("container CPU limits can be managed"):
250 set_config(f"container-{variant}1", "limits.cpu 1", restart=True)
251 wait_incus_exec_success(f"container-{variant}1", "nproc | grep '^1$'", timeout=90)
252
253
254 with subtest("container CPU limits can be hotplug changed"):
255 set_config(f"container-{variant}1", "limits.cpu 2")
256 wait_incus_exec_success(f"container-{variant}1", "nproc | grep '^2$'", timeout=90)
257
258
259 with subtest("container memory limits can be managed"):
260 set_config(f"container-{variant}1", "limits.memory 128MB", restart=True)
261 wait_incus_exec_success(f"container-{variant}1", "grep 'MemTotal:[[:space:]]*125000 kB' /proc/meminfo", timeout=90)
262
263
264 with subtest("container memory limits can be hotplug changed"):
265 set_config(f"container-{variant}1", "limits.memory 256MB")
266 wait_incus_exec_success(f"container-{variant}1", "grep 'MemTotal:[[:space:]]*250000 kB' /proc/meminfo", timeout=90)
267
268
269 with subtest("container software tpm can be configured"):
270 machine.succeed(f"incus config device add container-{variant}1 vtpm tpm path=/dev/tpm0 pathrm=/dev/tpmrm0")
271 machine.succeed(f"incus exec container-{variant}1 -- test -e /dev/tpm0")
272 machine.succeed(f"incus exec container-{variant}1 -- test -e /dev/tpmrm0")
273 machine.succeed(f"incus config device remove container-{variant}1 vtpm")
274 machine.fail(f"incus exec container-{variant}1 -- test -e /dev/tpm0")
275
276
277 with subtest("container lxc-generator compatibility"):
278 with subtest("lxc-container generator configures plain container"):
279 # default container is plain
280 machine.succeed(f"incus exec container-{variant}1 test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")
281
282 check_sysctl(f"container-{variant}1")
283
284 with subtest("lxc-container generator configures nested container"):
285 set_config(f"container-{variant}1", "security.nesting=true", restart=True)
286
287 machine.fail(f"incus exec container-{variant}1 test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")
288 target = machine.succeed(f"incus exec container-{variant}1 readlink -- -f /run/systemd/system/systemd-binfmt.service").strip()
289 assert target == "/dev/null", "lxc generator did not correctly mask /run/systemd/system/systemd-binfmt.service"
290
291 check_sysctl(f"container-{variant}1")
292
293 with subtest("lxc-container generator configures privileged container"):
294 # Create a new instance for a clean state
295 machine.succeed(f"incus launch {alias} container-{variant}2")
296 wait_for_instance(f"container-{variant}2")
297
298 machine.succeed(f"incus exec container-{variant}2 test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")
299
300 check_sysctl(f"container-{variant}2")
301
302 with subtest("container supports per-instance lxcfs"):
303 machine.succeed(f"incus stop container-{variant}1")
304 machine.fail(f"pgrep -a lxcfs | grep 'incus/devices/container-{variant}1/lxcfs'")
305
306 machine.succeed("incus config set instances.lxcfs.per_instance=true")
307
308 machine.succeed(f"incus start container-{variant}1")
309 wait_for_instance(f"container-{variant}1")
310 machine.succeed(f"pgrep -a lxcfs | grep 'incus/devices/container-{variant}1/lxcfs'")
311
312
313 with subtest("container can successfully restart"):
314 machine.succeed(f"incus restart container-{variant}1")
315 wait_for_instance(f"container-{variant}1")
316
317
318 with subtest("container remains running when softDaemonRestart is enabled and service is stopped"):
319 pid = machine.succeed(f"incus info container-{variant}1 | grep 'PID'").split(":")[1].strip()
320 machine.succeed(f"ps {pid}")
321 machine.succeed("systemctl stop incus")
322 machine.succeed(f"ps {pid}")
323 machine.succeed("systemctl start incus")
324
325 with subtest("containers stop with incus-startup.service"):
326 pid = machine.succeed(f"incus info container-{variant}1 | grep 'PID'").split(":")[1].strip()
327 machine.succeed(f"ps {pid}")
328 machine.succeed("systemctl stop incus-startup.service")
329 machine.wait_until_fails(f"ps {pid}", timeout=120)
330 machine.succeed("systemctl start incus-startup.service")
331
332
333 cleanup()
334 ''
335 ) "" initVariants
336 )
337 + lib.optionalString canTestVm (
338 (lib.foldl (
339 acc: variant:
340 acc
341 # python
342 + ''
343 metadata = "${(images variant).virtual-machine.metadata}"
344 disk = "${(images variant).virtual-machine.disk}"
345 alias = "nixos/virtual-machine/${variant}"
346 variant = "${variant}"
347
348 with subtest("virtual-machine image can be imported"):
349 machine.succeed(f"incus image import {metadata} {disk} --alias {alias}")
350
351
352 with subtest("virtual-machine can be created"):
353 machine.succeed(f"incus create {alias} vm-{variant}1 --vm --config limits.memory=512MB --config security.secureboot=false")
354
355
356 with subtest("virtual-machine software tpm can be configured"):
357 machine.succeed(f"incus config device add vm-{variant}1 vtpm tpm path=/dev/tpm0")
358
359
360 with subtest("virtual-machine can be launched and become available"):
361 machine.succeed(f"incus start vm-{variant}1")
362 wait_for_instance(f"vm-{variant}1")
363
364
365 with subtest("virtual-machine incus-agent is started"):
366 machine.succeed(f"incus exec vm-{variant}1 systemctl is-active incus-agent")
367
368
369 with subtest("virtual-machine incus-agent has a valid path"):
370 machine.succeed(f"incus exec vm-{variant}1 -- bash -c 'true'")
371
372
373 with subtest("virtual-machine CPU limits can be managed"):
374 set_config(f"vm-{variant}1", "limits.cpu 1", restart=True)
375 wait_incus_exec_success(f"vm-{variant}1", "nproc | grep '^1$'", timeout=90)
376
377
378 with subtest("virtual-machine CPU limits can be hotplug changed"):
379 set_config(f"vm-{variant}1", "limits.cpu 2")
380 wait_incus_exec_success(f"vm-{variant}1", "nproc | grep '^2$'", timeout=90)
381
382
383 with subtest("virtual-machine can successfully restart"):
384 machine.succeed(f"incus restart vm-{variant}1")
385 wait_for_instance(f"vm-{variant}1")
386
387
388 with subtest("virtual-machine remains running when softDaemonRestart is enabled and service is stopped"):
389 pid = machine.succeed(f"incus info vm-{variant}1 | grep 'PID'").split(":")[1].strip()
390 machine.succeed(f"ps {pid}")
391 machine.succeed("systemctl stop incus")
392 machine.succeed(f"ps {pid}")
393 machine.succeed("systemctl start incus")
394
395
396 with subtest("virtual-machines stop with incus-startup.service"):
397 pid = machine.succeed(f"incus info vm-{variant}1 | grep 'PID'").split(":")[1].strip()
398 machine.succeed(f"ps {pid}")
399 machine.succeed("systemctl stop incus-startup.service")
400 machine.wait_until_fails(f"ps {pid}", timeout=120)
401 machine.succeed("systemctl start incus-startup.service")
402
403
404 cleanup()
405 ''
406 ) "" initVariants)
407 +
408 # python
409 ''
410 with subtest("virtual-machine can launch CSM (BIOS)"):
411 machine.succeed("incus init csm --vm --empty -c security.csm=true -c security.secureboot=false")
412 machine.succeed("incus start csm")
413
414
415 cleanup()
416 ''
417 )
418 +
419 lib.optionalString featureUser # python
420 ''
421 with subtest("incus-user allows restricted access for users"):
422 machine.fail("incus project show user-1000")
423 machine.succeed("su - testuser bash -c 'incus list'")
424 # a project is created dynamically for the user
425 machine.succeed("incus project show user-1000")
426 # users shouldn't be able to list storage pools
427 machine.fail("su - testuser bash -c 'incus storage list'")
428
429
430 with subtest("incus-user allows users to launch instances"):
431 machine.succeed("su - testuser bash -c 'incus image import ${(images "systemd").container.metadata} ${(images "systemd").container.rootfs} --alias nixos'")
432 machine.succeed("su - testuser bash -c 'incus launch nixos instance2'")
433 wait_for_instance("instance2", "user-1000")
434
435 cleanup()
436 ''
437 +
438 lib.optionalString networkOvs # python
439 ''
440 with subtest("Verify openvswitch bridge"):
441 machine.succeed("incus network info ovsbr0")
442
443
444 with subtest("Verify openvswitch bridge"):
445 machine.succeed("ovs-vsctl br-exists ovsbr0")
446 ''
447
448 +
449 lib.optionalString storageZfs # python
450 ''
451 with subtest("Verify zfs pool created and usable"):
452 machine.succeed(
453 "zpool status",
454 "parted --script /dev/vdb mklabel gpt",
455 "zpool create zfs_pool /dev/vdb",
456 )
457
458 machine.succeed("incus storage create zfs_pool zfs source=zfs_pool/incus")
459 machine.succeed("zfs list zfs_pool/incus")
460
461 machine.succeed("incus storage volume create zfs_pool test_fs --type filesystem")
462 machine.succeed("incus storage volume create zfs_pool test_vol --type block")
463
464 machine.succeed("incus storage show zfs_pool")
465 machine.succeed("incus storage volume list zfs_pool")
466 machine.succeed("incus storage volume show zfs_pool test_fs")
467 machine.succeed("incus storage volume show zfs_pool test_vol")
468
469 machine.succeed("incus create zfs1 --empty --storage zfs_pool")
470 machine.succeed("incus list zfs1")
471 ''
472
473 +
474 lib.optionalString storageLvm # python
475 ''
476 with subtest("Verify lvm pool created and usable"):
477 machine.succeed("incus storage create lvm_pool lvm source=/dev/vdc lvm.vg_name=incus_pool")
478 machine.succeed("vgs incus_pool")
479
480 machine.succeed("incus storage volume create lvm_pool test_fs --type filesystem")
481 machine.succeed("incus storage volume create lvm_pool test_vol --type block")
482
483 machine.succeed("incus storage show lvm_pool")
484
485 machine.succeed("incus storage volume list lvm_pool")
486 machine.succeed("incus storage volume show lvm_pool test_fs")
487 machine.succeed("incus storage volume show lvm_pool test_vol")
488
489 machine.succeed("incus create lvm1 --empty --storage lvm_pool")
490 machine.succeed("incus list lvm1")
491 '';
492 }
493)