pyrox.dev
1import ../make-test-python.nix (
2 {
3 pkgs,
4 lib,
5
6 lts ? true,
7
8 allTests ? false,
9
10 appArmor ? false,
11 featureUser ? allTests,
12 initLegacy ? true,
13 initSystemd ? true,
14 instanceContainer ? allTests,
15 instanceVm ? allTests,
16 networkOvs ? allTests,
17 storageLvm ? allTests,
18 storageZfs ? allTests,
19 ...
20 }:
21
22 let
23 releases =
24 init:
25 import ../../release.nix {
26 configuration = {
27 # Building documentation makes the test unnecessarily take a longer time:
28 documentation.enable = lib.mkForce false;
29
30 boot.initrd.systemd.enable = init == "systemd";
31
32 # Arbitrary sysctl modification to ensure containers can update sysctl
33 boot.kernel.sysctl."net.ipv4.ip_forward" = "1";
34 };
35 };
36
37 images = init: {
38 container = {
39 metadata =
40 (releases init).incusContainerMeta.${pkgs.stdenv.hostPlatform.system}
41 + "/tarball/nixos-image-lxc-*-${pkgs.stdenv.hostPlatform.system}.tar.xz";
42
43 rootfs =
44 (releases init).incusContainerImage.${pkgs.stdenv.hostPlatform.system}
45 + "/nixos-lxc-image-${pkgs.stdenv.hostPlatform.system}.squashfs";
46 };
47
48 virtual-machine = {
49 metadata =
50 (releases init).incusVirtualMachineImageMeta.${pkgs.stdenv.hostPlatform.system} + "/*/*.tar.xz";
51 disk = (releases init).incusVirtualMachineImage.${pkgs.stdenv.hostPlatform.system} + "/nixos.qcow2";
52 };
53 };
54
55 initVariants = lib.optionals initLegacy [ "legacy" ] ++ lib.optionals initSystemd [ "systemd" ];
56
57 canTestVm = instanceVm && pkgs.stdenv.isLinux && pkgs.stdenv.isx86_64;
58 in
59 {
60 name = "incus" + lib.optionalString lts "-lts";
61
62 meta = {
63 maintainers = lib.teams.lxc.members;
64 };
65
66 nodes.machine = {
67 virtualisation = {
68 cores = 2;
69 memorySize = 2048;
70 diskSize = 12 * 1024;
71 emptyDiskImages = [
72 # vdb for zfs
73 2048
74 # vdc for lvm
75 2048
76 ];
77
78 incus = {
79 enable = true;
80 package = if lts then pkgs.incus-lts else pkgs.incus;
81
82 preseed = {
83 networks = [
84 {
85 name = "incusbr0";
86 type = "bridge";
87 config = {
88 "ipv4.address" = "10.0.10.1/24";
89 "ipv4.nat" = "true";
90 };
91 }
92 ]
93 ++ lib.optionals networkOvs [
94 {
95 name = "ovsbr0";
96 type = "bridge";
97 config = {
98 "bridge.driver" = "openvswitch";
99 "ipv4.address" = "10.0.20.1/24";
100 "ipv4.nat" = "true";
101 };
102 }
103 ];
104 profiles = [
105 {
106 name = "default";
107 devices = {
108 eth0 = {
109 name = "eth0";
110 network = "incusbr0";
111 type = "nic";
112 };
113 root = {
114 path = "/";
115 pool = "default";
116 size = "35GiB";
117 type = "disk";
118 };
119 };
120 }
121 ];
122 storage_pools = [
123 {
124 name = "default";
125 driver = "dir";
126 }
127 ];
128 };
129 };
130
131 vswitch.enable = networkOvs;
132 };
133
134 boot.supportedFilesystems = lib.optionals storageZfs [ "zfs" ];
135 boot.zfs.forceImportRoot = false;
136
137 environment.systemPackages = [ pkgs.parted ];
138
139 networking.hostId = "01234567";
140 networking.firewall.trustedInterfaces = [ "incusbr0" ];
141
142 security.apparmor.enable = appArmor;
143 services.dbus.apparmor = (if appArmor then "enabled" else "disabled");
144
145 services.lvm = {
146 boot.thin.enable = storageLvm;
147 dmeventd.enable = storageLvm;
148 };
149
150 networking.nftables.enable = true;
151
152 users.users.testuser = {
153 isNormalUser = true;
154 shell = pkgs.bashInteractive;
155 group = "incus";
156 uid = 1000;
157 };
158 };
159
160 testScript = # python
161 ''
162 import json
163
164 def wait_for_instance(name: str, project: str = "default"):
165 machine.wait_until_succeeds(f"incus exec {name} --disable-stdin --force-interactive --project {project} -- /run/current-system/sw/bin/systemctl is-system-running")
166
167
168 def wait_incus_exec_success(name: str, command: str, timeout: int = 900, project: str = "default"):
169 def check_command(_) -> bool:
170 status, _ = machine.execute(f"incus exec {name} --disable-stdin --force-interactive --project {project} -- {command}")
171 return status == 0
172
173 with machine.nested(f"Waiting for successful exec: {command}"):
174 retry(check_command, timeout)
175
176
177 def set_config(name: str, config: str, restart: bool = False, unset: bool = False):
178 if restart:
179 machine.succeed(f"incus stop {name}")
180
181 if unset:
182 machine.succeed(f"incus config unset {name} {config}")
183 else:
184 machine.succeed(f"incus config set {name} {config}")
185
186 if restart:
187 machine.succeed(f"incus start {name}")
188 wait_for_instance(name)
189 else:
190 # give a moment to settle
191 machine.sleep(1)
192
193
194 def cleanup():
195 # avoid conflict between preseed and cleanup operations
196 machine.execute("systemctl kill incus-preseed.service")
197
198 instances = json.loads(machine.succeed("incus list --format json --all-projects"))
199 with subtest("Stopping all running instances"):
200 for instance in [a for a in instances if a['status'] == 'Running']:
201 machine.execute(f"incus stop --force {instance['name']} --project {instance['project']}")
202 machine.execute(f"incus delete --force {instance['name']} --project {instance['project']}")
203
204
205 def check_sysctl(name: str):
206 with subtest("systemd sysctl settings are applied"):
207 machine.succeed(f"incus exec {name} -- systemctl status systemd-sysctl")
208 sysctl = machine.succeed(f"incus exec {name} -- sysctl net.ipv4.ip_forward").strip().split(" ")[-1]
209 assert "1" == sysctl, f"systemd-sysctl configuration not correctly applied, {sysctl} != 1"
210
211
212 with subtest("Wait for startup"):
213 machine.wait_for_unit("incus.service")
214 machine.wait_for_unit("incus-preseed.service")
215
216
217 with subtest("Verify preseed resources created"):
218 machine.succeed("incus profile show default")
219 machine.succeed("incus network info incusbr0")
220 machine.succeed("incus storage show default")
221
222 ''
223 + lib.optionalString appArmor ''
224 with subtest("Verify AppArmor service is started without issue"):
225 # restart AppArmor service since the Incus AppArmor folders are
226 # created after AA service is started
227 machine.systemctl("restart apparmor.service")
228 machine.succeed("systemctl --no-pager -l status apparmor.service")
229 machine.wait_for_unit("apparmor.service")
230 ''
231 + lib.optionalString instanceContainer (
232 lib.foldl (
233 acc: variant:
234 acc
235 # python
236 + ''
237 metadata = "${(images variant).container.metadata}"
238 rootfs = "${(images variant).container.rootfs}"
239 alias = "nixos/container/${variant}"
240 variant = "${variant}"
241
242 with subtest("container image can be imported"):
243 machine.succeed(f"incus image import {metadata} {rootfs} --alias {alias}")
244
245
246 with subtest("container can be launched and managed"):
247 machine.succeed(f"incus launch {alias} container-{variant}1")
248 wait_for_instance(f"container-{variant}1")
249
250
251 with subtest("container mounts lxcfs overlays"):
252 machine.succeed(f"incus exec container-{variant}1 mount | grep 'lxcfs on /proc/cpuinfo type fuse.lxcfs'")
253 machine.succeed(f"incus exec container-{variant}1 mount | grep 'lxcfs on /proc/meminfo type fuse.lxcfs'")
254
255
256 with subtest("container CPU limits can be managed"):
257 set_config(f"container-{variant}1", "limits.cpu 1", restart=True)
258 wait_incus_exec_success(f"container-{variant}1", "nproc | grep '^1$'", timeout=90)
259
260
261 with subtest("container CPU limits can be hotplug changed"):
262 set_config(f"container-{variant}1", "limits.cpu 2")
263 wait_incus_exec_success(f"container-{variant}1", "nproc | grep '^2$'", timeout=90)
264
265
266 with subtest("container memory limits can be managed"):
267 set_config(f"container-{variant}1", "limits.memory 128MB", restart=True)
268 wait_incus_exec_success(f"container-{variant}1", "grep 'MemTotal:[[:space:]]*125000 kB' /proc/meminfo", timeout=90)
269
270
271 with subtest("container memory limits can be hotplug changed"):
272 set_config(f"container-{variant}1", "limits.memory 256MB")
273 wait_incus_exec_success(f"container-{variant}1", "grep 'MemTotal:[[:space:]]*250000 kB' /proc/meminfo", timeout=90)
274
275
276 with subtest("container software tpm can be configured"):
277 machine.succeed(f"incus config device add container-{variant}1 vtpm tpm path=/dev/tpm0 pathrm=/dev/tpmrm0")
278 machine.succeed(f"incus exec container-{variant}1 -- test -e /dev/tpm0")
279 machine.succeed(f"incus exec container-{variant}1 -- test -e /dev/tpmrm0")
280 machine.succeed(f"incus config device remove container-{variant}1 vtpm")
281 machine.fail(f"incus exec container-{variant}1 -- test -e /dev/tpm0")
282
283
284 with subtest("container lxc-generator compatibility"):
285 with subtest("lxc-container generator configures plain container"):
286 # default container is plain
287 machine.succeed(f"incus exec container-{variant}1 test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")
288
289 check_sysctl(f"container-{variant}1")
290
291 with subtest("lxc-container generator configures nested container"):
292 set_config(f"container-{variant}1", "security.nesting=true", restart=True)
293
294 machine.fail(f"incus exec container-{variant}1 test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")
295 target = machine.succeed(f"incus exec container-{variant}1 readlink -- -f /run/systemd/system/systemd-binfmt.service").strip()
296 assert target == "/dev/null", "lxc generator did not correctly mask /run/systemd/system/systemd-binfmt.service"
297
298 check_sysctl(f"container-{variant}1")
299
300 with subtest("lxc-container generator configures privileged container"):
301 # Create a new instance for a clean state
302 machine.succeed(f"incus launch {alias} container-{variant}2")
303 wait_for_instance(f"container-{variant}2")
304
305 machine.succeed(f"incus exec container-{variant}2 test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")
306
307 check_sysctl(f"container-{variant}2")
308
309 with subtest("container supports per-instance lxcfs"):
310 machine.succeed(f"incus stop container-{variant}1")
311 machine.fail(f"pgrep -a lxcfs | grep 'incus/devices/container-{variant}1/lxcfs'")
312
313 machine.succeed("incus config set instances.lxcfs.per_instance=true")
314
315 machine.succeed(f"incus start container-{variant}1")
316 wait_for_instance(f"container-{variant}1")
317 machine.succeed(f"pgrep -a lxcfs | grep 'incus/devices/container-{variant}1/lxcfs'")
318
319
320 with subtest("container can successfully restart"):
321 machine.succeed(f"incus restart container-{variant}1")
322 wait_for_instance(f"container-{variant}1")
323
324
325 with subtest("container remains running when softDaemonRestart is enabled and service is stopped"):
326 pid = machine.succeed(f"incus info container-{variant}1 | grep 'PID'").split(":")[1].strip()
327 machine.succeed(f"ps {pid}")
328 machine.succeed("systemctl stop incus")
329 machine.succeed(f"ps {pid}")
330 machine.succeed("systemctl start incus")
331
332 with subtest("containers stop with incus-startup.service"):
333 pid = machine.succeed(f"incus info container-{variant}1 | grep 'PID'").split(":")[1].strip()
334 machine.succeed(f"ps {pid}")
335 machine.succeed("systemctl stop incus-startup.service")
336 machine.wait_until_fails(f"ps {pid}", timeout=120)
337 machine.succeed("systemctl start incus-startup.service")
338
339
340 cleanup()
341 ''
342 ) "" initVariants
343 )
344 + lib.optionalString canTestVm (
345 (lib.foldl (
346 acc: variant:
347 acc
348 # python
349 + ''
350 metadata = "${(images variant).virtual-machine.metadata}"
351 disk = "${(images variant).virtual-machine.disk}"
352 alias = "nixos/virtual-machine/${variant}"
353 variant = "${variant}"
354
355 with subtest("virtual-machine image can be imported"):
356 machine.succeed(f"incus image import {metadata} {disk} --alias {alias}")
357
358
359 with subtest("virtual-machine can be created"):
360 machine.succeed(f"incus create {alias} vm-{variant}1 --vm --config limits.memory=512MB --config security.secureboot=false")
361
362
363 with subtest("virtual-machine software tpm can be configured"):
364 machine.succeed(f"incus config device add vm-{variant}1 vtpm tpm path=/dev/tpm0")
365
366
367 with subtest("virtual-machine can be launched and become available"):
368 machine.succeed(f"incus start vm-{variant}1")
369 wait_for_instance(f"vm-{variant}1")
370
371
372 with subtest("virtual-machine incus-agent is started"):
373 machine.succeed(f"incus exec vm-{variant}1 systemctl is-active incus-agent")
374
375
376 with subtest("virtual-machine incus-agent has a valid path"):
377 machine.succeed(f"incus exec vm-{variant}1 -- bash -c 'true'")
378
379
380 with subtest("virtual-machine CPU limits can be managed"):
381 set_config(f"vm-{variant}1", "limits.cpu 1", restart=True)
382 wait_incus_exec_success(f"vm-{variant}1", "nproc | grep '^1$'", timeout=90)
383
384
385 with subtest("virtual-machine CPU limits can be hotplug changed"):
386 set_config(f"vm-{variant}1", "limits.cpu 2")
387 wait_incus_exec_success(f"vm-{variant}1", "nproc | grep '^2$'", timeout=90)
388
389
390 with subtest("virtual-machine can successfully restart"):
391 machine.succeed(f"incus restart vm-{variant}1")
392 wait_for_instance(f"vm-{variant}1")
393
394
395 with subtest("virtual-machine remains running when softDaemonRestart is enabled and service is stopped"):
396 pid = machine.succeed(f"incus info vm-{variant}1 | grep 'PID'").split(":")[1].strip()
397 machine.succeed(f"ps {pid}")
398 machine.succeed("systemctl stop incus")
399 machine.succeed(f"ps {pid}")
400 machine.succeed("systemctl start incus")
401
402
403 with subtest("virtual-machines stop with incus-startup.service"):
404 pid = machine.succeed(f"incus info vm-{variant}1 | grep 'PID'").split(":")[1].strip()
405 machine.succeed(f"ps {pid}")
406 machine.succeed("systemctl stop incus-startup.service")
407 machine.wait_until_fails(f"ps {pid}", timeout=120)
408 machine.succeed("systemctl start incus-startup.service")
409
410
411 cleanup()
412 ''
413 ) "" initVariants)
414 +
415 # python
416 ''
417 with subtest("virtual-machine can launch CSM (BIOS)"):
418 machine.succeed("incus init csm --vm --empty -c security.csm=true -c security.secureboot=false")
419 machine.succeed("incus start csm")
420
421
422 cleanup()
423 ''
424 )
425 +
426 lib.optionalString featureUser # python
427 ''
428 with subtest("incus-user allows restricted access for users"):
429 machine.fail("incus project show user-1000")
430 machine.succeed("su - testuser bash -c 'incus list'")
431 # a project is created dynamically for the user
432 machine.succeed("incus project show user-1000")
433 # users shouldn't be able to list storage pools
434 machine.fail("su - testuser bash -c 'incus storage list'")
435
436
437 with subtest("incus-user allows users to launch instances"):
438 machine.succeed("su - testuser bash -c 'incus image import ${(images "systemd").container.metadata} ${(images "systemd").container.rootfs} --alias nixos'")
439 machine.succeed("su - testuser bash -c 'incus launch nixos instance2'")
440 wait_for_instance("instance2", "user-1000")
441
442 cleanup()
443 ''
444 +
445 lib.optionalString networkOvs # python
446 ''
447 with subtest("Verify openvswitch bridge"):
448 machine.succeed("incus network info ovsbr0")
449
450
451 with subtest("Verify openvswitch bridge"):
452 machine.succeed("ovs-vsctl br-exists ovsbr0")
453 ''
454
455 +
456 lib.optionalString storageZfs # python
457 ''
458 with subtest("Verify zfs pool created and usable"):
459 machine.succeed(
460 "zpool status",
461 "parted --script /dev/vdb mklabel gpt",
462 "zpool create zfs_pool /dev/vdb",
463 )
464
465 machine.succeed("incus storage create zfs_pool zfs source=zfs_pool/incus")
466 machine.succeed("zfs list zfs_pool/incus")
467
468 machine.succeed("incus storage volume create zfs_pool test_fs --type filesystem")
469 machine.succeed("incus storage volume create zfs_pool test_vol --type block")
470
471 machine.succeed("incus storage show zfs_pool")
472 machine.succeed("incus storage volume list zfs_pool")
473 machine.succeed("incus storage volume show zfs_pool test_fs")
474 machine.succeed("incus storage volume show zfs_pool test_vol")
475
476 machine.succeed("incus create zfs1 --empty --storage zfs_pool")
477 machine.succeed("incus list zfs1")
478 ''
479
480 +
481 lib.optionalString storageLvm # python
482 ''
483 with subtest("Verify lvm pool created and usable"):
484 machine.succeed("incus storage create lvm_pool lvm source=/dev/vdc lvm.vg_name=incus_pool")
485 machine.succeed("vgs incus_pool")
486
487 machine.succeed("incus storage volume create lvm_pool test_fs --type filesystem")
488 machine.succeed("incus storage volume create lvm_pool test_vol --type block")
489
490 machine.succeed("incus storage show lvm_pool")
491
492 machine.succeed("incus storage volume list lvm_pool")
493 machine.succeed("incus storage volume show lvm_pool test_fs")
494 machine.succeed("incus storage volume show lvm_pool test_vol")
495
496 machine.succeed("incus create lvm1 --empty --storage lvm_pool")
497 machine.succeed("incus list lvm1")
498 '';
499 }
500)