pyrox.dev
1{ system ? builtins.currentSystem,
2 config ? {},
3 pkgs ? import ../.. { inherit system config; },
4 debug ? false,
5 enableUnfree ? false,
6 # Nested KVM virtualization (https://www.linux-kvm.org/page/Nested_Guests)
7 # requires a modprobe flag on the build machine: (kvm-amd for AMD CPUs)
8 # boot.extraModprobeConfig = "options kvm-intel nested=Y";
9 # Without this VirtualBox will use SW virtualization and will only be able
10 # to run 32-bit guests.
11 useKvmNestedVirt ? false,
12 # Whether to run 64-bit guests instead of 32-bit. Requires nested KVM.
13 use64bitGuest ? false
14}:
15
16assert use64bitGuest -> useKvmNestedVirt;
17
18with import ../lib/testing-python.nix { inherit system pkgs; };
19with pkgs.lib;
20
21let
22 testVMConfig = vmName: attrs: { config, pkgs, lib, ... }: let
23 guestAdditions = pkgs.linuxPackages.virtualboxGuestAdditions;
24
25 miniInit = ''
26 #!${pkgs.runtimeShell} -xe
27 export PATH="${lib.makeBinPath [ pkgs.coreutils pkgs.util-linux ]}"
28
29 mkdir -p /run/dbus
30 cat > /etc/passwd <<EOF
31 root:x:0:0::/root:/bin/false
32 messagebus:x:1:1::/run/dbus:/bin/false
33 EOF
34 cat > /etc/group <<EOF
35 root:x:0:
36 messagebus:x:1:
37 EOF
38
39 "${pkgs.dbus.daemon}/bin/dbus-daemon" --fork \
40 --config-file="${pkgs.dbus.daemon}/share/dbus-1/system.conf"
41
42 ${guestAdditions}/bin/VBoxService
43 ${(attrs.vmScript or (const "")) pkgs}
44
45 i=0
46 while [ ! -e /mnt-root/shutdown ]; do
47 sleep 10
48 i=$(($i + 10))
49 [ $i -le 120 ] || fail
50 done
51
52 rm -f /mnt-root/boot-done /mnt-root/shutdown
53 '';
54 in {
55 boot.kernelParams = [
56 "console=tty0" "console=ttyS0" "ignore_loglevel"
57 "boot.trace" "panic=1" "boot.panic_on_fail"
58 "init=${pkgs.writeScript "mini-init.sh" miniInit}"
59 ];
60
61 fileSystems."/" = {
62 device = "vboxshare";
63 fsType = "vboxsf";
64 };
65
66 virtualisation.virtualbox.guest.enable = true;
67
68 boot.initrd.kernelModules = [
69 "af_packet" "vboxsf"
70 "virtio" "virtio_pci" "virtio_ring" "virtio_net" "vboxguest"
71 ];
72
73 boot.initrd.extraUtilsCommands = ''
74 copy_bin_and_libs "${guestAdditions}/bin/mount.vboxsf"
75 copy_bin_and_libs "${pkgs.util-linux}/bin/unshare"
76 ${(attrs.extraUtilsCommands or (const "")) pkgs}
77 '';
78
79 boot.initrd.postMountCommands = ''
80 touch /mnt-root/boot-done
81 hostname "${vmName}"
82 mkdir -p /nix/store
83 unshare -m ${escapeShellArg pkgs.runtimeShell} -c '
84 mount -t vboxsf nixstore /nix/store
85 exec "$stage2Init"
86 '
87 poweroff -f
88 '';
89
90 system.requiredKernelConfig = with config.lib.kernelConfig; [
91 (isYes "SERIAL_8250_CONSOLE")
92 (isYes "SERIAL_8250")
93 ];
94
95 networking.usePredictableInterfaceNames = false;
96 };
97
98 mkLog = logfile: tag: let
99 rotated = map (i: "${logfile}.${toString i}") (range 1 9);
100 all = concatMapStringsSep " " (f: "\"${f}\"") ([logfile] ++ rotated);
101 logcmd = "tail -F ${all} 2> /dev/null | logger -t \"${tag}\"";
102 in if debug then "machine.execute(ru('${logcmd} & disown'))" else "pass";
103
104 testVM = vmName: vmScript: let
105 cfg = (import ../lib/eval-config.nix {
106 system = if use64bitGuest then "x86_64-linux" else "i686-linux";
107 modules = [
108 ../modules/profiles/minimal.nix
109 (testVMConfig vmName vmScript)
110 ];
111 }).config;
112 in pkgs.vmTools.runInLinuxVM (pkgs.runCommand "virtualbox-image" {
113 preVM = ''
114 mkdir -p "$out"
115 diskImage="$(pwd)/qimage"
116 ${pkgs.vmTools.qemu}/bin/qemu-img create -f raw "$diskImage" 100M
117 '';
118
119 postVM = ''
120 echo "creating VirtualBox disk image..."
121 ${pkgs.vmTools.qemu}/bin/qemu-img convert -f raw -O vdi \
122 "$diskImage" "$out/disk.vdi"
123 '';
124
125 buildInputs = [ pkgs.util-linux pkgs.perl ];
126 } ''
127 ${pkgs.parted}/sbin/parted --script /dev/vda mklabel msdos
128 ${pkgs.parted}/sbin/parted --script /dev/vda -- mkpart primary ext2 1M -1s
129 ${pkgs.e2fsprogs}/sbin/mkfs.ext4 /dev/vda1
130 ${pkgs.e2fsprogs}/sbin/tune2fs -c 0 -i 0 /dev/vda1
131 mkdir /mnt
132 mount /dev/vda1 /mnt
133 cp "${cfg.system.build.kernel}/bzImage" /mnt/linux
134 cp "${cfg.system.build.initialRamdisk}/initrd" /mnt/initrd
135
136 ${pkgs.grub2}/bin/grub-install --boot-directory=/mnt /dev/vda
137
138 cat > /mnt/grub/grub.cfg <<GRUB
139 set root=hd0,1
140 linux /linux ${concatStringsSep " " cfg.boot.kernelParams}
141 initrd /initrd
142 boot
143 GRUB
144 umount /mnt
145 '');
146
147 createVM = name: attrs: let
148 mkFlags = concatStringsSep " ";
149
150 sharePath = "/home/alice/vboxshare-${name}";
151
152 createFlags = mkFlags [
153 "--ostype ${if use64bitGuest then "Linux26_64" else "Linux26"}"
154 "--register"
155 ];
156
157 vmFlags = mkFlags ([
158 "--uart1 0x3F8 4"
159 "--uartmode1 client /run/virtualbox-log-${name}.sock"
160 "--memory 768"
161 "--audio none"
162 ] ++ (attrs.vmFlags or []));
163
164 controllerFlags = mkFlags [
165 "--name SATA"
166 "--add sata"
167 "--bootable on"
168 "--hostiocache on"
169 ];
170
171 diskFlags = mkFlags [
172 "--storagectl SATA"
173 "--port 0"
174 "--device 0"
175 "--type hdd"
176 "--mtype immutable"
177 "--medium ${testVM name attrs}/disk.vdi"
178 ];
179
180 sharedFlags = mkFlags [
181 "--name vboxshare"
182 "--hostpath ${sharePath}"
183 ];
184
185 nixstoreFlags = mkFlags [
186 "--name nixstore"
187 "--hostpath /nix/store"
188 "--readonly"
189 ];
190 in {
191 machine = {
192 systemd.sockets."vboxtestlog-${name}" = {
193 description = "VirtualBox Test Machine Log Socket For ${name}";
194 wantedBy = [ "sockets.target" ];
195 before = [ "multi-user.target" ];
196 socketConfig.ListenStream = "/run/virtualbox-log-${name}.sock";
197 socketConfig.Accept = true;
198 };
199
200 systemd.services."vboxtestlog-${name}@" = {
201 description = "VirtualBox Test Machine Log For ${name}";
202 serviceConfig.StandardInput = "socket";
203 serviceConfig.SyslogIdentifier = "GUEST-${name}";
204 serviceConfig.ExecStart = "${pkgs.coreutils}/bin/cat";
205 };
206 };
207
208 testSubs = ''
209
210
211 ${name}_sharepath = "${sharePath}"
212
213
214 def check_running_${name}():
215 cmd = "VBoxManage list runningvms | grep -q '^\"${name}\"'"
216 (status, _) = machine.execute(ru(cmd))
217 return status == 0
218
219
220 def cleanup_${name}():
221 if check_running_${name}():
222 machine.execute(ru("VBoxManage controlvm ${name} poweroff"))
223 machine.succeed("rm -rf ${sharePath}")
224 machine.succeed("mkdir -p ${sharePath}")
225 machine.succeed("chown alice.users ${sharePath}")
226
227
228 def create_vm_${name}():
229 vbm("createvm --name ${name} ${createFlags}")
230 vbm("modifyvm ${name} ${vmFlags}")
231 vbm("setextradata ${name} VBoxInternal/PDM/HaltOnReset 1")
232 vbm("storagectl ${name} ${controllerFlags}")
233 vbm("storageattach ${name} ${diskFlags}")
234 vbm("sharedfolder add ${name} ${sharedFlags}")
235 vbm("sharedfolder add ${name} ${nixstoreFlags}")
236 cleanup_${name}()
237
238 ${mkLog "$HOME/VirtualBox VMs/${name}/Logs/VBox.log" "HOST-${name}"}
239
240
241 def destroy_vm_${name}():
242 cleanup_${name}()
243 vbm("unregistervm ${name} --delete")
244
245
246 def wait_for_vm_boot_${name}():
247 machine.execute(
248 ru(
249 "set -e; i=0; "
250 "while ! test -e ${sharePath}/boot-done; do "
251 "sleep 10; i=$(($i + 10)); [ $i -le 3600 ]; "
252 "VBoxManage list runningvms | grep -q '^\"${name}\"'; "
253 "done"
254 )
255 )
256
257
258 def wait_for_ip_${name}(interface):
259 property = f"/VirtualBox/GuestInfo/Net/{interface}/V4/IP"
260 getip = f"VBoxManage guestproperty get ${name} {property} | sed -n -e 's/^Value: //p'"
261
262 ip = machine.succeed(
263 ru(
264 "for i in $(seq 1000); do "
265 f'if ipaddr="$({getip})" && [ -n "$ipaddr" ]; then '
266 'echo "$ipaddr"; exit 0; '
267 "fi; "
268 "sleep 1; "
269 "done; "
270 "echo 'Could not get IPv4 address for ${name}!' >&2; "
271 "exit 1"
272 )
273 ).strip()
274 return ip
275
276
277 def wait_for_startup_${name}(nudge=lambda: None):
278 for _ in range(0, 130, 10):
279 machine.sleep(10)
280 if check_running_${name}():
281 return
282 nudge()
283 raise Exception("VirtualBox VM didn't start up within 2 minutes")
284
285
286 def wait_for_shutdown_${name}():
287 for _ in range(0, 130, 10):
288 machine.sleep(10)
289 if not check_running_${name}():
290 return
291 raise Exception("VirtualBox VM didn't shut down within 2 minutes")
292
293
294 def shutdown_vm_${name}():
295 machine.succeed(ru("touch ${sharePath}/shutdown"))
296 machine.execute(
297 "set -e; i=0; "
298 "while test -e ${sharePath}/shutdown "
299 " -o -e ${sharePath}/boot-done; do "
300 "sleep 1; i=$(($i + 1)); [ $i -le 3600 ]; "
301 "done"
302 )
303 wait_for_shutdown_${name}()
304 '';
305 };
306
307 hostonlyVMFlags = [
308 "--nictype1 virtio"
309 "--nictype2 virtio"
310 "--nic2 hostonly"
311 "--hostonlyadapter2 vboxnet0"
312 ];
313
314 # The VirtualBox Oracle Extension Pack lets you use USB 3.0 (xHCI).
315 enableExtensionPackVMFlags = [
316 "--usbxhci on"
317 ];
318
319 dhcpScript = pkgs: ''
320 ${pkgs.dhcp}/bin/dhclient \
321 -lf /run/dhcp.leases \
322 -pf /run/dhclient.pid \
323 -v eth0 eth1
324
325 otherIP="$(${pkgs.netcat}/bin/nc -l 1234 || :)"
326 ${pkgs.iputils}/bin/ping -I eth1 -c1 "$otherIP"
327 echo "$otherIP reachable" | ${pkgs.netcat}/bin/nc -l 5678 || :
328 '';
329
330 sysdDetectVirt = pkgs: ''
331 ${pkgs.systemd}/bin/systemd-detect-virt > /mnt-root/result
332 '';
333
334 vboxVMs = mapAttrs createVM {
335 simple = {};
336
337 detectvirt.vmScript = sysdDetectVirt;
338
339 test1.vmFlags = hostonlyVMFlags;
340 test1.vmScript = dhcpScript;
341
342 test2.vmFlags = hostonlyVMFlags;
343 test2.vmScript = dhcpScript;
344
345 headless.virtualisation.virtualbox.headless = true;
346 headless.services.xserver.enable = false;
347 };
348
349 vboxVMsWithExtpack = mapAttrs createVM {
350 testExtensionPack.vmFlags = enableExtensionPackVMFlags;
351 };
352
353 mkVBoxTest = useExtensionPack: vms: name: testScript: makeTest {
354 name = "virtualbox-${name}";
355
356 machine = { lib, config, ... }: {
357 imports = let
358 mkVMConf = name: val: val.machine // { key = "${name}-config"; };
359 vmConfigs = mapAttrsToList mkVMConf vms;
360 in [ ./common/user-account.nix ./common/x11.nix ] ++ vmConfigs;
361 virtualisation.memorySize = 2048;
362 virtualisation.qemu.options =
363 if useKvmNestedVirt then ["-cpu" "kvm64,vmx=on"] else [];
364 virtualisation.virtualbox.host.enable = true;
365 test-support.displayManager.auto.user = "alice";
366 users.users.alice.extraGroups = let
367 inherit (config.virtualisation.virtualbox.host) enableHardening;
368 in lib.mkIf enableHardening (lib.singleton "vboxusers");
369 virtualisation.virtualbox.host.enableExtensionPack = useExtensionPack;
370 nixpkgs.config.allowUnfree = useExtensionPack;
371 };
372
373 testScript = ''
374 from shlex import quote
375 ${concatStrings (mapAttrsToList (_: getAttr "testSubs") vms)}
376
377 def ru(cmd: str) -> str:
378 return f"su - alice -c {quote(cmd)}"
379
380
381 def vbm(cmd: str) -> str:
382 return machine.succeed(ru(f"VBoxManage {cmd}"))
383
384
385 def remove_uuids(output: str) -> str:
386 return "\n".join(
387 [line for line in (output or "").splitlines() if not line.startswith("UUID:")]
388 )
389
390
391 machine.wait_for_x()
392
393 ${mkLog "$HOME/.config/VirtualBox/VBoxSVC.log" "HOST-SVC"}
394
395 ${testScript}
396 # (keep black happy)
397 '';
398
399 meta = with pkgs.lib.maintainers; {
400 maintainers = [ aszlig cdepillabout ];
401 };
402 };
403
404 unfreeTests = mapAttrs (mkVBoxTest true vboxVMsWithExtpack) {
405 enable-extension-pack = ''
406 create_vm_testExtensionPack()
407 vbm("startvm testExtensionPack")
408 wait_for_startup_testExtensionPack()
409 machine.screenshot("cli_started")
410 wait_for_vm_boot_testExtensionPack()
411 machine.screenshot("cli_booted")
412
413 with machine.nested("Checking for privilege escalation"):
414 machine.fail("test -e '/root/VirtualBox VMs'")
415 machine.fail("test -e '/root/.config/VirtualBox'")
416 machine.succeed("test -e '/home/alice/VirtualBox VMs'")
417
418 shutdown_vm_testExtensionPack()
419 destroy_vm_testExtensionPack()
420 '';
421 };
422
423in mapAttrs (mkVBoxTest false vboxVMs) {
424 simple-gui = ''
425 # Home to select Tools, down to move to the VM, enter to start it.
426 def send_vm_startup():
427 machine.send_key("home")
428 machine.send_key("down")
429 machine.send_key("ret")
430
431
432 create_vm_simple()
433 machine.succeed(ru("VirtualBox >&2 &"))
434 machine.wait_until_succeeds(ru("xprop -name 'Oracle VM VirtualBox Manager'"))
435 machine.sleep(5)
436 machine.screenshot("gui_manager_started")
437 send_vm_startup()
438 machine.screenshot("gui_manager_sent_startup")
439 wait_for_startup_simple(send_vm_startup)
440 machine.screenshot("gui_started")
441 wait_for_vm_boot_simple()
442 machine.screenshot("gui_booted")
443 shutdown_vm_simple()
444 machine.sleep(5)
445 machine.screenshot("gui_stopped")
446 machine.send_key("ctrl-q")
447 machine.sleep(5)
448 machine.screenshot("gui_manager_stopped")
449 destroy_vm_simple()
450 '';
451
452 simple-cli = ''
453 create_vm_simple()
454 vbm("startvm simple")
455 wait_for_startup_simple()
456 machine.screenshot("cli_started")
457 wait_for_vm_boot_simple()
458 machine.screenshot("cli_booted")
459
460 with machine.nested("Checking for privilege escalation"):
461 machine.fail("test -e '/root/VirtualBox VMs'")
462 machine.fail("test -e '/root/.config/VirtualBox'")
463 machine.succeed("test -e '/home/alice/VirtualBox VMs'")
464
465 shutdown_vm_simple()
466 destroy_vm_simple()
467 '';
468
469 headless = ''
470 create_vm_headless()
471 machine.succeed(ru("VBoxHeadless --startvm headless & disown %1"))
472 wait_for_startup_headless()
473 wait_for_vm_boot_headless()
474 shutdown_vm_headless()
475 destroy_vm_headless()
476 '';
477
478 host-usb-permissions = ''
479 user_usb = remove_uuids(vbm("list usbhost"))
480 print(user_usb, file=sys.stderr)
481 root_usb = remove_uuids(machine.succeed("VBoxManage list usbhost"))
482 print(root_usb, file=sys.stderr)
483
484 if user_usb != root_usb:
485 raise Exception("USB host devices differ for root and normal user")
486 if "<none>" in user_usb:
487 raise Exception("No USB host devices found")
488 '';
489
490 systemd-detect-virt = ''
491 create_vm_detectvirt()
492 vbm("startvm detectvirt")
493 wait_for_startup_detectvirt()
494 wait_for_vm_boot_detectvirt()
495 shutdown_vm_detectvirt()
496 result = machine.succeed(f"cat '{detectvirt_sharepath}/result'").strip()
497 destroy_vm_detectvirt()
498 if result != "oracle":
499 raise Exception(f'systemd-detect-virt returned "{result}" instead of "oracle"')
500 '';
501
502 net-hostonlyif = ''
503 create_vm_test1()
504 create_vm_test2()
505
506 vbm("startvm test1")
507 wait_for_startup_test1()
508 wait_for_vm_boot_test1()
509
510 vbm("startvm test2")
511 wait_for_startup_test2()
512 wait_for_vm_boot_test2()
513
514 machine.screenshot("net_booted")
515
516 test1_ip = wait_for_ip_test1(1)
517 test2_ip = wait_for_ip_test2(1)
518
519 machine.succeed(f"echo '{test2_ip}' | nc -N '{test1_ip}' 1234")
520 machine.succeed(f"echo '{test1_ip}' | nc -N '{test2_ip}' 1234")
521
522 machine.wait_until_succeeds(f"nc -N '{test1_ip}' 5678 < /dev/null >&2")
523 machine.wait_until_succeeds(f"nc -N '{test2_ip}' 5678 < /dev/null >&2")
524
525 shutdown_vm_test1()
526 shutdown_vm_test2()
527
528 destroy_vm_test1()
529 destroy_vm_test2()
530 '';
531} // (if enableUnfree then unfreeTests else {})