pyrox.dev
1# This module creates a virtual machine from the NixOS configuration.
2# Building the `config.system.build.vm' attribute gives you a command
3# that starts a KVM/QEMU VM running the NixOS configuration defined in
4# `config'. The Nix store is shared read-only with the host, which
5# makes (re)building VMs very efficient. However, it also means you
6# can't reconfigure the guest inside the guest - you need to rebuild
7# the VM in the host. On the other hand, the root filesystem is a
8# read/writable disk image persistent across VM reboots.
9
10{ config, lib, pkgs, ... }:
11
12with lib;
13with import ../../lib/qemu-flags.nix { inherit pkgs; };
14
15let
16
17 qemu = config.system.build.qemu or pkgs.qemu_test;
18
19 vmName =
20 if config.networking.hostName == ""
21 then "noname"
22 else config.networking.hostName;
23
24 cfg = config.virtualisation;
25
26 qemuGraphics = if cfg.graphics then "" else "-nographic";
27 kernelConsole = if cfg.graphics then "" else "console=${qemuSerialDevice}";
28 ttys = [ "tty1" "tty2" "tty3" "tty4" "tty5" "tty6" ];
29
30 # Shell script to start the VM.
31 startVM =
32 ''
33 #! ${pkgs.stdenv.shell}
34
35 NIX_DISK_IMAGE=$(readlink -f ''${NIX_DISK_IMAGE:-${config.virtualisation.diskImage}})
36
37 if ! test -e "$NIX_DISK_IMAGE"; then
38 ${qemu}/bin/qemu-img create -f qcow2 "$NIX_DISK_IMAGE" \
39 ${toString config.virtualisation.diskSize}M || exit 1
40 fi
41
42 # Create a directory for storing temporary data of the running VM.
43 if [ -z "$TMPDIR" -o -z "$USE_TMPDIR" ]; then
44 TMPDIR=$(mktemp -d nix-vm.XXXXXXXXXX --tmpdir)
45 fi
46
47 # Create a directory for exchanging data with the VM.
48 mkdir -p $TMPDIR/xchg
49
50 ${if cfg.useBootLoader then ''
51 # Create a writable copy/snapshot of the boot disk.
52 # A writable boot disk can be booted from automatically.
53 ${qemu}/bin/qemu-img create -f qcow2 -b ${bootDisk}/disk.img $TMPDIR/disk.img || exit 1
54
55 ${if cfg.useEFIBoot then ''
56 # VM needs a writable flash BIOS.
57 cp ${bootDisk}/bios.bin $TMPDIR || exit 1
58 chmod 0644 $TMPDIR/bios.bin || exit 1
59 '' else ''
60 ''}
61 '' else ''
62 ''}
63
64 cd $TMPDIR
65 idx=2
66 extraDisks=""
67 ${flip concatMapStrings cfg.emptyDiskImages (size: ''
68 if ! test -e "empty$idx.qcow2"; then
69 ${qemu}/bin/qemu-img create -f qcow2 "empty$idx.qcow2" "${toString size}M"
70 fi
71 extraDisks="$extraDisks -drive index=$idx,file=$(pwd)/empty$idx.qcow2,if=${cfg.qemu.diskInterface},werror=report"
72 idx=$((idx + 1))
73 '')}
74
75 # Start QEMU.
76 exec ${qemuBinary qemu} \
77 -name ${vmName} \
78 -m ${toString config.virtualisation.memorySize} \
79 -smp ${toString config.virtualisation.cores} \
80 ${concatStringsSep " " config.virtualisation.qemu.networkingOptions} \
81 -virtfs local,path=/nix/store,security_model=none,mount_tag=store \
82 -virtfs local,path=$TMPDIR/xchg,security_model=none,mount_tag=xchg \
83 -virtfs local,path=''${SHARED_DIR:-$TMPDIR/xchg},security_model=none,mount_tag=shared \
84 ${if cfg.useBootLoader then ''
85 -drive index=0,id=drive1,file=$NIX_DISK_IMAGE,if=${cfg.qemu.diskInterface},cache=writeback,werror=report \
86 -drive index=1,id=drive2,file=$TMPDIR/disk.img,media=disk \
87 ${if cfg.useEFIBoot then ''
88 -pflash $TMPDIR/bios.bin \
89 '' else ''
90 ''}
91 '' else ''
92 -drive index=0,id=drive1,file=$NIX_DISK_IMAGE,if=${cfg.qemu.diskInterface},cache=writeback,werror=report \
93 -kernel ${config.system.build.toplevel}/kernel \
94 -initrd ${config.system.build.toplevel}/initrd \
95 -append "$(cat ${config.system.build.toplevel}/kernel-params) init=${config.system.build.toplevel}/init regInfo=${regInfo}/registration ${kernelConsole} $QEMU_KERNEL_PARAMS" \
96 ''} \
97 $extraDisks \
98 ${qemuGraphics} \
99 ${toString config.virtualisation.qemu.options} \
100 $QEMU_OPTS \
101 $@
102 '';
103
104
105 regInfo = pkgs.closureInfo { rootPaths = config.virtualisation.pathsInNixDB; };
106
107
108 # Generate a hard disk image containing a /boot partition and GRUB
109 # in the MBR. Used when the `useBootLoader' option is set.
110 # FIXME: use nixos/lib/make-disk-image.nix.
111 bootDisk =
112 pkgs.vmTools.runInLinuxVM (
113 pkgs.runCommand "nixos-boot-disk"
114 { preVM =
115 ''
116 mkdir $out
117 diskImage=$out/disk.img
118 bootFlash=$out/bios.bin
119 ${qemu}/bin/qemu-img create -f qcow2 $diskImage "40M"
120 ${if cfg.useEFIBoot then ''
121 cp ${pkgs.OVMF-CSM.fd}/FV/OVMF.fd $bootFlash
122 chmod 0644 $bootFlash
123 '' else ''
124 ''}
125 '';
126 buildInputs = [ pkgs.utillinux ];
127 QEMU_OPTS = if cfg.useEFIBoot
128 then "-pflash $out/bios.bin -nographic -serial pty"
129 else "-nographic -serial pty";
130 }
131 ''
132 # Create a /boot EFI partition with 40M and arbitrary but fixed GUIDs for reproducibility
133 ${pkgs.gptfdisk}/bin/sgdisk \
134 --set-alignment=1 --new=1:34:2047 --change-name=1:BIOSBootPartition --typecode=1:ef02 \
135 --set-alignment=512 --largest-new=2 --change-name=2:EFISystem --typecode=2:ef00 \
136 --attributes=1:set:1 \
137 --attributes=2:set:2 \
138 --disk-guid=97FD5997-D90B-4AA3-8D16-C1723AEA73C1 \
139 --partition-guid=1:1C06F03B-704E-4657-B9CD-681A087A2FDC \
140 --partition-guid=2:970C694F-AFD0-4B99-B750-CDB7A329AB6F \
141 --hybrid 2 \
142 --recompute-chs /dev/vda
143 . /sys/class/block/vda2/uevent
144 mknod /dev/vda2 b $MAJOR $MINOR
145 . /sys/class/block/vda/uevent
146 ${pkgs.dosfstools}/bin/mkfs.fat -F16 /dev/vda2
147 export MTOOLS_SKIP_CHECK=1
148 ${pkgs.mtools}/bin/mlabel -i /dev/vda2 ::boot
149
150 # Mount /boot; load necessary modules first.
151 ${pkgs.kmod}/bin/insmod ${pkgs.linux}/lib/modules/*/kernel/fs/nls/nls_cp437.ko.xz || true
152 ${pkgs.kmod}/bin/insmod ${pkgs.linux}/lib/modules/*/kernel/fs/nls/nls_iso8859-1.ko.xz || true
153 ${pkgs.kmod}/bin/insmod ${pkgs.linux}/lib/modules/*/kernel/fs/fat/fat.ko.xz || true
154 ${pkgs.kmod}/bin/insmod ${pkgs.linux}/lib/modules/*/kernel/fs/fat/vfat.ko.xz || true
155 ${pkgs.kmod}/bin/insmod ${pkgs.linux}/lib/modules/*/kernel/fs/efivarfs/efivarfs.ko.xz || true
156 mkdir /boot
157 mount /dev/vda2 /boot
158
159 # This is needed for GRUB 0.97, which doesn't know about virtio devices.
160 mkdir /boot/grub
161 echo '(hd0) /dev/vda' > /boot/grub/device.map
162
163 # Install GRUB and generate the GRUB boot menu.
164 touch /etc/NIXOS
165 mkdir -p /nix/var/nix/profiles
166 ${config.system.build.toplevel}/bin/switch-to-configuration boot
167
168 umount /boot
169 '' # */
170 );
171
172in
173
174{
175 imports = [ ../profiles/qemu-guest.nix ];
176
177 options = {
178
179 virtualisation.memorySize =
180 mkOption {
181 default = 384;
182 description =
183 ''
184 Memory size (M) of virtual machine.
185 '';
186 };
187
188 virtualisation.diskSize =
189 mkOption {
190 default = 512;
191 description =
192 ''
193 Disk size (M) of virtual machine.
194 '';
195 };
196
197 virtualisation.diskImage =
198 mkOption {
199 default = "./${vmName}.qcow2";
200 description =
201 ''
202 Path to the disk image containing the root filesystem.
203 The image will be created on startup if it does not
204 exist.
205 '';
206 };
207
208 virtualisation.bootDevice =
209 mkOption {
210 type = types.str;
211 example = "/dev/vda";
212 description =
213 ''
214 The disk to be used for the root filesystem.
215 '';
216 };
217
218 virtualisation.emptyDiskImages =
219 mkOption {
220 default = [];
221 type = types.listOf types.int;
222 description =
223 ''
224 Additional disk images to provide to the VM. The value is
225 a list of size in megabytes of each disk. These disks are
226 writeable by the VM.
227 '';
228 };
229
230 virtualisation.graphics =
231 mkOption {
232 default = true;
233 description =
234 ''
235 Whether to run QEMU with a graphics window, or access
236 the guest computer serial port through the host tty.
237 '';
238 };
239
240 virtualisation.cores =
241 mkOption {
242 default = 1;
243 type = types.int;
244 description =
245 ''
246 Specify the number of cores the guest is permitted to use.
247 The number can be higher than the available cores on the
248 host system.
249 '';
250 };
251
252 virtualisation.pathsInNixDB =
253 mkOption {
254 default = [];
255 description =
256 ''
257 The list of paths whose closure is registered in the Nix
258 database in the VM. All other paths in the host Nix store
259 appear in the guest Nix store as well, but are considered
260 garbage (because they are not registered in the Nix
261 database in the guest).
262 '';
263 };
264
265 virtualisation.vlans =
266 mkOption {
267 default = [ 1 ];
268 example = [ 1 2 ];
269 description =
270 ''
271 Virtual networks to which the VM is connected. Each
272 number <replaceable>N</replaceable> in this list causes
273 the VM to have a virtual Ethernet interface attached to a
274 separate virtual network on which it will be assigned IP
275 address
276 <literal>192.168.<replaceable>N</replaceable>.<replaceable>M</replaceable></literal>,
277 where <replaceable>M</replaceable> is the index of this VM
278 in the list of VMs.
279 '';
280 };
281
282 virtualisation.writableStore =
283 mkOption {
284 default = true; # FIXME
285 description =
286 ''
287 If enabled, the Nix store in the VM is made writable by
288 layering an overlay filesystem on top of the host's Nix
289 store.
290 '';
291 };
292
293 virtualisation.writableStoreUseTmpfs =
294 mkOption {
295 default = true;
296 description =
297 ''
298 Use a tmpfs for the writable store instead of writing to the VM's
299 own filesystem.
300 '';
301 };
302
303 networking.primaryIPAddress =
304 mkOption {
305 default = "";
306 internal = true;
307 description = "Primary IP address used in /etc/hosts.";
308 };
309
310 virtualisation.qemu = {
311 options =
312 mkOption {
313 type = types.listOf types.unspecified;
314 default = [];
315 example = [ "-vga std" ];
316 description = "Options passed to QEMU.";
317 };
318
319 networkingOptions =
320 mkOption {
321 default = [
322 "-net nic,vlan=0,model=virtio"
323 "-net user,vlan=0\${QEMU_NET_OPTS:+,$QEMU_NET_OPTS}"
324 ];
325 type = types.listOf types.str;
326 description = ''
327 Networking-related command-line options that should be passed to qemu.
328 The default is to use userspace networking (slirp).
329
330 If you override this option, be advised to keep
331 ''${QEMU_NET_OPTS:+,$QEMU_NET_OPTS} (as seen in the default)
332 to keep the default runtime behaviour.
333 '';
334 };
335
336 diskInterface =
337 mkOption {
338 default = "virtio";
339 example = "scsi";
340 type = types.str;
341 description = ''
342 The interface used for the virtual hard disks
343 (<literal>virtio</literal> or <literal>scsi</literal>).
344 '';
345 };
346 };
347
348 virtualisation.useBootLoader =
349 mkOption {
350 default = false;
351 description =
352 ''
353 If enabled, the virtual machine will be booted using the
354 regular boot loader (i.e., GRUB 1 or 2). This allows
355 testing of the boot loader. If
356 disabled (the default), the VM directly boots the NixOS
357 kernel and initial ramdisk, bypassing the boot loader
358 altogether.
359 '';
360 };
361
362 virtualisation.useEFIBoot =
363 mkOption {
364 default = false;
365 description =
366 ''
367 If enabled, the virtual machine will provide a EFI boot
368 manager.
369 useEFIBoot is ignored if useBootLoader == false.
370 '';
371 };
372
373 };
374
375 config = {
376
377 boot.loader.grub.device = mkVMOverride cfg.bootDevice;
378
379 boot.initrd.extraUtilsCommands =
380 ''
381 # We need mke2fs in the initrd.
382 copy_bin_and_libs ${pkgs.e2fsprogs}/bin/mke2fs
383 '';
384
385 boot.initrd.postDeviceCommands =
386 ''
387 # If the disk image appears to be empty, run mke2fs to
388 # initialise.
389 FSTYPE=$(blkid -o value -s TYPE ${cfg.bootDevice} || true)
390 if test -z "$FSTYPE"; then
391 mke2fs -t ext4 ${cfg.bootDevice}
392 fi
393 '';
394
395 boot.initrd.postMountCommands =
396 ''
397 # Mark this as a NixOS machine.
398 mkdir -p $targetRoot/etc
399 echo -n > $targetRoot/etc/NIXOS
400
401 # Fix the permissions on /tmp.
402 chmod 1777 $targetRoot/tmp
403
404 mkdir -p $targetRoot/boot
405
406 ${optionalString cfg.writableStore ''
407 echo "mounting overlay filesystem on /nix/store..."
408 mkdir -p 0755 $targetRoot/nix/.rw-store/store $targetRoot/nix/.rw-store/work $targetRoot/nix/store
409 mount -t overlay overlay $targetRoot/nix/store \
410 -o lowerdir=$targetRoot/nix/.ro-store,upperdir=$targetRoot/nix/.rw-store/store,workdir=$targetRoot/nix/.rw-store/work || fail
411 ''}
412 '';
413
414 # After booting, register the closure of the paths in
415 # `virtualisation.pathsInNixDB' in the Nix database in the VM. This
416 # allows Nix operations to work in the VM. The path to the
417 # registration file is passed through the kernel command line to
418 # allow `system.build.toplevel' to be included. (If we had a direct
419 # reference to ${regInfo} here, then we would get a cyclic
420 # dependency.)
421 boot.postBootCommands =
422 ''
423 if [[ "$(cat /proc/cmdline)" =~ regInfo=([^ ]*) ]]; then
424 ${config.nix.package.out}/bin/nix-store --load-db < ''${BASH_REMATCH[1]}
425 fi
426 '';
427
428 boot.initrd.availableKernelModules =
429 optional cfg.writableStore "overlay"
430 ++ optional (cfg.qemu.diskInterface == "scsi") "sym53c8xx";
431
432 virtualisation.bootDevice =
433 mkDefault (if cfg.qemu.diskInterface == "scsi" then "/dev/sda" else "/dev/vda");
434
435 virtualisation.pathsInNixDB = [ config.system.build.toplevel ];
436
437 # FIXME: Figure out how to make this work on non-x86
438 virtualisation.qemu.options =
439 mkIf (pkgs.stdenv.isi686 || pkgs.stdenv.isx86_64) [ "-vga std" "-usbdevice tablet" ];
440
441 # Mount the host filesystem via 9P, and bind-mount the Nix store
442 # of the host into our own filesystem. We use mkVMOverride to
443 # allow this module to be applied to "normal" NixOS system
444 # configuration, where the regular value for the `fileSystems'
445 # attribute should be disregarded for the purpose of building a VM
446 # test image (since those filesystems don't exist in the VM).
447 fileSystems = mkVMOverride (
448 { "/".device = cfg.bootDevice;
449 ${if cfg.writableStore then "/nix/.ro-store" else "/nix/store"} =
450 { device = "store";
451 fsType = "9p";
452 options = [ "trans=virtio" "version=9p2000.L" "cache=loose" ];
453 neededForBoot = true;
454 };
455 "/tmp" = mkIf config.boot.tmpOnTmpfs
456 { device = "tmpfs";
457 fsType = "tmpfs";
458 neededForBoot = true;
459 # Sync with systemd's tmp.mount;
460 options = [ "mode=1777" "strictatime" "nosuid" "nodev" ];
461 };
462 "/tmp/xchg" =
463 { device = "xchg";
464 fsType = "9p";
465 options = [ "trans=virtio" "version=9p2000.L" "cache=loose" ];
466 neededForBoot = true;
467 };
468 "/tmp/shared" =
469 { device = "shared";
470 fsType = "9p";
471 options = [ "trans=virtio" "version=9p2000.L" ];
472 neededForBoot = true;
473 };
474 } // optionalAttrs (cfg.writableStore && cfg.writableStoreUseTmpfs)
475 { "/nix/.rw-store" =
476 { fsType = "tmpfs";
477 options = [ "mode=0755" ];
478 neededForBoot = true;
479 };
480 } // optionalAttrs cfg.useBootLoader
481 { "/boot" =
482 { device = "/dev/vdb2";
483 fsType = "vfat";
484 options = [ "ro" ];
485 noCheck = true; # fsck fails on a r/o filesystem
486 };
487 });
488
489 swapDevices = mkVMOverride [ ];
490 boot.initrd.luks.devices = mkVMOverride {};
491
492 # Don't run ntpd in the guest. It should get the correct time from KVM.
493 services.timesyncd.enable = false;
494
495 system.build.vm = pkgs.runCommand "nixos-vm" { preferLocalBuild = true; }
496 ''
497 mkdir -p $out/bin
498 ln -s ${config.system.build.toplevel} $out/system
499 ln -s ${pkgs.writeScript "run-nixos-vm" startVM} $out/bin/run-${vmName}-vm
500 '';
501
502 # When building a regular system configuration, override whatever
503 # video driver the host uses.
504 services.xserver.videoDrivers = mkVMOverride [ "modesetting" ];
505 services.xserver.defaultDepth = mkVMOverride 0;
506 services.xserver.resolutions = mkVMOverride [ { x = 1024; y = 768; } ];
507 services.xserver.monitorSection =
508 ''
509 # Set a higher refresh rate so that resolutions > 800x600 work.
510 HorizSync 30-140
511 VertRefresh 50-160
512 '';
513
514 # Wireless won't work in the VM.
515 networking.wireless.enable = mkVMOverride false;
516 networking.connman.enable = mkVMOverride false;
517
518 # Speed up booting by not waiting for ARP.
519 networking.dhcpcd.extraConfig = "noarp";
520
521 networking.usePredictableInterfaceNames = false;
522
523 system.requiredKernelConfig = with config.lib.kernelConfig;
524 [ (isEnabled "VIRTIO_BLK")
525 (isEnabled "VIRTIO_PCI")
526 (isEnabled "VIRTIO_NET")
527 (isEnabled "EXT4_FS")
528 (isYes "BLK_DEV")
529 (isYes "PCI")
530 (isYes "EXPERIMENTAL")
531 (isYes "NETDEVICES")
532 (isYes "NET_CORE")
533 (isYes "INET")
534 (isYes "NETWORK_FILESYSTEMS")
535 ] ++ optional (!cfg.graphics) [
536 (isYes "SERIAL_8250_CONSOLE")
537 (isYes "SERIAL_8250")
538 ];
539
540 };
541}