pyrox.dev
1# This module creates a virtual machine from the NixOS configuration.
2# Building the `config.system.build.vm' attribute gives you a command
3# that starts a KVM/QEMU VM running the NixOS configuration defined in
4# `config'. The Nix store is shared read-only with the host, which
5# makes (re)building VMs very efficient. However, it also means you
6# can't reconfigure the guest inside the guest - you need to rebuild
7# the VM in the host. On the other hand, the root filesystem is a
8# read/writable disk image persistent across VM reboots.
9
10{ config, lib, pkgs, ... }:
11
12with lib;
13
14let
15
16 qemu = config.system.build.qemu or pkgs.qemu_test;
17
18 vmName =
19 if config.networking.hostName == ""
20 then "noname"
21 else config.networking.hostName;
22
23 cfg = config.virtualisation;
24
25 qemuGraphics = if cfg.graphics then "" else "-nographic";
26 kernelConsole = if cfg.graphics then "" else "console=ttyS0";
27 ttys = [ "tty1" "tty2" "tty3" "tty4" "tty5" "tty6" ];
28
29 # Shell script to start the VM.
30 startVM =
31 ''
32 #! ${pkgs.stdenv.shell}
33
34 NIX_DISK_IMAGE=$(readlink -f ''${NIX_DISK_IMAGE:-${config.virtualisation.diskImage}})
35
36 if ! test -e "$NIX_DISK_IMAGE"; then
37 ${qemu}/bin/qemu-img create -f qcow2 "$NIX_DISK_IMAGE" \
38 ${toString config.virtualisation.diskSize}M || exit 1
39 fi
40
41 # Create a directory for storing temporary data of the running VM.
42 if [ -z "$TMPDIR" -o -z "$USE_TMPDIR" ]; then
43 TMPDIR=$(mktemp -d nix-vm.XXXXXXXXXX --tmpdir)
44 fi
45
46 # Create a directory for exchanging data with the VM.
47 mkdir -p $TMPDIR/xchg
48
49 ${if cfg.useBootLoader then ''
50 # Create a writable copy/snapshot of the boot disk.
51 # A writable boot disk can be booted from automatically.
52 ${qemu}/bin/qemu-img create -f qcow2 -b ${bootDisk}/disk.img $TMPDIR/disk.img || exit 1
53
54 ${if cfg.useEFIBoot then ''
55 # VM needs a writable flash BIOS.
56 cp ${bootDisk}/bios.bin $TMPDIR || exit 1
57 chmod 0644 $TMPDIR/bios.bin || exit 1
58 '' else ''
59 ''}
60 '' else ''
61 ''}
62
63 cd $TMPDIR
64 idx=2
65 extraDisks=""
66 ${flip concatMapStrings cfg.emptyDiskImages (size: ''
67 if ! test -e "empty$idx.qcow2"; then
68 ${qemu}/bin/qemu-img create -f qcow2 "empty$idx.qcow2" "${toString size}M"
69 fi
70 extraDisks="$extraDisks -drive index=$idx,file=$(pwd)/empty$idx.qcow2,if=${cfg.qemu.diskInterface},werror=report"
71 idx=$((idx + 1))
72 '')}
73
74 # Start QEMU.
75 exec ${qemu}/bin/qemu-kvm \
76 -name ${vmName} \
77 -m ${toString config.virtualisation.memorySize} \
78 -smp ${toString config.virtualisation.cores} \
79 ${optionalString (pkgs.stdenv.system == "x86_64-linux") "-cpu kvm64"} \
80 ${concatStringsSep " " config.virtualisation.qemu.networkingOptions} \
81 -virtfs local,path=/nix/store,security_model=none,mount_tag=store \
82 -virtfs local,path=$TMPDIR/xchg,security_model=none,mount_tag=xchg \
83 -virtfs local,path=''${SHARED_DIR:-$TMPDIR/xchg},security_model=none,mount_tag=shared \
84 ${if cfg.useBootLoader then ''
85 -drive index=0,id=drive1,file=$NIX_DISK_IMAGE,if=${cfg.qemu.diskInterface},cache=writeback,werror=report \
86 -drive index=1,id=drive2,file=$TMPDIR/disk.img,media=disk \
87 ${if cfg.useEFIBoot then ''
88 -pflash $TMPDIR/bios.bin \
89 '' else ''
90 ''}
91 '' else ''
92 -drive index=0,id=drive1,file=$NIX_DISK_IMAGE,if=${cfg.qemu.diskInterface},cache=writeback,werror=report \
93 -kernel ${config.system.build.toplevel}/kernel \
94 -initrd ${config.system.build.toplevel}/initrd \
95 -append "$(cat ${config.system.build.toplevel}/kernel-params) init=${config.system.build.toplevel}/init regInfo=${regInfo} ${kernelConsole} $QEMU_KERNEL_PARAMS" \
96 ''} \
97 $extraDisks \
98 ${qemuGraphics} \
99 ${toString config.virtualisation.qemu.options} \
100 $QEMU_OPTS \
101 $@
102 '';
103
104
105 regInfo = pkgs.runCommand "reginfo"
106 { exportReferencesGraph =
107 map (x: [("closure-" + baseNameOf x) x]) config.virtualisation.pathsInNixDB;
108 buildInputs = [ pkgs.perl ];
109 preferLocalBuild = true;
110 }
111 ''
112 printRegistration=1 perl ${pkgs.pathsFromGraph} closure-* > $out
113 '';
114
115
116 # Generate a hard disk image containing a /boot partition and GRUB
117 # in the MBR. Used when the `useBootLoader' option is set.
118 # FIXME: use nixos/lib/make-disk-image.nix.
119 bootDisk =
120 pkgs.vmTools.runInLinuxVM (
121 pkgs.runCommand "nixos-boot-disk"
122 { preVM =
123 ''
124 mkdir $out
125 diskImage=$out/disk.img
126 bootFlash=$out/bios.bin
127 ${qemu}/bin/qemu-img create -f qcow2 $diskImage "40M"
128 ${if cfg.useEFIBoot then ''
129 cp ${pkgs.OVMF-CSM.fd}/FV/OVMF.fd $bootFlash
130 chmod 0644 $bootFlash
131 '' else ''
132 ''}
133 '';
134 buildInputs = [ pkgs.utillinux ];
135 QEMU_OPTS = if cfg.useEFIBoot
136 then "-pflash $out/bios.bin -nographic -serial pty"
137 else "-nographic -serial pty";
138 }
139 ''
140 # Create a /boot EFI partition with 40M and arbitrary but fixed GUIDs for reproducibility
141 ${pkgs.gptfdisk}/bin/sgdisk \
142 --set-alignment=1 --new=1:34:2047 --change-name=1:BIOSBootPartition --typecode=1:ef02 \
143 --set-alignment=512 --largest-new=2 --change-name=2:EFISystem --typecode=2:ef00 \
144 --attributes=1:set:1 \
145 --attributes=2:set:2 \
146 --disk-guid=97FD5997-D90B-4AA3-8D16-C1723AEA73C1 \
147 --partition-guid=1:1C06F03B-704E-4657-B9CD-681A087A2FDC \
148 --partition-guid=2:970C694F-AFD0-4B99-B750-CDB7A329AB6F \
149 --hybrid 2 \
150 --recompute-chs /dev/vda
151 . /sys/class/block/vda2/uevent
152 mknod /dev/vda2 b $MAJOR $MINOR
153 . /sys/class/block/vda/uevent
154 ${pkgs.dosfstools}/bin/mkfs.fat -F16 /dev/vda2
155 export MTOOLS_SKIP_CHECK=1
156 ${pkgs.mtools}/bin/mlabel -i /dev/vda2 ::boot
157
158 # Mount /boot; load necessary modules first.
159 ${pkgs.kmod}/bin/insmod ${pkgs.linux}/lib/modules/*/kernel/fs/nls/nls_cp437.ko.xz || true
160 ${pkgs.kmod}/bin/insmod ${pkgs.linux}/lib/modules/*/kernel/fs/nls/nls_iso8859-1.ko.xz || true
161 ${pkgs.kmod}/bin/insmod ${pkgs.linux}/lib/modules/*/kernel/fs/fat/fat.ko.xz || true
162 ${pkgs.kmod}/bin/insmod ${pkgs.linux}/lib/modules/*/kernel/fs/fat/vfat.ko.xz || true
163 ${pkgs.kmod}/bin/insmod ${pkgs.linux}/lib/modules/*/kernel/fs/efivarfs/efivarfs.ko.xz || true
164 mkdir /boot
165 mount /dev/vda2 /boot
166
167 # This is needed for GRUB 0.97, which doesn't know about virtio devices.
168 mkdir /boot/grub
169 echo '(hd0) /dev/vda' > /boot/grub/device.map
170
171 # Install GRUB and generate the GRUB boot menu.
172 touch /etc/NIXOS
173 mkdir -p /nix/var/nix/profiles
174 ${config.system.build.toplevel}/bin/switch-to-configuration boot
175
176 umount /boot
177 '' # */
178 );
179
180in
181
182{
183 imports = [ ../profiles/qemu-guest.nix ];
184
185 options = {
186
187 virtualisation.memorySize =
188 mkOption {
189 default = 384;
190 description =
191 ''
192 Memory size (M) of virtual machine.
193 '';
194 };
195
196 virtualisation.diskSize =
197 mkOption {
198 default = 512;
199 description =
200 ''
201 Disk size (M) of virtual machine.
202 '';
203 };
204
205 virtualisation.diskImage =
206 mkOption {
207 default = "./${vmName}.qcow2";
208 description =
209 ''
210 Path to the disk image containing the root filesystem.
211 The image will be created on startup if it does not
212 exist.
213 '';
214 };
215
216 virtualisation.bootDevice =
217 mkOption {
218 type = types.str;
219 example = "/dev/vda";
220 description =
221 ''
222 The disk to be used for the root filesystem.
223 '';
224 };
225
226 virtualisation.emptyDiskImages =
227 mkOption {
228 default = [];
229 type = types.listOf types.int;
230 description =
231 ''
232 Additional disk images to provide to the VM. The value is
233 a list of size in megabytes of each disk. These disks are
234 writeable by the VM.
235 '';
236 };
237
238 virtualisation.graphics =
239 mkOption {
240 default = true;
241 description =
242 ''
243 Whether to run QEMU with a graphics window, or access
244 the guest computer serial port through the host tty.
245 '';
246 };
247
248 virtualisation.cores =
249 mkOption {
250 default = 1;
251 type = types.int;
252 description =
253 ''
254 Specify the number of cores the guest is permitted to use.
255 The number can be higher than the available cores on the
256 host system.
257 '';
258 };
259
260 virtualisation.pathsInNixDB =
261 mkOption {
262 default = [];
263 description =
264 ''
265 The list of paths whose closure is registered in the Nix
266 database in the VM. All other paths in the host Nix store
267 appear in the guest Nix store as well, but are considered
268 garbage (because they are not registered in the Nix
269 database in the guest).
270 '';
271 };
272
273 virtualisation.vlans =
274 mkOption {
275 default = [ 1 ];
276 example = [ 1 2 ];
277 description =
278 ''
279 Virtual networks to which the VM is connected. Each
280 number <replaceable>N</replaceable> in this list causes
281 the VM to have a virtual Ethernet interface attached to a
282 separate virtual network on which it will be assigned IP
283 address
284 <literal>192.168.<replaceable>N</replaceable>.<replaceable>M</replaceable></literal>,
285 where <replaceable>M</replaceable> is the index of this VM
286 in the list of VMs.
287 '';
288 };
289
290 virtualisation.writableStore =
291 mkOption {
292 default = true; # FIXME
293 description =
294 ''
295 If enabled, the Nix store in the VM is made writable by
296 layering an overlay filesystem on top of the host's Nix
297 store.
298 '';
299 };
300
301 virtualisation.writableStoreUseTmpfs =
302 mkOption {
303 default = true;
304 description =
305 ''
306 Use a tmpfs for the writable store instead of writing to the VM's
307 own filesystem.
308 '';
309 };
310
311 networking.primaryIPAddress =
312 mkOption {
313 default = "";
314 internal = true;
315 description = "Primary IP address used in /etc/hosts.";
316 };
317
318 virtualisation.qemu = {
319 options =
320 mkOption {
321 type = types.listOf types.unspecified;
322 default = [];
323 example = [ "-vga std" ];
324 description = "Options passed to QEMU.";
325 };
326
327 networkingOptions =
328 mkOption {
329 default = [
330 "-net nic,vlan=0,model=virtio"
331 "-net user,vlan=0\${QEMU_NET_OPTS:+,$QEMU_NET_OPTS}"
332 ];
333 type = types.listOf types.str;
334 description = ''
335 Networking-related command-line options that should be passed to qemu.
336 The default is to use userspace networking (slirp).
337
338 If you override this option, be advised to keep
339 ''${QEMU_NET_OPTS:+,$QEMU_NET_OPTS} (as seen in the default)
340 to keep the default runtime behaviour.
341 '';
342 };
343
344 diskInterface =
345 mkOption {
346 default = "virtio";
347 example = "scsi";
348 type = types.str;
349 description = ''
350 The interface used for the virtual hard disks
351 (<literal>virtio</literal> or <literal>scsi</literal>).
352 '';
353 };
354 };
355
356 virtualisation.useBootLoader =
357 mkOption {
358 default = false;
359 description =
360 ''
361 If enabled, the virtual machine will be booted using the
362 regular boot loader (i.e., GRUB 1 or 2). This allows
363 testing of the boot loader. If
364 disabled (the default), the VM directly boots the NixOS
365 kernel and initial ramdisk, bypassing the boot loader
366 altogether.
367 '';
368 };
369
370 virtualisation.useEFIBoot =
371 mkOption {
372 default = false;
373 description =
374 ''
375 If enabled, the virtual machine will provide a EFI boot
376 manager.
377 useEFIBoot is ignored if useBootLoader == false.
378 '';
379 };
380
381 };
382
383 config = {
384
385 boot.loader.grub.device = mkVMOverride cfg.bootDevice;
386
387 boot.initrd.extraUtilsCommands =
388 ''
389 # We need mke2fs in the initrd.
390 copy_bin_and_libs ${pkgs.e2fsprogs}/bin/mke2fs
391 '';
392
393 boot.initrd.postDeviceCommands =
394 ''
395 # If the disk image appears to be empty, run mke2fs to
396 # initialise.
397 FSTYPE=$(blkid -o value -s TYPE ${cfg.bootDevice} || true)
398 if test -z "$FSTYPE"; then
399 mke2fs -t ext4 ${cfg.bootDevice}
400 fi
401 '';
402
403 boot.initrd.postMountCommands =
404 ''
405 # Mark this as a NixOS machine.
406 mkdir -p $targetRoot/etc
407 echo -n > $targetRoot/etc/NIXOS
408
409 # Fix the permissions on /tmp.
410 chmod 1777 $targetRoot/tmp
411
412 mkdir -p $targetRoot/boot
413
414 ${optionalString cfg.writableStore ''
415 echo "mounting overlay filesystem on /nix/store..."
416 mkdir -p 0755 $targetRoot/nix/.rw-store/store $targetRoot/nix/.rw-store/work $targetRoot/nix/store
417 mount -t overlay overlay $targetRoot/nix/store \
418 -o lowerdir=$targetRoot/nix/.ro-store,upperdir=$targetRoot/nix/.rw-store/store,workdir=$targetRoot/nix/.rw-store/work || fail
419 ''}
420 '';
421
422 # After booting, register the closure of the paths in
423 # `virtualisation.pathsInNixDB' in the Nix database in the VM. This
424 # allows Nix operations to work in the VM. The path to the
425 # registration file is passed through the kernel command line to
426 # allow `system.build.toplevel' to be included. (If we had a direct
427 # reference to ${regInfo} here, then we would get a cyclic
428 # dependency.)
429 boot.postBootCommands =
430 ''
431 if [[ "$(cat /proc/cmdline)" =~ regInfo=([^ ]*) ]]; then
432 ${config.nix.package.out}/bin/nix-store --load-db < ''${BASH_REMATCH[1]}
433 fi
434 '';
435
436 boot.initrd.availableKernelModules =
437 optional cfg.writableStore "overlay"
438 ++ optional (cfg.qemu.diskInterface == "scsi") "sym53c8xx";
439
440 virtualisation.bootDevice =
441 mkDefault (if cfg.qemu.diskInterface == "scsi" then "/dev/sda" else "/dev/vda");
442
443 virtualisation.pathsInNixDB = [ config.system.build.toplevel ];
444
445 virtualisation.qemu.options = [ "-vga std" "-usbdevice tablet" ];
446
447 # Mount the host filesystem via 9P, and bind-mount the Nix store
448 # of the host into our own filesystem. We use mkVMOverride to
449 # allow this module to be applied to "normal" NixOS system
450 # configuration, where the regular value for the `fileSystems'
451 # attribute should be disregarded for the purpose of building a VM
452 # test image (since those filesystems don't exist in the VM).
453 fileSystems = mkVMOverride (
454 { "/".device = cfg.bootDevice;
455 ${if cfg.writableStore then "/nix/.ro-store" else "/nix/store"} =
456 { device = "store";
457 fsType = "9p";
458 options = [ "trans=virtio" "version=9p2000.L" "cache=loose" ];
459 neededForBoot = true;
460 };
461 "/tmp" = mkIf config.boot.tmpOnTmpfs
462 { device = "tmpfs";
463 fsType = "tmpfs";
464 neededForBoot = true;
465 # Sync with systemd's tmp.mount;
466 options = [ "mode=1777" "strictatime" "nosuid" "nodev" ];
467 };
468 "/tmp/xchg" =
469 { device = "xchg";
470 fsType = "9p";
471 options = [ "trans=virtio" "version=9p2000.L" "cache=loose" ];
472 neededForBoot = true;
473 };
474 "/tmp/shared" =
475 { device = "shared";
476 fsType = "9p";
477 options = [ "trans=virtio" "version=9p2000.L" ];
478 neededForBoot = true;
479 };
480 } // optionalAttrs (cfg.writableStore && cfg.writableStoreUseTmpfs)
481 { "/nix/.rw-store" =
482 { fsType = "tmpfs";
483 options = [ "mode=0755" ];
484 neededForBoot = true;
485 };
486 } // optionalAttrs cfg.useBootLoader
487 { "/boot" =
488 { device = "/dev/vdb2";
489 fsType = "vfat";
490 options = [ "ro" ];
491 noCheck = true; # fsck fails on a r/o filesystem
492 };
493 });
494
495 swapDevices = mkVMOverride [ ];
496 boot.initrd.luks.devices = mkVMOverride {};
497
498 # Don't run ntpd in the guest. It should get the correct time from KVM.
499 services.timesyncd.enable = false;
500
501 system.build.vm = pkgs.runCommand "nixos-vm" { preferLocalBuild = true; }
502 ''
503 mkdir -p $out/bin
504 ln -s ${config.system.build.toplevel} $out/system
505 ln -s ${pkgs.writeScript "run-nixos-vm" startVM} $out/bin/run-${vmName}-vm
506 '';
507
508 # When building a regular system configuration, override whatever
509 # video driver the host uses.
510 services.xserver.videoDrivers = mkVMOverride [ "modesetting" ];
511 services.xserver.defaultDepth = mkVMOverride 0;
512 services.xserver.resolutions = mkVMOverride [ { x = 1024; y = 768; } ];
513 services.xserver.monitorSection =
514 ''
515 # Set a higher refresh rate so that resolutions > 800x600 work.
516 HorizSync 30-140
517 VertRefresh 50-160
518 '';
519
520 # Wireless won't work in the VM.
521 networking.wireless.enable = mkVMOverride false;
522 networking.connman.enable = mkVMOverride false;
523
524 # Speed up booting by not waiting for ARP.
525 networking.dhcpcd.extraConfig = "noarp";
526
527 networking.usePredictableInterfaceNames = false;
528
529 system.requiredKernelConfig = with config.lib.kernelConfig;
530 [ (isEnabled "VIRTIO_BLK")
531 (isEnabled "VIRTIO_PCI")
532 (isEnabled "VIRTIO_NET")
533 (isEnabled "EXT4_FS")
534 (isYes "BLK_DEV")
535 (isYes "PCI")
536 (isYes "EXPERIMENTAL")
537 (isYes "NETDEVICES")
538 (isYes "NET_CORE")
539 (isYes "INET")
540 (isYes "NETWORK_FILESYSTEMS")
541 ] ++ optional (!cfg.graphics) [
542 (isYes "SERIAL_8250_CONSOLE")
543 (isYes "SERIAL_8250")
544 ];
545
546 };
547}