pyrox.dev
1# Note: This is a private API, internal to NixOS. Its interface is subject
2# to change without notice.
3#
4# The result of this builder is two disk images:
5#
6# * `boot` - a small disk formatted with FAT to be used for /boot. FAT is
7# chosen to support EFI.
8# * `root` - a larger disk with a zpool taking the entire disk.
9#
10# This two-disk approach is taken to satisfy ZFS's requirements for
11# autoexpand.
12#
13# # Why doesn't autoexpand work with ZFS in a partition?
14#
15# When ZFS owns the whole disk doesn’t really use a partition: it has
16# a marker partition at the start and a marker partition at the end of
17# the disk.
18#
19# If ZFS is constrained to a partition, ZFS leaves expanding the partition
20# up to the user. Obviously, the user may not choose to do so.
21#
22# Once the user expands the partition, calling zpool online -e expands the
23# vdev to use the whole partition. It doesn’t happen automatically
24# presumably because zed doesn’t get an event saying it’s partition grew,
25# whereas it can and does get an event saying the whole disk it is on is
26# now larger.
27{
28 lib,
29 pkgs,
30 # The NixOS configuration to be installed onto the disk image.
31 config,
32
33 # size of the FAT boot disk in MiB (1024*1024 bytes)
34 bootSize ? 1024,
35
36 # The size of the root disk in MiB (1024*1024 bytes)
37 rootSize ? 2048,
38
39 # The name of the ZFS pool
40 rootPoolName ? "tank",
41
42 # zpool properties
43 rootPoolProperties ? {
44 autoexpand = "on";
45 },
46 # pool-wide filesystem properties
47 rootPoolFilesystemProperties ? {
48 acltype = "posixacl";
49 atime = "off";
50 compression = "on";
51 mountpoint = "legacy";
52 xattr = "sa";
53 },
54
55 # datasets, with per-attribute options:
56 # mount: (optional) mount point in the VM
57 # properties: (optional) ZFS properties on the dataset, like filesystemProperties
58 # Notes:
59 # 1. datasets will be created from shorter to longer names as a simple topo-sort
60 # 2. you should define a root's dataset's mount for `/`
61 datasets ? { },
62
63 # The files and directories to be placed in the target file system.
64 # This is a list of attribute sets {source, target} where `source'
65 # is the file system object (regular file or directory) to be
66 # grafted in the file system at path `target'.
67 contents ? [ ],
68
69 # The initial NixOS configuration file to be copied to
70 # /etc/nixos/configuration.nix. This configuration will be embedded
71 # inside a configuration which includes the described ZFS fileSystems.
72 configFile ? null,
73
74 # Shell code executed after the VM has finished.
75 postVM ? "",
76
77 # Guest memory size in MiB (1024*1024 bytes)
78 memSize ? 1024,
79
80 name ? "nixos-disk-image",
81
82 # Disk image format, one of qcow2, qcow2-compressed, vdi, vpc, raw.
83 format ? "raw",
84
85 # Include a copy of Nixpkgs in the disk image
86 includeChannel ? true,
87}:
88let
89 formatOpt = if format == "qcow2-compressed" then "qcow2" else format;
90
91 compress = lib.optionalString (format == "qcow2-compressed") "-c";
92
93 filenameSuffix =
94 "."
95 + {
96 qcow2 = "qcow2";
97 vdi = "vdi";
98 vpc = "vhd";
99 raw = "img";
100 }
101 .${formatOpt} or formatOpt;
102 bootFilename = "nixos.boot${filenameSuffix}";
103 rootFilename = "nixos.root${filenameSuffix}";
104
105 # FIXME: merge with channel.nix / make-channel.nix.
106 channelSources =
107 let
108 nixpkgs = lib.cleanSource pkgs.path;
109 in
110 pkgs.runCommand "nixos-${config.system.nixos.version}" { } ''
111 mkdir -p $out
112 cp -prd ${nixpkgs.outPath} $out/nixos
113 chmod -R u+w $out/nixos
114 if [ ! -e $out/nixos/nixpkgs ]; then
115 ln -s . $out/nixos/nixpkgs
116 fi
117 rm -rf $out/nixos/.git
118 echo -n ${config.system.nixos.versionSuffix} > $out/nixos/.version-suffix
119 '';
120
121 closureInfo = pkgs.closureInfo {
122 rootPaths = [ config.system.build.toplevel ] ++ (lib.optional includeChannel channelSources);
123 };
124
125 modulesTree = pkgs.aggregateModules (
126 with config.boot;
127 [
128 kernelPackages.kernel
129 kernelPackages.${pkgs.zfs.kernelModuleAttribute}
130 ]
131 );
132
133 tools = lib.makeBinPath (
134 with pkgs;
135 [
136 nixos-enter
137 config.system.build.nixos-install
138 dosfstools
139 e2fsprogs
140 gptfdisk
141 nix
142 parted
143 util-linux
144 zfs
145 ]
146 );
147
148 hasDefinedMount = disk: ((disk.mount or null) != null);
149
150 stringifyProperties =
151 prefix: properties:
152 lib.concatStringsSep " \\\n" (
153 lib.mapAttrsToList (
154 property: value: "${prefix} ${lib.escapeShellArg property}=${lib.escapeShellArg value}"
155 ) properties
156 );
157
158 createDatasets =
159 let
160 datasetlist = lib.mapAttrsToList lib.nameValuePair datasets;
161 sorted = lib.sort (
162 left: right: (lib.stringLength left.name) < (lib.stringLength right.name)
163 ) datasetlist;
164 cmd =
165 { name, value }:
166 let
167 properties = stringifyProperties "-o" (value.properties or { });
168 in
169 "zfs create -p ${properties} ${name}";
170 in
171 lib.concatMapStringsSep "\n" cmd sorted;
172
173 mountDatasets =
174 let
175 datasetlist = lib.mapAttrsToList lib.nameValuePair datasets;
176 mounts = lib.filter ({ value, ... }: hasDefinedMount value) datasetlist;
177 sorted = lib.sort (
178 left: right: (lib.stringLength left.value.mount) < (lib.stringLength right.value.mount)
179 ) mounts;
180 cmd =
181 { name, value }:
182 ''
183 mkdir -p /mnt${lib.escapeShellArg value.mount}
184 mount -t zfs ${name} /mnt${lib.escapeShellArg value.mount}
185 '';
186 in
187 lib.concatMapStringsSep "\n" cmd sorted;
188
189 unmountDatasets =
190 let
191 datasetlist = lib.mapAttrsToList lib.nameValuePair datasets;
192 mounts = lib.filter ({ value, ... }: hasDefinedMount value) datasetlist;
193 sorted = lib.sort (
194 left: right: (lib.stringLength left.value.mount) > (lib.stringLength right.value.mount)
195 ) mounts;
196 cmd =
197 { name, value }:
198 ''
199 umount /mnt${lib.escapeShellArg value.mount}
200 '';
201 in
202 lib.concatMapStringsSep "\n" cmd sorted;
203
204 fileSystemsCfgFile =
205 let
206 mountable = lib.filterAttrs (_: value: hasDefinedMount value) datasets;
207 in
208 pkgs.runCommand "filesystem-config.nix"
209 {
210 buildInputs = with pkgs; [
211 jq
212 nixpkgs-fmt
213 ];
214 filesystems = builtins.toJSON {
215 fileSystems = lib.mapAttrs' (dataset: attrs: {
216 name = attrs.mount;
217 value = {
218 fsType = "zfs";
219 device = "${dataset}";
220 };
221 }) mountable;
222 };
223 passAsFile = [ "filesystems" ];
224 }
225 ''
226 (
227 echo "builtins.fromJSON '''"
228 jq . < "$filesystemsPath"
229 echo "'''"
230 ) > $out
231
232 nixpkgs-fmt $out
233 '';
234
235 mergedConfig =
236 if configFile == null then
237 fileSystemsCfgFile
238 else
239 pkgs.runCommand "configuration.nix"
240 {
241 buildInputs = with pkgs; [ nixpkgs-fmt ];
242 }
243 ''
244 (
245 echo '{ imports = ['
246 printf "(%s)\n" "$(cat ${fileSystemsCfgFile})";
247 printf "(%s)\n" "$(cat ${configFile})";
248 echo ']; }'
249 ) > $out
250
251 nixpkgs-fmt $out
252 '';
253
254 image =
255 (pkgs.vmTools.override {
256 rootModules = [
257 "9p"
258 "9pnet_virtio"
259 "virtio_blk"
260 "virtio_pci"
261 "virtiofs"
262 "zfs"
263 ];
264 kernel = modulesTree;
265 }).runInLinuxVM
266 (
267 pkgs.runCommand name
268 {
269 QEMU_OPTS =
270 "-drive file=$bootDiskImage,if=virtio,cache=unsafe,werror=report"
271 + " -drive file=$rootDiskImage,if=virtio,cache=unsafe,werror=report";
272 inherit memSize;
273 preVM = ''
274 PATH=$PATH:${pkgs.qemu_kvm}/bin
275 mkdir $out
276 bootDiskImage=boot.raw
277 qemu-img create -f raw $bootDiskImage ${toString bootSize}M
278
279 rootDiskImage=root.raw
280 qemu-img create -f raw $rootDiskImage ${toString rootSize}M
281 '';
282
283 postVM = ''
284 ${
285 if formatOpt == "raw" then
286 ''
287 mv $bootDiskImage $out/${bootFilename}
288 mv $rootDiskImage $out/${rootFilename}
289 ''
290 else
291 ''
292 ${pkgs.qemu_kvm}/bin/qemu-img convert -f raw -O ${formatOpt} ${compress} $bootDiskImage $out/${bootFilename}
293 ${pkgs.qemu_kvm}/bin/qemu-img convert -f raw -O ${formatOpt} ${compress} $rootDiskImage $out/${rootFilename}
294 ''
295 }
296 bootDiskImage=$out/${bootFilename}
297 rootDiskImage=$out/${rootFilename}
298 set -x
299 ${postVM}
300 '';
301 }
302 ''
303 export PATH=${tools}:$PATH
304 set -x
305
306 cp -sv /dev/vda /dev/sda
307 cp -sv /dev/vda /dev/xvda
308
309 parted --script /dev/vda -- \
310 mklabel gpt \
311 mkpart no-fs 1MiB 2MiB \
312 set 1 bios_grub on \
313 align-check optimal 1 \
314 mkpart ESP fat32 2MiB -1MiB \
315 align-check optimal 2 \
316 print
317
318 sfdisk --dump /dev/vda
319
320
321 zpool create \
322 ${stringifyProperties " -o" rootPoolProperties} \
323 ${stringifyProperties " -O" rootPoolFilesystemProperties} \
324 ${rootPoolName} /dev/vdb
325 parted --script /dev/vdb -- print
326
327 ${createDatasets}
328 ${mountDatasets}
329
330 mkdir -p /mnt/boot
331 mkfs.vfat -n ESP /dev/vda2
332 mount /dev/vda2 /mnt/boot
333
334 mount
335
336 # Install a configuration.nix
337 mkdir -p /mnt/etc/nixos
338 # `cat` so it is mutable on the fs
339 cat ${mergedConfig} > /mnt/etc/nixos/configuration.nix
340
341 export NIX_STATE_DIR=$TMPDIR/state
342 nix-store --load-db < ${closureInfo}/registration
343
344 nixos-install \
345 --root /mnt \
346 --no-root-passwd \
347 --system ${config.system.build.toplevel} \
348 --substituters "" \
349 ${lib.optionalString includeChannel ''--channel ${channelSources}''}
350
351 df -h
352
353 umount /mnt/boot
354 ${unmountDatasets}
355
356 zpool export ${rootPoolName}
357 ''
358 );
359in
360image