pyrox.dev
1# Note: This is a private API, internal to NixOS. Its interface is subject
2# to change without notice.
3#
4# The result of this builder is two disk images:
5#
6# * `boot` - a small disk formatted with FAT to be used for /boot. FAT is
7# chosen to support EFI.
8# * `root` - a larger disk with a zpool taking the entire disk.
9#
10# This two-disk approach is taken to satisfy ZFS's requirements for
11# autoexpand.
12#
13# # Why doesn't autoexpand work with ZFS in a partition?
14#
15# When ZFS owns the whole disk doesn’t really use a partition: it has
16# a marker partition at the start and a marker partition at the end of
17# the disk.
18#
19# If ZFS is constrained to a partition, ZFS leaves expanding the partition
20# up to the user. Obviously, the user may not choose to do so.
21#
22# Once the user expands the partition, calling zpool online -e expands the
23# vdev to use the whole partition. It doesn’t happen automatically
24# presumably because zed doesn’t get an event saying it’s partition grew,
25# whereas it can and does get an event saying the whole disk it is on is
26# now larger.
27{ lib
28, pkgs
29, # The NixOS configuration to be installed onto the disk image.
30 config
31
32, # size of the FAT boot disk, in megabytes.
33 bootSize ? 1024
34
35, # The size of the root disk, in megabytes.
36 rootSize ? 2048
37
38, # The name of the ZFS pool
39 rootPoolName ? "tank"
40
41, # zpool properties
42 rootPoolProperties ? {
43 autoexpand = "on";
44 }
45, # pool-wide filesystem properties
46 rootPoolFilesystemProperties ? {
47 acltype = "posixacl";
48 atime = "off";
49 compression = "on";
50 mountpoint = "legacy";
51 xattr = "sa";
52 }
53
54, # datasets, with per-attribute options:
55 # mount: (optional) mount point in the VM
56 # properties: (optional) ZFS properties on the dataset, like filesystemProperties
57 # Notes:
58 # 1. datasets will be created from shorter to longer names as a simple topo-sort
59 # 2. you should define a root's dataset's mount for `/`
60 datasets ? { }
61
62, # The files and directories to be placed in the target file system.
63 # This is a list of attribute sets {source, target} where `source'
64 # is the file system object (regular file or directory) to be
65 # grafted in the file system at path `target'.
66 contents ? []
67
68, # The initial NixOS configuration file to be copied to
69 # /etc/nixos/configuration.nix. This configuration will be embedded
70 # inside a configuration which includes the described ZFS fileSystems.
71 configFile ? null
72
73, # Shell code executed after the VM has finished.
74 postVM ? ""
75
76, name ? "nixos-disk-image"
77
78, # Disk image format, one of qcow2, qcow2-compressed, vdi, vpc, raw.
79 format ? "raw"
80
81, # Include a copy of Nixpkgs in the disk image
82 includeChannel ? true
83}:
84let
85 formatOpt = if format == "qcow2-compressed" then "qcow2" else format;
86
87 compress = lib.optionalString (format == "qcow2-compressed") "-c";
88
89 filenameSuffix = "." + {
90 qcow2 = "qcow2";
91 vdi = "vdi";
92 vpc = "vhd";
93 raw = "img";
94 }.${formatOpt} or formatOpt;
95 bootFilename = "nixos.boot${filenameSuffix}";
96 rootFilename = "nixos.root${filenameSuffix}";
97
98 # FIXME: merge with channel.nix / make-channel.nix.
99 channelSources =
100 let
101 nixpkgs = lib.cleanSource pkgs.path;
102 in
103 pkgs.runCommand "nixos-${config.system.nixos.version}" {} ''
104 mkdir -p $out
105 cp -prd ${nixpkgs.outPath} $out/nixos
106 chmod -R u+w $out/nixos
107 if [ ! -e $out/nixos/nixpkgs ]; then
108 ln -s . $out/nixos/nixpkgs
109 fi
110 rm -rf $out/nixos/.git
111 echo -n ${config.system.nixos.versionSuffix} > $out/nixos/.version-suffix
112 '';
113
114 closureInfo = pkgs.closureInfo {
115 rootPaths = [ config.system.build.toplevel ]
116 ++ (lib.optional includeChannel channelSources);
117 };
118
119 modulesTree = pkgs.aggregateModules
120 (with config.boot.kernelPackages; [ kernel zfs ]);
121
122 tools = lib.makeBinPath (
123 with pkgs; [
124 config.system.build.nixos-enter
125 config.system.build.nixos-install
126 dosfstools
127 e2fsprogs
128 gptfdisk
129 nix
130 parted
131 utillinux
132 zfs
133 ]
134 );
135
136 hasDefinedMount = disk: ((disk.mount or null) != null);
137
138 stringifyProperties = prefix: properties: lib.concatStringsSep " \\\n" (
139 lib.mapAttrsToList
140 (
141 property: value: "${prefix} ${lib.escapeShellArg property}=${lib.escapeShellArg value}"
142 )
143 properties
144 );
145
146 featuresToProperties = features:
147 lib.listToAttrs
148 (builtins.map (feature: {
149 name = "feature@${feature}";
150 value = "enabled";
151 }) features);
152
153 createDatasets =
154 let
155 datasetlist = lib.mapAttrsToList lib.nameValuePair datasets;
156 sorted = lib.sort (left: right: (lib.stringLength left.name) < (lib.stringLength right.name)) datasetlist;
157 cmd = { name, value }:
158 let
159 properties = stringifyProperties "-o" (value.properties or {});
160 in
161 "zfs create -p ${properties} ${name}";
162 in
163 lib.concatMapStringsSep "\n" cmd sorted;
164
165 mountDatasets =
166 let
167 datasetlist = lib.mapAttrsToList lib.nameValuePair datasets;
168 mounts = lib.filter ({ value, ... }: hasDefinedMount value) datasetlist;
169 sorted = lib.sort (left: right: (lib.stringLength left.value.mount) < (lib.stringLength right.value.mount)) mounts;
170 cmd = { name, value }:
171 ''
172 mkdir -p /mnt${lib.escapeShellArg value.mount}
173 mount -t zfs ${name} /mnt${lib.escapeShellArg value.mount}
174 '';
175 in
176 lib.concatMapStringsSep "\n" cmd sorted;
177
178 unmountDatasets =
179 let
180 datasetlist = lib.mapAttrsToList lib.nameValuePair datasets;
181 mounts = lib.filter ({ value, ... }: hasDefinedMount value) datasetlist;
182 sorted = lib.sort (left: right: (lib.stringLength left.value.mount) > (lib.stringLength right.value.mount)) mounts;
183 cmd = { name, value }:
184 ''
185 umount /mnt${lib.escapeShellArg value.mount}
186 '';
187 in
188 lib.concatMapStringsSep "\n" cmd sorted;
189
190
191 fileSystemsCfgFile =
192 let
193 mountable = lib.filterAttrs (_: value: hasDefinedMount value) datasets;
194 in
195 pkgs.runCommand "filesystem-config.nix" {
196 buildInputs = with pkgs; [ jq nixpkgs-fmt ];
197 filesystems = builtins.toJSON {
198 fileSystems = lib.mapAttrs'
199 (
200 dataset: attrs:
201 {
202 name = attrs.mount;
203 value = {
204 fsType = "zfs";
205 device = "${dataset}";
206 };
207 }
208 )
209 mountable;
210 };
211 passAsFile = [ "filesystems" ];
212 } ''
213 (
214 echo "builtins.fromJSON '''"
215 jq . < "$filesystemsPath"
216 echo "'''"
217 ) > $out
218
219 nixpkgs-fmt $out
220 '';
221
222 mergedConfig =
223 if configFile == null
224 then fileSystemsCfgFile
225 else
226 pkgs.runCommand "configuration.nix" {
227 buildInputs = with pkgs; [ nixpkgs-fmt ];
228 }
229 ''
230 (
231 echo '{ imports = ['
232 printf "(%s)\n" "$(cat ${fileSystemsCfgFile})";
233 printf "(%s)\n" "$(cat ${configFile})";
234 echo ']; }'
235 ) > $out
236
237 nixpkgs-fmt $out
238 '';
239
240 image = (
241 pkgs.vmTools.override {
242 rootModules =
243 [ "zfs" "9p" "9pnet_virtio" "virtio_pci" "virtio_blk" ] ++
244 (pkgs.lib.optional pkgs.stdenv.hostPlatform.isx86 "rtc_cmos");
245 kernel = modulesTree;
246 }
247 ).runInLinuxVM (
248 pkgs.runCommand name
249 {
250 QEMU_OPTS = "-drive file=$bootDiskImage,if=virtio,cache=unsafe,werror=report"
251 + " -drive file=$rootDiskImage,if=virtio,cache=unsafe,werror=report";
252 preVM = ''
253 PATH=$PATH:${pkgs.qemu_kvm}/bin
254 mkdir $out
255 bootDiskImage=boot.raw
256 qemu-img create -f raw $bootDiskImage ${toString bootSize}M
257
258 rootDiskImage=root.raw
259 qemu-img create -f raw $rootDiskImage ${toString rootSize}M
260 '';
261
262 postVM = ''
263 ${if formatOpt == "raw" then ''
264 mv $bootDiskImage $out/${bootFilename}
265 mv $rootDiskImage $out/${rootFilename}
266 '' else ''
267 ${pkgs.qemu}/bin/qemu-img convert -f raw -O ${formatOpt} ${compress} $bootDiskImage $out/${bootFilename}
268 ${pkgs.qemu}/bin/qemu-img convert -f raw -O ${formatOpt} ${compress} $rootDiskImage $out/${rootFilename}
269 ''}
270 bootDiskImage=$out/${bootFilename}
271 rootDiskImage=$out/${rootFilename}
272 set -x
273 ${postVM}
274 '';
275 } ''
276 export PATH=${tools}:$PATH
277 set -x
278
279 cp -sv /dev/vda /dev/sda
280 cp -sv /dev/vda /dev/xvda
281
282 parted --script /dev/vda -- \
283 mklabel gpt \
284 mkpart no-fs 1MiB 2MiB \
285 set 1 bios_grub on \
286 align-check optimal 1 \
287 mkpart ESP fat32 2MiB -1MiB \
288 align-check optimal 2 \
289 print
290
291 sfdisk --dump /dev/vda
292
293
294 zpool create \
295 ${stringifyProperties " -o" rootPoolProperties} \
296 ${stringifyProperties " -O" rootPoolFilesystemProperties} \
297 ${rootPoolName} /dev/vdb
298 parted --script /dev/vdb -- print
299
300 ${createDatasets}
301 ${mountDatasets}
302
303 mkdir -p /mnt/boot
304 mkfs.vfat -n ESP /dev/vda2
305 mount /dev/vda2 /mnt/boot
306
307 mount
308
309 # Install a configuration.nix
310 mkdir -p /mnt/etc/nixos
311 # `cat` so it is mutable on the fs
312 cat ${mergedConfig} > /mnt/etc/nixos/configuration.nix
313
314 export NIX_STATE_DIR=$TMPDIR/state
315 nix-store --load-db < ${closureInfo}/registration
316
317 nixos-install \
318 --root /mnt \
319 --no-root-passwd \
320 --system ${config.system.build.toplevel} \
321 --substituters "" \
322 ${lib.optionalString includeChannel ''--channel ${channelSources}''}
323
324 df -h
325
326 umount /mnt/boot
327 ${unmountDatasets}
328
329 zpool export ${rootPoolName}
330 ''
331 );
332in
333image