pyrox.dev
1# Note: This is a private API, internal to NixOS. Its interface is subject
2# to change without notice.
3#
4# The result of this builder is two disk images:
5#
6# * `boot` - a small disk formatted with FAT to be used for /boot. FAT is
7# chosen to support EFI.
8# * `root` - a larger disk with a zpool taking the entire disk.
9#
10# This two-disk approach is taken to satisfy ZFS's requirements for
11# autoexpand.
12#
13# # Why doesn't autoexpand work with ZFS in a partition?
14#
15# When ZFS owns the whole disk doesn’t really use a partition: it has
16# a marker partition at the start and a marker partition at the end of
17# the disk.
18#
19# If ZFS is constrained to a partition, ZFS leaves expanding the partition
20# up to the user. Obviously, the user may not choose to do so.
21#
22# Once the user expands the partition, calling zpool online -e expands the
23# vdev to use the whole partition. It doesn’t happen automatically
24# presumably because zed doesn’t get an event saying it’s partition grew,
25# whereas it can and does get an event saying the whole disk it is on is
26# now larger.
27{ lib
28, pkgs
29, # The NixOS configuration to be installed onto the disk image.
30 config
31
32, # size of the FAT boot disk, in megabytes.
33 bootSize ? 1024
34
35, # The size of the root disk, in megabytes.
36 rootSize ? 2048
37
38, # The name of the ZFS pool
39 rootPoolName ? "tank"
40
41, # zpool properties
42 rootPoolProperties ? {
43 autoexpand = "on";
44 }
45, # pool-wide filesystem properties
46 rootPoolFilesystemProperties ? {
47 acltype = "posixacl";
48 atime = "off";
49 compression = "on";
50 mountpoint = "legacy";
51 xattr = "sa";
52 }
53
54, # datasets, with per-attribute options:
55 # mount: (optional) mount point in the VM
56 # properties: (optional) ZFS properties on the dataset, like filesystemProperties
57 # Notes:
58 # 1. datasets will be created from shorter to longer names as a simple topo-sort
59 # 2. you should define a root's dataset's mount for `/`
60 datasets ? { }
61
62, # The files and directories to be placed in the target file system.
63 # This is a list of attribute sets {source, target} where `source'
64 # is the file system object (regular file or directory) to be
65 # grafted in the file system at path `target'.
66 contents ? []
67
68, # The initial NixOS configuration file to be copied to
69 # /etc/nixos/configuration.nix. This configuration will be embedded
70 # inside a configuration which includes the described ZFS fileSystems.
71 configFile ? null
72
73, # Shell code executed after the VM has finished.
74 postVM ? ""
75
76, name ? "nixos-disk-image"
77
78, # Disk image format, one of qcow2, qcow2-compressed, vdi, vpc, raw.
79 format ? "raw"
80
81, # Include a copy of Nixpkgs in the disk image
82 includeChannel ? true
83}:
84let
85 formatOpt = if format == "qcow2-compressed" then "qcow2" else format;
86
87 compress = lib.optionalString (format == "qcow2-compressed") "-c";
88
89 filenameSuffix = "." + {
90 qcow2 = "qcow2";
91 vdi = "vdi";
92 vpc = "vhd";
93 raw = "img";
94 }.${formatOpt} or formatOpt;
95 bootFilename = "nixos.boot${filenameSuffix}";
96 rootFilename = "nixos.root${filenameSuffix}";
97
98 # FIXME: merge with channel.nix / make-channel.nix.
99 channelSources =
100 let
101 nixpkgs = lib.cleanSource pkgs.path;
102 in
103 pkgs.runCommand "nixos-${config.system.nixos.version}" {} ''
104 mkdir -p $out
105 cp -prd ${nixpkgs.outPath} $out/nixos
106 chmod -R u+w $out/nixos
107 if [ ! -e $out/nixos/nixpkgs ]; then
108 ln -s . $out/nixos/nixpkgs
109 fi
110 rm -rf $out/nixos/.git
111 echo -n ${config.system.nixos.versionSuffix} > $out/nixos/.version-suffix
112 '';
113
114 closureInfo = pkgs.closureInfo {
115 rootPaths = [ config.system.build.toplevel ]
116 ++ (lib.optional includeChannel channelSources);
117 };
118
119 modulesTree = pkgs.aggregateModules
120 (with config.boot.kernelPackages; [ kernel zfs ]);
121
122 tools = lib.makeBinPath (
123 with pkgs; [
124 config.system.build.nixos-enter
125 config.system.build.nixos-install
126 dosfstools
127 e2fsprogs
128 gptfdisk
129 nix
130 parted
131 util-linux
132 zfs
133 ]
134 );
135
136 hasDefinedMount = disk: ((disk.mount or null) != null);
137
138 stringifyProperties = prefix: properties: lib.concatStringsSep " \\\n" (
139 lib.mapAttrsToList
140 (
141 property: value: "${prefix} ${lib.escapeShellArg property}=${lib.escapeShellArg value}"
142 )
143 properties
144 );
145
146 createDatasets =
147 let
148 datasetlist = lib.mapAttrsToList lib.nameValuePair datasets;
149 sorted = lib.sort (left: right: (lib.stringLength left.name) < (lib.stringLength right.name)) datasetlist;
150 cmd = { name, value }:
151 let
152 properties = stringifyProperties "-o" (value.properties or {});
153 in
154 "zfs create -p ${properties} ${name}";
155 in
156 lib.concatMapStringsSep "\n" cmd sorted;
157
158 mountDatasets =
159 let
160 datasetlist = lib.mapAttrsToList lib.nameValuePair datasets;
161 mounts = lib.filter ({ value, ... }: hasDefinedMount value) datasetlist;
162 sorted = lib.sort (left: right: (lib.stringLength left.value.mount) < (lib.stringLength right.value.mount)) mounts;
163 cmd = { name, value }:
164 ''
165 mkdir -p /mnt${lib.escapeShellArg value.mount}
166 mount -t zfs ${name} /mnt${lib.escapeShellArg value.mount}
167 '';
168 in
169 lib.concatMapStringsSep "\n" cmd sorted;
170
171 unmountDatasets =
172 let
173 datasetlist = lib.mapAttrsToList lib.nameValuePair datasets;
174 mounts = lib.filter ({ value, ... }: hasDefinedMount value) datasetlist;
175 sorted = lib.sort (left: right: (lib.stringLength left.value.mount) > (lib.stringLength right.value.mount)) mounts;
176 cmd = { name, value }:
177 ''
178 umount /mnt${lib.escapeShellArg value.mount}
179 '';
180 in
181 lib.concatMapStringsSep "\n" cmd sorted;
182
183
184 fileSystemsCfgFile =
185 let
186 mountable = lib.filterAttrs (_: value: hasDefinedMount value) datasets;
187 in
188 pkgs.runCommand "filesystem-config.nix" {
189 buildInputs = with pkgs; [ jq nixpkgs-fmt ];
190 filesystems = builtins.toJSON {
191 fileSystems = lib.mapAttrs'
192 (
193 dataset: attrs:
194 {
195 name = attrs.mount;
196 value = {
197 fsType = "zfs";
198 device = "${dataset}";
199 };
200 }
201 )
202 mountable;
203 };
204 passAsFile = [ "filesystems" ];
205 } ''
206 (
207 echo "builtins.fromJSON '''"
208 jq . < "$filesystemsPath"
209 echo "'''"
210 ) > $out
211
212 nixpkgs-fmt $out
213 '';
214
215 mergedConfig =
216 if configFile == null
217 then fileSystemsCfgFile
218 else
219 pkgs.runCommand "configuration.nix" {
220 buildInputs = with pkgs; [ nixpkgs-fmt ];
221 }
222 ''
223 (
224 echo '{ imports = ['
225 printf "(%s)\n" "$(cat ${fileSystemsCfgFile})";
226 printf "(%s)\n" "$(cat ${configFile})";
227 echo ']; }'
228 ) > $out
229
230 nixpkgs-fmt $out
231 '';
232
233 image = (
234 pkgs.vmTools.override {
235 rootModules =
236 [ "zfs" "9p" "9pnet_virtio" "virtio_pci" "virtio_blk" ] ++
237 (pkgs.lib.optional pkgs.stdenv.hostPlatform.isx86 "rtc_cmos");
238 kernel = modulesTree;
239 }
240 ).runInLinuxVM (
241 pkgs.runCommand name
242 {
243 QEMU_OPTS = "-drive file=$bootDiskImage,if=virtio,cache=unsafe,werror=report"
244 + " -drive file=$rootDiskImage,if=virtio,cache=unsafe,werror=report";
245 preVM = ''
246 PATH=$PATH:${pkgs.qemu_kvm}/bin
247 mkdir $out
248 bootDiskImage=boot.raw
249 qemu-img create -f raw $bootDiskImage ${toString bootSize}M
250
251 rootDiskImage=root.raw
252 qemu-img create -f raw $rootDiskImage ${toString rootSize}M
253 '';
254
255 postVM = ''
256 ${if formatOpt == "raw" then ''
257 mv $bootDiskImage $out/${bootFilename}
258 mv $rootDiskImage $out/${rootFilename}
259 '' else ''
260 ${pkgs.qemu}/bin/qemu-img convert -f raw -O ${formatOpt} ${compress} $bootDiskImage $out/${bootFilename}
261 ${pkgs.qemu}/bin/qemu-img convert -f raw -O ${formatOpt} ${compress} $rootDiskImage $out/${rootFilename}
262 ''}
263 bootDiskImage=$out/${bootFilename}
264 rootDiskImage=$out/${rootFilename}
265 set -x
266 ${postVM}
267 '';
268 } ''
269 export PATH=${tools}:$PATH
270 set -x
271
272 cp -sv /dev/vda /dev/sda
273 cp -sv /dev/vda /dev/xvda
274
275 parted --script /dev/vda -- \
276 mklabel gpt \
277 mkpart no-fs 1MiB 2MiB \
278 set 1 bios_grub on \
279 align-check optimal 1 \
280 mkpart ESP fat32 2MiB -1MiB \
281 align-check optimal 2 \
282 print
283
284 sfdisk --dump /dev/vda
285
286
287 zpool create \
288 ${stringifyProperties " -o" rootPoolProperties} \
289 ${stringifyProperties " -O" rootPoolFilesystemProperties} \
290 ${rootPoolName} /dev/vdb
291 parted --script /dev/vdb -- print
292
293 ${createDatasets}
294 ${mountDatasets}
295
296 mkdir -p /mnt/boot
297 mkfs.vfat -n ESP /dev/vda2
298 mount /dev/vda2 /mnt/boot
299
300 mount
301
302 # Install a configuration.nix
303 mkdir -p /mnt/etc/nixos
304 # `cat` so it is mutable on the fs
305 cat ${mergedConfig} > /mnt/etc/nixos/configuration.nix
306
307 export NIX_STATE_DIR=$TMPDIR/state
308 nix-store --load-db < ${closureInfo}/registration
309
310 nixos-install \
311 --root /mnt \
312 --no-root-passwd \
313 --system ${config.system.build.toplevel} \
314 --substituters "" \
315 ${lib.optionalString includeChannel ''--channel ${channelSources}''}
316
317 df -h
318
319 umount /mnt/boot
320 ${unmountDatasets}
321
322 zpool export ${rootPoolName}
323 ''
324 );
325in
326image