pyrox.dev
1# Note: This is a private API, internal to NixOS. Its interface is subject
2# to change without notice.
3#
4# The result of this builder is two disk images:
5#
6# * `boot` - a small disk formatted with FAT to be used for /boot. FAT is
7# chosen to support EFI.
8# * `root` - a larger disk with a zpool taking the entire disk.
9#
10# This two-disk approach is taken to satisfy ZFS's requirements for
11# autoexpand.
12#
13# # Why doesn't autoexpand work with ZFS in a partition?
14#
15# When ZFS owns the whole disk doesn’t really use a partition: it has
16# a marker partition at the start and a marker partition at the end of
17# the disk.
18#
19# If ZFS is constrained to a partition, ZFS leaves expanding the partition
20# up to the user. Obviously, the user may not choose to do so.
21#
22# Once the user expands the partition, calling zpool online -e expands the
23# vdev to use the whole partition. It doesn’t happen automatically
24# presumably because zed doesn’t get an event saying it’s partition grew,
25# whereas it can and does get an event saying the whole disk it is on is
26# now larger.
27{ lib
28, pkgs
29, # The NixOS configuration to be installed onto the disk image.
30 config
31
32, # size of the FAT boot disk, in megabytes.
33 bootSize ? 1024
34
35, # The size of the root disk, in megabytes.
36 rootSize ? 2048
37
38, # The name of the ZFS pool
39 rootPoolName ? "tank"
40
41, # zpool properties
42 rootPoolProperties ? {
43 autoexpand = "on";
44 }
45, # pool-wide filesystem properties
46 rootPoolFilesystemProperties ? {
47 acltype = "posixacl";
48 atime = "off";
49 compression = "on";
50 mountpoint = "legacy";
51 xattr = "sa";
52 }
53
54, # datasets, with per-attribute options:
55 # mount: (optional) mount point in the VM
56 # properties: (optional) ZFS properties on the dataset, like filesystemProperties
57 # Notes:
58 # 1. datasets will be created from shorter to longer names as a simple topo-sort
59 # 2. you should define a root's dataset's mount for `/`
60 datasets ? { }
61
62, # The files and directories to be placed in the target file system.
63 # This is a list of attribute sets {source, target} where `source'
64 # is the file system object (regular file or directory) to be
65 # grafted in the file system at path `target'.
66 contents ? []
67
68, # The initial NixOS configuration file to be copied to
69 # /etc/nixos/configuration.nix. This configuration will be embedded
70 # inside a configuration which includes the described ZFS fileSystems.
71 configFile ? null
72
73, # Shell code executed after the VM has finished.
74 postVM ? ""
75
76, # Guest memory size
77 memSize ? 1024
78
79, name ? "nixos-disk-image"
80
81, # Disk image format, one of qcow2, qcow2-compressed, vdi, vpc, raw.
82 format ? "raw"
83
84, # Include a copy of Nixpkgs in the disk image
85 includeChannel ? true
86}:
87let
88 formatOpt = if format == "qcow2-compressed" then "qcow2" else format;
89
90 compress = lib.optionalString (format == "qcow2-compressed") "-c";
91
92 filenameSuffix = "." + {
93 qcow2 = "qcow2";
94 vdi = "vdi";
95 vpc = "vhd";
96 raw = "img";
97 }.${formatOpt} or formatOpt;
98 bootFilename = "nixos.boot${filenameSuffix}";
99 rootFilename = "nixos.root${filenameSuffix}";
100
101 # FIXME: merge with channel.nix / make-channel.nix.
102 channelSources =
103 let
104 nixpkgs = lib.cleanSource pkgs.path;
105 in
106 pkgs.runCommand "nixos-${config.system.nixos.version}" {} ''
107 mkdir -p $out
108 cp -prd ${nixpkgs.outPath} $out/nixos
109 chmod -R u+w $out/nixos
110 if [ ! -e $out/nixos/nixpkgs ]; then
111 ln -s . $out/nixos/nixpkgs
112 fi
113 rm -rf $out/nixos/.git
114 echo -n ${config.system.nixos.versionSuffix} > $out/nixos/.version-suffix
115 '';
116
117 closureInfo = pkgs.closureInfo {
118 rootPaths = [ config.system.build.toplevel ]
119 ++ (lib.optional includeChannel channelSources);
120 };
121
122 modulesTree = pkgs.aggregateModules
123 (with config.boot.kernelPackages; [ kernel zfs ]);
124
125 tools = lib.makeBinPath (
126 with pkgs; [
127 config.system.build.nixos-enter
128 config.system.build.nixos-install
129 dosfstools
130 e2fsprogs
131 gptfdisk
132 nix
133 parted
134 util-linux
135 zfs
136 ]
137 );
138
139 hasDefinedMount = disk: ((disk.mount or null) != null);
140
141 stringifyProperties = prefix: properties: lib.concatStringsSep " \\\n" (
142 lib.mapAttrsToList
143 (
144 property: value: "${prefix} ${lib.escapeShellArg property}=${lib.escapeShellArg value}"
145 )
146 properties
147 );
148
149 createDatasets =
150 let
151 datasetlist = lib.mapAttrsToList lib.nameValuePair datasets;
152 sorted = lib.sort (left: right: (lib.stringLength left.name) < (lib.stringLength right.name)) datasetlist;
153 cmd = { name, value }:
154 let
155 properties = stringifyProperties "-o" (value.properties or {});
156 in
157 "zfs create -p ${properties} ${name}";
158 in
159 lib.concatMapStringsSep "\n" cmd sorted;
160
161 mountDatasets =
162 let
163 datasetlist = lib.mapAttrsToList lib.nameValuePair datasets;
164 mounts = lib.filter ({ value, ... }: hasDefinedMount value) datasetlist;
165 sorted = lib.sort (left: right: (lib.stringLength left.value.mount) < (lib.stringLength right.value.mount)) mounts;
166 cmd = { name, value }:
167 ''
168 mkdir -p /mnt${lib.escapeShellArg value.mount}
169 mount -t zfs ${name} /mnt${lib.escapeShellArg value.mount}
170 '';
171 in
172 lib.concatMapStringsSep "\n" cmd sorted;
173
174 unmountDatasets =
175 let
176 datasetlist = lib.mapAttrsToList lib.nameValuePair datasets;
177 mounts = lib.filter ({ value, ... }: hasDefinedMount value) datasetlist;
178 sorted = lib.sort (left: right: (lib.stringLength left.value.mount) > (lib.stringLength right.value.mount)) mounts;
179 cmd = { name, value }:
180 ''
181 umount /mnt${lib.escapeShellArg value.mount}
182 '';
183 in
184 lib.concatMapStringsSep "\n" cmd sorted;
185
186
187 fileSystemsCfgFile =
188 let
189 mountable = lib.filterAttrs (_: value: hasDefinedMount value) datasets;
190 in
191 pkgs.runCommand "filesystem-config.nix" {
192 buildInputs = with pkgs; [ jq nixpkgs-fmt ];
193 filesystems = builtins.toJSON {
194 fileSystems = lib.mapAttrs'
195 (
196 dataset: attrs:
197 {
198 name = attrs.mount;
199 value = {
200 fsType = "zfs";
201 device = "${dataset}";
202 };
203 }
204 )
205 mountable;
206 };
207 passAsFile = [ "filesystems" ];
208 } ''
209 (
210 echo "builtins.fromJSON '''"
211 jq . < "$filesystemsPath"
212 echo "'''"
213 ) > $out
214
215 nixpkgs-fmt $out
216 '';
217
218 mergedConfig =
219 if configFile == null
220 then fileSystemsCfgFile
221 else
222 pkgs.runCommand "configuration.nix" {
223 buildInputs = with pkgs; [ nixpkgs-fmt ];
224 }
225 ''
226 (
227 echo '{ imports = ['
228 printf "(%s)\n" "$(cat ${fileSystemsCfgFile})";
229 printf "(%s)\n" "$(cat ${configFile})";
230 echo ']; }'
231 ) > $out
232
233 nixpkgs-fmt $out
234 '';
235
236 image = (
237 pkgs.vmTools.override {
238 rootModules =
239 [ "zfs" "9p" "9pnet_virtio" "virtio_pci" "virtio_blk" ] ++
240 (pkgs.lib.optional pkgs.stdenv.hostPlatform.isx86 "rtc_cmos");
241 kernel = modulesTree;
242 }
243 ).runInLinuxVM (
244 pkgs.runCommand name
245 {
246 QEMU_OPTS = "-drive file=$bootDiskImage,if=virtio,cache=unsafe,werror=report"
247 + " -drive file=$rootDiskImage,if=virtio,cache=unsafe,werror=report";
248 inherit memSize;
249 preVM = ''
250 PATH=$PATH:${pkgs.qemu_kvm}/bin
251 mkdir $out
252 bootDiskImage=boot.raw
253 qemu-img create -f raw $bootDiskImage ${toString bootSize}M
254
255 rootDiskImage=root.raw
256 qemu-img create -f raw $rootDiskImage ${toString rootSize}M
257 '';
258
259 postVM = ''
260 ${if formatOpt == "raw" then ''
261 mv $bootDiskImage $out/${bootFilename}
262 mv $rootDiskImage $out/${rootFilename}
263 '' else ''
264 ${pkgs.qemu_kvm}/bin/qemu-img convert -f raw -O ${formatOpt} ${compress} $bootDiskImage $out/${bootFilename}
265 ${pkgs.qemu_kvm}/bin/qemu-img convert -f raw -O ${formatOpt} ${compress} $rootDiskImage $out/${rootFilename}
266 ''}
267 bootDiskImage=$out/${bootFilename}
268 rootDiskImage=$out/${rootFilename}
269 set -x
270 ${postVM}
271 '';
272 } ''
273 export PATH=${tools}:$PATH
274 set -x
275
276 cp -sv /dev/vda /dev/sda
277 cp -sv /dev/vda /dev/xvda
278
279 parted --script /dev/vda -- \
280 mklabel gpt \
281 mkpart no-fs 1MiB 2MiB \
282 set 1 bios_grub on \
283 align-check optimal 1 \
284 mkpart ESP fat32 2MiB -1MiB \
285 align-check optimal 2 \
286 print
287
288 sfdisk --dump /dev/vda
289
290
291 zpool create \
292 ${stringifyProperties " -o" rootPoolProperties} \
293 ${stringifyProperties " -O" rootPoolFilesystemProperties} \
294 ${rootPoolName} /dev/vdb
295 parted --script /dev/vdb -- print
296
297 ${createDatasets}
298 ${mountDatasets}
299
300 mkdir -p /mnt/boot
301 mkfs.vfat -n ESP /dev/vda2
302 mount /dev/vda2 /mnt/boot
303
304 mount
305
306 # Install a configuration.nix
307 mkdir -p /mnt/etc/nixos
308 # `cat` so it is mutable on the fs
309 cat ${mergedConfig} > /mnt/etc/nixos/configuration.nix
310
311 export NIX_STATE_DIR=$TMPDIR/state
312 nix-store --load-db < ${closureInfo}/registration
313
314 nixos-install \
315 --root /mnt \
316 --no-root-passwd \
317 --system ${config.system.build.toplevel} \
318 --substituters "" \
319 ${lib.optionalString includeChannel ''--channel ${channelSources}''}
320
321 df -h
322
323 umount /mnt/boot
324 ${unmountDatasets}
325
326 zpool export ${rootPoolName}
327 ''
328 );
329in
330image