pyrox.dev
1{ config, lib, pkgs, utils, ... }:
2#
3# todo:
4# - crontab for scrubs, etc
5# - zfs tunables
6
7with utils;
8with lib;
9
10let
11
12 cfgZfs = config.boot.zfs;
13 cfgSnapshots = config.services.zfs.autoSnapshot;
14 cfgSnapFlags = cfgSnapshots.flags;
15 cfgScrub = config.services.zfs.autoScrub;
16
17 inInitrd = any (fs: fs == "zfs") config.boot.initrd.supportedFilesystems;
18 inSystem = any (fs: fs == "zfs") config.boot.supportedFilesystems;
19
20 enableAutoSnapshots = cfgSnapshots.enable;
21 enableAutoScrub = cfgScrub.enable;
22 enableZfs = inInitrd || inSystem || enableAutoSnapshots || enableAutoScrub;
23
24 kernel = config.boot.kernelPackages;
25
26 packages = if config.boot.zfs.enableUnstable then {
27 spl = null;
28 zfs = kernel.zfsUnstable;
29 zfsUser = pkgs.zfsUnstable;
30 } else {
31 spl = kernel.spl;
32 zfs = kernel.zfs;
33 zfsUser = pkgs.zfs;
34 };
35
36 autosnapPkg = pkgs.zfstools.override {
37 zfs = packages.zfsUser;
38 };
39
40 zfsAutoSnap = "${autosnapPkg}/bin/zfs-auto-snapshot";
41
42 datasetToPool = x: elemAt (splitString "/" x) 0;
43
44 fsToPool = fs: datasetToPool fs.device;
45
46 zfsFilesystems = filter (x: x.fsType == "zfs") config.system.build.fileSystems;
47
48 allPools = unique ((map fsToPool zfsFilesystems) ++ cfgZfs.extraPools);
49
50 rootPools = unique (map fsToPool (filter fsNeededForBoot zfsFilesystems));
51
52 dataPools = unique (filter (pool: !(elem pool rootPools)) allPools);
53
54 snapshotNames = [ "frequent" "hourly" "daily" "weekly" "monthly" ];
55
56 # When importing ZFS pools, there's one difficulty: These scripts may run
57 # before the backing devices (physical HDDs, etc.) of the pool have been
58 # scanned and initialized.
59 #
60 # An attempted import with all devices missing will just fail, and can be
61 # retried, but an import where e.g. two out of three disks in a three-way
62 # mirror are missing, will succeed. This is a problem: When the missing disks
63 # are later discovered, they won't be automatically set online, rendering the
64 # pool redundancy-less (and far slower) until such time as the system reboots.
65 #
66 # The solution is the below. poolReady checks the status of an un-imported
67 # pool, to see if *every* device is available -- in which case the pool will be
68 # in state ONLINE, as opposed to DEGRADED, FAULTED or MISSING.
69 #
70 # The import scripts then loop over this, waiting until the pool is ready or a
71 # sufficient amount of time has passed that we can assume it won't be. In the
72 # latter case it makes one last attempt at importing, allowing the system to
73 # (eventually) boot even with a degraded pool.
74 importLib = {zpoolCmd, awkCmd, cfgZfs}: ''
75 poolReady() {
76 pool="$1"
77 state="$("${zpoolCmd}" import | "${awkCmd}" "/pool: $pool/ { found = 1 }; /state:/ { if (found == 1) { print \$2; exit } }; END { if (found == 0) { print \"MISSING\" } }")"
78 if [[ "$state" = "ONLINE" ]]; then
79 return 0
80 else
81 echo "Pool $pool in state $state, waiting"
82 return 1
83 fi
84 }
85 poolImported() {
86 pool="$1"
87 "${zpoolCmd}" list "$pool" >/dev/null 2>/dev/null
88 }
89 poolImport() {
90 pool="$1"
91 "${zpoolCmd}" import -d "${cfgZfs.devNodes}" -N $ZFS_FORCE "$pool"
92 }
93 '';
94
95in
96
97{
98
99 ###### interface
100
101 options = {
102 boot.zfs = {
103 enableUnstable = mkOption {
104 type = types.bool;
105 default = false;
106 description = ''
107 Use the unstable zfs package. This might be an option, if the latest
108 kernel is not yet supported by a published release of ZFS. Enabling
109 this option will install a development version of ZFS on Linux. The
110 version will have already passed an extensive test suite, but it is
111 more likely to hit an undiscovered bug compared to running a released
112 version of ZFS on Linux.
113 '';
114 };
115
116 extraPools = mkOption {
117 type = types.listOf types.str;
118 default = [];
119 example = [ "tank" "data" ];
120 description = ''
121 Name or GUID of extra ZFS pools that you wish to import during boot.
122
123 Usually this is not necessary. Instead, you should set the mountpoint property
124 of ZFS filesystems to <literal>legacy</literal> and add the ZFS filesystems to
125 NixOS's <option>fileSystems</option> option, which makes NixOS automatically
126 import the associated pool.
127
128 However, in some cases (e.g. if you have many filesystems) it may be preferable
129 to exclusively use ZFS commands to manage filesystems. If so, since NixOS/systemd
130 will not be managing those filesystems, you will need to specify the ZFS pool here
131 so that NixOS automatically imports it on every boot.
132 '';
133 };
134
135 devNodes = mkOption {
136 type = types.path;
137 default = "/dev/disk/by-id";
138 example = "/dev/disk/by-id";
139 description = ''
140 Name of directory from which to import ZFS devices.
141
142 This should be a path under /dev containing stable names for all devices needed, as
143 import may fail if device nodes are renamed concurrently with a device failing.
144 '';
145 };
146
147 forceImportRoot = mkOption {
148 type = types.bool;
149 default = true;
150 description = ''
151 Forcibly import the ZFS root pool(s) during early boot.
152
153 This is enabled by default for backwards compatibility purposes, but it is highly
154 recommended to disable this option, as it bypasses some of the safeguards ZFS uses
155 to protect your ZFS pools.
156
157 If you set this option to <literal>false</literal> and NixOS subsequently fails to
158 boot because it cannot import the root pool, you should boot with the
159 <literal>zfs_force=1</literal> option as a kernel parameter (e.g. by manually
160 editing the kernel params in grub during boot). You should only need to do this
161 once.
162 '';
163 };
164
165 forceImportAll = mkOption {
166 type = types.bool;
167 default = true;
168 description = ''
169 Forcibly import all ZFS pool(s).
170
171 This is enabled by default for backwards compatibility purposes, but it is highly
172 recommended to disable this option, as it bypasses some of the safeguards ZFS uses
173 to protect your ZFS pools.
174
175 If you set this option to <literal>false</literal> and NixOS subsequently fails to
176 import your non-root ZFS pool(s), you should manually import each pool with
177 "zpool import -f <pool-name>", and then reboot. You should only need to do
178 this once.
179 '';
180 };
181
182 requestEncryptionCredentials = mkOption {
183 type = types.bool;
184 default = config.boot.zfs.enableUnstable;
185 description = ''
186 Request encryption keys or passwords for all encrypted datasets on import.
187 Dataset encryption is only supported in zfsUnstable at the moment.
188 For root pools the encryption key can be supplied via both an
189 interactive prompt (keylocation=prompt) and from a file
190 (keylocation=file://). Note that for data pools the encryption key can
191 be only loaded from a file and not via interactive prompt since the
192 import is processed in a background systemd service.
193 '';
194 };
195
196 };
197
198 services.zfs.autoSnapshot = {
199 enable = mkOption {
200 default = false;
201 type = types.bool;
202 description = ''
203 Enable the (OpenSolaris-compatible) ZFS auto-snapshotting service.
204 Note that you must set the <literal>com.sun:auto-snapshot</literal>
205 property to <literal>true</literal> on all datasets which you wish
206 to auto-snapshot.
207
208 You can override a child dataset to use, or not use auto-snapshotting
209 by setting its flag with the given interval:
210 <literal>zfs set com.sun:auto-snapshot:weekly=false DATASET</literal>
211 '';
212 };
213
214 flags = mkOption {
215 default = "-k -p";
216 example = "-k -p --utc";
217 type = types.str;
218 description = ''
219 Flags to pass to the zfs-auto-snapshot command.
220
221 Run <literal>zfs-auto-snapshot</literal> (without any arguments) to
222 see available flags.
223
224 If it's not too inconvenient for snapshots to have timestamps in UTC,
225 it is suggested that you append <literal>--utc</literal> to the list
226 of default options (see example).
227
228 Otherwise, snapshot names can cause name conflicts or apparent time
229 reversals due to daylight savings, timezone or other date/time changes.
230 '';
231 };
232
233 frequent = mkOption {
234 default = 4;
235 type = types.int;
236 description = ''
237 Number of frequent (15-minute) auto-snapshots that you wish to keep.
238 '';
239 };
240
241 hourly = mkOption {
242 default = 24;
243 type = types.int;
244 description = ''
245 Number of hourly auto-snapshots that you wish to keep.
246 '';
247 };
248
249 daily = mkOption {
250 default = 7;
251 type = types.int;
252 description = ''
253 Number of daily auto-snapshots that you wish to keep.
254 '';
255 };
256
257 weekly = mkOption {
258 default = 4;
259 type = types.int;
260 description = ''
261 Number of weekly auto-snapshots that you wish to keep.
262 '';
263 };
264
265 monthly = mkOption {
266 default = 12;
267 type = types.int;
268 description = ''
269 Number of monthly auto-snapshots that you wish to keep.
270 '';
271 };
272 };
273
274 services.zfs.autoScrub = {
275 enable = mkOption {
276 default = false;
277 type = types.bool;
278 description = ''
279 Enables periodic scrubbing of ZFS pools.
280 '';
281 };
282
283 interval = mkOption {
284 default = "Sun, 02:00";
285 type = types.str;
286 example = "daily";
287 description = ''
288 Systemd calendar expression when to scrub ZFS pools. See
289 <citerefentry><refentrytitle>systemd.time</refentrytitle>
290 <manvolnum>7</manvolnum></citerefentry>.
291 '';
292 };
293
294 pools = mkOption {
295 default = [];
296 type = types.listOf types.str;
297 example = [ "tank" ];
298 description = ''
299 List of ZFS pools to periodically scrub. If empty, all pools
300 will be scrubbed.
301 '';
302 };
303 };
304 };
305
306 ###### implementation
307
308 config = mkMerge [
309 (mkIf enableZfs {
310 assertions = [
311 {
312 assertion = config.networking.hostId != null;
313 message = "ZFS requires networking.hostId to be set";
314 }
315 {
316 assertion = !cfgZfs.forceImportAll || cfgZfs.forceImportRoot;
317 message = "If you enable boot.zfs.forceImportAll, you must also enable boot.zfs.forceImportRoot";
318 }
319 {
320 assertion = cfgZfs.requestEncryptionCredentials -> cfgZfs.enableUnstable;
321 message = "This feature is only available for zfs unstable. Set the NixOS option boot.zfs.enableUnstable.";
322 }
323 ];
324
325 virtualisation.lxd.zfsSupport = true;
326
327 boot = {
328 kernelModules = [ "zfs" ] ++ optional (!cfgZfs.enableUnstable) "spl";
329 extraModulePackages = with packages; [ zfs ] ++ optional (!cfgZfs.enableUnstable) spl;
330 };
331
332 boot.initrd = mkIf inInitrd {
333 kernelModules = [ "zfs" ] ++ optional (!cfgZfs.enableUnstable) "spl";
334 extraUtilsCommands =
335 ''
336 copy_bin_and_libs ${packages.zfsUser}/sbin/zfs
337 copy_bin_and_libs ${packages.zfsUser}/sbin/zdb
338 copy_bin_and_libs ${packages.zfsUser}/sbin/zpool
339 '';
340 extraUtilsCommandsTest = mkIf inInitrd
341 ''
342 $out/bin/zfs --help >/dev/null 2>&1
343 $out/bin/zpool --help >/dev/null 2>&1
344 '';
345 postDeviceCommands = concatStringsSep "\n" ([''
346 ZFS_FORCE="${optionalString cfgZfs.forceImportRoot "-f"}"
347
348 for o in $(cat /proc/cmdline); do
349 case $o in
350 zfs_force|zfs_force=1)
351 ZFS_FORCE="-f"
352 ;;
353 esac
354 done
355 ''] ++ [(importLib {
356 # See comments at importLib definition.
357 zpoolCmd = "zpool";
358 awkCmd = "awk";
359 inherit cfgZfs;
360 })] ++ (map (pool: ''
361 echo -n "importing root ZFS pool \"${pool}\"..."
362 # Loop across the import until it succeeds, because the devices needed may not be discovered yet.
363 if ! poolImported "${pool}"; then
364 for trial in `seq 1 60`; do
365 poolReady "${pool}" > /dev/null && msg="$(poolImport "${pool}" 2>&1)" && break
366 sleep 1
367 echo -n .
368 done
369 echo
370 if [[ -n "$msg" ]]; then
371 echo "$msg";
372 fi
373 poolImported "${pool}" || poolImport "${pool}" # Try one last time, e.g. to import a degraded pool.
374 fi
375 ${lib.optionalString cfgZfs.requestEncryptionCredentials ''
376 zfs load-key -a
377 ''}
378 '') rootPools));
379 };
380
381 boot.loader.grub = mkIf inInitrd {
382 zfsSupport = true;
383 };
384
385 environment.etc."zfs/zed.d".source = "${packages.zfsUser}/etc/zfs/zed.d/";
386
387 system.fsPackages = [ packages.zfsUser ]; # XXX: needed? zfs doesn't have (need) a fsck
388 environment.systemPackages = [ packages.zfsUser ]
389 ++ optional enableAutoSnapshots autosnapPkg; # so the user can run the command to see flags
390
391 services.udev.packages = [ packages.zfsUser ]; # to hook zvol naming, etc.
392 systemd.packages = [ packages.zfsUser ];
393
394 systemd.services = let
395 getPoolFilesystems = pool:
396 filter (x: x.fsType == "zfs" && (fsToPool x) == pool) config.system.build.fileSystems;
397
398 getPoolMounts = pool:
399 let
400 mountPoint = fs: escapeSystemdPath fs.mountPoint;
401 in
402 map (x: "${mountPoint x}.mount") (getPoolFilesystems pool);
403
404 createImportService = pool:
405 nameValuePair "zfs-import-${pool}" {
406 description = "Import ZFS pool \"${pool}\"";
407 requires = [ "systemd-udev-settle.service" ];
408 after = [ "systemd-udev-settle.service" "systemd-modules-load.service" ];
409 wantedBy = (getPoolMounts pool) ++ [ "local-fs.target" ];
410 before = (getPoolMounts pool) ++ [ "local-fs.target" ];
411 unitConfig = {
412 DefaultDependencies = "no";
413 };
414 serviceConfig = {
415 Type = "oneshot";
416 RemainAfterExit = true;
417 };
418 script = (importLib {
419 # See comments at importLib definition.
420 zpoolCmd="${packages.zfsUser}/sbin/zpool";
421 awkCmd="${pkgs.gawk}/bin/awk";
422 inherit cfgZfs;
423 }) + ''
424 poolImported "${pool}" && exit
425 echo -n "importing ZFS pool \"${pool}\"..."
426 # Loop across the import until it succeeds, because the devices needed may not be discovered yet.
427 for trial in `seq 1 60`; do
428 poolReady "${pool}" && poolImport "${pool}" && break
429 sleep 1
430 done
431 poolImported "${pool}" || poolImport "${pool}" # Try one last time, e.g. to import a degraded pool.
432 if poolImported "${pool}"; then
433 ${optionalString cfgZfs.requestEncryptionCredentials "\"${packages.zfsUser}/sbin/zfs\" load-key -r \"${pool}\""}
434 echo "Successfully imported ${pool}"
435 else
436 exit 1
437 fi
438 '';
439 };
440
441 # This forces a sync of any ZFS pools prior to poweroff, even if they're set
442 # to sync=disabled.
443 createSyncService = pool:
444 nameValuePair "zfs-sync-${pool}" {
445 description = "Sync ZFS pool \"${pool}\"";
446 wantedBy = [ "shutdown.target" ];
447 unitConfig = {
448 DefaultDependencies = false;
449 };
450 serviceConfig = {
451 Type = "oneshot";
452 RemainAfterExit = true;
453 };
454 script = ''
455 ${packages.zfsUser}/sbin/zfs set nixos:shutdown-time="$(date)" "${pool}"
456 '';
457 };
458 createZfsService = serv:
459 nameValuePair serv {
460 after = [ "systemd-modules-load.service" ];
461 wantedBy = [ "zfs.target" ];
462 };
463
464 in listToAttrs (map createImportService dataPools ++
465 map createSyncService allPools ++
466 map createZfsService [ "zfs-mount" "zfs-share" "zfs-zed" ]);
467
468 systemd.targets."zfs-import" =
469 let
470 services = map (pool: "zfs-import-${pool}.service") dataPools;
471 in
472 {
473 requires = services;
474 after = services;
475 wantedBy = [ "zfs.target" ];
476 };
477
478 systemd.targets."zfs".wantedBy = [ "multi-user.target" ];
479 })
480
481 (mkIf enableAutoSnapshots {
482 systemd.services = let
483 descr = name: if name == "frequent" then "15 mins"
484 else if name == "hourly" then "hour"
485 else if name == "daily" then "day"
486 else if name == "weekly" then "week"
487 else if name == "monthly" then "month"
488 else throw "unknown snapshot name";
489 numSnapshots = name: builtins.getAttr name cfgSnapshots;
490 in builtins.listToAttrs (map (snapName:
491 {
492 name = "zfs-snapshot-${snapName}";
493 value = {
494 description = "ZFS auto-snapshotting every ${descr snapName}";
495 after = [ "zfs-import.target" ];
496 serviceConfig = {
497 Type = "oneshot";
498 ExecStart = "${zfsAutoSnap} ${cfgSnapFlags} ${snapName} ${toString (numSnapshots snapName)}";
499 };
500 restartIfChanged = false;
501 };
502 }) snapshotNames);
503
504 systemd.timers = let
505 timer = name: if name == "frequent" then "*:0,15,30,45" else name;
506 in builtins.listToAttrs (map (snapName:
507 {
508 name = "zfs-snapshot-${snapName}";
509 value = {
510 wantedBy = [ "timers.target" ];
511 timerConfig = {
512 OnCalendar = timer snapName;
513 Persistent = "yes";
514 };
515 };
516 }) snapshotNames);
517 })
518
519 (mkIf enableAutoScrub {
520 systemd.services.zfs-scrub = {
521 description = "ZFS pools scrubbing";
522 after = [ "zfs-import.target" ];
523 serviceConfig = {
524 Type = "oneshot";
525 };
526 script = ''
527 ${packages.zfsUser}/bin/zpool scrub ${
528 if cfgScrub.pools != [] then
529 (concatStringsSep " " cfgScrub.pools)
530 else
531 "$(${packages.zfsUser}/bin/zpool list -H -o name)"
532 }
533 '';
534 };
535
536 systemd.timers.zfs-scrub = {
537 wantedBy = [ "timers.target" ];
538 timerConfig = {
539 OnCalendar = cfgScrub.interval;
540 Persistent = "yes";
541 };
542 };
543 })
544 ];
545}