pyrox.dev
1{ config, lib, pkgs, utils, ... }:
2#
3# TODO: zfs tunables
4
5with utils;
6with lib;
7
8let
9
10 cfgZfs = config.boot.zfs;
11 cfgSnapshots = config.services.zfs.autoSnapshot;
12 cfgSnapFlags = cfgSnapshots.flags;
13 cfgScrub = config.services.zfs.autoScrub;
14 cfgTrim = config.services.zfs.trim;
15 cfgZED = config.services.zfs.zed;
16
17 inInitrd = any (fs: fs == "zfs") config.boot.initrd.supportedFilesystems;
18 inSystem = any (fs: fs == "zfs") config.boot.supportedFilesystems;
19
20 autosnapPkg = pkgs.zfstools.override {
21 zfs = cfgZfs.package;
22 };
23
24 zfsAutoSnap = "${autosnapPkg}/bin/zfs-auto-snapshot";
25
26 datasetToPool = x: elemAt (splitString "/" x) 0;
27
28 fsToPool = fs: datasetToPool fs.device;
29
30 zfsFilesystems = filter (x: x.fsType == "zfs") config.system.build.fileSystems;
31
32 allPools = unique ((map fsToPool zfsFilesystems) ++ cfgZfs.extraPools);
33
34 rootPools = unique (map fsToPool (filter fsNeededForBoot zfsFilesystems));
35
36 dataPools = unique (filter (pool: !(elem pool rootPools)) allPools);
37
38 snapshotNames = [ "frequent" "hourly" "daily" "weekly" "monthly" ];
39
40 # When importing ZFS pools, there's one difficulty: These scripts may run
41 # before the backing devices (physical HDDs, etc.) of the pool have been
42 # scanned and initialized.
43 #
44 # An attempted import with all devices missing will just fail, and can be
45 # retried, but an import where e.g. two out of three disks in a three-way
46 # mirror are missing, will succeed. This is a problem: When the missing disks
47 # are later discovered, they won't be automatically set online, rendering the
48 # pool redundancy-less (and far slower) until such time as the system reboots.
49 #
50 # The solution is the below. poolReady checks the status of an un-imported
51 # pool, to see if *every* device is available -- in which case the pool will be
52 # in state ONLINE, as opposed to DEGRADED, FAULTED or MISSING.
53 #
54 # The import scripts then loop over this, waiting until the pool is ready or a
55 # sufficient amount of time has passed that we can assume it won't be. In the
56 # latter case it makes one last attempt at importing, allowing the system to
57 # (eventually) boot even with a degraded pool.
58 importLib = {zpoolCmd, awkCmd, cfgZfs}: ''
59 poolReady() {
60 pool="$1"
61 state="$("${zpoolCmd}" import 2>/dev/null | "${awkCmd}" "/pool: $pool/ { found = 1 }; /state:/ { if (found == 1) { print \$2; exit } }; END { if (found == 0) { print \"MISSING\" } }")"
62 if [[ "$state" = "ONLINE" ]]; then
63 return 0
64 else
65 echo "Pool $pool in state $state, waiting"
66 return 1
67 fi
68 }
69 poolImported() {
70 pool="$1"
71 "${zpoolCmd}" list "$pool" >/dev/null 2>/dev/null
72 }
73 poolImport() {
74 pool="$1"
75 "${zpoolCmd}" import -d "${cfgZfs.devNodes}" -N $ZFS_FORCE "$pool"
76 }
77 '';
78
79 zedConf = generators.toKeyValue {
80 mkKeyValue = generators.mkKeyValueDefault {
81 mkValueString = v:
82 if isInt v then toString v
83 else if isString v then "\"${v}\""
84 else if true == v then "1"
85 else if false == v then "0"
86 else if isList v then "\"" + (concatStringsSep " " v) + "\""
87 else err "this value is" (toString v);
88 } "=";
89 } cfgZED.settings;
90in
91
92{
93
94 imports = [
95 (mkRemovedOptionModule [ "boot" "zfs" "enableLegacyCrypto" ] "The corresponding package was removed from nixpkgs.")
96 ];
97
98 ###### interface
99
100 options = {
101 boot.zfs = {
102 package = mkOption {
103 readOnly = true;
104 type = types.package;
105 default = if config.boot.zfs.enableUnstable then pkgs.zfsUnstable else pkgs.zfs;
106 defaultText = "if config.boot.zfs.enableUnstable then pkgs.zfsUnstable else pkgs.zfs";
107 description = "Configured ZFS userland tools package.";
108 };
109
110 enabled = mkOption {
111 readOnly = true;
112 type = types.bool;
113 default = inInitrd || inSystem;
114 description = "True if ZFS filesystem support is enabled";
115 };
116
117 enableUnstable = mkOption {
118 type = types.bool;
119 default = false;
120 description = ''
121 Use the unstable zfs package. This might be an option, if the latest
122 kernel is not yet supported by a published release of ZFS. Enabling
123 this option will install a development version of ZFS on Linux. The
124 version will have already passed an extensive test suite, but it is
125 more likely to hit an undiscovered bug compared to running a released
126 version of ZFS on Linux.
127 '';
128 };
129
130 extraPools = mkOption {
131 type = types.listOf types.str;
132 default = [];
133 example = [ "tank" "data" ];
134 description = ''
135 Name or GUID of extra ZFS pools that you wish to import during boot.
136
137 Usually this is not necessary. Instead, you should set the mountpoint property
138 of ZFS filesystems to <literal>legacy</literal> and add the ZFS filesystems to
139 NixOS's <option>fileSystems</option> option, which makes NixOS automatically
140 import the associated pool.
141
142 However, in some cases (e.g. if you have many filesystems) it may be preferable
143 to exclusively use ZFS commands to manage filesystems. If so, since NixOS/systemd
144 will not be managing those filesystems, you will need to specify the ZFS pool here
145 so that NixOS automatically imports it on every boot.
146 '';
147 };
148
149 devNodes = mkOption {
150 type = types.path;
151 default = "/dev/disk/by-id";
152 example = "/dev/disk/by-id";
153 description = ''
154 Name of directory from which to import ZFS devices.
155
156 This should be a path under /dev containing stable names for all devices needed, as
157 import may fail if device nodes are renamed concurrently with a device failing.
158 '';
159 };
160
161 forceImportRoot = mkOption {
162 type = types.bool;
163 default = true;
164 description = ''
165 Forcibly import the ZFS root pool(s) during early boot.
166
167 This is enabled by default for backwards compatibility purposes, but it is highly
168 recommended to disable this option, as it bypasses some of the safeguards ZFS uses
169 to protect your ZFS pools.
170
171 If you set this option to <literal>false</literal> and NixOS subsequently fails to
172 boot because it cannot import the root pool, you should boot with the
173 <literal>zfs_force=1</literal> option as a kernel parameter (e.g. by manually
174 editing the kernel params in grub during boot). You should only need to do this
175 once.
176 '';
177 };
178
179 forceImportAll = mkOption {
180 type = types.bool;
181 default = false;
182 description = ''
183 Forcibly import all ZFS pool(s).
184
185 If you set this option to <literal>false</literal> and NixOS subsequently fails to
186 import your non-root ZFS pool(s), you should manually import each pool with
187 "zpool import -f <pool-name>", and then reboot. You should only need to do
188 this once.
189 '';
190 };
191
192 requestEncryptionCredentials = mkOption {
193 type = types.either types.bool (types.listOf types.str);
194 default = true;
195 example = [ "tank" "data" ];
196 description = ''
197 If true on import encryption keys or passwords for all encrypted datasets
198 are requested. To only decrypt selected datasets supply a list of dataset
199 names instead. For root pools the encryption key can be supplied via both
200 an interactive prompt (keylocation=prompt) and from a file (keylocation=file://).
201 '';
202 };
203
204 };
205
206 services.zfs.autoSnapshot = {
207 enable = mkOption {
208 default = false;
209 type = types.bool;
210 description = ''
211 Enable the (OpenSolaris-compatible) ZFS auto-snapshotting service.
212 Note that you must set the <literal>com.sun:auto-snapshot</literal>
213 property to <literal>true</literal> on all datasets which you wish
214 to auto-snapshot.
215
216 You can override a child dataset to use, or not use auto-snapshotting
217 by setting its flag with the given interval:
218 <literal>zfs set com.sun:auto-snapshot:weekly=false DATASET</literal>
219 '';
220 };
221
222 flags = mkOption {
223 default = "-k -p";
224 example = "-k -p --utc";
225 type = types.str;
226 description = ''
227 Flags to pass to the zfs-auto-snapshot command.
228
229 Run <literal>zfs-auto-snapshot</literal> (without any arguments) to
230 see available flags.
231
232 If it's not too inconvenient for snapshots to have timestamps in UTC,
233 it is suggested that you append <literal>--utc</literal> to the list
234 of default options (see example).
235
236 Otherwise, snapshot names can cause name conflicts or apparent time
237 reversals due to daylight savings, timezone or other date/time changes.
238 '';
239 };
240
241 frequent = mkOption {
242 default = 4;
243 type = types.int;
244 description = ''
245 Number of frequent (15-minute) auto-snapshots that you wish to keep.
246 '';
247 };
248
249 hourly = mkOption {
250 default = 24;
251 type = types.int;
252 description = ''
253 Number of hourly auto-snapshots that you wish to keep.
254 '';
255 };
256
257 daily = mkOption {
258 default = 7;
259 type = types.int;
260 description = ''
261 Number of daily auto-snapshots that you wish to keep.
262 '';
263 };
264
265 weekly = mkOption {
266 default = 4;
267 type = types.int;
268 description = ''
269 Number of weekly auto-snapshots that you wish to keep.
270 '';
271 };
272
273 monthly = mkOption {
274 default = 12;
275 type = types.int;
276 description = ''
277 Number of monthly auto-snapshots that you wish to keep.
278 '';
279 };
280 };
281
282 services.zfs.trim = {
283 enable = mkOption {
284 description = "Whether to enable periodic TRIM on all ZFS pools.";
285 default = true;
286 example = false;
287 type = types.bool;
288 };
289
290 interval = mkOption {
291 default = "weekly";
292 type = types.str;
293 example = "daily";
294 description = ''
295 How often we run trim. For most desktop and server systems
296 a sufficient trimming frequency is once a week.
297
298 The format is described in
299 <citerefentry><refentrytitle>systemd.time</refentrytitle>
300 <manvolnum>7</manvolnum></citerefentry>.
301 '';
302 };
303 };
304
305 services.zfs.autoScrub = {
306 enable = mkEnableOption "periodic scrubbing of ZFS pools";
307
308 interval = mkOption {
309 default = "Sun, 02:00";
310 type = types.str;
311 example = "daily";
312 description = ''
313 Systemd calendar expression when to scrub ZFS pools. See
314 <citerefentry><refentrytitle>systemd.time</refentrytitle>
315 <manvolnum>7</manvolnum></citerefentry>.
316 '';
317 };
318
319 pools = mkOption {
320 default = [];
321 type = types.listOf types.str;
322 example = [ "tank" ];
323 description = ''
324 List of ZFS pools to periodically scrub. If empty, all pools
325 will be scrubbed.
326 '';
327 };
328 };
329
330 services.zfs.zed = {
331 enableMail = mkEnableOption "ZED's ability to send emails" // {
332 default = cfgZfs.package.enableMail;
333 };
334
335 settings = mkOption {
336 type = with types; attrsOf (oneOf [ str int bool (listOf str) ]);
337 example = literalExample ''
338 {
339 ZED_DEBUG_LOG = "/tmp/zed.debug.log";
340
341 ZED_EMAIL_ADDR = [ "root" ];
342 ZED_EMAIL_PROG = "mail";
343 ZED_EMAIL_OPTS = "-s '@SUBJECT@' @ADDRESS@";
344
345 ZED_NOTIFY_INTERVAL_SECS = 3600;
346 ZED_NOTIFY_VERBOSE = false;
347
348 ZED_USE_ENCLOSURE_LEDS = true;
349 ZED_SCRUB_AFTER_RESILVER = false;
350 }
351 '';
352 description = ''
353 ZFS Event Daemon /etc/zfs/zed.d/zed.rc content
354
355 See
356 <citerefentry><refentrytitle>zed</refentrytitle><manvolnum>8</manvolnum></citerefentry>
357 for details on ZED and the scripts in /etc/zfs/zed.d to find the possible variables
358 '';
359 };
360 };
361 };
362
363 ###### implementation
364
365 config = mkMerge [
366 (mkIf cfgZfs.enabled {
367 assertions = [
368 {
369 assertion = cfgZED.enableMail -> cfgZfs.package.enableMail;
370 message = ''
371 To allow ZED to send emails, ZFS needs to be configured to enable
372 this. To do so, one must override the `zfs` package and set
373 `enableMail` to true.
374 '';
375 }
376 {
377 assertion = config.networking.hostId != null;
378 message = "ZFS requires networking.hostId to be set";
379 }
380 {
381 assertion = !cfgZfs.forceImportAll || cfgZfs.forceImportRoot;
382 message = "If you enable boot.zfs.forceImportAll, you must also enable boot.zfs.forceImportRoot";
383 }
384 ];
385
386 boot = {
387 kernelModules = [ "zfs" ];
388
389 extraModulePackages = [
390 (if config.boot.zfs.enableUnstable then
391 config.boot.kernelPackages.zfsUnstable
392 else
393 config.boot.kernelPackages.zfs)
394 ];
395 };
396
397 boot.initrd = mkIf inInitrd {
398 kernelModules = [ "zfs" ] ++ optional (!cfgZfs.enableUnstable) "spl";
399 extraUtilsCommands =
400 ''
401 copy_bin_and_libs ${cfgZfs.package}/sbin/zfs
402 copy_bin_and_libs ${cfgZfs.package}/sbin/zdb
403 copy_bin_and_libs ${cfgZfs.package}/sbin/zpool
404 '';
405 extraUtilsCommandsTest = mkIf inInitrd
406 ''
407 $out/bin/zfs --help >/dev/null 2>&1
408 $out/bin/zpool --help >/dev/null 2>&1
409 '';
410 postDeviceCommands = concatStringsSep "\n" ([''
411 ZFS_FORCE="${optionalString cfgZfs.forceImportRoot "-f"}"
412
413 for o in $(cat /proc/cmdline); do
414 case $o in
415 zfs_force|zfs_force=1)
416 ZFS_FORCE="-f"
417 ;;
418 esac
419 done
420 ''] ++ [(importLib {
421 # See comments at importLib definition.
422 zpoolCmd = "zpool";
423 awkCmd = "awk";
424 inherit cfgZfs;
425 })] ++ (map (pool: ''
426 echo -n "importing root ZFS pool \"${pool}\"..."
427 # Loop across the import until it succeeds, because the devices needed may not be discovered yet.
428 if ! poolImported "${pool}"; then
429 for trial in `seq 1 60`; do
430 poolReady "${pool}" > /dev/null && msg="$(poolImport "${pool}" 2>&1)" && break
431 sleep 1
432 echo -n .
433 done
434 echo
435 if [[ -n "$msg" ]]; then
436 echo "$msg";
437 fi
438 poolImported "${pool}" || poolImport "${pool}" # Try one last time, e.g. to import a degraded pool.
439 fi
440 ${if isBool cfgZfs.requestEncryptionCredentials
441 then optionalString cfgZfs.requestEncryptionCredentials ''
442 zfs load-key -a
443 ''
444 else concatMapStrings (fs: ''
445 zfs load-key ${fs}
446 '') cfgZfs.requestEncryptionCredentials}
447 '') rootPools));
448 };
449
450 # TODO FIXME See https://github.com/NixOS/nixpkgs/pull/99386#issuecomment-798813567. To not break people's bootloader and as probably not everybody would read release notes that thoroughly add inSystem.
451 boot.loader.grub = mkIf (inInitrd || inSystem) {
452 zfsSupport = true;
453 };
454
455 services.zfs.zed.settings = {
456 ZED_EMAIL_PROG = mkIf cfgZED.enableMail (mkDefault "${pkgs.mailutils}/bin/mail");
457 PATH = lib.makeBinPath [
458 cfgZfs.package
459 pkgs.coreutils
460 pkgs.curl
461 pkgs.gawk
462 pkgs.gnugrep
463 pkgs.gnused
464 pkgs.nettools
465 pkgs.util-linux
466 ];
467 };
468
469 environment.etc = genAttrs
470 (map
471 (file: "zfs/zed.d/${file}")
472 [
473 "all-syslog.sh"
474 "pool_import-led.sh"
475 "resilver_finish-start-scrub.sh"
476 "statechange-led.sh"
477 "vdev_attach-led.sh"
478 "zed-functions.sh"
479 "data-notify.sh"
480 "resilver_finish-notify.sh"
481 "scrub_finish-notify.sh"
482 "statechange-notify.sh"
483 "vdev_clear-led.sh"
484 ]
485 )
486 (file: { source = "${cfgZfs.package}/etc/${file}"; })
487 // {
488 "zfs/zed.d/zed.rc".text = zedConf;
489 "zfs/zpool.d".source = "${cfgZfs.package}/etc/zfs/zpool.d/";
490 };
491
492 system.fsPackages = [ cfgZfs.package ]; # XXX: needed? zfs doesn't have (need) a fsck
493 environment.systemPackages = [ cfgZfs.package ]
494 ++ optional cfgSnapshots.enable autosnapPkg; # so the user can run the command to see flags
495
496 services.udev.packages = [ cfgZfs.package ]; # to hook zvol naming, etc.
497 systemd.packages = [ cfgZfs.package ];
498
499 systemd.services = let
500 getPoolFilesystems = pool:
501 filter (x: x.fsType == "zfs" && (fsToPool x) == pool) config.system.build.fileSystems;
502
503 getPoolMounts = pool:
504 let
505 mountPoint = fs: escapeSystemdPath fs.mountPoint;
506 in
507 map (x: "${mountPoint x}.mount") (getPoolFilesystems pool);
508
509 createImportService = pool:
510 nameValuePair "zfs-import-${pool}" {
511 description = "Import ZFS pool \"${pool}\"";
512 # we need systemd-udev-settle until https://github.com/zfsonlinux/zfs/pull/4943 is merged
513 requires = [ "systemd-udev-settle.service" ];
514 after = [
515 "systemd-udev-settle.service"
516 "systemd-modules-load.service"
517 "systemd-ask-password-console.service"
518 ];
519 wantedBy = (getPoolMounts pool) ++ [ "local-fs.target" ];
520 before = (getPoolMounts pool) ++ [ "local-fs.target" ];
521 unitConfig = {
522 DefaultDependencies = "no";
523 };
524 serviceConfig = {
525 Type = "oneshot";
526 RemainAfterExit = true;
527 };
528 environment.ZFS_FORCE = optionalString cfgZfs.forceImportAll "-f";
529 script = (importLib {
530 # See comments at importLib definition.
531 zpoolCmd = "${cfgZfs.package}/sbin/zpool";
532 awkCmd = "${pkgs.gawk}/bin/awk";
533 inherit cfgZfs;
534 }) + ''
535 poolImported "${pool}" && exit
536 echo -n "importing ZFS pool \"${pool}\"..."
537 # Loop across the import until it succeeds, because the devices needed may not be discovered yet.
538 for trial in `seq 1 60`; do
539 poolReady "${pool}" && poolImport "${pool}" && break
540 sleep 1
541 done
542 poolImported "${pool}" || poolImport "${pool}" # Try one last time, e.g. to import a degraded pool.
543 if poolImported "${pool}"; then
544 ${optionalString (if isBool cfgZfs.requestEncryptionCredentials
545 then cfgZfs.requestEncryptionCredentials
546 else cfgZfs.requestEncryptionCredentials != []) ''
547 ${cfgZfs.package}/sbin/zfs list -rHo name,keylocation ${pool} | while IFS=$'\t' read ds kl; do
548 (${optionalString (!isBool cfgZfs.requestEncryptionCredentials) ''
549 if ! echo '${concatStringsSep "\n" cfgZfs.requestEncryptionCredentials}' | grep -qFx "$ds"; then
550 continue
551 fi
552 ''}
553 case "$kl" in
554 none )
555 ;;
556 prompt )
557 ${config.systemd.package}/bin/systemd-ask-password "Enter key for $ds:" | ${cfgZfs.package}/sbin/zfs load-key "$ds"
558 ;;
559 * )
560 ${cfgZfs.package}/sbin/zfs load-key "$ds"
561 ;;
562 esac) < /dev/null # To protect while read ds kl in case anything reads stdin
563 done
564 ''}
565 echo "Successfully imported ${pool}"
566 else
567 exit 1
568 fi
569 '';
570 };
571
572 # This forces a sync of any ZFS pools prior to poweroff, even if they're set
573 # to sync=disabled.
574 createSyncService = pool:
575 nameValuePair "zfs-sync-${pool}" {
576 description = "Sync ZFS pool \"${pool}\"";
577 wantedBy = [ "shutdown.target" ];
578 unitConfig = {
579 DefaultDependencies = false;
580 };
581 serviceConfig = {
582 Type = "oneshot";
583 RemainAfterExit = true;
584 };
585 script = ''
586 ${cfgZfs.package}/sbin/zfs set nixos:shutdown-time="$(date)" "${pool}"
587 '';
588 };
589 createZfsService = serv:
590 nameValuePair serv {
591 after = [ "systemd-modules-load.service" ];
592 wantedBy = [ "zfs.target" ];
593 };
594
595 in listToAttrs (map createImportService dataPools ++
596 map createSyncService allPools ++
597 map createZfsService [ "zfs-mount" "zfs-share" "zfs-zed" ]);
598
599 systemd.targets.zfs-import =
600 let
601 services = map (pool: "zfs-import-${pool}.service") dataPools;
602 in
603 {
604 requires = services;
605 after = services;
606 wantedBy = [ "zfs.target" ];
607 };
608
609 systemd.targets.zfs.wantedBy = [ "multi-user.target" ];
610 })
611
612 (mkIf (cfgZfs.enabled && cfgSnapshots.enable) {
613 systemd.services = let
614 descr = name: if name == "frequent" then "15 mins"
615 else if name == "hourly" then "hour"
616 else if name == "daily" then "day"
617 else if name == "weekly" then "week"
618 else if name == "monthly" then "month"
619 else throw "unknown snapshot name";
620 numSnapshots = name: builtins.getAttr name cfgSnapshots;
621 in builtins.listToAttrs (map (snapName:
622 {
623 name = "zfs-snapshot-${snapName}";
624 value = {
625 description = "ZFS auto-snapshotting every ${descr snapName}";
626 after = [ "zfs-import.target" ];
627 serviceConfig = {
628 Type = "oneshot";
629 ExecStart = "${zfsAutoSnap} ${cfgSnapFlags} ${snapName} ${toString (numSnapshots snapName)}";
630 };
631 restartIfChanged = false;
632 };
633 }) snapshotNames);
634
635 systemd.timers = let
636 timer = name: if name == "frequent" then "*:0,15,30,45" else name;
637 in builtins.listToAttrs (map (snapName:
638 {
639 name = "zfs-snapshot-${snapName}";
640 value = {
641 wantedBy = [ "timers.target" ];
642 timerConfig = {
643 OnCalendar = timer snapName;
644 Persistent = "yes";
645 };
646 };
647 }) snapshotNames);
648 })
649
650 (mkIf (cfgZfs.enabled && cfgScrub.enable) {
651 systemd.services.zfs-scrub = {
652 description = "ZFS pools scrubbing";
653 after = [ "zfs-import.target" ];
654 serviceConfig = {
655 Type = "oneshot";
656 };
657 script = ''
658 ${cfgZfs.package}/bin/zpool scrub ${
659 if cfgScrub.pools != [] then
660 (concatStringsSep " " cfgScrub.pools)
661 else
662 "$(${cfgZfs.package}/bin/zpool list -H -o name)"
663 }
664 '';
665 };
666
667 systemd.timers.zfs-scrub = {
668 wantedBy = [ "timers.target" ];
669 after = [ "multi-user.target" ]; # Apparently scrubbing before boot is complete hangs the system? #53583
670 timerConfig = {
671 OnCalendar = cfgScrub.interval;
672 Persistent = "yes";
673 };
674 };
675 })
676
677 (mkIf (cfgZfs.enabled && cfgTrim.enable) {
678 systemd.services.zpool-trim = {
679 description = "ZFS pools trim";
680 after = [ "zfs-import.target" ];
681 path = [ cfgZfs.package ];
682 startAt = cfgTrim.interval;
683 # By default we ignore errors returned by the trim command, in case:
684 # - HDDs are mixed with SSDs
685 # - There is a SSDs in a pool that is currently trimmed.
686 # - There are only HDDs and we would set the system in a degraded state
687 serviceConfig.ExecStart = "${pkgs.runtimeShell} -c 'for pool in $(zpool list -H -o name); do zpool trim $pool; done || true' ";
688 };
689
690 systemd.timers.zpool-trim.timerConfig.Persistent = "yes";
691 })
692 ];
693}