pyrox.dev
1{ config, lib, pkgs, utils, ... }:
2#
3# todo:
4# - crontab for scrubs, etc
5# - zfs tunables
6
7with utils;
8with lib;
9
10let
11
12 cfgSpl = config.boot.spl;
13 cfgZfs = config.boot.zfs;
14 cfgSnapshots = config.services.zfs.autoSnapshot;
15 cfgSnapFlags = cfgSnapshots.flags;
16 cfgScrub = config.services.zfs.autoScrub;
17
18 inInitrd = any (fs: fs == "zfs") config.boot.initrd.supportedFilesystems;
19 inSystem = any (fs: fs == "zfs") config.boot.supportedFilesystems;
20
21 enableAutoSnapshots = cfgSnapshots.enable;
22 enableAutoScrub = cfgScrub.enable;
23 enableZfs = inInitrd || inSystem || enableAutoSnapshots || enableAutoScrub;
24
25 kernel = config.boot.kernelPackages;
26
27 packages = {
28 spl = kernel.spl;
29 zfs = kernel.zfs;
30 zfsUser = pkgs.zfs;
31 };
32
33 autosnapPkg = pkgs.zfstools.override {
34 zfs = packages.zfsUser;
35 };
36
37 zfsAutoSnap = "${autosnapPkg}/bin/zfs-auto-snapshot";
38
39 datasetToPool = x: elemAt (splitString "/" x) 0;
40
41 fsToPool = fs: datasetToPool fs.device;
42
43 zfsFilesystems = filter (x: x.fsType == "zfs") config.system.build.fileSystems;
44
45 allPools = unique ((map fsToPool zfsFilesystems) ++ cfgZfs.extraPools);
46
47 rootPools = unique (map fsToPool (filter fsNeededForBoot zfsFilesystems));
48
49 dataPools = unique (filter (pool: !(elem pool rootPools)) allPools);
50
51 snapshotNames = [ "frequent" "hourly" "daily" "weekly" "monthly" ];
52
53in
54
55{
56
57 ###### interface
58
59 options = {
60 boot.zfs = {
61 extraPools = mkOption {
62 type = types.listOf types.str;
63 default = [];
64 example = [ "tank" "data" ];
65 description = ''
66 Name or GUID of extra ZFS pools that you wish to import during boot.
67
68 Usually this is not necessary. Instead, you should set the mountpoint property
69 of ZFS filesystems to <literal>legacy</literal> and add the ZFS filesystems to
70 NixOS's <option>fileSystems</option> option, which makes NixOS automatically
71 import the associated pool.
72
73 However, in some cases (e.g. if you have many filesystems) it may be preferable
74 to exclusively use ZFS commands to manage filesystems. If so, since NixOS/systemd
75 will not be managing those filesystems, you will need to specify the ZFS pool here
76 so that NixOS automatically imports it on every boot.
77 '';
78 };
79
80 devNodes = mkOption {
81 type = types.path;
82 default = "/dev/disk/by-id";
83 example = "/dev/disk/by-id";
84 description = ''
85 Name of directory from which to import ZFS devices.
86
87 This should be a path under /dev containing stable names for all devices needed, as
88 import may fail if device nodes are renamed concurrently with a device failing.
89 '';
90 };
91
92 forceImportRoot = mkOption {
93 type = types.bool;
94 default = true;
95 description = ''
96 Forcibly import the ZFS root pool(s) during early boot.
97
98 This is enabled by default for backwards compatibility purposes, but it is highly
99 recommended to disable this option, as it bypasses some of the safeguards ZFS uses
100 to protect your ZFS pools.
101
102 If you set this option to <literal>false</literal> and NixOS subsequently fails to
103 boot because it cannot import the root pool, you should boot with the
104 <literal>zfs_force=1</literal> option as a kernel parameter (e.g. by manually
105 editing the kernel params in grub during boot). You should only need to do this
106 once.
107 '';
108 };
109
110 forceImportAll = mkOption {
111 type = types.bool;
112 default = true;
113 description = ''
114 Forcibly import all ZFS pool(s).
115
116 This is enabled by default for backwards compatibility purposes, but it is highly
117 recommended to disable this option, as it bypasses some of the safeguards ZFS uses
118 to protect your ZFS pools.
119
120 If you set this option to <literal>false</literal> and NixOS subsequently fails to
121 import your non-root ZFS pool(s), you should manually import each pool with
122 "zpool import -f <pool-name>", and then reboot. You should only need to do
123 this once.
124 '';
125 };
126 };
127
128 services.zfs.autoSnapshot = {
129 enable = mkOption {
130 default = false;
131 type = types.bool;
132 description = ''
133 Enable the (OpenSolaris-compatible) ZFS auto-snapshotting service.
134 Note that you must set the <literal>com.sun:auto-snapshot</literal>
135 property to <literal>true</literal> on all datasets which you wish
136 to auto-snapshot.
137
138 You can override a child dataset to use, or not use auto-snapshotting
139 by setting its flag with the given interval:
140 <literal>zfs set com.sun:auto-snapshot:weekly=false DATASET</literal>
141 '';
142 };
143
144 flags = mkOption {
145 default = "-k -p";
146 example = "-k -p --utc";
147 type = types.str;
148 description = ''
149 Flags to pass to the zfs-auto-snapshot command.
150
151 Run <literal>zfs-auto-snapshot</literal> (without any arguments) to
152 see available flags.
153
154 If it's not too inconvenient for snapshots to have timestamps in UTC,
155 it is suggested that you append <literal>--utc</literal> to the list
156 of default options (see example).
157
158 Otherwise, snapshot names can cause name conflicts or apparent time
159 reversals due to daylight savings, timezone or other date/time changes.
160 '';
161 };
162
163 frequent = mkOption {
164 default = 4;
165 type = types.int;
166 description = ''
167 Number of frequent (15-minute) auto-snapshots that you wish to keep.
168 '';
169 };
170
171 hourly = mkOption {
172 default = 24;
173 type = types.int;
174 description = ''
175 Number of hourly auto-snapshots that you wish to keep.
176 '';
177 };
178
179 daily = mkOption {
180 default = 7;
181 type = types.int;
182 description = ''
183 Number of daily auto-snapshots that you wish to keep.
184 '';
185 };
186
187 weekly = mkOption {
188 default = 4;
189 type = types.int;
190 description = ''
191 Number of weekly auto-snapshots that you wish to keep.
192 '';
193 };
194
195 monthly = mkOption {
196 default = 12;
197 type = types.int;
198 description = ''
199 Number of monthly auto-snapshots that you wish to keep.
200 '';
201 };
202 };
203
204 services.zfs.autoScrub = {
205 enable = mkOption {
206 default = false;
207 type = types.bool;
208 description = ''
209 Enables periodic scrubbing of ZFS pools.
210 '';
211 };
212
213 interval = mkOption {
214 default = "Sun, 02:00";
215 type = types.str;
216 example = "daily";
217 description = ''
218 Systemd calendar expression when to scrub ZFS pools. See
219 <citerefentry><refentrytitle>systemd.time</refentrytitle>
220 <manvolnum>7</manvolnum></citerefentry>.
221 '';
222 };
223
224 pools = mkOption {
225 default = [];
226 type = types.listOf types.str;
227 example = [ "tank" ];
228 description = ''
229 List of ZFS pools to periodically scrub. If empty, all pools
230 will be scrubbed.
231 '';
232 };
233 };
234 };
235
236 ###### implementation
237
238 config = mkMerge [
239 (mkIf enableZfs {
240 assertions = [
241 {
242 assertion = config.networking.hostId != null;
243 message = "ZFS requires config.networking.hostId to be set";
244 }
245 {
246 assertion = !cfgZfs.forceImportAll || cfgZfs.forceImportRoot;
247 message = "If you enable boot.zfs.forceImportAll, you must also enable boot.zfs.forceImportRoot";
248 }
249 ];
250
251 boot = {
252 kernelModules = [ "spl" "zfs" ] ;
253 extraModulePackages = with packages; [ spl zfs ];
254 };
255
256 boot.initrd = mkIf inInitrd {
257 kernelModules = [ "spl" "zfs" ];
258 extraUtilsCommands =
259 ''
260 copy_bin_and_libs ${packages.zfsUser}/sbin/zfs
261 copy_bin_and_libs ${packages.zfsUser}/sbin/zdb
262 copy_bin_and_libs ${packages.zfsUser}/sbin/zpool
263 '';
264 extraUtilsCommandsTest = mkIf inInitrd
265 ''
266 $out/bin/zfs --help >/dev/null 2>&1
267 $out/bin/zpool --help >/dev/null 2>&1
268 '';
269 postDeviceCommands = concatStringsSep "\n" ([''
270 ZFS_FORCE="${optionalString cfgZfs.forceImportRoot "-f"}"
271
272 for o in $(cat /proc/cmdline); do
273 case $o in
274 zfs_force|zfs_force=1)
275 ZFS_FORCE="-f"
276 ;;
277 esac
278 done
279 ''] ++ (map (pool: ''
280 echo -n "importing root ZFS pool \"${pool}\"..."
281 trial=0
282 until msg="$(zpool import -d ${cfgZfs.devNodes} -N $ZFS_FORCE '${pool}' 2>&1)"; do
283 sleep 0.25
284 echo -n .
285 trial=$(($trial + 1))
286 if [[ $trial -eq 60 ]]; then
287 break
288 fi
289 done
290 echo
291 if [[ -n "$msg" ]]; then echo "$msg"; fi
292 '') rootPools));
293 };
294
295 boot.loader.grub = mkIf inInitrd {
296 zfsSupport = true;
297 };
298
299 environment.etc."zfs/zed.d".source = "${packages.zfsUser}/etc/zfs/zed.d/";
300
301 system.fsPackages = [ packages.zfsUser ]; # XXX: needed? zfs doesn't have (need) a fsck
302 environment.systemPackages = [ packages.zfsUser ]
303 ++ optional enableAutoSnapshots autosnapPkg; # so the user can run the command to see flags
304
305 services.udev.packages = [ packages.zfsUser ]; # to hook zvol naming, etc.
306 systemd.packages = [ packages.zfsUser ];
307
308 systemd.services = let
309 getPoolFilesystems = pool:
310 filter (x: x.fsType == "zfs" && (fsToPool x) == pool) config.system.build.fileSystems;
311
312 getPoolMounts = pool:
313 let
314 mountPoint = fs: escapeSystemdPath fs.mountPoint;
315 in
316 map (x: "${mountPoint x}.mount") (getPoolFilesystems pool);
317
318 createImportService = pool:
319 nameValuePair "zfs-import-${pool}" {
320 description = "Import ZFS pool \"${pool}\"";
321 requires = [ "systemd-udev-settle.service" ];
322 after = [ "systemd-udev-settle.service" "systemd-modules-load.service" ];
323 wantedBy = (getPoolMounts pool) ++ [ "local-fs.target" ];
324 before = (getPoolMounts pool) ++ [ "local-fs.target" ];
325 unitConfig = {
326 DefaultDependencies = "no";
327 };
328 serviceConfig = {
329 Type = "oneshot";
330 RemainAfterExit = true;
331 };
332 script = ''
333 zpool_cmd="${packages.zfsUser}/sbin/zpool"
334 ("$zpool_cmd" list "${pool}" >/dev/null) || "$zpool_cmd" import -d ${cfgZfs.devNodes} -N ${optionalString cfgZfs.forceImportAll "-f"} "${pool}"
335 '';
336 };
337
338 # This forces a sync of any ZFS pools prior to poweroff, even if they're set
339 # to sync=disabled.
340 createSyncService = pool:
341 nameValuePair "zfs-sync-${pool}" {
342 description = "Sync ZFS pool \"${pool}\"";
343 wantedBy = [ "shutdown.target" ];
344 serviceConfig = {
345 Type = "oneshot";
346 RemainAfterExit = true;
347 };
348 script = ''
349 ${packages.zfsUser}/sbin/zfs set nixos:shutdown-time="$(date)" "${pool}"
350 '';
351 };
352
353 in listToAttrs (map createImportService dataPools ++ map createSyncService allPools) // {
354 "zfs-mount" = { after = [ "systemd-modules-load.service" ]; };
355 "zfs-share" = { after = [ "systemd-modules-load.service" ]; };
356 "zfs-zed" = { after = [ "systemd-modules-load.service" ]; };
357 };
358
359 systemd.targets."zfs-import" =
360 let
361 services = map (pool: "zfs-import-${pool}.service") dataPools;
362 in
363 {
364 requires = services;
365 after = services;
366 };
367
368 systemd.targets."zfs".wantedBy = [ "multi-user.target" ];
369 })
370
371 (mkIf enableAutoSnapshots {
372 systemd.services = let
373 descr = name: if name == "frequent" then "15 mins"
374 else if name == "hourly" then "hour"
375 else if name == "daily" then "day"
376 else if name == "weekly" then "week"
377 else if name == "monthly" then "month"
378 else throw "unknown snapshot name";
379 numSnapshots = name: builtins.getAttr name cfgSnapshots;
380 in builtins.listToAttrs (map (snapName:
381 {
382 name = "zfs-snapshot-${snapName}";
383 value = {
384 description = "ZFS auto-snapshotting every ${descr snapName}";
385 after = [ "zfs-import.target" ];
386 serviceConfig = {
387 Type = "oneshot";
388 ExecStart = "${zfsAutoSnap} ${cfgSnapFlags} ${snapName} ${toString (numSnapshots snapName)}";
389 };
390 restartIfChanged = false;
391 };
392 }) snapshotNames);
393
394 systemd.timers = let
395 timer = name: if name == "frequent" then "*:15,30,45" else name;
396 in builtins.listToAttrs (map (snapName:
397 {
398 name = "zfs-snapshot-${snapName}";
399 value = {
400 wantedBy = [ "timers.target" ];
401 timerConfig = {
402 OnCalendar = timer snapName;
403 Persistent = "yes";
404 };
405 };
406 }) snapshotNames);
407 })
408
409 (mkIf enableAutoScrub {
410 systemd.services.zfs-scrub = {
411 description = "ZFS pools scrubbing";
412 after = [ "zfs-import.target" ];
413 serviceConfig = {
414 Type = "oneshot";
415 };
416 script = ''
417 ${packages.zfsUser}/bin/zpool scrub ${
418 if cfgScrub.pools != [] then
419 (concatStringsSep " " cfgScrub.pools)
420 else
421 "$(${packages.zfsUser}/bin/zpool list -H -o name)"
422 }
423 '';
424 };
425
426 systemd.timers.zfs-scrub = {
427 wantedBy = [ "timers.target" ];
428 timerConfig = {
429 OnCalendar = cfgScrub.interval;
430 Persistent = "yes";
431 };
432 };
433 })
434 ];
435}