pyrox.dev
1{ config, lib, pkgs, ... }: with lib;
2
3# See http://christophe.varoqui.free.fr/usage.html and
4# https://github.com/opensvc/multipath-tools/blob/master/multipath/multipath.conf.5
5
6let
7 cfg = config.services.multipath;
8
9 indentLines = n: str: concatStringsSep "\n" (
10 map (line: "${fixedWidthString n " " " "}${line}") (
11 filter ( x: x != "" ) ( splitString "\n" str )
12 )
13 );
14
15 addCheckDesc = desc: elemType: check: types.addCheck elemType check
16 // { description = "${elemType.description} (with check: ${desc})"; };
17 hexChars = stringToCharacters "0123456789abcdef";
18 isHexString = s: all (c: elem c hexChars) (stringToCharacters (toLower s));
19 hexStr = addCheckDesc "hexadecimal string" types.str isHexString;
20
21in {
22
23 options.services.multipath = with types; {
24
25 enable = mkEnableOption (lib.mdDoc "the device mapper multipath (DM-MP) daemon");
26
27 package = mkOption {
28 type = package;
29 description = lib.mdDoc "multipath-tools package to use";
30 default = pkgs.multipath-tools;
31 defaultText = lib.literalExpression "pkgs.multipath-tools";
32 };
33
34 devices = mkOption {
35 default = [ ];
36 example = literalExpression ''
37 [
38 {
39 vendor = "\"COMPELNT\"";
40 product = "\"Compellent Vol\"";
41 path_checker = "tur";
42 no_path_retry = "queue";
43 max_sectors_kb = 256;
44 }, ...
45 ]
46 '';
47 description = lib.mdDoc ''
48 This option allows you to define arrays for use in multipath
49 groups.
50 '';
51 type = listOf (submodule {
52 options = {
53
54 vendor = mkOption {
55 type = str;
56 example = "COMPELNT";
57 description = lib.mdDoc "Regular expression to match the vendor name";
58 };
59
60 product = mkOption {
61 type = str;
62 example = "Compellent Vol";
63 description = lib.mdDoc "Regular expression to match the product name";
64 };
65
66 revision = mkOption {
67 type = nullOr str;
68 default = null;
69 description = lib.mdDoc "Regular expression to match the product revision";
70 };
71
72 product_blacklist = mkOption {
73 type = nullOr str;
74 default = null;
75 description = lib.mdDoc "Products with the given vendor matching this string are blacklisted";
76 };
77
78 alias_prefix = mkOption {
79 type = nullOr str;
80 default = null;
81 description = lib.mdDoc "The user_friendly_names prefix to use for this device type, instead of the default mpath";
82 };
83
84 vpd_vendor = mkOption {
85 type = nullOr str;
86 default = null;
87 description = lib.mdDoc "The vendor specific vpd page information, using the vpd page abbreviation";
88 };
89
90 hardware_handler = mkOption {
91 type = nullOr (enum [ "emc" "rdac" "hp_sw" "alua" "ana" ]);
92 default = null;
93 description = lib.mdDoc "The hardware handler to use for this device type";
94 };
95
96 # Optional arguments
97 path_grouping_policy = mkOption {
98 type = nullOr (enum [ "failover" "multibus" "group_by_serial" "group_by_prio" "group_by_node_name" ]);
99 default = null; # real default: "failover"
100 description = lib.mdDoc "The default path grouping policy to apply to unspecified multipaths";
101 };
102
103 uid_attribute = mkOption {
104 type = nullOr str;
105 default = null;
106 description = lib.mdDoc "The udev attribute providing a unique path identifier (WWID)";
107 };
108
109 getuid_callout = mkOption {
110 type = nullOr str;
111 default = null;
112 description = lib.mdDoc ''
113 (Superseded by uid_attribute) The default program and args to callout
114 to obtain a unique path identifier. Should be specified with an absolute path.
115 '';
116 };
117
118 path_selector = mkOption {
119 type = nullOr (enum [
120 ''"round-robin 0"''
121 ''"queue-length 0"''
122 ''"service-time 0"''
123 ''"historical-service-time 0"''
124 ]);
125 default = null; # real default: "service-time 0"
126 description = lib.mdDoc "The default path selector algorithm to use; they are offered by the kernel multipath target";
127 };
128
129 path_checker = mkOption {
130 type = enum [ "readsector0" "tur" "emc_clariion" "hp_sw" "rdac" "directio" "cciss_tur" "none" ];
131 default = "tur";
132 description = lib.mdDoc "The default method used to determine the paths state";
133 };
134
135 prio = mkOption {
136 type = nullOr (enum [
137 "none" "const" "sysfs" "emc" "alua" "ontap" "rdac" "hp_sw" "hds"
138 "random" "weightedpath" "path_latency" "ana" "datacore" "iet"
139 ]);
140 default = null; # real default: "const"
141 description = lib.mdDoc "The name of the path priority routine";
142 };
143
144 prio_args = mkOption {
145 type = nullOr str;
146 default = null;
147 description = lib.mdDoc "Arguments to pass to to the prio function";
148 };
149
150 features = mkOption {
151 type = nullOr str;
152 default = null;
153 description = lib.mdDoc "Specify any device-mapper features to be used";
154 };
155
156 failback = mkOption {
157 type = nullOr str;
158 default = null; # real default: "manual"
159 description = lib.mdDoc "Tell multipathd how to manage path group failback. Quote integers as strings";
160 };
161
162 rr_weight = mkOption {
163 type = nullOr (enum [ "priorities" "uniform" ]);
164 default = null; # real default: "uniform"
165 description = lib.mdDoc ''
166 If set to priorities the multipath configurator will assign path weights
167 as "path prio * rr_min_io".
168 '';
169 };
170
171 no_path_retry = mkOption {
172 type = nullOr str;
173 default = null; # real default: "fail"
174 description = lib.mdDoc "Specify what to do when all paths are down. Quote integers as strings";
175 };
176
177 rr_min_io = mkOption {
178 type = nullOr int;
179 default = null; # real default: 1000
180 description = lib.mdDoc ''
181 Number of I/O requests to route to a path before switching to the next in the
182 same path group. This is only for Block I/O (BIO) based multipath and
183 only apply to round-robin path_selector.
184 '';
185 };
186
187 rr_min_io_rq = mkOption {
188 type = nullOr int;
189 default = null; # real default: 1
190 description = lib.mdDoc ''
191 Number of I/O requests to route to a path before switching to the next in the
192 same path group. This is only for Request based multipath and
193 only apply to round-robin path_selector.
194 '';
195 };
196
197 fast_io_fail_tmo = mkOption {
198 type = nullOr str;
199 default = null; # real default: 5
200 description = lib.mdDoc ''
201 Specify the number of seconds the SCSI layer will wait after a problem has been
202 detected on a FC remote port before failing I/O to devices on that remote port.
203 This should be smaller than dev_loss_tmo. Setting this to "off" will disable
204 the timeout. Quote integers as strings.
205 '';
206 };
207
208 dev_loss_tmo = mkOption {
209 type = nullOr str;
210 default = null; # real default: 600
211 description = lib.mdDoc ''
212 Specify the number of seconds the SCSI layer will wait after a problem has
213 been detected on a FC remote port before removing it from the system. This
214 can be set to "infinity" which sets it to the max value of 2147483647
215 seconds, or 68 years. It will be automatically adjusted to the overall
216 retry interval no_path_retry * polling_interval
217 if a number of retries is given with no_path_retry and the
218 overall retry interval is longer than the specified dev_loss_tmo value.
219 The Linux kernel will cap this value to 600 if fast_io_fail_tmo
220 is not set.
221 '';
222 };
223
224 flush_on_last_del = mkOption {
225 type = nullOr (enum [ "yes" "no" ]);
226 default = null; # real default: "no"
227 description = lib.mdDoc ''
228 If set to "yes" multipathd will disable queueing when the last path to a
229 device has been deleted.
230 '';
231 };
232
233 user_friendly_names = mkOption {
234 type = nullOr (enum [ "yes" "no" ]);
235 default = null; # real default: "no"
236 description = lib.mdDoc ''
237 If set to "yes", using the bindings file /etc/multipath/bindings
238 to assign a persistent and unique alias to the multipath, in the
239 form of mpath. If set to "no" use the WWID as the alias. In either
240 case this be will be overridden by any specific aliases in the
241 multipaths section.
242 '';
243 };
244
245 detect_prio = mkOption {
246 type = nullOr (enum [ "yes" "no" ]);
247 default = null; # real default: "yes"
248 description = lib.mdDoc ''
249 If set to "yes", multipath will try to detect if the device supports
250 SCSI-3 ALUA. If so, the device will automatically use the sysfs
251 prioritizer if the required sysf attributes access_state and
252 preferred_path are supported, or the alua prioritizer if not. If set
253 to "no", the prioritizer will be selected as usual.
254 '';
255 };
256
257 detect_checker = mkOption {
258 type = nullOr (enum [ "yes" "no" ]);
259 default = null; # real default: "yes"
260 description = lib.mdDoc ''
261 If set to "yes", multipath will try to detect if the device supports
262 SCSI-3 ALUA. If so, the device will automatically use the tur checker.
263 If set to "no", the checker will be selected as usual.
264 '';
265 };
266
267 deferred_remove = mkOption {
268 type = nullOr (enum [ "yes" "no" ]);
269 default = null; # real default: "no"
270 description = lib.mdDoc ''
271 If set to "yes", multipathd will do a deferred remove instead of a
272 regular remove when the last path device has been deleted. This means
273 that if the multipath device is still in use, it will be freed when
274 the last user closes it. If path is added to the multipath device
275 before the last user closes it, the deferred remove will be canceled.
276 '';
277 };
278
279 san_path_err_threshold = mkOption {
280 type = nullOr str;
281 default = null;
282 description = lib.mdDoc ''
283 If set to a value greater than 0, multipathd will watch paths and check
284 how many times a path has been failed due to errors.If the number of
285 failures on a particular path is greater then the san_path_err_threshold,
286 then the path will not reinstate till san_path_err_recovery_time. These
287 path failures should occur within a san_path_err_forget_rate checks, if
288 not we will consider the path is good enough to reinstantate.
289 '';
290 };
291
292 san_path_err_forget_rate = mkOption {
293 type = nullOr str;
294 default = null;
295 description = lib.mdDoc ''
296 If set to a value greater than 0, multipathd will check whether the path
297 failures has exceeded the san_path_err_threshold within this many checks
298 i.e san_path_err_forget_rate. If so we will not reinstante the path till
299 san_path_err_recovery_time.
300 '';
301 };
302
303 san_path_err_recovery_time = mkOption {
304 type = nullOr str;
305 default = null;
306 description = lib.mdDoc ''
307 If set to a value greater than 0, multipathd will make sure that when
308 path failures has exceeded the san_path_err_threshold within
309 san_path_err_forget_rate then the path will be placed in failed state
310 for san_path_err_recovery_time duration. Once san_path_err_recovery_time
311 has timeout we will reinstante the failed path. san_path_err_recovery_time
312 value should be in secs.
313 '';
314 };
315
316 marginal_path_err_sample_time = mkOption {
317 type = nullOr int;
318 default = null;
319 description = lib.mdDoc "One of the four parameters of supporting path check based on accounting IO error such as intermittent error";
320 };
321
322 marginal_path_err_rate_threshold = mkOption {
323 type = nullOr int;
324 default = null;
325 description = lib.mdDoc "The error rate threshold as a permillage (1/1000)";
326 };
327
328 marginal_path_err_recheck_gap_time = mkOption {
329 type = nullOr str;
330 default = null;
331 description = lib.mdDoc "One of the four parameters of supporting path check based on accounting IO error such as intermittent error";
332 };
333
334 marginal_path_double_failed_time = mkOption {
335 type = nullOr str;
336 default = null;
337 description = lib.mdDoc "One of the four parameters of supporting path check based on accounting IO error such as intermittent error";
338 };
339
340 delay_watch_checks = mkOption {
341 type = nullOr str;
342 default = null;
343 description = lib.mdDoc "This option is deprecated, and mapped to san_path_err_forget_rate";
344 };
345
346 delay_wait_checks = mkOption {
347 type = nullOr str;
348 default = null;
349 description = lib.mdDoc "This option is deprecated, and mapped to san_path_err_recovery_time";
350 };
351
352 skip_kpartx = mkOption {
353 type = nullOr (enum [ "yes" "no" ]);
354 default = null; # real default: "no"
355 description = lib.mdDoc "If set to yes, kpartx will not automatically create partitions on the device";
356 };
357
358 max_sectors_kb = mkOption {
359 type = nullOr int;
360 default = null;
361 description = lib.mdDoc "Sets the max_sectors_kb device parameter on all path devices and the multipath device to the specified value";
362 };
363
364 ghost_delay = mkOption {
365 type = nullOr int;
366 default = null;
367 description = lib.mdDoc "Sets the number of seconds that multipath will wait after creating a device with only ghost paths before marking it ready for use in systemd";
368 };
369
370 all_tg_pt = mkOption {
371 type = nullOr str;
372 default = null;
373 description = lib.mdDoc "Set the 'all targets ports' flag when registering keys with mpathpersist";
374 };
375
376 };
377 });
378 };
379
380 defaults = mkOption {
381 type = nullOr str;
382 default = null;
383 description = lib.mdDoc ''
384 This section defines default values for attributes which are used
385 whenever no values are given in the appropriate device or multipath
386 sections.
387 '';
388 };
389
390 blacklist = mkOption {
391 type = nullOr str;
392 default = null;
393 description = lib.mdDoc ''
394 This section defines which devices should be excluded from the
395 multipath topology discovery.
396 '';
397 };
398
399 blacklist_exceptions = mkOption {
400 type = nullOr str;
401 default = null;
402 description = lib.mdDoc ''
403 This section defines which devices should be included in the
404 multipath topology discovery, despite being listed in the
405 blacklist section.
406 '';
407 };
408
409 overrides = mkOption {
410 type = nullOr str;
411 default = null;
412 description = lib.mdDoc ''
413 This section defines values for attributes that should override the
414 device-specific settings for all devices.
415 '';
416 };
417
418 extraConfig = mkOption {
419 type = nullOr str;
420 default = null;
421 description = lib.mdDoc "Lines to append to default multipath.conf";
422 };
423
424 extraConfigFile = mkOption {
425 type = nullOr str;
426 default = null;
427 description = lib.mdDoc "Append an additional file's contents to /etc/multipath.conf";
428 };
429
430 pathGroups = mkOption {
431 example = literalExpression ''
432 [
433 {
434 wwid = "360080e500043b35c0123456789abcdef";
435 alias = 10001234;
436 array = "bigarray.example.com";
437 fsType = "zfs"; # optional
438 options = "ro"; # optional
439 }, ...
440 ]
441 '';
442 description = lib.mdDoc ''
443 This option allows you to define multipath groups as described
444 in http://christophe.varoqui.free.fr/usage.html.
445 '';
446 type = listOf (submodule {
447 options = {
448
449 alias = mkOption {
450 type = int;
451 example = 1001234;
452 description = lib.mdDoc "The name of the multipath device";
453 };
454
455 wwid = mkOption {
456 type = hexStr;
457 example = "360080e500043b35c0123456789abcdef";
458 description = lib.mdDoc "The identifier for the multipath device";
459 };
460
461 array = mkOption {
462 type = str;
463 default = null;
464 example = "bigarray.example.com";
465 description = lib.mdDoc "The DNS name of the storage array";
466 };
467
468 fsType = mkOption {
469 type = nullOr str;
470 default = null;
471 example = "zfs";
472 description = lib.mdDoc "Type of the filesystem";
473 };
474
475 options = mkOption {
476 type = nullOr str;
477 default = null;
478 example = "ro";
479 description = lib.mdDoc "Options used to mount the file system";
480 };
481
482 };
483 });
484 };
485
486 };
487
488 config = mkIf cfg.enable {
489 environment.etc."multipath.conf".text =
490 let
491 inherit (cfg) defaults blacklist blacklist_exceptions overrides;
492
493 mkDeviceBlock = cfg: let
494 nonNullCfg = lib.filterAttrs (k: v: v != null) cfg;
495 attrs = lib.mapAttrsToList (name: value: " ${name} ${toString value}") nonNullCfg;
496 in ''
497 device {
498 ${lib.concatStringsSep "\n" attrs}
499 }
500 '';
501 devices = lib.concatMapStringsSep "\n" mkDeviceBlock cfg.devices;
502
503 mkMultipathBlock = m: ''
504 multipath {
505 wwid ${m.wwid}
506 alias ${toString m.alias}
507 }
508 '';
509 multipaths = lib.concatMapStringsSep "\n" mkMultipathBlock cfg.pathGroups;
510
511 in ''
512 devices {
513 ${indentLines 2 devices}
514 }
515
516 ${optionalString (defaults != null) ''
517 defaults {
518 ${indentLines 2 defaults}
519 }
520 ''}
521 ${optionalString (blacklist != null) ''
522 blacklist {
523 ${indentLines 2 blacklist}
524 }
525 ''}
526 ${optionalString (blacklist_exceptions != null) ''
527 blacklist_exceptions {
528 ${indentLines 2 blacklist_exceptions}
529 }
530 ''}
531 ${optionalString (overrides != null) ''
532 overrides {
533 ${indentLines 2 overrides}
534 }
535 ''}
536 multipaths {
537 ${indentLines 2 multipaths}
538 }
539 '';
540
541 systemd.packages = [ cfg.package ];
542
543 environment.systemPackages = [ cfg.package ];
544 boot.kernelModules = [ "dm-multipath" "dm-service-time" ];
545
546 # We do not have systemd in stage-1 boot so must invoke `multipathd`
547 # with the `-1` argument which disables systemd calls. Invoke `multipath`
548 # to display the multipath mappings in the output of `journalctl -b`.
549 boot.initrd.kernelModules = [ "dm-multipath" "dm-service-time" ];
550 boot.initrd.postDeviceCommands = ''
551 modprobe -a dm-multipath dm-service-time
552 multipathd -s
553 (set -x && sleep 1 && multipath -ll)
554 '';
555 };
556}